@dirgha/code 1.8.1 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/cli/config.d.ts +4 -3
- package/dist/cli/config.js +10 -3
- package/dist/cli/config.js.map +1 -1
- package/dist/cli/interactive.js +1 -1
- package/dist/cli/interactive.js.map +1 -1
- package/dist/cli/main.js.map +1 -1
- package/dist/cli/slash/index.d.ts +2 -1
- package/dist/cli/slash/index.js +3 -1
- package/dist/cli/slash/index.js.map +1 -1
- package/dist/cli/slash/paste.js +0 -1
- package/dist/cli/slash/paste.js.map +1 -1
- package/dist/cli/slash/provider.d.ts +17 -0
- package/dist/cli/slash/provider.js +101 -0
- package/dist/cli/slash/provider.js.map +1 -0
- package/dist/cli/subcommands/ask.js +14 -3
- package/dist/cli/subcommands/ask.js.map +1 -1
- package/dist/cli/subcommands/audit-codebase.js +15 -3
- package/dist/cli/subcommands/audit-codebase.js.map +1 -1
- package/dist/cli/subcommands/cost.js +4 -4
- package/dist/cli/subcommands/cost.js.map +1 -1
- package/dist/cli/subcommands/skills.js +2 -2
- package/dist/cli/subcommands/skills.js.map +1 -1
- package/dist/cli/submit-paper.js.map +1 -1
- package/dist/context/compaction.js +2 -1
- package/dist/context/compaction.js.map +1 -1
- package/dist/context/knowledge.js +1 -1
- package/dist/context/knowledge.js.map +1 -1
- package/dist/context/memory.js +1 -1
- package/dist/context/memory.js.map +1 -1
- package/dist/fleet/runner.js.map +1 -1
- package/dist/integrations/arniko-client.d.ts +2 -2
- package/dist/integrations/arniko-client.js +4 -4
- package/dist/integrations/arniko-client.js.map +1 -1
- package/dist/integrations/device-auth.js +1 -1
- package/dist/integrations/device-auth.js.map +1 -1
- package/dist/intelligence/prices.d.ts +1 -0
- package/dist/intelligence/prices.js +24 -0
- package/dist/intelligence/prices.js.map +1 -1
- package/dist/kernel/agent-loop.js +14 -2
- package/dist/kernel/agent-loop.js.map +1 -1
- package/dist/kernel/types.d.ts +1 -0
- package/dist/providers/deepseek.d.ts +27 -0
- package/dist/providers/deepseek.js +66 -0
- package/dist/providers/deepseek.js.map +1 -0
- package/dist/providers/dispatch.d.ts +2 -1
- package/dist/providers/dispatch.js +13 -3
- package/dist/providers/dispatch.js.map +1 -1
- package/dist/providers/index.d.ts +2 -0
- package/dist/providers/index.js +3 -0
- package/dist/providers/index.js.map +1 -1
- package/dist/providers/rate-limiter.js +0 -1
- package/dist/providers/rate-limiter.js.map +1 -1
- package/dist/skills/add-provider.md +134 -0
- package/dist/telemetry/sender.js +0 -1
- package/dist/telemetry/sender.js.map +1 -1
- package/dist/tools/browser.js +1 -2
- package/dist/tools/browser.js.map +1 -1
- package/dist/tools/registry.js +1 -1
- package/dist/tools/registry.js.map +1 -1
- package/dist/tui/ink/App.js +73 -3
- package/dist/tui/ink/App.js.map +1 -1
- package/dist/tui/ink/components/DenseToolMessage.d.ts +25 -0
- package/dist/tui/ink/components/DenseToolMessage.js +74 -0
- package/dist/tui/ink/components/DenseToolMessage.js.map +1 -0
- package/dist/tui/ink/components/InputBox.js +20 -1
- package/dist/tui/ink/components/InputBox.js.map +1 -1
- package/dist/tui/ink/components/ModelPicker.d.ts +12 -6
- package/dist/tui/ink/components/ModelPicker.js +70 -34
- package/dist/tui/ink/components/ModelPicker.js.map +1 -1
- package/dist/tui/ink/components/ModelSwitchPrompt.d.ts +21 -0
- package/dist/tui/ink/components/ModelSwitchPrompt.js +19 -0
- package/dist/tui/ink/components/ModelSwitchPrompt.js.map +1 -0
- package/dist/tui/ink/components/StreamingText.d.ts +7 -5
- package/dist/tui/ink/components/StreamingText.js +8 -3
- package/dist/tui/ink/components/StreamingText.js.map +1 -1
- package/dist/tui/ink/components/ToolGroup.d.ts +29 -0
- package/dist/tui/ink/components/ToolGroup.js +89 -0
- package/dist/tui/ink/components/ToolGroup.js.map +1 -0
- package/dist/tui/ink/icons.d.ts +20 -0
- package/dist/tui/ink/icons.js +19 -0
- package/dist/tui/ink/icons.js.map +1 -1
- package/dist/tui/ink/markdown/colorizer.d.ts +25 -0
- package/dist/tui/ink/markdown/colorizer.js +56 -0
- package/dist/tui/ink/markdown/colorizer.js.map +1 -0
- package/dist/tui/ink/markdown/display.d.ts +29 -0
- package/dist/tui/ink/markdown/display.js +50 -0
- package/dist/tui/ink/markdown/display.js.map +1 -0
- package/dist/tui/ink/markdown/index.d.ts +21 -0
- package/dist/tui/ink/markdown/index.js +20 -0
- package/dist/tui/ink/markdown/index.js.map +1 -0
- package/dist/tui/ink/markdown/inline.d.ts +29 -0
- package/dist/tui/ink/markdown/inline.js +105 -0
- package/dist/tui/ink/markdown/inline.js.map +1 -0
- package/dist/tui/ink/markdown/langs/index.d.ts +52 -0
- package/dist/tui/ink/markdown/langs/index.js +476 -0
- package/dist/tui/ink/markdown/langs/index.js.map +1 -0
- package/dist/tui/ink/markdown/langs/types.d.ts +20 -0
- package/dist/tui/ink/markdown/langs/types.js +2 -0
- package/dist/tui/ink/markdown/langs/types.js.map +1 -0
- package/dist/tui/ink/markdown/parser.d.ts +50 -0
- package/dist/tui/ink/markdown/parser.js +179 -0
- package/dist/tui/ink/markdown/parser.js.map +1 -0
- package/dist/tui/ink/markdown/table.d.ts +20 -0
- package/dist/tui/ink/markdown/table.js +54 -0
- package/dist/tui/ink/markdown/table.js.map +1 -0
- package/dist/tui/ink/theme-context.js +1 -1
- package/dist/tui/ink/theme-context.js.map +1 -1
- package/dist/tui/ink/use-event-projection.d.ts +1 -0
- package/dist/tui/ink/use-event-projection.js +6 -1
- package/dist/tui/ink/use-event-projection.js.map +1 -1
- package/dist/tui/theme-loader.js +2 -2
- package/dist/tui/theme-loader.js.map +1 -1
- package/dist/tui/theme.d.ts +55 -2
- package/dist/tui/theme.js +74 -14
- package/dist/tui/theme.js.map +1 -1
- package/package.json +4 -4
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
2
|
+
import { Text } from 'ink';
|
|
3
|
+
const BOLD_LEN = 2;
|
|
4
|
+
const ITALIC_LEN = 1;
|
|
5
|
+
const STRIKE_LEN = 2;
|
|
6
|
+
const CODE_LEN = 1;
|
|
7
|
+
const U_OPEN_LEN = 3; // <u>
|
|
8
|
+
const U_CLOSE_LEN = 4; // </u>
|
|
9
|
+
// Same alternation as gemini's parser. Order matters: ***foo*** must match
|
|
10
|
+
// before **foo** before *foo* / _foo_ / ~~foo~~ / [text](url) / `code` /
|
|
11
|
+
// <u>...</u> / bare URLs.
|
|
12
|
+
const INLINE_REGEX = /(\*\*\*.*?\*\*\*|\*\*.*?\*\*|\*.*?\*|_.*?_|~~.*?~~|\[.*?\]\(.*?\)|`+.+?`+|<u>.*?<\/u>|https?:\/\/\S+)/g;
|
|
13
|
+
/** Render one or more lines of markdown text with inline emphasis. */
|
|
14
|
+
export function RenderInline(props) {
|
|
15
|
+
const { text, palette, baseColor, keyPrefix = 'i' } = props;
|
|
16
|
+
const fg = baseColor ?? palette.text.primary;
|
|
17
|
+
// Fast path — no markdown markers anywhere.
|
|
18
|
+
if (!/[*_~`<[]|https?:/.test(text)) {
|
|
19
|
+
return _jsx(Text, { color: fg, children: text });
|
|
20
|
+
}
|
|
21
|
+
const out = [];
|
|
22
|
+
let lastIdx = 0;
|
|
23
|
+
let match;
|
|
24
|
+
let segIdx = 0;
|
|
25
|
+
// Reset regex state on each call (regex is module-level for perf, but
|
|
26
|
+
// RegExp.prototype.lastIndex persists across invocations).
|
|
27
|
+
INLINE_REGEX.lastIndex = 0;
|
|
28
|
+
while ((match = INLINE_REGEX.exec(text)) !== null) {
|
|
29
|
+
if (match.index > lastIdx) {
|
|
30
|
+
const plain = text.slice(lastIdx, match.index);
|
|
31
|
+
out.push(_jsx(Text, { color: fg, children: plain }, `${keyPrefix}-${segIdx++}`));
|
|
32
|
+
}
|
|
33
|
+
const full = match[0];
|
|
34
|
+
const node = renderSegment(full, palette, fg, `${keyPrefix}-${segIdx}`, text, match.index, INLINE_REGEX.lastIndex);
|
|
35
|
+
out.push(node);
|
|
36
|
+
segIdx += 1;
|
|
37
|
+
lastIdx = INLINE_REGEX.lastIndex;
|
|
38
|
+
}
|
|
39
|
+
if (lastIdx < text.length) {
|
|
40
|
+
out.push(_jsx(Text, { color: fg, children: text.slice(lastIdx) }, `${keyPrefix}-${segIdx}`));
|
|
41
|
+
}
|
|
42
|
+
return _jsx(Text, { children: out });
|
|
43
|
+
}
|
|
44
|
+
function renderSegment(full, palette, fg, key,
|
|
45
|
+
/** Source text, used for italic boundary heuristics. */
|
|
46
|
+
src, matchStart, matchEnd) {
|
|
47
|
+
// ***bold-italic***
|
|
48
|
+
if (full.startsWith('***') &&
|
|
49
|
+
full.endsWith('***') &&
|
|
50
|
+
full.length > (BOLD_LEN + ITALIC_LEN) * 2) {
|
|
51
|
+
const inner = full.slice(BOLD_LEN + ITALIC_LEN, -BOLD_LEN - ITALIC_LEN);
|
|
52
|
+
return (_jsx(Text, { bold: true, italic: true, children: _jsx(RenderInline, { text: inner, palette: palette, baseColor: fg, keyPrefix: `${key}b` }) }, key));
|
|
53
|
+
}
|
|
54
|
+
// **bold**
|
|
55
|
+
if (full.startsWith('**') && full.endsWith('**') && full.length > BOLD_LEN * 2) {
|
|
56
|
+
const inner = full.slice(BOLD_LEN, -BOLD_LEN);
|
|
57
|
+
return (_jsx(Text, { bold: true, children: _jsx(RenderInline, { text: inner, palette: palette, baseColor: fg, keyPrefix: `${key}b` }) }, key));
|
|
58
|
+
}
|
|
59
|
+
// *italic* / _italic_ — boundary heuristic from gemini: don't match if
|
|
60
|
+
// adjacent to word chars, and don't match path-like patterns.
|
|
61
|
+
if (full.length > ITALIC_LEN * 2 &&
|
|
62
|
+
((full.startsWith('*') && full.endsWith('*')) ||
|
|
63
|
+
(full.startsWith('_') && full.endsWith('_'))) &&
|
|
64
|
+
!/\w/.test(src.substring(matchStart - 1, matchStart)) &&
|
|
65
|
+
!/\w/.test(src.substring(matchEnd, matchEnd + 1)) &&
|
|
66
|
+
!/\S[./\\]/.test(src.substring(matchStart - 2, matchStart)) &&
|
|
67
|
+
!/[./\\]\S/.test(src.substring(matchEnd, matchEnd + 2))) {
|
|
68
|
+
const inner = full.slice(ITALIC_LEN, -ITALIC_LEN);
|
|
69
|
+
return (_jsx(Text, { italic: true, children: _jsx(RenderInline, { text: inner, palette: palette, baseColor: fg, keyPrefix: `${key}i` }) }, key));
|
|
70
|
+
}
|
|
71
|
+
// ~~strikethrough~~
|
|
72
|
+
if (full.startsWith('~~') && full.endsWith('~~') && full.length > STRIKE_LEN * 2) {
|
|
73
|
+
const inner = full.slice(STRIKE_LEN, -STRIKE_LEN);
|
|
74
|
+
return (_jsx(Text, { strikethrough: true, children: _jsx(RenderInline, { text: inner, palette: palette, baseColor: fg, keyPrefix: `${key}s` }) }, key));
|
|
75
|
+
}
|
|
76
|
+
// `inline code` — supports n-tick fences (`` x ``) by matching a balanced run.
|
|
77
|
+
if (full.startsWith('`') && full.endsWith('`') && full.length > CODE_LEN) {
|
|
78
|
+
const m = full.match(/^(`+)(.+?)\1$/s);
|
|
79
|
+
if (m && m[2]) {
|
|
80
|
+
return (_jsx(Text, { color: palette.text.accent, children: m[2] }, key));
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
// [text](url)
|
|
84
|
+
if (full.startsWith('[') && full.includes('](') && full.endsWith(')')) {
|
|
85
|
+
const m = full.match(/\[(.*?)\]\((.*?)\)/);
|
|
86
|
+
if (m) {
|
|
87
|
+
const [, linkText, url] = m;
|
|
88
|
+
return (_jsxs(Text, { children: [_jsx(RenderInline, { text: linkText, palette: palette, baseColor: fg, keyPrefix: `${key}l` }), _jsx(Text, { color: fg, children: " (" }), _jsx(Text, { color: palette.text.link, underline: true, children: url }), _jsx(Text, { color: fg, children: ")" })] }, key));
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
// <u>underline</u>
|
|
92
|
+
if (full.startsWith('<u>') &&
|
|
93
|
+
full.endsWith('</u>') &&
|
|
94
|
+
full.length > U_OPEN_LEN + U_CLOSE_LEN - 1) {
|
|
95
|
+
const inner = full.slice(U_OPEN_LEN, -U_CLOSE_LEN);
|
|
96
|
+
return (_jsx(Text, { underline: true, children: _jsx(RenderInline, { text: inner, palette: palette, baseColor: fg, keyPrefix: `${key}u` }) }, key));
|
|
97
|
+
}
|
|
98
|
+
// bare URL
|
|
99
|
+
if (/^https?:\/\//.test(full)) {
|
|
100
|
+
return (_jsx(Text, { color: palette.text.link, underline: true, children: full }, key));
|
|
101
|
+
}
|
|
102
|
+
// Fallback — render raw.
|
|
103
|
+
return (_jsx(Text, { color: fg, children: full }, key));
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=inline.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inline.js","sourceRoot":"","sources":["../../../../src/tui/ink/markdown/inline.tsx"],"names":[],"mappings":";AAkBA,OAAO,EAAE,IAAI,EAAE,MAAM,KAAK,CAAC;AAG3B,MAAM,QAAQ,GAAG,CAAC,CAAC;AACnB,MAAM,UAAU,GAAG,CAAC,CAAC;AACrB,MAAM,UAAU,GAAG,CAAC,CAAC;AACrB,MAAM,QAAQ,GAAG,CAAC,CAAC;AACnB,MAAM,UAAU,GAAG,CAAC,CAAC,CAAG,MAAM;AAC9B,MAAM,WAAW,GAAG,CAAC,CAAC,CAAE,OAAO;AAE/B,2EAA2E;AAC3E,yEAAyE;AACzE,0BAA0B;AAC1B,MAAM,YAAY,GAChB,wGAAwG,CAAC;AAW3G,sEAAsE;AACtE,MAAM,UAAU,YAAY,CAAC,KAAkB;IAC7C,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,GAAG,GAAG,EAAE,GAAG,KAAK,CAAC;IAC5D,MAAM,EAAE,GAAG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;IAE7C,4CAA4C;IAC5C,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACnC,OAAO,KAAC,IAAI,IAAC,KAAK,EAAE,EAAE,YAAG,IAAI,GAAQ,CAAC;IACxC,CAAC;IAED,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,KAA6B,CAAC;IAClC,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,sEAAsE;IACtE,2DAA2D;IAC3D,YAAY,CAAC,SAAS,GAAG,CAAC,CAAC;IAE3B,OAAO,CAAC,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAClD,IAAI,KAAK,CAAC,KAAK,GAAG,OAAO,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;YAC/C,GAAG,CAAC,IAAI,CACN,KAAC,IAAI,IAAkC,KAAK,EAAE,EAAE,YAC7C,KAAK,IADG,GAAG,SAAS,IAAI,MAAM,EAAE,EAAE,CAE9B,CACR,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG,SAAS,IAAI,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC;QACnH,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,MAAM,IAAI,CAAC,CAAC;QAEZ,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC;IACnC,CAAC;IAED,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC1B,GAAG,CAAC,IAAI,CACN,KAAC,IAAI,IAAgC,KAAK,EAAE,EAAE,YAC3C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IADX,GAAG,SAAS,IAAI,MAAM,EAAE,CAE5B,CACR,CAAC;IACJ,CAAC;IAED,OAAO,KAAC,IAAI,cAAE,GAAG,GAAQ,CAAC;AAC5B,CAAC;AAED,SAAS,aAAa,CACpB,IAAY,EACZ,OAAgB,EAChB,EAAU,EACV,GAAW;AACX,wDAAwD;AACxD,GAAW,EACX,UAAkB,EAClB,QAAgB;IAEhB,oBAAoB;IACpB,IACE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,CAAC,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,EACzC,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,UAAU,EAAE,CAAC,QAAQ,GAAG,UAAU,CAAC,CAAC;QACxE,OAAO,CACL,KAAC,IAAI,IAAW,IAAI,QAAC,MAAM,kBACzB,KAAC,YAAY,IAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,GAAG,GAAG,GAAI,IAD3E,GAAG,CAEP,CACR,CAAC;IACJ,CAAC;IAED,WAAW;IACX,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,QAAQ,GAAG,CAAC,EAAE,CAAC;QAC/E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC9C,OAAO,CACL,KAAC,IAAI,IAAW,IAAI,kBAClB,KAAC,YAAY,IAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,GAAG,GAAG,GAAI,IAD3E,GAAG,CAEP,CACR,CAAC;IACJ,CAAC;IAED,wEAAwE;IACxE,8DAA8D;IAC9D,IACE,IAAI,CAAC,MAAM,GAAG,UAAU,GAAG,CAAC;QAC5B,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;YAC3C,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/C,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,UAAU,GAAG,CAAC,EAAE,UAAU,CAAC,CAAC;QACrD,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;QACjD,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,UAAU,GAAG,CAAC,EAAE,UAAU,CAAC,CAAC;QAC3D,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,EACvD,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,UAAU,CAAC,CAAC;QAClD,OAAO,CACL,KAAC,IAAI,IAAW,MAAM,kBACpB,KAAC,YAAY,IAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,GAAG,GAAG,GAAI,IAD3E,GAAG,CAEP,CACR,CAAC;IACJ,CAAC;IAED,oBAAoB;IACpB,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;QACjF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,UAAU,CAAC,CAAC;QAClD,OAAO,CACL,KAAC,IAAI,IAAW,aAAa,kBAC3B,KAAC,YAAY,IAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,GAAG,GAAG,GAAI,IAD3E,GAAG,CAEP,CACR,CAAC;IACJ,CAAC;IAED,+EAA+E;IAC/E,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;QACzE,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACd,OAAO,CACL,KAAC,IAAI,IAAW,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,MAAM,YACvC,CAAC,CAAC,CAAC,CAAC,IADI,GAAG,CAEP,CACR,CAAC;QACJ,CAAC;IACH,CAAC;IAED,cAAc;IACd,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACtE,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QAC3C,IAAI,CAAC,EAAE,CAAC;YACN,MAAM,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,OAAO,CACL,MAAC,IAAI,eACH,KAAC,YAAY,IAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,GAAG,GAAG,GAAI,EACvF,KAAC,IAAI,IAAC,KAAK,EAAE,EAAE,mBAAW,EAC1B,KAAC,IAAI,IAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,kBACtC,GAAG,GACC,EACP,KAAC,IAAI,IAAC,KAAK,EAAE,EAAE,kBAAU,KANhB,GAAG,CAOP,CACR,CAAC;QACJ,CAAC;IACH,CAAC;IAED,mBAAmB;IACnB,IACE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,UAAU,GAAG,WAAW,GAAG,CAAC,EAC1C,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,WAAW,CAAC,CAAC;QACnD,OAAO,CACL,KAAC,IAAI,IAAW,SAAS,kBACvB,KAAC,YAAY,IAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,SAAS,EAAE,GAAG,GAAG,GAAG,GAAI,IAD3E,GAAG,CAEP,CACR,CAAC;IACJ,CAAC;IAED,WAAW;IACX,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,OAAO,CACL,KAAC,IAAI,IAAW,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,kBAChD,IAAI,IADI,GAAG,CAEP,CACR,CAAC;IACJ,CAAC;IAED,yBAAyB;IACzB,OAAO,CACL,KAAC,IAAI,IAAW,KAAK,EAAE,EAAE,YACtB,IAAI,IADI,GAAG,CAEP,CACR,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Native syntax highlighter — language registry + generic tokenizer.
|
|
3
|
+
*
|
|
4
|
+
* Replaces the `lowlight` dependency that gemini-cli pulls in. We accept
|
|
5
|
+
* less-than-perfect syntax fidelity in exchange for zero runtime deps.
|
|
6
|
+
* Each language contributes a small declarative spec (keywords, types,
|
|
7
|
+
* comment shape, string delimiters); the shared lexer below walks the
|
|
8
|
+
* source classifying chunks. Coverage targets the languages users most
|
|
9
|
+
* frequently see in agent output: ts/tsx/js/jsx/py/sh/bash/go/rust/json/
|
|
10
|
+
* yaml/md/diff/sql/html/css/c/cpp/java/ruby/php/rb.
|
|
11
|
+
*
|
|
12
|
+
* Rough equivalence to lowlight's class names:
|
|
13
|
+
* keyword → hljs-keyword
|
|
14
|
+
* string → hljs-string
|
|
15
|
+
* number → hljs-number
|
|
16
|
+
* comment → hljs-comment
|
|
17
|
+
* type → hljs-type / hljs-title
|
|
18
|
+
* builtin → hljs-built_in
|
|
19
|
+
* operator → hljs-operator
|
|
20
|
+
* tag → hljs-name
|
|
21
|
+
* attr → hljs-attr
|
|
22
|
+
* meta → hljs-meta
|
|
23
|
+
* punct → (default)
|
|
24
|
+
* addition / deletion → diff prefixes
|
|
25
|
+
*/
|
|
26
|
+
import type { Token } from './types.js';
|
|
27
|
+
export type { Token, TokenKind, Tokenizer } from './types.js';
|
|
28
|
+
interface LangSpec {
|
|
29
|
+
keywords: ReadonlySet<string>;
|
|
30
|
+
types?: ReadonlySet<string>;
|
|
31
|
+
builtins?: ReadonlySet<string>;
|
|
32
|
+
/** Line-comment prefix(es). Order them so longer prefixes come first. */
|
|
33
|
+
lineComment?: readonly string[];
|
|
34
|
+
/** Block comment open + close. */
|
|
35
|
+
blockComment?: readonly [string, string];
|
|
36
|
+
/** Quote chars for string literals. */
|
|
37
|
+
stringQuotes?: readonly string[];
|
|
38
|
+
/** Backtick template strings (ts/js). */
|
|
39
|
+
templateLiteral?: boolean;
|
|
40
|
+
/** Operator chars (single-char-classified). */
|
|
41
|
+
operatorChars?: string;
|
|
42
|
+
/** Whether to treat capitalised identifiers as `type` tokens (TS/Java/Rust). */
|
|
43
|
+
capitalIsType?: boolean;
|
|
44
|
+
/** Whether identifiers prefixed with `@` are decorators/meta (Py/TS/Java). */
|
|
45
|
+
atIsMeta?: boolean;
|
|
46
|
+
/** Whether `#` starts a line comment outside string context (Py/Rb/Sh). */
|
|
47
|
+
hashIsComment?: boolean;
|
|
48
|
+
}
|
|
49
|
+
declare function resolveLang(lang: string | null | undefined): LangSpec | null;
|
|
50
|
+
/** Public entry point. Returns a single plain-text token if the lang is unknown. */
|
|
51
|
+
export declare function tokenize(src: string, lang: string | null | undefined): Token[];
|
|
52
|
+
export { resolveLang };
|
|
@@ -0,0 +1,476 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Native syntax highlighter — language registry + generic tokenizer.
|
|
3
|
+
*
|
|
4
|
+
* Replaces the `lowlight` dependency that gemini-cli pulls in. We accept
|
|
5
|
+
* less-than-perfect syntax fidelity in exchange for zero runtime deps.
|
|
6
|
+
* Each language contributes a small declarative spec (keywords, types,
|
|
7
|
+
* comment shape, string delimiters); the shared lexer below walks the
|
|
8
|
+
* source classifying chunks. Coverage targets the languages users most
|
|
9
|
+
* frequently see in agent output: ts/tsx/js/jsx/py/sh/bash/go/rust/json/
|
|
10
|
+
* yaml/md/diff/sql/html/css/c/cpp/java/ruby/php/rb.
|
|
11
|
+
*
|
|
12
|
+
* Rough equivalence to lowlight's class names:
|
|
13
|
+
* keyword → hljs-keyword
|
|
14
|
+
* string → hljs-string
|
|
15
|
+
* number → hljs-number
|
|
16
|
+
* comment → hljs-comment
|
|
17
|
+
* type → hljs-type / hljs-title
|
|
18
|
+
* builtin → hljs-built_in
|
|
19
|
+
* operator → hljs-operator
|
|
20
|
+
* tag → hljs-name
|
|
21
|
+
* attr → hljs-attr
|
|
22
|
+
* meta → hljs-meta
|
|
23
|
+
* punct → (default)
|
|
24
|
+
* addition / deletion → diff prefixes
|
|
25
|
+
*/
|
|
26
|
+
const TS_KEYWORDS = new Set([
|
|
27
|
+
'abstract', 'as', 'async', 'await', 'break', 'case', 'catch', 'class',
|
|
28
|
+
'const', 'continue', 'debugger', 'declare', 'default', 'delete', 'do',
|
|
29
|
+
'else', 'enum', 'export', 'extends', 'false', 'finally', 'for', 'from',
|
|
30
|
+
'function', 'get', 'if', 'implements', 'import', 'in', 'infer', 'instanceof',
|
|
31
|
+
'interface', 'is', 'keyof', 'let', 'namespace', 'new', 'null', 'of',
|
|
32
|
+
'package', 'private', 'protected', 'public', 'readonly', 'require', 'return',
|
|
33
|
+
'satisfies', 'set', 'static', 'super', 'switch', 'this', 'throw', 'true',
|
|
34
|
+
'try', 'type', 'typeof', 'undefined', 'var', 'void', 'while', 'with', 'yield',
|
|
35
|
+
]);
|
|
36
|
+
const TS_TYPES = new Set([
|
|
37
|
+
'any', 'bigint', 'boolean', 'never', 'number', 'object', 'string', 'symbol',
|
|
38
|
+
'unknown', 'Array', 'Map', 'Set', 'Promise', 'Record', 'Partial', 'Readonly',
|
|
39
|
+
'Pick', 'Omit', 'Required', 'Awaited', 'ReturnType', 'Parameters',
|
|
40
|
+
]);
|
|
41
|
+
const TS_BUILTINS = new Set([
|
|
42
|
+
'console', 'process', 'globalThis', 'JSON', 'Math', 'Date', 'Error',
|
|
43
|
+
'RegExp', 'Object', 'Number', 'String', 'Boolean', 'Symbol',
|
|
44
|
+
]);
|
|
45
|
+
const PY_KEYWORDS = new Set([
|
|
46
|
+
'False', 'None', 'True', 'and', 'as', 'assert', 'async', 'await', 'break',
|
|
47
|
+
'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'finally',
|
|
48
|
+
'for', 'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'nonlocal',
|
|
49
|
+
'not', 'or', 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield',
|
|
50
|
+
'match', 'case',
|
|
51
|
+
]);
|
|
52
|
+
const PY_BUILTINS = new Set([
|
|
53
|
+
'abs', 'all', 'any', 'bin', 'bool', 'bytes', 'callable', 'chr', 'dict',
|
|
54
|
+
'dir', 'enumerate', 'eval', 'exec', 'filter', 'float', 'format', 'frozenset',
|
|
55
|
+
'getattr', 'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'isinstance',
|
|
56
|
+
'issubclass', 'iter', 'len', 'list', 'map', 'max', 'min', 'next', 'object',
|
|
57
|
+
'oct', 'open', 'ord', 'pow', 'print', 'property', 'range', 'repr', 'reversed',
|
|
58
|
+
'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod', 'str', 'sum',
|
|
59
|
+
'super', 'tuple', 'type', 'vars', 'zip', 'self', 'cls', '__init__', '__main__',
|
|
60
|
+
]);
|
|
61
|
+
const SH_KEYWORDS = new Set([
|
|
62
|
+
'if', 'then', 'else', 'elif', 'fi', 'case', 'esac', 'for', 'select', 'while',
|
|
63
|
+
'until', 'do', 'done', 'in', 'function', 'time', 'coproc', 'return', 'exit',
|
|
64
|
+
'break', 'continue', 'export', 'local', 'declare', 'readonly', 'unset',
|
|
65
|
+
'shift', 'set', 'source',
|
|
66
|
+
]);
|
|
67
|
+
const SH_BUILTINS = new Set([
|
|
68
|
+
'echo', 'printf', 'cd', 'pwd', 'ls', 'cat', 'mv', 'cp', 'rm', 'mkdir',
|
|
69
|
+
'rmdir', 'touch', 'chmod', 'chown', 'grep', 'sed', 'awk', 'find', 'xargs',
|
|
70
|
+
'head', 'tail', 'sort', 'uniq', 'wc', 'curl', 'wget', 'git', 'npm', 'node',
|
|
71
|
+
'python', 'python3', 'pip', 'docker', 'kubectl', 'tmux',
|
|
72
|
+
]);
|
|
73
|
+
const GO_KEYWORDS = new Set([
|
|
74
|
+
'break', 'case', 'chan', 'const', 'continue', 'default', 'defer', 'else',
|
|
75
|
+
'fallthrough', 'for', 'func', 'go', 'goto', 'if', 'import', 'interface',
|
|
76
|
+
'map', 'package', 'range', 'return', 'select', 'struct', 'switch', 'type',
|
|
77
|
+
'var', 'true', 'false', 'nil', 'iota',
|
|
78
|
+
]);
|
|
79
|
+
const GO_TYPES = new Set([
|
|
80
|
+
'bool', 'byte', 'complex64', 'complex128', 'error', 'float32', 'float64',
|
|
81
|
+
'int', 'int8', 'int16', 'int32', 'int64', 'rune', 'string', 'uint', 'uint8',
|
|
82
|
+
'uint16', 'uint32', 'uint64', 'uintptr',
|
|
83
|
+
]);
|
|
84
|
+
const RUST_KEYWORDS = new Set([
|
|
85
|
+
'as', 'async', 'await', 'break', 'const', 'continue', 'crate', 'dyn', 'else',
|
|
86
|
+
'enum', 'extern', 'false', 'fn', 'for', 'if', 'impl', 'in', 'let', 'loop',
|
|
87
|
+
'match', 'mod', 'move', 'mut', 'pub', 'ref', 'return', 'self', 'Self',
|
|
88
|
+
'static', 'struct', 'super', 'trait', 'true', 'type', 'unsafe', 'use',
|
|
89
|
+
'where', 'while', 'box', 'do', 'final', 'macro', 'override', 'priv',
|
|
90
|
+
'typeof', 'unsized', 'virtual', 'yield',
|
|
91
|
+
]);
|
|
92
|
+
const RUST_TYPES = new Set([
|
|
93
|
+
'i8', 'i16', 'i32', 'i64', 'i128', 'isize', 'u8', 'u16', 'u32', 'u64',
|
|
94
|
+
'u128', 'usize', 'f32', 'f64', 'bool', 'char', 'str', 'String', 'Vec',
|
|
95
|
+
'Option', 'Result', 'Box', 'Rc', 'Arc', 'HashMap', 'HashSet',
|
|
96
|
+
]);
|
|
97
|
+
const SQL_KEYWORDS = new Set([
|
|
98
|
+
'select', 'from', 'where', 'and', 'or', 'not', 'null', 'is', 'in', 'like',
|
|
99
|
+
'between', 'order', 'by', 'group', 'having', 'limit', 'offset', 'insert',
|
|
100
|
+
'into', 'values', 'update', 'set', 'delete', 'create', 'table', 'drop',
|
|
101
|
+
'alter', 'add', 'column', 'primary', 'key', 'foreign', 'references',
|
|
102
|
+
'unique', 'index', 'on', 'as', 'distinct', 'count', 'sum', 'avg', 'min',
|
|
103
|
+
'max', 'case', 'when', 'then', 'else', 'end', 'union', 'all', 'inner',
|
|
104
|
+
'left', 'right', 'outer', 'join', 'cross', 'with', 'as',
|
|
105
|
+
]);
|
|
106
|
+
const CSS_KEYWORDS = new Set([
|
|
107
|
+
'@media', '@import', '@keyframes', '@font-face', '@supports', '@charset',
|
|
108
|
+
'@namespace', '@page', '@layer', '@container',
|
|
109
|
+
]);
|
|
110
|
+
const SPECS = {
|
|
111
|
+
ts: {
|
|
112
|
+
keywords: TS_KEYWORDS,
|
|
113
|
+
types: TS_TYPES,
|
|
114
|
+
builtins: TS_BUILTINS,
|
|
115
|
+
lineComment: ['//'],
|
|
116
|
+
blockComment: ['/*', '*/'],
|
|
117
|
+
stringQuotes: ['"', "'"],
|
|
118
|
+
templateLiteral: true,
|
|
119
|
+
operatorChars: '+-*/%=<>!&|^~?:',
|
|
120
|
+
capitalIsType: true,
|
|
121
|
+
atIsMeta: true,
|
|
122
|
+
},
|
|
123
|
+
js: {
|
|
124
|
+
keywords: TS_KEYWORDS,
|
|
125
|
+
builtins: TS_BUILTINS,
|
|
126
|
+
lineComment: ['//'],
|
|
127
|
+
blockComment: ['/*', '*/'],
|
|
128
|
+
stringQuotes: ['"', "'"],
|
|
129
|
+
templateLiteral: true,
|
|
130
|
+
operatorChars: '+-*/%=<>!&|^~?:',
|
|
131
|
+
atIsMeta: true,
|
|
132
|
+
},
|
|
133
|
+
py: {
|
|
134
|
+
keywords: PY_KEYWORDS,
|
|
135
|
+
builtins: PY_BUILTINS,
|
|
136
|
+
stringQuotes: ['"', "'"],
|
|
137
|
+
operatorChars: '+-*/%=<>!&|^~',
|
|
138
|
+
capitalIsType: true,
|
|
139
|
+
atIsMeta: true,
|
|
140
|
+
hashIsComment: true,
|
|
141
|
+
},
|
|
142
|
+
sh: {
|
|
143
|
+
keywords: SH_KEYWORDS,
|
|
144
|
+
builtins: SH_BUILTINS,
|
|
145
|
+
stringQuotes: ['"', "'"],
|
|
146
|
+
operatorChars: '|&<>=',
|
|
147
|
+
hashIsComment: true,
|
|
148
|
+
},
|
|
149
|
+
go: {
|
|
150
|
+
keywords: GO_KEYWORDS,
|
|
151
|
+
types: GO_TYPES,
|
|
152
|
+
lineComment: ['//'],
|
|
153
|
+
blockComment: ['/*', '*/'],
|
|
154
|
+
stringQuotes: ['"', '`'],
|
|
155
|
+
operatorChars: '+-*/%=<>!&|^~:',
|
|
156
|
+
},
|
|
157
|
+
rust: {
|
|
158
|
+
keywords: RUST_KEYWORDS,
|
|
159
|
+
types: RUST_TYPES,
|
|
160
|
+
lineComment: ['//'],
|
|
161
|
+
blockComment: ['/*', '*/'],
|
|
162
|
+
stringQuotes: ['"'],
|
|
163
|
+
operatorChars: '+-*/%=<>!&|^~?:',
|
|
164
|
+
capitalIsType: true,
|
|
165
|
+
atIsMeta: true,
|
|
166
|
+
},
|
|
167
|
+
json: {
|
|
168
|
+
keywords: new Set(['true', 'false', 'null']),
|
|
169
|
+
stringQuotes: ['"'],
|
|
170
|
+
operatorChars: ':,',
|
|
171
|
+
},
|
|
172
|
+
yaml: {
|
|
173
|
+
keywords: new Set(['true', 'false', 'null', 'yes', 'no', 'on', 'off']),
|
|
174
|
+
stringQuotes: ['"', "'"],
|
|
175
|
+
operatorChars: ':-',
|
|
176
|
+
hashIsComment: true,
|
|
177
|
+
},
|
|
178
|
+
sql: {
|
|
179
|
+
keywords: SQL_KEYWORDS,
|
|
180
|
+
lineComment: ['--'],
|
|
181
|
+
blockComment: ['/*', '*/'],
|
|
182
|
+
stringQuotes: ["'"],
|
|
183
|
+
operatorChars: '+-*/%=<>!',
|
|
184
|
+
},
|
|
185
|
+
css: {
|
|
186
|
+
keywords: CSS_KEYWORDS,
|
|
187
|
+
blockComment: ['/*', '*/'],
|
|
188
|
+
stringQuotes: ['"', "'"],
|
|
189
|
+
operatorChars: ':;,',
|
|
190
|
+
},
|
|
191
|
+
};
|
|
192
|
+
// Aliases — so users can fence ```typescript or ```javascript etc.
|
|
193
|
+
const ALIASES = {
|
|
194
|
+
typescript: 'ts',
|
|
195
|
+
tsx: 'ts',
|
|
196
|
+
javascript: 'js',
|
|
197
|
+
jsx: 'js',
|
|
198
|
+
python: 'py',
|
|
199
|
+
python3: 'py',
|
|
200
|
+
bash: 'sh',
|
|
201
|
+
zsh: 'sh',
|
|
202
|
+
shell: 'sh',
|
|
203
|
+
console: 'sh',
|
|
204
|
+
golang: 'go',
|
|
205
|
+
rs: 'rust',
|
|
206
|
+
yml: 'yaml',
|
|
207
|
+
postgresql: 'sql',
|
|
208
|
+
mysql: 'sql',
|
|
209
|
+
scss: 'css',
|
|
210
|
+
sass: 'css',
|
|
211
|
+
};
|
|
212
|
+
function resolveLang(lang) {
|
|
213
|
+
if (!lang)
|
|
214
|
+
return null;
|
|
215
|
+
const key = lang.toLowerCase();
|
|
216
|
+
const resolved = ALIASES[key] ?? key;
|
|
217
|
+
return SPECS[resolved] ?? null;
|
|
218
|
+
}
|
|
219
|
+
/** Diff has its own dedicated tokenizer because lines have meaning, not chars. */
|
|
220
|
+
function tokenizeDiff(src) {
|
|
221
|
+
const out = [];
|
|
222
|
+
for (const line of src.split('\n')) {
|
|
223
|
+
if (line.startsWith('+') && !line.startsWith('+++')) {
|
|
224
|
+
out.push({ kind: 'addition', value: line + '\n' });
|
|
225
|
+
}
|
|
226
|
+
else if (line.startsWith('-') && !line.startsWith('---')) {
|
|
227
|
+
out.push({ kind: 'deletion', value: line + '\n' });
|
|
228
|
+
}
|
|
229
|
+
else if (line.startsWith('@@')) {
|
|
230
|
+
out.push({ kind: 'meta', value: line + '\n' });
|
|
231
|
+
}
|
|
232
|
+
else if (line.startsWith('+++') || line.startsWith('---')) {
|
|
233
|
+
out.push({ kind: 'meta', value: line + '\n' });
|
|
234
|
+
}
|
|
235
|
+
else if (line.startsWith('diff ') || line.startsWith('index ')) {
|
|
236
|
+
out.push({ kind: 'comment', value: line + '\n' });
|
|
237
|
+
}
|
|
238
|
+
else {
|
|
239
|
+
out.push({ kind: 'plain', value: line + '\n' });
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
return out;
|
|
243
|
+
}
|
|
244
|
+
/** HTML/XML — angle-tag aware. */
|
|
245
|
+
function tokenizeHtml(src) {
|
|
246
|
+
const out = [];
|
|
247
|
+
// Greedy split into tags / text.
|
|
248
|
+
const re = /(<!--[\s\S]*?-->|<[^>]+>|[^<]+)/g;
|
|
249
|
+
let m;
|
|
250
|
+
while ((m = re.exec(src)) !== null) {
|
|
251
|
+
const chunk = m[0];
|
|
252
|
+
if (chunk.startsWith('<!--')) {
|
|
253
|
+
out.push({ kind: 'comment', value: chunk });
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
if (chunk.startsWith('<')) {
|
|
257
|
+
// Split into <, tagname, attr=, value, >
|
|
258
|
+
const tagInner = chunk.slice(1, -1);
|
|
259
|
+
out.push({ kind: 'punct', value: '<' });
|
|
260
|
+
const tagM = tagInner.match(/^\/?([\w:-]+)/);
|
|
261
|
+
if (tagM) {
|
|
262
|
+
out.push({ kind: 'tag', value: tagM[0] });
|
|
263
|
+
const rest = tagInner.slice(tagM[0].length);
|
|
264
|
+
// Attrs.
|
|
265
|
+
const attrRe = /\s+([\w:-]+)(?:\s*=\s*("[^"]*"|'[^']*'|\S+))?/g;
|
|
266
|
+
let am;
|
|
267
|
+
let lastIdx = 0;
|
|
268
|
+
while ((am = attrRe.exec(rest)) !== null) {
|
|
269
|
+
if (am.index > lastIdx)
|
|
270
|
+
out.push({ kind: 'plain', value: rest.slice(lastIdx, am.index) });
|
|
271
|
+
out.push({ kind: 'plain', value: ' ' });
|
|
272
|
+
out.push({ kind: 'attr', value: am[1] });
|
|
273
|
+
if (am[2]) {
|
|
274
|
+
out.push({ kind: 'punct', value: '=' });
|
|
275
|
+
out.push({ kind: 'string', value: am[2] });
|
|
276
|
+
}
|
|
277
|
+
lastIdx = attrRe.lastIndex;
|
|
278
|
+
}
|
|
279
|
+
if (lastIdx < rest.length)
|
|
280
|
+
out.push({ kind: 'plain', value: rest.slice(lastIdx) });
|
|
281
|
+
}
|
|
282
|
+
else {
|
|
283
|
+
out.push({ kind: 'plain', value: tagInner });
|
|
284
|
+
}
|
|
285
|
+
out.push({ kind: 'punct', value: '>' });
|
|
286
|
+
}
|
|
287
|
+
else {
|
|
288
|
+
out.push({ kind: 'plain', value: chunk });
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
return out;
|
|
292
|
+
}
|
|
293
|
+
const IDENT_START = /[A-Za-z_$]/;
|
|
294
|
+
const IDENT_CONT = /[A-Za-z0-9_$]/;
|
|
295
|
+
const NUMBER_RE = /^(?:0x[\da-fA-F_]+|0b[01_]+|0o[0-7_]+|\d[\d_]*(?:\.\d[\d_]*)?(?:[eE][+-]?\d+)?)/;
|
|
296
|
+
/** Generic single-pass lexer driven by LangSpec. */
|
|
297
|
+
function tokenizeGeneric(src, spec) {
|
|
298
|
+
const out = [];
|
|
299
|
+
const len = src.length;
|
|
300
|
+
let i = 0;
|
|
301
|
+
let plainStart = 0;
|
|
302
|
+
const flushPlain = () => {
|
|
303
|
+
if (i > plainStart) {
|
|
304
|
+
out.push({ kind: 'plain', value: src.slice(plainStart, i) });
|
|
305
|
+
}
|
|
306
|
+
};
|
|
307
|
+
while (i < len) {
|
|
308
|
+
const ch = src[i];
|
|
309
|
+
// Whitespace — accumulate as plain.
|
|
310
|
+
if (ch === ' ' || ch === '\t' || ch === '\n' || ch === '\r') {
|
|
311
|
+
i += 1;
|
|
312
|
+
continue;
|
|
313
|
+
}
|
|
314
|
+
// Line comment.
|
|
315
|
+
if (spec.lineComment) {
|
|
316
|
+
const lc = spec.lineComment.find(p => src.startsWith(p, i));
|
|
317
|
+
if (lc) {
|
|
318
|
+
flushPlain();
|
|
319
|
+
const end = src.indexOf('\n', i);
|
|
320
|
+
const stop = end === -1 ? len : end;
|
|
321
|
+
out.push({ kind: 'comment', value: src.slice(i, stop) });
|
|
322
|
+
i = stop;
|
|
323
|
+
plainStart = i;
|
|
324
|
+
continue;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
// Hash comment (Py/Sh/Yaml/Rb).
|
|
328
|
+
if (spec.hashIsComment && ch === '#') {
|
|
329
|
+
flushPlain();
|
|
330
|
+
const end = src.indexOf('\n', i);
|
|
331
|
+
const stop = end === -1 ? len : end;
|
|
332
|
+
out.push({ kind: 'comment', value: src.slice(i, stop) });
|
|
333
|
+
i = stop;
|
|
334
|
+
plainStart = i;
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
// Block comment.
|
|
338
|
+
if (spec.blockComment && src.startsWith(spec.blockComment[0], i)) {
|
|
339
|
+
flushPlain();
|
|
340
|
+
const end = src.indexOf(spec.blockComment[1], i + spec.blockComment[0].length);
|
|
341
|
+
const stop = end === -1 ? len : end + spec.blockComment[1].length;
|
|
342
|
+
out.push({ kind: 'comment', value: src.slice(i, stop) });
|
|
343
|
+
i = stop;
|
|
344
|
+
plainStart = i;
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
347
|
+
// Strings.
|
|
348
|
+
if (spec.stringQuotes && spec.stringQuotes.includes(ch)) {
|
|
349
|
+
flushPlain();
|
|
350
|
+
const quote = ch;
|
|
351
|
+
let j = i + 1;
|
|
352
|
+
while (j < len) {
|
|
353
|
+
if (src[j] === '\\') {
|
|
354
|
+
j += 2;
|
|
355
|
+
continue;
|
|
356
|
+
}
|
|
357
|
+
if (src[j] === quote) {
|
|
358
|
+
j += 1;
|
|
359
|
+
break;
|
|
360
|
+
}
|
|
361
|
+
if (src[j] === '\n' && quote !== '`') {
|
|
362
|
+
break;
|
|
363
|
+
}
|
|
364
|
+
j += 1;
|
|
365
|
+
}
|
|
366
|
+
out.push({ kind: 'string', value: src.slice(i, j) });
|
|
367
|
+
i = j;
|
|
368
|
+
plainStart = i;
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
371
|
+
// Template literal (` ... `).
|
|
372
|
+
if (spec.templateLiteral && ch === '`') {
|
|
373
|
+
flushPlain();
|
|
374
|
+
let j = i + 1;
|
|
375
|
+
while (j < len) {
|
|
376
|
+
if (src[j] === '\\') {
|
|
377
|
+
j += 2;
|
|
378
|
+
continue;
|
|
379
|
+
}
|
|
380
|
+
if (src[j] === '`') {
|
|
381
|
+
j += 1;
|
|
382
|
+
break;
|
|
383
|
+
}
|
|
384
|
+
j += 1;
|
|
385
|
+
}
|
|
386
|
+
out.push({ kind: 'string', value: src.slice(i, j) });
|
|
387
|
+
i = j;
|
|
388
|
+
plainStart = i;
|
|
389
|
+
continue;
|
|
390
|
+
}
|
|
391
|
+
// Numbers.
|
|
392
|
+
if (ch >= '0' && ch <= '9') {
|
|
393
|
+
const m = src.slice(i).match(NUMBER_RE);
|
|
394
|
+
if (m) {
|
|
395
|
+
flushPlain();
|
|
396
|
+
out.push({ kind: 'number', value: m[0] });
|
|
397
|
+
i += m[0].length;
|
|
398
|
+
plainStart = i;
|
|
399
|
+
continue;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
// Identifiers — keyword/type/builtin/plain.
|
|
403
|
+
if (IDENT_START.test(ch)) {
|
|
404
|
+
let j = i + 1;
|
|
405
|
+
while (j < len && IDENT_CONT.test(src[j]))
|
|
406
|
+
j += 1;
|
|
407
|
+
const word = src.slice(i, j);
|
|
408
|
+
flushPlain();
|
|
409
|
+
let kind;
|
|
410
|
+
if (spec.keywords.has(word))
|
|
411
|
+
kind = 'keyword';
|
|
412
|
+
else if (spec.types?.has(word))
|
|
413
|
+
kind = 'type';
|
|
414
|
+
else if (spec.builtins?.has(word))
|
|
415
|
+
kind = 'builtin';
|
|
416
|
+
else if (spec.capitalIsType && /^[A-Z]/.test(word) && j < len && (src[j] === '<' || word.length > 1))
|
|
417
|
+
kind = 'type';
|
|
418
|
+
else
|
|
419
|
+
kind = 'plain';
|
|
420
|
+
out.push({ kind, value: word });
|
|
421
|
+
i = j;
|
|
422
|
+
plainStart = i;
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
425
|
+
// Decorator / meta (@foo).
|
|
426
|
+
if (spec.atIsMeta && ch === '@' && i + 1 < len && IDENT_START.test(src[i + 1])) {
|
|
427
|
+
flushPlain();
|
|
428
|
+
let j = i + 1;
|
|
429
|
+
while (j < len && IDENT_CONT.test(src[j]))
|
|
430
|
+
j += 1;
|
|
431
|
+
out.push({ kind: 'meta', value: src.slice(i, j) });
|
|
432
|
+
i = j;
|
|
433
|
+
plainStart = i;
|
|
434
|
+
continue;
|
|
435
|
+
}
|
|
436
|
+
// Operators.
|
|
437
|
+
if (spec.operatorChars && spec.operatorChars.includes(ch)) {
|
|
438
|
+
flushPlain();
|
|
439
|
+
let j = i;
|
|
440
|
+
while (j < len && spec.operatorChars.includes(src[j]))
|
|
441
|
+
j += 1;
|
|
442
|
+
out.push({ kind: 'operator', value: src.slice(i, j) });
|
|
443
|
+
i = j;
|
|
444
|
+
plainStart = i;
|
|
445
|
+
continue;
|
|
446
|
+
}
|
|
447
|
+
// Default — let it accumulate as plain.
|
|
448
|
+
i += 1;
|
|
449
|
+
}
|
|
450
|
+
flushPlain();
|
|
451
|
+
return out;
|
|
452
|
+
}
|
|
453
|
+
/** Public entry point. Returns a single plain-text token if the lang is unknown. */
|
|
454
|
+
export function tokenize(src, lang) {
|
|
455
|
+
if (!lang)
|
|
456
|
+
return [{ kind: 'plain', value: src }];
|
|
457
|
+
const norm = lang.toLowerCase();
|
|
458
|
+
const resolvedLang = ALIASES[norm] ?? norm;
|
|
459
|
+
// Special cases.
|
|
460
|
+
if (resolvedLang === 'diff' || resolvedLang === 'patch') {
|
|
461
|
+
return tokenizeDiff(src);
|
|
462
|
+
}
|
|
463
|
+
if (resolvedLang === 'html' || resolvedLang === 'xml' || resolvedLang === 'svg') {
|
|
464
|
+
return tokenizeHtml(src);
|
|
465
|
+
}
|
|
466
|
+
if (resolvedLang === 'md' || resolvedLang === 'markdown') {
|
|
467
|
+
// Recursive markdown rendering would loop; show as plain.
|
|
468
|
+
return [{ kind: 'plain', value: src }];
|
|
469
|
+
}
|
|
470
|
+
const spec = resolveLang(resolvedLang);
|
|
471
|
+
if (!spec)
|
|
472
|
+
return [{ kind: 'plain', value: src }];
|
|
473
|
+
return tokenizeGeneric(src, spec);
|
|
474
|
+
}
|
|
475
|
+
export { resolveLang };
|
|
476
|
+
//# sourceMappingURL=index.js.map
|