@scalar/highlight 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +49 -0
- package/README.md +52 -13
- package/dist/all.d.ts.map +1 -1
- package/dist/all.js +2 -0
- package/dist/compat/hast.d.ts.map +1 -1
- package/dist/compat/hast.js +4 -2
- package/dist/compat/hljs.d.ts.map +1 -1
- package/dist/compat/hljs.js +2 -1
- package/dist/compat/languages.d.ts.map +1 -1
- package/dist/compat/languages.js +2 -4
- package/dist/compat/syntax-highlight.d.ts.map +1 -1
- package/dist/compat/syntax-highlight.js +107 -29
- package/dist/core/compile.d.ts +9 -1
- package/dist/core/compile.d.ts.map +1 -1
- package/dist/core/compile.js +128 -10
- package/dist/core/index.d.ts +1 -0
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +1 -0
- package/dist/core/registry.d.ts.map +1 -1
- package/dist/core/registry.js +8 -4
- package/dist/core/render.d.ts.map +1 -1
- package/dist/core/render.js +12 -6
- package/dist/core/tokenize.d.ts.map +1 -1
- package/dist/core/tokenize.js +161 -17
- package/dist/core/types.d.ts +74 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js +5 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/langs/bash.d.ts.map +1 -1
- package/dist/langs/bash.js +60 -13
- package/dist/langs/csharp.d.ts.map +1 -1
- package/dist/langs/csharp.js +11 -5
- package/dist/langs/css.d.ts.map +1 -1
- package/dist/langs/css.js +84 -7
- package/dist/langs/go.d.ts.map +1 -1
- package/dist/langs/go.js +7 -1
- package/dist/langs/html.d.ts +0 -10
- package/dist/langs/html.d.ts.map +1 -1
- package/dist/langs/html.js +63 -24
- package/dist/langs/http.d.ts.map +1 -1
- package/dist/langs/http.js +25 -9
- package/dist/langs/javascript.d.ts.map +1 -1
- package/dist/langs/javascript.js +146 -30
- package/dist/langs/json.d.ts.map +1 -1
- package/dist/langs/json.js +7 -1
- package/dist/langs/julia.d.ts +4 -0
- package/dist/langs/julia.d.ts.map +1 -0
- package/dist/langs/julia.js +181 -0
- package/dist/langs/markdown.d.ts +9 -3
- package/dist/langs/markdown.d.ts.map +1 -1
- package/dist/langs/markdown.js +127 -6
- package/dist/langs/ocaml.d.ts.map +1 -1
- package/dist/langs/ocaml.js +16 -7
- package/dist/langs/php.d.ts.map +1 -1
- package/dist/langs/php.js +69 -19
- package/dist/langs/r.js +9 -9
- package/dist/langs/rust.d.ts.map +1 -1
- package/dist/langs/rust.js +19 -10
- package/dist/langs/sql.d.ts.map +1 -1
- package/dist/langs/sql.js +9 -3
- package/dist/langs/yaml.d.ts +0 -7
- package/dist/langs/yaml.d.ts.map +1 -1
- package/dist/langs/yaml.js +157 -28
- package/dist/lazy.d.ts +1 -0
- package/dist/lazy.d.ts.map +1 -1
- package/dist/lazy.js +6 -2
- package/package.json +3 -3
package/dist/core/compile.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { REMEMBERED } from './types.js';
|
|
1
2
|
/**
|
|
2
3
|
* Compiles a grammar into one merged regex per state.
|
|
3
4
|
*
|
|
@@ -21,24 +22,27 @@ const countGroups = (src) => {
|
|
|
21
22
|
continue;
|
|
22
23
|
}
|
|
23
24
|
if (inClass) {
|
|
24
|
-
if (c === 93 /* ] */)
|
|
25
|
+
if (c === 93 /* ] */) {
|
|
25
26
|
inClass = false;
|
|
27
|
+
}
|
|
26
28
|
continue;
|
|
27
29
|
}
|
|
28
30
|
if (c === 91 /* [ */) {
|
|
29
31
|
inClass = true;
|
|
30
32
|
continue;
|
|
31
33
|
}
|
|
32
|
-
if (c !== 40 /* ( */)
|
|
34
|
+
if (c !== 40 /* ( */) {
|
|
33
35
|
continue;
|
|
36
|
+
}
|
|
34
37
|
if (src.charCodeAt(i + 1) !== 63 /* ? */) {
|
|
35
38
|
count++;
|
|
36
39
|
}
|
|
37
40
|
else if (src.charCodeAt(i + 2) === 60 /* < */) {
|
|
38
41
|
// (?<name> captures; (?<= and (?<! are lookbehind.
|
|
39
42
|
const after = src.charCodeAt(i + 3);
|
|
40
|
-
if (after !== 61 /* = */ && after !== 33 /* ! */)
|
|
43
|
+
if (after !== 61 /* = */ && after !== 33 /* ! */) {
|
|
41
44
|
count++;
|
|
45
|
+
}
|
|
42
46
|
}
|
|
43
47
|
}
|
|
44
48
|
return count;
|
|
@@ -46,6 +50,60 @@ const countGroups = (src) => {
|
|
|
46
50
|
const patternSource = (match) => {
|
|
47
51
|
return typeof match === 'string' ? match : match.source;
|
|
48
52
|
};
|
|
53
|
+
/**
|
|
54
|
+
* Makes a pattern end lines at `\n` (or `\r\n`) only.
|
|
55
|
+
*
|
|
56
|
+
* With the `m` flag, JavaScript's `$` also matches before `\r`, U+2028 and
|
|
57
|
+
* U+2029, and `.` refuses to match them. Grammars are written as if lines end
|
|
58
|
+
* at `\n` — which is also where the line renderer splits — so a lone `\r` or a
|
|
59
|
+
* U+2028 inside a string literal used to end the string there and leave the
|
|
60
|
+
* rest of the line inverted. Rewriting the two tokens once, here, fixes every
|
|
61
|
+
* grammar at the same time:
|
|
62
|
+
*
|
|
63
|
+
* - `.` becomes `[^\r\n]`: it now matches U+2028 and U+2029, and still
|
|
64
|
+
* stops before the `\r` of a CRLF so an escape like `\\.` cannot swallow it
|
|
65
|
+
* - `$` becomes "before `\n` or `\r\n`, or at the end", so CRLF input still
|
|
66
|
+
* ends a line before its `\r`, exactly as it did before
|
|
67
|
+
*
|
|
68
|
+
* `^` stays native. A line that starts after a lone `\r` is harmless, and the
|
|
69
|
+
* engine's own anchor is measurably faster on large input than a lookbehind.
|
|
70
|
+
*
|
|
71
|
+
* Escapes and character classes match as themselves and are kept verbatim, so
|
|
72
|
+
* only a bare `.` or `$` is rewritten.
|
|
73
|
+
*/
|
|
74
|
+
const normalizeLineTerminators = (src) => {
|
|
75
|
+
// A single left-to-right scan rather than a tokenizing regex: a pattern for
|
|
76
|
+
// "a character class" retried at every `[` of an unclosed run is O(n²).
|
|
77
|
+
let out = '';
|
|
78
|
+
let inClass = false;
|
|
79
|
+
for (let i = 0; i < src.length; i++) {
|
|
80
|
+
const c = src[i];
|
|
81
|
+
if (c === '\\') {
|
|
82
|
+
out += c + (src[i + 1] ?? '');
|
|
83
|
+
i++;
|
|
84
|
+
}
|
|
85
|
+
else if (inClass) {
|
|
86
|
+
inClass = c !== ']';
|
|
87
|
+
out += c;
|
|
88
|
+
}
|
|
89
|
+
else if (c === '[') {
|
|
90
|
+
inClass = true;
|
|
91
|
+
out += c;
|
|
92
|
+
}
|
|
93
|
+
else if (c === '.') {
|
|
94
|
+
out += '[^\\r\\n]';
|
|
95
|
+
}
|
|
96
|
+
else if (c === '$') {
|
|
97
|
+
out += '(?=\\r?\\n|(?![\\s\\S]))';
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
out += c;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return out;
|
|
104
|
+
};
|
|
105
|
+
/** Escapes text so it can be spliced into a pattern and match literally. */
|
|
106
|
+
const escapeRegExp = (s) => s.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&');
|
|
49
107
|
/** Resolves `include` entries into a flat, priority-ordered rule list. */
|
|
50
108
|
const flatten = (grammarName, states, name, seen) => {
|
|
51
109
|
if (seen.has(name)) {
|
|
@@ -56,8 +114,9 @@ const flatten = (grammarName, states, name, seen) => {
|
|
|
56
114
|
// `constructor` would otherwise resolve off Object.prototype and sail past
|
|
57
115
|
// this check, failing later with something far less legible.
|
|
58
116
|
const state = Object.hasOwn(states, name) ? states[name] : undefined;
|
|
59
|
-
if (!state)
|
|
117
|
+
if (!state) {
|
|
60
118
|
throw new Error(`[${grammarName}] include of unknown state "${name}"`);
|
|
119
|
+
}
|
|
61
120
|
const out = [];
|
|
62
121
|
for (const entry of state.rules) {
|
|
63
122
|
if ('include' in entry) {
|
|
@@ -78,12 +137,17 @@ const compileState = (grammar, name) => {
|
|
|
78
137
|
// Group 0 is the whole match, so rule wrapper groups start at 1.
|
|
79
138
|
let group = 1;
|
|
80
139
|
const caseFlag = state.ignoreCase ? 'i' : '';
|
|
140
|
+
let dynamic = false;
|
|
141
|
+
const ownSources = [];
|
|
81
142
|
for (const rule of rules) {
|
|
82
|
-
const
|
|
83
|
-
if (BACKREF.test(
|
|
143
|
+
const raw = patternSource(rule.match);
|
|
144
|
+
if (BACKREF.test(raw)) {
|
|
84
145
|
throw new Error(`[${grammar.name}:${name}] backreferences are not supported (group numbers ` +
|
|
85
|
-
`shift when rules are merged): ${
|
|
146
|
+
`shift when rules are merged): ${raw}`);
|
|
86
147
|
}
|
|
148
|
+
const src = normalizeLineTerminators(raw);
|
|
149
|
+
const usesMemory = src.includes(REMEMBERED);
|
|
150
|
+
dynamic ||= usesMemory;
|
|
87
151
|
const captures = Array.isArray(rule.scope) ? rule.scope : null;
|
|
88
152
|
const inner = countGroups(src);
|
|
89
153
|
if (captures) {
|
|
@@ -91,17 +155,24 @@ const compileState = (grammar, name) => {
|
|
|
91
155
|
throw new Error(`[${grammar.name}:${name}] rule scopes ${captures.length} groups but the ` + `pattern has ${inner}: ${src}`);
|
|
92
156
|
}
|
|
93
157
|
}
|
|
158
|
+
if (rule.remember !== undefined && (!(rule.push || rule.set) || rule.remember < 1 || rule.remember > inner)) {
|
|
159
|
+
throw new Error(`[${grammar.name}:${name}] remember ${rule.remember} needs a push or set and one of ${inner} groups: ${raw}`);
|
|
160
|
+
}
|
|
94
161
|
compiled.push({
|
|
95
162
|
scope: typeof rule.scope === 'string' ? rule.scope : null,
|
|
96
163
|
captures,
|
|
97
164
|
// Sticky, so it can be re-run at the position the merged regex chose.
|
|
98
|
-
|
|
165
|
+
// A pattern that depends on the remembered text is built per value.
|
|
166
|
+
own: captures && !usesMemory ? new RegExp(src, `ymd${caseFlag}`) : null,
|
|
99
167
|
rest: rule.rest ?? null,
|
|
100
168
|
group,
|
|
101
169
|
push: rule.push ?? null,
|
|
102
170
|
set: rule.set ?? null,
|
|
103
171
|
pop: rule.pop === true ? 1 : (rule.pop ?? 0),
|
|
172
|
+
remember: rule.remember ?? 0,
|
|
173
|
+
rememberIndent: rule.rememberAs === 'indent',
|
|
104
174
|
});
|
|
175
|
+
ownSources.push(captures && usesMemory ? src : null);
|
|
105
176
|
sources.push(`(${src})`);
|
|
106
177
|
group += 1 + inner;
|
|
107
178
|
}
|
|
@@ -109,13 +180,60 @@ const compileState = (grammar, name) => {
|
|
|
109
180
|
// resolved per rule instead, which keeps the scan free of that cost.
|
|
110
181
|
const flags = `gm${caseFlag}`;
|
|
111
182
|
// An empty state matches nothing: everything falls through to `default`.
|
|
112
|
-
const
|
|
113
|
-
|
|
183
|
+
const source = sources.length ? sources.join('|') : '(?!)';
|
|
184
|
+
// A dynamic state still gets a regex of its own, built with nothing
|
|
185
|
+
// remembered, so it can be entered from the root of a fresh tokenize.
|
|
186
|
+
const re = new RegExp(dynamic ? source.split(REMEMBERED).join('') : source, flags);
|
|
187
|
+
return {
|
|
188
|
+
name,
|
|
189
|
+
re,
|
|
190
|
+
rules: compiled,
|
|
191
|
+
default: state.default ?? null,
|
|
192
|
+
dynamic: dynamic ? { source, flags, own: ownSources, cache: new Map() } : null,
|
|
193
|
+
embed: state.embed
|
|
194
|
+
? {
|
|
195
|
+
language: state.embed.language,
|
|
196
|
+
// Scan holes and the end marker together so a marker inside a hole
|
|
197
|
+
// cannot close the embedded body. Group 1 identifies a hole.
|
|
198
|
+
boundary: new RegExp(normalizeLineTerminators(state.embed.holes ? `(${state.embed.holes})|(?:${state.embed.until})` : state.embed.until), `gm${caseFlag}`),
|
|
199
|
+
holes: state.embed.holes ? new RegExp(normalizeLineTerminators(state.embed.holes), `gm${caseFlag}`) : null,
|
|
200
|
+
}
|
|
201
|
+
: null,
|
|
202
|
+
};
|
|
203
|
+
};
|
|
204
|
+
/**
|
|
205
|
+
* The merged regex of a dynamic state, and its per-rule sticky patterns, with
|
|
206
|
+
* `value` spliced in wherever a rule wrote `REMEMBERED`. Built once per value.
|
|
207
|
+
*/
|
|
208
|
+
export const dynamicRegExps = (state, value) => {
|
|
209
|
+
const dyn = state.dynamic;
|
|
210
|
+
let hit = dyn.cache.get(value);
|
|
211
|
+
if (hit === undefined) {
|
|
212
|
+
const literal = escapeRegExp(value);
|
|
213
|
+
const fill = (src) => src.split(REMEMBERED).join(literal);
|
|
214
|
+
const caseFlag = dyn.flags.includes('i') ? 'i' : '';
|
|
215
|
+
hit = {
|
|
216
|
+
re: new RegExp(fill(dyn.source), dyn.flags),
|
|
217
|
+
own: dyn.own.map((src) => (src === null ? null : new RegExp(fill(src), `ymd${caseFlag}`))),
|
|
218
|
+
};
|
|
219
|
+
// A document with thousands of distinct heredoc markers should not grow
|
|
220
|
+
// this without bound.
|
|
221
|
+
if (dyn.cache.size > 64) {
|
|
222
|
+
dyn.cache.clear();
|
|
223
|
+
}
|
|
224
|
+
dyn.cache.set(value, hit);
|
|
225
|
+
}
|
|
226
|
+
return hit;
|
|
114
227
|
};
|
|
115
228
|
export const compile = (grammar) => {
|
|
116
229
|
if (!Object.hasOwn(grammar.states, 'root')) {
|
|
117
230
|
throw new Error(`[${grammar.name}] grammar has no "root" state`);
|
|
118
231
|
}
|
|
232
|
+
// An embedding state pops itself once its body is done, and the root is
|
|
233
|
+
// never popped.
|
|
234
|
+
if (grammar.states.root?.embed) {
|
|
235
|
+
throw new Error(`[${grammar.name}] the "root" state cannot embed another language`);
|
|
236
|
+
}
|
|
119
237
|
// Null-prototype: state names come from grammar data, and the unknown-state
|
|
120
238
|
// check below has to see `constructor` and `__proto__` as missing rather
|
|
121
239
|
// than as inherited Object members. Tokenizing looks states up here too.
|
package/dist/core/index.d.ts
CHANGED
|
@@ -6,4 +6,5 @@ export type { Scope } from './scopes.js';
|
|
|
6
6
|
export { SCOPES, SCOPE_NAMES, scopeChain, scopeClass } from './scopes.js';
|
|
7
7
|
export { tokenize as tokenizeWith, tokenizeToArray } from './tokenize.js';
|
|
8
8
|
export type { CompiledGrammar, Emit, Grammar, Include, Rule, ScopeName, State, StateRule, Token, } from './types.js';
|
|
9
|
+
export { REMEMBERED } from './types.js';
|
|
9
10
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/core/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAC3D,OAAO,EACL,cAAc,EACd,eAAe,EACf,UAAU,EACV,SAAS,IAAI,aAAa,EAC1B,cAAc,IAAI,kBAAkB,GACrC,MAAM,UAAU,CAAA;AACjB,YAAY,EAAE,KAAK,EAAE,MAAM,UAAU,CAAA;AACrC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AACtE,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AACtE,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,SAAS,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAC3D,OAAO,EACL,cAAc,EACd,eAAe,EACf,UAAU,EACV,SAAS,IAAI,aAAa,EAC1B,cAAc,IAAI,kBAAkB,GACrC,MAAM,UAAU,CAAA;AACjB,YAAY,EAAE,KAAK,EAAE,MAAM,UAAU,CAAA;AACrC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AACtE,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AACtE,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,SAAS,CAAA;AAChB,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA"}
|
package/dist/core/index.js
CHANGED
|
@@ -3,3 +3,4 @@ export { getLanguage, isRegistered, listLanguages, registerLanguage, resolveGram
|
|
|
3
3
|
export { DEFAULT_PREFIX, escapeAttribute, escapeHtml, highlight as highlightWith, highlightBlock as highlightBlockWith, } from './render.js';
|
|
4
4
|
export { SCOPES, SCOPE_NAMES, scopeChain, scopeClass } from './scopes.js';
|
|
5
5
|
export { tokenize as tokenizeWith, tokenizeToArray } from './tokenize.js';
|
|
6
|
+
export { REMEMBERED } from './types.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/core/registry.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,SAAS,CAAA;AAYvD,eAAO,MAAM,gBAAgB,GAAI,GAAG,UAAU,OAAO,EAAE,KAAG,
|
|
1
|
+
{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/core/registry.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,SAAS,CAAA;AAYvD,eAAO,MAAM,gBAAgB,GAAI,GAAG,UAAU,OAAO,EAAE,KAAG,IAyBzD,CAAA;AAED,eAAO,MAAM,YAAY,GAAI,MAAM,MAAM,KAAG,OAE3C,CAAA;AAED,gEAAgE;AAChE,eAAO,MAAM,aAAa,QAAO,MAAM,EAEtC,CAAA;AAMD,qEAAqE;AACrE,eAAO,MAAM,WAAW,GAAI,MAAM,MAAM,KAAG,eAAe,GAAG,SAkB5D,CAAA;AAED;;;;GAIG;AACH,eAAO,MAAM,cAAc,GAAI,MAAM,MAAM,GAAG,OAAO,GAAG,eAAe,KAAG,eAgBzE,CAAA"}
|
package/dist/core/registry.js
CHANGED
|
@@ -19,8 +19,9 @@ export const registerLanguage = (...grammars) => {
|
|
|
19
19
|
if (previous) {
|
|
20
20
|
for (const key of [previous.name, ...(previous.aliases ?? [])]) {
|
|
21
21
|
const lower = key.toLowerCase();
|
|
22
|
-
if (sources.get(lower) === previous)
|
|
22
|
+
if (sources.get(lower) === previous) {
|
|
23
23
|
sources.delete(lower);
|
|
24
|
+
}
|
|
24
25
|
cache.delete(lower);
|
|
25
26
|
}
|
|
26
27
|
}
|
|
@@ -45,11 +46,13 @@ const isCompiled = (value) => {
|
|
|
45
46
|
export const getLanguage = (name) => {
|
|
46
47
|
const key = name.toLowerCase();
|
|
47
48
|
const hit = cache.get(key);
|
|
48
|
-
if (hit)
|
|
49
|
+
if (hit) {
|
|
49
50
|
return hit;
|
|
51
|
+
}
|
|
50
52
|
const grammar = sources.get(key);
|
|
51
|
-
if (!grammar)
|
|
53
|
+
if (!grammar) {
|
|
52
54
|
return undefined;
|
|
55
|
+
}
|
|
53
56
|
const built = compile(grammar);
|
|
54
57
|
// Cache under every alias so repeat lookups skip the alias hop.
|
|
55
58
|
for (const alias of [grammar.name, ...(grammar.aliases ?? [])]) {
|
|
@@ -67,8 +70,9 @@ export const resolveGrammar = (lang) => {
|
|
|
67
70
|
return isCompiled(lang) ? lang : compile(lang);
|
|
68
71
|
}
|
|
69
72
|
const found = getLanguage(lang);
|
|
70
|
-
if (found)
|
|
73
|
+
if (found) {
|
|
71
74
|
return found;
|
|
75
|
+
}
|
|
72
76
|
const known = listLanguages();
|
|
73
77
|
throw new Error(`Language "${lang}" is not registered. ` +
|
|
74
78
|
(known.length
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"render.d.ts","sourceRoot":"","sources":["../../src/core/render.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAE9C,MAAM,MAAM,aAAa,GAAG;IAC1B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,qDAAqD;IACrD,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACnC,CAAA;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,cAAc,SAAS,CAAA;AAMpC;;;;;;GAMG;AACH,eAAO,MAAM,UAAU,GAAI,GAAG,MAAM,KAAG,
|
|
1
|
+
{"version":3,"file":"render.d.ts","sourceRoot":"","sources":["../../src/core/render.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAE9C,MAAM,MAAM,aAAa,GAAG;IAC1B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,qDAAqD;IACrD,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACnC,CAAA;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,cAAc,SAAS,CAAA;AAMpC;;;;;;GAMG;AACH,eAAO,MAAM,UAAU,GAAI,GAAG,MAAM,KAAG,MAatC,CAAA;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,eAAe,GAAI,GAAG,MAAM,KAAG,MAE3C,CAAA;AA4BD;;;;;;GAMG;AACH,eAAO,MAAM,SAAS,GAAI,MAAM,MAAM,EAAE,SAAS,eAAe,EAAE,UAAS,aAAkB,KAAG,MAgC/F,CAAA;AA0FD,MAAM,MAAM,YAAY,GAAG,aAAa,GAAG;IACzC,oDAAoD;IACpD,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,oCAAoC;IACpC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,uEAAuE;IACvE,YAAY,CAAC,EAAE,OAAO,CAAA;CACvB,CAAA;AAED;;;;;;GAMG;AACH,eAAO,MAAM,cAAc,GAAI,MAAM,MAAM,EAAE,SAAS,eAAe,EAAE,UAAS,YAAiB,KAAG,MAkCnG,CAAA"}
|
package/dist/core/render.js
CHANGED
|
@@ -25,9 +25,11 @@ export const escapeHtml = (s) => {
|
|
|
25
25
|
let last = 0;
|
|
26
26
|
for (let i = 0; i < s.length; i++) {
|
|
27
27
|
const c = s.charCodeAt(i);
|
|
28
|
-
if (c !== AMP && c !== LT && c !== GT)
|
|
28
|
+
if (c !== AMP && c !== LT && c !== GT) {
|
|
29
29
|
continue;
|
|
30
|
-
|
|
30
|
+
}
|
|
31
|
+
const angleEntity = c === LT ? '<' : '>';
|
|
32
|
+
out += s.slice(last, i) + (c === AMP ? '&' : angleEntity);
|
|
31
33
|
last = i + 1;
|
|
32
34
|
}
|
|
33
35
|
return last === 0 ? s : out + s.slice(last);
|
|
@@ -75,8 +77,9 @@ const openTag = (tags, prefix, scope) => {
|
|
|
75
77
|
*/
|
|
76
78
|
export const highlight = (code, grammar, options = {}) => {
|
|
77
79
|
const prefix = escapeAttribute(options.classPrefix ?? DEFAULT_PREFIX);
|
|
78
|
-
if (options.lines)
|
|
80
|
+
if (options.lines) {
|
|
79
81
|
return renderLines(code, grammar, prefix, options.highlightLines).html;
|
|
82
|
+
}
|
|
80
83
|
const tags = tagsFor(prefix);
|
|
81
84
|
let out = '';
|
|
82
85
|
// Pending run of same-scoped ranges, flushed when the scope changes.
|
|
@@ -128,8 +131,9 @@ const renderLines = (code, grammar, prefix, highlightLines) => {
|
|
|
128
131
|
let from = 0;
|
|
129
132
|
let to = 0;
|
|
130
133
|
const flush = () => {
|
|
131
|
-
if (to > from)
|
|
134
|
+
if (to > from) {
|
|
132
135
|
line += span(code, scope, from, to, prefix);
|
|
136
|
+
}
|
|
133
137
|
from = to;
|
|
134
138
|
};
|
|
135
139
|
const endLine = () => {
|
|
@@ -150,8 +154,9 @@ const renderLines = (code, grammar, prefix, highlightLines) => {
|
|
|
150
154
|
}
|
|
151
155
|
to = stop;
|
|
152
156
|
}
|
|
153
|
-
if (stop === end)
|
|
157
|
+
if (stop === end) {
|
|
154
158
|
break;
|
|
159
|
+
}
|
|
155
160
|
endLine();
|
|
156
161
|
at = stop + 1;
|
|
157
162
|
scope = null;
|
|
@@ -163,8 +168,9 @@ const renderLines = (code, grammar, prefix, highlightLines) => {
|
|
|
163
168
|
// A trailing newline in the source closes the last line rather than starting
|
|
164
169
|
// an empty one.
|
|
165
170
|
const endsWithNewline = code.endsWith('\n');
|
|
166
|
-
if (endsWithNewline && lines[lines.length - 1] === '')
|
|
171
|
+
if (endsWithNewline && lines[lines.length - 1] === '') {
|
|
167
172
|
lines.pop();
|
|
173
|
+
}
|
|
168
174
|
let out = '';
|
|
169
175
|
for (let i = 0; i < lines.length; i++) {
|
|
170
176
|
const hl = marked?.has(i + 1) ? ' data-hl' : '';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../src/core/tokenize.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../src/core/tokenize.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAA+B,IAAI,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AAGxF;;;;;;GAMG;AACH,eAAO,MAAM,QAAQ,GAAI,MAAM,MAAM,EAAE,SAAS,eAAe,EAAE,MAAM,IAAI,KAAG,IA4G7E,CAAA;AAgLD;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,GAAI,MAAM,MAAM,EAAE,SAAS,eAAe,KAAG,KAAK,EAM7E,CAAA"}
|
package/dist/core/tokenize.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { dynamicRegExps } from './compile.js';
|
|
2
|
+
import { getLanguage } from './registry.js';
|
|
3
|
+
import { REMEMBERED } from './types.js';
|
|
1
4
|
/**
|
|
2
5
|
* Runs the state machine over `code`, calling `emit` for every range.
|
|
3
6
|
*
|
|
@@ -8,17 +11,36 @@
|
|
|
8
11
|
export const tokenize = (code, grammar, emit) => {
|
|
9
12
|
const len = code.length;
|
|
10
13
|
const stack = [grammar.root];
|
|
14
|
+
// What each stack entry remembered (see `Rule.remember`), in step with `stack`.
|
|
15
|
+
const memory = [''];
|
|
11
16
|
let top = grammar.root;
|
|
12
17
|
let pos = 0;
|
|
13
18
|
// Zero-length matches are legal (a pure state transition on a lookahead),
|
|
14
19
|
// but a rule that neither consumes nor eventually changes state would spin.
|
|
15
20
|
let spin = 0;
|
|
16
21
|
while (pos < len) {
|
|
17
|
-
|
|
22
|
+
if (top.embed !== null) {
|
|
23
|
+
pos = embedded(code, pos, top, memory[memory.length - 1], grammar, emit);
|
|
24
|
+
// An embedding state pops itself. A rule that `set` into it from the
|
|
25
|
+
// only state on the stack leaves nothing below, so the root takes over.
|
|
26
|
+
if (stack.length > 1) {
|
|
27
|
+
stack.length -= 1;
|
|
28
|
+
memory.length -= 1;
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
stack[0] = grammar.root;
|
|
32
|
+
memory[0] = '';
|
|
33
|
+
}
|
|
34
|
+
top = stack[stack.length - 1];
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
const dyn = top.dynamic === null ? null : dynamicRegExps(top, memory[memory.length - 1]);
|
|
38
|
+
const re = dyn === null ? top.re : dyn.re;
|
|
18
39
|
re.lastIndex = pos;
|
|
19
40
|
const m = re.exec(code);
|
|
20
|
-
if (m === null)
|
|
41
|
+
if (m === null) {
|
|
21
42
|
break;
|
|
43
|
+
}
|
|
22
44
|
const start = m.index;
|
|
23
45
|
if (start > pos) {
|
|
24
46
|
emit(top.default, pos, start);
|
|
@@ -26,34 +48,50 @@ export const tokenize = (code, grammar, emit) => {
|
|
|
26
48
|
}
|
|
27
49
|
const rules = top.rules;
|
|
28
50
|
let rule;
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
if (m[
|
|
32
|
-
rule =
|
|
51
|
+
let index = 0;
|
|
52
|
+
for (; index < rules.length; index++) {
|
|
53
|
+
if (m[rules[index].group] !== undefined) {
|
|
54
|
+
rule = rules[index];
|
|
33
55
|
break;
|
|
34
56
|
}
|
|
35
57
|
}
|
|
36
|
-
if (rule === undefined)
|
|
58
|
+
if (rule === undefined) {
|
|
37
59
|
break; // unreachable: some alternative matched
|
|
60
|
+
}
|
|
38
61
|
const end = start + m[0].length;
|
|
39
62
|
if (end > start) {
|
|
40
63
|
if (rule.captures !== null) {
|
|
41
|
-
|
|
64
|
+
// Only rules that use the remembered text are rebuilt per value; the
|
|
65
|
+
// rest of a dynamic state's capture rules keep their own pattern.
|
|
66
|
+
emitCaptures(code, start, rule, dyn?.own[index] ?? rule.own, top, emit);
|
|
42
67
|
}
|
|
43
68
|
else {
|
|
44
69
|
emit(rule.scope ?? top.default, start, end);
|
|
45
70
|
}
|
|
46
71
|
}
|
|
72
|
+
// Read before the stack changes: an entered state inherits the memory of
|
|
73
|
+
// the state that entered it unless the rule remembers something new.
|
|
74
|
+
let remembered = memory[memory.length - 1];
|
|
75
|
+
if (rule.remember > 0) {
|
|
76
|
+
const text = m[rule.group + rule.remember] ?? '';
|
|
77
|
+
remembered = rule.rememberIndent ? ' '.repeat(text.length) : text;
|
|
78
|
+
}
|
|
47
79
|
if (rule.pop > 0) {
|
|
48
80
|
// Never pop the root state off.
|
|
49
81
|
const n = Math.min(rule.pop, stack.length - 1);
|
|
50
82
|
stack.length -= n;
|
|
83
|
+
memory.length -= n;
|
|
84
|
+
if (rule.remember === 0) {
|
|
85
|
+
remembered = memory[memory.length - 1];
|
|
86
|
+
}
|
|
51
87
|
}
|
|
52
88
|
if (rule.set !== null) {
|
|
53
89
|
stack[stack.length - 1] = grammar.states[rule.set];
|
|
90
|
+
memory[memory.length - 1] = remembered;
|
|
54
91
|
}
|
|
55
92
|
if (rule.push !== null) {
|
|
56
93
|
stack.push(grammar.states[rule.push]);
|
|
94
|
+
memory.push(remembered);
|
|
57
95
|
}
|
|
58
96
|
top = stack[stack.length - 1];
|
|
59
97
|
if (end > pos) {
|
|
@@ -69,8 +107,109 @@ export const tokenize = (code, grammar, emit) => {
|
|
|
69
107
|
spin = 0;
|
|
70
108
|
}
|
|
71
109
|
}
|
|
72
|
-
if (pos < len)
|
|
110
|
+
if (pos < len) {
|
|
73
111
|
emit(top.default, pos, len);
|
|
112
|
+
}
|
|
113
|
+
};
|
|
114
|
+
/**
|
|
115
|
+
* How deep embeds may nest before a body falls back to its state's default
|
|
116
|
+
* scope. Real code nests two or three deep — Markdown, then HTML, then
|
|
117
|
+
* JavaScript — but a Markdown fence can embed Markdown, and a run of
|
|
118
|
+
* ` ```md ` lines would otherwise recurse once per line until the stack runs
|
|
119
|
+
* out.
|
|
120
|
+
*/
|
|
121
|
+
const MAX_EMBED_DEPTH = 8;
|
|
122
|
+
/** How many embedded bodies are being tokenized right now, one inside another. */
|
|
123
|
+
let depth = 0;
|
|
124
|
+
/**
|
|
125
|
+
* Tokenizes the body of an embedding state with its language and returns the
|
|
126
|
+
* position after it.
|
|
127
|
+
*
|
|
128
|
+
* The body runs to the first match of the state's `until`, which is left for
|
|
129
|
+
* the state below to tokenize. The slice is tokenized on its own, so the
|
|
130
|
+
* embedded grammar starts in its root state and cannot read past the body —
|
|
131
|
+
* an unterminated string in a `<script>` cannot swallow the `</script>`.
|
|
132
|
+
*/
|
|
133
|
+
const embedded = (code, pos, state, remembered, host, emit) => {
|
|
134
|
+
const boundary = state.embed.boundary;
|
|
135
|
+
boundary.lastIndex = pos;
|
|
136
|
+
let found = boundary.exec(code);
|
|
137
|
+
while (state.embed.holes !== null && found?.[1] !== undefined) {
|
|
138
|
+
// An empty hole must not prevent the scanner from making progress.
|
|
139
|
+
if (found[0].length === 0) {
|
|
140
|
+
boundary.lastIndex += code.codePointAt(boundary.lastIndex) > 0xffff ? 2 : 1;
|
|
141
|
+
}
|
|
142
|
+
found = boundary.exec(code);
|
|
143
|
+
}
|
|
144
|
+
const stop = found === null ? code.length : found.index;
|
|
145
|
+
if (stop === pos) {
|
|
146
|
+
return pos;
|
|
147
|
+
}
|
|
148
|
+
// `{{remembered}}` takes the language from the text the entering rule
|
|
149
|
+
// remembered, which is how a Markdown fence names its language at runtime.
|
|
150
|
+
const language = state.embed.language;
|
|
151
|
+
const grammar = getLanguage(language === REMEMBERED ? remembered : language);
|
|
152
|
+
if (grammar === undefined || depth >= MAX_EMBED_DEPTH) {
|
|
153
|
+
emit(state.default, pos, stop);
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
depth += 1;
|
|
157
|
+
try {
|
|
158
|
+
withHoles(code.slice(pos, stop), pos, grammar, state.embed.holes, host, emit);
|
|
159
|
+
}
|
|
160
|
+
finally {
|
|
161
|
+
depth -= 1;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
return stop;
|
|
165
|
+
};
|
|
166
|
+
/**
|
|
167
|
+
* Tokenizes an embedded body, around any `holes` that belong to the host
|
|
168
|
+
* grammar.
|
|
169
|
+
*
|
|
170
|
+
* The embedded grammar reads the body with every hole character except line
|
|
171
|
+
* breaks replaced by `_`, an identifier character in the languages that get
|
|
172
|
+
* embedded, so a hole reads as a name and the embedded state runs on across
|
|
173
|
+
* it. Its tokens are then emitted around the holes, and each hole is
|
|
174
|
+
* tokenized with the host grammar in its place.
|
|
175
|
+
*/
|
|
176
|
+
const withHoles = (body, offset, grammar, holes, host, emit) => {
|
|
177
|
+
// Flat [start, end, start, end, …] offsets into `body`.
|
|
178
|
+
const ranges = [];
|
|
179
|
+
const pieces = [];
|
|
180
|
+
let cursor = 0;
|
|
181
|
+
if (holes !== null) {
|
|
182
|
+
holes.lastIndex = 0;
|
|
183
|
+
for (const match of body.matchAll(holes)) {
|
|
184
|
+
const start = match.index;
|
|
185
|
+
const end = start + match[0].length;
|
|
186
|
+
if (start === end) {
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
ranges.push(start, end);
|
|
190
|
+
pieces.push(body.slice(cursor, start), match[0].replace(/[^\r\n]/g, '_'));
|
|
191
|
+
cursor = end;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
const masked = ranges.length === 0 ? body : pieces.join('') + body.slice(cursor);
|
|
195
|
+
let next = 0;
|
|
196
|
+
// Everything before `done` has been emitted, by one grammar or the other.
|
|
197
|
+
let done = 0;
|
|
198
|
+
tokenize(masked, grammar, (scope, from, to) => {
|
|
199
|
+
for (let at = Math.max(from, done); at < to; at = done) {
|
|
200
|
+
const start = ranges[next] ?? to;
|
|
201
|
+
if (start <= at) {
|
|
202
|
+
const end = ranges[next + 1];
|
|
203
|
+
tokenize(body.slice(start, end), host, (s, a, b) => emit(s, a + start + offset, b + start + offset));
|
|
204
|
+
next += 2;
|
|
205
|
+
done = end;
|
|
206
|
+
}
|
|
207
|
+
else {
|
|
208
|
+
done = Math.min(to, start);
|
|
209
|
+
emit(scope, at + offset, done + offset);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
});
|
|
74
213
|
};
|
|
75
214
|
/**
|
|
76
215
|
* Re-runs the winning rule on its own to find where its capture groups sit.
|
|
@@ -86,12 +225,12 @@ export const tokenize = (code, grammar, emit) => {
|
|
|
86
225
|
* either verbatim would repeat text, or claim text the tokenizer is about to
|
|
87
226
|
* visit again, so a group only ever scopes what the match still has left.
|
|
88
227
|
*/
|
|
89
|
-
const emitCaptures = (code, start, rule, state, emit) => {
|
|
90
|
-
const own = rule.own;
|
|
228
|
+
const emitCaptures = (code, start, rule, own, state, emit) => {
|
|
91
229
|
own.lastIndex = start;
|
|
92
230
|
const m = own.exec(code);
|
|
93
|
-
if (m === null)
|
|
231
|
+
if (m === null) {
|
|
94
232
|
return; // unreachable: the merged regex matched this rule here
|
|
233
|
+
}
|
|
95
234
|
const indices = m.indices;
|
|
96
235
|
const whole = indices[0];
|
|
97
236
|
const captures = rule.captures;
|
|
@@ -100,20 +239,25 @@ const emitCaptures = (code, start, rule, state, emit) => {
|
|
|
100
239
|
const limit = whole[1];
|
|
101
240
|
for (let i = 0; i < captures.length; i++) {
|
|
102
241
|
const span = indices[i + 1];
|
|
103
|
-
if (span === undefined)
|
|
242
|
+
if (span === undefined) {
|
|
104
243
|
continue; // group did not participate
|
|
244
|
+
}
|
|
105
245
|
const to = span[1] > limit ? limit : span[1]; // a lookahead group can overrun
|
|
106
|
-
if (to <= cursor)
|
|
246
|
+
if (to <= cursor) {
|
|
107
247
|
continue; // already covered by an enclosing group
|
|
248
|
+
}
|
|
108
249
|
const from = span[0] > cursor ? span[0] : cursor;
|
|
109
|
-
if (from > cursor)
|
|
250
|
+
if (from > cursor) {
|
|
110
251
|
emit(fallback, cursor, from);
|
|
111
|
-
|
|
252
|
+
}
|
|
253
|
+
if (to > from) {
|
|
112
254
|
emit(captures[i] ?? fallback, from, to); // a group can match empty
|
|
255
|
+
}
|
|
113
256
|
cursor = to;
|
|
114
257
|
}
|
|
115
|
-
if (limit > cursor)
|
|
258
|
+
if (limit > cursor) {
|
|
116
259
|
emit(fallback, cursor, limit);
|
|
260
|
+
}
|
|
117
261
|
};
|
|
118
262
|
/**
|
|
119
263
|
* Collects tokens into an array.
|