@scalar/highlight 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/CHANGELOG.md +49 -0
  2. package/README.md +52 -13
  3. package/dist/all.d.ts.map +1 -1
  4. package/dist/all.js +2 -0
  5. package/dist/compat/hast.d.ts.map +1 -1
  6. package/dist/compat/hast.js +4 -2
  7. package/dist/compat/hljs.d.ts.map +1 -1
  8. package/dist/compat/hljs.js +2 -1
  9. package/dist/compat/languages.d.ts.map +1 -1
  10. package/dist/compat/languages.js +2 -4
  11. package/dist/compat/syntax-highlight.d.ts.map +1 -1
  12. package/dist/compat/syntax-highlight.js +107 -29
  13. package/dist/core/compile.d.ts +9 -1
  14. package/dist/core/compile.d.ts.map +1 -1
  15. package/dist/core/compile.js +128 -10
  16. package/dist/core/index.d.ts +1 -0
  17. package/dist/core/index.d.ts.map +1 -1
  18. package/dist/core/index.js +1 -0
  19. package/dist/core/registry.d.ts.map +1 -1
  20. package/dist/core/registry.js +8 -4
  21. package/dist/core/render.d.ts.map +1 -1
  22. package/dist/core/render.js +12 -6
  23. package/dist/core/tokenize.d.ts.map +1 -1
  24. package/dist/core/tokenize.js +161 -17
  25. package/dist/core/types.d.ts +74 -0
  26. package/dist/core/types.d.ts.map +1 -1
  27. package/dist/core/types.js +5 -1
  28. package/dist/index.d.ts +1 -0
  29. package/dist/index.d.ts.map +1 -1
  30. package/dist/index.js +1 -0
  31. package/dist/langs/bash.d.ts.map +1 -1
  32. package/dist/langs/bash.js +60 -13
  33. package/dist/langs/csharp.d.ts.map +1 -1
  34. package/dist/langs/csharp.js +11 -5
  35. package/dist/langs/css.d.ts.map +1 -1
  36. package/dist/langs/css.js +84 -7
  37. package/dist/langs/go.d.ts.map +1 -1
  38. package/dist/langs/go.js +7 -1
  39. package/dist/langs/html.d.ts +0 -10
  40. package/dist/langs/html.d.ts.map +1 -1
  41. package/dist/langs/html.js +63 -24
  42. package/dist/langs/http.d.ts.map +1 -1
  43. package/dist/langs/http.js +25 -9
  44. package/dist/langs/javascript.d.ts.map +1 -1
  45. package/dist/langs/javascript.js +146 -30
  46. package/dist/langs/json.d.ts.map +1 -1
  47. package/dist/langs/json.js +7 -1
  48. package/dist/langs/julia.d.ts +4 -0
  49. package/dist/langs/julia.d.ts.map +1 -0
  50. package/dist/langs/julia.js +181 -0
  51. package/dist/langs/markdown.d.ts +9 -3
  52. package/dist/langs/markdown.d.ts.map +1 -1
  53. package/dist/langs/markdown.js +127 -6
  54. package/dist/langs/ocaml.d.ts.map +1 -1
  55. package/dist/langs/ocaml.js +16 -7
  56. package/dist/langs/php.d.ts.map +1 -1
  57. package/dist/langs/php.js +69 -19
  58. package/dist/langs/r.js +9 -9
  59. package/dist/langs/rust.d.ts.map +1 -1
  60. package/dist/langs/rust.js +19 -10
  61. package/dist/langs/sql.d.ts.map +1 -1
  62. package/dist/langs/sql.js +9 -3
  63. package/dist/langs/yaml.d.ts +0 -7
  64. package/dist/langs/yaml.d.ts.map +1 -1
  65. package/dist/langs/yaml.js +157 -28
  66. package/dist/lazy.d.ts +1 -0
  67. package/dist/lazy.d.ts.map +1 -1
  68. package/dist/lazy.js +6 -2
  69. package/package.json +3 -3
@@ -1,3 +1,4 @@
1
+ import { REMEMBERED } from './types.js';
1
2
  /**
2
3
  * Compiles a grammar into one merged regex per state.
3
4
  *
@@ -21,24 +22,27 @@ const countGroups = (src) => {
21
22
  continue;
22
23
  }
23
24
  if (inClass) {
24
- if (c === 93 /* ] */)
25
+ if (c === 93 /* ] */) {
25
26
  inClass = false;
27
+ }
26
28
  continue;
27
29
  }
28
30
  if (c === 91 /* [ */) {
29
31
  inClass = true;
30
32
  continue;
31
33
  }
32
- if (c !== 40 /* ( */)
34
+ if (c !== 40 /* ( */) {
33
35
  continue;
36
+ }
34
37
  if (src.charCodeAt(i + 1) !== 63 /* ? */) {
35
38
  count++;
36
39
  }
37
40
  else if (src.charCodeAt(i + 2) === 60 /* < */) {
38
41
  // (?<name> captures; (?<= and (?<! are lookbehind.
39
42
  const after = src.charCodeAt(i + 3);
40
- if (after !== 61 /* = */ && after !== 33 /* ! */)
43
+ if (after !== 61 /* = */ && after !== 33 /* ! */) {
41
44
  count++;
45
+ }
42
46
  }
43
47
  }
44
48
  return count;
@@ -46,6 +50,60 @@ const countGroups = (src) => {
46
50
  const patternSource = (match) => {
47
51
  return typeof match === 'string' ? match : match.source;
48
52
  };
53
+ /**
54
+ * Makes a pattern end lines at `\n` (or `\r\n`) only.
55
+ *
56
+ * With the `m` flag, JavaScript's `$` also matches before `\r`, U+2028 and
57
+ * U+2029, and `.` refuses to match them. Grammars are written as if lines end
58
+ * at `\n` — which is also where the line renderer splits — so a lone `\r` or a
59
+ * U+2028 inside a string literal used to end the string there and leave the
60
+ * rest of the line inverted. Rewriting the two tokens once, here, fixes every
61
+ * grammar at the same time:
62
+ *
63
+ * - `.` becomes `[^\r\n]`: it now matches U+2028 and U+2029, and still
64
+ * stops before the `\r` of a CRLF so an escape like `\\.` cannot swallow it
65
+ * - `$` becomes "before `\n` or `\r\n`, or at the end", so CRLF input still
66
+ * ends a line before its `\r`, exactly as it did before
67
+ *
68
+ * `^` stays native. A line that starts after a lone `\r` is harmless, and the
69
+ * engine's own anchor is measurably faster on large input than a lookbehind.
70
+ *
71
+ * Escapes and character classes match as themselves and are kept verbatim, so
72
+ * only a bare `.` or `$` is rewritten.
73
+ */
74
+ const normalizeLineTerminators = (src) => {
75
+ // A single left-to-right scan rather than a tokenizing regex: a pattern for
76
+ // "a character class" retried at every `[` of an unclosed run is O(n²).
77
+ let out = '';
78
+ let inClass = false;
79
+ for (let i = 0; i < src.length; i++) {
80
+ const c = src[i];
81
+ if (c === '\\') {
82
+ out += c + (src[i + 1] ?? '');
83
+ i++;
84
+ }
85
+ else if (inClass) {
86
+ inClass = c !== ']';
87
+ out += c;
88
+ }
89
+ else if (c === '[') {
90
+ inClass = true;
91
+ out += c;
92
+ }
93
+ else if (c === '.') {
94
+ out += '[^\\r\\n]';
95
+ }
96
+ else if (c === '$') {
97
+ out += '(?=\\r?\\n|(?![\\s\\S]))';
98
+ }
99
+ else {
100
+ out += c;
101
+ }
102
+ }
103
+ return out;
104
+ };
105
+ /** Escapes text so it can be spliced into a pattern and match literally. */
106
+ const escapeRegExp = (s) => s.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&');
49
107
  /** Resolves `include` entries into a flat, priority-ordered rule list. */
50
108
  const flatten = (grammarName, states, name, seen) => {
51
109
  if (seen.has(name)) {
@@ -56,8 +114,9 @@ const flatten = (grammarName, states, name, seen) => {
56
114
  // `constructor` would otherwise resolve off Object.prototype and sail past
57
115
  // this check, failing later with something far less legible.
58
116
  const state = Object.hasOwn(states, name) ? states[name] : undefined;
59
- if (!state)
117
+ if (!state) {
60
118
  throw new Error(`[${grammarName}] include of unknown state "${name}"`);
119
+ }
61
120
  const out = [];
62
121
  for (const entry of state.rules) {
63
122
  if ('include' in entry) {
@@ -78,12 +137,17 @@ const compileState = (grammar, name) => {
78
137
  // Group 0 is the whole match, so rule wrapper groups start at 1.
79
138
  let group = 1;
80
139
  const caseFlag = state.ignoreCase ? 'i' : '';
140
+ let dynamic = false;
141
+ const ownSources = [];
81
142
  for (const rule of rules) {
82
- const src = patternSource(rule.match);
83
- if (BACKREF.test(src)) {
143
+ const raw = patternSource(rule.match);
144
+ if (BACKREF.test(raw)) {
84
145
  throw new Error(`[${grammar.name}:${name}] backreferences are not supported (group numbers ` +
85
- `shift when rules are merged): ${src}`);
146
+ `shift when rules are merged): ${raw}`);
86
147
  }
148
+ const src = normalizeLineTerminators(raw);
149
+ const usesMemory = src.includes(REMEMBERED);
150
+ dynamic ||= usesMemory;
87
151
  const captures = Array.isArray(rule.scope) ? rule.scope : null;
88
152
  const inner = countGroups(src);
89
153
  if (captures) {
@@ -91,17 +155,24 @@ const compileState = (grammar, name) => {
91
155
  throw new Error(`[${grammar.name}:${name}] rule scopes ${captures.length} groups but the ` + `pattern has ${inner}: ${src}`);
92
156
  }
93
157
  }
158
+ if (rule.remember !== undefined && (!(rule.push || rule.set) || rule.remember < 1 || rule.remember > inner)) {
159
+ throw new Error(`[${grammar.name}:${name}] remember ${rule.remember} needs a push or set and one of ${inner} groups: ${raw}`);
160
+ }
94
161
  compiled.push({
95
162
  scope: typeof rule.scope === 'string' ? rule.scope : null,
96
163
  captures,
97
164
  // Sticky, so it can be re-run at the position the merged regex chose.
98
- own: captures ? new RegExp(src, `ymd${caseFlag}`) : null,
165
+ // A pattern that depends on the remembered text is built per value.
166
+ own: captures && !usesMemory ? new RegExp(src, `ymd${caseFlag}`) : null,
99
167
  rest: rule.rest ?? null,
100
168
  group,
101
169
  push: rule.push ?? null,
102
170
  set: rule.set ?? null,
103
171
  pop: rule.pop === true ? 1 : (rule.pop ?? 0),
172
+ remember: rule.remember ?? 0,
173
+ rememberIndent: rule.rememberAs === 'indent',
104
174
  });
175
+ ownSources.push(captures && usesMemory ? src : null);
105
176
  sources.push(`(${src})`);
106
177
  group += 1 + inner;
107
178
  }
@@ -109,13 +180,60 @@ const compileState = (grammar, name) => {
109
180
  // resolved per rule instead, which keeps the scan free of that cost.
110
181
  const flags = `gm${caseFlag}`;
111
182
  // An empty state matches nothing: everything falls through to `default`.
112
- const re = new RegExp(sources.length ? sources.join('|') : '(?!)', flags);
113
- return { name, re, rules: compiled, default: state.default ?? null };
183
+ const source = sources.length ? sources.join('|') : '(?!)';
184
+ // A dynamic state still gets a regex of its own, built with nothing
185
+ // remembered, so it can be entered from the root of a fresh tokenize.
186
+ const re = new RegExp(dynamic ? source.split(REMEMBERED).join('') : source, flags);
187
+ return {
188
+ name,
189
+ re,
190
+ rules: compiled,
191
+ default: state.default ?? null,
192
+ dynamic: dynamic ? { source, flags, own: ownSources, cache: new Map() } : null,
193
+ embed: state.embed
194
+ ? {
195
+ language: state.embed.language,
196
+ // Scan holes and the end marker together so a marker inside a hole
197
+ // cannot close the embedded body. Group 1 identifies a hole.
198
+ boundary: new RegExp(normalizeLineTerminators(state.embed.holes ? `(${state.embed.holes})|(?:${state.embed.until})` : state.embed.until), `gm${caseFlag}`),
199
+ holes: state.embed.holes ? new RegExp(normalizeLineTerminators(state.embed.holes), `gm${caseFlag}`) : null,
200
+ }
201
+ : null,
202
+ };
203
+ };
204
+ /**
205
+ * The merged regex of a dynamic state, and its per-rule sticky patterns, with
206
+ * `value` spliced in wherever a rule wrote `REMEMBERED`. Built once per value.
207
+ */
208
+ export const dynamicRegExps = (state, value) => {
209
+ const dyn = state.dynamic;
210
+ let hit = dyn.cache.get(value);
211
+ if (hit === undefined) {
212
+ const literal = escapeRegExp(value);
213
+ const fill = (src) => src.split(REMEMBERED).join(literal);
214
+ const caseFlag = dyn.flags.includes('i') ? 'i' : '';
215
+ hit = {
216
+ re: new RegExp(fill(dyn.source), dyn.flags),
217
+ own: dyn.own.map((src) => (src === null ? null : new RegExp(fill(src), `ymd${caseFlag}`))),
218
+ };
219
+ // A document with thousands of distinct heredoc markers should not grow
220
+ // this without bound.
221
+ if (dyn.cache.size > 64) {
222
+ dyn.cache.clear();
223
+ }
224
+ dyn.cache.set(value, hit);
225
+ }
226
+ return hit;
114
227
  };
115
228
  export const compile = (grammar) => {
116
229
  if (!Object.hasOwn(grammar.states, 'root')) {
117
230
  throw new Error(`[${grammar.name}] grammar has no "root" state`);
118
231
  }
232
+ // An embedding state pops itself once its body is done, and the root is
233
+ // never popped.
234
+ if (grammar.states.root?.embed) {
235
+ throw new Error(`[${grammar.name}] the "root" state cannot embed another language`);
236
+ }
119
237
  // Null-prototype: state names come from grammar data, and the unknown-state
120
238
  // check below has to see `constructor` and `__proto__` as missing rather
121
239
  // than as inherited Object members. Tokenizing looks states up here too.
@@ -6,4 +6,5 @@ export type { Scope } from './scopes.js';
6
6
  export { SCOPES, SCOPE_NAMES, scopeChain, scopeClass } from './scopes.js';
7
7
  export { tokenize as tokenizeWith, tokenizeToArray } from './tokenize.js';
8
8
  export type { CompiledGrammar, Emit, Grammar, Include, Rule, ScopeName, State, StateRule, Token, } from './types.js';
9
+ export { REMEMBERED } from './types.js';
9
10
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAC3D,OAAO,EACL,cAAc,EACd,eAAe,EACf,UAAU,EACV,SAAS,IAAI,aAAa,EAC1B,cAAc,IAAI,kBAAkB,GACrC,MAAM,UAAU,CAAA;AACjB,YAAY,EAAE,KAAK,EAAE,MAAM,UAAU,CAAA;AACrC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AACtE,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AACtE,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,SAAS,CAAA"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAC3D,OAAO,EACL,cAAc,EACd,eAAe,EACf,UAAU,EACV,SAAS,IAAI,aAAa,EAC1B,cAAc,IAAI,kBAAkB,GACrC,MAAM,UAAU,CAAA;AACjB,YAAY,EAAE,KAAK,EAAE,MAAM,UAAU,CAAA;AACrC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AACtE,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AACtE,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,SAAS,CAAA;AAChB,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA"}
@@ -3,3 +3,4 @@ export { getLanguage, isRegistered, listLanguages, registerLanguage, resolveGram
3
3
  export { DEFAULT_PREFIX, escapeAttribute, escapeHtml, highlight as highlightWith, highlightBlock as highlightBlockWith, } from './render.js';
4
4
  export { SCOPES, SCOPE_NAMES, scopeChain, scopeClass } from './scopes.js';
5
5
  export { tokenize as tokenizeWith, tokenizeToArray } from './tokenize.js';
6
+ export { REMEMBERED } from './types.js';
@@ -1 +1 @@
1
- {"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/core/registry.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,SAAS,CAAA;AAYvD,eAAO,MAAM,gBAAgB,GAAI,GAAG,UAAU,OAAO,EAAE,KAAG,IAuBzD,CAAA;AAED,eAAO,MAAM,YAAY,GAAI,MAAM,MAAM,KAAG,OAE3C,CAAA;AAED,gEAAgE;AAChE,eAAO,MAAM,aAAa,QAAO,MAAM,EAEtC,CAAA;AAMD,qEAAqE;AACrE,eAAO,MAAM,WAAW,GAAI,MAAM,MAAM,KAAG,eAAe,GAAG,SAc5D,CAAA;AAED;;;;GAIG;AACH,eAAO,MAAM,cAAc,GAAI,MAAM,MAAM,GAAG,OAAO,GAAG,eAAe,KAAG,eAczE,CAAA"}
1
+ {"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/core/registry.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,SAAS,CAAA;AAYvD,eAAO,MAAM,gBAAgB,GAAI,GAAG,UAAU,OAAO,EAAE,KAAG,IAyBzD,CAAA;AAED,eAAO,MAAM,YAAY,GAAI,MAAM,MAAM,KAAG,OAE3C,CAAA;AAED,gEAAgE;AAChE,eAAO,MAAM,aAAa,QAAO,MAAM,EAEtC,CAAA;AAMD,qEAAqE;AACrE,eAAO,MAAM,WAAW,GAAI,MAAM,MAAM,KAAG,eAAe,GAAG,SAkB5D,CAAA;AAED;;;;GAIG;AACH,eAAO,MAAM,cAAc,GAAI,MAAM,MAAM,GAAG,OAAO,GAAG,eAAe,KAAG,eAgBzE,CAAA"}
@@ -19,8 +19,9 @@ export const registerLanguage = (...grammars) => {
19
19
  if (previous) {
20
20
  for (const key of [previous.name, ...(previous.aliases ?? [])]) {
21
21
  const lower = key.toLowerCase();
22
- if (sources.get(lower) === previous)
22
+ if (sources.get(lower) === previous) {
23
23
  sources.delete(lower);
24
+ }
24
25
  cache.delete(lower);
25
26
  }
26
27
  }
@@ -45,11 +46,13 @@ const isCompiled = (value) => {
45
46
  export const getLanguage = (name) => {
46
47
  const key = name.toLowerCase();
47
48
  const hit = cache.get(key);
48
- if (hit)
49
+ if (hit) {
49
50
  return hit;
51
+ }
50
52
  const grammar = sources.get(key);
51
- if (!grammar)
53
+ if (!grammar) {
52
54
  return undefined;
55
+ }
53
56
  const built = compile(grammar);
54
57
  // Cache under every alias so repeat lookups skip the alias hop.
55
58
  for (const alias of [grammar.name, ...(grammar.aliases ?? [])]) {
@@ -67,8 +70,9 @@ export const resolveGrammar = (lang) => {
67
70
  return isCompiled(lang) ? lang : compile(lang);
68
71
  }
69
72
  const found = getLanguage(lang);
70
- if (found)
73
+ if (found) {
71
74
  return found;
75
+ }
72
76
  const known = listLanguages();
73
77
  throw new Error(`Language "${lang}" is not registered. ` +
74
78
  (known.length
@@ -1 +1 @@
1
- {"version":3,"file":"render.d.ts","sourceRoot":"","sources":["../../src/core/render.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAE9C,MAAM,MAAM,aAAa,GAAG;IAC1B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,qDAAqD;IACrD,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACnC,CAAA;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,cAAc,SAAS,CAAA;AAMpC;;;;;;GAMG;AACH,eAAO,MAAM,UAAU,GAAI,GAAG,MAAM,KAAG,MAUtC,CAAA;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,eAAe,GAAI,GAAG,MAAM,KAAG,MAE3C,CAAA;AA4BD;;;;;;GAMG;AACH,eAAO,MAAM,SAAS,GAAI,MAAM,MAAM,EAAE,SAAS,eAAe,EAAE,UAAS,aAAkB,KAAG,MA8B/F,CAAA;AAoFD,MAAM,MAAM,YAAY,GAAG,aAAa,GAAG;IACzC,oDAAoD;IACpD,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,oCAAoC;IACpC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,uEAAuE;IACvE,YAAY,CAAC,EAAE,OAAO,CAAA;CACvB,CAAA;AAED;;;;;;GAMG;AACH,eAAO,MAAM,cAAc,GAAI,MAAM,MAAM,EAAE,SAAS,eAAe,EAAE,UAAS,YAAiB,KAAG,MAkCnG,CAAA"}
1
+ {"version":3,"file":"render.d.ts","sourceRoot":"","sources":["../../src/core/render.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAE9C,MAAM,MAAM,aAAa,GAAG;IAC1B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,qDAAqD;IACrD,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACnC,CAAA;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,cAAc,SAAS,CAAA;AAMpC;;;;;;GAMG;AACH,eAAO,MAAM,UAAU,GAAI,GAAG,MAAM,KAAG,MAatC,CAAA;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,eAAe,GAAI,GAAG,MAAM,KAAG,MAE3C,CAAA;AA4BD;;;;;;GAMG;AACH,eAAO,MAAM,SAAS,GAAI,MAAM,MAAM,EAAE,SAAS,eAAe,EAAE,UAAS,aAAkB,KAAG,MAgC/F,CAAA;AA0FD,MAAM,MAAM,YAAY,GAAG,aAAa,GAAG;IACzC,oDAAoD;IACpD,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,oCAAoC;IACpC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,uEAAuE;IACvE,YAAY,CAAC,EAAE,OAAO,CAAA;CACvB,CAAA;AAED;;;;;;GAMG;AACH,eAAO,MAAM,cAAc,GAAI,MAAM,MAAM,EAAE,SAAS,eAAe,EAAE,UAAS,YAAiB,KAAG,MAkCnG,CAAA"}
@@ -25,9 +25,11 @@ export const escapeHtml = (s) => {
25
25
  let last = 0;
26
26
  for (let i = 0; i < s.length; i++) {
27
27
  const c = s.charCodeAt(i);
28
- if (c !== AMP && c !== LT && c !== GT)
28
+ if (c !== AMP && c !== LT && c !== GT) {
29
29
  continue;
30
- out += s.slice(last, i) + (c === AMP ? '&amp;' : c === LT ? '&lt;' : '&gt;');
30
+ }
31
+ const angleEntity = c === LT ? '&lt;' : '&gt;';
32
+ out += s.slice(last, i) + (c === AMP ? '&amp;' : angleEntity);
31
33
  last = i + 1;
32
34
  }
33
35
  return last === 0 ? s : out + s.slice(last);
@@ -75,8 +77,9 @@ const openTag = (tags, prefix, scope) => {
75
77
  */
76
78
  export const highlight = (code, grammar, options = {}) => {
77
79
  const prefix = escapeAttribute(options.classPrefix ?? DEFAULT_PREFIX);
78
- if (options.lines)
80
+ if (options.lines) {
79
81
  return renderLines(code, grammar, prefix, options.highlightLines).html;
82
+ }
80
83
  const tags = tagsFor(prefix);
81
84
  let out = '';
82
85
  // Pending run of same-scoped ranges, flushed when the scope changes.
@@ -128,8 +131,9 @@ const renderLines = (code, grammar, prefix, highlightLines) => {
128
131
  let from = 0;
129
132
  let to = 0;
130
133
  const flush = () => {
131
- if (to > from)
134
+ if (to > from) {
132
135
  line += span(code, scope, from, to, prefix);
136
+ }
133
137
  from = to;
134
138
  };
135
139
  const endLine = () => {
@@ -150,8 +154,9 @@ const renderLines = (code, grammar, prefix, highlightLines) => {
150
154
  }
151
155
  to = stop;
152
156
  }
153
- if (stop === end)
157
+ if (stop === end) {
154
158
  break;
159
+ }
155
160
  endLine();
156
161
  at = stop + 1;
157
162
  scope = null;
@@ -163,8 +168,9 @@ const renderLines = (code, grammar, prefix, highlightLines) => {
163
168
  // A trailing newline in the source closes the last line rather than starting
164
169
  // an empty one.
165
170
  const endsWithNewline = code.endsWith('\n');
166
- if (endsWithNewline && lines[lines.length - 1] === '')
171
+ if (endsWithNewline && lines[lines.length - 1] === '') {
167
172
  lines.pop();
173
+ }
168
174
  let out = '';
169
175
  for (let i = 0; i < lines.length; i++) {
170
176
  const hl = marked?.has(i + 1) ? ' data-hl' : '';
@@ -1 +1 @@
1
- {"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../src/core/tokenize.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAA+B,IAAI,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AAExF;;;;;;GAMG;AACH,eAAO,MAAM,QAAQ,GAAI,MAAM,MAAM,EAAE,SAAS,eAAe,EAAE,MAAM,IAAI,KAAG,IAoE7E,CAAA;AA2CD;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,GAAI,MAAM,MAAM,EAAE,SAAS,eAAe,KAAG,KAAK,EAM7E,CAAA"}
1
+ {"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../src/core/tokenize.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAA+B,IAAI,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AAGxF;;;;;;GAMG;AACH,eAAO,MAAM,QAAQ,GAAI,MAAM,MAAM,EAAE,SAAS,eAAe,EAAE,MAAM,IAAI,KAAG,IA4G7E,CAAA;AAgLD;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,GAAI,MAAM,MAAM,EAAE,SAAS,eAAe,KAAG,KAAK,EAM7E,CAAA"}
@@ -1,3 +1,6 @@
1
+ import { dynamicRegExps } from './compile.js';
2
+ import { getLanguage } from './registry.js';
3
+ import { REMEMBERED } from './types.js';
1
4
  /**
2
5
  * Runs the state machine over `code`, calling `emit` for every range.
3
6
  *
@@ -8,17 +11,36 @@
8
11
  export const tokenize = (code, grammar, emit) => {
9
12
  const len = code.length;
10
13
  const stack = [grammar.root];
14
+ // What each stack entry remembered (see `Rule.remember`), in step with `stack`.
15
+ const memory = [''];
11
16
  let top = grammar.root;
12
17
  let pos = 0;
13
18
  // Zero-length matches are legal (a pure state transition on a lookahead),
14
19
  // but a rule that neither consumes nor eventually changes state would spin.
15
20
  let spin = 0;
16
21
  while (pos < len) {
17
- const re = top.re;
22
+ if (top.embed !== null) {
23
+ pos = embedded(code, pos, top, memory[memory.length - 1], grammar, emit);
24
+ // An embedding state pops itself. A rule that `set` into it from the
25
+ // only state on the stack leaves nothing below, so the root takes over.
26
+ if (stack.length > 1) {
27
+ stack.length -= 1;
28
+ memory.length -= 1;
29
+ }
30
+ else {
31
+ stack[0] = grammar.root;
32
+ memory[0] = '';
33
+ }
34
+ top = stack[stack.length - 1];
35
+ continue;
36
+ }
37
+ const dyn = top.dynamic === null ? null : dynamicRegExps(top, memory[memory.length - 1]);
38
+ const re = dyn === null ? top.re : dyn.re;
18
39
  re.lastIndex = pos;
19
40
  const m = re.exec(code);
20
- if (m === null)
41
+ if (m === null) {
21
42
  break;
43
+ }
22
44
  const start = m.index;
23
45
  if (start > pos) {
24
46
  emit(top.default, pos, start);
@@ -26,34 +48,50 @@ export const tokenize = (code, grammar, emit) => {
26
48
  }
27
49
  const rules = top.rules;
28
50
  let rule;
29
- for (let i = 0; i < rules.length; i++) {
30
- const r = rules[i];
31
- if (m[r.group] !== undefined) {
32
- rule = r;
51
+ let index = 0;
52
+ for (; index < rules.length; index++) {
53
+ if (m[rules[index].group] !== undefined) {
54
+ rule = rules[index];
33
55
  break;
34
56
  }
35
57
  }
36
- if (rule === undefined)
58
+ if (rule === undefined) {
37
59
  break; // unreachable: some alternative matched
60
+ }
38
61
  const end = start + m[0].length;
39
62
  if (end > start) {
40
63
  if (rule.captures !== null) {
41
- emitCaptures(code, start, rule, top, emit);
64
+ // Only rules that use the remembered text are rebuilt per value; the
65
+ // rest of a dynamic state's capture rules keep their own pattern.
66
+ emitCaptures(code, start, rule, dyn?.own[index] ?? rule.own, top, emit);
42
67
  }
43
68
  else {
44
69
  emit(rule.scope ?? top.default, start, end);
45
70
  }
46
71
  }
72
+ // Read before the stack changes: an entered state inherits the memory of
73
+ // the state that entered it unless the rule remembers something new.
74
+ let remembered = memory[memory.length - 1];
75
+ if (rule.remember > 0) {
76
+ const text = m[rule.group + rule.remember] ?? '';
77
+ remembered = rule.rememberIndent ? ' '.repeat(text.length) : text;
78
+ }
47
79
  if (rule.pop > 0) {
48
80
  // Never pop the root state off.
49
81
  const n = Math.min(rule.pop, stack.length - 1);
50
82
  stack.length -= n;
83
+ memory.length -= n;
84
+ if (rule.remember === 0) {
85
+ remembered = memory[memory.length - 1];
86
+ }
51
87
  }
52
88
  if (rule.set !== null) {
53
89
  stack[stack.length - 1] = grammar.states[rule.set];
90
+ memory[memory.length - 1] = remembered;
54
91
  }
55
92
  if (rule.push !== null) {
56
93
  stack.push(grammar.states[rule.push]);
94
+ memory.push(remembered);
57
95
  }
58
96
  top = stack[stack.length - 1];
59
97
  if (end > pos) {
@@ -69,8 +107,109 @@ export const tokenize = (code, grammar, emit) => {
69
107
  spin = 0;
70
108
  }
71
109
  }
72
- if (pos < len)
110
+ if (pos < len) {
73
111
  emit(top.default, pos, len);
112
+ }
113
+ };
114
+ /**
115
+ * How deep embeds may nest before a body falls back to its state's default
116
+ * scope. Real code nests two or three deep — Markdown, then HTML, then
117
+ * JavaScript — but a Markdown fence can embed Markdown, and a run of
118
+ * ` ```md ` lines would otherwise recurse once per line until the stack runs
119
+ * out.
120
+ */
121
+ const MAX_EMBED_DEPTH = 8;
122
+ /** How many embedded bodies are being tokenized right now, one inside another. */
123
+ let depth = 0;
124
+ /**
125
+ * Tokenizes the body of an embedding state with its language and returns the
126
+ * position after it.
127
+ *
128
+ * The body runs to the first match of the state's `until`, which is left for
129
+ * the state below to tokenize. The slice is tokenized on its own, so the
130
+ * embedded grammar starts in its root state and cannot read past the body —
131
+ * an unterminated string in a `<script>` cannot swallow the `</script>`.
132
+ */
133
+ const embedded = (code, pos, state, remembered, host, emit) => {
134
+ const boundary = state.embed.boundary;
135
+ boundary.lastIndex = pos;
136
+ let found = boundary.exec(code);
137
+ while (state.embed.holes !== null && found?.[1] !== undefined) {
138
+ // An empty hole must not prevent the scanner from making progress.
139
+ if (found[0].length === 0) {
140
+ boundary.lastIndex += code.codePointAt(boundary.lastIndex) > 0xffff ? 2 : 1;
141
+ }
142
+ found = boundary.exec(code);
143
+ }
144
+ const stop = found === null ? code.length : found.index;
145
+ if (stop === pos) {
146
+ return pos;
147
+ }
148
+ // `{{remembered}}` takes the language from the text the entering rule
149
+ // remembered, which is how a Markdown fence names its language at runtime.
150
+ const language = state.embed.language;
151
+ const grammar = getLanguage(language === REMEMBERED ? remembered : language);
152
+ if (grammar === undefined || depth >= MAX_EMBED_DEPTH) {
153
+ emit(state.default, pos, stop);
154
+ }
155
+ else {
156
+ depth += 1;
157
+ try {
158
+ withHoles(code.slice(pos, stop), pos, grammar, state.embed.holes, host, emit);
159
+ }
160
+ finally {
161
+ depth -= 1;
162
+ }
163
+ }
164
+ return stop;
165
+ };
166
+ /**
167
+ * Tokenizes an embedded body, around any `holes` that belong to the host
168
+ * grammar.
169
+ *
170
+ * The embedded grammar reads the body with every hole character except line
171
+ * breaks replaced by `_`, an identifier character in the languages that get
172
+ * embedded, so a hole reads as a name and the embedded state runs on across
173
+ * it. Its tokens are then emitted around the holes, and each hole is
174
+ * tokenized with the host grammar in its place.
175
+ */
176
+ const withHoles = (body, offset, grammar, holes, host, emit) => {
177
+ // Flat [start, end, start, end, …] offsets into `body`.
178
+ const ranges = [];
179
+ const pieces = [];
180
+ let cursor = 0;
181
+ if (holes !== null) {
182
+ holes.lastIndex = 0;
183
+ for (const match of body.matchAll(holes)) {
184
+ const start = match.index;
185
+ const end = start + match[0].length;
186
+ if (start === end) {
187
+ continue;
188
+ }
189
+ ranges.push(start, end);
190
+ pieces.push(body.slice(cursor, start), match[0].replace(/[^\r\n]/g, '_'));
191
+ cursor = end;
192
+ }
193
+ }
194
+ const masked = ranges.length === 0 ? body : pieces.join('') + body.slice(cursor);
195
+ let next = 0;
196
+ // Everything before `done` has been emitted, by one grammar or the other.
197
+ let done = 0;
198
+ tokenize(masked, grammar, (scope, from, to) => {
199
+ for (let at = Math.max(from, done); at < to; at = done) {
200
+ const start = ranges[next] ?? to;
201
+ if (start <= at) {
202
+ const end = ranges[next + 1];
203
+ tokenize(body.slice(start, end), host, (s, a, b) => emit(s, a + start + offset, b + start + offset));
204
+ next += 2;
205
+ done = end;
206
+ }
207
+ else {
208
+ done = Math.min(to, start);
209
+ emit(scope, at + offset, done + offset);
210
+ }
211
+ }
212
+ });
74
213
  };
75
214
  /**
76
215
  * Re-runs the winning rule on its own to find where its capture groups sit.
@@ -86,12 +225,12 @@ export const tokenize = (code, grammar, emit) => {
86
225
  * either verbatim would repeat text, or claim text the tokenizer is about to
87
226
  * visit again, so a group only ever scopes what the match still has left.
88
227
  */
89
- const emitCaptures = (code, start, rule, state, emit) => {
90
- const own = rule.own;
228
+ const emitCaptures = (code, start, rule, own, state, emit) => {
91
229
  own.lastIndex = start;
92
230
  const m = own.exec(code);
93
- if (m === null)
231
+ if (m === null) {
94
232
  return; // unreachable: the merged regex matched this rule here
233
+ }
95
234
  const indices = m.indices;
96
235
  const whole = indices[0];
97
236
  const captures = rule.captures;
@@ -100,20 +239,25 @@ const emitCaptures = (code, start, rule, state, emit) => {
100
239
  const limit = whole[1];
101
240
  for (let i = 0; i < captures.length; i++) {
102
241
  const span = indices[i + 1];
103
- if (span === undefined)
242
+ if (span === undefined) {
104
243
  continue; // group did not participate
244
+ }
105
245
  const to = span[1] > limit ? limit : span[1]; // a lookahead group can overrun
106
- if (to <= cursor)
246
+ if (to <= cursor) {
107
247
  continue; // already covered by an enclosing group
248
+ }
108
249
  const from = span[0] > cursor ? span[0] : cursor;
109
- if (from > cursor)
250
+ if (from > cursor) {
110
251
  emit(fallback, cursor, from);
111
- if (to > from)
252
+ }
253
+ if (to > from) {
112
254
  emit(captures[i] ?? fallback, from, to); // a group can match empty
255
+ }
113
256
  cursor = to;
114
257
  }
115
- if (limit > cursor)
258
+ if (limit > cursor) {
116
259
  emit(fallback, cursor, limit);
260
+ }
117
261
  };
118
262
  /**
119
263
  * Collects tokens into an array.