@scalar/highlight 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +59 -0
- package/README.md +52 -13
- package/dist/all.d.ts.map +1 -1
- package/dist/all.js +2 -0
- package/dist/compat/languages.d.ts.map +1 -1
- package/dist/compat/languages.js +2 -4
- package/dist/compat/syntax-highlight.d.ts.map +1 -1
- package/dist/compat/syntax-highlight.js +101 -27
- package/dist/core/compile.d.ts +9 -1
- package/dist/core/compile.d.ts.map +1 -1
- package/dist/core/compile.js +168 -9
- package/dist/core/index.d.ts +1 -0
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +1 -0
- package/dist/core/tokenize.d.ts.map +1 -1
- package/dist/core/tokenize.js +147 -7
- package/dist/core/types.d.ts +76 -1
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js +5 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/langs/bash.d.ts.map +1 -1
- package/dist/langs/bash.js +60 -13
- package/dist/langs/cpp.d.ts.map +1 -1
- package/dist/langs/cpp.js +8 -0
- package/dist/langs/csharp.d.ts.map +1 -1
- package/dist/langs/csharp.js +11 -5
- package/dist/langs/css.d.ts.map +1 -1
- package/dist/langs/css.js +111 -9
- package/dist/langs/diff.d.ts +2 -1
- package/dist/langs/diff.d.ts.map +1 -1
- package/dist/langs/diff.js +19 -7
- package/dist/langs/go.d.ts.map +1 -1
- package/dist/langs/go.js +22 -1
- package/dist/langs/haskell.d.ts.map +1 -1
- package/dist/langs/haskell.js +33 -0
- package/dist/langs/html.d.ts +0 -10
- package/dist/langs/html.d.ts.map +1 -1
- package/dist/langs/html.js +63 -24
- package/dist/langs/http.d.ts.map +1 -1
- package/dist/langs/http.js +25 -9
- package/dist/langs/javascript.d.ts.map +1 -1
- package/dist/langs/javascript.js +279 -30
- package/dist/langs/json.d.ts.map +1 -1
- package/dist/langs/json.js +7 -1
- package/dist/langs/julia.d.ts +4 -0
- package/dist/langs/julia.d.ts.map +1 -0
- package/dist/langs/julia.js +181 -0
- package/dist/langs/kotlin.d.ts.map +1 -1
- package/dist/langs/kotlin.js +8 -1
- package/dist/langs/markdown.d.ts +9 -3
- package/dist/langs/markdown.d.ts.map +1 -1
- package/dist/langs/markdown.js +127 -6
- package/dist/langs/nginx.d.ts.map +1 -1
- package/dist/langs/nginx.js +26 -1
- package/dist/langs/ocaml.d.ts.map +1 -1
- package/dist/langs/ocaml.js +16 -7
- package/dist/langs/php.d.ts.map +1 -1
- package/dist/langs/php.js +69 -19
- package/dist/langs/python.d.ts.map +1 -1
- package/dist/langs/python.js +9 -2
- package/dist/langs/r.js +9 -9
- package/dist/langs/ruby.d.ts.map +1 -1
- package/dist/langs/ruby.js +8 -7
- package/dist/langs/rust.d.ts.map +1 -1
- package/dist/langs/rust.js +27 -11
- package/dist/langs/sql.d.ts.map +1 -1
- package/dist/langs/sql.js +9 -3
- package/dist/langs/swift.d.ts.map +1 -1
- package/dist/langs/swift.js +27 -1
- package/dist/langs/yaml.d.ts +0 -7
- package/dist/langs/yaml.d.ts.map +1 -1
- package/dist/langs/yaml.js +157 -28
- package/dist/lazy.d.ts +1 -0
- package/dist/lazy.d.ts.map +1 -1
- package/dist/lazy.js +2 -0
- package/package.json +2 -2
package/dist/core/compile.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { REMEMBERED } from './types.js';
|
|
1
2
|
/**
|
|
2
3
|
* Compiles a grammar into one merged regex per state.
|
|
3
4
|
*
|
|
@@ -7,7 +8,8 @@
|
|
|
7
8
|
* is skipped inside the regex engine rather than by a JS loop.
|
|
8
9
|
*
|
|
9
10
|
* Each rule is wrapped in its own group so that, after a match, we can tell
|
|
10
|
-
* which alternative won by finding the first defined wrapper group.
|
|
11
|
+
* which alternative won by finding the first defined wrapper group. That
|
|
12
|
+
* wrapper is the only group that captures; see `uncapture`.
|
|
11
13
|
*/
|
|
12
14
|
const BACKREF = /\\(?:[1-9]|k<)/;
|
|
13
15
|
/** Counts capture groups in a pattern, ignoring escapes and character classes. */
|
|
@@ -46,9 +48,107 @@ const countGroups = (src) => {
|
|
|
46
48
|
}
|
|
47
49
|
return count;
|
|
48
50
|
};
|
|
51
|
+
/**
|
|
52
|
+
* Turns every capture group in a pattern into a non-capturing one.
|
|
53
|
+
*
|
|
54
|
+
* Only the wrapper group around each rule has to capture in the merged regex,
|
|
55
|
+
* to say which rule won. A rule's own groups are read back from its sticky
|
|
56
|
+
* `own` pattern, so in the merged regex they would only cost: V8 keeps the
|
|
57
|
+
* match offsets of up to 63 groups in a preallocated buffer and allocates a
|
|
58
|
+
* fresh one on every `exec` past that, which is a fifth of the throughput of
|
|
59
|
+
* a large state such as JavaScript's.
|
|
60
|
+
*/
|
|
61
|
+
const uncapture = (src) => {
|
|
62
|
+
let out = '';
|
|
63
|
+
let inClass = false;
|
|
64
|
+
for (let i = 0; i < src.length; i++) {
|
|
65
|
+
const c = src[i];
|
|
66
|
+
if (c === '\\') {
|
|
67
|
+
out += c + (src[i + 1] ?? '');
|
|
68
|
+
i++;
|
|
69
|
+
}
|
|
70
|
+
else if (inClass) {
|
|
71
|
+
inClass = c !== ']';
|
|
72
|
+
out += c;
|
|
73
|
+
}
|
|
74
|
+
else if (c === '[') {
|
|
75
|
+
inClass = true;
|
|
76
|
+
out += c;
|
|
77
|
+
}
|
|
78
|
+
else if (c !== '(') {
|
|
79
|
+
out += c;
|
|
80
|
+
}
|
|
81
|
+
else if (src[i + 1] !== '?') {
|
|
82
|
+
out += '(?:';
|
|
83
|
+
}
|
|
84
|
+
else if (src[i + 2] === '<' && src[i + 3] !== '=' && src[i + 3] !== '!' && src.includes('>', i)) {
|
|
85
|
+
// `(?<name>` — drop the name along with the capture.
|
|
86
|
+
i = src.indexOf('>', i);
|
|
87
|
+
out += '(?:';
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
out += c;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
return out;
|
|
94
|
+
};
|
|
49
95
|
const patternSource = (match) => {
|
|
50
96
|
return typeof match === 'string' ? match : match.source;
|
|
51
97
|
};
|
|
98
|
+
/**
|
|
99
|
+
* Makes a pattern end lines at `\n` (or `\r\n`) only.
|
|
100
|
+
*
|
|
101
|
+
* With the `m` flag, JavaScript's `$` also matches before `\r`, U+2028 and
|
|
102
|
+
* U+2029, and `.` refuses to match them. Grammars are written as if lines end
|
|
103
|
+
* at `\n` — which is also where the line renderer splits — so a lone `\r` or a
|
|
104
|
+
* U+2028 inside a string literal used to end the string there and leave the
|
|
105
|
+
* rest of the line inverted. Rewriting the two tokens once, here, fixes every
|
|
106
|
+
* grammar at the same time:
|
|
107
|
+
*
|
|
108
|
+
* - `.` becomes `[^\r\n]`: it now matches U+2028 and U+2029, and still
|
|
109
|
+
* stops before the `\r` of a CRLF so an escape like `\\.` cannot swallow it
|
|
110
|
+
* - `$` becomes "before `\n` or `\r\n`, or at the end", so CRLF input still
|
|
111
|
+
* ends a line before its `\r`, exactly as it did before
|
|
112
|
+
*
|
|
113
|
+
* `^` stays native. A line that starts after a lone `\r` is harmless, and the
|
|
114
|
+
* engine's own anchor is measurably faster on large input than a lookbehind.
|
|
115
|
+
*
|
|
116
|
+
* Escapes and character classes match as themselves and are kept verbatim, so
|
|
117
|
+
* only a bare `.` or `$` is rewritten.
|
|
118
|
+
*/
|
|
119
|
+
const normalizeLineTerminators = (src) => {
|
|
120
|
+
// A single left-to-right scan rather than a tokenizing regex: a pattern for
|
|
121
|
+
// "a character class" retried at every `[` of an unclosed run is O(n²).
|
|
122
|
+
let out = '';
|
|
123
|
+
let inClass = false;
|
|
124
|
+
for (let i = 0; i < src.length; i++) {
|
|
125
|
+
const c = src[i];
|
|
126
|
+
if (c === '\\') {
|
|
127
|
+
out += c + (src[i + 1] ?? '');
|
|
128
|
+
i++;
|
|
129
|
+
}
|
|
130
|
+
else if (inClass) {
|
|
131
|
+
inClass = c !== ']';
|
|
132
|
+
out += c;
|
|
133
|
+
}
|
|
134
|
+
else if (c === '[') {
|
|
135
|
+
inClass = true;
|
|
136
|
+
out += c;
|
|
137
|
+
}
|
|
138
|
+
else if (c === '.') {
|
|
139
|
+
out += '[^\\r\\n]';
|
|
140
|
+
}
|
|
141
|
+
else if (c === '$') {
|
|
142
|
+
out += '(?=\\r?\\n|(?![\\s\\S]))';
|
|
143
|
+
}
|
|
144
|
+
else {
|
|
145
|
+
out += c;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return out;
|
|
149
|
+
};
|
|
150
|
+
/** Escapes text so it can be spliced into a pattern and match literally. */
|
|
151
|
+
const escapeRegExp = (s) => s.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&');
|
|
52
152
|
/** Resolves `include` entries into a flat, priority-ordered rule list. */
|
|
53
153
|
const flatten = (grammarName, states, name, seen) => {
|
|
54
154
|
if (seen.has(name)) {
|
|
@@ -82,12 +182,17 @@ const compileState = (grammar, name) => {
|
|
|
82
182
|
// Group 0 is the whole match, so rule wrapper groups start at 1.
|
|
83
183
|
let group = 1;
|
|
84
184
|
const caseFlag = state.ignoreCase ? 'i' : '';
|
|
185
|
+
let dynamic = false;
|
|
186
|
+
const ownSources = [];
|
|
85
187
|
for (const rule of rules) {
|
|
86
|
-
const
|
|
87
|
-
if (BACKREF.test(
|
|
188
|
+
const raw = patternSource(rule.match);
|
|
189
|
+
if (BACKREF.test(raw)) {
|
|
88
190
|
throw new Error(`[${grammar.name}:${name}] backreferences are not supported (group numbers ` +
|
|
89
|
-
`shift when rules are merged): ${
|
|
191
|
+
`shift when rules are merged): ${raw}`);
|
|
90
192
|
}
|
|
193
|
+
const src = normalizeLineTerminators(raw);
|
|
194
|
+
const usesMemory = src.includes(REMEMBERED);
|
|
195
|
+
dynamic ||= usesMemory;
|
|
91
196
|
const captures = Array.isArray(rule.scope) ? rule.scope : null;
|
|
92
197
|
const inner = countGroups(src);
|
|
93
198
|
if (captures) {
|
|
@@ -95,31 +200,85 @@ const compileState = (grammar, name) => {
|
|
|
95
200
|
throw new Error(`[${grammar.name}:${name}] rule scopes ${captures.length} groups but the ` + `pattern has ${inner}: ${src}`);
|
|
96
201
|
}
|
|
97
202
|
}
|
|
203
|
+
if (rule.remember !== undefined && (!(rule.push || rule.set) || rule.remember < 1 || rule.remember > inner)) {
|
|
204
|
+
throw new Error(`[${grammar.name}:${name}] remember ${rule.remember} needs a push or set and one of ${inner} groups: ${raw}`);
|
|
205
|
+
}
|
|
98
206
|
compiled.push({
|
|
99
207
|
scope: typeof rule.scope === 'string' ? rule.scope : null,
|
|
100
208
|
captures,
|
|
101
209
|
// Sticky, so it can be re-run at the position the merged regex chose.
|
|
102
|
-
|
|
210
|
+
// A pattern that depends on the remembered text is built per value.
|
|
211
|
+
own: (captures || rule.remember) && !usesMemory ? new RegExp(src, `ymd${caseFlag}`) : null,
|
|
103
212
|
rest: rule.rest ?? null,
|
|
104
213
|
group,
|
|
105
214
|
push: rule.push ?? null,
|
|
106
215
|
set: rule.set ?? null,
|
|
107
216
|
pop: rule.pop === true ? 1 : (rule.pop ?? 0),
|
|
217
|
+
remember: rule.remember ?? 0,
|
|
218
|
+
rememberIndent: rule.rememberAs === 'indent',
|
|
108
219
|
});
|
|
109
|
-
|
|
110
|
-
|
|
220
|
+
ownSources.push((captures || rule.remember) && usesMemory ? src : null);
|
|
221
|
+
sources.push(`(${uncapture(src)})`);
|
|
222
|
+
group += 1;
|
|
111
223
|
}
|
|
112
224
|
// `m` so grammars can anchor to lines. Deliberately no `d`: offsets are
|
|
113
225
|
// resolved per rule instead, which keeps the scan free of that cost.
|
|
114
226
|
const flags = `gm${caseFlag}`;
|
|
115
227
|
// An empty state matches nothing: everything falls through to `default`.
|
|
116
|
-
const
|
|
117
|
-
|
|
228
|
+
const source = sources.length ? sources.join('|') : '(?!)';
|
|
229
|
+
// A dynamic state still gets a regex of its own, built with nothing
|
|
230
|
+
// remembered, so it can be entered from the root of a fresh tokenize.
|
|
231
|
+
const re = new RegExp(dynamic ? source.split(REMEMBERED).join('') : source, flags);
|
|
232
|
+
return {
|
|
233
|
+
name,
|
|
234
|
+
re,
|
|
235
|
+
rules: compiled,
|
|
236
|
+
default: state.default ?? null,
|
|
237
|
+
dynamic: dynamic ? { source, flags, own: ownSources, cache: new Map() } : null,
|
|
238
|
+
embed: state.embed
|
|
239
|
+
? {
|
|
240
|
+
language: state.embed.language,
|
|
241
|
+
// Scan holes and the end marker together so a marker inside a hole
|
|
242
|
+
// cannot close the embedded body. Group 1 identifies a hole.
|
|
243
|
+
boundary: new RegExp(normalizeLineTerminators(state.embed.holes ? `(${state.embed.holes})|(?:${state.embed.until})` : state.embed.until), `gm${caseFlag}`),
|
|
244
|
+
holes: state.embed.holes ? new RegExp(normalizeLineTerminators(state.embed.holes), `gm${caseFlag}`) : null,
|
|
245
|
+
}
|
|
246
|
+
: null,
|
|
247
|
+
};
|
|
248
|
+
};
|
|
249
|
+
/**
|
|
250
|
+
* The merged regex of a dynamic state, and its per-rule sticky patterns, with
|
|
251
|
+
* `value` spliced in wherever a rule wrote `REMEMBERED`. Built once per value.
|
|
252
|
+
*/
|
|
253
|
+
export const dynamicRegExps = (state, value) => {
|
|
254
|
+
const dyn = state.dynamic;
|
|
255
|
+
let hit = dyn.cache.get(value);
|
|
256
|
+
if (hit === undefined) {
|
|
257
|
+
const literal = escapeRegExp(value);
|
|
258
|
+
const fill = (src) => src.split(REMEMBERED).join(literal);
|
|
259
|
+
const caseFlag = dyn.flags.includes('i') ? 'i' : '';
|
|
260
|
+
hit = {
|
|
261
|
+
re: new RegExp(fill(dyn.source), dyn.flags),
|
|
262
|
+
own: dyn.own.map((src) => (src === null ? null : new RegExp(fill(src), `ymd${caseFlag}`))),
|
|
263
|
+
};
|
|
264
|
+
// A document with thousands of distinct heredoc markers should not grow
|
|
265
|
+
// this without bound.
|
|
266
|
+
if (dyn.cache.size > 64) {
|
|
267
|
+
dyn.cache.clear();
|
|
268
|
+
}
|
|
269
|
+
dyn.cache.set(value, hit);
|
|
270
|
+
}
|
|
271
|
+
return hit;
|
|
118
272
|
};
|
|
119
273
|
export const compile = (grammar) => {
|
|
120
274
|
if (!Object.hasOwn(grammar.states, 'root')) {
|
|
121
275
|
throw new Error(`[${grammar.name}] grammar has no "root" state`);
|
|
122
276
|
}
|
|
277
|
+
// An embedding state pops itself once its body is done, and the root is
|
|
278
|
+
// never popped.
|
|
279
|
+
if (grammar.states.root?.embed) {
|
|
280
|
+
throw new Error(`[${grammar.name}] the "root" state cannot embed another language`);
|
|
281
|
+
}
|
|
123
282
|
// Null-prototype: state names come from grammar data, and the unknown-state
|
|
124
283
|
// check below has to see `constructor` and `__proto__` as missing rather
|
|
125
284
|
// than as inherited Object members. Tokenizing looks states up here too.
|
package/dist/core/index.d.ts
CHANGED
|
@@ -6,4 +6,5 @@ export type { Scope } from './scopes.js';
|
|
|
6
6
|
export { SCOPES, SCOPE_NAMES, scopeChain, scopeClass } from './scopes.js';
|
|
7
7
|
export { tokenize as tokenizeWith, tokenizeToArray } from './tokenize.js';
|
|
8
8
|
export type { CompiledGrammar, Emit, Grammar, Include, Rule, ScopeName, State, StateRule, Token, } from './types.js';
|
|
9
|
+
export { REMEMBERED } from './types.js';
|
|
9
10
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/core/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAC3D,OAAO,EACL,cAAc,EACd,eAAe,EACf,UAAU,EACV,SAAS,IAAI,aAAa,EAC1B,cAAc,IAAI,kBAAkB,GACrC,MAAM,UAAU,CAAA;AACjB,YAAY,EAAE,KAAK,EAAE,MAAM,UAAU,CAAA;AACrC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AACtE,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AACtE,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,SAAS,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAC3D,OAAO,EACL,cAAc,EACd,eAAe,EACf,UAAU,EACV,SAAS,IAAI,aAAa,EAC1B,cAAc,IAAI,kBAAkB,GACrC,MAAM,UAAU,CAAA;AACjB,YAAY,EAAE,KAAK,EAAE,MAAM,UAAU,CAAA;AACrC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AACtE,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AACtE,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,SAAS,CAAA;AAChB,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA"}
|
package/dist/core/index.js
CHANGED
|
@@ -3,3 +3,4 @@ export { getLanguage, isRegistered, listLanguages, registerLanguage, resolveGram
|
|
|
3
3
|
export { DEFAULT_PREFIX, escapeAttribute, escapeHtml, highlight as highlightWith, highlightBlock as highlightBlockWith, } from './render.js';
|
|
4
4
|
export { SCOPES, SCOPE_NAMES, scopeChain, scopeClass } from './scopes.js';
|
|
5
5
|
export { tokenize as tokenizeWith, tokenizeToArray } from './tokenize.js';
|
|
6
|
+
export { REMEMBERED } from './types.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../src/core/tokenize.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../src/core/tokenize.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAA+B,IAAI,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AAGxF;;;;;;GAMG;AACH,eAAO,MAAM,QAAQ,GAAI,MAAM,MAAM,EAAE,SAAS,eAAe,EAAE,MAAM,IAAI,KAAG,IAgH7E,CAAA;AAgLD;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,GAAI,MAAM,MAAM,EAAE,SAAS,eAAe,KAAG,KAAK,EAM7E,CAAA"}
|
package/dist/core/tokenize.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { dynamicRegExps } from './compile.js';
|
|
2
|
+
import { getLanguage } from './registry.js';
|
|
3
|
+
import { REMEMBERED } from './types.js';
|
|
1
4
|
/**
|
|
2
5
|
* Runs the state machine over `code`, calling `emit` for every range.
|
|
3
6
|
*
|
|
@@ -8,13 +11,31 @@
|
|
|
8
11
|
export const tokenize = (code, grammar, emit) => {
|
|
9
12
|
const len = code.length;
|
|
10
13
|
const stack = [grammar.root];
|
|
14
|
+
// What each stack entry remembered (see `Rule.remember`), in step with `stack`.
|
|
15
|
+
const memory = [''];
|
|
11
16
|
let top = grammar.root;
|
|
12
17
|
let pos = 0;
|
|
13
18
|
// Zero-length matches are legal (a pure state transition on a lookahead),
|
|
14
19
|
// but a rule that neither consumes nor eventually changes state would spin.
|
|
15
20
|
let spin = 0;
|
|
16
21
|
while (pos < len) {
|
|
17
|
-
|
|
22
|
+
if (top.embed !== null) {
|
|
23
|
+
pos = embedded(code, pos, top, memory[memory.length - 1], grammar, emit);
|
|
24
|
+
// An embedding state pops itself. A rule that `set` into it from the
|
|
25
|
+
// only state on the stack leaves nothing below, so the root takes over.
|
|
26
|
+
if (stack.length > 1) {
|
|
27
|
+
stack.length -= 1;
|
|
28
|
+
memory.length -= 1;
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
stack[0] = grammar.root;
|
|
32
|
+
memory[0] = '';
|
|
33
|
+
}
|
|
34
|
+
top = stack[stack.length - 1];
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
const dyn = top.dynamic === null ? null : dynamicRegExps(top, memory[memory.length - 1]);
|
|
38
|
+
const re = dyn === null ? top.re : dyn.re;
|
|
18
39
|
re.lastIndex = pos;
|
|
19
40
|
const m = re.exec(code);
|
|
20
41
|
if (m === null) {
|
|
@@ -27,9 +48,10 @@ export const tokenize = (code, grammar, emit) => {
|
|
|
27
48
|
}
|
|
28
49
|
const rules = top.rules;
|
|
29
50
|
let rule;
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
51
|
+
let index = 0;
|
|
52
|
+
for (; index < rules.length; index++) {
|
|
53
|
+
if (m[rules[index].group] !== undefined) {
|
|
54
|
+
rule = rules[index];
|
|
33
55
|
break;
|
|
34
56
|
}
|
|
35
57
|
}
|
|
@@ -39,22 +61,41 @@ export const tokenize = (code, grammar, emit) => {
|
|
|
39
61
|
const end = start + m[0].length;
|
|
40
62
|
if (end > start) {
|
|
41
63
|
if (rule.captures !== null) {
|
|
42
|
-
|
|
64
|
+
// Only rules that use the remembered text are rebuilt per value; the
|
|
65
|
+
// rest of a dynamic state's capture rules keep their own pattern.
|
|
66
|
+
emitCaptures(code, start, rule, dyn?.own[index] ?? rule.own, top, emit);
|
|
43
67
|
}
|
|
44
68
|
else {
|
|
45
69
|
emit(rule.scope ?? top.default, start, end);
|
|
46
70
|
}
|
|
47
71
|
}
|
|
72
|
+
// Read before the stack changes: an entered state inherits the memory of
|
|
73
|
+
// the state that entered it unless the rule remembers something new.
|
|
74
|
+
let remembered = memory[memory.length - 1];
|
|
75
|
+
if (rule.remember > 0) {
|
|
76
|
+
// The merged regex captures only which rule won, so the group comes from
|
|
77
|
+
// the rule's own pattern, re-run where the merged one matched.
|
|
78
|
+
const own = dyn?.own[index] ?? rule.own;
|
|
79
|
+
own.lastIndex = start;
|
|
80
|
+
const text = own.exec(code)?.[rule.remember] ?? '';
|
|
81
|
+
remembered = rule.rememberIndent ? ' '.repeat(text.length) : text;
|
|
82
|
+
}
|
|
48
83
|
if (rule.pop > 0) {
|
|
49
84
|
// Never pop the root state off.
|
|
50
85
|
const n = Math.min(rule.pop, stack.length - 1);
|
|
51
86
|
stack.length -= n;
|
|
87
|
+
memory.length -= n;
|
|
88
|
+
if (rule.remember === 0) {
|
|
89
|
+
remembered = memory[memory.length - 1];
|
|
90
|
+
}
|
|
52
91
|
}
|
|
53
92
|
if (rule.set !== null) {
|
|
54
93
|
stack[stack.length - 1] = grammar.states[rule.set];
|
|
94
|
+
memory[memory.length - 1] = remembered;
|
|
55
95
|
}
|
|
56
96
|
if (rule.push !== null) {
|
|
57
97
|
stack.push(grammar.states[rule.push]);
|
|
98
|
+
memory.push(remembered);
|
|
58
99
|
}
|
|
59
100
|
top = stack[stack.length - 1];
|
|
60
101
|
if (end > pos) {
|
|
@@ -74,6 +115,106 @@ export const tokenize = (code, grammar, emit) => {
|
|
|
74
115
|
emit(top.default, pos, len);
|
|
75
116
|
}
|
|
76
117
|
};
|
|
118
|
+
/**
|
|
119
|
+
* How deep embeds may nest before a body falls back to its state's default
|
|
120
|
+
* scope. Real code nests two or three deep — Markdown, then HTML, then
|
|
121
|
+
* JavaScript — but a Markdown fence can embed Markdown, and a run of
|
|
122
|
+
* ` ```md ` lines would otherwise recurse once per line until the stack runs
|
|
123
|
+
* out.
|
|
124
|
+
*/
|
|
125
|
+
const MAX_EMBED_DEPTH = 8;
|
|
126
|
+
/** How many embedded bodies are being tokenized right now, one inside another. */
|
|
127
|
+
let depth = 0;
|
|
128
|
+
/**
|
|
129
|
+
* Tokenizes the body of an embedding state with its language and returns the
|
|
130
|
+
* position after it.
|
|
131
|
+
*
|
|
132
|
+
* The body runs to the first match of the state's `until`, which is left for
|
|
133
|
+
* the state below to tokenize. The slice is tokenized on its own, so the
|
|
134
|
+
* embedded grammar starts in its root state and cannot read past the body —
|
|
135
|
+
* an unterminated string in a `<script>` cannot swallow the `</script>`.
|
|
136
|
+
*/
|
|
137
|
+
const embedded = (code, pos, state, remembered, host, emit) => {
|
|
138
|
+
const boundary = state.embed.boundary;
|
|
139
|
+
boundary.lastIndex = pos;
|
|
140
|
+
let found = boundary.exec(code);
|
|
141
|
+
while (state.embed.holes !== null && found?.[1] !== undefined) {
|
|
142
|
+
// An empty hole must not prevent the scanner from making progress.
|
|
143
|
+
if (found[0].length === 0) {
|
|
144
|
+
boundary.lastIndex += code.codePointAt(boundary.lastIndex) > 0xffff ? 2 : 1;
|
|
145
|
+
}
|
|
146
|
+
found = boundary.exec(code);
|
|
147
|
+
}
|
|
148
|
+
const stop = found === null ? code.length : found.index;
|
|
149
|
+
if (stop === pos) {
|
|
150
|
+
return pos;
|
|
151
|
+
}
|
|
152
|
+
// `{{remembered}}` takes the language from the text the entering rule
|
|
153
|
+
// remembered, which is how a Markdown fence names its language at runtime.
|
|
154
|
+
const language = state.embed.language;
|
|
155
|
+
const grammar = getLanguage(language === REMEMBERED ? remembered : language);
|
|
156
|
+
if (grammar === undefined || depth >= MAX_EMBED_DEPTH) {
|
|
157
|
+
emit(state.default, pos, stop);
|
|
158
|
+
}
|
|
159
|
+
else {
|
|
160
|
+
depth += 1;
|
|
161
|
+
try {
|
|
162
|
+
withHoles(code.slice(pos, stop), pos, grammar, state.embed.holes, host, emit);
|
|
163
|
+
}
|
|
164
|
+
finally {
|
|
165
|
+
depth -= 1;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return stop;
|
|
169
|
+
};
|
|
170
|
+
/**
|
|
171
|
+
* Tokenizes an embedded body, around any `holes` that belong to the host
|
|
172
|
+
* grammar.
|
|
173
|
+
*
|
|
174
|
+
* The embedded grammar reads the body with every hole character except line
|
|
175
|
+
* breaks replaced by `_`, an identifier character in the languages that get
|
|
176
|
+
* embedded, so a hole reads as a name and the embedded state runs on across
|
|
177
|
+
* it. Its tokens are then emitted around the holes, and each hole is
|
|
178
|
+
* tokenized with the host grammar in its place.
|
|
179
|
+
*/
|
|
180
|
+
const withHoles = (body, offset, grammar, holes, host, emit) => {
|
|
181
|
+
// Flat [start, end, start, end, …] offsets into `body`.
|
|
182
|
+
const ranges = [];
|
|
183
|
+
const pieces = [];
|
|
184
|
+
let cursor = 0;
|
|
185
|
+
if (holes !== null) {
|
|
186
|
+
holes.lastIndex = 0;
|
|
187
|
+
for (const match of body.matchAll(holes)) {
|
|
188
|
+
const start = match.index;
|
|
189
|
+
const end = start + match[0].length;
|
|
190
|
+
if (start === end) {
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
ranges.push(start, end);
|
|
194
|
+
pieces.push(body.slice(cursor, start), match[0].replace(/[^\r\n]/g, '_'));
|
|
195
|
+
cursor = end;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
const masked = ranges.length === 0 ? body : pieces.join('') + body.slice(cursor);
|
|
199
|
+
let next = 0;
|
|
200
|
+
// Everything before `done` has been emitted, by one grammar or the other.
|
|
201
|
+
let done = 0;
|
|
202
|
+
tokenize(masked, grammar, (scope, from, to) => {
|
|
203
|
+
for (let at = Math.max(from, done); at < to; at = done) {
|
|
204
|
+
const start = ranges[next] ?? to;
|
|
205
|
+
if (start <= at) {
|
|
206
|
+
const end = ranges[next + 1];
|
|
207
|
+
tokenize(body.slice(start, end), host, (s, a, b) => emit(s, a + start + offset, b + start + offset));
|
|
208
|
+
next += 2;
|
|
209
|
+
done = end;
|
|
210
|
+
}
|
|
211
|
+
else {
|
|
212
|
+
done = Math.min(to, start);
|
|
213
|
+
emit(scope, at + offset, done + offset);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
});
|
|
217
|
+
};
|
|
77
218
|
/**
|
|
78
219
|
* Re-runs the winning rule on its own to find where its capture groups sit.
|
|
79
220
|
*
|
|
@@ -88,8 +229,7 @@ export const tokenize = (code, grammar, emit) => {
|
|
|
88
229
|
* either verbatim would repeat text, or claim text the tokenizer is about to
|
|
89
230
|
* visit again, so a group only ever scopes what the match still has left.
|
|
90
231
|
*/
|
|
91
|
-
const emitCaptures = (code, start, rule, state, emit) => {
|
|
92
|
-
const own = rule.own;
|
|
232
|
+
const emitCaptures = (code, start, rule, own, state, emit) => {
|
|
93
233
|
own.lastIndex = start;
|
|
94
234
|
const m = own.exec(code);
|
|
95
235
|
if (m === null) {
|
package/dist/core/types.d.ts
CHANGED
|
@@ -35,7 +35,31 @@ export type Rule = {
|
|
|
35
35
|
set?: string;
|
|
36
36
|
/** Pop `n` states (`true` means 1). */
|
|
37
37
|
pop?: number | true;
|
|
38
|
+
/**
|
|
39
|
+
* Remember the text of this capture group (1-based, within this rule's
|
|
40
|
+
* pattern) for the state that `push` or `set` enters.
|
|
41
|
+
*
|
|
42
|
+
* That state's rules can then match it again by writing `REMEMBERED`
|
|
43
|
+
* (`{{remembered}}`) in a pattern. It is the stand-in for a backreference
|
|
44
|
+
* across rules: a heredoc remembers its terminator, a raw string the run of
|
|
45
|
+
* quotes that opened it, a YAML block scalar the indentation of its key.
|
|
46
|
+
*
|
|
47
|
+
* A state entered without `remember` inherits the value of the state that
|
|
48
|
+
* pushed it, so helper states nested inside keep seeing it.
|
|
49
|
+
*/
|
|
50
|
+
remember?: number;
|
|
51
|
+
/**
|
|
52
|
+
* `'text'` (the default) remembers the group verbatim. `'indent'` remembers
|
|
53
|
+
* a run of spaces as wide as the group, which is what an indentation-based
|
|
54
|
+
* end rule wants: `- key: |` remembers the column of `key`, not `- `.
|
|
55
|
+
*/
|
|
56
|
+
rememberAs?: 'text' | 'indent';
|
|
38
57
|
};
|
|
58
|
+
/**
|
|
59
|
+
* The placeholder a pattern uses to match the remembered text again. The text
|
|
60
|
+
* is inserted escaped, so it matches literally.
|
|
61
|
+
*/
|
|
62
|
+
export declare const REMEMBERED = "{{remembered}}";
|
|
39
63
|
/** Splices another state's rules in at this position, keeping priority order. */
|
|
40
64
|
export type Include = {
|
|
41
65
|
include: string;
|
|
@@ -46,6 +70,34 @@ export type State = {
|
|
|
46
70
|
default?: ScopeName;
|
|
47
71
|
/** Compile this state's patterns case-insensitively. */
|
|
48
72
|
ignoreCase?: boolean;
|
|
73
|
+
/**
|
|
74
|
+
* Highlight the body of this state with another registered language.
|
|
75
|
+
*
|
|
76
|
+
* When the state is entered, the text up to the first match of `until` (or
|
|
77
|
+
* the end of the input) is tokenized with `language`, then the state pops.
|
|
78
|
+
* `until` is not consumed, so the state below sees it: a `<script>` body
|
|
79
|
+
* ends where `</script>` starts, and the HTML grammar scopes the tag.
|
|
80
|
+
*
|
|
81
|
+
* The language is looked up in the registry when the state is entered, so
|
|
82
|
+
* embedding costs nothing until that language is registered. Without it,
|
|
83
|
+
* the body is emitted with this state's `default` scope.
|
|
84
|
+
*
|
|
85
|
+
* `language` can be `REMEMBERED` (`{{remembered}}`) to use the text the rule
|
|
86
|
+
* that entered the state remembered: a Markdown fence remembers its info
|
|
87
|
+
* string and embeds whatever language it names.
|
|
88
|
+
*
|
|
89
|
+
* `holes` marks regions inside the body that belong to this grammar rather
|
|
90
|
+
* than the embedded one, such as `<?= $x ?>` inside a PHP template's
|
|
91
|
+
* `<script>`. The embedded language reads the body with each hole masked,
|
|
92
|
+
* so its state carries across (a hole inside a JavaScript string leaves the
|
|
93
|
+
* rest of the string a string), and each hole is tokenized with this
|
|
94
|
+
* grammar from its root.
|
|
95
|
+
*/
|
|
96
|
+
embed?: {
|
|
97
|
+
language: string;
|
|
98
|
+
until: string;
|
|
99
|
+
holes?: string;
|
|
100
|
+
};
|
|
49
101
|
rules: StateRule[];
|
|
50
102
|
};
|
|
51
103
|
export type Grammar = {
|
|
@@ -70,7 +122,8 @@ export type CompiledRule = {
|
|
|
70
122
|
captures: (ScopeName | null)[] | null;
|
|
71
123
|
/**
|
|
72
124
|
* This rule alone, sticky and with `hasIndices`, used to locate its capture
|
|
73
|
-
* groups after the merged regex has
|
|
125
|
+
* groups — and to read the one it remembers — after the merged regex has
|
|
126
|
+
* already picked it. The merged regex does not capture them at all.
|
|
74
127
|
*
|
|
75
128
|
* Asking the merged regex for indices would make every match in the state
|
|
76
129
|
* pay to have offsets computed for every group of every rule; re-running one
|
|
@@ -83,12 +136,34 @@ export type CompiledRule = {
|
|
|
83
136
|
push: string | null;
|
|
84
137
|
set: string | null;
|
|
85
138
|
pop: number;
|
|
139
|
+
/** Capture group of `own` to remember, 1-based; `0` for none. */
|
|
140
|
+
remember: number;
|
|
141
|
+
rememberIndent: boolean;
|
|
86
142
|
};
|
|
87
143
|
export type CompiledState = {
|
|
88
144
|
name: string;
|
|
89
145
|
re: RegExp;
|
|
90
146
|
rules: CompiledRule[];
|
|
91
147
|
default: ScopeName | null;
|
|
148
|
+
/**
|
|
149
|
+
* Set when a rule uses `REMEMBERED`: the merged regex then depends on the
|
|
150
|
+
* remembered text and is built per value, from these sources.
|
|
151
|
+
*/
|
|
152
|
+
dynamic: {
|
|
153
|
+
source: string;
|
|
154
|
+
flags: string;
|
|
155
|
+
/** Per-rule sticky sources for capture offsets, `null` where unused. */
|
|
156
|
+
own: (string | null)[];
|
|
157
|
+
cache: Map<string, {
|
|
158
|
+
re: RegExp;
|
|
159
|
+
own: (RegExp | null)[];
|
|
160
|
+
}>;
|
|
161
|
+
} | null;
|
|
162
|
+
embed: {
|
|
163
|
+
language: string;
|
|
164
|
+
boundary: RegExp;
|
|
165
|
+
holes: RegExp | null;
|
|
166
|
+
} | null;
|
|
92
167
|
};
|
|
93
168
|
export type CompiledGrammar = {
|
|
94
169
|
name: string;
|
package/dist/core/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,4DAA4D;AAC5D,MAAM,MAAM,SAAS,GAAG,MAAM,CAAA;AAE9B,MAAM,MAAM,IAAI,GAAG;IACjB;;;;;;OAMG;IACH,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;IAEtB;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,SAAS,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,EAAE,CAAA;IAExC,kEAAkE;IAClE,IAAI,CAAC,EAAE,SAAS,CAAA;IAEhB,mCAAmC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,gEAAgE;IAChE,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,uCAAuC;IACvC,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,4DAA4D;AAC5D,MAAM,MAAM,SAAS,GAAG,MAAM,CAAA;AAE9B,MAAM,MAAM,IAAI,GAAG;IACjB;;;;;;OAMG;IACH,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;IAEtB;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,SAAS,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,EAAE,CAAA;IAExC,kEAAkE;IAClE,IAAI,CAAC,EAAE,SAAS,CAAA;IAEhB,mCAAmC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,gEAAgE;IAChE,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,uCAAuC;IACvC,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAEnB;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAA;CAC/B,CAAA;AAED;;;GAGG;AACH,eAAO,MAAM,UAAU,mBAAmB,CAAA;AAE1C,iFAAiF;AACjF,MAAM,MAAM,OAAO,GAAG;IACpB,OAAO,EAAE,MAAM,CAAA;CAChB,CAAA;AAED,MAAM,MAAM,SAAS,GAAG,IAAI,GAAG,OAAO,CAAA;AAEtC,MAAM,MAAM,KAAK,GAAG;IAClB,2EAA2E;IAC3E,OAAO,CAAC,EAAE,SAAS,CAAA;IACnB,wDAAwD;IACxD,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,KAAK,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;IAC3D,KAAK,EAAE,SAAS,EAAE,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,CAAC,EAAE,MAAM,EAAE,CAAA;IAClB,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;CAC9B,CAAA;AAED,MAAM,MAAM,KAAK,GAAG;IAClB,qCAAqC;IACrC,KAAK,EAAE,SAAS,GAAG,IAAI,CAAA;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;CACZ,CAAA;AAED,gFAAgF;AAChF,MAAM,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,SAAS,GAAG,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,IAAI,CAAA;AAIhF,MAAM,MAAM,YAAY,GAAG;IACzB,4DAA4D;IAC5D,KAAK,EAAE,SAAS,GAAG,IAAI,CAAA;IACvB,kEAAkE;IAClE,QAAQ,EAAE,CAAC,SAAS,GAAG,IAAI,CAAC,EAAE,GAAG,IAAI,CAAA;IACrC;;;;;;;;OAQG;IACH,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;IAClB,IAAI,EAAE,SAAS,GAAG,IAAI,CAAA;IACtB,gEAAgE;IAChE,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;IAClB,GAAG,EAAE,MAAM,CAAA;IACX,iEAAiE;IACjE,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,OAAO,CAAA;CACxB,CAAA;AAED,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,YAAY,EAAE,CAAA;IACrB,OAAO,EAAE,SAAS,GAAG,IAAI,CAAA;IACzB;;;OAGG;IACH,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAA;QACd,KAAK,EAAE,MAAM,CAAA;QACb,wEAAwE;QACxE,GAAG,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAA;QACtB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,GAAG,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAA;SAAE,CAAC,CAAA;KAC3D,GAAG,IAAI,CAAA;IACR,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAA;CAC3E,CAAA;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;IACrC,IAAI,EAAE,aAAa,CAAA;CACpB,CAAA"}
|
package/dist/core/types.js
CHANGED
|
@@ -7,4 +7,8 @@
|
|
|
7
7
|
* a string state needs three rules (escape, interpolation, closing quote), not
|
|
8
8
|
* a rule for "string content".
|
|
9
9
|
*/
|
|
10
|
-
|
|
10
|
+
/**
|
|
11
|
+
* The placeholder a pattern uses to match the remembered text again. The text
|
|
12
|
+
* is inserted escaped, so it matches literally.
|
|
13
|
+
*/
|
|
14
|
+
export const REMEMBERED = '{{remembered}}';
|
package/dist/index.d.ts
CHANGED
|
@@ -16,4 +16,5 @@ export type { Scope } from './core/scopes.js';
|
|
|
16
16
|
export { SCOPES, SCOPE_NAMES, scopeChain, scopeClass } from './core/scopes.js';
|
|
17
17
|
export { tokenize as tokenizeStream } from './core/tokenize.js';
|
|
18
18
|
export type { CompiledGrammar, Emit, Grammar, Include, Rule, ScopeName, State, StateRule, Token, } from './core/types.js';
|
|
19
|
+
export { REMEMBERED } from './core/types.js';
|
|
19
20
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAGhE,OAAO,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,cAAc,CAAA;AAEnE,iFAAiF;AACjF,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,OAAO,GAAG,eAAe,CAAA;AAE9D,yEAAyE;AACzE,eAAO,MAAM,SAAS,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,EAAE,UAAU,aAAa,KAAG,MAEtF,CAAA;AAED,yDAAyD;AACzD,eAAO,MAAM,cAAc,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,EAAE,UAAU,YAAY,KAAG,MAE1F,CAAA;AAED,uEAAuE;AACvE,eAAO,MAAM,QAAQ,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,KAAG,KAAK,EAEjE,CAAA;AAED,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,iBAAiB,CAAA;AACxB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAChE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,YAAY,EAAE,KAAK,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,OAAO,EAAE,QAAQ,IAAI,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAC5D,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,cAAc,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAGhE,OAAO,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,cAAc,CAAA;AAEnE,iFAAiF;AACjF,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,OAAO,GAAG,eAAe,CAAA;AAE9D,yEAAyE;AACzE,eAAO,MAAM,SAAS,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,EAAE,UAAU,aAAa,KAAG,MAEtF,CAAA;AAED,yDAAyD;AACzD,eAAO,MAAM,cAAc,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,EAAE,UAAU,YAAY,KAAG,MAE1F,CAAA;AAED,uEAAuE;AACvE,eAAO,MAAM,QAAQ,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,KAAG,KAAK,EAEjE,CAAA;AAED,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,iBAAiB,CAAA;AACxB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAChE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,YAAY,EAAE,KAAK,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,OAAO,EAAE,QAAQ,IAAI,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAC5D,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA"}
|
package/dist/index.js
CHANGED
|
@@ -18,3 +18,4 @@ export { getLanguage, isRegistered, listLanguages, registerLanguage, resolveGram
|
|
|
18
18
|
export { DEFAULT_PREFIX, escapeAttribute, escapeHtml } from './core/render.js';
|
|
19
19
|
export { SCOPES, SCOPE_NAMES, scopeChain, scopeClass } from './core/scopes.js';
|
|
20
20
|
export { tokenize as tokenizeStream } from './core/tokenize.js';
|
|
21
|
+
export { REMEMBERED } from './core/types.js';
|
package/dist/langs/bash.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bash.d.ts","sourceRoot":"","sources":["../../src/langs/bash.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"bash.d.ts","sourceRoot":"","sources":["../../src/langs/bash.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAoB5C,QAAA,MAAM,IAAI,EAAE,OAqKX,CAAA;AAED,eAAe,IAAI,CAAA"}
|