@scalar/highlight 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/README.md +52 -13
- package/dist/all.d.ts.map +1 -1
- package/dist/all.js +2 -0
- package/dist/compat/languages.d.ts.map +1 -1
- package/dist/compat/languages.js +2 -4
- package/dist/compat/syntax-highlight.d.ts.map +1 -1
- package/dist/compat/syntax-highlight.js +101 -27
- package/dist/core/compile.d.ts +9 -1
- package/dist/core/compile.d.ts.map +1 -1
- package/dist/core/compile.js +120 -6
- package/dist/core/index.d.ts +1 -0
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +1 -0
- package/dist/core/tokenize.d.ts.map +1 -1
- package/dist/core/tokenize.js +143 -7
- package/dist/core/types.d.ts +74 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js +5 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/langs/bash.d.ts.map +1 -1
- package/dist/langs/bash.js +60 -13
- package/dist/langs/csharp.d.ts.map +1 -1
- package/dist/langs/csharp.js +11 -5
- package/dist/langs/css.d.ts.map +1 -1
- package/dist/langs/css.js +84 -7
- package/dist/langs/go.d.ts.map +1 -1
- package/dist/langs/go.js +7 -1
- package/dist/langs/html.d.ts +0 -10
- package/dist/langs/html.d.ts.map +1 -1
- package/dist/langs/html.js +63 -24
- package/dist/langs/http.d.ts.map +1 -1
- package/dist/langs/http.js +25 -9
- package/dist/langs/javascript.d.ts.map +1 -1
- package/dist/langs/javascript.js +146 -30
- package/dist/langs/json.d.ts.map +1 -1
- package/dist/langs/json.js +7 -1
- package/dist/langs/julia.d.ts +4 -0
- package/dist/langs/julia.d.ts.map +1 -0
- package/dist/langs/julia.js +181 -0
- package/dist/langs/markdown.d.ts +9 -3
- package/dist/langs/markdown.d.ts.map +1 -1
- package/dist/langs/markdown.js +127 -6
- package/dist/langs/ocaml.d.ts.map +1 -1
- package/dist/langs/ocaml.js +16 -7
- package/dist/langs/php.d.ts.map +1 -1
- package/dist/langs/php.js +69 -19
- package/dist/langs/r.js +9 -9
- package/dist/langs/rust.d.ts.map +1 -1
- package/dist/langs/rust.js +19 -10
- package/dist/langs/sql.d.ts.map +1 -1
- package/dist/langs/sql.js +9 -3
- package/dist/langs/yaml.d.ts +0 -7
- package/dist/langs/yaml.d.ts.map +1 -1
- package/dist/langs/yaml.js +157 -28
- package/dist/lazy.d.ts +1 -0
- package/dist/lazy.d.ts.map +1 -1
- package/dist/lazy.js +2 -0
- package/package.json +2 -2
package/dist/core/tokenize.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { dynamicRegExps } from './compile.js';
|
|
2
|
+
import { getLanguage } from './registry.js';
|
|
3
|
+
import { REMEMBERED } from './types.js';
|
|
1
4
|
/**
|
|
2
5
|
* Runs the state machine over `code`, calling `emit` for every range.
|
|
3
6
|
*
|
|
@@ -8,13 +11,31 @@
|
|
|
8
11
|
export const tokenize = (code, grammar, emit) => {
|
|
9
12
|
const len = code.length;
|
|
10
13
|
const stack = [grammar.root];
|
|
14
|
+
// What each stack entry remembered (see `Rule.remember`), in step with `stack`.
|
|
15
|
+
const memory = [''];
|
|
11
16
|
let top = grammar.root;
|
|
12
17
|
let pos = 0;
|
|
13
18
|
// Zero-length matches are legal (a pure state transition on a lookahead),
|
|
14
19
|
// but a rule that neither consumes nor eventually changes state would spin.
|
|
15
20
|
let spin = 0;
|
|
16
21
|
while (pos < len) {
|
|
17
|
-
|
|
22
|
+
if (top.embed !== null) {
|
|
23
|
+
pos = embedded(code, pos, top, memory[memory.length - 1], grammar, emit);
|
|
24
|
+
// An embedding state pops itself. A rule that `set` into it from the
|
|
25
|
+
// only state on the stack leaves nothing below, so the root takes over.
|
|
26
|
+
if (stack.length > 1) {
|
|
27
|
+
stack.length -= 1;
|
|
28
|
+
memory.length -= 1;
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
stack[0] = grammar.root;
|
|
32
|
+
memory[0] = '';
|
|
33
|
+
}
|
|
34
|
+
top = stack[stack.length - 1];
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
const dyn = top.dynamic === null ? null : dynamicRegExps(top, memory[memory.length - 1]);
|
|
38
|
+
const re = dyn === null ? top.re : dyn.re;
|
|
18
39
|
re.lastIndex = pos;
|
|
19
40
|
const m = re.exec(code);
|
|
20
41
|
if (m === null) {
|
|
@@ -27,9 +48,10 @@ export const tokenize = (code, grammar, emit) => {
|
|
|
27
48
|
}
|
|
28
49
|
const rules = top.rules;
|
|
29
50
|
let rule;
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
51
|
+
let index = 0;
|
|
52
|
+
for (; index < rules.length; index++) {
|
|
53
|
+
if (m[rules[index].group] !== undefined) {
|
|
54
|
+
rule = rules[index];
|
|
33
55
|
break;
|
|
34
56
|
}
|
|
35
57
|
}
|
|
@@ -39,22 +61,37 @@ export const tokenize = (code, grammar, emit) => {
|
|
|
39
61
|
const end = start + m[0].length;
|
|
40
62
|
if (end > start) {
|
|
41
63
|
if (rule.captures !== null) {
|
|
42
|
-
|
|
64
|
+
// Only rules that use the remembered text are rebuilt per value; the
|
|
65
|
+
// rest of a dynamic state's capture rules keep their own pattern.
|
|
66
|
+
emitCaptures(code, start, rule, dyn?.own[index] ?? rule.own, top, emit);
|
|
43
67
|
}
|
|
44
68
|
else {
|
|
45
69
|
emit(rule.scope ?? top.default, start, end);
|
|
46
70
|
}
|
|
47
71
|
}
|
|
72
|
+
// Read before the stack changes: an entered state inherits the memory of
|
|
73
|
+
// the state that entered it unless the rule remembers something new.
|
|
74
|
+
let remembered = memory[memory.length - 1];
|
|
75
|
+
if (rule.remember > 0) {
|
|
76
|
+
const text = m[rule.group + rule.remember] ?? '';
|
|
77
|
+
remembered = rule.rememberIndent ? ' '.repeat(text.length) : text;
|
|
78
|
+
}
|
|
48
79
|
if (rule.pop > 0) {
|
|
49
80
|
// Never pop the root state off.
|
|
50
81
|
const n = Math.min(rule.pop, stack.length - 1);
|
|
51
82
|
stack.length -= n;
|
|
83
|
+
memory.length -= n;
|
|
84
|
+
if (rule.remember === 0) {
|
|
85
|
+
remembered = memory[memory.length - 1];
|
|
86
|
+
}
|
|
52
87
|
}
|
|
53
88
|
if (rule.set !== null) {
|
|
54
89
|
stack[stack.length - 1] = grammar.states[rule.set];
|
|
90
|
+
memory[memory.length - 1] = remembered;
|
|
55
91
|
}
|
|
56
92
|
if (rule.push !== null) {
|
|
57
93
|
stack.push(grammar.states[rule.push]);
|
|
94
|
+
memory.push(remembered);
|
|
58
95
|
}
|
|
59
96
|
top = stack[stack.length - 1];
|
|
60
97
|
if (end > pos) {
|
|
@@ -74,6 +111,106 @@ export const tokenize = (code, grammar, emit) => {
|
|
|
74
111
|
emit(top.default, pos, len);
|
|
75
112
|
}
|
|
76
113
|
};
|
|
114
|
+
/**
|
|
115
|
+
* How deep embeds may nest before a body falls back to its state's default
|
|
116
|
+
* scope. Real code nests two or three deep — Markdown, then HTML, then
|
|
117
|
+
* JavaScript — but a Markdown fence can embed Markdown, and a run of
|
|
118
|
+
* ` ```md ` lines would otherwise recurse once per line until the stack runs
|
|
119
|
+
* out.
|
|
120
|
+
*/
|
|
121
|
+
const MAX_EMBED_DEPTH = 8;
|
|
122
|
+
/** How many embedded bodies are being tokenized right now, one inside another. */
|
|
123
|
+
let depth = 0;
|
|
124
|
+
/**
|
|
125
|
+
* Tokenizes the body of an embedding state with its language and returns the
|
|
126
|
+
* position after it.
|
|
127
|
+
*
|
|
128
|
+
* The body runs to the first match of the state's `until`, which is left for
|
|
129
|
+
* the state below to tokenize. The slice is tokenized on its own, so the
|
|
130
|
+
* embedded grammar starts in its root state and cannot read past the body —
|
|
131
|
+
* an unterminated string in a `<script>` cannot swallow the `</script>`.
|
|
132
|
+
*/
|
|
133
|
+
const embedded = (code, pos, state, remembered, host, emit) => {
|
|
134
|
+
const boundary = state.embed.boundary;
|
|
135
|
+
boundary.lastIndex = pos;
|
|
136
|
+
let found = boundary.exec(code);
|
|
137
|
+
while (state.embed.holes !== null && found?.[1] !== undefined) {
|
|
138
|
+
// An empty hole must not prevent the scanner from making progress.
|
|
139
|
+
if (found[0].length === 0) {
|
|
140
|
+
boundary.lastIndex += code.codePointAt(boundary.lastIndex) > 0xffff ? 2 : 1;
|
|
141
|
+
}
|
|
142
|
+
found = boundary.exec(code);
|
|
143
|
+
}
|
|
144
|
+
const stop = found === null ? code.length : found.index;
|
|
145
|
+
if (stop === pos) {
|
|
146
|
+
return pos;
|
|
147
|
+
}
|
|
148
|
+
// `{{remembered}}` takes the language from the text the entering rule
|
|
149
|
+
// remembered, which is how a Markdown fence names its language at runtime.
|
|
150
|
+
const language = state.embed.language;
|
|
151
|
+
const grammar = getLanguage(language === REMEMBERED ? remembered : language);
|
|
152
|
+
if (grammar === undefined || depth >= MAX_EMBED_DEPTH) {
|
|
153
|
+
emit(state.default, pos, stop);
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
depth += 1;
|
|
157
|
+
try {
|
|
158
|
+
withHoles(code.slice(pos, stop), pos, grammar, state.embed.holes, host, emit);
|
|
159
|
+
}
|
|
160
|
+
finally {
|
|
161
|
+
depth -= 1;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
return stop;
|
|
165
|
+
};
|
|
166
|
+
/**
|
|
167
|
+
* Tokenizes an embedded body, around any `holes` that belong to the host
|
|
168
|
+
* grammar.
|
|
169
|
+
*
|
|
170
|
+
* The embedded grammar reads the body with every hole character except line
|
|
171
|
+
* breaks replaced by `_`, an identifier character in the languages that get
|
|
172
|
+
* embedded, so a hole reads as a name and the embedded state runs on across
|
|
173
|
+
* it. Its tokens are then emitted around the holes, and each hole is
|
|
174
|
+
* tokenized with the host grammar in its place.
|
|
175
|
+
*/
|
|
176
|
+
const withHoles = (body, offset, grammar, holes, host, emit) => {
|
|
177
|
+
// Flat [start, end, start, end, …] offsets into `body`.
|
|
178
|
+
const ranges = [];
|
|
179
|
+
const pieces = [];
|
|
180
|
+
let cursor = 0;
|
|
181
|
+
if (holes !== null) {
|
|
182
|
+
holes.lastIndex = 0;
|
|
183
|
+
for (const match of body.matchAll(holes)) {
|
|
184
|
+
const start = match.index;
|
|
185
|
+
const end = start + match[0].length;
|
|
186
|
+
if (start === end) {
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
ranges.push(start, end);
|
|
190
|
+
pieces.push(body.slice(cursor, start), match[0].replace(/[^\r\n]/g, '_'));
|
|
191
|
+
cursor = end;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
const masked = ranges.length === 0 ? body : pieces.join('') + body.slice(cursor);
|
|
195
|
+
let next = 0;
|
|
196
|
+
// Everything before `done` has been emitted, by one grammar or the other.
|
|
197
|
+
let done = 0;
|
|
198
|
+
tokenize(masked, grammar, (scope, from, to) => {
|
|
199
|
+
for (let at = Math.max(from, done); at < to; at = done) {
|
|
200
|
+
const start = ranges[next] ?? to;
|
|
201
|
+
if (start <= at) {
|
|
202
|
+
const end = ranges[next + 1];
|
|
203
|
+
tokenize(body.slice(start, end), host, (s, a, b) => emit(s, a + start + offset, b + start + offset));
|
|
204
|
+
next += 2;
|
|
205
|
+
done = end;
|
|
206
|
+
}
|
|
207
|
+
else {
|
|
208
|
+
done = Math.min(to, start);
|
|
209
|
+
emit(scope, at + offset, done + offset);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
});
|
|
213
|
+
};
|
|
77
214
|
/**
|
|
78
215
|
* Re-runs the winning rule on its own to find where its capture groups sit.
|
|
79
216
|
*
|
|
@@ -88,8 +225,7 @@ export const tokenize = (code, grammar, emit) => {
|
|
|
88
225
|
* either verbatim would repeat text, or claim text the tokenizer is about to
|
|
89
226
|
* visit again, so a group only ever scopes what the match still has left.
|
|
90
227
|
*/
|
|
91
|
-
const emitCaptures = (code, start, rule, state, emit) => {
|
|
92
|
-
const own = rule.own;
|
|
228
|
+
const emitCaptures = (code, start, rule, own, state, emit) => {
|
|
93
229
|
own.lastIndex = start;
|
|
94
230
|
const m = own.exec(code);
|
|
95
231
|
if (m === null) {
|
package/dist/core/types.d.ts
CHANGED
|
@@ -35,7 +35,31 @@ export type Rule = {
|
|
|
35
35
|
set?: string;
|
|
36
36
|
/** Pop `n` states (`true` means 1). */
|
|
37
37
|
pop?: number | true;
|
|
38
|
+
/**
|
|
39
|
+
* Remember the text of this capture group (1-based, within this rule's
|
|
40
|
+
* pattern) for the state that `push` or `set` enters.
|
|
41
|
+
*
|
|
42
|
+
* That state's rules can then match it again by writing `REMEMBERED`
|
|
43
|
+
* (`{{remembered}}`) in a pattern. It is the stand-in for a backreference
|
|
44
|
+
* across rules: a heredoc remembers its terminator, a raw string the run of
|
|
45
|
+
* quotes that opened it, a YAML block scalar the indentation of its key.
|
|
46
|
+
*
|
|
47
|
+
* A state entered without `remember` inherits the value of the state that
|
|
48
|
+
* pushed it, so helper states nested inside keep seeing it.
|
|
49
|
+
*/
|
|
50
|
+
remember?: number;
|
|
51
|
+
/**
|
|
52
|
+
* `'text'` (the default) remembers the group verbatim. `'indent'` remembers
|
|
53
|
+
* a run of spaces as wide as the group, which is what an indentation-based
|
|
54
|
+
* end rule wants: `- key: |` remembers the column of `key`, not `- `.
|
|
55
|
+
*/
|
|
56
|
+
rememberAs?: 'text' | 'indent';
|
|
38
57
|
};
|
|
58
|
+
/**
|
|
59
|
+
* The placeholder a pattern uses to match the remembered text again. The text
|
|
60
|
+
* is inserted escaped, so it matches literally.
|
|
61
|
+
*/
|
|
62
|
+
export declare const REMEMBERED = "{{remembered}}";
|
|
39
63
|
/** Splices another state's rules in at this position, keeping priority order. */
|
|
40
64
|
export type Include = {
|
|
41
65
|
include: string;
|
|
@@ -46,6 +70,34 @@ export type State = {
|
|
|
46
70
|
default?: ScopeName;
|
|
47
71
|
/** Compile this state's patterns case-insensitively. */
|
|
48
72
|
ignoreCase?: boolean;
|
|
73
|
+
/**
|
|
74
|
+
* Highlight the body of this state with another registered language.
|
|
75
|
+
*
|
|
76
|
+
* When the state is entered, the text up to the first match of `until` (or
|
|
77
|
+
* the end of the input) is tokenized with `language`, then the state pops.
|
|
78
|
+
* `until` is not consumed, so the state below sees it: a `<script>` body
|
|
79
|
+
* ends where `</script>` starts, and the HTML grammar scopes the tag.
|
|
80
|
+
*
|
|
81
|
+
* The language is looked up in the registry when the state is entered, so
|
|
82
|
+
* embedding costs nothing until that language is registered. Without it,
|
|
83
|
+
* the body is emitted with this state's `default` scope.
|
|
84
|
+
*
|
|
85
|
+
* `language` can be `REMEMBERED` (`{{remembered}}`) to use the text the rule
|
|
86
|
+
* that entered the state remembered: a Markdown fence remembers its info
|
|
87
|
+
* string and embeds whatever language it names.
|
|
88
|
+
*
|
|
89
|
+
* `holes` marks regions inside the body that belong to this grammar rather
|
|
90
|
+
* than the embedded one, such as `<?= $x ?>` inside a PHP template's
|
|
91
|
+
* `<script>`. The embedded language reads the body with each hole masked,
|
|
92
|
+
* so its state carries across (a hole inside a JavaScript string leaves the
|
|
93
|
+
* rest of the string a string), and each hole is tokenized with this
|
|
94
|
+
* grammar from its root.
|
|
95
|
+
*/
|
|
96
|
+
embed?: {
|
|
97
|
+
language: string;
|
|
98
|
+
until: string;
|
|
99
|
+
holes?: string;
|
|
100
|
+
};
|
|
49
101
|
rules: StateRule[];
|
|
50
102
|
};
|
|
51
103
|
export type Grammar = {
|
|
@@ -83,12 +135,34 @@ export type CompiledRule = {
|
|
|
83
135
|
push: string | null;
|
|
84
136
|
set: string | null;
|
|
85
137
|
pop: number;
|
|
138
|
+
/** Capture group to remember, as an offset from `group`; `0` for none. */
|
|
139
|
+
remember: number;
|
|
140
|
+
rememberIndent: boolean;
|
|
86
141
|
};
|
|
87
142
|
export type CompiledState = {
|
|
88
143
|
name: string;
|
|
89
144
|
re: RegExp;
|
|
90
145
|
rules: CompiledRule[];
|
|
91
146
|
default: ScopeName | null;
|
|
147
|
+
/**
|
|
148
|
+
* Set when a rule uses `REMEMBERED`: the merged regex then depends on the
|
|
149
|
+
* remembered text and is built per value, from these sources.
|
|
150
|
+
*/
|
|
151
|
+
dynamic: {
|
|
152
|
+
source: string;
|
|
153
|
+
flags: string;
|
|
154
|
+
/** Per-rule sticky sources for capture offsets, `null` where unused. */
|
|
155
|
+
own: (string | null)[];
|
|
156
|
+
cache: Map<string, {
|
|
157
|
+
re: RegExp;
|
|
158
|
+
own: (RegExp | null)[];
|
|
159
|
+
}>;
|
|
160
|
+
} | null;
|
|
161
|
+
embed: {
|
|
162
|
+
language: string;
|
|
163
|
+
boundary: RegExp;
|
|
164
|
+
holes: RegExp | null;
|
|
165
|
+
} | null;
|
|
92
166
|
};
|
|
93
167
|
export type CompiledGrammar = {
|
|
94
168
|
name: string;
|
package/dist/core/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,4DAA4D;AAC5D,MAAM,MAAM,SAAS,GAAG,MAAM,CAAA;AAE9B,MAAM,MAAM,IAAI,GAAG;IACjB;;;;;;OAMG;IACH,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;IAEtB;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,SAAS,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,EAAE,CAAA;IAExC,kEAAkE;IAClE,IAAI,CAAC,EAAE,SAAS,CAAA;IAEhB,mCAAmC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,gEAAgE;IAChE,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,uCAAuC;IACvC,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,4DAA4D;AAC5D,MAAM,MAAM,SAAS,GAAG,MAAM,CAAA;AAE9B,MAAM,MAAM,IAAI,GAAG;IACjB;;;;;;OAMG;IACH,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;IAEtB;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,SAAS,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,EAAE,CAAA;IAExC,kEAAkE;IAClE,IAAI,CAAC,EAAE,SAAS,CAAA;IAEhB,mCAAmC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,gEAAgE;IAChE,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,uCAAuC;IACvC,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAEnB;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAA;CAC/B,CAAA;AAED;;;GAGG;AACH,eAAO,MAAM,UAAU,mBAAmB,CAAA;AAE1C,iFAAiF;AACjF,MAAM,MAAM,OAAO,GAAG;IACpB,OAAO,EAAE,MAAM,CAAA;CAChB,CAAA;AAED,MAAM,MAAM,SAAS,GAAG,IAAI,GAAG,OAAO,CAAA;AAEtC,MAAM,MAAM,KAAK,GAAG;IAClB,2EAA2E;IAC3E,OAAO,CAAC,EAAE,SAAS,CAAA;IACnB,wDAAwD;IACxD,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,KAAK,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;IAC3D,KAAK,EAAE,SAAS,EAAE,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,CAAC,EAAE,MAAM,EAAE,CAAA;IAClB,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;CAC9B,CAAA;AAED,MAAM,MAAM,KAAK,GAAG;IAClB,qCAAqC;IACrC,KAAK,EAAE,SAAS,GAAG,IAAI,CAAA;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;CACZ,CAAA;AAED,gFAAgF;AAChF,MAAM,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,SAAS,GAAG,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,IAAI,CAAA;AAIhF,MAAM,MAAM,YAAY,GAAG;IACzB,4DAA4D;IAC5D,KAAK,EAAE,SAAS,GAAG,IAAI,CAAA;IACvB,kEAAkE;IAClE,QAAQ,EAAE,CAAC,SAAS,GAAG,IAAI,CAAC,EAAE,GAAG,IAAI,CAAA;IACrC;;;;;;;OAOG;IACH,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;IAClB,IAAI,EAAE,SAAS,GAAG,IAAI,CAAA;IACtB,gEAAgE;IAChE,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;IAClB,GAAG,EAAE,MAAM,CAAA;IACX,0EAA0E;IAC1E,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,OAAO,CAAA;CACxB,CAAA;AAED,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,YAAY,EAAE,CAAA;IACrB,OAAO,EAAE,SAAS,GAAG,IAAI,CAAA;IACzB;;;OAGG;IACH,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAA;QACd,KAAK,EAAE,MAAM,CAAA;QACb,wEAAwE;QACxE,GAAG,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAA;QACtB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,GAAG,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAA;SAAE,CAAC,CAAA;KAC3D,GAAG,IAAI,CAAA;IACR,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAA;CAC3E,CAAA;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;IACrC,IAAI,EAAE,aAAa,CAAA;CACpB,CAAA"}
|
package/dist/core/types.js
CHANGED
|
@@ -7,4 +7,8 @@
|
|
|
7
7
|
* a string state needs three rules (escape, interpolation, closing quote), not
|
|
8
8
|
* a rule for "string content".
|
|
9
9
|
*/
|
|
10
|
-
|
|
10
|
+
/**
|
|
11
|
+
* The placeholder a pattern uses to match the remembered text again. The text
|
|
12
|
+
* is inserted escaped, so it matches literally.
|
|
13
|
+
*/
|
|
14
|
+
export const REMEMBERED = '{{remembered}}';
|
package/dist/index.d.ts
CHANGED
|
@@ -16,4 +16,5 @@ export type { Scope } from './core/scopes.js';
|
|
|
16
16
|
export { SCOPES, SCOPE_NAMES, scopeChain, scopeClass } from './core/scopes.js';
|
|
17
17
|
export { tokenize as tokenizeStream } from './core/tokenize.js';
|
|
18
18
|
export type { CompiledGrammar, Emit, Grammar, Include, Rule, ScopeName, State, StateRule, Token, } from './core/types.js';
|
|
19
|
+
export { REMEMBERED } from './core/types.js';
|
|
19
20
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAGhE,OAAO,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,cAAc,CAAA;AAEnE,iFAAiF;AACjF,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,OAAO,GAAG,eAAe,CAAA;AAE9D,yEAAyE;AACzE,eAAO,MAAM,SAAS,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,EAAE,UAAU,aAAa,KAAG,MAEtF,CAAA;AAED,yDAAyD;AACzD,eAAO,MAAM,cAAc,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,EAAE,UAAU,YAAY,KAAG,MAE1F,CAAA;AAED,uEAAuE;AACvE,eAAO,MAAM,QAAQ,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,KAAG,KAAK,EAEjE,CAAA;AAED,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,iBAAiB,CAAA;AACxB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAChE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,YAAY,EAAE,KAAK,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,OAAO,EAAE,QAAQ,IAAI,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAC5D,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,cAAc,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAGhE,OAAO,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,cAAc,CAAA;AAEnE,iFAAiF;AACjF,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,OAAO,GAAG,eAAe,CAAA;AAE9D,yEAAyE;AACzE,eAAO,MAAM,SAAS,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,EAAE,UAAU,aAAa,KAAG,MAEtF,CAAA;AAED,yDAAyD;AACzD,eAAO,MAAM,cAAc,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,EAAE,UAAU,YAAY,KAAG,MAE1F,CAAA;AAED,uEAAuE;AACvE,eAAO,MAAM,QAAQ,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,KAAG,KAAK,EAEjE,CAAA;AAED,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,iBAAiB,CAAA;AACxB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAChE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,YAAY,EAAE,KAAK,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,OAAO,EAAE,QAAQ,IAAI,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAC5D,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA"}
|
package/dist/index.js
CHANGED
|
@@ -18,3 +18,4 @@ export { getLanguage, isRegistered, listLanguages, registerLanguage, resolveGram
|
|
|
18
18
|
export { DEFAULT_PREFIX, escapeAttribute, escapeHtml } from './core/render.js';
|
|
19
19
|
export { SCOPES, SCOPE_NAMES, scopeChain, scopeClass } from './core/scopes.js';
|
|
20
20
|
export { tokenize as tokenizeStream } from './core/tokenize.js';
|
|
21
|
+
export { REMEMBERED } from './core/types.js';
|
package/dist/langs/bash.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bash.d.ts","sourceRoot":"","sources":["../../src/langs/bash.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"bash.d.ts","sourceRoot":"","sources":["../../src/langs/bash.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAoB5C,QAAA,MAAM,IAAI,EAAE,OAqKX,CAAA;AAED,eAAe,IAAI,CAAA"}
|
package/dist/langs/bash.js
CHANGED
|
@@ -7,6 +7,12 @@
|
|
|
7
7
|
* the way the shell does.
|
|
8
8
|
*/
|
|
9
9
|
const VAR = '\\$(?:[A-Za-z_]\\w*|[0-9!@#?*$-])';
|
|
10
|
+
/**
|
|
11
|
+
* Where a command name can stand: at the start of a line that does not
|
|
12
|
+
* continue the one before it, after a prompt, after a pipe, list operator or
|
|
13
|
+
* subshell opener, and after the keywords and wrappers that run a command.
|
|
14
|
+
*/
|
|
15
|
+
const BEFORE_COMMAND = '(?:^(?<!\\\\\\r?\\n)(?:[ \\t]*\\$(?=[ \\t]))?|[|&;(`{]|\\$\\(|\\b(?:then|do|else|elif|sudo|time|exec|xargs|nohup|env)\\b|!)';
|
|
10
16
|
const bash = {
|
|
11
17
|
name: 'bash',
|
|
12
18
|
aliases: ['sh', 'shell', 'zsh', 'console'],
|
|
@@ -16,31 +22,58 @@ const bash = {
|
|
|
16
22
|
{ match: '^#!.*', scope: 'comment' },
|
|
17
23
|
{ match: '(?:^|[ \\t])#[^\\n]*', scope: 'comment' },
|
|
18
24
|
{ include: 'expansion' },
|
|
25
|
+
// An escaped character outside quotes is literal. It has to come
|
|
26
|
+
// before the quote rules: `'it'\\''s'` is how a single quote is written
|
|
27
|
+
// inside a single-quoted string — close, escaped quote, reopen — and
|
|
28
|
+
// reading that `\\'` as the start of a string inverts the rest of the
|
|
29
|
+
// line. snippetz writes it for every curl body with an apostrophe.
|
|
30
|
+
// A backslash before a newline is a line continuation, left unscoped.
|
|
31
|
+
{ match: '\\\\[^\\r\\n]', scope: 'string.escape' },
|
|
19
32
|
{ match: '"', scope: 'string', push: 'string-double' },
|
|
20
33
|
{ match: "'", scope: 'string', push: 'string-single' },
|
|
21
34
|
{ match: "\\$'", scope: 'string', push: 'string-ansi' },
|
|
22
35
|
{ match: '`', scope: 'string', push: 'backtick' },
|
|
23
36
|
// Heredocs: the body is scoped as a string up to a line holding only
|
|
24
|
-
// the terminator.
|
|
25
|
-
//
|
|
37
|
+
// the remembered terminator. A quoted marker (`<<'EOF'`) turns
|
|
38
|
+
// expansion off, so `$HOME` in its body is text.
|
|
26
39
|
{
|
|
27
|
-
match: '(<<-?)(\\
|
|
40
|
+
match: '(<<-?)([ \\t]*)([\'"])([A-Za-z_]\\w*)([\'"])',
|
|
28
41
|
scope: ['operator', null, 'string', 'string.special', 'string'],
|
|
42
|
+
push: 'heredoc-literal',
|
|
43
|
+
remember: 4,
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
match: '(<<-?)([ \\t]*)([A-Za-z_]\\w*)',
|
|
47
|
+
scope: ['operator', null, 'string.special'],
|
|
29
48
|
push: 'heredoc',
|
|
49
|
+
remember: 3,
|
|
30
50
|
},
|
|
31
51
|
{
|
|
32
52
|
match: '\\b(?:if|then|else|elif|fi|for|while|until|do|done|case|esac|select|break|continue|return|exit)\\b',
|
|
33
53
|
scope: 'keyword.control',
|
|
34
54
|
},
|
|
55
|
+
// The name a declaration sets gets a state of its own, so the argument
|
|
56
|
+
// rule below cannot read `API_KEY=…` in `export API_KEY=…` as a string.
|
|
35
57
|
{
|
|
36
|
-
match: '\\b(?:
|
|
58
|
+
match: '\\b(?:local|declare|typeset|readonly|export)\\b',
|
|
37
59
|
scope: 'keyword.declaration',
|
|
60
|
+
push: 'declared-name',
|
|
38
61
|
},
|
|
62
|
+
{ match: '\\b(?:function|alias|unset)\\b', scope: 'keyword.declaration' },
|
|
39
63
|
{ match: '\\b(?:in|then)\\b', scope: 'keyword.operator' },
|
|
40
64
|
{
|
|
41
65
|
match: '\\b(?:echo|printf|read|cd|pwd|source|eval|exec|test|trap|set|shift|wait|kill|jobs|getopts|command|type|hash|umask|ulimit)\\b',
|
|
42
66
|
scope: 'function.builtin',
|
|
43
67
|
},
|
|
68
|
+
// The command itself: `curl`, `./build.sh`, `/usr/bin/env`. Keywords
|
|
69
|
+
// and builtins start at the same position and come first, so they keep
|
|
70
|
+
// their own scopes. The lookbehind sits after the first character so
|
|
71
|
+
// the rule still starts with a literal class, which keeps the engine's
|
|
72
|
+
// first-character scan.
|
|
73
|
+
{
|
|
74
|
+
match: `[A-Za-z_./~](?<=${BEFORE_COMMAND}[ \\t]*[A-Za-z_./~])[\\w./~+-]{0,128}(?=[ \\t]|$|[;|&)\`])`,
|
|
75
|
+
scope: 'function.call',
|
|
76
|
+
},
|
|
44
77
|
{
|
|
45
78
|
match: '^([ \\t]*)([A-Za-z_]\\w*)(\\s*)(\\(\\))',
|
|
46
79
|
scope: [null, 'function', null, 'punctuation.bracket'],
|
|
@@ -49,12 +82,24 @@ const bash = {
|
|
|
49
82
|
// Flags are a large share of a shell line and benefit from their own color.
|
|
50
83
|
{ match: '(?:^|\\s)(--?[A-Za-z][\\w-]*)', scope: ['constant'] },
|
|
51
84
|
{ match: '\\b\\d+\\b', scope: 'number' },
|
|
85
|
+
// An unquoted argument is a string to the shell, and themes colour it
|
|
86
|
+
// like one. Command names, keywords and flags start at the same place
|
|
87
|
+
// and come first.
|
|
88
|
+
{ match: '[^\\s$\'"`\\\\|&;()<>{}\\[\\]#=-][^\\s$\'"`\\\\|&;()<>{}]{0,256}', scope: 'string' },
|
|
52
89
|
// `/` is left out: at the top level it is almost always a path
|
|
53
90
|
// separator, and coloring every path segment boundary is noise.
|
|
54
91
|
{ match: '&&|\\|\\||[|&;]|[<>]{1,2}|[-+*%!=]=?|=~', scope: 'operator' },
|
|
55
92
|
{ match: '[()\\[\\]{}]', scope: 'punctuation.bracket' },
|
|
56
93
|
],
|
|
57
94
|
},
|
|
95
|
+
/** After `export`, `local` and the like: flags, then the one name it sets. */
|
|
96
|
+
'declared-name': {
|
|
97
|
+
rules: [
|
|
98
|
+
{ match: '(?:^|\\s)(--?[A-Za-z][\\w-]*)', scope: ['constant'] },
|
|
99
|
+
{ match: '[A-Za-z_]\\w*', scope: 'variable', pop: true },
|
|
100
|
+
{ match: '(?=[^ \\t])', pop: true },
|
|
101
|
+
],
|
|
102
|
+
},
|
|
58
103
|
expansion: {
|
|
59
104
|
rules: [
|
|
60
105
|
{ match: '\\$\\{', scope: 'variable', push: 'brace-expansion' },
|
|
@@ -84,7 +129,7 @@ const bash = {
|
|
|
84
129
|
'string-double': {
|
|
85
130
|
default: 'string',
|
|
86
131
|
rules: [
|
|
87
|
-
{ match: '\\\\[\\\\$`"\\n]', scope: 'string.escape' },
|
|
132
|
+
{ match: '\\\\(?:[\\\\$`"\\n]|\\r\\n)', scope: 'string.escape' },
|
|
88
133
|
{ include: 'expansion' },
|
|
89
134
|
{ match: '`', scope: 'string', push: 'backtick' },
|
|
90
135
|
{ match: '"', scope: 'string', pop: true },
|
|
@@ -109,14 +154,16 @@ const bash = {
|
|
|
109
154
|
},
|
|
110
155
|
heredoc: {
|
|
111
156
|
default: 'string',
|
|
112
|
-
rules: [
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
157
|
+
rules: [{ include: 'heredoc-end' }, { include: 'expansion' }],
|
|
158
|
+
},
|
|
159
|
+
'heredoc-literal': {
|
|
160
|
+
default: 'string',
|
|
161
|
+
rules: [{ include: 'heredoc-end' }],
|
|
162
|
+
},
|
|
163
|
+
'heredoc-end': {
|
|
164
|
+
// `<<-` lets the terminator be indented with tabs; allowing any indent
|
|
165
|
+
// costs nothing and also covers bodies pasted with spaces.
|
|
166
|
+
rules: [{ match: '^[ \\t]*{{remembered}}[ \\t]*$', scope: 'string.special', pop: true }],
|
|
120
167
|
},
|
|
121
168
|
},
|
|
122
169
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"csharp.d.ts","sourceRoot":"","sources":["../../src/langs/csharp.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAuD5C,QAAA,MAAM,MAAM,EAAE,
|
|
1
|
+
{"version":3,"file":"csharp.d.ts","sourceRoot":"","sources":["../../src/langs/csharp.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAuD5C,QAAA,MAAM,MAAM,EAAE,OAkSb,CAAA;AAED,eAAe,MAAM,CAAA"}
|
package/dist/langs/csharp.js
CHANGED
|
@@ -192,12 +192,18 @@ const csharp = {
|
|
|
192
192
|
// rather than as part of it. Longest opener first: `"""` before `"`.
|
|
193
193
|
strings: {
|
|
194
194
|
rules: [
|
|
195
|
+
// A raw literal opens with three or more quotes and closes with the
|
|
196
|
+
// same number, so it can hold a shorter run: `""""` wraps text that
|
|
197
|
+
// contains `"""`. The opening run is remembered for the close.
|
|
195
198
|
{
|
|
196
|
-
|
|
199
|
+
// The lookbehind keeps the `$` run from being retried at each of its
|
|
200
|
+
// characters, which is O(n²) on a long line of `$`.
|
|
201
|
+
match: '(\\$(?<!\\$\\$)\\$*)("{3,})',
|
|
197
202
|
scope: ['string.special', 'string'],
|
|
198
203
|
push: 'raw-interpolated',
|
|
204
|
+
remember: 2,
|
|
199
205
|
},
|
|
200
|
-
{ match: '"
|
|
206
|
+
{ match: '("{3,})', scope: 'string', push: 'raw', remember: 1 },
|
|
201
207
|
{
|
|
202
208
|
match: '(\\$@|@\\$)(")',
|
|
203
209
|
scope: ['string.special', 'string'],
|
|
@@ -261,15 +267,15 @@ const csharp = {
|
|
|
261
267
|
{ match: '"', scope: 'string', pop: true },
|
|
262
268
|
],
|
|
263
269
|
},
|
|
264
|
-
// Raw literals have no escapes at all: the closing
|
|
270
|
+
// Raw literals have no escapes at all: the closing run of quotes is the only thing
|
|
265
271
|
// that ends them, which is what lets them hold quotes verbatim.
|
|
266
272
|
raw: {
|
|
267
273
|
default: 'string',
|
|
268
|
-
rules: [{ match: '
|
|
274
|
+
rules: [{ match: '{{remembered}}', scope: 'string', pop: true }],
|
|
269
275
|
},
|
|
270
276
|
'raw-interpolated': {
|
|
271
277
|
default: 'string',
|
|
272
|
-
rules: [{ include: 'holes' }, { match: '
|
|
278
|
+
rules: [{ include: 'holes' }, { match: '{{remembered}}', scope: 'string', pop: true }],
|
|
273
279
|
},
|
|
274
280
|
/** The `{…}` holes shared by every interpolated flavour. */
|
|
275
281
|
holes: {
|
package/dist/langs/css.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"css.d.ts","sourceRoot":"","sources":["../../src/langs/css.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"css.d.ts","sourceRoot":"","sources":["../../src/langs/css.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAc5C,QAAA,MAAM,GAAG,EAAE,OAwKV,CAAA;AAED,eAAe,GAAG,CAAA"}
|