@scalar/highlight 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/CHANGELOG.md +43 -0
  2. package/README.md +52 -13
  3. package/dist/all.d.ts.map +1 -1
  4. package/dist/all.js +2 -0
  5. package/dist/compat/languages.d.ts.map +1 -1
  6. package/dist/compat/languages.js +2 -4
  7. package/dist/compat/syntax-highlight.d.ts.map +1 -1
  8. package/dist/compat/syntax-highlight.js +101 -27
  9. package/dist/core/compile.d.ts +9 -1
  10. package/dist/core/compile.d.ts.map +1 -1
  11. package/dist/core/compile.js +120 -6
  12. package/dist/core/index.d.ts +1 -0
  13. package/dist/core/index.d.ts.map +1 -1
  14. package/dist/core/index.js +1 -0
  15. package/dist/core/tokenize.d.ts.map +1 -1
  16. package/dist/core/tokenize.js +143 -7
  17. package/dist/core/types.d.ts +74 -0
  18. package/dist/core/types.d.ts.map +1 -1
  19. package/dist/core/types.js +5 -1
  20. package/dist/index.d.ts +1 -0
  21. package/dist/index.d.ts.map +1 -1
  22. package/dist/index.js +1 -0
  23. package/dist/langs/bash.d.ts.map +1 -1
  24. package/dist/langs/bash.js +60 -13
  25. package/dist/langs/csharp.d.ts.map +1 -1
  26. package/dist/langs/csharp.js +11 -5
  27. package/dist/langs/css.d.ts.map +1 -1
  28. package/dist/langs/css.js +84 -7
  29. package/dist/langs/go.d.ts.map +1 -1
  30. package/dist/langs/go.js +7 -1
  31. package/dist/langs/html.d.ts +0 -10
  32. package/dist/langs/html.d.ts.map +1 -1
  33. package/dist/langs/html.js +63 -24
  34. package/dist/langs/http.d.ts.map +1 -1
  35. package/dist/langs/http.js +25 -9
  36. package/dist/langs/javascript.d.ts.map +1 -1
  37. package/dist/langs/javascript.js +146 -30
  38. package/dist/langs/json.d.ts.map +1 -1
  39. package/dist/langs/json.js +7 -1
  40. package/dist/langs/julia.d.ts +4 -0
  41. package/dist/langs/julia.d.ts.map +1 -0
  42. package/dist/langs/julia.js +181 -0
  43. package/dist/langs/markdown.d.ts +9 -3
  44. package/dist/langs/markdown.d.ts.map +1 -1
  45. package/dist/langs/markdown.js +127 -6
  46. package/dist/langs/ocaml.d.ts.map +1 -1
  47. package/dist/langs/ocaml.js +16 -7
  48. package/dist/langs/php.d.ts.map +1 -1
  49. package/dist/langs/php.js +69 -19
  50. package/dist/langs/r.js +9 -9
  51. package/dist/langs/rust.d.ts.map +1 -1
  52. package/dist/langs/rust.js +19 -10
  53. package/dist/langs/sql.d.ts.map +1 -1
  54. package/dist/langs/sql.js +9 -3
  55. package/dist/langs/yaml.d.ts +0 -7
  56. package/dist/langs/yaml.d.ts.map +1 -1
  57. package/dist/langs/yaml.js +157 -28
  58. package/dist/lazy.d.ts +1 -0
  59. package/dist/lazy.d.ts.map +1 -1
  60. package/dist/lazy.js +2 -0
  61. package/package.json +2 -2
@@ -1,3 +1,6 @@
1
+ import { dynamicRegExps } from './compile.js';
2
+ import { getLanguage } from './registry.js';
3
+ import { REMEMBERED } from './types.js';
1
4
  /**
2
5
  * Runs the state machine over `code`, calling `emit` for every range.
3
6
  *
@@ -8,13 +11,31 @@
8
11
  export const tokenize = (code, grammar, emit) => {
9
12
  const len = code.length;
10
13
  const stack = [grammar.root];
14
+ // What each stack entry remembered (see `Rule.remember`), in step with `stack`.
15
+ const memory = [''];
11
16
  let top = grammar.root;
12
17
  let pos = 0;
13
18
  // Zero-length matches are legal (a pure state transition on a lookahead),
14
19
  // but a rule that neither consumes nor eventually changes state would spin.
15
20
  let spin = 0;
16
21
  while (pos < len) {
17
- const re = top.re;
22
+ if (top.embed !== null) {
23
+ pos = embedded(code, pos, top, memory[memory.length - 1], grammar, emit);
24
+ // An embedding state pops itself. A rule that `set` into it from the
25
+ // only state on the stack leaves nothing below, so the root takes over.
26
+ if (stack.length > 1) {
27
+ stack.length -= 1;
28
+ memory.length -= 1;
29
+ }
30
+ else {
31
+ stack[0] = grammar.root;
32
+ memory[0] = '';
33
+ }
34
+ top = stack[stack.length - 1];
35
+ continue;
36
+ }
37
+ const dyn = top.dynamic === null ? null : dynamicRegExps(top, memory[memory.length - 1]);
38
+ const re = dyn === null ? top.re : dyn.re;
18
39
  re.lastIndex = pos;
19
40
  const m = re.exec(code);
20
41
  if (m === null) {
@@ -27,9 +48,10 @@ export const tokenize = (code, grammar, emit) => {
27
48
  }
28
49
  const rules = top.rules;
29
50
  let rule;
30
- for (const r of rules) {
31
- if (m[r.group] !== undefined) {
32
- rule = r;
51
+ let index = 0;
52
+ for (; index < rules.length; index++) {
53
+ if (m[rules[index].group] !== undefined) {
54
+ rule = rules[index];
33
55
  break;
34
56
  }
35
57
  }
@@ -39,22 +61,37 @@ export const tokenize = (code, grammar, emit) => {
39
61
  const end = start + m[0].length;
40
62
  if (end > start) {
41
63
  if (rule.captures !== null) {
42
- emitCaptures(code, start, rule, top, emit);
64
+ // Only rules that use the remembered text are rebuilt per value; the
65
+ // rest of a dynamic state's capture rules keep their own pattern.
66
+ emitCaptures(code, start, rule, dyn?.own[index] ?? rule.own, top, emit);
43
67
  }
44
68
  else {
45
69
  emit(rule.scope ?? top.default, start, end);
46
70
  }
47
71
  }
72
+ // Read before the stack changes: an entered state inherits the memory of
73
+ // the state that entered it unless the rule remembers something new.
74
+ let remembered = memory[memory.length - 1];
75
+ if (rule.remember > 0) {
76
+ const text = m[rule.group + rule.remember] ?? '';
77
+ remembered = rule.rememberIndent ? ' '.repeat(text.length) : text;
78
+ }
48
79
  if (rule.pop > 0) {
49
80
  // Never pop the root state off.
50
81
  const n = Math.min(rule.pop, stack.length - 1);
51
82
  stack.length -= n;
83
+ memory.length -= n;
84
+ if (rule.remember === 0) {
85
+ remembered = memory[memory.length - 1];
86
+ }
52
87
  }
53
88
  if (rule.set !== null) {
54
89
  stack[stack.length - 1] = grammar.states[rule.set];
90
+ memory[memory.length - 1] = remembered;
55
91
  }
56
92
  if (rule.push !== null) {
57
93
  stack.push(grammar.states[rule.push]);
94
+ memory.push(remembered);
58
95
  }
59
96
  top = stack[stack.length - 1];
60
97
  if (end > pos) {
@@ -74,6 +111,106 @@ export const tokenize = (code, grammar, emit) => {
74
111
  emit(top.default, pos, len);
75
112
  }
76
113
  };
114
+ /**
115
+ * How deep embeds may nest before a body falls back to its state's default
116
+ * scope. Real code nests two or three deep — Markdown, then HTML, then
117
+ * JavaScript — but a Markdown fence can embed Markdown, and a run of
118
+ * ` ```md ` lines would otherwise recurse once per line until the stack runs
119
+ * out.
120
+ */
121
+ const MAX_EMBED_DEPTH = 8;
122
+ /** How many embedded bodies are being tokenized right now, one inside another. */
123
+ let depth = 0;
124
+ /**
125
+ * Tokenizes the body of an embedding state with its language and returns the
126
+ * position after it.
127
+ *
128
+ * The body runs to the first match of the state's `until`, which is left for
129
+ * the state below to tokenize. The slice is tokenized on its own, so the
130
+ * embedded grammar starts in its root state and cannot read past the body —
131
+ * an unterminated string in a `<script>` cannot swallow the `</script>`.
132
+ */
133
+ const embedded = (code, pos, state, remembered, host, emit) => {
134
+ const boundary = state.embed.boundary;
135
+ boundary.lastIndex = pos;
136
+ let found = boundary.exec(code);
137
+ while (state.embed.holes !== null && found?.[1] !== undefined) {
138
+ // An empty hole must not prevent the scanner from making progress.
139
+ if (found[0].length === 0) {
140
+ boundary.lastIndex += code.codePointAt(boundary.lastIndex) > 0xffff ? 2 : 1;
141
+ }
142
+ found = boundary.exec(code);
143
+ }
144
+ const stop = found === null ? code.length : found.index;
145
+ if (stop === pos) {
146
+ return pos;
147
+ }
148
+ // `{{remembered}}` takes the language from the text the entering rule
149
+ // remembered, which is how a Markdown fence names its language at runtime.
150
+ const language = state.embed.language;
151
+ const grammar = getLanguage(language === REMEMBERED ? remembered : language);
152
+ if (grammar === undefined || depth >= MAX_EMBED_DEPTH) {
153
+ emit(state.default, pos, stop);
154
+ }
155
+ else {
156
+ depth += 1;
157
+ try {
158
+ withHoles(code.slice(pos, stop), pos, grammar, state.embed.holes, host, emit);
159
+ }
160
+ finally {
161
+ depth -= 1;
162
+ }
163
+ }
164
+ return stop;
165
+ };
166
+ /**
167
+ * Tokenizes an embedded body, around any `holes` that belong to the host
168
+ * grammar.
169
+ *
170
+ * The embedded grammar reads the body with every hole character except line
171
+ * breaks replaced by `_`, an identifier character in the languages that get
172
+ * embedded, so a hole reads as a name and the embedded state runs on across
173
+ * it. Its tokens are then emitted around the holes, and each hole is
174
+ * tokenized with the host grammar in its place.
175
+ */
176
+ const withHoles = (body, offset, grammar, holes, host, emit) => {
177
+ // Flat [start, end, start, end, …] offsets into `body`.
178
+ const ranges = [];
179
+ const pieces = [];
180
+ let cursor = 0;
181
+ if (holes !== null) {
182
+ holes.lastIndex = 0;
183
+ for (const match of body.matchAll(holes)) {
184
+ const start = match.index;
185
+ const end = start + match[0].length;
186
+ if (start === end) {
187
+ continue;
188
+ }
189
+ ranges.push(start, end);
190
+ pieces.push(body.slice(cursor, start), match[0].replace(/[^\r\n]/g, '_'));
191
+ cursor = end;
192
+ }
193
+ }
194
+ const masked = ranges.length === 0 ? body : pieces.join('') + body.slice(cursor);
195
+ let next = 0;
196
+ // Everything before `done` has been emitted, by one grammar or the other.
197
+ let done = 0;
198
+ tokenize(masked, grammar, (scope, from, to) => {
199
+ for (let at = Math.max(from, done); at < to; at = done) {
200
+ const start = ranges[next] ?? to;
201
+ if (start <= at) {
202
+ const end = ranges[next + 1];
203
+ tokenize(body.slice(start, end), host, (s, a, b) => emit(s, a + start + offset, b + start + offset));
204
+ next += 2;
205
+ done = end;
206
+ }
207
+ else {
208
+ done = Math.min(to, start);
209
+ emit(scope, at + offset, done + offset);
210
+ }
211
+ }
212
+ });
213
+ };
77
214
  /**
78
215
  * Re-runs the winning rule on its own to find where its capture groups sit.
79
216
  *
@@ -88,8 +225,7 @@ export const tokenize = (code, grammar, emit) => {
88
225
  * either verbatim would repeat text, or claim text the tokenizer is about to
89
226
  * visit again, so a group only ever scopes what the match still has left.
90
227
  */
91
- const emitCaptures = (code, start, rule, state, emit) => {
92
- const own = rule.own;
228
+ const emitCaptures = (code, start, rule, own, state, emit) => {
93
229
  own.lastIndex = start;
94
230
  const m = own.exec(code);
95
231
  if (m === null) {
@@ -35,7 +35,31 @@ export type Rule = {
35
35
  set?: string;
36
36
  /** Pop `n` states (`true` means 1). */
37
37
  pop?: number | true;
38
+ /**
39
+ * Remember the text of this capture group (1-based, within this rule's
40
+ * pattern) for the state that `push` or `set` enters.
41
+ *
42
+ * That state's rules can then match it again by writing `REMEMBERED`
43
+ * (`{{remembered}}`) in a pattern. It is the stand-in for a backreference
44
+ * across rules: a heredoc remembers its terminator, a raw string the run of
45
+ * quotes that opened it, a YAML block scalar the indentation of its key.
46
+ *
47
+ * A state entered without `remember` inherits the value of the state that
48
+ * pushed it, so helper states nested inside keep seeing it.
49
+ */
50
+ remember?: number;
51
+ /**
52
+ * `'text'` (the default) remembers the group verbatim. `'indent'` remembers
53
+ * a run of spaces as wide as the group, which is what an indentation-based
54
+ * end rule wants: `- key: |` remembers the column of `key`, not `- `.
55
+ */
56
+ rememberAs?: 'text' | 'indent';
38
57
  };
58
+ /**
59
+ * The placeholder a pattern uses to match the remembered text again. The text
60
+ * is inserted escaped, so it matches literally.
61
+ */
62
+ export declare const REMEMBERED = "{{remembered}}";
39
63
  /** Splices another state's rules in at this position, keeping priority order. */
40
64
  export type Include = {
41
65
  include: string;
@@ -46,6 +70,34 @@ export type State = {
46
70
  default?: ScopeName;
47
71
  /** Compile this state's patterns case-insensitively. */
48
72
  ignoreCase?: boolean;
73
+ /**
74
+ * Highlight the body of this state with another registered language.
75
+ *
76
+ * When the state is entered, the text up to the first match of `until` (or
77
+ * the end of the input) is tokenized with `language`, then the state pops.
78
+ * `until` is not consumed, so the state below sees it: a `<script>` body
79
+ * ends where `</script>` starts, and the HTML grammar scopes the tag.
80
+ *
81
+ * The language is looked up in the registry when the state is entered, so
82
+ * embedding costs nothing until that language is registered. Without it,
83
+ * the body is emitted with this state's `default` scope.
84
+ *
85
+ * `language` can be `REMEMBERED` (`{{remembered}}`) to use the text the rule
86
+ * that entered the state remembered: a Markdown fence remembers its info
87
+ * string and embeds whatever language it names.
88
+ *
89
+ * `holes` marks regions inside the body that belong to this grammar rather
90
+ * than the embedded one, such as `<?= $x ?>` inside a PHP template's
91
+ * `<script>`. The embedded language reads the body with each hole masked,
92
+ * so its state carries across (a hole inside a JavaScript string leaves the
93
+ * rest of the string a string), and each hole is tokenized with this
94
+ * grammar from its root.
95
+ */
96
+ embed?: {
97
+ language: string;
98
+ until: string;
99
+ holes?: string;
100
+ };
49
101
  rules: StateRule[];
50
102
  };
51
103
  export type Grammar = {
@@ -83,12 +135,34 @@ export type CompiledRule = {
83
135
  push: string | null;
84
136
  set: string | null;
85
137
  pop: number;
138
+ /** Capture group to remember, as an offset from `group`; `0` for none. */
139
+ remember: number;
140
+ rememberIndent: boolean;
86
141
  };
87
142
  export type CompiledState = {
88
143
  name: string;
89
144
  re: RegExp;
90
145
  rules: CompiledRule[];
91
146
  default: ScopeName | null;
147
+ /**
148
+ * Set when a rule uses `REMEMBERED`: the merged regex then depends on the
149
+ * remembered text and is built per value, from these sources.
150
+ */
151
+ dynamic: {
152
+ source: string;
153
+ flags: string;
154
+ /** Per-rule sticky sources for capture offsets, `null` where unused. */
155
+ own: (string | null)[];
156
+ cache: Map<string, {
157
+ re: RegExp;
158
+ own: (RegExp | null)[];
159
+ }>;
160
+ } | null;
161
+ embed: {
162
+ language: string;
163
+ boundary: RegExp;
164
+ holes: RegExp | null;
165
+ } | null;
92
166
  };
93
167
  export type CompiledGrammar = {
94
168
  name: string;
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,4DAA4D;AAC5D,MAAM,MAAM,SAAS,GAAG,MAAM,CAAA;AAE9B,MAAM,MAAM,IAAI,GAAG;IACjB;;;;;;OAMG;IACH,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;IAEtB;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,SAAS,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,EAAE,CAAA;IAExC,kEAAkE;IAClE,IAAI,CAAC,EAAE,SAAS,CAAA;IAEhB,mCAAmC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,gEAAgE;IAChE,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,uCAAuC;IACvC,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CACpB,CAAA;AAED,iFAAiF;AACjF,MAAM,MAAM,OAAO,GAAG;IACpB,OAAO,EAAE,MAAM,CAAA;CAChB,CAAA;AAED,MAAM,MAAM,SAAS,GAAG,IAAI,GAAG,OAAO,CAAA;AAEtC,MAAM,MAAM,KAAK,GAAG;IAClB,2EAA2E;IAC3E,OAAO,CAAC,EAAE,SAAS,CAAA;IACnB,wDAAwD;IACxD,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB,KAAK,EAAE,SAAS,EAAE,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,CAAC,EAAE,MAAM,EAAE,CAAA;IAClB,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;CAC9B,CAAA;AAED,MAAM,MAAM,KAAK,GAAG;IAClB,qCAAqC;IACrC,KAAK,EAAE,SAAS,GAAG,IAAI,CAAA;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;CACZ,CAAA;AAED,gFAAgF;AAChF,MAAM,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,SAAS,GAAG,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,IAAI,CAAA;AAIhF,MAAM,MAAM,YAAY,GAAG;IACzB,4DAA4D;IAC5D,KAAK,EAAE,SAAS,GAAG,IAAI,CAAA;IACvB,kEAAkE;IAClE,QAAQ,EAAE,CAAC,SAAS,GAAG,IAAI,CAAC,EAAE,GAAG,IAAI,CAAA;IACrC;;;;;;;OAOG;IACH,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;IAClB,IAAI,EAAE,SAAS,GAAG,IAAI,CAAA;IACtB,gEAAgE;IAChE,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;IAClB,GAAG,EAAE,MAAM,CAAA;CACZ,CAAA;AAED,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,YAAY,EAAE,CAAA;IACrB,OAAO,EAAE,SAAS,GAAG,IAAI,CAAA;CAC1B,CAAA;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;IACrC,IAAI,EAAE,aAAa,CAAA;CACpB,CAAA"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,4DAA4D;AAC5D,MAAM,MAAM,SAAS,GAAG,MAAM,CAAA;AAE9B,MAAM,MAAM,IAAI,GAAG;IACjB;;;;;;OAMG;IACH,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;IAEtB;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,SAAS,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,EAAE,CAAA;IAExC,kEAAkE;IAClE,IAAI,CAAC,EAAE,SAAS,CAAA;IAEhB,mCAAmC;IACnC,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,gEAAgE;IAChE,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,uCAAuC;IACvC,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAEnB;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IAEjB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAA;CAC/B,CAAA;AAED;;;GAGG;AACH,eAAO,MAAM,UAAU,mBAAmB,CAAA;AAE1C,iFAAiF;AACjF,MAAM,MAAM,OAAO,GAAG;IACpB,OAAO,EAAE,MAAM,CAAA;CAChB,CAAA;AAED,MAAM,MAAM,SAAS,GAAG,IAAI,GAAG,OAAO,CAAA;AAEtC,MAAM,MAAM,KAAK,GAAG;IAClB,2EAA2E;IAC3E,OAAO,CAAC,EAAE,SAAS,CAAA;IACnB,wDAAwD;IACxD,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACH,KAAK,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;IAC3D,KAAK,EAAE,SAAS,EAAE,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,OAAO,GAAG;IACpB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,CAAC,EAAE,MAAM,EAAE,CAAA;IAClB,6DAA6D;IAC7D,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;CAC9B,CAAA;AAED,MAAM,MAAM,KAAK,GAAG;IAClB,qCAAqC;IACrC,KAAK,EAAE,SAAS,GAAG,IAAI,CAAA;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;CACZ,CAAA;AAED,gFAAgF;AAChF,MAAM,MAAM,IAAI,GAAG,CAAC,KAAK,EAAE,SAAS,GAAG,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,IAAI,CAAA;AAIhF,MAAM,MAAM,YAAY,GAAG;IACzB,4DAA4D;IAC5D,KAAK,EAAE,SAAS,GAAG,IAAI,CAAA;IACvB,kEAAkE;IAClE,QAAQ,EAAE,CAAC,SAAS,GAAG,IAAI,CAAC,EAAE,GAAG,IAAI,CAAA;IACrC;;;;;;;OAOG;IACH,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;IAClB,IAAI,EAAE,SAAS,GAAG,IAAI,CAAA;IACtB,gEAAgE;IAChE,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;IAClB,GAAG,EAAE,MAAM,CAAA;IACX,0EAA0E;IAC1E,QAAQ,EAAE,MAAM,CAAA;IAChB,cAAc,EAAE,OAAO,CAAA;CACxB,CAAA;AAED,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,YAAY,EAAE,CAAA;IACrB,OAAO,EAAE,SAAS,GAAG,IAAI,CAAA;IACzB;;;OAGG;IACH,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAA;QACd,KAAK,EAAE,MAAM,CAAA;QACb,wEAAwE;QACxE,GAAG,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAA;QACtB,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,GAAG,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAA;SAAE,CAAC,CAAA;KAC3D,GAAG,IAAI,CAAA;IACR,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAA;CAC3E,CAAA;AAED,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;IACrC,IAAI,EAAE,aAAa,CAAA;CACpB,CAAA"}
@@ -7,4 +7,8 @@
7
7
  * a string state needs three rules (escape, interpolation, closing quote), not
8
8
  * a rule for "string content".
9
9
  */
10
- export {};
10
+ /**
11
+ * The placeholder a pattern uses to match the remembered text again. The text
12
+ * is inserted escaped, so it matches literally.
13
+ */
14
+ export const REMEMBERED = '{{remembered}}';
package/dist/index.d.ts CHANGED
@@ -16,4 +16,5 @@ export type { Scope } from './core/scopes.js';
16
16
  export { SCOPES, SCOPE_NAMES, scopeChain, scopeClass } from './core/scopes.js';
17
17
  export { tokenize as tokenizeStream } from './core/tokenize.js';
18
18
  export type { CompiledGrammar, Emit, Grammar, Include, Rule, ScopeName, State, StateRule, Token, } from './core/types.js';
19
+ export { REMEMBERED } from './core/types.js';
19
20
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAGhE,OAAO,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,cAAc,CAAA;AAEnE,iFAAiF;AACjF,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,OAAO,GAAG,eAAe,CAAA;AAE9D,yEAAyE;AACzE,eAAO,MAAM,SAAS,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,EAAE,UAAU,aAAa,KAAG,MAEtF,CAAA;AAED,yDAAyD;AACzD,eAAO,MAAM,cAAc,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,EAAE,UAAU,YAAY,KAAG,MAE1F,CAAA;AAED,uEAAuE;AACvE,eAAO,MAAM,QAAQ,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,KAAG,KAAK,EAEjE,CAAA;AAED,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,iBAAiB,CAAA;AACxB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAChE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,YAAY,EAAE,KAAK,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,OAAO,EAAE,QAAQ,IAAI,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAC5D,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,cAAc,CAAA"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAGhE,OAAO,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,cAAc,CAAA;AAEnE,iFAAiF;AACjF,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,OAAO,GAAG,eAAe,CAAA;AAE9D,yEAAyE;AACzE,eAAO,MAAM,SAAS,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,EAAE,UAAU,aAAa,KAAG,MAEtF,CAAA;AAED,yDAAyD;AACzD,eAAO,MAAM,cAAc,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,EAAE,UAAU,YAAY,KAAG,MAE1F,CAAA;AAED,uEAAuE;AACvE,eAAO,MAAM,QAAQ,GAAI,MAAM,MAAM,EAAE,MAAM,aAAa,KAAG,KAAK,EAEjE,CAAA;AAED,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,iBAAiB,CAAA;AACxB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAChE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,YAAY,EAAE,KAAK,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,OAAO,EAAE,QAAQ,IAAI,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAC5D,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA"}
package/dist/index.js CHANGED
@@ -18,3 +18,4 @@ export { getLanguage, isRegistered, listLanguages, registerLanguage, resolveGram
18
18
  export { DEFAULT_PREFIX, escapeAttribute, escapeHtml } from './core/render.js';
19
19
  export { SCOPES, SCOPE_NAMES, scopeChain, scopeClass } from './core/scopes.js';
20
20
  export { tokenize as tokenizeStream } from './core/tokenize.js';
21
+ export { REMEMBERED } from './core/types.js';
@@ -1 +1 @@
1
- {"version":3,"file":"bash.d.ts","sourceRoot":"","sources":["../../src/langs/bash.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAY5C,QAAA,MAAM,IAAI,EAAE,OA0HX,CAAA;AAED,eAAe,IAAI,CAAA"}
1
+ {"version":3,"file":"bash.d.ts","sourceRoot":"","sources":["../../src/langs/bash.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAoB5C,QAAA,MAAM,IAAI,EAAE,OAqKX,CAAA;AAED,eAAe,IAAI,CAAA"}
@@ -7,6 +7,12 @@
7
7
  * the way the shell does.
8
8
  */
9
9
  const VAR = '\\$(?:[A-Za-z_]\\w*|[0-9!@#?*$-])';
10
+ /**
11
+ * Where a command name can stand: at the start of a line that does not
12
+ * continue the one before it, after a prompt, after a pipe, list operator or
13
+ * subshell opener, and after the keywords and wrappers that run a command.
14
+ */
15
+ const BEFORE_COMMAND = '(?:^(?<!\\\\\\r?\\n)(?:[ \\t]*\\$(?=[ \\t]))?|[|&;(`{]|\\$\\(|\\b(?:then|do|else|elif|sudo|time|exec|xargs|nohup|env)\\b|!)';
10
16
  const bash = {
11
17
  name: 'bash',
12
18
  aliases: ['sh', 'shell', 'zsh', 'console'],
@@ -16,31 +22,58 @@ const bash = {
16
22
  { match: '^#!.*', scope: 'comment' },
17
23
  { match: '(?:^|[ \\t])#[^\\n]*', scope: 'comment' },
18
24
  { include: 'expansion' },
25
+ // An escaped character outside quotes is literal. It has to come
26
+ // before the quote rules: `'it'\\''s'` is how a single quote is written
27
+ // inside a single-quoted string — close, escaped quote, reopen — and
28
+ // reading that `\\'` as the start of a string inverts the rest of the
29
+ // line. snippetz writes it for every curl body with an apostrophe.
30
+ // A backslash before a newline is a line continuation, left unscoped.
31
+ { match: '\\\\[^\\r\\n]', scope: 'string.escape' },
19
32
  { match: '"', scope: 'string', push: 'string-double' },
20
33
  { match: "'", scope: 'string', push: 'string-single' },
21
34
  { match: "\\$'", scope: 'string', push: 'string-ansi' },
22
35
  { match: '`', scope: 'string', push: 'backtick' },
23
36
  // Heredocs: the body is scoped as a string up to a line holding only
24
- // the terminator. The terminator name is not tracked (that needs a
25
- // backreference), so an inner line equal to another terminator ends it.
37
+ // the remembered terminator. A quoted marker (`<<'EOF'`) turns
38
+ // expansion off, so `$HOME` in its body is text.
26
39
  {
27
- match: '(<<-?)(\\s*)([\'"]?)([A-Za-z_]\\w*)([\'"]?)',
40
+ match: '(<<-?)([ \\t]*)([\'"])([A-Za-z_]\\w*)([\'"])',
28
41
  scope: ['operator', null, 'string', 'string.special', 'string'],
42
+ push: 'heredoc-literal',
43
+ remember: 4,
44
+ },
45
+ {
46
+ match: '(<<-?)([ \\t]*)([A-Za-z_]\\w*)',
47
+ scope: ['operator', null, 'string.special'],
29
48
  push: 'heredoc',
49
+ remember: 3,
30
50
  },
31
51
  {
32
52
  match: '\\b(?:if|then|else|elif|fi|for|while|until|do|done|case|esac|select|break|continue|return|exit)\\b',
33
53
  scope: 'keyword.control',
34
54
  },
55
+ // The name a declaration sets gets a state of its own, so the argument
56
+ // rule below cannot read `API_KEY=…` in `export API_KEY=…` as a string.
35
57
  {
36
- match: '\\b(?:function|local|declare|typeset|readonly|export|alias|unset)\\b',
58
+ match: '\\b(?:local|declare|typeset|readonly|export)\\b',
37
59
  scope: 'keyword.declaration',
60
+ push: 'declared-name',
38
61
  },
62
+ { match: '\\b(?:function|alias|unset)\\b', scope: 'keyword.declaration' },
39
63
  { match: '\\b(?:in|then)\\b', scope: 'keyword.operator' },
40
64
  {
41
65
  match: '\\b(?:echo|printf|read|cd|pwd|source|eval|exec|test|trap|set|shift|wait|kill|jobs|getopts|command|type|hash|umask|ulimit)\\b',
42
66
  scope: 'function.builtin',
43
67
  },
68
+ // The command itself: `curl`, `./build.sh`, `/usr/bin/env`. Keywords
69
+ // and builtins start at the same position and come first, so they keep
70
+ // their own scopes. The lookbehind sits after the first character so
71
+ // the rule still starts with a literal class, which keeps the engine's
72
+ // first-character scan.
73
+ {
74
+ match: `[A-Za-z_./~](?<=${BEFORE_COMMAND}[ \\t]*[A-Za-z_./~])[\\w./~+-]{0,128}(?=[ \\t]|$|[;|&)\`])`,
75
+ scope: 'function.call',
76
+ },
44
77
  {
45
78
  match: '^([ \\t]*)([A-Za-z_]\\w*)(\\s*)(\\(\\))',
46
79
  scope: [null, 'function', null, 'punctuation.bracket'],
@@ -49,12 +82,24 @@ const bash = {
49
82
  // Flags are a large share of a shell line and benefit from their own color.
50
83
  { match: '(?:^|\\s)(--?[A-Za-z][\\w-]*)', scope: ['constant'] },
51
84
  { match: '\\b\\d+\\b', scope: 'number' },
85
+ // An unquoted argument is a string to the shell, and themes colour it
86
+ // like one. Command names, keywords and flags start at the same place
87
+ // and come first.
88
+ { match: '[^\\s$\'"`\\\\|&;()<>{}\\[\\]#=-][^\\s$\'"`\\\\|&;()<>{}]{0,256}', scope: 'string' },
52
89
  // `/` is left out: at the top level it is almost always a path
53
90
  // separator, and coloring every path segment boundary is noise.
54
91
  { match: '&&|\\|\\||[|&;]|[<>]{1,2}|[-+*%!=]=?|=~', scope: 'operator' },
55
92
  { match: '[()\\[\\]{}]', scope: 'punctuation.bracket' },
56
93
  ],
57
94
  },
95
+ /** After `export`, `local` and the like: flags, then the one name it sets. */
96
+ 'declared-name': {
97
+ rules: [
98
+ { match: '(?:^|\\s)(--?[A-Za-z][\\w-]*)', scope: ['constant'] },
99
+ { match: '[A-Za-z_]\\w*', scope: 'variable', pop: true },
100
+ { match: '(?=[^ \\t])', pop: true },
101
+ ],
102
+ },
58
103
  expansion: {
59
104
  rules: [
60
105
  { match: '\\$\\{', scope: 'variable', push: 'brace-expansion' },
@@ -84,7 +129,7 @@ const bash = {
84
129
  'string-double': {
85
130
  default: 'string',
86
131
  rules: [
87
- { match: '\\\\[\\\\$`"\\n]', scope: 'string.escape' },
132
+ { match: '\\\\(?:[\\\\$`"\\n]|\\r\\n)', scope: 'string.escape' },
88
133
  { include: 'expansion' },
89
134
  { match: '`', scope: 'string', push: 'backtick' },
90
135
  { match: '"', scope: 'string', pop: true },
@@ -109,14 +154,16 @@ const bash = {
109
154
  },
110
155
  heredoc: {
111
156
  default: 'string',
112
- rules: [
113
- {
114
- match: '^[ \\t]*[A-Za-z_]\\w*[ \\t]*$',
115
- scope: 'string.special',
116
- pop: true,
117
- },
118
- { include: 'expansion' },
119
- ],
157
+ rules: [{ include: 'heredoc-end' }, { include: 'expansion' }],
158
+ },
159
+ 'heredoc-literal': {
160
+ default: 'string',
161
+ rules: [{ include: 'heredoc-end' }],
162
+ },
163
+ 'heredoc-end': {
164
+ // `<<-` lets the terminator be indented with tabs; allowing any indent
165
+ // costs nothing and also covers bodies pasted with spaces.
166
+ rules: [{ match: '^[ \\t]*{{remembered}}[ \\t]*$', scope: 'string.special', pop: true }],
120
167
  },
121
168
  },
122
169
  };
@@ -1 +1 @@
1
- {"version":3,"file":"csharp.d.ts","sourceRoot":"","sources":["../../src/langs/csharp.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAuD5C,QAAA,MAAM,MAAM,EAAE,OA4Rb,CAAA;AAED,eAAe,MAAM,CAAA"}
1
+ {"version":3,"file":"csharp.d.ts","sourceRoot":"","sources":["../../src/langs/csharp.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAuD5C,QAAA,MAAM,MAAM,EAAE,OAkSb,CAAA;AAED,eAAe,MAAM,CAAA"}
@@ -192,12 +192,18 @@ const csharp = {
192
192
  // rather than as part of it. Longest opener first: `"""` before `"`.
193
193
  strings: {
194
194
  rules: [
195
+ // A raw literal opens with three or more quotes and closes with the
196
+ // same number, so it can hold a shorter run: `""""` wraps text that
197
+ // contains `"""`. The opening run is remembered for the close.
195
198
  {
196
- match: '(\\$)(""")',
199
+ // The lookbehind keeps the `$` run from being retried at each of its
200
+ // characters, which is O(n²) on a long line of `$`.
201
+ match: '(\\$(?<!\\$\\$)\\$*)("{3,})',
197
202
  scope: ['string.special', 'string'],
198
203
  push: 'raw-interpolated',
204
+ remember: 2,
199
205
  },
200
- { match: '"""', scope: 'string', push: 'raw' },
206
+ { match: '("{3,})', scope: 'string', push: 'raw', remember: 1 },
201
207
  {
202
208
  match: '(\\$@|@\\$)(")',
203
209
  scope: ['string.special', 'string'],
@@ -261,15 +267,15 @@ const csharp = {
261
267
  { match: '"', scope: 'string', pop: true },
262
268
  ],
263
269
  },
264
- // Raw literals have no escapes at all: the closing `"""` is the only thing
270
+ // Raw literals have no escapes at all: the closing run of quotes is the only thing
265
271
  // that ends them, which is what lets them hold quotes verbatim.
266
272
  raw: {
267
273
  default: 'string',
268
- rules: [{ match: '"""', scope: 'string', pop: true }],
274
+ rules: [{ match: '{{remembered}}', scope: 'string', pop: true }],
269
275
  },
270
276
  'raw-interpolated': {
271
277
  default: 'string',
272
- rules: [{ include: 'holes' }, { match: '"""', scope: 'string', pop: true }],
278
+ rules: [{ include: 'holes' }, { match: '{{remembered}}', scope: 'string', pop: true }],
273
279
  },
274
280
  /** The `{…}` holes shared by every interpolated flavour. */
275
281
  holes: {
@@ -1 +1 @@
1
- {"version":3,"file":"css.d.ts","sourceRoot":"","sources":["../../src/langs/css.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAW5C,QAAA,MAAM,GAAG,EAAE,OA0FV,CAAA;AAED,eAAe,GAAG,CAAA"}
1
+ {"version":3,"file":"css.d.ts","sourceRoot":"","sources":["../../src/langs/css.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAc5C,QAAA,MAAM,GAAG,EAAE,OAwKV,CAAA;AAED,eAAe,GAAG,CAAA"}