@scalar/highlight 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/README.md +52 -13
- package/dist/all.d.ts.map +1 -1
- package/dist/all.js +2 -0
- package/dist/compat/languages.d.ts.map +1 -1
- package/dist/compat/languages.js +2 -4
- package/dist/compat/syntax-highlight.d.ts.map +1 -1
- package/dist/compat/syntax-highlight.js +101 -27
- package/dist/core/compile.d.ts +9 -1
- package/dist/core/compile.d.ts.map +1 -1
- package/dist/core/compile.js +120 -6
- package/dist/core/index.d.ts +1 -0
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +1 -0
- package/dist/core/tokenize.d.ts.map +1 -1
- package/dist/core/tokenize.js +143 -7
- package/dist/core/types.d.ts +74 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js +5 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/langs/bash.d.ts.map +1 -1
- package/dist/langs/bash.js +60 -13
- package/dist/langs/csharp.d.ts.map +1 -1
- package/dist/langs/csharp.js +11 -5
- package/dist/langs/css.d.ts.map +1 -1
- package/dist/langs/css.js +84 -7
- package/dist/langs/go.d.ts.map +1 -1
- package/dist/langs/go.js +7 -1
- package/dist/langs/html.d.ts +0 -10
- package/dist/langs/html.d.ts.map +1 -1
- package/dist/langs/html.js +63 -24
- package/dist/langs/http.d.ts.map +1 -1
- package/dist/langs/http.js +25 -9
- package/dist/langs/javascript.d.ts.map +1 -1
- package/dist/langs/javascript.js +146 -30
- package/dist/langs/json.d.ts.map +1 -1
- package/dist/langs/json.js +7 -1
- package/dist/langs/julia.d.ts +4 -0
- package/dist/langs/julia.d.ts.map +1 -0
- package/dist/langs/julia.js +181 -0
- package/dist/langs/markdown.d.ts +9 -3
- package/dist/langs/markdown.d.ts.map +1 -1
- package/dist/langs/markdown.js +127 -6
- package/dist/langs/ocaml.d.ts.map +1 -1
- package/dist/langs/ocaml.js +16 -7
- package/dist/langs/php.d.ts.map +1 -1
- package/dist/langs/php.js +69 -19
- package/dist/langs/r.js +9 -9
- package/dist/langs/rust.d.ts.map +1 -1
- package/dist/langs/rust.js +19 -10
- package/dist/langs/sql.d.ts.map +1 -1
- package/dist/langs/sql.js +9 -3
- package/dist/langs/yaml.d.ts +0 -7
- package/dist/langs/yaml.d.ts.map +1 -1
- package/dist/langs/yaml.js +157 -28
- package/dist/lazy.d.ts +1 -0
- package/dist/lazy.d.ts.map +1 -1
- package/dist/lazy.js +2 -0
- package/package.json +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,48 @@
|
|
|
1
1
|
# @scalar/highlight
|
|
2
2
|
|
|
3
|
+
## 0.4.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- [#10621](https://github.com/scalar/scalar/pull/10621): Close more of the highlighting gap with TextMate grammars, without slowing anything outside Markdown.
|
|
8
|
+
- **Markdown** fenced code blocks are highlighted with the language they name (` ```ts `, `~~~sh`, any registered language or alias). Fences that name no registered language, and fences of four or more backticks, stay plain strings.
|
|
9
|
+
- `embed.language` can be `REMEMBERED`, which embeds whatever language the rule that entered the state remembered. Markdown fences use it.
|
|
10
|
+
- **SQL** scopes common T-SQL and PL/SQL keywords and types (`DECLARE`, `PRINT`, `GO`, `EXEC`, `RETURN`, `VARCHAR2`, `NUMBER`, …).
|
|
11
|
+
- **JavaScript and TypeScript** scope the name a `const`, `let` or `var` declares, read a regex after a control condition (`if (ok) /re/.test(s)`), and colour a tagged template's tag (`` gql`…` ``) as a call.
|
|
12
|
+
- **Shell** scopes unquoted arguments as strings, as TextMate grammars do.
|
|
13
|
+
|
|
14
|
+
TypeScript const assertions keep the following statement highlighted as code instead of consuming its keyword as a declared name.
|
|
15
|
+
|
|
16
|
+
- [#10622](https://github.com/scalar/scalar/pull/10622): feat: highlight more embedded languages and keyword-named keys
|
|
17
|
+
- Markdown: front matter is highlighted as YAML, and HTML that starts a line as HTML. A `<script>`, `<pre>` or `<style>` block runs to its closing tag. The last line of front matter is no longer read as a heading.
|
|
18
|
+
- HTTP: a body that starts with `<` is highlighted as XML/HTML.
|
|
19
|
+
- PHP: `<script>` and `<style>` bodies in templates are highlighted as JavaScript and CSS, and a `<?= ?>` region inside them keeps the surrounding JavaScript or CSS state, so `"<?= $name ?>"` stays one string.
|
|
20
|
+
- Engine: an `embed` can declare `holes`, regions of the body the host grammar tokenizes while the embedded language reads across them.
|
|
21
|
+
- CSS: the condition of `@media`, `@supports`, `@container` and `@import` gets keywords, features, numbers and values.
|
|
22
|
+
- JavaScript/TypeScript: a keyword used as an object key (`{ type: 'object' }`, `{ default: 1 }`) reads as a property, `static of()` and `get(key)` read as method names, and `#private` fields are scoped.
|
|
23
|
+
|
|
24
|
+
Preserve source text when hole patterns contain capture groups, and keep PHP strings and comments from prematurely closing script or style bodies.
|
|
25
|
+
|
|
26
|
+
- [#10612](https://github.com/scalar/scalar/pull/10612): Fix the highlighting breakages a comparison against Shiki and `@scalar/code-highlight` found, and add Julia.
|
|
27
|
+
- **PHP** snippets without `<?php` (every Guzzle, Laravel and curl sample snippetz generates) are read as code instead of rendering as plain text. Heredocs close only on their own marker.
|
|
28
|
+
- **Julia** has a grammar, so `julia/http` code samples are highlighted.
|
|
29
|
+
- **Shell**: `'\''`, the way snippetz writes an apostrophe inside a single-quoted curl body, no longer flips strings and code for the rest of the sample. Heredocs close only on their terminator, and a quoted marker (`<<'EOF'`) turns expansion off.
|
|
30
|
+
- **YAML** block scalars end where their indentation does, so prose such as `Note: …` inside a `description: |` stays a string. `#` starts a comment only after whitespace, commas stay inside plain scalars, and flow collections (`{a: 1, b: [x, y]}`) get their own state.
|
|
31
|
+
- **HTML, XML, SVG and Vue** highlight `<script>` and `<style>` bodies with the JavaScript and CSS grammars when those are registered. `<?xml …?>` is scoped as metadata instead of a comment, and unquoted attribute values as strings.
|
|
32
|
+
- **C#** raw strings close only on the run of quotes that opened them, so `""""…""""` can hold `"""`. **Rust** raw strings close on their own `#` run, and block comments nest. **R** raw strings and **OCaml** quoted strings close on their own delimiter, and OCaml no longer ends a comment at `"*)"`.
|
|
33
|
+
- **CSS** reads an unquoted `url(https://…)` as a string instead of starting a `//` comment.
|
|
34
|
+
- A lone `\r`, U+2028 or U+2029 inside a string no longer ends it early in any grammar.
|
|
35
|
+
- Unterminated `/*`, `<!--` and quotes, and runs of blank lines in JavaScript, no longer cost quadratic time.
|
|
36
|
+
- **JavaScript, TypeScript and JSX**: JSX children are read as text, so an apostrophe in `<p>Don't</p>` no longer opens a string to the end of the line. An element opens only where an expression can start, and TypeScript generics (`<T>(x: T) => x`, `<T extends X>`) and `<string>` assertions are not read as tags. A function assigned to a name (`const App = () => …`) is scoped as a function, and a capitalised name followed by a call (`function App()`, `Error('x')`, `new Date()`) is a call rather than a type.
|
|
37
|
+
- **Shell** command names (`curl`, `grep`, `./build.sh`) are scoped, except on a line that continues the one before it.
|
|
38
|
+
- **YAML** plain scalars that wrap onto the next line keep their string color.
|
|
39
|
+
- **Markdown** fences close only on a run at least as long as the opener, and HTML comments are scoped.
|
|
40
|
+
- `@scalar/highlight/compat` masks credentials on the token stream: a secret the tokenizer splits (`sk_live.abc123` unquoted) is masked, one containing `&` or `<` is masked, and a credential that reads like markup (`span`, `class`) no longer corrupts the HTML.
|
|
41
|
+
|
|
42
|
+
Grammars gain two features. `remember` lets a rule that enters a state remember a capture group, which that state matches again with `REMEMBERED` (`{{remembered}}`) — the cross-state stand-in for a backreference. `embed` hands a state's body to another registered language until a closing pattern.
|
|
43
|
+
|
|
44
|
+
Preserve multiline credential masking, including when line numbers are enabled. Keep defaulted TypeScript generics in code mode, and only embed JavaScript or CSS for complete `script` and `style` tag names so custom elements retain HTML highlighting.
|
|
45
|
+
|
|
3
46
|
## 0.3.1
|
|
4
47
|
|
|
5
48
|
### Patch Changes
|
package/README.md
CHANGED
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
Scalar's code highlighter.
|
|
4
4
|
|
|
5
|
-
- **Small.**
|
|
5
|
+
- **Small.** 3.3 KB core, ~1.7 KB for Python, gzipped. Nothing you do not import.
|
|
6
6
|
- **No theme system.** Colors come from the `--scalar-*` variables `@scalar/themes` already defines, so a code block follows the app's theme and switches with `.light-mode` / `.dark-mode` — without being re-highlighted.
|
|
7
7
|
- **Zero runtime dependencies.**
|
|
8
8
|
|
|
9
|
-
Forty languages ship today: Python, JavaScript/TypeScript/JSX/TSX, Rust, Go, C, C++, C#, F#, Java, Kotlin, Scala, Swift, Objective-C, OCaml, Haskell, Elixir, Clojure, Dart, Mojo, PHP, Perl, Ruby, Lua, R, MATLAB, Bash, PowerShell, SQL, GraphQL, HTTP, YAML, JSON, TOML/INI, CSS, HTML, Markdown, Dockerfiles, Makefiles, nginx configs, and diffs.
|
|
9
|
+
Forty-one languages ship today: Python, JavaScript/TypeScript/JSX/TSX, Rust, Go, C, C++, C#, F#, Java, Kotlin, Scala, Swift, Objective-C, OCaml, Haskell, Elixir, Clojure, Dart, Julia, Mojo, PHP, Perl, Ruby, Lua, R, MATLAB, Bash, PowerShell, SQL, GraphQL, HTTP, YAML, JSON, TOML/INI, CSS, HTML, Markdown, Dockerfiles, Makefiles, nginx configs, and diffs.
|
|
10
10
|
|
|
11
11
|
```bash
|
|
12
12
|
npm install @scalar/highlight
|
|
@@ -43,7 +43,7 @@ highlight(source, 'rust');
|
|
|
43
43
|
```
|
|
44
44
|
|
|
45
45
|
Every entry in the loader is a static `import()`, so bundlers split each language into its
|
|
46
|
-
own chunk. The initial chunk with the loader and no languages is
|
|
46
|
+
own chunk. The initial chunk with the loader and no languages is 4.9 KB.
|
|
47
47
|
|
|
48
48
|
### Entry points
|
|
49
49
|
|
|
@@ -83,12 +83,12 @@ re-exporting it, so tree-shaking runs the way it does in a real app.
|
|
|
83
83
|
|
|
84
84
|
| Bundle | Gzipped |
|
|
85
85
|
| --- | ---: |
|
|
86
|
-
| `@scalar/highlight` core |
|
|
87
|
-
| …+ Python |
|
|
88
|
-
| …lazy loader, initial chunk |
|
|
86
|
+
| `@scalar/highlight` core | 3.32 KB |
|
|
87
|
+
| …+ Python | 5.25 KB |
|
|
88
|
+
| …lazy loader, initial chunk | 4.88 KB |
|
|
89
89
|
| …`style.css` | 1.26 KB |
|
|
90
90
|
|
|
91
|
-
Per language, on top of the core: `diff` 0.14 KB · `json` 0.
|
|
91
|
+
Per language, on top of the core: `diff` 0.14 KB · `json` 0.29 KB · `html` 0.53 KB · `nginx` 0.54 KB · `http` 0.67 KB · `ini` 0.70 KB · `graphql` 0.72 KB · `css` 0.73 KB · `go` 0.80 KB · `yaml` 0.82 KB · `rust` 0.85 KB · `makefile` 0.90 KB · `markdown` 0.91 KB · `dockerfile` 0.94 KB · `bash` 1.03 KB · `sql` 1.10 KB · `java` 1.15 KB · `dart` 1.17 KB · `lua` 1.19 KB · `r` 1.28 KB · `swift` 1.29 KB · `c` 1.31 KB · `clojure` 1.31 KB · `kotlin` 1.33 KB · `julia` 1.36 KB · `scala` 1.36 KB · `powershell` 1.37 KB · `elixir` 1.38 KB · `haskell` 1.41 KB · `matlab` 1.43 KB · `objectivec` 1.43 KB · `ocaml` 1.44 KB · `csharp` 1.52 KB · `mojo` 1.54 KB · `fsharp` 1.68 KB · `python` 1.69 KB · `ruby` 1.70 KB · `cpp` 1.75 KB · `javascript` 1.78 KB · `perl` 1.86 KB · `php` 1.91 KB.
|
|
92
92
|
|
|
93
93
|
Reproduce with `pnpm size`.
|
|
94
94
|
|
|
@@ -199,6 +199,34 @@ state names, circular includes, backreferences (group numbers shift when rules m
|
|
|
199
199
|
scope arrays longer than the pattern's capture count — all at compile time, not mid-tokenize
|
|
200
200
|
on a rare branch.
|
|
201
201
|
|
|
202
|
+
What a backreference would do, `remember` does across states: a rule that enters a state
|
|
203
|
+
can remember one of its capture groups, and that state's patterns match it again with
|
|
204
|
+
`{{remembered}}`. A heredoc closes only on its own marker, a raw string only on the run of
|
|
205
|
+
quotes that opened it, a YAML block scalar at the first line indented no further than its key:
|
|
206
|
+
|
|
207
|
+
```ts
|
|
208
|
+
{ match: '<<(\\w+)', scope: 'string.special', push: 'heredoc', remember: 1 },
|
|
209
|
+
// …
|
|
210
|
+
heredoc: {
|
|
211
|
+
default: 'string',
|
|
212
|
+
rules: [{ match: '^{{remembered}}$', scope: 'string.special', pop: true }],
|
|
213
|
+
},
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
A state can also hand its body to another language: `embed: { language: 'javascript',
|
|
217
|
+
until: '</script' }` tokenizes everything up to `</script` with the JavaScript grammar, if
|
|
218
|
+
it is registered, and leaves the closing tag to the state below. The HTML grammar does this
|
|
219
|
+
for `<script>` and `<style>` without importing either grammar. `language` can also be
|
|
220
|
+
`REMEMBERED`, which embeds whatever language the entering rule remembered: that is how a
|
|
221
|
+
Markdown fence such as ` ```ts ` highlights its body as TypeScript. `holes` marks regions of
|
|
222
|
+
the body that belong to the host grammar instead: the embedded language reads across them
|
|
223
|
+
as if each were a name, so its state carries over, and the host grammar tokenizes each hole.
|
|
224
|
+
A PHP template's `<script>` uses it for `<?= $x ?>`, which keeps `"<?= $name ?>"` one string.
|
|
225
|
+
Hole patterns may contain capture groups. An end marker inside a hole does not close the embedded body.
|
|
226
|
+
|
|
227
|
+
Lines end at `\n` (or `\r\n`) and nowhere else: the compiler rewrites `$` and `.` so a lone
|
|
228
|
+
`\r` or a U+2028 inside a string literal cannot end it early, as JavaScript's `m` flag would.
|
|
229
|
+
|
|
202
230
|
The test suite asserts, for every language and every sample, that emitted tokens
|
|
203
231
|
reconstruct the source exactly, that scopes come from the shared vocabulary, and that
|
|
204
232
|
tokenizing every truncated prefix terminates and stays consistent — which is what an editor
|
|
@@ -255,7 +283,7 @@ today. Because the comparison runs against the workspace package rather than a c
|
|
|
255
283
|
a change on either side shows up here.
|
|
256
284
|
|
|
257
285
|
Colour is the part that does *not* match exactly, and `pnpm compat:report --detail`
|
|
258
|
-
quantifies it: across the whole
|
|
286
|
+
quantifies it: across the whole 42-sample corpus 68.6% of visible characters keep the colour
|
|
259
287
|
`code.css` gives them, and 82% across the thirteen languages the corpus started with. The
|
|
260
288
|
rest is our finer scope vocabulary showing through — f-string interiors scoped as code
|
|
261
289
|
rather than string, docstrings as comments rather than strings, punctuation given its own
|
|
@@ -268,8 +296,8 @@ For the same three languages `ScalarCodeBlock` shows most often:
|
|
|
268
296
|
|
|
269
297
|
| | gzipped |
|
|
270
298
|
| --- | ---: |
|
|
271
|
-
| `@scalar/code-highlight` | 127.
|
|
272
|
-
| `@scalar/highlight/compat` |
|
|
299
|
+
| `@scalar/code-highlight` | 127.58 KB |
|
|
300
|
+
| `@scalar/highlight/compat` | 7.18 KB |
|
|
273
301
|
|
|
274
302
|
## Trade-offs
|
|
275
303
|
|
|
@@ -287,9 +315,20 @@ spacing (`f(x=1)` versus `x = 1`); a triple-quoted string at the start of a line
|
|
|
287
315
|
as a docstring; `CapWords` reads as a type and `SCREAMING_CASE` as a constant. These are
|
|
288
316
|
right on ordinary code and wrong on code written to defeat them.
|
|
289
317
|
|
|
290
|
-
**
|
|
291
|
-
|
|
292
|
-
|
|
318
|
+
**Embedding needs the embedded language registered.** Some grammars hand part of their input
|
|
319
|
+
to another grammar, when that one is registered:
|
|
320
|
+
|
|
321
|
+
- `<script>` and `<style>` bodies in HTML (and so in XML, SVG and Vue) and in PHP templates go
|
|
322
|
+
to JavaScript and CSS.
|
|
323
|
+
- A Markdown fence goes to the language it names, front matter to YAML, and HTML at the start
|
|
324
|
+
of a line to HTML.
|
|
325
|
+
- An HTTP body that starts with `<` goes to HTML.
|
|
326
|
+
|
|
327
|
+
When the grammar is not registered, the body is a plain string or unscoped text. A fence of four
|
|
328
|
+
or more backticks stays a plain string either way, so a block that shows ` ``` ` inside it
|
|
329
|
+
keeps closing on its own run. HTML in Markdown takes its line, or a `<script>`, `<pre>` or
|
|
330
|
+
`<style>` block to its closing tag, rather than running to the next blank line as CommonMark
|
|
331
|
+
says: MDX and most docs renderers read Markdown inside components.
|
|
293
332
|
|
|
294
333
|
**No automatic language detection.** Pass a language name. Detection means shipping every
|
|
295
334
|
grammar, which is the thing this library exists to avoid.
|
package/dist/all.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"all.d.ts","sourceRoot":"","sources":["../src/all.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;
|
|
1
|
+
{"version":3,"file":"all.d.ts","sourceRoot":"","sources":["../src/all.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AA2C3C;;;;;;GAMG;AACH,eAAO,MAAM,SAAS,EAAE,OAAO,EA0C9B,CAAA;AAID;;;;;;;GAOG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,iBAAiB,CAAA;AACxB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAChE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,YAAY,EAAE,KAAK,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,OAAO,EAAE,QAAQ,IAAI,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAC5D,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,cAAc,CAAA;AACrB,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAC5C,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA"}
|
package/dist/all.js
CHANGED
|
@@ -19,6 +19,7 @@ import ini from './langs/ini.js';
|
|
|
19
19
|
import java from './langs/java.js';
|
|
20
20
|
import javascript from './langs/javascript.js';
|
|
21
21
|
import json from './langs/json.js';
|
|
22
|
+
import julia from './langs/julia.js';
|
|
22
23
|
import kotlin from './langs/kotlin.js';
|
|
23
24
|
import lua from './langs/lua.js';
|
|
24
25
|
import makefile from './langs/makefile.js';
|
|
@@ -67,6 +68,7 @@ export const languages = [
|
|
|
67
68
|
java,
|
|
68
69
|
javascript,
|
|
69
70
|
json,
|
|
71
|
+
julia,
|
|
70
72
|
kotlin,
|
|
71
73
|
lua,
|
|
72
74
|
makefile,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"languages.d.ts","sourceRoot":"","sources":["../../src/compat/languages.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAEpD;;;;;;GAMG;AACH,eAAO,MAAM,wBAAwB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAQ3D,CAAA;
|
|
1
|
+
{"version":3,"file":"languages.d.ts","sourceRoot":"","sources":["../../src/compat/languages.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAEpD;;;;;;GAMG;AACH,eAAO,MAAM,wBAAwB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAQ3D,CAAA;AA6HD;;;;;;;;GAQG;AACH,eAAO,MAAM,uBAAuB,keAoD1B,CAAA;AAEV,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,uBAAuB,CAAC,CAAC,MAAM,CAAC,CAAA;AAE1E;;;;;;;;GAQG;AACH,eAAO,MAAM,iBAAiB,EAEzB,MAAM,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC,CAAA;AAE/C,gFAAgF;AAChF,eAAO,MAAM,oBAAoB,EAAE,MAAM,EAAiE,CAAA;AAE1G;;;GAGG;AACH,eAAO,MAAM,aAAa,GAAI,MAAM,MAAM,KAAG,MAK5C,CAAA;AAED;;;;;;GAMG;AACH,eAAO,MAAM,UAAU,GAAI,MAAM,MAAM,KAAG,eAAe,GAAG,SAG3D,CAAA"}
|
package/dist/compat/languages.js
CHANGED
|
@@ -70,8 +70,10 @@ const GRAMMAR_FOR = {
|
|
|
70
70
|
js: 'javascript',
|
|
71
71
|
json: 'json',
|
|
72
72
|
json5: 'json',
|
|
73
|
+
jl: 'julia',
|
|
73
74
|
jsonc: 'json',
|
|
74
75
|
jsx: 'javascript',
|
|
76
|
+
julia: 'julia',
|
|
75
77
|
kotlin: 'kotlin',
|
|
76
78
|
kt: 'kotlin',
|
|
77
79
|
less: 'css',
|
|
@@ -167,10 +169,6 @@ export const STANDARD_LANGUAGE_NAMES = [
|
|
|
167
169
|
'java',
|
|
168
170
|
'javascript',
|
|
169
171
|
'json',
|
|
170
|
-
// Kept in `StandardLanguageKey` for parity with `@scalar/code-highlight`,
|
|
171
|
-
// which highlights Julia via highlight.js. This zero-dependency package has
|
|
172
|
-
// no hand-written Julia grammar yet, so it stays in `unsupportedLanguages`
|
|
173
|
-
// and renders as plain escaped text until one is added.
|
|
174
172
|
'julia',
|
|
175
173
|
'kotlin',
|
|
176
174
|
'less',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"syntax-highlight.d.ts","sourceRoot":"","sources":["../../src/compat/syntax-highlight.ts"],"names":[],"mappings":"AAMA,MAAM,MAAM,sBAAsB,GAAG;IACnC,iDAAiD;IACjD,IAAI,EAAE,MAAM,CAAA;IACZ;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,4EAA4E;IAC5E,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,2DAA2D;IAC3D,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;CACpC,CAAA;
|
|
1
|
+
{"version":3,"file":"syntax-highlight.d.ts","sourceRoot":"","sources":["../../src/compat/syntax-highlight.ts"],"names":[],"mappings":"AAMA,MAAM,MAAM,sBAAsB,GAAG;IACnC,iDAAiD;IACjD,IAAI,EAAE,MAAM,CAAA;IACZ;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,4EAA4E;IAC5E,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,2DAA2D;IAC3D,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;CACpC,CAAA;AAkND;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,eAAe,GAAI,YAAY,MAAM,EAAE,SAAS,sBAAsB,KAAG,MAUrF,CAAA"}
|
|
@@ -12,6 +12,44 @@ const openTag = (cls) => {
|
|
|
12
12
|
}
|
|
13
13
|
return tag;
|
|
14
14
|
};
|
|
15
|
+
/**
|
|
16
|
+
* Credentials shorter than this are not masked — too likely to appear inside
|
|
17
|
+
* unrelated tokens, and a two-character secret is not a secret.
|
|
18
|
+
*/
|
|
19
|
+
const MIN_CREDENTIAL_LENGTH = 3;
|
|
20
|
+
/**
|
|
21
|
+
* Where the credentials sit in the source, as non-overlapping ranges in order.
|
|
22
|
+
*
|
|
23
|
+
* Masking works on these source ranges rather than on the finished HTML. A
|
|
24
|
+
* search over the HTML misses a secret the tokenizer split into several spans
|
|
25
|
+
* (`sk_live.abc123` is a name, a `.` and a member), and it also matches inside
|
|
26
|
+
* the markup itself — a credential that happens to read `span` or `class`
|
|
27
|
+
* used to corrupt the output.
|
|
28
|
+
*/
|
|
29
|
+
const findCredentials = (code, credentials) => {
|
|
30
|
+
const list = (typeof credentials === 'string' ? [credentials] : (credentials ?? [])).filter((credential) => credential.length >= MIN_CREDENTIAL_LENGTH);
|
|
31
|
+
const found = [];
|
|
32
|
+
for (const credential of list) {
|
|
33
|
+
let at = code.indexOf(credential);
|
|
34
|
+
while (at !== -1) {
|
|
35
|
+
found.push({ start: at, end: at + credential.length });
|
|
36
|
+
at = code.indexOf(credential, at + credential.length);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
// Earliest first, and the longer of two that start together.
|
|
40
|
+
found.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
41
|
+
const merged = [];
|
|
42
|
+
for (const range of found) {
|
|
43
|
+
const last = merged[merged.length - 1];
|
|
44
|
+
if (last && range.start < last.end) {
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
merged.push(range);
|
|
48
|
+
}
|
|
49
|
+
return merged;
|
|
50
|
+
};
|
|
51
|
+
const MASK_OPEN = '<span class="credential"><span class="credential-value">';
|
|
52
|
+
const MASK_CLOSE = '</span></span>';
|
|
15
53
|
/**
|
|
16
54
|
* Renders `code` to highlight.js-shaped markup, optionally split into lines.
|
|
17
55
|
*
|
|
@@ -24,18 +62,48 @@ const openTag = (cls) => {
|
|
|
24
62
|
* vocabulary is finer than highlight.js's, so `keyword.declaration` followed
|
|
25
63
|
* by `keyword` has to collapse into one `hljs-keyword` span rather than the
|
|
26
64
|
* two neighbours lowlight would never have emitted.
|
|
65
|
+
*
|
|
66
|
+
* A masked credential takes the class of its first character for its whole
|
|
67
|
+
* length, so it always lands inside a single span, the way a credential inside
|
|
68
|
+
* a string literal did with lowlight.
|
|
27
69
|
*/
|
|
28
|
-
const render = (code, grammar, lineMode) => {
|
|
70
|
+
const render = (code, grammar, lineMode, masked) => {
|
|
29
71
|
const lines = [];
|
|
30
72
|
let line = '';
|
|
31
73
|
// Pending run of same-classed ranges, flushed when the class changes.
|
|
32
74
|
let cls = null;
|
|
33
75
|
let start = 0;
|
|
34
76
|
let end = 0;
|
|
77
|
+
// The next masked range, and the class its first character was given.
|
|
78
|
+
let maskIndex = 0;
|
|
79
|
+
let maskClass = null;
|
|
80
|
+
// Flushes move forward through the source, and the masked ranges are
|
|
81
|
+
// sorted and disjoint, so a cursor keeps each flush from rescanning them all.
|
|
82
|
+
let textIndex = 0;
|
|
83
|
+
/** Escaped text of `[from, to)`, with masked ranges wrapped. */
|
|
84
|
+
const text = (from, to) => {
|
|
85
|
+
while (textIndex < masked.length && masked[textIndex].end <= from) {
|
|
86
|
+
textIndex++;
|
|
87
|
+
}
|
|
88
|
+
let out = '';
|
|
89
|
+
let cursor = from;
|
|
90
|
+
for (let i = textIndex; i < masked.length; i++) {
|
|
91
|
+
const range = masked[i];
|
|
92
|
+
if (range.start >= to) {
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
// A multiline credential is wrapped separately inside each line element.
|
|
96
|
+
const start = Math.max(from, range.start);
|
|
97
|
+
const end = Math.min(to, range.end);
|
|
98
|
+
out += escapeHtml(code.slice(cursor, start)) + MASK_OPEN + escapeHtml(code.slice(start, end)) + MASK_CLOSE;
|
|
99
|
+
cursor = end;
|
|
100
|
+
}
|
|
101
|
+
return out + escapeHtml(code.slice(cursor, to));
|
|
102
|
+
};
|
|
35
103
|
const flush = () => {
|
|
36
104
|
if (end > start) {
|
|
37
|
-
const
|
|
38
|
-
line += cls === null ?
|
|
105
|
+
const body = masked.length ? text(start, end) : escapeHtml(code.slice(start, end));
|
|
106
|
+
line += cls === null ? body : `${openTag(cls)}${body}</span>`;
|
|
39
107
|
}
|
|
40
108
|
start = end;
|
|
41
109
|
};
|
|
@@ -44,8 +112,7 @@ const render = (code, grammar, lineMode) => {
|
|
|
44
112
|
lines.push(line);
|
|
45
113
|
line = '';
|
|
46
114
|
};
|
|
47
|
-
const
|
|
48
|
-
const next = scope === null ? null : hljsClass(scope);
|
|
115
|
+
const place = (next, s, e) => {
|
|
49
116
|
let at = s;
|
|
50
117
|
while (at < e) {
|
|
51
118
|
const nl = lineMode ? code.indexOf('\n', at) : -1;
|
|
@@ -70,6 +137,32 @@ const render = (code, grammar, lineMode) => {
|
|
|
70
137
|
end = at;
|
|
71
138
|
}
|
|
72
139
|
};
|
|
140
|
+
const emit = (scope, s, e) => {
|
|
141
|
+
const next = scope === null ? null : hljsClass(scope);
|
|
142
|
+
// Split the range where it enters or leaves a masked range; the inside
|
|
143
|
+
// parts take the class the masked range started with.
|
|
144
|
+
let at = s;
|
|
145
|
+
while (at < e) {
|
|
146
|
+
while (maskIndex < masked.length && masked[maskIndex].end <= at) {
|
|
147
|
+
maskIndex++;
|
|
148
|
+
}
|
|
149
|
+
const range = masked[maskIndex];
|
|
150
|
+
if (range === undefined || range.start >= e) {
|
|
151
|
+
place(next, at, e);
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
if (at < range.start) {
|
|
155
|
+
place(next, at, range.start);
|
|
156
|
+
at = range.start;
|
|
157
|
+
}
|
|
158
|
+
if (at === range.start) {
|
|
159
|
+
maskClass = next;
|
|
160
|
+
}
|
|
161
|
+
const stop = range.end < e ? range.end : e;
|
|
162
|
+
place(maskClass, at, stop);
|
|
163
|
+
at = stop;
|
|
164
|
+
}
|
|
165
|
+
};
|
|
73
166
|
if (grammar) {
|
|
74
167
|
tokenize(code, grammar, emit);
|
|
75
168
|
}
|
|
@@ -88,26 +181,6 @@ const render = (code, grammar, lineMode) => {
|
|
|
88
181
|
}
|
|
89
182
|
return { html, count: lines.length };
|
|
90
183
|
};
|
|
91
|
-
/**
|
|
92
|
-
* Credentials shorter than this are not masked — too likely to appear inside
|
|
93
|
-
* unrelated tokens, and a two-character secret is not a secret.
|
|
94
|
-
*/
|
|
95
|
-
const MIN_CREDENTIAL_LENGTH = 3;
|
|
96
|
-
/**
|
|
97
|
-
* Replaces credential substrings in the finished HTML.
|
|
98
|
-
*
|
|
99
|
-
* Post-processing the serialized string, rather than the token stream, is what
|
|
100
|
-
* `@scalar/code-highlight` does. It is kept here deliberately: a credential
|
|
101
|
-
* that straddles a token boundary masks or fails to mask exactly as it does
|
|
102
|
-
* today, so a swap cannot change what is visible on screen.
|
|
103
|
-
*/
|
|
104
|
-
const applyCredentialMask = (html, credentials) => {
|
|
105
|
-
const list = (typeof credentials === 'string' ? [credentials] : (credentials ?? [])).filter((credential) => credential.length >= MIN_CREDENTIAL_LENGTH);
|
|
106
|
-
if (!list.length) {
|
|
107
|
-
return html;
|
|
108
|
-
}
|
|
109
|
-
return list.reduce((acc, credential) => acc.split(credential).join(`<span class="credential"><span class="credential-value">${credential}</span></span>`), html);
|
|
110
|
-
};
|
|
111
184
|
/**
|
|
112
185
|
* Drop-in replacement for `syntaxHighlight` from `@scalar/code-highlight`.
|
|
113
186
|
*
|
|
@@ -131,7 +204,8 @@ export const syntaxHighlight = (codeString, options) => {
|
|
|
131
204
|
const name = canonicalName(options.lang);
|
|
132
205
|
const grammar = grammarFor(name);
|
|
133
206
|
const lineMode = options.lineNumbers === true;
|
|
134
|
-
const
|
|
207
|
+
const masked = findCredentials(codeString, options.maskCredentials);
|
|
208
|
+
const { html, count } = render(codeString, grammar, lineMode, masked);
|
|
135
209
|
const style = lineMode ? ` style="--line-count: ${count}; --line-digits: ${count.toString().length};"` : '';
|
|
136
|
-
return
|
|
210
|
+
return `<pre><code class="hljs language-${escapeAttribute(name)}"${style}>${html}</code></pre>`;
|
|
137
211
|
};
|
package/dist/core/compile.d.ts
CHANGED
|
@@ -1,3 +1,11 @@
|
|
|
1
|
-
import type { CompiledGrammar, Grammar } from './types.js';
|
|
1
|
+
import type { CompiledGrammar, CompiledState, Grammar } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* The merged regex of a dynamic state, and its per-rule sticky patterns, with
|
|
4
|
+
* `value` spliced in wherever a rule wrote `REMEMBERED`. Built once per value.
|
|
5
|
+
*/
|
|
6
|
+
export declare const dynamicRegExps: (state: CompiledState, value: string) => {
|
|
7
|
+
re: RegExp;
|
|
8
|
+
own: (RegExp | null)[];
|
|
9
|
+
};
|
|
2
10
|
export declare const compile: (grammar: Grammar) => CompiledGrammar;
|
|
3
11
|
//# sourceMappingURL=compile.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compile.d.ts","sourceRoot":"","sources":["../../src/core/compile.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,
|
|
1
|
+
{"version":3,"file":"compile.d.ts","sourceRoot":"","sources":["../../src/core/compile.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAgB,aAAa,EAAE,OAAO,EAAmB,MAAM,SAAS,CAAA;AAuOrG;;;GAGG;AACH,eAAO,MAAM,cAAc,GAAI,OAAO,aAAa,EAAE,OAAO,MAAM,KAAG;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAA;CAmBxG,CAAA;AAED,eAAO,MAAM,OAAO,GAAI,SAAS,OAAO,KAAG,eAmC1C,CAAA"}
|
package/dist/core/compile.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { REMEMBERED } from './types.js';
|
|
1
2
|
/**
|
|
2
3
|
* Compiles a grammar into one merged regex per state.
|
|
3
4
|
*
|
|
@@ -49,6 +50,60 @@ const countGroups = (src) => {
|
|
|
49
50
|
const patternSource = (match) => {
|
|
50
51
|
return typeof match === 'string' ? match : match.source;
|
|
51
52
|
};
|
|
53
|
+
/**
|
|
54
|
+
* Makes a pattern end lines at `\n` (or `\r\n`) only.
|
|
55
|
+
*
|
|
56
|
+
* With the `m` flag, JavaScript's `$` also matches before `\r`, U+2028 and
|
|
57
|
+
* U+2029, and `.` refuses to match them. Grammars are written as if lines end
|
|
58
|
+
* at `\n` — which is also where the line renderer splits — so a lone `\r` or a
|
|
59
|
+
* U+2028 inside a string literal used to end the string there and leave the
|
|
60
|
+
* rest of the line inverted. Rewriting the two tokens once, here, fixes every
|
|
61
|
+
* grammar at the same time:
|
|
62
|
+
*
|
|
63
|
+
* - `.` becomes `[^\r\n]`: it now matches U+2028 and U+2029, and still
|
|
64
|
+
* stops before the `\r` of a CRLF so an escape like `\\.` cannot swallow it
|
|
65
|
+
* - `$` becomes "before `\n` or `\r\n`, or at the end", so CRLF input still
|
|
66
|
+
* ends a line before its `\r`, exactly as it did before
|
|
67
|
+
*
|
|
68
|
+
* `^` stays native. A line that starts after a lone `\r` is harmless, and the
|
|
69
|
+
* engine's own anchor is measurably faster on large input than a lookbehind.
|
|
70
|
+
*
|
|
71
|
+
* Escapes and character classes match as themselves and are kept verbatim, so
|
|
72
|
+
* only a bare `.` or `$` is rewritten.
|
|
73
|
+
*/
|
|
74
|
+
const normalizeLineTerminators = (src) => {
|
|
75
|
+
// A single left-to-right scan rather than a tokenizing regex: a pattern for
|
|
76
|
+
// "a character class" retried at every `[` of an unclosed run is O(n²).
|
|
77
|
+
let out = '';
|
|
78
|
+
let inClass = false;
|
|
79
|
+
for (let i = 0; i < src.length; i++) {
|
|
80
|
+
const c = src[i];
|
|
81
|
+
if (c === '\\') {
|
|
82
|
+
out += c + (src[i + 1] ?? '');
|
|
83
|
+
i++;
|
|
84
|
+
}
|
|
85
|
+
else if (inClass) {
|
|
86
|
+
inClass = c !== ']';
|
|
87
|
+
out += c;
|
|
88
|
+
}
|
|
89
|
+
else if (c === '[') {
|
|
90
|
+
inClass = true;
|
|
91
|
+
out += c;
|
|
92
|
+
}
|
|
93
|
+
else if (c === '.') {
|
|
94
|
+
out += '[^\\r\\n]';
|
|
95
|
+
}
|
|
96
|
+
else if (c === '$') {
|
|
97
|
+
out += '(?=\\r?\\n|(?![\\s\\S]))';
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
out += c;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return out;
|
|
104
|
+
};
|
|
105
|
+
/** Escapes text so it can be spliced into a pattern and match literally. */
|
|
106
|
+
const escapeRegExp = (s) => s.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&');
|
|
52
107
|
/** Resolves `include` entries into a flat, priority-ordered rule list. */
|
|
53
108
|
const flatten = (grammarName, states, name, seen) => {
|
|
54
109
|
if (seen.has(name)) {
|
|
@@ -82,12 +137,17 @@ const compileState = (grammar, name) => {
|
|
|
82
137
|
// Group 0 is the whole match, so rule wrapper groups start at 1.
|
|
83
138
|
let group = 1;
|
|
84
139
|
const caseFlag = state.ignoreCase ? 'i' : '';
|
|
140
|
+
let dynamic = false;
|
|
141
|
+
const ownSources = [];
|
|
85
142
|
for (const rule of rules) {
|
|
86
|
-
const
|
|
87
|
-
if (BACKREF.test(
|
|
143
|
+
const raw = patternSource(rule.match);
|
|
144
|
+
if (BACKREF.test(raw)) {
|
|
88
145
|
throw new Error(`[${grammar.name}:${name}] backreferences are not supported (group numbers ` +
|
|
89
|
-
`shift when rules are merged): ${
|
|
146
|
+
`shift when rules are merged): ${raw}`);
|
|
90
147
|
}
|
|
148
|
+
const src = normalizeLineTerminators(raw);
|
|
149
|
+
const usesMemory = src.includes(REMEMBERED);
|
|
150
|
+
dynamic ||= usesMemory;
|
|
91
151
|
const captures = Array.isArray(rule.scope) ? rule.scope : null;
|
|
92
152
|
const inner = countGroups(src);
|
|
93
153
|
if (captures) {
|
|
@@ -95,17 +155,24 @@ const compileState = (grammar, name) => {
|
|
|
95
155
|
throw new Error(`[${grammar.name}:${name}] rule scopes ${captures.length} groups but the ` + `pattern has ${inner}: ${src}`);
|
|
96
156
|
}
|
|
97
157
|
}
|
|
158
|
+
if (rule.remember !== undefined && (!(rule.push || rule.set) || rule.remember < 1 || rule.remember > inner)) {
|
|
159
|
+
throw new Error(`[${grammar.name}:${name}] remember ${rule.remember} needs a push or set and one of ${inner} groups: ${raw}`);
|
|
160
|
+
}
|
|
98
161
|
compiled.push({
|
|
99
162
|
scope: typeof rule.scope === 'string' ? rule.scope : null,
|
|
100
163
|
captures,
|
|
101
164
|
// Sticky, so it can be re-run at the position the merged regex chose.
|
|
102
|
-
|
|
165
|
+
// A pattern that depends on the remembered text is built per value.
|
|
166
|
+
own: captures && !usesMemory ? new RegExp(src, `ymd${caseFlag}`) : null,
|
|
103
167
|
rest: rule.rest ?? null,
|
|
104
168
|
group,
|
|
105
169
|
push: rule.push ?? null,
|
|
106
170
|
set: rule.set ?? null,
|
|
107
171
|
pop: rule.pop === true ? 1 : (rule.pop ?? 0),
|
|
172
|
+
remember: rule.remember ?? 0,
|
|
173
|
+
rememberIndent: rule.rememberAs === 'indent',
|
|
108
174
|
});
|
|
175
|
+
ownSources.push(captures && usesMemory ? src : null);
|
|
109
176
|
sources.push(`(${src})`);
|
|
110
177
|
group += 1 + inner;
|
|
111
178
|
}
|
|
@@ -113,13 +180,60 @@ const compileState = (grammar, name) => {
|
|
|
113
180
|
// resolved per rule instead, which keeps the scan free of that cost.
|
|
114
181
|
const flags = `gm${caseFlag}`;
|
|
115
182
|
// An empty state matches nothing: everything falls through to `default`.
|
|
116
|
-
const
|
|
117
|
-
|
|
183
|
+
const source = sources.length ? sources.join('|') : '(?!)';
|
|
184
|
+
// A dynamic state still gets a regex of its own, built with nothing
|
|
185
|
+
// remembered, so it can be entered from the root of a fresh tokenize.
|
|
186
|
+
const re = new RegExp(dynamic ? source.split(REMEMBERED).join('') : source, flags);
|
|
187
|
+
return {
|
|
188
|
+
name,
|
|
189
|
+
re,
|
|
190
|
+
rules: compiled,
|
|
191
|
+
default: state.default ?? null,
|
|
192
|
+
dynamic: dynamic ? { source, flags, own: ownSources, cache: new Map() } : null,
|
|
193
|
+
embed: state.embed
|
|
194
|
+
? {
|
|
195
|
+
language: state.embed.language,
|
|
196
|
+
// Scan holes and the end marker together so a marker inside a hole
|
|
197
|
+
// cannot close the embedded body. Group 1 identifies a hole.
|
|
198
|
+
boundary: new RegExp(normalizeLineTerminators(state.embed.holes ? `(${state.embed.holes})|(?:${state.embed.until})` : state.embed.until), `gm${caseFlag}`),
|
|
199
|
+
holes: state.embed.holes ? new RegExp(normalizeLineTerminators(state.embed.holes), `gm${caseFlag}`) : null,
|
|
200
|
+
}
|
|
201
|
+
: null,
|
|
202
|
+
};
|
|
203
|
+
};
|
|
204
|
+
/**
|
|
205
|
+
* The merged regex of a dynamic state, and its per-rule sticky patterns, with
|
|
206
|
+
* `value` spliced in wherever a rule wrote `REMEMBERED`. Built once per value.
|
|
207
|
+
*/
|
|
208
|
+
export const dynamicRegExps = (state, value) => {
|
|
209
|
+
const dyn = state.dynamic;
|
|
210
|
+
let hit = dyn.cache.get(value);
|
|
211
|
+
if (hit === undefined) {
|
|
212
|
+
const literal = escapeRegExp(value);
|
|
213
|
+
const fill = (src) => src.split(REMEMBERED).join(literal);
|
|
214
|
+
const caseFlag = dyn.flags.includes('i') ? 'i' : '';
|
|
215
|
+
hit = {
|
|
216
|
+
re: new RegExp(fill(dyn.source), dyn.flags),
|
|
217
|
+
own: dyn.own.map((src) => (src === null ? null : new RegExp(fill(src), `ymd${caseFlag}`))),
|
|
218
|
+
};
|
|
219
|
+
// A document with thousands of distinct heredoc markers should not grow
|
|
220
|
+
// this without bound.
|
|
221
|
+
if (dyn.cache.size > 64) {
|
|
222
|
+
dyn.cache.clear();
|
|
223
|
+
}
|
|
224
|
+
dyn.cache.set(value, hit);
|
|
225
|
+
}
|
|
226
|
+
return hit;
|
|
118
227
|
};
|
|
119
228
|
export const compile = (grammar) => {
|
|
120
229
|
if (!Object.hasOwn(grammar.states, 'root')) {
|
|
121
230
|
throw new Error(`[${grammar.name}] grammar has no "root" state`);
|
|
122
231
|
}
|
|
232
|
+
// An embedding state pops itself once its body is done, and the root is
|
|
233
|
+
// never popped.
|
|
234
|
+
if (grammar.states.root?.embed) {
|
|
235
|
+
throw new Error(`[${grammar.name}] the "root" state cannot embed another language`);
|
|
236
|
+
}
|
|
123
237
|
// Null-prototype: state names come from grammar data, and the unknown-state
|
|
124
238
|
// check below has to see `constructor` and `__proto__` as missing rather
|
|
125
239
|
// than as inherited Object members. Tokenizing looks states up here too.
|
package/dist/core/index.d.ts
CHANGED
|
@@ -6,4 +6,5 @@ export type { Scope } from './scopes.js';
|
|
|
6
6
|
export { SCOPES, SCOPE_NAMES, scopeChain, scopeClass } from './scopes.js';
|
|
7
7
|
export { tokenize as tokenizeWith, tokenizeToArray } from './tokenize.js';
|
|
8
8
|
export type { CompiledGrammar, Emit, Grammar, Include, Rule, ScopeName, State, StateRule, Token, } from './types.js';
|
|
9
|
+
export { REMEMBERED } from './types.js';
|
|
9
10
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/core/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAC3D,OAAO,EACL,cAAc,EACd,eAAe,EACf,UAAU,EACV,SAAS,IAAI,aAAa,EAC1B,cAAc,IAAI,kBAAkB,GACrC,MAAM,UAAU,CAAA;AACjB,YAAY,EAAE,KAAK,EAAE,MAAM,UAAU,CAAA;AACrC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AACtE,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AACtE,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,SAAS,CAAA"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AACnC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAC3D,OAAO,EACL,cAAc,EACd,eAAe,EACf,UAAU,EACV,SAAS,IAAI,aAAa,EAC1B,cAAc,IAAI,kBAAkB,GACrC,MAAM,UAAU,CAAA;AACjB,YAAY,EAAE,KAAK,EAAE,MAAM,UAAU,CAAA;AACrC,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AACtE,OAAO,EAAE,QAAQ,IAAI,YAAY,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AACtE,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,SAAS,CAAA;AAChB,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA"}
|
package/dist/core/index.js
CHANGED
|
@@ -3,3 +3,4 @@ export { getLanguage, isRegistered, listLanguages, registerLanguage, resolveGram
|
|
|
3
3
|
export { DEFAULT_PREFIX, escapeAttribute, escapeHtml, highlight as highlightWith, highlightBlock as highlightBlockWith, } from './render.js';
|
|
4
4
|
export { SCOPES, SCOPE_NAMES, scopeChain, scopeClass } from './scopes.js';
|
|
5
5
|
export { tokenize as tokenizeWith, tokenizeToArray } from './tokenize.js';
|
|
6
|
+
export { REMEMBERED } from './types.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../src/core/tokenize.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"tokenize.d.ts","sourceRoot":"","sources":["../../src/core/tokenize.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAA+B,IAAI,EAAE,KAAK,EAAE,MAAM,SAAS,CAAA;AAGxF;;;;;;GAMG;AACH,eAAO,MAAM,QAAQ,GAAI,MAAM,MAAM,EAAE,SAAS,eAAe,EAAE,MAAM,IAAI,KAAG,IA4G7E,CAAA;AAgLD;;;;;;GAMG;AACH,eAAO,MAAM,eAAe,GAAI,MAAM,MAAM,EAAE,SAAS,eAAe,KAAG,KAAK,EAM7E,CAAA"}
|