@scalar/highlight 0.3.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/CHANGELOG.md +59 -0
  2. package/README.md +52 -13
  3. package/dist/all.d.ts.map +1 -1
  4. package/dist/all.js +2 -0
  5. package/dist/compat/languages.d.ts.map +1 -1
  6. package/dist/compat/languages.js +2 -4
  7. package/dist/compat/syntax-highlight.d.ts.map +1 -1
  8. package/dist/compat/syntax-highlight.js +101 -27
  9. package/dist/core/compile.d.ts +9 -1
  10. package/dist/core/compile.d.ts.map +1 -1
  11. package/dist/core/compile.js +168 -9
  12. package/dist/core/index.d.ts +1 -0
  13. package/dist/core/index.d.ts.map +1 -1
  14. package/dist/core/index.js +1 -0
  15. package/dist/core/tokenize.d.ts.map +1 -1
  16. package/dist/core/tokenize.js +147 -7
  17. package/dist/core/types.d.ts +76 -1
  18. package/dist/core/types.d.ts.map +1 -1
  19. package/dist/core/types.js +5 -1
  20. package/dist/index.d.ts +1 -0
  21. package/dist/index.d.ts.map +1 -1
  22. package/dist/index.js +1 -0
  23. package/dist/langs/bash.d.ts.map +1 -1
  24. package/dist/langs/bash.js +60 -13
  25. package/dist/langs/cpp.d.ts.map +1 -1
  26. package/dist/langs/cpp.js +8 -0
  27. package/dist/langs/csharp.d.ts.map +1 -1
  28. package/dist/langs/csharp.js +11 -5
  29. package/dist/langs/css.d.ts.map +1 -1
  30. package/dist/langs/css.js +111 -9
  31. package/dist/langs/diff.d.ts +2 -1
  32. package/dist/langs/diff.d.ts.map +1 -1
  33. package/dist/langs/diff.js +19 -7
  34. package/dist/langs/go.d.ts.map +1 -1
  35. package/dist/langs/go.js +22 -1
  36. package/dist/langs/haskell.d.ts.map +1 -1
  37. package/dist/langs/haskell.js +33 -0
  38. package/dist/langs/html.d.ts +0 -10
  39. package/dist/langs/html.d.ts.map +1 -1
  40. package/dist/langs/html.js +63 -24
  41. package/dist/langs/http.d.ts.map +1 -1
  42. package/dist/langs/http.js +25 -9
  43. package/dist/langs/javascript.d.ts.map +1 -1
  44. package/dist/langs/javascript.js +279 -30
  45. package/dist/langs/json.d.ts.map +1 -1
  46. package/dist/langs/json.js +7 -1
  47. package/dist/langs/julia.d.ts +4 -0
  48. package/dist/langs/julia.d.ts.map +1 -0
  49. package/dist/langs/julia.js +181 -0
  50. package/dist/langs/kotlin.d.ts.map +1 -1
  51. package/dist/langs/kotlin.js +8 -1
  52. package/dist/langs/markdown.d.ts +9 -3
  53. package/dist/langs/markdown.d.ts.map +1 -1
  54. package/dist/langs/markdown.js +127 -6
  55. package/dist/langs/nginx.d.ts.map +1 -1
  56. package/dist/langs/nginx.js +26 -1
  57. package/dist/langs/ocaml.d.ts.map +1 -1
  58. package/dist/langs/ocaml.js +16 -7
  59. package/dist/langs/php.d.ts.map +1 -1
  60. package/dist/langs/php.js +69 -19
  61. package/dist/langs/python.d.ts.map +1 -1
  62. package/dist/langs/python.js +9 -2
  63. package/dist/langs/r.js +9 -9
  64. package/dist/langs/ruby.d.ts.map +1 -1
  65. package/dist/langs/ruby.js +8 -7
  66. package/dist/langs/rust.d.ts.map +1 -1
  67. package/dist/langs/rust.js +27 -11
  68. package/dist/langs/sql.d.ts.map +1 -1
  69. package/dist/langs/sql.js +9 -3
  70. package/dist/langs/swift.d.ts.map +1 -1
  71. package/dist/langs/swift.js +27 -1
  72. package/dist/langs/yaml.d.ts +0 -7
  73. package/dist/langs/yaml.d.ts.map +1 -1
  74. package/dist/langs/yaml.js +157 -28
  75. package/dist/lazy.d.ts +1 -0
  76. package/dist/lazy.d.ts.map +1 -1
  77. package/dist/lazy.js +2 -0
  78. package/package.json +2 -2
package/CHANGELOG.md CHANGED
@@ -1,5 +1,64 @@
1
1
  # @scalar/highlight
2
2
 
3
+ ## 0.5.0
4
+
5
+ ### Minor Changes
6
+
7
+ - [#10668](https://github.com/scalar/scalar/pull/10668): feat: close more gaps with TextMate grammars, and highlight faster
8
+ - Engine: only the wrapper group around each rule captures in a state's merged regex. Past 63 capture groups V8 allocates on every match, which cost large grammars such as C, C++, C#, PHP, Ruby and Python a third of their throughput or more.
9
+ - CSS: `var(--brand)` has the same scope as the `--brand:` it refers to. SCSS `$name` and Less `@name` read whole where they are used, and `$brand: red` at the top level reads as a value rather than a selector.
10
+ - JavaScript/TypeScript: the names in a parameter list read as parameters, destructuring and annotations included; `function f()` names a definition; `with { type: 'json' }` on an import is a keyword.
11
+ - Swift: `#/…/#` and `/…/` regex literals, and the `nonisolated`, `isolated`, `distributed`, `borrowing`, `consuming`, `sending` and `macro` modifiers.
12
+ - Diff: context (`diff -c`) and normal (`diff`) formats, svn `Index:` headers, and `#` comment lines.
13
+ - Ruby: a method called without parentheses, `list.length`, reads as a call, and `p`, `printf`, `warn` and friends as builtins.
14
+ - Go: a package-qualified type, `w http.ResponseWriter` or `*testing.T`, reads as a type where a type goes.
15
+ - nginx: log levels, event methods, rewrite flags, and the names of `upstream` and `map` blocks.
16
+ - Haskell: the names in an import list. Kotlin: `package` and `import` paths, and `tailrec`, `external` and `value class`. C++: calls with explicit template arguments, `any_cast<int>(x)`.
17
+ - Rust: `.await` is a keyword, `self` the receiver and `Self` a type, and `r#type` stays a name. Python: `case 200: return x` with its body on the same line is a keyword. TypeScript: `const id = <T,>(x: T) => x` names a function.
18
+
19
+ ## 0.4.0
20
+
21
+ ### Minor Changes
22
+
23
+ - [#10621](https://github.com/scalar/scalar/pull/10621): Close more of the highlighting gap with TextMate grammars, without slowing anything outside Markdown.
24
+ - **Markdown** fenced code blocks are highlighted with the language they name (` ```ts `, `~~~sh`, any registered language or alias). Fences that name no registered language, and fences of four or more backticks, stay plain strings.
25
+ - `embed.language` can be `REMEMBERED`, which embeds whatever language the rule that entered the state remembered. Markdown fences use it.
26
+ - **SQL** scopes common T-SQL and PL/SQL keywords and types (`DECLARE`, `PRINT`, `GO`, `EXEC`, `RETURN`, `VARCHAR2`, `NUMBER`, …).
27
+ - **JavaScript and TypeScript** scope the name a `const`, `let` or `var` declares, read a regex after a control condition (`if (ok) /re/.test(s)`), and colour a tagged template's tag (`` gql`…` ``) as a call.
28
+ - **Shell** scopes unquoted arguments as strings, as TextMate grammars do.
29
+
30
+ TypeScript const assertions keep the following statement highlighted as code instead of consuming its keyword as a declared name.
31
+
32
+ - [#10622](https://github.com/scalar/scalar/pull/10622): feat: highlight more embedded languages and keyword-named keys
33
+ - Markdown: front matter is highlighted as YAML, and HTML that starts a line as HTML. A `<script>`, `<pre>` or `<style>` block runs to its closing tag. The last line of front matter is no longer read as a heading.
34
+ - HTTP: a body that starts with `<` is highlighted as XML/HTML.
35
+ - PHP: `<script>` and `<style>` bodies in templates are highlighted as JavaScript and CSS, and a `<?= ?>` region inside them keeps the surrounding JavaScript or CSS state, so `"<?= $name ?>"` stays one string.
36
+ - Engine: an `embed` can declare `holes`, regions of the body the host grammar tokenizes while the embedded language reads across them.
37
+ - CSS: the condition of `@media`, `@supports`, `@container` and `@import` gets keywords, features, numbers and values.
38
+ - JavaScript/TypeScript: a keyword used as an object key (`{ type: 'object' }`, `{ default: 1 }`) reads as a property, `static of()` and `get(key)` read as method names, and `#private` fields are scoped.
39
+
40
+ Preserve source text when hole patterns contain capture groups, and keep PHP strings and comments from prematurely closing script or style bodies.
41
+
42
+ - [#10612](https://github.com/scalar/scalar/pull/10612): Fix the highlighting breakages a comparison against Shiki and `@scalar/code-highlight` found, and add Julia.
43
+ - **PHP** snippets without `<?php` (every Guzzle, Laravel and curl sample snippetz generates) are read as code instead of rendering as plain text. Heredocs close only on their own marker.
44
+ - **Julia** has a grammar, so `julia/http` code samples are highlighted.
45
+ - **Shell**: `'\''`, the way snippetz writes an apostrophe inside a single-quoted curl body, no longer flips strings and code for the rest of the sample. Heredocs close only on their terminator, and a quoted marker (`<<'EOF'`) turns expansion off.
46
+ - **YAML** block scalars end where their indentation does, so prose such as `Note: …` inside a `description: |` stays a string. `#` starts a comment only after whitespace, commas stay inside plain scalars, and flow collections (`{a: 1, b: [x, y]}`) get their own state.
47
+ - **HTML, XML, SVG and Vue** highlight `<script>` and `<style>` bodies with the JavaScript and CSS grammars when those are registered. `<?xml …?>` is scoped as metadata instead of a comment, and unquoted attribute values as strings.
48
+ - **C#** raw strings close only on the run of quotes that opened them, so `""""…""""` can hold `"""`. **Rust** raw strings close on their own `#` run, and block comments nest. **R** raw strings and **OCaml** quoted strings close on their own delimiter, and OCaml no longer ends a comment at `"*)"`.
49
+ - **CSS** reads an unquoted `url(https://…)` as a string instead of starting a `//` comment.
50
+ - A lone `\r`, U+2028 or U+2029 inside a string no longer ends it early in any grammar.
51
+ - Unterminated `/*`, `<!--` and quotes, and runs of blank lines in JavaScript, no longer cost quadratic time.
52
+ - **JavaScript, TypeScript and JSX**: JSX children are read as text, so an apostrophe in `<p>Don't</p>` no longer opens a string to the end of the line. An element opens only where an expression can start, and TypeScript generics (`<T>(x: T) => x`, `<T extends X>`) and `<string>` assertions are not read as tags. A function assigned to a name (`const App = () => …`) is scoped as a function, and a capitalised name followed by a call (`function App()`, `Error('x')`, `new Date()`) is a call rather than a type.
53
+ - **Shell** command names (`curl`, `grep`, `./build.sh`) are scoped, except on a line that continues the one before it.
54
+ - **YAML** plain scalars that wrap onto the next line keep their string color.
55
+ - **Markdown** fences close only on a run at least as long as the opener, and HTML comments are scoped.
56
+ - `@scalar/highlight/compat` masks credentials on the token stream: a secret the tokenizer splits (`sk_live.abc123` unquoted) is masked, one containing `&` or `<` is masked, and a credential that reads like markup (`span`, `class`) no longer corrupts the HTML.
57
+
58
+ Grammars gain two features. `remember` lets a rule that enters a state remember a capture group, which that state matches again with `REMEMBERED` (`{{remembered}}`) — the cross-state stand-in for a backreference. `embed` hands a state's body to another registered language until a closing pattern.
59
+
60
+ Preserve multiline credential masking, including when line numbers are enabled. Keep defaulted TypeScript generics in code mode, and only embed JavaScript or CSS for complete `script` and `style` tag names so custom elements retain HTML highlighting.
61
+
3
62
  ## 0.3.1
4
63
 
5
64
  ### Patch Changes
package/README.md CHANGED
@@ -2,11 +2,11 @@
2
2
 
3
3
  Scalar's code highlighter.
4
4
 
5
- - **Small.** 2.3 KB core, ~1.7 KB for Python, gzipped. Nothing you do not import.
5
+ - **Small.** 3.5 KB core, ~1.7 KB for Python, gzipped. Nothing you do not import.
6
6
  - **No theme system.** Colors come from the `--scalar-*` variables `@scalar/themes` already defines, so a code block follows the app's theme and switches with `.light-mode` / `.dark-mode` — without being re-highlighted.
7
7
  - **Zero runtime dependencies.**
8
8
 
9
- Forty languages ship today: Python, JavaScript/TypeScript/JSX/TSX, Rust, Go, C, C++, C#, F#, Java, Kotlin, Scala, Swift, Objective-C, OCaml, Haskell, Elixir, Clojure, Dart, Mojo, PHP, Perl, Ruby, Lua, R, MATLAB, Bash, PowerShell, SQL, GraphQL, HTTP, YAML, JSON, TOML/INI, CSS, HTML, Markdown, Dockerfiles, Makefiles, nginx configs, and diffs.
9
+ Forty-one languages ship today: Python, JavaScript/TypeScript/JSX/TSX, Rust, Go, C, C++, C#, F#, Java, Kotlin, Scala, Swift, Objective-C, OCaml, Haskell, Elixir, Clojure, Dart, Julia, Mojo, PHP, Perl, Ruby, Lua, R, MATLAB, Bash, PowerShell, SQL, GraphQL, HTTP, YAML, JSON, TOML/INI, CSS, HTML, Markdown, Dockerfiles, Makefiles, nginx configs, and diffs.
10
10
 
11
11
  ```bash
12
12
  npm install @scalar/highlight
@@ -43,7 +43,7 @@ highlight(source, 'rust');
43
43
  ```
44
44
 
45
45
  Every entry in the loader is a static `import()`, so bundlers split each language into its
46
- own chunk. The initial chunk with the loader and no languages is 3.9 KB.
46
+ own chunk. The initial chunk with the loader and no languages is 5.0 KB.
47
47
 
48
48
  ### Entry points
49
49
 
@@ -83,12 +83,12 @@ re-exporting it, so tree-shaking runs the way it does in a real app.
83
83
 
84
84
  | Bundle | Gzipped |
85
85
  | --- | ---: |
86
- | `@scalar/highlight` core | 2.32 KB |
87
- | …+ Python | 4.33 KB |
88
- | …lazy loader, initial chunk | 3.93 KB |
86
+ | `@scalar/highlight` core | 3.50 KB |
87
+ | …+ Python | 5.44 KB |
88
+ | …lazy loader, initial chunk | 5.05 KB |
89
89
  | …`style.css` | 1.26 KB |
90
90
 
91
- Per language, on top of the core: `diff` 0.14 KB · `json` 0.28 KB · `html` 0.32 KB · `markdown` 0.46 KB · `yaml` 0.52 KB · `css` 0.53 KB · `nginx` 0.54 KB · `http` 0.64 KB · `ini` 0.71 KB · `graphql` 0.73 KB · `go` 0.79 KB · `bash` 0.80 KB · `rust` 0.83 KB · `makefile` 0.90 KB · `dockerfile` 0.95 KB · `sql` 0.97 KB · `java` 1.15 KB · `dart` 1.18 KB · `lua` 1.20 KB · `r` 1.28 KB · `swift` 1.30 KB · `c` 1.32 KB · `clojure` 1.32 KB · `kotlin` 1.33 KB · `javascript` 1.36 KB · `scala` 1.37 KB · `elixir` 1.38 KB · `powershell` 1.38 KB · `haskell` 1.41 KB · `ocaml` 1.41 KB · `matlab` 1.44 KB · `objectivec` 1.44 KB · `csharp` 1.50 KB · `mojo` 1.55 KB · `fsharp` 1.69 KB · `python` 1.70 KB · `ruby` 1.71 KB · `php` 1.75 KB · `cpp` 1.76 KB · `perl` 1.86 KB.
91
+ Per language, on top of the core: `diff` 0.21 KB · `json` 0.29 KB · `html` 0.53 KB · `http` 0.67 KB · `nginx` 0.69 KB · `ini` 0.70 KB · `graphql` 0.73 KB · `css` 0.79 KB · `yaml` 0.82 KB · `rust` 0.88 KB · `go` 0.90 KB · `makefile` 0.90 KB · `markdown` 0.91 KB · `dockerfile` 0.95 KB · `bash` 1.03 KB · `sql` 1.11 KB · `java` 1.15 KB · `dart` 1.17 KB · `lua` 1.19 KB · `r` 1.28 KB · `c` 1.31 KB · `clojure` 1.32 KB · `scala` 1.36 KB · `julia` 1.37 KB · `elixir` 1.38 KB · `kotlin` 1.38 KB · `powershell` 1.38 KB · `matlab` 1.43 KB · `objectivec` 1.43 KB · `swift` 1.44 KB · `ocaml` 1.45 KB · `haskell` 1.50 KB · `csharp` 1.52 KB · `mojo` 1.54 KB · `fsharp` 1.68 KB · `python` 1.71 KB · `ruby` 1.74 KB · `cpp` 1.78 KB · `perl` 1.87 KB · `php` 2.01 KB · `javascript` 2.11 KB.
92
92
 
93
93
  Reproduce with `pnpm size`.
94
94
 
@@ -199,6 +199,34 @@ state names, circular includes, backreferences (group numbers shift when rules m
199
199
  scope arrays longer than the pattern's capture count — all at compile time, not mid-tokenize
200
200
  on a rare branch.
201
201
 
202
+ What a backreference would do, `remember` does across states: a rule that enters a state
203
+ can remember one of its capture groups, and that state's patterns match it again with
204
+ `{{remembered}}`. A heredoc closes only on its own marker, a raw string only on the run of
205
+ quotes that opened it, a YAML block scalar at the first line indented no further than its key:
206
+
207
+ ```ts
208
+ { match: '<<(\\w+)', scope: 'string.special', push: 'heredoc', remember: 1 },
209
+ // …
210
+ heredoc: {
211
+ default: 'string',
212
+ rules: [{ match: '^{{remembered}}$', scope: 'string.special', pop: true }],
213
+ },
214
+ ```
215
+
216
+ A state can also hand its body to another language: `embed: { language: 'javascript',
217
+ until: '</script' }` tokenizes everything up to `</script` with the JavaScript grammar, if
218
+ it is registered, and leaves the closing tag to the state below. The HTML grammar does this
219
+ for `<script>` and `<style>` without importing either grammar. `language` can also be
220
+ `REMEMBERED`, which embeds whatever language the entering rule remembered: that is how a
221
+ Markdown fence such as ` ```ts ` highlights its body as TypeScript. `holes` marks regions of
222
+ the body that belong to the host grammar instead: the embedded language reads across them
223
+ as if each were a name, so its state carries over, and the host grammar tokenizes each hole.
224
+ A PHP template's `<script>` uses it for `<?= $x ?>`, which keeps `"<?= $name ?>"` one string.
225
+ Hole patterns may contain capture groups. An end marker inside a hole does not close the embedded body.
226
+
227
+ Lines end at `\n` (or `\r\n`) and nowhere else: the compiler rewrites `$` and `.` so a lone
228
+ `\r` or a U+2028 inside a string literal cannot end it early, as JavaScript's `m` flag would.
229
+
202
230
  The test suite asserts, for every language and every sample, that emitted tokens
203
231
  reconstruct the source exactly, that scopes come from the shared vocabulary, and that
204
232
  tokenizing every truncated prefix terminates and stays consistent — which is what an editor
@@ -255,7 +283,7 @@ today. Because the comparison runs against the workspace package rather than a c
255
283
  a change on either side shows up here.
256
284
 
257
285
  Colour is the part that does *not* match exactly, and `pnpm compat:report --detail`
258
- quantifies it: across the whole 41-sample corpus 68.5% of visible characters keep the colour
286
+ quantifies it: across the whole 42-sample corpus 68.6% of visible characters keep the colour
259
287
  `code.css` gives them, and 82% across the thirteen languages the corpus started with. The
260
288
  rest is our finer scope vocabulary showing through — f-string interiors scoped as code
261
289
  rather than string, docstrings as comments rather than strings, punctuation given its own
@@ -268,8 +296,8 @@ For the same three languages `ScalarCodeBlock` shows most often:
268
296
 
269
297
  | | gzipped |
270
298
  | --- | ---: |
271
- | `@scalar/code-highlight` | 127.34 KB |
272
- | `@scalar/highlight/compat` | 5.36 KB |
299
+ | `@scalar/code-highlight` | 127.58 KB |
300
+ | `@scalar/highlight/compat` | 7.67 KB |
273
301
 
274
302
  ## Trade-offs
275
303
 
@@ -287,9 +315,20 @@ spacing (`f(x=1)` versus `x = 1`); a triple-quoted string at the start of a line
287
315
  as a docstring; `CapWords` reads as a type and `SCREAMING_CASE` as a constant. These are
288
316
  right on ordinary code and wrong on code written to defeat them.
289
317
 
290
- **No cross-language embedding yet.** `<script>` bodies in HTML and fenced code blocks in
291
- Markdown are scoped as plain text rather than highlighted with their own grammar. The engine
292
- has the state machinery for it; the registry hook is not written.
318
+ **Embedding needs the embedded language registered.** Some grammars hand part of their input
319
+ to another grammar, when that one is registered:
320
+
321
+ - `<script>` and `<style>` bodies in HTML (and so in XML, SVG and Vue) and in PHP templates go
322
+ to JavaScript and CSS.
323
+ - A Markdown fence goes to the language it names, front matter to YAML, and HTML at the start
324
+ of a line to HTML.
325
+ - An HTTP body that starts with `<` goes to HTML.
326
+
327
+ When the grammar is not registered, the body is a plain string or unscoped text. A fence of four
328
+ or more backticks stays a plain string either way, so a block that shows ` ``` ` inside it
329
+ keeps closing on its own run. HTML in Markdown takes its line, or a `<script>`, `<pre>` or
330
+ `<style>` block to its closing tag, rather than running to the next blank line as CommonMark
331
+ says: MDX and most docs renderers read Markdown inside components.
293
332
 
294
333
  **No automatic language detection.** Pass a language name. Detection means shipping every
295
334
  grammar, which is the thing this library exists to avoid.
package/dist/all.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"all.d.ts","sourceRoot":"","sources":["../src/all.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AA0C3C;;;;;;GAMG;AACH,eAAO,MAAM,SAAS,EAAE,OAAO,EAyC9B,CAAA;AAID;;;;;;;GAOG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,iBAAiB,CAAA;AACxB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAChE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,YAAY,EAAE,KAAK,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,OAAO,EAAE,QAAQ,IAAI,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAC5D,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,cAAc,CAAA;AACrB,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAC5C,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA"}
1
+ {"version":3,"file":"all.d.ts","sourceRoot":"","sources":["../src/all.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AA2C3C;;;;;;GAMG;AACH,eAAO,MAAM,SAAS,EAAE,OAAO,EA0C9B,CAAA;AAID;;;;;;;GAOG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,iBAAiB,CAAA;AACxB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAChE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,YAAY,EAAE,KAAK,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,OAAO,EAAE,QAAQ,IAAI,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAC5D,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,cAAc,CAAA;AACrB,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAC5C,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA"}
package/dist/all.js CHANGED
@@ -19,6 +19,7 @@ import ini from './langs/ini.js';
19
19
  import java from './langs/java.js';
20
20
  import javascript from './langs/javascript.js';
21
21
  import json from './langs/json.js';
22
+ import julia from './langs/julia.js';
22
23
  import kotlin from './langs/kotlin.js';
23
24
  import lua from './langs/lua.js';
24
25
  import makefile from './langs/makefile.js';
@@ -67,6 +68,7 @@ export const languages = [
67
68
  java,
68
69
  javascript,
69
70
  json,
71
+ julia,
70
72
  kotlin,
71
73
  lua,
72
74
  makefile,
@@ -1 +1 @@
1
- {"version":3,"file":"languages.d.ts","sourceRoot":"","sources":["../../src/compat/languages.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAEpD;;;;;;GAMG;AACH,eAAO,MAAM,wBAAwB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAQ3D,CAAA;AA2HD;;;;;;;;GAQG;AACH,eAAO,MAAM,uBAAuB,keAwD1B,CAAA;AAEV,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,uBAAuB,CAAC,CAAC,MAAM,CAAC,CAAA;AAE1E;;;;;;;;GAQG;AACH,eAAO,MAAM,iBAAiB,EAEzB,MAAM,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC,CAAA;AAE/C,gFAAgF;AAChF,eAAO,MAAM,oBAAoB,EAAE,MAAM,EAAiE,CAAA;AAE1G;;;GAGG;AACH,eAAO,MAAM,aAAa,GAAI,MAAM,MAAM,KAAG,MAK5C,CAAA;AAED;;;;;;GAMG;AACH,eAAO,MAAM,UAAU,GAAI,MAAM,MAAM,KAAG,eAAe,GAAG,SAG3D,CAAA"}
1
+ {"version":3,"file":"languages.d.ts","sourceRoot":"","sources":["../../src/compat/languages.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAEpD;;;;;;GAMG;AACH,eAAO,MAAM,wBAAwB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAQ3D,CAAA;AA6HD;;;;;;;;GAQG;AACH,eAAO,MAAM,uBAAuB,keAoD1B,CAAA;AAEV,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,uBAAuB,CAAC,CAAC,MAAM,CAAC,CAAA;AAE1E;;;;;;;;GAQG;AACH,eAAO,MAAM,iBAAiB,EAEzB,MAAM,CAAC,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC,CAAA;AAE/C,gFAAgF;AAChF,eAAO,MAAM,oBAAoB,EAAE,MAAM,EAAiE,CAAA;AAE1G;;;GAGG;AACH,eAAO,MAAM,aAAa,GAAI,MAAM,MAAM,KAAG,MAK5C,CAAA;AAED;;;;;;GAMG;AACH,eAAO,MAAM,UAAU,GAAI,MAAM,MAAM,KAAG,eAAe,GAAG,SAG3D,CAAA"}
@@ -70,8 +70,10 @@ const GRAMMAR_FOR = {
70
70
  js: 'javascript',
71
71
  json: 'json',
72
72
  json5: 'json',
73
+ jl: 'julia',
73
74
  jsonc: 'json',
74
75
  jsx: 'javascript',
76
+ julia: 'julia',
75
77
  kotlin: 'kotlin',
76
78
  kt: 'kotlin',
77
79
  less: 'css',
@@ -167,10 +169,6 @@ export const STANDARD_LANGUAGE_NAMES = [
167
169
  'java',
168
170
  'javascript',
169
171
  'json',
170
- // Kept in `StandardLanguageKey` for parity with `@scalar/code-highlight`,
171
- // which highlights Julia via highlight.js. This zero-dependency package has
172
- // no hand-written Julia grammar yet, so it stays in `unsupportedLanguages`
173
- // and renders as plain escaped text until one is added.
174
172
  'julia',
175
173
  'kotlin',
176
174
  'less',
@@ -1 +1 @@
1
- {"version":3,"file":"syntax-highlight.d.ts","sourceRoot":"","sources":["../../src/compat/syntax-highlight.ts"],"names":[],"mappings":"AAMA,MAAM,MAAM,sBAAsB,GAAG;IACnC,iDAAiD;IACjD,IAAI,EAAE,MAAM,CAAA;IACZ;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,4EAA4E;IAC5E,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,2DAA2D;IAC3D,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;CACpC,CAAA;AAuID;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,eAAe,GAAI,YAAY,MAAM,EAAE,SAAS,sBAAsB,KAAG,MAYrF,CAAA"}
1
+ {"version":3,"file":"syntax-highlight.d.ts","sourceRoot":"","sources":["../../src/compat/syntax-highlight.ts"],"names":[],"mappings":"AAMA,MAAM,MAAM,sBAAsB,GAAG;IACnC,iDAAiD;IACjD,IAAI,EAAE,MAAM,CAAA;IACZ;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,4EAA4E;IAC5E,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,2DAA2D;IAC3D,eAAe,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;CACpC,CAAA;AAkND;;;;;;;;;;;;;;;;;;GAkBG;AACH,eAAO,MAAM,eAAe,GAAI,YAAY,MAAM,EAAE,SAAS,sBAAsB,KAAG,MAUrF,CAAA"}
@@ -12,6 +12,44 @@ const openTag = (cls) => {
12
12
  }
13
13
  return tag;
14
14
  };
15
+ /**
16
+ * Credentials shorter than this are not masked — too likely to appear inside
17
+ * unrelated tokens, and a two-character secret is not a secret.
18
+ */
19
+ const MIN_CREDENTIAL_LENGTH = 3;
20
+ /**
21
+ * Where the credentials sit in the source, as non-overlapping ranges in order.
22
+ *
23
+ * Masking works on these source ranges rather than on the finished HTML. A
24
+ * search over the HTML misses a secret the tokenizer split into several spans
25
+ * (`sk_live.abc123` is a name, a `.` and a member), and it also matches inside
26
+ * the markup itself — a credential that happens to read `span` or `class`
27
+ * used to corrupt the output.
28
+ */
29
+ const findCredentials = (code, credentials) => {
30
+ const list = (typeof credentials === 'string' ? [credentials] : (credentials ?? [])).filter((credential) => credential.length >= MIN_CREDENTIAL_LENGTH);
31
+ const found = [];
32
+ for (const credential of list) {
33
+ let at = code.indexOf(credential);
34
+ while (at !== -1) {
35
+ found.push({ start: at, end: at + credential.length });
36
+ at = code.indexOf(credential, at + credential.length);
37
+ }
38
+ }
39
+ // Earliest first, and the longer of two that start together.
40
+ found.sort((a, b) => a.start - b.start || b.end - a.end);
41
+ const merged = [];
42
+ for (const range of found) {
43
+ const last = merged[merged.length - 1];
44
+ if (last && range.start < last.end) {
45
+ continue;
46
+ }
47
+ merged.push(range);
48
+ }
49
+ return merged;
50
+ };
51
+ const MASK_OPEN = '<span class="credential"><span class="credential-value">';
52
+ const MASK_CLOSE = '</span></span>';
15
53
  /**
16
54
  * Renders `code` to highlight.js-shaped markup, optionally split into lines.
17
55
  *
@@ -24,18 +62,48 @@ const openTag = (cls) => {
24
62
  * vocabulary is finer than highlight.js's, so `keyword.declaration` followed
25
63
  * by `keyword` has to collapse into one `hljs-keyword` span rather than the
26
64
  * two neighbours lowlight would never have emitted.
65
+ *
66
+ * A masked credential takes the class of its first character for its whole
67
+ * length, so it always lands inside a single span, the way a credential inside
68
+ * a string literal did with lowlight.
27
69
  */
28
- const render = (code, grammar, lineMode) => {
70
+ const render = (code, grammar, lineMode, masked) => {
29
71
  const lines = [];
30
72
  let line = '';
31
73
  // Pending run of same-classed ranges, flushed when the class changes.
32
74
  let cls = null;
33
75
  let start = 0;
34
76
  let end = 0;
77
+ // The next masked range, and the class its first character was given.
78
+ let maskIndex = 0;
79
+ let maskClass = null;
80
+ // Flushes move forward through the source, and the masked ranges are
81
+ // sorted and disjoint, so a cursor keeps each flush from rescanning them all.
82
+ let textIndex = 0;
83
+ /** Escaped text of `[from, to)`, with masked ranges wrapped. */
84
+ const text = (from, to) => {
85
+ while (textIndex < masked.length && masked[textIndex].end <= from) {
86
+ textIndex++;
87
+ }
88
+ let out = '';
89
+ let cursor = from;
90
+ for (let i = textIndex; i < masked.length; i++) {
91
+ const range = masked[i];
92
+ if (range.start >= to) {
93
+ break;
94
+ }
95
+ // A multiline credential is wrapped separately inside each line element.
96
+ const start = Math.max(from, range.start);
97
+ const end = Math.min(to, range.end);
98
+ out += escapeHtml(code.slice(cursor, start)) + MASK_OPEN + escapeHtml(code.slice(start, end)) + MASK_CLOSE;
99
+ cursor = end;
100
+ }
101
+ return out + escapeHtml(code.slice(cursor, to));
102
+ };
35
103
  const flush = () => {
36
104
  if (end > start) {
37
- const text = escapeHtml(code.slice(start, end));
38
- line += cls === null ? text : `${openTag(cls)}${text}</span>`;
105
+ const body = masked.length ? text(start, end) : escapeHtml(code.slice(start, end));
106
+ line += cls === null ? body : `${openTag(cls)}${body}</span>`;
39
107
  }
40
108
  start = end;
41
109
  };
@@ -44,8 +112,7 @@ const render = (code, grammar, lineMode) => {
44
112
  lines.push(line);
45
113
  line = '';
46
114
  };
47
- const emit = (scope, s, e) => {
48
- const next = scope === null ? null : hljsClass(scope);
115
+ const place = (next, s, e) => {
49
116
  let at = s;
50
117
  while (at < e) {
51
118
  const nl = lineMode ? code.indexOf('\n', at) : -1;
@@ -70,6 +137,32 @@ const render = (code, grammar, lineMode) => {
70
137
  end = at;
71
138
  }
72
139
  };
140
+ const emit = (scope, s, e) => {
141
+ const next = scope === null ? null : hljsClass(scope);
142
+ // Split the range where it enters or leaves a masked range; the inside
143
+ // parts take the class the masked range started with.
144
+ let at = s;
145
+ while (at < e) {
146
+ while (maskIndex < masked.length && masked[maskIndex].end <= at) {
147
+ maskIndex++;
148
+ }
149
+ const range = masked[maskIndex];
150
+ if (range === undefined || range.start >= e) {
151
+ place(next, at, e);
152
+ return;
153
+ }
154
+ if (at < range.start) {
155
+ place(next, at, range.start);
156
+ at = range.start;
157
+ }
158
+ if (at === range.start) {
159
+ maskClass = next;
160
+ }
161
+ const stop = range.end < e ? range.end : e;
162
+ place(maskClass, at, stop);
163
+ at = stop;
164
+ }
165
+ };
73
166
  if (grammar) {
74
167
  tokenize(code, grammar, emit);
75
168
  }
@@ -88,26 +181,6 @@ const render = (code, grammar, lineMode) => {
88
181
  }
89
182
  return { html, count: lines.length };
90
183
  };
91
- /**
92
- * Credentials shorter than this are not masked — too likely to appear inside
93
- * unrelated tokens, and a two-character secret is not a secret.
94
- */
95
- const MIN_CREDENTIAL_LENGTH = 3;
96
- /**
97
- * Replaces credential substrings in the finished HTML.
98
- *
99
- * Post-processing the serialized string, rather than the token stream, is what
100
- * `@scalar/code-highlight` does. It is kept here deliberately: a credential
101
- * that straddles a token boundary masks or fails to mask exactly as it does
102
- * today, so a swap cannot change what is visible on screen.
103
- */
104
- const applyCredentialMask = (html, credentials) => {
105
- const list = (typeof credentials === 'string' ? [credentials] : (credentials ?? [])).filter((credential) => credential.length >= MIN_CREDENTIAL_LENGTH);
106
- if (!list.length) {
107
- return html;
108
- }
109
- return list.reduce((acc, credential) => acc.split(credential).join(`<span class="credential"><span class="credential-value">${credential}</span></span>`), html);
110
- };
111
184
  /**
112
185
  * Drop-in replacement for `syntaxHighlight` from `@scalar/code-highlight`.
113
186
  *
@@ -131,7 +204,8 @@ export const syntaxHighlight = (codeString, options) => {
131
204
  const name = canonicalName(options.lang);
132
205
  const grammar = grammarFor(name);
133
206
  const lineMode = options.lineNumbers === true;
134
- const { html, count } = render(codeString, grammar, lineMode);
207
+ const masked = findCredentials(codeString, options.maskCredentials);
208
+ const { html, count } = render(codeString, grammar, lineMode, masked);
135
209
  const style = lineMode ? ` style="--line-count: ${count}; --line-digits: ${count.toString().length};"` : '';
136
- return applyCredentialMask(`<pre><code class="hljs language-${escapeAttribute(name)}"${style}>${html}</code></pre>`, options.maskCredentials);
210
+ return `<pre><code class="hljs language-${escapeAttribute(name)}"${style}>${html}</code></pre>`;
137
211
  };
@@ -1,3 +1,11 @@
1
- import type { CompiledGrammar, Grammar } from './types.js';
1
+ import type { CompiledGrammar, CompiledState, Grammar } from './types.js';
2
+ /**
3
+ * The merged regex of a dynamic state, and its per-rule sticky patterns, with
4
+ * `value` spliced in wherever a rule wrote `REMEMBERED`. Built once per value.
5
+ */
6
+ export declare const dynamicRegExps: (state: CompiledState, value: string) => {
7
+ re: RegExp;
8
+ own: (RegExp | null)[];
9
+ };
2
10
  export declare const compile: (grammar: Grammar) => CompiledGrammar;
3
11
  //# sourceMappingURL=compile.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"compile.d.ts","sourceRoot":"","sources":["../../src/core/compile.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAA+B,OAAO,EAAmB,MAAM,SAAS,CAAA;AA4IrG,eAAO,MAAM,OAAO,GAAI,SAAS,OAAO,KAAG,eA8B1C,CAAA"}
1
+ {"version":3,"file":"compile.d.ts","sourceRoot":"","sources":["../../src/core/compile.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAgB,aAAa,EAAE,OAAO,EAAmB,MAAM,SAAS,CAAA;AA+QrG;;;GAGG;AACH,eAAO,MAAM,cAAc,GAAI,OAAO,aAAa,EAAE,OAAO,MAAM,KAAG;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAA;CAmBxG,CAAA;AAED,eAAO,MAAM,OAAO,GAAI,SAAS,OAAO,KAAG,eAmC1C,CAAA"}