@scalar/highlight 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. package/CHANGELOG.md +13 -0
  2. package/README.md +324 -0
  3. package/dist/all.d.ts +28 -0
  4. package/dist/all.d.ts.map +1 -0
  5. package/dist/all.js +105 -0
  6. package/dist/compat/hljs.d.ts +24 -0
  7. package/dist/compat/hljs.d.ts.map +1 -0
  8. package/dist/compat/hljs.js +95 -0
  9. package/dist/compat/index.d.ts +24 -0
  10. package/dist/compat/index.d.ts.map +1 -0
  11. package/dist/compat/index.js +21 -0
  12. package/dist/compat/languages.d.ts +46 -0
  13. package/dist/compat/languages.d.ts.map +1 -0
  14. package/dist/compat/languages.js +236 -0
  15. package/dist/compat/syntax-highlight.d.ts +37 -0
  16. package/dist/compat/syntax-highlight.d.ts.map +1 -0
  17. package/dist/compat/syntax-highlight.js +133 -0
  18. package/dist/core/compile.d.ts +3 -0
  19. package/dist/core/compile.d.ts.map +1 -0
  20. package/dist/core/compile.js +142 -0
  21. package/dist/core/index.d.ts +9 -0
  22. package/dist/core/index.d.ts.map +1 -0
  23. package/dist/core/index.js +5 -0
  24. package/dist/core/registry.d.ts +14 -0
  25. package/dist/core/registry.d.ts.map +1 -0
  26. package/dist/core/registry.js +77 -0
  27. package/dist/core/render.d.ts +67 -0
  28. package/dist/core/render.d.ts.map +1 -0
  29. package/dist/core/render.js +216 -0
  30. package/dist/core/scopes.d.ts +94 -0
  31. package/dist/core/scopes.d.ts.map +1 -0
  32. package/dist/core/scopes.js +113 -0
  33. package/dist/core/tokenize.d.ts +18 -0
  34. package/dist/core/tokenize.d.ts.map +1 -0
  35. package/dist/core/tokenize.js +131 -0
  36. package/dist/core/types.d.ts +99 -0
  37. package/dist/core/types.d.ts.map +1 -0
  38. package/dist/core/types.js +10 -0
  39. package/dist/index.d.ts +19 -0
  40. package/dist/index.d.ts.map +1 -0
  41. package/dist/index.js +20 -0
  42. package/dist/langs/bash.d.ts +4 -0
  43. package/dist/langs/bash.d.ts.map +1 -0
  44. package/dist/langs/bash.js +123 -0
  45. package/dist/langs/c.d.ts +4 -0
  46. package/dist/langs/c.d.ts.map +1 -0
  47. package/dist/langs/c.js +196 -0
  48. package/dist/langs/clojure.d.ts +4 -0
  49. package/dist/langs/clojure.d.ts.map +1 -0
  50. package/dist/langs/clojure.js +213 -0
  51. package/dist/langs/cpp.d.ts +4 -0
  52. package/dist/langs/cpp.d.ts.map +1 -0
  53. package/dist/langs/cpp.js +391 -0
  54. package/dist/langs/csharp.d.ts +4 -0
  55. package/dist/langs/csharp.d.ts.map +1 -0
  56. package/dist/langs/csharp.js +307 -0
  57. package/dist/langs/css.d.ts +4 -0
  58. package/dist/langs/css.d.ts.map +1 -0
  59. package/dist/langs/css.js +94 -0
  60. package/dist/langs/dart.d.ts +4 -0
  61. package/dist/langs/dart.d.ts.map +1 -0
  62. package/dist/langs/dart.js +262 -0
  63. package/dist/langs/diff.d.ts +10 -0
  64. package/dist/langs/diff.d.ts.map +1 -0
  65. package/dist/langs/diff.js +26 -0
  66. package/dist/langs/dockerfile.d.ts +4 -0
  67. package/dist/langs/dockerfile.d.ts.map +1 -0
  68. package/dist/langs/dockerfile.js +308 -0
  69. package/dist/langs/elixir.d.ts +4 -0
  70. package/dist/langs/elixir.d.ts.map +1 -0
  71. package/dist/langs/elixir.js +343 -0
  72. package/dist/langs/fsharp.d.ts +4 -0
  73. package/dist/langs/fsharp.d.ts.map +1 -0
  74. package/dist/langs/fsharp.js +330 -0
  75. package/dist/langs/go.d.ts +5 -0
  76. package/dist/langs/go.d.ts.map +1 -0
  77. package/dist/langs/go.js +90 -0
  78. package/dist/langs/graphql.d.ts +4 -0
  79. package/dist/langs/graphql.d.ts.map +1 -0
  80. package/dist/langs/graphql.js +185 -0
  81. package/dist/langs/haskell.d.ts +4 -0
  82. package/dist/langs/haskell.d.ts.map +1 -0
  83. package/dist/langs/haskell.js +375 -0
  84. package/dist/langs/html.d.ts +14 -0
  85. package/dist/langs/html.d.ts.map +1 -0
  86. package/dist/langs/html.js +71 -0
  87. package/dist/langs/http.d.ts +4 -0
  88. package/dist/langs/http.d.ts.map +1 -0
  89. package/dist/langs/http.js +185 -0
  90. package/dist/langs/ini.d.ts +4 -0
  91. package/dist/langs/ini.d.ts.map +1 -0
  92. package/dist/langs/ini.js +227 -0
  93. package/dist/langs/java.d.ts +4 -0
  94. package/dist/langs/java.d.ts.map +1 -0
  95. package/dist/langs/java.js +178 -0
  96. package/dist/langs/javascript.d.ts +4 -0
  97. package/dist/langs/javascript.d.ts.map +1 -0
  98. package/dist/langs/javascript.js +226 -0
  99. package/dist/langs/json.d.ts +10 -0
  100. package/dist/langs/json.d.ts.map +1 -0
  101. package/dist/langs/json.js +46 -0
  102. package/dist/langs/kotlin.d.ts +4 -0
  103. package/dist/langs/kotlin.d.ts.map +1 -0
  104. package/dist/langs/kotlin.js +220 -0
  105. package/dist/langs/lua.d.ts +4 -0
  106. package/dist/langs/lua.d.ts.map +1 -0
  107. package/dist/langs/lua.js +330 -0
  108. package/dist/langs/makefile.d.ts +4 -0
  109. package/dist/langs/makefile.d.ts.map +1 -0
  110. package/dist/langs/makefile.js +266 -0
  111. package/dist/langs/markdown.d.ts +11 -0
  112. package/dist/langs/markdown.d.ts.map +1 -0
  113. package/dist/langs/markdown.js +71 -0
  114. package/dist/langs/matlab.d.ts +4 -0
  115. package/dist/langs/matlab.d.ts.map +1 -0
  116. package/dist/langs/matlab.js +352 -0
  117. package/dist/langs/mojo.d.ts +4 -0
  118. package/dist/langs/mojo.d.ts.map +1 -0
  119. package/dist/langs/mojo.js +337 -0
  120. package/dist/langs/nginx.d.ts +4 -0
  121. package/dist/langs/nginx.d.ts.map +1 -0
  122. package/dist/langs/nginx.js +168 -0
  123. package/dist/langs/objectivec.d.ts +4 -0
  124. package/dist/langs/objectivec.d.ts.map +1 -0
  125. package/dist/langs/objectivec.js +287 -0
  126. package/dist/langs/ocaml.d.ts +4 -0
  127. package/dist/langs/ocaml.d.ts.map +1 -0
  128. package/dist/langs/ocaml.js +309 -0
  129. package/dist/langs/perl.d.ts +4 -0
  130. package/dist/langs/perl.d.ts.map +1 -0
  131. package/dist/langs/perl.js +435 -0
  132. package/dist/langs/php.d.ts +4 -0
  133. package/dist/langs/php.d.ts.map +1 -0
  134. package/dist/langs/php.js +325 -0
  135. package/dist/langs/powershell.d.ts +4 -0
  136. package/dist/langs/powershell.d.ts.map +1 -0
  137. package/dist/langs/powershell.js +308 -0
  138. package/dist/langs/python.d.ts +4 -0
  139. package/dist/langs/python.d.ts.map +1 -0
  140. package/dist/langs/python.js +350 -0
  141. package/dist/langs/r.d.ts +4 -0
  142. package/dist/langs/r.d.ts.map +1 -0
  143. package/dist/langs/r.js +265 -0
  144. package/dist/langs/ruby.d.ts +4 -0
  145. package/dist/langs/ruby.d.ts.map +1 -0
  146. package/dist/langs/ruby.js +365 -0
  147. package/dist/langs/rust.d.ts +10 -0
  148. package/dist/langs/rust.d.ts.map +1 -0
  149. package/dist/langs/rust.js +116 -0
  150. package/dist/langs/scala.d.ts +4 -0
  151. package/dist/langs/scala.d.ts.map +1 -0
  152. package/dist/langs/scala.js +308 -0
  153. package/dist/langs/sql.d.ts +5 -0
  154. package/dist/langs/sql.d.ts.map +1 -0
  155. package/dist/langs/sql.js +57 -0
  156. package/dist/langs/swift.d.ts +33 -0
  157. package/dist/langs/swift.d.ts.map +1 -0
  158. package/dist/langs/swift.js +256 -0
  159. package/dist/langs/yaml.d.ts +11 -0
  160. package/dist/langs/yaml.d.ts.map +1 -0
  161. package/dist/langs/yaml.js +123 -0
  162. package/dist/lazy.d.ts +69 -0
  163. package/dist/lazy.d.ts.map +1 -0
  164. package/dist/lazy.js +200 -0
  165. package/dist/style.css +308 -0
  166. package/package.json +86 -0
package/CHANGELOG.md ADDED
@@ -0,0 +1,13 @@
1
+ # @scalar/highlight
2
+
3
+ ## 0.2.0
4
+
5
+ ### Minor Changes
6
+
7
+ - [#9923](https://github.com/scalar/scalar/pull/9923): Add `@scalar/highlight`, a zero-dependency syntax highlighter. A grammar is plain data describing a small state machine, each state compiles to one merged regular expression, and rendering goes straight to an HTML string without allocating token objects. Languages are separate modules, so importing `python` does not cost you `scala`.
8
+
9
+ Entry points are `.` (`highlight`, `highlightBlock`, `tokenize`, the registry and the scope vocabulary), `./core` for the engine alone, `./all` and `./lazy` for eager and code-split registration, `./langs/*` for a single grammar, `./style.css` for the stylesheet, and `./compat`, a drop-in for `@scalar/code-highlight`'s `syntaxHighlight` that emits the same `hljs-*` markup. Nothing currently consumes it — this only adds the package.
10
+
11
+ ### Patch Changes
12
+
13
+ - [#9913](https://github.com/scalar/scalar/pull/9913): Add Julia (HTTP.jl) as a code example target. The new `julia/http` client generates HTTP.jl snippets, including headers, query parameters, cookies, basic auth, JSON bodies (as `Dict`s serialized with `JSON.json`), url-encoded bodies and `HTTP.Form` multipart uploads. Julia syntax highlighting and a Julia icon are included, so the client shows up in the code example picker like any other language.
package/README.md ADDED
@@ -0,0 +1,324 @@
1
+ # @scalar/highlight
2
+
3
+ Scalar's code highlighter.
4
+
5
+ - **Small.** 2.3 KB core, ~1.7 KB for Python, gzipped. Nothing you do not import.
6
+ - **No theme system.** Colors come from the `--scalar-*` variables `@scalar/themes` already defines, so a code block follows the app's theme and switches with `.light-mode` / `.dark-mode` — without being re-highlighted.
7
+ - **Zero runtime dependencies.**
8
+
9
+ Forty languages ship today: Python, JavaScript/TypeScript/JSX/TSX, Rust, Go, C, C++, C#, F#, Java, Kotlin, Scala, Swift, Objective-C, OCaml, Haskell, Elixir, Clojure, Dart, Mojo, PHP, Perl, Ruby, Lua, R, MATLAB, Bash, PowerShell, SQL, GraphQL, HTTP, YAML, JSON, TOML/INI, CSS, HTML, Markdown, Dockerfiles, Makefiles, nginx configs, and diffs.
10
+
11
+ ```bash
12
+ npm install @scalar/highlight
13
+ ```
14
+
15
+ ## Quick start
16
+
17
+ ```js
18
+ import { highlightBlock, registerLanguage } from '@scalar/highlight';
19
+ import python from '@scalar/highlight/langs/python';
20
+ import '@scalar/highlight/style.css';
21
+
22
+ registerLanguage(python);
23
+
24
+ const html = highlightBlock(source, 'python', { lineNumbers: true });
25
+ ```
26
+
27
+ That is the whole setup. `html` is a `<pre class="shl-code">…</pre>` block, and the
28
+ stylesheet colors it from the theme already loaded on the page.
29
+
30
+ > **Requires a mode class.** `@scalar/themes` defines its palette only under
31
+ > `.light-mode` or `.dark-mode`, so one of them must be on a container — the same
32
+ > contract every other Scalar component has. Without it the block still renders,
33
+ > readable but unstyled.
34
+
35
+ ### Loading languages on demand
36
+
37
+ ```js
38
+ import { highlight } from '@scalar/highlight';
39
+ import { loadLanguage } from '@scalar/highlight/lazy';
40
+
41
+ await loadLanguage('rust'); // one dynamic import, one bundler chunk
42
+ highlight(source, 'rust');
43
+ ```
44
+
45
+ Every entry in the loader is a static `import()`, so bundlers split each language into its
46
+ own chunk. The initial chunk with the loader and no languages is 3.9 KB.
47
+
48
+ ### Entry points
49
+
50
+ | Import | What you get |
51
+ | --- | --- |
52
+ | `@scalar/highlight` | `highlight`, `highlightBlock`, `tokenize`, the registry |
53
+ | `@scalar/highlight/style.css` | The stylesheet. Required for color |
54
+ | `@scalar/highlight/core` | Just the engine |
55
+ | `@scalar/highlight/langs/<name>` | One grammar |
56
+ | `@scalar/highlight/lazy` | `loadLanguage`, for code splitting |
57
+ | `@scalar/highlight/all` | Every language, registered. For build-time rendering |
58
+ | `@scalar/highlight/compat` | Drop-in `syntaxHighlight` for `@scalar/code-highlight` |
59
+
60
+ ### Rendering somewhere other than HTML
61
+
62
+ ```js
63
+ import { tokenize } from '@scalar/highlight';
64
+
65
+ for (const { scope, text } of tokenize(source, 'python')) {
66
+ // scope is 'keyword.control' | 'string' | … | null
67
+ }
68
+ ```
69
+
70
+ For hot paths there is a streaming form that allocates nothing per token — it is what the
71
+ HTML renderer uses:
72
+
73
+ ```js
74
+ import { tokenizeStream, resolveGrammar } from '@scalar/highlight';
75
+
76
+ tokenizeStream(source, resolveGrammar('python'), (scope, start, end) => { … });
77
+ ```
78
+
79
+ ## Bundle size
80
+
81
+ Bundled with Rolldown, minified, gzipped. Entry points *call* the API rather than
82
+ re-exporting it, so tree-shaking runs the way it does in a real app.
83
+
84
+ | Bundle | Gzipped |
85
+ | --- | ---: |
86
+ | `@scalar/highlight` core | 2.32 KB |
87
+ | …+ Python | 4.33 KB |
88
+ | …lazy loader, initial chunk | 3.93 KB |
89
+ | …`style.css` | 1.26 KB |
90
+
91
+ Per language, on top of the core: `diff` 0.14 KB · `json` 0.28 KB · `html` 0.32 KB · `markdown` 0.46 KB · `yaml` 0.52 KB · `css` 0.53 KB · `nginx` 0.54 KB · `http` 0.64 KB · `ini` 0.71 KB · `graphql` 0.73 KB · `go` 0.79 KB · `bash` 0.80 KB · `rust` 0.83 KB · `makefile` 0.90 KB · `dockerfile` 0.95 KB · `sql` 0.97 KB · `java` 1.15 KB · `dart` 1.18 KB · `lua` 1.20 KB · `r` 1.28 KB · `swift` 1.30 KB · `c` 1.32 KB · `clojure` 1.32 KB · `kotlin` 1.33 KB · `javascript` 1.36 KB · `scala` 1.37 KB · `elixir` 1.38 KB · `powershell` 1.38 KB · `haskell` 1.41 KB · `ocaml` 1.41 KB · `matlab` 1.44 KB · `objectivec` 1.44 KB · `csharp` 1.50 KB · `mojo` 1.55 KB · `fsharp` 1.69 KB · `python` 1.70 KB · `ruby` 1.71 KB · `php` 1.75 KB · `cpp` 1.76 KB · `perl` 1.86 KB.
92
+
93
+ Reproduce with `pnpm size`.
94
+
95
+ ## How it works
96
+
97
+ A grammar is a small **state machine**, and each state is compiled to **one merged regular
98
+ expression**.
99
+
100
+ ```js
101
+ {
102
+ match: '\\b(def)([ \\t]+)(\\w+)([ \\t]*)(\\()',
103
+ scope: ['keyword.declaration', null, 'function', null, 'punctuation.bracket'],
104
+ push: 'params',
105
+ }
106
+ ```
107
+
108
+ Three decisions follow from that shape and explain most of the performance:
109
+
110
+ **States describe only what is interesting.** Anything a state does not match is emitted
111
+ with that state's `default` scope. A string state needs three rules — escape,
112
+ interpolation, closing quote — not a rule for "string content". Sparse grammars are why
113
+ each language is around a kilobyte.
114
+
115
+ **One `exec` finds the next token.** Rules are merged into a single ordered alternation, so
116
+ the regex engine skips plain text in native code instead of a JavaScript loop trying
117
+ patterns at every position. Rule order is priority order, exactly as alternation already
118
+ works.
119
+
120
+ **Capture offsets are resolved off the hot path.** Scoping capture groups needs the `d`
121
+ flag, which makes the engine compute offsets for *every* group on *every* match — measured
122
+ at ~40% of scan throughput. Instead the merged regex runs without `d`, and the handful of
123
+ rules that need offsets re-run their own small sticky pattern at the position that already
124
+ matched.
125
+
126
+ Rendering streams: `tokenize` hands ranges to a callback, and the HTML renderer
127
+ concatenates directly from the source string. No token objects are allocated, adjacent
128
+ ranges sharing a scope merge into one span, and unscoped text is written bare.
129
+
130
+ ## Scopes and styling
131
+
132
+ Grammars emit scopes from a [fixed vocabulary](src/core/scopes.ts) — `keyword.control`,
133
+ `string.escape`, `variable.parameter`, and so on. Each maps to a short class
134
+ (`<span class="shl-kd">def</span>`), and [`style.css`](src/style.css) points those
135
+ classes at fourteen semantic slots:
136
+
137
+ ```css
138
+ /* the entire theming surface, per token kind */
139
+ .shl-code .shl-s { color: var(--scalar-hl-string); }
140
+
141
+ /* recolor every string in the app, in one line */
142
+ .shl-code { --scalar-hl-string: var(--scalar-color-purple); }
143
+ ```
144
+
145
+ Because the color lives in a variable that `@scalar/themes` already swaps, changing theme
146
+ is a class change on a container. The HTML never moves — which also means a page can be
147
+ rendered once at build time and still respect a runtime theme toggle.
148
+
149
+ ### About the `color-mix()`
150
+
151
+ Scalar's semantic palette is tuned for UI — badges, labels, large text — and several hues
152
+ are too light for 13px code on a light background. Against `--scalar-background-2` the
153
+ default theme's light values give yellow 1.6:1, accent 2.8:1 and orange 2.9:1, where WCAG
154
+ AA wants 4.5:1.
155
+
156
+ So each hue is pulled toward `--scalar-color-1`, the theme's own text color. That preserves
157
+ the hue's identity — a custom theme's green still drives strings — while guaranteeing it
158
+ moves toward whatever contrasts with that theme's background. `src/style.test.ts`
159
+ recomputes every resolved color in oklab and fails if one drops below AA, so the guarantee
160
+ stays checked rather than checked once.
161
+
162
+ `--scalar-hl-punctuation` is the one deliberate exception, held to the 3:1 non-text bar.
163
+ Brackets and commas are structural; lifting them to 4.5:1 would make them as loud as the
164
+ code they punctuate.
165
+
166
+ ## Adding a language
167
+
168
+ A grammar is plain data with no imports beyond its type:
169
+
170
+ ```ts
171
+ import type { Grammar } from '@scalar/highlight/core';
172
+
173
+ const toml: Grammar = {
174
+ name: 'toml',
175
+ aliases: ['tml'],
176
+ states: {
177
+ root: {
178
+ rules: [
179
+ { match: '#[^\\n]*', scope: 'comment' },
180
+ { match: '^\\[[^\\]]+\\]', scope: 'namespace' },
181
+ { match: '(\\w+)(\\s*)(=)', scope: ['property', null, 'operator'] },
182
+ { match: '"', scope: 'string', push: 'string' },
183
+ ],
184
+ },
185
+ string: {
186
+ default: 'string',
187
+ rules: [
188
+ { match: '\\\\.', scope: 'string.escape' },
189
+ { match: '"', scope: 'string', pop: true },
190
+ ],
191
+ },
192
+ },
193
+ };
194
+ ```
195
+
196
+ Rules may `push`, `pop` or `set` states, and `{ include: 'other-state' }` splices another
197
+ state's rules in at that position, keeping priority order. The compiler rejects unknown
198
+ state names, circular includes, backreferences (group numbers shift when rules merge) and
199
+ scope arrays longer than the pattern's capture count — all at compile time, not mid-tokenize
200
+ on a rare branch.
201
+
202
+ The test suite asserts, for every language and every sample, that emitted tokens
203
+ reconstruct the source exactly, that scopes come from the shared vocabulary, and that
204
+ tokenizing every truncated prefix terminates and stays consistent — which is what an editor
205
+ does on every keystroke.
206
+
207
+ ## Replacing `@scalar/code-highlight`
208
+
209
+ `@scalar/highlight/compat` is a drop-in `syntaxHighlight`. Same signature, same
210
+ `<pre><code class="hljs language-x">` envelope, same `hljs-*` token classes — so
211
+ [`code.css`](https://github.com/scalar/scalar/blob/main/packages/code-highlight/src/css/code.css)
212
+ keeps working unchanged and `ScalarCodeBlock` needs one import line changed:
213
+
214
+ ```diff
215
+ -import { standardLanguages, syntaxHighlight } from '@scalar/code-highlight';
216
+ +import '@scalar/highlight/all';
217
+ +import { standardLanguages, syntaxHighlight } from '@scalar/highlight/compat';
218
+ ```
219
+
220
+ `standardLanguages` is re-exported with the same key set, so `StandardLanguageKey` is
221
+ unchanged and the component's props keep their types.
222
+
223
+ ### What it covers, and what it does not
224
+
225
+ `@scalar/code-highlight` is two packages wearing one name. The compat layer replaces the
226
+ first half only:
227
+
228
+ | export | status |
229
+ | --- | --- |
230
+ | `syntaxHighlight` | covered |
231
+ | `standardLanguages` | key-compatible; 49 of 50 names have a grammar |
232
+ | `basicLanguages`, `jsonYamlLanguages` | **not covered** — never exported here |
233
+ | `lowlightLanguageMappings` | covered |
234
+ | `htmlFromMarkdown`, `getHeadings`, `splitContent`, `isHeading`, `textFromNode` | **not covered** |
235
+ | `rehypeHighlight`, `rehypeAlert` | **not covered** |
236
+
237
+ The Markdown half is a remark/rehype pipeline — GFM, sanitization, GitHub alerts, external
238
+ links — that has nothing to do with tokenizing code, and it is not something this library
239
+ should grow. A swap keeps `@scalar/code-highlight` as a dependency for
240
+ `ScalarMarkdown` and drops it from `ScalarCodeBlock`.
241
+
242
+ The one `standardLanguages` name with no grammar — `plaintext` — renders as escaped,
243
+ unhighlighted code, which is exactly what the lowlight pipeline already does for a language
244
+ it cannot resolve. `curl` resolves to the Bash grammar, since a curl invocation is a shell
245
+ command line; the `language-curl` class `code.css` targets is unaffected, because that
246
+ class comes from the name the caller asked for.
247
+
248
+ ### Verifying it
249
+
250
+ [`src/compat/compat.test.ts`](src/compat/compat.test.ts) runs the real lowlight pipeline —
251
+ imported straight from [`@scalar/code-highlight`](../code-highlight) — beside ours over the
252
+ shared sample corpus, and asserts that the rendered text, the `<code>` envelope and the line
253
+ elements match exactly. It also ports the four assertions `@scalar/code-highlight` ships
254
+ today. Because the comparison runs against the workspace package rather than a copy of it,
255
+ a change on either side shows up here.
256
+
257
+ Colour is the part that does *not* match exactly, and `pnpm compat:report --detail`
258
+ quantifies it: across the whole 41-sample corpus 68.5% of visible characters keep the colour
259
+ `code.css` gives them, and 82% across the thirteen languages the corpus started with. The
260
+ rest is our finer scope vocabulary showing through — f-string interiors scoped as code
261
+ rather than string, docstrings as comments rather than strings, punctuation given its own
262
+ scope — and it is widest where highlight.js scopes least, as on `http`, where it leaves
263
+ header values, `#` comments and status numbers unstyled. Each round of grammars keeps its
264
+ own floor — the whole corpus, the twenty-six samples before the latest round, and the
265
+ original thirteen — so a new language cannot dilute a regression in an older one.
266
+
267
+ For the same three languages `ScalarCodeBlock` shows most often:
268
+
269
+ | | gzipped |
270
+ | --- | ---: |
271
+ | `@scalar/code-highlight` | 127.34 KB |
272
+ | `@scalar/highlight/compat` | 5.36 KB |
273
+
274
+ ## Trade-offs
275
+
276
+ **It emits more markup.** Punctuation, operators and brackets get their own scopes rather
277
+ than inheriting the foreground, which is roughly 45% of the spans. That is the deliberate
278
+ cost of the way it looks. If you want it back, drop the punctuation rules from a grammar —
279
+ it is data.
280
+
281
+ **Hand-written grammars are approximations.** A tokenizer built on real TextMate grammars
282
+ will get things right on unusual syntax that these do not. The trade is startup cost and
283
+ bundle weight, and this library sits at the small end of it.
284
+
285
+ **Python's heuristics are heuristics.** Keyword arguments are told from assignments by PEP 8
286
+ spacing (`f(x=1)` versus `x = 1`); a triple-quoted string at the start of a line is treated
287
+ as a docstring; `CapWords` reads as a type and `SCREAMING_CASE` as a constant. These are
288
+ right on ordinary code and wrong on code written to defeat them.
289
+
290
+ **No cross-language embedding yet.** `<script>` bodies in HTML and fenced code blocks in
291
+ Markdown are scoped as plain text rather than highlighted with their own grammar. The engine
292
+ has the state machinery for it; the registry hook is not written.
293
+
294
+ **No automatic language detection.** Pass a language name. Detection means shipping every
295
+ grammar, which is the thing this library exists to avoid.
296
+
297
+ ## Development
298
+
299
+ From the repository root:
300
+
301
+ ```bash
302
+ pnpm install
303
+ pnpm vitest packages/highlight --run # the suite
304
+ pnpm --filter @scalar/highlight build # tsc, plus a copy of style.css
305
+ pnpm --filter @scalar/highlight types:check
306
+ ```
307
+
308
+ From this directory:
309
+
310
+ ```bash
311
+ pnpm bench # throughput per language, for comparing a branch against main
312
+ pnpm size # bundle size per entry point and per language
313
+ pnpm compat:report # @scalar/code-highlight swap report; --detail for colour drift
314
+ ```
315
+
316
+ The published package ships zero dependencies. `rolldown` measures bundle size and
317
+ `highlight.js` supplies the reference grammars the compat tests differ against; both are
318
+ devDependencies and neither reaches consumers.
319
+
320
+ Repository-wide conventions live in the root [`AGENTS.md`](../../AGENTS.md).
321
+
322
+ ## License
323
+
324
+ MIT
package/dist/all.d.ts ADDED
@@ -0,0 +1,28 @@
1
+ import type { Grammar } from './core/types.js';
2
+ /**
3
+ * Every bundled language, registered on import.
4
+ *
5
+ * Meant for build-time rendering — static site generators, docs pipelines,
6
+ * anything where bundle size is not a constraint. In a browser bundle prefer
7
+ * `@scalar/highlight/lazy`, or import the two or three languages you actually use.
8
+ */
9
+ export declare const languages: Grammar[];
10
+ /**
11
+ * The root entry's surface, re-exported by name.
12
+ *
13
+ * `import { highlight } from '@scalar/highlight/all'` has to work without a
14
+ * second import, so everything `.` exports is listed here. Spelled out rather
15
+ * than `export *` so the entry's surface is greppable and Biome's barrel-file
16
+ * rules stay satisfied.
17
+ */
18
+ export { compile } from './core/compile.js';
19
+ export { getLanguage, isRegistered, listLanguages, registerLanguage, resolveGrammar, } from './core/registry.js';
20
+ export type { BlockOptions, RenderOptions } from './core/render.js';
21
+ export { DEFAULT_PREFIX, escapeAttribute, escapeHtml } from './core/render.js';
22
+ export type { Scope } from './core/scopes.js';
23
+ export { SCOPES, SCOPE_NAMES, scopeChain, scopeClass } from './core/scopes.js';
24
+ export { tokenize as tokenizeStream } from './core/tokenize.js';
25
+ export type { CompiledGrammar, Emit, Grammar, Include, Rule, ScopeName, State, StateRule, Token, } from './core/types.js';
26
+ export type { LanguageInput } from './index.js';
27
+ export { highlight, highlightBlock, tokenize } from './index.js';
28
+ //# sourceMappingURL=all.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"all.d.ts","sourceRoot":"","sources":["../src/all.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAA;AA0C3C;;;;;;GAMG;AACH,eAAO,MAAM,SAAS,EAAE,OAAO,EAyC9B,CAAA;AAID;;;;;;;GAOG;AACH,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAA;AACxC,OAAO,EACL,WAAW,EACX,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,iBAAiB,CAAA;AACxB,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAChE,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,YAAY,EAAE,KAAK,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC3E,OAAO,EAAE,QAAQ,IAAI,cAAc,EAAE,MAAM,iBAAiB,CAAA;AAC5D,YAAY,EACV,eAAe,EACf,IAAI,EACJ,OAAO,EACP,OAAO,EACP,IAAI,EACJ,SAAS,EACT,KAAK,EACL,SAAS,EACT,KAAK,GACN,MAAM,cAAc,CAAA;AACrB,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAC5C,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA"}
package/dist/all.js ADDED
@@ -0,0 +1,105 @@
1
+ import { registerLanguage } from './core/registry.js';
2
+ import bash from './langs/bash.js';
3
+ import c from './langs/c.js';
4
+ import clojure from './langs/clojure.js';
5
+ import cpp from './langs/cpp.js';
6
+ import csharp from './langs/csharp.js';
7
+ import css from './langs/css.js';
8
+ import dart from './langs/dart.js';
9
+ import diff from './langs/diff.js';
10
+ import dockerfile from './langs/dockerfile.js';
11
+ import elixir from './langs/elixir.js';
12
+ import fsharp from './langs/fsharp.js';
13
+ import go from './langs/go.js';
14
+ import graphql from './langs/graphql.js';
15
+ import haskell from './langs/haskell.js';
16
+ import html from './langs/html.js';
17
+ import http from './langs/http.js';
18
+ import ini from './langs/ini.js';
19
+ import java from './langs/java.js';
20
+ import javascript from './langs/javascript.js';
21
+ import json from './langs/json.js';
22
+ import kotlin from './langs/kotlin.js';
23
+ import lua from './langs/lua.js';
24
+ import makefile from './langs/makefile.js';
25
+ import markdown from './langs/markdown.js';
26
+ import matlab from './langs/matlab.js';
27
+ import mojo from './langs/mojo.js';
28
+ import nginx from './langs/nginx.js';
29
+ import objectivec from './langs/objectivec.js';
30
+ import ocaml from './langs/ocaml.js';
31
+ import perl from './langs/perl.js';
32
+ import php from './langs/php.js';
33
+ import powershell from './langs/powershell.js';
34
+ import python from './langs/python.js';
35
+ import r from './langs/r.js';
36
+ import ruby from './langs/ruby.js';
37
+ import rust from './langs/rust.js';
38
+ import scala from './langs/scala.js';
39
+ import sql from './langs/sql.js';
40
+ import swift from './langs/swift.js';
41
+ import yaml from './langs/yaml.js';
42
+ /**
43
+ * Every bundled language, registered on import.
44
+ *
45
+ * Meant for build-time rendering — static site generators, docs pipelines,
46
+ * anything where bundle size is not a constraint. In a browser bundle prefer
47
+ * `@scalar/highlight/lazy`, or import the two or three languages you actually use.
48
+ */
49
+ export const languages = [
50
+ bash,
51
+ c,
52
+ clojure,
53
+ cpp,
54
+ csharp,
55
+ css,
56
+ dart,
57
+ diff,
58
+ dockerfile,
59
+ elixir,
60
+ fsharp,
61
+ go,
62
+ graphql,
63
+ haskell,
64
+ html,
65
+ http,
66
+ ini,
67
+ java,
68
+ javascript,
69
+ json,
70
+ kotlin,
71
+ lua,
72
+ makefile,
73
+ markdown,
74
+ matlab,
75
+ mojo,
76
+ nginx,
77
+ objectivec,
78
+ ocaml,
79
+ perl,
80
+ php,
81
+ powershell,
82
+ python,
83
+ r,
84
+ ruby,
85
+ rust,
86
+ scala,
87
+ sql,
88
+ swift,
89
+ yaml,
90
+ ];
91
+ registerLanguage(...languages);
92
+ /**
93
+ * The root entry's surface, re-exported by name.
94
+ *
95
+ * `import { highlight } from '@scalar/highlight/all'` has to work without a
96
+ * second import, so everything `.` exports is listed here. Spelled out rather
97
+ * than `export *` so the entry's surface is greppable and Biome's barrel-file
98
+ * rules stay satisfied.
99
+ */
100
+ export { compile } from './core/compile.js';
101
+ export { getLanguage, isRegistered, listLanguages, registerLanguage, resolveGrammar, } from './core/registry.js';
102
+ export { DEFAULT_PREFIX, escapeAttribute, escapeHtml } from './core/render.js';
103
+ export { SCOPES, SCOPE_NAMES, scopeChain, scopeClass } from './core/scopes.js';
104
+ export { tokenize as tokenizeStream } from './core/tokenize.js';
105
+ export { highlight, highlightBlock, tokenize } from './index.js';
@@ -0,0 +1,24 @@
1
+ /**
2
+ * Scope vocabulary -> highlight.js class names.
3
+ *
4
+ * `@scalar/code-highlight` renders through lowlight, so every stylesheet and
5
+ * screenshot in the ecosystem is written against `hljs-*` classes. Emitting
6
+ * those classes is what makes a swap invisible: `packages/code-highlight/src/css/code.css`
7
+ * keeps working unchanged, and nothing downstream has to be restyled.
8
+ *
9
+ * Values are full class attributes, not suffixes, because highlight.js uses
10
+ * two-class tokens for a few scopes (`hljs-title function_`).
11
+ *
12
+ * `null` means "emit bare text". highlight.js leaves operators unscoped, and a
13
+ * span nobody styles is bytes for nothing.
14
+ */
15
+ declare const HLJS_CLASSES: Record<string, string | null>;
16
+ /**
17
+ * Resolves a scope to its highlight.js class attribute.
18
+ *
19
+ * Unknown scopes walk up the chain — a grammar that emits `string.heredoc`
20
+ * still lands on `hljs-string` rather than falling out of the theme.
21
+ */
22
+ export declare const hljsClass: (scope: string) => string | null;
23
+ export { HLJS_CLASSES };
24
+ //# sourceMappingURL=hljs.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"hljs.d.ts","sourceRoot":"","sources":["../../src/compat/hljs.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;GAaG;AACH,QAAA,MAAM,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAmE/C,CAAA;AAED;;;;;GAKG;AACH,eAAO,MAAM,SAAS,GAAI,OAAO,MAAM,KAAG,MAAM,GAAG,IAUlD,CAAA;AAED,OAAO,EAAE,YAAY,EAAE,CAAA"}
@@ -0,0 +1,95 @@
1
+ import { scopeChain } from '../core/scopes.js';
2
+ /**
3
+ * Scope vocabulary -> highlight.js class names.
4
+ *
5
+ * `@scalar/code-highlight` renders through lowlight, so every stylesheet and
6
+ * screenshot in the ecosystem is written against `hljs-*` classes. Emitting
7
+ * those classes is what makes a swap invisible: `packages/code-highlight/src/css/code.css`
8
+ * keeps working unchanged, and nothing downstream has to be restyled.
9
+ *
10
+ * Values are full class attributes, not suffixes, because highlight.js uses
11
+ * two-class tokens for a few scopes (`hljs-title function_`).
12
+ *
13
+ * `null` means "emit bare text". highlight.js leaves operators unscoped, and a
14
+ * span nobody styles is bytes for nothing.
15
+ */
16
+ const HLJS_CLASSES = {
17
+ // --- comments -------------------------------------------------------------
18
+ comment: 'hljs-comment',
19
+ 'comment.doc': 'hljs-comment',
20
+ // --- literals -------------------------------------------------------------
21
+ string: 'hljs-string',
22
+ 'string.escape': 'hljs-string',
23
+ 'string.special': 'hljs-string',
24
+ interpolation: 'hljs-subst',
25
+ regexp: 'hljs-regexp',
26
+ number: 'hljs-number',
27
+ boolean: 'hljs-literal',
28
+ // --- names ----------------------------------------------------------------
29
+ constant: 'hljs-variable',
30
+ 'constant.builtin': 'hljs-literal',
31
+ variable: 'hljs-variable',
32
+ 'variable.builtin': 'hljs-variable',
33
+ 'variable.parameter': 'hljs-params',
34
+ 'variable.member': 'hljs-property',
35
+ 'variable.special': 'hljs-variable',
36
+ // --- keywords -------------------------------------------------------------
37
+ keyword: 'hljs-keyword',
38
+ 'keyword.control': 'hljs-keyword',
39
+ 'keyword.declaration': 'hljs-keyword',
40
+ 'keyword.import': 'hljs-keyword',
41
+ 'keyword.operator': 'hljs-keyword',
42
+ // --- callables and types --------------------------------------------------
43
+ function: 'hljs-title function_',
44
+ 'function.call': 'hljs-title function_',
45
+ 'function.builtin': 'hljs-built_in',
46
+ 'function.method': 'hljs-title function_',
47
+ // highlight.js reserves `hljs-type` for type *keywords* and puts named
48
+ // types under `hljs-title class_`, which is what `code.css` colours as a
49
+ // class name. `int`, `str`, `string`, `number` land on `hljs-built_in`
50
+ // there, so they do here too.
51
+ type: 'hljs-title class_',
52
+ 'type.builtin': 'hljs-built_in',
53
+ class: 'hljs-title class_',
54
+ namespace: 'hljs-title class_',
55
+ decorator: 'hljs-meta',
56
+ // --- syntax ---------------------------------------------------------------
57
+ operator: null,
58
+ punctuation: 'hljs-punctuation',
59
+ 'punctuation.bracket': 'hljs-punctuation',
60
+ 'punctuation.delimiter': 'hljs-punctuation',
61
+ // --- markup ---------------------------------------------------------------
62
+ tag: 'hljs-name',
63
+ 'tag.attribute': 'hljs-attr',
64
+ property: 'hljs-attr',
65
+ selector: 'hljs-selector-tag',
66
+ unit: 'hljs-number',
67
+ heading: 'hljs-section',
68
+ link: 'hljs-link',
69
+ emphasis: 'hljs-emphasis',
70
+ strong: 'hljs-strong',
71
+ quote: 'hljs-quote',
72
+ list: 'hljs-bullet',
73
+ 'diff.plus': 'hljs-addition',
74
+ 'diff.minus': 'hljs-deletion',
75
+ invalid: null,
76
+ };
77
+ /**
78
+ * Resolves a scope to its highlight.js class attribute.
79
+ *
80
+ * Unknown scopes walk up the chain — a grammar that emits `string.heredoc`
81
+ * still lands on `hljs-string` rather than falling out of the theme.
82
+ */
83
+ export const hljsClass = (scope) => {
84
+ for (const name of scopeChain(scope)) {
85
+ // `hasOwn` because scope names come from grammar data: `constructor` or
86
+ // `toString` would otherwise resolve off Object.prototype instead of
87
+ // falling through to the next, less specific scope. An owned value is
88
+ // `string | null` (null means bare text); `?? null` only satisfies the
89
+ // index-access type, it never fires for a key we have confirmed we own.
90
+ if (Object.hasOwn(HLJS_CLASSES, name))
91
+ return HLJS_CLASSES[name] ?? null;
92
+ }
93
+ return null;
94
+ };
95
+ export { HLJS_CLASSES };
@@ -0,0 +1,24 @@
1
+ /**
2
+ * `@scalar/code-highlight` compatibility layer.
3
+ *
4
+ * Enough of that package's surface to swap this highlighter in underneath
5
+ * `ScalarCodeBlock` without touching the component, the stylesheet, or any
6
+ * caller of `syntaxHighlight`. Registering languages is left to the importer,
7
+ * the same as everywhere else in this package:
8
+ *
9
+ * ```js
10
+ * import '@scalar/highlight/all'; // or register the few you need
11
+ * import { syntaxHighlight } from '@scalar/highlight/compat';
12
+ * ```
13
+ *
14
+ * What this layer deliberately does *not* cover is the Markdown half of
15
+ * `@scalar/code-highlight` — `htmlFromMarkdown`, `getHeadings`, `splitContent`,
16
+ * `rehypeAlert`, `rehypeHighlight`. Those are a remark/rehype pipeline that has
17
+ * nothing to do with tokenizing code, and they stay where they are.
18
+ */
19
+ export { HLJS_CLASSES, hljsClass } from './hljs.js';
20
+ export type { StandardLanguageKey } from './languages.js';
21
+ export { STANDARD_LANGUAGE_NAMES, canonicalName, grammarFor, lowlightLanguageMappings, standardLanguages, unsupportedLanguages, } from './languages.js';
22
+ export type { SyntaxHighlightOptions } from './syntax-highlight.js';
23
+ export { syntaxHighlight } from './syntax-highlight.js';
24
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/compat/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AAChD,YAAY,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AACtD,OAAO,EACL,uBAAuB,EACvB,aAAa,EACb,UAAU,EACV,wBAAwB,EACxB,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,aAAa,CAAA;AACpB,YAAY,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAA;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA"}
@@ -0,0 +1,21 @@
1
+ /**
2
+ * `@scalar/code-highlight` compatibility layer.
3
+ *
4
+ * Enough of that package's surface to swap this highlighter in underneath
5
+ * `ScalarCodeBlock` without touching the component, the stylesheet, or any
6
+ * caller of `syntaxHighlight`. Registering languages is left to the importer,
7
+ * the same as everywhere else in this package:
8
+ *
9
+ * ```js
10
+ * import '@scalar/highlight/all'; // or register the few you need
11
+ * import { syntaxHighlight } from '@scalar/highlight/compat';
12
+ * ```
13
+ *
14
+ * What this layer deliberately does *not* cover is the Markdown half of
15
+ * `@scalar/code-highlight` — `htmlFromMarkdown`, `getHeadings`, `splitContent`,
16
+ * `rehypeAlert`, `rehypeHighlight`. Those are a remark/rehype pipeline that has
17
+ * nothing to do with tokenizing code, and they stay where they are.
18
+ */
19
+ export { HLJS_CLASSES, hljsClass } from './hljs.js';
20
+ export { STANDARD_LANGUAGE_NAMES, canonicalName, grammarFor, lowlightLanguageMappings, standardLanguages, unsupportedLanguages, } from './languages.js';
21
+ export { syntaxHighlight } from './syntax-highlight.js';