@allons-y/webc-code-sample 0.1.0-beta.0 → 0.1.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +103 -9
- package/components/norn-code-block.webc +306 -0
- package/index.js +93 -1
- package/llms.txt +70 -0
- package/markdown.js +25 -0
- package/package.json +17 -5
- package/tokenize.js +231 -0
package/README.md
CHANGED
|
@@ -1,31 +1,125 @@
|
|
|
1
1
|
# @allons-y/webc-code-sample
|
|
2
2
|
|
|
3
|
-
`code-
|
|
3
|
+
Part of [Norn](https://github.com/allonsy-studio/webc-resources#readme). `norn-code-block` WebC component with build-time syntax highlighting. Shiki runs at build; the browser receives token ranges for the CSS Custom Highlight API & a copy button, with no highlighter shipped.
|
|
4
|
+
|
|
5
|
+
See it in the [component preview](https://allonsy-studio.github.io/webc-resources/).
|
|
6
|
+
|
|
7
|
+
Using an AI coding agent? Point it at [`llms.txt`](llms.txt): the rules for calling & styling the components, & a reference of every prop, slot, & token.
|
|
4
8
|
|
|
5
9
|
## Status
|
|
6
10
|
|
|
7
|
-
|
|
11
|
+
Beta. Versions are `0.1.0-beta.N` until 1.0.
|
|
8
12
|
|
|
9
13
|
## Install
|
|
10
14
|
|
|
11
15
|
```sh
|
|
12
|
-
npm install --save-dev @allons-y/webc-code-sample
|
|
16
|
+
npm install --save-dev @11ty/eleventy @11ty/eleventy-plugin-webc @allons-y/webc-patterns @allons-y/webc-code-sample
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Node 22 or later. Evergreen browsers from about 2024. Features newer than that, such as the CSS Custom Highlight API, degrade gracefully: the code stays readable plain text.
|
|
20
|
+
|
|
21
|
+
The package registers nothing itself. It is an optional peer dependency of `@allons-y/webc-patterns`: with it installed, the Norn plugin registers `norn-code-block`, the `nornTokenRanges` helper, & the Markdown add-on, & the Norn layouts show their samples with it. The copy button is the patterns package's `norn-copy-button`.
|
|
22
|
+
|
|
23
|
+
## Usage
|
|
24
|
+
|
|
25
|
+
Install it beside the patterns package & add the Norn plugin (or a theme that adds it, such as `@allons-y/webc-oss-site`). There is nothing else to register:
|
|
26
|
+
|
|
27
|
+
```js
|
|
28
|
+
import norn from "@allons-y/webc-patterns/plugin";
|
|
29
|
+
|
|
30
|
+
export default function (config) {
|
|
31
|
+
config.addPlugin(norn, { props: (data) => ({ title: data.title, name: "My project" }) });
|
|
32
|
+
|
|
33
|
+
return { markdownTemplateEngine: false };
|
|
34
|
+
}
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Without the Norn plugin, add `codeBlock` to the project's own WebC `components` list beside `patterns()` from `@allons-y/webc-patterns`, & register `tokenRanges` as `nornTokenRanges`. Without `patterns()` in the list, `norn-copy-button` is an unknown tag & the code blocks render with no copy button & no error.
|
|
38
|
+
|
|
39
|
+
Call the component from a WebC page. Props take the `@` prefix:
|
|
40
|
+
|
|
41
|
+
```html
|
|
42
|
+
<norn-code-block @language="js" @caption="example.js" @code="const answer = 42;"></norn-code-block>
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
### Markdown
|
|
46
|
+
|
|
47
|
+
The Norn plugin adds the Markdown add-on for fenced code. Without the plugin, amend the library yourself:
|
|
48
|
+
|
|
49
|
+
```js
|
|
50
|
+
import { highlightFences } from "@allons-y/webc-code-sample/markdown";
|
|
51
|
+
|
|
52
|
+
config.amendLibrary("md", highlightFences);
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
Each fence renders as the `<norn-code-block>` an author would write, with the fence's first word as `@language`. The add-on fills no slots, so the labels & button texts come from the component's fallbacks, as in Translating below. The fence's tag is compiled by WebC, so the page's layout renders the Markdown content with `@html="content"`. Every Norn layout does, & `norn-base` does for a Markdown page that uses it directly.
|
|
56
|
+
|
|
57
|
+
### Translating
|
|
58
|
+
|
|
59
|
+
Every string a visitor sees or hears is content in the page, in a named slot, hidden until the script uses it: slot `label` names the code when there is no caption, `copy-label` names the copy button (`{name}` stands in for the code's name, filled in by the script), & `copy`, `copied`, & `failed` are the button's texts:
|
|
60
|
+
|
|
61
|
+
```html
|
|
62
|
+
<norn-code-block @language="sh" @code="npm install">
|
|
63
|
+
<span slot="label">Commande d'installation</span>
|
|
64
|
+
<span slot="copy-label">Copier la {name}</span>
|
|
65
|
+
<span slot="copy">Copier</span>
|
|
66
|
+
</norn-code-block>
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
An empty slot falls back to the site's strings from the patterns package's `componentStrings`, under `norn-code-block` (`label`, with `{language}` for the language, & `copy-label`) & `norn-copy-button` (`copy`, `copied`, `failed`), then to English. That covers Markdown fences too: `highlightFences` writes no slots of its own.
|
|
70
|
+
|
|
71
|
+
### Layout requirements
|
|
72
|
+
|
|
73
|
+
The Norn layouts meet these. A layout of the project's own:
|
|
74
|
+
|
|
75
|
+
- Links the page's bundles from the layout head, with `webc:keep` on both tags. WebC removes `<link>` & `<script src>` without it. `norn-head` does this.
|
|
76
|
+
|
|
77
|
+
```html
|
|
78
|
+
<link rel="stylesheet" :href="getBundleFileUrl('css')" webc:keep />
|
|
79
|
+
<script :src="getBundleFileUrl('js')" defer webc:keep></script>
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
- Sets `markdownTemplateEngine: false`, or the default Liquid pass rewrites `{{ }}` inside fences.
|
|
83
|
+
- Drops the `href=""` & `src=""` tags a page with no bundles is left with: `webc:if` can't guard them, because the URL resolves after render. The Norn plugin registers `dropEmptyBundleTags` from the patterns package as a transform for this.
|
|
84
|
+
|
|
85
|
+
### Without JavaScript
|
|
86
|
+
|
|
87
|
+
The code stays readable plain text, & is not a tab stop: the script makes it a named, focusable region only while it scrolls sideways. The highlight colors need the CSS Custom Highlight API & a script. The copy button is an icon over the top end corner of the code, which names itself when the pointer or focus is on it, & needs a script & the Clipboard API, which browsers expose only on secure origins (HTTPS or localhost): elsewhere it stays hidden. The code always reads left to right, in a right-to-left page too. The script registers `norn-code-block` as a custom element, so a sample added to the page later is enhanced too.
|
|
88
|
+
|
|
89
|
+
### Customizing
|
|
90
|
+
|
|
91
|
+
`@chrome="terminal"` frames the code with a header bar for the caption:
|
|
92
|
+
|
|
93
|
+
```html
|
|
94
|
+
<norn-code-block @chrome="terminal" @language="sh" @caption="Terminal" @code="npm install"></norn-code-block>
|
|
13
95
|
```
|
|
14
96
|
|
|
15
|
-
The
|
|
97
|
+
The sample reads shared tokens only, each falling back to its default compiled into the component, so it keeps its look with no stylesheet beside it. Set them in a project `tokens.css`:
|
|
98
|
+
|
|
99
|
+
| Token | Controls |
|
|
100
|
+
| --- | --- |
|
|
101
|
+
| `--norn-surface-code`, `--norn-text-base` | Surface & text colors |
|
|
102
|
+
| `--norn-ui-border-radius`, `--norn-space-m` | Corner radius & inner spacing |
|
|
103
|
+
| `--norn-font-mono` | Font stack |
|
|
104
|
+
| `--norn-code-comment`, `--norn-code-string`, `--norn-code-keyword`, `--norn-code-function`, `--norn-code-number`, `--norn-code-flag` | Syntax colors |
|
|
16
105
|
|
|
17
106
|
## Exports
|
|
18
107
|
|
|
19
108
|
| Export | Purpose |
|
|
20
109
|
| --- | --- |
|
|
21
|
-
| `
|
|
22
|
-
| `tokenRanges(code, language)` |
|
|
23
|
-
| `
|
|
24
|
-
| `loadLanguages(
|
|
110
|
+
| `codeBlock` | Absolute path to `components/norn-code-block.webc` |
|
|
111
|
+
| `tokenRanges(code, language)` | Shiki tokens as the compact JSON that `data-tokens` carries, or undefined with no tokens: `[groups, numbers]`, the group names once each, then three whole numbers per range, the group's index in `groups`, the gap from the end of the range before (from 0 for the first), & the length: `[["keyword","string"],[0,0,5,1,3,4]]` is `keyword` from 0 to 5 & `string` from 8 to 12. The ranges are character offsets in order, & none overlaps the next. The groups are `comment`, `string`, `keyword`, `function`, `number`, `flag`, & `punctuation`. For `sh`, `bash`, `shell`, & `shell-session`, the command is a `function`, its subcommand a `keyword`, a flag a `flag`, & an argument a `string`. The plugin registers it as `nornTokenRanges` |
|
|
112
|
+
| `decodeRanges(value)` | Reads a `tokenRanges` string back into `[group, start, end][]` character offsets. Undefined reads as `[]`; any other value that is not the compact form throws a `TypeError` that says what is wrong |
|
|
113
|
+
| `loadLanguages(...languages)` | Adds Shiki language registrations beyond the defaults (`bash`, `css`, `diff`, `html`, `javascript`, `json`, `markdown`, `typescript`, `yaml`), such as `import rust from "@shikijs/langs/rust"`. Call it once, before the build highlights code |
|
|
114
|
+
|
|
115
|
+
`@allons-y/webc-code-sample/markdown` exports `highlightFences(md)`, a markdown-it amend that renders fenced code as `<norn-code-block>`.
|
|
25
116
|
|
|
26
117
|
## License
|
|
27
118
|
|
|
28
|
-
[MPL-2.0](LICENSE): use freely, modify as needed; changes to MPL-licensed files
|
|
119
|
+
> [MPL-2.0](LICENSE): use freely, modify as needed; changes to MPL-licensed files
|
|
120
|
+
> should be shared back under the same license.
|
|
121
|
+
|
|
122
|
+
---
|
|
29
123
|
|
|
30
124
|
<p align="center">
|
|
31
125
|
<a href="https://allons-y.studio">
|
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
<!---
|
|
2
|
+
/*
|
|
3
|
+
* SPDX-License-Identifier: MPL-2.0
|
|
4
|
+
* This Source Code Form is subject to the terms of the Mozilla Public
|
|
5
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
6
|
+
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
7
|
+
*/
|
|
8
|
+
--->
|
|
9
|
+
<script webc:setup>
|
|
10
|
+
/** The code as it should paste: a leading `$ ` prompt comes off each line, and `$HOME` is untouched. */
|
|
11
|
+
const pasteable = (code) =>
|
|
12
|
+
String(code)
|
|
13
|
+
.split("\n")
|
|
14
|
+
.map((line) => line.replace(/^\$\s+/, ""))
|
|
15
|
+
.join("\n");
|
|
16
|
+
|
|
17
|
+
/** The code's default name: the label template with the language in it. */
|
|
18
|
+
const nameOf = (language, label) =>
|
|
19
|
+
(typeof label === "string" && label ? label : "{language} code")
|
|
20
|
+
.replaceAll("{language}", typeof language === "string" ? language : "")
|
|
21
|
+
.replace(/\s+/g, " ")
|
|
22
|
+
.trim();
|
|
23
|
+
</script>
|
|
24
|
+
<figure :data-chrome="chrome === 'terminal' ? 'terminal' : undefined">
|
|
25
|
+
<figcaption webc:if="caption" @text="caption"></figcaption>
|
|
26
|
+
<norn-copy-button :@value="pasteable(code)" :@strings="'norn-code-block-' + uid" :@compact="true" @surface="code"></norn-copy-button>
|
|
27
|
+
<pre dir="ltr" :data-language="typeof language === 'string' && language ? language : undefined"><code :data-tokens="nornTokenRanges(code, language)" @text="code"></code></pre>
|
|
28
|
+
<span hidden :id="'norn-code-block-' + uid">
|
|
29
|
+
<span data-string="label" :id="'norn-code-block-' + uid + '-copy'"><slot name="copy-label"><span webc:nokeep @text="nornComponentStrings?.('norn-code-block', 'copy-label', $data.page) || 'Copy {name} to clipboard'"></span></slot></span>
|
|
30
|
+
<span data-string="copy"><slot name="copy"><span webc:nokeep @text="nornComponentStrings?.('norn-copy-button', 'copy', $data.page) || 'Copy'"></span></slot></span>
|
|
31
|
+
<span data-string="copied"><slot name="copied"><span webc:nokeep @text="nornComponentStrings?.('norn-copy-button', 'copied', $data.page) || 'Copied'"></span></slot></span>
|
|
32
|
+
<span data-string="failed"><slot name="failed"><span webc:nokeep @text="nornComponentStrings?.('norn-copy-button', 'failed', $data.page) || 'Copy failed'"></span></slot></span>
|
|
33
|
+
</span>
|
|
34
|
+
<span data-name hidden><slot name="label"><span data-default @text="nameOf(language, nornComponentStrings?.('norn-code-block', 'label', $data.page))"></span></slot></span>
|
|
35
|
+
</figure>
|
|
36
|
+
<style>
|
|
37
|
+
@layer norn.components {
|
|
38
|
+
norn-code-block {
|
|
39
|
+
display: block;
|
|
40
|
+
|
|
41
|
+
/* The caption sits above the code. The copy button shares the code's cell, over its top end corner. */
|
|
42
|
+
& figure {
|
|
43
|
+
display: grid;
|
|
44
|
+
grid-template-columns: minmax(0, 1fr);
|
|
45
|
+
margin: 0;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
& figure:has(> figcaption) {
|
|
49
|
+
row-gap: var(--norn-space-xs, 0.5rem);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
& figcaption {
|
|
53
|
+
grid-area: 1 / 1;
|
|
54
|
+
color: var(--norn-text-muted, light-dark(#4b5160, #a4abba));
|
|
55
|
+
font-family: var(--norn-font-mono, ui-monospace, SFMono-Regular, Menlo, Consolas, monospace);
|
|
56
|
+
font-size: var(--norn-font-size-s, 0.875rem);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
& norn-copy-button {
|
|
60
|
+
grid-area: 2 / 1;
|
|
61
|
+
z-index: 1;
|
|
62
|
+
justify-self: end;
|
|
63
|
+
align-self: start;
|
|
64
|
+
margin: var(--norn-space-2xs, 0.25rem);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
& pre {
|
|
68
|
+
box-sizing: border-box;
|
|
69
|
+
grid-area: 2 / 1;
|
|
70
|
+
margin: 0;
|
|
71
|
+
overflow-x: auto;
|
|
72
|
+
padding: var(--norn-space-m, 1.25rem);
|
|
73
|
+
border-radius: var(--norn-ui-border-radius, 6px);
|
|
74
|
+
background: var(--norn-surface-code, light-dark(hsl(40, 14%, 94%), #2c313b));
|
|
75
|
+
color: var(--norn-text-base, light-dark(#16181d, #eceef3));
|
|
76
|
+
font-family: var(--norn-font-mono, ui-monospace, SFMono-Regular, Menlo, Consolas, monospace);
|
|
77
|
+
font-size: var(--norn-font-size-s, 0.875rem);
|
|
78
|
+
line-height: 1.5;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/* With the copy button showing, the code's scrolling area stops short of it. A transparent border is outside the scrolling area and under the background, so no code runs beneath the button. The button sits at the figure's end, and the code is always left to right, so a right-to-left page takes the border on the other side. */
|
|
82
|
+
& figure:has(> norn-copy-button > button:not([hidden])) {
|
|
83
|
+
--_chip: calc(2 * var(--norn-space-xs, 0.5rem) + var(--norn-ui-icon-s, 1em) + var(--norn-space-2xs, 0.25rem) + var(--norn-space-xs, 0.5rem));
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
& figure:has(> norn-copy-button > button:not([hidden])) > pre {
|
|
87
|
+
border-inline-end: var(--_chip) solid transparent;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
& figure:dir(rtl):has(> norn-copy-button > button:not([hidden])) > pre {
|
|
91
|
+
border-inline-end: 0;
|
|
92
|
+
border-inline-start: var(--_chip) solid transparent;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
& code {
|
|
96
|
+
padding: 0;
|
|
97
|
+
background: none;
|
|
98
|
+
font: inherit;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/* A frame with a header bar for the caption, for `@chrome="terminal"`. `:where()` keeps each rule as specific as the plain one it follows, so a container's rules still win. */
|
|
102
|
+
& figure:where([data-chrome="terminal"]),
|
|
103
|
+
& figure:where([data-chrome="terminal"]):has(> figcaption) {
|
|
104
|
+
overflow: hidden;
|
|
105
|
+
row-gap: 0;
|
|
106
|
+
border-radius: var(--norn-ui-border-radius, 6px);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
& figure:where([data-chrome="terminal"]) > figcaption {
|
|
110
|
+
box-sizing: border-box;
|
|
111
|
+
display: flex;
|
|
112
|
+
align-items: center;
|
|
113
|
+
block-size: 2rem;
|
|
114
|
+
padding-inline: var(--norn-space-s, 0.75rem);
|
|
115
|
+
background: var(--norn-surface-raised, light-dark(hsl(40, 14%, 96%), #242932));
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
& figure:where([data-chrome="terminal"]) > pre {
|
|
119
|
+
border-radius: 0;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/* Colors for the ranges the script registers, set on `code` as Firefox ignores them on an ancestor. Plain text without Custom Highlight API support. */
|
|
123
|
+
& code::highlight(tok-comment) {
|
|
124
|
+
color: var(--norn-code-comment, light-dark(#566070, #9aa3b5));
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
& code::highlight(tok-string) {
|
|
128
|
+
color: var(--norn-code-string, light-dark(#0b6b3a, #7ed6a1));
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
& code::highlight(tok-keyword) {
|
|
132
|
+
color: var(--norn-code-keyword, light-dark(#6b2fc2, #c4a3ff));
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
& code::highlight(tok-function) {
|
|
136
|
+
color: var(--norn-code-function, light-dark(#1b4fc4, #8fb0ff));
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
& code::highlight(tok-number) {
|
|
140
|
+
color: var(--norn-code-number, light-dark(#9a4a00, #ffb36b));
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
& code::highlight(tok-flag) {
|
|
144
|
+
color: var(--norn-code-flag, light-dark(#00687a, #5ccfe0));
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
& code::highlight(tok-punctuation) {
|
|
148
|
+
color: var(--norn-text-muted, light-dark(#4b5160, #a4abba));
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
@media print {
|
|
152
|
+
break-inside: avoid;
|
|
153
|
+
|
|
154
|
+
& pre {
|
|
155
|
+
white-space: pre-wrap;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
</style>
|
|
161
|
+
<script>
|
|
162
|
+
{
|
|
163
|
+
/** The ranges of every connected sample, by sample. The Custom Highlight API registry is shared by the page. */
|
|
164
|
+
const rangesBySample = new Map();
|
|
165
|
+
const supported = typeof CSS !== "undefined" && "highlights" in CSS;
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Read `data-tokens`: `[groups, numbers]`, the group names, then a group index, the gap from the previous range's end, and a length per range.
|
|
169
|
+
*
|
|
170
|
+
* @param {string} tokens
|
|
171
|
+
* @returns {[string, number, number][]}
|
|
172
|
+
*/
|
|
173
|
+
const decode = (tokens) => {
|
|
174
|
+
const [groups, numbers] = JSON.parse(tokens);
|
|
175
|
+
const ranges = [];
|
|
176
|
+
let end = 0;
|
|
177
|
+
for (let at = 0; at < numbers.length; at += 3) {
|
|
178
|
+
const group = groups[numbers[at]];
|
|
179
|
+
const start = end + numbers[at + 1];
|
|
180
|
+
end = start + numbers[at + 2];
|
|
181
|
+
if (typeof group !== "string" || !Number.isInteger(start) || !(end > start)) throw new TypeError("Unusable data-tokens");
|
|
182
|
+
ranges.push([group, start, end]);
|
|
183
|
+
}
|
|
184
|
+
return ranges;
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
/** Add a sample's ranges to the page's highlights, one group at a time, so each connect costs only its own ranges. */
|
|
188
|
+
const paint = (sample, found) => {
|
|
189
|
+
rangesBySample.set(sample, found);
|
|
190
|
+
if (!supported) return;
|
|
191
|
+
for (const [group, range] of found) {
|
|
192
|
+
const name = `tok-${group}`;
|
|
193
|
+
if (!CSS.highlights.has(name)) CSS.highlights.set(name, new Highlight());
|
|
194
|
+
CSS.highlights.get(name).add(range);
|
|
195
|
+
}
|
|
196
|
+
};
|
|
197
|
+
|
|
198
|
+
/** Take a sample's ranges back out, and drop a group left empty. */
|
|
199
|
+
const unpaint = (sample) => {
|
|
200
|
+
const found = rangesBySample.get(sample);
|
|
201
|
+
rangesBySample.delete(sample);
|
|
202
|
+
if (!found || !supported) return;
|
|
203
|
+
for (const [group, range] of found) {
|
|
204
|
+
const name = `tok-${group}`;
|
|
205
|
+
const highlight = CSS.highlights.get(name);
|
|
206
|
+
highlight?.delete(range);
|
|
207
|
+
if (highlight?.size === 0) CSS.highlights.delete(name);
|
|
208
|
+
}
|
|
209
|
+
};
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Code sample enhancements. The server-rendered figure, caption, and code are the whole component without a script:
|
|
213
|
+
* this adds the highlight colors, the copy button's name, and a tab stop only while the code scrolls.
|
|
214
|
+
*/
|
|
215
|
+
class CodeBlock extends HTMLElement {
|
|
216
|
+
#observer;
|
|
217
|
+
|
|
218
|
+
connectedCallback() {
|
|
219
|
+
if (document.readyState === "loading") {
|
|
220
|
+
document.addEventListener("DOMContentLoaded", () => this.isConnected && this.#enhance(), { once: true });
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
this.#enhance();
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
disconnectedCallback() {
|
|
227
|
+
unpaint(this);
|
|
228
|
+
this.#observer?.disconnect();
|
|
229
|
+
this.#observer = undefined;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** Safe to run again: a move reconnects the element. */
|
|
233
|
+
#enhance() {
|
|
234
|
+
const pre = this.querySelector("pre");
|
|
235
|
+
if (!pre) return;
|
|
236
|
+
const name = this.#name();
|
|
237
|
+
this.#nameCopyButton(name);
|
|
238
|
+
this.#highlight(pre.querySelector("code[data-tokens]"));
|
|
239
|
+
this.#watchOverflow(pre, name);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** What names the code: its caption, else the `label` text. Either gets an id from the texts' holder, for `aria-labelledby`. */
|
|
243
|
+
#name() {
|
|
244
|
+
const holder = this.querySelector(":scope > figure > span[hidden][id]");
|
|
245
|
+
const named = this.querySelector(":scope > figure > figcaption") ?? this.querySelector(":scope > figure > [data-name]");
|
|
246
|
+
if (named && holder) named.id ||= `${holder.id}-name`;
|
|
247
|
+
return named;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/** Put the code's name in the copy button's, where it has `{name}`. Once filled, it has none left. */
|
|
251
|
+
#nameCopyButton(name) {
|
|
252
|
+
const label = this.querySelector(':scope > figure > [hidden] > [data-string="label"]');
|
|
253
|
+
if (!label || !name) return;
|
|
254
|
+
const walker = document.createTreeWalker(label, NodeFilter.SHOW_TEXT);
|
|
255
|
+
const words = name.textContent.trim();
|
|
256
|
+
while (walker.nextNode()) walker.currentNode.data = walker.currentNode.data.replaceAll("{name}", words);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/** A sample whose token data is unusable stays plain. */
|
|
260
|
+
#highlight(code) {
|
|
261
|
+
unpaint(this);
|
|
262
|
+
try {
|
|
263
|
+
const text = code?.firstChild;
|
|
264
|
+
if (text) {
|
|
265
|
+
paint(
|
|
266
|
+
this,
|
|
267
|
+
decode(code.dataset.tokens).map(([group, start, end]) => {
|
|
268
|
+
const range = new Range();
|
|
269
|
+
range.setStart(text, start);
|
|
270
|
+
range.setEnd(text, end);
|
|
271
|
+
return [group, range];
|
|
272
|
+
}),
|
|
273
|
+
);
|
|
274
|
+
}
|
|
275
|
+
} catch {
|
|
276
|
+
unpaint(this);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
/** Keep the code a named, focusable region only while it scrolls: a tab stop is for something to scroll. */
|
|
282
|
+
#watchOverflow(pre, name) {
|
|
283
|
+
const fit = () => {
|
|
284
|
+
if (pre.scrollWidth > pre.clientWidth) {
|
|
285
|
+
pre.setAttribute("tabindex", "0");
|
|
286
|
+
if (name?.id) {
|
|
287
|
+
pre.setAttribute("role", "region");
|
|
288
|
+
pre.setAttribute("aria-labelledby", name.id);
|
|
289
|
+
}
|
|
290
|
+
} else {
|
|
291
|
+
pre.removeAttribute("tabindex");
|
|
292
|
+
pre.removeAttribute("role");
|
|
293
|
+
pre.removeAttribute("aria-labelledby");
|
|
294
|
+
}
|
|
295
|
+
};
|
|
296
|
+
document.fonts?.ready.then(fit);
|
|
297
|
+
this.#observer?.disconnect();
|
|
298
|
+
if (typeof ResizeObserver === "undefined") return;
|
|
299
|
+
this.#observer = new ResizeObserver(fit);
|
|
300
|
+
this.#observer.observe(pre);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
if (!customElements.get("norn-code-block")) customElements.define("norn-code-block", CodeBlock);
|
|
305
|
+
}
|
|
306
|
+
</script>
|
package/index.js
CHANGED
|
@@ -5,4 +5,96 @@
|
|
|
5
5
|
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
import path from "node:path";
|
|
9
|
+
|
|
10
|
+
import { addLanguages, tokenize } from "./tokenize.js";
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Ranges by language and code, for the life of the process. The code is the key, so a watch-mode rebuild with changed code tokenizes again.
|
|
14
|
+
* Holds at most `CACHE_LIMIT` samples and drops the oldest first.
|
|
15
|
+
*/
|
|
16
|
+
const cache = new Map();
|
|
17
|
+
const CACHE_LIMIT = 1000;
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Add languages beyond the defaults.
|
|
21
|
+
*
|
|
22
|
+
* @param {...import("shiki/core").LanguageRegistration[]} languages - Shiki language registrations, such as the default export of `@shikijs/langs/rust`
|
|
23
|
+
*/
|
|
24
|
+
export function loadLanguages(...languages) {
|
|
25
|
+
addLanguages(languages.flat());
|
|
26
|
+
cache.clear();
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Absolute glob-safe path to `components/norn-code-block.webc`, for the project's WebC `components` list. */
|
|
30
|
+
export const codeBlock = path.join(import.meta.dirname, "components", "norn-code-block.webc").split(path.sep).join("/");
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Tokenize code with Shiki into highlight ranges, in the compact form `data-tokens` carries (see `encodeRanges`).
|
|
34
|
+
*
|
|
35
|
+
* @param {string} code
|
|
36
|
+
* @param {string} [language]
|
|
37
|
+
* @returns {string | undefined} JSON `[groups, numbers]`; undefined when the language isn't loaded or nothing matched
|
|
38
|
+
*/
|
|
39
|
+
export function tokenRanges(code, language = "") {
|
|
40
|
+
const key = `${language}\0${code}`;
|
|
41
|
+
if (cache.has(key)) return cache.get(key);
|
|
42
|
+
const found = tokenize(code, language);
|
|
43
|
+
const ranges = found && encodeRanges(found);
|
|
44
|
+
if (cache.size >= CACHE_LIMIT) cache.delete(cache.keys().next().value);
|
|
45
|
+
cache.set(key, ranges);
|
|
46
|
+
return ranges;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Write ranges in the compact `data-tokens` form: `[groups, numbers]`, the group names once each, in order of first use,
|
|
51
|
+
* then three numbers per range: the group's index in `groups`, the gap from the end of the range before (or from 0), and the length.
|
|
52
|
+
*
|
|
53
|
+
* @param {[string, number, number][]} ranges - `[group, start, end]`, ordered and apart
|
|
54
|
+
* @returns {string}
|
|
55
|
+
*/
|
|
56
|
+
function encodeRanges(ranges) {
|
|
57
|
+
const groups = [];
|
|
58
|
+
const numbers = [];
|
|
59
|
+
let previousEnd = 0;
|
|
60
|
+
for (const [group, start, end] of ranges) {
|
|
61
|
+
if (!groups.includes(group)) groups.push(group);
|
|
62
|
+
numbers.push(groups.indexOf(group), start - previousEnd, end - start);
|
|
63
|
+
previousEnd = end;
|
|
64
|
+
}
|
|
65
|
+
return JSON.stringify([groups, numbers]);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Read a `tokenRanges` string, the `data-tokens` value, back into `[group, start, end][]` character offsets.
|
|
70
|
+
*
|
|
71
|
+
* @param {string | undefined} value - The compact form; undefined, as `tokenRanges` returns for no ranges, reads as none
|
|
72
|
+
* @returns {[string, number, number][]}
|
|
73
|
+
* @throws {TypeError} When the value is not the compact form, naming what is wrong
|
|
74
|
+
*/
|
|
75
|
+
export function decodeRanges(value) {
|
|
76
|
+
if (value === undefined) return [];
|
|
77
|
+
if (typeof value !== "string") throw new TypeError(`decodeRanges takes a string, not ${typeof value}.`);
|
|
78
|
+
let parsed;
|
|
79
|
+
try {
|
|
80
|
+
parsed = JSON.parse(value);
|
|
81
|
+
} catch {
|
|
82
|
+
throw new TypeError("decodeRanges: the value is not JSON.");
|
|
83
|
+
}
|
|
84
|
+
const [groups, numbers] = Array.isArray(parsed) ? parsed : [];
|
|
85
|
+
if (parsed?.length !== 2 || !Array.isArray(groups) || !Array.isArray(numbers)) throw new TypeError("decodeRanges: the value is not a group table and a list of numbers.");
|
|
86
|
+
if (!groups.every((group) => typeof group === "string")) throw new TypeError("decodeRanges: the group table holds group names only.");
|
|
87
|
+
if (numbers.length % 3) throw new TypeError("decodeRanges: the list of numbers is not a multiple of three long.");
|
|
88
|
+
const ranges = [];
|
|
89
|
+
let end = 0;
|
|
90
|
+
for (let at = 0; at < numbers.length; at += 3) {
|
|
91
|
+
const [index, gap, length] = numbers.slice(at, at + 3);
|
|
92
|
+
if (!Number.isInteger(index) || !Object.hasOwn(groups, index)) throw new TypeError(`decodeRanges: range ${at / 3} names group ${index}, which the table lacks.`);
|
|
93
|
+
if (!Number.isInteger(gap) || gap < 0) throw new TypeError(`decodeRanges: range ${at / 3} has gap ${gap}, not a whole number of 0 or more.`);
|
|
94
|
+
if (!Number.isInteger(length) || length < 1) throw new TypeError(`decodeRanges: range ${at / 3} has length ${length}, not a whole number of 1 or more.`);
|
|
95
|
+
const start = end + gap;
|
|
96
|
+
end = start + length;
|
|
97
|
+
ranges.push([groups[index], start, end]);
|
|
98
|
+
}
|
|
99
|
+
return ranges;
|
|
100
|
+
}
|
package/llms.txt
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# @allons-y/webc-code-sample
|
|
2
|
+
|
|
3
|
+
> `norn-code-block`: a WebC code sample highlighted at build time. Shiki runs during the build. The browser gets token ranges for the CSS Custom Highlight API and a copy button, and no highlighter. The reference at the end is generated from the component source.
|
|
4
|
+
|
|
5
|
+
## What you are working with
|
|
6
|
+
|
|
7
|
+
- One component, `norn-code-block`, and one Markdown add-on, `highlightFences`.
|
|
8
|
+
- It is an optional peer of `@allons-y/webc-patterns`. Installed beside it, the Norn plugin registers the component, the `nornTokenRanges` helper, and the add-on, and the Norn layouts show their code with it. Without it, the plugin registers a plain `norn-code-block` that renders `<pre><code>`, so every call site works either way.
|
|
9
|
+
- It registers nothing itself. Do not add it to a project's WebC `components` list when the Norn plugin is in use.
|
|
10
|
+
|
|
11
|
+
## Use it
|
|
12
|
+
|
|
13
|
+
```html
|
|
14
|
+
<norn-code-block @language="js" @caption="example.js" @code="const answer = 42;"></norn-code-block>
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
- Props take the `@` prefix. `@code` is required and holds the code as a string. `@language`, `@caption`, and `@chrome` are optional. The component has no default slot for code.
|
|
18
|
+
- In Markdown, a fenced block becomes a `norn-code-block` with the fence's first word as `@language`. Write fences as normal.
|
|
19
|
+
- Every visitor-facing text is a named slot: `label` (names the code when there is no caption), `copy-label` (names the button; `{name}` stands for the code's name), `copy`, `copied`, `failed`. An empty slot falls back to the site's `componentStrings`, then English.
|
|
20
|
+
|
|
21
|
+
```html
|
|
22
|
+
<norn-code-block @language="sh" @code="npm install">
|
|
23
|
+
<span slot="label">Commande d'installation</span>
|
|
24
|
+
<span slot="copy">Copier</span>
|
|
25
|
+
</norn-code-block>
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Rules
|
|
29
|
+
|
|
30
|
+
- An unknown or empty language renders plain, readable text with no highlighting. Use `loadLanguages` from the package to register more Shiki languages at build time.
|
|
31
|
+
- Theme the colors with the shared tokens (`--norn-code-keyword`, `--norn-code-string`, `--norn-code-flag`, and the rest of the `code` group) in the project's own stylesheet. The component has no custom property of its own.
|
|
32
|
+
- `@chrome="terminal"` frames the code with a flat header bar for the caption. It is a prop, not a custom property.
|
|
33
|
+
- `data-tokens` is an implementation detail: a compact group table plus integer triples. Read it with `decodeRanges` if you must. Do not write it by hand.
|
|
34
|
+
- Highlighting needs the CSS Custom Highlight API. A browser without it shows the code as plain text.
|
|
35
|
+
|
|
36
|
+
## Where to look next
|
|
37
|
+
|
|
38
|
+
- README: install, usage without the Norn plugin, translating, layout requirements.
|
|
39
|
+
- Component preview: https://allonsy-studio.github.io/webc-resources/
|
|
40
|
+
|
|
41
|
+
## Reference
|
|
42
|
+
|
|
43
|
+
Generated from the component sources and their CSF files. Do not edit by hand: run `yarn llms`.
|
|
44
|
+
|
|
45
|
+
### Component components
|
|
46
|
+
|
|
47
|
+
A self-contained element: a link, a code block, a lockup.
|
|
48
|
+
|
|
49
|
+
#### norn-code-block
|
|
50
|
+
|
|
51
|
+
Tag: `<norn-code-block>`.
|
|
52
|
+
|
|
53
|
+
Code sample tokenized at build time. Its script registers `norn-code-block` as a custom element that adds the highlight colors; without the script the markup is the whole sample. A compact `norn-copy-button` from `@allons-y/webc-patterns` sits over the top end corner of the code, as its icon until the pointer or focus is on it, and copies the code without any leading `$ ` prompts. While the button shows, the code scrolls in an area that stops short of it, so no code runs beneath it. A caption, when there is one, sits above the code. The code reads left to right in any page direction. While the code scrolls sideways, the script makes it a tab stop, so a keyboard can scroll it. Shell samples (`sh`, `bash`, `shell`, `shell-session`) color the command, its subcommand, its flags, and its arguments apart. Needs `tokenRanges` registered as the `nornTokenRanges` JavaScript function, and the patterns package's components. The names and the copy button's texts are in the page, hidden: the script names the code while it scrolls, and shows the button. Each falls back to the site's `componentStrings`, under `norn-code-block` for `label` and `copy-label` and under `norn-copy-button` for the rest, then English.
|
|
54
|
+
|
|
55
|
+
Props:
|
|
56
|
+
|
|
57
|
+
- `code` (string, required): The code to show.
|
|
58
|
+
- `language` (string, optional): A language Shiki has loaded: the defaults or one added with `loadLanguages`. Unknown or empty renders plain text.
|
|
59
|
+
- `caption` (string, optional): Also names the code and its copy button.
|
|
60
|
+
- `chrome` (string, optional, one of `terminal`): `terminal` frames the code with a header bar for the caption. Without it the code sits on its surface with the caption above.
|
|
61
|
+
|
|
62
|
+
Slots:
|
|
63
|
+
|
|
64
|
+
- `label`: The code's name without a caption, as one element. Falls back to "{language} code", with the language filled in, or "code" with no language.
|
|
65
|
+
- `copy-label`: The copy button's name, with `{name}` for the code's name, as one element. Falls back to "Copy {name} to clipboard".
|
|
66
|
+
- `copy`: The copy button's text. Falls back to "Copy".
|
|
67
|
+
- `copied`: The copy button's text after a copy. Falls back to "Copied".
|
|
68
|
+
- `failed`: The copy button's text when the copy fails. Falls back to "Copy failed".
|
|
69
|
+
|
|
70
|
+
Global tokens it reads: `--norn-ui-border-radius`, `--norn-ui-icon-s`, `--norn-surface-raised`, `--norn-surface-code`, `--norn-text-base`, `--norn-text-muted`, `--norn-font-mono`, `--norn-font-size-s`, `--norn-code-comment`, `--norn-code-string`, `--norn-code-keyword`, `--norn-code-function`, `--norn-code-number`, `--norn-code-flag`, `--norn-space-2xs`, `--norn-space-xs`, `--norn-space-s`, `--norn-space-m`.
|
package/markdown.js
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* SPDX-License-Identifier: MPL-2.0
|
|
3
|
+
* This Source Code Form is subject to the terms of the Mozilla Public
|
|
4
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
5
|
+
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/*
|
|
9
|
+
* The Markdown add-on. It writes the `<norn-code-block>` tag an author would write by hand
|
|
10
|
+
* and no text of its own: the component's slot fallbacks and `componentStrings` supply every string.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* markdown-it amend: render fenced code as `<norn-code-block>`, with the fence's first word as its `language`.
|
|
15
|
+
* It fills no slots: the component names the code ("js code") and its copy button, and takes the button texts from the site's `componentStrings`, then English.
|
|
16
|
+
*
|
|
17
|
+
* @param {import("markdown-it")} md
|
|
18
|
+
*/
|
|
19
|
+
export function highlightFences(md) {
|
|
20
|
+
md.renderer.rules.fence = (tokens, idx) => {
|
|
21
|
+
const { content, info } = tokens[idx];
|
|
22
|
+
const language = info.trim().split(/\s+/)[0];
|
|
23
|
+
return `<norn-code-block @code="${md.utils.escapeHtml(content)}" @language="${md.utils.escapeHtml(language)}"></norn-code-block>\n`;
|
|
24
|
+
};
|
|
25
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@allons-y/webc-code-sample",
|
|
3
|
-
"version": "0.1.0-beta.
|
|
3
|
+
"version": "0.1.0-beta.3",
|
|
4
4
|
"description": "WebC code sample component with build-time syntax highlighting and no highlighter shipped to the browser.",
|
|
5
5
|
"license": "MPL-2.0",
|
|
6
6
|
"author": "Cassondra Roberts <castastrophe@users.noreply.github.com> (https://allons-y.studio)",
|
|
@@ -20,20 +20,28 @@
|
|
|
20
20
|
"type": "module",
|
|
21
21
|
"exports": {
|
|
22
22
|
".": "./index.js",
|
|
23
|
+
"./markdown": "./markdown.js",
|
|
23
24
|
"./package.json": "./package.json"
|
|
24
25
|
},
|
|
25
26
|
"files": [
|
|
26
27
|
"components/",
|
|
27
28
|
"index.js",
|
|
29
|
+
"markdown.js",
|
|
30
|
+
"tokenize.js",
|
|
28
31
|
"LICENSE",
|
|
29
|
-
"README.md"
|
|
32
|
+
"README.md",
|
|
33
|
+
"llms.txt"
|
|
30
34
|
],
|
|
31
|
-
"
|
|
32
|
-
"
|
|
35
|
+
"scripts": {
|
|
36
|
+
"prepack": "node ../../scripts/build-components.js webc-code-sample"
|
|
37
|
+
},
|
|
38
|
+
"devDependencies": {
|
|
39
|
+
"markdown-it": "^15.0.3"
|
|
33
40
|
},
|
|
34
41
|
"peerDependencies": {
|
|
35
42
|
"@11ty/eleventy": "^3.0.0",
|
|
36
|
-
"@11ty/eleventy-plugin-webc": "^0.11.0"
|
|
43
|
+
"@11ty/eleventy-plugin-webc": "^0.11.0",
|
|
44
|
+
"@allons-y/webc-patterns": "^0.1.0-beta.0"
|
|
37
45
|
},
|
|
38
46
|
"keywords": [
|
|
39
47
|
"webc",
|
|
@@ -42,5 +50,9 @@
|
|
|
42
50
|
],
|
|
43
51
|
"engines": {
|
|
44
52
|
"node": ">=22"
|
|
53
|
+
},
|
|
54
|
+
"dependencies": {
|
|
55
|
+
"@shikijs/langs": "^4.6.0",
|
|
56
|
+
"shiki": "^4.6.0"
|
|
45
57
|
}
|
|
46
58
|
}
|
package/tokenize.js
ADDED
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* SPDX-License-Identifier: MPL-2.0
|
|
3
|
+
* This Source Code Form is subject to the terms of the Mozilla Public
|
|
4
|
+
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
5
|
+
* file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/** Shiki tokenizing behind `tokenRanges`. Internal: the package exports map leaves it out. */
|
|
9
|
+
|
|
10
|
+
import bash from "@shikijs/langs/shellscript";
|
|
11
|
+
import css from "@shikijs/langs/css";
|
|
12
|
+
import diff from "@shikijs/langs/diff";
|
|
13
|
+
import html from "@shikijs/langs/html";
|
|
14
|
+
import javascript from "@shikijs/langs/javascript";
|
|
15
|
+
import json from "@shikijs/langs/json";
|
|
16
|
+
import markdown from "@shikijs/langs/markdown";
|
|
17
|
+
import typescript from "@shikijs/langs/typescript";
|
|
18
|
+
import yaml from "@shikijs/langs/yaml";
|
|
19
|
+
import { createHighlighterCoreSync } from "shiki/core";
|
|
20
|
+
import { createJavaScriptRegexEngine } from "shiki/engine/javascript";
|
|
21
|
+
|
|
22
|
+
/** Shiki needs a theme to tokenize. The colors come from the page's CSS, so this one has none. */
|
|
23
|
+
const THEME = { name: "norn", type: "light", fg: "#000", bg: "#fff", settings: [] };
|
|
24
|
+
|
|
25
|
+
const highlighter = createHighlighterCoreSync({
|
|
26
|
+
themes: [THEME],
|
|
27
|
+
langs: [bash, css, diff, html, javascript, json, markdown, typescript, yaml],
|
|
28
|
+
engine: createJavaScriptRegexEngine(),
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Add Shiki languages to the highlighter.
|
|
33
|
+
*
|
|
34
|
+
* @param {import("shiki/core").LanguageRegistration[]} languages
|
|
35
|
+
*/
|
|
36
|
+
export function addLanguages(languages) {
|
|
37
|
+
highlighter.loadLanguageSync(languages);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Highlight group by TextMate scope prefix, most specific first. The deepest scope of a token that matches a prefix decides. */
|
|
41
|
+
const SCOPES = [
|
|
42
|
+
["comment", "comment"],
|
|
43
|
+
["punctuation.definition.comment", "comment"],
|
|
44
|
+
["punctuation.definition.string", "string"],
|
|
45
|
+
["string.unquoted", undefined],
|
|
46
|
+
["string", "string"],
|
|
47
|
+
["markup.inserted", "string"],
|
|
48
|
+
["constant.character.escape.line-continuation", "punctuation"],
|
|
49
|
+
["constant.numeric", "number"],
|
|
50
|
+
["constant.language", "number"],
|
|
51
|
+
["keyword.operator", "punctuation"],
|
|
52
|
+
["keyword", "keyword"],
|
|
53
|
+
["storage", "keyword"],
|
|
54
|
+
["entity.name.tag", "keyword"],
|
|
55
|
+
["markup.heading", "keyword"],
|
|
56
|
+
["entity.name.function", "function"],
|
|
57
|
+
["entity.name.command", "function"],
|
|
58
|
+
["entity.name.type", "function"],
|
|
59
|
+
["entity.name.class", "function"],
|
|
60
|
+
["entity.other.attribute-name", "function"],
|
|
61
|
+
["entity.other.inherited-class", "function"],
|
|
62
|
+
["support.function", "function"],
|
|
63
|
+
["support.class", "function"],
|
|
64
|
+
["support.type.property-name", "function"],
|
|
65
|
+
["meta.object-literal.key", "function"],
|
|
66
|
+
["punctuation", "punctuation"],
|
|
67
|
+
];
|
|
68
|
+
|
|
69
|
+
/** The highlight group of a stack of scopes, from the outermost to the deepest. A prefix with no group, such as an unquoted word, stops the search: it stays plain. */
|
|
70
|
+
const groupOf = (scopes) => {
|
|
71
|
+
for (const { scopeName } of scopes.toReversed()) {
|
|
72
|
+
const match = SCOPES.find(([prefix]) => scopeName === prefix || scopeName.startsWith(`${prefix}.`));
|
|
73
|
+
if (match) return match[1];
|
|
74
|
+
}
|
|
75
|
+
return undefined;
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
/** Languages that read as shell commands, and the ones whose lines are commands only after a `$ ` prompt. */
|
|
79
|
+
const SHELL = new Set(["sh", "bash", "shell"]);
|
|
80
|
+
const SESSION = new Set(["shell-session", "sh-session", "shellsession"]);
|
|
81
|
+
|
|
82
|
+
/** Words that start a command list again: after one, the next word is a command. */
|
|
83
|
+
const CONTROL = new Set(["if", "then", "else", "elif", "do", "while", "until", "!", "time"]);
|
|
84
|
+
|
|
85
|
+
/** Words made only of operator characters: a pipe or a list separator starts a command, a redirect does not. */
|
|
86
|
+
const OPERATOR = /^[<>|&;()0-9]*[<>|&;()]+$/;
|
|
87
|
+
const ASSIGNMENT = /^[A-Za-z_]\w*=/;
|
|
88
|
+
const SEPARATOR = /^(\|\|?|&&|&|;|\(|\|&)$/;
|
|
89
|
+
|
|
90
|
+
/** Whether a line ends with a lone backslash, so its command goes on in the next line. */
|
|
91
|
+
const continues = (line) => line.trimEnd().split(/\s+/).at(-1) === "\\";
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Tokenize code with Shiki into highlight ranges, ordered and apart: each range starts at or after the end of the one before.
|
|
96
|
+
*
|
|
97
|
+
* @param {string} code
|
|
98
|
+
* @param {string} [language]
|
|
99
|
+
* @returns {[string, number, number][] | undefined} `[group, start, end]` character offsets; undefined when the language isn't loaded or nothing matched
|
|
100
|
+
*/
|
|
101
|
+
export function tokenize(code, language = "") {
|
|
102
|
+
const session = SESSION.has(language);
|
|
103
|
+
const lang = session ? "bash" : language;
|
|
104
|
+
if (!lang || !highlighter.getLoadedLanguages().includes(lang)) return undefined;
|
|
105
|
+
let ranges = [];
|
|
106
|
+
// No time limit: Shiki tokenizes each line twice, and a limit that stops one pass early, as on a busy machine, breaks the explanation.
|
|
107
|
+
const { tokens } = highlighter.codeToTokens(code, { lang, theme: THEME.name, includeExplanation: "scopeName", tokenizeTimeLimit: 0 });
|
|
108
|
+
for (const lineTokens of tokens) {
|
|
109
|
+
for (const token of lineTokens) {
|
|
110
|
+
let at = token.offset;
|
|
111
|
+
for (const { content, scopes } of token.explanation ?? []) {
|
|
112
|
+
const group = groupOf(scopes);
|
|
113
|
+
const end = at + content.length;
|
|
114
|
+
if (group && content.trim()) {
|
|
115
|
+
const previous = ranges.at(-1);
|
|
116
|
+
if (previous?.[0] === group && previous[2] === at) previous[2] = end;
|
|
117
|
+
else ranges.push([group, at, end]);
|
|
118
|
+
}
|
|
119
|
+
at = end;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
if (SHELL.has(language) || session) ranges = shellWords(code, ranges, session);
|
|
124
|
+
return ranges.length ? ranges : undefined;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Give the words of each shell line their roles over Shiki's ranges: the command is a `function`, the first plain word after it a `keyword` (the subcommand), a flag a `flag`, and any other word a `string`.
|
|
129
|
+
* Comments, quoted strings, and variables keep what Shiki gave them. A `$ ` prompt stays punctuation.
|
|
130
|
+
*
|
|
131
|
+
* @param {string} code
|
|
132
|
+
* @param {[string, number, number][]} ranges - Shiki's ranges, in order
|
|
133
|
+
* @param {boolean} session - Only lines that start with a `$ ` prompt are commands
|
|
134
|
+
* @returns {[string, number, number][]}
|
|
135
|
+
*/
|
|
136
|
+
function shellWords(code, ranges, session) {
|
|
137
|
+
const roles = [];
|
|
138
|
+
if (session) {
|
|
139
|
+
// Output lines are plain: keep Shiki's ranges only on the lines of a command, which start with a prompt and go on while a line ends with a backslash.
|
|
140
|
+
let at = 0;
|
|
141
|
+
let running = false;
|
|
142
|
+
const kept = new Set();
|
|
143
|
+
for (const line of code.split("\n")) {
|
|
144
|
+
const command = running || /^\s*\$\s/.test(line);
|
|
145
|
+
if (command) kept.add(at);
|
|
146
|
+
running = command && continues(line);
|
|
147
|
+
at += line.length + 1;
|
|
148
|
+
}
|
|
149
|
+
const lineStart = (offset) => code.lastIndexOf("\n", offset - 1) + 1;
|
|
150
|
+
ranges = ranges.filter(([, start]) => kept.has(lineStart(start)));
|
|
151
|
+
}
|
|
152
|
+
const within = (at, groups) => ranges.some(([group, start, end]) => groups.includes(group) && at >= start && at < end);
|
|
153
|
+
let command = true;
|
|
154
|
+
let sub = true;
|
|
155
|
+
let dashes = false;
|
|
156
|
+
let continued = false;
|
|
157
|
+
let offset = 0;
|
|
158
|
+
for (const line of code.split("\n")) {
|
|
159
|
+
const words = [...line.matchAll(/\S+/g)];
|
|
160
|
+
const prompted = /^\s*\$\s/.test(line);
|
|
161
|
+
const continuing = continued;
|
|
162
|
+
if (!continuing) {
|
|
163
|
+
command = !session || prompted;
|
|
164
|
+
sub = false;
|
|
165
|
+
dashes = false;
|
|
166
|
+
}
|
|
167
|
+
continued = (!session || prompted || continuing) && continues(line);
|
|
168
|
+
for (const [index, match] of words.entries()) {
|
|
169
|
+
const [word] = match;
|
|
170
|
+
const start = offset + match.index;
|
|
171
|
+
const end = start + word.length;
|
|
172
|
+
if (within(start, ["comment", "string"])) {
|
|
173
|
+
if (within(start, ["comment"])) break;
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
if (index === 0 && prompted && word === "$") {
|
|
177
|
+
roles.push(["punctuation", start, end]);
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
if (!command && session && !prompted && !continuing) continue;
|
|
181
|
+
if (SEPARATOR.test(word)) {
|
|
182
|
+
command = true;
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
185
|
+
if (OPERATOR.test(word) || word === "\\") continue;
|
|
186
|
+
if (command && ASSIGNMENT.test(word)) {
|
|
187
|
+
const equals = word.indexOf("=");
|
|
188
|
+
const value = word.slice(equals + 1);
|
|
189
|
+
roles.push(["punctuation", start + equals, start + equals + 1]);
|
|
190
|
+
if (value && !/^["']/.test(value)) roles.push([/^\d+$/.test(value) ? "number" : "string", start + equals + 1, end]);
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
if (word.startsWith("$")) {
|
|
194
|
+
if (!command) roles.push(["string", start, end]);
|
|
195
|
+
continue;
|
|
196
|
+
}
|
|
197
|
+
if (command) {
|
|
198
|
+
if (CONTROL.has(word)) {
|
|
199
|
+
roles.push(["keyword", start, end]);
|
|
200
|
+
} else {
|
|
201
|
+
roles.push(["function", start, end]);
|
|
202
|
+
command = false;
|
|
203
|
+
sub = false;
|
|
204
|
+
dashes = false;
|
|
205
|
+
}
|
|
206
|
+
} else if (word === "--") {
|
|
207
|
+
dashes = true;
|
|
208
|
+
roles.push(["punctuation", start, end]);
|
|
209
|
+
} else if (word.startsWith("-") && word.length > 1) {
|
|
210
|
+
const equals = word.indexOf("=");
|
|
211
|
+
const stop = equals === -1 ? end : start + equals;
|
|
212
|
+
roles.push(["flag", start, stop]);
|
|
213
|
+
if (equals !== -1) roles.push(["punctuation", stop, stop + 1]);
|
|
214
|
+
if (equals !== -1 && equals + 1 < word.length) roles.push(["string", start + equals + 1, end]);
|
|
215
|
+
} else if (!sub && !dashes) {
|
|
216
|
+
roles.push(["keyword", start, end]);
|
|
217
|
+
sub = true;
|
|
218
|
+
} else {
|
|
219
|
+
roles.push(["string", start, end]);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
offset += line.length + 1;
|
|
223
|
+
}
|
|
224
|
+
// The roles replace whatever Shiki gave the same characters.
|
|
225
|
+
const cut = ranges.flatMap(([group, start, end]) => {
|
|
226
|
+
let pieces = [[start, end]];
|
|
227
|
+
for (const [, from, to] of roles) pieces = pieces.flatMap(([a, b]) => (to <= a || from >= b ? [[a, b]] : [[a, Math.min(b, from)], [Math.max(a, to), b]].filter(([x, y]) => y > x)));
|
|
228
|
+
return pieces.map(([a, b]) => [group, a, b]);
|
|
229
|
+
});
|
|
230
|
+
return [...cut, ...roles].sort((a, b) => a[1] - b[1]);
|
|
231
|
+
}
|