foldkit-richtext-code-shiki 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Patrick Glenn
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,82 @@
1
+ # foldkit-richtext-code-shiki
2
+
3
+ [Shiki](https://shiki.style)'s grammars as tokenizers for a
4
+ [`foldkit-richtext`](../richtext) `CodeBlock`.
5
+
6
+ A code block's document holds its `language` and plain text; what a highlighter finds there is
7
+ a set of decorations, derived for a render and discarded (`RichText.codeDecorations`). This
8
+ package turns a Shiki highlighter the application already made into those tokenizers — one per
9
+ language it loaded — so TypeScript, CSS, Python, or anything else Shiki has a grammar for
10
+ highlights the way `foldkit-richtext-code`'s JSON grammar does.
11
+
12
+ ## What it owns
13
+
14
+ The mapping from Shiki to the tokenizer contract, and nothing else. The application owns the
15
+ highlighter: which grammars and theme it loads, and when. The document stays
16
+ `foldkit-richtext`'s, and drawing stays the renderer's (`span[data-decoration=<kind>]`).
17
+
18
+ Shiki's colours are not used. Its TextMate scopes are read into the same `syntax-*` kinds the
19
+ JSON grammar names, so one stylesheet colours every language:
20
+
21
+ ```text
22
+ syntax-comment syntax-property syntax-string syntax-number syntax-keyword
23
+ syntax-operator syntax-function syntax-type syntax-punctuation
24
+ ```
25
+
26
+ A scope matches by whole segments (`string` and `string.quoted`, never `stringy`), and the
27
+ first kind that matches wins: a JSON key sits inside `string` and is a property, and a string's
28
+ quotes are part of the string. Whitespace and plain identifiers are not decorated.
29
+
30
+ ## Install
31
+
32
+ ```bash
33
+ pnpm add foldkit-richtext-code-shiki shiki
34
+ ```
35
+
36
+ ## Highlight TypeScript
37
+
38
+ ```ts
39
+ import * as RichText from 'foldkit-richtext'
40
+ import { shikiTokenizers } from 'foldkit-richtext-code-shiki'
41
+ import { createHighlighterCoreSync } from 'shiki/core'
42
+ import { createJavaScriptRegexEngine } from 'shiki/engine/javascript'
43
+ import typescript from 'shiki/langs/typescript.mjs'
44
+ import nord from 'shiki/themes/nord.mjs'
45
+
46
+ const highlighter = createHighlighterCoreSync({
47
+ themes: [nord],
48
+ langs: [typescript],
49
+ engine: createJavaScriptRegexEngine(),
50
+ })
51
+
52
+ const tokenizers = shikiTokenizers(highlighter)
53
+
54
+ const decorate = (document: RichText.Document) => RichText.codeDecorations(document, tokenizers)
55
+ ```
56
+
57
+ `shikiTokenizers` gives a `Map` from every name the highlighter loaded (`typescript`, `ts`,
58
+ `cts`, `mts`) to its tokenizer, which is what `codeDecorations` takes. `decorate` is what an
59
+ editor placement names, `editorAt(hostId, { decorate })` in `foldkit-richtext-dom`, and what a
60
+ read-only render passes to `renderDocument`.
61
+
62
+ The highlighter must be synchronous, which is why the example builds it with
63
+ `createHighlighterCoreSync` and Shiki's JavaScript regex engine: a tokenizer is called on
64
+ every render. Loading grammars is startup work that belongs to the application, before it
65
+ places an editor. `shikiTokenizer(highlighter, language)` makes one tokenizer, and it throws
66
+ when it is made if the highlighter has not loaded that language or any theme. Shiki needs a
67
+ theme to tokenize at all, even though its colours are discarded.
68
+
69
+ ## Cost
70
+
71
+ Each tokenizer remembers the last 64 texts it read, so an editor that retokenizes every code
72
+ block on every patch pays for a block only when its text changes. Reading scopes
73
+ (`includeExplanation`) is Shiki's slow path, so a very large block still costs a full parse on
74
+ each edit to it. Shiki's per-line time limit is turned off: with explanations on, a line cut
75
+ short by it made Shiki throw, so a slow line now costs time rather than a failed render.
76
+
77
+ ## Limits
78
+
79
+ - **No asynchronous loading.** A grammar loaded after an editor is placed is not picked up;
80
+ build the tokenizers again and place them again.
81
+ - **The kinds are fixed.** A scope outside the table above is not decorated, and there is no
82
+ hook to map it; say what it should be and the table grows.
@@ -0,0 +1,14 @@
1
+ import { CodeTokenizer } from "foldkit-richtext";
2
+ import { HighlighterCore } from "shiki/core";
3
+ //#region src/shiki.d.ts
4
+ /**
5
+ * The tokenizer for one language the highlighter has loaded. It throws here, when it is made,
6
+ * if the language or any theme is missing, rather than on the first render. Adjacent pieces of
7
+ * the same kind become one token. Results are remembered by text, so an unchanged block is not
8
+ * tokenized again.
9
+ */
10
+ export declare const shikiTokenizer: (highlighter: HighlighterCore, language: string) => CodeTokenizer;
11
+ /** A tokenizer for every language the highlighter has loaded, under each of its names. */
12
+ export declare const shikiTokenizers: (highlighter: HighlighterCore) => ReadonlyMap<string, CodeTokenizer>;
13
+ //#endregion
14
+ //# sourceMappingURL=index.d.mts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.mts","names":[],"sources":["../src/shiki.ts"],"mappings":";;;;;;;;;qBAyDa,iBAAkB,aAAa,iBAAiB,qBAAmB;;qBAiDnE,kBAAmB,aAAa,oBAAkB,oBAAoB"}
package/dist/index.mjs ADDED
@@ -0,0 +1,94 @@
1
+ //#region src/shiki.ts
2
+ /**
3
+ * Which kind a scope stack reads as, first match wins. The order is the point: a JSON key sits
4
+ * inside `string` and carries `support.type.property-name`, so property must be tried before
5
+ * string and before `support.type` makes it a type; a string's quote
6
+ * (`punctuation.definition.string`, inside `string`) belongs to the string, not punctuation;
7
+ * and `keyword.operator` is an operator before every other `keyword.*` is a keyword.
8
+ */
9
+ const KINDS = [
10
+ ["syntax-comment", ["comment"]],
11
+ ["syntax-property", ["support.type.property-name", "meta.object-literal.key"]],
12
+ ["syntax-string", ["string"]],
13
+ ["syntax-number", ["constant.numeric"]],
14
+ ["syntax-operator", ["keyword.operator"]],
15
+ ["syntax-keyword", [
16
+ "keyword",
17
+ "storage",
18
+ "constant.language"
19
+ ]],
20
+ ["syntax-function", ["entity.name.function", "support.function"]],
21
+ ["syntax-type", [
22
+ "entity.name.type",
23
+ "entity.name.class",
24
+ "support.type",
25
+ "support.class"
26
+ ]],
27
+ ["syntax-punctuation", ["punctuation"]]
28
+ ];
29
+ /** The kind a scope stack reads as, or none: plain text and whitespace are not decorated. */
30
+ const kindForScopes = (scopes) => {
31
+ for (const [kind, prefixes] of KINDS) if (scopes.some((scope) => prefixes.some((prefix) => `${scope}.`.startsWith(`${prefix}.`)))) return kind;
32
+ };
33
+ /**
34
+ * How many texts one tokenizer remembers, least recently read forgotten first. An editor
35
+ * retokenizes every block on every patch, and typing in one block makes a new text each
36
+ * keystroke, so a block that did not change stays remembered while it keeps being read.
37
+ */
38
+ const CACHED_TEXTS = 64;
39
+ /**
40
+ * The tokenizer for one language the highlighter has loaded. It throws here, when it is made,
41
+ * if the language or any theme is missing, rather than on the first render. Adjacent pieces of
42
+ * the same kind become one token. Results are remembered by text, so an unchanged block is not
43
+ * tokenized again.
44
+ */
45
+ const shikiTokenizer = (highlighter, language) => {
46
+ if (!highlighter.getLoadedLanguages().includes(language)) throw new Error(`shikiTokenizer: the highlighter has not loaded "${language}"`);
47
+ const [theme] = highlighter.getLoadedThemes();
48
+ if (theme === void 0) throw new Error("shikiTokenizer: the highlighter has no theme loaded");
49
+ const cache = /* @__PURE__ */ new Map();
50
+ return (text) => {
51
+ const cached = cache.get(text);
52
+ if (cached !== void 0) {
53
+ cache.delete(text);
54
+ cache.set(text, cached);
55
+ return cached;
56
+ }
57
+ const tokens = [];
58
+ const { tokens: lines } = highlighter.codeToTokens(text, {
59
+ lang: language,
60
+ theme,
61
+ includeExplanation: true,
62
+ tokenizeTimeLimit: 0
63
+ });
64
+ for (const line of lines) for (const token of line) {
65
+ let at = token.offset;
66
+ for (const part of token.explanation ?? []) {
67
+ const kind = kindForScopes(part.scopes.map((scope) => scope.scopeName));
68
+ const to = at + part.content.length;
69
+ const previous = tokens.at(-1);
70
+ if (kind !== void 0) {
71
+ if (previous?.kind === kind && previous.to === at) tokens[tokens.length - 1] = {
72
+ ...previous,
73
+ to
74
+ };
75
+ else tokens.push({
76
+ from: at,
77
+ to,
78
+ kind
79
+ });
80
+ }
81
+ at = to;
82
+ }
83
+ }
84
+ if (cache.size >= CACHED_TEXTS) cache.delete(cache.keys().next().value);
85
+ cache.set(text, tokens);
86
+ return tokens;
87
+ };
88
+ };
89
+ /** A tokenizer for every language the highlighter has loaded, under each of its names. */
90
+ const shikiTokenizers = (highlighter) => new Map(highlighter.getLoadedLanguages().map((language) => [language, shikiTokenizer(highlighter, language)]));
91
+ //#endregion
92
+ export { shikiTokenizer, shikiTokenizers };
93
+
94
+ //# sourceMappingURL=index.mjs.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.mjs","names":[],"sources":["../src/shiki.ts"],"sourcesContent":["/**\n * Shiki's grammars as code tokenizers (§130): the application creates a highlighter with the\n * languages it wants, and each becomes a `CodeTokenizer` whose tokens are decorations. Shiki's\n * colours are not used; its TextMate scopes are read into the `syntax-*` kinds the JSON grammar\n * already names, so one stylesheet colours every language, and a theme stays the stylesheet's.\n *\n * The highlighter must be synchronous — `createHighlighterCoreSync` with the JavaScript regex\n * engine — because a tokenizer is a pure call made on every render. Loading grammars is the\n * application's startup work, not the editor's.\n */\nimport type { CodeToken, CodeTokenizer } from 'foldkit-richtext'\nimport type { HighlighterCore } from 'shiki/core'\n\n/**\n * Which kind a scope stack reads as, first match wins. The order is the point: a JSON key sits\n * inside `string` and carries `support.type.property-name`, so property must be tried before\n * string and before `support.type` makes it a type; a string's quote\n * (`punctuation.definition.string`, inside `string`) belongs to the string, not punctuation;\n * and `keyword.operator` is an operator before every other `keyword.*` is a keyword.\n */\nconst KINDS: ReadonlyArray<readonly [kind: string, prefixes: ReadonlyArray<string>]> = [\n ['syntax-comment', ['comment']],\n ['syntax-property', ['support.type.property-name', 'meta.object-literal.key']],\n ['syntax-string', ['string']],\n ['syntax-number', ['constant.numeric']],\n ['syntax-operator', ['keyword.operator']],\n ['syntax-keyword', ['keyword', 'storage', 'constant.language']],\n ['syntax-function', ['entity.name.function', 'support.function']],\n ['syntax-type', ['entity.name.type', 'entity.name.class', 'support.type', 'support.class']],\n ['syntax-punctuation', ['punctuation']],\n]\n\n/** The kind a scope stack reads as, or none: plain text and whitespace are not decorated. */\nconst kindForScopes = (scopes: ReadonlyArray<string>): string | undefined => {\n for (const [kind, prefixes] of KINDS) {\n // A prefix matches whole scope segments: `string` and `string.quoted`, never `stringy`.\n const matches = scopes.some(scope =>\n prefixes.some(prefix => `${scope}.`.startsWith(`${prefix}.`)),\n )\n if (matches) return kind\n }\n return undefined\n}\n\n/**\n * How many texts one tokenizer remembers, least recently read forgotten first. An editor\n * retokenizes every block on every patch, and typing in one block makes a new text each\n * keystroke, so a block that did not change stays remembered while it keeps being read.\n */\nconst CACHED_TEXTS = 64\n\n/**\n * The tokenizer for one language the highlighter has loaded. It throws here, when it is made,\n * if the language or any theme is missing, rather than on the first render. Adjacent pieces of\n * the same kind become one token. Results are remembered by text, so an unchanged block is not\n * tokenized again.\n */\nexport const shikiTokenizer = (highlighter: HighlighterCore, language: string): CodeTokenizer => {\n if (!highlighter.getLoadedLanguages().includes(language)) {\n throw new Error(`shikiTokenizer: the highlighter has not loaded \"${language}\"`)\n }\n // Tokenizing needs a theme even though its colours are discarded.\n const [theme] = highlighter.getLoadedThemes()\n if (theme === undefined) throw new Error('shikiTokenizer: the highlighter has no theme loaded')\n const cache = new Map<string, ReadonlyArray<CodeToken>>()\n return text => {\n const cached = cache.get(text)\n if (cached !== undefined) {\n // A Map keeps insertion order, so re-inserting marks this text as the latest read.\n cache.delete(text)\n cache.set(text, cached)\n return cached\n }\n const tokens: Array<CodeToken> = []\n const { tokens: lines } = highlighter.codeToTokens(text, {\n lang: language,\n theme,\n includeExplanation: true,\n // Explanations tokenize each line twice, each pass under this limit; when one pass is\n // cut short and the other is not, Shiki reads past the shorter one and throws. With no\n // limit both passes finish, so a slow line costs time instead of a failed render.\n tokenizeTimeLimit: 0,\n })\n for (const line of lines) {\n for (const token of line) {\n let at = token.offset\n for (const part of token.explanation ?? []) {\n const kind = kindForScopes(part.scopes.map(scope => scope.scopeName))\n const to = at + part.content.length\n const previous = tokens.at(-1)\n if (kind !== undefined) {\n if (previous?.kind === kind && previous.to === at)\n tokens[tokens.length - 1] = { ...previous, to }\n else tokens.push({ from: at, to, kind })\n }\n at = to\n }\n }\n }\n if (cache.size >= CACHED_TEXTS) cache.delete(cache.keys().next().value!)\n cache.set(text, tokens)\n return tokens\n }\n}\n\n/** A tokenizer for every language the highlighter has loaded, under each of its names. */\nexport const shikiTokenizers = (highlighter: HighlighterCore): ReadonlyMap<string, CodeTokenizer> =>\n new Map(\n highlighter\n .getLoadedLanguages()\n .map(language => [language, shikiTokenizer(highlighter, language)]),\n )\n"],"mappings":";;;;;;;;AAoBA,MAAM,QAAiF;CACrF,CAAC,kBAAkB,CAAC,SAAS,CAAC;CAC9B,CAAC,mBAAmB,CAAC,8BAA8B,yBAAyB,CAAC;CAC7E,CAAC,iBAAiB,CAAC,QAAQ,CAAC;CAC5B,CAAC,iBAAiB,CAAC,kBAAkB,CAAC;CACtC,CAAC,mBAAmB,CAAC,kBAAkB,CAAC;CACxC,CAAC,kBAAkB;EAAC;EAAW;EAAW;CAAmB,CAAC;CAC9D,CAAC,mBAAmB,CAAC,wBAAwB,kBAAkB,CAAC;CAChE,CAAC,eAAe;EAAC;EAAoB;EAAqB;EAAgB;CAAe,CAAC;CAC1F,CAAC,sBAAsB,CAAC,aAAa,CAAC;AACxC;;AAGA,MAAM,iBAAiB,WAAsD;CAC3E,KAAK,MAAM,CAAC,MAAM,aAAa,OAK7B,IAHgB,OAAO,MAAK,UAC1B,SAAS,MAAK,WAAU,GAAG,MAAM,GAAG,WAAW,GAAG,OAAO,EAAE,CAAC,CAEpD,GAAG,OAAO;AAGxB;;;;;;AAOA,MAAM,eAAe;;;;;;;AAQrB,MAAa,kBAAkB,aAA8B,aAAoC;CAC/F,IAAI,CAAC,YAAY,mBAAmB,CAAC,CAAC,SAAS,QAAQ,GACrD,MAAM,IAAI,MAAM,mDAAmD,SAAS,EAAE;CAGhF,MAAM,CAAC,SAAS,YAAY,gBAAgB;CAC5C,IAAI,UAAU,KAAA,GAAW,MAAM,IAAI,MAAM,qDAAqD;CAC9F,MAAM,wBAAQ,IAAI,IAAsC;CACxD,QAAO,SAAQ;EACb,MAAM,SAAS,MAAM,IAAI,IAAI;EAC7B,IAAI,WAAW,KAAA,GAAW;GAExB,MAAM,OAAO,IAAI;GACjB,MAAM,IAAI,MAAM,MAAM;GACtB,OAAO;EACT;EACA,MAAM,SAA2B,CAAC;EAClC,MAAM,EAAE,QAAQ,UAAU,YAAY,aAAa,MAAM;GACvD,MAAM;GACN;GACA,oBAAoB;GAIpB,mBAAmB;EACrB,CAAC;EACD,KAAK,MAAM,QAAQ,OACjB,KAAK,MAAM,SAAS,MAAM;GACxB,IAAI,KAAK,MAAM;GACf,KAAK,MAAM,QAAQ,MAAM,eAAe,CAAC,GAAG;IAC1C,MAAM,OAAO,cAAc,KAAK,OAAO,KAAI,UAAS,MAAM,SAAS,CAAC;IACpE,MAAM,KAAK,KAAK,KAAK,QAAQ;IAC7B,MAAM,WAAW,OAAO,GAAG,EAAE;IAC7B,IAAI,SAAS,KAAA,GAAW;KACtB,IAAI,UAAU,SAAS,QAAQ,SAAS,OAAO,IAC7C,OAAO,OAAO,SAAS,KAAK;MAAE,GAAG;MAAU;KAAG;UAC3C,OAAO,KAAK;MAAE,MAAM;MAAI;MAAI;KAAK,CAAC;IACzC;IACA,KAAK;GACP;EACF;EAEF,IAAI,MAAM,QAAQ,cAAc,MAAM,OAAO,MAAM,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,KAAM;EACvE,MAAM,IAAI,MAAM,MAAM;EACtB,OAAO;CACT;AACF;;AAGA,MAAa,mBAAmB,gBAC9B,IAAI,IACF,YACG,mBAAmB,CAAC,CACpB,KAAI,aAAY,CAAC,UAAU,eAAe,aAAa,QAAQ,CAAC,CAAC,CACtE"}
package/package.json ADDED
@@ -0,0 +1,58 @@
1
+ {
2
+ "name": "foldkit-richtext-code-shiki",
3
+ "version": "0.1.0",
4
+ "description": "Shiki grammars as foldkit-richtext code tokenizers, producing syntax decorations.",
5
+ "keywords": [
6
+ "foldkit",
7
+ "effect",
8
+ "rich-text",
9
+ "syntax-highlighting",
10
+ "shiki",
11
+ "typescript"
12
+ ],
13
+ "type": "module",
14
+ "license": "MIT",
15
+ "publishConfig": {
16
+ "access": "public"
17
+ },
18
+ "author": "Patrick Glenn <doeixd@gmail.com>",
19
+ "repository": {
20
+ "type": "git",
21
+ "url": "git+https://github.com/doeixd/foldkit-plus.git",
22
+ "directory": "packages/richtext-code-shiki"
23
+ },
24
+ "homepage": "https://github.com/doeixd/foldkit-plus/tree/main/packages/richtext-code-shiki#readme",
25
+ "bugs": {
26
+ "url": "https://github.com/doeixd/foldkit-plus/issues"
27
+ },
28
+ "sideEffects": false,
29
+ "exports": {
30
+ ".": {
31
+ "foldkit-plus:source": "./src/index.ts",
32
+ "types": "./dist/index.d.mts",
33
+ "import": "./dist/index.mjs"
34
+ }
35
+ },
36
+ "files": [
37
+ "dist",
38
+ "README.md",
39
+ "LICENSE"
40
+ ],
41
+ "dependencies": {
42
+ "foldkit-richtext": "0.2.0"
43
+ },
44
+ "peerDependencies": {
45
+ "effect": ">=4.0.0-rc.116 <4.0.0-rc.118",
46
+ "shiki": "^4.4.3"
47
+ },
48
+ "devDependencies": {
49
+ "effect": "4.0.0-rc.116",
50
+ "shiki": "4.4.3",
51
+ "tsdown": "^0.23.0",
52
+ "typescript": "5.9.3"
53
+ },
54
+ "scripts": {
55
+ "build": "tsdown",
56
+ "typecheck": "tsc -b"
57
+ }
58
+ }