@dogsbay/highlight 0.0.0-stage → 0.2.0-beta.117

package/README.md

@@ -1,3 +1,48 @@
-# Temporary Holding Version
+# @dogsbay/highlight
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Syntax highlighting for `@dogsbay/render-html`. It implements the package's
+`Highlighter` interface with Shiki's core build and JavaScript regex engine.
+
+- **No WebAssembly**, so it bundles into a Cloudflare Worker.
+- **Grammars load per language**, one `import()` each, so a split bundle reads only the grammars a site uses.
+- **Both themes from one tokenization** (`one-pass.ts`).
+- **A bounded cache** for identical blocks.
+
+```ts
+import { createHighlighter } from "@dogsbay/highlight";
+import { highlightInputs } from "@dogsbay/render-html";
+const highlighter = await createHighlighter({ cacheChars: 8_000_000 });
+// before each page renders (rendering is synchronous):
+await highlighter.load(highlightInputs(tree));
+// pass it as RenderContext.highlighter
+```
+
+38 grammars, about 2.1 MB minified in all, each loaded when `load` (or the
+`langs` option; `langs: "all"` loads every one) first names one of its
+spellings: the grammar's name, its aliases (`sh`, `zsh`), or ours (`py`, `yml`,
+`dita`). `src/grammars.ts` maps every spelling to its loader; it is generated by
+`node scripts/grammars.mjs` and a test fails when it is stale. A block whose
+grammar was not loaded renders as plain text and is counted in
+`cacheStats().unloaded`, so a caller can warn. Unknown languages render as plain
+text, the same way `@dogsbay/render-html` renders all code when no highlighter
+is passed.
+
+`load` takes blocks (tag and code), not only tags, because markdown embeds other
+grammars lazily: its fences and front matter colour from whatever grammars are
+registered when the block is tokenized, and its module imports none of them.
+`src/grammars.ts` also lists, for each such grammar (`LAZY_EMBEDS`: markdown and
+haml), the trigger patterns of each embed, taken from the grammar with their
+anchors removed so they match anywhere the rule could start, and which grammars
+reach one of them (`LAZY_VIA`: ruby reaches markdown through haml, whose
+`:markdown` filter a HAML heredoc can hold). `load` scans a block's lines with
+them (skipping lines Shiki does not tokenize, 1,000 characters or more) and
+loads what they reach; a block renders
+only once all of that is loaded (else plain, and counted), so its colours never
+depend on what other pages happened to load. A tag passed without code loads
+everything that grammar can embed.
+
+`tests/eager-golden.json` is the output of the highlighter as it was, with every
+grammar imported statically (`scripts/eager-golden.mjs`); `tests/lazy.test.ts`
+holds every spelling and every grammar, each loaded alone, to it, and markdown
+blocks with fences, front matter and raw HTML, loaded for their own content, to
+the output with every grammar loaded.

package/dist/grammars.d.ts

@@ -0,0 +1,24 @@
+import type { LanguageRegistration } from "shiki/core";
+type Loader = () => Promise<{
+    default: LanguageRegistration[];
+}>;
+/** One literal import() per grammar, so a bundler splits each into its own chunk. */
+export declare const LOADERS: Record<string, Loader>;
+/** Every spelling the grammars declare → the grammar in LOADERS that brings it. */
+export declare const SPELLINGS: Record<string, string>;
+/**
+ * Grammars that embed others lazily (Shiki's embeddedLangsLazy: they resolve
+ * an embedded scope from whatever is registered when a block is tokenized, and
+ * import nothing), and for each embed among ours, the patterns of which one
+ * must match a line of a block for the block to reach it ("" for always).
+ * The patterns are the grammar's own, with the anchors ^, \A and \G removed,
+ * so they match anywhere a nested rule could start them, and never less.
+ */
+export declare const LAZY_EMBEDS: Record<string, Record<string, string[]>>;
+/**
+ * Every grammar that can reach a grammar in LAZY_EMBEDS (itself, or through
+ * the grammars it includes: ruby includes haml, which embeds markdown lazily),
+ * and those it reaches. A block in it needs whatever their triggers find.
+ */
+export declare const LAZY_VIA: Record<string, string[]>;
+export {};

package/dist/grammars.js

@@ -0,0 +1,173 @@
+/** One literal import() per grammar, so a bundler splits each into its own chunk. */
+export const LOADERS = {
+    "bash": () => import("@shikijs/langs/bash"),
+    "shellsession": () => import("@shikijs/langs/shellsession"),
+    "powershell": () => import("@shikijs/langs/powershell"),
+    "bat": () => import("@shikijs/langs/bat"),
+    "javascript": () => import("@shikijs/langs/javascript"),
+    "typescript": () => import("@shikijs/langs/typescript"),
+    "jsx": () => import("@shikijs/langs/jsx"),
+    "tsx": () => import("@shikijs/langs/tsx"),
+    "json": () => import("@shikijs/langs/json"),
+    "jsonc": () => import("@shikijs/langs/jsonc"),
+    "yaml": () => import("@shikijs/langs/yaml"),
+    "toml": () => import("@shikijs/langs/toml"),
+    "ini": () => import("@shikijs/langs/ini"),
+    "properties": () => import("@shikijs/langs/properties"),
+    "xml": () => import("@shikijs/langs/xml"),
+    "xsl": () => import("@shikijs/langs/xsl"),
+    "html": () => import("@shikijs/langs/html"),
+    "css": () => import("@shikijs/langs/css"),
+    "scss": () => import("@shikijs/langs/scss"),
+    "markdown": () => import("@shikijs/langs/markdown"),
+    "java": () => import("@shikijs/langs/java"),
+    "python": () => import("@shikijs/langs/python"),
+    "go": () => import("@shikijs/langs/go"),
+    "rust": () => import("@shikijs/langs/rust"),
+    "c": () => import("@shikijs/langs/c"),
+    "cpp": () => import("@shikijs/langs/cpp"),
+    "csharp": () => import("@shikijs/langs/csharp"),
+    "sql": () => import("@shikijs/langs/sql"),
+    "diff": () => import("@shikijs/langs/diff"),
+    "docker": () => import("@shikijs/langs/docker"),
+    "makefile": () => import("@shikijs/langs/makefile"),
+    "groovy": () => import("@shikijs/langs/groovy"),
+    "kotlin": () => import("@shikijs/langs/kotlin"),
+    "ruby": () => import("@shikijs/langs/ruby"),
+    "graphql": () => import("@shikijs/langs/graphql"),
+    "nginx": () => import("@shikijs/langs/nginx"),
+    "hcl": () => import("@shikijs/langs/hcl"),
+    "http": () => import("@shikijs/langs/http"),
+};
+/** Every spelling the grammars declare → the grammar in LOADERS that brings it. */
+export const SPELLINGS = {
+    "bash": "bash",
+    "bat": "bat",
+    "batch": "bat",
+    "c": "c",
+    "c#": "csharp",
+    "c++": "cpp",
+    "cjs": "javascript",
+    "console": "shellsession",
+    "cpp": "cpp",
+    "cpp-macro": "cpp",
+    "cs": "csharp",
+    "csharp": "csharp",
+    "css": "css",
+    "cts": "typescript",
+    "diff": "diff",
+    "docker": "docker",
+    "dockerfile": "docker",
+    "glsl": "cpp",
+    "go": "go",
+    "gql": "graphql",
+    "graphql": "graphql",
+    "groovy": "groovy",
+    "haml": "ruby",
+    "hcl": "hcl",
+    "html": "html",
+    "http": "http",
+    "ini": "ini",
+    "java": "java",
+    "javascript": "javascript",
+    "js": "javascript",
+    "json": "json",
+    "jsonc": "jsonc",
+    "jsx": "jsx",
+    "kotlin": "kotlin",
+    "kt": "kotlin",
+    "kts": "kotlin",
+    "lua": "ruby",
+    "make": "makefile",
+    "makefile": "makefile",
+    "markdown": "markdown",
+    "md": "markdown",
+    "mjs": "javascript",
+    "mts": "typescript",
+    "nginx": "nginx",
+    "powershell": "powershell",
+    "properties": "ini",
+    "ps": "powershell",
+    "ps1": "powershell",
+    "py": "python",
+    "python": "python",
+    "rb": "ruby",
+    "regex": "cpp",
+    "regexp": "cpp",
+    "rs": "rust",
+    "ruby": "ruby",
+    "rust": "rust",
+    "scss": "scss",
+    "sh": "bash",
+    "shell": "bash",
+    "shellscript": "bash",
+    "shellsession": "shellsession",
+    "sql": "sql",
+    "toml": "toml",
+    "ts": "typescript",
+    "tsx": "tsx",
+    "typescript": "typescript",
+    "xml": "xml",
+    "xsl": "xsl",
+    "yaml": "yaml",
+    "yml": "yaml",
+    "zsh": "bash",
+};
+/**
+ * Grammars that embed others lazily (Shiki's embeddedLangsLazy: they resolve
+ * an embedded scope from whatever is registered when a block is tokenized, and
+ * import nothing), and for each embed among ours, the patterns of which one
+ * must match a line of a block for the block to reach it ("" for always).
+ * The patterns are the grammar's own, with the anchors ^, \A and \G removed,
+ * so they match anywhere a nested rule could start them, and never less.
+ */
+export const LAZY_EMBEDS = {
+    "markdown": {
+        "bat": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(bat(?:|ch))((\\s+|[,:?{])[^`]*)?$)"],
+        "c": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:([ch])((\\s+|[,:?{])[^`]*)?$)"],
+        "cpp": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(c(?:pp|\\+\\+|xx))((\\s+|[,:?{])[^`]*)?$)"],
+        "csharp": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(c(?:s|sharp|#))((\\s+|[,:?{])[^`]*)?$)"],
+        "css": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(css(?:|.erb))((\\s+|[,:?{])[^`]*)?$)"],
+        "diff": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(patch|diff|rej)((\\s+|[,:?{])[^`]*)?$)"],
+        "docker": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:([Dd]ockerfile)((\\s+|[,:?{])[^`]*)?$)"],
+        "go": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(go(?:|lang))((\\s+|[,:?{])[^`]*)?$)"],
+        "groovy": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(g(?:roovy|vy))((\\s+|[,:?{])[^`]*)?$)"],
+        "html": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(html?|shtml|xhtml|inc|tmpl|tpl)((\\s+|[,:?{])[^`]*)?$)", "(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(php3??|php4|php5|phpt|phtml|aw|ctp)((\\s+|[,:?{])[^`]*)?$)"],
+        "ini": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(ini|conf)((\\s+|[,:?{])[^`]*)?$)"],
+        "java": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(java|bsh)((\\s+|[,:?{])[^`]*)?$)"],
+        "javascript": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(jsx??|javascript|es6|mjs|cjs|dataviewjs|\\{\\.js.+?})((\\s+|[,:?{])[^`]*)?$)"],
+        "json": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(json5??|sublime-settings|sublime-menu|sublime-keymap|sublime-mousemap|sublime-theme|sublime-build|sublime-project|sublime-completions)((\\s+|[,:?{])[^`]*)?$)"],
+        "jsonc": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(jsonc)((\\s+|[,:?{])[^`]*)?$)"],
+        "lua": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(lua)((\\s+|[,:?{])[^`]*)?$)"],
+        "make": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:((?:[Mm]|GNUm|OCamlM)akefile)((\\s+|[,:?{])[^`]*)?$)"],
+        "powershell": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(p(?:owershell|s1|sm1|sd1|wsh))((\\s+|[,:?{])[^`]*)?$)"],
+        "python": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(python|py3??|rpy|pyw|cpy|SConstruct|Sconstruct|sconstruct|SConscript|gypi??|\\{\\.python.+?})((\\s+|[,:?{])[^`]*)?$)"],
+        "regexp": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(re)((\\s+|[,:?{])[^`]*)?$)"],
+        "ruby": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(ruby|rbx??|rjs|Rakefile|rake|cgi|fcgi|gemspec|irbrc|Capfile|ru|prawn|Cheffile|Gemfile|Guardfile|Hobofile|Vagrantfile|Appraisals|Rantfile|Berksfile|Berksfile.lock|Thorfile|Puppetfile)((\\s+|[,:?{])[^`]*)?$)"],
+        "rust": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(rust|rs|\\{\\.rust.+?})((\\s+|[,:?{])[^`]*)?$)"],
+        "scss": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(scss)((\\s+|[,:?{])[^`]*)?$)"],
+        "shellscript": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(shell|sh|bash|zsh|bashrc|bash_profile|bash_login|profile|bash_logout|.textmate_init|\\{\\.bash.+?})((\\s+|[,:?{])[^`]*)?$)"],
+        "sql": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(sql|ddl|dml)((\\s+|[,:?{])[^`]*)?$)"],
+        "tsx": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(tsx)((\\s+|[,:?{])[^`]*)?$)"],
+        "typescript": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(t(?:ypescript|s))((\\s+|[,:?{])[^`]*)?$)"],
+        "xml": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(xml|xsd|tld|jsp|pt|cpt|dtml|rss|opml)((\\s+|[,:?{])[^`]*)?$)"],
+        "xsl": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(xslt??)((\\s+|[,:?{])[^`]*)?$)"],
+        "yaml": ["(|)(\\s*)(`{3,}|~{3,})\\s*(?i:(ya?ml)((\\s+|[,:?{])[^`]*)?$)", "(?=(-{3,}))"],
+    },
+    "haml": {
+        "css": ["(\\s*)(:(css|styles?))$"],
+        "javascript": ["(\\s*):javascript", "(\\s*)%script"],
+        "markdown": ["(\\s*)(:markdown)"],
+        "ruby": ["(\\s*):ruby", "(\\s*):ruby$", "\\s*(?:((%)([-:\\w]+))|(?=[#.]))", "(\\s*):(ruby|opal)$", "(\\s*)(:ruby)", "#\\{", "!?==", "\\{", "(&amp|!)?([-=~])"],
+    },
+};
+/**
+ * Every grammar that can reach a grammar in LAZY_EMBEDS (itself, or through
+ * the grammars it includes: ruby includes haml, which embeds markdown lazily),
+ * and those it reaches. A block in it needs whatever their triggers find.
+ */
+export const LAZY_VIA = {
+    "haml": ["haml"],
+    "markdown": ["markdown"],
+    "ruby": ["haml"],
+};

package/dist/highlight.d.ts

@@ -0,0 +1,24 @@
+import { type Highlighter } from "@dogsbay/render-html";
+export { parseLineRanges } from "@dogsbay/render-html";
+export type { CodeOptions, Highlighter } from "@dogsbay/render-html";
+/**
+ * Characters of rendered blocks (and their keys) the highlighter keeps by
+ * default, for library callers: 8M characters is 8–16 MB (a string with any
+ * character outside Latin-1 takes two bytes per character). The CLI passes
+ * 64M, the Worker 4M.
+ */
+export declare const DEFAULT_HIGHLIGHT_CACHE_CHARS = 8000000;
+export interface HighlighterOptions {
+    cacheChars?: number;
+    onePass?: boolean;
+    /** Grammars to load now: tags as written (`py`, `yml`), or blocks (see `load`), or "all" for every grammar (today's start-up cost, for a caller that cannot list its languages first). */
+    langs?: Iterable<LoadItem> | "all";
+}
+/** What `load` takes: a block (its tag as written and its code), or a tag alone, which for markdown loads every grammar markdown can embed. */
+export type LoadItem = string | undefined | {
+    lang: string | undefined;
+    code: string;
+};
+export declare function createHighlighter(o?: HighlighterOptions): Promise<Highlighter & {
+    load(items: Iterable<LoadItem>): Promise<void>;
+}>;

package/dist/highlight.js

@@ -0,0 +1,375 @@
+/**
+ * Syntax highlighting with Shiki's core build and JavaScript regex
+ * engine: no WebAssembly to instantiate, so it bundles into a Worker.
+ * Grammars load per language (`load`, before the code that needs them
+ * renders), one literal import() each (grammars.ts), so a split bundle reads
+ * only the grammars a site uses; the set covers the languages found in the
+ * Dogsbay test corpora. Unknown languages render as escaped plain text.
+ * Markdown embeds other grammars lazily (its fences name grammars it does not
+ * import), and ruby reaches it through haml, so what such a block needs
+ * depends on its content: `load` scans it (LAZY_EMBEDS, LAZY_VIA), and a block
+ * renders only once all of it is loaded, so its output never depends on what
+ * other pages loaded.
+ *
+ * Output uses dual themes with `defaultColor: false`, so colours come
+ * from `--shiki-light` / `--shiki-dark` CSS variables and the theme
+ * toggle switches them with no re-render.
+ *
+ * Output must not depend on machine load. Shiki's default guard is a
+ * 500 ms WALL-CLOCK limit per line, after which the rest of the line gets
+ * the default colour. On OpenShift a 4,445-character base64 line took
+ * ~310 ms idle and crossed 500 ms under build load, so a page's colours
+ * changed between identical builds. The bound is now a line LENGTH
+ * (deterministic): longer lines are left untokenized. The time limit is
+ * kept only as a Worker safety net far above any real line.
+ */
+import { createHighlighterCore } from "shiki/core";
+import { createJavaScriptRegexEngine, defaultJavaScriptRegexConstructor } from "shiki/engine/javascript";
+import { onePassRenderer } from "./one-pass.js";
+import githubLight from "@shikijs/themes/github-light-default";
+import githubDark from "@shikijs/themes/github-dark-default";
+import { LAZY_EMBEDS, LAZY_VIA, LOADERS, SPELLINGS } from "./grammars.js";
+import { transformerNotationDiff, transformerNotationErrorLevel, transformerNotationFocus, transformerNotationHighlight, transformerNotationWordHighlight, } from "@shikijs/transformers";
+import { esc, PLAIN, plainBlock } from "@dogsbay/render-html";
+import { stripUnprocessableMarkers } from "@dogsbay/render-html/code-markers";
+export { parseLineRanges } from "@dogsbay/render-html";
+/** Aliases the grammars do not declare themselves. */
+const ALIASES = {
+    terminal: "bash",
+    console: "shellsession",
+    "shell-session": "shellsession",
+    cmd: "bat",
+    dos: "bat",
+    batch: "bat",
+    dita: "xml",
+    ditamap: "xml",
+    ditaval: "xml",
+    xhtml: "html",
+    svg: "xml",
+    xslt: "xsl",
+    yml: "yaml",
+    conf: "ini",
+    cfg: "ini",
+    env: "properties",
+    dockerfile: "docker",
+    containerfile: "docker",
+    "c++": "cpp",
+    "c#": "csharp",
+    cs: "csharp",
+    gradle: "groovy",
+    kt: "kotlin",
+    py: "python",
+    rb: "ruby",
+    md: "markdown",
+    terraform: "hcl",
+    tf: "hcl",
+};
+/** Lines longer than this are not tokenized: 14 of 167,153 OpenShift code lines, all base64 or certificates. */
+const MAX_LINE_LENGTH = 1000;
+/** Safety net against a pathological grammar regex; a 1,000-character line takes ~70 ms. */
+const TIME_LIMIT_MS = 5000;
+let corePromise;
+/** One core per process, shared by every highlighter: the themes, and the grammars loaded so far. */
+function core() {
+    corePromise ??= createHighlighterCore({
+        themes: [githubLight, githubDark],
+        langs: [],
+        engine: createJavaScriptRegexEngine({ forgiving: true }),
+    });
+    return corePromise;
+}
+/** Grammar module → its load, kept while it succeeds (a failed import is tried again). */
+const grammarLoads = new Map();
+/** Ids the core has registered (names and aliases), for every highlighter on it. */
+const loaded = new Set();
+function loadGrammar(hl, module) {
+    let p = grammarLoads.get(module);
+    if (!p) {
+        p = LOADERS[module]()
+            .then(async (m) => {
+            await hl.loadLanguage(...m.default);
+            for (const id of hl.getLoadedLanguages())
+                loaded.add(id);
+        })
+            .catch((e) => {
+            grammarLoads.delete(module);
+            throw e;
+        });
+        grammarLoads.set(module, p);
+    }
+    return p;
+}
+/** A trigger pattern compiled as the core compiles grammar patterns; null for one the engine rejects, which then always triggers. */
+const triggers = new Map();
+function trigger(src) {
+    if (!triggers.has(src)) {
+        let re = null;
+        try {
+            re = defaultJavaScriptRegexConstructor(src);
+        }
+        catch {
+            re = null;
+        }
+        triggers.set(src, re);
+    }
+    return triggers.get(src);
+}
+/**
+ * The grammar ids a block needs: its own, and for a grammar that reaches one
+ * that embeds others lazily (LAZY_VIA: markdown itself; ruby through haml),
+ * each embed one of whose triggers matches a line of the block (all of them
+ * when the block's code is not known). An embed that reaches lazy grammars in
+ * turn is followed over the same code.
+ */
+function needs(id, code, out = new Set(), lines = blockLines(code)) {
+    if (id === "text" || out.has(id))
+        return out;
+    out.add(id);
+    for (const lazy of LAZY_VIA[id] ?? []) {
+        for (const [embed, pats] of Object.entries(LAZY_EMBEDS[lazy])) {
+            if (out.has(embed))
+                continue;
+            const hit = !lines ||
+                pats.some((p) => {
+                    if (p === "")
+                        return true;
+                    const re = trigger(p);
+                    return !re || lines.some((l) => ((re.lastIndex = 0), re.test(l)));
+                });
+            if (hit)
+                needs(embed, code, out, lines);
+        }
+    }
+    return out;
+}
+/**
+ * A block's lines as `render` has Shiki tokenize them (markers stripped, the
+ * last newline dropped, split on \r?\n), each with the newline the tokenizer
+ * adds. Lines at or over MAX_LINE_LENGTH are left out: Shiki does not
+ * tokenize them and keeps its state across them, so no rule starts there (and
+ * the unanchored triggers would be quadratic on a long run of spaces).
+ */
+function blockLines(code) {
+    if (code === undefined)
+        return undefined;
+    return stripUnprocessableMarkers(code.replace(/\n$/, ""), false)
+        .split(/\r?\n/)
+        .filter((l) => l.length < MAX_LINE_LENGTH)
+        .map((l) => l + "\n");
+}
+/** Everything a block in this grammar could need, whatever its code; when all of it is loaded, no scan is needed. */
+const reachable = new Map();
+function canReach(id) {
+    let r = reachable.get(id);
+    if (!r)
+        reachable.set(id, (r = [...needs(id, undefined)]));
+    return r;
+}
+/** Whether everything a block needs is loaded, so it renders as with every grammar loaded. */
+function ready(id, code) {
+    if (!loaded.has(id))
+        return false;
+    if (!LAZY_VIA[id] || canReach(id).every((n) => loaded.has(n)))
+        return true;
+    for (const n of needs(id, code))
+        if (!loaded.has(n))
+            return false;
+    return true;
+}
+/** The language id a block's tag means: "text" for plain or unknown tags. */
+function resolveId(lang) {
+    const l = (lang ?? "").toLowerCase().trim();
+    if (PLAIN.has(l))
+        return "text";
+    const a = ALIASES[l] ?? l;
+    return Object.hasOwn(SPELLINGS, a) ? a : "text";
+}
+/**
+ * Characters of rendered blocks (and their keys) the highlighter keeps by
+ * default, for library callers: 8M characters is 8–16 MB (a string with any
+ * character outside Latin-1 takes two bytes per character). The CLI passes
+ * 64M, the Worker 4M.
+ */
+export const DEFAULT_HIGHLIGHT_CACHE_CHARS = 8_000_000;
+export async function createHighlighter(o = {}) {
+    const hl = await core();
+    // Both themes from one tokenization (src/render/one-pass.ts); `onePass: false` keeps Shiki's codeToHtml, for comparison.
+    const THEMES = { light: "github-light-default", dark: "github-dark-default" };
+    const onePass = o.onePass === false ? undefined : onePassRenderer(hl, THEMES);
+    const resolve = resolveId;
+    let unloaded = 0;
+    /** Whether the last `render` had everything its block needs (so `code` may cache it). */
+    let lastReady = true;
+    const load = async (items) => {
+        const modules = new Set();
+        for (const item of items) {
+            const block = typeof item === "object" ? item : { lang: item, code: undefined };
+            const id = resolveId(block.lang);
+            if (id === "text")
+                continue;
+            const all = canReach(id);
+            if (all.every((n) => loaded.has(n)))
+                continue;
+            for (const n of block.code === undefined || all.length === 1 ? all : needs(id, block.code))
+                if (!loaded.has(n))
+                    modules.add(SPELLINGS[n]);
+        }
+        if (!modules.size)
+            return;
+        await Promise.all([...modules].map((m) => loadGrammar(hl, m)));
+    };
+    if (o.langs === "all")
+        await load(Object.keys(SPELLINGS));
+    else if (o.langs)
+        await load(o.langs);
+    // Identical blocks (install commands, shared snippets) are common across a
+    // corpus; tokenizing dominates highlight time, so memoise by input. An LRU:
+    // a Map keeps insertion order, a hit moves the entry to the end, and the
+    // oldest entries go when the budget (characters of key and HTML) is spent.
+    // The key is the full input, not a hash, so a collision cannot return the
+    // wrong block. A hit re-inserts the entry under its own flattened key: the
+    // caller's key is a rope around `code`, which may be a slice of its page.
+    const cache = new Map();
+    let cacheChars = 0;
+    let hits = 0;
+    let misses = 0;
+    const CACHE_MAX = Number.isFinite(o.cacheChars) && o.cacheChars >= 0 ? o.cacheChars : DEFAULT_HIGHLIGHT_CACHE_CHARS;
+    const keyOf = (code, lang, opts) => `${lang ?? ""}\0${opts.lineNumbers ? 1 : 0}\0${(opts.highlights ?? []).join(",")}\0${(opts.diffAdd ?? []).join(",")}\0${(opts.diffRemove ?? []).join(",")}\0${code}`;
+    const keep = (key, html) => {
+        const size = key.length + html.length;
+        // A block over a quarter of the budget is not kept, so one block cannot empty the cache (as TextCache does).
+        if (size > CACHE_MAX / 4)
+            return;
+        for (const [k, v] of cache) {
+            if (cacheChars + size <= CACHE_MAX)
+                break;
+            cache.delete(k);
+            cacheChars -= v.key.length + v.html.length;
+        }
+        // Flatten before keeping: the rendered string is a rope of slices that
+        // would pin every token and source line it was built from.
+        const kept = { key: flatten(key), html: flatten(html) };
+        cache.set(kept.key, kept);
+        cacheChars += size;
+    };
+    const render = (code, lang, opts = {}) => {
+        const id = resolve(lang);
+        // Markers Shiki cannot see (plain text has no comment tokens; mid-line markers) are removed, as CodeRich does.
+        const src = stripUnprocessableMarkers(code.replace(/\n$/, ""), id === "text");
+        const marks = new Set(opts.highlights ?? []);
+        const adds = new Set(opts.diffAdd ?? []);
+        const removes = new Set(opts.diffRemove ?? []);
+        const cls = opts.lineNumbers ? " line-numbers" : "";
+        lastReady = true;
+        if (id === "text")
+            return plainBlock(src, opts);
+        // A grammar nobody loaded first (a caller missed `load`), or one a markdown block embeds: plain, counted, and not cached by `code`.
+        if (!ready(id, code)) {
+            lastReady = false;
+            unloaded++;
+            return plainBlock(src, opts);
+        }
+        // The notation transformers each walk every line: only for code that has markers.
+        const notation = src.includes("[!code");
+        const transformers = [
+            ...(notation
+                ? [transformerNotationDiff(), transformerNotationHighlight(), transformerNotationFocus(), transformerNotationErrorLevel(), transformerNotationWordHighlight()]
+                : []),
+            {
+                name: "dogsbay:lines",
+                line(node, line) {
+                    if (marks.has(line))
+                        this.addClassToHast(node, "highlighted");
+                    if (adds.has(line))
+                        this.addClassToHast(node, ["diff", "add"]);
+                    else if (removes.has(line))
+                        this.addClassToHast(node, ["diff", "remove"]);
+                },
+            },
+        ];
+        if (opts.lineNumbers || adds.size || removes.size || notation)
+            transformers.push(gutter(!!opts.lineNumbers));
+        try {
+            const shiki = () => hl.codeToHtml(src, { lang: id, themes: THEMES, defaultColor: false, tokenizeMaxLineLength: MAX_LINE_LENGTH, tokenizeTimeLimit: TIME_LIMIT_MS, transformers });
+            let out;
+            if (onePass && !notation) {
+                try {
+                    // undefined: a block nested too deep for it, which Shiki's own path handles.
+                    out = onePass(src, { lang: id, tokenizeMaxLineLength: MAX_LINE_LENGTH, tokenizeTimeLimit: TIME_LIMIT_MS, transformers });
+                }
+                catch {
+                    out = undefined; // an unexpected failure: Shiki's path gives the color, before falling back to plain text
+                }
+            }
+            out ??= shiki();
+            return cls ? out.replace('<pre class="shiki', `<pre class="shiki${cls}`) : out;
+        }
+        catch {
+            return `<pre class="shiki shiki-plain${cls}" tabindex="0"><code>${esc(src)}</code></pre>`;
+        }
+    };
+    return {
+        resolve,
+        render,
+        load,
+        cacheStats: () => ({ entries: cache.size, chars: cacheChars, hits, misses, unloaded }),
+        has: (code, lang, opts = {}) => cache.has(keyOf(code, lang, opts)),
+        prime(code, lang, opts, html) {
+            const key = keyOf(code, lang, opts);
+            // Trusted as given: a thread that rendered it loaded what the block needs first (this highlighter may not have yet).
+            if (!cache.has(key))
+                keep(key, html);
+        },
+        code(code, lang, opts = {}) {
+            const key = keyOf(code, lang, opts);
+            const hit = cache.get(key);
+            if (hit !== undefined) {
+                hits++;
+                cache.delete(key);
+                cache.set(hit.key, hit);
+                return hit.html;
+            }
+            misses++;
+            const html = render(code, lang, opts);
+            if (lastReady)
+                keep(key, html);
+            return html;
+        },
+    };
+}
+/**
+ * A string's own copy. V8 keeps a sliced or concatenated string as a rope that
+ * pins every string it was built from, so the cache flattens what it keeps.
+ */
+const flatten = (s) => JSON.parse(JSON.stringify(s));
+/** The gutter as a Shiki transformer, after the notation transformers have marked diff lines. */
+function gutter(lineNumbers) {
+    return {
+        name: "dogsbay:gutter",
+        code(node) {
+            const cls = (n) => {
+                const c = n.properties?.class;
+                return Array.isArray(c) ? c.join(" ") : String(c ?? "");
+            };
+            const lines = node.children.filter((c) => c.type === "element" && c.tagName === "span");
+            const diffs = lines.some((l) => /\bdiff\b/.test(cls(l)));
+            if (!lineNumbers && !diffs)
+                return;
+            let n = 1;
+            for (const line of lines) {
+                const c = cls(line);
+                if (!/\bline\b/.test(c))
+                    continue;
+                const kind = /\badd\b/.test(c) ? "add" : /\bremove\b/.test(c) ? "remove" : "";
+                const parts = [];
+                if (lineNumbers)
+                    parts.push(String(n).padStart(3));
+                if (diffs)
+                    parts.push(kind === "add" ? "+" : kind === "remove" ? "−" : " ");
+                // Not text of the code: Pagefind skips it (it indexed "1function", "5const").
+                line.children.unshift({ type: "element", tagName: "span", properties: { class: `gutter${kind ? ` ${kind}` : ""}`, "data-pagefind-ignore": "" }, children: [{ type: "text", value: parts.join(" ") }] });
+                n++;
+            }
+        },
+    };
+}

package/dist/one-pass.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Dual-theme highlighting with one tokenization (plans/build-performance.md,
+ * step 3). Shiki's `codeToHtml` with `themes: { light, dark }` runs the
+ * grammar once per theme (`codeToTokensWithThemes` maps `codeToTokensBase`
+ * over them), and the grammar is most of the cost. The scopes a grammar
+ * assigns do not depend on the theme, so this tokenizes each line once with
+ * `tokenizeLine` (scopes) and colors every token for both themes with the
+ * TextMate theme matcher, the same `Theme` Shiki's registry uses.
+ *
+ * The result goes through Shiki's own `tokensToHast` and `hastToHtml`, so
+ * transformers that work on lines and code (our line classes, gutter) run
+ * as before. Colors per character are the same as Shiki's; the span
+ * boundaries can differ (Shiki also splits where a token's type changes but
+ * its colors do not), which does not change what is shown.
+ *
+ * Not used for code with `[!code …]` notation, whose transformers work on
+ * the source before tokenization: those blocks keep Shiki's `codeToHtml`.
+ */
+import { type HighlighterCore, type ShikiTransformer } from "shiki/core";
+/** A token nested deeper than this sends its block to Shiki's own path, which has no per-path cost. */
+export declare const MAX_SCOPE_DEPTH = 100;
+export interface OnePassOptions {
+    lang: string;
+    tokenizeMaxLineLength: number;
+    tokenizeTimeLimit: number;
+    transformers: ShikiTransformer[];
+}
+/**
+ * A renderer bound to one highlighter and its two themes; `render` replaces
+ * `codeToHtml`, and returns undefined for a block it leaves to Shiki (tokens
+ * nested deeper than MAX_SCOPE_DEPTH).
+ */
+export declare function onePassRenderer(hl: HighlighterCore, themes: {
+    light: string;
+    dark: string;
+}): (code: string, o: OnePassOptions) => string | undefined;

package/dist/one-pass.js

@@ -0,0 +1,193 @@
+/**
+ * Dual-theme highlighting with one tokenization (plans/build-performance.md,
+ * step 3). Shiki's `codeToHtml` with `themes: { light, dark }` runs the
+ * grammar once per theme (`codeToTokensWithThemes` maps `codeToTokensBase`
+ * over them), and the grammar is most of the cost. The scopes a grammar
+ * assigns do not depend on the theme, so this tokenizes each line once with
+ * `tokenizeLine` (scopes) and colors every token for both themes with the
+ * TextMate theme matcher, the same `Theme` Shiki's registry uses.
+ *
+ * The result goes through Shiki's own `tokensToHast` and `hastToHtml`, so
+ * transformers that work on lines and code (our line classes, gutter) run
+ * as before. Colors per character are the same as Shiki's; the span
+ * boundaries can differ (Shiki also splits where a token's type changes but
+ * its colors do not), which does not change what is shown.
+ *
+ * Not used for code with `[!code …]` notation, whose transformers work on
+ * the source before tokenization: those blocks keep Shiki's `codeToHtml`.
+ */
+import { applyColorReplacements, flatTokenVariants, hastToHtml, resolveColorReplacements, splitLines, tokensToHast, } from "shiki/core";
+import { INITIAL, Theme } from "shiki/textmate";
+/** vscode-textmate's FontStyle (a const enum, so not importable here). */
+const FontStyle = { NotSet: -1, Underline: 4, Strikethrough: 8 };
+function themeInfo(hl, themeName) {
+    const resolved = hl.getTheme(themeName);
+    const theme = Theme.createFromRawTheme(resolved);
+    const replacements = resolveColorReplacements(resolved, {});
+    return {
+        name: resolved.name,
+        fg: applyColorReplacements(resolved.fg, replacements) ?? "",
+        bg: applyColorReplacements(resolved.bg, replacements) ?? "",
+        theme,
+        colorMap: theme.getColorMap(),
+        replacements,
+        defaults: theme.getDefaults(),
+    };
+}
+/** Nodes kept before the trie is cleared (a corpus has a few thousand distinct paths). */
+const MAX_NODES = 50_000;
+/** A token nested deeper than this sends its block to Shiki's own path, which has no per-path cost. */
+export const MAX_SCOPE_DEPTH = 100;
+class DeepNesting extends Error {
+}
+/**
+ * A renderer bound to one highlighter and its two themes; `render` replaces
+ * `codeToHtml`, and returns undefined for a block it leaves to Shiki (tokens
+ * nested deeper than MAX_SCOPE_DEPTH).
+ */
+export function onePassRenderer(hl, themes) {
+    const t = [themeInfo(hl, themes.light), themeInfo(hl, themes.dark)];
+    const order = ["light", "dark"];
+    const fg = `--shiki-light:${t[0].fg || "inherit"};--shiki-dark:${t[1].fg || "inherit"}`;
+    const bg = `--shiki-light-bg:${t[0].bg || "inherit"};--shiki-dark-bg:${t[1].bg || "inherit"}`;
+    const themeName = `shiki-themes ${t[0].name} ${t[1].name}`;
+    const newRoot = () => ({ path: null, fontStyle: [t[0].defaults.fontStyle, t[1].defaults.fontStyle], fg: [t[0].defaults.foregroundId, t[1].defaults.foregroundId], children: new Map() });
+    let root = newRoot();
+    let nodes = 0;
+    const child = (parent, scopeName) => {
+        let n = parent.children.get(scopeName);
+        if (n)
+            return n;
+        const path = { parent: parent.path, scopeName };
+        n = { path, fontStyle: [...parent.fontStyle], fg: [...parent.fg], children: new Map() };
+        for (const k of [0, 1]) {
+            const m = t[k].theme.match(path);
+            if (!m)
+                continue;
+            if (m.fontStyle !== FontStyle.NotSet)
+                n.fontStyle[k] = m.fontStyle;
+            if (m.foregroundId !== 0)
+                n.fg[k] = m.foregroundId;
+        }
+        parent.children.set(scopeName, n);
+        nodes++;
+        return n;
+    };
+    const styleOf = (n) => (n.style ??= [0, 1].map((k) => ({
+        color: applyColorReplacements(t[k].colorMap[n.fg[k]], t[k].replacements) ?? "",
+        fontStyle: n.fontStyle[k] === FontStyle.NotSet ? 0 : n.fontStyle[k],
+    })));
+    const nodeFor = (scopes) => {
+        if (scopes.length > MAX_SCOPE_DEPTH)
+            throw new DeepNesting();
+        let n = root;
+        for (const s of scopes)
+            n = child(n, s);
+        return n;
+    };
+    return function render(code, o) {
+        if (nodes > MAX_NODES) {
+            root = newRoot();
+            nodes = 0;
+        }
+        const grammar = hl.getLanguage(o.lang);
+        let stack = INITIAL;
+        const lines = [];
+        try {
+            for (const [line, offset] of splitLines(code)) {
+                if (line === "") {
+                    lines.push([]);
+                    continue;
+                }
+                if (o.tokenizeMaxLineLength > 0 && line.length >= o.tokenizeMaxLineLength) {
+                    // As Shiki does: a line this long is not tokenized, and gets no color.
+                    lines.push([flatTokenVariants({ content: line, offset, variants: { light: { color: "", fontStyle: 0 }, dark: { color: "", fontStyle: 0 } } }, order, "--shiki-", false)]);
+                    continue;
+                }
+                const r = grammar.tokenizeLine(line, stack, o.tokenizeTimeLimit);
+                const out = [];
+                let prev;
+                const flush = () => {
+                    if (!prev)
+                        return;
+                    out.push(flatTokenVariants({ content: line.slice(prev.start, prev.end), offset: offset + prev.start, variants: { light: prev.l, dark: prev.d } }, order, "--shiki-", false));
+                };
+                for (let i = 0; i < r.tokens.length; i++) {
+                    const tok = r.tokens[i];
+                    // As Shiki reads tokenizeLine2: a token runs to the next one's start, and the last to the end of the line,
+                    // so a line cut short by the time limit keeps its text (the rest takes the last token's style).
+                    const end = i + 1 < r.tokens.length ? r.tokens[i + 1].startIndex : line.length;
+                    if (end <= tok.startIndex)
+                        continue;
+                    const [l, d] = styleOf(nodeFor(tok.scopes));
+                    // Adjacent tokens styled the same in both themes are one span, as Shiki's merge of the two themes' tokens gives.
+                    // Shiki's spans can split differently around whitespace (its tokens also break where a token's type changes),
+                    // so a space may take the color of the token before it rather than after: invisible, since spaces are not
+                    // underlined or struck through here unless the whole token is.
+                    if (prev && prev.end === tok.startIndex && same(prev.l, l) && same(prev.d, d))
+                        prev.end = end;
+                    else {
+                        flush();
+                        prev = { l, d, start: tok.startIndex, end };
+                    }
+                }
+                flush();
+                lines.push(out);
+                stack = r.ruleStack;
+            }
+        }
+        catch (e) {
+            if (e instanceof DeepNesting)
+                return undefined;
+            throw e;
+        }
+        const context = {
+            meta: {},
+            options: { lang: o.lang, themes, defaultColor: false, transformers: o.transformers },
+            codeToHast: () => {
+                throw new Error("codeToHast is not available in the one-pass renderer");
+            },
+            codeToTokens: () => {
+                throw new Error("codeToTokens is not available in the one-pass renderer");
+            },
+            get source() {
+                return code;
+            },
+        };
+        const hast = tokensToHast(mergeWhitespaceTokens(lines), { ...context.options, fg, bg, themeName, rootStyle: `${fg};${bg}` }, context);
+        return hastToHtml(hast);
+    };
+}
+const same = (a, b) => a.color === b.color && a.fontStyle === b.fontStyle;
+/**
+ * Shiki's default `mergeWhitespaces: true` (its `mergeWhitespaceTokens`, not
+ * exported): a whitespace-only token joins the next one, and trailing
+ * whitespace joins the previous one, unless it is underlined or struck through.
+ */
+function mergeWhitespaceTokens(lines) {
+    const WS = /^\s+$/;
+    return lines.map((line) => {
+        const out = [];
+        let carry = "";
+        let firstOffset;
+        line.forEach((token, idx) => {
+            const couldMerge = !(token.fontStyle && (token.fontStyle & FontStyle.Underline || token.fontStyle & FontStyle.Strikethrough));
+            if (couldMerge && WS.test(token.content) && line[idx + 1]) {
+                if (firstOffset === undefined)
+                    firstOffset = token.offset;
+                carry += token.content;
+            }
+            else if (carry) {
+                if (couldMerge)
+                    out.push({ ...token, offset: firstOffset, content: carry + token.content });
+                else
+                    out.push({ content: carry, offset: firstOffset }, token);
+                firstOffset = undefined;
+                carry = "";
+            }
+            else
+                out.push(token);
+        });
+        return out;
+    });
+}

package/package.json

@@ -1,6 +1,48 @@
 {
   "name": "@dogsbay/highlight",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.2.0-beta.117",
+  "description": "Syntax highlighting for @dogsbay/render-html: Shiki core with the JavaScript regex engine, both themes from one tokenization, a bounded cache. No WebAssembly; Worker-safe",
+  "type": "module",
+  "main": "./dist/highlight.js",
+  "types": "./dist/highlight.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/highlight.d.ts",
+      "import": "./dist/highlight.js"
+    },
+    "./one-pass": {
+      "types": "./dist/one-pass.d.ts",
+      "import": "./dist/one-pass.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "dependencies": {
+    "@shikijs/engine-javascript": "4.3.1",
+    "@shikijs/langs": "4.3.1",
+    "@shikijs/themes": "4.3.1",
+    "@shikijs/transformers": "4.3.1",
+    "shiki": "4.3.1",
+    "@dogsbay/render-html": "0.2.0-beta.117"
+  },
+  "devDependencies": {
+    "@types/node": "^22.20.1",
+    "typescript": "^5.9.3",
+    "vitest": "^4.1.10",
+    "esbuild": "^0.25.0"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dogsbay/dogsbay.git",
+    "directory": "packages/highlight"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit"
+  }
 }