@motionscript/code 0.1.0 → 0.1.1-alpha.0

package/src/node.ts

@@ -52,11 +52,7 @@ import {
 const ROW_BAND_COLOR = "rgba(255, 255, 255, 0.12)";
 const ROW_BAND_RADIUS = 6;
 
-/**
- * Which token became which across an edit, and when each part of it happens.
- * Arguments of {@link defineCode}: a strategy is a function, which a prop would
- * take for a binding, so it is part of the component's definition instead.
- */
+/** Arguments of {@link defineCode}: a strategy is a function, which a prop would take for a binding, so it belongs to the definition. */
 export interface CodeStrategies {
     /** Decides which token became which across an edit. Defaults to `defaultCodeDiff`. */
     diffStrategy?: CodeDiffStrategy;
@@ -123,12 +119,9 @@ const TARGET = Types.custom<CodeTarget | undefined>(undefined, { kind: "selector
 const TEXT = Types.text("", { multiline: true });
 
 /**
- * A `Code` whose edits are paired and timed by `strategies`. {@link Code} is
- * the one with the defaults; a project wanting another hands its own to the
- * engine in its place (the key is the same).
- *
- * A listing and nothing else: no paint slots, and no box behind the tokens. A
- * listing set on a card is a `Code` inside a hugging `Rect`.
+ * A `Code` whose edits are paired and timed by `strategies`; {@link Code} has the
+ * defaults, and a project wanting another hands its own to the engine (same key).
+ * A listing only, with no box behind it: a card is a `Code` inside a hugging `Rect`.
  */
 export function defineCode(strategies: CodeStrategies = {}) {
     const kit: Kit = {
@@ -304,8 +297,6 @@ export function findFirstRange(source: CodeSource, text: string): CodeRange | nu
     return findRangeAt(source, text, 0);
 }
 
-// ── What a listing shows ────────────────────────────────────────────────────
-
 interface Kit {
     readonly diff: CodeDiffStrategy;
     readonly phases: CodePhaseStrategy;
@@ -318,7 +309,7 @@ interface Edited {
     readonly listing: CodeListing;
 }
 
-/** What the structure resolves from. A command's snapshot and a node's own state view both are one. */
+/** What the structure resolves from; a command's snapshot and a node's state view both qualify. */
 interface ListingState {
     readonly code: CodeSource;
     readonly language: string;
@@ -350,10 +341,7 @@ interface Shown {
     readonly progress: number;
 }
 
-/**
- * A `to({ code })` mid-flight comes first: it wrote `code` after any edit
- * command did. Then an edit command in flight, then the settled listing.
- */
+/** A `to({ code })` mid-flight wins, as it wrote `code` after any edit command; then an edit in flight, then the settled listing. */
 function shown(s: ListingState, kit: Kit): Shown {
     const code = s.code;
     if (typeof code !== "string") {
@@ -375,10 +363,9 @@ function current(s: ListingState, kit: Kit): CodeListing {
 }
 
 /**
- * The structure `source` settles on. Until a listing command has acted, the
- * props are all there is; after one, it is the listing that command left,
- * recoloured keeping its ids when only the theme moved, and diffed onward
- * when the source has (a `to({ code })` that landed, a binding).
+ * The structure `source` settles on: from the props until a listing command has
+ * acted, then the listing it left, recoloured (ids kept) when only the theme moved
+ * and diffed onward when the source has.
  */
 function settled(s: ListingState, source: string, kit: Kit): CodeListing {
     const basis = basisOf(s);
@@ -391,7 +378,7 @@ function settled(s: ListingState, source: string, kit: Kit): CodeListing {
 }
 
 function basisOf(s: { readonly language: string; readonly theme: CodeTheme }): CodeTokenBasis {
-    return { language: s.language, theme: s.theme, highlighted: canHighlight(s.language, s.theme) };
+    return { language: s.language, theme: s.theme, highlighted: canHighlight(s.language) };
 }
 
 function sameBasis(a: CodeTokenBasis, b: CodeTokenBasis): boolean {
@@ -403,10 +390,7 @@ function sourceText(code: CodeSource): string {
     return typeof code === "string" ? code : code.to;
 }
 
-/**
- * Mid-tween `code` holds both sources, so the frame can draw the edit between
- * them; at either end it is a source again.
- */
+/** Mid-tween `code` holds both sources so the frame can draw the edit between them; at either end it is a source again. */
 function blendSource(from: CodeSource, to: CodeSource, t: number): CodeSource {
     const leaving = sourceText(from);
     const arriving = sourceText(to);
@@ -468,7 +452,7 @@ function buildEdit(kit: Kit, from: IdLine[], next: string, basis: CodeTokenBasis
 
 function tokenize(source: string, basis: CodeTokenBasis): IdLine[] {
     // Best effort outside the asset pipeline (a test, a headless measure): on the timeline `load` has the grammar in first.
-    if (!basis.highlighted) ensureHighlighter(undefined, [basis.language]).catch(() => { });
+    if (!basis.highlighted) ensureHighlighter([basis.language]).catch(() => { });
     return tokenizeCodeToIdLines(source, basis.language, basis.theme);
 }
 
@@ -517,8 +501,6 @@ function adoptIds(from: IdLine[], to: IdLine[]): void {
     }
 }
 
-// ── Commands ────────────────────────────────────────────────────────────────
-
 /** What a listing command writes at a point of its own eased time. */
 interface ListingWrites {
     readonly code?: string;
@@ -530,11 +512,7 @@ interface ListingWrites {
 
 type Frame = (t: number) => ListingWrites;
 
-/**
- * Each command writes the listing it acted on, at every time it is asked for,
- * so the ids it captured are the ones drawn; before it starts the timeline has
- * put back the props, and the listing is derived from them again.
- */
+/** Each command writes the listing it acted on at every time asked, so the ids it captured are the ones drawn. */
 function editFrame(kit: Kit, seen: CodeListing, source: string, basis: CodeTokenBasis): Frame | null {
     const next = normalizeSource(source);
     if (next === seen.source) return null;
@@ -670,10 +648,7 @@ function rowBandsIn(lines: IdLine[], codeRanges: CodeRange[], color: NormalizedC
     return bands;
 }
 
-/**
- * A single range (a plain object) as a one-element array; a chain or an array
- * of ranges as itself; the `{ list }` a chain serialises as, as that chain.
- */
+/** One range as a one-element array; a chain or array as itself; the `{ list }` a chain serialises as, as that chain. */
 function normalizeCodeRanges(codeRange: CodeTarget | undefined): CodeRange[] {
     const maybeIterable = codeRange as Partial<Iterable<CodeRange>>;
     if (typeof maybeIterable[Symbol.iterator] === "function") {
@@ -686,28 +661,22 @@ function normalizeCodeRanges(codeRange: CodeTarget | undefined): CodeRange[] {
 
 const grammarLoaders = new Map<string, () => Promise<void>>();
 
-// One function per language: blocks sharing a grammar declare one key, and a
-// second load function for a key already declared is ignored with a warning.
+// One function per language: a second load function for an already-declared task key is ignored with a warning.
 function grammarLoader(language: string): () => Promise<void> {
     let load = grammarLoaders.get(language);
     if (load === undefined) {
-        load = () => ensureHighlighter(undefined, [language]);
+        load = () => ensureHighlighter([language]);
         grammarLoaders.set(language, load);
     }
     return load;
 }
 
-// ── Geometry and drawing ────────────────────────────────────────────────────
-
 interface Layouts {
     readonly from: CodeLayout;
     readonly to: CodeLayout;
 }
 
-/**
- * A node's measurements, kept between frames: the advance of each token, and
- * the layouts last built, reused while what they were built from is unchanged.
- */
+/** A node's measurements between frames: token advances, and the layouts last built, reused while their inputs are unchanged. */
 interface Measured {
     readonly advances: TokenAdvanceCache;
     measurer: Measurer | null;
@@ -738,11 +707,7 @@ function metricsOf(s: LayoutState): CodeMetrics {
     };
 }
 
-/**
- * The one or two layouts a frame is drawn from: a settled frame has one, a
- * frame inside an edit two (where every token was, and where it is going).
- * `null` for a node never measured, which has nothing to measure them with.
- */
+/** One layout for a settled frame, two (where every token was, where it goes) inside an edit; `null` for a node never measured. */
 function layoutsOf(measured: Measured, view: Shown, m: CodeMetrics): Layouts | null {
     const signature = measured.advances.signature(m.fontSize, m.fontFamily);
     measured.advances.sync(signature);
@@ -817,10 +782,9 @@ function drawListing(ctx: RenderContext, s: DrawState, kit: Kit): void {
 }
 
 /**
- * A listing is grabbed by its text, line by ragged line: each line a slot as
- * wide as its ink, plus the gutter where numbers show, inside the box. Code is
- * mostly whitespace, and its empty corners would swallow clicks meant for what
- * is behind it. Mid-edit, or before it is laid out, the box.
+ * A listing is picked by its ink, line by ragged line (plus the gutter), inside the
+ * box: code is mostly whitespace and empty corners would swallow clicks meant for
+ * what is behind. Mid-edit, or before layout, the box.
  */
 function outlineOf(view: Shown, layouts: Layouts | null, m: CodeMetrics, size: Size2D): Graphics2D {
     const box = { x: 0, y: 0, width: size.width, height: size.height };

package/src/nodes.ts

@@ -1,16 +1,4 @@
 import { Code } from "./node";
 
-/**
- * Every node type this package publishes.
- *
- * A host registers these by handing the array to an engine —
- * `new Engine(platform, { nodes: NODES })` — which reads each component's key.
- * Defining a component only *declares* that key: nothing is registered by
- * importing a module, so a registry holds exactly what its owner asked for.
- *
- * Listing the component **values** is also what survives bundling: this package
- * declares `sideEffects: false`, and a document names a node type by string, so
- * an import kept only for its side effect is exactly what gets shaken out.
- */
 // Typed by name, so the declaration names each component rather than spelling out its type through core.
 export const NODES: (typeof Code)[] = [Code];

package/src/render.ts

@@ -5,14 +5,10 @@ import { windowProgress, moveProgress } from "./transitions";
 import type { TokenState, StructuralTransition, RowBand, RowsTransition } from "./transitions";
 import type { TokenAdvanceCache } from "./measure-cache";
 
-/** Colour of a token the grammar had no opinion about. */
 const DEFAULT_TOKEN_COLOR: NormalizedColor = [0.82, 0.84, 0.86, 1];
-/** Gutter line-number colour: a muted version of the standard text colour. */
 const LINE_NUMBER_COLOR: NormalizedColor = [0.45, 0.5, 0.55, 1];
 
-// `parseColor` walks a CSS string; a listing asks for the same handful of theme
-// colours on every token of every frame, so the parse is memoized. Module-level,
-// because the answer depends only on the string.
+// A listing asks for the same few theme colours on every token of every frame.
 const colorCache = new Map<string, NormalizedColor>();
 function tokenColor(css: string | undefined): NormalizedColor {
     if (!css) return DEFAULT_TOKEN_COLOR;
@@ -35,10 +31,7 @@ function lerpColor(from: NormalizedColor, to: NormalizedColor, t: number): Norma
     ];
 }
 
-// Opacity multiplier for a line's number under the active highlight. The
-// number stays bright only when the WHOLE line is highlighted, so we take the
-// min token opacity on the line. Returns 1 (no dimming) when no highlight is
-// active, so an edit's entries don't drag the number down.
+// A number stays bright only when the whole line is highlighted, so it takes the line's minimum token opacity.
 function lineHighlightOpacity(line: IdLine, stateById: Map<number, TokenState>, highlightActive: boolean): number {
     if (!highlightActive) return 1;
     let min = 1;
@@ -50,36 +43,27 @@ function lineHighlightOpacity(line: IdLine, stateById: Map<number, TokenState>,
     return min;
 }
 
-/** Everything a `Code` node's own instance state supplies to draw one frame. */
+/** What a `Code` node supplies to draw one frame. */
 export interface DrawCodeState {
     tokenLines: IdLine[];
     metrics: CodeMetrics;
     advanceCache: TokenAdvanceCache;
-    /** The one or two layouts this frame draws from — see `frameLayout` in `layout.ts`. */
     from: CodeLayout;
     to: CodeLayout;
     edit: StructuralTransition | null;
-    /** Per-token opacity from `resolveTokenStates` (persistent highlight + any active dim transition). */
     tokenStates: Map<number, TokenState>;
-    /** Whether a highlight (persistent dim, or a highlight()/resetHighlight() cross-fade) is currently engaged. */
+    /** A persistent dim or a highlight cross-fade is engaged. */
     highlightActive: boolean;
-    /** The settled row bands, and the change between two sets of them in flight, if any. */
     rowBands?: RowBand[];
     rowsEdit?: RowsTransition | null;
-    /** The node's laid-out width, which a band spans when it is wider than the block. */
+    /** A band spans this when it is wider than the block. */
     boxW?: number;
 }
 
-/** What a frame is drawn onto: somewhere to draw, and the face's measurements. A render context is one. */
+/** Somewhere to draw and the face's measurements; a render context is one. */
 export type CodeCanvas = Pick<RenderContext2D, "draw" | "measureText">;
 
-/**
- * Draw one frame of a code listing: the settled structure, or — mid
- * structural edit — the two structures it is interpolating between (what
- * stays reflows, what is leaving fades out where it was, what is arriving
- * fades/slides in where it lands). Pure over `state`; touches nothing but
- * `draw`.
- */
+/** Draws the settled structure, or mid-edit the two it interpolates between. Pure over `state`. */
 export function drawCode(draw: CodeCanvas, state: DrawCodeState): void {
     const { tokenLines, metrics, advanceCache, from, to, edit, tokenStates: dim } = state;
 
@@ -91,8 +75,6 @@ export function drawCode(draw: CodeCanvas, state: DrawCodeState): void {
     const blockW = edit ? lerpNumber(from.blockW, to.blockW, pMove) : to.blockW;
     const gutter = edit ? lerpNumber(from.gutter, to.gutter, pMove) : to.gutter;
     const gutterGap = to.gutterGap || from.gutterGap;
-    // Right edge of the line-number column: one gap to the left of where the
-    // code text begins.
     const numberRight = -blockW / 2 + metrics.padding.left + gutter - gutterGap;
 
     drawRowBands(draw, state, Math.max(blockW, state.boxW ?? 0), pMove);
@@ -136,8 +118,7 @@ export function drawCode(draw: CodeCanvas, state: DrawCodeState): void {
             .fill([{ type: "solid", color, opacity }]));
     };
 
-    // What survived the edit, and what it brought with it — drawn from the
-    // structure the edit lands on.
+    // Survivors and arrivals, from the structure the edit lands on.
     for (let i = 0; i < tokenLines.length; i++) {
         const line = tokenLines[i];
         const toIndex = to.lineIndex.get(line.id);
@@ -148,10 +129,7 @@ export function drawCode(draw: CodeCanvas, state: DrawCodeState): void {
         let rowY = toY;
         let rowAlpha = 1;
         if (edit && entering) {
-            // A wholly new row fades as one piece, in the place it lands — the
-            // line arrives, it does not travel to get there, and it is not a
-            // spray of glyphs each finding its own way in. See
-            // `StructuralTransition.enteringLineIds`.
+            // A new row fades as one piece where it lands (see `enteringLineIds`).
             rowAlpha = pIn;
         } else if (edit) {
             const fromIndex = from.lineIndex.get(line.id);
@@ -168,8 +146,7 @@ export function drawCode(draw: CodeCanvas, state: DrawCodeState): void {
 
             if (edit && !entering) {
                 if (edit.addedIds.has(token.id)) {
-                    // Spliced into a row that already existed: the reflow
-                    // already opened the space, so this only has to appear.
+                    // Spliced into an existing row: the reflow opened the space already.
                     alpha = pIn;
                 } else {
                     const was = from.tokens.get(token.id);
@@ -184,17 +161,14 @@ export function drawCode(draw: CodeCanvas, state: DrawCodeState): void {
         }
     }
 
-    // What the edit takes away, drawn from the structure it is leaving — it
-    // has no place in the new one, which is the whole reason it is fading.
+    // What the edit takes away, from the structure it is leaving.
     if (edit && pOut < 1) {
         for (let i = 0; i < edit.from.length; i++) {
             const line = edit.from[i];
             const fromIndex = from.lineIndex.get(line.id);
             if (fromIndex === undefined) continue;
             const toIndex = to.lineIndex.get(line.id);
-            // A row that survived the edit is still moving, so its outgoing
-            // tokens travel with it; a deleted row stays where it was and
-            // dissolves while the rows below close over it.
+            // A surviving row's outgoing tokens travel with it; a deleted row dissolves in place.
             const y = toIndex === undefined
                 ? from.lineY[fromIndex]
                 : lerpNumber(from.lineY[fromIndex], to.lineY[toIndex], pMove);
@@ -228,11 +202,7 @@ function bandEdges(band: RowBand, state: DrawCodeState, pMove: number): { top: n
     return { top: (first ?? last as number) + half, bottom: (last ?? first as number) - half };
 }
 
-/**
- * Draw the row bands, under the text and across the whole block. A band rides
- * its rows through a structural edit; between two sets, bands paired by position
- * slide and recolour while an unpaired one fades.
- */
+/** Bands ride their rows through an edit; between two sets, those paired by position slide and recolour and an unpaired one fades. */
 function drawRowBands(draw: CodeCanvas, state: DrawCodeState, width: number, pMove: number): void {
     const { rowBands = [], rowsEdit = null } = state;
     const paint = (top: number, bottom: number, color: NormalizedColor, radius: number, opacity: number): void => {
@@ -270,22 +240,9 @@ function drawRowBands(draw: CodeCanvas, state: DrawCodeState, width: number, pMo
 }
 
 /**
- * Draw the gutter — one number per *slot*, not one per row.
- *
- * That distinction is the whole of it. A line number is an ordinal of the
- * block: "3" means the third line, and it is there for exactly as long as the
- * listing has a third line. Drawing it alongside the row it labels — the
- * obvious thing, since that is where it appears — tied it to the row's identity
- * instead, so a line rewritten past the point the diff could pair it took its
- * number down with it and brought an identical one back up. The number blinked
- * although it had never gone away, and every substantially rewritten line did
- * it at once.
- *
- * Numbering the slots means only the numbers at the end move at all: those the
- * block grew to reach fade in, those it shrank past fade out, and every number
- * both structures have simply travels to its new row height. Which is also the
- * honest reading of the edit — inserting a line in the middle does not make a
- * new number appear there, it makes the block one line longer.
+ * One number per slot, not per row: "3" lasts as long as the listing has a third
+ * line. Tied to a row's identity, a line rewritten past what the diff could pair
+ * would take its number down and bring an identical one back up (a blink).
  */
 function drawLineNumbers(
     state: DrawCodeState,
@@ -302,8 +259,7 @@ function drawLineNumbers(
     for (let i = 0; i < Math.max(fromCount, toCount); i++) {
         const inFrom = i < fromCount;
         const inTo = i < toCount;
-        // Highlight dimming follows whatever content occupies the slot, which
-        // is the new listing wherever there is one.
+        // Dimming follows the content in the slot: the new listing where there is one.
         const line = inTo ? tokenLines[i] : edit?.from[i];
         const dimming = line ? lineHighlightOpacity(line, dim, highlightActive) : 1;
 

package/src/style.ts

@@ -1,40 +1,22 @@
 import { Highlighter, Tag, tagHighlighter, tags as t } from "@lezer/highlight";
 
-/**
- * A single style rule: map one or more Lezer highlight {@link Tag}s to a color.
- * This mirrors a `@codemirror/language` HighlightStyle entry, but instead of
- * emitting CSS class names it carries the color directly — we render to a canvas,
- * not the DOM, so there is no stylesheet to resolve against.
- */
+/** Lezer highlight {@link Tag}s mapped to a color; a canvas has no stylesheet, so the color is carried directly. */
 export interface CodeStyleRule {
     tag: Tag | readonly Tag[];
     color: string;
 }
 
-/**
- * A code theme: an ordered list of tag→color rules plus a fallback color for any
- * text the parser produces no highlight tag for (operators, punctuation, plain
- * identifiers in some grammars, etc.). Pass a custom one via the Code node's
- * `theme` prop to recolor highlighting without touching the highlighter wiring.
- *
- * @remarks
- * Later rules win when several match the same tag, matching how
- * `@lezer/highlight`'s {@link tagHighlighter} resolves precedence.
- */
+/** Ordered tag rules plus a fallback color; later rules win when several match a tag, as in {@link tagHighlighter}. */
 export interface CodeHighlightStyle {
-    /** Stable name, used to register the style and to refer to it from a `theme` prop. */
+    /** Registers the style and names it in a `theme` prop. */
     name: string;
-    /** Color applied to text with no matching tag. */
+    /** For text with no matching tag. */
     defaultColor: string;
     rules: CodeStyleRule[];
 }
 
-/**
- * Motion Canvas's default highlight palette (a Nord-derived theme), ported from
- * `@motion-canvas/2d`'s DefaultHighlightStyle so highlighting matches Motion
- * Canvas token-for-token. Colors and tag groupings are kept identical; only the
- * shape changes (color strings in place of CSS rules).
- */
+// The default palette is ported from `@motion-canvas/2d`; the GitHub and VS Code ones are adapted from
+// `@uiw/codemirror-theme-github` and `-vscode`, with their DOM-only style props dropped.
 export const DefaultHighlightStyle: CodeHighlightStyle = {
     name: "motion-canvas-default",
     defaultColor: "#d8dee9",
@@ -77,12 +59,6 @@ export const DefaultHighlightStyle: CodeHighlightStyle = {
     ],
 };
 
-/**
- * GitHub Dark highlight palette, adapted from `@uiw/codemirror-theme-github`.
- * `defaultColor` is GitHub's editor foreground (`#c9d1d9`), used for any text the
- * parser tags don't cover. DOM-only style props (backgroundColor, textDecoration,
- * fontWeight, fontStyle) are dropped — we only color tokens on the canvas.
- */
 export const GithubDarkStyle: CodeHighlightStyle = {
     name: "github-dark",
     defaultColor: "#c9d1d9",
@@ -102,11 +78,6 @@ export const GithubDarkStyle: CodeHighlightStyle = {
     ],
 };
 
-/**
- * GitHub Light highlight palette, adapted from `@uiw/codemirror-theme-github`.
- * `defaultColor` is GitHub's editor foreground (`#24292e`). See {@link GithubDarkStyle}
- * for the note on dropped DOM-only style props.
- */
 export const GithubLightStyle: CodeHighlightStyle = {
     name: "github-light",
     defaultColor: "#24292e",
@@ -127,11 +98,6 @@ export const GithubLightStyle: CodeHighlightStyle = {
     ],
 };
 
-/**
- * VS Code Dark+ highlight palette, adapted from the `@uiw/codemirror-theme-vscode`
- * snippet. `defaultColor` is VS Code's editor foreground (`#9cdcfe`). DOM-only
- * style props are dropped; later rules win, matching {@link tagHighlighter}.
- */
 export const VscodeDarkStyle: CodeHighlightStyle = {
     name: "vscode-dark",
     defaultColor: "#9cdcfe",
@@ -166,11 +132,6 @@ export const VscodeDarkStyle: CodeHighlightStyle = {
     ],
 };
 
-/**
- * VS Code Light+ highlight palette, adapted from the `@uiw/codemirror-theme-vscode`
- * snippet. `defaultColor` is VS Code's editor foreground (`#383a42`). See
- * {@link VscodeDarkStyle} for the note on dropped DOM-only style props.
- */
 export const VscodeLightStyle: CodeHighlightStyle = {
     name: "vscode-light",
     defaultColor: "#383a42",
@@ -205,11 +166,6 @@ export const VscodeLightStyle: CodeHighlightStyle = {
     ],
 };
 
-/**
- * The built-in themes, keyed by their string name. {@link CodeThemeName} is the
- * union of these keys — pass one of those strings (or any {@link CodeHighlightStyle}
- * object) to a Code node's `theme` prop.
- */
 export const BUILTIN_THEMES = {
     // Literal keys (not computed from `.name`) so `keyof` yields a string-literal
     // union for CodeThemeName; these must stay in sync with each style's `name`.
@@ -223,27 +179,16 @@ export const BUILTIN_THEMES = {
 /** Names of the built-in themes, e.g. `'github-dark' | 'github-light'`. */
 export type CodeThemeName = keyof typeof BUILTIN_THEMES;
 
-/**
- * What a Code node's `theme` prop accepts: a built-in theme name (with
- * autocomplete), any other registered theme name, or a {@link CodeHighlightStyle}
- * object passed inline. The `string & {}` keeps the literal-name suggestions while
- * still allowing arbitrary registered names.
- */
+/** A built-in name, any registered name, or a style inline; `string & {}` keeps the autocomplete for the built-in names. */
 export type CodeTheme = CodeThemeName | (string & {}) | CodeHighlightStyle;
 
-// Cache one compiled Highlighter per style *object*. Compiling builds a tag-rule
-// matcher; the parsed result is pure data so it's safe to share across Code nodes.
-// Keyed by the object rather than its `name`: two styles may share a name (an
+// One compiled Highlighter per style *object*, keyed by the object rather than its `name`: two styles may share a name (an
 // inline theme, or one re-registered under the same name with new colours), and
 // the second must not colour with the first one's rules. A WeakMap, so a style
 // nothing refers to any more takes its matcher with it.
 const compiledByStyle = new WeakMap<CodeHighlightStyle, Highlighter>();
 
-/**
- * Compile a {@link CodeHighlightStyle} into a `@lezer/highlight` {@link Highlighter}
- * whose "class" output is the literal color string for each tag. Cached per style
- * object so repeated lookups (every tokenize) don't rebuild the matcher.
- */
+/** A {@link Highlighter} whose "class" output is the color string for each tag. */
 export function compileStyle(style: CodeHighlightStyle): Highlighter {
     const cached = compiledByStyle.get(style);
     if (cached) return cached;

package/src/tokens.ts

@@ -1,26 +1,18 @@
 import { canHighlight, highlightToTokens, normalizeSource, resolveTheme } from "./highlight";
 import { CodeTheme } from "./style";
 
-// IdToken is our internal wrapper around a highlighter token that gives every
-// token a stable identity across line insertions/removals. All animation state
-// is keyed by `id`, so concurrent edits at different positions don't clobber
-// each other while one is mid-animation.
+/** A highlighter token with a stable identity: animation state is keyed by `id`, so concurrent edits at different positions don't clobber each other. */
 export interface IdToken {
     id: number;
     content: string;
     color?: string;
 }
 
-// IdLine wraps a row of tokens with its own stable id, so per-line animations
-// (heightScale during insert/remove) survive line insertions/removals.
 export interface IdLine {
     id: number;
     tokens: IdToken[];
 }
 
-// Module-global, monotonic id counters shared across all Code nodes — ids only
-// need to be unique within a node, but a global counter is the simplest way to
-// guarantee that and never collides.
 let nextTokenId = 1;
 let nextLineId = 1;
 
@@ -33,47 +25,13 @@ export function makeIdLine(tokens: IdToken[]): IdLine {
 }
 
 export function tokenizeCodeToIdLines(code: string, language: string, theme: CodeTheme): IdLine[] {
-    // Until the language parser has loaded (it streams in via the asset loader),
-    // fall back to flat, uncolored lines instead of throwing on an unloaded
-    // grammar. The node re-tokenizes once the load completes.
     code = normalizeSource(code);
-    if (!canHighlight(language, theme)) {
-        // Resolved for its warning: an unknown theme is reported whether or not
-        // the grammar has landed yet.
-        resolveTheme(theme);
+    if (!canHighlight(language)) {
+        // Flat lines until the grammar loads; the node re-tokenizes then.
+        resolveTheme(theme); // so an unknown theme still warns before the grammar lands
         return code.split('\n').map(line => makeIdLine([makeIdToken(line, '#d1d5db')]));
     }
     return highlightToTokens(code, language, theme).map(line =>
         makeIdLine(line.map(tok => makeIdToken(tok.content, tok.color))),
     );
 }
-
-/**
- * Split `tokens` at character offset `col` (relative to the joined content of
- * the tokens). If `col` falls inside a token, that token is split into two new
- * tokens with fresh ids (the original id is discarded — it didn't exist as a
- * single unit on either side of the cut).
- */
-export function splitTokensAt(
-    tokens: IdToken[],
-    col: number,
-): { before: IdToken[]; after: IdToken[] } {
-    const before: IdToken[] = [];
-    const after: IdToken[] = [];
-    let off = 0;
-    for (const tok of tokens) {
-        const tStart = off;
-        const tEnd = off + tok.content.length;
-        if (tEnd <= col) {
-            before.push(tok);
-        } else if (tStart >= col) {
-            after.push(tok);
-        } else {
-            const cut = col - tStart;
-            before.push(makeIdToken(tok.content.slice(0, cut), tok.color));
-            after.push(makeIdToken(tok.content.slice(cut), tok.color));
-        }
-        off = tEnd;
-    }
-    return { before, after };
-}