@motionscript/code 0.0.0-stage → 0.1.0-alpha.0

package/src/nodes.ts

@@ -0,0 +1,18 @@
+import { Code } from "./node";
+
+/**
+ * Every node type this package publishes.
+ *
+ * A host registers these by handing the array to an engine —
+ * `new Engine(platform, { nodes: NODES })` — which reads each class's `@node()`
+ * key. The decorator only *declares* that key: nothing is registered by the mere
+ * act of importing a module, so a registry holds exactly what its owner asked
+ * for and two engines in one process can differ about it.
+ *
+ * Listing class **values** is also what survives bundling. This package declares
+ * `sideEffects: false`, which licenses a bundler to drop a module nothing
+ * imports a value from, and a document names a node type by string — so
+ * `import "./node"` for its side effect is exactly what gets shaken out, where an
+ * array of bindings is a data dependency that cannot be.
+ */
+export const NODES = [Code];

package/src/render.ts

@@ -0,0 +1,247 @@
+import { RenderContext2D, Graphics2D, NormalizedColor, parseColor, lerpNumber } from "@motionscript/core";
+import type { CodeLayout, CodeMetrics } from "./layout";
+import type { IdLine } from "./tokens";
+import { windowProgress, moveProgress } from "./transitions";
+import type { TokenState, StructuralTransition } from "./transitions";
+import type { TokenAdvanceCache } from "./measure-cache";
+
+/** Colour of a token the grammar had no opinion about. */
+const DEFAULT_TOKEN_COLOR: NormalizedColor = [0.82, 0.84, 0.86, 1];
+/** Gutter line-number colour: a muted version of the standard text colour. */
+const LINE_NUMBER_COLOR: NormalizedColor = [0.45, 0.5, 0.55, 1];
+
+// `parseColor` walks a CSS string; a listing asks for the same handful of theme
+// colours on every token of every frame, so the parse is memoized. Module-level,
+// because the answer depends only on the string.
+const colorCache = new Map<string, NormalizedColor>();
+function tokenColor(css: string | undefined): NormalizedColor {
+    if (!css) return DEFAULT_TOKEN_COLOR;
+    let hit = colorCache.get(css);
+    if (!hit) {
+        hit = parseColor(css);
+        colorCache.set(css, hit);
+    }
+    return hit;
+}
+
+function lerpColor(from: NormalizedColor, to: NormalizedColor, t: number): NormalizedColor {
+    if (t <= 0) return from;
+    if (t >= 1) return to;
+    return [
+        from[0] + (to[0] - from[0]) * t,
+        from[1] + (to[1] - from[1]) * t,
+        from[2] + (to[2] - from[2]) * t,
+        from[3] + (to[3] - from[3]) * t,
+    ];
+}
+
+// Opacity multiplier for a line's number under the active highlight. The
+// number stays bright only when the WHOLE line is highlighted, so we take the
+// min token opacity on the line. Returns 1 (no dimming) when no highlight is
+// active, so an edit's entries don't drag the number down.
+function lineHighlightOpacity(line: IdLine, stateById: Map<number, TokenState>, highlightActive: boolean): number {
+    if (!highlightActive) return 1;
+    let min = 1;
+    for (const tok of line.tokens) {
+        if (tok.content.length === 0) continue;
+        const op = stateById.get(tok.id)?.opacity ?? 1;
+        if (op < min) min = op;
+    }
+    return min;
+}
+
+/** Everything a `Code` node's own instance state supplies to draw one frame. */
+export interface DrawCodeState {
+    tokenLines: IdLine[];
+    metrics: CodeMetrics;
+    advanceCache: TokenAdvanceCache;
+    /** The one or two layouts this frame draws from — see `frameLayout` in `layout.ts`. */
+    from: CodeLayout;
+    to: CodeLayout;
+    edit: StructuralTransition | null;
+    /** Per-token opacity from `resolveTokenStates` (persistent highlight + any active dim transition). */
+    tokenStates: Map<number, TokenState>;
+    /** Whether a highlight (persistent dim, or a highlight()/resetHighlight() cross-fade) is currently engaged. */
+    highlightActive: boolean;
+}
+
+/**
+ * Draw one frame of a code listing: the settled structure, or — mid
+ * structural edit — the two structures it is interpolating between (what
+ * stays reflows, what is leaving fades out where it was, what is arriving
+ * fades/slides in where it lands). Pure over `state`; touches nothing but
+ * `draw`.
+ */
+export function drawCode(draw: RenderContext2D, state: DrawCodeState): void {
+    const { tokenLines, metrics, advanceCache, from, to, edit, tokenStates: dim } = state;
+
+    const progress = edit ? edit.progress : 1;
+    const pOut = edit ? windowProgress(edit.phases.out, progress) : 1;
+    const pMove = moveProgress(edit);
+    const pIn = edit ? windowProgress(edit.phases.in, progress) : 1;
+
+    const blockW = edit ? lerpNumber(from.blockW, to.blockW, pMove) : to.blockW;
+    const gutter = edit ? lerpNumber(from.gutter, to.gutter, pMove) : to.gutter;
+    const gutterGap = to.gutterGap || from.gutterGap;
+    // Right edge of the line-number column: one gap to the left of where the
+    // code text begins.
+    const numberRight = -blockW / 2 + metrics.padding.left + gutter - gutterGap;
+
+    const drawNumber = (label: string, y: number, opacity: number): void => {
+        if (opacity <= 0) return;
+        const labelW = advanceCache.advance(draw, label, metrics.fontSize, metrics.fontFamily, 0);
+        draw.draw(new Graphics2D()
+            .text({
+                text: label,
+                fontSize: metrics.fontSize,
+                fontFamily: metrics.fontFamily,
+                lineHeight: metrics.lineHeight,
+                x: numberRight - labelW / 2,
+                y,
+                textAlign: 'left',
+            })
+            .fill([{ type: "solid", color: LINE_NUMBER_COLOR, opacity }]));
+    };
+
+    const drawToken = (text: string, x: number, y: number, color: NormalizedColor, opacity: number): void => {
+        if (opacity <= 0 || text.length === 0) return;
+        const width = advanceCache.advance(draw, text, metrics.fontSize, metrics.fontFamily, metrics.letterSpacing);
+        draw.draw(new Graphics2D()
+            .text({
+                text,
+                fontSize: metrics.fontSize,
+                fontFamily: metrics.fontFamily,
+                lineHeight: metrics.lineHeight,
+                letterSpacing: metrics.letterSpacing,
+                // The renderer centres a single-token block on the (x, y) it is
+                // given, so every token is anchored at the centre of its cell
+                // rather than at its left edge. Passing lineHeight makes the
+                // block's height deterministic (fontSize × lineHeight) rather
+                // than the font's natural metrics, so the vertical centre
+                // lands exactly on the slot centre.
+                x: x + width / 2,
+                y,
+                textAlign: 'left',
+            })
+            .fill([{ type: "solid", color, opacity }]));
+    };
+
+    // What survived the edit, and what it brought with it — drawn from the
+    // structure the edit lands on.
+    for (let i = 0; i < tokenLines.length; i++) {
+        const line = tokenLines[i];
+        const toIndex = to.lineIndex.get(line.id);
+        if (toIndex === undefined) continue;
+        const toY = to.lineY[toIndex];
+
+        const entering = edit?.enteringLineIds.has(line.id) ?? false;
+        let rowY = toY;
+        let rowAlpha = 1;
+        if (edit && entering) {
+            // A wholly new row fades as one piece, in the place it lands — the
+            // line arrives, it does not travel to get there, and it is not a
+            // spray of glyphs each finding its own way in. See
+            // `StructuralTransition.enteringLineIds`.
+            rowAlpha = pIn;
+        } else if (edit) {
+            const fromIndex = from.lineIndex.get(line.id);
+            if (fromIndex !== undefined) rowY = lerpNumber(from.lineY[fromIndex], toY, pMove);
+        }
+
+        for (const token of line.tokens) {
+            const box = to.tokens.get(token.id);
+            if (!box) continue;
+
+            let x = box.x;
+            let alpha = rowAlpha;
+            let color = tokenColor(token.color);
+
+            if (edit && !entering) {
+                if (edit.addedIds.has(token.id)) {
+                    // Spliced into a row that already existed: the reflow
+                    // already opened the space, so this only has to appear.
+                    alpha = pIn;
+                } else {
+                    const was = from.tokens.get(token.id);
+                    if (was) x = lerpNumber(was.x, box.x, pMove);
+                    if (edit.fromColorById.has(token.id)) {
+                        color = lerpColor(tokenColor(edit.fromColorById.get(token.id)), color, pMove);
+                    }
+                }
+            }
+
+            drawToken(token.content, x, rowY, color, alpha * (dim.get(token.id)?.opacity ?? 1));
+        }
+    }
+
+    // What the edit takes away, drawn from the structure it is leaving — it
+    // has no place in the new one, which is the whole reason it is fading.
+    if (edit && pOut < 1) {
+        for (let i = 0; i < edit.from.length; i++) {
+            const line = edit.from[i];
+            const fromIndex = from.lineIndex.get(line.id);
+            if (fromIndex === undefined) continue;
+            const toIndex = to.lineIndex.get(line.id);
+            // A row that survived the edit is still moving, so its outgoing
+            // tokens travel with it; a deleted row stays where it was and
+            // dissolves while the rows below close over it.
+            const y = toIndex === undefined
+                ? from.lineY[fromIndex]
+                : lerpNumber(from.lineY[fromIndex], to.lineY[toIndex], pMove);
+
+            for (const token of line.tokens) {
+                if (!edit.removedIds.has(token.id)) continue;
+                const box = from.tokens.get(token.id);
+                if (!box) continue;
+                drawToken(token.content, box.x, y, tokenColor(token.color), 1 - pOut);
+            }
+        }
+    }
+
+    if (metrics.showLineNumbers) drawLineNumbers(state, drawNumber, pMove, pIn, pOut);
+}
+
+/**
+ * Draw the gutter — one number per *slot*, not one per row.
+ *
+ * That distinction is the whole of it. A line number is an ordinal of the
+ * block: "3" means the third line, and it is there for exactly as long as the
+ * listing has a third line. Drawing it alongside the row it labels — the
+ * obvious thing, since that is where it appears — tied it to the row's identity
+ * instead, so a line rewritten past the point the diff could pair it took its
+ * number down with it and brought an identical one back up. The number blinked
+ * although it had never gone away, and every substantially rewritten line did
+ * it at once.
+ *
+ * Numbering the slots means only the numbers at the end move at all: those the
+ * block grew to reach fade in, those it shrank past fade out, and every number
+ * both structures have simply travels to its new row height. Which is also the
+ * honest reading of the edit — inserting a line in the middle does not make a
+ * new number appear there, it makes the block one line longer.
+ */
+function drawLineNumbers(
+    state: DrawCodeState,
+    drawNumber: (label: string, y: number, opacity: number) => void,
+    pMove: number,
+    pIn: number,
+    pOut: number,
+): void {
+    const { tokenLines, from, to, edit, tokenStates: dim, highlightActive } = state;
+
+    const toCount = tokenLines.length;
+    const fromCount = edit ? edit.from.length : toCount;
+
+    for (let i = 0; i < Math.max(fromCount, toCount); i++) {
+        const inFrom = i < fromCount;
+        const inTo = i < toCount;
+        // Highlight dimming follows whatever content occupies the slot, which
+        // is the new listing wherever there is one.
+        const line = inTo ? tokenLines[i] : edit?.from[i];
+        const dimming = line ? lineHighlightOpacity(line, dim, highlightActive) : 1;
+
+        if (!edit) drawNumber(String(i + 1), to.lineY[i], dimming);
+        else if (inFrom && inTo) drawNumber(String(i + 1), lerpNumber(from.lineY[i], to.lineY[i], pMove), dimming);
+        else if (inTo) drawNumber(String(i + 1), to.lineY[i], pIn * dimming);
+        else drawNumber(String(i + 1), from.lineY[i], (1 - pOut) * dimming);
+    }
+}

package/src/style.ts

@@ -0,0 +1,251 @@
+import { Highlighter, Tag, tagHighlighter, tags as t } from "@lezer/highlight";
+
+/**
+ * A single style rule: map one or more Lezer highlight {@link Tag}s to a color.
+ * This mirrors a `@codemirror/language` HighlightStyle entry, but instead of
+ * emitting CSS class names it carries the color directly — we render to a canvas,
+ * not the DOM, so there is no stylesheet to resolve against.
+ */
+export interface CodeStyleRule {
+    tag: Tag | readonly Tag[];
+    color: string;
+}
+
+/**
+ * A code theme: an ordered list of tag→color rules plus a fallback color for any
+ * text the parser produces no highlight tag for (operators, punctuation, plain
+ * identifiers in some grammars, etc.). Pass a custom one via the Code node's
+ * `theme` prop to recolor highlighting without touching the highlighter wiring.
+ *
+ * @remarks
+ * Later rules win when several match the same tag, matching how
+ * `@lezer/highlight`'s {@link tagHighlighter} resolves precedence.
+ */
+export interface CodeHighlightStyle {
+    /** Stable name, used as the highlighter cache key. */
+    name: string;
+    /** Color applied to text with no matching tag. */
+    defaultColor: string;
+    rules: CodeStyleRule[];
+}
+
+/**
+ * Motion Canvas's default highlight palette (a Nord-derived theme), ported from
+ * `@motion-canvas/2d`'s DefaultHighlightStyle so highlighting matches Motion
+ * Canvas token-for-token. Colors and tag groupings are kept identical; only the
+ * shape changes (color strings in place of CSS rules).
+ */
+export const DefaultHighlightStyle: CodeHighlightStyle = {
+    name: "motion-canvas-default",
+    defaultColor: "#d8dee9",
+    rules: [
+        { tag: t.keyword, color: "#5e81ac" },
+        { tag: [t.name, t.deleted, t.character, t.propertyName, t.macroName], color: "#88c0d0" },
+        { tag: [t.variableName], color: "#8fbcbb" },
+        { tag: [t.function(t.variableName)], color: "#8fbcbb" },
+        { tag: [t.labelName], color: "#81a1c1" },
+        { tag: [t.color, t.constant(t.name), t.standard(t.name)], color: "#5e81ac" },
+        { tag: [t.definition(t.name), t.separator], color: "#a3be8c" },
+        { tag: [t.brace], color: "#8fbcbb" },
+        { tag: [t.annotation], color: "#d30102" },
+        { tag: [t.number, t.changed, t.annotation, t.modifier, t.self, t.namespace], color: "#b48ead" },
+        { tag: [t.typeName, t.className], color: "#ECEFF4" },
+        { tag: [t.operator, t.operatorKeyword], color: "#a3be8c" },
+        { tag: [t.tagName], color: "#b48ead" },
+        { tag: [t.squareBracket], color: "#ECEFF4" },
+        { tag: [t.angleBracket], color: "#ECEFF4" },
+        { tag: [t.attributeName], color: "#eceff4" },
+        { tag: [t.regexp], color: "#5e81ac" },
+        { tag: [t.quote], color: "#b48ead" },
+        { tag: [t.string], color: "#a3be8c" },
+        { tag: t.link, color: "#a3be8c" },
+        { tag: [t.url, t.escape, t.special(t.string)], color: "#8fbcbb" },
+        { tag: [t.meta], color: "#88c0d0" },
+        { tag: [t.monospace], color: "#d8dee9" },
+        { tag: [t.comment], color: "#4c566a" },
+        { tag: t.strong, color: "#5e81ac" },
+        { tag: t.emphasis, color: "#5e81ac" },
+        { tag: t.heading, color: "#5e81ac" },
+        { tag: t.special(t.heading1), color: "#5e81ac" },
+        { tag: t.heading1, color: "#5e81ac" },
+        { tag: [t.heading2, t.heading3, t.heading4], color: "#5e81ac" },
+        { tag: [t.heading5, t.heading6], color: "#5e81ac" },
+        { tag: [t.atom, t.bool, t.special(t.variableName)], color: "#d08770" },
+        { tag: [t.processingInstruction, t.inserted], color: "#8fbcbb" },
+        { tag: [t.contentSeparator], color: "#ebcb8b" },
+        { tag: t.invalid, color: "#434c5e" },
+    ],
+};
+
+/**
+ * GitHub Dark highlight palette, adapted from `@uiw/codemirror-theme-github`.
+ * `defaultColor` is GitHub's editor foreground (`#c9d1d9`), used for any text the
+ * parser tags don't cover. DOM-only style props (backgroundColor, textDecoration,
+ * fontWeight, fontStyle) are dropped — we only color tokens on the canvas.
+ */
+export const GithubDarkStyle: CodeHighlightStyle = {
+    name: "github-dark",
+    defaultColor: "#c9d1d9",
+    rules: [
+        { tag: [t.standard(t.tagName), t.tagName], color: "#7ee787" },
+        { tag: [t.comment, t.bracket], color: "#8b949e" },
+        { tag: [t.className, t.propertyName], color: "#d2a8ff" },
+        { tag: [t.variableName, t.attributeName, t.number, t.operator], color: "#79c0ff" },
+        { tag: [t.keyword, t.typeName, t.typeOperator], color: "#ff7b72" },
+        { tag: [t.string, t.meta, t.regexp], color: "#a5d6ff" },
+        { tag: [t.name, t.quote], color: "#7ee787" },
+        { tag: [t.heading, t.strong], color: "#d2a8ff" },
+        { tag: [t.emphasis], color: "#d2a8ff" },
+        { tag: [t.deleted], color: "#ffdcd7" },
+        { tag: [t.atom, t.bool, t.special(t.variableName)], color: "#ffab70" },
+        { tag: t.invalid, color: "#f97583" },
+    ],
+};
+
+/**
+ * GitHub Light highlight palette, adapted from `@uiw/codemirror-theme-github`.
+ * `defaultColor` is GitHub's editor foreground (`#24292e`). See {@link GithubDarkStyle}
+ * for the note on dropped DOM-only style props.
+ */
+export const GithubLightStyle: CodeHighlightStyle = {
+    name: "github-light",
+    defaultColor: "#24292e",
+    rules: [
+        { tag: [t.standard(t.tagName), t.tagName], color: "#116329" },
+        { tag: [t.comment, t.bracket], color: "#6a737d" },
+        { tag: [t.className, t.propertyName], color: "#6f42c1" },
+        { tag: [t.variableName, t.attributeName, t.number, t.operator], color: "#005cc5" },
+        { tag: [t.keyword, t.typeName, t.typeOperator], color: "#d73a49" },
+        { tag: [t.string, t.meta, t.regexp], color: "#032f62" },
+        { tag: [t.name, t.quote], color: "#22863a" },
+        { tag: [t.heading, t.strong], color: "#24292e" },
+        { tag: [t.emphasis], color: "#24292e" },
+        { tag: [t.deleted], color: "#b31d28" },
+        { tag: [t.atom, t.bool, t.special(t.variableName)], color: "#e36209" },
+        { tag: [t.url, t.escape, t.regexp, t.link], color: "#032f62" },
+        { tag: t.invalid, color: "#cb2431" },
+    ],
+};
+
+/**
+ * VS Code Dark+ highlight palette, adapted from the `@uiw/codemirror-theme-vscode`
+ * snippet. `defaultColor` is VS Code's editor foreground (`#9cdcfe`). DOM-only
+ * style props are dropped; later rules win, matching {@link tagHighlighter}.
+ */
+export const VscodeDarkStyle: CodeHighlightStyle = {
+    name: "vscode-dark",
+    defaultColor: "#9cdcfe",
+    rules: [
+        {
+            tag: [
+                t.keyword, t.operatorKeyword, t.modifier, t.color,
+                t.constant(t.name), t.standard(t.name), t.standard(t.tagName),
+                t.special(t.brace), t.atom, t.bool, t.special(t.variableName),
+            ],
+            color: "#569cd6",
+        },
+        { tag: [t.controlKeyword, t.moduleKeyword], color: "#c586c0" },
+        {
+            tag: [
+                t.name, t.deleted, t.character, t.macroName, t.propertyName,
+                t.variableName, t.labelName, t.definition(t.name),
+            ],
+            color: "#9cdcfe",
+        },
+        { tag: t.heading, color: "#9cdcfe" },
+        { tag: [t.typeName, t.className, t.tagName, t.changed, t.annotation, t.self, t.namespace], color: "#4ec9b0" },
+        { tag: [t.function(t.variableName), t.function(t.propertyName)], color: "#dcdcaa" },
+        { tag: [t.number], color: "#b5cea8" },
+        { tag: [t.operator, t.punctuation, t.separator, t.url, t.escape], color: "#d4d4d4" },
+        { tag: [t.regexp], color: "#d16969" },
+        { tag: [t.special(t.string), t.processingInstruction, t.string, t.inserted], color: "#ce9178" },
+        { tag: [t.angleBracket], color: "#808080" },
+        { tag: [t.meta, t.comment], color: "#6a9955" },
+        { tag: t.link, color: "#6a9955" },
+        { tag: t.invalid, color: "#ff0000" },
+    ],
+};
+
+/**
+ * VS Code Light+ highlight palette, adapted from the `@uiw/codemirror-theme-vscode`
+ * snippet. `defaultColor` is VS Code's editor foreground (`#383a42`). See
+ * {@link VscodeDarkStyle} for the note on dropped DOM-only style props.
+ */
+export const VscodeLightStyle: CodeHighlightStyle = {
+    name: "vscode-light",
+    defaultColor: "#383a42",
+    rules: [
+        {
+            tag: [
+                t.keyword, t.operatorKeyword, t.modifier, t.color,
+                t.constant(t.name), t.standard(t.name), t.standard(t.tagName),
+                t.special(t.brace), t.atom, t.bool, t.special(t.variableName),
+            ],
+            color: "#0000ff",
+        },
+        { tag: [t.moduleKeyword, t.controlKeyword], color: "#af00db" },
+        {
+            tag: [
+                t.name, t.deleted, t.character, t.macroName, t.propertyName,
+                t.variableName, t.labelName, t.definition(t.name),
+            ],
+            color: "#0070c1",
+        },
+        { tag: t.heading, color: "#0070c1" },
+        { tag: [t.typeName, t.className, t.tagName, t.changed, t.annotation, t.self, t.namespace], color: "#267f99" },
+        { tag: [t.function(t.variableName), t.function(t.propertyName)], color: "#795e26" },
+        { tag: [t.number], color: "#098658" },
+        { tag: [t.operator, t.punctuation, t.separator, t.url, t.escape], color: "#383a42" },
+        { tag: [t.regexp], color: "#af00db" },
+        { tag: [t.special(t.string), t.processingInstruction, t.string, t.inserted], color: "#a31515" },
+        { tag: [t.angleBracket], color: "#383a42" },
+        { tag: [t.meta, t.comment], color: "#008000" },
+        { tag: t.link, color: "#4078f2" },
+        { tag: t.invalid, color: "#e45649" },
+    ],
+};
+
+/**
+ * The built-in themes, keyed by their string name. {@link CodeThemeName} is the
+ * union of these keys — pass one of those strings (or any {@link CodeHighlightStyle}
+ * object) to a Code node's `theme` prop.
+ */
+export const BUILTIN_THEMES = {
+    // Literal keys (not computed from `.name`) so `keyof` yields a string-literal
+    // union for CodeThemeName; these must stay in sync with each style's `name`.
+    "motion-canvas-default": DefaultHighlightStyle,
+    "github-dark": GithubDarkStyle,
+    "github-light": GithubLightStyle,
+    "vscode-dark": VscodeDarkStyle,
+    "vscode-light": VscodeLightStyle,
+} satisfies Record<string, CodeHighlightStyle>;
+
+/** Names of the built-in themes, e.g. `'github-dark' | 'github-light'`. */
+export type CodeThemeName = keyof typeof BUILTIN_THEMES;
+
+/**
+ * What a Code node's `theme` prop accepts: a built-in theme name (with
+ * autocomplete), any other registered theme name, or a {@link CodeHighlightStyle}
+ * object passed inline. The `string & {}` keeps the literal-name suggestions while
+ * still allowing arbitrary registered names.
+ */
+export type CodeTheme = CodeThemeName | (string & {}) | CodeHighlightStyle;
+
+// Cache one compiled Highlighter per style name. Compiling builds a tag-rule
+// matcher; the parsed result is pure data so it's safe to share across Code nodes.
+const compiledByName = new Map<string, Highlighter>();
+
+/**
+ * Compile a {@link CodeHighlightStyle} into a `@lezer/highlight` {@link Highlighter}
+ * whose "class" output is the literal color string for each tag. Cached by style
+ * name so repeated lookups (every tokenize) don't rebuild the matcher.
+ */
+export function compileStyle(style: CodeHighlightStyle): Highlighter {
+    const cached = compiledByName.get(style.name);
+    if (cached) return cached;
+    const highlighter = tagHighlighter(
+        style.rules.map(rule => ({ tag: rule.tag, class: rule.color })),
+    );
+    compiledByName.set(style.name, highlighter);
+    return highlighter;
+}

package/src/tokens.ts

@@ -0,0 +1,75 @@
+import { canHighlight, highlightToTokens } from "./highlight";
+import { CodeTheme } from "./style";
+
+// IdToken is our internal wrapper around a highlighter token that gives every
+// token a stable identity across line insertions/removals. All animation state
+// is keyed by `id`, so concurrent edits at different positions don't clobber
+// each other while one is mid-animation.
+export interface IdToken {
+    id: number;
+    content: string;
+    color?: string;
+}
+
+// IdLine wraps a row of tokens with its own stable id, so per-line animations
+// (heightScale during insert/remove) survive line insertions/removals.
+export interface IdLine {
+    id: number;
+    tokens: IdToken[];
+}
+
+// Module-global, monotonic id counters shared across all Code nodes — ids only
+// need to be unique within a node, but a global counter is the simplest way to
+// guarantee that and never collides.
+let nextTokenId = 1;
+let nextLineId = 1;
+
+export function makeIdToken(content: string, color?: string): IdToken {
+    return { id: nextTokenId++, content, color };
+}
+
+export function makeIdLine(tokens: IdToken[]): IdLine {
+    return { id: nextLineId++, tokens };
+}
+
+export function tokenizeCodeToIdLines(code: string, language: string, theme: CodeTheme): IdLine[] {
+    // Until the language parser has loaded (it streams in via the asset loader),
+    // fall back to flat, uncolored lines instead of throwing on an unloaded
+    // grammar. The node re-tokenizes once the load completes.
+    if (!canHighlight(language, theme)) {
+        return code.split('\n').map(line => makeIdLine([makeIdToken(line, '#d1d5db')]));
+    }
+    return highlightToTokens(code, language, theme).map(line =>
+        makeIdLine(line.map(tok => makeIdToken(tok.content, tok.color))),
+    );
+}
+
+/**
+ * Split `tokens` at character offset `col` (relative to the joined content of
+ * the tokens). If `col` falls inside a token, that token is split into two new
+ * tokens with fresh ids (the original id is discarded — it didn't exist as a
+ * single unit on either side of the cut).
+ */
+export function splitTokensAt(
+    tokens: IdToken[],
+    col: number,
+): { before: IdToken[]; after: IdToken[] } {
+    const before: IdToken[] = [];
+    const after: IdToken[] = [];
+    let off = 0;
+    for (const tok of tokens) {
+        const tStart = off;
+        const tEnd = off + tok.content.length;
+        if (tEnd <= col) {
+            before.push(tok);
+        } else if (tStart >= col) {
+            after.push(tok);
+        } else {
+            const cut = col - tStart;
+            before.push(makeIdToken(tok.content.slice(0, cut), tok.color));
+            after.push(makeIdToken(tok.content.slice(cut), tok.color));
+        }
+        off = tEnd;
+    }
+    return { before, after };
+}