@motionscript/code 0.0.0-stage → 0.1.0-alpha.0

package/src/highlight.ts

@@ -0,0 +1,291 @@
+import { Parser, Tree } from "@lezer/common";
+import { highlightTree } from "@lezer/highlight";
+import {
+    BUILTIN_THEMES,
+    CodeHighlightStyle,
+    CodeTheme,
+    DefaultHighlightStyle,
+    compileStyle,
+} from "./style";
+
+/**
+ * Lezer-backed syntax highlighting for the Code component, replacing the former
+ * Shiki wiring. This mirrors Motion Canvas's `LezerHighlighter`:
+ *
+ *  - {@link prepareHighlight} parses code once and builds a `from:to` → color map
+ *    by walking {@link highlightTree}.
+ *  - {@link tokenizeCode} splits code at parse-tree leaf boundaries.
+ *  - {@link highlightAt} resolves the color (and run length) for a character index.
+ *
+ * Parsers load lazily and asynchronously (dynamic import) so the bundle doesn't
+ * pull in every grammar. The node's asset loader awaits {@link ensureHighlighter}
+ * before tokenizing; until a parser resolves, callers fall back to plain text and
+ * re-tokenize once {@link canHighlight} returns true.
+ */
+
+/** Dynamic-import factories for each supported language → a Lezer {@link Parser}. */
+const LANGUAGE_LOADERS: Record<string, () => Promise<Parser>> = {
+    typescript: async () => (await import("@lezer/javascript")).parser.configure({ dialect: "ts" }),
+    tsx: async () => (await import("@lezer/javascript")).parser.configure({ dialect: "ts jsx" }),
+    javascript: async () => (await import("@lezer/javascript")).parser,
+    jsx: async () => (await import("@lezer/javascript")).parser.configure({ dialect: "jsx" }),
+    python: async () => (await import("@lezer/python")).parser,
+    json: async () => (await import("@lezer/json")).parser,
+    html: async () => (await import("@lezer/html")).parser,
+    css: async () => (await import("@lezer/css")).parser,
+    rust: async () => (await import("@lezer/rust")).parser,
+    go: async () => (await import("@lezer/go")).parser,
+    java: async () => (await import("@lezer/java")).parser,
+    cpp: async () => (await import("@lezer/cpp")).parser,
+    php: async () => (await import("@lezer/php")).parser,
+    markdown: async () => (await import("@lezer/markdown")).parser,
+    xml: async () => (await import("@lezer/xml")).parser,
+    yaml: async () => (await import("@lezer/yaml")).parser,
+};
+
+// Aliases so common language strings resolve to a loader above.
+const LANGUAGE_ALIASES: Record<string, string> = {
+    ts: "typescript",
+    js: "javascript",
+    py: "python",
+    "c++": "cpp",
+    md: "markdown",
+    yml: "yaml",
+    shell: "javascript", // no shell grammar bundled; fall back rather than throw
+};
+
+function resolveLanguageKey(language: string): string {
+    const key = language.toLowerCase();
+    return LANGUAGE_ALIASES[key] ?? key;
+}
+
+// Loaded parsers and in-flight loads, keyed by resolved language key, so
+// concurrent Code nodes asking for the same language share one dynamic import.
+const loadedParsers = new Map<string, Parser>();
+const inFlightLangs = new Map<string, Promise<Parser | null>>();
+
+export const DEFAULT_THEMES = [DefaultHighlightStyle.name];
+export const DEFAULT_LANGS = ["typescript", "javascript", "json", "python"];
+
+// Theme registry, seeded with the built-ins (Motion Canvas default + GitHub
+// dark/light). The `theme` prop is a string name or a CodeHighlightStyle object;
+// apps can register additional styles by name and reference them by that name.
+const themesByName = new Map<string, CodeHighlightStyle>(Object.entries(BUILTIN_THEMES));
+
+/** Register a custom {@link CodeHighlightStyle} so Code nodes can reference it by name. */
+export function registerCodeTheme(style: CodeHighlightStyle): void {
+    themesByName.set(style.name, style);
+}
+
+/**
+ * Resolve a theme reference (a registered name, or a CodeHighlightStyle object)
+ * to a concrete style. Unknown names fall back to the Motion Canvas default so a
+ * stray theme string never throws.
+ */
+export function resolveTheme(theme: CodeTheme): CodeHighlightStyle {
+    if (typeof theme !== "string") return theme;
+    return themesByName.get(theme) ?? DefaultHighlightStyle;
+}
+
+/** Has `language` finished loading its parser? */
+function isLanguageLoaded(language: string): boolean {
+    return loadedParsers.has(resolveLanguageKey(language));
+}
+
+/**
+ * Load the parsers for `langs` (themes are synchronous, so they're a no-op here
+ * but kept in the signature for API parity with the former Shiki implementation).
+ * Resolves once every requested language's parser is available. Unknown languages
+ * resolve without loading anything — {@link canHighlight} stays false for them and
+ * the node renders them as plain text.
+ */
+export function ensureHighlighter(
+    _themes: string[] = DEFAULT_THEMES,
+    langs: string[] = DEFAULT_LANGS,
+): Promise<void> {
+    const jobs: Promise<unknown>[] = [];
+
+    for (const lang of langs) {
+        const key = resolveLanguageKey(lang);
+        if (loadedParsers.has(key)) continue;
+
+        const loader = LANGUAGE_LOADERS[key];
+        if (!loader) continue; // unsupported language; leave it unhighlighted
+
+        let job = inFlightLangs.get(key);
+        if (!job) {
+            job = loader()
+                .then(parser => {
+                    loadedParsers.set(key, parser);
+                    return parser;
+                })
+                .catch(() => null)
+                .finally(() => inFlightLangs.delete(key));
+            inFlightLangs.set(key, job);
+        }
+        jobs.push(job);
+    }
+
+    return jobs.length === 0 ? Promise.resolve() : Promise.all(jobs).then(() => undefined);
+}
+
+/**
+ * Load a syntax-highlight language into the shared highlighter. Exposed so a Code
+ * node's asset loader can run it on the timeline, and so apps can preload
+ * languages explicitly. Returns once the parser is ready (or immediately for an
+ * unsupported language). The `theme` arg is accepted for backwards compatibility.
+ */
+export async function loadCodeLanguage(language: string, _theme?: string): Promise<void> {
+    await ensureHighlighter(DEFAULT_THEMES, [language]);
+}
+
+export async function initSyntaxHighlighter(
+    themes: string[] = DEFAULT_THEMES,
+    langs: string[] = DEFAULT_LANGS,
+): Promise<void> {
+    await ensureHighlighter(themes, langs);
+}
+
+/** True once `language` can be tokenized (its parser has loaded). */
+export function canHighlight(language: string, _theme?: CodeTheme): boolean {
+    return isLanguageLoaded(language);
+}
+
+/** A parsed-and-prepared highlight cache for one snapshot of code. */
+interface HighlightCache {
+    tree: Tree;
+    code: string;
+    // `${from}:${to}` node id → color, exactly as Motion Canvas keys it.
+    colorLookup: Map<string, string>;
+}
+
+function nodeId(from: number, to: number): string {
+    return `${from}:${to}`;
+}
+
+/**
+ * Parse `code` and build the per-node color map. Mirrors Motion Canvas's
+ * `LezerHighlighter.prepare`: for each highlighted range, walk the subtree and
+ * record the color against every contained node's `from:to` id, so a later
+ * {@link highlightAt} lookup by `resolveInner` index finds it.
+ */
+function prepareHighlight(code: string, language: string, style: CodeHighlightStyle): HighlightCache | null {
+    const parser = loadedParsers.get(resolveLanguageKey(language));
+    if (!parser) return null;
+
+    const tree = parser.parse(code);
+    const highlighter = compileStyle(style);
+    const colorLookup = new Map<string, string>();
+
+    highlightTree(tree, highlighter, (from, to, classes) => {
+        // tagHighlighter emits our color string as the "class".
+        const color = classes;
+        if (!color) return;
+        const cursor = tree.cursorAt(from, 1);
+        do {
+            colorLookup.set(nodeId(cursor.from, cursor.to), color);
+        } while (cursor.next() && cursor.to <= to);
+    });
+
+    return { tree, code, colorLookup };
+}
+
+/**
+ * Resolve the color and run length for the token starting at `index`. Mirrors
+ * Motion Canvas's `LezerHighlighter.highlight`: resolve the innermost node, look
+ * up its color, and report how far the run extends (`skipAhead`).
+ */
+function highlightAt(index: number, cache: HighlightCache): { color: string | null; skipAhead: number } {
+    const node = cache.tree.resolveInner(index, 1);
+    const color = cache.colorLookup.get(nodeId(node.from, node.to));
+    if (color) {
+        return { color, skipAhead: node.to - index };
+    }
+    // No color: skip past a childless (leaf) node so we don't re-resolve every char.
+    const skipAhead = node.firstChild ? 0 : node.to - index;
+    return { color: null, skipAhead };
+}
+
+/**
+ * Split `code` into structural tokens at parse-tree leaf boundaries, including the
+ * gaps between leaves (whitespace/punctuation). Mirrors Motion Canvas's
+ * `LezerHighlighter.tokenize`; this segmentation is what the token-level
+ * append/insert/erase/replace animations diff against, so matching it keeps the
+ * animation behavior identical to Motion Canvas.
+ */
+function tokenizeCode(code: string, language: string): string[] {
+    const parser = loadedParsers.get(resolveLanguageKey(language));
+    if (!parser) return [code];
+
+    const tree = parser.parse(code);
+    const cursor = tree.cursor();
+    const tokens: string[] = [];
+    let current = 0;
+
+    do {
+        if (!cursor.node.firstChild) {
+            if (cursor.from > current) tokens.push(code.slice(current, cursor.from));
+            if (cursor.from < cursor.to) tokens.push(code.slice(cursor.from, cursor.to));
+            current = cursor.to;
+        }
+    } while (cursor.next());
+
+    if (current < code.length) tokens.push(code.slice(current));
+    return tokens;
+}
+
+/**
+ * Tokenize `code` into per-line, colored tokens — the contract the rest of the
+ * Code component consumes. Combines {@link tokenizeCode} (structural split) with
+ * {@link highlightAt} (per-run color), then breaks tokens at newlines so each
+ * output line is a row of single-line, single-color tokens.
+ *
+ * Callers must have checked {@link canHighlight} first; this returns a single
+ * uncolored line per source line if the parser isn't loaded.
+ */
+export function highlightToTokens(
+    code: string,
+    language: string,
+    theme: CodeTheme = DefaultHighlightStyle,
+): { content: string; color?: string }[][] {
+    const style = resolveTheme(theme);
+    const cache = prepareHighlight(code, language, style);
+
+    const lines: { content: string; color?: string }[][] = [[]];
+    const pushLine = () => lines.push([]);
+
+    if (!cache) {
+        // Parser unavailable: one uncolored token per source line.
+        return code.split("\n").map(line => [{ content: line, color: style.defaultColor }]);
+    }
+
+    // Walk the structural tokens, coloring each by its starting index and
+    // splitting on newlines so no token spans rows.
+    let index = 0;
+    for (const token of tokenizeCode(code, language)) {
+        let offset = 0;
+        while (offset < token.length) {
+            const { color, skipAhead } = highlightAt(index + offset, cache);
+            // Extend the run as far as the highlight says, but never past this
+            // structural token and never across a newline (lines are drawn rows).
+            let runEnd = offset + Math.max(skipAhead, 1);
+            if (runEnd > token.length) runEnd = token.length;
+            const newlineRel = token.indexOf("\n", offset);
+            if (newlineRel !== -1 && newlineRel < runEnd) runEnd = newlineRel;
+
+            const slice = token.slice(offset, runEnd);
+            if (slice.length > 0) {
+                lines[lines.length - 1].push({ content: slice, color: color ?? style.defaultColor });
+            }
+
+            if (newlineRel === runEnd) {
+                pushLine();
+                runEnd += 1; // consume the newline itself
+            }
+            offset = runEnd;
+        }
+        index += token.length;
+    }
+
+    return lines;
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export * from './node';
+export * from './engine';
+export { NODES } from "./nodes";

package/src/layout.ts

@@ -0,0 +1,157 @@
+import type { InsetsResolved, MeasureContext2D, RenderContext2D } from "@motionscript/core";
+import type { TokenAdvanceCache } from "./measure-cache";
+import type { IdLine } from "./tokens";
+
+/**
+ * Everything about a listing's geometry that is *not* its content — the type,
+ * the padding, whether there is a gutter. Split out from the node so a layout
+ * can be computed for a token structure the node is no longer holding: during a
+ * structural transition the frame is the interpolation of two of them, and the
+ * one that is going away is not reachable through `this` any more.
+ */
+export interface CodeMetrics {
+    fontSize: number;
+    fontFamily: string;
+    lineHeight: number;
+    letterSpacing: number;
+    padding: InsetsResolved;
+    showLineNumbers: boolean;
+    lineNumberGap: number;
+}
+
+/** Where one token sits: its LEFT edge, and the vertical CENTRE of its line's slot. */
+export interface TokenBox {
+    x: number;
+    y: number;
+}
+
+/**
+ * A resolved listing geometry: every token placed, in the node's own space
+ * (origin at the block's centre, y-up).
+ *
+ * Positions are absolute rather than "a running sum plus a width scale", which
+ * is the whole point. A token's place in the *next* structure is knowable
+ * without drawing it, so a transition can move a token from where it was to
+ * where it will be instead of faking the motion by collapsing the advance of
+ * its neighbours — which is what made an insert look like a pile of glyphs at
+ * the left margin springing apart.
+ */
+export interface CodeLayout {
+    /** Token id → placement. */
+    tokens: Map<number, TokenBox>;
+    /** Vertical centre of each line's slot, by line index. */
+    lineY: number[];
+    /**
+     * Width of each line's ink, by line index — the sum of its tokens' advances,
+     * so a blank line is `0` and a short line is short.
+     *
+     * Returned rather than kept local because it is the only thing that knows a
+     * listing is **ragged**: `innerW` is the longest line, and everything that
+     * asks where the text actually is (hit testing, for one) would otherwise have
+     * to treat the block as a rectangle the width of its longest row.
+     */
+    lineW: number[];
+    /** Line id → its index in this structure (i.e. its line *number* minus one). */
+    lineIndex: Map<number, number>;
+    gutter: number;
+    gutterGap: number;
+    /** Content size, no padding. */
+    innerW: number;
+    innerH: number;
+    /** Content size plus padding — what the block draws and measures as. */
+    blockW: number;
+    blockH: number;
+    /** Left edge of the code column (after padding and gutter). */
+    startX: number;
+}
+
+/**
+ * Signature of everything a layout depends on other than the token structure
+ * itself. A cached layout survives exactly as long as this is unchanged.
+ */
+export function metricsSignature(m: CodeMetrics): string {
+    const p = m.padding;
+    return [
+        m.fontSize, m.fontFamily, m.lineHeight, m.letterSpacing,
+        p.left, p.right, p.top, p.bottom,
+        m.showLineNumbers ? 1 : 0, m.lineNumberGap,
+    ].join("|");
+}
+
+/**
+ * Horizontal gap between the line-number column and the code text. Sized in
+ * space-widths so it scales with the font, and measured with letterSpacing 0 —
+ * line numbers and the gap don't carry the code's letter-spacing.
+ */
+function gutterGapOf(scope: MeasureContext2D | RenderContext2D, m: CodeMetrics, cache: TokenAdvanceCache): number {
+    return cache.advance(scope, " ", m.fontSize, m.fontFamily, 0) * m.lineNumberGap;
+}
+
+/** Width of the line-number column for a structure of `lineCount` lines, gap included. */
+function gutterOf(
+    scope: MeasureContext2D | RenderContext2D,
+    m: CodeMetrics,
+    cache: TokenAdvanceCache,
+    lineCount: number,
+    gap: number,
+): number {
+    if (!m.showLineNumbers) return 0;
+    const sample = String(Math.max(1, lineCount));
+    return cache.advance(scope, sample, m.fontSize, m.fontFamily, 0) + gap;
+}
+
+/**
+ * Place every token of `lines`.
+ *
+ * Pure in (lines, metrics, font measurements), which is what lets a caller cache
+ * one per structure rather than recomputing it per frame — a transition's two
+ * endpoints are both fixed for its whole duration.
+ */
+export function layoutCode(
+    lines: IdLine[],
+    m: CodeMetrics,
+    cache: TokenAdvanceCache,
+    scope: MeasureContext2D | RenderContext2D,
+): CodeLayout {
+    const lineH = m.fontSize * m.lineHeight;
+    const gutterGap = m.showLineNumbers ? gutterGapOf(scope, m, cache) : 0;
+    const gutter = gutterOf(scope, m, cache, lines.length, gutterGap);
+
+    const widths: number[] = new Array(lines.length);
+    let maxLineWidth = 0;
+    for (let i = 0; i < lines.length; i++) {
+        let w = 0;
+        for (const tok of lines[i].tokens) {
+            w += cache.advance(scope, tok.content, m.fontSize, m.fontFamily, m.letterSpacing);
+        }
+        widths[i] = w;
+        if (w > maxLineWidth) maxLineWidth = w;
+    }
+
+    const innerW = maxLineWidth + gutter;
+    const innerH = lines.length * lineH;
+    const blockW = innerW + m.padding.left + m.padding.right;
+    const blockH = innerH + m.padding.top + m.padding.bottom;
+    const startX = -blockW / 2 + m.padding.left + gutter;
+    // y-up author space: the first line sits at the TOP of the block, so the
+    // cursor starts at the top edge (+half height, minus top padding) and steps
+    // DOWNWARD by subtracting a line height per row.
+    const startY = blockH / 2 - m.padding.top;
+
+    const tokens = new Map<number, TokenBox>();
+    const lineY: number[] = new Array(lines.length);
+    const lineIndex = new Map<number, number>();
+
+    for (let i = 0; i < lines.length; i++) {
+        const centerY = startY - i * lineH - lineH / 2;
+        lineY[i] = centerY;
+        lineIndex.set(lines[i].id, i);
+        let x = startX;
+        for (const tok of lines[i].tokens) {
+            tokens.set(tok.id, { x, y: centerY });
+            x += cache.advance(scope, tok.content, m.fontSize, m.fontFamily, m.letterSpacing);
+        }
+    }
+
+    return { tokens, lineY, lineW: widths, lineIndex, gutter, gutterGap, innerW, innerH, blockW, blockH, startX };
+}