@motionscript/code 0.0.0-stage → 0.1.0-alpha.0

package/package.json

@@ -1,6 +1,85 @@
 {
   "name": "@motionscript/code",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.1.0-alpha.0",
+  "type": "module",
+  "description": "Animated, syntax-highlighted code blocks for Motion Script, with token-level append, insert, remove, replace, and highlight transitions.",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/motionscript-dev/motionscript.git",
+    "directory": "packages/components/code"
+  },
+  "homepage": "https://motionscript.dev",
+  "bugs": {
+    "url": "https://github.com/motionscript-dev/motionscript/issues"
+  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "sideEffects": false,
+  "files": [
+    "dist",
+    "src",
+    "registry.json",
+    "CHANGELOG.md",
+    "!dist/**/*.tsbuildinfo",
+    "!src/**/tests/**",
+    "!src/**/*.test.ts",
+    "!src/**/*.test.tsx",
+    "!src/**/*.fixtures.ts"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "keywords": [
+    "motion-script",
+    "motion script",
+    "code",
+    "syntax-highlighting",
+    "animation"
+  ],
+  "license": "Apache-2.0",
+  "peerDependencies": {
+    "@motionscript/core": "^0.1.0-alpha.0"
+  },
+  "dependencies": {
+    "@lezer/common": "^1.5.2",
+    "@lezer/cpp": "^1.1.6",
+    "@lezer/css": "^1.3.3",
+    "@lezer/go": "^1.0.1",
+    "@lezer/highlight": "^1.2.3",
+    "@lezer/html": "^1.3.13",
+    "@lezer/java": "^1.1.3",
+    "@lezer/javascript": "^1.5.4",
+    "@lezer/json": "^1.0.3",
+    "@lezer/markdown": "^1.6.4",
+    "@lezer/php": "^1.0.5",
+    "@lezer/python": "^1.1.19",
+    "@lezer/rust": "^1.0.2",
+    "@lezer/xml": "^1.0.6",
+    "@lezer/yaml": "^1.0.4"
+  },
+  "devDependencies": {
+    "@types/node": "^25.6.2",
+    "eslint": "^9.39.4",
+    "eslint-import-resolver-typescript": "^4.4.4",
+    "eslint-plugin-import-x": "^4.16.1",
+    "tsc-alias": "^1.8.16",
+    "typescript": "^6.0.3",
+    "typescript-eslint": "^8.48.0",
+    "vitest": "^4.1.5",
+    "@motionscript/core": "0.1.0-alpha.0"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json --resolve-full-paths && node ../../../scripts/build-browser.mjs",
+    "dev": "tsc -p tsconfig.build.json --watch & tsc-alias -p tsconfig.build.json --resolve-full-paths -w",
+    "clean": "rimraf --glob dist .turbo *.tsbuildinfo",
+    "test": "vitest --passWithNoTests",
+    "lint": "eslint .",
+    "typecheck": "tsc -p tsconfig.json --noEmit"
+  }
 }

package/registry.json

@@ -0,0 +1,6 @@
+{
+  "kit": { "barrel": "src/engine.ts", "specifier": "@motionscript/code" },
+  "items": [
+    { "name": "code", "root": "src", "files": ["node.ts"], "flatten": true }
+  ]
+}

package/src/code-range.ts

@@ -0,0 +1,156 @@
+/**
+ * A range over a code document. Lines and columns are 1-indexed.
+ *
+ * `endLine`/`endCol` may be `Infinity` to mean "to the end of the line" or
+ * "to the end of the document".
+ */
+export interface CodeRange {
+    startLine: number;
+    startCol: number;
+    endLine: number;
+    endCol: number;
+}
+
+/**
+ * Create a range that spans `length` characters starting at (line, col).
+ * If `length` is omitted, the range extends to the end of the line.
+ *
+ * Example:
+ *   word(1, 3, 3)   // line 1, column 3, spans 3 characters
+ *   word(1, 3)      // line 1, column 3, to end of line
+ */
+export function word(line: number, col: number, length?: number): CodeRange {
+    return {
+        startLine: line,
+        startCol: col,
+        endLine: line,
+        endCol: length === undefined ? Infinity : col + length,
+    };
+}
+
+/**
+ * Create a range that covers entire lines from `from` to `to` (inclusive).
+ * If `to` is omitted, only line `from` is covered.
+ *
+ * Example:
+ *   lines(1, 3)   // lines 1 through 3
+ *   lines(2)      // line 2 only
+ */
+export function lines(from: number, to?: number): CodeRange {
+    return {
+        startLine: from,
+        startCol: 1,
+        endLine: to ?? from,
+        endCol: Infinity,
+    };
+}
+
+
+
+/**
+ * Immutable, chainable, iterable builder over multiple ranges — mirrors
+ * {@link FillChain}/`Fills` in `@motionscript/core`. Every method returns a
+ * *new* chain with one more range appended, so the next range-kind method is
+ * called directly rather than through a separate combinator:
+ *
+ * @example
+ * CodeRanges.lines(2).word(5, 1, 3)   // a 2-range selection, one expression
+ */
+export class CodeRangeChain {
+    constructor(public readonly list: CodeRange[] = []) { }
+
+    lines(from: number, to?: number): CodeRangeChain {
+        return new CodeRangeChain([...this.list, lines(from, to)]);
+    }
+
+    word(line: number, col: number, length?: number): CodeRangeChain {
+        return new CodeRangeChain([...this.list, word(line, col, length)]);
+    }
+
+    *[Symbol.iterator](): Iterator<CodeRange> {
+        yield* this.list;
+    }
+}
+
+/**
+ * Entry point for building multi-range selections fluently — an empty
+ * {@link CodeRangeChain}, so `CodeRanges.lines(2)` *is* `new
+ * CodeRangeChain().lines(2)`. `word`/`lines` stay directly importable and
+ * usable standalone for a single range; reach for `CodeRanges` when a call
+ * needs more than one.
+ */
+export const CodeRanges = new CodeRangeChain();
+
+/**
+ * Convert a CodeRange to absolute character offsets [start, end) in the joined
+ * source text, given the per-line lengths of that source. Lines/columns are
+ * clamped into the document so out-of-bounds inputs produce a valid range.
+ */
+export function rangeToCharOffsets(
+    r: CodeRange,
+    lineLengths: number[],
+): { start: number; end: number } {
+    if (lineLengths.length === 0) return { start: 0, end: 0 };
+
+    // Cumulative line starts in the joined string (lines joined with '\n').
+    const lineStart: number[] = new Array(lineLengths.length);
+    let acc = 0;
+    for (let i = 0; i < lineLengths.length; i++) {
+        lineStart[i] = acc;
+        acc += lineLengths[i] + 1; // +1 for the joining newline
+    }
+    const docEnd = acc > 0 ? acc - 1 : 0; // strip trailing newline
+
+    const clampLine = (n: number) =>
+        Math.max(0, Math.min(lineLengths.length - 1, n - 1)); // to 0-indexed
+
+    // A range read back from JSON holds `Infinity` as `null`; both mean "to the end".
+    const endLine = r.endLine ?? Infinity;
+    const endCol = r.endCol ?? Infinity;
+
+    const sLine = clampLine(r.startLine);
+    const eLine = clampLine(endLine);
+    const sColMax = lineLengths[sLine];
+    const eColMax = lineLengths[eLine];
+
+    // Columns are 1-indexed; clamp to [1, lineLength+1] so endCol can sit past
+    // the last char (one-past-end).
+    const sCol = Math.max(0, Math.min(sColMax, r.startCol - 1));
+    const eCol = endCol === Infinity
+        ? eColMax
+        : Math.max(0, Math.min(eColMax, endCol - 1));
+
+    const start = Math.min(docEnd, lineStart[sLine] + sCol);
+    const end = Math.min(docEnd, lineStart[eLine] + eCol);
+    return { start: Math.min(start, end), end: Math.max(start, end) };
+}
+
+/**
+ * Convert [start, end) character offsets in joined source to a CodeRange.
+ * Lines/cols in the returned range are 1-indexed.
+ */
+export function charOffsetsToRange(
+    start: number,
+    end: number,
+    lineLengths: number[],
+): CodeRange {
+    let off = 0;
+    let startLine = 1, startCol = 1, endLine = 1, endCol = 1;
+    let foundStart = false;
+    for (let li = 0; li < lineLengths.length; li++) {
+        const lineStart = off;
+        const lineEnd = off + lineLengths[li];
+        if (!foundStart && start >= lineStart && start <= lineEnd) {
+            startLine = li + 1;
+            startCol = (start - lineStart) + 1;
+            foundStart = true;
+        }
+        if (end >= lineStart && end <= lineEnd) {
+            endLine = li + 1;
+            endCol = (end - lineStart) + 1;
+            break;
+        }
+        off = lineEnd + 1;
+    }
+    return { startLine, startCol, endLine, endCol };
+}

package/src/diff.ts

@@ -0,0 +1,216 @@
+import type { IdLine, IdToken } from "./tokens";
+
+/**
+ * The result of diffing one listing structure against another: the *next*
+ * structure, with token and line ids carried over from the previous one
+ * wherever the content survived, plus the three sets a transition needs to know
+ * which role each token plays.
+ *
+ * Identity is the whole product here. `const variable = 3` becoming
+ * `const number = 3` should move `const` and `= 3` rather than dissolve and
+ * rebuild them, and a token can only *move* if the frame after the edit still
+ * knows it is the same token.
+ */
+/**
+ * How a `Code` node decides which token became which across an edit. Given
+ * the old and new tokenized structures, return the carried-over structure
+ * plus the added/removed/renamed-line bookkeeping a transition needs.
+ * `diffCode` below is the default implementation (exported as
+ * `defaultCodeDiff`) — pass a `diffStrategy` of this shape to `<Code>` to
+ * replace it with your own.
+ */
+export type CodeDiffStrategy = (from: IdLine[], to: IdLine[]) => CodeEdit;
+
+export interface CodeEdit {
+    /** The new structure. Ids are rewritten in place onto the tokens handed in. */
+    lines: IdLine[];
+    /** Ids present only in the old structure — these fade out. */
+    removedIds: Set<number>;
+    /** Ids present only in the new structure — these fade in. */
+    addedIds: Set<number>;
+    /** Ids of {@link lines} that did not exist before — these enter as a whole row. */
+    newLineIds: Set<number>;
+    /** Previous colour of a surviving token whose highlighting changed, for the cross-fade. */
+    fromColorById: Map<number, string | undefined>;
+}
+
+/**
+ * How similar two lines must be before they are treated as *one line that was
+ * edited* rather than as one line deleted and another inserted.
+ *
+ * Below this, a row's tokens have so little in common that carrying ids across
+ * would animate glyphs sliding between unrelated words. Above it, the edit reads
+ * the way it was made.
+ */
+const LINE_MATCH_FLOOR = 0.45;
+
+/** Weight a token carries when judging similarity: its visible characters. */
+function tokenWeight(tok: IdToken): number {
+    return tok.content.trim().length;
+}
+
+function lineWeight(line: IdLine): number {
+    let sum = 0;
+    for (const tok of line.tokens) sum += tokenWeight(tok);
+    return sum;
+}
+
+/**
+ * Dice coefficient over the two lines' token multisets, weighted by visible
+ * characters so `variable` counts for far more than a run of spaces.
+ *
+ * Two blank lines are identical (1); a blank line against a non-blank one shares
+ * nothing (0), which keeps the blank rows a reformat introduces from anchoring
+ * themselves onto real code.
+ */
+function similarity(a: IdLine, b: IdLine): number {
+    const wa = lineWeight(a);
+    const wb = lineWeight(b);
+    if (wa === 0 && wb === 0) return 1;
+    if (wa === 0 || wb === 0) return 0;
+
+    const bag = new Map<string, number>();
+    for (const tok of a.tokens) {
+        if (tokenWeight(tok) === 0) continue;
+        bag.set(tok.content, (bag.get(tok.content) ?? 0) + 1);
+    }
+    let common = 0;
+    for (const tok of b.tokens) {
+        const w = tokenWeight(tok);
+        if (w === 0) continue;
+        const left = bag.get(tok.content) ?? 0;
+        if (left > 0) {
+            bag.set(tok.content, left - 1);
+            common += w;
+        }
+    }
+    return (2 * common) / (wa + wb);
+}
+
+/**
+ * Align two sequences by maximising total `score`, skipping is free.
+ *
+ * A plain LCS would only ever pair *identical* elements, which is the wrong tool
+ * twice over: two lines that differ by one word are the same line, and two
+ * tokens that differ only in colour are the same token. Scoring the pairing and
+ * letting anything that scores ≤ 0 fall out expresses that directly, and because
+ * skips cost nothing the alignment stays in order.
+ *
+ * Returns the matched index pairs, in order.
+ */
+function align(
+    aLength: number,
+    bLength: number,
+    score: (i: number, j: number) => number,
+): Array<[number, number]> {
+    // dp[i][j] = best score aligning a[0..i) with b[0..j).
+    const width = bLength + 1;
+    const dp = new Float64Array((aLength + 1) * width);
+    const pairScore = new Float64Array(aLength * bLength);
+
+    for (let i = 1; i <= aLength; i++) {
+        for (let j = 1; j <= bLength; j++) {
+            const s = score(i - 1, j - 1);
+            pairScore[(i - 1) * bLength + (j - 1)] = s;
+            const skipA = dp[(i - 1) * width + j];
+            const skipB = dp[i * width + (j - 1)];
+            let best = skipA > skipB ? skipA : skipB;
+            if (s > 0) {
+                const paired = dp[(i - 1) * width + (j - 1)] + s;
+                if (paired > best) best = paired;
+            }
+            dp[i * width + j] = best;
+        }
+    }
+
+    const pairs: Array<[number, number]> = [];
+    let i = aLength;
+    let j = bLength;
+    while (i > 0 && j > 0) {
+        const s = pairScore[(i - 1) * bLength + (j - 1)];
+        if (s > 0 && dp[i * width + j] === dp[(i - 1) * width + (j - 1)] + s) {
+            pairs.push([i - 1, j - 1]);
+            i--; j--;
+        } else if (dp[(i - 1) * width + j] >= dp[i * width + (j - 1)]) {
+            i--;
+        } else {
+            j--;
+        }
+    }
+    pairs.reverse();
+    return pairs;
+}
+
+/**
+ * Match the tokens of one line against another's.
+ *
+ * Scored by visible characters rather than token count, so an alignment that
+ * keeps `items.reduce` together beats one that keeps four more spaces. A token
+ * pairs only when its content is unchanged — a changed word is a different word,
+ * and should be seen to be replaced.
+ */
+function alignTokens(from: IdToken[], to: IdToken[]): Array<[number, number]> {
+    return align(from.length, to.length, (i, j) => {
+        if (from[i].content !== to[j].content) return 0;
+        // Whitespace still pairs (it holds indentation together), just weakly
+        // enough that it never outranks real content.
+        return tokenWeight(to[j]) || 0.25;
+    });
+}
+
+/**
+ * Carry `from`'s identities onto `to`, and report what changed.
+ *
+ * `to` is freshly tokenized and owns ids nobody has seen yet, so rewriting them
+ * in place is safe and is what makes the transition cheap: after this, the two
+ * structures share an id for every token that survived the edit.
+ */
+export function diffCode(from: IdLine[], to: IdLine[]): CodeEdit {
+    const removedIds = new Set<number>();
+    const addedIds = new Set<number>();
+    const newLineIds = new Set<number>();
+    const fromColorById = new Map<number, string | undefined>();
+
+    const linePairs = align(from.length, to.length, (i, j) => similarity(from[i], to[j]) - LINE_MATCH_FLOOR);
+
+    const pairedTo = new Set<number>();
+    const pairedFrom = new Set<number>();
+
+    for (const [fi, ti] of linePairs) {
+        pairedFrom.add(fi);
+        pairedTo.add(ti);
+        const fromLine = from[fi];
+        const toLine = to[ti];
+        toLine.id = fromLine.id;
+
+        const matched = alignTokens(fromLine.tokens, toLine.tokens);
+        const keptFrom = new Set<number>();
+        const keptTo = new Set<number>();
+        for (const [a, b] of matched) {
+            keptFrom.add(a);
+            keptTo.add(b);
+            const oldTok = fromLine.tokens[a];
+            const newTok = toLine.tokens[b];
+            newTok.id = oldTok.id;
+            if (oldTok.color !== newTok.color) fromColorById.set(oldTok.id, oldTok.color);
+        }
+        for (let a = 0; a < fromLine.tokens.length; a++) {
+            if (!keptFrom.has(a)) removedIds.add(fromLine.tokens[a].id);
+        }
+        for (let b = 0; b < toLine.tokens.length; b++) {
+            if (!keptTo.has(b)) addedIds.add(toLine.tokens[b].id);
+        }
+    }
+
+    for (let i = 0; i < from.length; i++) {
+        if (pairedFrom.has(i)) continue;
+        for (const tok of from[i].tokens) removedIds.add(tok.id);
+    }
+    for (let j = 0; j < to.length; j++) {
+        if (pairedTo.has(j)) continue;
+        newLineIds.add(to[j].id);
+        for (const tok of to[j].tokens) addedIds.add(tok.id);
+    }
+
+    return { lines: to, removedIds, addedIds, newLineIds, fromColorById };
+}

package/src/engine.ts

@@ -0,0 +1,21 @@
+// The Code "engine": syntax highlighting, tokenizing, layout, diffing, and
+// transition/phase scheduling — kept separate from node.ts so `ms add code`
+// can copy just the node declaration and import this behind
+// `@motionscript/code` instead. What gets copied is whatever this
+// package's registry.json declares, which for this component is node.ts;
+// packages/site/scripts/build-registry.mjs enforces that anything it
+// reaches for is either copied alongside it or re-exported from here.
+export * from './transitions';
+export * from './layout';
+export { drawCode } from './render';
+export type { DrawCodeState } from './render';
+export { diffCode as defaultCodeDiff } from './diff';
+export type { CodeDiffStrategy, CodeEdit } from './diff';
+export { loadCodeLanguage, initSyntaxHighlighter, ensureHighlighter, registerCodeTheme, resolveTheme, canHighlight } from './highlight';
+export { tokenizeCodeToIdLines } from './tokens';
+export type { IdLine, IdToken } from './tokens';
+export { DefaultHighlightStyle, GithubDarkStyle, GithubLightStyle, VscodeDarkStyle, VscodeLightStyle, BUILTIN_THEMES, compileStyle } from './style';
+export type { CodeHighlightStyle, CodeStyleRule, CodeTheme, CodeThemeName } from './style';
+export { TokenAdvanceCache } from './measure-cache';
+export { word, lines, rangeToCharOffsets, charOffsetsToRange, CodeRanges, CodeRangeChain } from './code-range';
+export type { CodeRange } from './code-range';