@cube-drone/marquee-parser 0.1.0

package/dist/inlines.js

@@ -0,0 +1,400 @@
+// The inline algorithm (SPEC.md, "The inline algorithm"): a faithful port of
+// rust/parser/src/inlines.rs. One left-to-right pass per container; escape >
+// code span > bracket > emoji > delimiter run; top-of-stack delimiter
+// matching; everything unmatched reverts to literal.
+//
+// Works on code points (Array.from), mirroring Rust's Vec<char>, so the two
+// implementations scan identical units.
+import { isName, nameLen, parseAttrs, parseValue, utf8Len } from "./attrs.js";
+import { MAX_TARGET_BYTES } from "./blocks.js";
+export const MAX_INLINE_DEPTH = 16;
+export const MAX_EMOJI_SLUG_BYTES = 64;
+function rootFrame() {
+    return { openerRaw: "", name: "", attrs: {}, children: [], delims: [] };
+}
+const ASCII_PUNCT = new Set("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~");
+/** Grammar whitespace is ASCII only (SPEC.md, front-door normalization):
+ * Unicode spaces are content, not structure. */
+function isWs(c) {
+    return c === " " || c === "\t" || c === "\n";
+}
+function isSlugChar(c) {
+    return /^[a-z0-9_+-]$/.test(c);
+}
+export function parseInlines(text) {
+    return parseInlinesAt(text, 0);
+}
+/** `base` is the inline depth already spent by enclosing link text: spans,
+ * delimiters, and link nesting share the one <= 16 cap (a per-construct cap
+ * would let composition multiply past it). */
+function parseInlinesAt(text, base) {
+    const chars = Array.from(text);
+    const frames = [rootFrame()];
+    let i = 0;
+    while (i < chars.length) {
+        const c = chars[i];
+        if (c === "\\") {
+            const n = chars[i + 1];
+            if (n === "\n") {
+                top(frames).children.push({ type: "hard_break" });
+                i += 2;
+            }
+            else if (n !== undefined && ASCII_PUNCT.has(n)) {
+                pushStr(frames, n);
+                i += 2;
+            }
+            else {
+                pushStr(frames, "\\");
+                i += 1;
+            }
+        }
+        else if (c === "`") {
+            const n = runLen(chars, i, "`");
+            const k = findBacktickCloser(chars, i + n, n);
+            if (k !== null) {
+                top(frames).children.push({ type: "code_span", text: chars.slice(i + n, k).join("") });
+                i = k + n;
+            }
+            else {
+                pushStr(frames, "`".repeat(n));
+                i += n;
+            }
+        }
+        else if (c === "[") {
+            i = bracket(chars, i, false, frames, base);
+        }
+        else if (c === "!" && chars[i + 1] === "[") {
+            i = bracket(chars, i + 1, true, frames, base);
+        }
+        else if (c === ":") {
+            let k = i + 1;
+            while (k < chars.length && isSlugChar(chars[k])) {
+                k += 1;
+            }
+            const slugLen = k - (i + 1);
+            if (slugLen >= 1 && slugLen <= MAX_EMOJI_SLUG_BYTES && chars[k] === ":") {
+                top(frames).children.push({ type: "emoji", slug: chars.slice(i + 1, k).join("") });
+                i = k + 1;
+            }
+            else {
+                pushStr(frames, ":");
+                i += 1;
+            }
+        }
+        else if (c === "*") {
+            const n = runLen(chars, i, "*");
+            if (n === 1) {
+                i = delimiter(chars, i, n, "em", "*", frames, base);
+            }
+            else if (n === 2) {
+                i = delimiter(chars, i, n, "strong", "**", frames, base);
+            }
+            else {
+                pushStr(frames, "*".repeat(n));
+                i += n;
+            }
+        }
+        else if (c === "~") {
+            const n = runLen(chars, i, "~");
+            if (n === 2) {
+                i = delimiter(chars, i, n, "strike", "~~", frames, base);
+            }
+            else {
+                pushStr(frames, "~".repeat(n));
+                i += n;
+            }
+        }
+        else {
+            pushStr(frames, c);
+            i += 1;
+        }
+    }
+    // Container end: unclosed spans and delimiters revert to literal text.
+    while (frames.length > 1) {
+        const frame = frames.pop();
+        top(frames).children.push(...revertFrame(frame));
+    }
+    const root = frames.pop();
+    return normalize(revertDelims(root.children, root.delims));
+}
+function top(frames) {
+    return frames[frames.length - 1];
+}
+function pushStr(frames, s) {
+    const frame = top(frames);
+    // An open delimiter sits (invisibly, until it closes) at its recorded
+    // index: text on its far side must not merge into text before it.
+    const barrier = frame.delims.length > 0 ? frame.delims[frame.delims.length - 1].idx : 0;
+    const children = frame.children;
+    if (children.length > barrier) {
+        const last = children[children.length - 1];
+        if (last.type === "text") {
+            last.value += s;
+            return;
+        }
+    }
+    children.push({ type: "text", value: s });
+}
+function runLen(chars, i, c) {
+    let n = 0;
+    while (i + n < chars.length && chars[i + n] === c) {
+        n += 1;
+    }
+    return n;
+}
+function findBacktickCloser(chars, from, n) {
+    let k = from;
+    while (k < chars.length) {
+        if (chars[k] === "`") {
+            const m = runLen(chars, k, "`");
+            if (m === n) {
+                return k;
+            }
+            k += m;
+        }
+        else {
+            k += 1;
+        }
+    }
+    return null;
+}
+function totalDepth(frames) {
+    let depth = frames.length - 1;
+    for (const f of frames) {
+        depth += f.delims.length;
+    }
+    return depth;
+}
+function delimiter(chars, i, n, kind, raw, frames, base) {
+    const canClose = i > 0 && !isWs(chars[i - 1]);
+    const next = chars[i + n];
+    const canOpen = next !== undefined && !isWs(next);
+    const deep = base + totalDepth(frames) >= MAX_INLINE_DEPTH;
+    const frame = top(frames);
+    const innermost = frame.delims[frame.delims.length - 1];
+    if (canClose && innermost !== undefined && innermost.kind === kind) {
+        frame.delims.pop();
+        const inner = normalize(frame.children.splice(innermost.idx));
+        frame.children.push(kind === "em"
+            ? { type: "emphasis", children: inner }
+            : kind === "strong"
+                ? { type: "strong", children: inner }
+                : { type: "strikethrough", children: inner });
+    }
+    else if (canOpen && !deep) {
+        frame.delims.push({ kind, idx: frame.children.length, raw });
+    }
+    else {
+        pushStr(frames, raw);
+    }
+    return i + n;
+}
+/** Handle a bracket construct starting at `chars[open]` (which is `[`).
+ * `embed` means a `!` sits just before it. Returns the new position. */
+function bracket(chars, open, embed, frames, base) {
+    const bang = embed ? open - 1 : open;
+    const fallback = () => {
+        pushStr(frames, chars[bang]);
+        return bang + 1;
+    };
+    // Find the matching `]` (balanced, escape-aware).
+    let depth = 1;
+    let k = open + 1;
+    while (k < chars.length) {
+        const c = chars[k];
+        if (c === "\\") {
+            k += 1;
+        }
+        else if (c === "[") {
+            depth += 1;
+        }
+        else if (c === "]") {
+            depth -= 1;
+            if (depth === 0) {
+                break;
+            }
+        }
+        k += 1;
+    }
+    if (k >= chars.length) {
+        return fallback();
+    }
+    const interior = chars.slice(open + 1, k).join("");
+    // Link / embed: `](` with a lexable target.
+    if (chars[k + 1] === "(") {
+        // A link's text is one nesting level (embeds don't recurse: alt is a
+        // plain string). Over the shared inline cap, fall back to literal.
+        if (!embed && base + totalDepth(frames) >= MAX_INLINE_DEPTH) {
+            return fallback();
+        }
+        const lexed = lexTarget(chars, k + 2);
+        if (lexed !== null) {
+            const depth = base + totalDepth(frames) + 1;
+            top(frames).children.push(embed
+                ? { type: "embed", target: lexed.target, alt: resolveEscapes(interior) }
+                : { type: "link", target: lexed.target, children: parseInlinesAt(interior, depth) });
+            return lexed.end;
+        }
+        return fallback();
+    }
+    // Span closer: `[/name]` must name the innermost open span.
+    if (interior.startsWith("/")) {
+        const name = interior.slice(1);
+        if (isName(name)) {
+            if (embed) {
+                pushStr(frames, "!"); // the ! belongs to links/embeds, not spans
+            }
+            if (frames.length > 1 && top(frames).name === name) {
+                const frame = frames.pop();
+                const children = normalize(revertDelims(frame.children, frame.delims));
+                top(frames).children.push({ type: "span", name: frame.name, attrs: frame.attrs, children });
+            }
+            else {
+                // Well-formed but mismatched or orphan: the characters back.
+                pushStr(frames, `[/${name}]`);
+            }
+            return k + 1;
+        }
+        return fallback();
+    }
+    // Span opener: `[name ...]`, with the BBCode default-parameter idiom
+    // (`[color=red]` puts `color=red` in the span's own attrs).
+    const opener = parseSpanOpener(interior);
+    if (opener !== null) {
+        if (embed) {
+            pushStr(frames, "!"); // the ! belongs to links/embeds, not spans
+        }
+        if (base + totalDepth(frames) >= MAX_INLINE_DEPTH) {
+            pushStr(frames, chars.slice(open, k + 1).join(""));
+        }
+        else {
+            frames.push({
+                openerRaw: chars.slice(open, k + 1).join(""),
+                name: opener.name,
+                attrs: opener.attrs,
+                children: [],
+                delims: [],
+            });
+        }
+        return k + 1;
+    }
+    return fallback();
+}
+function parseSpanOpener(interior) {
+    const nlen = nameLen(interior);
+    if (nlen === 0) {
+        return null;
+    }
+    const name = interior.slice(0, nlen);
+    const pairs = new Map();
+    let rest = interior.slice(nlen);
+    if (rest.startsWith("=")) {
+        const v = parseValue(rest.slice(1));
+        if (!v.ok) {
+            return null;
+        }
+        if (!(v.rest === "" || v.rest.startsWith(" ") || v.rest.startsWith("\t"))) {
+            return null;
+        }
+        pairs.set(name, v.value);
+        rest = v.rest;
+    }
+    else if (!(rest === "" || rest.startsWith(" ") || rest.startsWith("\t"))) {
+        return null;
+    }
+    const parsed = parseAttrs(rest);
+    if (!parsed.ok) {
+        return null;
+    }
+    for (const [key, value] of Object.entries(parsed.attrs)) {
+        if (!pairs.has(key)) {
+            pairs.set(key, value);
+        }
+    }
+    return { name, attrs: Object.fromEntries(pairs) };
+}
+/** Lex a link/embed target from `chars[from]`: no whitespace, balanced
+ * parens, an unbalanced `)` ends it. Returns the position after `)`. */
+function lexTarget(chars, from) {
+    let depth = 0;
+    let k = from;
+    while (k < chars.length) {
+        const c = chars[k];
+        if (c === ")" && depth === 0) {
+            const target = chars.slice(from, k).join("");
+            if (utf8Len(target) > MAX_TARGET_BYTES) {
+                return null;
+            }
+            return { target, end: k + 1 };
+        }
+        if (c === ")") {
+            depth -= 1;
+        }
+        else if (c === "(") {
+            depth += 1;
+        }
+        else if (isWs(c)) {
+            return null;
+        }
+        k += 1;
+    }
+    return null;
+}
+function resolveEscapes(s) {
+    let out = "";
+    const chars = Array.from(s);
+    let i = 0;
+    while (i < chars.length) {
+        const c = chars[i];
+        if (c === "\\") {
+            const n = chars[i + 1];
+            if (n !== undefined && ASCII_PUNCT.has(n)) {
+                out += n;
+                i += 2;
+                continue;
+            }
+            out += "\\";
+            if (n !== undefined) {
+                out += n;
+                i += 2;
+                continue;
+            }
+            i += 1;
+            continue;
+        }
+        out += c;
+        i += 1;
+    }
+    return out;
+}
+/** An unclosed span reverts: its opener text, then its children, flattened
+ * into the parent (its own unmatched delimiters reverting first). */
+function revertFrame(frame) {
+    const out = [{ type: "text", value: frame.openerRaw }];
+    out.push(...revertDelims(frame.children, frame.delims));
+    return out;
+}
+function revertDelims(children, delims) {
+    for (let d = delims.length - 1; d >= 0; d -= 1) {
+        children.splice(delims[d].idx, 0, { type: "text", value: delims[d].raw });
+    }
+    return children;
+}
+/** Canonical text: adjacent literals merge, empty text nodes vanish. */
+function normalize(children) {
+    const out = [];
+    for (const node of children) {
+        if (node.type === "text") {
+            if (node.value === "") {
+                continue;
+            }
+            const last = out[out.length - 1];
+            if (last !== undefined && last.type === "text") {
+                last.value += node.value;
+                continue;
+            }
+        }
+        out.push(node);
+    }
+    return out;
+}

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@cube-drone/marquee-parser",
+  "version": "0.1.0",
+  "description": "Reference parser for the Marquee markup language",
+  "license": "MPL-2.0",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/cube-drone/marqueemarkup.git",
+    "directory": "ts/parser"
+  },
+  "type": "module",
+  "engines": {
+    "node": ">=22.6.0"
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "exports": {
+    ".": {
+      "marquee-src": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "test": "node --conditions=marquee-src --test test/*.test.ts",
+    "check": "tsc --noEmit",
+    "build": "tsc -p tsconfig.build.json",
+    "prepack": "npm run build"
+  },
+  "devDependencies": {
+    "@types/node": "^24.0.0",
+    "typescript": "^5.9.0"
+  }
+}

package/src/ast.ts

@@ -0,0 +1,39 @@
+// The AST is the contract: these shapes mirror SPEC.md's node inventory
+// exactly, and structural equality against the vectors is conformance.
+
+export type Attrs = Record<string, string>;
+
+/** The closed `invalid_directive` reason enum (SPEC.md, "The AST"). */
+export type Reason =
+  | "bad_name"
+  | "bad_attribute"
+  | "attribute_too_long"
+  | "depth_exceeded"
+  | "mismatched_close"
+  | "stray_close";
+
+export type Node =
+  // Blocks
+  | { type: "document"; version: number; children: Node[] }
+  | { type: "paragraph"; children: Node[] }
+  | { type: "heading"; level: number; children: Node[] }
+  | { type: "code_block"; info?: string; text: string }
+  | { type: "blockquote"; children: Node[] }
+  | { type: "list"; ordered: boolean; children: Node[] }
+  | { type: "list_item"; children: Node[] }
+  | { type: "thematic_break" }
+  | { type: "directive"; name: string; attrs: Attrs; children: Node[] }
+  | { type: "invalid_directive"; reason: Reason; children: Node[] }
+  | { type: "comment"; text: string }
+  // Inlines
+  | { type: "text"; value: string }
+  | { type: "emphasis"; children: Node[] }
+  | { type: "strong"; children: Node[] }
+  | { type: "strikethrough"; children: Node[] }
+  | { type: "code_span"; text: string }
+  | { type: "link"; target: string; children: Node[] }
+  | { type: "embed"; target: string; alt: string }
+  | { type: "turbolink"; target: string }
+  | { type: "span"; name: string; attrs: Attrs; children: Node[] }
+  | { type: "emoji"; slug: string }
+  | { type: "hard_break" };

package/src/attrs.ts

@@ -0,0 +1,114 @@
+// The attribute grammar (SPEC.md, "The attribute grammar"). Shared by
+// directive open lines and span openers; only the failure mode differs
+// (blocks get invalid_directive nodes, spans fall back to literal text).
+
+import type { Attrs, Reason } from "./ast.ts";
+
+export const MAX_ATTR_VALUE_BYTES = 2048;
+
+const encoder = new TextEncoder();
+
+/** Caps are byte counts (SPEC.md): measure UTF-8, not UTF-16 units. */
+export function utf8Len(s: string): number {
+  return encoder.encode(s).length;
+}
+
+export function isName(s: string): boolean {
+  return /^[a-z][a-z0-9_-]*$/.test(s);
+}
+
+/** Length of the leading name in `s`, or 0. (Names are ASCII, so this
+ * length is the same in UTF-16 units and bytes.) */
+export function nameLen(s: string): number {
+  const m = /^[a-z][a-z0-9_-]*/.exec(s);
+  return m ? m[0].length : 0;
+}
+
+export type ValueResult =
+  | { ok: true; value: string; rest: string }
+  | { ok: false; reason: Reason };
+
+export type AttrsResult = { ok: true; attrs: Attrs } | { ok: false; reason: Reason };
+
+/**
+ * Parse a whitespace-separated `key=value` list. Duplicate keys resolve
+ * first-writer-wins. Any deviation from the grammar is an error.
+ */
+export function parseAttrs(src: string): AttrsResult {
+  // A Map sidesteps object-prototype traps (`__proto__` is a legal key);
+  // Object.fromEntries defines own properties, so the traps stay closed.
+  const attrs = new Map<string, string>();
+  let rest = src;
+  for (;;) {
+    rest = rest.replace(/^[ \t]+/, "");
+    if (rest === "") {
+      return { ok: true, attrs: Object.fromEntries(attrs) };
+    }
+    const klen = nameLen(rest);
+    if (klen === 0) {
+      return { ok: false, reason: "bad_attribute" };
+    }
+    const key = rest.slice(0, klen);
+    rest = rest.slice(klen);
+    if (!rest.startsWith("=")) {
+      return { ok: false, reason: "bad_attribute" };
+    }
+    const v = parseValue(rest.slice(1));
+    if (!v.ok) {
+      return { ok: false, reason: v.reason };
+    }
+    rest = v.rest;
+    // After a value: end of input or whitespace.
+    if (!(rest === "" || rest.startsWith(" ") || rest.startsWith("\t"))) {
+      return { ok: false, reason: "bad_attribute" };
+    }
+    if (!attrs.has(key)) {
+      attrs.set(key, v.value);
+    }
+  }
+}
+
+/** Parse one attribute value at the start of `s`; returns (value, rest). */
+export function parseValue(s: string): ValueResult {
+  if (s.startsWith('"')) {
+    let value = "";
+    let i = 1;
+    while (i < s.length) {
+      const c = s[i]!;
+      if (c === '"') {
+        if (utf8Len(value) > MAX_ATTR_VALUE_BYTES) {
+          return { ok: false, reason: "attribute_too_long" };
+        }
+        return { ok: true, value, rest: s.slice(i + 1) };
+      }
+      if (c === "\\") {
+        const n = s[i + 1];
+        if (n === '"' || n === "\\") {
+          value += n;
+          i += 2;
+          continue;
+        }
+        return { ok: false, reason: "bad_attribute" };
+      }
+      value += c;
+      i += 1;
+    }
+    return { ok: false, reason: "bad_attribute" }; // unterminated quote
+  }
+  let end = 0;
+  while (end < s.length && s[end] !== " " && s[end] !== "\t" && s[end] !== '"') {
+    end += 1;
+  }
+  if (end === 0) {
+    return { ok: false, reason: "bad_attribute" }; // empty bare value
+  }
+  const rest = s.slice(end);
+  if (rest.startsWith('"')) {
+    return { ok: false, reason: "bad_attribute" }; // quote inside a bare value
+  }
+  const value = s.slice(0, end);
+  if (utf8Len(value) > MAX_ATTR_VALUE_BYTES) {
+    return { ok: false, reason: "attribute_too_long" };
+  }
+  return { ok: true, value, rest };
+}