@miliastry/quasar 1.0.0

package/src/Syntax/greenEdit.ts

@@ -0,0 +1,110 @@
+/**
+ * DocumentEngine — Green tree editing primitives
+ *
+ * The green tree is immutable, so an edit is not a mutation: it is a new tree
+ * that shares everything the edit did not touch.
+ *
+ * ─── What used to be here ───────────────────────────────────────────────────
+ *
+ * `shiftGreen(node, delta)` — a deep copy of a subtree with every range moved.
+ * It existed because green nodes carried absolute positions, so inserting three
+ * characters near the top of a document meant rebuilding every node after the
+ * insertion point just to renumber it. On a 19.6 KB document that was 14% of
+ * the cost of an incremental reparse, spent entirely on producing structurally
+ * identical copies of nodes that had not changed.
+ *
+ * It is gone, and there is nothing to replace it with. Green nodes have widths,
+ * not positions (see `GreenNode.ts`), so text inserted before a subtree changes
+ * where it is — which is the red tree's business — and not what it is. A
+ * sibling after an edit is now shared by reference, exactly like a sibling
+ * before it always was.
+ */
+
+import { GreenNode, greenNode } from './GreenNode'
+
+/** One step of the descent from the root to the node being replaced. */
+export interface SpineStep {
+  /** The ancestor. */
+  node: GreenNode
+  /** Index of the child that the descent continued into. */
+  index: number
+}
+
+/**
+ * Rebuild the ancestor spine after replacing one node.
+ *
+ * `spine` runs root-first; `replacement` takes the place of
+ * `spine[spine.length - 1].node.children[spine[spine.length - 1].index]`.
+ *
+ * Only the spine itself is rebuilt — one new node per level of nesting, and
+ * every other subtree in the document is carried over by reference. The
+ * ancestors' widths follow from their new children automatically, so there is
+ * no delta to propagate and no way to get the arithmetic wrong.
+ */
+export function spliceGreen(
+  spine: readonly SpineStep[],
+  replacement: GreenNode,
+): GreenNode {
+  let current = replacement
+
+  for (let i = spine.length - 1; i >= 0; i--) {
+    const { node, index } = spine[i]
+    const oldChildren = node.children as readonly GreenNode[]
+    const children = new Array<GreenNode>(oldChildren.length)
+
+    for (let j = 0; j < oldChildren.length; j++) children[j] = oldChildren[j]
+    children[index] = current
+
+    current = greenNode(
+      node.kind,
+      node.text,
+      children,
+      node.leadingWidth,
+      node.trailingWidth,
+    )
+  }
+
+  return current
+}
+
+/**
+ * Replace a node's children while keeping the node itself — its kind, its
+ * attributes and, above all, its delimiters.
+ *
+ * The incremental parser re-parses a span strictly INSIDE a node, so that
+ * node's own `[centre]` and `[/centre]` never pass through the parser again and
+ * cannot be reinterpreted. `leadingWidth`/`trailingWidth` carry across
+ * untouched; the new width follows from the new children.
+ */
+export function withChildren(node: GreenNode, children: GreenNode[]): GreenNode {
+  return greenNode(node.kind, node.text, children, node.leadingWidth, node.trailingWidth)
+}
+
+/**
+ * Replace the children `[from, to)` of `node` with `replacement`.
+ *
+ * The unit an edit actually touches is a RUN OF SIBLINGS, not a whole node.
+ * Re-parsing a container's entire contents because one character changed
+ * somewhere inside it meant that typing into a large `[notice]` re-lexed 66% of
+ * the document per keystroke; and an edit at document level had no enclosing
+ * container at all, so it fell back to a full rebuild — which is where the
+ * caret sits while you write the end of a post, the single most common case.
+ *
+ * Everything outside `[from, to)` is carried over by reference.
+ */
+export function withChildrenSpliced(
+  node: GreenNode,
+  from: number,
+  to: number,
+  replacement: readonly GreenNode[],
+): GreenNode {
+  const old = node.children as readonly GreenNode[]
+  const children = new Array<GreenNode>(from + replacement.length + (old.length - to))
+
+  let w = 0
+  for (let i = 0; i < from; i++) children[w++] = old[i]
+  for (let i = 0; i < replacement.length; i++) children[w++] = replacement[i]
+  for (let i = to; i < old.length; i++) children[w++] = old[i]
+
+  return greenNode(node.kind, node.text, children, node.leadingWidth, node.trailingWidth)
+}

package/src/Syntax/hash.ts

@@ -0,0 +1,30 @@
+/**
+ * Quasar — Structural Hashing Primitives (FNV-1a)
+ *
+ * Shared by GreenNode (`_hash`, for interning and structural sharing) and
+ * NodeMatcher (subtree fingerprints, for cross-version node matching).
+ *
+ * FNV-1a is chosen for the same reason Roslyn uses a cheap non-cryptographic
+ * hash: this runs once per node on every keystroke, so throughput matters far
+ * more than adversarial collision resistance. `Math.imul` keeps the multiply
+ * in 32-bit integer space instead of promoting to doubles.
+ */
+
+export const FNV_OFFSET = 0x811c9dc5
+export const FNV_PRIME = 0x01000193
+
+/** Secondary basis, used to build a 64-bit key from two independent lanes. */
+export const FNV_OFFSET_ALT = 0x27d4eb2f
+
+export function hashString(seed: number, str: string): number {
+  let h = seed
+  for (let i = 0; i < str.length; i++) {
+    h ^= str.charCodeAt(i)
+    h = Math.imul(h, FNV_PRIME)
+  }
+  return h >>> 0
+}
+
+export function hashUint32(seed: number, value: number): number {
+  return Math.imul(seed ^ value, FNV_PRIME) >>> 0
+}

package/src/Syntax/index.ts

@@ -0,0 +1,12 @@
+export { GreenNode, greenNode, greenLeaf } from './GreenNode'
+export { RedNode } from './RedNode'
+export { TreeBuilder } from './TreeBuilder'
+export type { BuildResult, NodeFactory } from './TreeBuilder'
+export { checkPartition, assertPartition } from './partition'
+export type {
+  PartitionViolation,
+  PartitionViolationKind,
+  CheckPartitionOptions,
+} from './partition'
+export { NodeMatcher } from './NodeMatcher'
+export type { MatchResult, MatchStatus, NodeMatch } from './NodeMatcher'

package/src/Syntax/partition.ts

@@ -0,0 +1,161 @@
+/**
+ * DocumentEngine — Partition Invariant
+ *
+ * The green tree must PARTITION the source text: every character of the input
+ * is accounted for exactly once, and the tree can answer "which node owns
+ * offset N?" without ambiguity.
+ *
+ * Before the width model this was false in three ways, all measured:
+ *
+ *  1. Delimiters were not represented at all. `[b]hola[/b]` produced a `bold`
+ *     spanning `[0..13]` whose only child spanned `[3..7]` — 6 characters
+ *     inside the parent belonged to no node. 32.630 occurrences of
+ *     `parent.start != firstChild.start` in a single real document.
+ *  2. Siblings could occupy the same range. `"hola\n\nmundo"` produced
+ *     `spacing [4..6]` AND `empty_line [4..6]`, so offset 5 had two owners.
+ *  3. Orphaned closing tags were dropped silently, leaving holes in the root.
+ *
+ * ─── What this module still checks, and what it no longer needs to ──────────
+ *
+ * Most of the invariant is now STRUCTURAL. A green node has no position, and
+ * its width is computed by its own constructor as
+ * `leadingWidth + Σ child widths + trailingWidth`, so gaps, overlaps and
+ * mismatched delimiters are not states the type can be in. The checks for them
+ * were deleted along with the ability to fail them.
+ *
+ * Two things remain genuinely checkable, and they are the two that matter:
+ *
+ *  - **Coverage.** The root's width must equal the source length. If the parser
+ *    drops a token — which is exactly what the orphaned-closing-tag bug did —
+ *    the tree is internally consistent but describes a shorter document.
+ *  - **Leaf honesty.** A `text` token's width must equal the text it holds.
+ *    Nothing derives this, so nothing enforces it.
+ */
+
+import type { GreenNode } from './GreenNode'
+
+export type PartitionViolationKind =
+  /** The root's width does not match the source length. */
+  | 'root-coverage'
+  /** A leaf's width does not match the text it claims to hold. */
+  | 'leaf-width'
+  /** A node's width disagrees with its own children plus delimiters. */
+  | 'width-mismatch'
+
+export interface PartitionViolation {
+  kind: PartitionViolationKind
+  /** Structural path to the offending node, e.g. `document/paragraph[0]/bold[1]`. */
+  path: string
+  nodeKind: string
+  detail: string
+}
+
+export interface CheckPartitionOptions {
+  /** Stop after this many violations. Default 200 — enough to diagnose, cheap to print. */
+  limit?: number
+  /**
+   * Also require leaf widths to match `text.length`. Off by default: `text` on
+   * a leaf is overloaded (it holds tag attributes on element nodes and is empty
+   * on `spacing`/`empty_line`), so only `text`-kind leaves are checked even
+   * when this is on.
+   */
+  checkLeafWidths?: boolean
+}
+
+/**
+ * Verify that `root` partitions `[0..sourceLength]`.
+ *
+ * Returns an empty array when the tree is well formed. Never throws.
+ */
+export function checkPartition(
+  root: GreenNode,
+  sourceLength: number,
+  options: CheckPartitionOptions = {},
+): PartitionViolation[] {
+  const limit = options.limit ?? 200
+  const checkLeafWidths = options.checkLeafWidths ?? false
+  const violations: PartitionViolation[] = []
+
+  const report = (
+    kind: PartitionViolationKind,
+    path: string,
+    nodeKind: string,
+    detail: string,
+  ): void => {
+    if (violations.length < limit) {
+      violations.push({ kind, path, nodeKind, detail })
+    }
+  }
+
+  if (root.width !== sourceLength) {
+    report(
+      'root-coverage',
+      root.kind,
+      root.kind,
+      `root width ${root.width}, source length ${sourceLength}`,
+    )
+  }
+
+  // Iterative walk: documents nest deeply enough (lists inside boxes inside
+  // centres) that recursion here is a needless risk in a validator.
+  const stack: { node: GreenNode; path: string }[] = [{ node: root, path: root.kind }]
+
+  while (stack.length > 0 && violations.length < limit) {
+    const { node, path } = stack.pop()!
+    const children = node.children as readonly GreenNode[]
+
+    if (children.length === 0) {
+      // A childless node is either a token whose whole width is its own
+      // content, or an empty element (`[b][/b]`) whose whole width is its two
+      // delimiters. Either way the delimiters cannot claim more room than the
+      // node occupies.
+      if (node.leadingWidth + node.trailingWidth > node.width) {
+        report(
+          'width-mismatch',
+          path,
+          node.kind,
+          `width ${node.width} but leading ${node.leadingWidth} + trailing ${node.trailingWidth}`,
+        )
+      }
+      if (
+        checkLeafWidths &&
+        node.kind === 'text' &&
+        node.width !== node.text.length
+      ) {
+        report(
+          'leaf-width',
+          path,
+          node.kind,
+          `width ${node.width} but text.length ${node.text.length}`,
+        )
+      }
+      continue
+    }
+
+    // Belt and braces: the constructor computes this, so a failure here means
+    // somebody built a GreenNode by a route that bypassed it.
+    let sum = node.leadingWidth + node.trailingWidth
+    for (let i = 0; i < children.length; i++) {
+      sum += children[i].width
+      stack.push({ node: children[i], path: `${path}/${children[i].kind}[${i}]` })
+    }
+    if (sum !== node.width) {
+      report(
+        'width-mismatch',
+        path,
+        node.kind,
+        `width ${node.width} but children + delimiters sum to ${sum}`,
+      )
+    }
+  }
+
+  return violations
+}
+
+/** Convenience for tests: throw a readable error if the invariant is broken. */
+export function assertPartition(root: GreenNode, sourceLength: number): void {
+  const violations = checkPartition(root, sourceLength, { limit: 10 })
+  if (violations.length === 0) return
+  const lines = violations.map(v => `  ${v.kind} at ${v.path}: ${v.detail}`)
+  throw new Error(`Tree does not partition the source:\n${lines.join('\n')}`)
+}

package/src/Syntax/preserveNodeIds.ts

@@ -0,0 +1,201 @@
+/**
+ * DocumentEngine — preserveNodeIds
+ *
+ * Carry `RedNode.id` across a reparse, so a node that did not change keeps
+ * the identity it had in the previous tree.
+ *
+ * Why this exists: every element in the rendered HTML embeds `data-node-id`
+ * (the preview→Monaco click mapping reads it), and ids used to be regenerated
+ * on every parse. That made the HTML of *unchanged* subtrees differ between
+ * keystrokes, which defeated the DOMMorpher's `isEqualNode` fast path —
+ * measured on a one-character edit, only 2 of 5 top-level subtrees compared
+ * equal with ids embedded, versus 4 of 5 without them. Stable ids also make
+ * any future node-identity feature (selection preservation, collaboration)
+ * possible at all: two versions of a document can refer to the same node.
+ *
+ * The pairing strategy mirrors `NodeMatcher`'s Phase 0, which is validated at
+ * scale: walk both trees in lockstep, trim the common prefix and suffix of
+ * every child list, and only descend into the changed window when it is the
+ * single-child shape a keystroke produces. Three tiers of matching:
+ *
+ *   1. Reference-equal greens — the incremental splice shares untouched
+ *      subtrees by reference, so this covers almost everything while typing.
+ *      Identical by construction; adopt the whole subtree, no checks.
+ *   2. Hash-equal greens — the full-rebuild path reparses everything, so
+ *      nothing is reference-equal but unchanged subtrees hash the same.
+ *      Verified structurally before adopting (a 32-bit hash is not proof).
+ *   3. Same kind at the same position — the node itself changed (it is the
+ *      one being typed into) but it is still "the same element", so it keeps
+ *      its id and its children get the prefix/suffix treatment.
+ *
+ * Uniqueness is preserved by construction: the walk is strictly positional
+ * (old child i pairs with at most one new child), so an old id is adopted at
+ * most once, and ids never adopted stay as freshly generated — the global
+ * counter guarantees those cannot collide with anything older.
+ */
+
+import type { RedNode } from './RedNode'
+import type { NodeId } from '../Types/core'
+
+export interface PreserveIdsStats {
+  /** Nodes that kept their previous id */
+  adopted: number
+  /** Nodes in the new tree that kept a fresh id */
+  fresh: number
+}
+
+/**
+ * Copy ids from `oldRoot`'s tree onto the matching nodes of `newRoot`'s tree.
+ * Returns how many nodes were matched, for tests and diagnostics.
+ */
+export function preserveNodeIds(oldRoot: RedNode, newRoot: RedNode): PreserveIdsStats {
+  const stats: PreserveIdsStats = { adopted: 0, fresh: 0 }
+  preserve(oldRoot, newRoot, stats)
+  stats.fresh = countNodes(newRoot) - stats.adopted
+  return stats
+}
+
+function preserve(oldNode: RedNode, newNode: RedNode, stats: PreserveIdsStats): void {
+  // Tier 0: with red-subtree reuse, "both" nodes are often the SAME object,
+  // adopted from the old tree — its ids are already its own. Note these
+  // subtrees are not counted in `adopted`.
+  if (oldNode === newNode) return
+
+  // Tier 1: the splice shared this subtree by reference — identical by
+  // construction, adopt wholesale.
+  if (oldNode.green === newNode.green) {
+    adoptSubtree(oldNode, newNode, stats)
+    return
+  }
+
+  // Tier 2: same hash across a full reparse. Verify before trusting it —
+  // a false adoption would be harmless for correctness (the morpher compares
+  // content, not ids) but would pair ids across genuinely different nodes.
+  if (oldNode.green._hash === newNode.green._hash && verifyAndAdopt(oldNode, newNode, stats)) {
+    return
+  }
+
+  // Tier 3: the changed path itself. Same kind at the same position is the
+  // same element with different content — keep its identity and descend.
+  if (oldNode.kind !== newNode.kind) return
+  adoptId(oldNode, newNode, stats)
+
+  const oldKids = oldNode.children
+  const newKids = newNode.children
+  const limit = Math.min(oldKids.length, newKids.length)
+
+  // Common prefix.
+  let lo = 0
+  while (lo < limit && matchable(oldKids[lo], newKids[lo])) {
+    preserve(oldKids[lo], newKids[lo], stats)
+    lo++
+  }
+
+  // Common suffix, stopping before the prefix already consumed.
+  let oldHi = oldKids.length - 1
+  let newHi = newKids.length - 1
+  while (oldHi >= lo && newHi >= lo && matchable(oldKids[oldHi], newKids[newHi])) {
+    preserve(oldKids[oldHi], newKids[newHi], stats)
+    oldHi--
+    newHi--
+  }
+
+  // A single changed child on both sides is the shape a keystroke produces:
+  // descend so its own untouched children still keep their ids. A wider
+  // window (multi-block paste, reordering) keeps fresh ids — guessing there
+  // would risk pairing unrelated nodes.
+  if (oldHi === lo && newHi === lo) {
+    preserve(oldKids[lo], newKids[lo], stats)
+  }
+}
+
+/**
+ * Whether two positionally aligned children are the SAME content, and so can
+ * be walked in lockstep while trimming the common prefix and suffix.
+ *
+ * Deliberately not "same kind": that is true of any two paragraphs, so a
+ * single inserted block would let the trim march through the whole list
+ * pairing each survivor with its neighbour — renumbering every block after
+ * the insertion and, worse, assigning ids that are still in use elsewhere.
+ * Evidence of sameness has to be content-based; kind similarity is only
+ * enough for the one changed child in the middle, which `preserve` handles.
+ */
+function matchable(oldNode: RedNode, newNode: RedNode): boolean {
+  return (
+    oldNode.green === newNode.green ||
+    oldNode.green._hash === newNode.green._hash
+  )
+}
+
+/**
+ * Adopt every id of a subtree pair whose greens are the same object.
+ *
+ * Reference-equal greens guarantee identical *green* structure, but a red
+ * tree can be mutated directly (`transact` deletes red children without
+ * touching green, and with interning on, an old mutated red can still sit on
+ * a pool-shared green). When the red child lists disagree, fall back to the
+ * full tiered walk for that level instead of indexing out of bounds.
+ */
+function adoptSubtree(oldNode: RedNode, newNode: RedNode, stats: PreserveIdsStats): void {
+  if (oldNode === newNode) return
+  adoptId(oldNode, newNode, stats)
+  const oldKids = oldNode.children
+  const newKids = newNode.children
+  if (oldKids.length !== newKids.length) {
+    const limit = Math.min(oldKids.length, newKids.length)
+    for (let i = 0; i < limit; i++) {
+      if (!matchable(oldKids[i], newKids[i])) break
+      preserve(oldKids[i], newKids[i], stats)
+    }
+    return
+  }
+  for (let i = 0; i < oldKids.length; i++) {
+    adoptSubtree(oldKids[i], newKids[i], stats)
+  }
+}
+
+/**
+ * Structurally verify a hash match and, only if the whole subtree confirms,
+ * adopt every id. Mirrors `NodeMatcher.verifyIdentical`: pairs are collected
+ * during the check and committed atomically, so a mismatch (hash collision)
+ * leaves nothing half-adopted before the caller falls through to Tier 3.
+ */
+function verifyAndAdopt(oldNode: RedNode, newNode: RedNode, stats: PreserveIdsStats): boolean {
+  const pairs: RedNode[] = []
+  if (!verifyIdentical(oldNode, newNode, pairs)) return false
+  for (let i = 0; i < pairs.length; i += 2) {
+    adoptId(pairs[i], pairs[i + 1], stats)
+  }
+  return true
+}
+
+function verifyIdentical(oldNode: RedNode, newNode: RedNode, pairs: RedNode[]): boolean {
+  if (
+    oldNode.kind !== newNode.kind ||
+    oldNode.text !== newNode.text ||
+    oldNode.children.length !== newNode.children.length
+  ) {
+    return false
+  }
+
+  pairs.push(oldNode, newNode)
+
+  const oldKids = oldNode.children
+  const newKids = newNode.children
+  for (let i = 0; i < oldKids.length; i++) {
+    if (!verifyIdentical(oldKids[i], newKids[i], pairs)) return false
+  }
+  return true
+}
+
+function adoptId(oldNode: RedNode, newNode: RedNode, stats: PreserveIdsStats): void {
+  // `id` is readonly for consumers; this module is the one sanctioned writer.
+  ;(newNode as { id: NodeId }).id = oldNode.id
+  stats.adopted++
+}
+
+function countNodes(root: RedNode): number {
+  let n = 1
+  for (const child of root.children) n += countNodes(child)
+  return n
+}

package/src/Tests/ASTOptimizerIdempotence.test.ts

@@ -0,0 +1,77 @@
+import { describe, it, expect } from 'vitest'
+import { BBCodeDocumentModel } from '../BBCode/BBCodeDocumentModel'
+import { BBCodeExporter } from '../Visitors/BBCodeExporter'
+import { ASTOptimizer } from '../Transformers/ASTOptimizer'
+import { TagRegistry } from '../Model/TagRegistry'
+
+/**
+ * Regression: ASTOptimizer must be idempotent — optimizing its own output has to
+ * be a no-op.
+ *
+ * The fuzz suite originally found this but could not report it: it ran on an
+ * unseeded `Math.random()`, so the failure showed up in ~50% of runs and the
+ * payload that caused it was discarded. Seeding the fuzzer made CI
+ * deterministic, but the default seed happens to miss this case, so the
+ * minimized repro is pinned here.
+ *
+ * The bug: paragraphs exist because a block element splits inline content. When
+ * a rule deleted that block — here an empty `[notice]` — the two halves were
+ * left as adjacent `paragraph` siblings with nothing between them, and
+ * `paragraph` is unmergeable. Exporting that tree emits no separator, so
+ * re-parsing folded the paragraphs back into one and a *second* optimizer pass
+ * found more work. Fixed by `ASTOptimizer.isOrphanedParagraphPair`, which merges
+ * that specific shape — one the parser itself never produces.
+ *
+ * Other seeds that reproduced it: 2, 7, 42, 99999.
+ * Repro a whole corpus with: QUASAR_FUZZ_SEED=2 npm test
+ */
+describe('ASTOptimizer — idempotence', () => {
+  const registry = new TagRegistry()
+  const REPRO = '[url=https://osu.ppy.sh][/url][notice][/notice][url=https://osu.ppy.sh]'
+
+  /** Optimize → export → re-parse → re-optimize. Should need zero mutations. */
+  function reoptimizationOps(source: string): number {
+    const model = new BBCodeDocumentModel({ source, strictMode: false })
+    new ASTOptimizer().transform(model)
+
+    const exported = new BBCodeExporter(registry).export(model.redRoot!)
+
+    const reparsed = new BBCodeDocumentModel({ source: exported, strictMode: false })
+    const { transaction } = new ASTOptimizer().transform(reparsed)
+    return transaction?.operations.length ?? 0
+  }
+
+  function optimizeAndExport(source: string): string {
+    const model = new BBCodeDocumentModel({ source, strictMode: false })
+    new ASTOptimizer().transform(model)
+    return new BBCodeExporter(registry).export(model.redRoot!)
+  }
+
+  it('optimizing already-optimized output is a no-op', () => {
+    expect(reoptimizationOps(REPRO)).toBe(0)
+  })
+
+  it('reaches its fixed point in a single pass', () => {
+    const pass1 = optimizeAndExport(REPRO)
+    expect(optimizeAndExport(pass1)).toBe(pass1)
+  })
+
+  it('never loses or alters visible text', () => {
+    const model = new BBCodeDocumentModel({ source: REPRO, strictMode: false })
+    const before = model.redRoot!.green.text
+    new ASTOptimizer().transform(model)
+    expect(model.redRoot!.green.text).toBe(before)
+  })
+
+  // The fix is deliberately narrow: only paragraphs with *nothing* between them
+  // collapse. An authored blank line is a real break and must survive.
+  it('preserves paragraphs separated by a blank line', () => {
+    const source = '[b]uno[/b]\n\n[b]dos[/b]'
+    const optimized = optimizeAndExport(source)
+    expect(optimized).toContain('uno')
+    expect(optimized).toContain('dos')
+    expect(optimized).toMatch(/\n/)
+    // And it is still a fixed point.
+    expect(optimizeAndExport(optimized)).toBe(optimized)
+  })
+})