@miliastry/quasar 1.0.0

package/src/Syntax/GreenNode.ts

@@ -0,0 +1,324 @@
+/**
+ * DocumentEngine — GreenNode (Immutable Syntax Tree)
+ *
+ * The Green Tree is IMMUTABLE. It never changes after creation.
+ * This is the "compiled" representation of the source text.
+ *
+ * Properties:
+ * - Fully immutable (readonly)
+ * - Contains only structural information (kind, text, children, widths)
+ * - **No absolute position** — see below
+ * - No parent references
+ * - No identity (IDs live in Red Tree)
+ * - Can be shared across document versions
+ * - Safe to cache
+ *
+ * ─── Why there is no `range` ────────────────────────────────────────────────
+ *
+ * A green node knows how WIDE it is, not WHERE it is. That is the whole point:
+ * the two `[color=#e8b04b]` runs in a gradient are the same structure, and if
+ * each carried its own offsets they could never be the same object. Measured on
+ * the reference document: 1736 nodes, but only 702 distinct structures — 59.6%
+ * of the tree is duplicate.
+ *
+ * Position is a property of a node's PLACE in a tree, so it lives on the red
+ * node, which is the thing that has a place. `RedNode.range` computes it once
+ * at construction by accumulating widths from the root and caches it, so
+ * consumers see exactly the same `{start, end}` they always did.
+ *
+ * This also makes incremental editing cheaper: text inserted before a subtree
+ * changes where it is, not what it is, so nothing about it needs rebuilding.
+ * `shiftGreen` — which used to deep-copy every following sibling to bump its
+ * offsets — no longer exists, because there is nothing left to shift.
+ *
+ * ─── Why `width` is derived and never passed ────────────────────────────────
+ *
+ * `width = leadingWidth + (children ? Σ child widths : ownWidth) + trailingWidth`
+ *
+ * The partition invariant of roadmap point 14 — every character owned exactly
+ * once — used to be a runtime check that a careless caller could violate. Now
+ * it is arithmetic the constructor performs, so a tree that does not partition
+ * its source cannot be built. `Syntax/partition.ts` still exists to verify the
+ * one thing this cannot: that the root's width matches the source length.
+ *
+ * Inspired by Roslyn's Green Tree and SwiftSyntax.
+ */
+
+import { FNV_OFFSET, hashString, hashUint32 } from './hash'
+
+/**
+ * Shared frozen empty array for childless nodes.
+ *
+ * Over half the nodes in a document are leaves, and each was allocating and
+ * freezing its own empty array — 930 of them on the reference document, for
+ * 17% of the parser. They cannot differ from one another; one instance does.
+ */
+const NO_CHILDREN: readonly GreenNode[] = Object.freeze([])
+
+/**
+ * Whether to freeze children lists at all.
+ *
+ * `Object.freeze` per node cost 32% of the parser, buying a runtime guard
+ * against a mutation that `readonly GreenNode[]` already rejects at compile
+ * time. Keeping it outside production means the throw still happens where
+ * mistakes are made and caught — in development and in the test suite — and
+ * production pays nothing for a check that has never once fired there.
+ *
+ * Evaluated once at module load, so the hot path sees a constant. The `typeof`
+ * guard is because this engine also runs inside Workers, where a bundler may
+ * not have shimmed `process`.
+ */
+const FREEZE_CHILDREN =
+  typeof process === 'undefined' || process.env?.NODE_ENV !== 'production'
+
+// ─── Green Node ────────────────────────────────────────────────
+
+export class GreenNode {
+  /** Syntax kind — maps to TokenKind for leaves, NodeKind for internals */
+  readonly kind: string
+  /** Raw text of this node (empty for internal nodes) */
+  readonly text: string
+  /** Child nodes */
+  readonly children: readonly GreenNode[]
+  /** Total width of this node in characters, delimiters included */
+  readonly width: number
+  /** Whether this is a leaf (token) or internal node */
+  readonly isLeaf: boolean
+
+  /**
+   * Characters this node owns BEFORE its first child — its opening delimiter.
+   * For `[b]hola[/b]` that is the 3 characters of `[b]`.
+   *
+   * `declare` is load-bearing here for the same reason as `_hashCache` below:
+   * a plain field declaration would emit a `defineProperty` per construction.
+   */
+  declare readonly leadingWidth: number
+  /** Characters this node owns AFTER its last child — its closing delimiter. */
+  declare readonly trailingWidth: number
+
+  /**
+   * Memoized structural hash — computed on first access, never invalidated.
+   *
+   * `declare` is load-bearing: with `target: ES2022`, TypeScript defaults
+   * `useDefineForClassFields` to true, so a plain field declaration emits a
+   * per-construction `defineProperty`. Measured on a 19.6 KB document that
+   * cost 23% of the parse phase (0.583 ms → 0.718 ms) for a slot most nodes
+   * never read. `declare` emits nothing; the constructor assigns it directly,
+   * which keeps the object shape monomorphic without the field-init overhead.
+   */
+  private declare _hashCache: number
+
+  /**
+   * @param ownWidth  Width of a childless node's own content: the length of a
+   *                  `text` token, or the 1-2 characters of a newline. Ignored
+   *                  when there are children, whose widths sum to the same
+   *                  thing. An empty element like `[b][/b]` passes 0 and gets
+   *                  its width from its two delimiters.
+   */
+  constructor(
+    kind: string,
+    text: string,
+    children: GreenNode[] = [],
+    leadingWidth: number = 0,
+    trailingWidth: number = 0,
+    ownWidth: number = 0,
+  ) {
+    this.kind = kind
+    this.text = text
+    const count = children.length
+    this.children =
+      count === 0
+        ? NO_CHILDREN
+        : FREEZE_CHILDREN
+          ? Object.freeze([...children])
+          : [...children]
+    this.isLeaf = count === 0
+    this.leadingWidth = leadingWidth
+    this.trailingWidth = trailingWidth
+
+    let inner = ownWidth
+    if (children.length > 0) {
+      inner = 0
+      for (let i = 0; i < children.length; i++) inner += children[i].width
+    }
+    this.width = leadingWidth + inner + trailingWidth
+
+    // -1 is the "not yet computed" sentinel: `_hash` is always `>>> 0`, so it
+    // can never legitimately be negative.
+    this._hashCache = -1
+  }
+
+  // ─── Structural Identity ─────────────────────────────────
+
+  /**
+   * Structural hash of this subtree: `(kind, text, widths, children hashes)`.
+   *
+   * Two subtrees that render identically at different offsets ARE structurally
+   * identical — that is the entire premise of structural sharing, and now that
+   * green nodes carry no position it is simply true rather than something the
+   * hash had to be careful to arrange.
+   *
+   * The widths ARE folded in: they are structure, not position. A `text` leaf
+   * of width 4 and one of width 5 are different nodes.
+   *
+   * Computed lazily: parsing paths that never intern pay nothing. Safe to
+   * memoize because a GreenNode is immutable once constructed.
+   */
+  get _hash(): number {
+    let h = this._hashCache
+    if (h === -1) {
+      h = hashString(FNV_OFFSET, this.kind)
+      h = hashString(h, this.text)
+      h = hashUint32(h, this.width)
+      h = hashUint32(h, this.leadingWidth)
+      h = hashUint32(h, this.trailingWidth)
+      h = hashUint32(h, this.children.length)
+      for (const child of this.children as GreenNode[]) {
+        h = hashUint32(h, child._hash)
+      }
+      this._hashCache = h
+    }
+    return h
+  }
+
+  // ─── Query Methods ───────────────────────────────────────
+
+  /** Get child at index */
+  childAt(index: number): GreenNode | undefined {
+    return this.children[index] as GreenNode | undefined
+  }
+
+  /** Find first child matching predicate */
+  findChild(predicate: (n: GreenNode) => boolean): GreenNode | undefined {
+    return (this.children as GreenNode[]).find(predicate)
+  }
+
+  /** Count of direct children */
+  get childCount(): number {
+    return this.children.length
+  }
+
+  /** Check if this node has a specific kind */
+  isKind(kind: string): boolean {
+    return this.kind === kind
+  }
+
+  /**
+   * Offset of this node's first child, relative to this node's own start.
+   * Equal to `leadingWidth`; named for the places that read it as a position.
+   */
+  get innerOffset(): number {
+    return this.leadingWidth
+  }
+
+  /** Width of the span between this node's delimiters. */
+  get innerWidth(): number {
+    return this.width - this.leadingWidth - this.trailingWidth
+  }
+
+  /** Walk all descendants in pre-order */
+  walk(visitor: (node: GreenNode, depth: number) => void | 'skip', depth: number = 0): void {
+    const result = visitor(this, depth)
+    if (result !== 'skip') {
+      for (const child of this.children as GreenNode[]) {
+        child.walk(visitor, depth + 1)
+      }
+    }
+  }
+
+  /**
+   * Walk all descendants in pre-order, with each node's absolute offset.
+   *
+   * The replacement for the walks that used to read `node.range`. Offsets are
+   * accumulated on the way down, which is the only place they exist.
+   */
+  walkWithOffset(
+    visitor: (node: GreenNode, start: number, depth: number) => void | 'skip',
+    start: number = 0,
+    depth: number = 0,
+  ): void {
+    if (visitor(this, start, depth) === 'skip') return
+    let offset = start + this.leadingWidth
+    for (const child of this.children as GreenNode[]) {
+      child.walkWithOffset(visitor, offset, depth + 1)
+      offset += child.width
+    }
+  }
+
+  /** Convert to a debug string */
+  toString(depth: number = 0, start: number = 0): string {
+    const indent = '  '.repeat(depth)
+    const rangeStr = `[${start}..${start + this.width}]`
+    const textPreview = this.text.length > 20
+      ? this.text.slice(0, 20) + '...'
+      : this.text
+
+    let result = `${indent}${this.kind} ${rangeStr} "${textPreview}"`
+    if (this.children.length > 0) {
+      let offset = start + this.leadingWidth
+      const parts: string[] = []
+      for (const child of this.children as GreenNode[]) {
+        parts.push(child.toString(depth + 1, offset))
+        offset += child.width
+      }
+      result += '\n' + parts.join('\n')
+    }
+    return result
+  }
+
+  /** Get the source text for this node, given where it starts. */
+  getSourceText(source: string, start: number): string {
+    return source.slice(start, start + this.width)
+  }
+}
+
+// ─── Green Tree Factory ────────────────────────────────────────
+
+/**
+ * Build an internal node. Its width comes from its children plus its own
+ * delimiters — there is no way to declare a width that disagrees with them.
+ */
+export function greenNode(
+  kind: string,
+  text: string,
+  children: GreenNode[] = [],
+  leadingWidth: number = 0,
+  trailingWidth: number = 0,
+  ownWidth: number = 0,
+): GreenNode {
+  return new GreenNode(kind, text, children, leadingWidth, trailingWidth, ownWidth)
+}
+
+/**
+ * Absolute offsets of `node`'s children, given where `node` itself starts.
+ *
+ * Returns `children.length + 1` entries: entry `i` is where child `i` begins
+ * and entry `i + 1` is where it ends, so a run of children `[a, b)` spans
+ * `offsets[a] .. offsets[b]`.
+ *
+ * For code that walks the green tree and needs source positions — analyzers
+ * reporting diagnostics, transforms recording what they collapsed. Green nodes
+ * have no positions of their own, so a walk that needs them accumulates them,
+ * and this is the one-liner for doing that at a single level.
+ */
+export function childOffsets(node: GreenNode, start: number): number[] {
+  const children = node.children as readonly GreenNode[]
+  const offsets = new Array<number>(children.length + 1)
+  let offset = start + node.leadingWidth
+  for (let i = 0; i < children.length; i++) {
+    offsets[i] = offset
+    offset += children[i].width
+  }
+  offsets[children.length] = offset
+  return offsets
+}
+
+/**
+ * Build a token. `width` defaults to the text's own length, which is right for
+ * every leaf whose text IS its source (`text` nodes). Leaves whose `text` holds
+ * something else — `spacing` and `empty_line` carry `''` but occupy 1-2
+ * characters, element nodes carry their attributes — must pass it.
+ */
+export function greenLeaf(kind: string, text: string, width: number = text.length): GreenNode {
+  return new GreenNode(kind, text, [], 0, 0, width)
+}

package/src/Syntax/GreenNodePool.ts

@@ -0,0 +1,269 @@
+/**
+ * DocumentEngine — GreenNodePool (Structural Sharing via Interning)
+ *
+ * Roslyn-grade GreenNode Deduplication Pool.
+ *
+ * Ensures that structurally identical subtrees share the EXACT SAME
+ * GreenNode memory address in RAM. This is structural sharing:
+ * when the same BBCode block is pasted 10 times, the parser creates
+ * GreenNodes for it ONCE, and all 10 copies reference the same objects.
+ *
+ * Architecture:
+ * - Flyweight Pattern: GreenNodes are pure structural data without parent pointers.
+ * - Hash-based lookup: O(1) amortized via _hash property on GreenNode.
+ * - Reference-equality verification: identical children = same reference (because
+ *   children are interned first, bottom-up construction guarantees this).
+ * - Per-document scope: interners are created per BBCodeDocumentModel, cleared on dispose.
+ *
+ * Key insight: because GreenNodes are immutable and children are interned bottom-up,
+ * two nodes with the same (kind, text, widths, children-references) are structurally
+ * identical. We use _hash for fast lookup, then verify with reference equality on
+ * children.
+ *
+ * ─── Why this used to be broken (roadmap S2) ────────────────────────────────
+ *
+ * Green nodes used to carry an absolute `range`, and the two halves of the pool
+ * disagreed about what to do with it:
+ *
+ *  - `internLeaf` keyed on `kind:text` alone, so the second `"Hello"` in a
+ *    document got back the node built for the FIRST one — carrying the first
+ *    one's offsets. Enabling the interner made every position downstream lie
+ *    silently: incremental reparse, `findNodeAtOffset`, diagnostics, the
+ *    Monaco↔AST mapping.
+ *  - `internNode` did the opposite and required identical start AND end before
+ *    deduplicating, which no two distinct occurrences can have. The same block
+ *    pasted ten times gave 0% dedup and 100% of the scanning cost.
+ *
+ * Neither was fixable while position lived on the green node, because those are
+ * the only two options: ignore it and lie, or respect it and never match. The
+ * fix was to remove position from the green tree entirely — see `GreenNode.ts`.
+ * Now `kind:text:width` genuinely identifies a leaf, and interning is simply
+ * correct.
+ */
+
+import { GreenNode } from './GreenNode'
+
+export interface GreenNodePoolStats {
+  size: number
+  hits: number
+  misses: number
+  deduplicatedCount: number
+  /** Number of hash collisions that were verified as true duplicates */
+  collisionHits: number
+  /** Number of hash collisions that were false positives */
+  collisionMisses: number
+}
+
+/**
+ * How much of the tree to deduplicate.
+ *
+ * Measured on the 19.6 KB reference document, against no interning at all:
+ *
+ * | mode     | parse time | green tree memory |
+ * |----------|------------|-------------------|
+ * | `leaves` | +16%       | −42%              |
+ * | `full`   | +110%      | −56%              |
+ *
+ * `leaves` is the default because it buys three quarters of the memory for a
+ * seventh of the time. Interning an internal node means hashing its whole
+ * subtree — `_hash` is lazy, so a parse that never interns never pays for it —
+ * and then walking its children to verify a bucket hit. Leaves cost one string
+ * key and one Map lookup, and they are where the duplication is: the reference
+ * document has 930 leaves and only 122 distinct ones.
+ */
+export type InterningMode = 'leaves' | 'full'
+
+export class GreenNodePool {
+  private static _instance: GreenNodePool | null = null
+  private readonly _mode: InterningMode
+  private _leafCache = new Map<string, GreenNode>()
+  private _nodeCache = new Map<number, GreenNode[]>()
+  private _hits = 0
+  private _misses = 0
+  private _collisionHits = 0
+  private _collisionMisses = 0
+
+  /** Singleton instance accessor (for backward compatibility) */
+  static get instance(): GreenNodePool {
+    if (!GreenNodePool._instance) {
+      GreenNodePool._instance = new GreenNodePool()
+    }
+    return GreenNodePool._instance
+  }
+
+  constructor(mode: InterningMode = 'leaves') {
+    this._mode = mode
+  }
+
+  /**
+   * Create a new isolated interner (for per-document use).
+   * The returned pool shares no cache with the singleton.
+   */
+  static create(mode: InterningMode = 'leaves'): GreenNodePool {
+    return new GreenNodePool(mode)
+  }
+
+  /**
+   * Intern (deduplicate) a leaf GreenNode.
+   * If an identical leaf exists in the pool, returns the cached instance.
+   * Leaf key: `kind:text` — deterministic, no collision possible.
+   */
+  internLeaf(kind: string, text: string, width = text.length): GreenNode {
+    // `width` belongs in the key: `spacing` carries no text but occupies one or
+    // two characters depending on whether the source used `\n` or `\r\n`, and
+    // those are different tokens.
+    const key = `${kind}:${width}:${text}`
+    const existing = this._leafCache.get(key)
+    if (existing) {
+      this._hits++
+      return existing
+    }
+
+    this._misses++
+    const newNode = new GreenNode(kind, text, [], 0, 0, width)
+    this._leafCache.set(key, newNode)
+    return newNode
+  }
+
+  /**
+   * Intern an internal GreenNode (structural sharing).
+   *
+   * Strategy:
+   * 1. Compute hash from (kind, text, children's _hash values) — O(n) where n = children count
+   * 2. Look up hash in bucket — O(1) amortized
+   * 3. Verify with reference equality on children — O(n) but cache-friendly
+   * 4. If verified, return existing (structural sharing achieved)
+   * 5. If not, store and return new node
+   *
+   * Because children are interned bottom-up, identical children are the same reference.
+   * This makes reference equality a perfect structural equality check.
+   */
+  internNode(
+    kind: string,
+    text: string,
+    children: GreenNode[],
+    leadingWidth = 0,
+    trailingWidth = 0,
+  ): GreenNode {
+    if (this._mode === 'leaves') {
+      return new GreenNode(kind, text, children, leadingWidth, trailingWidth)
+    }
+    // Fast path: check if we already have this exact node by hash
+    // For nodes with same kind+text+children-count, the hash is likely unique.
+    // We use the hash as the bucket key, then verify all children by reference.
+    const bucketKey = this.computeBucketKey(kind, text, children, leadingWidth, trailingWidth)
+    const bucket = this._nodeCache.get(bucketKey)
+    if (bucket) {
+      for (const existing of bucket) {
+        if (existing.kind === kind
+          && existing.text === text
+          && existing.children.length === children.length
+          && existing.leadingWidth === leadingWidth
+          && existing.trailingWidth === trailingWidth) {
+          // Verify children by reference (O(n) but cache-linear)
+          let allMatch = true
+          for (let i = 0; i < children.length; i++) {
+            if (existing.children[i] !== children[i]) {
+              allMatch = false
+              break
+            }
+          }
+          if (allMatch) {
+            this._hits++
+            this._collisionHits++
+            return existing
+          }
+        }
+      }
+      // Hash collision — different node with same bucket key
+      this._collisionMisses++
+    }
+
+    this._misses++
+    const newNode = new GreenNode(kind, text, children, leadingWidth, trailingWidth)
+
+    if (bucket) {
+      bucket.push(newNode)
+    } else {
+      this._nodeCache.set(bucketKey, [newNode])
+    }
+
+    return newNode
+  }
+
+  /**
+   * Compute a bucket key for internal nodes.
+   * Uses (kind, text, children-count, first-child-hash) for fast bucketing.
+   * This gives O(1) bucket lookup in the common case.
+   */
+  private computeBucketKey(
+    kind: string,
+    text: string,
+    children: GreenNode[],
+    leadingWidth: number,
+    trailingWidth: number,
+  ): number {
+    let h = this.fnv1a(kind)
+    h = this.hashCombine(h, this.fnv1a(text))
+    h = this.hashCombine(h, leadingWidth)
+    h = this.hashCombine(h, trailingWidth)
+    h = this.hashCombine(h, children.length)
+    if (children.length > 0) {
+      h = this.hashCombine(h, children[0]._hash)
+    }
+    return h
+  }
+
+  private fnv1a(str: string): number {
+    let hash = 0x811c9dc5
+    for (let i = 0; i < str.length; i++) {
+      hash ^= str.charCodeAt(i)
+      hash = Math.imul(hash, 0x01000193)
+    }
+    return hash >>> 0
+  }
+
+  private hashCombine(seed: number, value: number): number {
+    seed ^= value
+    seed = Math.imul(seed, 0x01000193)
+    return seed >>> 0
+  }
+
+  /** Clear the pool cache */
+  clear(): void {
+    this._leafCache.clear()
+    this._nodeCache.clear()
+    this._hits = 0
+    this._misses = 0
+    this._collisionHits = 0
+    this._collisionMisses = 0
+  }
+
+  /** Get current pool statistics */
+  get stats(): GreenNodePoolStats {
+    return {
+      size: this._leafCache.size + this._nodeCache.size,
+      hits: this._hits,
+      misses: this._misses,
+      deduplicatedCount: this._hits,
+      collisionHits: this._collisionHits,
+      collisionMisses: this._collisionMisses,
+    }
+  }
+}
+
+/** Convenience helper for interning green leaves */
+export function internGreenLeaf(kind: string, text: string, width = text.length): GreenNode {
+  return GreenNodePool.instance.internLeaf(kind, text, width)
+}
+
+/** Convenience helper for interning green nodes */
+export function internGreenNode(
+  kind: string,
+  text: string,
+  children: GreenNode[],
+  leadingWidth = 0,
+  trailingWidth = 0,
+): GreenNode {
+  return GreenNodePool.instance.internNode(kind, text, children, leadingWidth, trailingWidth)
+}