@px-lsp/server 0.1.0

package/src/parser/cst.ts

@@ -0,0 +1,257 @@
+// Concrete Syntax Tree node types and utilities for Paradox script.
+//
+// All offsets are UTF-16 code-unit offsets into the source string (i.e. the
+// same units JavaScript string indexing / `String.prototype.slice` use, and
+// the same units the VS Code LSP uses for `character` positions on a line).
+
+export interface Range {
+  start: number;
+  end: number;
+}
+
+export type Operator = "=" | "?=" | "==" | "!=" | "<" | "<=" | ">" | ">=";
+
+export interface RootNode {
+  kind: "root";
+  statements: Statement[];
+  range: Range;
+}
+
+export interface AssignmentNode {
+  kind: "assignment";
+  key: ScalarNode;
+  op: Operator | null; // null for GUI-style `key { ... }` with no operator
+  value: ValueNode | null; // null when the value is missing (error recorded)
+  range: Range;
+}
+
+// A bare list element, e.g. the `brave` / `ambitious` in `traits = { brave ambitious }`
+// or an anonymous block in `{ 1 2 } { 3 4 }`.
+export interface ValueStatementNode {
+  kind: "value";
+  value: ValueNode;
+  range: Range;
+}
+
+export type Statement = AssignmentNode | ValueStatementNode;
+
+export interface ScalarNode {
+  kind: "scalar";
+  text: string; // the raw text of the scalar; for quoted scalars, WITHOUT surrounding quotes
+  quoted: boolean;
+  range: Range; // for quoted scalars, INCLUDES the surrounding quotes
+}
+
+export interface BlockNode {
+  kind: "block";
+  statements: Statement[];
+  range: Range;
+  openBrace: number; // offset of `{`
+  closeBrace: number | null; // offset of `}`, or null if missing (error recorded)
+}
+
+// e.g. `color = rgb { 255 0 0 }` — a scalar tag immediately followed by a block.
+export interface TaggedBlockNode {
+  kind: "tagged-block";
+  tag: ScalarNode;
+  block: BlockNode;
+  range: Range;
+}
+
+export type ValueNode = ScalarNode | BlockNode | TaggedBlockNode;
+
+export interface CommentNode {
+  text: string; // includes the leading `#`
+  range: Range;
+  line: number; // 0-based line number
+}
+
+export type ParseErrorCode =
+  | "unclosed-brace" // report at the OPENING brace of the unclosed block
+  | "stray-close" // `}` with no open block
+  | "unterminated-string" // recover at end of line
+  | "missing-value"; // `key =` with nothing parseable after
+
+export interface ParseError {
+  code: ParseErrorCode;
+  message: string;
+  range: Range;
+}
+
+export interface ParseResult {
+  root: RootNode;
+  errors: ParseError[];
+  comments: CommentNode[];
+}
+
+// -------------------------------------------------------------------------
+// LineIndex — maps between offsets and (line, character) positions.
+// -------------------------------------------------------------------------
+
+export class LineIndex {
+  // lineStarts[i] is the offset of the first character of line i (0-based).
+  private readonly lineStarts: number[];
+  private readonly length: number;
+
+  constructor(text: string) {
+    this.length = text.length;
+    const starts: number[] = [0];
+    for (let i = 0; i < text.length; i++) {
+      const c = text.charCodeAt(i);
+      if (c === 10 /* \n */) {
+        starts.push(i + 1);
+      } else if (c === 13 /* \r */) {
+        // Treat \r\n as a single break; a lone \r also breaks a line.
+        if (i + 1 < text.length && text.charCodeAt(i + 1) === 10) {
+          starts.push(i + 2);
+          i++;
+        } else {
+          starts.push(i + 1);
+        }
+      }
+    }
+    this.lineStarts = starts;
+  }
+
+  get lineCount(): number {
+    return this.lineStarts.length;
+  }
+
+  lineStart(line: number): number {
+    if (line < 0) return 0;
+    if (line >= this.lineStarts.length) return this.length;
+    return this.lineStarts[line];
+  }
+
+  positionAt(offset: number): { line: number; character: number } {
+    let o = offset;
+    if (o < 0) o = 0;
+    if (o > this.length) o = this.length;
+    // Binary search for the greatest lineStart <= o.
+    let lo = 0;
+    let hi = this.lineStarts.length - 1;
+    while (lo < hi) {
+      const mid = (lo + hi + 1) >> 1;
+      if (this.lineStarts[mid] <= o) {
+        lo = mid;
+      } else {
+        hi = mid - 1;
+      }
+    }
+    return { line: lo, character: o - this.lineStarts[lo] };
+  }
+
+  offsetAt(pos: { line: number; character: number }): number {
+    let line = pos.line;
+    if (line < 0) line = 0;
+    if (line >= this.lineStarts.length) {
+      return this.length;
+    }
+    const start = this.lineStarts[line];
+    // Clamp character to the end of this line (start of next line, or EOF).
+    const nextStart = line + 1 < this.lineStarts.length ? this.lineStarts[line + 1] : this.length;
+    let ch = pos.character;
+    if (ch < 0) ch = 0;
+    let offset = start + ch;
+    if (offset > nextStart) offset = nextStart;
+    if (offset > this.length) offset = this.length;
+    return offset;
+  }
+}
+
+// -------------------------------------------------------------------------
+// Walk helpers
+// -------------------------------------------------------------------------
+
+function statementChildBlock(stmt: Statement): BlockNode | null {
+  if (stmt.kind === "assignment") {
+    const v = stmt.value;
+    if (v && v.kind === "block") return v;
+    if (v && v.kind === "tagged-block") return v.block;
+    return null;
+  }
+  // value statement
+  const v = stmt.value;
+  if (v.kind === "block") return v;
+  if (v.kind === "tagged-block") return v.block;
+  return null;
+}
+
+/**
+ * Depth-first walk over every statement in the tree. The callback receives the
+ * statement plus the chain of ancestor assignment/block nodes (outermost first).
+ */
+export function walkStatements(
+  root: RootNode | BlockNode,
+  cb: (stmt: Statement, ancestors: readonly (AssignmentNode | BlockNode)[]) => void
+): void {
+  const ancestors: (AssignmentNode | BlockNode)[] = [];
+
+  const visitBlock = (block: BlockNode): void => {
+    for (const stmt of block.statements) {
+      cb(stmt, ancestors);
+      const child = statementChildBlock(stmt);
+      if (child) {
+        if (stmt.kind === "assignment") {
+          ancestors.push(stmt);
+        }
+        ancestors.push(child);
+        visitBlock(child);
+        ancestors.pop();
+        if (stmt.kind === "assignment") {
+          ancestors.pop();
+        }
+      }
+    }
+  };
+
+  if (root.kind === "root") {
+    for (const stmt of root.statements) {
+      cb(stmt, ancestors);
+      const child = statementChildBlock(stmt);
+      if (child) {
+        if (stmt.kind === "assignment") {
+          ancestors.push(stmt);
+        }
+        ancestors.push(child);
+        visitBlock(child);
+        ancestors.pop();
+        if (stmt.kind === "assignment") {
+          ancestors.pop();
+        }
+      }
+    }
+  } else {
+    visitBlock(root);
+  }
+}
+
+function offsetInRange(offset: number, range: Range): boolean {
+  return offset >= range.start && offset <= range.end;
+}
+
+/**
+ * Returns the innermost-last chain of statements whose ranges contain `offset`.
+ * For a cursor sitting between statements inside a block, the path ends at the
+ * enclosing statement chain (i.e. the assignment/value-statement that owns the
+ * block the cursor is inside). Returns null if offset is outside all statements.
+ */
+export function nodeAtOffset(root: RootNode, offset: number): { path: Statement[] } | null {
+  const path: Statement[] = [];
+
+  const searchStatements = (statements: Statement[]): boolean => {
+    for (const stmt of statements) {
+      if (!offsetInRange(offset, stmt.range)) continue;
+      path.push(stmt);
+      const child = statementChildBlock(stmt);
+      if (child && offsetInRange(offset, child.range)) {
+        searchStatements(child.statements);
+      }
+      return true;
+    }
+    return false;
+  };
+
+  searchStatements(root.statements);
+  return path.length > 0 ? { path } : null;
+}

package/src/parser/encoding.ts

@@ -0,0 +1,106 @@
+// Encoding helpers for reading Paradox script / localization files.
+//
+// Paradox files are UTF-8 (often with a BOM). Some older or hand-edited files
+// contain invalid UTF-8 byte sequences (typically Latin-1 / Windows-1252 text);
+// for those we fall back to a latin1 decode so we still get usable text rather
+// than U+FFFD replacement soup.
+
+const UTF8_BOM_0 = 0xef;
+const UTF8_BOM_1 = 0xbb;
+const UTF8_BOM_2 = 0xbf;
+
+export function hasUtf8Bom(buf: Uint8Array): boolean {
+  return buf.length >= 3 && buf[0] === UTF8_BOM_0 && buf[1] === UTF8_BOM_1 && buf[2] === UTF8_BOM_2;
+}
+
+/**
+ * Returns true if `buf` is entirely valid UTF-8 (ignoring a leading BOM).
+ * Single pass, no allocations.
+ */
+export function isValidUtf8(buf: Uint8Array): boolean {
+  let i = hasUtf8Bom(buf) ? 3 : 0;
+  const len = buf.length;
+  while (i < len) {
+    const b0 = buf[i];
+    if (b0 < 0x80) {
+      i++;
+      continue;
+    }
+    let extra: number;
+    let min: number;
+    let codepointHigh: number;
+    if (b0 >= 0xc2 && b0 <= 0xdf) {
+      extra = 1;
+      min = 0x80;
+      codepointHigh = b0 & 0x1f;
+    } else if (b0 >= 0xe0 && b0 <= 0xef) {
+      extra = 2;
+      min = 0x800;
+      codepointHigh = b0 & 0x0f;
+    } else if (b0 >= 0xf0 && b0 <= 0xf4) {
+      extra = 3;
+      min = 0x10000;
+      codepointHigh = b0 & 0x07;
+    } else {
+      // 0x80-0xC1 or 0xF5-0xFF: invalid lead byte.
+      return false;
+    }
+    if (i + extra >= len) return false;
+    let cp = codepointHigh;
+    for (let k = 1; k <= extra; k++) {
+      const b = buf[i + k];
+      if ((b & 0xc0) !== 0x80) return false; // not a continuation byte
+      cp = (cp << 6) | (b & 0x3f);
+    }
+    // Reject overlong encodings, surrogates, and out-of-range code points.
+    if (cp < min) return false;
+    if (cp >= 0xd800 && cp <= 0xdfff) return false;
+    if (cp > 0x10ffff) return false;
+    i += extra + 1;
+  }
+  return true;
+}
+
+/**
+ * Decode a buffer to a string. Prefers UTF-8 (stripping a BOM). If the buffer
+ * is not valid UTF-8, falls back to a latin1 decode and reports that.
+ */
+export function decode(buf: Uint8Array): {
+  text: string;
+  hadBom: boolean;
+  encoding: "utf8" | "utf8-bom" | "latin1-fallback";
+} {
+  const bom = hasUtf8Bom(buf);
+  if (isValidUtf8(buf)) {
+    const body = bom ? buf.subarray(3) : buf;
+    const text = utf8Decode(body);
+    return { text, hadBom: bom, encoding: bom ? "utf8-bom" : "utf8" };
+  }
+  // latin1 fallback: each byte maps 1:1 to U+00xx.
+  const text = latin1Decode(buf);
+  return { text, hadBom: false, encoding: "latin1-fallback" };
+}
+
+function utf8Decode(body: Uint8Array): string {
+  // Prefer Node's TextDecoder when available (fast, correct).
+  if (typeof TextDecoder !== "undefined") {
+    return new TextDecoder("utf-8", { fatal: false }).decode(body);
+  }
+  // Fallback (should not be hit in Node/vitest): naive latin1.
+  return latin1Decode(body);
+}
+
+function latin1Decode(buf: Uint8Array): string {
+  // Build in chunks to avoid apply() stack limits on large buffers.
+  let out = "";
+  const CHUNK = 0x8000;
+  for (let i = 0; i < buf.length; i += CHUNK) {
+    const end = Math.min(i + CHUNK, buf.length);
+    let piece = "";
+    for (let j = i; j < end; j++) {
+      piece += String.fromCharCode(buf[j]);
+    }
+    out += piece;
+  }
+  return out;
+}

package/src/parser/index.ts

@@ -0,0 +1,7 @@
+// Public API of the parser layer. Import from here.
+
+export * from "./cst.js";
+export * from "./lexer.js";
+export * from "./parser.js";
+export * from "./locParser.js";
+export * from "./encoding.js";

package/src/parser/lexer.ts

@@ -0,0 +1,245 @@
+// Hand-written, allocation-light, error-tolerant lexer for Paradox script.
+//
+// Single pass over the source; no per-character regex. Offsets are UTF-16
+// code-unit offsets into the source string.
+
+export type TokenKind =
+  | "lbrace" // {
+  | "rbrace" // }
+  | "op" // = ?= == != < <= > >=
+  | "string" // quoted "..." (text WITHOUT quotes stored separately via range)
+  | "comment" // # to end of line (text includes leading #)
+  | "word" // scalar run
+  | "eof";
+
+export interface Token {
+  kind: TokenKind;
+  start: number;
+  end: number;
+  // For "op": the operator text ("=", "?=", ...).
+  // For "word"/"string"/"comment": not populated (slice the source instead),
+  // but we DO record `unterminated` on unterminated strings.
+  value?: string;
+  // Only meaningful for "string" tokens: true if the closing quote was missing
+  // and the token was terminated at end-of-line / EOF.
+  unterminated?: boolean;
+}
+
+/**
+ * How many newlines one quoted string may cross while a `[` inside it is still
+ * open. The longest multi-line data function measured across the shipped gui
+ * trees is four lines; this is the backstop that keeps a genuinely broken `"[`
+ * from turning the remainder of a file into one token.
+ */
+const MAX_STRING_SPAN = 32;
+
+// Character classification helpers ----------------------------------------
+
+function isWhitespace(c: number): boolean {
+  // space, tab, newline, carriage return, form feed, vertical tab, and the BOM.
+  // Callers strip a leading BOM where they can, but not every one does, and a
+  // BOM glued to the first top-level key would swallow that whole block.
+  return c === 32 || c === 9 || c === 10 || c === 13 || c === 12 || c === 11 || c === 0xfeff;
+}
+
+// Characters that terminate a bare word.
+function isWordTerminator(c: number): boolean {
+  return (
+    isWhitespace(c) ||
+    c === 123 /* { */ ||
+    c === 125 /* } */ ||
+    c === 35 /* # */ ||
+    c === 34 /* " */ ||
+    c === 61 /* = */ ||
+    c === 60 /* < */ ||
+    c === 62 /* > */ ||
+    c === 33 /* ! */
+    // NOTE: `?` is intentionally NOT a terminator on its own. `?=` is handled
+    // specially: a `?` only starts an operator when immediately followed by `=`.
+  );
+}
+
+/**
+ * Lex the entire source into a token array (including a trailing eof token).
+ * Never throws. Comments and strings are emitted as tokens; the parser decides
+ * what to do with them.
+ */
+export function tokenize(text: string): Token[] {
+  const tokens: Token[] = [];
+  const len = text.length;
+  let i = 0;
+
+  while (i < len) {
+    const c = text.charCodeAt(i);
+
+    // Whitespace (newlines are plain whitespace).
+    if (isWhitespace(c)) {
+      i++;
+      continue;
+    }
+
+    // Comment: # to end of line.
+    if (c === 35 /* # */) {
+      const start = i;
+      i++;
+      while (i < len) {
+        const cc = text.charCodeAt(i);
+        if (cc === 10 || cc === 13) break;
+        i++;
+      }
+      tokens.push({ kind: "comment", start, end: i });
+      continue;
+    }
+
+    // Braces.
+    if (c === 123 /* { */) {
+      tokens.push({ kind: "lbrace", start: i, end: i + 1 });
+      i++;
+      continue;
+    }
+    if (c === 125 /* } */) {
+      tokens.push({ kind: "rbrace", start: i, end: i + 1 });
+      i++;
+      continue;
+    }
+
+    // Quoted string.
+    if (c === 34 /* " */) {
+      const start = i;
+      i++;
+      let unterminated = false;
+      // A `[ … ]` data-function expression may be written across several lines
+      //, `visible = "[And(\n\tA,\n\tB\n)]"`, and the games accept it: one
+      // shipped gui tree writes it, and the largest community framework on it
+      // does so in 23 of its 52 .gui files. A newline therefore continues the
+      // string ONLY while a `[` is still open: an ordinary unterminated string
+      // recovers at end of line exactly as before, and a stray `"[` gives up
+      // after MAX_STRING_SPAN lines instead of swallowing the rest of the file.
+      let brackets = 0;
+      let spanned = 0;
+      while (true) {
+        if (i >= len) {
+          unterminated = true;
+          break;
+        }
+        const cc = text.charCodeAt(i);
+        if (cc === 92 /* backslash */) {
+          // Escape: skip the next char (handles \" and \\ etc.).
+          i += 2;
+          continue;
+        }
+        if (cc === 34 /* " */) {
+          i++; // consume closing quote
+          break;
+        }
+        if (cc === 10 || cc === 13) {
+          if (brackets === 0 || spanned >= MAX_STRING_SPAN) {
+            // Unterminated: recover at end of line.
+            unterminated = true;
+            break;
+          }
+          if (cc === 10) spanned++;
+          i++;
+          continue;
+        }
+        if (cc === 91 /* [ */) brackets++;
+        else if (cc === 93 /* ] */ && brackets > 0) brackets--;
+        i++;
+      }
+      const tok: Token = { kind: "string", start, end: i };
+      if (unterminated) tok.unterminated = true;
+      tokens.push(tok);
+      continue;
+    }
+
+    // Operators.
+    if (c === 61 /* = */) {
+      if (i + 1 < len && text.charCodeAt(i + 1) === 61 /* = */) {
+        tokens.push({ kind: "op", start: i, end: i + 2, value: "==" });
+        i += 2;
+      } else {
+        tokens.push({ kind: "op", start: i, end: i + 1, value: "=" });
+        i += 1;
+      }
+      continue;
+    }
+    if (c === 33 /* ! */) {
+      if (i + 1 < len && text.charCodeAt(i + 1) === 61 /* = */) {
+        tokens.push({ kind: "op", start: i, end: i + 2, value: "!=" });
+        i += 2;
+        continue;
+      }
+      // Lone `!`, treat as a one-char word (rare; be tolerant).
+      tokens.push({ kind: "word", start: i, end: i + 1 });
+      i += 1;
+      continue;
+    }
+    if (c === 60 /* < */) {
+      if (i + 1 < len && text.charCodeAt(i + 1) === 61 /* = */) {
+        tokens.push({ kind: "op", start: i, end: i + 2, value: "<=" });
+        i += 2;
+      } else {
+        tokens.push({ kind: "op", start: i, end: i + 1, value: "<" });
+        i += 1;
+      }
+      continue;
+    }
+    if (c === 62 /* > */) {
+      if (i + 1 < len && text.charCodeAt(i + 1) === 61 /* = */) {
+        tokens.push({ kind: "op", start: i, end: i + 2, value: ">=" });
+        i += 2;
+      } else {
+        tokens.push({ kind: "op", start: i, end: i + 1, value: ">" });
+        i += 1;
+      }
+      continue;
+    }
+    if (c === 63 /* ? */) {
+      if (i + 1 < len && text.charCodeAt(i + 1) === 61 /* = */) {
+        tokens.push({ kind: "op", start: i, end: i + 2, value: "?=" });
+        i += 2;
+        continue;
+      }
+      // Lone `?` not followed by `=`, fall through and treat as a word char.
+      // (Do not `continue`; let the word scanner below pick it up.)
+    }
+
+    // Word (scalar). Includes inline-math `@[ ... ]` which may contain spaces.
+    {
+      const start = i;
+      while (i < len) {
+        const cc = text.charCodeAt(i);
+        // Inline math bracket: `@[` ... `]` is ONE token, spaces allowed inside.
+        if (cc === 64 /* @ */ && i + 1 < len && text.charCodeAt(i + 1) === 91 /* [ */) {
+          i += 2;
+          while (i < len && text.charCodeAt(i) !== 93 /* ] */) {
+            // stop runaway at newline to stay tolerant
+            const inner = text.charCodeAt(i);
+            if (inner === 10 || inner === 13) break;
+            i++;
+          }
+          if (i < len && text.charCodeAt(i) === 93) i++; // consume `]`
+          continue;
+        }
+        // A lone `?` (not `?=`) is part of the word.
+        if (cc === 63 /* ? */) {
+          if (i + 1 < len && text.charCodeAt(i + 1) === 61 /* = */) break;
+          i++;
+          continue;
+        }
+        if (isWordTerminator(cc)) break;
+        i++;
+      }
+      if (i === start) {
+        // Defensive: unknown char we didn't advance past, consume one char as a
+        // word so we never loop forever on pathological input.
+        i++;
+      }
+      tokens.push({ kind: "word", start, end: i });
+      continue;
+    }
+  }
+
+  tokens.push({ kind: "eof", start: len, end: len });
+  return tokens;
+}