@px-lsp/server 0.1.0

package/src/parser/locParser.ts

@@ -0,0 +1,276 @@
+// Parser for Paradox's localization "YAML" dialect (it is NOT real YAML).
+//
+// Shape:
+//   l_english:
+//    key:0 "value with $variables$, [GetPlayer.GetName], #bold#!, £gold£"
+//    other_key: "no version number"   # trailing comment
+//
+// Never throws. Offsets are UTF-16 code-unit offsets into the ORIGINAL text
+// (including the BOM, if present) so ranges map back onto the source buffer.
+
+import { Range } from "./cst.js";
+
+export interface LocEntry {
+  key: string;
+  keyRange: Range;
+  version: number | null;
+  value: string; // text between the opening quote and the LAST quote on the line, verbatim
+  valueRange: Range; // covers the text inside the quotes
+  line: number; // 0-based
+}
+
+export type LocErrorCode =
+  "no-header" | "bad-entry" | "tab-indent" | "unterminated-value" | "content-before-header";
+
+export interface LocError {
+  code: LocErrorCode;
+  message: string;
+  range: Range;
+}
+
+export interface LocParseResult {
+  language: string | null; // "english" from `l_english:`; null if none found
+  headerRange: Range | null;
+  entries: LocEntry[];
+  errors: LocError[];
+  hadBom: boolean; // text started with the UTF-8 BOM char U+FEFF
+}
+
+// A header line: optional leading whitespace, `l_<name>:`, then only trailing
+// whitespace / comment.
+const HEADER_RE = /^[ \t]*l_([A-Za-z_]+):[ \t]*(#.*)?$/;
+
+// Key characters: letters, digits, `_ . - '`.
+function isKeyChar(c: string): boolean {
+  return (
+    (c >= "a" && c <= "z") ||
+    (c >= "A" && c <= "Z") ||
+    (c >= "0" && c <= "9") ||
+    c === "_" ||
+    c === "." ||
+    c === "-" ||
+    c === "'"
+  );
+}
+
+export function parseLoc(text: string): LocParseResult {
+  const hadBom = text.length > 0 && text.charCodeAt(0) === 0xfeff;
+  // Work on the raw text but skip the BOM offset when scanning line content.
+  // Offsets we emit are relative to the ORIGINAL text.
+  const entries: LocEntry[] = [];
+  const errors: LocError[] = [];
+  let language: string | null = null;
+  let headerRange: Range | null = null;
+
+  const len = text.length;
+  let lineStart = 0;
+  let lineNo = 0;
+  let headerFound = false;
+
+  while (lineStart <= len) {
+    // Find end of this line (exclusive of the newline).
+    let lineEnd = lineStart;
+    while (lineEnd < len) {
+      const cc = text.charCodeAt(lineEnd);
+      if (cc === 10 || cc === 13) break;
+      lineEnd++;
+    }
+
+    // Compute where the next line starts (handle \r\n).
+    let nextStart = lineEnd;
+    if (nextStart < len) {
+      if (text.charCodeAt(nextStart) === 13 && nextStart + 1 < len && text.charCodeAt(nextStart + 1) === 10) {
+        nextStart += 2;
+      } else {
+        nextStart += 1;
+      }
+    } else {
+      nextStart = len + 1; // ensure the loop terminates after the last line
+    }
+
+    // Slice the line; strip a leading BOM on the very first line.
+    let sliceStart = lineStart;
+    if (lineNo === 0 && hadBom && text.charCodeAt(sliceStart) === 0xfeff) {
+      sliceStart += 1;
+    }
+    const rawLine = text.slice(sliceStart, lineEnd);
+
+    processLine(rawLine, sliceStart);
+
+    lineStart = nextStart;
+    lineNo++;
+    if (lineStart > len) break;
+  }
+
+  if (!headerFound) {
+    errors.push({
+      code: "no-header",
+      message: "No localization header line (e.g. `l_english:`) found.",
+      range: { start: hadBom ? 1 : 0, end: hadBom ? 1 : 0 },
+    });
+  }
+
+  return { language, headerRange, entries, errors, hadBom };
+
+  // ---- inner helpers (closures over the accumulators) ----
+
+  function processLine(line: string, base: number): void {
+    // base = offset in `text` of `line[0]`.
+    // Determine indentation and whether tabs were used for it.
+    let i = 0;
+    let sawTab = false;
+    while (i < line.length) {
+      const ch = line[i];
+      if (ch === " ") {
+        i++;
+      } else if (ch === "\t") {
+        sawTab = true;
+        i++;
+      } else {
+        break;
+      }
+    }
+
+    const contentStart = i; // index within `line`
+    // Blank line?
+    if (contentStart >= line.length) return;
+    // Comment-only line?
+    if (line[contentStart] === "#") return;
+
+    const trimmed = line.slice(contentStart);
+
+    // Header?
+    if (!headerFound) {
+      const m = HEADER_RE.exec(line);
+      if (m) {
+        headerFound = true;
+        language = m[1];
+        // headerRange covers the `l_<name>:` token.
+        const idx = line.indexOf("l_");
+        const start = base + idx;
+        const colon = line.indexOf(":", idx);
+        const end = base + (colon >= 0 ? colon + 1 : line.length);
+        headerRange = { start, end };
+        return;
+      }
+      // Non-blank, non-comment content before the header.
+      errors.push({
+        code: "content-before-header",
+        message: "Content appears before the localization header (e.g. `l_english:`).",
+        range: { start: base + contentStart, end: base + line.length },
+      });
+      return;
+    }
+
+    // A second header-looking line after the first is just ignored as content;
+    // but if it matches the header pattern we skip it silently.
+    if (HEADER_RE.test(line)) {
+      return;
+    }
+
+    // Tab indentation is rejected by the game.
+    if (sawTab) {
+      errors.push({
+        code: "tab-indent",
+        message: "Tabs are not allowed for indentation in localization files.",
+        range: { start: base, end: base + contentStart },
+      });
+      // Continue trying to parse the entry anyway (best-effort).
+    }
+
+    // Parse an entry: key (:version)? "value"
+    parseEntry(trimmed, base + contentStart, line, base);
+  }
+
+  function parseEntry(entryText: string, entryBase: number, fullLine: string, lineBase: number): void {
+    // entryBase = offset in `text` of entryText[0].
+    let j = 0;
+    // Read key.
+    while (j < entryText.length && isKeyChar(entryText[j])) j++;
+    if (j === 0) {
+      badEntry(fullLine, lineBase);
+      return;
+    }
+    const key = entryText.slice(0, j);
+    const keyRange: Range = { start: entryBase, end: entryBase + j };
+
+    // Expect a colon.
+    if (j >= entryText.length || entryText[j] !== ":") {
+      badEntry(fullLine, lineBase);
+      return;
+    }
+    j++; // consume ':'
+
+    // Optional version number (digits).
+    let version: number | null = null;
+    const verStart = j;
+    while (j < entryText.length && entryText[j] >= "0" && entryText[j] <= "9") {
+      j++;
+    }
+    if (j > verStart) {
+      version = Number(entryText.slice(verStart, j));
+    }
+
+    // Skip whitespace before the opening quote.
+    while (j < entryText.length && (entryText[j] === " " || entryText[j] === "\t")) {
+      j++;
+    }
+
+    // Expect opening quote.
+    if (j >= entryText.length || entryText[j] !== '"') {
+      badEntry(fullLine, lineBase);
+      return;
+    }
+    const quoteOpen = j;
+    const valueInnerStart = entryBase + quoteOpen + 1;
+
+    // The game closes the value at the LAST quote on the line. Inner quotes
+    // are literal — vanilla wraps direct speech as `""…""` and embeds bare
+    // quotes mid-value without escaping — so scanning to the first close would
+    // read those values as empty/truncated.
+    const quoteClose = entryText.lastIndexOf('"');
+
+    if (quoteClose === quoteOpen) {
+      errors.push({
+        code: "unterminated-value",
+        message: "Unterminated localization value (missing closing quote).",
+        range: {
+          start: entryBase + quoteOpen,
+          end: lineBase + fullLine.length,
+        },
+      });
+      // Still record the entry with whatever value we captured.
+      const valueInnerEnd = entryBase + entryText.length;
+      entries.push({
+        key,
+        keyRange,
+        version,
+        value: entryText.slice(quoteOpen + 1),
+        valueRange: { start: valueInnerStart, end: valueInnerEnd },
+        line: lineNo,
+      });
+      return;
+    }
+
+    const valueInnerEnd = entryBase + quoteClose;
+    const value = entryText.slice(quoteOpen + 1, quoteClose);
+    // Anything after the closing quote is ignored (trailing comments etc.).
+
+    entries.push({
+      key,
+      keyRange,
+      version,
+      value,
+      valueRange: { start: valueInnerStart, end: valueInnerEnd },
+      line: lineNo,
+    });
+  }
+
+  function badEntry(fullLine: string, lineBase: number): void {
+    errors.push({
+      code: "bad-entry",
+      message: 'Malformed localization entry; expected `key: "value"`.',
+      range: { start: lineBase, end: lineBase + fullLine.length },
+    });
+  }
+}

package/src/parser/parser.ts

@@ -0,0 +1,360 @@
+// Recursive-descent, error-tolerant parser for Paradox script.
+//
+// Always produces a usable CST plus a list of structural errors. Never throws.
+
+import {
+  AssignmentNode,
+  BlockNode,
+  CommentNode,
+  Operator,
+  ParseError,
+  ParseResult,
+  RootNode,
+  ScalarNode,
+  Statement,
+  TaggedBlockNode,
+  ValueNode,
+  ValueStatementNode,
+} from "./cst.js";
+import { Token, tokenize } from "./lexer.js";
+// Display prose only (game name in the unclosed-brace hint) — the parser
+// itself stays game-agnostic.
+import { activeProfile } from "../games/active";
+
+const OPERATORS: ReadonlySet<string> = new Set(["=", "?=", "==", "!=", "<", "<=", ">", ">="]);
+
+class Parser {
+  private readonly text: string;
+  private readonly tokens: Token[];
+  private pos = 0;
+  private readonly errors: ParseError[] = [];
+  private readonly comments: CommentNode[] = [];
+
+  constructor(text: string) {
+    this.text = text;
+    this.tokens = tokenize(text);
+  }
+
+  parse(): ParseResult {
+    const statements: Statement[] = [];
+    while (!this.atEof()) {
+      const tok = this.peek();
+      if (tok.kind === "rbrace") {
+        // Stray close at root level.
+        this.errors.push({
+          code: "stray-close",
+          message: "Unexpected '}' with no matching open brace.",
+          range: { start: tok.start, end: tok.end },
+        });
+        this.advance();
+        continue;
+      }
+      const stmt = this.parseStatement();
+      if (stmt) {
+        statements.push(stmt);
+      } else if (!this.atEof()) {
+        // Defensive: nothing consumed — skip a token to guarantee progress.
+        this.advance();
+      }
+    }
+
+    const root: RootNode = {
+      kind: "root",
+      statements,
+      range: { start: 0, end: this.text.length },
+    };
+    return { root, errors: this.errors, comments: this.comments };
+  }
+
+  // --- token stream helpers (skip comments transparently) ---
+
+  private peek(): Token {
+    this.skipComments();
+    return this.tokens[this.pos];
+  }
+
+  private peekAhead(): Token {
+    // Peek the token AFTER the current significant token, skipping comments.
+    this.skipComments();
+    let j = this.pos + 1;
+    while (j < this.tokens.length && this.tokens[j].kind === "comment") j++;
+    return this.tokens[j] ?? this.tokens[this.tokens.length - 1];
+  }
+
+  private advance(): Token {
+    this.skipComments();
+    const t = this.tokens[this.pos];
+    if (this.pos < this.tokens.length - 1) this.pos++;
+    return t;
+  }
+
+  private skipComments(): void {
+    while (this.pos < this.tokens.length && this.tokens[this.pos].kind === "comment") {
+      const c = this.tokens[this.pos];
+      this.comments.push({
+        text: this.text.slice(c.start, c.end),
+        range: { start: c.start, end: c.end },
+        line: this.lineAt(c.start),
+      });
+      this.pos++;
+    }
+  }
+
+  // Lazily-built newline offset table for cheap 0-based line lookups.
+  private newlineOffsets: number[] | null = null;
+
+  private lineAt(offset: number): number {
+    if (this.newlineOffsets === null) {
+      const nl: number[] = [];
+      for (let k = 0; k < this.text.length; k++) {
+        if (this.text.charCodeAt(k) === 10 /* \n */) nl.push(k);
+      }
+      this.newlineOffsets = nl;
+    }
+    const nl = this.newlineOffsets;
+    // Count newlines strictly before `offset` via binary search.
+    let lo = 0;
+    let hi = nl.length;
+    while (lo < hi) {
+      const mid = (lo + hi) >> 1;
+      if (nl[mid] < offset) lo = mid + 1;
+      else hi = mid;
+    }
+    return lo;
+  }
+
+  private atEof(): boolean {
+    this.skipComments();
+    return this.tokens[this.pos].kind === "eof";
+  }
+
+  // --- grammar ---
+
+  // statement := key (op value | block)? | value
+  private parseStatement(): Statement | null {
+    const tok = this.peek();
+
+    if (tok.kind === "eof" || tok.kind === "rbrace") {
+      return null;
+    }
+
+    // A block starting here is an anonymous list element (ValueStatementNode).
+    if (tok.kind === "lbrace") {
+      const block = this.parseBlock();
+      const vs: ValueStatementNode = {
+        kind: "value",
+        value: block,
+        range: { ...block.range },
+      };
+      return vs;
+    }
+
+    // Otherwise the statement starts with a scalar (word or string).
+    if (tok.kind === "word" || tok.kind === "string") {
+      const key = this.makeScalar(this.advance());
+      const next = this.peek();
+
+      // key op value
+      if (next.kind === "op") {
+        const opTok = this.advance();
+        const op = opTok.value as Operator;
+        const value = this.parseValueAfterOperator();
+        if (value === null) {
+          this.errors.push({
+            code: "missing-value",
+            message: `Missing value after '${op}'.`,
+            range: { start: opTok.start, end: opTok.end },
+          });
+        }
+        const end = value ? value.range.end : opTok.end;
+        const assign: AssignmentNode = {
+          kind: "assignment",
+          key,
+          op,
+          value,
+          range: { start: key.range.start, end },
+        };
+        return assign;
+      }
+
+      // key block (GUI style, no operator) — e.g. `template MyT { ... }`
+      if (next.kind === "lbrace") {
+        const block = this.parseBlock();
+        const assign: AssignmentNode = {
+          kind: "assignment",
+          key,
+          op: null,
+          value: block,
+          range: { start: key.range.start, end: block.range.end },
+        };
+        return assign;
+      }
+
+      // Bare scalar — list element.
+      const vs: ValueStatementNode = {
+        kind: "value",
+        value: key,
+        range: { ...key.range },
+      };
+      return vs;
+    }
+
+    // Unexpected token kind (shouldn't normally happen). Consume and skip.
+    this.advance();
+    return null;
+  }
+
+  // value := scalar | block | scalar-immediately-followed-by-`{` (TaggedBlock)
+  // Used after an operator. Returns null if no value is parseable.
+  private parseValueAfterOperator(): ValueNode | null {
+    const tok = this.peek();
+
+    if (tok.kind === "lbrace") {
+      return this.parseBlock();
+    }
+
+    // `OPERATOR = <=` and friends: a comparison operator used as a VALUE.
+    // Vanilla uses this in list/any triggers (RELATION/OPERATOR/COUNT blocks).
+    // Accept the operator token as a bare scalar value.
+    if (tok.kind === "op") {
+      const opTok = this.advance();
+      return {
+        kind: "scalar",
+        text: this.text.slice(opTok.start, opTok.end),
+        quoted: false,
+        range: { start: opTok.start, end: opTok.end },
+      };
+    }
+
+    if (tok.kind === "word" || tok.kind === "string") {
+      // Disambiguate `key =\n nextKey = ...`: if the scalar here is immediately
+      // followed by an operator, it is the NEXT statement's key, not this
+      // value. Treat the current value as missing so the next statement parses.
+      // But `word op word` where the trailing op is a value (rare) is handled
+      // by the caller re-entering; here we only bail when the following token
+      // is an operator AND the current scalar is a plausible key.
+      const after = this.peekAhead();
+      if (after.kind === "op") {
+        return null;
+      }
+      const scalar = this.makeScalar(this.advance());
+      // Tagged block: scalar immediately followed by `{`.
+      const next = this.peek();
+      if (next.kind === "lbrace") {
+        const block = this.parseBlock();
+        const tb: TaggedBlockNode = {
+          kind: "tagged-block",
+          tag: scalar,
+          block,
+          range: { start: scalar.range.start, end: block.range.end },
+        };
+        return tb;
+      }
+      return scalar;
+    }
+
+    // op / rbrace / eof → no value.
+    return null;
+  }
+
+  // block := `{` statement* `}`
+  private parseBlock(): BlockNode {
+    const open = this.advance(); // consume `{`
+    const openBrace = open.start;
+    const statements: Statement[] = [];
+    let closeBrace: number | null = null;
+    let end = open.end;
+
+    while (true) {
+      const tok = this.peek();
+      if (tok.kind === "eof") {
+        // Unclosed block: report at the opening brace, swallow the rest.
+        this.errors.push({
+          code: "unclosed-brace",
+          message:
+            "Unclosed '{': the rest of the file is swallowed by this block. " +
+            `${activeProfile().shortName} silently ignores everything after an unbalanced brace.`,
+          range: { start: openBrace, end: openBrace + 1 },
+        });
+        closeBrace = null;
+        end = tok.start;
+        break;
+      }
+      if (tok.kind === "rbrace") {
+        closeBrace = tok.start;
+        end = tok.end;
+        this.advance();
+        break;
+      }
+      const stmt = this.parseStatement();
+      if (stmt) {
+        statements.push(stmt);
+      } else if (this.peek().kind !== "rbrace" && this.peek().kind !== "eof") {
+        // Guarantee progress.
+        this.advance();
+      }
+    }
+
+    return {
+      kind: "block",
+      statements,
+      range: { start: openBrace, end },
+      openBrace,
+      closeBrace,
+    };
+  }
+
+  // --- helpers ---
+
+  private makeScalar(tok: Token): ScalarNode {
+    if (tok.kind === "string") {
+      // Strip surrounding quotes for `text`; keep range covering the quotes.
+      // For unterminated strings there's only an opening quote.
+      let inner: string;
+      if (tok.unterminated) {
+        // Unterminated: report the diagnostic and take everything after `"`.
+        this.errors.push({
+          code: "unterminated-string",
+          message: "Unterminated string; recovered at end of line.",
+          range: { start: tok.start, end: tok.end },
+        });
+        inner = this.text.slice(tok.start + 1, tok.end);
+      } else {
+        inner = this.text.slice(tok.start + 1, tok.end - 1);
+      }
+      return {
+        kind: "scalar",
+        text: inner,
+        quoted: true,
+        range: { start: tok.start, end: tok.end },
+      };
+    }
+    // word
+    return {
+      kind: "scalar",
+      text: this.text.slice(tok.start, tok.end),
+      quoted: false,
+      range: { start: tok.start, end: tok.end },
+    };
+  }
+}
+
+export function parseScript(text: string): ParseResult {
+  try {
+    return new Parser(text).parse();
+  } catch {
+    // Absolute last-resort guard: should be unreachable, but the contract is
+    // "never throw on any input".
+    return {
+      root: {
+        kind: "root",
+        statements: [],
+        range: { start: 0, end: text.length },
+      },
+      errors: [],
+      comments: [],
+    };
+  }
+}
+
+export { OPERATORS };

package/src/schema/freqs.ts

@@ -0,0 +1,70 @@
+/**
+ * Bundled corpus-frequency tables (update plan v1.1 §C3), the ranking input for
+ * Workstream C. Emitted by scripts/build-freqs.ts from a vanilla scan (+ optional
+ * mod corpus) and shipped as packages/server/data/<game>/freqs.json. Loaded fail-soft: a
+ * missing/corrupt file yields empty tables and the extension ranks with a neutral
+ * frequency bucket everywhere.
+ *
+ * `contexts` are the same six completion contexts the eval harness reports on;
+ * each maps a key name to its occurrence count in that context. `tokens` is a
+ * context-independent global table (a key's count summed across all contexts),
+ * used for effect/trigger tokens and definitions that aren't structural keys.
+ *
+ * No `vscode` imports: plain data + a Node fs loader.
+ */
+import * as fs from "fs";
+import * as path from "path";
+
+/** The frequency contexts. Mirrors the eval harness's EvalContext set. */
+export type FreqContext =
+  "event_top" | "event_option" | "interaction_top" | "decision_top" | "effect_block" | "trigger_block";
+
+export const FREQ_CONTEXTS: FreqContext[] = [
+  "event_top",
+  "event_option",
+  "interaction_top",
+  "decision_top",
+  "effect_block",
+  "trigger_block",
+];
+
+export interface FreqData {
+  /** Provenance note (which corpora, when) — display only. */
+  meta?: { generated?: string; sources?: string[] };
+  /** Per-context name -> count (top-N per context). */
+  contexts: Record<FreqContext, Record<string, number>>;
+  /** Global name -> count across all contexts (top-N). */
+  tokens: Record<string, number>;
+}
+
+export function emptyFreqData(): FreqData {
+  const contexts = {} as Record<FreqContext, Record<string, number>>;
+  for (const c of FREQ_CONTEXTS) contexts[c] = {};
+  return { contexts, tokens: {} };
+}
+
+/**
+ * Load the bundled freqs.json from `dir`. Fail-soft: any error (missing file,
+ * bad JSON, wrong shape) returns empty tables so completion still works.
+ */
+export function loadFreqs(dir: string): FreqData {
+  try {
+    const raw = fs.readFileSync(path.join(dir, "freqs.json"), "utf8");
+    const parsed = JSON.parse(raw) as Partial<FreqData>;
+    if (!parsed || typeof parsed !== "object" || !parsed.contexts || !parsed.tokens) {
+      return emptyFreqData();
+    }
+    const out = emptyFreqData();
+    for (const c of FREQ_CONTEXTS) {
+      const table = (parsed.contexts as Record<string, unknown>)[c];
+      if (table && typeof table === "object") {
+        out.contexts[c] = table as Record<string, number>;
+      }
+    }
+    if (parsed.tokens && typeof parsed.tokens === "object") out.tokens = parsed.tokens;
+    if (parsed.meta) out.meta = parsed.meta;
+    return out;
+  } catch {
+    return emptyFreqData();
+  }
+}