@words-lang/parser 0.1.0

package/dist/lexer/lexer.js

@@ -0,0 +1,365 @@
+"use strict";
+/**
+ * lexer.ts
+ *
+ * The WORDS lexer. Converts a raw `.wds` source string into a flat array
+ * of tokens that the parser consumes.
+ *
+ * Design principles:
+ *
+ * - Single-pass, character-by-character. No regular expressions at runtime —
+ *   all character classification is done with simple comparisons.
+ *
+ * - Never throws. Unrecognised characters are emitted as `Unknown` tokens so
+ *   the parser can continue and collect all errors in one pass rather than
+ *   stopping at the first problem.
+ *
+ * - `is not` is normalised into a single `IsNot` token during lexing.
+ *   This simplifies the parser — it never has to handle a two-token sequence
+ *   in conditional expressions.
+ *
+ * - Newlines are emitted as `Newline` tokens. The parser uses them to
+ *   distinguish a bare ownership declaration (`module AuthModule` on its own
+ *   line) from a construct body opening (`module AuthModule "..." (`).
+ *
+ * - Comments are included in the token stream (not silently discarded) so the
+ *   parser can attach them to adjacent nodes for hover documentation.
+ *
+ * - Method names, callback prop names, and handler method names are all plain
+ *   camelCase identifiers. Names like `switch`, `onLoad`, `onSubmit` carry no
+ *   special meaning to the lexer — they are emitted as CamelIdent tokens.
+ *
+ * - Position tracking (line, column, offset) is maintained for every token
+ *   so the LSP can report diagnostics and resolve go-to-definition requests
+ *   at the exact source location.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Lexer = void 0;
+const token_1 = require("./token");
+// ── Keyword table ─────────────────────────────────────────────────────────────
+/**
+ * Maps every reserved word in WORDS to its token type.
+ * Identifiers not found in this table are classified as PascalIdent or
+ * CamelIdent based on their first character.
+ *
+ * Note: `true` and `false` are listed here so they are never accidentally
+ * emitted as plain identifiers.
+ *
+ * Intentionally NOT in this table — these are all plain camelCase names
+ * chosen by the designer with no special language meaning:
+ *   - Method names (e.g. `switch`, `login`, `getProducts`, `clear`)
+ *   - Callback prop names (e.g. `onSubmit`, `onConfirm`, `onLoad`, `onDismiss`)
+ *   - Handler method names (e.g. `switch` on a handler interface)
+ *   - Iteration variables (e.g. `notification`, `category`)
+ */
+const KEYWORDS = {
+    system: token_1.TokenType.System,
+    module: token_1.TokenType.Module,
+    process: token_1.TokenType.Process,
+    state: token_1.TokenType.State,
+    context: token_1.TokenType.Context,
+    screen: token_1.TokenType.Screen,
+    view: token_1.TokenType.View,
+    provider: token_1.TokenType.Provider,
+    adapter: token_1.TokenType.Adapter,
+    interface: token_1.TokenType.Interface,
+    modules: token_1.TokenType.Modules,
+    props: token_1.TokenType.Props,
+    uses: token_1.TokenType.Uses,
+    returns: token_1.TokenType.Returns,
+    receives: token_1.TokenType.Receives,
+    start: token_1.TokenType.Start,
+    implements: token_1.TokenType.Implements,
+    when: token_1.TokenType.When,
+    enter: token_1.TokenType.Enter,
+    if: token_1.TokenType.If,
+    for: token_1.TokenType.For,
+    as: token_1.TokenType.As,
+    is: token_1.TokenType.Is,
+    true: token_1.TokenType.BooleanLit,
+    false: token_1.TokenType.BooleanLit,
+    string: token_1.TokenType.TString,
+    integer: token_1.TokenType.TInteger,
+    float: token_1.TokenType.TFloat,
+    boolean: token_1.TokenType.TBoolean,
+    list: token_1.TokenType.TList,
+    map: token_1.TokenType.TMap,
+};
+// ── Lexer class ───────────────────────────────────────────────────────────────
+class Lexer {
+    constructor(source) {
+        /** Current byte offset into `source`. */
+        this.pos = 0;
+        /** Current 1-based line number. Incremented each time a `\n` is consumed. */
+        this.line = 1;
+        /**
+         * Current 1-based column number.
+         * Reset to 1 after each newline; incremented after each other character.
+         */
+        this.column = 1;
+        /** Accumulated token stream. Populated by `tokenize()`. */
+        this.tokens = [];
+        this.source = source;
+    }
+    // ── Public API ─────────────────────────────────────────────────────────────
+    /**
+     * Tokenizes the entire source string and returns the token stream.
+     * The last token in the stream is always an `EOF` token.
+     *
+     * Calling `tokenize()` more than once on the same instance returns a new
+     * stream from scratch (internal state is reset on construction, not here —
+     * create a new Lexer for each source string).
+     */
+    tokenize() {
+        while (!this.isAtEnd()) {
+            // Skip horizontal whitespace between tokens.
+            // Newlines are NOT skipped here — they are emitted as Newline tokens.
+            this.skipWhitespace();
+            if (this.isAtEnd())
+                break;
+            const start = this.pos;
+            const startLine = this.line;
+            const startCol = this.column;
+            const ch = this.current();
+            // ── Line comment ─────────────────────────────────────────────────────
+            if (ch === '/' && this.peek(1) === '/') {
+                const comment = this.readLineComment();
+                this.tokens.push((0, token_1.token)(token_1.TokenType.Comment, comment, startLine, startCol, start));
+                continue;
+            }
+            // ── String literal ───────────────────────────────────────────────────
+            if (ch === '"') {
+                const str = this.readString();
+                this.tokens.push((0, token_1.token)(token_1.TokenType.StringLit, str, startLine, startCol, start));
+                continue;
+            }
+            // ── Number literal ───────────────────────────────────────────────────
+            // Integers and floats are distinguished by the presence of a decimal point.
+            if (this.isDigit(ch)) {
+                const num = this.readNumber();
+                const type = num.includes('.') ? token_1.TokenType.FloatLit : token_1.TokenType.IntegerLit;
+                this.tokens.push((0, token_1.token)(type, num, startLine, startCol, start));
+                continue;
+            }
+            // ── Identifier or keyword ────────────────────────────────────────────
+            if (this.isAlpha(ch) || ch === '_') {
+                const ident = this.readIdent();
+                // Special case: 'is not' — look ahead past any whitespace to see if
+                // the next word is 'not'. If so, consume it and emit a single IsNot
+                // token. This keeps the parser free from two-token handling in conditions.
+                if (ident === 'is') {
+                    const savedPos = this.pos;
+                    const savedLine = this.line;
+                    const savedCol = this.column;
+                    this.skipWhitespace();
+                    if (this.source.startsWith('not', this.pos) &&
+                        !this.isAlphaNumeric(this.source[this.pos + 3] ?? '')) {
+                        this.pos += 3;
+                        this.column += 3;
+                        this.tokens.push((0, token_1.token)(token_1.TokenType.IsNot, 'is not', startLine, startCol, start));
+                        continue;
+                    }
+                    // Not 'is not' — restore position and emit plain Is.
+                    this.pos = savedPos;
+                    this.line = savedLine;
+                    this.column = savedCol;
+                }
+                // Look up keyword table; fall through to identifier classification.
+                const kwType = KEYWORDS[ident];
+                if (kwType !== undefined) {
+                    this.tokens.push((0, token_1.token)(kwType, ident, startLine, startCol, start));
+                }
+                else if (/^[A-Z]/.test(ident)) {
+                    // PascalCase → construct name or type reference
+                    this.tokens.push((0, token_1.token)(token_1.TokenType.PascalIdent, ident, startLine, startCol, start));
+                }
+                else {
+                    // camelCase → prop name, method name, handler method name, or
+                    // iteration variable. This includes all designer-chosen names
+                    // such as `switch`, `onLoad`, `onSubmit`, `onConfirm`, etc.
+                    this.tokens.push((0, token_1.token)(token_1.TokenType.CamelIdent, ident, startLine, startCol, start));
+                }
+                continue;
+            }
+            // ── Optional marker ──────────────────────────────────────────────────
+            // `?` always immediately precedes a PascalCase type name.
+            if (ch === '?') {
+                this.advance();
+                this.tokens.push((0, token_1.token)(token_1.TokenType.Question, '?', startLine, startCol, start));
+                continue;
+            }
+            // ── Punctuation ──────────────────────────────────────────────────────
+            if (ch === '(') {
+                this.advance();
+                this.tokens.push((0, token_1.token)(token_1.TokenType.LParen, '(', startLine, startCol, start));
+                continue;
+            }
+            if (ch === ')') {
+                this.advance();
+                this.tokens.push((0, token_1.token)(token_1.TokenType.RParen, ')', startLine, startCol, start));
+                continue;
+            }
+            if (ch === ',') {
+                this.advance();
+                this.tokens.push((0, token_1.token)(token_1.TokenType.Comma, ',', startLine, startCol, start));
+                continue;
+            }
+            if (ch === '.') {
+                this.advance();
+                this.tokens.push((0, token_1.token)(token_1.TokenType.Dot, '.', startLine, startCol, start));
+                continue;
+            }
+            // ── Newline ──────────────────────────────────────────────────────────
+            // Emitted as a token so the parser can detect line boundaries.
+            // The line counter is incremented inside `advance()`.
+            if (ch === '\n') {
+                this.tokens.push((0, token_1.token)(token_1.TokenType.Newline, '\n', startLine, startCol, start));
+                this.advance();
+                continue;
+            }
+            // ── Unknown ──────────────────────────────────────────────────────────
+            // Emit and continue rather than throwing, so all errors can be collected.
+            this.tokens.push((0, token_1.token)(token_1.TokenType.Unknown, ch, startLine, startCol, start));
+            this.advance();
+        }
+        // EOF sentinel — always the last token.
+        this.tokens.push((0, token_1.token)(token_1.TokenType.EOF, '', this.line, this.column, this.pos));
+        return this.tokens;
+    }
+    // ── Private helpers ────────────────────────────────────────────────────────
+    /**
+     * Returns the character at the current position without consuming it.
+     * Returns an empty string if at end of input.
+     */
+    current() {
+        return this.source[this.pos] ?? '';
+    }
+    /**
+     * Returns the character at `pos + offset` without consuming it.
+     * Used for one-character lookahead (e.g. distinguishing `//` from `/`).
+     * Returns an empty string if the offset is out of bounds.
+     */
+    peek(offset) {
+        return this.source[this.pos + offset] ?? '';
+    }
+    /**
+     * Consumes the current character, advances the position, and updates
+     * line/column tracking. Returns the consumed character.
+     * Line is incremented and column reset to 1 when a `\n` is consumed.
+     */
+    advance() {
+        const ch = this.source[this.pos];
+        if (ch === '\n') {
+            this.line++;
+            this.column = 1;
+        }
+        else {
+            this.column++;
+        }
+        this.pos++;
+        return ch ?? '';
+    }
+    /** Returns true when all characters have been consumed. */
+    isAtEnd() {
+        return this.pos >= this.source.length;
+    }
+    /** Returns true for ASCII decimal digit characters. */
+    isDigit(ch) {
+        return ch >= '0' && ch <= '9';
+    }
+    /** Returns true for ASCII letters and underscore. */
+    isAlpha(ch) {
+        return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch === '_';
+    }
+    /** Returns true for characters valid inside an identifier (letters, digits, underscore). */
+    isAlphaNumeric(ch) {
+        return this.isAlpha(ch) || this.isDigit(ch);
+    }
+    /**
+     * Advances past spaces, tabs, and carriage returns.
+     * Newlines are NOT skipped — they are significant and emitted as tokens.
+     */
+    skipWhitespace() {
+        while (!this.isAtEnd()) {
+            const ch = this.current();
+            if (ch === ' ' || ch === '\t' || ch === '\r') {
+                this.advance();
+            }
+            else {
+                break;
+            }
+        }
+    }
+    /**
+     * Reads a `//` line comment from the current position to the end of the line.
+     * The returned value includes the `//` prefix.
+     * The terminating `\n` is NOT consumed — it will be emitted as a Newline token
+     * on the next iteration.
+     */
+    readLineComment() {
+        let result = '';
+        while (!this.isAtEnd() && this.current() !== '\n') {
+            result += this.advance();
+        }
+        return result;
+    }
+    /**
+     * Reads a double-quoted string literal from the current position.
+     * Handles backslash escape sequences by consuming both the `\` and the
+     * following character as a unit.
+     * The returned value includes the surrounding quotes.
+     * Unclosed strings (EOF before closing `"`) are returned as-is — the
+     * parser will report the error from context.
+     */
+    readString() {
+        let result = '"';
+        this.advance(); // consume opening quote
+        while (!this.isAtEnd() && this.current() !== '"') {
+            if (this.current() === '\\') {
+                result += this.advance(); // backslash
+                result += this.advance(); // escaped character
+            }
+            else {
+                result += this.advance();
+            }
+        }
+        if (!this.isAtEnd()) {
+            result += this.advance(); // consume closing quote
+        }
+        return result;
+    }
+    /**
+     * Reads an integer or float literal from the current position.
+     * A decimal point followed by at least one digit triggers float mode.
+     * The returned string is the raw source text — conversion to a number
+     * happens in the parser.
+     */
+    readNumber() {
+        let result = '';
+        while (!this.isAtEnd() && this.isDigit(this.current())) {
+            result += this.advance();
+        }
+        // Check for decimal point followed by a digit — if so, continue as float.
+        if (!this.isAtEnd() && this.current() === '.' && this.isDigit(this.peek(1))) {
+            result += this.advance(); // consume '.'
+            while (!this.isAtEnd() && this.isDigit(this.current())) {
+                result += this.advance();
+            }
+        }
+        return result;
+    }
+    /**
+     * Reads an identifier (keyword or user-defined name) from the current position.
+     * Identifiers consist of letters, digits, and underscores.
+     * The caller is responsible for classifying the result via the keyword table.
+     */
+    readIdent() {
+        let result = '';
+        while (!this.isAtEnd() && this.isAlphaNumeric(this.current())) {
+            result += this.advance();
+        }
+        return result;
+    }
+}
+exports.Lexer = Lexer;
+//# sourceMappingURL=lexer.js.map

package/dist/lexer/lexer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lexer.js","sourceRoot":"","sources":["../../src/lexer/lexer.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;;;AAEH,mCAAiD;AAEjD,iFAAiF;AAEjF;;;;;;;;;;;;;;GAcG;AACH,MAAM,QAAQ,GAA8B;IACxC,MAAM,EAAE,iBAAS,CAAC,MAAM;IACxB,MAAM,EAAE,iBAAS,CAAC,MAAM;IACxB,OAAO,EAAE,iBAAS,CAAC,OAAO;IAC1B,KAAK,EAAE,iBAAS,CAAC,KAAK;IACtB,OAAO,EAAE,iBAAS,CAAC,OAAO;IAC1B,MAAM,EAAE,iBAAS,CAAC,MAAM;IACxB,IAAI,EAAE,iBAAS,CAAC,IAAI;IACpB,QAAQ,EAAE,iBAAS,CAAC,QAAQ;IAC5B,OAAO,EAAE,iBAAS,CAAC,OAAO;IAC1B,SAAS,EAAE,iBAAS,CAAC,SAAS;IAC9B,OAAO,EAAE,iBAAS,CAAC,OAAO;IAC1B,KAAK,EAAE,iBAAS,CAAC,KAAK;IACtB,IAAI,EAAE,iBAAS,CAAC,IAAI;IACpB,OAAO,EAAE,iBAAS,CAAC,OAAO;IAC1B,QAAQ,EAAE,iBAAS,CAAC,QAAQ;IAC5B,KAAK,EAAE,iBAAS,CAAC,KAAK;IACtB,UAAU,EAAE,iBAAS,CAAC,UAAU;IAChC,IAAI,EAAE,iBAAS,CAAC,IAAI;IACpB,KAAK,EAAE,iBAAS,CAAC,KAAK;IACtB,EAAE,EAAE,iBAAS,CAAC,EAAE;IAChB,GAAG,EAAE,iBAAS,CAAC,GAAG;IAClB,EAAE,EAAE,iBAAS,CAAC,EAAE;IAChB,EAAE,EAAE,iBAAS,CAAC,EAAE;IAChB,IAAI,EAAE,iBAAS,CAAC,UAAU;IAC1B,KAAK,EAAE,iBAAS,CAAC,UAAU;IAC3B,MAAM,EAAE,iBAAS,CAAC,OAAO;IACzB,OAAO,EAAE,iBAAS,CAAC,QAAQ;IAC3B,KAAK,EAAE,iBAAS,CAAC,MAAM;IACvB,OAAO,EAAE,iBAAS,CAAC,QAAQ;IAC3B,IAAI,EAAE,iBAAS,CAAC,KAAK;IACrB,GAAG,EAAE,iBAAS,CAAC,IAAI;CACtB,CAAA;AAED,iFAAiF;AAEjF,MAAa,KAAK;IAmBd,YAAY,MAAc;QAf1B,yCAAyC;QACjC,QAAG,GAAW,CAAC,CAAA;QAEvB,6EAA6E;QACrE,SAAI,GAAW,CAAC,CAAA;QAExB;;;WAGG;QACK,WAAM,GAAW,CAAC,CAAA;QAE1B,2DAA2D;QACnD,WAAM,GAAY,EAAE,CAAA;QAGxB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACxB,CAAC;IAED,8EAA8E;IAE9E;;;;;;;OAOG;IACH,QAAQ;QACJ,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACrB,6CAA6C;YAC7C,sEAAsE;YACtE,IAAI,CAAC,cAAc,EAAE,CAAA;YACrB,IAAI,IAAI,CAAC,OAAO,EAAE;gBAAE,MAAK;YAEzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAA;YACtB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAA;YAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAA;YAC5B,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAA;YAEzB,wEAAwE;YACxE,IAAI,EAAE,KAAK,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;gBACrC,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,EAAE,CAAA;gBACtC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAA,aAAK,EAAC,iBAAS,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAA;gBAC/E,SAAQ;YACZ,CAAC;YAED,wEAAwE;YACxE,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBACb,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAA;gBAC7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAA,aAAK,EAAC,iBAAS,CAAC,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAA;gBAC7E,SAAQ;YACZ,CAAC;YAED,wEAAwE;YACxE,4EAA4E;YAC5E,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;gBACnB,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,EAAE,CAAA;gBAC7B,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,iBAAS,CAAC,UAAU,CAAA;gBAC1E,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAA,aAAK,EAAC,IAAI,EAAE,GAAG,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAA;gBAC9D,SAAQ;YACZ,CAAC;YAED,wEAAwE;YACxE,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBACjC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAA;gBAE9B,oEAAoE;gBACpE,oEAAoE;gBACpE,2EAA2E;gBAC3E,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;oBACjB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAA;oBACzB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAA;oBAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAA;oBAC5B,IAAI,CAAC,cAAc,EAAE,CAAA;oBACrB,IACI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;wBACvC,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EACvD,CAAC;wBACC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAA;wBACb,IAAI,CAAC,MAAM,IAAI,CAAC,CAAA;wBAChB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAA,aAAK,EAAC,iBAAS,CAAC,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAA;wBAC9E,SAAQ;oBACZ,CAAC;oBACD,qDAAqD;oBACrD,IAAI,CAAC,GAAG,GAAG,QAAQ,CAAA;oBACnB,IAAI,CAAC,IAAI,GAAG,SAAS,CAAA;oBACrB,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAA;gBAC1B,CAAC;gBAED,oEAAoE;gBACpE,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;gBAC9B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;oBACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAA,aAAK,EAAC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAA;gBACtE,CAAC;qBAAM,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC9B,gDAAgD;oBAChD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAA,aAAK,EAAC,iBAAS,CAAC,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAA;gBACrF,CAAC;qBAAM,CAAC;oBACJ,8DAA8D;oBAC9D,8DAA8D;oBAC9D,4DAA4D;oBAC5D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAA,aAAK,EAAC,iBAAS,CAAC,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAA;gBACpF,CAAC;gBACD,SAAQ;YACZ,CAAC;YAED,wEAAwE;YACxE,0DAA0D;YAC1D,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,OAAO,EAAE,CAAA;gBACd,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAA,aAAK,EAAC,iBAAS,CAAC,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAA;gBAC5E,SAAQ;YACZ,CAAC;YAED,wEAAwE;YACxE,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAA,aAAK,EAAC,iBAAS,CAAC,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;gBAAC,SAAQ;YAAC,CAAC;YACxH,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAA,aAAK,EAAC,iBAAS,CAAC,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;gBAAC,SAAQ;YAAC,CAAC;YACxH,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAA,aAAK,EAAC,iBAAS,CAAC,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;gBAAC,SAAQ;YAAC,CAAC;YACvH,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAA,aAAK,EAAC,iBAAS,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;gBAAC,SAAQ;YAAC,CAAC;YAErH,wEAAwE;YACxE,+DAA+D;YAC/D,sDAAsD;YACtD,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;gBACd,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAA,aAAK,EAAC,iBAAS,CAAC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAA;gBAC5E,IAAI,CAAC,OAAO,EAAE,CAAA;gBACd,SAAQ;YACZ,CAAC;YAED,wEAAwE;YACxE,0EAA0E;YAC1E,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAA,aAAK,EAAC,iBAAS,CAAC,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAA;YAC1E,IAAI,CAAC,OAAO,EAAE,CAAA;QAClB,CAAC;QAED,wCAAwC;QACxC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAA,aAAK,EAAC,iBAAS,CAAC,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;QAC5E,OAAO,IAAI,CAAC,MAAM,CAAA;IACtB,CAAC;IAED,8EAA8E;IAE9E;;;OAGG;IACK,OAAO;QACX,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAA;IACtC,CAAC;IAED;;;;OAIG;IACK,IAAI,CAAC,MAAc;QACvB,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,IAAI,EAAE,CAAA;IAC/C,CAAC;IAED;;;;OAIG;IACK,OAAO;QACX,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAChC,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YACd,IAAI,CAAC,IAAI,EAAE,CAAA;YACX,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;QACnB,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,MAAM,EAAE,CAAA;QACjB,CAAC;QACD,IAAI,CAAC,GAAG,EAAE,CAAA;QACV,OAAO,EAAE,IAAI,EAAE,CAAA;IACnB,CAAC;IAED,2DAA2D;IACnD,OAAO;QACX,OAAO,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;IACzC,CAAC;IAED,uDAAuD;IAC/C,OAAO,CAAC,EAAU;QACtB,OAAO,EAAE,IAAI,GAAG,IAAI,EAAE,IAAI,GAAG,CAAA;IACjC,CAAC;IAED,qDAAqD;IAC7C,OAAO,CAAC,EAAU;QACtB,OAAO,CAAC,EAAE,IAAI,GAAG,IAAI,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,GAAG,IAAI,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,CAAA;IAC7E,CAAC;IAED,4FAA4F;IACpF,cAAc,CAAC,EAAU;QAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IAC/C,CAAC;IAED;;;OAGG;IACK,cAAc;QAClB,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YACrB,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,CAAA;YACzB,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;gBAC3C,IAAI,CAAC,OAAO,EAAE,CAAA;YAClB,CAAC;iBAAM,CAAC;gBACJ,MAAK;YACT,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACK,eAAe;QACnB,IAAI,MAAM,GAAG,EAAE,CAAA;QACf,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;YAChD,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAA;QAC5B,CAAC;QACD,OAAO,MAAM,CAAA;IACjB,CAAC;IAED;;;;;;;OAOG;IACK,UAAU;QACd,IAAI,MAAM,GAAG,GAAG,CAAA;QAChB,IAAI,CAAC,OAAO,EAAE,CAAA,CAAC,wBAAwB;QACvC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,OAAO,EAAE,KAAK,GAAG,EAAE,CAAC;YAC/C,IAAI,IAAI,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;gBAC1B,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAA,CAAC,YAAY;gBACrC,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAA,CAAC,oBAAoB;YACjD,CAAC;iBAAM,CAAC;gBACJ,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAA;YAC5B,CAAC;QACL,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;YAClB,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAA,CAAC,wBAAwB;QACrD,CAAC;QACD,OAAO,MAAM,CAAA;IACjB,CAAC;IAED;;;;;OAKG;IACK,UAAU;QACd,IAAI,MAAM,GAAG,EAAE,CAAA;QACf,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;YACrD,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAA;QAC5B,CAAC;QACD,0EAA0E;QAC1E,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,OAAO,EAAE,KAAK,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1E,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAA,CAAC,cAAc;YACvC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;gBACrD,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAA;YAC5B,CAAC;QACL,CAAC;QACD,OAAO,MAAM,CAAA;IACjB,CAAC;IAED;;;;OAIG;IACK,SAAS;QACb,IAAI,MAAM,GAAG,EAAE,CAAA;QACf,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;YAC5D,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAA;QAC5B,CAAC;QACD,OAAO,MAAM,CAAA;IACjB,CAAC;CACJ;AAhSD,sBAgSC"}

package/dist/lexer/token.d.ts

@@ -0,0 +1,247 @@
+/**
+ * token.ts
+ *
+ * Defines every token type the WORDS lexer can produce, the Token interface
+ * that carries a token's value and source location, and a convenience
+ * constructor for creating tokens in tests and the lexer itself.
+ */
+/**
+ * Every token type the WORDS lexer can produce.
+ *
+ * Grouped by category:
+ *   - Top-level construct keywords  (system, module, state, ...)
+ *   - Block keywords                (modules, props, uses, returns, ...)
+ *   - Process / transition keywords (when, enter)
+ *   - Control flow keywords         (if, for, as, is, is not)
+ *   - Primitive types               (tstring, tinteger, ...)
+ *   - Literals                      (string_lit, integer_lit, ...)
+ *   - Identifiers                   (pascal_ident, camel_ident)
+ *   - Punctuation                   ((, ), ,, .)
+ *   - Special                       (comment, newline, eof, unknown)
+ *
+ * Enum values are the string representation of each token — useful for
+ * readable error messages and debug output without a separate lookup table.
+ *
+ * Note on naming conventions: method names, callback prop names, and handler
+ * method names (including names like `switch`, `onLoad`, `onSubmit`) are all
+ * plain camelCase identifiers chosen by the designer. None of them are
+ * reserved keywords in the WORDS language.
+ */
+export declare enum TokenType {
+    /** `system` — opens the system declaration. Appears exactly once per project. */
+    System = "system",
+    /**
+     * `module` — opens a module definition body, or appears alone on a line
+     * as an ownership declaration at the top of component files.
+     * e.g. `module AuthModule` (ownership) or `module AuthModule "..." (` (definition)
+     */
+    Module = "module",
+    /** `process` — opens a process definition inside a module body. */
+    Process = "process",
+    /** `state` — opens a state definition. */
+    State = "state",
+    /** `context` — opens a context definition. */
+    Context = "context",
+    /** `screen` — opens a screen definition, or activates a screen inside a `uses` block. */
+    Screen = "screen",
+    /**
+     * `view` — opens a view definition, or activates a view inside a `uses` block.
+     * e.g. `view LoginFormSection "..." (` or `view UIModule.LoginForm (`
+     */
+    View = "view",
+    /** `provider` — opens a provider definition, or activates a provider in a `uses` block. */
+    Provider = "provider",
+    /**
+     * `adapter` — opens an adapter definition, or activates an adapter in a `uses` block.
+     * e.g. `adapter AuthAdapter.login credentials is state.context`
+     */
+    Adapter = "adapter",
+    /**
+     * `interface` — opens an interface component definition, activates one in a
+     * `uses` block, or introduces an inline interface block inside a module body.
+     */
+    Interface = "interface",
+    /** `modules` — opens the module list inside a system body. */
+    Modules = "modules",
+    /** `props` — opens the props block inside a view, provider, adapter, or interface. */
+    Props = "props",
+    /** `uses` — opens the uses block inside a state, screen, view, provider, or interface. */
+    Uses = "uses",
+    /**
+     * `returns` — appears in two roles:
+     * 1. Opens the returns clause inside a state: `returns AccountCredentials`
+     * 2. Declares the return type of a method: `getProducts returns(list(Product))`
+     */
+    Returns = "returns",
+    /**
+     * `receives` — opens the receives clause on a state definition.
+     * e.g. `state Unauthenticated receives ?AuthError`
+     */
+    Receives = "receives",
+    /**
+     * `start` — names the initial state of a module.
+     * e.g. `start Unauthenticated`
+     */
+    Start = "start",
+    /**
+     * `implements` — opens a cross-module handler implementation block
+     * inside a module body.
+     * e.g. `implements RoutingModule.RouteSwitchHandler (`
+     */
+    Implements = "implements",
+    /**
+     * `when` — opens a transition rule inside a process body.
+     * e.g. `when Unauthenticated returns AccountCredentials`
+     */
+    When = "when",
+    /**
+     * `enter` — names the next state to transition to in a `when` rule
+     * or an `implements` branch.
+     * e.g. `enter StartAuthenticating "The user tries to authenticate"`
+     */
+    Enter = "enter",
+    /**
+     * `if` — opens a conditional block inside a `uses` block or an
+     * `implements` handler body.
+     * e.g. `if state.context is AccountDeauthenticated (`
+     */
+    If = "if",
+    /**
+     * `for` — opens an iteration block inside a `uses` block.
+     * e.g. `for state.context.notifications as notification (`
+     */
+    For = "for",
+    /**
+     * `as` — binds the iteration variable(s) in a `for` block.
+     * For lists: `as notification` — one binding.
+     * For maps:  `as category, products` — two bindings.
+     */
+    As = "as",
+    /**
+     * `is` — the assignment and equality operator.
+     * As assignment: `type is "warning"`, `path is "/home"`
+     * As comparison: `if state.context is AccountDeauthenticated`
+     * Context determines which role it plays.
+     */
+    Is = "is",
+    /**
+     * `is not` — the inequality operator, normalised from the two-word
+     * sequence `is not` into a single token during lexing.
+     * e.g. `if state.context is not AccountRecovered`
+     */
+    IsNot = "is_not",
+    /** `string` — text value type. Default: `""` */
+    TString = "tstring",
+    /** `integer` — whole number type. Default: `0` */
+    TInteger = "tinteger",
+    /** `float` — decimal number type. Default: `0.0` */
+    TFloat = "tfloat",
+    /** `boolean` — true/false type. Default: `false` */
+    TBoolean = "tboolean",
+    /**
+     * `context` — opaque context type used only in the system interface
+     * declaration for `getContext` and `setContext`.
+     */
+    TContext = "tcontext",
+    /** `list` — ordered collection type. Parameterised: `list(Product)`. Default: `[]` */
+    TList = "tlist",
+    /** `map` — key-value collection type. Parameterised: `map(string, OrderSummary)`. Default: `{}` */
+    TMap = "tmap",
+    /**
+     * A double-quoted string literal.
+     * The token value includes the surrounding quotes.
+     * e.g. `"The user authenticated successfully"`, `"/home"`
+     */
+    StringLit = "string_lit",
+    /**
+     * A whole number literal.
+     * e.g. `42`, `5000`, `0`
+     */
+    IntegerLit = "integer_lit",
+    /**
+     * A decimal number literal.
+     * e.g. `0.0`, `3.14`
+     */
+    FloatLit = "float_lit",
+    /**
+     * A boolean literal — `true` or `false`.
+     * Lexed as keyword-style tokens with value `"true"` or `"false"`.
+     */
+    BooleanLit = "boolean_lit",
+    /**
+     * A PascalCase identifier — names a construct (state, context, module, etc.)
+     * or a type reference.
+     * e.g. `AuthModule`, `AccountCredentials`, `LoginScreen`, `Product`
+     */
+    PascalIdent = "pascal_ident",
+    /**
+     * A camelCase identifier — names a prop, method, argument, or iteration variable.
+     * This includes all designer-chosen names: callback prop names, method names,
+     * and handler method names regardless of what they are called.
+     * e.g. `onSubmit`, `credentials`, `fullName`, `notification`, `switch`, `onLoad`
+     */
+    CamelIdent = "camel_ident",
+    /** `(` — opens a body block or argument list. */
+    LParen = "(",
+    /** `)` — closes a body block or argument list. */
+    RParen = ")",
+    /** `,` — separates entries in a list — props, arguments, context names. */
+    Comma = ",",
+    /** `.` — separates parts of a qualified name or access expression. */
+    Dot = ".",
+    /**
+     * `?` — marks an optional type or receives clause.
+     * e.g. `?AuthError`, `?Product`
+     * Always immediately precedes a PascalIdent.
+     */
+    Question = "?",
+    /**
+     * A `//` line comment. The token value includes the `//` prefix and
+     * everything up to (but not including) the newline.
+     * Comments are included in the token stream so the parser can attach
+     * them to adjacent nodes for hover documentation.
+     */
+    Comment = "comment",
+    /**
+     * A newline character `\n`.
+     * Newlines are significant in WORDS for distinguishing an ownership
+     * declaration (`module AuthModule` on its own line) from a construct
+     * opening (`module AuthModule "..." (`).
+     */
+    Newline = "newline",
+    /**
+     * End of file sentinel. Always the last token in the stream.
+     * The parser uses this to detect unexpected end of input.
+     */
+    EOF = "eof",
+    /**
+     * An unrecognised character. Emitted rather than throwing so the lexer
+     * can continue and collect all errors in a single pass.
+     */
+    Unknown = "unknown"
+}
+/**
+ * A single token produced by the lexer.
+ *
+ * `type`   — the token's category, used by the parser for matching.
+ * `value`  — the raw source text that produced this token. For string
+ *            literals this includes the surrounding quotes. For `is not`
+ *            this is the normalised string `"is not"`.
+ * `line`   — 1-based line number of the token's first character.
+ * `column` — 1-based column number of the token's first character.
+ * `offset` — byte offset from the start of the source string. Used by the
+ *            LSP to convert between source positions and editor positions.
+ */
+export interface Token {
+    type: TokenType;
+    value: string;
+    line: number;
+    column: number;
+    offset: number;
+}
+/**
+ * Creates a Token with the given fields.
+ * Used internally by the lexer and in tests to avoid repetitive object literals.
+ */
+export declare function token(type: TokenType, value: string, line: number, column: number, offset: number): Token;
+//# sourceMappingURL=token.d.ts.map

package/dist/lexer/token.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../src/lexer/token.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,oBAAY,SAAS;IAMnB,iFAAiF;IACjF,MAAM,WAAW;IAEjB;;;;OAIG;IACH,MAAM,WAAW;IAEjB,mEAAmE;IACnE,OAAO,YAAY;IAEnB,0CAA0C;IAC1C,KAAK,UAAU;IAEf,8CAA8C;IAC9C,OAAO,YAAY;IAEnB,yFAAyF;IACzF,MAAM,WAAW;IAEjB;;;OAGG;IACH,IAAI,SAAS;IAEb,2FAA2F;IAC3F,QAAQ,aAAa;IAErB;;;OAGG;IACH,OAAO,YAAY;IAEnB;;;OAGG;IACH,SAAS,cAAc;IAKvB,8DAA8D;IAC9D,OAAO,YAAY;IAEnB,sFAAsF;IACtF,KAAK,UAAU;IAEf,0FAA0F;IAC1F,IAAI,SAAS;IAEb;;;;OAIG;IACH,OAAO,YAAY;IAEnB;;;OAGG;IACH,QAAQ,aAAa;IAErB;;;OAGG;IACH,KAAK,UAAU;IAEf;;;;OAIG;IACH,UAAU,eAAe;IAIzB;;;OAGG;IACH,IAAI,SAAS;IAEb;;;;OAIG;IACH,KAAK,UAAU;IAIf;;;;OAIG;IACH,EAAE,OAAO;IAET;;;OAGG;IACH,GAAG,QAAQ;IAEX;;;;OAIG;IACH,EAAE,OAAO;IAET;;;;;OAKG;IACH,EAAE,OAAO;IAET;;;;OAIG;IACH,KAAK,WAAW;IAKhB,gDAAgD;IAChD,OAAO,YAAY;IAEnB,kDAAkD;IAClD,QAAQ,aAAa;IAErB,oDAAoD;IACpD,MAAM,WAAW;IAEjB,oDAAoD;IACpD,QAAQ,aAAa;IAErB;;;OAGG;IACH,QAAQ,aAAa;IAErB,sFAAsF;IACtF,KAAK,UAAU;IAEf,mGAAmG;IACnG,IAAI,SAAS;IAIb;;;;OAIG;IACH,SAAS,eAAe;IAExB;;;OAGG;IACH,UAAU,gBAAgB;IAE1B;;;OAGG;IACH,QAAQ,cAAc;IAEtB;;;OAGG;IACH,UAAU,gBAAgB;IAI1B;;;;OAIG;IACH,WAAW,iBAAiB;IAE5B;;;;;OAKG;IACH,UAAU,gBAAgB;IAI1B,iDAAiD;IACjD,MAAM,MAAM;IAEZ,kDAAkD;IAClD,MAAM,MAAM;IAEZ,2EAA2E;IAC3E,KAAK,MAAM;IAEX,sEAAsE;IACtE,GAAG,MAAM;IAET;;;;OAIG;IACH,QAAQ,MAAM;IAId;;;;;OAKG;IACH,OAAO,YAAY;IAEnB;;;;;OAKG;IACH,OAAO,YAAY;IAEnB;;;OAGG;IACH,GAAG,QAAQ;IAEX;;;OAGG;IACH,OAAO,YAAY;CACpB;AAID;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,KAAK;IACpB,IAAI,EAAE,SAAS,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;CACf;AAID;;;GAGG;AACH,wBAAgB,KAAK,CACnB,IAAI,EAAE,SAAS,EACf,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,GACb,KAAK,CAEP"}