english-lang 0.1.0

package/dist/src/lexer/Lexer.js

@@ -0,0 +1,167 @@
+"use strict";
+// ============================================================
+// Lexer.ts — Tokenizes .eng source into an INDENT-aware stream
+//
+// Token stream rules:
+//   - Blank lines and comment lines (--) produce NO tokens
+//   - Each non-blank line ends with NEWLINE
+//   - Indentation increases produce INDENT before the line tokens
+//   - Indentation decreases produce one or more DEDENT before the line tokens
+//   - Tabs are normalized to 4 spaces
+// ============================================================
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Lexer = void 0;
+class Lexer {
+    constructor(source) {
+        this.tokens = [];
+        this.indentStack = [0];
+        this.source = source;
+        this.lines = source.split('\n');
+    }
+    tokenize() {
+        for (let lineIdx = 0; lineIdx < this.lines.length; lineIdx++) {
+            this.processLine(this.lines[lineIdx], lineIdx + 1);
+        }
+        // Flush remaining DEDENT levels at EOF
+        while (this.indentStack.length > 1) {
+            this.indentStack.pop();
+            this.push('DEDENT', '', this.lines.length, 1);
+        }
+        this.push('EOF', '', this.lines.length + 1, 1);
+        return this.tokens;
+    }
+    // ── Line processing ───────────────────────────────────────
+    processLine(raw, lineNum) {
+        // Measure leading indentation (tabs → 4 spaces)
+        let indent = 0;
+        let i = 0;
+        while (i < raw.length && (raw[i] === ' ' || raw[i] === '\t')) {
+            indent += raw[i] === '\t' ? 4 : 1;
+            i++;
+        }
+        const content = raw.slice(i);
+        // Skip blank lines and comment lines entirely
+        if (content === '' || content.startsWith('--'))
+            return;
+        // Emit INDENT / DEDENT tokens before the line's own tokens
+        const current = this.indentStack[this.indentStack.length - 1];
+        if (indent > current) {
+            this.indentStack.push(indent);
+            this.push('INDENT', '', lineNum, 1);
+        }
+        else if (indent < current) {
+            while (this.indentStack.length > 1 && this.indentStack[this.indentStack.length - 1] > indent) {
+                this.indentStack.pop();
+                this.push('DEDENT', '', lineNum, 1);
+            }
+        }
+        // Tokenize content characters
+        this.tokenizeContent(content, lineNum, i + 1);
+        // Every logical line ends with NEWLINE
+        this.push('NEWLINE', '\n', lineNum, raw.length + 1);
+    }
+    // ── Character-level tokenizer ─────────────────────────────
+    tokenizeContent(content, lineNum, startCol) {
+        let i = 0;
+        let col = startCol;
+        while (i < content.length) {
+            const ch = content[i];
+            // Whitespace
+            if (ch === ' ' || ch === '\t') {
+                const step = ch === '\t' ? 4 : 1;
+                i++;
+                col += step;
+                continue;
+            }
+            // Comment: -- to end of line
+            if (ch === '-' && content[i + 1] === '-')
+                break;
+            // Arrow: ->
+            if (ch === '-' && content[i + 1] === '>') {
+                this.push('ARROW', '->', lineNum, col);
+                i += 2;
+                col += 2;
+                continue;
+            }
+            // String literal: "..."
+            if (ch === '"') {
+                let j = i + 1;
+                while (j < content.length && content[j] !== '"')
+                    j++;
+                const value = content.slice(i + 1, j);
+                this.push('STRING', value, lineNum, col);
+                col += j - i + 1;
+                i = j + 1;
+                continue;
+            }
+            // Possessive: 's
+            if (ch === '\'' && content[i + 1] === 's' && (i + 2 >= content.length || !/[a-zA-Z0-9_]/.test(content[i + 2]))) {
+                this.push('APOSTROPHE_S', "'s", lineNum, col);
+                i += 2;
+                col += 2;
+                continue;
+            }
+            // Number: optional leading minus only if not after a word char
+            if ((ch >= '0' && ch <= '9') ||
+                (ch === '-' && i + 1 < content.length && content[i + 1] >= '0' && content[i + 1] <= '9' &&
+                    (i === 0 || content[i - 1] === ' ' || content[i - 1] === '\t'))) {
+                let j = i + (ch === '-' ? 1 : 0);
+                while (j < content.length && (content[j] >= '0' && content[j] <= '9' || content[j] === '.'))
+                    j++;
+                this.push('NUMBER', content.slice(i, j), lineNum, col);
+                col += j - i;
+                i = j;
+                continue;
+            }
+            // Single-character tokens
+            if (ch === ':') {
+                this.push('COLON', ':', lineNum, col);
+                i++;
+                col++;
+                continue;
+            }
+            if (ch === '(') {
+                this.push('LPAREN', '(', lineNum, col);
+                i++;
+                col++;
+                continue;
+            }
+            if (ch === ')') {
+                this.push('RPAREN', ')', lineNum, col);
+                i++;
+                col++;
+                continue;
+            }
+            if (ch === ',') {
+                this.push('COMMA', ',', lineNum, col);
+                i++;
+                col++;
+                continue;
+            }
+            if (ch === '.') {
+                this.push('DOT', '.', lineNum, col);
+                i++;
+                col++;
+                continue;
+            }
+            // Word: identifier or keyword
+            if (/[a-zA-Z_]/.test(ch)) {
+                let j = i;
+                while (j < content.length && /[a-zA-Z0-9_]/.test(content[j]))
+                    j++;
+                this.push('WORD', content.slice(i, j), lineNum, col);
+                col += j - i;
+                i = j;
+                continue;
+            }
+            // Unknown character — skip
+            i++;
+            col++;
+        }
+    }
+    // ── Helpers ────────────────────────────────────────────────
+    push(type, value, line, col) {
+        this.tokens.push({ type, value, line, col });
+    }
+}
+exports.Lexer = Lexer;