@lewin671/python-vm 0.1.0

package/dist/lexer/lexer.js

@@ -0,0 +1,403 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Lexer = void 0;
+const types_1 = require("../types");
+/**
+ * 词法分析器 - 将源代码转换为 token 流
+ */
+class Lexer {
+    constructor(code) {
+        this.pos = 0;
+        this.line = 1;
+        this.column = 1;
+        this.tokens = [];
+        this.indentStack = [0];
+        this.atLineStart = true;
+        this.code = code;
+    }
+    tokenize() {
+        // Helper function to create a token
+        const createToken = (type, value) => ({
+            type,
+            value,
+            line: this.line,
+            column: this.column - value.length
+        });
+        // Helper function to advance position
+        const advance = (n = 1) => {
+            for (let i = 0; i < n; i++) {
+                if (this.code[this.pos] === '\n') {
+                    this.line++;
+                    this.column = 1;
+                }
+                else {
+                    this.column++;
+                }
+                this.pos++;
+            }
+        };
+        // Helper function to peek ahead
+        const peek = (n = 0) => this.code[this.pos + n] || '';
+        const emitIndentTokens = (indent) => {
+            const currentIndent = this.indentStack[this.indentStack.length - 1];
+            if (indent > currentIndent) {
+                this.tokens.push(createToken(types_1.TokenType.INDENT, ''));
+                this.indentStack.push(indent);
+            }
+            else if (indent < currentIndent) {
+                while (indent < this.indentStack[this.indentStack.length - 1]) {
+                    this.tokens.push(createToken(types_1.TokenType.DEDENT, ''));
+                    this.indentStack.pop();
+                }
+                if (indent !== this.indentStack[this.indentStack.length - 1]) {
+                    throw new Error(`Indentation error at line ${this.line}`);
+                }
+            }
+        };
+        // Main tokenization loop
+        while (this.pos < this.code.length) {
+            if (this.atLineStart) {
+                let indent = 0;
+                while (peek() === ' ' || peek() === '\t') {
+                    indent += peek() === '\t' ? 4 : 1;
+                    advance();
+                }
+                if (peek() === '\n') {
+                    this.tokens.push(createToken(types_1.TokenType.NEWLINE, '\n'));
+                    advance();
+                    this.atLineStart = true;
+                    continue;
+                }
+                if (peek() === '#') {
+                    while (this.pos < this.code.length && peek() !== '\n') {
+                        advance();
+                    }
+                    continue;
+                }
+                emitIndentTokens(indent);
+                this.atLineStart = false;
+            }
+            if (this.pos >= this.code.length) {
+                break;
+            }
+            const char = peek();
+            if (char === ' ' || char === '\t') {
+                advance();
+                continue;
+            }
+            if (char === '#') {
+                while (this.pos < this.code.length && peek() !== '\n') {
+                    advance();
+                }
+                continue;
+            }
+            if (char === '\n') {
+                this.tokens.push(createToken(types_1.TokenType.NEWLINE, '\n'));
+                advance();
+                this.atLineStart = true;
+                continue;
+            }
+            // Numbers
+            if (/[0-9]/.test(char) || (char === '.' && /[0-9]/.test(peek(1)))) {
+                let num = '';
+                let hasDot = false;
+                if (char === '.') {
+                    hasDot = true;
+                    num += '.';
+                    advance();
+                }
+                while (this.pos < this.code.length && /[0-9]/.test(peek())) {
+                    num += peek();
+                    advance();
+                }
+                if (peek() === '.' && !hasDot) {
+                    hasDot = true;
+                    num += '.';
+                    advance();
+                    while (this.pos < this.code.length && /[0-9]/.test(peek())) {
+                        num += peek();
+                        advance();
+                    }
+                }
+                if (peek() === 'j' || peek() === 'J') {
+                    num += peek();
+                    advance();
+                }
+                this.tokens.push(createToken(types_1.TokenType.NUMBER, num));
+                continue;
+            }
+            // Strings
+            if (char === '"' || char === "'" || ((char === 'f' || char === 'F') && (peek(1) === '"' || peek(1) === "'"))) {
+                let prefix = '';
+                let quote = char;
+                if (char === 'f' || char === 'F') {
+                    prefix = char;
+                    quote = peek(1);
+                    advance();
+                }
+                let str = prefix + quote;
+                advance();
+                const isTriple = peek() === quote && peek(1) === quote;
+                if (isTriple) {
+                    str += quote + quote;
+                    advance(2);
+                }
+                while (this.pos < this.code.length) {
+                    if (!isTriple && peek() === quote) {
+                        break;
+                    }
+                    if (isTriple && peek() === quote && peek(1) === quote && peek(2) === quote) {
+                        break;
+                    }
+                    if (peek() === '\\') {
+                        str += peek();
+                        advance();
+                        if (this.pos < this.code.length) {
+                            str += peek();
+                            advance();
+                        }
+                    }
+                    else {
+                        str += peek();
+                        advance();
+                    }
+                }
+                if (!isTriple && peek() === quote) {
+                    str += quote;
+                    advance();
+                }
+                else if (isTriple && peek() === quote && peek(1) === quote && peek(2) === quote) {
+                    str += quote + quote + quote;
+                    advance(3);
+                }
+                else {
+                    throw new Error(`Unterminated string at line ${this.line}`);
+                }
+                this.tokens.push(createToken(types_1.TokenType.STRING, str));
+                continue;
+            }
+            // Identifiers and keywords
+            if (/[a-zA-Z_]/.test(char)) {
+                let ident = '';
+                while (this.pos < this.code.length && /[a-zA-Z0-9_]/.test(peek())) {
+                    ident += peek();
+                    advance();
+                }
+                // Check for keywords and boolean literals
+                if (ident === 'def' || ident === 'class' || ident === 'if' || ident === 'elif' || ident === 'else' ||
+                    ident === 'for' || ident === 'while' || ident === 'return' || ident === 'break' || ident === 'continue' ||
+                    ident === 'pass' || ident === 'in' || ident === 'is' || ident === 'and' || ident === 'or' || ident === 'not' ||
+                    ident === 'lambda' || ident === 'yield' || ident === 'try' || ident === 'except' || ident === 'finally' ||
+                    ident === 'with' || ident === 'as' || ident === 'global' || ident === 'nonlocal' || ident === 'assert' ||
+                    ident === 'raise' || ident === 'del' || ident === 'match' || ident === 'case' || ident === 'import' ||
+                    ident === 'async') {
+                    this.tokens.push(createToken(types_1.TokenType.KEYWORD, ident));
+                }
+                else if (ident === 'True' || ident === 'False') {
+                    this.tokens.push(createToken(types_1.TokenType.BOOLEAN, ident));
+                }
+                else if (ident === 'None') {
+                    this.tokens.push(createToken(types_1.TokenType.NONE, ident));
+                }
+                else {
+                    this.tokens.push(createToken(types_1.TokenType.IDENTIFIER, ident));
+                }
+                continue;
+            }
+            // Operators and delimiters
+            switch (char) {
+                case '+':
+                    if (peek(1) === '=') {
+                        this.tokens.push(createToken(types_1.TokenType.OPERATOR, '+='));
+                        advance(2);
+                    }
+                    else {
+                        this.tokens.push(createToken(types_1.TokenType.OPERATOR, '+'));
+                        advance();
+                    }
+                    break;
+                case '-':
+                    if (peek(1) === '=') {
+                        this.tokens.push(createToken(types_1.TokenType.OPERATOR, '-='));
+                        advance(2);
+                    }
+                    else {
+                        this.tokens.push(createToken(types_1.TokenType.OPERATOR, '-'));
+                        advance();
+                    }
+                    break;
+                case '*':
+                    if (peek(1) === '*') {
+                        if (peek(2) === '=') {
+                            this.tokens.push(createToken(types_1.TokenType.OPERATOR, '**='));
+                            advance(3);
+                        }
+                        else {
+                            this.tokens.push(createToken(types_1.TokenType.OPERATOR, '**'));
+                            advance(2);
+                        }
+                    }
+                    else {
+                        if (peek(1) === '=') {
+                            this.tokens.push(createToken(types_1.TokenType.OPERATOR, '*='));
+                            advance(2);
+                        }
+                        else {
+                            this.tokens.push(createToken(types_1.TokenType.OPERATOR, '*'));
+                            advance();
+                        }
+                    }
+                    break;
+                case '/':
+                    if (peek(1) === '/') {
+                        if (peek(2) === '=') {
+                            this.tokens.push(createToken(types_1.TokenType.OPERATOR, '//='));
+                            advance(3);
+                        }
+                        else {
+                            this.tokens.push(createToken(types_1.TokenType.OPERATOR, '//'));
+                            advance(2);
+                        }
+                    }
+                    else if (peek(1) === '=') {
+                        this.tokens.push(createToken(types_1.TokenType.OPERATOR, '/='));
+                        advance(2);
+                    }
+                    else {
+                        this.tokens.push(createToken(types_1.TokenType.OPERATOR, '/'));
+                        advance();
+                    }
+                    break;
+                case '%':
+                    if (peek(1) === '=') {
+                        this.tokens.push(createToken(types_1.TokenType.OPERATOR, '%='));
+                        advance(2);
+                    }
+                    else {
+                        this.tokens.push(createToken(types_1.TokenType.OPERATOR, '%'));
+                        advance();
+                    }
+                    break;
+                case '=':
+                    if (peek(1) === '=') {
+                        this.tokens.push(createToken(types_1.TokenType.OPERATOR, '=='));
+                        advance(2);
+                    }
+                    else {
+                        this.tokens.push(createToken(types_1.TokenType.ASSIGN, '='));
+                        advance();
+                    }
+                    break;
+                case '!':
+                    if (peek(1) === '=') {
+                        this.tokens.push(createToken(types_1.TokenType.OPERATOR, '!='));
+                        advance(2);
+                    }
+                    else {
+                        throw new Error(`Unexpected character '!' at line ${this.line}`);
+                    }
+                    break;
+                case '&':
+                    this.tokens.push(createToken(types_1.TokenType.OPERATOR, '&'));
+                    advance();
+                    break;
+                case '|':
+                    this.tokens.push(createToken(types_1.TokenType.OPERATOR, '|'));
+                    advance();
+                    break;
+                case '^':
+                    this.tokens.push(createToken(types_1.TokenType.OPERATOR, '^'));
+                    advance();
+                    break;
+                case '~':
+                    this.tokens.push(createToken(types_1.TokenType.OPERATOR, '~'));
+                    advance();
+                    break;
+                case '<':
+                    if (peek(1) === '<') {
+                        this.tokens.push(createToken(types_1.TokenType.OPERATOR, '<<'));
+                        advance(2);
+                    }
+                    else if (peek(1) === '=') {
+                        this.tokens.push(createToken(types_1.TokenType.OPERATOR, '<='));
+                        advance(2);
+                    }
+                    else {
+                        this.tokens.push(createToken(types_1.TokenType.OPERATOR, '<'));
+                        advance();
+                    }
+                    break;
+                case '>':
+                    if (peek(1) === '>') {
+                        this.tokens.push(createToken(types_1.TokenType.OPERATOR, '>>'));
+                        advance(2);
+                    }
+                    else if (peek(1) === '=') {
+                        this.tokens.push(createToken(types_1.TokenType.OPERATOR, '>='));
+                        advance(2);
+                    }
+                    else {
+                        this.tokens.push(createToken(types_1.TokenType.OPERATOR, '>'));
+                        advance();
+                    }
+                    break;
+                case '(':
+                    this.tokens.push(createToken(types_1.TokenType.LPAREN, '('));
+                    advance();
+                    break;
+                case ')':
+                    this.tokens.push(createToken(types_1.TokenType.RPAREN, ')'));
+                    advance();
+                    break;
+                case '[':
+                    this.tokens.push(createToken(types_1.TokenType.LBRACKET, '['));
+                    advance();
+                    break;
+                case ']':
+                    this.tokens.push(createToken(types_1.TokenType.RBRACKET, ']'));
+                    advance();
+                    break;
+                case '{':
+                    this.tokens.push(createToken(types_1.TokenType.LBRACE, '{'));
+                    advance();
+                    break;
+                case '}':
+                    this.tokens.push(createToken(types_1.TokenType.RBRACE, '}'));
+                    advance();
+                    break;
+                case ':':
+                    this.tokens.push(createToken(types_1.TokenType.COLON, ':'));
+                    advance();
+                    break;
+                case ',':
+                    this.tokens.push(createToken(types_1.TokenType.COMMA, ','));
+                    advance();
+                    break;
+                case '.':
+                    this.tokens.push(createToken(types_1.TokenType.DOT, '.'));
+                    advance();
+                    break;
+                case '@':
+                    this.tokens.push(createToken(types_1.TokenType.AT, '@'));
+                    advance();
+                    break;
+                case ' ':
+                case '\t':
+                    // Should be handled by skipWhitespace
+                    advance();
+                    break;
+                default:
+                    throw new Error(`Unexpected character '${char}' at line ${this.line}, column ${this.column}`);
+            }
+        }
+        while (this.indentStack.length > 1) {
+            this.tokens.push(createToken(types_1.TokenType.DEDENT, ''));
+            this.indentStack.pop();
+        }
+        // Add EOF
+        this.tokens.push(createToken(types_1.TokenType.EOF, ''));
+        return this.tokens;
+    }
+}
+exports.Lexer = Lexer;
+//# sourceMappingURL=lexer.js.map

package/dist/parser/expressions.d.ts

@@ -0,0 +1,30 @@
+import type { Parser } from './parser';
+import { ASTNode } from '../types';
+export declare function parseStringLiteral(this: Parser): ASTNode;
+export declare function parseLiteral(this: Parser): ASTNode;
+export declare function parseIdentifier(this: Parser): ASTNode;
+export declare function parseArguments(this: Parser): ASTNode[];
+export declare function parseSlice(this: Parser): ASTNode;
+export declare function parsePatternAtom(this: Parser): ASTNode;
+export declare function parsePattern(this: Parser): ASTNode;
+export declare function parseLambdaParameters(this: Parser): string[];
+export declare function parseLambda(this: Parser): ASTNode;
+export declare function parseAtom(this: Parser): ASTNode;
+export declare function parsePostfix(this: Parser): ASTNode;
+export declare function parseUnary(this: Parser): ASTNode;
+export declare function parsePower(this: Parser): ASTNode;
+export declare function parseFactor(this: Parser): ASTNode;
+export declare function parseTerm(this: Parser): ASTNode;
+export declare function parseShift(this: Parser): ASTNode;
+export declare function parseBitAnd(this: Parser): ASTNode;
+export declare function parseBitXor(this: Parser): ASTNode;
+export declare function parseBitOr(this: Parser): ASTNode;
+export declare function parseComparison(this: Parser): ASTNode;
+export declare function parseNot(this: Parser): ASTNode;
+export declare function parseAnd(this: Parser): ASTNode;
+export declare function parseOr(this: Parser): ASTNode;
+export declare function parseIfExpression(this: Parser): ASTNode;
+export declare function parseExpression(this: Parser): ASTNode;
+export declare function parseExpressionNoIf(this: Parser): ASTNode;
+export declare function parseComprehension(this: Parser, expression: ASTNode): ASTNode;
+//# sourceMappingURL=expressions.d.ts.map