natalie_parser 2.0.0 → 2.2.0

data/src/lexer.cpp

@@ -80,7 +80,47 @@ Token Lexer::next_token() {
     m_whitespace_precedes = skip_whitespace();
     m_token_line = m_cursor_line;
     m_token_column = m_cursor_column;
-    return build_next_token();
+    Token token = build_next_token();
+    switch (token.type()) {
+    case Token::Type::AliasKeyword:
+        m_remaining_method_names = 2;
+        break;
+    case Token::Type::ConstantResolution:
+    case Token::Type::DefKeyword:
+        m_remaining_method_names = 1;
+        m_allow_assignment_method = true;
+        break;
+    case Token::Type::Dot:
+        m_remaining_method_names = 1;
+        break;
+    case Token::Type::UndefKeyword:
+        m_remaining_method_names = std::numeric_limits<size_t>::max();
+        m_method_name_separator = Token::Type::Comma;
+        break;
+    default:
+        if (m_method_name_separator != Token::Type::Invalid) {
+            if (m_last_method_name) {
+                m_last_method_name = {};
+                if (token.type() != m_method_name_separator) {
+                    m_remaining_method_names = 0;
+                    m_method_name_separator = Token::Type::Invalid;
+                }
+            } else {
+                m_last_method_name = token;
+            }
+        } else if (m_remaining_method_names > 0) {
+            m_remaining_method_names--;
+        } else {
+            m_allow_assignment_method = false;
+        }
+        break;
+    }
+    return token;
+}
+
+bool is_name_start_char(char c) {
+    if (!c) return false;
+    return (c >= 'a' && c <= 'z') || c == '_' || (unsigned int)c >= 128;
 }
 
 bool is_identifier_char(char c) {
@@ -204,10 +244,10 @@ Token Lexer::build_next_token() {
             advance();
             return Token { Token::Type::PlusEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         case '@':
-            if (m_last_token.is_def_keyword() || m_last_token.is_dot()) {
+            if (m_remaining_method_names > 0) {
                 advance();
                 SharedPtr<String> lit = new String("+@");
-                return Token { Token::Type::BareName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+                return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             } else {
                 return Token { Token::Type::Plus, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             }
@@ -224,10 +264,10 @@ Token Lexer::build_next_token() {
             advance();
             return Token { Token::Type::MinusEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         case '@':
-            if (m_last_token.is_def_keyword() || m_last_token.is_dot()) {
+            if (m_remaining_method_names > 0) {
                 advance();
                 SharedPtr<String> lit = new String("-@");
-                return Token { Token::Type::BareName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+                return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             } else {
                 return Token { Token::Type::Minus, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             }
@@ -256,21 +296,28 @@ Token Lexer::build_next_token() {
         advance();
         if (!m_last_token)
             return consume_regexp('/', '/');
+        if (m_remaining_method_names > 0)
+            return Token { Token::Type::Slash, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         switch (m_last_token.type()) {
         case Token::Type::Comma:
         case Token::Type::Doc:
+        case Token::Type::Equal:
         case Token::Type::LBracket:
         case Token::Type::LCurlyBrace:
         case Token::Type::LParen:
         case Token::Type::Match:
         case Token::Type::Newline:
+        case Token::Type::Not:
+        case Token::Type::Pipe:
             return consume_regexp('/', '/');
-        case Token::Type::DefKeyword:
-            return Token { Token::Type::Slash, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         default: {
             switch (current_char()) {
             case ' ':
-                return Token { Token::Type::Slash, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+                if (m_last_token.is_keyword() && m_last_token.can_precede_regexp_literal()) {
+                    return consume_regexp('/', '/');
+                } else {
+                    return Token { Token::Type::Slash, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+                }
             case '=':
                 advance();
                 return Token { Token::Type::SlashEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };
@@ -291,216 +338,26 @@ Token Lexer::build_next_token() {
             advance();
             return Token { Token::Type::PercentEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         case 'q':
-            switch (peek()) {
-            case '[':
-                advance(2);
-                return consume_single_quoted_string('[', ']');
-            case '{':
-                advance(2);
-                return consume_single_quoted_string('{', '}');
-            case '<':
-                advance(2);
-                return consume_single_quoted_string('<', '>');
-            case '(':
-                advance(2);
-                return consume_single_quoted_string('(', ')');
-            default: {
-                char c = peek();
-                if (char_can_be_string_or_regexp_delimiter(c)) {
-                    advance(2);
-                    return consume_single_quoted_string(c, c);
-                } else {
-                    return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
-                }
-            }
-            }
+            return consume_percent_string(&Lexer::consume_single_quoted_string);
         case 'Q':
-            switch (peek()) {
-            case '[':
-                advance(2);
-                return consume_double_quoted_string('[', ']');
-            case '{':
-                advance(2);
-                return consume_double_quoted_string('{', '}');
-            case '<':
-                advance(2);
-                return consume_double_quoted_string('<', '>');
-            case '(':
-                advance(2);
-                return consume_double_quoted_string('(', ')');
-            default: {
-                char c = peek();
-                if (char_can_be_string_or_regexp_delimiter(c)) {
-                    advance(2);
-                    return consume_double_quoted_string(c, c);
-                } else {
-                    return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
-                }
-            }
-            }
+            return consume_percent_string(&Lexer::consume_interpolated_string);
         case 'r':
-            switch (peek()) {
-            case '[':
-                advance(2);
-                return consume_regexp('[', ']');
-            case '{':
-                advance(2);
-                return consume_regexp('{', '}');
-            case '(':
-                advance(2);
-                return consume_regexp('(', ')');
-            case '<':
-                advance(2);
-                return consume_regexp('<', '>');
-            default: {
-                char c = peek();
-                if (char_can_be_string_or_regexp_delimiter(c)) {
-                    advance(2);
-                    return consume_regexp(c, c);
-                } else {
-                    return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
-                }
-            }
-            }
+            return consume_percent_string(&Lexer::consume_regexp);
+        case 's':
+            return consume_percent_string(&Lexer::consume_percent_symbol);
         case 'x':
-            switch (peek()) {
-            case '/': {
-                advance(2);
-                return consume_double_quoted_string('/', '/', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
-            }
-            case '[': {
-                advance(2);
-                return consume_double_quoted_string('[', ']', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
-            }
-            case '{': {
-                advance(2);
-                return consume_double_quoted_string('{', '}', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
-            }
-            case '(': {
-                advance(2);
-                return consume_double_quoted_string('(', ')', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
-            }
-            default:
-                return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
-            }
+            return consume_percent_string(&Lexer::consume_interpolated_shell);
         case 'w':
-            switch (peek()) {
-            case '/':
-            case '|': {
-                char c = next();
-                advance();
-                return consume_quoted_array_without_interpolation(c, c, Token::Type::PercentLowerW);
-            }
-            case '[':
-                advance(2);
-                return consume_quoted_array_without_interpolation('[', ']', Token::Type::PercentLowerW);
-            case '{':
-                advance(2);
-                return consume_quoted_array_without_interpolation('{', '}', Token::Type::PercentLowerW);
-            case '<':
-                advance(2);
-                return consume_quoted_array_without_interpolation('<', '>', Token::Type::PercentLowerW);
-            case '(':
-                advance(2);
-                return consume_quoted_array_without_interpolation('(', ')', Token::Type::PercentLowerW);
-            default:
-                return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
-            }
+            return consume_percent_string(&Lexer::consume_percent_lower_w);
         case 'W':
-            switch (peek()) {
-            case '/':
-            case '|': {
-                char c = next();
-                advance();
-                return consume_quoted_array_with_interpolation(0, c, Token::Type::PercentUpperW);
-            }
-            case '[':
-                advance(2);
-                return consume_quoted_array_with_interpolation('[', ']', Token::Type::PercentUpperW);
-            case '{':
-                advance(2);
-                return consume_quoted_array_with_interpolation('{', '}', Token::Type::PercentUpperW);
-            case '<':
-                advance(2);
-                return consume_quoted_array_with_interpolation('<', '>', Token::Type::PercentUpperW);
-            case '(':
-                advance(2);
-                return consume_quoted_array_with_interpolation('(', ')', Token::Type::PercentUpperW);
-            default:
-                return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
-            }
+            return consume_percent_string(&Lexer::consume_percent_upper_w);
         case 'i':
-            switch (peek()) {
-            case '|':
-            case '/': {
-                char c = next();
-                advance();
-                return consume_quoted_array_without_interpolation(c, c, Token::Type::PercentLowerI);
-            }
-            case '[':
-                advance(2);
-                return consume_quoted_array_without_interpolation('[', ']', Token::Type::PercentLowerI);
-            case '{':
-                advance(2);
-                return consume_quoted_array_without_interpolation('{', '}', Token::Type::PercentLowerI);
-            case '<':
-                advance(2);
-                return consume_quoted_array_without_interpolation('<', '>', Token::Type::PercentLowerI);
-            case '(':
-                advance(2);
-                return consume_quoted_array_without_interpolation('(', ')', Token::Type::PercentLowerI);
-            default:
-                return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
-            }
+            return consume_percent_string(&Lexer::consume_percent_lower_i);
         case 'I':
-            switch (peek()) {
-            case '|':
-            case '/': {
-                char c = next();
-                advance();
-                return consume_quoted_array_with_interpolation(0, c, Token::Type::PercentUpperI);
-            }
-            case '[':
-                advance(2);
-                return consume_quoted_array_with_interpolation('[', ']', Token::Type::PercentUpperI);
-            case '{':
-                advance(2);
-                return consume_quoted_array_with_interpolation('{', '}', Token::Type::PercentUpperI);
-            case '<':
-                advance(2);
-                return consume_quoted_array_with_interpolation('<', '>', Token::Type::PercentUpperI);
-            case '(':
-                advance(2);
-                return consume_quoted_array_with_interpolation('(', ')', Token::Type::PercentUpperI);
-            default:
-                return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
-            }
-        case '[':
-            advance();
-            return consume_double_quoted_string('[', ']');
-        case '{':
-            advance();
-            return consume_double_quoted_string('{', '}');
-        case '<':
-            advance();
-            return consume_double_quoted_string('<', '>');
-        case '(':
-            if (m_last_token.type() == Token::Type::DefKeyword || m_last_token.type() == Token::Type::Dot) {
-                // It's a trap! This looks like a %(string) but it's a method def/call!
-                break;
-            }
-            advance();
-            return consume_double_quoted_string('(', ')');
-        default: {
-            auto c = current_char();
-            if (char_can_be_string_or_regexp_delimiter(c)) {
-                advance();
-                return consume_double_quoted_string(c, c);
-            }
-            break;
-        }
+            return consume_percent_string(&Lexer::consume_percent_upper_i);
+        default:
+            return consume_percent_string(&Lexer::consume_interpolated_string, false);
         }
-        return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     case '!':
         advance();
         switch (current_char()) {
@@ -511,10 +368,10 @@ Token Lexer::build_next_token() {
             advance();
             return Token { Token::Type::NotMatch, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         case '@':
-            if (m_last_token.is_def_keyword() || m_last_token.is_dot()) {
+            if (m_remaining_method_names > 0) {
                 advance();
                 SharedPtr<String> lit = new String("!@");
-                return Token { Token::Type::BareName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+                return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             } else {
                 return Token { Token::Type::Not, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             }
@@ -649,10 +506,10 @@ Token Lexer::build_next_token() {
         advance();
         switch (current_char()) {
         case '@':
-            if (m_last_token.is_def_keyword() || m_last_token.is_dot()) {
+            if (m_remaining_method_names > 0) {
                 advance();
                 SharedPtr<String> lit = new String("~@");
-                return Token { Token::Type::BareName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+                return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             } else {
                 return Token { Token::Type::Tilde, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             }
@@ -661,7 +518,7 @@ Token Lexer::build_next_token() {
         }
     case '?': {
         auto c = next();
-        if (isspace(c)) {
+        if (isspace(c) || c == 0) {
             m_open_ternary = true;
             return Token { Token::Type::TernaryQuestion, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         } else {
@@ -691,7 +548,7 @@ Token Lexer::build_next_token() {
             advance();
             auto string = consume_single_quoted_string('\'', '\'');
             return Token { Token::Type::Symbol, string.literal(), m_file, m_token_line, m_token_column, m_whitespace_precedes };
-        } else if (isspace(c)) {
+        } else if (isspace(c) || c == 0) {
             m_open_ternary = false;
             auto token = Token { Token::Type::TernaryColon, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             return token;
@@ -789,13 +646,18 @@ Token Lexer::build_next_token() {
         return Token { Token::Type::Comma, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     case '"':
         advance();
-        return consume_double_quoted_string('"', '"');
+        return consume_interpolated_string('"', '"');
     case '\'':
         advance();
         return consume_single_quoted_string('\'', '\'');
     case '`': {
         advance();
-        return consume_double_quoted_string('`', '`', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
+        if (m_remaining_method_names > 0) {
+            SharedPtr<String> lit = new String("`");
+            return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+        } else {
+            return consume_interpolated_shell('`', '`');
+        }
     }
     case '#':
         if (token_is_first_on_line()) {
@@ -858,14 +720,14 @@ Token Lexer::build_next_token() {
 
     Token keyword_token;
 
-    if (!m_last_token.is_dot() && match(4, "self")) {
-        if (current_char() == '.')
+    if (!m_last_token.is_dot() && !m_last_token.is_constant_resolution() && match(4, "self")) {
+        if (current_char() == '.' || (current_char() == ':' && peek() == ':'))
             keyword_token = { Token::Type::SelfKeyword, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         else
             rewind(4);
     }
 
-    if (!m_last_token.is_dot() && !m_last_token.is_def_keyword()) {
+    if (m_remaining_method_names == 0) {
         if (match(12, "__ENCODING__"))
             keyword_token = { Token::Type::ENCODINGKeyword, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         else if (match(8, "__LINE__"))
@@ -960,10 +822,10 @@ Token Lexer::build_next_token() {
     }
 
     auto c = current_char();
-    if ((c >= 'a' && c <= 'z') || c == '_') {
-        return consume_bare_name();
+    if (is_name_start_char(c)) {
+        return consume_bare_name_or_constant(Token::Type::BareName);
     } else if (c >= 'A' && c <= 'Z') {
-        return consume_constant();
+        return consume_bare_name_or_constant(Token::Type::Constant);
     } else {
         auto buf = consume_non_whitespace();
         auto token = Token { Token::Type::Invalid, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
@@ -1093,45 +955,47 @@ Token Lexer::consume_symbol() {
     return Token { Token::Type::Symbol, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
-Token Lexer::consume_word(Token::Type type) {
+SharedPtr<String> Lexer::consume_word() {
     char c = current_char();
     SharedPtr<String> buf = new String("");
     do {
         buf->append_char(c);
         c = next();
     } while (is_identifier_char(c));
+    return buf;
+}
+
+Token Lexer::consume_word(Token::Type type) {
+    return Token { type, consume_word(), m_file, m_token_line, m_token_column, m_whitespace_precedes };
+}
+
+Token Lexer::consume_bare_name_or_constant(Token::Type type) {
+    auto buf = consume_word();
+    auto c = current_char();
     switch (c) {
     case '?':
     case '!':
         advance();
         buf->append_char(c);
         break;
+    case '=':
+        if (m_allow_assignment_method || (!m_last_token.is_dot() && m_remaining_method_names > 0)) {
+            advance();
+            buf->append_char(c);
+        }
+        break;
+    case ':':
+        if (peek() != ':' && m_last_token.can_precede_symbol_key()) {
+            advance();
+            type = Token::Type::SymbolKey;
+        }
+        break;
     default:
         break;
     }
     return Token { type, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
-Token Lexer::consume_bare_name() {
-    auto token = consume_word(Token::Type::BareName);
-    auto c = current_char();
-    if (c == ':' && peek() != ':' && m_last_token.can_precede_symbol_key()) {
-        advance();
-        token.set_type(Token::Type::SymbolKey);
-    }
-    return token;
-}
-
-Token Lexer::consume_constant() {
-    auto token = consume_word(Token::Type::Constant);
-    auto c = current_char();
-    if (c == ':' && peek() != ':' && m_last_token.can_precede_symbol_key()) {
-        advance();
-        token.set_type(Token::Type::SymbolKey);
-    }
-    return token;
-}
-
 Token Lexer::consume_global_variable() {
     switch (peek()) {
     case '?':
@@ -1153,7 +1017,6 @@ Token Lexer::consume_global_variable() {
     case '.':
     case ',':
     case ':':
-    case '_':
     case '~': {
         advance();
         SharedPtr<String> buf = new String("$");
@@ -1277,7 +1140,7 @@ Token Lexer::consume_heredoc() {
         }
         advance();
     } else {
-        heredoc_name = String(consume_word(Token::Type::BareName).literal());
+        heredoc_name = *consume_word();
     }
 
     SharedPtr<String> doc = new String("");
@@ -1673,7 +1536,7 @@ Token Lexer::consume_single_quoted_string(char start_char, char stop_char) {
     SharedPtr<String> buf = new String("");
     char c = current_char();
     while (c) {
-        if (c == '\\') {
+        if (c == '\\' && stop_char != '\\') {
             c = next();
             if (c == stop_char || c == '\\') {
                 buf->append_char(c);
@@ -1720,6 +1583,65 @@ Token Lexer::consume_regexp(char start_char, char stop_char) {
     return Token { Token::Type::InterpolatedRegexpBegin, start_char, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
+Token Lexer::consume_percent_symbol(char start_char, char stop_char) {
+    Token token = consume_single_quoted_string(start_char, stop_char);
+    token.set_type(Token::Type::Symbol);
+    return token;
+}
+
+Token Lexer::consume_interpolated_string(char start_char, char stop_char) {
+    return consume_double_quoted_string(start_char, stop_char, Token::Type::InterpolatedStringBegin, Token::Type::InterpolatedStringEnd);
+}
+
+Token Lexer::consume_interpolated_shell(char start_char, char stop_char) {
+    return consume_double_quoted_string(start_char, stop_char, Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
+}
+
+Token Lexer::consume_percent_lower_w(char start_char, char stop_char) {
+    return consume_quoted_array_without_interpolation(start_char, stop_char, Token::Type::PercentLowerW);
+}
+
+Token Lexer::consume_percent_upper_w(char start_char, char stop_char) {
+    return consume_quoted_array_with_interpolation(start_char, stop_char, Token::Type::PercentUpperW);
+}
+
+Token Lexer::consume_percent_lower_i(char start_char, char stop_char) {
+    return consume_quoted_array_without_interpolation(start_char, stop_char, Token::Type::PercentLowerI);
+}
+
+Token Lexer::consume_percent_upper_i(char start_char, char stop_char) {
+    return consume_quoted_array_with_interpolation(start_char, stop_char, Token::Type::PercentUpperI);
+}
+
+Token Lexer::consume_percent_string(Token (Lexer::*consumer)(char start_char, char stop_char), bool is_lettered) {
+    if (m_remaining_method_names > 0) {
+        return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+    }
+    char c = is_lettered ? peek() : current_char();
+    size_t bytes = is_lettered ? 2 : 1;
+    switch (c) {
+    case '[':
+        advance(bytes);
+        return (this->*consumer)('[', ']');
+    case '{':
+        advance(bytes);
+        return (this->*consumer)('{', '}');
+    case '<':
+        advance(bytes);
+        return (this->*consumer)('<', '>');
+    case '(':
+        advance(bytes);
+        return (this->*consumer)('(', ')');
+    default:
+        if (char_can_be_string_or_regexp_delimiter(c)) {
+            advance(bytes);
+            return (this->*consumer)(c, c);
+        } else {
+            return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+        }
+    }
+}
+
 SharedPtr<String> Lexer::consume_non_whitespace() {
     char c = current_char();
     SharedPtr<String> buf = new String("");

data/src/node/match_node.cpp

@@ -3,6 +3,11 @@
 namespace NatalieParser {
 
 void MatchNode::transform(Creator *creator) const {
+    if (!m_arg) {
+        creator->set_type("match");
+        creator->append(m_regexp.ref());
+        return;
+    }
     if (m_regexp_on_left)
         creator->set_type("match2");
     else