natalie_parser 2.1.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0598d42a8e0b83aa5a340d1615474a4c5a8e7b209032bf574a230778e4d3ce5e'
-  data.tar.gz: 475cce9e2f078ab440674b48e3b99ba48f163f34b889b376cf681f74a489ce53
+  metadata.gz: 31bee2746c1e7a36eca16194e27467264dfbad5131075090e0b94c675ff150b1
+  data.tar.gz: 7f9a2a77aa2e34c56faeb740900779ff4682471cb02dbd51403eb1b7f72b13a7
 SHA512:
-  metadata.gz: 59cd20f85484845036f934ca5b40c020ad35655824950c036a697bf0bc437d3ebd35eb25bacc7c6f062d096c01b58dea5a1ebd688d0d101192ee7f43b9881b25
-  data.tar.gz: 53bcc9b34ab02943dda1602141ae949e4fa1dcf1c229dfeda8ef710b01d1d9d9c97137ce7e5b08e88b121025009c9a9fd71edbc77eead37f69f37b794fab02f8
+  metadata.gz: 74335871743111340267d41631f3eeae0281ea5f93bb2bb4a6536682a55f325162d428d986ba0182e14adfb27e830f9622048fb443f1561ee9611c988b07bd6b
+  data.tar.gz: 2fd301c60b32db6c074946adceaa54fca3736f6e64335d5bb68c9ad01f092bf19f1ce2810fb75e4641e4b6cf7ebf7e77165538282107f214fa116254b0d37ac0

data/CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## 2.2.0 (2022-10-24)
+
+- FIX: Allow backreference as first arg of implicit call
+- FIX: Exclude `?` and `!` from class/instance/global variable name
+- FIX: Fix parsing of hashes containing assignments
+- FIX: Parse aliases of unary operators and methods named as keywords
+- FIX: Parse alias of keyword-like method to assignment method
+- FIX: Parse alias/undef of assignment methods
+- FIX: Parse backtick as method name
+- FIX: Parse class/module definition in dynamic class/module
+- FIX: Parse `def !`
+- FIX: Parse `def foo::bar`
+- FIX: Parse `foo(<<FOO)`
+- FIX: Parse `for ... do` and `while ... do`
+- FIX: Parse global variables beginning with `$_`
+- FIX: Parse keywords/operators as method names for `::`
+- FIX: Parse __LINE__ constant
+- FIX: Parse more percent strings
+- FIX: Parse more regexes
+- FIX: Parse more stabby procs without parentheses
+- FIX: Parse multiple parameters set to the same default value
+- FIX: Parse parentheses-less stabby proc with keyword arg
+- FIX: Parse undef of methods with special names
+- FIX: Parse `unless ... then`
+- FIX: Parse variable names starting with extended ASCII characters
+- FIX: Raise SyntaxError for `def ====`
+- FIX: Raise syntax error for `def +@.foo`
+- FIX: Tokenize unterminated ternary at EOF
+- FIX: Use lower precedence for block pass
+
 ## 2.1.0 (2022-08-12)
 
 - FEAT: Parse for loops

data/ext/natalie_parser/natalie_parser.cpp

@@ -80,6 +80,7 @@ VALUE token_to_ruby(NatalieParser::Token token, bool include_location_info) {
     case NatalieParser::Token::Type::Constant:
     case NatalieParser::Token::Type::GlobalVariable:
     case NatalieParser::Token::Type::InstanceVariable:
+    case NatalieParser::Token::Type::OperatorName:
     case NatalieParser::Token::Type::Symbol:
     case NatalieParser::Token::Type::SymbolKey: {
         auto literal = token.literal_string();

data/include/natalie_parser/lexer.hpp

@@ -70,9 +70,9 @@ protected:
     virtual bool skip_whitespace();
     virtual Token build_next_token();
     Token consume_symbol();
+    SharedPtr<String> consume_word();
     Token consume_word(Token::Type type);
-    Token consume_bare_name();
-    Token consume_constant();
+    Token consume_bare_name_or_constant(Token::Type type);
     Token consume_global_variable();
     Token consume_heredoc();
     Token consume_numeric();
@@ -80,11 +80,19 @@ protected:
     Token consume_nth_ref();
     long long consume_hex_number(int max_length = 0, bool allow_underscore = false);
     long long consume_octal_number(int max_length = 0, bool allow_underscore = false);
-    Token consume_double_quoted_string(char, char, Token::Type begin_type = Token::Type::InterpolatedStringBegin, Token::Type end_type = Token::Type::InterpolatedStringEnd);
+    Token consume_double_quoted_string(char, char, Token::Type begin_type, Token::Type end_type);
     Token consume_single_quoted_string(char, char);
     Token consume_quoted_array_without_interpolation(char start_char, char stop_char, Token::Type type);
     Token consume_quoted_array_with_interpolation(char start_char, char stop_char, Token::Type type);
     Token consume_regexp(char start_char, char stop_char);
+    Token consume_percent_symbol(char start_char, char stop_char);
+    Token consume_interpolated_string(char start_char, char stop_char);
+    Token consume_interpolated_shell(char start_char, char stop_char);
+    Token consume_percent_lower_w(char start_char, char stop_char);
+    Token consume_percent_upper_w(char start_char, char stop_char);
+    Token consume_percent_lower_i(char start_char, char stop_char);
+    Token consume_percent_upper_i(char start_char, char stop_char);
+    Token consume_percent_string(Token (Lexer::*consumer)(char start_char, char stop_char), bool is_lettered = true);
     SharedPtr<String> consume_non_whitespace();
 
     void utf32_codepoint_to_utf8(String &buf, long long codepoint);
@@ -95,7 +103,7 @@ protected:
     bool token_is_first_on_line() const;
 
     bool char_can_be_string_or_regexp_delimiter(char c) const {
-        return (c >= '!' && c <= '/') || c == ':' || c == '?' || c == '@' || c == '~' || c == '|' || (c >= '^' && c <= '`');
+        return (c >= '!' && c <= '/') || c == ':' || c == ';' || c == '=' || c == '?' || c == '@' || c == '\\' || c == '~' || c == '|' || (c >= '^' && c <= '`');
     }
 
     SharedPtr<String> m_input;
@@ -131,5 +139,10 @@ protected:
     // then increment m_pair_depth
     char m_start_char { 0 };
     int m_pair_depth { 0 };
+
+    size_t m_remaining_method_names { 0 };
+    bool m_allow_assignment_method { false };
+    Token::Type m_method_name_separator { Token::Type::Invalid };
+    Token m_last_method_name {};
 };
 }

data/include/natalie_parser/parser.hpp

@@ -49,16 +49,22 @@ public:
 
     enum class Precedence;
 
+    enum class IterAllow {
+        NONE,
+        CURLY_ONLY,
+        CURLY_AND_BLOCK,
+    };
+
     SharedPtr<Node> tree();
 
 private:
-    bool higher_precedence(Token &token, SharedPtr<Node> left, Precedence current_precedence);
+    bool higher_precedence(Token &token, SharedPtr<Node> left, Precedence current_precedence, IterAllow iter_allow);
 
     Precedence get_precedence(Token &token, SharedPtr<Node> left = {});
 
     bool is_first_arg_of_call_without_parens(SharedPtr<Node>, Token &);
 
-    SharedPtr<Node> parse_expression(Precedence, LocalsHashmap &);
+    SharedPtr<Node> parse_expression(Precedence, LocalsHashmap &, IterAllow = IterAllow::CURLY_AND_BLOCK);
 
     SharedPtr<BlockNode> parse_body(LocalsHashmap &, Precedence, std::function<bool(Token::Type)>, bool = false);
     SharedPtr<BlockNode> parse_body(LocalsHashmap &, Precedence, Token::Type = Token::Type::EndKeyword, bool = false);
@@ -67,8 +73,9 @@ private:
     SharedPtr<Node> parse_if_body(LocalsHashmap &);
     SharedPtr<BlockNode> parse_def_body(LocalsHashmap &);
 
+    void reinsert_collapsed_newline();
     SharedPtr<Node> parse_alias(LocalsHashmap &);
-    SharedPtr<SymbolNode> parse_alias_arg(LocalsHashmap &, const char *, bool);
+    SharedPtr<SymbolNode> parse_alias_arg(LocalsHashmap &, const char *);
     SharedPtr<Node> parse_array(LocalsHashmap &);
     SharedPtr<Node> parse_back_ref(LocalsHashmap &);
     SharedPtr<Node> parse_begin_block(LocalsHashmap &);
@@ -96,7 +103,8 @@ private:
         Method,
         Proc,
     };
-    void parse_def_single_arg(Vector<SharedPtr<Node>> &, LocalsHashmap &, ArgsContext);
+    void parse_def_single_arg(Vector<SharedPtr<Node>> &, LocalsHashmap &, ArgsContext, IterAllow = IterAllow::CURLY_AND_BLOCK);
+    SharedPtr<Node> parse_arg_default_value(LocalsHashmap &, IterAllow);
 
     SharedPtr<Node> parse_encoding(LocalsHashmap &);
     SharedPtr<Node> parse_end_block(LocalsHashmap &);
@@ -115,6 +123,7 @@ private:
     SharedPtr<Node> parse_interpolated_shell(LocalsHashmap &);
     SharedPtr<Node> parse_interpolated_string(LocalsHashmap &);
     SharedPtr<Node> parse_interpolated_symbol(LocalsHashmap &);
+    SharedPtr<Node> parse_line_constant(LocalsHashmap &);
     SharedPtr<Node> parse_lit(LocalsHashmap &);
     SharedPtr<Node> parse_keyword_splat(LocalsHashmap &);
     SharedPtr<Node> parse_keyword_splat_wrapped_in_hash(LocalsHashmap &);
@@ -124,7 +133,7 @@ private:
     SharedPtr<Node> parse_nil(LocalsHashmap &);
     SharedPtr<Node> parse_not(LocalsHashmap &);
     SharedPtr<Node> parse_nth_ref(LocalsHashmap &);
-    void parse_proc_args(Vector<SharedPtr<Node>> &, LocalsHashmap &);
+    void parse_proc_args(Vector<SharedPtr<Node>> &, LocalsHashmap &, IterAllow);
     SharedPtr<Node> parse_redo(LocalsHashmap &);
     SharedPtr<Node> parse_retry(LocalsHashmap &);
     SharedPtr<Node> parse_return(LocalsHashmap &);

data/include/natalie_parser/token.hpp

@@ -103,6 +103,7 @@ public:
         NotKeyword,
         NotMatch,
         NthRef,
+        OperatorName,
         OrKeyword,
         Percent,
         PercentEqual,
@@ -431,6 +432,8 @@ public:
             return "!";
         case Type::NthRef:
             return "nth_ref";
+        case Type::OperatorName:
+            return "operator";
         case Type::OrKeyword:
             return "or";
         case Type::Percent:
@@ -614,6 +617,7 @@ public:
         case Token::Type::LessThanOrEqual:
         case Token::Type::Match:
         case Token::Type::Minus:
+        case Token::Type::Not:
         case Token::Type::NotEqual:
         case Token::Type::NotMatch:
         case Token::Type::Percent:
@@ -635,6 +639,7 @@ public:
     bool is_closing_token() const { return m_type == Type::RBracket || m_type == Type::RCurlyBrace || m_type == Type::RParen; }
     bool is_comma() const { return m_type == Type::Comma; }
     bool is_comment() const { return m_type == Type::Comment; }
+    bool is_constant_resolution() const { return m_type == Type::ConstantResolution; }
     bool is_def_keyword() const { return m_type == Type::DefKeyword; }
     bool is_doc() const { return m_type == Type::Doc; }
     bool is_dot() const { return m_type == Type::Dot; }
@@ -743,9 +748,23 @@ public:
         }
     }
 
+    bool can_be_first_arg_of_def() const {
+        switch (m_type) {
+        case Token::Type::Ampersand:
+        case Token::Type::BareName:
+        case Token::Type::Star:
+        case Token::Type::StarStar:
+        case Token::Type::SymbolKey:
+            return true;
+        default:
+            return false;
+        }
+    }
+
     bool can_be_first_arg_of_implicit_call() const {
         switch (m_type) {
         case Token::Type::Arrow:
+        case Token::Type::BackRef:
         case Token::Type::BareName:
         case Token::Type::BeginKeyword:
         case Token::Type::Bignum:
@@ -800,6 +819,7 @@ public:
         switch (m_type) {
         case Token::Type::Equal:
         case Token::Type::LBracket:
+        case Token::Type::LParen:
             return true;
         default:
             if (is_operator())
@@ -826,6 +846,7 @@ public:
 
     bool can_precede_symbol_key() const {
         switch (m_type) {
+        case Type::Arrow:
         case Type::BareName:
         case Type::Comma:
         case Type::Constant:

data/lib/natalie_parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class NatalieParser
-  VERSION = '2.1.0'
+  VERSION = '2.2.0'
 end

data/src/lexer/interpolated_string_lexer.cpp

@@ -22,7 +22,7 @@ Token InterpolatedStringLexer::build_next_token() {
 Token InterpolatedStringLexer::consume_string() {
     SharedPtr<String> buf = new String;
     while (auto c = current_char()) {
-        if (c == '\\') {
+        if (c == '\\' && m_stop_char != '\\') {
             advance(); // backslash
             auto result = consume_escaped_byte(*buf);
             if (!result.first)

data/src/lexer/regexp_lexer.cpp

@@ -38,7 +38,7 @@ Token RegexpLexer::build_next_token() {
 Token RegexpLexer::consume_regexp() {
     SharedPtr<String> buf = new String;
     while (auto c = current_char()) {
-        if (c == '\\') {
+        if (c == '\\' && m_stop_char != '\\') {
             c = next();
             switch (c) {
             case '/':

data/src/lexer/word_array_lexer.cpp

@@ -38,7 +38,7 @@ Token WordArrayLexer::build_next_token() {
 Token WordArrayLexer::consume_array() {
     m_buffer = new String;
     while (auto c = current_char()) {
-        if (c == '\\') {
+        if (c == '\\' && m_stop_char != '\\') {
             c = next();
             advance();
             if (c == ' ') {

data/src/lexer.cpp

@@ -80,7 +80,47 @@ Token Lexer::next_token() {
     m_whitespace_precedes = skip_whitespace();
     m_token_line = m_cursor_line;
     m_token_column = m_cursor_column;
-    return build_next_token();
+    Token token = build_next_token();
+    switch (token.type()) {
+    case Token::Type::AliasKeyword:
+        m_remaining_method_names = 2;
+        break;
+    case Token::Type::ConstantResolution:
+    case Token::Type::DefKeyword:
+        m_remaining_method_names = 1;
+        m_allow_assignment_method = true;
+        break;
+    case Token::Type::Dot:
+        m_remaining_method_names = 1;
+        break;
+    case Token::Type::UndefKeyword:
+        m_remaining_method_names = std::numeric_limits<size_t>::max();
+        m_method_name_separator = Token::Type::Comma;
+        break;
+    default:
+        if (m_method_name_separator != Token::Type::Invalid) {
+            if (m_last_method_name) {
+                m_last_method_name = {};
+                if (token.type() != m_method_name_separator) {
+                    m_remaining_method_names = 0;
+                    m_method_name_separator = Token::Type::Invalid;
+                }
+            } else {
+                m_last_method_name = token;
+            }
+        } else if (m_remaining_method_names > 0) {
+            m_remaining_method_names--;
+        } else {
+            m_allow_assignment_method = false;
+        }
+        break;
+    }
+    return token;
+}
+
+bool is_name_start_char(char c) {
+    if (!c) return false;
+    return (c >= 'a' && c <= 'z') || c == '_' || (unsigned int)c >= 128;
 }
 
 bool is_identifier_char(char c) {
@@ -204,10 +244,10 @@ Token Lexer::build_next_token() {
             advance();
             return Token { Token::Type::PlusEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         case '@':
-            if (m_last_token.is_def_keyword() || m_last_token.is_dot()) {
+            if (m_remaining_method_names > 0) {
                 advance();
                 SharedPtr<String> lit = new String("+@");
-                return Token { Token::Type::BareName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+                return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             } else {
                 return Token { Token::Type::Plus, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             }
@@ -224,10 +264,10 @@ Token Lexer::build_next_token() {
             advance();
             return Token { Token::Type::MinusEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         case '@':
-            if (m_last_token.is_def_keyword() || m_last_token.is_dot()) {
+            if (m_remaining_method_names > 0) {
                 advance();
                 SharedPtr<String> lit = new String("-@");
-                return Token { Token::Type::BareName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+                return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             } else {
                 return Token { Token::Type::Minus, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             }
@@ -256,17 +296,20 @@ Token Lexer::build_next_token() {
         advance();
         if (!m_last_token)
             return consume_regexp('/', '/');
+        if (m_remaining_method_names > 0)
+            return Token { Token::Type::Slash, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         switch (m_last_token.type()) {
         case Token::Type::Comma:
         case Token::Type::Doc:
+        case Token::Type::Equal:
         case Token::Type::LBracket:
         case Token::Type::LCurlyBrace:
         case Token::Type::LParen:
         case Token::Type::Match:
         case Token::Type::Newline:
+        case Token::Type::Not:
+        case Token::Type::Pipe:
             return consume_regexp('/', '/');
-        case Token::Type::DefKeyword:
-            return Token { Token::Type::Slash, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         default: {
             switch (current_char()) {
             case ' ':
@@ -295,216 +338,26 @@ Token Lexer::build_next_token() {
             advance();
             return Token { Token::Type::PercentEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         case 'q':
-            switch (peek()) {
-            case '[':
-                advance(2);
-                return consume_single_quoted_string('[', ']');
-            case '{':
-                advance(2);
-                return consume_single_quoted_string('{', '}');
-            case '<':
-                advance(2);
-                return consume_single_quoted_string('<', '>');
-            case '(':
-                advance(2);
-                return consume_single_quoted_string('(', ')');
-            default: {
-                char c = peek();
-                if (char_can_be_string_or_regexp_delimiter(c)) {
-                    advance(2);
-                    return consume_single_quoted_string(c, c);
-                } else {
-                    return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
-                }
-            }
-            }
+            return consume_percent_string(&Lexer::consume_single_quoted_string);
         case 'Q':
-            switch (peek()) {
-            case '[':
-                advance(2);
-                return consume_double_quoted_string('[', ']');
-            case '{':
-                advance(2);
-                return consume_double_quoted_string('{', '}');
-            case '<':
-                advance(2);
-                return consume_double_quoted_string('<', '>');
-            case '(':
-                advance(2);
-                return consume_double_quoted_string('(', ')');
-            default: {
-                char c = peek();
-                if (char_can_be_string_or_regexp_delimiter(c)) {
-                    advance(2);
-                    return consume_double_quoted_string(c, c);
-                } else {
-                    return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
-                }
-            }
-            }
+            return consume_percent_string(&Lexer::consume_interpolated_string);
         case 'r':
-            switch (peek()) {
-            case '[':
-                advance(2);
-                return consume_regexp('[', ']');
-            case '{':
-                advance(2);
-                return consume_regexp('{', '}');
-            case '(':
-                advance(2);
-                return consume_regexp('(', ')');
-            case '<':
-                advance(2);
-                return consume_regexp('<', '>');
-            default: {
-                char c = peek();
-                if (char_can_be_string_or_regexp_delimiter(c)) {
-                    advance(2);
-                    return consume_regexp(c, c);
-                } else {
-                    return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
-                }
-            }
-            }
+            return consume_percent_string(&Lexer::consume_regexp);
+        case 's':
+            return consume_percent_string(&Lexer::consume_percent_symbol);
         case 'x':
-            switch (peek()) {
-            case '/': {
-                advance(2);
-                return consume_double_quoted_string('/', '/', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
-            }
-            case '[': {
-                advance(2);
-                return consume_double_quoted_string('[', ']', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
-            }
-            case '{': {
-                advance(2);
-                return consume_double_quoted_string('{', '}', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
-            }
-            case '(': {
-                advance(2);
-                return consume_double_quoted_string('(', ')', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
-            }
-            default:
-                return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
-            }
+            return consume_percent_string(&Lexer::consume_interpolated_shell);
         case 'w':
-            switch (peek()) {
-            case '/':
-            case '|': {
-                char c = next();
-                advance();
-                return consume_quoted_array_without_interpolation(c, c, Token::Type::PercentLowerW);
-            }
-            case '[':
-                advance(2);
-                return consume_quoted_array_without_interpolation('[', ']', Token::Type::PercentLowerW);
-            case '{':
-                advance(2);
-                return consume_quoted_array_without_interpolation('{', '}', Token::Type::PercentLowerW);
-            case '<':
-                advance(2);
-                return consume_quoted_array_without_interpolation('<', '>', Token::Type::PercentLowerW);
-            case '(':
-                advance(2);
-                return consume_quoted_array_without_interpolation('(', ')', Token::Type::PercentLowerW);
-            default:
-                return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
-            }
+            return consume_percent_string(&Lexer::consume_percent_lower_w);
         case 'W':
-            switch (peek()) {
-            case '/':
-            case '|': {
-                char c = next();
-                advance();
-                return consume_quoted_array_with_interpolation(0, c, Token::Type::PercentUpperW);
-            }
-            case '[':
-                advance(2);
-                return consume_quoted_array_with_interpolation('[', ']', Token::Type::PercentUpperW);
-            case '{':
-                advance(2);
-                return consume_quoted_array_with_interpolation('{', '}', Token::Type::PercentUpperW);
-            case '<':
-                advance(2);
-                return consume_quoted_array_with_interpolation('<', '>', Token::Type::PercentUpperW);
-            case '(':
-                advance(2);
-                return consume_quoted_array_with_interpolation('(', ')', Token::Type::PercentUpperW);
-            default:
-                return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
-            }
+            return consume_percent_string(&Lexer::consume_percent_upper_w);
         case 'i':
-            switch (peek()) {
-            case '|':
-            case '/': {
-                char c = next();
-                advance();
-                return consume_quoted_array_without_interpolation(c, c, Token::Type::PercentLowerI);
-            }
-            case '[':
-                advance(2);
-                return consume_quoted_array_without_interpolation('[', ']', Token::Type::PercentLowerI);
-            case '{':
-                advance(2);
-                return consume_quoted_array_without_interpolation('{', '}', Token::Type::PercentLowerI);
-            case '<':
-                advance(2);
-                return consume_quoted_array_without_interpolation('<', '>', Token::Type::PercentLowerI);
-            case '(':
-                advance(2);
-                return consume_quoted_array_without_interpolation('(', ')', Token::Type::PercentLowerI);
-            default:
-                return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
-            }
+            return consume_percent_string(&Lexer::consume_percent_lower_i);
         case 'I':
-            switch (peek()) {
-            case '|':
-            case '/': {
-                char c = next();
-                advance();
-                return consume_quoted_array_with_interpolation(0, c, Token::Type::PercentUpperI);
-            }
-            case '[':
-                advance(2);
-                return consume_quoted_array_with_interpolation('[', ']', Token::Type::PercentUpperI);
-            case '{':
-                advance(2);
-                return consume_quoted_array_with_interpolation('{', '}', Token::Type::PercentUpperI);
-            case '<':
-                advance(2);
-                return consume_quoted_array_with_interpolation('<', '>', Token::Type::PercentUpperI);
-            case '(':
-                advance(2);
-                return consume_quoted_array_with_interpolation('(', ')', Token::Type::PercentUpperI);
-            default:
-                return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
-            }
-        case '[':
-            advance();
-            return consume_double_quoted_string('[', ']');
-        case '{':
-            advance();
-            return consume_double_quoted_string('{', '}');
-        case '<':
-            advance();
-            return consume_double_quoted_string('<', '>');
-        case '(':
-            if (m_last_token.type() == Token::Type::DefKeyword || m_last_token.type() == Token::Type::Dot) {
-                // It's a trap! This looks like a %(string) but it's a method def/call!
-                break;
-            }
-            advance();
-            return consume_double_quoted_string('(', ')');
-        default: {
-            auto c = current_char();
-            if (char_can_be_string_or_regexp_delimiter(c)) {
-                advance();
-                return consume_double_quoted_string(c, c);
-            }
-            break;
-        }
+            return consume_percent_string(&Lexer::consume_percent_upper_i);
+        default:
+            return consume_percent_string(&Lexer::consume_interpolated_string, false);
         }
-        return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     case '!':
         advance();
         switch (current_char()) {
@@ -515,10 +368,10 @@ Token Lexer::build_next_token() {
             advance();
             return Token { Token::Type::NotMatch, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         case '@':
-            if (m_last_token.is_def_keyword() || m_last_token.is_dot()) {
+            if (m_remaining_method_names > 0) {
                 advance();
                 SharedPtr<String> lit = new String("!@");
-                return Token { Token::Type::BareName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+                return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             } else {
                 return Token { Token::Type::Not, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             }
@@ -653,10 +506,10 @@ Token Lexer::build_next_token() {
         advance();
         switch (current_char()) {
         case '@':
-            if (m_last_token.is_def_keyword() || m_last_token.is_dot()) {
+            if (m_remaining_method_names > 0) {
                 advance();
                 SharedPtr<String> lit = new String("~@");
-                return Token { Token::Type::BareName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+                return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             } else {
                 return Token { Token::Type::Tilde, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             }
@@ -665,7 +518,7 @@ Token Lexer::build_next_token() {
         }
     case '?': {
         auto c = next();
-        if (isspace(c)) {
+        if (isspace(c) || c == 0) {
             m_open_ternary = true;
             return Token { Token::Type::TernaryQuestion, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         } else {
@@ -695,7 +548,7 @@ Token Lexer::build_next_token() {
             advance();
             auto string = consume_single_quoted_string('\'', '\'');
             return Token { Token::Type::Symbol, string.literal(), m_file, m_token_line, m_token_column, m_whitespace_precedes };
-        } else if (isspace(c)) {
+        } else if (isspace(c) || c == 0) {
             m_open_ternary = false;
             auto token = Token { Token::Type::TernaryColon, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             return token;
@@ -793,13 +646,18 @@ Token Lexer::build_next_token() {
         return Token { Token::Type::Comma, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     case '"':
         advance();
-        return consume_double_quoted_string('"', '"');
+        return consume_interpolated_string('"', '"');
     case '\'':
         advance();
         return consume_single_quoted_string('\'', '\'');
     case '`': {
         advance();
-        return consume_double_quoted_string('`', '`', Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
+        if (m_remaining_method_names > 0) {
+            SharedPtr<String> lit = new String("`");
+            return Token { Token::Type::OperatorName, lit, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+        } else {
+            return consume_interpolated_shell('`', '`');
+        }
     }
     case '#':
         if (token_is_first_on_line()) {
@@ -862,14 +720,14 @@ Token Lexer::build_next_token() {
 
     Token keyword_token;
 
-    if (!m_last_token.is_dot() && match(4, "self")) {
-        if (current_char() == '.')
+    if (!m_last_token.is_dot() && !m_last_token.is_constant_resolution() && match(4, "self")) {
+        if (current_char() == '.' || (current_char() == ':' && peek() == ':'))
             keyword_token = { Token::Type::SelfKeyword, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         else
             rewind(4);
     }
 
-    if (!m_last_token.is_dot() && !m_last_token.is_def_keyword()) {
+    if (m_remaining_method_names == 0) {
         if (match(12, "__ENCODING__"))
             keyword_token = { Token::Type::ENCODINGKeyword, m_file, m_token_line, m_token_column, m_whitespace_precedes };
         else if (match(8, "__LINE__"))
@@ -964,10 +822,10 @@ Token Lexer::build_next_token() {
     }
 
     auto c = current_char();
-    if ((c >= 'a' && c <= 'z') || c == '_') {
-        return consume_bare_name();
+    if (is_name_start_char(c)) {
+        return consume_bare_name_or_constant(Token::Type::BareName);
     } else if (c >= 'A' && c <= 'Z') {
-        return consume_constant();
+        return consume_bare_name_or_constant(Token::Type::Constant);
     } else {
         auto buf = consume_non_whitespace();
         auto token = Token { Token::Type::Invalid, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
@@ -1097,45 +955,47 @@ Token Lexer::consume_symbol() {
     return Token { Token::Type::Symbol, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
-Token Lexer::consume_word(Token::Type type) {
+SharedPtr<String> Lexer::consume_word() {
     char c = current_char();
     SharedPtr<String> buf = new String("");
     do {
         buf->append_char(c);
         c = next();
     } while (is_identifier_char(c));
+    return buf;
+}
+
+Token Lexer::consume_word(Token::Type type) {
+    return Token { type, consume_word(), m_file, m_token_line, m_token_column, m_whitespace_precedes };
+}
+
+Token Lexer::consume_bare_name_or_constant(Token::Type type) {
+    auto buf = consume_word();
+    auto c = current_char();
     switch (c) {
     case '?':
     case '!':
         advance();
         buf->append_char(c);
         break;
+    case '=':
+        if (m_allow_assignment_method || (!m_last_token.is_dot() && m_remaining_method_names > 0)) {
+            advance();
+            buf->append_char(c);
+        }
+        break;
+    case ':':
+        if (peek() != ':' && m_last_token.can_precede_symbol_key()) {
+            advance();
+            type = Token::Type::SymbolKey;
+        }
+        break;
     default:
         break;
     }
     return Token { type, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
-Token Lexer::consume_bare_name() {
-    auto token = consume_word(Token::Type::BareName);
-    auto c = current_char();
-    if (c == ':' && peek() != ':' && m_last_token.can_precede_symbol_key()) {
-        advance();
-        token.set_type(Token::Type::SymbolKey);
-    }
-    return token;
-}
-
-Token Lexer::consume_constant() {
-    auto token = consume_word(Token::Type::Constant);
-    auto c = current_char();
-    if (c == ':' && peek() != ':' && m_last_token.can_precede_symbol_key()) {
-        advance();
-        token.set_type(Token::Type::SymbolKey);
-    }
-    return token;
-}
-
 Token Lexer::consume_global_variable() {
     switch (peek()) {
     case '?':
@@ -1157,7 +1017,6 @@ Token Lexer::consume_global_variable() {
     case '.':
     case ',':
     case ':':
-    case '_':
     case '~': {
         advance();
         SharedPtr<String> buf = new String("$");
@@ -1281,7 +1140,7 @@ Token Lexer::consume_heredoc() {
         }
         advance();
     } else {
-        heredoc_name = String(consume_word(Token::Type::BareName).literal());
+        heredoc_name = *consume_word();
     }
 
     SharedPtr<String> doc = new String("");
@@ -1677,7 +1536,7 @@ Token Lexer::consume_single_quoted_string(char start_char, char stop_char) {
     SharedPtr<String> buf = new String("");
     char c = current_char();
     while (c) {
-        if (c == '\\') {
+        if (c == '\\' && stop_char != '\\') {
             c = next();
             if (c == stop_char || c == '\\') {
                 buf->append_char(c);
@@ -1724,6 +1583,65 @@ Token Lexer::consume_regexp(char start_char, char stop_char) {
     return Token { Token::Type::InterpolatedRegexpBegin, start_char, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
+Token Lexer::consume_percent_symbol(char start_char, char stop_char) {
+    Token token = consume_single_quoted_string(start_char, stop_char);
+    token.set_type(Token::Type::Symbol);
+    return token;
+}
+
+Token Lexer::consume_interpolated_string(char start_char, char stop_char) {
+    return consume_double_quoted_string(start_char, stop_char, Token::Type::InterpolatedStringBegin, Token::Type::InterpolatedStringEnd);
+}
+
+Token Lexer::consume_interpolated_shell(char start_char, char stop_char) {
+    return consume_double_quoted_string(start_char, stop_char, Token::Type::InterpolatedShellBegin, Token::Type::InterpolatedShellEnd);
+}
+
+Token Lexer::consume_percent_lower_w(char start_char, char stop_char) {
+    return consume_quoted_array_without_interpolation(start_char, stop_char, Token::Type::PercentLowerW);
+}
+
+Token Lexer::consume_percent_upper_w(char start_char, char stop_char) {
+    return consume_quoted_array_with_interpolation(start_char, stop_char, Token::Type::PercentUpperW);
+}
+
+Token Lexer::consume_percent_lower_i(char start_char, char stop_char) {
+    return consume_quoted_array_without_interpolation(start_char, stop_char, Token::Type::PercentLowerI);
+}
+
+Token Lexer::consume_percent_upper_i(char start_char, char stop_char) {
+    return consume_quoted_array_with_interpolation(start_char, stop_char, Token::Type::PercentUpperI);
+}
+
+Token Lexer::consume_percent_string(Token (Lexer::*consumer)(char start_char, char stop_char), bool is_lettered) {
+    if (m_remaining_method_names > 0) {
+        return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+    }
+    char c = is_lettered ? peek() : current_char();
+    size_t bytes = is_lettered ? 2 : 1;
+    switch (c) {
+    case '[':
+        advance(bytes);
+        return (this->*consumer)('[', ']');
+    case '{':
+        advance(bytes);
+        return (this->*consumer)('{', '}');
+    case '<':
+        advance(bytes);
+        return (this->*consumer)('<', '>');
+    case '(':
+        advance(bytes);
+        return (this->*consumer)('(', ')');
+    default:
+        if (char_can_be_string_or_regexp_delimiter(c)) {
+            advance(bytes);
+            return (this->*consumer)(c, c);
+        } else {
+            return Token { Token::Type::Percent, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+        }
+    }
+}
+
 SharedPtr<String> Lexer::consume_non_whitespace() {
     char c = current_char();
     SharedPtr<String> buf = new String("");

data/src/parser.cpp

@@ -46,7 +46,7 @@ enum class Parser::Precedence {
     REF, // foo[1] / foo[1] = 2
 };
 
-bool Parser::higher_precedence(Token &token, SharedPtr<Node> left, Precedence current_precedence) {
+bool Parser::higher_precedence(Token &token, SharedPtr<Node> left, Precedence current_precedence, IterAllow iter_allow) {
     auto next_precedence = get_precedence(token, left);
 
     // printf("token %d, left %d, current_precedence %d, next_precedence %d\n", (int)token.type(), (int)left->type(), (int)current_precedence, (int)next_precedence);
@@ -83,11 +83,11 @@ bool Parser::higher_precedence(Token &token, SharedPtr<Node> left, Precedence cu
         // NOTE: `m_call_depth` should probably be called
         // `m_call_that_can_accept_a_block_depth`, but that's a bit long.
         //
-        return m_call_depth.last() == 0;
+        return iter_allow == IterAllow::CURLY_AND_BLOCK && m_call_depth.last() == 0;
     }
 
     if (next_precedence == Precedence::ITER_CURLY)
-        return left->is_callable();
+        return iter_allow >= IterAllow::CURLY_ONLY && left->is_callable();
 
     return next_precedence > current_precedence;
 }
@@ -198,7 +198,7 @@ Parser::Precedence Parser::get_precedence(Token &token, SharedPtr<Node> left) {
     return Precedence::LOWEST;
 }
 
-SharedPtr<Node> Parser::parse_expression(Parser::Precedence precedence, LocalsHashmap &locals) {
+SharedPtr<Node> Parser::parse_expression(Parser::Precedence precedence, LocalsHashmap &locals, IterAllow iter_allow) {
     skip_newlines();
 
     m_precedence_stack.push(precedence);
@@ -211,7 +211,7 @@ SharedPtr<Node> Parser::parse_expression(Parser::Precedence precedence, LocalsHa
 
     while (current_token().is_valid()) {
         auto token = current_token();
-        if (!higher_precedence(token, left, precedence))
+        if (!higher_precedence(token, left, precedence, iter_allow))
             break;
         auto left_fn = left_denotation(token, left, precedence);
         if (!left_fn)
@@ -278,44 +278,47 @@ SharedPtr<BlockNode> Parser::parse_def_body(LocalsHashmap &locals) {
     return parse_body(locals, Precedence::LOWEST, Token::Type::EndKeyword, true);
 }
 
+void Parser::reinsert_collapsed_newline() {
+    auto token = previous_token();
+    if (token.can_precede_collapsible_newline()) {
+        // Some operators at the end of a line cause the newlines to be collapsed:
+        //
+        //     foo <<
+        //       bar
+        //
+        // ...but in this case (an alias), collapsing the newline was a mistake:
+        //
+        //     alias foo <<
+        //     def bar; end
+        //
+        // So, we'll put the newline back.
+        m_tokens->insert(m_index, Token { Token::Type::Newline, token.file(), token.line(), token.column(), token.whitespace_precedes() });
+    }
+}
+
 SharedPtr<Node> Parser::parse_alias(LocalsHashmap &locals) {
     auto token = current_token();
     advance();
-    SharedPtr<SymbolNode> new_name = parse_alias_arg(locals, "alias new name (first argument)", false);
-    auto existing_name = parse_alias_arg(locals, "alias existing name (second argument)", true);
+    auto new_name = parse_alias_arg(locals, "alias new name (first argument)");
+    auto existing_name = parse_alias_arg(locals, "alias existing name (second argument)");
+    reinsert_collapsed_newline();
     return new AliasNode { token, new_name, existing_name };
 }
 
-SharedPtr<SymbolNode> Parser::parse_alias_arg(LocalsHashmap &locals, const char *expected_message, bool reinsert_collapsed_newline) {
+SharedPtr<SymbolNode> Parser::parse_alias_arg(LocalsHashmap &locals, const char *expected_message) {
     auto token = current_token();
     switch (token.type()) {
-        // TODO: handle Constant too
-    case Token::Type::BareName: {
-        advance();
-        return new SymbolNode { token, token.literal_string() };
-    }
+    case Token::Type::BareName:
+    case Token::Type::Constant:
+    case Token::Type::OperatorName:
+        return new SymbolNode { token, parse_method_name(locals) };
     case Token::Type::Symbol:
         return parse_symbol(locals).static_cast_as<SymbolNode>();
     case Token::Type::InterpolatedSymbolBegin:
         return parse_interpolated_symbol(locals).static_cast_as<SymbolNode>();
     default:
         if (token.is_operator() || token.is_keyword()) {
-            advance();
-            if (token.can_precede_collapsible_newline() && reinsert_collapsed_newline) {
-                // Some operators at the end of a line cause the newlines to be collapsed:
-                //
-                //     foo <<
-                //       bar
-                //
-                // ...but in this case (an alias), collapsing the newline was a mistake:
-                //
-                //     alias foo <<
-                //     def bar; end
-                //
-                // So, we'll put the newline back.
-                m_tokens->insert(m_index, Token { Token::Type::Newline, token.file(), token.line(), token.column(), token.whitespace_precedes() });
-            }
-            return new SymbolNode { token, new String(token.type_value()) };
+            return new SymbolNode { token, parse_method_name(locals) };
         } else {
             throw_unexpected(expected_message);
         }
@@ -501,7 +504,7 @@ SharedPtr<Node> Parser::parse_beginless_range(LocalsHashmap &locals) {
 SharedPtr<Node> Parser::parse_block_pass(LocalsHashmap &locals) {
     auto token = current_token();
     advance();
-    auto value = parse_expression(Precedence::UNARY_PLUS, locals);
+    auto value = parse_expression(Precedence::LOWEST, locals);
     return new BlockPassNode { token, value };
 }
 
@@ -865,15 +868,19 @@ SharedPtr<BlockNode> Parser::parse_case_when_body(LocalsHashmap &locals) {
 }
 
 SharedPtr<Node> Parser::parse_class_or_module_name(LocalsHashmap &locals) {
-    Token name_token;
-    if (current_token().type() == Token::Type::ConstantResolution) {
-        name_token = peek_token();
-    } else {
-        name_token = current_token();
-    }
-    if (name_token.type() != Token::Type::Constant)
+    auto name_token = current_token();
+    auto exp = parse_expression(Precedence::LESS_GREATER, locals);
+    switch (exp->type()) {
+    case Node::Type::Colon2:
+    case Node::Type::Colon3:
+        return exp;
+    case Node::Type::Identifier:
+        if (name_token.type() == Token::Type::Constant)
+            return exp;
+        [[fallthrough]];
+    default:
         throw SyntaxError { "class/module name must be CONSTANT" };
-    return parse_expression(Precedence::LESS_GREATER, locals);
+    }
 }
 
 SharedPtr<Node> Parser::parse_class(LocalsHashmap &locals) {
@@ -991,21 +998,24 @@ SharedPtr<Node> Parser::parse_def(LocalsHashmap &locals) {
     auto token = current_token();
     switch (token.type()) {
     case Token::Type::BareName:
-        if (peek_token().type() == Token::Type::Dot) {
+        if (peek_token().is_dot() || peek_token().is_constant_resolution()) {
             self_node = parse_identifier(locals);
             advance(); // dot
         }
         name = parse_method_name(locals);
         break;
     case Token::Type::Constant:
-        if (peek_token().type() == Token::Type::Dot) {
+        if (peek_token().is_dot() || peek_token().is_constant_resolution()) {
             self_node = parse_constant(locals);
             advance(); // dot
         }
         name = parse_method_name(locals);
         break;
+    case Token::Type::OperatorName:
+        name = parse_method_name(locals);
+        break;
     case Token::Type::SelfKeyword:
-        if (peek_token().type() == Token::Type::Dot) {
+        if (peek_token().is_dot() || peek_token().is_constant_resolution()) {
             self_node = new SelfNode { current_token() };
             advance(); // self
             advance(); // dot
@@ -1023,10 +1033,6 @@ SharedPtr<Node> Parser::parse_def(LocalsHashmap &locals) {
         }
     }
     }
-    if (current_token().is_equal() && !current_token().whitespace_precedes()) {
-        advance();
-        name->append_char('=');
-    }
     auto args = Vector<SharedPtr<Node>> {};
     if (current_token().is_lparen()) {
         advance();
@@ -1037,7 +1043,7 @@ SharedPtr<Node> Parser::parse_def(LocalsHashmap &locals) {
             expect(Token::Type::RParen, "args closing paren");
             advance();
         }
-    } else if (current_token().is_bare_name() || current_token().is_splat() || current_token().is_symbol_key()) {
+    } else if (current_token().can_be_first_arg_of_def()) {
         parse_def_args(args, our_locals);
     }
     SharedPtr<BlockNode> body;
@@ -1085,7 +1091,21 @@ void Parser::parse_def_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals
     }
 }
 
-void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals, ArgsContext context) {
+SharedPtr<Node> Parser::parse_arg_default_value(LocalsHashmap &locals, IterAllow iter_allow) {
+    auto token = current_token();
+    if (token.is_bare_name() && peek_token().is_equal()) {
+        SharedPtr<ArgNode> arg = new ArgNode { token, token.literal_string() };
+        advance();
+        advance(); // =
+        arg->add_to_locals(locals);
+        arg->set_value(parse_arg_default_value(locals, iter_allow));
+        return arg.static_cast_as<Node>();
+    } else {
+        return parse_expression(Precedence::DEF_ARG, locals, iter_allow);
+    }
+}
+
+void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals, ArgsContext context, IterAllow iter_allow) {
     auto args_have_any_splat = [&]() { return !args.is_empty() && args.last()->type() == Node::Type::Arg && args.last().static_cast_as<ArgNode>()->splat_or_kwsplat(); };
     auto args_have_keyword = [&]() { return !args.is_empty() && args.last()->type() == Node::Type::KeywordArg; };
 
@@ -1105,7 +1125,7 @@ void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &
             if (args_have_any_splat())
                 throw_error(token, "default value after splat");
             advance(); // =
-            arg->set_value(parse_expression(Precedence::DEF_ARG, locals));
+            arg->set_value(parse_arg_default_value(locals, iter_allow));
         }
         args.push(arg.static_cast_as<Node>());
         return;
@@ -1176,8 +1196,12 @@ void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &
         case Token::Type::RParen:
         case Token::Type::Semicolon:
             break;
+        case Token::Type::LCurlyBrace:
+            if (iter_allow < IterAllow::CURLY_ONLY)
+                break;
+            [[fallthrough]];
         default:
-            arg->set_value(parse_expression(Precedence::DEF_ARG, locals));
+            arg->set_value(parse_expression(Precedence::DEF_ARG, locals, iter_allow));
         }
         arg->add_to_locals(locals);
         args.push(arg.static_cast_as<Node>());
@@ -1263,6 +1287,12 @@ SharedPtr<Node> Parser::parse_file_constant(LocalsHashmap &) {
     return new StringNode { token, token.file() };
 }
 
+SharedPtr<Node> Parser::parse_line_constant(LocalsHashmap &) {
+    auto token = current_token();
+    advance();
+    return new FixnumNode { token, static_cast<long long>(token.line() + 1) };
+}
+
 SharedPtr<Node> Parser::parse_for(LocalsHashmap &locals) {
     auto token = current_token();
     advance();
@@ -1272,7 +1302,10 @@ SharedPtr<Node> Parser::parse_for(LocalsHashmap &locals) {
     }
     expect(Token::Type::InKeyword, "for in");
     advance();
-    auto expr = parse_expression(Precedence::LOWEST, locals);
+    auto expr = parse_expression(Precedence::LOWEST, locals, IterAllow::CURLY_ONLY);
+    if (current_token().type() == Token::Type::DoKeyword) {
+        advance();
+    }
     auto body = parse_body(locals, Precedence::LOWEST);
     expect(Token::Type::EndKeyword, "for end");
     advance();
@@ -1690,13 +1723,15 @@ SharedPtr<String> Parser::parse_method_name(LocalsHashmap &) {
     switch (token.type()) {
     case Token::Type::BareName:
     case Token::Type::Constant:
+    case Token::Type::OperatorName:
         name = current_token().literal_string();
         break;
     default:
-        if (token.is_operator() || token.is_keyword())
+        if (token.is_operator() || token.is_keyword()) {
             name = new String(current_token().type_value());
-        else
+        } else {
             throw_unexpected("method name");
+        }
     }
     advance();
     return name;
@@ -1766,15 +1801,15 @@ SharedPtr<Node> Parser::parse_nth_ref(LocalsHashmap &) {
     return new NthRefNode { token, token.get_fixnum() };
 }
 
-void Parser::parse_proc_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals) {
+void Parser::parse_proc_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals, IterAllow iter_allow) {
     if (current_token().is_semicolon()) {
         parse_shadow_variables_in_args(args, locals);
         return;
     }
-    parse_def_single_arg(args, locals, ArgsContext::Proc);
+    parse_def_single_arg(args, locals, ArgsContext::Proc, iter_allow);
     while (current_token().is_comma()) {
         advance();
-        parse_def_single_arg(args, locals, ArgsContext::Proc);
+        parse_def_single_arg(args, locals, ArgsContext::Proc, iter_allow);
     }
     if (current_token().is_semicolon()) {
         parse_shadow_variables_in_args(args, locals);
@@ -1887,13 +1922,13 @@ SharedPtr<Node> Parser::parse_stabby_proc(LocalsHashmap &locals) {
         if (current_token().is_rparen()) {
             advance(); // )
         } else {
-            parse_proc_args(args, locals);
+            parse_proc_args(args, locals, IterAllow::CURLY_AND_BLOCK);
             expect(Token::Type::RParen, "proc args closing paren");
             advance(); // )
         }
-    } else if (current_token().is_bare_name() || current_token().type() == Token::Type::Star) {
+    } else if (current_token().can_be_first_arg_of_def()) {
         has_args = true;
-        parse_proc_args(args, locals);
+        parse_proc_args(args, locals, IterAllow::NONE);
     }
     if (current_token().type() != Token::Type::DoKeyword && current_token().type() != Token::Type::LCurlyBrace)
         throw_unexpected("block");
@@ -2065,36 +2100,23 @@ SharedPtr<Node> Parser::parse_unary_operator(LocalsHashmap &locals) {
 SharedPtr<Node> Parser::parse_undef(LocalsHashmap &locals) {
     auto undef_token = current_token();
     advance();
-    auto symbol_from_token = [&](Token &token) -> SharedPtr<Node> {
-        switch (token.type()) {
-        case Token::Type::BareName:
-        case Token::Type::Constant:
-            advance();
-            return new SymbolNode { token, token.literal_string() };
-        case Token::Type::Symbol:
-            return parse_symbol(locals);
-        case Token::Type::InterpolatedSymbolBegin: {
-            return parse_interpolated_symbol(locals);
-        }
-        default:
-            throw_unexpected("method name for undef");
-        }
-    };
     SharedPtr<UndefNode> undef_node = new UndefNode { undef_token };
-    auto token = current_token();
-    undef_node->add_arg(symbol_from_token(token));
+    auto arg = parse_alias_arg(locals, "method name for undef");
+    undef_node->add_arg(arg.static_cast_as<Node>());
     if (current_token().is_comma()) {
         SharedPtr<BlockNode> block = new BlockNode { undef_token };
         block->add_node(undef_node.static_cast_as<Node>());
         while (current_token().is_comma()) {
             advance();
-            token = current_token();
             SharedPtr<UndefNode> undef_node = new UndefNode { undef_token };
-            undef_node->add_arg(symbol_from_token(token));
+            auto arg = parse_alias_arg(locals, "method name for undef");
+            undef_node->add_arg(arg.static_cast_as<Node>());
             block->add_node(undef_node.static_cast_as<Node>());
         }
+        reinsert_collapsed_newline();
         return block.static_cast_as<Node>();
     }
+    reinsert_collapsed_newline();
     return undef_node.static_cast_as<Node>();
 };
 
@@ -2416,6 +2438,7 @@ SharedPtr<Node> Parser::parse_constant_resolution_expression(SharedPtr<Node> lef
     SharedPtr<Node> node;
     switch (name_token.type()) {
     case Token::Type::BareName:
+    case Token::Type::OperatorName:
         advance();
         node = new CallNode { name_token, left, name_token.literal_string() };
         break;
@@ -2434,7 +2457,12 @@ SharedPtr<Node> Parser::parse_constant_resolution_expression(SharedPtr<Node> lef
         break;
     }
     default:
-        throw_unexpected(name_token, ":: identifier name");
+        if (name_token.is_operator() || name_token.is_keyword()) {
+            advance();
+            node = new CallNode { name_token, left, new String(name_token.type_value()) };
+        } else {
+            throw_unexpected(name_token, ":: identifier name");
+        }
     }
     return node;
 }
@@ -2677,6 +2705,7 @@ SharedPtr<Node> Parser::parse_safe_send_expression(SharedPtr<Node> left, LocalsH
         break;
     case Token::Type::BareName:
     case Token::Type::Constant:
+    case Token::Type::OperatorName:
         name = name_token.literal_string();
         advance();
         break;
@@ -2703,6 +2732,7 @@ SharedPtr<Node> Parser::parse_send_expression(SharedPtr<Node> left, LocalsHashma
     switch (name_token.type()) {
     case Token::Type::BareName:
     case Token::Type::Constant:
+    case Token::Type::OperatorName:
         name = name_token.literal_string();
         advance();
         break;
@@ -2745,7 +2775,11 @@ SharedPtr<Node> Parser::parse_unless(LocalsHashmap &locals) {
     if (condition->type() == Node::Type::Regexp) {
         condition = new MatchNode { condition->token(), condition.static_cast_as<RegexpNode>() };
     }
-    next_expression();
+    if (current_token().type() == Token::Type::ThenKeyword) {
+        advance(); // then
+    } else {
+        next_expression();
+    }
     SharedPtr<Node> false_expr = parse_if_body(locals);
     SharedPtr<Node> true_expr;
     if (current_token().is_else_keyword()) {
@@ -2762,11 +2796,15 @@ SharedPtr<Node> Parser::parse_unless(LocalsHashmap &locals) {
 SharedPtr<Node> Parser::parse_while(LocalsHashmap &locals) {
     auto token = current_token();
     advance();
-    SharedPtr<Node> condition = parse_expression(Precedence::LOWEST, locals);
+    SharedPtr<Node> condition = parse_expression(Precedence::LOWEST, locals, IterAllow::CURLY_ONLY);
     if (condition->type() == Node::Type::Regexp) {
         condition = new MatchNode { condition->token(), condition.static_cast_as<RegexpNode>() };
     }
-    next_expression();
+    if (current_token().type() == Token::Type::DoKeyword) {
+        advance();
+    } else {
+        next_expression();
+    }
     SharedPtr<BlockNode> body = parse_body(locals, Precedence::LOWEST);
     expect(Token::Type::EndKeyword, "while end");
     advance();
@@ -2824,6 +2862,8 @@ Parser::parse_null_fn Parser::null_denotation(Token::Type type) {
         return &Parser::parse_group;
     case Type::LCurlyBrace:
         return &Parser::parse_hash;
+    case Type::LINEKeyword:
+        return &Parser::parse_line_constant;
     case Type::BareName:
     case Type::ClassVariable:
     case Type::Constant:
@@ -2907,7 +2947,7 @@ Parser::parse_left_fn Parser::left_denotation(Token &token, SharedPtr<Node> left
     using Type = Token::Type;
     switch (token.type()) {
     case Type::Equal:
-        if (precedence == Precedence::ARRAY || precedence == Precedence::BARE_CALL_ARG || precedence == Precedence::CALL_ARG)
+        if (precedence == Precedence::ARRAY || precedence == Precedence::HASH || precedence == Precedence::BARE_CALL_ARG || precedence == Precedence::CALL_ARG)
             return &Parser::parse_assignment_expression_without_multiple_values;
         else
             return &Parser::parse_assignment_expression;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: natalie_parser
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.0
 platform: ruby
 authors:
 - Tim Morgan
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-08-12 00:00:00.000000000 Z
+date: 2022-10-25 00:00:00.000000000 Z
 dependencies: []
 description: NatalieParser is a zero-dependency, from-scratch, hand-written recursive
   descent parser for the Ruby Programming Language.