natalie_parser 2.1.0 → 2.3.0

data/src/parser.cpp

@@ -46,7 +46,7 @@ enum class Parser::Precedence {
     REF, // foo[1] / foo[1] = 2
 };
 
-bool Parser::higher_precedence(Token &token, SharedPtr<Node> left, Precedence current_precedence) {
+bool Parser::higher_precedence(Token &token, SharedPtr<Node> left, Precedence current_precedence, IterAllow iter_allow) {
     auto next_precedence = get_precedence(token, left);
 
     // printf("token %d, left %d, current_precedence %d, next_precedence %d\n", (int)token.type(), (int)left->type(), (int)current_precedence, (int)next_precedence);
@@ -83,11 +83,11 @@ bool Parser::higher_precedence(Token &token, SharedPtr<Node> left, Precedence cu
         // NOTE: `m_call_depth` should probably be called
         // `m_call_that_can_accept_a_block_depth`, but that's a bit long.
         //
-        return m_call_depth.last() == 0;
+        return iter_allow == IterAllow::CURLY_AND_BLOCK && m_call_depth.last() == 0;
     }
 
     if (next_precedence == Precedence::ITER_CURLY)
-        return left->is_callable();
+        return iter_allow >= IterAllow::CURLY_ONLY && left->is_callable();
 
     return next_precedence > current_precedence;
 }
@@ -198,7 +198,7 @@ Parser::Precedence Parser::get_precedence(Token &token, SharedPtr<Node> left) {
     return Precedence::LOWEST;
 }
 
-SharedPtr<Node> Parser::parse_expression(Parser::Precedence precedence, LocalsHashmap &locals) {
+SharedPtr<Node> Parser::parse_expression(Parser::Precedence precedence, LocalsHashmap &locals, IterAllow iter_allow) {
     skip_newlines();
 
     m_precedence_stack.push(precedence);
@@ -211,7 +211,7 @@ SharedPtr<Node> Parser::parse_expression(Parser::Precedence precedence, LocalsHa
 
     while (current_token().is_valid()) {
         auto token = current_token();
-        if (!higher_precedence(token, left, precedence))
+        if (!higher_precedence(token, left, precedence, iter_allow))
             break;
         auto left_fn = left_denotation(token, left, precedence);
         if (!left_fn)
@@ -278,44 +278,47 @@ SharedPtr<BlockNode> Parser::parse_def_body(LocalsHashmap &locals) {
     return parse_body(locals, Precedence::LOWEST, Token::Type::EndKeyword, true);
 }
 
+void Parser::reinsert_collapsed_newline() {
+    auto token = previous_token();
+    if (token.can_precede_collapsible_newline()) {
+        // Some operators at the end of a line cause the newlines to be collapsed:
+        //
+        //     foo <<
+        //       bar
+        //
+        // ...but in this case (an alias), collapsing the newline was a mistake:
+        //
+        //     alias foo <<
+        //     def bar; end
+        //
+        // So, we'll put the newline back.
+        m_tokens->insert(m_index, Token { Token::Type::Newline, token.file(), token.line(), token.column(), token.whitespace_precedes() });
+    }
+}
+
 SharedPtr<Node> Parser::parse_alias(LocalsHashmap &locals) {
     auto token = current_token();
     advance();
-    SharedPtr<SymbolNode> new_name = parse_alias_arg(locals, "alias new name (first argument)", false);
-    auto existing_name = parse_alias_arg(locals, "alias existing name (second argument)", true);
+    auto new_name = parse_alias_arg(locals, "alias new name (first argument)");
+    auto existing_name = parse_alias_arg(locals, "alias existing name (second argument)");
+    reinsert_collapsed_newline();
     return new AliasNode { token, new_name, existing_name };
 }
 
-SharedPtr<SymbolNode> Parser::parse_alias_arg(LocalsHashmap &locals, const char *expected_message, bool reinsert_collapsed_newline) {
+SharedPtr<SymbolNode> Parser::parse_alias_arg(LocalsHashmap &locals, const char *expected_message) {
     auto token = current_token();
     switch (token.type()) {
-        // TODO: handle Constant too
-    case Token::Type::BareName: {
-        advance();
-        return new SymbolNode { token, token.literal_string() };
-    }
+    case Token::Type::BareName:
+    case Token::Type::Constant:
+    case Token::Type::OperatorName:
+        return new SymbolNode { token, parse_method_name(locals) };
     case Token::Type::Symbol:
         return parse_symbol(locals).static_cast_as<SymbolNode>();
     case Token::Type::InterpolatedSymbolBegin:
         return parse_interpolated_symbol(locals).static_cast_as<SymbolNode>();
     default:
         if (token.is_operator() || token.is_keyword()) {
-            advance();
-            if (token.can_precede_collapsible_newline() && reinsert_collapsed_newline) {
-                // Some operators at the end of a line cause the newlines to be collapsed:
-                //
-                //     foo <<
-                //       bar
-                //
-                // ...but in this case (an alias), collapsing the newline was a mistake:
-                //
-                //     alias foo <<
-                //     def bar; end
-                //
-                // So, we'll put the newline back.
-                m_tokens->insert(m_index, Token { Token::Type::Newline, token.file(), token.line(), token.column(), token.whitespace_precedes() });
-            }
-            return new SymbolNode { token, new String(token.type_value()) };
+            return new SymbolNode { token, parse_method_name(locals) };
         } else {
             throw_unexpected(expected_message);
         }
@@ -431,16 +434,8 @@ void Parser::parse_rest_of_begin(BeginNode &begin_node, LocalsHashmap &locals) {
             }
             if (current_token().is_hash_rocket()) {
                 advance();
-                SharedPtr<IdentifierNode> name;
-                switch (current_token().type()) {
-                case Token::Type::BareName:
-                    name = new IdentifierNode { current_token(), current_token().literal_string() };
-                    advance();
-                    break;
-                default:
-                    throw_unexpected("exception name");
-                }
-                name->add_to_locals(locals);
+                auto name = parse_expression(Precedence::ASSIGNMENT_LHS, locals);
+                add_assignment_locals(name, locals);
                 rescue_node->set_exception_name(name);
             }
             next_expression();
@@ -501,7 +496,7 @@ SharedPtr<Node> Parser::parse_beginless_range(LocalsHashmap &locals) {
 SharedPtr<Node> Parser::parse_block_pass(LocalsHashmap &locals) {
     auto token = current_token();
     advance();
-    auto value = parse_expression(Precedence::UNARY_PLUS, locals);
+    auto value = parse_expression(Precedence::LOWEST, locals);
     return new BlockPassNode { token, value };
 }
 
@@ -865,15 +860,19 @@ SharedPtr<BlockNode> Parser::parse_case_when_body(LocalsHashmap &locals) {
 }
 
 SharedPtr<Node> Parser::parse_class_or_module_name(LocalsHashmap &locals) {
-    Token name_token;
-    if (current_token().type() == Token::Type::ConstantResolution) {
-        name_token = peek_token();
-    } else {
-        name_token = current_token();
-    }
-    if (name_token.type() != Token::Type::Constant)
+    auto name_token = current_token();
+    auto exp = parse_expression(Precedence::LESS_GREATER, locals);
+    switch (exp->type()) {
+    case Node::Type::Colon2:
+    case Node::Type::Colon3:
+        return exp;
+    case Node::Type::Identifier:
+        if (name_token.type() == Token::Type::Constant)
+            return exp;
+        [[fallthrough]];
+    default:
         throw SyntaxError { "class/module name must be CONSTANT" };
-    return parse_expression(Precedence::LESS_GREATER, locals);
+    }
 }
 
 SharedPtr<Node> Parser::parse_class(LocalsHashmap &locals) {
@@ -903,7 +902,7 @@ SharedPtr<Node> Parser::parse_multiple_assignment_expression(SharedPtr<Node> lef
     list->add_node(left);
     while (current_token().is_comma()) {
         advance();
-        if (current_token().is_rparen() || current_token().is_equal()) {
+        if (current_token().is_rparen() || current_token().is_equal() || current_token().type() == Token::Type::InKeyword) {
             // trailing comma with no additional identifier
             break;
         }
@@ -991,21 +990,24 @@ SharedPtr<Node> Parser::parse_def(LocalsHashmap &locals) {
     auto token = current_token();
     switch (token.type()) {
     case Token::Type::BareName:
-        if (peek_token().type() == Token::Type::Dot) {
+        if (peek_token().is_dot() || peek_token().is_constant_resolution()) {
             self_node = parse_identifier(locals);
             advance(); // dot
         }
         name = parse_method_name(locals);
         break;
     case Token::Type::Constant:
-        if (peek_token().type() == Token::Type::Dot) {
+        if (peek_token().is_dot() || peek_token().is_constant_resolution()) {
             self_node = parse_constant(locals);
             advance(); // dot
         }
         name = parse_method_name(locals);
         break;
+    case Token::Type::OperatorName:
+        name = parse_method_name(locals);
+        break;
     case Token::Type::SelfKeyword:
-        if (peek_token().type() == Token::Type::Dot) {
+        if (peek_token().is_dot() || peek_token().is_constant_resolution()) {
             self_node = new SelfNode { current_token() };
             advance(); // self
             advance(); // dot
@@ -1023,10 +1025,6 @@ SharedPtr<Node> Parser::parse_def(LocalsHashmap &locals) {
         }
     }
     }
-    if (current_token().is_equal() && !current_token().whitespace_precedes()) {
-        advance();
-        name->append_char('=');
-    }
     auto args = Vector<SharedPtr<Node>> {};
     if (current_token().is_lparen()) {
         advance();
@@ -1037,7 +1035,7 @@ SharedPtr<Node> Parser::parse_def(LocalsHashmap &locals) {
             expect(Token::Type::RParen, "args closing paren");
             advance();
         }
-    } else if (current_token().is_bare_name() || current_token().is_splat() || current_token().is_symbol_key()) {
+    } else if (current_token().can_be_first_arg_of_def()) {
         parse_def_args(args, our_locals);
     }
     SharedPtr<BlockNode> body;
@@ -1085,7 +1083,21 @@ void Parser::parse_def_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals
     }
 }
 
-void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals, ArgsContext context) {
+SharedPtr<Node> Parser::parse_arg_default_value(LocalsHashmap &locals, IterAllow iter_allow) {
+    auto token = current_token();
+    if (token.is_bare_name() && peek_token().is_equal()) {
+        SharedPtr<ArgNode> arg = new ArgNode { token, token.literal_string() };
+        advance();
+        advance(); // =
+        arg->add_to_locals(locals);
+        arg->set_value(parse_arg_default_value(locals, iter_allow));
+        return arg.static_cast_as<Node>();
+    } else {
+        return parse_expression(Precedence::DEF_ARG, locals, iter_allow);
+    }
+}
+
+void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals, ArgsContext context, IterAllow iter_allow) {
     auto args_have_any_splat = [&]() { return !args.is_empty() && args.last()->type() == Node::Type::Arg && args.last().static_cast_as<ArgNode>()->splat_or_kwsplat(); };
     auto args_have_keyword = [&]() { return !args.is_empty() && args.last()->type() == Node::Type::KeywordArg; };
 
@@ -1105,7 +1117,7 @@ void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &
             if (args_have_any_splat())
                 throw_error(token, "default value after splat");
             advance(); // =
-            arg->set_value(parse_expression(Precedence::DEF_ARG, locals));
+            arg->set_value(parse_arg_default_value(locals, iter_allow));
         }
         args.push(arg.static_cast_as<Node>());
         return;
@@ -1176,8 +1188,12 @@ void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &
         case Token::Type::RParen:
         case Token::Type::Semicolon:
             break;
+        case Token::Type::LCurlyBrace:
+            if (iter_allow < IterAllow::CURLY_ONLY)
+                break;
+            [[fallthrough]];
         default:
-            arg->set_value(parse_expression(Precedence::DEF_ARG, locals));
+            arg->set_value(parse_expression(Precedence::DEF_ARG, locals, iter_allow));
         }
         arg->add_to_locals(locals);
         args.push(arg.static_cast_as<Node>());
@@ -1263,16 +1279,28 @@ SharedPtr<Node> Parser::parse_file_constant(LocalsHashmap &) {
     return new StringNode { token, token.file() };
 }
 
+SharedPtr<Node> Parser::parse_line_constant(LocalsHashmap &) {
+    auto token = current_token();
+    advance();
+    return new FixnumNode { token, static_cast<long long>(token.line() + 1) };
+}
+
 SharedPtr<Node> Parser::parse_for(LocalsHashmap &locals) {
     auto token = current_token();
     advance();
-    auto vars = parse_assignment_identifier(false, locals);
-    if (current_token().type() == Token::Type::Comma) {
+    auto vars = parse_assignment_identifier(true, locals);
+    if (current_token().is_comma() || vars->type() == Node::Type::Splat) {
         vars = parse_multiple_assignment_expression(vars, locals);
     }
+    add_assignment_locals(vars, locals);
     expect(Token::Type::InKeyword, "for in");
     advance();
-    auto expr = parse_expression(Precedence::LOWEST, locals);
+    m_call_depth.last()++;
+    auto expr = parse_expression(Precedence::BARE_CALL_ARG, locals, IterAllow::CURLY_ONLY);
+    m_call_depth.last()--;
+    if (current_token().type() == Token::Type::DoKeyword) {
+        advance();
+    }
     auto body = parse_body(locals, Precedence::LOWEST);
     expect(Token::Type::EndKeyword, "for end");
     advance();
@@ -1690,13 +1718,15 @@ SharedPtr<String> Parser::parse_method_name(LocalsHashmap &) {
     switch (token.type()) {
     case Token::Type::BareName:
     case Token::Type::Constant:
+    case Token::Type::OperatorName:
         name = current_token().literal_string();
         break;
     default:
-        if (token.is_operator() || token.is_keyword())
+        if (token.is_operator() || token.is_keyword()) {
             name = new String(current_token().type_value());
-        else
+        } else {
             throw_unexpected("method name");
+        }
     }
     advance();
     return name;
@@ -1766,15 +1796,15 @@ SharedPtr<Node> Parser::parse_nth_ref(LocalsHashmap &) {
     return new NthRefNode { token, token.get_fixnum() };
 }
 
-void Parser::parse_proc_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals) {
+void Parser::parse_proc_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals, IterAllow iter_allow) {
     if (current_token().is_semicolon()) {
         parse_shadow_variables_in_args(args, locals);
         return;
     }
-    parse_def_single_arg(args, locals, ArgsContext::Proc);
+    parse_def_single_arg(args, locals, ArgsContext::Proc, iter_allow);
     while (current_token().is_comma()) {
         advance();
-        parse_def_single_arg(args, locals, ArgsContext::Proc);
+        parse_def_single_arg(args, locals, ArgsContext::Proc, iter_allow);
     }
     if (current_token().is_semicolon()) {
         parse_shadow_variables_in_args(args, locals);
@@ -1887,13 +1917,13 @@ SharedPtr<Node> Parser::parse_stabby_proc(LocalsHashmap &locals) {
         if (current_token().is_rparen()) {
             advance(); // )
         } else {
-            parse_proc_args(args, locals);
+            parse_proc_args(args, locals, IterAllow::CURLY_AND_BLOCK);
             expect(Token::Type::RParen, "proc args closing paren");
             advance(); // )
         }
-    } else if (current_token().is_bare_name() || current_token().type() == Token::Type::Star) {
+    } else if (current_token().can_be_first_arg_of_def()) {
         has_args = true;
-        parse_proc_args(args, locals);
+        parse_proc_args(args, locals, IterAllow::NONE);
     }
     if (current_token().type() != Token::Type::DoKeyword && current_token().type() != Token::Type::LCurlyBrace)
         throw_unexpected("block");
@@ -2065,36 +2095,23 @@ SharedPtr<Node> Parser::parse_unary_operator(LocalsHashmap &locals) {
 SharedPtr<Node> Parser::parse_undef(LocalsHashmap &locals) {
     auto undef_token = current_token();
     advance();
-    auto symbol_from_token = [&](Token &token) -> SharedPtr<Node> {
-        switch (token.type()) {
-        case Token::Type::BareName:
-        case Token::Type::Constant:
-            advance();
-            return new SymbolNode { token, token.literal_string() };
-        case Token::Type::Symbol:
-            return parse_symbol(locals);
-        case Token::Type::InterpolatedSymbolBegin: {
-            return parse_interpolated_symbol(locals);
-        }
-        default:
-            throw_unexpected("method name for undef");
-        }
-    };
     SharedPtr<UndefNode> undef_node = new UndefNode { undef_token };
-    auto token = current_token();
-    undef_node->add_arg(symbol_from_token(token));
+    auto arg = parse_alias_arg(locals, "method name for undef");
+    undef_node->add_arg(arg.static_cast_as<Node>());
     if (current_token().is_comma()) {
         SharedPtr<BlockNode> block = new BlockNode { undef_token };
         block->add_node(undef_node.static_cast_as<Node>());
         while (current_token().is_comma()) {
             advance();
-            token = current_token();
             SharedPtr<UndefNode> undef_node = new UndefNode { undef_token };
-            undef_node->add_arg(symbol_from_token(token));
+            auto arg = parse_alias_arg(locals, "method name for undef");
+            undef_node->add_arg(arg.static_cast_as<Node>());
             block->add_node(undef_node.static_cast_as<Node>());
         }
+        reinsert_collapsed_newline();
         return block.static_cast_as<Node>();
     }
+    reinsert_collapsed_newline();
     return undef_node.static_cast_as<Node>();
 };
 
@@ -2155,32 +2172,34 @@ SharedPtr<Node> Parser::parse_assignment_expression(SharedPtr<Node> left, Locals
     if (left->type() == Node::Type::Splat) {
         return parse_multiple_assignment_expression(left, locals);
     }
+    add_assignment_locals(left, locals);
+    advance();
+    bool to_array = left->type() == Node::Type::MultipleAssignment;
+    auto value = parse_assignment_expression_value(to_array, locals, allow_multiple);
+    return new AssignmentNode { token, left, value };
+}
+
+void Parser::add_assignment_locals(SharedPtr<Node> left, LocalsHashmap &locals) {
     switch (left->type()) {
     case Node::Type::Identifier: {
         auto left_identifier = left.static_cast_as<IdentifierNode>();
         left_identifier->add_to_locals(locals);
-        advance();
-        auto value = parse_assignment_expression_value(false, locals, allow_multiple);
-        return new AssignmentNode { token, left, value };
+        break;
     }
     case Node::Type::Call:
     case Node::Type::Colon2:
     case Node::Type::Colon3:
     case Node::Type::SafeCall: {
-        advance();
-        auto value = parse_assignment_expression_value(false, locals, allow_multiple);
-        return new AssignmentNode { token, left, value };
+        break;
     }
     case Node::Type::MultipleAssignment: {
         left.static_cast_as<MultipleAssignmentNode>()->add_locals(locals);
-        advance();
-        auto value = parse_assignment_expression_value(true, locals, allow_multiple);
-        return new AssignmentNode { token, left, value };
+        break;
     }
     default:
-        throw_unexpected(left->token(), "left side of assignment");
+        throw_unexpected(left->token(), "assignment identifier");
     }
-};
+}
 
 SharedPtr<Node> Parser::parse_assignment_expression_value(bool to_array, LocalsHashmap &locals, bool allow_multiple) {
     auto token = current_token();
@@ -2416,6 +2435,7 @@ SharedPtr<Node> Parser::parse_constant_resolution_expression(SharedPtr<Node> lef
     SharedPtr<Node> node;
     switch (name_token.type()) {
     case Token::Type::BareName:
+    case Token::Type::OperatorName:
         advance();
         node = new CallNode { name_token, left, name_token.literal_string() };
         break;
@@ -2434,7 +2454,12 @@ SharedPtr<Node> Parser::parse_constant_resolution_expression(SharedPtr<Node> lef
         break;
     }
     default:
-        throw_unexpected(name_token, ":: identifier name");
+        if (name_token.is_operator() || name_token.is_keyword()) {
+            advance();
+            node = new CallNode { name_token, left, new String(name_token.type_value()) };
+        } else {
+            throw_unexpected(name_token, ":: identifier name");
+        }
     }
     return node;
 }
@@ -2677,6 +2702,7 @@ SharedPtr<Node> Parser::parse_safe_send_expression(SharedPtr<Node> left, LocalsH
         break;
     case Token::Type::BareName:
     case Token::Type::Constant:
+    case Token::Type::OperatorName:
         name = name_token.literal_string();
         advance();
         break;
@@ -2703,6 +2729,7 @@ SharedPtr<Node> Parser::parse_send_expression(SharedPtr<Node> left, LocalsHashma
     switch (name_token.type()) {
     case Token::Type::BareName:
     case Token::Type::Constant:
+    case Token::Type::OperatorName:
         name = name_token.literal_string();
         advance();
         break;
@@ -2745,7 +2772,11 @@ SharedPtr<Node> Parser::parse_unless(LocalsHashmap &locals) {
     if (condition->type() == Node::Type::Regexp) {
         condition = new MatchNode { condition->token(), condition.static_cast_as<RegexpNode>() };
     }
-    next_expression();
+    if (current_token().type() == Token::Type::ThenKeyword) {
+        advance(); // then
+    } else {
+        next_expression();
+    }
     SharedPtr<Node> false_expr = parse_if_body(locals);
     SharedPtr<Node> true_expr;
     if (current_token().is_else_keyword()) {
@@ -2762,11 +2793,15 @@ SharedPtr<Node> Parser::parse_unless(LocalsHashmap &locals) {
 SharedPtr<Node> Parser::parse_while(LocalsHashmap &locals) {
     auto token = current_token();
     advance();
-    SharedPtr<Node> condition = parse_expression(Precedence::LOWEST, locals);
+    SharedPtr<Node> condition = parse_expression(Precedence::LOWEST, locals, IterAllow::CURLY_ONLY);
     if (condition->type() == Node::Type::Regexp) {
         condition = new MatchNode { condition->token(), condition.static_cast_as<RegexpNode>() };
     }
-    next_expression();
+    if (current_token().type() == Token::Type::DoKeyword) {
+        advance();
+    } else {
+        next_expression();
+    }
     SharedPtr<BlockNode> body = parse_body(locals, Precedence::LOWEST);
     expect(Token::Type::EndKeyword, "while end");
     advance();
@@ -2824,6 +2859,8 @@ Parser::parse_null_fn Parser::null_denotation(Token::Type type) {
         return &Parser::parse_group;
     case Type::LCurlyBrace:
         return &Parser::parse_hash;
+    case Type::LINEKeyword:
+        return &Parser::parse_line_constant;
     case Type::BareName:
     case Type::ClassVariable:
     case Type::Constant:
@@ -2907,7 +2944,7 @@ Parser::parse_left_fn Parser::left_denotation(Token &token, SharedPtr<Node> left
     using Type = Token::Type;
     switch (token.type()) {
     case Type::Equal:
-        if (precedence == Precedence::ARRAY || precedence == Precedence::BARE_CALL_ARG || precedence == Precedence::CALL_ARG)
+        if (precedence == Precedence::ARRAY || precedence == Precedence::HASH || precedence == Precedence::BARE_CALL_ARG || precedence == Precedence::CALL_ARG)
             return &Parser::parse_assignment_expression_without_multiple_values;
         else
             return &Parser::parse_assignment_expression;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: natalie_parser
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.3.0
 platform: ruby
 authors:
 - Tim Morgan
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-08-12 00:00:00.000000000 Z
+date: 2022-12-31 00:00:00.000000000 Z
 dependencies: []
 description: NatalieParser is a zero-dependency, from-scratch, hand-written recursive
   descent parser for the Ruby Programming Language.
@@ -151,6 +151,7 @@ files:
 - src/node/begin_node.cpp
 - src/node/begin_rescue_node.cpp
 - src/node/class_node.cpp
+- src/node/for_node.cpp
 - src/node/interpolated_regexp_node.cpp
 - src/node/interpolated_shell_node.cpp
 - src/node/interpolated_string_node.cpp