natalie_parser 1.1.1 → 2.0.0

data/src/node/begin_rescue_node.cpp

@@ -10,7 +10,7 @@ SharedPtr<Node> BeginRescueNode::name_to_node() const {
         token(),
         m_name.static_cast_as<Node>(),
         new IdentifierNode {
-            Token { Token::Type::GlobalVariable, "$!", file(), line(), column() },
+            Token { Token::Type::GlobalVariable, "$!", file(), line(), column(), false },
             false },
     };
 }

data/src/node/node.cpp

@@ -1,4 +1,5 @@
 #include "natalie_parser/node.hpp"
+#include "natalie_parser/creator/debug_creator.hpp"
 
 namespace NatalieParser {
 
@@ -7,4 +8,10 @@ BlockNode &Node::as_block_node() {
     return *static_cast<BlockNode *>(this);
 }
 
+void Node::debug() {
+    DebugCreator creator;
+    transform(&creator);
+    printf("DEBUG[type=%d]: %s\n", (int)type(), creator.to_string().c_str());
+}
+
 }

data/src/parser.cpp

@@ -249,7 +249,7 @@ SharedPtr<BlockNode> Parser::parse_body(LocalsHashmap &locals, Precedence preced
     validate_current_token();
     skip_newlines();
     while (!current_token().is_eof() && !is_end(current_token().type())) {
-        if (allow_rescue && current_token().type() == Token::Type::RescueKeyword) {
+        if (allow_rescue && (current_token().is_rescue() || current_token().is_ensure())) {
             auto token = body->token();
             SharedPtr<BeginNode> begin_node = new BeginNode { body->token(), body };
             parse_rest_of_begin(begin_node.ref(), locals);
@@ -313,7 +313,7 @@ SharedPtr<SymbolNode> Parser::parse_alias_arg(LocalsHashmap &locals, const char
                 //     def bar; end
                 //
                 // So, we'll put the newline back.
-                m_tokens->insert(m_index, Token { Token::Type::Newline, token.file(), token.line(), token.column() });
+                m_tokens->insert(m_index, Token { Token::Type::Newline, token.file(), token.line(), token.column(), token.whitespace_precedes() });
             }
             return new SymbolNode { token, new String(token.type_value()) };
         } else {
@@ -337,7 +337,7 @@ SharedPtr<Node> Parser::parse_array(LocalsHashmap &locals) {
             return array.static_cast_as<Node>();
         }
         if (token.type() == Token::Type::SymbolKey) {
-            array->add_node(parse_hash_inner(locals, Precedence::HASH, Token::Type::RBracket));
+            array->add_node(parse_hash_inner(locals, Precedence::HASH, Token::Type::RBracket, true));
             expect(Token::Type::RBracket, "array closing bracket");
             advance();
             return array.static_cast_as<Node>();
@@ -345,7 +345,7 @@ SharedPtr<Node> Parser::parse_array(LocalsHashmap &locals) {
         auto value = parse_expression(Precedence::ARRAY, locals);
         token = current_token();
         if (token.is_hash_rocket()) {
-            array->add_node(parse_hash_inner(locals, Precedence::HASH, Token::Type::RBracket, value));
+            array->add_node(parse_hash_inner(locals, Precedence::HASH, Token::Type::RBracket, true, value));
             expect(Token::Type::RBracket, "array closing bracket");
             advance();
             return array.static_cast_as<Node>();
@@ -624,6 +624,11 @@ SharedPtr<Node> Parser::parse_case_in_pattern(LocalsHashmap &locals) {
         advance();
         node = new IdentifierNode { token, true };
         break;
+    case Token::Type::Bignum:
+    case Token::Type::Fixnum:
+    case Token::Type::Float:
+        node = parse_lit(locals);
+        break;
     case Token::Type::Caret:
         advance();
         expect(Token::Type::BareName, "pinned variable name");
@@ -633,12 +638,18 @@ SharedPtr<Node> Parser::parse_case_in_pattern(LocalsHashmap &locals) {
     case Token::Type::Constant:
         node = parse_constant(locals);
         break;
+    case Token::Type::DotDot:
+    case Token::Type::DotDotDot:
+        node = parse_beginless_range(locals);
+        break;
     case Token::Type::LBracketRBracket:
         advance();
         node = new ArrayPatternNode { token };
         break;
+    case Token::Type::InterpolatedStringBegin:
+        node = parse_interpolated_string(locals);
+        break;
     case Token::Type::LBracket: {
-        // TODO: might need to keep track of and pass along precedence value?
         advance();
         SharedPtr<ArrayPatternNode> array = new ArrayPatternNode { token };
         if (current_token().is_rbracket()) {
@@ -649,7 +660,10 @@ SharedPtr<Node> Parser::parse_case_in_pattern(LocalsHashmap &locals) {
         array->add_node(parse_case_in_pattern(locals));
         while (current_token().is_comma()) {
             advance();
-            array->add_node(parse_case_in_pattern(locals));
+            if (current_token().is_rbracket())
+                array->add_node(new SplatNode { current_token() });
+            else
+                array->add_node(parse_case_in_pattern(locals));
         }
         expect(Token::Type::RBracket, "array pattern closing bracket");
         advance();
@@ -659,47 +673,71 @@ SharedPtr<Node> Parser::parse_case_in_pattern(LocalsHashmap &locals) {
     case Token::Type::LCurlyBrace: {
         advance();
         SharedPtr<HashPatternNode> hash = new HashPatternNode { token };
+        node = hash.static_cast_as<Node>();
         if (current_token().type() == Token::Type::RCurlyBrace) {
             advance();
             node = hash.static_cast_as<Node>();
             break;
         }
-        expect(Token::Type::SymbolKey, "hash pattern symbol key");
-        hash->add_node(parse_symbol(locals));
-        hash->add_node(parse_case_in_pattern(locals));
+
+        auto add_pair = [&]() {
+            auto key = parse_case_in_pattern_hash_symbol_key(locals);
+            hash->add_node(key);
+            if (key->type() == Node::Type::KeywordRestPattern) {
+                // nothing else to do
+            } else if (current_token().type() == Token::Type::RCurlyBrace || current_token().type() == Token::Type::Comma) {
+                if (key->type() == Node::Type::SymbolKey)
+                    locals.set(key.static_cast_as<SymbolKeyNode>()->name().ref());
+                hash->add_node(new NilNode { current_token() });
+            } else {
+                hash->add_node(parse_case_in_pattern(locals));
+            }
+        };
+
+        add_pair();
+
         while (current_token().is_comma()) {
             advance(); // ,
-            expect(Token::Type::Symbol, "hash pattern symbol");
-            hash->add_node(parse_symbol(locals));
-            hash->add_node(parse_case_in_pattern(locals));
+            add_pair();
         }
+
         expect(Token::Type::RCurlyBrace, "hash pattern closing brace");
         advance();
-        node = hash.static_cast_as<Node>();
         break;
     }
-    case Token::Type::Bignum:
-    case Token::Type::Fixnum:
-    case Token::Type::Float:
-        node = parse_lit(locals);
+    case Token::Type::LParen:
+        advance(); // (
+        node = parse_case_in_pattern(locals);
+        expect(Token::Type::RParen, "closing paren for pattern");
+        advance();
+        break;
+    case Token::Type::Minus:
+        node = parse_unary_operator(locals);
+        break;
+    case Token::Type::NilKeyword:
+        node = parse_nil(locals);
         break;
     case Token::Type::Star: {
-        auto splat_token = current_token();
-        advance();
-        SharedPtr<ArrayPatternNode> array = new ArrayPatternNode { token };
+        advance(); // *
         switch (current_token().type()) {
         case Token::Type::BareName:
         case Token::Type::Constant: {
-            auto name = new IdentifierNode { current_token(), true };
-            name->prepend_to_name('*');
-            array->add_node(name);
+            SharedPtr<String> name = current_token().literal_string();
+            auto symbol = new SymbolNode { current_token(), name };
+            node = new SplatNode { symbol->token(), symbol };
             advance();
             break;
         }
         default:
-            array->add_node(new SplatNode { splat_token });
+            node = new SplatNode { current_token() };
         }
-        node = array.static_cast_as<Node>();
+        break;
+    }
+    case Token::Type::StarStar: {
+        SharedPtr<HashPatternNode> hash = new HashPatternNode { token };
+        auto key = parse_case_in_pattern_hash_symbol_key(locals);
+        hash->add_node(key);
+        node = hash.static_cast_as<Node>();
         break;
     }
     case Token::Type::String:
@@ -711,6 +749,11 @@ SharedPtr<Node> Parser::parse_case_in_pattern(LocalsHashmap &locals) {
     default:
         throw_unexpected("case in pattern");
     }
+
+    if (current_token().type() == Token::Type::DotDot || current_token().type() == Token::Type::DotDotDot) {
+        node = parse_range_expression(node, locals);
+    }
+
     token = current_token();
     if (token.is_hash_rocket()) {
         advance();
@@ -723,26 +766,67 @@ SharedPtr<Node> Parser::parse_case_in_pattern(LocalsHashmap &locals) {
     return node;
 }
 
+SharedPtr<Node> Parser::parse_case_in_pattern_alternation(LocalsHashmap &locals) {
+    SharedPtr<ArrayPatternNode> array_pattern = new ArrayPatternNode { current_token() };
+    array_pattern->add_node(parse_case_in_pattern(locals));
+    while (current_token().is_comma()) {
+        advance();
+        array_pattern->add_node(parse_case_in_pattern(locals));
+    }
+    if (array_pattern->nodes().size() == 1)
+        return array_pattern->nodes().first();
+    return array_pattern.static_cast_as<Node>();
+}
+
+SharedPtr<Node> Parser::parse_case_in_pattern_hash_symbol_key(LocalsHashmap &locals) {
+    auto token = current_token();
+    SharedPtr<Node> node;
+    switch (token.type()) {
+    case Token::Type::InterpolatedStringBegin:
+        node = parse_interpolated_string(locals);
+        if (node->type() != Node::Type::SymbolKey)
+            throw_unexpected(token, "hash pattern symbol key");
+        break;
+    case Token::Type::StarStar:
+        advance(); // **
+        switch (current_token().type()) {
+        case Token::Type::NilKeyword:
+            node = new KeywordRestPatternNode { token, current_token().type_value() };
+            advance();
+            break;
+        case Token::Type::BareName: {
+            auto name = current_token().literal_string();
+            node = new KeywordRestPatternNode { token, name };
+            locals.set(name.ref());
+            advance();
+            break;
+        }
+        default:
+            node = new KeywordRestPatternNode { token };
+            break;
+        }
+        break;
+    case Token::Type::SymbolKey:
+        node = parse_symbol_key(locals);
+        break;
+    default:
+        throw_unexpected("hash pattern symbol key");
+    }
+    return node;
+}
+
 SharedPtr<Node> Parser::parse_case_in_patterns(LocalsHashmap &locals) {
     Vector<SharedPtr<Node>> patterns;
-    patterns.push(parse_case_in_pattern(locals));
+    auto pattern = parse_case_in_pattern_alternation(locals);
+    if (pattern->type() == Node::Type::Splat)
+        pattern = new ArrayPatternNode { pattern->token(), pattern };
+    patterns.push(pattern);
     while (current_token().type() == Token::Type::Pipe) {
         advance();
-        patterns.push(parse_case_in_pattern(locals));
-    }
-    while (current_token().is_comma()) {
-        advance();
-        auto last_pattern = patterns.last();
-        auto next_pattern = parse_case_in_pattern(locals);
-        if (last_pattern->type() == Node::Type::ArrayPattern) {
-            last_pattern.static_cast_as<ArrayPatternNode>()->add_node(next_pattern);
-        } else {
-            patterns.pop();
-            auto array_pattern = new ArrayPatternNode { last_pattern->token() };
-            array_pattern->add_node(last_pattern);
-            array_pattern->add_node(next_pattern);
-            patterns.push(array_pattern);
-        }
+        auto pattern = parse_case_in_pattern_alternation(locals);
+        if (pattern->type() == Node::Type::Splat)
+            pattern = new ArrayPatternNode { pattern->token(), pattern };
+        patterns.push(pattern);
     }
     assert(patterns.size() > 0);
     if (patterns.size() == 1) {
@@ -849,13 +933,12 @@ SharedPtr<Node> Parser::parse_assignment_identifier(bool allow_splat, LocalsHash
     case Token::Type::Star: {
         if (!allow_splat)
             expect(Token::Type::BareName, "assignment identifier");
-        auto splat_token = current_token();
         advance();
         if (current_token().is_assignable()) {
             auto id = parse_assignment_identifier(false, locals);
-            node = new SplatNode { splat_token, id };
+            node = new SplatNode { token, id };
         } else {
-            node = new SplatNode { splat_token };
+            node = new SplatNode { token };
         }
         break;
     }
@@ -1216,17 +1299,17 @@ SharedPtr<Node> Parser::parse_hash(LocalsHashmap &locals) {
     advance();
     SharedPtr<Node> hash;
     if (current_token().type() == Token::Type::RCurlyBrace)
-        hash = new HashNode { token };
+        hash = new HashNode { token, false };
     else
-        hash = parse_hash_inner(locals, Precedence::HASH, Token::Type::RCurlyBrace);
+        hash = parse_hash_inner(locals, Precedence::HASH, Token::Type::RCurlyBrace, false);
     expect(Token::Type::RCurlyBrace, "hash closing curly brace");
     advance();
     return hash;
 }
 
-SharedPtr<Node> Parser::parse_hash_inner(LocalsHashmap &locals, Precedence precedence, Token::Type closing_token_type, SharedPtr<Node> first_key) {
+SharedPtr<Node> Parser::parse_hash_inner(LocalsHashmap &locals, Precedence precedence, Token::Type closing_token_type, bool bare, SharedPtr<Node> first_key) {
     auto token = current_token();
-    SharedPtr<HashNode> hash = new HashNode { token };
+    SharedPtr<HashNode> hash = new HashNode { token, bare };
     if (!first_key)
         first_key = parse_expression(precedence, locals);
     hash->add_node(first_key);
@@ -1557,7 +1640,7 @@ SharedPtr<Node> Parser::parse_keyword_splat(LocalsHashmap &locals) {
 }
 
 SharedPtr<Node> Parser::parse_keyword_splat_wrapped_in_hash(LocalsHashmap &locals) {
-    SharedPtr<HashNode> hash = new HashNode { current_token() };
+    SharedPtr<HashNode> hash = new HashNode { current_token(), true };
     hash->add_node(parse_keyword_splat(locals));
     return hash.static_cast_as<Node>();
 }
@@ -1886,11 +1969,14 @@ SharedPtr<Node> Parser::parse_unary_operator(LocalsHashmap &locals) {
     advance();
     auto precedence = get_precedence(token);
     auto receiver = parse_expression(precedence, locals);
+
     if ((token.type() == Token::Type::Minus || token.type() == Token::Type::Plus) && receiver->is_numeric()) {
         switch (receiver->type()) {
         case Node::Type::Bignum: {
             if (token.type() == Token::Type::Minus) {
                 auto num = receiver.static_cast_as<BignumNode>();
+                if (num->negative())
+                    break;
                 num->negate();
                 return num.static_cast_as<Node>();
             }
@@ -1899,6 +1985,8 @@ SharedPtr<Node> Parser::parse_unary_operator(LocalsHashmap &locals) {
         case Node::Type::Fixnum: {
             if (token.type() == Token::Type::Minus) {
                 auto num = receiver.static_cast_as<FixnumNode>();
+                if (num->negative())
+                    break;
                 num->negate();
                 return num.static_cast_as<Node>();
             }
@@ -1907,6 +1995,8 @@ SharedPtr<Node> Parser::parse_unary_operator(LocalsHashmap &locals) {
         case Node::Type::Float: {
             if (token.type() == Token::Type::Minus) {
                 auto num = receiver.static_cast_as<FloatNode>();
+                if (num->negative())
+                    break;
                 num->negate();
                 return num.static_cast_as<Node>();
             }
@@ -2100,6 +2190,12 @@ SharedPtr<Node> Parser::parse_iter_expression(SharedPtr<Node> left, LocalsHashma
     bool curly_brace = current_token().type() == Token::Type::LCurlyBrace;
     bool has_args = false;
     auto args = Vector<SharedPtr<Node>> {};
+
+    if (curly_brace) {
+        if (left->type() == Node::Type::Call && !left.static_cast_as<CallNode>()->args().is_empty() && !previous_token().is_rparen())
+            throw_unexpected("nearest object cannot accept a { ... } block");
+    }
+
     if (left->type() == Node::Type::StabbyProc) {
         advance(); // { or do
         auto stabby_proc_node = left.static_cast_as<StabbyProcNode>();
@@ -2193,6 +2289,15 @@ SharedPtr<NodeWithArgs> Parser::to_node_with_args(SharedPtr<Node> node) {
         };
         return call_node;
     }
+    case Node::Type::Colon2: {
+        auto colon2 = node.static_cast_as<Colon2Node>();
+        auto call_node = new CallNode {
+            colon2->token(),
+            colon2->left(),
+            colon2->name(),
+        };
+        return call_node;
+    }
     case Node::Type::Call:
     case Node::Type::SafeCall:
     case Node::Type::Super:
@@ -2205,7 +2310,7 @@ SharedPtr<NodeWithArgs> Parser::to_node_with_args(SharedPtr<Node> node) {
 }
 
 void Parser::parse_call_args(NodeWithArgs &node, LocalsHashmap &locals, bool bare, Token::Type closing_token_type) {
-    if (node.can_accept_a_block())
+    if (bare && node.can_accept_a_block())
         m_call_depth.last()++;
     auto arg = parse_expression(bare ? Precedence::BARE_CALL_ARG : Precedence::CALL_ARG, locals);
     if (current_token().is_hash_rocket() || arg->is_symbol_key()) {
@@ -2228,16 +2333,17 @@ void Parser::parse_call_args(NodeWithArgs &node, LocalsHashmap &locals, bool bar
             }
         }
     }
-    if (node.can_accept_a_block())
+    if (bare && node.can_accept_a_block())
         m_call_depth.last()--;
 }
 
-SharedPtr<Node> Parser::parse_call_hash_args(LocalsHashmap &locals, bool bare, Token::Type closing_token_type, SharedPtr<Node> first_arg) {
+SharedPtr<Node> Parser::parse_call_hash_args(LocalsHashmap &locals, bool bare_call, Token::Type closing_token_type, SharedPtr<Node> first_arg) {
+    bool bare_hash = true; // we got here via foo(1, a: 'b') so it's always a "bare" hash
     SharedPtr<Node> hash;
-    if (bare)
-        hash = parse_hash_inner(locals, Precedence::BARE_CALL_ARG, closing_token_type, first_arg);
+    if (bare_call)
+        hash = parse_hash_inner(locals, Precedence::BARE_CALL_ARG, closing_token_type, bare_hash, first_arg);
     else
-        hash = parse_hash_inner(locals, Precedence::CALL_ARG, closing_token_type, first_arg);
+        hash = parse_hash_inner(locals, Precedence::CALL_ARG, closing_token_type, bare_hash, first_arg);
     if (current_token().type() == Token::Type::StarStar)
         hash.static_cast_as<HashNode>()->add_node(parse_keyword_splat(locals));
     return hash;
@@ -2472,15 +2578,19 @@ SharedPtr<Node> Parser::parse_proc_call_expression(SharedPtr<Node> left, LocalsH
 
 SharedPtr<Node> Parser::parse_range_expression(SharedPtr<Node> left, LocalsHashmap &locals) {
     auto token = current_token();
-    advance();
+    advance(); // .. or ...
+    skip_newlines();
     SharedPtr<Node> right;
-    try {
+    if (current_token().can_be_range_arg_token()) {
         right = parse_expression(Precedence::RANGE, locals);
-    } catch (SyntaxError &e) {
-        // NOTE: I'm not sure if this is the "right" way to handle an endless range,
-        // but it seems to be effective for the tests I threw at it. ¯\_(ツ)_/¯
+    } else {
+        // endless range
         right = new NilNode { token };
+        // HACK: insert a newline here so subsequent expressions parse ok
+        if (!current_token().can_follow_collapsible_newline())
+            m_tokens->insert(m_index, Token { Token::Type::Newline, current_token().file(), current_token().line(), current_token().column(), current_token().whitespace_precedes() });
     }
+
     return new RangeNode { token, left, right, token.type() == Token::Type::DotDotDot };
 }
 
@@ -2767,16 +2877,19 @@ Parser::parse_left_fn Parser::left_denotation(Token &token, SharedPtr<Node> left
     case Type::LeftShift:
     case Type::LessThan:
     case Type::LessThanOrEqual:
-    case Type::Minus:
     case Type::NotEqual:
     case Type::Percent:
     case Type::Pipe:
-    case Type::Plus:
     case Type::RightShift:
     case Type::Slash:
     case Type::Star:
     case Type::StarStar:
         return &Parser::parse_infix_expression;
+    case Type::Minus:
+    case Type::Plus:
+        if (!token.whitespace_precedes() || peek_token().whitespace_precedes() || !left->is_callable())
+            return &Parser::parse_infix_expression;
+        break;
     case Type::DoKeyword:
     case Type::LCurlyBrace:
         return &Parser::parse_iter_expression;
@@ -2843,20 +2956,22 @@ bool Parser::is_first_arg_of_call_without_parens(SharedPtr<Node> left, Token &to
     return left->is_callable() && token.can_be_first_arg_of_implicit_call();
 }
 
+Token &Parser::previous_token() const {
+    if (m_index > 0)
+        return (*m_tokens)[m_index - 1];
+    return Token::invalid();
+}
+
 Token &Parser::current_token() const {
-    if (m_index < m_tokens->size()) {
+    if (m_index < m_tokens->size())
         return m_tokens->at(m_index);
-    } else {
-        return Token::invalid();
-    }
+    return Token::invalid();
 }
 
 Token &Parser::peek_token() const {
-    if (m_index + 1 < m_tokens->size()) {
+    if (m_index + 1 < m_tokens->size())
         return (*m_tokens)[m_index + 1];
-    } else {
-        return Token::invalid();
-    }
+    return Token::invalid();
 }
 
 void Parser::next_expression() {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: natalie_parser
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 2.0.0
 platform: ruby
 authors:
 - Tim Morgan
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-04 00:00:00.000000000 Z
+date: 2022-06-24 00:00:00.000000000 Z
 dependencies: []
 description: NatalieParser is a zero-dependency, from-scratch, hand-written recursive
   descent parser for the Ruby Programming Language.
@@ -79,6 +79,7 @@ files:
 - include/natalie_parser/node/interpolated_symbol_node.hpp
 - include/natalie_parser/node/iter_node.hpp
 - include/natalie_parser/node/keyword_arg_node.hpp
+- include/natalie_parser/node/keyword_rest_pattern_node.hpp
 - include/natalie_parser/node/keyword_splat_node.hpp
 - include/natalie_parser/node/logical_and_node.hpp
 - include/natalie_parser/node/logical_or_node.hpp