natalie_parser 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b35f9b98f6bbe4ec3bf8d5d9bce2c7e082504db23f442964f2a58e1ac9e11dc7
-  data.tar.gz: 1fbf06aa6fae8400855ea0d363d77a1b27d78869266bfb17b0f45cf9b8a30d41
+  metadata.gz: '0598d42a8e0b83aa5a340d1615474a4c5a8e7b209032bf574a230778e4d3ce5e'
+  data.tar.gz: 475cce9e2f078ab440674b48e3b99ba48f163f34b889b376cf681f74a489ce53
 SHA512:
-  metadata.gz: 936ec0ef70541afd5839dbe4450401d7e8713499c3536d9ac7e53eb5617671dd308eee8e4f458188a9b0890f5a7ed5d855b3011c5539b64eb31639b4140e07fc
-  data.tar.gz: f6deb35c75e17e5c9fb49c6092e48453e904e7c7d4b515e2c7b5fe7fe69dc656da2faa6096b665b6021557ae8c47844ec65a9e8aca1933712f35d72f4db47c50
+  metadata.gz: 59cd20f85484845036f934ca5b40c020ad35655824950c036a697bf0bc437d3ebd35eb25bacc7c6f062d096c01b58dea5a1ebd688d0d101192ee7f43b9881b25
+  data.tar.gz: 53bcc9b34ab02943dda1602141ae949e4fa1dcf1c229dfeda8ef710b01d1d9d9c97137ce7e5b08e88b121025009c9a9fd71edbc77eead37f69f37b794fab02f8

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 2.1.0 (2022-08-12)
+
+- FEAT: Parse for loops
+- FIX: Fix bug parsing defined? with parens
+- FIX: Fix parsing of keyword splat next to other keyword args
+- FIX: Parse block pass after bare/implicit hash
+- FIX: Parse if statements with match conditions
+- FIX: Parse regexps with leading space preceeded by keywords
+- FIX: Parse symbol key after super keyword
+- FIX: Parse unless statements with match conditions
+- FIX: Parse while/until statements with match conditions
+- FIX: Reset block association level inside array
+
 ## 2.0.0 (2022-06-24)
 
 - FEAT: Differentiate between bare/implicit hash and explicit one

data/include/natalie_parser/node/for_node.hpp

@@ -0,0 +1,50 @@
+#pragma once
+
+#include "natalie_parser/node/block_node.hpp"
+#include "natalie_parser/node/node.hpp"
+#include "natalie_parser/node/node_with_args.hpp"
+
+namespace NatalieParser {
+
+using namespace TM;
+
+class ForNode : public Node {
+public:
+    ForNode(const Token &token, SharedPtr<Node> expr, SharedPtr<Node> vars, SharedPtr<BlockNode> body)
+        : Node { token }
+        , m_expr { expr }
+        , m_vars { vars }
+        , m_body { body } {
+        assert(m_expr);
+        assert(m_vars);
+    }
+
+    virtual Type type() const override { return Type::For; }
+
+    const SharedPtr<Node> expr() const { return m_expr; }
+    const SharedPtr<Node> vars() const { return m_vars; }
+    const SharedPtr<BlockNode> body() const { return m_body; }
+
+    virtual void transform(Creator *creator) const override {
+        creator->set_type("for");
+        creator->append(m_expr);
+        switch (m_vars->type()) {
+        case Node::Type::Identifier:
+            creator->with_assignment(true, [&]() { creator->append(*m_vars); });
+            break;
+        case Node::Type::MultipleAssignment:
+            creator->append(m_vars);
+            break;
+        default:
+            TM_UNREACHABLE();
+        }
+        if (!m_body->is_empty())
+            creator->append(m_body->without_unnecessary_nesting());
+    }
+
+protected:
+    SharedPtr<Node> m_expr {};
+    SharedPtr<Node> m_vars {};
+    SharedPtr<BlockNode> m_body {};
+};
+}

data/include/natalie_parser/node/match_node.hpp

@@ -13,6 +13,12 @@ using namespace TM;
 
 class MatchNode : public Node {
 public:
+    MatchNode(const Token &token, SharedPtr<RegexpNode> regexp)
+        : Node { token }
+        , m_regexp { regexp } {
+        assert(m_regexp);
+    }
+
     MatchNode(const Token &token, SharedPtr<RegexpNode> regexp, SharedPtr<Node> arg, bool regexp_on_left)
         : Node { token }
         , m_regexp { regexp }

data/include/natalie_parser/node/node.hpp

@@ -43,6 +43,7 @@ public:
         False,
         Fixnum,
         Float,
+        For,
         ForwardArgs,
         Hash,
         HashPattern,

data/include/natalie_parser/node.hpp

@@ -31,6 +31,7 @@
 #include "natalie_parser/node/false_node.hpp"
 #include "natalie_parser/node/fixnum_node.hpp"
 #include "natalie_parser/node/float_node.hpp"
+#include "natalie_parser/node/for_node.hpp"
 #include "natalie_parser/node/forward_args_node.hpp"
 #include "natalie_parser/node/hash_node.hpp"
 #include "natalie_parser/node/hash_pattern_node.hpp"

data/include/natalie_parser/parser.hpp

@@ -101,12 +101,14 @@ private:
     SharedPtr<Node> parse_encoding(LocalsHashmap &);
     SharedPtr<Node> parse_end_block(LocalsHashmap &);
     SharedPtr<Node> parse_file_constant(LocalsHashmap &);
+    SharedPtr<Node> parse_for(LocalsHashmap &);
     SharedPtr<Node> parse_forward_args(LocalsHashmap &);
     SharedPtr<Node> parse_group(LocalsHashmap &);
     SharedPtr<Node> parse_hash(LocalsHashmap &);
     SharedPtr<Node> parse_hash_inner(LocalsHashmap &, Precedence, Token::Type, bool, SharedPtr<Node> = {});
     SharedPtr<Node> parse_identifier(LocalsHashmap &);
     SharedPtr<Node> parse_if(LocalsHashmap &);
+    SharedPtr<Node> parse_if_branch(LocalsHashmap &, bool);
     void parse_interpolated_body(LocalsHashmap &, InterpolatedNode &, Token::Type);
     SharedPtr<Node> parse_interpolated_regexp(LocalsHashmap &);
     int parse_regexp_options(String &);

data/include/natalie_parser/token.hpp

@@ -808,6 +808,22 @@ public:
         }
     }
 
+    bool can_precede_regexp_literal() const {
+        switch (m_type) {
+        case Type::ElsifKeyword:
+        case Type::IfKeyword:
+        case Type::RescueKeyword:
+        case Type::ReturnKeyword:
+        case Type::UnlessKeyword:
+        case Type::UntilKeyword:
+        case Type::WhenKeyword:
+        case Type::WhileKeyword:
+            return true;
+        default:
+            return false;
+        }
+    }
+
     bool can_precede_symbol_key() const {
         switch (m_type) {
         case Type::BareName:
@@ -818,6 +834,7 @@ public:
         case Type::LParen:
         case Type::Pipe:
         case Type::PipePipe:
+        case Type::SuperKeyword:
             return true;
         default:
             return false;

data/lib/natalie_parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class NatalieParser
-  VERSION = '2.0.0'
+  VERSION = '2.1.0'
 end

data/src/lexer.cpp

@@ -270,7 +270,11 @@ Token Lexer::build_next_token() {
         default: {
             switch (current_char()) {
             case ' ':
-                return Token { Token::Type::Slash, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+                if (m_last_token.is_keyword() && m_last_token.can_precede_regexp_literal()) {
+                    return consume_regexp('/', '/');
+                } else {
+                    return Token { Token::Type::Slash, m_file, m_token_line, m_token_column, m_whitespace_precedes };
+                }
             case '=':
                 advance();
                 return Token { Token::Type::SlashEqual, m_file, m_token_line, m_token_column, m_whitespace_precedes };

data/src/node/match_node.cpp

@@ -3,6 +3,11 @@
 namespace NatalieParser {
 
 void MatchNode::transform(Creator *creator) const {
+    if (!m_arg) {
+        creator->set_type("match");
+        creator->append(m_regexp.ref());
+        return;
+    }
     if (m_regexp_on_left)
         creator->set_type("match2");
     else

data/src/parser.cpp

@@ -354,11 +354,17 @@ SharedPtr<Node> Parser::parse_array(LocalsHashmap &locals) {
         return {};
     };
     auto ret = add_node();
-    if (ret) return ret;
+    if (ret) {
+        m_call_depth.pop();
+        return ret;
+    }
     while (current_token().is_comma()) {
         advance();
         ret = add_node();
-        if (ret) return ret;
+        if (ret) {
+            m_call_depth.pop();
+            return ret;
+        }
     }
     expect(Token::Type::RBracket, "array closing bracket");
     advance(); // ]
@@ -1058,7 +1064,16 @@ SharedPtr<Node> Parser::parse_def(LocalsHashmap &locals) {
 SharedPtr<Node> Parser::parse_defined(LocalsHashmap &locals) {
     auto token = current_token();
     advance();
-    auto arg = parse_expression(Precedence::BARE_CALL_ARG, locals);
+    bool bare = true;
+    if (current_token().is_lparen()) {
+        advance();
+        bare = false;
+    }
+    auto arg = parse_expression(bare ? Precedence::BARE_CALL_ARG : Precedence::CALL_ARG, locals);
+    if (!bare) {
+        expect(Token::Type::RParen, "defined? closing paren");
+        advance();
+    }
     return new DefinedNode { token, arg };
 }
 
@@ -1248,6 +1263,22 @@ SharedPtr<Node> Parser::parse_file_constant(LocalsHashmap &) {
     return new StringNode { token, token.file() };
 }
 
+SharedPtr<Node> Parser::parse_for(LocalsHashmap &locals) {
+    auto token = current_token();
+    advance();
+    auto vars = parse_assignment_identifier(false, locals);
+    if (current_token().type() == Token::Type::Comma) {
+        vars = parse_multiple_assignment_expression(vars, locals);
+    }
+    expect(Token::Type::InKeyword, "for in");
+    advance();
+    auto expr = parse_expression(Precedence::LOWEST, locals);
+    auto body = parse_body(locals, Precedence::LOWEST);
+    expect(Token::Type::EndKeyword, "for end");
+    advance();
+    return new ForNode { token, expr, vars, body };
+}
+
 SharedPtr<Node> Parser::parse_forward_args(LocalsHashmap &locals) {
     auto token = current_token();
     if (!locals.get("..."))
@@ -1310,28 +1341,35 @@ SharedPtr<Node> Parser::parse_hash(LocalsHashmap &locals) {
 SharedPtr<Node> Parser::parse_hash_inner(LocalsHashmap &locals, Precedence precedence, Token::Type closing_token_type, bool bare, SharedPtr<Node> first_key) {
     auto token = current_token();
     SharedPtr<HashNode> hash = new HashNode { token, bare };
+
+    auto add_value = [&](SharedPtr<Node> key) {
+        if (key->is_symbol_key()) {
+            hash->add_node(parse_expression(precedence, locals));
+        } else if (key->type() == Node::Type::KeywordSplat) {
+            // kwsplat already added
+        } else {
+            expect(Token::Type::HashRocket, "hash rocket");
+            advance(); // =>
+            hash->add_node(parse_expression(precedence, locals));
+        }
+    };
+
     if (!first_key)
         first_key = parse_expression(precedence, locals);
     hash->add_node(first_key);
-    if (!first_key->is_symbol_key()) {
-        expect(Token::Type::HashRocket, "hash rocket");
-        advance();
-    }
-    hash->add_node(parse_expression(precedence, locals));
+    add_value(first_key);
+
     while (current_token().is_comma()) {
         advance();
         if (current_token().type() == closing_token_type)
             break;
-        if (current_token().type() == Token::Type::StarStar) // **kwsplat
+        if (current_token().type() == Token::Type::Ampersand) // &block
             break;
         auto key = parse_expression(precedence, locals);
         hash->add_node(key);
-        if (!key->is_symbol_key()) {
-            expect(Token::Type::HashRocket, "hash rocket");
-            advance();
-        }
-        hash->add_node(parse_expression(precedence, locals));
+        add_value(key);
     }
+
     return hash.static_cast_as<Node>();
 }
 
@@ -1345,9 +1383,16 @@ SharedPtr<Node> Parser::parse_identifier(LocalsHashmap &locals) {
 };
 
 SharedPtr<Node> Parser::parse_if(LocalsHashmap &locals) {
+    return parse_if_branch(locals, true);
+}
+
+SharedPtr<Node> Parser::parse_if_branch(LocalsHashmap &locals, bool parse_match_condition) {
     auto token = current_token();
     advance();
     SharedPtr<Node> condition = parse_expression(Precedence::LOWEST, locals);
+    if (parse_match_condition && condition->type() == Node::Type::Regexp) {
+        condition = new MatchNode { condition->token(), condition.static_cast_as<RegexpNode>() };
+    }
     if (current_token().type() == Token::Type::ThenKeyword) {
         advance(); // then
     } else {
@@ -1356,7 +1401,7 @@ SharedPtr<Node> Parser::parse_if(LocalsHashmap &locals) {
     SharedPtr<Node> true_expr = parse_if_body(locals);
     SharedPtr<Node> false_expr;
     if (current_token().is_elsif_keyword()) {
-        false_expr = parse_if(locals);
+        false_expr = parse_if_branch(locals, false);
         return new IfNode { current_token(), condition, true_expr, false_expr };
     } else {
         if (current_token().is_else_keyword()) {
@@ -1639,12 +1684,6 @@ SharedPtr<Node> Parser::parse_keyword_splat(LocalsHashmap &locals) {
     return new KeywordSplatNode { token, parse_expression(Precedence::SPLAT, locals) };
 }
 
-SharedPtr<Node> Parser::parse_keyword_splat_wrapped_in_hash(LocalsHashmap &locals) {
-    SharedPtr<HashNode> hash = new HashNode { current_token(), true };
-    hash->add_node(parse_keyword_splat(locals));
-    return hash.static_cast_as<Node>();
-}
-
 SharedPtr<String> Parser::parse_method_name(LocalsHashmap &) {
     SharedPtr<String> name = new String("");
     auto token = current_token();
@@ -2313,7 +2352,7 @@ void Parser::parse_call_args(NodeWithArgs &node, LocalsHashmap &locals, bool bar
     if (bare && node.can_accept_a_block())
         m_call_depth.last()++;
     auto arg = parse_expression(bare ? Precedence::BARE_CALL_ARG : Precedence::CALL_ARG, locals);
-    if (current_token().is_hash_rocket() || arg->is_symbol_key()) {
+    if (current_token().is_hash_rocket() || arg->is_symbol_key() || arg->type() == Node::Type::KeywordSplat) {
         node.add_arg(parse_call_hash_args(locals, bare, closing_token_type, arg));
     } else {
         node.add_arg(arg);
@@ -2325,7 +2364,7 @@ void Parser::parse_call_args(NodeWithArgs &node, LocalsHashmap &locals, bool bar
                 break;
             }
             arg = parse_expression(bare ? Precedence::BARE_CALL_ARG : Precedence::CALL_ARG, locals);
-            if (current_token().is_hash_rocket() || arg->is_symbol_key()) {
+            if (current_token().is_hash_rocket() || arg->is_symbol_key() || arg->type() == Node::Type::KeywordSplat) {
                 node.add_arg(parse_call_hash_args(locals, bare, closing_token_type, arg));
                 break;
             } else {
@@ -2333,6 +2372,9 @@ void Parser::parse_call_args(NodeWithArgs &node, LocalsHashmap &locals, bool bar
             }
         }
     }
+    if (current_token().type() == Token::Type::Ampersand) {
+        node.add_arg(parse_block_pass(locals));
+    }
     if (bare && node.can_accept_a_block())
         m_call_depth.last()--;
 }
@@ -2700,6 +2742,9 @@ SharedPtr<Node> Parser::parse_unless(LocalsHashmap &locals) {
     auto token = current_token();
     advance();
     SharedPtr<Node> condition = parse_expression(Precedence::LOWEST, locals);
+    if (condition->type() == Node::Type::Regexp) {
+        condition = new MatchNode { condition->token(), condition.static_cast_as<RegexpNode>() };
+    }
     next_expression();
     SharedPtr<Node> false_expr = parse_if_body(locals);
     SharedPtr<Node> true_expr;
@@ -2718,6 +2763,9 @@ SharedPtr<Node> Parser::parse_while(LocalsHashmap &locals) {
     auto token = current_token();
     advance();
     SharedPtr<Node> condition = parse_expression(Precedence::LOWEST, locals);
+    if (condition->type() == Node::Type::Regexp) {
+        condition = new MatchNode { condition->token(), condition.static_cast_as<RegexpNode>() };
+    }
     next_expression();
     SharedPtr<BlockNode> body = parse_body(locals, Precedence::LOWEST);
     expect(Token::Type::EndKeyword, "while end");
@@ -2770,6 +2818,8 @@ Parser::parse_null_fn Parser::null_denotation(Token::Type type) {
         return &Parser::parse_end_block;
     case Type::FILEKeyword:
         return &Parser::parse_file_constant;
+    case Type::ForKeyword:
+        return &Parser::parse_for;
     case Type::LParen:
         return &Parser::parse_group;
     case Type::LCurlyBrace:
@@ -2791,7 +2841,7 @@ Parser::parse_null_fn Parser::null_denotation(Token::Type type) {
     case Type::InterpolatedSymbolBegin:
         return &Parser::parse_interpolated_symbol;
     case Type::StarStar:
-        return &Parser::parse_keyword_splat_wrapped_in_hash;
+        return &Parser::parse_keyword_splat;
     case Type::Bignum:
     case Type::Fixnum:
     case Type::Float:

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: natalie_parser
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Tim Morgan
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-06-24 00:00:00.000000000 Z
+date: 2022-08-12 00:00:00.000000000 Z
 dependencies: []
 description: NatalieParser is a zero-dependency, from-scratch, hand-written recursive
   descent parser for the Ruby Programming Language.
@@ -65,6 +65,7 @@ files:
 - include/natalie_parser/node/false_node.hpp
 - include/natalie_parser/node/fixnum_node.hpp
 - include/natalie_parser/node/float_node.hpp
+- include/natalie_parser/node/for_node.hpp
 - include/natalie_parser/node/forward_args_node.hpp
 - include/natalie_parser/node/hash_node.hpp
 - include/natalie_parser/node/hash_pattern_node.hpp