natalie_parser 2.2.0 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 31bee2746c1e7a36eca16194e27467264dfbad5131075090e0b94c675ff150b1
-  data.tar.gz: 7f9a2a77aa2e34c56faeb740900779ff4682471cb02dbd51403eb1b7f72b13a7
+  metadata.gz: 2de813d6f0cb6ab94d5a6bb2dd64dc8be719567d401f19ea2b6ac9e71ebaed65
+  data.tar.gz: 668f9574d0968781f077e407f6476f2b320195964f8c10342ab2761beb3f7587
 SHA512:
-  metadata.gz: 74335871743111340267d41631f3eeae0281ea5f93bb2bb4a6536682a55f325162d428d986ba0182e14adfb27e830f9622048fb443f1561ee9611c988b07bd6b
-  data.tar.gz: 2fd301c60b32db6c074946adceaa54fca3736f6e64335d5bb68c9ad01f092bf19f1ce2810fb75e4641e4b6cf7ebf7e77165538282107f214fa116254b0d37ac0
+  metadata.gz: 02da9b2afc4bc76c5b76b03abb48a40bb9242e2884146bedd61b7b944f358401655aeb077bf7900eaf6c3936c3d06000115d20200b378d17bf98cc18ffd28174
+  data.tar.gz: 5e42458d1ab43bce763d3b929b095f9df5ab620e0c9ba25caac5abb8bd44673ded90bcbcd943b94ffe6cadf65d28f5bfebb54bb50942909c4055feb5c613cb2f

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 2.3.0 (2022-12-31)
+
+- FIX: Add for loop variables to local scope
+- FIX: Add lexer support for leading-zero (bare) octal - e.g. 0777
+- FIX: Allow non-local variable in rescue
+- FIX: Allow trailing comma after for loop variables
+- FIX: Fix for-loop with attr assignment
+- FIX: Fix lexing of symbol keys with trailing punctuation
+- FIX: Fix precedence of 'do' keyword in use with 'for'
+- FIX: Parse for loop with splat assignment
+- FIX: Use proper make on OpenBSD
+
 ## 2.2.0 (2022-10-24)
 
 - FIX: Allow backreference as first arg of implicit call

data/Dockerfile

@@ -8,7 +8,7 @@ ENV LC_ALL C.UTF-8
 
 WORKDIR natalie_parser
 
-COPY Gemfile /natalie_parser/ 
+COPY Gemfile /natalie_parser/
 RUN bundle install
 
 ARG CC=gcc

data/Gemfile

@@ -6,5 +6,5 @@ source 'https://rubygems.org'
 gem 'minitest'
 gem 'minitest-focus'
 gem 'minitest-reporters'
-gem 'ruby_parser'
+gem 'ruby_parser', '3.19.1'
 gem 'rake'

data/README.md

@@ -1,5 +1,6 @@
 # Natalie Parser
 
+[![Gem Version](https://badge.fury.io/rb/natalie_parser.svg)](https://badge.fury.io/rb/natalie_parser)
 [![github build status](https://github.com/natalie-lang/natalie_parser/actions/workflows/build.yml/badge.svg)](https://github.com/natalie-lang/natalie_parser/actions?query=workflow%3ABuild+branch%3Amaster)
 [![MIT License](https://img.shields.io/badge/license-MIT-blue)](https://github.com/natalie-lang/natalie_parser/blob/master/LICENSE)
 

data/Rakefile

@@ -185,7 +185,8 @@ file "ext/natalie_parser/natalie_parser.#{so_ext}" => [
   else
     cxx_hacky = cxx
   end
-  sh "CC=#{cc.inspect} CXX=#{cxx_hacky.inspect} make -C #{build_dir} -j -e V=1 2>&1 | tee #{log_file}"
+  make = system('which gmake 2>&1 > /dev/null') ? 'gmake' : 'make'
+  sh "CC=#{cc.inspect} CXX=#{cxx_hacky.inspect} #{make} -C #{build_dir} -j -e V=1 2>&1 | tee #{log_file}"
 end
 
   file 'build/fragments.hpp' => ['test/parser_test.rb', 'test/support/extract_parser_test_fragments.rb'] do

data/ext/natalie_parser/mri_creator.hpp

@@ -25,7 +25,7 @@ public:
 
     virtual void reset_sexp() override {
         m_sexp = rb_class_new_instance(0, nullptr, Sexp);
-        rb_ivar_set(m_sexp, rb_intern("@file"), get_file_string(file()));
+        rb_ivar_set(m_sexp, rb_intern("@file"), get_file_string(*file()));
         rb_ivar_set(m_sexp, rb_intern("@line"), rb_int_new(line() + 1));
         rb_ivar_set(m_sexp, rb_intern("@column"), rb_int_new(column() + 1));
     }
@@ -139,13 +139,13 @@ public:
 private:
     VALUE m_sexp { Qnil };
 
-    static VALUE get_file_string(SharedPtr<const String> file) {
-        auto file_string = s_file_cache.get(*file);
+    static VALUE get_file_string(const String &file) {
+        auto file_string = s_file_cache.get(file);
         if (!file_string) {
-            file_string = rb_str_new(file->c_str(), file->length());
+            file_string = rb_str_new(file.c_str(), file.length());
             // FIXME: Seems there is no way to un-register and object. :-(
             rb_gc_register_mark_object(file_string);
-            s_file_cache.put(*file, file_string);
+            s_file_cache.put(file, file_string);
         }
         return file_string;
     }

data/ext/natalie_parser/natalie_parser.cpp

@@ -27,9 +27,9 @@ VALUE initialize(int argc, VALUE *argv, VALUE self) {
     return self;
 }
 
-VALUE node_to_ruby(TM::SharedPtr<NatalieParser::Node> node) {
-    NatalieParser::MRICreator creator { node.ref() };
-    node->transform(&creator);
+VALUE node_to_ruby(const NatalieParser::Node &node) {
+    NatalieParser::MRICreator creator { node };
+    node.transform(&creator);
     return creator.sexp();
 }
 
@@ -41,7 +41,7 @@ VALUE parse_on_instance(VALUE self) {
     auto parser = NatalieParser::Parser { code_string, path_string };
     try {
         auto tree = parser.tree();
-        VALUE ast = node_to_ruby(tree);
+        VALUE ast = node_to_ruby(*tree);
         return ast;
     } catch (NatalieParser::Parser::SyntaxError &error) {
         rb_raise(rb_eSyntaxError, "%s", error.message());

data/include/natalie_parser/node/begin_rescue_node.hpp

@@ -1,7 +1,6 @@
 #pragma once
 
 #include "natalie_parser/node/block_node.hpp"
-#include "natalie_parser/node/identifier_node.hpp"
 #include "natalie_parser/node/node.hpp"
 #include "natalie_parser/node/node_with_args.hpp"
 #include "tm/hashmap.hpp"
@@ -23,24 +22,24 @@ public:
         m_exceptions.push(node);
     }
 
-    void set_exception_name(SharedPtr<IdentifierNode> name) {
+    void set_exception_name(SharedPtr<Node> name) {
         m_name = name;
     }
 
     void set_body(SharedPtr<BlockNode> body) { m_body = body; }
 
-    SharedPtr<Node> name_to_node() const;
+    SharedPtr<Node> name_to_assignment() const;
 
     bool has_name() const { return m_name; }
 
-    const SharedPtr<IdentifierNode> name() const { return m_name; }
+    const SharedPtr<Node> name() const { return m_name; }
     const Vector<SharedPtr<Node>> &exceptions() const { return m_exceptions; }
     const SharedPtr<BlockNode> body() const { return m_body; }
 
     virtual void transform(Creator *creator) const override;
 
 protected:
-    SharedPtr<IdentifierNode> m_name {};
+    SharedPtr<Node> m_name {};
     Vector<SharedPtr<Node>> m_exceptions {};
     SharedPtr<BlockNode> m_body {};
 };

data/include/natalie_parser/node/for_node.hpp

@@ -25,22 +25,7 @@ public:
     const SharedPtr<Node> vars() const { return m_vars; }
     const SharedPtr<BlockNode> body() const { return m_body; }
 
-    virtual void transform(Creator *creator) const override {
-        creator->set_type("for");
-        creator->append(m_expr);
-        switch (m_vars->type()) {
-        case Node::Type::Identifier:
-            creator->with_assignment(true, [&]() { creator->append(*m_vars); });
-            break;
-        case Node::Type::MultipleAssignment:
-            creator->append(m_vars);
-            break;
-        default:
-            TM_UNREACHABLE();
-        }
-        if (!m_body->is_empty())
-            creator->append(m_body->without_unnecessary_nesting());
-    }
+    virtual void transform(Creator *creator) const override;
 
 protected:
     SharedPtr<Node> m_expr {};

data/include/natalie_parser/parser.hpp

@@ -161,6 +161,7 @@ private:
     SharedPtr<Node> parse_assignment_expression(SharedPtr<Node>, LocalsHashmap &);
     SharedPtr<Node> parse_assignment_expression_without_multiple_values(SharedPtr<Node>, LocalsHashmap &);
     SharedPtr<Node> parse_assignment_expression(SharedPtr<Node>, LocalsHashmap &, bool);
+    void add_assignment_locals(SharedPtr<Node>, LocalsHashmap &);
     SharedPtr<Node> parse_assignment_expression_value(bool, LocalsHashmap &, bool);
     SharedPtr<Node> parse_assignment_identifier(bool, LocalsHashmap &);
     SharedPtr<Node> parse_call_expression_without_parens(SharedPtr<Node>, LocalsHashmap &);

data/lib/natalie_parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class NatalieParser
-  VERSION = '2.2.0'
+  VERSION = '2.3.0'
 end

data/src/lexer.cpp

@@ -975,6 +975,10 @@ Token Lexer::consume_bare_name_or_constant(Token::Type type) {
     switch (c) {
     case '?':
     case '!':
+        if (peek() == ':' && m_last_token.can_precede_symbol_key()) {
+            advance();
+            type = Token::Type::SymbolKey;
+        }
         advance();
         buf->append_char(c);
         break;
@@ -1284,7 +1288,26 @@ Token Lexer::consume_numeric() {
             break;
         }
         default:
-            token = consume_decimal_digits_and_build_token();
+            char c = peek();
+
+            if (isdigit(c)) {
+                // bare octal case, e.g. 0777.
+                // If starts with a 0 but next number is not 0..7 then that's an error.
+                if (!(c >= '0' && c <= '7'))
+                    return Token { Token::Type::Invalid, c, m_file, m_cursor_line,
+                        m_cursor_column, m_whitespace_precedes };
+                chars->append_char(c);
+                c = next();
+                do {
+                    chars->append_char(c);
+                    c = next();
+                    if (c == '_')
+                        c = next();
+                } while (c >= '0' && c <= '7');
+                token = chars_to_fixnum_or_bignum_token(chars, 8, 1);
+            } else {
+                token = consume_decimal_digits_and_build_token();
+            }
         }
     } else {
         token = consume_decimal_digits_and_build_token();

data/src/node/begin_rescue_node.cpp

@@ -1,14 +1,15 @@
 #include "natalie_parser/node/begin_rescue_node.hpp"
 #include "natalie_parser/node/array_node.hpp"
 #include "natalie_parser/node/assignment_node.hpp"
+#include "natalie_parser/node/identifier_node.hpp"
 
 namespace NatalieParser {
 
-SharedPtr<Node> BeginRescueNode::name_to_node() const {
+SharedPtr<Node> BeginRescueNode::name_to_assignment() const {
     assert(m_name);
     return new AssignmentNode {
         token(),
-        m_name.static_cast_as<Node>(),
+        m_name,
         new IdentifierNode {
             Token { Token::Type::GlobalVariable, "$!", file(), line(), column(), false },
             false },
@@ -21,7 +22,7 @@ void BeginRescueNode::transform(Creator *creator) const {
     for (auto exception_node : m_exceptions)
         array.add_node(exception_node);
     if (m_name)
-        array.add_node(name_to_node());
+        array.add_node(name_to_assignment());
     creator->append(array);
     if (m_body->nodes().is_empty())
         creator->append_nil();

data/src/node/for_node.cpp

@@ -0,0 +1,13 @@
+#include "natalie_parser/node/for_node.hpp"
+
+namespace NatalieParser {
+
+void ForNode::transform(Creator *creator) const {
+    creator->set_type("for");
+    creator->append(m_expr);
+    creator->with_assignment(true, [&]() { creator->append(*m_vars); });
+    if (!m_body->is_empty())
+        creator->append(m_body->without_unnecessary_nesting());
+}
+
+}

data/src/parser.cpp

@@ -434,16 +434,8 @@ void Parser::parse_rest_of_begin(BeginNode &begin_node, LocalsHashmap &locals) {
             }
             if (current_token().is_hash_rocket()) {
                 advance();
-                SharedPtr<IdentifierNode> name;
-                switch (current_token().type()) {
-                case Token::Type::BareName:
-                    name = new IdentifierNode { current_token(), current_token().literal_string() };
-                    advance();
-                    break;
-                default:
-                    throw_unexpected("exception name");
-                }
-                name->add_to_locals(locals);
+                auto name = parse_expression(Precedence::ASSIGNMENT_LHS, locals);
+                add_assignment_locals(name, locals);
                 rescue_node->set_exception_name(name);
             }
             next_expression();
@@ -910,7 +902,7 @@ SharedPtr<Node> Parser::parse_multiple_assignment_expression(SharedPtr<Node> lef
     list->add_node(left);
     while (current_token().is_comma()) {
         advance();
-        if (current_token().is_rparen() || current_token().is_equal()) {
+        if (current_token().is_rparen() || current_token().is_equal() || current_token().type() == Token::Type::InKeyword) {
             // trailing comma with no additional identifier
             break;
         }
@@ -1296,13 +1288,16 @@ SharedPtr<Node> Parser::parse_line_constant(LocalsHashmap &) {
 SharedPtr<Node> Parser::parse_for(LocalsHashmap &locals) {
     auto token = current_token();
     advance();
-    auto vars = parse_assignment_identifier(false, locals);
-    if (current_token().type() == Token::Type::Comma) {
+    auto vars = parse_assignment_identifier(true, locals);
+    if (current_token().is_comma() || vars->type() == Node::Type::Splat) {
         vars = parse_multiple_assignment_expression(vars, locals);
     }
+    add_assignment_locals(vars, locals);
     expect(Token::Type::InKeyword, "for in");
     advance();
-    auto expr = parse_expression(Precedence::LOWEST, locals, IterAllow::CURLY_ONLY);
+    m_call_depth.last()++;
+    auto expr = parse_expression(Precedence::BARE_CALL_ARG, locals, IterAllow::CURLY_ONLY);
+    m_call_depth.last()--;
     if (current_token().type() == Token::Type::DoKeyword) {
         advance();
     }
@@ -2177,32 +2172,34 @@ SharedPtr<Node> Parser::parse_assignment_expression(SharedPtr<Node> left, Locals
     if (left->type() == Node::Type::Splat) {
         return parse_multiple_assignment_expression(left, locals);
     }
+    add_assignment_locals(left, locals);
+    advance();
+    bool to_array = left->type() == Node::Type::MultipleAssignment;
+    auto value = parse_assignment_expression_value(to_array, locals, allow_multiple);
+    return new AssignmentNode { token, left, value };
+}
+
+void Parser::add_assignment_locals(SharedPtr<Node> left, LocalsHashmap &locals) {
     switch (left->type()) {
     case Node::Type::Identifier: {
         auto left_identifier = left.static_cast_as<IdentifierNode>();
         left_identifier->add_to_locals(locals);
-        advance();
-        auto value = parse_assignment_expression_value(false, locals, allow_multiple);
-        return new AssignmentNode { token, left, value };
+        break;
     }
     case Node::Type::Call:
     case Node::Type::Colon2:
     case Node::Type::Colon3:
     case Node::Type::SafeCall: {
-        advance();
-        auto value = parse_assignment_expression_value(false, locals, allow_multiple);
-        return new AssignmentNode { token, left, value };
+        break;
     }
     case Node::Type::MultipleAssignment: {
         left.static_cast_as<MultipleAssignmentNode>()->add_locals(locals);
-        advance();
-        auto value = parse_assignment_expression_value(true, locals, allow_multiple);
-        return new AssignmentNode { token, left, value };
+        break;
     }
     default:
-        throw_unexpected(left->token(), "left side of assignment");
+        throw_unexpected(left->token(), "assignment identifier");
     }
-};
+}
 
 SharedPtr<Node> Parser::parse_assignment_expression_value(bool to_array, LocalsHashmap &locals, bool allow_multiple) {
     auto token = current_token();

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: natalie_parser
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.3.0
 platform: ruby
 authors:
 - Tim Morgan
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-25 00:00:00.000000000 Z
+date: 2022-12-31 00:00:00.000000000 Z
 dependencies: []
 description: NatalieParser is a zero-dependency, from-scratch, hand-written recursive
   descent parser for the Ruby Programming Language.
@@ -151,6 +151,7 @@ files:
 - src/node/begin_node.cpp
 - src/node/begin_rescue_node.cpp
 - src/node/class_node.cpp
+- src/node/for_node.cpp
 - src/node/interpolated_regexp_node.cpp
 - src/node/interpolated_shell_node.cpp
 - src/node/interpolated_string_node.cpp