natalie_parser 2.1.0 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0598d42a8e0b83aa5a340d1615474a4c5a8e7b209032bf574a230778e4d3ce5e'
-  data.tar.gz: 475cce9e2f078ab440674b48e3b99ba48f163f34b889b376cf681f74a489ce53
+  metadata.gz: 2de813d6f0cb6ab94d5a6bb2dd64dc8be719567d401f19ea2b6ac9e71ebaed65
+  data.tar.gz: 668f9574d0968781f077e407f6476f2b320195964f8c10342ab2761beb3f7587
 SHA512:
-  metadata.gz: 59cd20f85484845036f934ca5b40c020ad35655824950c036a697bf0bc437d3ebd35eb25bacc7c6f062d096c01b58dea5a1ebd688d0d101192ee7f43b9881b25
-  data.tar.gz: 53bcc9b34ab02943dda1602141ae949e4fa1dcf1c229dfeda8ef710b01d1d9d9c97137ce7e5b08e88b121025009c9a9fd71edbc77eead37f69f37b794fab02f8
+  metadata.gz: 02da9b2afc4bc76c5b76b03abb48a40bb9242e2884146bedd61b7b944f358401655aeb077bf7900eaf6c3936c3d06000115d20200b378d17bf98cc18ffd28174
+  data.tar.gz: 5e42458d1ab43bce763d3b929b095f9df5ab620e0c9ba25caac5abb8bd44673ded90bcbcd943b94ffe6cadf65d28f5bfebb54bb50942909c4055feb5c613cb2f

data/CHANGELOG.md

@@ -1,5 +1,47 @@
 # Changelog
 
+## 2.3.0 (2022-12-31)
+
+- FIX: Add for loop variables to local scope
+- FIX: Add lexer support for leading-zero (bare) octal - e.g. 0777
+- FIX: Allow non-local variable in rescue
+- FIX: Allow trailing comma after for loop variables
+- FIX: Fix for-loop with attr assignment
+- FIX: Fix lexing of symbol keys with trailing punctuation
+- FIX: Fix precedence of 'do' keyword in use with 'for'
+- FIX: Parse for loop with splat assignment
+- FIX: Use proper make on OpenBSD
+
+## 2.2.0 (2022-10-24)
+
+- FIX: Allow backreference as first arg of implicit call
+- FIX: Exclude `?` and `!` from class/instance/global variable name
+- FIX: Fix parsing of hashes containing assignments
+- FIX: Parse aliases of unary operators and methods named as keywords
+- FIX: Parse alias of keyword-like method to assignment method
+- FIX: Parse alias/undef of assignment methods
+- FIX: Parse backtick as method name
+- FIX: Parse class/module definition in dynamic class/module
+- FIX: Parse `def !`
+- FIX: Parse `def foo::bar`
+- FIX: Parse `foo(<<FOO)`
+- FIX: Parse `for ... do` and `while ... do`
+- FIX: Parse global variables beginning with `$_`
+- FIX: Parse keywords/operators as method names for `::`
+- FIX: Parse __LINE__ constant
+- FIX: Parse more percent strings
+- FIX: Parse more regexes
+- FIX: Parse more stabby procs without parentheses
+- FIX: Parse multiple parameters set to the same default value
+- FIX: Parse parentheses-less stabby proc with keyword arg
+- FIX: Parse undef of methods with special names
+- FIX: Parse `unless ... then`
+- FIX: Parse variable names starting with extended ASCII characters
+- FIX: Raise SyntaxError for `def ====`
+- FIX: Raise syntax error for `def +@.foo`
+- FIX: Tokenize unterminated ternary at EOF
+- FIX: Use lower precedence for block pass
+
 ## 2.1.0 (2022-08-12)
 
 - FEAT: Parse for loops

data/Dockerfile

@@ -8,7 +8,7 @@ ENV LC_ALL C.UTF-8
 
 WORKDIR natalie_parser
 
-COPY Gemfile /natalie_parser/ 
+COPY Gemfile /natalie_parser/
 RUN bundle install
 
 ARG CC=gcc

data/Gemfile

@@ -6,5 +6,5 @@ source 'https://rubygems.org'
 gem 'minitest'
 gem 'minitest-focus'
 gem 'minitest-reporters'
-gem 'ruby_parser'
+gem 'ruby_parser', '3.19.1'
 gem 'rake'

data/README.md

@@ -1,5 +1,6 @@
 # Natalie Parser
 
+[![Gem Version](https://badge.fury.io/rb/natalie_parser.svg)](https://badge.fury.io/rb/natalie_parser)
 [![github build status](https://github.com/natalie-lang/natalie_parser/actions/workflows/build.yml/badge.svg)](https://github.com/natalie-lang/natalie_parser/actions?query=workflow%3ABuild+branch%3Amaster)
 [![MIT License](https://img.shields.io/badge/license-MIT-blue)](https://github.com/natalie-lang/natalie_parser/blob/master/LICENSE)
 

data/Rakefile

@@ -185,7 +185,8 @@ file "ext/natalie_parser/natalie_parser.#{so_ext}" => [
   else
     cxx_hacky = cxx
   end
-  sh "CC=#{cc.inspect} CXX=#{cxx_hacky.inspect} make -C #{build_dir} -j -e V=1 2>&1 | tee #{log_file}"
+  make = system('which gmake 2>&1 > /dev/null') ? 'gmake' : 'make'
+  sh "CC=#{cc.inspect} CXX=#{cxx_hacky.inspect} #{make} -C #{build_dir} -j -e V=1 2>&1 | tee #{log_file}"
 end
 
   file 'build/fragments.hpp' => ['test/parser_test.rb', 'test/support/extract_parser_test_fragments.rb'] do

data/ext/natalie_parser/mri_creator.hpp

@@ -25,7 +25,7 @@ public:
 
     virtual void reset_sexp() override {
         m_sexp = rb_class_new_instance(0, nullptr, Sexp);
-        rb_ivar_set(m_sexp, rb_intern("@file"), get_file_string(file()));
+        rb_ivar_set(m_sexp, rb_intern("@file"), get_file_string(*file()));
         rb_ivar_set(m_sexp, rb_intern("@line"), rb_int_new(line() + 1));
         rb_ivar_set(m_sexp, rb_intern("@column"), rb_int_new(column() + 1));
     }
@@ -139,13 +139,13 @@ public:
 private:
     VALUE m_sexp { Qnil };
 
-    static VALUE get_file_string(SharedPtr<const String> file) {
-        auto file_string = s_file_cache.get(*file);
+    static VALUE get_file_string(const String &file) {
+        auto file_string = s_file_cache.get(file);
         if (!file_string) {
-            file_string = rb_str_new(file->c_str(), file->length());
+            file_string = rb_str_new(file.c_str(), file.length());
             // FIXME: Seems there is no way to un-register and object. :-(
             rb_gc_register_mark_object(file_string);
-            s_file_cache.put(*file, file_string);
+            s_file_cache.put(file, file_string);
         }
         return file_string;
     }

data/ext/natalie_parser/natalie_parser.cpp

@@ -27,9 +27,9 @@ VALUE initialize(int argc, VALUE *argv, VALUE self) {
     return self;
 }
 
-VALUE node_to_ruby(TM::SharedPtr<NatalieParser::Node> node) {
-    NatalieParser::MRICreator creator { node.ref() };
-    node->transform(&creator);
+VALUE node_to_ruby(const NatalieParser::Node &node) {
+    NatalieParser::MRICreator creator { node };
+    node.transform(&creator);
     return creator.sexp();
 }
 
@@ -41,7 +41,7 @@ VALUE parse_on_instance(VALUE self) {
     auto parser = NatalieParser::Parser { code_string, path_string };
     try {
         auto tree = parser.tree();
-        VALUE ast = node_to_ruby(tree);
+        VALUE ast = node_to_ruby(*tree);
         return ast;
     } catch (NatalieParser::Parser::SyntaxError &error) {
         rb_raise(rb_eSyntaxError, "%s", error.message());
@@ -80,6 +80,7 @@ VALUE token_to_ruby(NatalieParser::Token token, bool include_location_info) {
     case NatalieParser::Token::Type::Constant:
     case NatalieParser::Token::Type::GlobalVariable:
     case NatalieParser::Token::Type::InstanceVariable:
+    case NatalieParser::Token::Type::OperatorName:
     case NatalieParser::Token::Type::Symbol:
     case NatalieParser::Token::Type::SymbolKey: {
         auto literal = token.literal_string();

data/include/natalie_parser/lexer.hpp

@@ -70,9 +70,9 @@ protected:
     virtual bool skip_whitespace();
     virtual Token build_next_token();
     Token consume_symbol();
+    SharedPtr<String> consume_word();
     Token consume_word(Token::Type type);
-    Token consume_bare_name();
-    Token consume_constant();
+    Token consume_bare_name_or_constant(Token::Type type);
     Token consume_global_variable();
     Token consume_heredoc();
     Token consume_numeric();
@@ -80,11 +80,19 @@ protected:
     Token consume_nth_ref();
     long long consume_hex_number(int max_length = 0, bool allow_underscore = false);
     long long consume_octal_number(int max_length = 0, bool allow_underscore = false);
-    Token consume_double_quoted_string(char, char, Token::Type begin_type = Token::Type::InterpolatedStringBegin, Token::Type end_type = Token::Type::InterpolatedStringEnd);
+    Token consume_double_quoted_string(char, char, Token::Type begin_type, Token::Type end_type);
     Token consume_single_quoted_string(char, char);
     Token consume_quoted_array_without_interpolation(char start_char, char stop_char, Token::Type type);
     Token consume_quoted_array_with_interpolation(char start_char, char stop_char, Token::Type type);
     Token consume_regexp(char start_char, char stop_char);
+    Token consume_percent_symbol(char start_char, char stop_char);
+    Token consume_interpolated_string(char start_char, char stop_char);
+    Token consume_interpolated_shell(char start_char, char stop_char);
+    Token consume_percent_lower_w(char start_char, char stop_char);
+    Token consume_percent_upper_w(char start_char, char stop_char);
+    Token consume_percent_lower_i(char start_char, char stop_char);
+    Token consume_percent_upper_i(char start_char, char stop_char);
+    Token consume_percent_string(Token (Lexer::*consumer)(char start_char, char stop_char), bool is_lettered = true);
     SharedPtr<String> consume_non_whitespace();
 
     void utf32_codepoint_to_utf8(String &buf, long long codepoint);
@@ -95,7 +103,7 @@ protected:
     bool token_is_first_on_line() const;
 
     bool char_can_be_string_or_regexp_delimiter(char c) const {
-        return (c >= '!' && c <= '/') || c == ':' || c == '?' || c == '@' || c == '~' || c == '|' || (c >= '^' && c <= '`');
+        return (c >= '!' && c <= '/') || c == ':' || c == ';' || c == '=' || c == '?' || c == '@' || c == '\\' || c == '~' || c == '|' || (c >= '^' && c <= '`');
     }
 
     SharedPtr<String> m_input;
@@ -131,5 +139,10 @@ protected:
     // then increment m_pair_depth
     char m_start_char { 0 };
     int m_pair_depth { 0 };
+
+    size_t m_remaining_method_names { 0 };
+    bool m_allow_assignment_method { false };
+    Token::Type m_method_name_separator { Token::Type::Invalid };
+    Token m_last_method_name {};
 };
 }

data/include/natalie_parser/node/begin_rescue_node.hpp

@@ -1,7 +1,6 @@
 #pragma once
 
 #include "natalie_parser/node/block_node.hpp"
-#include "natalie_parser/node/identifier_node.hpp"
 #include "natalie_parser/node/node.hpp"
 #include "natalie_parser/node/node_with_args.hpp"
 #include "tm/hashmap.hpp"
@@ -23,24 +22,24 @@ public:
         m_exceptions.push(node);
     }
 
-    void set_exception_name(SharedPtr<IdentifierNode> name) {
+    void set_exception_name(SharedPtr<Node> name) {
         m_name = name;
     }
 
     void set_body(SharedPtr<BlockNode> body) { m_body = body; }
 
-    SharedPtr<Node> name_to_node() const;
+    SharedPtr<Node> name_to_assignment() const;
 
     bool has_name() const { return m_name; }
 
-    const SharedPtr<IdentifierNode> name() const { return m_name; }
+    const SharedPtr<Node> name() const { return m_name; }
     const Vector<SharedPtr<Node>> &exceptions() const { return m_exceptions; }
     const SharedPtr<BlockNode> body() const { return m_body; }
 
     virtual void transform(Creator *creator) const override;
 
 protected:
-    SharedPtr<IdentifierNode> m_name {};
+    SharedPtr<Node> m_name {};
     Vector<SharedPtr<Node>> m_exceptions {};
     SharedPtr<BlockNode> m_body {};
 };

data/include/natalie_parser/node/for_node.hpp

@@ -25,22 +25,7 @@ public:
     const SharedPtr<Node> vars() const { return m_vars; }
     const SharedPtr<BlockNode> body() const { return m_body; }
 
-    virtual void transform(Creator *creator) const override {
-        creator->set_type("for");
-        creator->append(m_expr);
-        switch (m_vars->type()) {
-        case Node::Type::Identifier:
-            creator->with_assignment(true, [&]() { creator->append(*m_vars); });
-            break;
-        case Node::Type::MultipleAssignment:
-            creator->append(m_vars);
-            break;
-        default:
-            TM_UNREACHABLE();
-        }
-        if (!m_body->is_empty())
-            creator->append(m_body->without_unnecessary_nesting());
-    }
+    virtual void transform(Creator *creator) const override;
 
 protected:
     SharedPtr<Node> m_expr {};

data/include/natalie_parser/parser.hpp

@@ -49,16 +49,22 @@ public:
 
     enum class Precedence;
 
+    enum class IterAllow {
+        NONE,
+        CURLY_ONLY,
+        CURLY_AND_BLOCK,
+    };
+
     SharedPtr<Node> tree();
 
 private:
-    bool higher_precedence(Token &token, SharedPtr<Node> left, Precedence current_precedence);
+    bool higher_precedence(Token &token, SharedPtr<Node> left, Precedence current_precedence, IterAllow iter_allow);
 
     Precedence get_precedence(Token &token, SharedPtr<Node> left = {});
 
     bool is_first_arg_of_call_without_parens(SharedPtr<Node>, Token &);
 
-    SharedPtr<Node> parse_expression(Precedence, LocalsHashmap &);
+    SharedPtr<Node> parse_expression(Precedence, LocalsHashmap &, IterAllow = IterAllow::CURLY_AND_BLOCK);
 
     SharedPtr<BlockNode> parse_body(LocalsHashmap &, Precedence, std::function<bool(Token::Type)>, bool = false);
     SharedPtr<BlockNode> parse_body(LocalsHashmap &, Precedence, Token::Type = Token::Type::EndKeyword, bool = false);
@@ -67,8 +73,9 @@ private:
     SharedPtr<Node> parse_if_body(LocalsHashmap &);
     SharedPtr<BlockNode> parse_def_body(LocalsHashmap &);
 
+    void reinsert_collapsed_newline();
     SharedPtr<Node> parse_alias(LocalsHashmap &);
-    SharedPtr<SymbolNode> parse_alias_arg(LocalsHashmap &, const char *, bool);
+    SharedPtr<SymbolNode> parse_alias_arg(LocalsHashmap &, const char *);
     SharedPtr<Node> parse_array(LocalsHashmap &);
     SharedPtr<Node> parse_back_ref(LocalsHashmap &);
     SharedPtr<Node> parse_begin_block(LocalsHashmap &);
@@ -96,7 +103,8 @@ private:
         Method,
         Proc,
     };
-    void parse_def_single_arg(Vector<SharedPtr<Node>> &, LocalsHashmap &, ArgsContext);
+    void parse_def_single_arg(Vector<SharedPtr<Node>> &, LocalsHashmap &, ArgsContext, IterAllow = IterAllow::CURLY_AND_BLOCK);
+    SharedPtr<Node> parse_arg_default_value(LocalsHashmap &, IterAllow);
 
     SharedPtr<Node> parse_encoding(LocalsHashmap &);
     SharedPtr<Node> parse_end_block(LocalsHashmap &);
@@ -115,6 +123,7 @@ private:
     SharedPtr<Node> parse_interpolated_shell(LocalsHashmap &);
     SharedPtr<Node> parse_interpolated_string(LocalsHashmap &);
     SharedPtr<Node> parse_interpolated_symbol(LocalsHashmap &);
+    SharedPtr<Node> parse_line_constant(LocalsHashmap &);
     SharedPtr<Node> parse_lit(LocalsHashmap &);
     SharedPtr<Node> parse_keyword_splat(LocalsHashmap &);
     SharedPtr<Node> parse_keyword_splat_wrapped_in_hash(LocalsHashmap &);
@@ -124,7 +133,7 @@ private:
     SharedPtr<Node> parse_nil(LocalsHashmap &);
     SharedPtr<Node> parse_not(LocalsHashmap &);
     SharedPtr<Node> parse_nth_ref(LocalsHashmap &);
-    void parse_proc_args(Vector<SharedPtr<Node>> &, LocalsHashmap &);
+    void parse_proc_args(Vector<SharedPtr<Node>> &, LocalsHashmap &, IterAllow);
     SharedPtr<Node> parse_redo(LocalsHashmap &);
     SharedPtr<Node> parse_retry(LocalsHashmap &);
     SharedPtr<Node> parse_return(LocalsHashmap &);
@@ -152,6 +161,7 @@ private:
     SharedPtr<Node> parse_assignment_expression(SharedPtr<Node>, LocalsHashmap &);
     SharedPtr<Node> parse_assignment_expression_without_multiple_values(SharedPtr<Node>, LocalsHashmap &);
     SharedPtr<Node> parse_assignment_expression(SharedPtr<Node>, LocalsHashmap &, bool);
+    void add_assignment_locals(SharedPtr<Node>, LocalsHashmap &);
     SharedPtr<Node> parse_assignment_expression_value(bool, LocalsHashmap &, bool);
     SharedPtr<Node> parse_assignment_identifier(bool, LocalsHashmap &);
     SharedPtr<Node> parse_call_expression_without_parens(SharedPtr<Node>, LocalsHashmap &);

data/include/natalie_parser/token.hpp

@@ -103,6 +103,7 @@ public:
         NotKeyword,
         NotMatch,
         NthRef,
+        OperatorName,
         OrKeyword,
         Percent,
         PercentEqual,
@@ -431,6 +432,8 @@ public:
             return "!";
         case Type::NthRef:
             return "nth_ref";
+        case Type::OperatorName:
+            return "operator";
         case Type::OrKeyword:
             return "or";
         case Type::Percent:
@@ -614,6 +617,7 @@ public:
         case Token::Type::LessThanOrEqual:
         case Token::Type::Match:
         case Token::Type::Minus:
+        case Token::Type::Not:
         case Token::Type::NotEqual:
         case Token::Type::NotMatch:
         case Token::Type::Percent:
@@ -635,6 +639,7 @@ public:
     bool is_closing_token() const { return m_type == Type::RBracket || m_type == Type::RCurlyBrace || m_type == Type::RParen; }
     bool is_comma() const { return m_type == Type::Comma; }
     bool is_comment() const { return m_type == Type::Comment; }
+    bool is_constant_resolution() const { return m_type == Type::ConstantResolution; }
     bool is_def_keyword() const { return m_type == Type::DefKeyword; }
     bool is_doc() const { return m_type == Type::Doc; }
     bool is_dot() const { return m_type == Type::Dot; }
@@ -743,9 +748,23 @@ public:
         }
     }
 
+    bool can_be_first_arg_of_def() const {
+        switch (m_type) {
+        case Token::Type::Ampersand:
+        case Token::Type::BareName:
+        case Token::Type::Star:
+        case Token::Type::StarStar:
+        case Token::Type::SymbolKey:
+            return true;
+        default:
+            return false;
+        }
+    }
+
     bool can_be_first_arg_of_implicit_call() const {
         switch (m_type) {
         case Token::Type::Arrow:
+        case Token::Type::BackRef:
         case Token::Type::BareName:
         case Token::Type::BeginKeyword:
         case Token::Type::Bignum:
@@ -800,6 +819,7 @@ public:
         switch (m_type) {
         case Token::Type::Equal:
         case Token::Type::LBracket:
+        case Token::Type::LParen:
             return true;
         default:
             if (is_operator())
@@ -826,6 +846,7 @@ public:
 
     bool can_precede_symbol_key() const {
         switch (m_type) {
+        case Type::Arrow:
         case Type::BareName:
         case Type::Comma:
         case Type::Constant:

data/lib/natalie_parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class NatalieParser
-  VERSION = '2.1.0'
+  VERSION = '2.3.0'
 end

data/src/lexer/interpolated_string_lexer.cpp

@@ -22,7 +22,7 @@ Token InterpolatedStringLexer::build_next_token() {
 Token InterpolatedStringLexer::consume_string() {
     SharedPtr<String> buf = new String;
     while (auto c = current_char()) {
-        if (c == '\\') {
+        if (c == '\\' && m_stop_char != '\\') {
             advance(); // backslash
             auto result = consume_escaped_byte(*buf);
             if (!result.first)

data/src/lexer/regexp_lexer.cpp

@@ -38,7 +38,7 @@ Token RegexpLexer::build_next_token() {
 Token RegexpLexer::consume_regexp() {
     SharedPtr<String> buf = new String;
     while (auto c = current_char()) {
-        if (c == '\\') {
+        if (c == '\\' && m_stop_char != '\\') {
             c = next();
             switch (c) {
             case '/':

data/src/lexer/word_array_lexer.cpp

@@ -38,7 +38,7 @@ Token WordArrayLexer::build_next_token() {
 Token WordArrayLexer::consume_array() {
     m_buffer = new String;
     while (auto c = current_char()) {
-        if (c == '\\') {
+        if (c == '\\' && m_stop_char != '\\') {
             c = next();
             advance();
             if (c == ' ') {