natalie_parser 2.0.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b35f9b98f6bbe4ec3bf8d5d9bce2c7e082504db23f442964f2a58e1ac9e11dc7
-  data.tar.gz: 1fbf06aa6fae8400855ea0d363d77a1b27d78869266bfb17b0f45cf9b8a30d41
+  metadata.gz: 31bee2746c1e7a36eca16194e27467264dfbad5131075090e0b94c675ff150b1
+  data.tar.gz: 7f9a2a77aa2e34c56faeb740900779ff4682471cb02dbd51403eb1b7f72b13a7
 SHA512:
-  metadata.gz: 936ec0ef70541afd5839dbe4450401d7e8713499c3536d9ac7e53eb5617671dd308eee8e4f458188a9b0890f5a7ed5d855b3011c5539b64eb31639b4140e07fc
-  data.tar.gz: f6deb35c75e17e5c9fb49c6092e48453e904e7c7d4b515e2c7b5fe7fe69dc656da2faa6096b665b6021557ae8c47844ec65a9e8aca1933712f35d72f4db47c50
+  metadata.gz: 74335871743111340267d41631f3eeae0281ea5f93bb2bb4a6536682a55f325162d428d986ba0182e14adfb27e830f9622048fb443f1561ee9611c988b07bd6b
+  data.tar.gz: 2fd301c60b32db6c074946adceaa54fca3736f6e64335d5bb68c9ad01f092bf19f1ce2810fb75e4641e4b6cf7ebf7e77165538282107f214fa116254b0d37ac0

data/CHANGELOG.md

@@ -1,5 +1,48 @@
 # Changelog
 
+## 2.2.0 (2022-10-24)
+
+- FIX: Allow backreference as first arg of implicit call
+- FIX: Exclude `?` and `!` from class/instance/global variable name
+- FIX: Fix parsing of hashes containing assignments
+- FIX: Parse aliases of unary operators and methods named as keywords
+- FIX: Parse alias of keyword-like method to assignment method
+- FIX: Parse alias/undef of assignment methods
+- FIX: Parse backtick as method name
+- FIX: Parse class/module definition in dynamic class/module
+- FIX: Parse `def !`
+- FIX: Parse `def foo::bar`
+- FIX: Parse `foo(<<FOO)`
+- FIX: Parse `for ... do` and `while ... do`
+- FIX: Parse global variables beginning with `$_`
+- FIX: Parse keywords/operators as method names for `::`
+- FIX: Parse __LINE__ constant
+- FIX: Parse more percent strings
+- FIX: Parse more regexes
+- FIX: Parse more stabby procs without parentheses
+- FIX: Parse multiple parameters set to the same default value
+- FIX: Parse parentheses-less stabby proc with keyword arg
+- FIX: Parse undef of methods with special names
+- FIX: Parse `unless ... then`
+- FIX: Parse variable names starting with extended ASCII characters
+- FIX: Raise SyntaxError for `def ====`
+- FIX: Raise syntax error for `def +@.foo`
+- FIX: Tokenize unterminated ternary at EOF
+- FIX: Use lower precedence for block pass
+
+## 2.1.0 (2022-08-12)
+
+- FEAT: Parse for loops
+- FIX: Fix bug parsing defined? with parens
+- FIX: Fix parsing of keyword splat next to other keyword args
+- FIX: Parse block pass after bare/implicit hash
+- FIX: Parse if statements with match conditions
+- FIX: Parse regexps with leading space preceeded by keywords
+- FIX: Parse symbol key after super keyword
+- FIX: Parse unless statements with match conditions
+- FIX: Parse while/until statements with match conditions
+- FIX: Reset block association level inside array
+
 ## 2.0.0 (2022-06-24)
 
 - FEAT: Differentiate between bare/implicit hash and explicit one

data/ext/natalie_parser/natalie_parser.cpp

@@ -80,6 +80,7 @@ VALUE token_to_ruby(NatalieParser::Token token, bool include_location_info) {
     case NatalieParser::Token::Type::Constant:
     case NatalieParser::Token::Type::GlobalVariable:
     case NatalieParser::Token::Type::InstanceVariable:
+    case NatalieParser::Token::Type::OperatorName:
     case NatalieParser::Token::Type::Symbol:
     case NatalieParser::Token::Type::SymbolKey: {
         auto literal = token.literal_string();

data/include/natalie_parser/lexer.hpp

@@ -70,9 +70,9 @@ protected:
     virtual bool skip_whitespace();
     virtual Token build_next_token();
     Token consume_symbol();
+    SharedPtr<String> consume_word();
     Token consume_word(Token::Type type);
-    Token consume_bare_name();
-    Token consume_constant();
+    Token consume_bare_name_or_constant(Token::Type type);
     Token consume_global_variable();
     Token consume_heredoc();
     Token consume_numeric();
@@ -80,11 +80,19 @@ protected:
     Token consume_nth_ref();
     long long consume_hex_number(int max_length = 0, bool allow_underscore = false);
     long long consume_octal_number(int max_length = 0, bool allow_underscore = false);
-    Token consume_double_quoted_string(char, char, Token::Type begin_type = Token::Type::InterpolatedStringBegin, Token::Type end_type = Token::Type::InterpolatedStringEnd);
+    Token consume_double_quoted_string(char, char, Token::Type begin_type, Token::Type end_type);
     Token consume_single_quoted_string(char, char);
     Token consume_quoted_array_without_interpolation(char start_char, char stop_char, Token::Type type);
     Token consume_quoted_array_with_interpolation(char start_char, char stop_char, Token::Type type);
     Token consume_regexp(char start_char, char stop_char);
+    Token consume_percent_symbol(char start_char, char stop_char);
+    Token consume_interpolated_string(char start_char, char stop_char);
+    Token consume_interpolated_shell(char start_char, char stop_char);
+    Token consume_percent_lower_w(char start_char, char stop_char);
+    Token consume_percent_upper_w(char start_char, char stop_char);
+    Token consume_percent_lower_i(char start_char, char stop_char);
+    Token consume_percent_upper_i(char start_char, char stop_char);
+    Token consume_percent_string(Token (Lexer::*consumer)(char start_char, char stop_char), bool is_lettered = true);
     SharedPtr<String> consume_non_whitespace();
 
     void utf32_codepoint_to_utf8(String &buf, long long codepoint);
@@ -95,7 +103,7 @@ protected:
     bool token_is_first_on_line() const;
 
     bool char_can_be_string_or_regexp_delimiter(char c) const {
-        return (c >= '!' && c <= '/') || c == ':' || c == '?' || c == '@' || c == '~' || c == '|' || (c >= '^' && c <= '`');
+        return (c >= '!' && c <= '/') || c == ':' || c == ';' || c == '=' || c == '?' || c == '@' || c == '\\' || c == '~' || c == '|' || (c >= '^' && c <= '`');
     }
 
     SharedPtr<String> m_input;
@@ -131,5 +139,10 @@ protected:
     // then increment m_pair_depth
     char m_start_char { 0 };
     int m_pair_depth { 0 };
+
+    size_t m_remaining_method_names { 0 };
+    bool m_allow_assignment_method { false };
+    Token::Type m_method_name_separator { Token::Type::Invalid };
+    Token m_last_method_name {};
 };
 }

data/include/natalie_parser/node/for_node.hpp

@@ -0,0 +1,50 @@
+#pragma once
+
+#include "natalie_parser/node/block_node.hpp"
+#include "natalie_parser/node/node.hpp"
+#include "natalie_parser/node/node_with_args.hpp"
+
+namespace NatalieParser {
+
+using namespace TM;
+
+class ForNode : public Node {
+public:
+    ForNode(const Token &token, SharedPtr<Node> expr, SharedPtr<Node> vars, SharedPtr<BlockNode> body)
+        : Node { token }
+        , m_expr { expr }
+        , m_vars { vars }
+        , m_body { body } {
+        assert(m_expr);
+        assert(m_vars);
+    }
+
+    virtual Type type() const override { return Type::For; }
+
+    const SharedPtr<Node> expr() const { return m_expr; }
+    const SharedPtr<Node> vars() const { return m_vars; }
+    const SharedPtr<BlockNode> body() const { return m_body; }
+
+    virtual void transform(Creator *creator) const override {
+        creator->set_type("for");
+        creator->append(m_expr);
+        switch (m_vars->type()) {
+        case Node::Type::Identifier:
+            creator->with_assignment(true, [&]() { creator->append(*m_vars); });
+            break;
+        case Node::Type::MultipleAssignment:
+            creator->append(m_vars);
+            break;
+        default:
+            TM_UNREACHABLE();
+        }
+        if (!m_body->is_empty())
+            creator->append(m_body->without_unnecessary_nesting());
+    }
+
+protected:
+    SharedPtr<Node> m_expr {};
+    SharedPtr<Node> m_vars {};
+    SharedPtr<BlockNode> m_body {};
+};
+}

data/include/natalie_parser/node/match_node.hpp

@@ -13,6 +13,12 @@ using namespace TM;
 
 class MatchNode : public Node {
 public:
+    MatchNode(const Token &token, SharedPtr<RegexpNode> regexp)
+        : Node { token }
+        , m_regexp { regexp } {
+        assert(m_regexp);
+    }
+
     MatchNode(const Token &token, SharedPtr<RegexpNode> regexp, SharedPtr<Node> arg, bool regexp_on_left)
         : Node { token }
         , m_regexp { regexp }

data/include/natalie_parser/node/node.hpp

@@ -43,6 +43,7 @@ public:
         False,
         Fixnum,
         Float,
+        For,
         ForwardArgs,
         Hash,
         HashPattern,

data/include/natalie_parser/node.hpp

@@ -31,6 +31,7 @@
 #include "natalie_parser/node/false_node.hpp"
 #include "natalie_parser/node/fixnum_node.hpp"
 #include "natalie_parser/node/float_node.hpp"
+#include "natalie_parser/node/for_node.hpp"
 #include "natalie_parser/node/forward_args_node.hpp"
 #include "natalie_parser/node/hash_node.hpp"
 #include "natalie_parser/node/hash_pattern_node.hpp"

data/include/natalie_parser/parser.hpp

@@ -49,16 +49,22 @@ public:
 
     enum class Precedence;
 
+    enum class IterAllow {
+        NONE,
+        CURLY_ONLY,
+        CURLY_AND_BLOCK,
+    };
+
     SharedPtr<Node> tree();
 
 private:
-    bool higher_precedence(Token &token, SharedPtr<Node> left, Precedence current_precedence);
+    bool higher_precedence(Token &token, SharedPtr<Node> left, Precedence current_precedence, IterAllow iter_allow);
 
     Precedence get_precedence(Token &token, SharedPtr<Node> left = {});
 
     bool is_first_arg_of_call_without_parens(SharedPtr<Node>, Token &);
 
-    SharedPtr<Node> parse_expression(Precedence, LocalsHashmap &);
+    SharedPtr<Node> parse_expression(Precedence, LocalsHashmap &, IterAllow = IterAllow::CURLY_AND_BLOCK);
 
     SharedPtr<BlockNode> parse_body(LocalsHashmap &, Precedence, std::function<bool(Token::Type)>, bool = false);
     SharedPtr<BlockNode> parse_body(LocalsHashmap &, Precedence, Token::Type = Token::Type::EndKeyword, bool = false);
@@ -67,8 +73,9 @@ private:
     SharedPtr<Node> parse_if_body(LocalsHashmap &);
     SharedPtr<BlockNode> parse_def_body(LocalsHashmap &);
 
+    void reinsert_collapsed_newline();
     SharedPtr<Node> parse_alias(LocalsHashmap &);
-    SharedPtr<SymbolNode> parse_alias_arg(LocalsHashmap &, const char *, bool);
+    SharedPtr<SymbolNode> parse_alias_arg(LocalsHashmap &, const char *);
     SharedPtr<Node> parse_array(LocalsHashmap &);
     SharedPtr<Node> parse_back_ref(LocalsHashmap &);
     SharedPtr<Node> parse_begin_block(LocalsHashmap &);
@@ -96,23 +103,27 @@ private:
         Method,
         Proc,
     };
-    void parse_def_single_arg(Vector<SharedPtr<Node>> &, LocalsHashmap &, ArgsContext);
+    void parse_def_single_arg(Vector<SharedPtr<Node>> &, LocalsHashmap &, ArgsContext, IterAllow = IterAllow::CURLY_AND_BLOCK);
+    SharedPtr<Node> parse_arg_default_value(LocalsHashmap &, IterAllow);
 
     SharedPtr<Node> parse_encoding(LocalsHashmap &);
     SharedPtr<Node> parse_end_block(LocalsHashmap &);
     SharedPtr<Node> parse_file_constant(LocalsHashmap &);
+    SharedPtr<Node> parse_for(LocalsHashmap &);
     SharedPtr<Node> parse_forward_args(LocalsHashmap &);
     SharedPtr<Node> parse_group(LocalsHashmap &);
     SharedPtr<Node> parse_hash(LocalsHashmap &);
     SharedPtr<Node> parse_hash_inner(LocalsHashmap &, Precedence, Token::Type, bool, SharedPtr<Node> = {});
     SharedPtr<Node> parse_identifier(LocalsHashmap &);
     SharedPtr<Node> parse_if(LocalsHashmap &);
+    SharedPtr<Node> parse_if_branch(LocalsHashmap &, bool);
     void parse_interpolated_body(LocalsHashmap &, InterpolatedNode &, Token::Type);
     SharedPtr<Node> parse_interpolated_regexp(LocalsHashmap &);
     int parse_regexp_options(String &);
     SharedPtr<Node> parse_interpolated_shell(LocalsHashmap &);
     SharedPtr<Node> parse_interpolated_string(LocalsHashmap &);
     SharedPtr<Node> parse_interpolated_symbol(LocalsHashmap &);
+    SharedPtr<Node> parse_line_constant(LocalsHashmap &);
     SharedPtr<Node> parse_lit(LocalsHashmap &);
     SharedPtr<Node> parse_keyword_splat(LocalsHashmap &);
     SharedPtr<Node> parse_keyword_splat_wrapped_in_hash(LocalsHashmap &);
@@ -122,7 +133,7 @@ private:
     SharedPtr<Node> parse_nil(LocalsHashmap &);
     SharedPtr<Node> parse_not(LocalsHashmap &);
     SharedPtr<Node> parse_nth_ref(LocalsHashmap &);
-    void parse_proc_args(Vector<SharedPtr<Node>> &, LocalsHashmap &);
+    void parse_proc_args(Vector<SharedPtr<Node>> &, LocalsHashmap &, IterAllow);
     SharedPtr<Node> parse_redo(LocalsHashmap &);
     SharedPtr<Node> parse_retry(LocalsHashmap &);
     SharedPtr<Node> parse_return(LocalsHashmap &);

data/include/natalie_parser/token.hpp

@@ -103,6 +103,7 @@ public:
         NotKeyword,
         NotMatch,
         NthRef,
+        OperatorName,
         OrKeyword,
         Percent,
         PercentEqual,
@@ -431,6 +432,8 @@ public:
             return "!";
         case Type::NthRef:
             return "nth_ref";
+        case Type::OperatorName:
+            return "operator";
         case Type::OrKeyword:
             return "or";
         case Type::Percent:
@@ -614,6 +617,7 @@ public:
         case Token::Type::LessThanOrEqual:
         case Token::Type::Match:
         case Token::Type::Minus:
+        case Token::Type::Not:
         case Token::Type::NotEqual:
         case Token::Type::NotMatch:
         case Token::Type::Percent:
@@ -635,6 +639,7 @@ public:
     bool is_closing_token() const { return m_type == Type::RBracket || m_type == Type::RCurlyBrace || m_type == Type::RParen; }
     bool is_comma() const { return m_type == Type::Comma; }
     bool is_comment() const { return m_type == Type::Comment; }
+    bool is_constant_resolution() const { return m_type == Type::ConstantResolution; }
     bool is_def_keyword() const { return m_type == Type::DefKeyword; }
     bool is_doc() const { return m_type == Type::Doc; }
     bool is_dot() const { return m_type == Type::Dot; }
@@ -743,9 +748,23 @@ public:
         }
     }
 
+    bool can_be_first_arg_of_def() const {
+        switch (m_type) {
+        case Token::Type::Ampersand:
+        case Token::Type::BareName:
+        case Token::Type::Star:
+        case Token::Type::StarStar:
+        case Token::Type::SymbolKey:
+            return true;
+        default:
+            return false;
+        }
+    }
+
     bool can_be_first_arg_of_implicit_call() const {
         switch (m_type) {
         case Token::Type::Arrow:
+        case Token::Type::BackRef:
         case Token::Type::BareName:
         case Token::Type::BeginKeyword:
         case Token::Type::Bignum:
@@ -800,6 +819,7 @@ public:
         switch (m_type) {
         case Token::Type::Equal:
         case Token::Type::LBracket:
+        case Token::Type::LParen:
             return true;
         default:
             if (is_operator())
@@ -808,8 +828,25 @@ public:
         }
     }
 
+    bool can_precede_regexp_literal() const {
+        switch (m_type) {
+        case Type::ElsifKeyword:
+        case Type::IfKeyword:
+        case Type::RescueKeyword:
+        case Type::ReturnKeyword:
+        case Type::UnlessKeyword:
+        case Type::UntilKeyword:
+        case Type::WhenKeyword:
+        case Type::WhileKeyword:
+            return true;
+        default:
+            return false;
+        }
+    }
+
     bool can_precede_symbol_key() const {
         switch (m_type) {
+        case Type::Arrow:
         case Type::BareName:
         case Type::Comma:
         case Type::Constant:
@@ -818,6 +855,7 @@ public:
         case Type::LParen:
         case Type::Pipe:
         case Type::PipePipe:
+        case Type::SuperKeyword:
             return true;
         default:
             return false;

data/lib/natalie_parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class NatalieParser
-  VERSION = '2.0.0'
+  VERSION = '2.2.0'
 end

data/src/lexer/interpolated_string_lexer.cpp

@@ -22,7 +22,7 @@ Token InterpolatedStringLexer::build_next_token() {
 Token InterpolatedStringLexer::consume_string() {
     SharedPtr<String> buf = new String;
     while (auto c = current_char()) {
-        if (c == '\\') {
+        if (c == '\\' && m_stop_char != '\\') {
             advance(); // backslash
             auto result = consume_escaped_byte(*buf);
             if (!result.first)

data/src/lexer/regexp_lexer.cpp

@@ -38,7 +38,7 @@ Token RegexpLexer::build_next_token() {
 Token RegexpLexer::consume_regexp() {
     SharedPtr<String> buf = new String;
     while (auto c = current_char()) {
-        if (c == '\\') {
+        if (c == '\\' && m_stop_char != '\\') {
             c = next();
             switch (c) {
             case '/':

data/src/lexer/word_array_lexer.cpp

@@ -38,7 +38,7 @@ Token WordArrayLexer::build_next_token() {
 Token WordArrayLexer::consume_array() {
     m_buffer = new String;
     while (auto c = current_char()) {
-        if (c == '\\') {
+        if (c == '\\' && m_stop_char != '\\') {
             c = next();
             advance();
             if (c == ' ') {