natalie_parser 1.0.0 → 1.2.0

data/include/natalie_parser/token.hpp

@@ -36,6 +36,7 @@ public:
         Comma,
         Comment,
         Comparison,
+        Complex,
         Constant,
         ConstantResolution,
         DefinedKeyword,
@@ -115,6 +116,8 @@ public:
         PipePipeEqual,
         Plus,
         PlusEqual,
+        Rational,
+        RationalComplex,
         RCurlyBrace,
         RBracket,
         RedoKeyword,
@@ -155,58 +158,64 @@ public:
 
     Token() { }
 
-    Token(Type type, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(file);
     }
 
-    Token(Type type, const char *literal, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, const char *literal, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_literal { new String(literal) }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(literal);
         assert(file);
     }
 
-    Token(Type type, SharedPtr<String> literal, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, SharedPtr<String> literal, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_literal { literal }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(literal);
         assert(file);
     }
 
-    Token(Type type, char literal, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, char literal, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_literal { new String(literal) }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(file);
     }
 
-    Token(Type type, long long fixnum, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, long long fixnum, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_fixnum { fixnum }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(file);
     }
 
-    Token(Type type, double dbl, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, double dbl, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_double { dbl }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(file);
     }
 
@@ -286,6 +295,8 @@ public:
             return "comment";
         case Type::Comparison:
             return "<=>";
+        case Type::Complex:
+            return "complex";
         case Type::ConstantResolution:
             return "::";
         case Type::Constant:
@@ -446,6 +457,10 @@ public:
             return "+=";
         case Type::Plus:
             return "+";
+        case Type::Rational:
+            return "rational";
+        case Type::RationalComplex:
+            return "rational_complex";
         case Type::RCurlyBrace:
             return "}";
         case Type::RBracket:
@@ -627,6 +642,7 @@ public:
     bool is_elsif_keyword() const { return m_type == Type::ElsifKeyword; }
     bool is_end_keyword() const { return m_type == Type::EndKeyword; }
     bool is_end_of_expression() const { return m_type == Type::EndKeyword || m_type == Type::RCurlyBrace || m_type == Type::Newline || m_type == Type::Semicolon || m_type == Type::Eof || is_expression_modifier(); }
+    bool is_ensure() const { return m_type == Type::EnsureKeyword; }
     bool is_eof() const { return m_type == Type::Eof; }
     bool is_end_of_line() const { return m_type == Type::Newline || m_type == Type::Semicolon; }
     bool is_equal() const { return m_type == Type::Equal; }
@@ -635,6 +651,7 @@ public:
     bool is_lparen() const { return m_type == Type::LParen; }
     bool is_newline() const { return m_type == Type::Newline; }
     bool is_rbracket() const { return m_type == Type::RBracket; }
+    bool is_rescue() const { return m_type == Type::RescueKeyword; }
     bool is_rparen() const { return m_type == Type::RParen; }
     bool is_semicolon() const { return m_type == Type::Semicolon; }
     bool is_splat() const { return m_type == Type::Star || m_type == Type::StarStar; }
@@ -653,6 +670,7 @@ public:
         case Token::Type::RParen:
         case Token::Type::SafeNavigation:
         case Token::Type::TernaryColon:
+        case Token::Type::ThenKeyword:
             return true;
         default:
             return false;
@@ -671,7 +689,6 @@ public:
         case Token::Type::Comparison:
         case Token::Type::ConstantResolution:
         case Token::Type::Dot:
-        case Token::Type::DotDot:
         case Token::Type::Equal:
         case Token::Type::EqualEqual:
         case Token::Type::EqualEqualEqual:
@@ -754,6 +771,7 @@ public:
         case Token::Type::LBracketRBracket:
         case Token::Type::LINEKeyword:
         case Token::Type::LParen:
+        case Token::Type::Minus:
         case Token::Type::NilKeyword:
         case Token::Type::Not:
         case Token::Type::NotKeyword:
@@ -761,6 +779,7 @@ public:
         case Token::Type::PercentLowerW:
         case Token::Type::PercentUpperI:
         case Token::Type::PercentUpperW:
+        case Token::Type::Plus:
         case Token::Type::SelfKeyword:
         case Token::Type::Star:
         case Token::Type::String:
@@ -804,6 +823,36 @@ public:
         }
     }
 
+    bool can_be_complex_or_rational() const {
+        switch (m_type) {
+        case Type::Bignum:
+        case Type::Fixnum:
+        case Type::Float:
+            return true;
+        default:
+            return false;
+        }
+    }
+
+    bool can_be_range_arg_token() const {
+        if (is_closing_token())
+            return false;
+        if (is_semicolon() || is_eof())
+            return false;
+        switch (m_type) {
+        case Type::ElseKeyword:
+        case Type::ElsifKeyword:
+        case Type::EndKeyword:
+        case Type::InKeyword:
+        case Type::ThenKeyword:
+        case Type::WhenKeyword:
+            // TODO: likely many more cases!
+            return false;
+        default:
+            return true;
+        }
+    }
+
     void set_literal(const char *literal) { m_literal = new String(literal); }
     void set_literal(SharedPtr<String> literal) { m_literal = literal; }
     void set_literal(String literal) { m_literal = new String(literal); }

data/lib/natalie_parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class NatalieParser
-  VERSION = '1.0.0'
+  VERSION = '1.2.0'
 end

data/src/lexer/interpolated_string_lexer.cpp

@@ -14,7 +14,7 @@ Token InterpolatedStringLexer::build_next_token() {
     case State::EndToken:
         return finish();
     case State::Done:
-        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column };
+        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
     }
     TM_UNREACHABLE();
 }
@@ -26,13 +26,13 @@ Token InterpolatedStringLexer::consume_string() {
             advance(); // backslash
             auto result = consume_escaped_byte(*buf);
             if (!result.first)
-                return Token { result.second, current_char(), m_file, m_cursor_line, m_cursor_column };
+                return Token { result.second, current_char(), m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
         } else if (c == '#' && peek() == '{') {
             if (buf->is_empty()) {
                 advance(2);
                 return start_evaluation();
             }
-            auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+            auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             advance(2);
             m_state = State::EvaluateBegin;
             return token;
@@ -49,7 +49,7 @@ Token InterpolatedStringLexer::consume_string() {
                 return finish();
             } else {
                 m_state = State::EndToken;
-                return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+                return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             }
         } else {
             buf->append_char(c);
@@ -62,27 +62,27 @@ Token InterpolatedStringLexer::consume_string() {
     if (m_stop_char == 0) {
         advance();
         m_state = State::EndToken;
-        return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+        return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     }
 
-    return Token { Token::Type::UnterminatedString, buf, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::UnterminatedString, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 Token InterpolatedStringLexer::start_evaluation() {
     m_nested_lexer = new Lexer { *this, '{', '}' };
     m_state = State::EvaluateEnd;
-    return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 Token InterpolatedStringLexer::stop_evaluation() {
     advance(); // }
     m_state = State::InProgress;
-    return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 Token InterpolatedStringLexer::finish() {
     m_state = State::Done;
-    return Token { m_end_type, m_file, m_cursor_line, m_cursor_column };
+    return Token { m_end_type, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
 }
 
 };

data/src/lexer/regexp_lexer.cpp

@@ -11,7 +11,7 @@ Token RegexpLexer::build_next_token() {
         m_nested_lexer = new Lexer { *this };
         m_nested_lexer->set_stop_char('}');
         m_state = State::EvaluateEnd;
-        return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column };
+        return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     case State::EvaluateEnd:
         advance(); // }
         if (current_char() == m_stop_char) {
@@ -21,16 +21,16 @@ Token RegexpLexer::build_next_token() {
         } else {
             m_state = State::InProgress;
         }
-        return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column };
+        return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     case State::EndToken: {
         m_state = State::Done;
-        auto token = Token { Token::Type::InterpolatedRegexpEnd, m_file, m_cursor_line, m_cursor_column };
+        auto token = Token { Token::Type::InterpolatedRegexpEnd, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
         if (m_options && !m_options->is_empty())
             token.set_literal(m_options);
         return token;
     }
     case State::Done:
-        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column };
+        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
     }
     TM_UNREACHABLE();
 }
@@ -55,7 +55,7 @@ Token RegexpLexer::consume_regexp() {
             }
             advance();
         } else if (c == '#' && peek() == '{') {
-            auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+            auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             buf = new String;
             advance(2);
             m_state = State::EvaluateBegin;
@@ -72,14 +72,14 @@ Token RegexpLexer::consume_regexp() {
             } else {
                 m_options = consume_options();
                 m_state = State::EndToken;
-                return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+                return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             }
         } else {
             buf->append_char(c);
             advance();
         }
     }
-    return Token { Token::Type::UnterminatedRegexp, buf, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::UnterminatedRegexp, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 String *RegexpLexer::consume_options() {

data/src/lexer/word_array_lexer.cpp

@@ -11,7 +11,7 @@ Token WordArrayLexer::build_next_token() {
         return consume_array();
     case State::DynamicStringBegin:
         m_state = State::EvaluateBegin;
-        return Token { Token::Type::String, m_buffer, m_file, m_token_line, m_token_column };
+        return Token { Token::Type::String, m_buffer, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     case State::DynamicStringEnd:
         if (current_char() == m_stop_char) {
             advance();
@@ -19,18 +19,18 @@ Token WordArrayLexer::build_next_token() {
         } else {
             m_state = State::InProgress;
         }
-        return Token { Token::Type::InterpolatedStringEnd, m_file, m_token_line, m_token_column };
+        return Token { Token::Type::InterpolatedStringEnd, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     case State::EvaluateBegin:
         return start_evaluation();
     case State::EvaluateEnd:
         advance(); // }
         m_state = State::DynamicStringInProgress;
-        return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column };
+        return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     case State::EndToken:
         m_state = State::Done;
-        return Token { Token::Type::RBracket, m_file, m_cursor_line, m_cursor_column };
+        return Token { Token::Type::RBracket, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
     case State::Done:
-        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column };
+        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
     }
     TM_UNREACHABLE();
 }
@@ -70,7 +70,7 @@ Token WordArrayLexer::consume_array() {
                 return dynamic_string_finish();
             }
             if (!m_buffer->is_empty()) {
-                auto token = Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column };
+                auto token = Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
                 advance();
                 return token;
             }
@@ -97,38 +97,38 @@ Token WordArrayLexer::consume_array() {
         }
     }
 
-    return Token { Token::Type::UnterminatedWordArray, m_buffer, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::UnterminatedWordArray, m_buffer, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 Token WordArrayLexer::in_progress_start_dynamic_string() {
     advance(2); // #{
     m_state = State::DynamicStringBegin;
-    return Token { Token::Type::InterpolatedStringBegin, m_file, m_cursor_line, m_cursor_column };
+    return Token { Token::Type::InterpolatedStringBegin, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
 }
 
 Token WordArrayLexer::start_evaluation() {
     m_nested_lexer = new Lexer { *this, '{', '}' };
     m_state = State::EvaluateEnd;
-    return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 Token WordArrayLexer::dynamic_string_finish() {
     if (!m_buffer->is_empty()) {
         m_state = State::DynamicStringEnd;
-        return Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column };
+        return Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
     }
     m_state = State::InProgress;
-    return Token { Token::Type::InterpolatedStringEnd, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::InterpolatedStringEnd, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 Token WordArrayLexer::in_progress_finish() {
     advance(); // ) or ] or } or whatever
     if (!m_buffer->is_empty()) {
         m_state = State::EndToken;
-        return Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column };
+        return Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
     }
     m_state = State::Done;
-    return Token { Token::Type::RBracket, m_file, m_cursor_line, m_cursor_column };
+    return Token { Token::Type::RBracket, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
 }
 
 };