natalie_parser 1.1.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a06594f8ef75760689083616aefd09c426554ac59e07e37958102ef7bcf514b4
-  data.tar.gz: 2bc38fd3e4dd891b1b5251ed9ba6853d9cc5aa1ca3afa8ffe1af614cb5aa318f
+  metadata.gz: b35f9b98f6bbe4ec3bf8d5d9bce2c7e082504db23f442964f2a58e1ac9e11dc7
+  data.tar.gz: 1fbf06aa6fae8400855ea0d363d77a1b27d78869266bfb17b0f45cf9b8a30d41
 SHA512:
-  metadata.gz: 05515d488cd67f0bbe50c24724248b73cb026b2fc73580419fd1fc017146706ac877fe8a103b14005649695732ca10f293aba8b48d6aee0eb9cd72087e44d339
-  data.tar.gz: 7d55f78cd2e539b4d28951fd50bc3f278df631c63c7aba82d1af6af6b6e085790ecdacc426747f097fb6047f13f178d66c4a751c882f99a115004e39ce45cd70
+  metadata.gz: 936ec0ef70541afd5839dbe4450401d7e8713499c3536d9ac7e53eb5617671dd308eee8e4f458188a9b0890f5a7ed5d855b3011c5539b64eb31639b4140e07fc
+  data.tar.gz: f6deb35c75e17e5c9fb49c6092e48453e904e7c7d4b515e2c7b5fe7fe69dc656da2faa6096b665b6021557ae8c47844ec65a9e8aca1933712f35d72f4db47c50

data/CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## 2.0.0 (2022-06-24)
+
+- FEAT: Differentiate between bare/implicit hash and explicit one
+- FIX: Fix calling colon2
+- FIX: Parse implicit method calls with nth ref argument
+
+## 1.2.1 (2022-06-16)
+
+- FIX: Fix regression with unary/infix operators (+/-)
+
+## 1.2.0 (2022-06-16)
+
+- CHORE: Enable true random fuzzing
+- FEAT: Add Node::debug() function to help with debugging
+- FEAT: Parse more pattern matching cases
+- FIX: Don't error if ext/natalie_parser/build.log isn't written yet
+- FIX: Fix block association inside call with parentheses
+- FIX: Fix bug negating an already-negative number
+- FIX: Fix bug parsing unary operator as an infix operation
+- FIX: Fix method definition with lonely ensure
+- FIX: Fix parsing endless ranges inside case/when and other odd places
+- FIX: Make sure every token knows if it has preceding whitespace
+- FIX: Parse method calls with constant receiver
+
 ## 1.1.1 (2022-06-04)
 
 - FIX: Workaround for clang declspec bug
@@ -8,7 +32,7 @@
 
 - CHORE: Add ccache and compiledb for the ext/natalie_parser directory
 - CHORE: Add tests for numbered block arg shorthand
-- FEAT Parse arg forwarding (...) shorthand
+- FEAT: Parse arg forwarding (...) shorthand
 - FEAT: Parse complex and rational numbers
 - FIX: Fix panic when closing word array delimiter is not found
 - FIX: Fix precedence bug with op assign operators (+= et al)

data/Rakefile

@@ -127,7 +127,9 @@ if system('which compiledb 2>&1 >/dev/null')
     if $compiledb_out.any?
       File.write('build/build.log', $compiledb_out.join("\n"))
       sh 'compiledb < build/build.log'
-      sh 'cd ext/natalie_parser && compiledb < build.log'
+      if File.exist?('ext/natalie_parser/build.log')
+        sh 'cd ext/natalie_parser && compiledb < build.log'
+      end
     end
   end
 else

data/include/natalie_parser/node/array_pattern_node.hpp

@@ -3,6 +3,8 @@
 #include "natalie_parser/node/array_node.hpp"
 #include "natalie_parser/node/node.hpp"
 #include "natalie_parser/node/node_with_args.hpp"
+#include "natalie_parser/node/splat_node.hpp"
+#include "natalie_parser/node/symbol_node.hpp"
 #include "tm/hashmap.hpp"
 #include "tm/string.hpp"
 
@@ -15,14 +17,30 @@ public:
     ArrayPatternNode(const Token &token)
         : ArrayNode { token } { }
 
+    ArrayPatternNode(const Token &token, SharedPtr<Node> node)
+        : ArrayNode { token } {
+        m_nodes.push(node);
+    }
+
     virtual Type type() const override { return Type::ArrayPattern; }
 
     virtual void transform(Creator *creator) const override {
         creator->set_type("array_pat");
         if (!m_nodes.is_empty())
             creator->append_nil(); // NOTE: I don't know what this nil is for
-        for (auto node : m_nodes)
-            creator->append(node);
+        for (auto node : m_nodes) {
+            if (node->type() == Node::Type::Splat) {
+                auto splat_node = node.static_cast_as<SplatNode>();
+                auto name = String("*");
+                if (splat_node->node()) {
+                    assert(splat_node->node()->type() == Node::Type::Symbol);
+                    name.append(splat_node->node().static_cast_as<SymbolNode>()->name().ref());
+                }
+                creator->append_symbol(name);
+            } else {
+                creator->append(node);
+            }
+        }
     }
 };
 }

data/include/natalie_parser/node/bignum_node.hpp

@@ -31,6 +31,10 @@ public:
         m_number->prepend_char('-');
     }
 
+    bool negative() const {
+        return m_number->at(0) == '-';
+    }
+
 protected:
     SharedPtr<String> m_number;
 };

data/include/natalie_parser/node/case_in_node.hpp

@@ -29,8 +29,11 @@ public:
     virtual void transform(Creator *creator) const override {
         creator->set_type("in");
         creator->append(m_pattern.ref());
-        for (auto node : m_body->nodes())
-            creator->append(node);
+        if (!m_body->is_empty())
+            for (auto node : m_body->nodes())
+                creator->append(node);
+        else
+            creator->append_nil();
     }
 
 protected:

data/include/natalie_parser/node/colon2_node.hpp

@@ -23,6 +23,7 @@ public:
     virtual Type type() const override { return Type::Colon2; }
 
     virtual bool is_assignable() const override { return true; }
+    virtual bool is_callable() const override { return true; }
 
     const SharedPtr<Node> left() const { return m_left; }
     SharedPtr<String> name() const { return m_name; }

data/include/natalie_parser/node/fixnum_node.hpp

@@ -30,6 +30,10 @@ public:
         m_number *= -1;
     }
 
+    bool negative() const {
+        return m_number < 0;
+    }
+
 protected:
     long long m_number;
 };

data/include/natalie_parser/node/float_node.hpp

@@ -30,6 +30,10 @@ public:
         m_number *= -1;
     }
 
+    bool negative() const {
+        return m_number < 0.0;
+    }
+
 protected:
     double m_number;
 };

data/include/natalie_parser/node/hash_node.hpp

@@ -11,8 +11,9 @@ using namespace TM;
 
 class HashNode : public Node {
 public:
-    HashNode(const Token &token)
-        : Node { token } { }
+    HashNode(const Token &token, bool bare)
+        : Node { token }
+        , m_bare { bare } { }
 
     virtual Type type() const override { return Type::Hash; }
 
@@ -23,12 +24,16 @@ public:
     const Vector<SharedPtr<Node>> &nodes() const { return m_nodes; }
 
     virtual void transform(Creator *creator) const override {
-        creator->set_type("hash");
+        if (m_bare)
+            creator->set_type("bare_hash");
+        else
+            creator->set_type("hash");
         for (auto node : m_nodes)
             creator->append(node);
     }
 
 protected:
     Vector<SharedPtr<Node>> m_nodes {};
+    bool m_bare { false };
 };
 }

data/include/natalie_parser/node/hash_pattern_node.hpp

@@ -3,6 +3,7 @@
 #include "natalie_parser/node/hash_node.hpp"
 #include "natalie_parser/node/node.hpp"
 #include "natalie_parser/node/node_with_args.hpp"
+#include "natalie_parser/node/symbol_key_node.hpp"
 #include "tm/hashmap.hpp"
 #include "tm/string.hpp"
 
@@ -13,7 +14,7 @@ using namespace TM;
 class HashPatternNode : public HashNode {
 public:
     HashPatternNode(const Token &token)
-        : HashNode { token } { }
+        : HashNode { token, true } { }
 
     virtual Type type() const override { return Type::HashPattern; }
 

data/include/natalie_parser/node/infix_op_node.hpp

@@ -22,7 +22,7 @@ public:
 
     virtual Type type() const override { return Type::InfixOp; }
 
-    virtual bool is_callable() const override { return true; }
+    virtual bool is_callable() const override { return false; }
     virtual bool can_accept_a_block() const override { return false; }
 
     const SharedPtr<Node> left() const { return m_left; }

data/include/natalie_parser/node/keyword_rest_pattern_node.hpp

@@ -0,0 +1,43 @@
+#pragma once
+
+#include "natalie_parser/node/array_node.hpp"
+#include "natalie_parser/node/node.hpp"
+#include "natalie_parser/node/node_with_args.hpp"
+#include "natalie_parser/node/splat_node.hpp"
+#include "natalie_parser/node/symbol_node.hpp"
+#include "tm/hashmap.hpp"
+#include "tm/string.hpp"
+
+namespace NatalieParser {
+
+using namespace TM;
+
+class KeywordRestPatternNode : public Node {
+public:
+    KeywordRestPatternNode(const Token &token)
+        : Node { token } { }
+
+    KeywordRestPatternNode(const Token &token, String name)
+        : Node { token }
+        , m_name { new String(name) } { }
+
+    KeywordRestPatternNode(const Token &token, SharedPtr<String> name)
+        : Node { token }
+        , m_name { name } { }
+
+    virtual Type type() const override { return Type::KeywordRestPattern; }
+
+    const SharedPtr<String> name() const { return m_name; }
+
+    virtual void transform(Creator *creator) const override {
+        creator->set_type("kwrest");
+        auto name = String("**");
+        if (m_name)
+            name.append(m_name.ref());
+        creator->append_symbol(name);
+    }
+
+private:
+    SharedPtr<String> m_name;
+};
+}

data/include/natalie_parser/node/node.hpp

@@ -56,6 +56,7 @@ public:
         InterpolatedSymbol,
         InterpolatedSymbolKey,
         KeywordArg,
+        KeywordRestPattern,
         KeywordSplat,
         LogicalAnd,
         LogicalOr,
@@ -150,6 +151,8 @@ public:
         return type() != Type::Invalid;
     }
 
+    void debug();
+
 protected:
     static inline SharedPtr<Node> s_invalid {};
     Token m_token {};

data/include/natalie_parser/node/unary_op_node.hpp

@@ -20,7 +20,7 @@ public:
 
     virtual Type type() const override { return Type::UnaryOp; }
 
-    virtual bool is_callable() const override { return true; }
+    virtual bool is_callable() const override { return false; }
     virtual bool can_accept_a_block() const override { return false; }
 
     const SharedPtr<String> op() const { return m_op; }

data/include/natalie_parser/node.hpp

@@ -45,6 +45,7 @@
 #include "natalie_parser/node/interpolated_symbol_node.hpp"
 #include "natalie_parser/node/iter_node.hpp"
 #include "natalie_parser/node/keyword_arg_node.hpp"
+#include "natalie_parser/node/keyword_rest_pattern_node.hpp"
 #include "natalie_parser/node/keyword_splat_node.hpp"
 #include "natalie_parser/node/logical_and_node.hpp"
 #include "natalie_parser/node/logical_or_node.hpp"

data/include/natalie_parser/parser.hpp

@@ -82,6 +82,8 @@ private:
     SharedPtr<Node> parse_class_or_module_name(LocalsHashmap &);
     SharedPtr<Node> parse_case(LocalsHashmap &);
     SharedPtr<Node> parse_case_in_pattern(LocalsHashmap &);
+    SharedPtr<Node> parse_case_in_pattern_alternation(LocalsHashmap &);
+    SharedPtr<Node> parse_case_in_pattern_hash_symbol_key(LocalsHashmap &);
     SharedPtr<Node> parse_case_in_patterns(LocalsHashmap &);
     void parse_comma_separated_expressions(ArrayNode &, LocalsHashmap &);
     SharedPtr<Node> parse_constant(LocalsHashmap &);
@@ -102,7 +104,7 @@ private:
     SharedPtr<Node> parse_forward_args(LocalsHashmap &);
     SharedPtr<Node> parse_group(LocalsHashmap &);
     SharedPtr<Node> parse_hash(LocalsHashmap &);
-    SharedPtr<Node> parse_hash_inner(LocalsHashmap &, Precedence, Token::Type, SharedPtr<Node> = {});
+    SharedPtr<Node> parse_hash_inner(LocalsHashmap &, Precedence, Token::Type, bool, SharedPtr<Node> = {});
     SharedPtr<Node> parse_identifier(LocalsHashmap &);
     SharedPtr<Node> parse_if(LocalsHashmap &);
     void parse_interpolated_body(LocalsHashmap &, InterpolatedNode &, Token::Type);
@@ -197,6 +199,7 @@ private:
 
     SharedPtr<NodeWithArgs> to_node_with_args(SharedPtr<Node> node);
 
+    Token &previous_token() const;
     Token &current_token() const;
     Token &peek_token() const;
 

data/include/natalie_parser/token.hpp

@@ -158,58 +158,64 @@ public:
 
     Token() { }
 
-    Token(Type type, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(file);
     }
 
-    Token(Type type, const char *literal, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, const char *literal, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_literal { new String(literal) }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(literal);
         assert(file);
     }
 
-    Token(Type type, SharedPtr<String> literal, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, SharedPtr<String> literal, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_literal { literal }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(literal);
         assert(file);
     }
 
-    Token(Type type, char literal, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, char literal, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_literal { new String(literal) }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(file);
     }
 
-    Token(Type type, long long fixnum, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, long long fixnum, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_fixnum { fixnum }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(file);
     }
 
-    Token(Type type, double dbl, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, double dbl, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_double { dbl }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(file);
     }
 
@@ -636,6 +642,7 @@ public:
     bool is_elsif_keyword() const { return m_type == Type::ElsifKeyword; }
     bool is_end_keyword() const { return m_type == Type::EndKeyword; }
     bool is_end_of_expression() const { return m_type == Type::EndKeyword || m_type == Type::RCurlyBrace || m_type == Type::Newline || m_type == Type::Semicolon || m_type == Type::Eof || is_expression_modifier(); }
+    bool is_ensure() const { return m_type == Type::EnsureKeyword; }
     bool is_eof() const { return m_type == Type::Eof; }
     bool is_end_of_line() const { return m_type == Type::Newline || m_type == Type::Semicolon; }
     bool is_equal() const { return m_type == Type::Equal; }
@@ -644,6 +651,7 @@ public:
     bool is_lparen() const { return m_type == Type::LParen; }
     bool is_newline() const { return m_type == Type::Newline; }
     bool is_rbracket() const { return m_type == Type::RBracket; }
+    bool is_rescue() const { return m_type == Type::RescueKeyword; }
     bool is_rparen() const { return m_type == Type::RParen; }
     bool is_semicolon() const { return m_type == Type::Semicolon; }
     bool is_splat() const { return m_type == Type::Star || m_type == Type::StarStar; }
@@ -662,6 +670,7 @@ public:
         case Token::Type::RParen:
         case Token::Type::SafeNavigation:
         case Token::Type::TernaryColon:
+        case Token::Type::ThenKeyword:
             return true;
         default:
             return false;
@@ -680,7 +689,6 @@ public:
         case Token::Type::Comparison:
         case Token::Type::ConstantResolution:
         case Token::Type::Dot:
-        case Token::Type::DotDot:
         case Token::Type::Equal:
         case Token::Type::EqualEqual:
         case Token::Type::EqualEqualEqual:
@@ -763,13 +771,16 @@ public:
         case Token::Type::LBracketRBracket:
         case Token::Type::LINEKeyword:
         case Token::Type::LParen:
+        case Token::Type::Minus:
         case Token::Type::NilKeyword:
         case Token::Type::Not:
         case Token::Type::NotKeyword:
+        case Token::Type::NthRef:
         case Token::Type::PercentLowerI:
         case Token::Type::PercentLowerW:
         case Token::Type::PercentUpperI:
         case Token::Type::PercentUpperW:
+        case Token::Type::Plus:
         case Token::Type::SelfKeyword:
         case Token::Type::Star:
         case Token::Type::String:
@@ -824,6 +835,25 @@ public:
         }
     }
 
+    bool can_be_range_arg_token() const {
+        if (is_closing_token())
+            return false;
+        if (is_semicolon() || is_eof())
+            return false;
+        switch (m_type) {
+        case Type::ElseKeyword:
+        case Type::ElsifKeyword:
+        case Type::EndKeyword:
+        case Type::InKeyword:
+        case Type::ThenKeyword:
+        case Type::WhenKeyword:
+            // TODO: likely many more cases!
+            return false;
+        default:
+            return true;
+        }
+    }
+
     void set_literal(const char *literal) { m_literal = new String(literal); }
     void set_literal(SharedPtr<String> literal) { m_literal = literal; }
     void set_literal(String literal) { m_literal = new String(literal); }

data/lib/natalie_parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class NatalieParser
-  VERSION = '1.1.1'
+  VERSION = '2.0.0'
 end

data/src/lexer/interpolated_string_lexer.cpp

@@ -14,7 +14,7 @@ Token InterpolatedStringLexer::build_next_token() {
     case State::EndToken:
         return finish();
     case State::Done:
-        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column };
+        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
     }
     TM_UNREACHABLE();
 }
@@ -26,13 +26,13 @@ Token InterpolatedStringLexer::consume_string() {
             advance(); // backslash
             auto result = consume_escaped_byte(*buf);
             if (!result.first)
-                return Token { result.second, current_char(), m_file, m_cursor_line, m_cursor_column };
+                return Token { result.second, current_char(), m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
         } else if (c == '#' && peek() == '{') {
             if (buf->is_empty()) {
                 advance(2);
                 return start_evaluation();
             }
-            auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+            auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             advance(2);
             m_state = State::EvaluateBegin;
             return token;
@@ -49,7 +49,7 @@ Token InterpolatedStringLexer::consume_string() {
                 return finish();
             } else {
                 m_state = State::EndToken;
-                return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+                return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             }
         } else {
             buf->append_char(c);
@@ -62,27 +62,27 @@ Token InterpolatedStringLexer::consume_string() {
     if (m_stop_char == 0) {
         advance();
         m_state = State::EndToken;
-        return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+        return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     }
 
-    return Token { Token::Type::UnterminatedString, buf, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::UnterminatedString, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 Token InterpolatedStringLexer::start_evaluation() {
     m_nested_lexer = new Lexer { *this, '{', '}' };
     m_state = State::EvaluateEnd;
-    return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 Token InterpolatedStringLexer::stop_evaluation() {
     advance(); // }
     m_state = State::InProgress;
-    return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 Token InterpolatedStringLexer::finish() {
     m_state = State::Done;
-    return Token { m_end_type, m_file, m_cursor_line, m_cursor_column };
+    return Token { m_end_type, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
 }
 
 };

data/src/lexer/regexp_lexer.cpp

@@ -11,7 +11,7 @@ Token RegexpLexer::build_next_token() {
         m_nested_lexer = new Lexer { *this };
         m_nested_lexer->set_stop_char('}');
         m_state = State::EvaluateEnd;
-        return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column };
+        return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     case State::EvaluateEnd:
         advance(); // }
         if (current_char() == m_stop_char) {
@@ -21,16 +21,16 @@ Token RegexpLexer::build_next_token() {
         } else {
             m_state = State::InProgress;
         }
-        return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column };
+        return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     case State::EndToken: {
         m_state = State::Done;
-        auto token = Token { Token::Type::InterpolatedRegexpEnd, m_file, m_cursor_line, m_cursor_column };
+        auto token = Token { Token::Type::InterpolatedRegexpEnd, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
         if (m_options && !m_options->is_empty())
             token.set_literal(m_options);
         return token;
     }
     case State::Done:
-        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column };
+        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
     }
     TM_UNREACHABLE();
 }
@@ -55,7 +55,7 @@ Token RegexpLexer::consume_regexp() {
             }
             advance();
         } else if (c == '#' && peek() == '{') {
-            auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+            auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             buf = new String;
             advance(2);
             m_state = State::EvaluateBegin;
@@ -72,14 +72,14 @@ Token RegexpLexer::consume_regexp() {
             } else {
                 m_options = consume_options();
                 m_state = State::EndToken;
-                return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+                return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             }
         } else {
             buf->append_char(c);
             advance();
         }
     }
-    return Token { Token::Type::UnterminatedRegexp, buf, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::UnterminatedRegexp, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 String *RegexpLexer::consume_options() {