natalie_parser 1.1.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 00b7efc8434fd5d0ade0fe162d09284e3db60417fd0d2beb8a0526df7d52d903
-  data.tar.gz: 749a3d63bfdcb53e4273afe4a0601eb353e9337ba66d4c1f321661bb3f83135e
+  metadata.gz: 88d3e8539a0c9f60203b9d0c914091e0ea4f436629b26b1dca6926edf0488e0a
+  data.tar.gz: 0d165c15a569ec4c76c8cb8de2b2aae13fa502956ff1a38adafd358b7cdecea0
 SHA512:
-  metadata.gz: e975dbfc660b163bf235c8efb4cc2e0c2e9b7924c4a7ed92382a594fdb4443bcf75f967b09fc0fe8af1eafaa4482793c04c07bc5605128fddfe3a8be24174b73
-  data.tar.gz: 121523ba176eec494681009e3863f80f09691234bcc294c26257b5e86544890787c5a40adc4665c9a2c59de37667596957745bc88293b4d8bfc96ccec26e20b0
+  metadata.gz: 9f7f6fad13b1c17cf5efde219fc13ce4dd70e47448f65ea13b0e92f98587acec95355bea454c37de60b5bf9a367862cb8611f6a2fa60a3adb7d4fc9e31e62d2c
+  data.tar.gz: 682ad83ec17e2e1077e4ec826aed547a781a89a4fadccd00c34613e20d945b0795396cb7efbd356430002792e1fea0099dcf3ec72a2cb174a3a5e28adb4290d0

data/CHANGELOG.md

@@ -1,10 +1,32 @@
 # Changelog
 
+## 1.2.1 (2022-06-16)
+
+- FIX: Fix regression with unary/infix operators (+/-)
+
+## 1.2.0 (2022-06-16)
+
+- CHORE: Enable true random fuzzing
+- FEAT: Add Node::debug() function to help with debugging
+- FEAT: Parse more pattern matching cases
+- FIX: Don't error if ext/natalie_parser/build.log isn't written yet
+- FIX: Fix block association inside call with parentheses
+- FIX: Fix bug negating an already-negative number
+- FIX: Fix bug parsing unary operator as an infix operation
+- FIX: Fix method definition with lonely ensure
+- FIX: Fix parsing endless ranges inside case/when and other odd places
+- FIX: Make sure every token knows if it has preceding whitespace
+- FIX: Parse method calls with constant receiver
+
+## 1.1.1 (2022-06-04)
+
+- FIX: Workaround for clang declspec bug
+
 ## 1.1.0 (2022-06-04)
 
 - CHORE: Add ccache and compiledb for the ext/natalie_parser directory
 - CHORE: Add tests for numbered block arg shorthand
-- FEAT Parse arg forwarding (...) shorthand
+- FEAT: Parse arg forwarding (...) shorthand
 - FEAT: Parse complex and rational numbers
 - FIX: Fix panic when closing word array delimiter is not found
 - FIX: Fix precedence bug with op assign operators (+= et al)

data/Rakefile

@@ -127,7 +127,9 @@ if system('which compiledb 2>&1 >/dev/null')
     if $compiledb_out.any?
       File.write('build/build.log', $compiledb_out.join("\n"))
       sh 'compiledb < build/build.log'
-      sh 'cd ext/natalie_parser && compiledb < build.log'
+      if File.exist?('ext/natalie_parser/build.log')
+        sh 'cd ext/natalie_parser && compiledb < build.log'
+      end
     end
   end
 else
@@ -177,7 +179,13 @@ file "ext/natalie_parser/natalie_parser.#{so_ext}" => [
   Rake::FileList['ext/natalie_parser/*.o'].each { |path| rm path }
   rm_rf 'ext/natalie_parser/natalie_parser.so'
   sh "cd #{build_dir} && ruby extconf.rb"
-  sh "CC=#{cc.inspect} CXX=#{cxx.inspect} make -C #{build_dir} -j -e V=1 2>&1 | tee #{log_file}"
+  if `#{cxx} -v 2>&1` =~ /clang/
+    # workaround for clang bug: https://bugs.ruby-lang.org/issues/18616
+    cxx_hacky = "#{cxx} -fdeclspec"
+  else
+    cxx_hacky = cxx
+  end
+  sh "CC=#{cc.inspect} CXX=#{cxx_hacky.inspect} make -C #{build_dir} -j -e V=1 2>&1 | tee #{log_file}"
 end
 
   file 'build/fragments.hpp' => ['test/parser_test.rb', 'test/support/extract_parser_test_fragments.rb'] do

data/include/natalie_parser/node/array_pattern_node.hpp

@@ -3,6 +3,8 @@
 #include "natalie_parser/node/array_node.hpp"
 #include "natalie_parser/node/node.hpp"
 #include "natalie_parser/node/node_with_args.hpp"
+#include "natalie_parser/node/splat_node.hpp"
+#include "natalie_parser/node/symbol_node.hpp"
 #include "tm/hashmap.hpp"
 #include "tm/string.hpp"
 
@@ -15,14 +17,30 @@ public:
     ArrayPatternNode(const Token &token)
         : ArrayNode { token } { }
 
+    ArrayPatternNode(const Token &token, SharedPtr<Node> node)
+        : ArrayNode { token } {
+        m_nodes.push(node);
+    }
+
     virtual Type type() const override { return Type::ArrayPattern; }
 
     virtual void transform(Creator *creator) const override {
         creator->set_type("array_pat");
         if (!m_nodes.is_empty())
             creator->append_nil(); // NOTE: I don't know what this nil is for
-        for (auto node : m_nodes)
-            creator->append(node);
+        for (auto node : m_nodes) {
+            if (node->type() == Node::Type::Splat) {
+                auto splat_node = node.static_cast_as<SplatNode>();
+                auto name = String("*");
+                if (splat_node->node()) {
+                    assert(splat_node->node()->type() == Node::Type::Symbol);
+                    name.append(splat_node->node().static_cast_as<SymbolNode>()->name().ref());
+                }
+                creator->append_symbol(name);
+            } else {
+                creator->append(node);
+            }
+        }
     }
 };
 }

data/include/natalie_parser/node/bignum_node.hpp

@@ -31,6 +31,10 @@ public:
         m_number->prepend_char('-');
     }
 
+    bool negative() const {
+        return m_number->at(0) == '-';
+    }
+
 protected:
     SharedPtr<String> m_number;
 };

data/include/natalie_parser/node/case_in_node.hpp

@@ -29,8 +29,11 @@ public:
     virtual void transform(Creator *creator) const override {
         creator->set_type("in");
         creator->append(m_pattern.ref());
-        for (auto node : m_body->nodes())
-            creator->append(node);
+        if (!m_body->is_empty())
+            for (auto node : m_body->nodes())
+                creator->append(node);
+        else
+            creator->append_nil();
     }
 
 protected:

data/include/natalie_parser/node/fixnum_node.hpp

@@ -30,6 +30,10 @@ public:
         m_number *= -1;
     }
 
+    bool negative() const {
+        return m_number < 0;
+    }
+
 protected:
     long long m_number;
 };

data/include/natalie_parser/node/float_node.hpp

@@ -30,6 +30,10 @@ public:
         m_number *= -1;
     }
 
+    bool negative() const {
+        return m_number < 0.0;
+    }
+
 protected:
     double m_number;
 };

data/include/natalie_parser/node/hash_pattern_node.hpp

@@ -3,6 +3,7 @@
 #include "natalie_parser/node/hash_node.hpp"
 #include "natalie_parser/node/node.hpp"
 #include "natalie_parser/node/node_with_args.hpp"
+#include "natalie_parser/node/symbol_key_node.hpp"
 #include "tm/hashmap.hpp"
 #include "tm/string.hpp"
 

data/include/natalie_parser/node/infix_op_node.hpp

@@ -22,7 +22,7 @@ public:
 
     virtual Type type() const override { return Type::InfixOp; }
 
-    virtual bool is_callable() const override { return true; }
+    virtual bool is_callable() const override { return false; }
     virtual bool can_accept_a_block() const override { return false; }
 
     const SharedPtr<Node> left() const { return m_left; }

data/include/natalie_parser/node/keyword_rest_pattern_node.hpp

@@ -0,0 +1,43 @@
+#pragma once
+
+#include "natalie_parser/node/array_node.hpp"
+#include "natalie_parser/node/node.hpp"
+#include "natalie_parser/node/node_with_args.hpp"
+#include "natalie_parser/node/splat_node.hpp"
+#include "natalie_parser/node/symbol_node.hpp"
+#include "tm/hashmap.hpp"
+#include "tm/string.hpp"
+
+namespace NatalieParser {
+
+using namespace TM;
+
+class KeywordRestPatternNode : public Node {
+public:
+    KeywordRestPatternNode(const Token &token)
+        : Node { token } { }
+
+    KeywordRestPatternNode(const Token &token, String name)
+        : Node { token }
+        , m_name { new String(name) } { }
+
+    KeywordRestPatternNode(const Token &token, SharedPtr<String> name)
+        : Node { token }
+        , m_name { name } { }
+
+    virtual Type type() const override { return Type::KeywordRestPattern; }
+
+    const SharedPtr<String> name() const { return m_name; }
+
+    virtual void transform(Creator *creator) const override {
+        creator->set_type("kwrest");
+        auto name = String("**");
+        if (m_name)
+            name.append(m_name.ref());
+        creator->append_symbol(name);
+    }
+
+private:
+    SharedPtr<String> m_name;
+};
+}

data/include/natalie_parser/node/node.hpp

@@ -56,6 +56,7 @@ public:
         InterpolatedSymbol,
         InterpolatedSymbolKey,
         KeywordArg,
+        KeywordRestPattern,
         KeywordSplat,
         LogicalAnd,
         LogicalOr,
@@ -150,6 +151,8 @@ public:
         return type() != Type::Invalid;
     }
 
+    void debug();
+
 protected:
     static inline SharedPtr<Node> s_invalid {};
     Token m_token {};

data/include/natalie_parser/node/unary_op_node.hpp

@@ -20,7 +20,7 @@ public:
 
     virtual Type type() const override { return Type::UnaryOp; }
 
-    virtual bool is_callable() const override { return true; }
+    virtual bool is_callable() const override { return false; }
     virtual bool can_accept_a_block() const override { return false; }
 
     const SharedPtr<String> op() const { return m_op; }

data/include/natalie_parser/node.hpp

@@ -45,6 +45,7 @@
 #include "natalie_parser/node/interpolated_symbol_node.hpp"
 #include "natalie_parser/node/iter_node.hpp"
 #include "natalie_parser/node/keyword_arg_node.hpp"
+#include "natalie_parser/node/keyword_rest_pattern_node.hpp"
 #include "natalie_parser/node/keyword_splat_node.hpp"
 #include "natalie_parser/node/logical_and_node.hpp"
 #include "natalie_parser/node/logical_or_node.hpp"

data/include/natalie_parser/parser.hpp

@@ -82,6 +82,8 @@ private:
     SharedPtr<Node> parse_class_or_module_name(LocalsHashmap &);
     SharedPtr<Node> parse_case(LocalsHashmap &);
     SharedPtr<Node> parse_case_in_pattern(LocalsHashmap &);
+    SharedPtr<Node> parse_case_in_pattern_alternation(LocalsHashmap &);
+    SharedPtr<Node> parse_case_in_pattern_hash_symbol_key(LocalsHashmap &);
     SharedPtr<Node> parse_case_in_patterns(LocalsHashmap &);
     void parse_comma_separated_expressions(ArrayNode &, LocalsHashmap &);
     SharedPtr<Node> parse_constant(LocalsHashmap &);
@@ -197,6 +199,7 @@ private:
 
     SharedPtr<NodeWithArgs> to_node_with_args(SharedPtr<Node> node);
 
+    Token &previous_token() const;
     Token &current_token() const;
     Token &peek_token() const;
 

data/include/natalie_parser/token.hpp

@@ -158,58 +158,64 @@ public:
 
     Token() { }
 
-    Token(Type type, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(file);
     }
 
-    Token(Type type, const char *literal, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, const char *literal, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_literal { new String(literal) }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(literal);
         assert(file);
     }
 
-    Token(Type type, SharedPtr<String> literal, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, SharedPtr<String> literal, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_literal { literal }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(literal);
         assert(file);
     }
 
-    Token(Type type, char literal, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, char literal, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_literal { new String(literal) }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(file);
     }
 
-    Token(Type type, long long fixnum, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, long long fixnum, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_fixnum { fixnum }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(file);
     }
 
-    Token(Type type, double dbl, SharedPtr<String> file, size_t line, size_t column)
+    Token(Type type, double dbl, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
         : m_type { type }
         , m_double { dbl }
         , m_file { file }
         , m_line { line }
-        , m_column { column } {
+        , m_column { column }
+        , m_whitespace_precedes { whitespace_precedes } {
         assert(file);
     }
 
@@ -636,6 +642,7 @@ public:
     bool is_elsif_keyword() const { return m_type == Type::ElsifKeyword; }
     bool is_end_keyword() const { return m_type == Type::EndKeyword; }
     bool is_end_of_expression() const { return m_type == Type::EndKeyword || m_type == Type::RCurlyBrace || m_type == Type::Newline || m_type == Type::Semicolon || m_type == Type::Eof || is_expression_modifier(); }
+    bool is_ensure() const { return m_type == Type::EnsureKeyword; }
     bool is_eof() const { return m_type == Type::Eof; }
     bool is_end_of_line() const { return m_type == Type::Newline || m_type == Type::Semicolon; }
     bool is_equal() const { return m_type == Type::Equal; }
@@ -644,6 +651,7 @@ public:
     bool is_lparen() const { return m_type == Type::LParen; }
     bool is_newline() const { return m_type == Type::Newline; }
     bool is_rbracket() const { return m_type == Type::RBracket; }
+    bool is_rescue() const { return m_type == Type::RescueKeyword; }
     bool is_rparen() const { return m_type == Type::RParen; }
     bool is_semicolon() const { return m_type == Type::Semicolon; }
     bool is_splat() const { return m_type == Type::Star || m_type == Type::StarStar; }
@@ -662,6 +670,7 @@ public:
         case Token::Type::RParen:
         case Token::Type::SafeNavigation:
         case Token::Type::TernaryColon:
+        case Token::Type::ThenKeyword:
             return true;
         default:
             return false;
@@ -680,7 +689,6 @@ public:
         case Token::Type::Comparison:
         case Token::Type::ConstantResolution:
         case Token::Type::Dot:
-        case Token::Type::DotDot:
         case Token::Type::Equal:
         case Token::Type::EqualEqual:
         case Token::Type::EqualEqualEqual:
@@ -763,6 +771,7 @@ public:
         case Token::Type::LBracketRBracket:
         case Token::Type::LINEKeyword:
         case Token::Type::LParen:
+        case Token::Type::Minus:
         case Token::Type::NilKeyword:
         case Token::Type::Not:
         case Token::Type::NotKeyword:
@@ -770,6 +779,7 @@ public:
         case Token::Type::PercentLowerW:
         case Token::Type::PercentUpperI:
         case Token::Type::PercentUpperW:
+        case Token::Type::Plus:
         case Token::Type::SelfKeyword:
         case Token::Type::Star:
         case Token::Type::String:
@@ -824,6 +834,25 @@ public:
         }
     }
 
+    bool can_be_range_arg_token() const {
+        if (is_closing_token())
+            return false;
+        if (is_semicolon() || is_eof())
+            return false;
+        switch (m_type) {
+        case Type::ElseKeyword:
+        case Type::ElsifKeyword:
+        case Type::EndKeyword:
+        case Type::InKeyword:
+        case Type::ThenKeyword:
+        case Type::WhenKeyword:
+            // TODO: likely many more cases!
+            return false;
+        default:
+            return true;
+        }
+    }
+
     void set_literal(const char *literal) { m_literal = new String(literal); }
     void set_literal(SharedPtr<String> literal) { m_literal = literal; }
     void set_literal(String literal) { m_literal = new String(literal); }

data/lib/natalie_parser/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class NatalieParser
-  VERSION = '1.1.0'
+  VERSION = '1.2.1'
 end

data/src/lexer/interpolated_string_lexer.cpp

@@ -14,7 +14,7 @@ Token InterpolatedStringLexer::build_next_token() {
     case State::EndToken:
         return finish();
     case State::Done:
-        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column };
+        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
     }
     TM_UNREACHABLE();
 }
@@ -26,13 +26,13 @@ Token InterpolatedStringLexer::consume_string() {
             advance(); // backslash
             auto result = consume_escaped_byte(*buf);
             if (!result.first)
-                return Token { result.second, current_char(), m_file, m_cursor_line, m_cursor_column };
+                return Token { result.second, current_char(), m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
         } else if (c == '#' && peek() == '{') {
             if (buf->is_empty()) {
                 advance(2);
                 return start_evaluation();
             }
-            auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+            auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             advance(2);
             m_state = State::EvaluateBegin;
             return token;
@@ -49,7 +49,7 @@ Token InterpolatedStringLexer::consume_string() {
                 return finish();
             } else {
                 m_state = State::EndToken;
-                return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+                return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             }
         } else {
             buf->append_char(c);
@@ -62,27 +62,27 @@ Token InterpolatedStringLexer::consume_string() {
     if (m_stop_char == 0) {
         advance();
         m_state = State::EndToken;
-        return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+        return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     }
 
-    return Token { Token::Type::UnterminatedString, buf, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::UnterminatedString, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 Token InterpolatedStringLexer::start_evaluation() {
     m_nested_lexer = new Lexer { *this, '{', '}' };
     m_state = State::EvaluateEnd;
-    return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 Token InterpolatedStringLexer::stop_evaluation() {
     advance(); // }
     m_state = State::InProgress;
-    return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 Token InterpolatedStringLexer::finish() {
     m_state = State::Done;
-    return Token { m_end_type, m_file, m_cursor_line, m_cursor_column };
+    return Token { m_end_type, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
 }
 
 };

data/src/lexer/regexp_lexer.cpp

@@ -11,7 +11,7 @@ Token RegexpLexer::build_next_token() {
         m_nested_lexer = new Lexer { *this };
         m_nested_lexer->set_stop_char('}');
         m_state = State::EvaluateEnd;
-        return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column };
+        return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     case State::EvaluateEnd:
         advance(); // }
         if (current_char() == m_stop_char) {
@@ -21,16 +21,16 @@ Token RegexpLexer::build_next_token() {
         } else {
             m_state = State::InProgress;
         }
-        return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column };
+        return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     case State::EndToken: {
         m_state = State::Done;
-        auto token = Token { Token::Type::InterpolatedRegexpEnd, m_file, m_cursor_line, m_cursor_column };
+        auto token = Token { Token::Type::InterpolatedRegexpEnd, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
         if (m_options && !m_options->is_empty())
             token.set_literal(m_options);
         return token;
     }
     case State::Done:
-        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column };
+        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
     }
     TM_UNREACHABLE();
 }
@@ -55,7 +55,7 @@ Token RegexpLexer::consume_regexp() {
             }
             advance();
         } else if (c == '#' && peek() == '{') {
-            auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+            auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             buf = new String;
             advance(2);
             m_state = State::EvaluateBegin;
@@ -72,14 +72,14 @@ Token RegexpLexer::consume_regexp() {
             } else {
                 m_options = consume_options();
                 m_state = State::EndToken;
-                return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+                return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
             }
         } else {
             buf->append_char(c);
             advance();
         }
     }
-    return Token { Token::Type::UnterminatedRegexp, buf, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::UnterminatedRegexp, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 String *RegexpLexer::consume_options() {

data/src/lexer/word_array_lexer.cpp

@@ -11,7 +11,7 @@ Token WordArrayLexer::build_next_token() {
         return consume_array();
     case State::DynamicStringBegin:
         m_state = State::EvaluateBegin;
-        return Token { Token::Type::String, m_buffer, m_file, m_token_line, m_token_column };
+        return Token { Token::Type::String, m_buffer, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     case State::DynamicStringEnd:
         if (current_char() == m_stop_char) {
             advance();
@@ -19,18 +19,18 @@ Token WordArrayLexer::build_next_token() {
         } else {
             m_state = State::InProgress;
         }
-        return Token { Token::Type::InterpolatedStringEnd, m_file, m_token_line, m_token_column };
+        return Token { Token::Type::InterpolatedStringEnd, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     case State::EvaluateBegin:
         return start_evaluation();
     case State::EvaluateEnd:
         advance(); // }
         m_state = State::DynamicStringInProgress;
-        return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column };
+        return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column, m_whitespace_precedes };
     case State::EndToken:
         m_state = State::Done;
-        return Token { Token::Type::RBracket, m_file, m_cursor_line, m_cursor_column };
+        return Token { Token::Type::RBracket, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
     case State::Done:
-        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column };
+        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
     }
     TM_UNREACHABLE();
 }
@@ -70,7 +70,7 @@ Token WordArrayLexer::consume_array() {
                 return dynamic_string_finish();
             }
             if (!m_buffer->is_empty()) {
-                auto token = Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column };
+                auto token = Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
                 advance();
                 return token;
             }
@@ -97,38 +97,38 @@ Token WordArrayLexer::consume_array() {
         }
     }
 
-    return Token { Token::Type::UnterminatedWordArray, m_buffer, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::UnterminatedWordArray, m_buffer, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 Token WordArrayLexer::in_progress_start_dynamic_string() {
     advance(2); // #{
     m_state = State::DynamicStringBegin;
-    return Token { Token::Type::InterpolatedStringBegin, m_file, m_cursor_line, m_cursor_column };
+    return Token { Token::Type::InterpolatedStringBegin, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
 }
 
 Token WordArrayLexer::start_evaluation() {
     m_nested_lexer = new Lexer { *this, '{', '}' };
     m_state = State::EvaluateEnd;
-    return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 Token WordArrayLexer::dynamic_string_finish() {
     if (!m_buffer->is_empty()) {
         m_state = State::DynamicStringEnd;
-        return Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column };
+        return Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
     }
     m_state = State::InProgress;
-    return Token { Token::Type::InterpolatedStringEnd, m_file, m_token_line, m_token_column };
+    return Token { Token::Type::InterpolatedStringEnd, m_file, m_token_line, m_token_column, m_whitespace_precedes };
 }
 
 Token WordArrayLexer::in_progress_finish() {
     advance(); // ) or ] or } or whatever
     if (!m_buffer->is_empty()) {
         m_state = State::EndToken;
-        return Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column };
+        return Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
     }
     m_state = State::Done;
-    return Token { Token::Type::RBracket, m_file, m_cursor_line, m_cursor_column };
+    return Token { Token::Type::RBracket, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
 }
 
 };