natalie_parser 1.1.0 → 1.2.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 00b7efc8434fd5d0ade0fe162d09284e3db60417fd0d2beb8a0526df7d52d903
4
- data.tar.gz: 749a3d63bfdcb53e4273afe4a0601eb353e9337ba66d4c1f321661bb3f83135e
3
+ metadata.gz: 88d3e8539a0c9f60203b9d0c914091e0ea4f436629b26b1dca6926edf0488e0a
4
+ data.tar.gz: 0d165c15a569ec4c76c8cb8de2b2aae13fa502956ff1a38adafd358b7cdecea0
5
5
  SHA512:
6
- metadata.gz: e975dbfc660b163bf235c8efb4cc2e0c2e9b7924c4a7ed92382a594fdb4443bcf75f967b09fc0fe8af1eafaa4482793c04c07bc5605128fddfe3a8be24174b73
7
- data.tar.gz: 121523ba176eec494681009e3863f80f09691234bcc294c26257b5e86544890787c5a40adc4665c9a2c59de37667596957745bc88293b4d8bfc96ccec26e20b0
6
+ metadata.gz: 9f7f6fad13b1c17cf5efde219fc13ce4dd70e47448f65ea13b0e92f98587acec95355bea454c37de60b5bf9a367862cb8611f6a2fa60a3adb7d4fc9e31e62d2c
7
+ data.tar.gz: 682ad83ec17e2e1077e4ec826aed547a781a89a4fadccd00c34613e20d945b0795396cb7efbd356430002792e1fea0099dcf3ec72a2cb174a3a5e28adb4290d0
data/CHANGELOG.md CHANGED
@@ -1,10 +1,32 @@
1
1
  # Changelog
2
2
 
3
+ ## 1.2.1 (2022-06-16)
4
+
5
+ - FIX: Fix regression with unary/infix operators (+/-)
6
+
7
+ ## 1.2.0 (2022-06-16)
8
+
9
+ - CHORE: Enable true random fuzzing
10
+ - FEAT: Add Node::debug() function to help with debugging
11
+ - FEAT: Parse more pattern matching cases
12
+ - FIX: Don't error if ext/natalie_parser/build.log isn't written yet
13
+ - FIX: Fix block association inside call with parentheses
14
+ - FIX: Fix bug negating an already-negative number
15
+ - FIX: Fix bug parsing unary operator as an infix operation
16
+ - FIX: Fix method definition with lonely ensure
17
+ - FIX: Fix parsing endless ranges inside case/when and other odd places
18
+ - FIX: Make sure every token knows if it has preceding whitespace
19
+ - FIX: Parse method calls with constant receiver
20
+
21
+ ## 1.1.1 (2022-06-04)
22
+
23
+ - FIX: Workaround for clang declspec bug
24
+
3
25
  ## 1.1.0 (2022-06-04)
4
26
 
5
27
  - CHORE: Add ccache and compiledb for the ext/natalie_parser directory
6
28
  - CHORE: Add tests for numbered block arg shorthand
7
- - FEAT Parse arg forwarding (...) shorthand
29
+ - FEAT: Parse arg forwarding (...) shorthand
8
30
  - FEAT: Parse complex and rational numbers
9
31
  - FIX: Fix panic when closing word array delimiter is not found
10
32
  - FIX: Fix precedence bug with op assign operators (+= et al)
data/Rakefile CHANGED
@@ -127,7 +127,9 @@ if system('which compiledb 2>&1 >/dev/null')
127
127
  if $compiledb_out.any?
128
128
  File.write('build/build.log', $compiledb_out.join("\n"))
129
129
  sh 'compiledb < build/build.log'
130
- sh 'cd ext/natalie_parser && compiledb < build.log'
130
+ if File.exist?('ext/natalie_parser/build.log')
131
+ sh 'cd ext/natalie_parser && compiledb < build.log'
132
+ end
131
133
  end
132
134
  end
133
135
  else
@@ -177,7 +179,13 @@ file "ext/natalie_parser/natalie_parser.#{so_ext}" => [
177
179
  Rake::FileList['ext/natalie_parser/*.o'].each { |path| rm path }
178
180
  rm_rf 'ext/natalie_parser/natalie_parser.so'
179
181
  sh "cd #{build_dir} && ruby extconf.rb"
180
- sh "CC=#{cc.inspect} CXX=#{cxx.inspect} make -C #{build_dir} -j -e V=1 2>&1 | tee #{log_file}"
182
+ if `#{cxx} -v 2>&1` =~ /clang/
183
+ # workaround for clang bug: https://bugs.ruby-lang.org/issues/18616
184
+ cxx_hacky = "#{cxx} -fdeclspec"
185
+ else
186
+ cxx_hacky = cxx
187
+ end
188
+ sh "CC=#{cc.inspect} CXX=#{cxx_hacky.inspect} make -C #{build_dir} -j -e V=1 2>&1 | tee #{log_file}"
181
189
  end
182
190
 
183
191
  file 'build/fragments.hpp' => ['test/parser_test.rb', 'test/support/extract_parser_test_fragments.rb'] do
@@ -3,6 +3,8 @@
3
3
  #include "natalie_parser/node/array_node.hpp"
4
4
  #include "natalie_parser/node/node.hpp"
5
5
  #include "natalie_parser/node/node_with_args.hpp"
6
+ #include "natalie_parser/node/splat_node.hpp"
7
+ #include "natalie_parser/node/symbol_node.hpp"
6
8
  #include "tm/hashmap.hpp"
7
9
  #include "tm/string.hpp"
8
10
 
@@ -15,14 +17,30 @@ public:
15
17
  ArrayPatternNode(const Token &token)
16
18
  : ArrayNode { token } { }
17
19
 
20
+ ArrayPatternNode(const Token &token, SharedPtr<Node> node)
21
+ : ArrayNode { token } {
22
+ m_nodes.push(node);
23
+ }
24
+
18
25
  virtual Type type() const override { return Type::ArrayPattern; }
19
26
 
20
27
  virtual void transform(Creator *creator) const override {
21
28
  creator->set_type("array_pat");
22
29
  if (!m_nodes.is_empty())
23
30
  creator->append_nil(); // NOTE: I don't know what this nil is for
24
- for (auto node : m_nodes)
25
- creator->append(node);
31
+ for (auto node : m_nodes) {
32
+ if (node->type() == Node::Type::Splat) {
33
+ auto splat_node = node.static_cast_as<SplatNode>();
34
+ auto name = String("*");
35
+ if (splat_node->node()) {
36
+ assert(splat_node->node()->type() == Node::Type::Symbol);
37
+ name.append(splat_node->node().static_cast_as<SymbolNode>()->name().ref());
38
+ }
39
+ creator->append_symbol(name);
40
+ } else {
41
+ creator->append(node);
42
+ }
43
+ }
26
44
  }
27
45
  };
28
46
  }
@@ -31,6 +31,10 @@ public:
31
31
  m_number->prepend_char('-');
32
32
  }
33
33
 
34
+ bool negative() const {
35
+ return m_number->at(0) == '-';
36
+ }
37
+
34
38
  protected:
35
39
  SharedPtr<String> m_number;
36
40
  };
@@ -29,8 +29,11 @@ public:
29
29
  virtual void transform(Creator *creator) const override {
30
30
  creator->set_type("in");
31
31
  creator->append(m_pattern.ref());
32
- for (auto node : m_body->nodes())
33
- creator->append(node);
32
+ if (!m_body->is_empty())
33
+ for (auto node : m_body->nodes())
34
+ creator->append(node);
35
+ else
36
+ creator->append_nil();
34
37
  }
35
38
 
36
39
  protected:
@@ -30,6 +30,10 @@ public:
30
30
  m_number *= -1;
31
31
  }
32
32
 
33
+ bool negative() const {
34
+ return m_number < 0;
35
+ }
36
+
33
37
  protected:
34
38
  long long m_number;
35
39
  };
@@ -30,6 +30,10 @@ public:
30
30
  m_number *= -1;
31
31
  }
32
32
 
33
+ bool negative() const {
34
+ return m_number < 0.0;
35
+ }
36
+
33
37
  protected:
34
38
  double m_number;
35
39
  };
@@ -3,6 +3,7 @@
3
3
  #include "natalie_parser/node/hash_node.hpp"
4
4
  #include "natalie_parser/node/node.hpp"
5
5
  #include "natalie_parser/node/node_with_args.hpp"
6
+ #include "natalie_parser/node/symbol_key_node.hpp"
6
7
  #include "tm/hashmap.hpp"
7
8
  #include "tm/string.hpp"
8
9
 
@@ -22,7 +22,7 @@ public:
22
22
 
23
23
  virtual Type type() const override { return Type::InfixOp; }
24
24
 
25
- virtual bool is_callable() const override { return true; }
25
+ virtual bool is_callable() const override { return false; }
26
26
  virtual bool can_accept_a_block() const override { return false; }
27
27
 
28
28
  const SharedPtr<Node> left() const { return m_left; }
@@ -0,0 +1,43 @@
1
+ #pragma once
2
+
3
+ #include "natalie_parser/node/array_node.hpp"
4
+ #include "natalie_parser/node/node.hpp"
5
+ #include "natalie_parser/node/node_with_args.hpp"
6
+ #include "natalie_parser/node/splat_node.hpp"
7
+ #include "natalie_parser/node/symbol_node.hpp"
8
+ #include "tm/hashmap.hpp"
9
+ #include "tm/string.hpp"
10
+
11
+ namespace NatalieParser {
12
+
13
+ using namespace TM;
14
+
15
+ class KeywordRestPatternNode : public Node {
16
+ public:
17
+ KeywordRestPatternNode(const Token &token)
18
+ : Node { token } { }
19
+
20
+ KeywordRestPatternNode(const Token &token, String name)
21
+ : Node { token }
22
+ , m_name { new String(name) } { }
23
+
24
+ KeywordRestPatternNode(const Token &token, SharedPtr<String> name)
25
+ : Node { token }
26
+ , m_name { name } { }
27
+
28
+ virtual Type type() const override { return Type::KeywordRestPattern; }
29
+
30
+ const SharedPtr<String> name() const { return m_name; }
31
+
32
+ virtual void transform(Creator *creator) const override {
33
+ creator->set_type("kwrest");
34
+ auto name = String("**");
35
+ if (m_name)
36
+ name.append(m_name.ref());
37
+ creator->append_symbol(name);
38
+ }
39
+
40
+ private:
41
+ SharedPtr<String> m_name;
42
+ };
43
+ }
@@ -56,6 +56,7 @@ public:
56
56
  InterpolatedSymbol,
57
57
  InterpolatedSymbolKey,
58
58
  KeywordArg,
59
+ KeywordRestPattern,
59
60
  KeywordSplat,
60
61
  LogicalAnd,
61
62
  LogicalOr,
@@ -150,6 +151,8 @@ public:
150
151
  return type() != Type::Invalid;
151
152
  }
152
153
 
154
+ void debug();
155
+
153
156
  protected:
154
157
  static inline SharedPtr<Node> s_invalid {};
155
158
  Token m_token {};
@@ -20,7 +20,7 @@ public:
20
20
 
21
21
  virtual Type type() const override { return Type::UnaryOp; }
22
22
 
23
- virtual bool is_callable() const override { return true; }
23
+ virtual bool is_callable() const override { return false; }
24
24
  virtual bool can_accept_a_block() const override { return false; }
25
25
 
26
26
  const SharedPtr<String> op() const { return m_op; }
@@ -45,6 +45,7 @@
45
45
  #include "natalie_parser/node/interpolated_symbol_node.hpp"
46
46
  #include "natalie_parser/node/iter_node.hpp"
47
47
  #include "natalie_parser/node/keyword_arg_node.hpp"
48
+ #include "natalie_parser/node/keyword_rest_pattern_node.hpp"
48
49
  #include "natalie_parser/node/keyword_splat_node.hpp"
49
50
  #include "natalie_parser/node/logical_and_node.hpp"
50
51
  #include "natalie_parser/node/logical_or_node.hpp"
@@ -82,6 +82,8 @@ private:
82
82
  SharedPtr<Node> parse_class_or_module_name(LocalsHashmap &);
83
83
  SharedPtr<Node> parse_case(LocalsHashmap &);
84
84
  SharedPtr<Node> parse_case_in_pattern(LocalsHashmap &);
85
+ SharedPtr<Node> parse_case_in_pattern_alternation(LocalsHashmap &);
86
+ SharedPtr<Node> parse_case_in_pattern_hash_symbol_key(LocalsHashmap &);
85
87
  SharedPtr<Node> parse_case_in_patterns(LocalsHashmap &);
86
88
  void parse_comma_separated_expressions(ArrayNode &, LocalsHashmap &);
87
89
  SharedPtr<Node> parse_constant(LocalsHashmap &);
@@ -197,6 +199,7 @@ private:
197
199
 
198
200
  SharedPtr<NodeWithArgs> to_node_with_args(SharedPtr<Node> node);
199
201
 
202
+ Token &previous_token() const;
200
203
  Token &current_token() const;
201
204
  Token &peek_token() const;
202
205
 
@@ -158,58 +158,64 @@ public:
158
158
 
159
159
  Token() { }
160
160
 
161
- Token(Type type, SharedPtr<String> file, size_t line, size_t column)
161
+ Token(Type type, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
162
162
  : m_type { type }
163
163
  , m_file { file }
164
164
  , m_line { line }
165
- , m_column { column } {
165
+ , m_column { column }
166
+ , m_whitespace_precedes { whitespace_precedes } {
166
167
  assert(file);
167
168
  }
168
169
 
169
- Token(Type type, const char *literal, SharedPtr<String> file, size_t line, size_t column)
170
+ Token(Type type, const char *literal, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
170
171
  : m_type { type }
171
172
  , m_literal { new String(literal) }
172
173
  , m_file { file }
173
174
  , m_line { line }
174
- , m_column { column } {
175
+ , m_column { column }
176
+ , m_whitespace_precedes { whitespace_precedes } {
175
177
  assert(literal);
176
178
  assert(file);
177
179
  }
178
180
 
179
- Token(Type type, SharedPtr<String> literal, SharedPtr<String> file, size_t line, size_t column)
181
+ Token(Type type, SharedPtr<String> literal, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
180
182
  : m_type { type }
181
183
  , m_literal { literal }
182
184
  , m_file { file }
183
185
  , m_line { line }
184
- , m_column { column } {
186
+ , m_column { column }
187
+ , m_whitespace_precedes { whitespace_precedes } {
185
188
  assert(literal);
186
189
  assert(file);
187
190
  }
188
191
 
189
- Token(Type type, char literal, SharedPtr<String> file, size_t line, size_t column)
192
+ Token(Type type, char literal, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
190
193
  : m_type { type }
191
194
  , m_literal { new String(literal) }
192
195
  , m_file { file }
193
196
  , m_line { line }
194
- , m_column { column } {
197
+ , m_column { column }
198
+ , m_whitespace_precedes { whitespace_precedes } {
195
199
  assert(file);
196
200
  }
197
201
 
198
- Token(Type type, long long fixnum, SharedPtr<String> file, size_t line, size_t column)
202
+ Token(Type type, long long fixnum, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
199
203
  : m_type { type }
200
204
  , m_fixnum { fixnum }
201
205
  , m_file { file }
202
206
  , m_line { line }
203
- , m_column { column } {
207
+ , m_column { column }
208
+ , m_whitespace_precedes { whitespace_precedes } {
204
209
  assert(file);
205
210
  }
206
211
 
207
- Token(Type type, double dbl, SharedPtr<String> file, size_t line, size_t column)
212
+ Token(Type type, double dbl, SharedPtr<String> file, size_t line, size_t column, bool whitespace_precedes)
208
213
  : m_type { type }
209
214
  , m_double { dbl }
210
215
  , m_file { file }
211
216
  , m_line { line }
212
- , m_column { column } {
217
+ , m_column { column }
218
+ , m_whitespace_precedes { whitespace_precedes } {
213
219
  assert(file);
214
220
  }
215
221
 
@@ -636,6 +642,7 @@ public:
636
642
  bool is_elsif_keyword() const { return m_type == Type::ElsifKeyword; }
637
643
  bool is_end_keyword() const { return m_type == Type::EndKeyword; }
638
644
  bool is_end_of_expression() const { return m_type == Type::EndKeyword || m_type == Type::RCurlyBrace || m_type == Type::Newline || m_type == Type::Semicolon || m_type == Type::Eof || is_expression_modifier(); }
645
+ bool is_ensure() const { return m_type == Type::EnsureKeyword; }
639
646
  bool is_eof() const { return m_type == Type::Eof; }
640
647
  bool is_end_of_line() const { return m_type == Type::Newline || m_type == Type::Semicolon; }
641
648
  bool is_equal() const { return m_type == Type::Equal; }
@@ -644,6 +651,7 @@ public:
644
651
  bool is_lparen() const { return m_type == Type::LParen; }
645
652
  bool is_newline() const { return m_type == Type::Newline; }
646
653
  bool is_rbracket() const { return m_type == Type::RBracket; }
654
+ bool is_rescue() const { return m_type == Type::RescueKeyword; }
647
655
  bool is_rparen() const { return m_type == Type::RParen; }
648
656
  bool is_semicolon() const { return m_type == Type::Semicolon; }
649
657
  bool is_splat() const { return m_type == Type::Star || m_type == Type::StarStar; }
@@ -662,6 +670,7 @@ public:
662
670
  case Token::Type::RParen:
663
671
  case Token::Type::SafeNavigation:
664
672
  case Token::Type::TernaryColon:
673
+ case Token::Type::ThenKeyword:
665
674
  return true;
666
675
  default:
667
676
  return false;
@@ -680,7 +689,6 @@ public:
680
689
  case Token::Type::Comparison:
681
690
  case Token::Type::ConstantResolution:
682
691
  case Token::Type::Dot:
683
- case Token::Type::DotDot:
684
692
  case Token::Type::Equal:
685
693
  case Token::Type::EqualEqual:
686
694
  case Token::Type::EqualEqualEqual:
@@ -763,6 +771,7 @@ public:
763
771
  case Token::Type::LBracketRBracket:
764
772
  case Token::Type::LINEKeyword:
765
773
  case Token::Type::LParen:
774
+ case Token::Type::Minus:
766
775
  case Token::Type::NilKeyword:
767
776
  case Token::Type::Not:
768
777
  case Token::Type::NotKeyword:
@@ -770,6 +779,7 @@ public:
770
779
  case Token::Type::PercentLowerW:
771
780
  case Token::Type::PercentUpperI:
772
781
  case Token::Type::PercentUpperW:
782
+ case Token::Type::Plus:
773
783
  case Token::Type::SelfKeyword:
774
784
  case Token::Type::Star:
775
785
  case Token::Type::String:
@@ -824,6 +834,25 @@ public:
824
834
  }
825
835
  }
826
836
 
837
+ bool can_be_range_arg_token() const {
838
+ if (is_closing_token())
839
+ return false;
840
+ if (is_semicolon() || is_eof())
841
+ return false;
842
+ switch (m_type) {
843
+ case Type::ElseKeyword:
844
+ case Type::ElsifKeyword:
845
+ case Type::EndKeyword:
846
+ case Type::InKeyword:
847
+ case Type::ThenKeyword:
848
+ case Type::WhenKeyword:
849
+ // TODO: likely many more cases!
850
+ return false;
851
+ default:
852
+ return true;
853
+ }
854
+ }
855
+
827
856
  void set_literal(const char *literal) { m_literal = new String(literal); }
828
857
  void set_literal(SharedPtr<String> literal) { m_literal = literal; }
829
858
  void set_literal(String literal) { m_literal = new String(literal); }
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  class NatalieParser
4
- VERSION = '1.1.0'
4
+ VERSION = '1.2.1'
5
5
  end
@@ -14,7 +14,7 @@ Token InterpolatedStringLexer::build_next_token() {
14
14
  case State::EndToken:
15
15
  return finish();
16
16
  case State::Done:
17
- return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column };
17
+ return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
18
18
  }
19
19
  TM_UNREACHABLE();
20
20
  }
@@ -26,13 +26,13 @@ Token InterpolatedStringLexer::consume_string() {
26
26
  advance(); // backslash
27
27
  auto result = consume_escaped_byte(*buf);
28
28
  if (!result.first)
29
- return Token { result.second, current_char(), m_file, m_cursor_line, m_cursor_column };
29
+ return Token { result.second, current_char(), m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
30
30
  } else if (c == '#' && peek() == '{') {
31
31
  if (buf->is_empty()) {
32
32
  advance(2);
33
33
  return start_evaluation();
34
34
  }
35
- auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
35
+ auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
36
36
  advance(2);
37
37
  m_state = State::EvaluateBegin;
38
38
  return token;
@@ -49,7 +49,7 @@ Token InterpolatedStringLexer::consume_string() {
49
49
  return finish();
50
50
  } else {
51
51
  m_state = State::EndToken;
52
- return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
52
+ return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
53
53
  }
54
54
  } else {
55
55
  buf->append_char(c);
@@ -62,27 +62,27 @@ Token InterpolatedStringLexer::consume_string() {
62
62
  if (m_stop_char == 0) {
63
63
  advance();
64
64
  m_state = State::EndToken;
65
- return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
65
+ return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
66
66
  }
67
67
 
68
- return Token { Token::Type::UnterminatedString, buf, m_file, m_token_line, m_token_column };
68
+ return Token { Token::Type::UnterminatedString, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
69
69
  }
70
70
 
71
71
  Token InterpolatedStringLexer::start_evaluation() {
72
72
  m_nested_lexer = new Lexer { *this, '{', '}' };
73
73
  m_state = State::EvaluateEnd;
74
- return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column };
74
+ return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column, m_whitespace_precedes };
75
75
  }
76
76
 
77
77
  Token InterpolatedStringLexer::stop_evaluation() {
78
78
  advance(); // }
79
79
  m_state = State::InProgress;
80
- return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column };
80
+ return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column, m_whitespace_precedes };
81
81
  }
82
82
 
83
83
  Token InterpolatedStringLexer::finish() {
84
84
  m_state = State::Done;
85
- return Token { m_end_type, m_file, m_cursor_line, m_cursor_column };
85
+ return Token { m_end_type, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
86
86
  }
87
87
 
88
88
  };
@@ -11,7 +11,7 @@ Token RegexpLexer::build_next_token() {
11
11
  m_nested_lexer = new Lexer { *this };
12
12
  m_nested_lexer->set_stop_char('}');
13
13
  m_state = State::EvaluateEnd;
14
- return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column };
14
+ return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column, m_whitespace_precedes };
15
15
  case State::EvaluateEnd:
16
16
  advance(); // }
17
17
  if (current_char() == m_stop_char) {
@@ -21,16 +21,16 @@ Token RegexpLexer::build_next_token() {
21
21
  } else {
22
22
  m_state = State::InProgress;
23
23
  }
24
- return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column };
24
+ return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column, m_whitespace_precedes };
25
25
  case State::EndToken: {
26
26
  m_state = State::Done;
27
- auto token = Token { Token::Type::InterpolatedRegexpEnd, m_file, m_cursor_line, m_cursor_column };
27
+ auto token = Token { Token::Type::InterpolatedRegexpEnd, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
28
28
  if (m_options && !m_options->is_empty())
29
29
  token.set_literal(m_options);
30
30
  return token;
31
31
  }
32
32
  case State::Done:
33
- return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column };
33
+ return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
34
34
  }
35
35
  TM_UNREACHABLE();
36
36
  }
@@ -55,7 +55,7 @@ Token RegexpLexer::consume_regexp() {
55
55
  }
56
56
  advance();
57
57
  } else if (c == '#' && peek() == '{') {
58
- auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
58
+ auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
59
59
  buf = new String;
60
60
  advance(2);
61
61
  m_state = State::EvaluateBegin;
@@ -72,14 +72,14 @@ Token RegexpLexer::consume_regexp() {
72
72
  } else {
73
73
  m_options = consume_options();
74
74
  m_state = State::EndToken;
75
- return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
75
+ return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
76
76
  }
77
77
  } else {
78
78
  buf->append_char(c);
79
79
  advance();
80
80
  }
81
81
  }
82
- return Token { Token::Type::UnterminatedRegexp, buf, m_file, m_token_line, m_token_column };
82
+ return Token { Token::Type::UnterminatedRegexp, buf, m_file, m_token_line, m_token_column, m_whitespace_precedes };
83
83
  }
84
84
 
85
85
  String *RegexpLexer::consume_options() {
@@ -11,7 +11,7 @@ Token WordArrayLexer::build_next_token() {
11
11
  return consume_array();
12
12
  case State::DynamicStringBegin:
13
13
  m_state = State::EvaluateBegin;
14
- return Token { Token::Type::String, m_buffer, m_file, m_token_line, m_token_column };
14
+ return Token { Token::Type::String, m_buffer, m_file, m_token_line, m_token_column, m_whitespace_precedes };
15
15
  case State::DynamicStringEnd:
16
16
  if (current_char() == m_stop_char) {
17
17
  advance();
@@ -19,18 +19,18 @@ Token WordArrayLexer::build_next_token() {
19
19
  } else {
20
20
  m_state = State::InProgress;
21
21
  }
22
- return Token { Token::Type::InterpolatedStringEnd, m_file, m_token_line, m_token_column };
22
+ return Token { Token::Type::InterpolatedStringEnd, m_file, m_token_line, m_token_column, m_whitespace_precedes };
23
23
  case State::EvaluateBegin:
24
24
  return start_evaluation();
25
25
  case State::EvaluateEnd:
26
26
  advance(); // }
27
27
  m_state = State::DynamicStringInProgress;
28
- return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column };
28
+ return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column, m_whitespace_precedes };
29
29
  case State::EndToken:
30
30
  m_state = State::Done;
31
- return Token { Token::Type::RBracket, m_file, m_cursor_line, m_cursor_column };
31
+ return Token { Token::Type::RBracket, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
32
32
  case State::Done:
33
- return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column };
33
+ return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
34
34
  }
35
35
  TM_UNREACHABLE();
36
36
  }
@@ -70,7 +70,7 @@ Token WordArrayLexer::consume_array() {
70
70
  return dynamic_string_finish();
71
71
  }
72
72
  if (!m_buffer->is_empty()) {
73
- auto token = Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column };
73
+ auto token = Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
74
74
  advance();
75
75
  return token;
76
76
  }
@@ -97,38 +97,38 @@ Token WordArrayLexer::consume_array() {
97
97
  }
98
98
  }
99
99
 
100
- return Token { Token::Type::UnterminatedWordArray, m_buffer, m_file, m_token_line, m_token_column };
100
+ return Token { Token::Type::UnterminatedWordArray, m_buffer, m_file, m_token_line, m_token_column, m_whitespace_precedes };
101
101
  }
102
102
 
103
103
  Token WordArrayLexer::in_progress_start_dynamic_string() {
104
104
  advance(2); // #{
105
105
  m_state = State::DynamicStringBegin;
106
- return Token { Token::Type::InterpolatedStringBegin, m_file, m_cursor_line, m_cursor_column };
106
+ return Token { Token::Type::InterpolatedStringBegin, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
107
107
  }
108
108
 
109
109
  Token WordArrayLexer::start_evaluation() {
110
110
  m_nested_lexer = new Lexer { *this, '{', '}' };
111
111
  m_state = State::EvaluateEnd;
112
- return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column };
112
+ return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column, m_whitespace_precedes };
113
113
  }
114
114
 
115
115
  Token WordArrayLexer::dynamic_string_finish() {
116
116
  if (!m_buffer->is_empty()) {
117
117
  m_state = State::DynamicStringEnd;
118
- return Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column };
118
+ return Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
119
119
  }
120
120
  m_state = State::InProgress;
121
- return Token { Token::Type::InterpolatedStringEnd, m_file, m_token_line, m_token_column };
121
+ return Token { Token::Type::InterpolatedStringEnd, m_file, m_token_line, m_token_column, m_whitespace_precedes };
122
122
  }
123
123
 
124
124
  Token WordArrayLexer::in_progress_finish() {
125
125
  advance(); // ) or ] or } or whatever
126
126
  if (!m_buffer->is_empty()) {
127
127
  m_state = State::EndToken;
128
- return Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column };
128
+ return Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
129
129
  }
130
130
  m_state = State::Done;
131
- return Token { Token::Type::RBracket, m_file, m_cursor_line, m_cursor_column };
131
+ return Token { Token::Type::RBracket, m_file, m_cursor_line, m_cursor_column, m_whitespace_precedes };
132
132
  }
133
133
 
134
134
  };