natalie_parser 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '048e048ede42652f8a81298a3310313f8bf9929ba5cd952a89d3269d8333b363'
4
- data.tar.gz: 5c2388c8125b1444c454c0ee3a0d4a07f305cc3fd668d8e03dbc4404201a50f7
3
+ metadata.gz: 00b7efc8434fd5d0ade0fe162d09284e3db60417fd0d2beb8a0526df7d52d903
4
+ data.tar.gz: 749a3d63bfdcb53e4273afe4a0601eb353e9337ba66d4c1f321661bb3f83135e
5
5
  SHA512:
6
- metadata.gz: 55932c3dc24ceeeab109a4a980ddf96babf276ced15dffb68d7c013ca5cc246d2551b59acc94f82c1ec5c0073859267ace3a264a3d8b9470465672ea5aebb59d
7
- data.tar.gz: bcbaaad396877bcbf6453e8d55ad3f46dcb68ba09bcfe2f6cf0f1fa6d96966c646f530e3872d77736ef40b5818feed888d69fe33e62beeb95dcfd1c6f5115203
6
+ metadata.gz: e975dbfc660b163bf235c8efb4cc2e0c2e9b7924c4a7ed92382a594fdb4443bcf75f967b09fc0fe8af1eafaa4482793c04c07bc5605128fddfe3a8be24174b73
7
+ data.tar.gz: 121523ba176eec494681009e3863f80f09691234bcc294c26257b5e86544890787c5a40adc4665c9a2c59de37667596957745bc88293b4d8bfc96ccec26e20b0
data/CHANGELOG.md CHANGED
@@ -1,5 +1,14 @@
1
1
  # Changelog
2
2
 
3
+ ## 1.1.0 (2022-06-04)
4
+
5
+ - CHORE: Add ccache and compiledb for the ext/natalie_parser directory
6
+ - CHORE: Add tests for numbered block arg shorthand
7
+ - FEAT Parse arg forwarding (...) shorthand
8
+ - FEAT: Parse complex and rational numbers
9
+ - FIX: Fix panic when closing word array delimiter is not found
10
+ - FIX: Fix precedence bug with op assign operators (+= et al)
11
+
3
12
  ## 1.0.0 (2022-06-03)
4
13
 
5
14
  ### Summary
data/README.md CHANGED
@@ -25,14 +25,21 @@ production applications.**
25
25
  - [ ] Support different source encodings
26
26
  - [ ] Support more of the Ruby 3.0 syntax
27
27
  - [x] "Endless" method definition (`def foo = bar`)
28
- - [ ] Argument forwarding (`...`)
29
- - [ ] Pattern matching
30
- - [ ] Numbered block parameters (`_1`, `_2`, etc.)
28
+ - [x] Argument forwarding (`...`)
29
+ - [x] Numbered block parameters (`_1`, `_2`, etc.)
30
+ - [x] Rational and Complex literals (`1r` and `2i`)
31
31
  - [ ] Non-ASCII identifiers
32
- - [ ] Rational and Complex literals (`1r` and `2i`)
32
+ - [ ] Pattern matching
33
33
 
34
34
  ## Development
35
35
 
36
+ You'll need:
37
+
38
+ - gcc or clang
39
+ - ruby-dev (dev headers)
40
+ - ccache (optional)
41
+ - compiledb (optional)
42
+
36
43
  ```sh
37
44
  rake
38
45
  ruby -I lib:ext -r natalie_parser -e "p NatalieParser.parse('1 + 2')"
data/Rakefile CHANGED
@@ -127,6 +127,7 @@ if system('which compiledb 2>&1 >/dev/null')
127
127
  if $compiledb_out.any?
128
128
  File.write('build/build.log', $compiledb_out.join("\n"))
129
129
  sh 'compiledb < build/build.log'
130
+ sh 'cd ext/natalie_parser && compiledb < build.log'
130
131
  end
131
132
  end
132
133
  else
@@ -172,13 +173,11 @@ file "ext/natalie_parser/natalie_parser.#{so_ext}" => [
172
173
  'ext/natalie_parser/mri_creator.hpp',
173
174
  ] + SOURCES + HEADERS do |t|
174
175
  build_dir = File.expand_path('ext/natalie_parser', __dir__)
176
+ log_file = File.join(build_dir, 'build.log')
175
177
  Rake::FileList['ext/natalie_parser/*.o'].each { |path| rm path }
176
178
  rm_rf 'ext/natalie_parser/natalie_parser.so'
177
- sh <<-SH
178
- cd #{build_dir} && \
179
- ruby extconf.rb && \
180
- make -j
181
- SH
179
+ sh "cd #{build_dir} && ruby extconf.rb"
180
+ sh "CC=#{cc.inspect} CXX=#{cxx.inspect} make -C #{build_dir} -j -e V=1 2>&1 | tee #{log_file}"
182
181
  end
183
182
 
184
183
  file 'build/fragments.hpp' => ['test/parser_test.rb', 'test/support/extract_parser_test_fragments.rb'] do
@@ -61,17 +61,20 @@ public:
61
61
  rb_ary_push(m_sexp, Qfalse);
62
62
  }
63
63
 
64
- virtual void append_float(double number) override {
65
- rb_ary_push(m_sexp, rb_float_new(number));
64
+ virtual void append_bignum(TM::String &number) override {
65
+ auto string_obj = rb_utf8_str_new(number.c_str(), number.length());
66
+ auto num = rb_Integer(string_obj);
67
+ rb_ary_push(m_sexp, num);
66
68
  }
67
69
 
68
- virtual void append_integer(long long number) override {
69
- rb_ary_push(m_sexp, rb_int_new(number));
70
+ virtual void append_fixnum(long long number) override {
71
+ auto num = rb_int_new(number);
72
+ rb_ary_push(m_sexp, num);
70
73
  }
71
74
 
72
- virtual void append_integer(TM::String &number) override {
73
- auto string_obj = rb_utf8_str_new(number.c_str(), number.length());
74
- rb_ary_push(m_sexp, rb_Integer(string_obj));
75
+ virtual void append_float(double number) override {
76
+ auto num = rb_float_new(number);
77
+ rb_ary_push(m_sexp, num);
75
78
  }
76
79
 
77
80
  virtual void append_nil() override {
@@ -109,6 +112,21 @@ public:
109
112
  rb_ary_push(m_sexp, Qtrue);
110
113
  }
111
114
 
115
+ virtual void make_complex_number() override {
116
+ auto num = rb_ary_pop(m_sexp);
117
+ num = rb_Complex(INT2FIX(0), num);
118
+ rb_ary_push(m_sexp, num);
119
+ }
120
+
121
+ virtual void make_rational_number() override {
122
+ auto num = rb_ary_pop(m_sexp);
123
+ if (TYPE(num) == T_FLOAT)
124
+ num = rb_flt_rationalize(num);
125
+ else
126
+ num = rb_Rational(num, INT2FIX(1));
127
+ rb_ary_push(m_sexp, num);
128
+ }
129
+
112
130
  virtual void wrap(const char *type) override {
113
131
  auto inner = m_sexp;
114
132
  reset_sexp();
@@ -42,15 +42,15 @@ public:
42
42
  m_nodes.push("false");
43
43
  }
44
44
 
45
- virtual void append_float(double number) override {
45
+ virtual void append_bignum(TM::String &number) override {
46
46
  m_nodes.push(String(number));
47
47
  }
48
48
 
49
- virtual void append_integer(long long number) override {
49
+ virtual void append_fixnum(long long number) override {
50
50
  m_nodes.push(String(number));
51
51
  }
52
52
 
53
- virtual void append_integer(TM::String &number) override {
53
+ virtual void append_float(double number) override {
54
54
  m_nodes.push(String(number));
55
55
  }
56
56
 
@@ -89,6 +89,16 @@ public:
89
89
  m_nodes.push("true");
90
90
  }
91
91
 
92
+ virtual void make_complex_number() override {
93
+ auto num = m_nodes.pop();
94
+ m_nodes.push(String::format("Complex(0, {})", num));
95
+ }
96
+
97
+ virtual void make_rational_number() override {
98
+ auto num = m_nodes.pop();
99
+ m_nodes.push(String::format("Rational({}, 1)", num));
100
+ }
101
+
92
102
  virtual void wrap(const char *type) override {
93
103
  auto inner = to_string();
94
104
  m_nodes.clear();
@@ -27,9 +27,9 @@ public:
27
27
  virtual void append_array(const TM::SharedPtr<ArrayNode> array) { append_array(*array); }
28
28
  virtual void append_array(const ArrayNode &array) = 0;
29
29
  virtual void append_false() = 0;
30
+ virtual void append_bignum(TM::String &number) = 0;
31
+ virtual void append_fixnum(long long number) = 0;
30
32
  virtual void append_float(double number) = 0;
31
- virtual void append_integer(long long number) = 0;
32
- virtual void append_integer(TM::String &number) = 0;
33
33
  virtual void append_nil() = 0;
34
34
  virtual void append_range(long long first, long long last, bool exclude_end) = 0;
35
35
  virtual void append_regexp(TM::String &pattern, int options) = 0;
@@ -37,6 +37,8 @@ public:
37
37
  virtual void append_string(TM::String &string) = 0;
38
38
  virtual void append_symbol(TM::String &symbol) = 0;
39
39
  virtual void append_true() = 0;
40
+ virtual void make_complex_number() = 0;
41
+ virtual void make_rational_number() = 0;
40
42
  virtual void wrap(const char *type) = 0;
41
43
 
42
44
  virtual ~Creator() { }
@@ -23,7 +23,7 @@ public:
23
23
 
24
24
  virtual void transform(Creator *creator) const override {
25
25
  creator->set_type("lit");
26
- creator->append_integer(*m_number);
26
+ creator->append_bignum(*m_number);
27
27
  }
28
28
 
29
29
  void negate() {
@@ -0,0 +1,49 @@
1
+ #pragma once
2
+
3
+ #include "natalie_parser/node/bignum_node.hpp"
4
+ #include "natalie_parser/node/fixnum_node.hpp"
5
+ #include "natalie_parser/node/float_node.hpp"
6
+ #include "natalie_parser/node/node.hpp"
7
+ #include "natalie_parser/node/rational_node.hpp"
8
+
9
+ namespace NatalieParser {
10
+
11
+ using namespace TM;
12
+
13
+ class ComplexNode : public Node {
14
+ public:
15
+ ComplexNode(const Token &token, SharedPtr<Node> value)
16
+ : Node { token }
17
+ , m_value { value } { }
18
+
19
+ virtual Type type() const override { return Type::Complex; }
20
+
21
+ virtual void transform(Creator *creator) const override {
22
+ creator->set_type("lit");
23
+ transform_number(creator);
24
+ creator->make_complex_number();
25
+ }
26
+
27
+ void transform_number(Creator *creator) const {
28
+ switch (m_value->type()) {
29
+ case Node::Type::Bignum:
30
+ creator->append_bignum(m_value.static_cast_as<BignumNode>()->number().ref());
31
+ break;
32
+ case Node::Type::Fixnum:
33
+ creator->append_fixnum(m_value.static_cast_as<FixnumNode>()->number());
34
+ break;
35
+ case Node::Type::Float:
36
+ creator->append_float(m_value.static_cast_as<FloatNode>()->number());
37
+ break;
38
+ case Node::Type::Rational:
39
+ m_value.static_cast_as<RationalNode>()->transform_number(creator);
40
+ break;
41
+ default:
42
+ TM_UNREACHABLE();
43
+ }
44
+ }
45
+
46
+ protected:
47
+ SharedPtr<Node> m_value;
48
+ };
49
+ }
@@ -23,7 +23,7 @@ public:
23
23
 
24
24
  virtual void transform(Creator *creator) const override {
25
25
  creator->set_type("lit");
26
- creator->append_integer(m_number);
26
+ creator->append_fixnum(m_number);
27
27
  }
28
28
 
29
29
  void negate() {
@@ -0,0 +1,26 @@
1
+ #pragma once
2
+
3
+ #include "natalie_parser/node/node.hpp"
4
+ #include "tm/hashmap.hpp"
5
+
6
+ namespace NatalieParser {
7
+
8
+ using namespace TM;
9
+
10
+ class ForwardArgsNode : public Node {
11
+ public:
12
+ ForwardArgsNode(const Token &token)
13
+ : Node { token } { }
14
+
15
+ virtual Type type() const override { return Type::ForwardArgs; }
16
+
17
+ void add_to_locals(TM::Hashmap<TM::String> &locals) {
18
+ locals.set("...");
19
+ }
20
+
21
+ virtual void transform(Creator *creator) const override {
22
+ creator->set_type("forward_args");
23
+ }
24
+ };
25
+
26
+ }
@@ -32,7 +32,7 @@ public:
32
32
  if (m_has_args)
33
33
  append_method_or_block_args(creator);
34
34
  else
35
- creator->append_integer(0);
35
+ creator->append_fixnum(0);
36
36
  if (!m_body->is_empty())
37
37
  creator->append(m_body->without_unnecessary_nesting());
38
38
  }
@@ -33,6 +33,7 @@ public:
33
33
  Class,
34
34
  Colon2,
35
35
  Colon3,
36
+ Complex,
36
37
  Constant,
37
38
  Def,
38
39
  Defined,
@@ -42,6 +43,7 @@ public:
42
43
  False,
43
44
  Fixnum,
44
45
  Float,
46
+ ForwardArgs,
45
47
  Hash,
46
48
  HashPattern,
47
49
  Identifier,
@@ -73,6 +75,7 @@ public:
73
75
  OpAssignOr,
74
76
  Pin,
75
77
  Range,
78
+ Rational,
76
79
  Redo,
77
80
  Regexp,
78
81
  Retry,
@@ -122,7 +125,7 @@ public:
122
125
 
123
126
  virtual void transform(Creator *creator) const {
124
127
  creator->set_type("NOT_YET_IMPLEMENTED");
125
- creator->append_integer((int)type());
128
+ creator->append_fixnum((int)type());
126
129
  }
127
130
 
128
131
  SharedPtr<String> file() const { return m_token.file(); }
@@ -18,7 +18,7 @@ public:
18
18
 
19
19
  virtual void transform(Creator *creator) const override {
20
20
  creator->set_type("nth_ref");
21
- creator->append_integer(m_num);
21
+ creator->append_fixnum(m_num);
22
22
  }
23
23
 
24
24
  protected:
@@ -0,0 +1,45 @@
1
+ #pragma once
2
+
3
+ #include "natalie_parser/node/bignum_node.hpp"
4
+ #include "natalie_parser/node/fixnum_node.hpp"
5
+ #include "natalie_parser/node/float_node.hpp"
6
+ #include "natalie_parser/node/node.hpp"
7
+
8
+ namespace NatalieParser {
9
+
10
+ using namespace TM;
11
+
12
+ class RationalNode : public Node {
13
+ public:
14
+ RationalNode(const Token &token, SharedPtr<Node> value)
15
+ : Node { token }
16
+ , m_value { value } { }
17
+
18
+ virtual Type type() const override { return Type::Rational; }
19
+
20
+ virtual void transform(Creator *creator) const override {
21
+ creator->set_type("lit");
22
+ transform_number(creator);
23
+ creator->make_rational_number();
24
+ }
25
+
26
+ void transform_number(Creator *creator) const {
27
+ switch (m_value->type()) {
28
+ case Node::Type::Bignum:
29
+ creator->append_bignum(m_value.static_cast_as<BignumNode>()->number().ref());
30
+ break;
31
+ case Node::Type::Fixnum:
32
+ creator->append_fixnum(m_value.static_cast_as<FixnumNode>()->number());
33
+ break;
34
+ case Node::Type::Float:
35
+ creator->append_float(m_value.static_cast_as<FloatNode>()->number());
36
+ break;
37
+ default:
38
+ TM_UNREACHABLE();
39
+ }
40
+ }
41
+
42
+ protected:
43
+ SharedPtr<Node> m_value;
44
+ };
45
+ }
@@ -21,6 +21,7 @@
21
21
  #include "natalie_parser/node/class_node.hpp"
22
22
  #include "natalie_parser/node/colon2_node.hpp"
23
23
  #include "natalie_parser/node/colon3_node.hpp"
24
+ #include "natalie_parser/node/complex_node.hpp"
24
25
  #include "natalie_parser/node/constant_node.hpp"
25
26
  #include "natalie_parser/node/def_node.hpp"
26
27
  #include "natalie_parser/node/defined_node.hpp"
@@ -30,6 +31,7 @@
30
31
  #include "natalie_parser/node/false_node.hpp"
31
32
  #include "natalie_parser/node/fixnum_node.hpp"
32
33
  #include "natalie_parser/node/float_node.hpp"
34
+ #include "natalie_parser/node/forward_args_node.hpp"
33
35
  #include "natalie_parser/node/hash_node.hpp"
34
36
  #include "natalie_parser/node/hash_pattern_node.hpp"
35
37
  #include "natalie_parser/node/identifier_node.hpp"
@@ -63,6 +65,7 @@
63
65
  #include "natalie_parser/node/op_assign_or_node.hpp"
64
66
  #include "natalie_parser/node/pin_node.hpp"
65
67
  #include "natalie_parser/node/range_node.hpp"
68
+ #include "natalie_parser/node/rational_node.hpp"
66
69
  #include "natalie_parser/node/redo_node.hpp"
67
70
  #include "natalie_parser/node/regexp_node.hpp"
68
71
  #include "natalie_parser/node/retry_node.hpp"
@@ -87,11 +87,19 @@ private:
87
87
  SharedPtr<Node> parse_constant(LocalsHashmap &);
88
88
  SharedPtr<Node> parse_def(LocalsHashmap &);
89
89
  SharedPtr<Node> parse_defined(LocalsHashmap &);
90
+
90
91
  void parse_def_args(Vector<SharedPtr<Node>> &, LocalsHashmap &);
91
- void parse_def_single_arg(Vector<SharedPtr<Node>> &, LocalsHashmap &);
92
+ enum class ArgsContext {
93
+ Block,
94
+ Method,
95
+ Proc,
96
+ };
97
+ void parse_def_single_arg(Vector<SharedPtr<Node>> &, LocalsHashmap &, ArgsContext);
98
+
92
99
  SharedPtr<Node> parse_encoding(LocalsHashmap &);
93
100
  SharedPtr<Node> parse_end_block(LocalsHashmap &);
94
101
  SharedPtr<Node> parse_file_constant(LocalsHashmap &);
102
+ SharedPtr<Node> parse_forward_args(LocalsHashmap &);
95
103
  SharedPtr<Node> parse_group(LocalsHashmap &);
96
104
  SharedPtr<Node> parse_hash(LocalsHashmap &);
97
105
  SharedPtr<Node> parse_hash_inner(LocalsHashmap &, Precedence, Token::Type, SharedPtr<Node> = {});
@@ -128,6 +136,7 @@ private:
128
136
  SharedPtr<Node> parse_symbol_key(LocalsHashmap &);
129
137
  SharedPtr<Node> parse_statement_keyword(LocalsHashmap &);
130
138
  SharedPtr<Node> parse_top_level_constant(LocalsHashmap &);
139
+ SharedPtr<Node> parse_triple_dot(LocalsHashmap &);
131
140
  SharedPtr<Node> parse_unary_operator(LocalsHashmap &);
132
141
  SharedPtr<Node> parse_undef(LocalsHashmap &);
133
142
  SharedPtr<Node> parse_unless(LocalsHashmap &);
@@ -195,6 +204,7 @@ private:
195
204
  void skip_newlines();
196
205
 
197
206
  void expect(Token::Type, const char *);
207
+ [[noreturn]] void throw_error(const Token &, const char *);
198
208
  [[noreturn]] void throw_unexpected(const Token &, const char *, const char * = nullptr);
199
209
  [[noreturn]] void throw_unexpected(const char *);
200
210
  [[noreturn]] void throw_unterminated_thing(Token, Token = {});
@@ -36,6 +36,7 @@ public:
36
36
  Comma,
37
37
  Comment,
38
38
  Comparison,
39
+ Complex,
39
40
  Constant,
40
41
  ConstantResolution,
41
42
  DefinedKeyword,
@@ -115,6 +116,8 @@ public:
115
116
  PipePipeEqual,
116
117
  Plus,
117
118
  PlusEqual,
119
+ Rational,
120
+ RationalComplex,
118
121
  RCurlyBrace,
119
122
  RBracket,
120
123
  RedoKeyword,
@@ -286,6 +289,8 @@ public:
286
289
  return "comment";
287
290
  case Type::Comparison:
288
291
  return "<=>";
292
+ case Type::Complex:
293
+ return "complex";
289
294
  case Type::ConstantResolution:
290
295
  return "::";
291
296
  case Type::Constant:
@@ -446,6 +451,10 @@ public:
446
451
  return "+=";
447
452
  case Type::Plus:
448
453
  return "+";
454
+ case Type::Rational:
455
+ return "rational";
456
+ case Type::RationalComplex:
457
+ return "rational_complex";
449
458
  case Type::RCurlyBrace:
450
459
  return "}";
451
460
  case Type::RBracket:
@@ -804,6 +813,17 @@ public:
804
813
  }
805
814
  }
806
815
 
816
+ bool can_be_complex_or_rational() const {
817
+ switch (m_type) {
818
+ case Type::Bignum:
819
+ case Type::Fixnum:
820
+ case Type::Float:
821
+ return true;
822
+ default:
823
+ return false;
824
+ }
825
+ }
826
+
807
827
  void set_literal(const char *literal) { m_literal = new String(literal); }
808
828
  void set_literal(SharedPtr<String> literal) { m_literal = literal; }
809
829
  void set_literal(String literal) { m_literal = new String(literal); }
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  class NatalieParser
4
- VERSION = '1.0.0'
4
+ VERSION = '1.1.0'
5
5
  end
data/src/lexer.cpp CHANGED
@@ -842,6 +842,23 @@ Token Lexer::build_next_token() {
842
842
  auto token = consume_numeric();
843
843
  return token;
844
844
  }
845
+ case 'i':
846
+ if (m_last_token.can_be_complex_or_rational() && !isalnum(peek())) {
847
+ advance();
848
+ return Token { Token::Type::Complex, m_file, m_token_line, m_token_column };
849
+ }
850
+ break;
851
+ case 'r':
852
+ if (m_last_token.can_be_complex_or_rational()) {
853
+ if (peek() == 'i') {
854
+ advance(2);
855
+ return Token { Token::Type::RationalComplex, m_file, m_token_line, m_token_column };
856
+ } else if (!isalnum(peek())) {
857
+ advance();
858
+ return Token { Token::Type::Rational, m_file, m_token_line, m_token_column };
859
+ }
860
+ }
861
+ break;
845
862
  };
846
863
 
847
864
  Token keyword_token;
@@ -1322,7 +1339,25 @@ Token Lexer::consume_heredoc() {
1322
1339
 
1323
1340
  Token Lexer::consume_numeric() {
1324
1341
  SharedPtr<String> chars = new String;
1342
+
1343
+ auto consume_decimal_digits_and_build_token = [&]() {
1344
+ char c = current_char();
1345
+ do {
1346
+ chars->append_char(c);
1347
+ c = next();
1348
+ if (c == '_')
1349
+ c = next();
1350
+ } while (isdigit(c));
1351
+ if ((c == '.' && isdigit(peek())) || (c == 'e' || c == 'E'))
1352
+ return consume_numeric_as_float(chars);
1353
+ else
1354
+ return chars_to_fixnum_or_bignum_token(chars, 10, 0);
1355
+ };
1356
+
1357
+ Token token;
1358
+
1325
1359
  if (current_char() == '0') {
1360
+ // special-prefixed literals 0d, 0x, etc.
1326
1361
  switch (peek()) {
1327
1362
  case 'd':
1328
1363
  case 'D': {
@@ -1336,7 +1371,8 @@ Token Lexer::consume_numeric() {
1336
1371
  if (c == '_')
1337
1372
  c = next();
1338
1373
  } while (isdigit(c));
1339
- return chars_to_fixnum_or_bignum_token(chars, 10, 0);
1374
+ token = chars_to_fixnum_or_bignum_token(chars, 10, 0);
1375
+ break;
1340
1376
  }
1341
1377
  case 'o':
1342
1378
  case 'O': {
@@ -1352,7 +1388,8 @@ Token Lexer::consume_numeric() {
1352
1388
  if (c == '_')
1353
1389
  c = next();
1354
1390
  } while (c >= '0' && c <= '7');
1355
- return chars_to_fixnum_or_bignum_token(chars, 8, 2);
1391
+ token = chars_to_fixnum_or_bignum_token(chars, 8, 2);
1392
+ break;
1356
1393
  }
1357
1394
  case 'x':
1358
1395
  case 'X': {
@@ -1368,7 +1405,8 @@ Token Lexer::consume_numeric() {
1368
1405
  if (c == '_')
1369
1406
  c = next();
1370
1407
  } while (isxdigit(c));
1371
- return chars_to_fixnum_or_bignum_token(chars, 16, 2);
1408
+ token = chars_to_fixnum_or_bignum_token(chars, 16, 2);
1409
+ break;
1372
1410
  }
1373
1411
  case 'b':
1374
1412
  case 'B': {
@@ -1384,21 +1422,17 @@ Token Lexer::consume_numeric() {
1384
1422
  if (c == '_')
1385
1423
  c = next();
1386
1424
  } while (c == '0' || c == '1');
1387
- return chars_to_fixnum_or_bignum_token(chars, 2, 2);
1425
+ token = chars_to_fixnum_or_bignum_token(chars, 2, 2);
1426
+ break;
1388
1427
  }
1428
+ default:
1429
+ token = consume_decimal_digits_and_build_token();
1389
1430
  }
1431
+ } else {
1432
+ token = consume_decimal_digits_and_build_token();
1390
1433
  }
1391
- char c = current_char();
1392
- do {
1393
- chars->append_char(c);
1394
- c = next();
1395
- if (c == '_')
1396
- c = next();
1397
- } while (isdigit(c));
1398
- if ((c == '.' && isdigit(peek())) || (c == 'e' || c == 'E'))
1399
- return consume_numeric_as_float(chars);
1400
- else
1401
- return chars_to_fixnum_or_bignum_token(chars, 10, 0);
1434
+
1435
+ return token;
1402
1436
  }
1403
1437
 
1404
1438
  const long long max_fixnum = std::numeric_limits<long long>::max() / 2; // 63 bits for MRI
@@ -13,7 +13,7 @@ void InterpolatedRegexpNode::transform(Creator *creator) const {
13
13
  creator->append(node);
14
14
  }
15
15
  if (m_options != 0)
16
- creator->append_integer(m_options);
16
+ creator->append_fixnum(m_options);
17
17
  }
18
18
 
19
19
  }
@@ -17,6 +17,7 @@ void NodeWithArgs::append_method_or_block_args(Creator *creator) const {
17
17
  arg.static_cast_as<ArgNode>()->append_name(c);
18
18
  break;
19
19
  }
20
+ case Node::Type::ForwardArgs:
20
21
  case Node::Type::KeywordArg:
21
22
  case Node::Type::MultipleAssignmentArg:
22
23
  case Node::Type::ShadowArg:
data/src/parser.cpp CHANGED
@@ -16,7 +16,7 @@ enum class Parser::Precedence {
16
16
  INLINE_RESCUE, // foo rescue 2
17
17
  ITER_BLOCK, // do |n| ... end
18
18
  BARE_CALL_ARG, // foo (_), b
19
- OP_ASSIGNMENT, // += -= *= **= /= %= |= &= ^= >>= <<= ||= &&=
19
+ OP_ASSIGNMENT_RHS, // x += (_)
20
20
  TERNARY_TRUE, // _ ? (_) : _
21
21
  CALL_ARG, // foo( (_), b )
22
22
  TERNARY_QUESTION, // (_) ? _ : _
@@ -28,6 +28,7 @@ enum class Parser::Precedence {
28
28
  LOGICAL_NOT, // not
29
29
  EQUALITY, // <=> == === != =~ !~
30
30
  LESS_GREATER, // <= < > >=
31
+ OP_ASSIGNMENT_LHS, // (_) += 1
31
32
  BITWISE_OR, // ^ |
32
33
  BITWISE_AND, // &
33
34
  BITWISE_SHIFT, // << >>
@@ -112,7 +113,7 @@ Parser::Precedence Parser::get_precedence(Token &token, SharedPtr<Node> left) {
112
113
  case Token::Type::SlashEqual:
113
114
  case Token::Type::StarEqual:
114
115
  case Token::Type::StarStarEqual:
115
- return Precedence::OP_ASSIGNMENT;
116
+ return Precedence::OP_ASSIGNMENT_LHS;
116
117
  case Token::Type::Ampersand:
117
118
  return Precedence::BITWISE_AND;
118
119
  case Token::Type::Caret:
@@ -979,28 +980,32 @@ SharedPtr<Node> Parser::parse_defined(LocalsHashmap &locals) {
979
980
  }
980
981
 
981
982
  void Parser::parse_def_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals) {
982
- parse_def_single_arg(args, locals);
983
+ parse_def_single_arg(args, locals, ArgsContext::Method);
983
984
  while (current_token().is_comma()) {
984
985
  advance();
985
- parse_def_single_arg(args, locals);
986
+ parse_def_single_arg(args, locals, ArgsContext::Method);
986
987
  }
987
988
  }
988
989
 
989
- void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals) {
990
+ void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals, ArgsContext context) {
990
991
  auto args_have_any_splat = [&]() { return !args.is_empty() && args.last()->type() == Node::Type::Arg && args.last().static_cast_as<ArgNode>()->splat_or_kwsplat(); };
991
992
  auto args_have_keyword = [&]() { return !args.is_empty() && args.last()->type() == Node::Type::KeywordArg; };
992
993
 
993
994
  auto token = current_token();
995
+
996
+ if (!args.is_empty() && args.last()->type() == Node::Type::ForwardArgs)
997
+ throw_error(token, "anything after arg forwarding (...) shorthand");
998
+
994
999
  switch (token.type()) {
995
1000
  case Token::Type::BareName: {
996
1001
  if (args_have_keyword())
997
- throw_unexpected(token, nullptr, "normal arg after keyword arg");
1002
+ throw_error(token, "normal arg after keyword arg");
998
1003
  SharedPtr<ArgNode> arg = new ArgNode { token, token.literal_string() };
999
1004
  advance();
1000
1005
  arg->add_to_locals(locals);
1001
1006
  if (current_token().is_equal()) {
1002
1007
  if (args_have_any_splat())
1003
- throw_unexpected(token, nullptr, "default value after splat");
1008
+ throw_error(token, "default value after splat");
1004
1009
  advance(); // =
1005
1010
  arg->set_value(parse_expression(Precedence::DEF_ARG, locals));
1006
1011
  }
@@ -1022,7 +1027,7 @@ void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &
1022
1027
  }
1023
1028
  case Token::Type::Star: {
1024
1029
  if (args_have_any_splat())
1025
- throw_unexpected(token, nullptr, "splat after keyword splat");
1030
+ throw_error(token, "splat after keyword splat");
1026
1031
  advance();
1027
1032
  SharedPtr<ArgNode> arg;
1028
1033
  if (current_token().is_bare_name()) {
@@ -1080,6 +1085,15 @@ void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &
1080
1085
  args.push(arg.static_cast_as<Node>());
1081
1086
  return;
1082
1087
  }
1088
+ case Token::Type::DotDotDot: {
1089
+ if (context == ArgsContext::Block)
1090
+ throw_error(token, "arg forwarding (...) shorthand not allowed in block");
1091
+ SharedPtr<ForwardArgsNode> arg = new ForwardArgsNode { token };
1092
+ advance();
1093
+ arg->add_to_locals(locals);
1094
+ args.push(arg.static_cast_as<Node>());
1095
+ return;
1096
+ }
1083
1097
  default:
1084
1098
  throw_unexpected("argument");
1085
1099
  }
@@ -1151,6 +1165,15 @@ SharedPtr<Node> Parser::parse_file_constant(LocalsHashmap &) {
1151
1165
  return new StringNode { token, token.file() };
1152
1166
  }
1153
1167
 
1168
+ SharedPtr<Node> Parser::parse_forward_args(LocalsHashmap &locals) {
1169
+ auto token = current_token();
1170
+ if (!locals.get("..."))
1171
+ throw_error(token, "forwarding args without ... shorthand in method definition");
1172
+ advance(); // ...
1173
+ SharedPtr<ForwardArgsNode> node = new ForwardArgsNode { token };
1174
+ return node.static_cast_as<Node>();
1175
+ }
1176
+
1154
1177
  SharedPtr<Node> Parser::parse_group(LocalsHashmap &locals) {
1155
1178
  auto token = current_token();
1156
1179
  advance(); // (
@@ -1491,19 +1514,40 @@ SharedPtr<Node> Parser::parse_interpolated_symbol(LocalsHashmap &locals) {
1491
1514
 
1492
1515
  SharedPtr<Node> Parser::parse_lit(LocalsHashmap &) {
1493
1516
  auto token = current_token();
1517
+ SharedPtr<Node> node;
1494
1518
  switch (token.type()) {
1495
1519
  case Token::Type::Bignum:
1496
1520
  advance();
1497
- return new BignumNode { token, token.literal_string() };
1521
+ node = new BignumNode { token, token.literal_string() };
1522
+ break;
1498
1523
  case Token::Type::Fixnum:
1499
1524
  advance();
1500
- return new FixnumNode { token, token.get_fixnum() };
1525
+ node = new FixnumNode { token, token.get_fixnum() };
1526
+ break;
1501
1527
  case Token::Type::Float:
1502
1528
  advance();
1503
- return new FloatNode { token, token.get_double() };
1529
+ node = new FloatNode { token, token.get_double() };
1530
+ break;
1504
1531
  default:
1505
1532
  TM_UNREACHABLE();
1506
1533
  }
1534
+ switch (current_token().type()) {
1535
+ case Token::Type::Complex:
1536
+ advance();
1537
+ node = new ComplexNode { token, node };
1538
+ break;
1539
+ case Token::Type::Rational:
1540
+ advance();
1541
+ node = new RationalNode { token, node };
1542
+ break;
1543
+ case Token::Type::RationalComplex:
1544
+ advance();
1545
+ node = new ComplexNode { token, new RationalNode { token, node } };
1546
+ break;
1547
+ default:
1548
+ break;
1549
+ }
1550
+ return node;
1507
1551
  };
1508
1552
 
1509
1553
  SharedPtr<Node> Parser::parse_keyword_splat(LocalsHashmap &locals) {
@@ -1605,10 +1649,10 @@ void Parser::parse_proc_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &local
1605
1649
  parse_shadow_variables_in_args(args, locals);
1606
1650
  return;
1607
1651
  }
1608
- parse_def_single_arg(args, locals);
1652
+ parse_def_single_arg(args, locals, ArgsContext::Proc);
1609
1653
  while (current_token().is_comma()) {
1610
1654
  advance();
1611
- parse_def_single_arg(args, locals);
1655
+ parse_def_single_arg(args, locals, ArgsContext::Proc);
1612
1656
  }
1613
1657
  if (current_token().is_semicolon()) {
1614
1658
  parse_shadow_variables_in_args(args, locals);
@@ -2101,7 +2145,7 @@ void Parser::parse_iter_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &local
2101
2145
  parse_shadow_variables_in_args(args, locals);
2102
2146
  return;
2103
2147
  }
2104
- parse_def_single_arg(args, locals);
2148
+ parse_def_single_arg(args, locals, ArgsContext::Block);
2105
2149
  while (current_token().is_comma()) {
2106
2150
  advance();
2107
2151
  if (current_token().is_block_arg_delimiter()) {
@@ -2109,7 +2153,7 @@ void Parser::parse_iter_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &local
2109
2153
  args.push(new NilNode { current_token() });
2110
2154
  break;
2111
2155
  }
2112
- parse_def_single_arg(args, locals);
2156
+ parse_def_single_arg(args, locals, ArgsContext::Block);
2113
2157
  }
2114
2158
  if (current_token().is_semicolon()) {
2115
2159
  parse_shadow_variables_in_args(args, locals);
@@ -2156,7 +2200,7 @@ SharedPtr<NodeWithArgs> Parser::to_node_with_args(SharedPtr<Node> node) {
2156
2200
  case Node::Type::Yield:
2157
2201
  return node.static_cast_as<NodeWithArgs>();
2158
2202
  default:
2159
- throw_unexpected(current_token(), nullptr, "left-hand-side is not callable");
2203
+ throw_error(current_token(), "left-hand-side is not callable");
2160
2204
  }
2161
2205
  }
2162
2206
 
@@ -2381,7 +2425,7 @@ SharedPtr<Node> Parser::parse_op_attr_assign_expression(SharedPtr<Node> left, Lo
2381
2425
  auto left_call = left.static_cast_as<CallNode>();
2382
2426
  auto token = current_token();
2383
2427
  advance();
2384
- auto value = parse_expression(Precedence::OP_ASSIGNMENT, locals);
2428
+ auto value = parse_expression(Precedence::OP_ASSIGNMENT_RHS, locals);
2385
2429
 
2386
2430
  if (*left_call->message() != "[]") {
2387
2431
  if (token.type() == Token::Type::AmpersandAmpersandEqual) {
@@ -2536,6 +2580,12 @@ SharedPtr<Node> Parser::parse_ternary_expression(SharedPtr<Node> left, LocalsHas
2536
2580
  return new IfNode { token, left, true_expr, false_expr };
2537
2581
  }
2538
2582
 
2583
+ SharedPtr<Node> Parser::parse_triple_dot(LocalsHashmap &locals) {
2584
+ if (peek_token().is_rparen())
2585
+ return parse_forward_args(locals);
2586
+ return parse_beginless_range(locals);
2587
+ }
2588
+
2539
2589
  SharedPtr<Node> Parser::parse_unless(LocalsHashmap &locals) {
2540
2590
  auto token = current_token();
2541
2591
  advance();
@@ -2603,7 +2653,7 @@ Parser::parse_null_fn Parser::null_denotation(Token::Type type) {
2603
2653
  return &Parser::parse_defined;
2604
2654
  case Type::DotDot:
2605
2655
  case Type::DotDotDot:
2606
- return &Parser::parse_beginless_range;
2656
+ return &Parser::parse_triple_dot;
2607
2657
  case Type::ENCODINGKeyword:
2608
2658
  return &Parser::parse_encoding;
2609
2659
  case Type::ENDKeyword:
@@ -2826,6 +2876,10 @@ void Parser::expect(Token::Type type, const char *expected) {
2826
2876
  throw_unexpected(expected);
2827
2877
  }
2828
2878
 
2879
+ void Parser::throw_error(const Token &token, const char *error) {
2880
+ throw_unexpected(token, nullptr, error);
2881
+ }
2882
+
2829
2883
  void Parser::throw_unexpected(const Token &token, const char *expected, const char *error) {
2830
2884
  auto file = token.file() ? String(*token.file()) : String("(unknown)");
2831
2885
  auto line = token.line() + 1;
@@ -2881,20 +2935,22 @@ void Parser::throw_unterminated_thing(Token token, Token start_token) {
2881
2935
  auto indent = String { start_token.column(), ' ' };
2882
2936
  String expected;
2883
2937
  const char *lit = start_token.literal();
2884
- assert(lit);
2885
- if (strcmp(lit, "(") == 0)
2886
- expected = "')'";
2887
- else if (strcmp(lit, "[") == 0)
2888
- expected = "']'";
2889
- else if (strcmp(lit, "{") == 0)
2890
- expected = "'}'";
2891
- else if (strcmp(lit, "<") == 0)
2892
- expected = "'>'";
2893
- else if (strcmp(lit, "'") == 0)
2894
- expected = "\"'\"";
2895
- else
2896
- expected = String::format("'{}'", lit);
2897
- assert(!expected.is_empty());
2938
+ if (lit) {
2939
+ if (strcmp(lit, "(") == 0)
2940
+ expected = "')'";
2941
+ else if (strcmp(lit, "[") == 0)
2942
+ expected = "']'";
2943
+ else if (strcmp(lit, "{") == 0)
2944
+ expected = "'}'";
2945
+ else if (strcmp(lit, "<") == 0)
2946
+ expected = "'>'";
2947
+ else if (strcmp(lit, "'") == 0)
2948
+ expected = "\"'\"";
2949
+ else
2950
+ expected = String::format("'{}'", lit);
2951
+ } else {
2952
+ expected = "delimiter"; // FIXME: why do we not know what this delimiter is?
2953
+ }
2898
2954
  const char *thing = nullptr;
2899
2955
  switch (token.type()) {
2900
2956
  case Token::Type::InterpolatedRegexpBegin:
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: natalie_parser
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tim Morgan
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-06-03 00:00:00.000000000 Z
11
+ date: 2022-06-04 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: NatalieParser is a zero-dependency, from-scratch, hand-written recursive
14
14
  descent parser for the Ruby Programming Language.
@@ -55,6 +55,7 @@ files:
55
55
  - include/natalie_parser/node/class_node.hpp
56
56
  - include/natalie_parser/node/colon2_node.hpp
57
57
  - include/natalie_parser/node/colon3_node.hpp
58
+ - include/natalie_parser/node/complex_node.hpp
58
59
  - include/natalie_parser/node/constant_node.hpp
59
60
  - include/natalie_parser/node/def_node.hpp
60
61
  - include/natalie_parser/node/defined_node.hpp
@@ -64,6 +65,7 @@ files:
64
65
  - include/natalie_parser/node/false_node.hpp
65
66
  - include/natalie_parser/node/fixnum_node.hpp
66
67
  - include/natalie_parser/node/float_node.hpp
68
+ - include/natalie_parser/node/forward_args_node.hpp
67
69
  - include/natalie_parser/node/hash_node.hpp
68
70
  - include/natalie_parser/node/hash_pattern_node.hpp
69
71
  - include/natalie_parser/node/identifier_node.hpp
@@ -98,6 +100,7 @@ files:
98
100
  - include/natalie_parser/node/op_assign_or_node.hpp
99
101
  - include/natalie_parser/node/pin_node.hpp
100
102
  - include/natalie_parser/node/range_node.hpp
103
+ - include/natalie_parser/node/rational_node.hpp
101
104
  - include/natalie_parser/node/redo_node.hpp
102
105
  - include/natalie_parser/node/regexp_node.hpp
103
106
  - include/natalie_parser/node/retry_node.hpp