natalie_parser 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/README.md +11 -4
- data/Rakefile +4 -5
- data/ext/natalie_parser/mri_creator.hpp +25 -7
- data/include/natalie_parser/creator/debug_creator.hpp +13 -3
- data/include/natalie_parser/creator.hpp +4 -2
- data/include/natalie_parser/node/bignum_node.hpp +1 -1
- data/include/natalie_parser/node/complex_node.hpp +49 -0
- data/include/natalie_parser/node/fixnum_node.hpp +1 -1
- data/include/natalie_parser/node/forward_args_node.hpp +26 -0
- data/include/natalie_parser/node/iter_node.hpp +1 -1
- data/include/natalie_parser/node/node.hpp +4 -1
- data/include/natalie_parser/node/nth_ref_node.hpp +1 -1
- data/include/natalie_parser/node/rational_node.hpp +45 -0
- data/include/natalie_parser/node.hpp +3 -0
- data/include/natalie_parser/parser.hpp +11 -1
- data/include/natalie_parser/token.hpp +20 -0
- data/lib/natalie_parser/version.rb +1 -1
- data/src/lexer.cpp +49 -15
- data/src/node/interpolated_regexp_node.cpp +1 -1
- data/src/node/node_with_args.cpp +1 -0
- data/src/parser.cpp +88 -32
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 00b7efc8434fd5d0ade0fe162d09284e3db60417fd0d2beb8a0526df7d52d903
|
4
|
+
data.tar.gz: 749a3d63bfdcb53e4273afe4a0601eb353e9337ba66d4c1f321661bb3f83135e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e975dbfc660b163bf235c8efb4cc2e0c2e9b7924c4a7ed92382a594fdb4443bcf75f967b09fc0fe8af1eafaa4482793c04c07bc5605128fddfe3a8be24174b73
|
7
|
+
data.tar.gz: 121523ba176eec494681009e3863f80f09691234bcc294c26257b5e86544890787c5a40adc4665c9a2c59de37667596957745bc88293b4d8bfc96ccec26e20b0
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,14 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
+
## 1.1.0 (2022-06-04)
|
4
|
+
|
5
|
+
- CHORE: Add ccache and compiledb for the ext/natalie_parser directory
|
6
|
+
- CHORE: Add tests for numbered block arg shorthand
|
7
|
+
- FEAT Parse arg forwarding (...) shorthand
|
8
|
+
- FEAT: Parse complex and rational numbers
|
9
|
+
- FIX: Fix panic when closing word array delimiter is not found
|
10
|
+
- FIX: Fix precedence bug with op assign operators (+= et al)
|
11
|
+
|
3
12
|
## 1.0.0 (2022-06-03)
|
4
13
|
|
5
14
|
### Summary
|
data/README.md
CHANGED
@@ -25,14 +25,21 @@ production applications.**
|
|
25
25
|
- [ ] Support different source encodings
|
26
26
|
- [ ] Support more of the Ruby 3.0 syntax
|
27
27
|
- [x] "Endless" method definition (`def foo = bar`)
|
28
|
-
- [
|
29
|
-
- [
|
30
|
-
- [
|
28
|
+
- [x] Argument forwarding (`...`)
|
29
|
+
- [x] Numbered block parameters (`_1`, `_2`, etc.)
|
30
|
+
- [x] Rational and Complex literals (`1r` and `2i`)
|
31
31
|
- [ ] Non-ASCII identifiers
|
32
|
-
- [ ]
|
32
|
+
- [ ] Pattern matching
|
33
33
|
|
34
34
|
## Development
|
35
35
|
|
36
|
+
You'll need:
|
37
|
+
|
38
|
+
- gcc or clang
|
39
|
+
- ruby-dev (dev headers)
|
40
|
+
- ccache (optional)
|
41
|
+
- compiledb (optional)
|
42
|
+
|
36
43
|
```sh
|
37
44
|
rake
|
38
45
|
ruby -I lib:ext -r natalie_parser -e "p NatalieParser.parse('1 + 2')"
|
data/Rakefile
CHANGED
@@ -127,6 +127,7 @@ if system('which compiledb 2>&1 >/dev/null')
|
|
127
127
|
if $compiledb_out.any?
|
128
128
|
File.write('build/build.log', $compiledb_out.join("\n"))
|
129
129
|
sh 'compiledb < build/build.log'
|
130
|
+
sh 'cd ext/natalie_parser && compiledb < build.log'
|
130
131
|
end
|
131
132
|
end
|
132
133
|
else
|
@@ -172,13 +173,11 @@ file "ext/natalie_parser/natalie_parser.#{so_ext}" => [
|
|
172
173
|
'ext/natalie_parser/mri_creator.hpp',
|
173
174
|
] + SOURCES + HEADERS do |t|
|
174
175
|
build_dir = File.expand_path('ext/natalie_parser', __dir__)
|
176
|
+
log_file = File.join(build_dir, 'build.log')
|
175
177
|
Rake::FileList['ext/natalie_parser/*.o'].each { |path| rm path }
|
176
178
|
rm_rf 'ext/natalie_parser/natalie_parser.so'
|
177
|
-
sh
|
178
|
-
|
179
|
-
ruby extconf.rb && \
|
180
|
-
make -j
|
181
|
-
SH
|
179
|
+
sh "cd #{build_dir} && ruby extconf.rb"
|
180
|
+
sh "CC=#{cc.inspect} CXX=#{cxx.inspect} make -C #{build_dir} -j -e V=1 2>&1 | tee #{log_file}"
|
182
181
|
end
|
183
182
|
|
184
183
|
file 'build/fragments.hpp' => ['test/parser_test.rb', 'test/support/extract_parser_test_fragments.rb'] do
|
@@ -61,17 +61,20 @@ public:
|
|
61
61
|
rb_ary_push(m_sexp, Qfalse);
|
62
62
|
}
|
63
63
|
|
64
|
-
virtual void
|
65
|
-
|
64
|
+
virtual void append_bignum(TM::String &number) override {
|
65
|
+
auto string_obj = rb_utf8_str_new(number.c_str(), number.length());
|
66
|
+
auto num = rb_Integer(string_obj);
|
67
|
+
rb_ary_push(m_sexp, num);
|
66
68
|
}
|
67
69
|
|
68
|
-
virtual void
|
69
|
-
|
70
|
+
virtual void append_fixnum(long long number) override {
|
71
|
+
auto num = rb_int_new(number);
|
72
|
+
rb_ary_push(m_sexp, num);
|
70
73
|
}
|
71
74
|
|
72
|
-
virtual void
|
73
|
-
auto
|
74
|
-
rb_ary_push(m_sexp,
|
75
|
+
virtual void append_float(double number) override {
|
76
|
+
auto num = rb_float_new(number);
|
77
|
+
rb_ary_push(m_sexp, num);
|
75
78
|
}
|
76
79
|
|
77
80
|
virtual void append_nil() override {
|
@@ -109,6 +112,21 @@ public:
|
|
109
112
|
rb_ary_push(m_sexp, Qtrue);
|
110
113
|
}
|
111
114
|
|
115
|
+
virtual void make_complex_number() override {
|
116
|
+
auto num = rb_ary_pop(m_sexp);
|
117
|
+
num = rb_Complex(INT2FIX(0), num);
|
118
|
+
rb_ary_push(m_sexp, num);
|
119
|
+
}
|
120
|
+
|
121
|
+
virtual void make_rational_number() override {
|
122
|
+
auto num = rb_ary_pop(m_sexp);
|
123
|
+
if (TYPE(num) == T_FLOAT)
|
124
|
+
num = rb_flt_rationalize(num);
|
125
|
+
else
|
126
|
+
num = rb_Rational(num, INT2FIX(1));
|
127
|
+
rb_ary_push(m_sexp, num);
|
128
|
+
}
|
129
|
+
|
112
130
|
virtual void wrap(const char *type) override {
|
113
131
|
auto inner = m_sexp;
|
114
132
|
reset_sexp();
|
@@ -42,15 +42,15 @@ public:
|
|
42
42
|
m_nodes.push("false");
|
43
43
|
}
|
44
44
|
|
45
|
-
virtual void
|
45
|
+
virtual void append_bignum(TM::String &number) override {
|
46
46
|
m_nodes.push(String(number));
|
47
47
|
}
|
48
48
|
|
49
|
-
virtual void
|
49
|
+
virtual void append_fixnum(long long number) override {
|
50
50
|
m_nodes.push(String(number));
|
51
51
|
}
|
52
52
|
|
53
|
-
virtual void
|
53
|
+
virtual void append_float(double number) override {
|
54
54
|
m_nodes.push(String(number));
|
55
55
|
}
|
56
56
|
|
@@ -89,6 +89,16 @@ public:
|
|
89
89
|
m_nodes.push("true");
|
90
90
|
}
|
91
91
|
|
92
|
+
virtual void make_complex_number() override {
|
93
|
+
auto num = m_nodes.pop();
|
94
|
+
m_nodes.push(String::format("Complex(0, {})", num));
|
95
|
+
}
|
96
|
+
|
97
|
+
virtual void make_rational_number() override {
|
98
|
+
auto num = m_nodes.pop();
|
99
|
+
m_nodes.push(String::format("Rational({}, 1)", num));
|
100
|
+
}
|
101
|
+
|
92
102
|
virtual void wrap(const char *type) override {
|
93
103
|
auto inner = to_string();
|
94
104
|
m_nodes.clear();
|
@@ -27,9 +27,9 @@ public:
|
|
27
27
|
virtual void append_array(const TM::SharedPtr<ArrayNode> array) { append_array(*array); }
|
28
28
|
virtual void append_array(const ArrayNode &array) = 0;
|
29
29
|
virtual void append_false() = 0;
|
30
|
+
virtual void append_bignum(TM::String &number) = 0;
|
31
|
+
virtual void append_fixnum(long long number) = 0;
|
30
32
|
virtual void append_float(double number) = 0;
|
31
|
-
virtual void append_integer(long long number) = 0;
|
32
|
-
virtual void append_integer(TM::String &number) = 0;
|
33
33
|
virtual void append_nil() = 0;
|
34
34
|
virtual void append_range(long long first, long long last, bool exclude_end) = 0;
|
35
35
|
virtual void append_regexp(TM::String &pattern, int options) = 0;
|
@@ -37,6 +37,8 @@ public:
|
|
37
37
|
virtual void append_string(TM::String &string) = 0;
|
38
38
|
virtual void append_symbol(TM::String &symbol) = 0;
|
39
39
|
virtual void append_true() = 0;
|
40
|
+
virtual void make_complex_number() = 0;
|
41
|
+
virtual void make_rational_number() = 0;
|
40
42
|
virtual void wrap(const char *type) = 0;
|
41
43
|
|
42
44
|
virtual ~Creator() { }
|
@@ -0,0 +1,49 @@
|
|
1
|
+
#pragma once
|
2
|
+
|
3
|
+
#include "natalie_parser/node/bignum_node.hpp"
|
4
|
+
#include "natalie_parser/node/fixnum_node.hpp"
|
5
|
+
#include "natalie_parser/node/float_node.hpp"
|
6
|
+
#include "natalie_parser/node/node.hpp"
|
7
|
+
#include "natalie_parser/node/rational_node.hpp"
|
8
|
+
|
9
|
+
namespace NatalieParser {
|
10
|
+
|
11
|
+
using namespace TM;
|
12
|
+
|
13
|
+
class ComplexNode : public Node {
|
14
|
+
public:
|
15
|
+
ComplexNode(const Token &token, SharedPtr<Node> value)
|
16
|
+
: Node { token }
|
17
|
+
, m_value { value } { }
|
18
|
+
|
19
|
+
virtual Type type() const override { return Type::Complex; }
|
20
|
+
|
21
|
+
virtual void transform(Creator *creator) const override {
|
22
|
+
creator->set_type("lit");
|
23
|
+
transform_number(creator);
|
24
|
+
creator->make_complex_number();
|
25
|
+
}
|
26
|
+
|
27
|
+
void transform_number(Creator *creator) const {
|
28
|
+
switch (m_value->type()) {
|
29
|
+
case Node::Type::Bignum:
|
30
|
+
creator->append_bignum(m_value.static_cast_as<BignumNode>()->number().ref());
|
31
|
+
break;
|
32
|
+
case Node::Type::Fixnum:
|
33
|
+
creator->append_fixnum(m_value.static_cast_as<FixnumNode>()->number());
|
34
|
+
break;
|
35
|
+
case Node::Type::Float:
|
36
|
+
creator->append_float(m_value.static_cast_as<FloatNode>()->number());
|
37
|
+
break;
|
38
|
+
case Node::Type::Rational:
|
39
|
+
m_value.static_cast_as<RationalNode>()->transform_number(creator);
|
40
|
+
break;
|
41
|
+
default:
|
42
|
+
TM_UNREACHABLE();
|
43
|
+
}
|
44
|
+
}
|
45
|
+
|
46
|
+
protected:
|
47
|
+
SharedPtr<Node> m_value;
|
48
|
+
};
|
49
|
+
}
|
@@ -0,0 +1,26 @@
|
|
1
|
+
#pragma once
|
2
|
+
|
3
|
+
#include "natalie_parser/node/node.hpp"
|
4
|
+
#include "tm/hashmap.hpp"
|
5
|
+
|
6
|
+
namespace NatalieParser {
|
7
|
+
|
8
|
+
using namespace TM;
|
9
|
+
|
10
|
+
class ForwardArgsNode : public Node {
|
11
|
+
public:
|
12
|
+
ForwardArgsNode(const Token &token)
|
13
|
+
: Node { token } { }
|
14
|
+
|
15
|
+
virtual Type type() const override { return Type::ForwardArgs; }
|
16
|
+
|
17
|
+
void add_to_locals(TM::Hashmap<TM::String> &locals) {
|
18
|
+
locals.set("...");
|
19
|
+
}
|
20
|
+
|
21
|
+
virtual void transform(Creator *creator) const override {
|
22
|
+
creator->set_type("forward_args");
|
23
|
+
}
|
24
|
+
};
|
25
|
+
|
26
|
+
}
|
@@ -33,6 +33,7 @@ public:
|
|
33
33
|
Class,
|
34
34
|
Colon2,
|
35
35
|
Colon3,
|
36
|
+
Complex,
|
36
37
|
Constant,
|
37
38
|
Def,
|
38
39
|
Defined,
|
@@ -42,6 +43,7 @@ public:
|
|
42
43
|
False,
|
43
44
|
Fixnum,
|
44
45
|
Float,
|
46
|
+
ForwardArgs,
|
45
47
|
Hash,
|
46
48
|
HashPattern,
|
47
49
|
Identifier,
|
@@ -73,6 +75,7 @@ public:
|
|
73
75
|
OpAssignOr,
|
74
76
|
Pin,
|
75
77
|
Range,
|
78
|
+
Rational,
|
76
79
|
Redo,
|
77
80
|
Regexp,
|
78
81
|
Retry,
|
@@ -122,7 +125,7 @@ public:
|
|
122
125
|
|
123
126
|
virtual void transform(Creator *creator) const {
|
124
127
|
creator->set_type("NOT_YET_IMPLEMENTED");
|
125
|
-
creator->
|
128
|
+
creator->append_fixnum((int)type());
|
126
129
|
}
|
127
130
|
|
128
131
|
SharedPtr<String> file() const { return m_token.file(); }
|
@@ -0,0 +1,45 @@
|
|
1
|
+
#pragma once
|
2
|
+
|
3
|
+
#include "natalie_parser/node/bignum_node.hpp"
|
4
|
+
#include "natalie_parser/node/fixnum_node.hpp"
|
5
|
+
#include "natalie_parser/node/float_node.hpp"
|
6
|
+
#include "natalie_parser/node/node.hpp"
|
7
|
+
|
8
|
+
namespace NatalieParser {
|
9
|
+
|
10
|
+
using namespace TM;
|
11
|
+
|
12
|
+
class RationalNode : public Node {
|
13
|
+
public:
|
14
|
+
RationalNode(const Token &token, SharedPtr<Node> value)
|
15
|
+
: Node { token }
|
16
|
+
, m_value { value } { }
|
17
|
+
|
18
|
+
virtual Type type() const override { return Type::Rational; }
|
19
|
+
|
20
|
+
virtual void transform(Creator *creator) const override {
|
21
|
+
creator->set_type("lit");
|
22
|
+
transform_number(creator);
|
23
|
+
creator->make_rational_number();
|
24
|
+
}
|
25
|
+
|
26
|
+
void transform_number(Creator *creator) const {
|
27
|
+
switch (m_value->type()) {
|
28
|
+
case Node::Type::Bignum:
|
29
|
+
creator->append_bignum(m_value.static_cast_as<BignumNode>()->number().ref());
|
30
|
+
break;
|
31
|
+
case Node::Type::Fixnum:
|
32
|
+
creator->append_fixnum(m_value.static_cast_as<FixnumNode>()->number());
|
33
|
+
break;
|
34
|
+
case Node::Type::Float:
|
35
|
+
creator->append_float(m_value.static_cast_as<FloatNode>()->number());
|
36
|
+
break;
|
37
|
+
default:
|
38
|
+
TM_UNREACHABLE();
|
39
|
+
}
|
40
|
+
}
|
41
|
+
|
42
|
+
protected:
|
43
|
+
SharedPtr<Node> m_value;
|
44
|
+
};
|
45
|
+
}
|
@@ -21,6 +21,7 @@
|
|
21
21
|
#include "natalie_parser/node/class_node.hpp"
|
22
22
|
#include "natalie_parser/node/colon2_node.hpp"
|
23
23
|
#include "natalie_parser/node/colon3_node.hpp"
|
24
|
+
#include "natalie_parser/node/complex_node.hpp"
|
24
25
|
#include "natalie_parser/node/constant_node.hpp"
|
25
26
|
#include "natalie_parser/node/def_node.hpp"
|
26
27
|
#include "natalie_parser/node/defined_node.hpp"
|
@@ -30,6 +31,7 @@
|
|
30
31
|
#include "natalie_parser/node/false_node.hpp"
|
31
32
|
#include "natalie_parser/node/fixnum_node.hpp"
|
32
33
|
#include "natalie_parser/node/float_node.hpp"
|
34
|
+
#include "natalie_parser/node/forward_args_node.hpp"
|
33
35
|
#include "natalie_parser/node/hash_node.hpp"
|
34
36
|
#include "natalie_parser/node/hash_pattern_node.hpp"
|
35
37
|
#include "natalie_parser/node/identifier_node.hpp"
|
@@ -63,6 +65,7 @@
|
|
63
65
|
#include "natalie_parser/node/op_assign_or_node.hpp"
|
64
66
|
#include "natalie_parser/node/pin_node.hpp"
|
65
67
|
#include "natalie_parser/node/range_node.hpp"
|
68
|
+
#include "natalie_parser/node/rational_node.hpp"
|
66
69
|
#include "natalie_parser/node/redo_node.hpp"
|
67
70
|
#include "natalie_parser/node/regexp_node.hpp"
|
68
71
|
#include "natalie_parser/node/retry_node.hpp"
|
@@ -87,11 +87,19 @@ private:
|
|
87
87
|
SharedPtr<Node> parse_constant(LocalsHashmap &);
|
88
88
|
SharedPtr<Node> parse_def(LocalsHashmap &);
|
89
89
|
SharedPtr<Node> parse_defined(LocalsHashmap &);
|
90
|
+
|
90
91
|
void parse_def_args(Vector<SharedPtr<Node>> &, LocalsHashmap &);
|
91
|
-
|
92
|
+
enum class ArgsContext {
|
93
|
+
Block,
|
94
|
+
Method,
|
95
|
+
Proc,
|
96
|
+
};
|
97
|
+
void parse_def_single_arg(Vector<SharedPtr<Node>> &, LocalsHashmap &, ArgsContext);
|
98
|
+
|
92
99
|
SharedPtr<Node> parse_encoding(LocalsHashmap &);
|
93
100
|
SharedPtr<Node> parse_end_block(LocalsHashmap &);
|
94
101
|
SharedPtr<Node> parse_file_constant(LocalsHashmap &);
|
102
|
+
SharedPtr<Node> parse_forward_args(LocalsHashmap &);
|
95
103
|
SharedPtr<Node> parse_group(LocalsHashmap &);
|
96
104
|
SharedPtr<Node> parse_hash(LocalsHashmap &);
|
97
105
|
SharedPtr<Node> parse_hash_inner(LocalsHashmap &, Precedence, Token::Type, SharedPtr<Node> = {});
|
@@ -128,6 +136,7 @@ private:
|
|
128
136
|
SharedPtr<Node> parse_symbol_key(LocalsHashmap &);
|
129
137
|
SharedPtr<Node> parse_statement_keyword(LocalsHashmap &);
|
130
138
|
SharedPtr<Node> parse_top_level_constant(LocalsHashmap &);
|
139
|
+
SharedPtr<Node> parse_triple_dot(LocalsHashmap &);
|
131
140
|
SharedPtr<Node> parse_unary_operator(LocalsHashmap &);
|
132
141
|
SharedPtr<Node> parse_undef(LocalsHashmap &);
|
133
142
|
SharedPtr<Node> parse_unless(LocalsHashmap &);
|
@@ -195,6 +204,7 @@ private:
|
|
195
204
|
void skip_newlines();
|
196
205
|
|
197
206
|
void expect(Token::Type, const char *);
|
207
|
+
[[noreturn]] void throw_error(const Token &, const char *);
|
198
208
|
[[noreturn]] void throw_unexpected(const Token &, const char *, const char * = nullptr);
|
199
209
|
[[noreturn]] void throw_unexpected(const char *);
|
200
210
|
[[noreturn]] void throw_unterminated_thing(Token, Token = {});
|
@@ -36,6 +36,7 @@ public:
|
|
36
36
|
Comma,
|
37
37
|
Comment,
|
38
38
|
Comparison,
|
39
|
+
Complex,
|
39
40
|
Constant,
|
40
41
|
ConstantResolution,
|
41
42
|
DefinedKeyword,
|
@@ -115,6 +116,8 @@ public:
|
|
115
116
|
PipePipeEqual,
|
116
117
|
Plus,
|
117
118
|
PlusEqual,
|
119
|
+
Rational,
|
120
|
+
RationalComplex,
|
118
121
|
RCurlyBrace,
|
119
122
|
RBracket,
|
120
123
|
RedoKeyword,
|
@@ -286,6 +289,8 @@ public:
|
|
286
289
|
return "comment";
|
287
290
|
case Type::Comparison:
|
288
291
|
return "<=>";
|
292
|
+
case Type::Complex:
|
293
|
+
return "complex";
|
289
294
|
case Type::ConstantResolution:
|
290
295
|
return "::";
|
291
296
|
case Type::Constant:
|
@@ -446,6 +451,10 @@ public:
|
|
446
451
|
return "+=";
|
447
452
|
case Type::Plus:
|
448
453
|
return "+";
|
454
|
+
case Type::Rational:
|
455
|
+
return "rational";
|
456
|
+
case Type::RationalComplex:
|
457
|
+
return "rational_complex";
|
449
458
|
case Type::RCurlyBrace:
|
450
459
|
return "}";
|
451
460
|
case Type::RBracket:
|
@@ -804,6 +813,17 @@ public:
|
|
804
813
|
}
|
805
814
|
}
|
806
815
|
|
816
|
+
bool can_be_complex_or_rational() const {
|
817
|
+
switch (m_type) {
|
818
|
+
case Type::Bignum:
|
819
|
+
case Type::Fixnum:
|
820
|
+
case Type::Float:
|
821
|
+
return true;
|
822
|
+
default:
|
823
|
+
return false;
|
824
|
+
}
|
825
|
+
}
|
826
|
+
|
807
827
|
void set_literal(const char *literal) { m_literal = new String(literal); }
|
808
828
|
void set_literal(SharedPtr<String> literal) { m_literal = literal; }
|
809
829
|
void set_literal(String literal) { m_literal = new String(literal); }
|
data/src/lexer.cpp
CHANGED
@@ -842,6 +842,23 @@ Token Lexer::build_next_token() {
|
|
842
842
|
auto token = consume_numeric();
|
843
843
|
return token;
|
844
844
|
}
|
845
|
+
case 'i':
|
846
|
+
if (m_last_token.can_be_complex_or_rational() && !isalnum(peek())) {
|
847
|
+
advance();
|
848
|
+
return Token { Token::Type::Complex, m_file, m_token_line, m_token_column };
|
849
|
+
}
|
850
|
+
break;
|
851
|
+
case 'r':
|
852
|
+
if (m_last_token.can_be_complex_or_rational()) {
|
853
|
+
if (peek() == 'i') {
|
854
|
+
advance(2);
|
855
|
+
return Token { Token::Type::RationalComplex, m_file, m_token_line, m_token_column };
|
856
|
+
} else if (!isalnum(peek())) {
|
857
|
+
advance();
|
858
|
+
return Token { Token::Type::Rational, m_file, m_token_line, m_token_column };
|
859
|
+
}
|
860
|
+
}
|
861
|
+
break;
|
845
862
|
};
|
846
863
|
|
847
864
|
Token keyword_token;
|
@@ -1322,7 +1339,25 @@ Token Lexer::consume_heredoc() {
|
|
1322
1339
|
|
1323
1340
|
Token Lexer::consume_numeric() {
|
1324
1341
|
SharedPtr<String> chars = new String;
|
1342
|
+
|
1343
|
+
auto consume_decimal_digits_and_build_token = [&]() {
|
1344
|
+
char c = current_char();
|
1345
|
+
do {
|
1346
|
+
chars->append_char(c);
|
1347
|
+
c = next();
|
1348
|
+
if (c == '_')
|
1349
|
+
c = next();
|
1350
|
+
} while (isdigit(c));
|
1351
|
+
if ((c == '.' && isdigit(peek())) || (c == 'e' || c == 'E'))
|
1352
|
+
return consume_numeric_as_float(chars);
|
1353
|
+
else
|
1354
|
+
return chars_to_fixnum_or_bignum_token(chars, 10, 0);
|
1355
|
+
};
|
1356
|
+
|
1357
|
+
Token token;
|
1358
|
+
|
1325
1359
|
if (current_char() == '0') {
|
1360
|
+
// special-prefixed literals 0d, 0x, etc.
|
1326
1361
|
switch (peek()) {
|
1327
1362
|
case 'd':
|
1328
1363
|
case 'D': {
|
@@ -1336,7 +1371,8 @@ Token Lexer::consume_numeric() {
|
|
1336
1371
|
if (c == '_')
|
1337
1372
|
c = next();
|
1338
1373
|
} while (isdigit(c));
|
1339
|
-
|
1374
|
+
token = chars_to_fixnum_or_bignum_token(chars, 10, 0);
|
1375
|
+
break;
|
1340
1376
|
}
|
1341
1377
|
case 'o':
|
1342
1378
|
case 'O': {
|
@@ -1352,7 +1388,8 @@ Token Lexer::consume_numeric() {
|
|
1352
1388
|
if (c == '_')
|
1353
1389
|
c = next();
|
1354
1390
|
} while (c >= '0' && c <= '7');
|
1355
|
-
|
1391
|
+
token = chars_to_fixnum_or_bignum_token(chars, 8, 2);
|
1392
|
+
break;
|
1356
1393
|
}
|
1357
1394
|
case 'x':
|
1358
1395
|
case 'X': {
|
@@ -1368,7 +1405,8 @@ Token Lexer::consume_numeric() {
|
|
1368
1405
|
if (c == '_')
|
1369
1406
|
c = next();
|
1370
1407
|
} while (isxdigit(c));
|
1371
|
-
|
1408
|
+
token = chars_to_fixnum_or_bignum_token(chars, 16, 2);
|
1409
|
+
break;
|
1372
1410
|
}
|
1373
1411
|
case 'b':
|
1374
1412
|
case 'B': {
|
@@ -1384,21 +1422,17 @@ Token Lexer::consume_numeric() {
|
|
1384
1422
|
if (c == '_')
|
1385
1423
|
c = next();
|
1386
1424
|
} while (c == '0' || c == '1');
|
1387
|
-
|
1425
|
+
token = chars_to_fixnum_or_bignum_token(chars, 2, 2);
|
1426
|
+
break;
|
1388
1427
|
}
|
1428
|
+
default:
|
1429
|
+
token = consume_decimal_digits_and_build_token();
|
1389
1430
|
}
|
1431
|
+
} else {
|
1432
|
+
token = consume_decimal_digits_and_build_token();
|
1390
1433
|
}
|
1391
|
-
|
1392
|
-
|
1393
|
-
chars->append_char(c);
|
1394
|
-
c = next();
|
1395
|
-
if (c == '_')
|
1396
|
-
c = next();
|
1397
|
-
} while (isdigit(c));
|
1398
|
-
if ((c == '.' && isdigit(peek())) || (c == 'e' || c == 'E'))
|
1399
|
-
return consume_numeric_as_float(chars);
|
1400
|
-
else
|
1401
|
-
return chars_to_fixnum_or_bignum_token(chars, 10, 0);
|
1434
|
+
|
1435
|
+
return token;
|
1402
1436
|
}
|
1403
1437
|
|
1404
1438
|
const long long max_fixnum = std::numeric_limits<long long>::max() / 2; // 63 bits for MRI
|
data/src/node/node_with_args.cpp
CHANGED
@@ -17,6 +17,7 @@ void NodeWithArgs::append_method_or_block_args(Creator *creator) const {
|
|
17
17
|
arg.static_cast_as<ArgNode>()->append_name(c);
|
18
18
|
break;
|
19
19
|
}
|
20
|
+
case Node::Type::ForwardArgs:
|
20
21
|
case Node::Type::KeywordArg:
|
21
22
|
case Node::Type::MultipleAssignmentArg:
|
22
23
|
case Node::Type::ShadowArg:
|
data/src/parser.cpp
CHANGED
@@ -16,7 +16,7 @@ enum class Parser::Precedence {
|
|
16
16
|
INLINE_RESCUE, // foo rescue 2
|
17
17
|
ITER_BLOCK, // do |n| ... end
|
18
18
|
BARE_CALL_ARG, // foo (_), b
|
19
|
-
|
19
|
+
OP_ASSIGNMENT_RHS, // x += (_)
|
20
20
|
TERNARY_TRUE, // _ ? (_) : _
|
21
21
|
CALL_ARG, // foo( (_), b )
|
22
22
|
TERNARY_QUESTION, // (_) ? _ : _
|
@@ -28,6 +28,7 @@ enum class Parser::Precedence {
|
|
28
28
|
LOGICAL_NOT, // not
|
29
29
|
EQUALITY, // <=> == === != =~ !~
|
30
30
|
LESS_GREATER, // <= < > >=
|
31
|
+
OP_ASSIGNMENT_LHS, // (_) += 1
|
31
32
|
BITWISE_OR, // ^ |
|
32
33
|
BITWISE_AND, // &
|
33
34
|
BITWISE_SHIFT, // << >>
|
@@ -112,7 +113,7 @@ Parser::Precedence Parser::get_precedence(Token &token, SharedPtr<Node> left) {
|
|
112
113
|
case Token::Type::SlashEqual:
|
113
114
|
case Token::Type::StarEqual:
|
114
115
|
case Token::Type::StarStarEqual:
|
115
|
-
return Precedence::
|
116
|
+
return Precedence::OP_ASSIGNMENT_LHS;
|
116
117
|
case Token::Type::Ampersand:
|
117
118
|
return Precedence::BITWISE_AND;
|
118
119
|
case Token::Type::Caret:
|
@@ -979,28 +980,32 @@ SharedPtr<Node> Parser::parse_defined(LocalsHashmap &locals) {
|
|
979
980
|
}
|
980
981
|
|
981
982
|
void Parser::parse_def_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals) {
|
982
|
-
parse_def_single_arg(args, locals);
|
983
|
+
parse_def_single_arg(args, locals, ArgsContext::Method);
|
983
984
|
while (current_token().is_comma()) {
|
984
985
|
advance();
|
985
|
-
parse_def_single_arg(args, locals);
|
986
|
+
parse_def_single_arg(args, locals, ArgsContext::Method);
|
986
987
|
}
|
987
988
|
}
|
988
989
|
|
989
|
-
void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals) {
|
990
|
+
void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals, ArgsContext context) {
|
990
991
|
auto args_have_any_splat = [&]() { return !args.is_empty() && args.last()->type() == Node::Type::Arg && args.last().static_cast_as<ArgNode>()->splat_or_kwsplat(); };
|
991
992
|
auto args_have_keyword = [&]() { return !args.is_empty() && args.last()->type() == Node::Type::KeywordArg; };
|
992
993
|
|
993
994
|
auto token = current_token();
|
995
|
+
|
996
|
+
if (!args.is_empty() && args.last()->type() == Node::Type::ForwardArgs)
|
997
|
+
throw_error(token, "anything after arg forwarding (...) shorthand");
|
998
|
+
|
994
999
|
switch (token.type()) {
|
995
1000
|
case Token::Type::BareName: {
|
996
1001
|
if (args_have_keyword())
|
997
|
-
|
1002
|
+
throw_error(token, "normal arg after keyword arg");
|
998
1003
|
SharedPtr<ArgNode> arg = new ArgNode { token, token.literal_string() };
|
999
1004
|
advance();
|
1000
1005
|
arg->add_to_locals(locals);
|
1001
1006
|
if (current_token().is_equal()) {
|
1002
1007
|
if (args_have_any_splat())
|
1003
|
-
|
1008
|
+
throw_error(token, "default value after splat");
|
1004
1009
|
advance(); // =
|
1005
1010
|
arg->set_value(parse_expression(Precedence::DEF_ARG, locals));
|
1006
1011
|
}
|
@@ -1022,7 +1027,7 @@ void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &
|
|
1022
1027
|
}
|
1023
1028
|
case Token::Type::Star: {
|
1024
1029
|
if (args_have_any_splat())
|
1025
|
-
|
1030
|
+
throw_error(token, "splat after keyword splat");
|
1026
1031
|
advance();
|
1027
1032
|
SharedPtr<ArgNode> arg;
|
1028
1033
|
if (current_token().is_bare_name()) {
|
@@ -1080,6 +1085,15 @@ void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &
|
|
1080
1085
|
args.push(arg.static_cast_as<Node>());
|
1081
1086
|
return;
|
1082
1087
|
}
|
1088
|
+
case Token::Type::DotDotDot: {
|
1089
|
+
if (context == ArgsContext::Block)
|
1090
|
+
throw_error(token, "arg forwarding (...) shorthand not allowed in block");
|
1091
|
+
SharedPtr<ForwardArgsNode> arg = new ForwardArgsNode { token };
|
1092
|
+
advance();
|
1093
|
+
arg->add_to_locals(locals);
|
1094
|
+
args.push(arg.static_cast_as<Node>());
|
1095
|
+
return;
|
1096
|
+
}
|
1083
1097
|
default:
|
1084
1098
|
throw_unexpected("argument");
|
1085
1099
|
}
|
@@ -1151,6 +1165,15 @@ SharedPtr<Node> Parser::parse_file_constant(LocalsHashmap &) {
|
|
1151
1165
|
return new StringNode { token, token.file() };
|
1152
1166
|
}
|
1153
1167
|
|
1168
|
+
SharedPtr<Node> Parser::parse_forward_args(LocalsHashmap &locals) {
|
1169
|
+
auto token = current_token();
|
1170
|
+
if (!locals.get("..."))
|
1171
|
+
throw_error(token, "forwarding args without ... shorthand in method definition");
|
1172
|
+
advance(); // ...
|
1173
|
+
SharedPtr<ForwardArgsNode> node = new ForwardArgsNode { token };
|
1174
|
+
return node.static_cast_as<Node>();
|
1175
|
+
}
|
1176
|
+
|
1154
1177
|
SharedPtr<Node> Parser::parse_group(LocalsHashmap &locals) {
|
1155
1178
|
auto token = current_token();
|
1156
1179
|
advance(); // (
|
@@ -1491,19 +1514,40 @@ SharedPtr<Node> Parser::parse_interpolated_symbol(LocalsHashmap &locals) {
|
|
1491
1514
|
|
1492
1515
|
SharedPtr<Node> Parser::parse_lit(LocalsHashmap &) {
|
1493
1516
|
auto token = current_token();
|
1517
|
+
SharedPtr<Node> node;
|
1494
1518
|
switch (token.type()) {
|
1495
1519
|
case Token::Type::Bignum:
|
1496
1520
|
advance();
|
1497
|
-
|
1521
|
+
node = new BignumNode { token, token.literal_string() };
|
1522
|
+
break;
|
1498
1523
|
case Token::Type::Fixnum:
|
1499
1524
|
advance();
|
1500
|
-
|
1525
|
+
node = new FixnumNode { token, token.get_fixnum() };
|
1526
|
+
break;
|
1501
1527
|
case Token::Type::Float:
|
1502
1528
|
advance();
|
1503
|
-
|
1529
|
+
node = new FloatNode { token, token.get_double() };
|
1530
|
+
break;
|
1504
1531
|
default:
|
1505
1532
|
TM_UNREACHABLE();
|
1506
1533
|
}
|
1534
|
+
switch (current_token().type()) {
|
1535
|
+
case Token::Type::Complex:
|
1536
|
+
advance();
|
1537
|
+
node = new ComplexNode { token, node };
|
1538
|
+
break;
|
1539
|
+
case Token::Type::Rational:
|
1540
|
+
advance();
|
1541
|
+
node = new RationalNode { token, node };
|
1542
|
+
break;
|
1543
|
+
case Token::Type::RationalComplex:
|
1544
|
+
advance();
|
1545
|
+
node = new ComplexNode { token, new RationalNode { token, node } };
|
1546
|
+
break;
|
1547
|
+
default:
|
1548
|
+
break;
|
1549
|
+
}
|
1550
|
+
return node;
|
1507
1551
|
};
|
1508
1552
|
|
1509
1553
|
SharedPtr<Node> Parser::parse_keyword_splat(LocalsHashmap &locals) {
|
@@ -1605,10 +1649,10 @@ void Parser::parse_proc_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &local
|
|
1605
1649
|
parse_shadow_variables_in_args(args, locals);
|
1606
1650
|
return;
|
1607
1651
|
}
|
1608
|
-
parse_def_single_arg(args, locals);
|
1652
|
+
parse_def_single_arg(args, locals, ArgsContext::Proc);
|
1609
1653
|
while (current_token().is_comma()) {
|
1610
1654
|
advance();
|
1611
|
-
parse_def_single_arg(args, locals);
|
1655
|
+
parse_def_single_arg(args, locals, ArgsContext::Proc);
|
1612
1656
|
}
|
1613
1657
|
if (current_token().is_semicolon()) {
|
1614
1658
|
parse_shadow_variables_in_args(args, locals);
|
@@ -2101,7 +2145,7 @@ void Parser::parse_iter_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &local
|
|
2101
2145
|
parse_shadow_variables_in_args(args, locals);
|
2102
2146
|
return;
|
2103
2147
|
}
|
2104
|
-
parse_def_single_arg(args, locals);
|
2148
|
+
parse_def_single_arg(args, locals, ArgsContext::Block);
|
2105
2149
|
while (current_token().is_comma()) {
|
2106
2150
|
advance();
|
2107
2151
|
if (current_token().is_block_arg_delimiter()) {
|
@@ -2109,7 +2153,7 @@ void Parser::parse_iter_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &local
|
|
2109
2153
|
args.push(new NilNode { current_token() });
|
2110
2154
|
break;
|
2111
2155
|
}
|
2112
|
-
parse_def_single_arg(args, locals);
|
2156
|
+
parse_def_single_arg(args, locals, ArgsContext::Block);
|
2113
2157
|
}
|
2114
2158
|
if (current_token().is_semicolon()) {
|
2115
2159
|
parse_shadow_variables_in_args(args, locals);
|
@@ -2156,7 +2200,7 @@ SharedPtr<NodeWithArgs> Parser::to_node_with_args(SharedPtr<Node> node) {
|
|
2156
2200
|
case Node::Type::Yield:
|
2157
2201
|
return node.static_cast_as<NodeWithArgs>();
|
2158
2202
|
default:
|
2159
|
-
|
2203
|
+
throw_error(current_token(), "left-hand-side is not callable");
|
2160
2204
|
}
|
2161
2205
|
}
|
2162
2206
|
|
@@ -2381,7 +2425,7 @@ SharedPtr<Node> Parser::parse_op_attr_assign_expression(SharedPtr<Node> left, Lo
|
|
2381
2425
|
auto left_call = left.static_cast_as<CallNode>();
|
2382
2426
|
auto token = current_token();
|
2383
2427
|
advance();
|
2384
|
-
auto value = parse_expression(Precedence::
|
2428
|
+
auto value = parse_expression(Precedence::OP_ASSIGNMENT_RHS, locals);
|
2385
2429
|
|
2386
2430
|
if (*left_call->message() != "[]") {
|
2387
2431
|
if (token.type() == Token::Type::AmpersandAmpersandEqual) {
|
@@ -2536,6 +2580,12 @@ SharedPtr<Node> Parser::parse_ternary_expression(SharedPtr<Node> left, LocalsHas
|
|
2536
2580
|
return new IfNode { token, left, true_expr, false_expr };
|
2537
2581
|
}
|
2538
2582
|
|
2583
|
+
SharedPtr<Node> Parser::parse_triple_dot(LocalsHashmap &locals) {
|
2584
|
+
if (peek_token().is_rparen())
|
2585
|
+
return parse_forward_args(locals);
|
2586
|
+
return parse_beginless_range(locals);
|
2587
|
+
}
|
2588
|
+
|
2539
2589
|
SharedPtr<Node> Parser::parse_unless(LocalsHashmap &locals) {
|
2540
2590
|
auto token = current_token();
|
2541
2591
|
advance();
|
@@ -2603,7 +2653,7 @@ Parser::parse_null_fn Parser::null_denotation(Token::Type type) {
|
|
2603
2653
|
return &Parser::parse_defined;
|
2604
2654
|
case Type::DotDot:
|
2605
2655
|
case Type::DotDotDot:
|
2606
|
-
return &Parser::
|
2656
|
+
return &Parser::parse_triple_dot;
|
2607
2657
|
case Type::ENCODINGKeyword:
|
2608
2658
|
return &Parser::parse_encoding;
|
2609
2659
|
case Type::ENDKeyword:
|
@@ -2826,6 +2876,10 @@ void Parser::expect(Token::Type type, const char *expected) {
|
|
2826
2876
|
throw_unexpected(expected);
|
2827
2877
|
}
|
2828
2878
|
|
2879
|
+
void Parser::throw_error(const Token &token, const char *error) {
|
2880
|
+
throw_unexpected(token, nullptr, error);
|
2881
|
+
}
|
2882
|
+
|
2829
2883
|
void Parser::throw_unexpected(const Token &token, const char *expected, const char *error) {
|
2830
2884
|
auto file = token.file() ? String(*token.file()) : String("(unknown)");
|
2831
2885
|
auto line = token.line() + 1;
|
@@ -2881,20 +2935,22 @@ void Parser::throw_unterminated_thing(Token token, Token start_token) {
|
|
2881
2935
|
auto indent = String { start_token.column(), ' ' };
|
2882
2936
|
String expected;
|
2883
2937
|
const char *lit = start_token.literal();
|
2884
|
-
|
2885
|
-
|
2886
|
-
|
2887
|
-
|
2888
|
-
|
2889
|
-
|
2890
|
-
|
2891
|
-
|
2892
|
-
|
2893
|
-
|
2894
|
-
|
2895
|
-
|
2896
|
-
|
2897
|
-
|
2938
|
+
if (lit) {
|
2939
|
+
if (strcmp(lit, "(") == 0)
|
2940
|
+
expected = "')'";
|
2941
|
+
else if (strcmp(lit, "[") == 0)
|
2942
|
+
expected = "']'";
|
2943
|
+
else if (strcmp(lit, "{") == 0)
|
2944
|
+
expected = "'}'";
|
2945
|
+
else if (strcmp(lit, "<") == 0)
|
2946
|
+
expected = "'>'";
|
2947
|
+
else if (strcmp(lit, "'") == 0)
|
2948
|
+
expected = "\"'\"";
|
2949
|
+
else
|
2950
|
+
expected = String::format("'{}'", lit);
|
2951
|
+
} else {
|
2952
|
+
expected = "delimiter"; // FIXME: why do we not know what this delimiter is?
|
2953
|
+
}
|
2898
2954
|
const char *thing = nullptr;
|
2899
2955
|
switch (token.type()) {
|
2900
2956
|
case Token::Type::InterpolatedRegexpBegin:
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: natalie_parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tim Morgan
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-06-
|
11
|
+
date: 2022-06-04 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: NatalieParser is a zero-dependency, from-scratch, hand-written recursive
|
14
14
|
descent parser for the Ruby Programming Language.
|
@@ -55,6 +55,7 @@ files:
|
|
55
55
|
- include/natalie_parser/node/class_node.hpp
|
56
56
|
- include/natalie_parser/node/colon2_node.hpp
|
57
57
|
- include/natalie_parser/node/colon3_node.hpp
|
58
|
+
- include/natalie_parser/node/complex_node.hpp
|
58
59
|
- include/natalie_parser/node/constant_node.hpp
|
59
60
|
- include/natalie_parser/node/def_node.hpp
|
60
61
|
- include/natalie_parser/node/defined_node.hpp
|
@@ -64,6 +65,7 @@ files:
|
|
64
65
|
- include/natalie_parser/node/false_node.hpp
|
65
66
|
- include/natalie_parser/node/fixnum_node.hpp
|
66
67
|
- include/natalie_parser/node/float_node.hpp
|
68
|
+
- include/natalie_parser/node/forward_args_node.hpp
|
67
69
|
- include/natalie_parser/node/hash_node.hpp
|
68
70
|
- include/natalie_parser/node/hash_pattern_node.hpp
|
69
71
|
- include/natalie_parser/node/identifier_node.hpp
|
@@ -98,6 +100,7 @@ files:
|
|
98
100
|
- include/natalie_parser/node/op_assign_or_node.hpp
|
99
101
|
- include/natalie_parser/node/pin_node.hpp
|
100
102
|
- include/natalie_parser/node/range_node.hpp
|
103
|
+
- include/natalie_parser/node/rational_node.hpp
|
101
104
|
- include/natalie_parser/node/redo_node.hpp
|
102
105
|
- include/natalie_parser/node/regexp_node.hpp
|
103
106
|
- include/natalie_parser/node/retry_node.hpp
|