natalie_parser 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/README.md +11 -4
- data/Rakefile +4 -5
- data/ext/natalie_parser/mri_creator.hpp +25 -7
- data/include/natalie_parser/creator/debug_creator.hpp +13 -3
- data/include/natalie_parser/creator.hpp +4 -2
- data/include/natalie_parser/node/bignum_node.hpp +1 -1
- data/include/natalie_parser/node/complex_node.hpp +49 -0
- data/include/natalie_parser/node/fixnum_node.hpp +1 -1
- data/include/natalie_parser/node/forward_args_node.hpp +26 -0
- data/include/natalie_parser/node/iter_node.hpp +1 -1
- data/include/natalie_parser/node/node.hpp +4 -1
- data/include/natalie_parser/node/nth_ref_node.hpp +1 -1
- data/include/natalie_parser/node/rational_node.hpp +45 -0
- data/include/natalie_parser/node.hpp +3 -0
- data/include/natalie_parser/parser.hpp +11 -1
- data/include/natalie_parser/token.hpp +20 -0
- data/lib/natalie_parser/version.rb +1 -1
- data/src/lexer.cpp +49 -15
- data/src/node/interpolated_regexp_node.cpp +1 -1
- data/src/node/node_with_args.cpp +1 -0
- data/src/parser.cpp +88 -32
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 00b7efc8434fd5d0ade0fe162d09284e3db60417fd0d2beb8a0526df7d52d903
|
4
|
+
data.tar.gz: 749a3d63bfdcb53e4273afe4a0601eb353e9337ba66d4c1f321661bb3f83135e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e975dbfc660b163bf235c8efb4cc2e0c2e9b7924c4a7ed92382a594fdb4443bcf75f967b09fc0fe8af1eafaa4482793c04c07bc5605128fddfe3a8be24174b73
|
7
|
+
data.tar.gz: 121523ba176eec494681009e3863f80f09691234bcc294c26257b5e86544890787c5a40adc4665c9a2c59de37667596957745bc88293b4d8bfc96ccec26e20b0
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,14 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
+
## 1.1.0 (2022-06-04)
|
4
|
+
|
5
|
+
- CHORE: Add ccache and compiledb for the ext/natalie_parser directory
|
6
|
+
- CHORE: Add tests for numbered block arg shorthand
|
7
|
+
- FEAT Parse arg forwarding (...) shorthand
|
8
|
+
- FEAT: Parse complex and rational numbers
|
9
|
+
- FIX: Fix panic when closing word array delimiter is not found
|
10
|
+
- FIX: Fix precedence bug with op assign operators (+= et al)
|
11
|
+
|
3
12
|
## 1.0.0 (2022-06-03)
|
4
13
|
|
5
14
|
### Summary
|
data/README.md
CHANGED
@@ -25,14 +25,21 @@ production applications.**
|
|
25
25
|
- [ ] Support different source encodings
|
26
26
|
- [ ] Support more of the Ruby 3.0 syntax
|
27
27
|
- [x] "Endless" method definition (`def foo = bar`)
|
28
|
-
- [
|
29
|
-
- [
|
30
|
-
- [
|
28
|
+
- [x] Argument forwarding (`...`)
|
29
|
+
- [x] Numbered block parameters (`_1`, `_2`, etc.)
|
30
|
+
- [x] Rational and Complex literals (`1r` and `2i`)
|
31
31
|
- [ ] Non-ASCII identifiers
|
32
|
-
- [ ]
|
32
|
+
- [ ] Pattern matching
|
33
33
|
|
34
34
|
## Development
|
35
35
|
|
36
|
+
You'll need:
|
37
|
+
|
38
|
+
- gcc or clang
|
39
|
+
- ruby-dev (dev headers)
|
40
|
+
- ccache (optional)
|
41
|
+
- compiledb (optional)
|
42
|
+
|
36
43
|
```sh
|
37
44
|
rake
|
38
45
|
ruby -I lib:ext -r natalie_parser -e "p NatalieParser.parse('1 + 2')"
|
data/Rakefile
CHANGED
@@ -127,6 +127,7 @@ if system('which compiledb 2>&1 >/dev/null')
|
|
127
127
|
if $compiledb_out.any?
|
128
128
|
File.write('build/build.log', $compiledb_out.join("\n"))
|
129
129
|
sh 'compiledb < build/build.log'
|
130
|
+
sh 'cd ext/natalie_parser && compiledb < build.log'
|
130
131
|
end
|
131
132
|
end
|
132
133
|
else
|
@@ -172,13 +173,11 @@ file "ext/natalie_parser/natalie_parser.#{so_ext}" => [
|
|
172
173
|
'ext/natalie_parser/mri_creator.hpp',
|
173
174
|
] + SOURCES + HEADERS do |t|
|
174
175
|
build_dir = File.expand_path('ext/natalie_parser', __dir__)
|
176
|
+
log_file = File.join(build_dir, 'build.log')
|
175
177
|
Rake::FileList['ext/natalie_parser/*.o'].each { |path| rm path }
|
176
178
|
rm_rf 'ext/natalie_parser/natalie_parser.so'
|
177
|
-
sh
|
178
|
-
|
179
|
-
ruby extconf.rb && \
|
180
|
-
make -j
|
181
|
-
SH
|
179
|
+
sh "cd #{build_dir} && ruby extconf.rb"
|
180
|
+
sh "CC=#{cc.inspect} CXX=#{cxx.inspect} make -C #{build_dir} -j -e V=1 2>&1 | tee #{log_file}"
|
182
181
|
end
|
183
182
|
|
184
183
|
file 'build/fragments.hpp' => ['test/parser_test.rb', 'test/support/extract_parser_test_fragments.rb'] do
|
@@ -61,17 +61,20 @@ public:
|
|
61
61
|
rb_ary_push(m_sexp, Qfalse);
|
62
62
|
}
|
63
63
|
|
64
|
-
virtual void
|
65
|
-
|
64
|
+
virtual void append_bignum(TM::String &number) override {
|
65
|
+
auto string_obj = rb_utf8_str_new(number.c_str(), number.length());
|
66
|
+
auto num = rb_Integer(string_obj);
|
67
|
+
rb_ary_push(m_sexp, num);
|
66
68
|
}
|
67
69
|
|
68
|
-
virtual void
|
69
|
-
|
70
|
+
virtual void append_fixnum(long long number) override {
|
71
|
+
auto num = rb_int_new(number);
|
72
|
+
rb_ary_push(m_sexp, num);
|
70
73
|
}
|
71
74
|
|
72
|
-
virtual void
|
73
|
-
auto
|
74
|
-
rb_ary_push(m_sexp,
|
75
|
+
virtual void append_float(double number) override {
|
76
|
+
auto num = rb_float_new(number);
|
77
|
+
rb_ary_push(m_sexp, num);
|
75
78
|
}
|
76
79
|
|
77
80
|
virtual void append_nil() override {
|
@@ -109,6 +112,21 @@ public:
|
|
109
112
|
rb_ary_push(m_sexp, Qtrue);
|
110
113
|
}
|
111
114
|
|
115
|
+
virtual void make_complex_number() override {
|
116
|
+
auto num = rb_ary_pop(m_sexp);
|
117
|
+
num = rb_Complex(INT2FIX(0), num);
|
118
|
+
rb_ary_push(m_sexp, num);
|
119
|
+
}
|
120
|
+
|
121
|
+
virtual void make_rational_number() override {
|
122
|
+
auto num = rb_ary_pop(m_sexp);
|
123
|
+
if (TYPE(num) == T_FLOAT)
|
124
|
+
num = rb_flt_rationalize(num);
|
125
|
+
else
|
126
|
+
num = rb_Rational(num, INT2FIX(1));
|
127
|
+
rb_ary_push(m_sexp, num);
|
128
|
+
}
|
129
|
+
|
112
130
|
virtual void wrap(const char *type) override {
|
113
131
|
auto inner = m_sexp;
|
114
132
|
reset_sexp();
|
@@ -42,15 +42,15 @@ public:
|
|
42
42
|
m_nodes.push("false");
|
43
43
|
}
|
44
44
|
|
45
|
-
virtual void
|
45
|
+
virtual void append_bignum(TM::String &number) override {
|
46
46
|
m_nodes.push(String(number));
|
47
47
|
}
|
48
48
|
|
49
|
-
virtual void
|
49
|
+
virtual void append_fixnum(long long number) override {
|
50
50
|
m_nodes.push(String(number));
|
51
51
|
}
|
52
52
|
|
53
|
-
virtual void
|
53
|
+
virtual void append_float(double number) override {
|
54
54
|
m_nodes.push(String(number));
|
55
55
|
}
|
56
56
|
|
@@ -89,6 +89,16 @@ public:
|
|
89
89
|
m_nodes.push("true");
|
90
90
|
}
|
91
91
|
|
92
|
+
virtual void make_complex_number() override {
|
93
|
+
auto num = m_nodes.pop();
|
94
|
+
m_nodes.push(String::format("Complex(0, {})", num));
|
95
|
+
}
|
96
|
+
|
97
|
+
virtual void make_rational_number() override {
|
98
|
+
auto num = m_nodes.pop();
|
99
|
+
m_nodes.push(String::format("Rational({}, 1)", num));
|
100
|
+
}
|
101
|
+
|
92
102
|
virtual void wrap(const char *type) override {
|
93
103
|
auto inner = to_string();
|
94
104
|
m_nodes.clear();
|
@@ -27,9 +27,9 @@ public:
|
|
27
27
|
virtual void append_array(const TM::SharedPtr<ArrayNode> array) { append_array(*array); }
|
28
28
|
virtual void append_array(const ArrayNode &array) = 0;
|
29
29
|
virtual void append_false() = 0;
|
30
|
+
virtual void append_bignum(TM::String &number) = 0;
|
31
|
+
virtual void append_fixnum(long long number) = 0;
|
30
32
|
virtual void append_float(double number) = 0;
|
31
|
-
virtual void append_integer(long long number) = 0;
|
32
|
-
virtual void append_integer(TM::String &number) = 0;
|
33
33
|
virtual void append_nil() = 0;
|
34
34
|
virtual void append_range(long long first, long long last, bool exclude_end) = 0;
|
35
35
|
virtual void append_regexp(TM::String &pattern, int options) = 0;
|
@@ -37,6 +37,8 @@ public:
|
|
37
37
|
virtual void append_string(TM::String &string) = 0;
|
38
38
|
virtual void append_symbol(TM::String &symbol) = 0;
|
39
39
|
virtual void append_true() = 0;
|
40
|
+
virtual void make_complex_number() = 0;
|
41
|
+
virtual void make_rational_number() = 0;
|
40
42
|
virtual void wrap(const char *type) = 0;
|
41
43
|
|
42
44
|
virtual ~Creator() { }
|
@@ -0,0 +1,49 @@
|
|
1
|
+
#pragma once
|
2
|
+
|
3
|
+
#include "natalie_parser/node/bignum_node.hpp"
|
4
|
+
#include "natalie_parser/node/fixnum_node.hpp"
|
5
|
+
#include "natalie_parser/node/float_node.hpp"
|
6
|
+
#include "natalie_parser/node/node.hpp"
|
7
|
+
#include "natalie_parser/node/rational_node.hpp"
|
8
|
+
|
9
|
+
namespace NatalieParser {
|
10
|
+
|
11
|
+
using namespace TM;
|
12
|
+
|
13
|
+
class ComplexNode : public Node {
|
14
|
+
public:
|
15
|
+
ComplexNode(const Token &token, SharedPtr<Node> value)
|
16
|
+
: Node { token }
|
17
|
+
, m_value { value } { }
|
18
|
+
|
19
|
+
virtual Type type() const override { return Type::Complex; }
|
20
|
+
|
21
|
+
virtual void transform(Creator *creator) const override {
|
22
|
+
creator->set_type("lit");
|
23
|
+
transform_number(creator);
|
24
|
+
creator->make_complex_number();
|
25
|
+
}
|
26
|
+
|
27
|
+
void transform_number(Creator *creator) const {
|
28
|
+
switch (m_value->type()) {
|
29
|
+
case Node::Type::Bignum:
|
30
|
+
creator->append_bignum(m_value.static_cast_as<BignumNode>()->number().ref());
|
31
|
+
break;
|
32
|
+
case Node::Type::Fixnum:
|
33
|
+
creator->append_fixnum(m_value.static_cast_as<FixnumNode>()->number());
|
34
|
+
break;
|
35
|
+
case Node::Type::Float:
|
36
|
+
creator->append_float(m_value.static_cast_as<FloatNode>()->number());
|
37
|
+
break;
|
38
|
+
case Node::Type::Rational:
|
39
|
+
m_value.static_cast_as<RationalNode>()->transform_number(creator);
|
40
|
+
break;
|
41
|
+
default:
|
42
|
+
TM_UNREACHABLE();
|
43
|
+
}
|
44
|
+
}
|
45
|
+
|
46
|
+
protected:
|
47
|
+
SharedPtr<Node> m_value;
|
48
|
+
};
|
49
|
+
}
|
@@ -0,0 +1,26 @@
|
|
1
|
+
#pragma once
|
2
|
+
|
3
|
+
#include "natalie_parser/node/node.hpp"
|
4
|
+
#include "tm/hashmap.hpp"
|
5
|
+
|
6
|
+
namespace NatalieParser {
|
7
|
+
|
8
|
+
using namespace TM;
|
9
|
+
|
10
|
+
class ForwardArgsNode : public Node {
|
11
|
+
public:
|
12
|
+
ForwardArgsNode(const Token &token)
|
13
|
+
: Node { token } { }
|
14
|
+
|
15
|
+
virtual Type type() const override { return Type::ForwardArgs; }
|
16
|
+
|
17
|
+
void add_to_locals(TM::Hashmap<TM::String> &locals) {
|
18
|
+
locals.set("...");
|
19
|
+
}
|
20
|
+
|
21
|
+
virtual void transform(Creator *creator) const override {
|
22
|
+
creator->set_type("forward_args");
|
23
|
+
}
|
24
|
+
};
|
25
|
+
|
26
|
+
}
|
@@ -33,6 +33,7 @@ public:
|
|
33
33
|
Class,
|
34
34
|
Colon2,
|
35
35
|
Colon3,
|
36
|
+
Complex,
|
36
37
|
Constant,
|
37
38
|
Def,
|
38
39
|
Defined,
|
@@ -42,6 +43,7 @@ public:
|
|
42
43
|
False,
|
43
44
|
Fixnum,
|
44
45
|
Float,
|
46
|
+
ForwardArgs,
|
45
47
|
Hash,
|
46
48
|
HashPattern,
|
47
49
|
Identifier,
|
@@ -73,6 +75,7 @@ public:
|
|
73
75
|
OpAssignOr,
|
74
76
|
Pin,
|
75
77
|
Range,
|
78
|
+
Rational,
|
76
79
|
Redo,
|
77
80
|
Regexp,
|
78
81
|
Retry,
|
@@ -122,7 +125,7 @@ public:
|
|
122
125
|
|
123
126
|
virtual void transform(Creator *creator) const {
|
124
127
|
creator->set_type("NOT_YET_IMPLEMENTED");
|
125
|
-
creator->
|
128
|
+
creator->append_fixnum((int)type());
|
126
129
|
}
|
127
130
|
|
128
131
|
SharedPtr<String> file() const { return m_token.file(); }
|
@@ -0,0 +1,45 @@
|
|
1
|
+
#pragma once
|
2
|
+
|
3
|
+
#include "natalie_parser/node/bignum_node.hpp"
|
4
|
+
#include "natalie_parser/node/fixnum_node.hpp"
|
5
|
+
#include "natalie_parser/node/float_node.hpp"
|
6
|
+
#include "natalie_parser/node/node.hpp"
|
7
|
+
|
8
|
+
namespace NatalieParser {
|
9
|
+
|
10
|
+
using namespace TM;
|
11
|
+
|
12
|
+
class RationalNode : public Node {
|
13
|
+
public:
|
14
|
+
RationalNode(const Token &token, SharedPtr<Node> value)
|
15
|
+
: Node { token }
|
16
|
+
, m_value { value } { }
|
17
|
+
|
18
|
+
virtual Type type() const override { return Type::Rational; }
|
19
|
+
|
20
|
+
virtual void transform(Creator *creator) const override {
|
21
|
+
creator->set_type("lit");
|
22
|
+
transform_number(creator);
|
23
|
+
creator->make_rational_number();
|
24
|
+
}
|
25
|
+
|
26
|
+
void transform_number(Creator *creator) const {
|
27
|
+
switch (m_value->type()) {
|
28
|
+
case Node::Type::Bignum:
|
29
|
+
creator->append_bignum(m_value.static_cast_as<BignumNode>()->number().ref());
|
30
|
+
break;
|
31
|
+
case Node::Type::Fixnum:
|
32
|
+
creator->append_fixnum(m_value.static_cast_as<FixnumNode>()->number());
|
33
|
+
break;
|
34
|
+
case Node::Type::Float:
|
35
|
+
creator->append_float(m_value.static_cast_as<FloatNode>()->number());
|
36
|
+
break;
|
37
|
+
default:
|
38
|
+
TM_UNREACHABLE();
|
39
|
+
}
|
40
|
+
}
|
41
|
+
|
42
|
+
protected:
|
43
|
+
SharedPtr<Node> m_value;
|
44
|
+
};
|
45
|
+
}
|
@@ -21,6 +21,7 @@
|
|
21
21
|
#include "natalie_parser/node/class_node.hpp"
|
22
22
|
#include "natalie_parser/node/colon2_node.hpp"
|
23
23
|
#include "natalie_parser/node/colon3_node.hpp"
|
24
|
+
#include "natalie_parser/node/complex_node.hpp"
|
24
25
|
#include "natalie_parser/node/constant_node.hpp"
|
25
26
|
#include "natalie_parser/node/def_node.hpp"
|
26
27
|
#include "natalie_parser/node/defined_node.hpp"
|
@@ -30,6 +31,7 @@
|
|
30
31
|
#include "natalie_parser/node/false_node.hpp"
|
31
32
|
#include "natalie_parser/node/fixnum_node.hpp"
|
32
33
|
#include "natalie_parser/node/float_node.hpp"
|
34
|
+
#include "natalie_parser/node/forward_args_node.hpp"
|
33
35
|
#include "natalie_parser/node/hash_node.hpp"
|
34
36
|
#include "natalie_parser/node/hash_pattern_node.hpp"
|
35
37
|
#include "natalie_parser/node/identifier_node.hpp"
|
@@ -63,6 +65,7 @@
|
|
63
65
|
#include "natalie_parser/node/op_assign_or_node.hpp"
|
64
66
|
#include "natalie_parser/node/pin_node.hpp"
|
65
67
|
#include "natalie_parser/node/range_node.hpp"
|
68
|
+
#include "natalie_parser/node/rational_node.hpp"
|
66
69
|
#include "natalie_parser/node/redo_node.hpp"
|
67
70
|
#include "natalie_parser/node/regexp_node.hpp"
|
68
71
|
#include "natalie_parser/node/retry_node.hpp"
|
@@ -87,11 +87,19 @@ private:
|
|
87
87
|
SharedPtr<Node> parse_constant(LocalsHashmap &);
|
88
88
|
SharedPtr<Node> parse_def(LocalsHashmap &);
|
89
89
|
SharedPtr<Node> parse_defined(LocalsHashmap &);
|
90
|
+
|
90
91
|
void parse_def_args(Vector<SharedPtr<Node>> &, LocalsHashmap &);
|
91
|
-
|
92
|
+
enum class ArgsContext {
|
93
|
+
Block,
|
94
|
+
Method,
|
95
|
+
Proc,
|
96
|
+
};
|
97
|
+
void parse_def_single_arg(Vector<SharedPtr<Node>> &, LocalsHashmap &, ArgsContext);
|
98
|
+
|
92
99
|
SharedPtr<Node> parse_encoding(LocalsHashmap &);
|
93
100
|
SharedPtr<Node> parse_end_block(LocalsHashmap &);
|
94
101
|
SharedPtr<Node> parse_file_constant(LocalsHashmap &);
|
102
|
+
SharedPtr<Node> parse_forward_args(LocalsHashmap &);
|
95
103
|
SharedPtr<Node> parse_group(LocalsHashmap &);
|
96
104
|
SharedPtr<Node> parse_hash(LocalsHashmap &);
|
97
105
|
SharedPtr<Node> parse_hash_inner(LocalsHashmap &, Precedence, Token::Type, SharedPtr<Node> = {});
|
@@ -128,6 +136,7 @@ private:
|
|
128
136
|
SharedPtr<Node> parse_symbol_key(LocalsHashmap &);
|
129
137
|
SharedPtr<Node> parse_statement_keyword(LocalsHashmap &);
|
130
138
|
SharedPtr<Node> parse_top_level_constant(LocalsHashmap &);
|
139
|
+
SharedPtr<Node> parse_triple_dot(LocalsHashmap &);
|
131
140
|
SharedPtr<Node> parse_unary_operator(LocalsHashmap &);
|
132
141
|
SharedPtr<Node> parse_undef(LocalsHashmap &);
|
133
142
|
SharedPtr<Node> parse_unless(LocalsHashmap &);
|
@@ -195,6 +204,7 @@ private:
|
|
195
204
|
void skip_newlines();
|
196
205
|
|
197
206
|
void expect(Token::Type, const char *);
|
207
|
+
[[noreturn]] void throw_error(const Token &, const char *);
|
198
208
|
[[noreturn]] void throw_unexpected(const Token &, const char *, const char * = nullptr);
|
199
209
|
[[noreturn]] void throw_unexpected(const char *);
|
200
210
|
[[noreturn]] void throw_unterminated_thing(Token, Token = {});
|
@@ -36,6 +36,7 @@ public:
|
|
36
36
|
Comma,
|
37
37
|
Comment,
|
38
38
|
Comparison,
|
39
|
+
Complex,
|
39
40
|
Constant,
|
40
41
|
ConstantResolution,
|
41
42
|
DefinedKeyword,
|
@@ -115,6 +116,8 @@ public:
|
|
115
116
|
PipePipeEqual,
|
116
117
|
Plus,
|
117
118
|
PlusEqual,
|
119
|
+
Rational,
|
120
|
+
RationalComplex,
|
118
121
|
RCurlyBrace,
|
119
122
|
RBracket,
|
120
123
|
RedoKeyword,
|
@@ -286,6 +289,8 @@ public:
|
|
286
289
|
return "comment";
|
287
290
|
case Type::Comparison:
|
288
291
|
return "<=>";
|
292
|
+
case Type::Complex:
|
293
|
+
return "complex";
|
289
294
|
case Type::ConstantResolution:
|
290
295
|
return "::";
|
291
296
|
case Type::Constant:
|
@@ -446,6 +451,10 @@ public:
|
|
446
451
|
return "+=";
|
447
452
|
case Type::Plus:
|
448
453
|
return "+";
|
454
|
+
case Type::Rational:
|
455
|
+
return "rational";
|
456
|
+
case Type::RationalComplex:
|
457
|
+
return "rational_complex";
|
449
458
|
case Type::RCurlyBrace:
|
450
459
|
return "}";
|
451
460
|
case Type::RBracket:
|
@@ -804,6 +813,17 @@ public:
|
|
804
813
|
}
|
805
814
|
}
|
806
815
|
|
816
|
+
bool can_be_complex_or_rational() const {
|
817
|
+
switch (m_type) {
|
818
|
+
case Type::Bignum:
|
819
|
+
case Type::Fixnum:
|
820
|
+
case Type::Float:
|
821
|
+
return true;
|
822
|
+
default:
|
823
|
+
return false;
|
824
|
+
}
|
825
|
+
}
|
826
|
+
|
807
827
|
void set_literal(const char *literal) { m_literal = new String(literal); }
|
808
828
|
void set_literal(SharedPtr<String> literal) { m_literal = literal; }
|
809
829
|
void set_literal(String literal) { m_literal = new String(literal); }
|
data/src/lexer.cpp
CHANGED
@@ -842,6 +842,23 @@ Token Lexer::build_next_token() {
|
|
842
842
|
auto token = consume_numeric();
|
843
843
|
return token;
|
844
844
|
}
|
845
|
+
case 'i':
|
846
|
+
if (m_last_token.can_be_complex_or_rational() && !isalnum(peek())) {
|
847
|
+
advance();
|
848
|
+
return Token { Token::Type::Complex, m_file, m_token_line, m_token_column };
|
849
|
+
}
|
850
|
+
break;
|
851
|
+
case 'r':
|
852
|
+
if (m_last_token.can_be_complex_or_rational()) {
|
853
|
+
if (peek() == 'i') {
|
854
|
+
advance(2);
|
855
|
+
return Token { Token::Type::RationalComplex, m_file, m_token_line, m_token_column };
|
856
|
+
} else if (!isalnum(peek())) {
|
857
|
+
advance();
|
858
|
+
return Token { Token::Type::Rational, m_file, m_token_line, m_token_column };
|
859
|
+
}
|
860
|
+
}
|
861
|
+
break;
|
845
862
|
};
|
846
863
|
|
847
864
|
Token keyword_token;
|
@@ -1322,7 +1339,25 @@ Token Lexer::consume_heredoc() {
|
|
1322
1339
|
|
1323
1340
|
Token Lexer::consume_numeric() {
|
1324
1341
|
SharedPtr<String> chars = new String;
|
1342
|
+
|
1343
|
+
auto consume_decimal_digits_and_build_token = [&]() {
|
1344
|
+
char c = current_char();
|
1345
|
+
do {
|
1346
|
+
chars->append_char(c);
|
1347
|
+
c = next();
|
1348
|
+
if (c == '_')
|
1349
|
+
c = next();
|
1350
|
+
} while (isdigit(c));
|
1351
|
+
if ((c == '.' && isdigit(peek())) || (c == 'e' || c == 'E'))
|
1352
|
+
return consume_numeric_as_float(chars);
|
1353
|
+
else
|
1354
|
+
return chars_to_fixnum_or_bignum_token(chars, 10, 0);
|
1355
|
+
};
|
1356
|
+
|
1357
|
+
Token token;
|
1358
|
+
|
1325
1359
|
if (current_char() == '0') {
|
1360
|
+
// special-prefixed literals 0d, 0x, etc.
|
1326
1361
|
switch (peek()) {
|
1327
1362
|
case 'd':
|
1328
1363
|
case 'D': {
|
@@ -1336,7 +1371,8 @@ Token Lexer::consume_numeric() {
|
|
1336
1371
|
if (c == '_')
|
1337
1372
|
c = next();
|
1338
1373
|
} while (isdigit(c));
|
1339
|
-
|
1374
|
+
token = chars_to_fixnum_or_bignum_token(chars, 10, 0);
|
1375
|
+
break;
|
1340
1376
|
}
|
1341
1377
|
case 'o':
|
1342
1378
|
case 'O': {
|
@@ -1352,7 +1388,8 @@ Token Lexer::consume_numeric() {
|
|
1352
1388
|
if (c == '_')
|
1353
1389
|
c = next();
|
1354
1390
|
} while (c >= '0' && c <= '7');
|
1355
|
-
|
1391
|
+
token = chars_to_fixnum_or_bignum_token(chars, 8, 2);
|
1392
|
+
break;
|
1356
1393
|
}
|
1357
1394
|
case 'x':
|
1358
1395
|
case 'X': {
|
@@ -1368,7 +1405,8 @@ Token Lexer::consume_numeric() {
|
|
1368
1405
|
if (c == '_')
|
1369
1406
|
c = next();
|
1370
1407
|
} while (isxdigit(c));
|
1371
|
-
|
1408
|
+
token = chars_to_fixnum_or_bignum_token(chars, 16, 2);
|
1409
|
+
break;
|
1372
1410
|
}
|
1373
1411
|
case 'b':
|
1374
1412
|
case 'B': {
|
@@ -1384,21 +1422,17 @@ Token Lexer::consume_numeric() {
|
|
1384
1422
|
if (c == '_')
|
1385
1423
|
c = next();
|
1386
1424
|
} while (c == '0' || c == '1');
|
1387
|
-
|
1425
|
+
token = chars_to_fixnum_or_bignum_token(chars, 2, 2);
|
1426
|
+
break;
|
1388
1427
|
}
|
1428
|
+
default:
|
1429
|
+
token = consume_decimal_digits_and_build_token();
|
1389
1430
|
}
|
1431
|
+
} else {
|
1432
|
+
token = consume_decimal_digits_and_build_token();
|
1390
1433
|
}
|
1391
|
-
|
1392
|
-
|
1393
|
-
chars->append_char(c);
|
1394
|
-
c = next();
|
1395
|
-
if (c == '_')
|
1396
|
-
c = next();
|
1397
|
-
} while (isdigit(c));
|
1398
|
-
if ((c == '.' && isdigit(peek())) || (c == 'e' || c == 'E'))
|
1399
|
-
return consume_numeric_as_float(chars);
|
1400
|
-
else
|
1401
|
-
return chars_to_fixnum_or_bignum_token(chars, 10, 0);
|
1434
|
+
|
1435
|
+
return token;
|
1402
1436
|
}
|
1403
1437
|
|
1404
1438
|
const long long max_fixnum = std::numeric_limits<long long>::max() / 2; // 63 bits for MRI
|
data/src/node/node_with_args.cpp
CHANGED
@@ -17,6 +17,7 @@ void NodeWithArgs::append_method_or_block_args(Creator *creator) const {
|
|
17
17
|
arg.static_cast_as<ArgNode>()->append_name(c);
|
18
18
|
break;
|
19
19
|
}
|
20
|
+
case Node::Type::ForwardArgs:
|
20
21
|
case Node::Type::KeywordArg:
|
21
22
|
case Node::Type::MultipleAssignmentArg:
|
22
23
|
case Node::Type::ShadowArg:
|
data/src/parser.cpp
CHANGED
@@ -16,7 +16,7 @@ enum class Parser::Precedence {
|
|
16
16
|
INLINE_RESCUE, // foo rescue 2
|
17
17
|
ITER_BLOCK, // do |n| ... end
|
18
18
|
BARE_CALL_ARG, // foo (_), b
|
19
|
-
|
19
|
+
OP_ASSIGNMENT_RHS, // x += (_)
|
20
20
|
TERNARY_TRUE, // _ ? (_) : _
|
21
21
|
CALL_ARG, // foo( (_), b )
|
22
22
|
TERNARY_QUESTION, // (_) ? _ : _
|
@@ -28,6 +28,7 @@ enum class Parser::Precedence {
|
|
28
28
|
LOGICAL_NOT, // not
|
29
29
|
EQUALITY, // <=> == === != =~ !~
|
30
30
|
LESS_GREATER, // <= < > >=
|
31
|
+
OP_ASSIGNMENT_LHS, // (_) += 1
|
31
32
|
BITWISE_OR, // ^ |
|
32
33
|
BITWISE_AND, // &
|
33
34
|
BITWISE_SHIFT, // << >>
|
@@ -112,7 +113,7 @@ Parser::Precedence Parser::get_precedence(Token &token, SharedPtr<Node> left) {
|
|
112
113
|
case Token::Type::SlashEqual:
|
113
114
|
case Token::Type::StarEqual:
|
114
115
|
case Token::Type::StarStarEqual:
|
115
|
-
return Precedence::
|
116
|
+
return Precedence::OP_ASSIGNMENT_LHS;
|
116
117
|
case Token::Type::Ampersand:
|
117
118
|
return Precedence::BITWISE_AND;
|
118
119
|
case Token::Type::Caret:
|
@@ -979,28 +980,32 @@ SharedPtr<Node> Parser::parse_defined(LocalsHashmap &locals) {
|
|
979
980
|
}
|
980
981
|
|
981
982
|
void Parser::parse_def_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals) {
|
982
|
-
parse_def_single_arg(args, locals);
|
983
|
+
parse_def_single_arg(args, locals, ArgsContext::Method);
|
983
984
|
while (current_token().is_comma()) {
|
984
985
|
advance();
|
985
|
-
parse_def_single_arg(args, locals);
|
986
|
+
parse_def_single_arg(args, locals, ArgsContext::Method);
|
986
987
|
}
|
987
988
|
}
|
988
989
|
|
989
|
-
void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals) {
|
990
|
+
void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &locals, ArgsContext context) {
|
990
991
|
auto args_have_any_splat = [&]() { return !args.is_empty() && args.last()->type() == Node::Type::Arg && args.last().static_cast_as<ArgNode>()->splat_or_kwsplat(); };
|
991
992
|
auto args_have_keyword = [&]() { return !args.is_empty() && args.last()->type() == Node::Type::KeywordArg; };
|
992
993
|
|
993
994
|
auto token = current_token();
|
995
|
+
|
996
|
+
if (!args.is_empty() && args.last()->type() == Node::Type::ForwardArgs)
|
997
|
+
throw_error(token, "anything after arg forwarding (...) shorthand");
|
998
|
+
|
994
999
|
switch (token.type()) {
|
995
1000
|
case Token::Type::BareName: {
|
996
1001
|
if (args_have_keyword())
|
997
|
-
|
1002
|
+
throw_error(token, "normal arg after keyword arg");
|
998
1003
|
SharedPtr<ArgNode> arg = new ArgNode { token, token.literal_string() };
|
999
1004
|
advance();
|
1000
1005
|
arg->add_to_locals(locals);
|
1001
1006
|
if (current_token().is_equal()) {
|
1002
1007
|
if (args_have_any_splat())
|
1003
|
-
|
1008
|
+
throw_error(token, "default value after splat");
|
1004
1009
|
advance(); // =
|
1005
1010
|
arg->set_value(parse_expression(Precedence::DEF_ARG, locals));
|
1006
1011
|
}
|
@@ -1022,7 +1027,7 @@ void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &
|
|
1022
1027
|
}
|
1023
1028
|
case Token::Type::Star: {
|
1024
1029
|
if (args_have_any_splat())
|
1025
|
-
|
1030
|
+
throw_error(token, "splat after keyword splat");
|
1026
1031
|
advance();
|
1027
1032
|
SharedPtr<ArgNode> arg;
|
1028
1033
|
if (current_token().is_bare_name()) {
|
@@ -1080,6 +1085,15 @@ void Parser::parse_def_single_arg(Vector<SharedPtr<Node>> &args, LocalsHashmap &
|
|
1080
1085
|
args.push(arg.static_cast_as<Node>());
|
1081
1086
|
return;
|
1082
1087
|
}
|
1088
|
+
case Token::Type::DotDotDot: {
|
1089
|
+
if (context == ArgsContext::Block)
|
1090
|
+
throw_error(token, "arg forwarding (...) shorthand not allowed in block");
|
1091
|
+
SharedPtr<ForwardArgsNode> arg = new ForwardArgsNode { token };
|
1092
|
+
advance();
|
1093
|
+
arg->add_to_locals(locals);
|
1094
|
+
args.push(arg.static_cast_as<Node>());
|
1095
|
+
return;
|
1096
|
+
}
|
1083
1097
|
default:
|
1084
1098
|
throw_unexpected("argument");
|
1085
1099
|
}
|
@@ -1151,6 +1165,15 @@ SharedPtr<Node> Parser::parse_file_constant(LocalsHashmap &) {
|
|
1151
1165
|
return new StringNode { token, token.file() };
|
1152
1166
|
}
|
1153
1167
|
|
1168
|
+
SharedPtr<Node> Parser::parse_forward_args(LocalsHashmap &locals) {
|
1169
|
+
auto token = current_token();
|
1170
|
+
if (!locals.get("..."))
|
1171
|
+
throw_error(token, "forwarding args without ... shorthand in method definition");
|
1172
|
+
advance(); // ...
|
1173
|
+
SharedPtr<ForwardArgsNode> node = new ForwardArgsNode { token };
|
1174
|
+
return node.static_cast_as<Node>();
|
1175
|
+
}
|
1176
|
+
|
1154
1177
|
SharedPtr<Node> Parser::parse_group(LocalsHashmap &locals) {
|
1155
1178
|
auto token = current_token();
|
1156
1179
|
advance(); // (
|
@@ -1491,19 +1514,40 @@ SharedPtr<Node> Parser::parse_interpolated_symbol(LocalsHashmap &locals) {
|
|
1491
1514
|
|
1492
1515
|
SharedPtr<Node> Parser::parse_lit(LocalsHashmap &) {
|
1493
1516
|
auto token = current_token();
|
1517
|
+
SharedPtr<Node> node;
|
1494
1518
|
switch (token.type()) {
|
1495
1519
|
case Token::Type::Bignum:
|
1496
1520
|
advance();
|
1497
|
-
|
1521
|
+
node = new BignumNode { token, token.literal_string() };
|
1522
|
+
break;
|
1498
1523
|
case Token::Type::Fixnum:
|
1499
1524
|
advance();
|
1500
|
-
|
1525
|
+
node = new FixnumNode { token, token.get_fixnum() };
|
1526
|
+
break;
|
1501
1527
|
case Token::Type::Float:
|
1502
1528
|
advance();
|
1503
|
-
|
1529
|
+
node = new FloatNode { token, token.get_double() };
|
1530
|
+
break;
|
1504
1531
|
default:
|
1505
1532
|
TM_UNREACHABLE();
|
1506
1533
|
}
|
1534
|
+
switch (current_token().type()) {
|
1535
|
+
case Token::Type::Complex:
|
1536
|
+
advance();
|
1537
|
+
node = new ComplexNode { token, node };
|
1538
|
+
break;
|
1539
|
+
case Token::Type::Rational:
|
1540
|
+
advance();
|
1541
|
+
node = new RationalNode { token, node };
|
1542
|
+
break;
|
1543
|
+
case Token::Type::RationalComplex:
|
1544
|
+
advance();
|
1545
|
+
node = new ComplexNode { token, new RationalNode { token, node } };
|
1546
|
+
break;
|
1547
|
+
default:
|
1548
|
+
break;
|
1549
|
+
}
|
1550
|
+
return node;
|
1507
1551
|
};
|
1508
1552
|
|
1509
1553
|
SharedPtr<Node> Parser::parse_keyword_splat(LocalsHashmap &locals) {
|
@@ -1605,10 +1649,10 @@ void Parser::parse_proc_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &local
|
|
1605
1649
|
parse_shadow_variables_in_args(args, locals);
|
1606
1650
|
return;
|
1607
1651
|
}
|
1608
|
-
parse_def_single_arg(args, locals);
|
1652
|
+
parse_def_single_arg(args, locals, ArgsContext::Proc);
|
1609
1653
|
while (current_token().is_comma()) {
|
1610
1654
|
advance();
|
1611
|
-
parse_def_single_arg(args, locals);
|
1655
|
+
parse_def_single_arg(args, locals, ArgsContext::Proc);
|
1612
1656
|
}
|
1613
1657
|
if (current_token().is_semicolon()) {
|
1614
1658
|
parse_shadow_variables_in_args(args, locals);
|
@@ -2101,7 +2145,7 @@ void Parser::parse_iter_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &local
|
|
2101
2145
|
parse_shadow_variables_in_args(args, locals);
|
2102
2146
|
return;
|
2103
2147
|
}
|
2104
|
-
parse_def_single_arg(args, locals);
|
2148
|
+
parse_def_single_arg(args, locals, ArgsContext::Block);
|
2105
2149
|
while (current_token().is_comma()) {
|
2106
2150
|
advance();
|
2107
2151
|
if (current_token().is_block_arg_delimiter()) {
|
@@ -2109,7 +2153,7 @@ void Parser::parse_iter_args(Vector<SharedPtr<Node>> &args, LocalsHashmap &local
|
|
2109
2153
|
args.push(new NilNode { current_token() });
|
2110
2154
|
break;
|
2111
2155
|
}
|
2112
|
-
parse_def_single_arg(args, locals);
|
2156
|
+
parse_def_single_arg(args, locals, ArgsContext::Block);
|
2113
2157
|
}
|
2114
2158
|
if (current_token().is_semicolon()) {
|
2115
2159
|
parse_shadow_variables_in_args(args, locals);
|
@@ -2156,7 +2200,7 @@ SharedPtr<NodeWithArgs> Parser::to_node_with_args(SharedPtr<Node> node) {
|
|
2156
2200
|
case Node::Type::Yield:
|
2157
2201
|
return node.static_cast_as<NodeWithArgs>();
|
2158
2202
|
default:
|
2159
|
-
|
2203
|
+
throw_error(current_token(), "left-hand-side is not callable");
|
2160
2204
|
}
|
2161
2205
|
}
|
2162
2206
|
|
@@ -2381,7 +2425,7 @@ SharedPtr<Node> Parser::parse_op_attr_assign_expression(SharedPtr<Node> left, Lo
|
|
2381
2425
|
auto left_call = left.static_cast_as<CallNode>();
|
2382
2426
|
auto token = current_token();
|
2383
2427
|
advance();
|
2384
|
-
auto value = parse_expression(Precedence::
|
2428
|
+
auto value = parse_expression(Precedence::OP_ASSIGNMENT_RHS, locals);
|
2385
2429
|
|
2386
2430
|
if (*left_call->message() != "[]") {
|
2387
2431
|
if (token.type() == Token::Type::AmpersandAmpersandEqual) {
|
@@ -2536,6 +2580,12 @@ SharedPtr<Node> Parser::parse_ternary_expression(SharedPtr<Node> left, LocalsHas
|
|
2536
2580
|
return new IfNode { token, left, true_expr, false_expr };
|
2537
2581
|
}
|
2538
2582
|
|
2583
|
+
SharedPtr<Node> Parser::parse_triple_dot(LocalsHashmap &locals) {
|
2584
|
+
if (peek_token().is_rparen())
|
2585
|
+
return parse_forward_args(locals);
|
2586
|
+
return parse_beginless_range(locals);
|
2587
|
+
}
|
2588
|
+
|
2539
2589
|
SharedPtr<Node> Parser::parse_unless(LocalsHashmap &locals) {
|
2540
2590
|
auto token = current_token();
|
2541
2591
|
advance();
|
@@ -2603,7 +2653,7 @@ Parser::parse_null_fn Parser::null_denotation(Token::Type type) {
|
|
2603
2653
|
return &Parser::parse_defined;
|
2604
2654
|
case Type::DotDot:
|
2605
2655
|
case Type::DotDotDot:
|
2606
|
-
return &Parser::
|
2656
|
+
return &Parser::parse_triple_dot;
|
2607
2657
|
case Type::ENCODINGKeyword:
|
2608
2658
|
return &Parser::parse_encoding;
|
2609
2659
|
case Type::ENDKeyword:
|
@@ -2826,6 +2876,10 @@ void Parser::expect(Token::Type type, const char *expected) {
|
|
2826
2876
|
throw_unexpected(expected);
|
2827
2877
|
}
|
2828
2878
|
|
2879
|
+
void Parser::throw_error(const Token &token, const char *error) {
|
2880
|
+
throw_unexpected(token, nullptr, error);
|
2881
|
+
}
|
2882
|
+
|
2829
2883
|
void Parser::throw_unexpected(const Token &token, const char *expected, const char *error) {
|
2830
2884
|
auto file = token.file() ? String(*token.file()) : String("(unknown)");
|
2831
2885
|
auto line = token.line() + 1;
|
@@ -2881,20 +2935,22 @@ void Parser::throw_unterminated_thing(Token token, Token start_token) {
|
|
2881
2935
|
auto indent = String { start_token.column(), ' ' };
|
2882
2936
|
String expected;
|
2883
2937
|
const char *lit = start_token.literal();
|
2884
|
-
|
2885
|
-
|
2886
|
-
|
2887
|
-
|
2888
|
-
|
2889
|
-
|
2890
|
-
|
2891
|
-
|
2892
|
-
|
2893
|
-
|
2894
|
-
|
2895
|
-
|
2896
|
-
|
2897
|
-
|
2938
|
+
if (lit) {
|
2939
|
+
if (strcmp(lit, "(") == 0)
|
2940
|
+
expected = "')'";
|
2941
|
+
else if (strcmp(lit, "[") == 0)
|
2942
|
+
expected = "']'";
|
2943
|
+
else if (strcmp(lit, "{") == 0)
|
2944
|
+
expected = "'}'";
|
2945
|
+
else if (strcmp(lit, "<") == 0)
|
2946
|
+
expected = "'>'";
|
2947
|
+
else if (strcmp(lit, "'") == 0)
|
2948
|
+
expected = "\"'\"";
|
2949
|
+
else
|
2950
|
+
expected = String::format("'{}'", lit);
|
2951
|
+
} else {
|
2952
|
+
expected = "delimiter"; // FIXME: why do we not know what this delimiter is?
|
2953
|
+
}
|
2898
2954
|
const char *thing = nullptr;
|
2899
2955
|
switch (token.type()) {
|
2900
2956
|
case Token::Type::InterpolatedRegexpBegin:
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: natalie_parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tim Morgan
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-06-
|
11
|
+
date: 2022-06-04 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: NatalieParser is a zero-dependency, from-scratch, hand-written recursive
|
14
14
|
descent parser for the Ruby Programming Language.
|
@@ -55,6 +55,7 @@ files:
|
|
55
55
|
- include/natalie_parser/node/class_node.hpp
|
56
56
|
- include/natalie_parser/node/colon2_node.hpp
|
57
57
|
- include/natalie_parser/node/colon3_node.hpp
|
58
|
+
- include/natalie_parser/node/complex_node.hpp
|
58
59
|
- include/natalie_parser/node/constant_node.hpp
|
59
60
|
- include/natalie_parser/node/def_node.hpp
|
60
61
|
- include/natalie_parser/node/defined_node.hpp
|
@@ -64,6 +65,7 @@ files:
|
|
64
65
|
- include/natalie_parser/node/false_node.hpp
|
65
66
|
- include/natalie_parser/node/fixnum_node.hpp
|
66
67
|
- include/natalie_parser/node/float_node.hpp
|
68
|
+
- include/natalie_parser/node/forward_args_node.hpp
|
67
69
|
- include/natalie_parser/node/hash_node.hpp
|
68
70
|
- include/natalie_parser/node/hash_pattern_node.hpp
|
69
71
|
- include/natalie_parser/node/identifier_node.hpp
|
@@ -98,6 +100,7 @@ files:
|
|
98
100
|
- include/natalie_parser/node/op_assign_or_node.hpp
|
99
101
|
- include/natalie_parser/node/pin_node.hpp
|
100
102
|
- include/natalie_parser/node/range_node.hpp
|
103
|
+
- include/natalie_parser/node/rational_node.hpp
|
101
104
|
- include/natalie_parser/node/redo_node.hpp
|
102
105
|
- include/natalie_parser/node/regexp_node.hpp
|
103
106
|
- include/natalie_parser/node/retry_node.hpp
|