parser 2.0.0.beta6 → 2.0.0.beta7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +15 -0
- data/CHANGELOG.md +12 -0
- data/README.md +1 -1
- data/Rakefile +15 -0
- data/lib/parser.rb +39 -39
- data/lib/parser/base.rb +2 -5
- data/lib/parser/builders/default.rb +25 -5
- data/lib/parser/lexer/explanation.rb +1 -1
- data/lib/parser/ruby18.y +4 -9
- data/lib/parser/ruby19.y +4 -9
- data/lib/parser/ruby20.y +4 -9
- data/lib/parser/ruby21.y +4 -9
- data/lib/parser/runner.rb +7 -2
- data/lib/parser/runner/ruby_parse.rb +3 -3
- data/lib/parser/runner/ruby_rewrite.rb +1 -1
- data/lib/parser/source/buffer.rb +11 -9
- data/lib/parser/source/rewriter.rb +1 -1
- data/lib/parser/source/rewriter/action.rb +2 -2
- data/lib/parser/version.rb +1 -1
- data/test/helper.rb +2 -2
- data/test/parse_helper.rb +5 -5
- data/test/test_diagnostic.rb +7 -7
- data/test/test_diagnostic_engine.rb +5 -5
- data/test/test_encoding.rb +7 -7
- data/test/test_lexer.rb +2 -2
- data/test/test_parse_helper.rb +2 -2
- data/test/test_parser.rb +89 -56
- data/test/test_source_buffer.rb +6 -6
- data/test/test_source_range.rb +3 -3
- data/test/test_source_rewriter.rb +1 -1
- data/test/test_source_rewriter_action.rb +8 -8
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2ddc24b45d61fd0c8dbd0796770c2932bdd6b6e2
|
4
|
+
data.tar.gz: 25b8bbe0798e066f1fa005a88e5ea3e8cdf95a20
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3454dc2eb47f0555bb29c4cafe6ef65ecf18003b2dfc4406e00190652af8098f2a48160d1225ad36d655755a537de3115ccad033594bf0b946fe65687caa73ca
|
7
|
+
data.tar.gz: 05e32b3d2f67f0ff2c7d745efc63336f81ae83f22841f4cddb8cfe76e8a311886eeefe53eaed814c17655b05422899590afa141dca73a3e08e1daaa90f2dc0c6
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,18 @@
|
|
1
1
|
Changelog
|
2
2
|
=========
|
3
3
|
|
4
|
+
v2.0.0.beta7 (2013-06-22)
|
5
|
+
-------------------------
|
6
|
+
|
7
|
+
API modifications:
|
8
|
+
* Implement a much more sane encoding model (closes #60). (Peter Zotov)
|
9
|
+
|
10
|
+
Features implemented:
|
11
|
+
* builders/default: (while-post) and (until-post); (kwbegin) (fixes #70). (Peter Zotov)
|
12
|
+
|
13
|
+
Bugs fixed:
|
14
|
+
* builders/default: don't swallow (begin) in "if (foo); end" (fixes #75). (Peter Zotov)
|
15
|
+
|
4
16
|
v2.0.0.beta6 (2013-06-17)
|
5
17
|
-------------------------
|
6
18
|
|
data/README.md
CHANGED
@@ -133,7 +133,7 @@ The Bison parser rules are derived from [Ruby MRI](http://github.com/ruby/ruby)
|
|
133
133
|
|
134
134
|
## Contributing
|
135
135
|
|
136
|
-
1. Make sure you have [Ragel 6.
|
136
|
+
1. Make sure you have [Ragel ~> 6.7](http://www.complang.org/ragel/) installed
|
137
137
|
2. Fork it
|
138
138
|
3. Create your feature branch (`git checkout -b my-new-feature`)
|
139
139
|
4. Commit your changes (`git commit -am 'Add some feature'`)
|
data/Rakefile
CHANGED
@@ -23,6 +23,7 @@ CLEAN.include(GENERATED_FILES)
|
|
23
23
|
|
24
24
|
desc 'Generate the Ragel lexer and Bison parser.'
|
25
25
|
task :generate => GENERATED_FILES do
|
26
|
+
Rake::Task[:ragel_check].invoke
|
26
27
|
GENERATED_FILES.each do |filename|
|
27
28
|
content = File.read(filename)
|
28
29
|
content = "# -*- encoding:utf-8; warn-indent:false -*-\n" + content
|
@@ -42,6 +43,20 @@ task :clean_env do
|
|
42
43
|
ENV.delete 'RACC_DEBUG'
|
43
44
|
end
|
44
45
|
|
46
|
+
task :ragel_check do
|
47
|
+
major_req, minor_req = 6, 7
|
48
|
+
|
49
|
+
ragel_check = `which ragel && ragel --version`
|
50
|
+
ragel_version = ragel_check.match(/version (([0-9]+)\.([0-9]+))/)
|
51
|
+
raise 'command-line dependency ragel not installed!' unless ragel_version
|
52
|
+
|
53
|
+
_, version_str, major, minor = *ragel_version
|
54
|
+
if (major.to_i != major_req || minor.to_i < minor_req) # ~> major.minor
|
55
|
+
raise "command-line dependency ragel must be " +
|
56
|
+
"~> #{major_req}.#{minor_req}; got #{version_str}"
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
45
60
|
desc 'Generate YARD documentation'
|
46
61
|
task :yard => :generate do
|
47
62
|
sh('yard doc')
|
data/lib/parser.rb
CHANGED
@@ -57,61 +57,61 @@ module Parser
|
|
57
57
|
|
58
58
|
ERRORS = {
|
59
59
|
# Lexer errors
|
60
|
-
:unicode_point_too_large =>
|
61
|
-
:invalid_escape =>
|
62
|
-
:invalid_escape_use =>
|
63
|
-
:incomplete_escape =>
|
64
|
-
:invalid_hex_escape =>
|
65
|
-
:invalid_unicode_escape =>
|
66
|
-
:unterminated_unicode =>
|
67
|
-
:escape_eof =>
|
68
|
-
:string_eof =>
|
69
|
-
:regexp_options =>
|
60
|
+
:unicode_point_too_large => 'invalid Unicode codepoint (too large)',
|
61
|
+
:invalid_escape => 'invalid escape character syntax',
|
62
|
+
:invalid_escape_use => 'invalid character syntax; use ?%{escape}',
|
63
|
+
:incomplete_escape => 'incomplete character syntax',
|
64
|
+
:invalid_hex_escape => 'invalid hex escape',
|
65
|
+
:invalid_unicode_escape => 'invalid Unicode escape',
|
66
|
+
:unterminated_unicode => 'unterminated Unicode escape',
|
67
|
+
:escape_eof => 'escape sequence meets end of file',
|
68
|
+
:string_eof => 'unterminated string meets end of file',
|
69
|
+
:regexp_options => 'unknown regexp options: %{options}',
|
70
70
|
:cvar_name => "`%{name}' is not allowed as a class variable name",
|
71
71
|
:ivar_name => "`%{name}' is not allowed as an instance variable name",
|
72
72
|
:trailing_underscore => "trailing `_' in number",
|
73
|
-
:empty_numeric =>
|
74
|
-
:invalid_octal =>
|
75
|
-
:no_dot_digit_literal =>
|
76
|
-
:bare_backslash =>
|
73
|
+
:empty_numeric => 'numeric literal without digits',
|
74
|
+
:invalid_octal => 'invalid octal digit',
|
75
|
+
:no_dot_digit_literal => 'no .<digit> floating literal anymore; put 0 before dot',
|
76
|
+
:bare_backslash => 'bare backslash only allowed before newline',
|
77
77
|
:unexpected => "unexpected `%{character}'",
|
78
|
-
:embedded_document =>
|
78
|
+
:embedded_document => 'embedded document meats end of file (and they embark on a romantic journey)',
|
79
79
|
|
80
80
|
# Lexer warnings
|
81
|
-
:ambiguous_literal =>
|
81
|
+
:ambiguous_literal => 'ambiguous first argument; parenthesize arguments or add whitespace to the right',
|
82
82
|
:ambiguous_prefix => "`%{prefix}' interpreted as argument prefix",
|
83
83
|
|
84
84
|
# Parser errors
|
85
|
-
:nth_ref_alias =>
|
86
|
-
:begin_in_method =>
|
87
|
-
:backref_assignment =>
|
88
|
-
:invalid_assignment =>
|
89
|
-
:module_name_const =>
|
90
|
-
:unexpected_token =>
|
91
|
-
:argument_const =>
|
92
|
-
:argument_ivar =>
|
93
|
-
:argument_gvar =>
|
94
|
-
:argument_cvar =>
|
95
|
-
:duplicate_argument =>
|
96
|
-
:empty_symbol =>
|
97
|
-
:odd_hash =>
|
98
|
-
:singleton_literal =>
|
99
|
-
:dynamic_const =>
|
100
|
-
:module_in_def =>
|
101
|
-
:class_in_def =>
|
102
|
-
:unexpected_percent_str =>
|
103
|
-
:block_and_blockarg =>
|
104
|
-
:masgn_as_condition =>
|
85
|
+
:nth_ref_alias => 'cannot define an alias for a back-reference variable',
|
86
|
+
:begin_in_method => 'BEGIN in method',
|
87
|
+
:backref_assignment => 'cannot assign to a back-reference variable',
|
88
|
+
:invalid_assignment => 'cannot assign to a keyword',
|
89
|
+
:module_name_const => 'class or module name must be a constant literal',
|
90
|
+
:unexpected_token => 'unexpected token %{token}',
|
91
|
+
:argument_const => 'formal argument cannot be a constant',
|
92
|
+
:argument_ivar => 'formal argument cannot be an instance variable',
|
93
|
+
:argument_gvar => 'formal argument cannot be a global variable',
|
94
|
+
:argument_cvar => 'formal argument cannot be a class variable',
|
95
|
+
:duplicate_argument => 'duplicate argument name',
|
96
|
+
:empty_symbol => 'empty symbol literal',
|
97
|
+
:odd_hash => 'odd number of entries for a hash',
|
98
|
+
:singleton_literal => 'cannot define a singleton method for a literal',
|
99
|
+
:dynamic_const => 'dynamic constant assignment',
|
100
|
+
:module_in_def => 'module definition in method body',
|
101
|
+
:class_in_def => 'class definition in method body',
|
102
|
+
:unexpected_percent_str => '%{type}: unknown type of percent-literal',
|
103
|
+
:block_and_blockarg => 'both block argument and literal block are passed',
|
104
|
+
:masgn_as_condition => 'multiple assignment in conditional context',
|
105
105
|
|
106
106
|
# Parser warnings
|
107
|
-
:end_in_method =>
|
107
|
+
:end_in_method => 'END in method; use at_exit',
|
108
108
|
:space_before_lparen => "don't put space before argument parentheses",
|
109
|
-
:useless_else =>
|
109
|
+
:useless_else => 'else without rescue is useless',
|
110
110
|
}.freeze
|
111
111
|
|
112
112
|
def self.check_for_encoding_support
|
113
113
|
unless defined?(Encoding)
|
114
|
-
raise RuntimeError,
|
114
|
+
raise RuntimeError, 'Parsing 1.9 and later versions of Ruby is not supported on 1.8 due to the lack of Encoding support'
|
115
115
|
end
|
116
116
|
end
|
117
117
|
end
|
data/lib/parser/base.rb
CHANGED
@@ -12,6 +12,8 @@ module Parser
|
|
12
12
|
$stderr.puts(diagnostic.render)
|
13
13
|
end
|
14
14
|
|
15
|
+
string = string.dup.force_encoding(parser.default_encoding)
|
16
|
+
|
15
17
|
source_buffer = Source::Buffer.new(file, line)
|
16
18
|
source_buffer.source = string
|
17
19
|
|
@@ -106,11 +108,6 @@ module Parser
|
|
106
108
|
@lexer.advance
|
107
109
|
end
|
108
110
|
|
109
|
-
def value_expr(v)
|
110
|
-
#p 'value_expr', v
|
111
|
-
v
|
112
|
-
end
|
113
|
-
|
114
111
|
def check_kwarg_name(name_t)
|
115
112
|
case name_t[0]
|
116
113
|
when /^[a-z_]/
|
@@ -718,6 +718,10 @@ module Parser
|
|
718
718
|
end
|
719
719
|
|
720
720
|
def loop_mod(type, body, keyword_t, cond)
|
721
|
+
if body.type == :kwbegin
|
722
|
+
type = :"#{type}_post"
|
723
|
+
end
|
724
|
+
|
721
725
|
n(type, [ check_condition(cond), body ],
|
722
726
|
keyword_mod_map(body, keyword_t, cond))
|
723
727
|
end
|
@@ -802,7 +806,7 @@ module Parser
|
|
802
806
|
|
803
807
|
def begin(begin_t, body, end_t)
|
804
808
|
if body.nil?
|
805
|
-
# A nil expression: `()'
|
809
|
+
# A nil expression: `()'.
|
806
810
|
n0(:begin,
|
807
811
|
collection_map(begin_t, nil, end_t))
|
808
812
|
elsif body.type == :mlhs ||
|
@@ -818,7 +822,21 @@ module Parser
|
|
818
822
|
end
|
819
823
|
end
|
820
824
|
|
821
|
-
|
825
|
+
def begin_keyword(begin_t, body, end_t)
|
826
|
+
if body.nil?
|
827
|
+
# A nil expression: `begin end'.
|
828
|
+
n0(:kwbegin,
|
829
|
+
collection_map(begin_t, nil, end_t))
|
830
|
+
elsif (body.type == :begin &&
|
831
|
+
body.loc.begin.nil? && body.loc.end.nil?)
|
832
|
+
# Synthesized (begin) from compstmt "a; b".
|
833
|
+
n(:kwbegin, body.children,
|
834
|
+
collection_map(begin_t, body.children, end_t))
|
835
|
+
else
|
836
|
+
n(:kwbegin, [ body ],
|
837
|
+
collection_map(begin_t, [ body ], end_t))
|
838
|
+
end
|
839
|
+
end
|
822
840
|
|
823
841
|
private
|
824
842
|
|
@@ -834,7 +852,9 @@ module Parser
|
|
834
852
|
|
835
853
|
when :begin
|
836
854
|
if cond.children.count == 1
|
837
|
-
|
855
|
+
cond.updated(nil, [
|
856
|
+
check_condition(cond.children.last)
|
857
|
+
])
|
838
858
|
else
|
839
859
|
cond
|
840
860
|
end
|
@@ -843,8 +863,8 @@ module Parser
|
|
843
863
|
lhs, rhs = *cond
|
844
864
|
|
845
865
|
type = case cond.type
|
846
|
-
when :irange
|
847
|
-
when :erange
|
866
|
+
when :irange then :iflipflop
|
867
|
+
when :erange then :eflipflop
|
848
868
|
end
|
849
869
|
|
850
870
|
cond.updated(type, [
|
@@ -30,7 +30,7 @@ module Parser
|
|
30
30
|
line[from...to] = "\e[4m#{line[from...to]}\e[0m"
|
31
31
|
|
32
32
|
tail_len = to - from - 1
|
33
|
-
tail =
|
33
|
+
tail = '~' * (tail_len >= 0 ? tail_len : 0)
|
34
34
|
decoration = "#{" " * from}\e[1;31m^#{tail}\e[0m #{token} ".
|
35
35
|
ljust(70) + info
|
36
36
|
|
data/lib/parser/ruby18.y
CHANGED
@@ -236,9 +236,6 @@ rule
|
|
236
236
|
| arg
|
237
237
|
|
238
238
|
expr_value: expr
|
239
|
-
{
|
240
|
-
result = value_expr(val[0])
|
241
|
-
}
|
242
239
|
|
243
240
|
command_call: command
|
244
241
|
| block_command
|
@@ -748,9 +745,6 @@ rule
|
|
748
745
|
| primary
|
749
746
|
|
750
747
|
arg_value: arg
|
751
|
-
{
|
752
|
-
result = value_expr(val[0])
|
753
|
-
}
|
754
748
|
|
755
749
|
aref_args: none
|
756
750
|
| command opt_nl
|
@@ -1245,9 +1239,6 @@ rule
|
|
1245
1239
|
}
|
1246
1240
|
|
1247
1241
|
primary_value: primary
|
1248
|
-
{
|
1249
|
-
result = value_expr(val[0])
|
1250
|
-
}
|
1251
1242
|
|
1252
1243
|
then: term
|
1253
1244
|
| tCOLON
|
@@ -1942,3 +1933,7 @@ require 'parser'
|
|
1942
1933
|
def version
|
1943
1934
|
18
|
1944
1935
|
end
|
1936
|
+
|
1937
|
+
def default_encoding
|
1938
|
+
Encoding::BINARY
|
1939
|
+
end
|
data/lib/parser/ruby19.y
CHANGED
@@ -267,9 +267,6 @@ rule
|
|
267
267
|
| arg
|
268
268
|
|
269
269
|
expr_value: expr
|
270
|
-
{
|
271
|
-
result = value_expr(val[0])
|
272
|
-
}
|
273
270
|
|
274
271
|
command_call: command
|
275
272
|
| block_command
|
@@ -820,9 +817,6 @@ rule
|
|
820
817
|
| primary
|
821
818
|
|
822
819
|
arg_value: arg
|
823
|
-
{
|
824
|
-
result = value_expr(val[0])
|
825
|
-
}
|
826
820
|
|
827
821
|
aref_args: none
|
828
822
|
| args trailer
|
@@ -1202,9 +1196,6 @@ rule
|
|
1202
1196
|
}
|
1203
1197
|
|
1204
1198
|
primary_value: primary
|
1205
|
-
{
|
1206
|
-
result = value_expr(val[0])
|
1207
|
-
}
|
1208
1199
|
|
1209
1200
|
then: term
|
1210
1201
|
| kTHEN
|
@@ -2142,3 +2133,7 @@ Parser.check_for_encoding_support
|
|
2142
2133
|
def version
|
2143
2134
|
19
|
2144
2135
|
end
|
2136
|
+
|
2137
|
+
def default_encoding
|
2138
|
+
Encoding::BINARY
|
2139
|
+
end
|
data/lib/parser/ruby20.y
CHANGED
@@ -278,9 +278,6 @@ rule
|
|
278
278
|
| arg
|
279
279
|
|
280
280
|
expr_value: expr
|
281
|
-
{
|
282
|
-
result = value_expr(val[0])
|
283
|
-
}
|
284
281
|
|
285
282
|
command_call: command
|
286
283
|
| block_command
|
@@ -840,9 +837,6 @@ rule
|
|
840
837
|
| primary
|
841
838
|
|
842
839
|
arg_value: arg
|
843
|
-
{
|
844
|
-
result = value_expr(val[0])
|
845
|
-
}
|
846
840
|
|
847
841
|
aref_args: none
|
848
842
|
| args trailer
|
@@ -1239,9 +1233,6 @@ rule
|
|
1239
1233
|
}
|
1240
1234
|
|
1241
1235
|
primary_value: primary
|
1242
|
-
{
|
1243
|
-
result = value_expr(val[0])
|
1244
|
-
}
|
1245
1236
|
|
1246
1237
|
then: term
|
1247
1238
|
| kTHEN
|
@@ -2327,3 +2318,7 @@ Parser.check_for_encoding_support
|
|
2327
2318
|
def version
|
2328
2319
|
20
|
2329
2320
|
end
|
2321
|
+
|
2322
|
+
def default_encoding
|
2323
|
+
Encoding::UTF_8
|
2324
|
+
end
|
data/lib/parser/ruby21.y
CHANGED
@@ -273,9 +273,6 @@ rule
|
|
273
273
|
| arg
|
274
274
|
|
275
275
|
expr_value: expr
|
276
|
-
{
|
277
|
-
result = value_expr(val[0])
|
278
|
-
}
|
279
276
|
|
280
277
|
command_call: command
|
281
278
|
| block_command
|
@@ -835,9 +832,6 @@ rule
|
|
835
832
|
| primary
|
836
833
|
|
837
834
|
arg_value: arg
|
838
|
-
{
|
839
|
-
result = value_expr(val[0])
|
840
|
-
}
|
841
835
|
|
842
836
|
aref_args: none
|
843
837
|
| args trailer
|
@@ -1240,9 +1234,6 @@ rule
|
|
1240
1234
|
}
|
1241
1235
|
|
1242
1236
|
primary_value: primary
|
1243
|
-
{
|
1244
|
-
result = value_expr(val[0])
|
1245
|
-
}
|
1246
1237
|
|
1247
1238
|
then: term
|
1248
1239
|
| kTHEN
|
@@ -2347,3 +2338,7 @@ warn "warning: Ruby 2.1 is not released yet and parser support may be incomplete
|
|
2347
2338
|
def version
|
2348
2339
|
21
|
2349
2340
|
end
|
2341
|
+
|
2342
|
+
def default_encoding
|
2343
|
+
Encoding::BINARY
|
2344
|
+
end
|
data/lib/parser/runner.rb
CHANGED
@@ -102,7 +102,7 @@ module Parser
|
|
102
102
|
end
|
103
103
|
|
104
104
|
if @files.empty? && @fragments.empty?
|
105
|
-
$stderr.puts
|
105
|
+
$stderr.puts 'Need something to parse!'
|
106
106
|
exit 1
|
107
107
|
end
|
108
108
|
|
@@ -141,6 +141,8 @@ module Parser
|
|
141
141
|
|
142
142
|
def process_fragments
|
143
143
|
@fragments.each_with_index do |fragment, index|
|
144
|
+
fragment = fragment.dup.force_encoding(@parser.default_encoding)
|
145
|
+
|
144
146
|
buffer = Source::Buffer.new("(fragment:#{index})")
|
145
147
|
buffer.source = fragment
|
146
148
|
|
@@ -150,8 +152,11 @@ module Parser
|
|
150
152
|
|
151
153
|
def process_files
|
152
154
|
@files.each do |filename|
|
155
|
+
source = File.read(filename)
|
156
|
+
source.force_encoding(@parser.default_encoding)
|
157
|
+
|
153
158
|
buffer = Parser::Source::Buffer.new(filename)
|
154
|
-
buffer.
|
159
|
+
buffer.source = source
|
155
160
|
|
156
161
|
process_buffer(buffer)
|
157
162
|
end
|