parser 1.1.0 → 1.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (48) hide show
  1. checksums.yaml +7 -7
  2. data/.gitignore +0 -1
  3. data/README.md +4 -2
  4. data/bin/{parse → ruby-parse} +2 -2
  5. data/bin/ruby-rewrite +6 -0
  6. data/{AST_FORMAT.md → doc/AST_FORMAT.md} +45 -29
  7. data/doc/CUSTOMIZATION.md +37 -0
  8. data/doc/INTERNALS.md +21 -0
  9. data/lib/parser.rb +14 -3
  10. data/lib/parser/ast/node.rb +6 -0
  11. data/lib/parser/ast/processor.rb +216 -0
  12. data/lib/parser/builders/default.rb +613 -215
  13. data/lib/parser/compatibility/slop.rb +12 -0
  14. data/lib/parser/lexer.rl +30 -10
  15. data/lib/parser/lexer/explanation.rb +1 -1
  16. data/lib/parser/lexer/literal.rb +5 -6
  17. data/lib/parser/ruby18.y +31 -24
  18. data/lib/parser/ruby19.y +26 -19
  19. data/lib/parser/ruby20.y +27 -20
  20. data/lib/parser/ruby21.y +27 -20
  21. data/lib/parser/runner.rb +198 -0
  22. data/lib/parser/runner/ruby_parse.rb +87 -0
  23. data/lib/parser/runner/ruby_rewrite.rb +13 -0
  24. data/lib/parser/source/buffer.rb +1 -0
  25. data/lib/parser/source/map.rb +20 -0
  26. data/lib/parser/source/map/block.rb +16 -0
  27. data/lib/parser/source/map/collection.rb +16 -0
  28. data/lib/parser/source/map/condition.rb +19 -0
  29. data/lib/parser/source/map/constant.rb +27 -0
  30. data/lib/parser/source/map/definition.rb +21 -0
  31. data/lib/parser/source/map/for.rb +17 -0
  32. data/lib/parser/source/map/keyword.rb +18 -0
  33. data/lib/parser/source/map/rescue_body.rb +19 -0
  34. data/lib/parser/source/map/send.rb +29 -0
  35. data/lib/parser/source/map/ternary.rb +16 -0
  36. data/lib/parser/source/map/variable.rb +26 -0
  37. data/lib/parser/source/range.rb +25 -24
  38. data/lib/parser/version.rb +3 -0
  39. data/parser.gemspec +4 -2
  40. data/test/parse_helper.rb +13 -10
  41. data/test/test_lexer.rb +32 -11
  42. data/test/test_parse_helper.rb +1 -0
  43. data/test/test_parser.rb +176 -128
  44. data/test/test_source_range.rb +18 -6
  45. metadata +161 -91
  46. data/bin/benchmark +0 -47
  47. data/bin/explain-parse +0 -14
  48. data/lib/parser/source/map/variable_assignment.rb +0 -15
@@ -8,8 +8,8 @@ class TestSourceRange < MiniTest::Unit::TestCase
8
8
 
9
9
  def test_initialize
10
10
  sr = Parser::Source::Range.new(@sfile, 1, 2)
11
- assert_equal 1, sr.begin
12
- assert_equal 2, sr.end
11
+ assert_equal 1, sr.begin_pos
12
+ assert_equal 2, sr.end_pos
13
13
  assert sr.frozen?
14
14
  end
15
15
 
@@ -23,8 +23,8 @@ class TestSourceRange < MiniTest::Unit::TestCase
23
23
  sr2 = Parser::Source::Range.new(@sfile, 5, 8)
24
24
  sr = sr1.join(sr2)
25
25
 
26
- assert_equal 1, sr.begin
27
- assert_equal 8, sr.end
26
+ assert_equal 1, sr.begin_pos
27
+ assert_equal 8, sr.end_pos
28
28
  end
29
29
 
30
30
  def test_line
@@ -39,11 +39,23 @@ class TestSourceRange < MiniTest::Unit::TestCase
39
39
 
40
40
  def test_columns
41
41
  sr = Parser::Source::Range.new(@sfile, 7, 8)
42
- assert_equal 0, sr.begin_column
43
- assert_equal 1, sr.end_column
42
+ assert_equal 0, sr.begin.column
43
+ assert_equal 1, sr.end.column
44
44
  assert_equal 0..1, sr.column_range
45
45
  end
46
46
 
47
+ def test_begin_end
48
+ sr = Parser::Source::Range.new(@sfile, 1, 5)
49
+
50
+ sr_beg = sr.begin
51
+ assert_equal 1, sr_beg.begin_pos
52
+ assert_equal 1, sr_beg.end_pos
53
+
54
+ sr_end = sr.end
55
+ assert_equal 5, sr_end.begin_pos
56
+ assert_equal 5, sr_end.end_pos
57
+ end
58
+
47
59
  def test_to_s
48
60
  sr = Parser::Source::Range.new(@sfile, 8, 9)
49
61
  assert_equal "(string):2:2", sr.to_s
metadata CHANGED
@@ -1,121 +1,175 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: parser
3
- version: !ruby/object:Gem::Version
4
- version: 1.1.0
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.2.0
5
5
  platform: ruby
6
- authors:
6
+ authors:
7
7
  - Peter Zotov
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
-
12
- date: 2013-04-18 00:00:00 Z
13
- dependencies:
14
- - !ruby/object:Gem::Dependency
11
+ date: 2013-04-25 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
15
14
  name: ast
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: '1.0'
20
+ type: :runtime
16
21
  prerelease: false
17
- requirement: &id001 !ruby/object:Gem::Requirement
18
- requirements:
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
19
24
  - - ~>
20
- - !ruby/object:Gem::Version
21
- version: "1.0"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: slop
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ~>
32
+ - !ruby/object:Gem::Version
33
+ version: '3.4'
22
34
  type: :runtime
23
- version_requirements: *id001
24
- - !ruby/object:Gem::Dependency
25
- name: bundler
26
35
  prerelease: false
27
- requirement: &id002 !ruby/object:Gem::Requirement
28
- requirements:
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
29
38
  - - ~>
30
- - !ruby/object:Gem::Version
31
- version: "1.2"
39
+ - !ruby/object:Gem::Version
40
+ version: '3.4'
41
+ - !ruby/object:Gem::Dependency
42
+ name: bundler
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ~>
46
+ - !ruby/object:Gem::Version
47
+ version: '1.2'
32
48
  type: :development
33
- version_requirements: *id002
34
- - !ruby/object:Gem::Dependency
35
- name: rake
36
49
  prerelease: false
37
- requirement: &id003 !ruby/object:Gem::Requirement
38
- requirements:
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
39
52
  - - ~>
40
- - !ruby/object:Gem::Version
41
- version: "0.9"
53
+ - !ruby/object:Gem::Version
54
+ version: '1.2'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rake
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ~>
60
+ - !ruby/object:Gem::Version
61
+ version: '0.9'
42
62
  type: :development
43
- version_requirements: *id003
44
- - !ruby/object:Gem::Dependency
45
- name: racc
46
63
  prerelease: false
47
- requirement: &id004 !ruby/object:Gem::Requirement
48
- requirements:
49
- - &id007
50
- - ">="
51
- - !ruby/object:Gem::Version
52
- version: "0"
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ~>
67
+ - !ruby/object:Gem::Version
68
+ version: '0.9'
69
+ - !ruby/object:Gem::Dependency
70
+ name: racc
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - '>='
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
53
76
  type: :development
54
- version_requirements: *id004
55
- - !ruby/object:Gem::Dependency
56
- name: minitest
57
77
  prerelease: false
58
- requirement: &id005 !ruby/object:Gem::Requirement
59
- requirements:
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - '>='
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: minitest
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
60
87
  - - ~>
61
- - !ruby/object:Gem::Version
88
+ - !ruby/object:Gem::Version
62
89
  version: 4.7.0
63
90
  type: :development
64
- version_requirements: *id005
65
- - !ruby/object:Gem::Dependency
66
- name: simplecov
67
91
  prerelease: false
68
- requirement: &id006 !ruby/object:Gem::Requirement
69
- requirements:
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
70
94
  - - ~>
71
- - !ruby/object:Gem::Version
72
- version: "0.7"
95
+ - !ruby/object:Gem::Version
96
+ version: 4.7.0
97
+ - !ruby/object:Gem::Dependency
98
+ name: simplecov
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ~>
102
+ - !ruby/object:Gem::Version
103
+ version: '0.7'
73
104
  type: :development
74
- version_requirements: *id006
75
- - !ruby/object:Gem::Dependency
76
- name: coveralls
77
105
  prerelease: false
78
- requirement: &id008 !ruby/object:Gem::Requirement
79
- requirements:
80
- - *id007
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ~>
109
+ - !ruby/object:Gem::Version
110
+ version: '0.7'
111
+ - !ruby/object:Gem::Dependency
112
+ name: coveralls
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - '>='
116
+ - !ruby/object:Gem::Version
117
+ version: '0'
81
118
  type: :development
82
- version_requirements: *id008
83
- - !ruby/object:Gem::Dependency
84
- name: json_pure
85
119
  prerelease: false
86
- requirement: &id009 !ruby/object:Gem::Requirement
87
- requirements:
88
- - *id007
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - '>='
123
+ - !ruby/object:Gem::Version
124
+ version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: json_pure
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - '>='
130
+ - !ruby/object:Gem::Version
131
+ version: '0'
89
132
  type: :development
90
- version_requirements: *id009
91
- - !ruby/object:Gem::Dependency
92
- name: simplecov-sublime-ruby-coverage
93
133
  prerelease: false
94
- requirement: &id010 !ruby/object:Gem::Requirement
95
- requirements:
96
- - *id007
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - '>='
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
139
+ - !ruby/object:Gem::Dependency
140
+ name: simplecov-sublime-ruby-coverage
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - '>='
144
+ - !ruby/object:Gem::Version
145
+ version: '0'
97
146
  type: :development
98
- version_requirements: *id010
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - '>='
151
+ - !ruby/object:Gem::Version
152
+ version: '0'
99
153
  description: A Ruby parser written in pure Ruby.
100
- email:
154
+ email:
101
155
  - whitequark@whitequark.org
102
- executables: []
103
-
156
+ executables:
157
+ - ruby-parse
158
+ - ruby-rewrite
104
159
  extensions: []
105
-
106
160
  extra_rdoc_files: []
107
-
108
- files:
161
+ files:
109
162
  - .gitignore
110
163
  - .travis.yml
111
- - AST_FORMAT.md
112
164
  - Gemfile
113
165
  - LICENSE.txt
114
166
  - README.md
115
167
  - Rakefile
116
- - bin/benchmark
117
- - bin/explain-parse
118
- - bin/parse
168
+ - bin/ruby-parse
169
+ - bin/ruby-rewrite
170
+ - doc/AST_FORMAT.md
171
+ - doc/CUSTOMIZATION.md
172
+ - doc/INTERNALS.md
119
173
  - lib/parser.rb
120
174
  - lib/parser/all.rb
121
175
  - lib/parser/ast/node.rb
@@ -123,6 +177,7 @@ files:
123
177
  - lib/parser/base.rb
124
178
  - lib/parser/builders/default.rb
125
179
  - lib/parser/compatibility/ruby1_8.rb
180
+ - lib/parser/compatibility/slop.rb
126
181
  - lib/parser/current.rb
127
182
  - lib/parser/diagnostic.rb
128
183
  - lib/parser/diagnostic/engine.rb
@@ -134,13 +189,27 @@ files:
134
189
  - lib/parser/ruby19.y
135
190
  - lib/parser/ruby20.y
136
191
  - lib/parser/ruby21.y
192
+ - lib/parser/runner.rb
193
+ - lib/parser/runner/ruby_parse.rb
194
+ - lib/parser/runner/ruby_rewrite.rb
137
195
  - lib/parser/source/buffer.rb
138
196
  - lib/parser/source/map.rb
197
+ - lib/parser/source/map/block.rb
198
+ - lib/parser/source/map/collection.rb
199
+ - lib/parser/source/map/condition.rb
200
+ - lib/parser/source/map/constant.rb
201
+ - lib/parser/source/map/definition.rb
202
+ - lib/parser/source/map/for.rb
203
+ - lib/parser/source/map/keyword.rb
139
204
  - lib/parser/source/map/operator.rb
140
- - lib/parser/source/map/variable_assignment.rb
205
+ - lib/parser/source/map/rescue_body.rb
206
+ - lib/parser/source/map/send.rb
207
+ - lib/parser/source/map/ternary.rb
208
+ - lib/parser/source/map/variable.rb
141
209
  - lib/parser/source/range.rb
142
210
  - lib/parser/static_environment.rb
143
211
  - lib/parser/syntax_error.rb
212
+ - lib/parser/version.rb
144
213
  - parser.gemspec
145
214
  - test/helper.rb
146
215
  - test/parse_helper.rb
@@ -162,29 +231,30 @@ files:
162
231
  - lib/parser/ruby20.rb
163
232
  - lib/parser/ruby21.rb
164
233
  homepage: http://github.com/whitequark/parser
165
- licenses:
234
+ licenses:
166
235
  - MIT
167
236
  metadata: {}
168
-
169
237
  post_install_message:
170
238
  rdoc_options: []
171
-
172
- require_paths:
239
+ require_paths:
173
240
  - lib
174
- required_ruby_version: !ruby/object:Gem::Requirement
175
- requirements:
176
- - *id007
177
- required_rubygems_version: !ruby/object:Gem::Requirement
178
- requirements:
179
- - *id007
241
+ required_ruby_version: !ruby/object:Gem::Requirement
242
+ requirements:
243
+ - - '>='
244
+ - !ruby/object:Gem::Version
245
+ version: '0'
246
+ required_rubygems_version: !ruby/object:Gem::Requirement
247
+ requirements:
248
+ - - '>='
249
+ - !ruby/object:Gem::Version
250
+ version: '0'
180
251
  requirements: []
181
-
182
252
  rubyforge_project:
183
- rubygems_version: 2.0.3
253
+ rubygems_version: 2.0.0
184
254
  signing_key:
185
255
  specification_version: 4
186
256
  summary: A Ruby parser written in pure Ruby.
187
- test_files:
257
+ test_files:
188
258
  - test/helper.rb
189
259
  - test/parse_helper.rb
190
260
  - test/racc_coverage_helper.rb
data/bin/benchmark DELETED
@@ -1,47 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- require 'benchmark'
4
- require 'ruby_parser'
5
-
6
- $LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
7
- require 'parser/all'
8
-
9
- def measure(what)
10
- rp_failures = 0
11
- p_failures = 0
12
-
13
- Benchmark.bm(10) do |x|
14
- x.report "ruby_parser" do
15
- what.each do |file, src|
16
- begin
17
- Ruby18Parser.new.parse(src)
18
- rescue => e
19
- puts file
20
- puts "RP: #{e.class}: #{e.message}"
21
- rp_failures += 1
22
- end
23
- end
24
- end
25
-
26
- x.report "parser" do
27
- what.each do |file, src|
28
- begin
29
- Parser::Ruby20.parse(src, file)
30
- rescue Parser::SyntaxError
31
- p_failures += 1
32
- rescue => e
33
- puts file
34
- puts "P: #{e.class}: #{e.message}"
35
- puts e.backtrace
36
- p_failures += 1
37
- end
38
- end
39
- end
40
- end
41
-
42
- puts
43
- puts "Failures: RP: #{rp_failures} P: #{p_failures}"
44
- end
45
-
46
- files = Hash[ARGV.map { |f| [f, File.read(f)] }]
47
- measure(files)
data/bin/explain-parse DELETED
@@ -1,14 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- $LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
4
- require 'parser/all'
5
-
6
- ENV['RACC_DEBUG'] = '1'
7
-
8
- class Parser::Base
9
- def next_token
10
- @lexer.advance_and_explain
11
- end
12
- end
13
-
14
- p Parser::Ruby20.parse(ARGF.read)
@@ -1,15 +0,0 @@
1
- module Parser
2
- module Source
3
-
4
- class Map::VariableAssignment < Map::Operator
5
- attr_reader :name
6
-
7
- def initialize(name, operator, expression)
8
- @name = name
9
-
10
- super(operator, expression)
11
- end
12
- end
13
-
14
- end
15
- end