ruby_parser 3.15.0 → 3.19.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/History.rdoc +151 -0
- data/Manifest.txt +7 -0
- data/README.rdoc +9 -6
- data/Rakefile +141 -31
- data/bin/ruby_parse_extract_error +1 -1
- data/compare/normalize.rb +8 -3
- data/debugging.md +133 -0
- data/gauntlet.md +107 -0
- data/lib/rp_extensions.rb +15 -36
- data/lib/rp_stringscanner.rb +20 -51
- data/lib/ruby20_parser.rb +7544 -3633
- data/lib/ruby20_parser.y +335 -257
- data/lib/ruby21_parser.rb +7518 -3678
- data/lib/ruby21_parser.y +330 -254
- data/lib/ruby22_parser.rb +7652 -3689
- data/lib/ruby22_parser.y +334 -256
- data/lib/ruby23_parser.rb +7659 -3702
- data/lib/ruby23_parser.y +334 -256
- data/lib/ruby24_parser.rb +7748 -3721
- data/lib/ruby24_parser.y +334 -256
- data/lib/ruby25_parser.rb +7748 -3721
- data/lib/ruby25_parser.y +334 -256
- data/lib/ruby26_parser.rb +7755 -3726
- data/lib/ruby26_parser.y +334 -255
- data/lib/ruby27_parser.rb +10290 -4518
- data/lib/ruby27_parser.y +933 -254
- data/lib/ruby30_parser.rb +13258 -0
- data/lib/ruby30_parser.y +3459 -0
- data/lib/ruby31_parser.rb +13638 -0
- data/lib/ruby31_parser.y +3493 -0
- data/lib/ruby3_parser.yy +3548 -0
- data/lib/ruby_lexer.rb +277 -599
- data/lib/ruby_lexer.rex +28 -21
- data/lib/ruby_lexer.rex.rb +60 -24
- data/lib/ruby_lexer_strings.rb +638 -0
- data/lib/ruby_parser.rb +4 -0
- data/lib/ruby_parser.yy +974 -261
- data/lib/ruby_parser_extras.rb +355 -114
- data/test/test_ruby_lexer.rb +226 -129
- data/test/test_ruby_parser.rb +1653 -267
- data/tools/munge.rb +36 -8
- data/tools/ripper.rb +15 -10
- data.tar.gz.sig +0 -0
- metadata +55 -37
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 82ff82f1d9693451e0d2f740f5e7a9cbeaa31034bee66ffa61486bc0f930d60b
|
4
|
+
data.tar.gz: 15d8b71b9533534ffaa4184e50f0e5a795e4a7f190bf13a55858f59b17c1ed99
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 78332be9c05d511d05f1c47977c74390658fcc00fc412009dba172d0e53c0e698b8e28c4803c4969720f2d2d29ed3fc891be2a990618098ae0cf04d00ad28e9f
|
7
|
+
data.tar.gz: 72f7e1c089bc80662ea2eb8c8056d7bb3bf84ff27224c2d450ffde94abd2fb5ba2317d1405a880bebb2c91ac34bcb7023685065f1a8b2e69f62ed8f6b0362e77
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/History.rdoc
CHANGED
@@ -1,3 +1,154 @@
|
|
1
|
+
=== 3.19.2 / 2022-12-03
|
2
|
+
|
3
|
+
* 5 bug fixes:
|
4
|
+
|
5
|
+
* Fixed bug parsing array patterns using %w, %i, etc.
|
6
|
+
* Fixed dedenting when heredoc has only empty line and interpolation. (mvz)
|
7
|
+
* Fixed dsym unescaping. (mvz)
|
8
|
+
* Fixed vars in pattern matching that should be lasgn, not lvar.
|
9
|
+
* find_pat mid-patterns are now splatted inline, not wrapped in an array_pat.
|
10
|
+
|
11
|
+
=== 3.19.1 / 2022-04-05
|
12
|
+
|
13
|
+
* 2 bug fixes:
|
14
|
+
|
15
|
+
* Added comments to endless defn and defs. (mvz)
|
16
|
+
* Fixed endless method bug handling attrset names.
|
17
|
+
|
18
|
+
=== 3.19.0 / 2022-03-29
|
19
|
+
|
20
|
+
* 1 major enhancement:
|
21
|
+
|
22
|
+
* Added tentative 3.1 support.
|
23
|
+
|
24
|
+
* 7 minor enhancements:
|
25
|
+
|
26
|
+
* 3.1: bare RHS assoc: { y: } => s(:hash, s(:lit, :y), nil)
|
27
|
+
* 3.1: calls w/ unnamed block args (bare &)
|
28
|
+
* 3.1: endless defn/defs w/ paren-less calls (aka commands)
|
29
|
+
* 3.1: pattern capture to nonlocal vars, eg: ^@a, ^$b, ^@@c
|
30
|
+
* 3.1: pattern: ^(expr) => expr
|
31
|
+
* Improved steps for adding new versions.
|
32
|
+
* Improved steps for running gauntlets.
|
33
|
+
|
34
|
+
* 2 bug fixes:
|
35
|
+
|
36
|
+
* Bumped 2.6+ cached versions for rake compare.
|
37
|
+
* Skip test_regexp_esc_C_slash on ruby 3.1.0 because of MRI bug.
|
38
|
+
|
39
|
+
=== 3.18.1 / 2021-11-10
|
40
|
+
|
41
|
+
* 1 minor enhancement:
|
42
|
+
|
43
|
+
* All parser tests are now explicitly testing line numbers at every level.
|
44
|
+
|
45
|
+
* 3 bug fixes:
|
46
|
+
|
47
|
+
* Fixed endless method with noargs. (mitsuru)
|
48
|
+
* Fixed line numbers on some yield forms.
|
49
|
+
* Handle and clearly report if unifdef is missing.
|
50
|
+
|
51
|
+
=== 3.18.0 / 2021-10-27
|
52
|
+
|
53
|
+
Holy crap... 58 commits! 2.7 and 3.0 are feature complete. Strings
|
54
|
+
& heredocs have been rewritten.
|
55
|
+
|
56
|
+
* 9 major enhancements:
|
57
|
+
|
58
|
+
* !!! Rewrote lexer (and friends) for strings, heredocs, and %*[] constructs.
|
59
|
+
* Massive overhaul on line numbers.
|
60
|
+
* Freeze input! Finally!!! No more modifying the input string for heredocs.
|
61
|
+
* Overhauled RPStringScanner. Removed OLD compatibility methods!
|
62
|
+
* Removed Sexp methods: value, to_sym, add, add_all, node_type, values.
|
63
|
+
* value moved to sexp_processor.
|
64
|
+
* Removed String#grep monkey-patch.
|
65
|
+
* Removed String#lineno monkey-patch.
|
66
|
+
* Removed string_to_pos, charpos, etc hacks for ancient ruby versions.
|
67
|
+
* Removed unread_many... NO! NO EDITING THE INPUT STRING!
|
68
|
+
|
69
|
+
* 31 minor enhancements:
|
70
|
+
|
71
|
+
* 2.7/3.0: many more pattern edge cases
|
72
|
+
* 2.7: Added `mlhs = rhs rescue expr`
|
73
|
+
* 2.7: refactored destructured args (`|(k,v)|`) and unfactored(?!) case_body/args.
|
74
|
+
* 3.0: excessed_comma
|
75
|
+
* 3.0: finished most everything: endless methods, patterns, etc.
|
76
|
+
* 3.0: refactored / added new pattern changes
|
77
|
+
* Added RubyLexer#in_heredoc? (ie, is there old_ss ?)
|
78
|
+
* Added RubyLexer#old_ss and old_lineno and removed much of SSStack(ish).
|
79
|
+
* Added Symbol#end_with? when necessary
|
80
|
+
* Added TALLY and DEBUG options for ss.getch and ss.scan
|
81
|
+
* Added ignore_body_comments to make parser productions more clear.
|
82
|
+
* Added support for no_kwarg (eg `def f(**nil)`).
|
83
|
+
* Added support for no_kwarg in blocks (eg `f { |**nil| }`).
|
84
|
+
* Augmented generated parser files to have frozen_string_literal comments and fixed tests.
|
85
|
+
* Broke out 3.0 parser into its own to ease development.
|
86
|
+
* Bumped dependencies on sexp_processor and oedipus_lex.
|
87
|
+
* Clean generated 3.x files.
|
88
|
+
* Extracted all string scanner methods to their own module.
|
89
|
+
* Fixed some precedence decls.
|
90
|
+
* Implemented most of pattern matching for 2.7+.
|
91
|
+
* Improve lex_state= to report location in verbose debug mode.
|
92
|
+
* Made it easier to debug with a particular version of ruby via rake.
|
93
|
+
* Make sure ripper uses the same version of ruby we specified.
|
94
|
+
* Moved all string/heredoc/etc code to ruby_lexer_strings.rb
|
95
|
+
* Remove warning from newer bisons.
|
96
|
+
* Sprinkled in some frozen_string_literal, but mostly helped by oedipus bump.
|
97
|
+
* Switch to comparing against ruby binary since ripper is buggy.
|
98
|
+
* bugs task should try both bug*.rb and bad*.rb.
|
99
|
+
* endless methods
|
100
|
+
* f_any_kwrest refactoring.
|
101
|
+
* refactored defn/defs
|
102
|
+
|
103
|
+
* 15 bug fixes:
|
104
|
+
|
105
|
+
* Cleaned a bunch of old hacks. Initializing RubyLexer w/ Parser is cleaner now.
|
106
|
+
* Corrected some lex_state errors in process_token_keyword.
|
107
|
+
* Fixed ancient ruby2 change (use #lines) in ruby_parse_extract_error.
|
108
|
+
* Fixed bug where else without rescue only raises on 2.6+
|
109
|
+
* Fixed caller for getch and scan when DEBUG=1
|
110
|
+
* Fixed comments in the middle of message cascades.
|
111
|
+
* Fixed differences w/ symbol productions against ruby 2.7.
|
112
|
+
* Fixed dsym to use string_contents production.
|
113
|
+
* Fixed error in bdot2/3 in some edge cases. Fixed p_alt line.
|
114
|
+
* Fixed heredoc dedenting in the presence of empty lines. (mvz)
|
115
|
+
* Fixed some leading whitespace / comment processing
|
116
|
+
* Fixed up how class/module/defn/defs comments were collected.
|
117
|
+
* Overhauled ripper.rb to deal with buggy ripper w/ yydebug.
|
118
|
+
* Removed dsym from literal.
|
119
|
+
* Removed tUBANG lexeme but kept it distinct as a method name (eg: `def !@`).
|
120
|
+
|
121
|
+
=== 3.17.0 / 2021-08-03
|
122
|
+
|
123
|
+
* 1 minor enhancement:
|
124
|
+
|
125
|
+
* Added support for arg forwarding (eg `def f(...); m(...); end`) (presidentbeef)
|
126
|
+
|
127
|
+
=== 3.16.0 / 2021-05-15
|
128
|
+
|
129
|
+
* 1 major enhancement:
|
130
|
+
|
131
|
+
* Added tentative 3.0 support.
|
132
|
+
|
133
|
+
* 3 minor enhancements:
|
134
|
+
|
135
|
+
* Added lexing for "beginless range" (bdots).
|
136
|
+
* Added parsing for bdots.
|
137
|
+
* Updated rake compare task to download xz files, bumped versions, etc
|
138
|
+
|
139
|
+
* 4 bug fixes:
|
140
|
+
|
141
|
+
* Bump rake dependency to >= 10, < 15. (presidentbeef)
|
142
|
+
* Bump sexp_processor dependency to 4.15.1+. (pravi)
|
143
|
+
* Fixed minor state mismatch at the end of parsing to make diffing a little cleaner.
|
144
|
+
* Fixed normalizer to deal with new bison token syntax
|
145
|
+
|
146
|
+
=== 3.15.1 / 2021-01-10
|
147
|
+
|
148
|
+
* 1 bug fix:
|
149
|
+
|
150
|
+
* Bumped ruby version to include < 4 (trunk).
|
151
|
+
|
1
152
|
=== 3.15.0 / 2020-08-31
|
2
153
|
|
3
154
|
* 1 major enhancement:
|
data/Manifest.txt
CHANGED
@@ -7,6 +7,7 @@ bin/ruby_parse
|
|
7
7
|
bin/ruby_parse_extract_error
|
8
8
|
compare/normalize.rb
|
9
9
|
debugging.md
|
10
|
+
gauntlet.md
|
10
11
|
lib/.document
|
11
12
|
lib/rp_extensions.rb
|
12
13
|
lib/rp_stringscanner.rb
|
@@ -26,9 +27,15 @@ lib/ruby26_parser.rb
|
|
26
27
|
lib/ruby26_parser.y
|
27
28
|
lib/ruby27_parser.rb
|
28
29
|
lib/ruby27_parser.y
|
30
|
+
lib/ruby30_parser.rb
|
31
|
+
lib/ruby30_parser.y
|
32
|
+
lib/ruby31_parser.rb
|
33
|
+
lib/ruby31_parser.y
|
34
|
+
lib/ruby3_parser.yy
|
29
35
|
lib/ruby_lexer.rb
|
30
36
|
lib/ruby_lexer.rex
|
31
37
|
lib/ruby_lexer.rex.rb
|
38
|
+
lib/ruby_lexer_strings.rb
|
32
39
|
lib/ruby_parser.rb
|
33
40
|
lib/ruby_parser.yy
|
34
41
|
lib/ruby_parser_extras.rb
|
data/README.rdoc
CHANGED
@@ -32,6 +32,10 @@ Tested against 801,039 files from the latest of all rubygems (as of 2013-05):
|
|
32
32
|
* 1.8 parser is at 99.9739% accuracy, 3.651 sigma
|
33
33
|
* 1.9 parser is at 99.9940% accuracy, 4.013 sigma
|
34
34
|
* 2.0 parser is at 99.9939% accuracy, 4.008 sigma
|
35
|
+
* 2.6 parser is at 99.9972% accuracy, 4.191 sigma
|
36
|
+
* 3.0 parser has a 100% parse rate.
|
37
|
+
* Tested against 2,672,412 unique ruby files across 167k gems.
|
38
|
+
* As do all the others now, basically.
|
35
39
|
|
36
40
|
== FEATURES/PROBLEMS:
|
37
41
|
|
@@ -61,15 +65,14 @@ You can also use Ruby19Parser, Ruby18Parser, or RubyParser.for_current_ruby:
|
|
61
65
|
|
62
66
|
To add a new version:
|
63
67
|
|
64
|
-
* New parser should be generated from lib/
|
65
|
-
* Extend lib/
|
66
|
-
* Add new version number to V2 in Rakefile for rule creation.
|
68
|
+
* New parser should be generated from lib/ruby[3]_parser.yy.
|
69
|
+
* Extend lib/ruby[3]_parser.yy with new class name.
|
70
|
+
* Add new version number to V2/V3 in Rakefile for rule creation.
|
71
|
+
* Add new (full) version to `ruby_parse` section of Rakefile for rake compare
|
67
72
|
* Require generated parser in lib/ruby_parser.rb.
|
68
73
|
* Add empty TestRubyParserShared##Plus module and TestRubyParserV## to test/test_ruby_parser.rb.
|
69
74
|
* Extend Manifest.txt with generated file names.
|
70
|
-
*
|
71
|
-
* add_19tests needs to have the version added
|
72
|
-
* VER_RE needs to have the regexp expanded
|
75
|
+
* Add new version number to sexp_processor's pt_testcase.rb in all_versions
|
73
76
|
|
74
77
|
Until all of these are done, you won't have a clean test run.
|
75
78
|
|
data/Rakefile
CHANGED
@@ -12,27 +12,40 @@ Hoe.add_include_dirs "lib"
|
|
12
12
|
Hoe.add_include_dirs "../../sexp_processor/dev/lib"
|
13
13
|
Hoe.add_include_dirs "../../minitest/dev/lib"
|
14
14
|
Hoe.add_include_dirs "../../oedipus_lex/dev/lib"
|
15
|
+
Hoe.add_include_dirs "../../ruby2ruby/dev/lib"
|
15
16
|
|
16
17
|
V2 = %w[20 21 22 23 24 25 26 27]
|
17
|
-
|
18
|
+
V3 = %w[30 31]
|
19
|
+
|
20
|
+
VERS = V2 + V3
|
21
|
+
|
22
|
+
ENV["FAST"] = VERS.last if ENV["FAST"] && !VERS.include?(ENV["FAST"])
|
23
|
+
VERS.replace [ENV["FAST"]] if ENV["FAST"]
|
18
24
|
|
19
25
|
Hoe.spec "ruby_parser" do
|
20
26
|
developer "Ryan Davis", "ryand-ruby@zenspider.com"
|
21
27
|
|
22
28
|
license "MIT"
|
23
29
|
|
24
|
-
dependency "sexp_processor", "~> 4.
|
25
|
-
dependency "rake", "<
|
26
|
-
dependency "oedipus_lex", "~> 2.
|
30
|
+
dependency "sexp_processor", "~> 4.16"
|
31
|
+
dependency "rake", [">= 10", "< 15"], :developer
|
32
|
+
dependency "oedipus_lex", "~> 2.6", :developer
|
33
|
+
|
34
|
+
# NOTE: Ryan!!! Stop trying to fix this dependency! Isolate just
|
35
|
+
# can't handle having a faux-gem half-installed! Stop! Just `gem
|
36
|
+
# install racc` and move on. Revisit this ONLY once racc-compiler
|
37
|
+
# gets split out.
|
27
38
|
|
28
|
-
|
39
|
+
dependency "racc", "~> 1.5", :developer
|
40
|
+
|
41
|
+
require_ruby_version [">= 2.1", "< 4"]
|
29
42
|
|
30
43
|
if plugin? :perforce then # generated files
|
31
|
-
|
44
|
+
VERS.each do |n|
|
32
45
|
self.perforce_ignore << "lib/ruby#{n}_parser.rb"
|
33
46
|
end
|
34
47
|
|
35
|
-
|
48
|
+
VERS.each do |n|
|
36
49
|
self.perforce_ignore << "lib/ruby#{n}_parser.y"
|
37
50
|
end
|
38
51
|
|
@@ -46,8 +59,44 @@ Hoe.spec "ruby_parser" do
|
|
46
59
|
end
|
47
60
|
end
|
48
61
|
|
62
|
+
def maybe_add_to_top path, string
|
63
|
+
file = File.read path
|
64
|
+
|
65
|
+
return if file.start_with? string
|
66
|
+
|
67
|
+
warn "Altering top of #{path}"
|
68
|
+
tmp_path = "#{path}.tmp"
|
69
|
+
File.open(tmp_path, "w") do |f|
|
70
|
+
f.puts string
|
71
|
+
f.puts
|
72
|
+
|
73
|
+
f.write file
|
74
|
+
# TODO: make this deal with encoding comments properly?
|
75
|
+
end
|
76
|
+
File.rename tmp_path, path
|
77
|
+
end
|
78
|
+
|
79
|
+
def unifdef?
|
80
|
+
@unifdef ||= system("which unifdef") or abort <<~EOM
|
81
|
+
unifdef not found!
|
82
|
+
|
83
|
+
Please install 'unifdef' package on your system or `rake generate` on a mac.
|
84
|
+
EOM
|
85
|
+
end
|
86
|
+
|
49
87
|
V2.each do |n|
|
50
88
|
file "lib/ruby#{n}_parser.y" => "lib/ruby_parser.yy" do |t|
|
89
|
+
unifdef?
|
90
|
+
cmd = 'unifdef -tk -DV=%s -UDEAD %s > %s || true' % [n, t.source, t.name]
|
91
|
+
sh cmd
|
92
|
+
end
|
93
|
+
|
94
|
+
file "lib/ruby#{n}_parser.rb" => "lib/ruby#{n}_parser.y"
|
95
|
+
end
|
96
|
+
|
97
|
+
V3.each do |n|
|
98
|
+
file "lib/ruby#{n}_parser.y" => "lib/ruby3_parser.yy" do |t|
|
99
|
+
unifdef?
|
51
100
|
cmd = 'unifdef -tk -DV=%s -UDEAD %s > %s || true' % [n, t.source, t.name]
|
52
101
|
sh cmd
|
53
102
|
end
|
@@ -57,6 +106,12 @@ end
|
|
57
106
|
|
58
107
|
file "lib/ruby_lexer.rex.rb" => "lib/ruby_lexer.rex"
|
59
108
|
|
109
|
+
task :parser do |t|
|
110
|
+
t.prerequisite_tasks.grep(Rake::FileTask).select(&:already_invoked).each do |f|
|
111
|
+
maybe_add_to_top f.name, "# frozen_string_literal: true"
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
60
115
|
task :generate => [:lexer, :parser]
|
61
116
|
|
62
117
|
task :clean do
|
@@ -65,6 +120,7 @@ task :clean do
|
|
65
120
|
Dir["coverage.info"] +
|
66
121
|
Dir["coverage"] +
|
67
122
|
Dir["lib/ruby2*_parser.y"] +
|
123
|
+
Dir["lib/ruby3*_parser.y"] +
|
68
124
|
Dir["lib/*.output"])
|
69
125
|
end
|
70
126
|
|
@@ -92,7 +148,7 @@ end
|
|
92
148
|
|
93
149
|
def dl v
|
94
150
|
dir = v[/^\d+\.\d+/]
|
95
|
-
url = "https://cache.ruby-lang.org/pub/ruby/#{dir}/ruby-#{v}.tar.
|
151
|
+
url = "https://cache.ruby-lang.org/pub/ruby/#{dir}/ruby-#{v}.tar.xz"
|
96
152
|
path = File.basename url
|
97
153
|
unless File.exist? path then
|
98
154
|
system "curl -O #{url}"
|
@@ -104,7 +160,7 @@ def ruby_parse version
|
|
104
160
|
rp_txt = "rp#{v}.txt"
|
105
161
|
mri_txt = "mri#{v}.txt"
|
106
162
|
parse_y = "parse#{v}.y"
|
107
|
-
tarball = "ruby-#{version}.tar.
|
163
|
+
tarball = "ruby-#{version}.tar.xz"
|
108
164
|
ruby_dir = "ruby-#{version}"
|
109
165
|
diff = "diff#{v}.diff"
|
110
166
|
rp_out = "lib/ruby#{v}_parser.output"
|
@@ -124,15 +180,18 @@ def ruby_parse version
|
|
124
180
|
end
|
125
181
|
end
|
126
182
|
|
183
|
+
desc "fetch all tarballs"
|
184
|
+
task :fetch => c_tarball
|
185
|
+
|
127
186
|
file c_parse_y => c_tarball do
|
128
187
|
in_compare do
|
129
|
-
extract_glob = case
|
130
|
-
when
|
188
|
+
extract_glob = case
|
189
|
+
when version > "2.7" then
|
131
190
|
"{id.h,parse.y,tool/{id2token.rb,lib/vpath.rb}}"
|
132
191
|
else
|
133
192
|
"{id.h,parse.y,tool/{id2token.rb,vpath.rb}}"
|
134
193
|
end
|
135
|
-
system "tar
|
194
|
+
system "tar Jxf #{tarball} #{ruby_dir}/#{extract_glob}"
|
136
195
|
|
137
196
|
Dir.chdir ruby_dir do
|
138
197
|
if File.exist? "tool/id2token.rb" then
|
@@ -141,15 +200,20 @@ def ruby_parse version
|
|
141
200
|
sh "expand parse.y > ../#{parse_y}"
|
142
201
|
end
|
143
202
|
|
144
|
-
ruby "-pi", "-e", 'gsub(/^%
|
203
|
+
ruby "-pi", "-e", 'gsub(/^%pure-parser/, "%define api.pure")', "../#{parse_y}"
|
145
204
|
end
|
146
205
|
sh "rm -rf #{ruby_dir}"
|
147
206
|
end
|
148
207
|
end
|
149
208
|
|
209
|
+
bison = Dir["/opt/homebrew/opt/bison/bin/bison",
|
210
|
+
"/usr/local/opt/bison/bin/bison",
|
211
|
+
`which bison`.chomp,
|
212
|
+
].first
|
213
|
+
|
150
214
|
file c_mri_txt => [c_parse_y, normalize] do
|
151
215
|
in_compare do
|
152
|
-
sh "bison -r all #{parse_y}"
|
216
|
+
sh "#{bison} -r all #{parse_y}"
|
153
217
|
sh "./normalize.rb parse#{v}.output > #{mri_txt}"
|
154
218
|
rm ["parse#{v}.output", "parse#{v}.tab.c"]
|
155
219
|
end
|
@@ -190,17 +254,51 @@ def ruby_parse version
|
|
190
254
|
end
|
191
255
|
end
|
192
256
|
|
257
|
+
task :versions do
|
258
|
+
require "open-uri"
|
259
|
+
require "net/http" # avoid require issues in threads
|
260
|
+
require "net/https"
|
261
|
+
|
262
|
+
versions = VERS.map { |s| s.split(//).join "." }
|
263
|
+
|
264
|
+
base_url = "https://cache.ruby-lang.org/pub/ruby"
|
265
|
+
|
266
|
+
class Array
|
267
|
+
def human_sort
|
268
|
+
sort_by { |item| item.to_s.split(/(\d+)/).map { |e| [e.to_i, e] } }
|
269
|
+
end
|
270
|
+
end
|
271
|
+
|
272
|
+
versions = versions.map { |ver|
|
273
|
+
Thread.new {
|
274
|
+
URI
|
275
|
+
.parse("#{base_url}/#{ver}/")
|
276
|
+
.read
|
277
|
+
.scan(/ruby-\d+\.\d+\.\d+[-\w.]*?.tar.gz/)
|
278
|
+
.reject { |s| s =~ /-(?:rc|preview)\d/ }
|
279
|
+
.human_sort
|
280
|
+
.last
|
281
|
+
.delete_prefix("ruby-")
|
282
|
+
.delete_suffix ".tar.gz"
|
283
|
+
}
|
284
|
+
}.map(&:value).sort
|
285
|
+
|
286
|
+
puts versions.map { |v| "ruby_parse %p" % [v] }
|
287
|
+
end
|
288
|
+
|
193
289
|
ruby_parse "2.0.0-p648"
|
194
|
-
ruby_parse "2.1.
|
195
|
-
ruby_parse "2.2.
|
290
|
+
ruby_parse "2.1.10"
|
291
|
+
ruby_parse "2.2.10"
|
196
292
|
ruby_parse "2.3.8"
|
197
|
-
ruby_parse "2.4.
|
198
|
-
ruby_parse "2.5.
|
199
|
-
ruby_parse "2.6.
|
200
|
-
ruby_parse "2.7.
|
293
|
+
ruby_parse "2.4.10"
|
294
|
+
ruby_parse "2.5.9"
|
295
|
+
ruby_parse "2.6.9"
|
296
|
+
ruby_parse "2.7.5"
|
297
|
+
ruby_parse "3.0.3"
|
298
|
+
ruby_parse "3.1.1"
|
201
299
|
|
202
300
|
task :debug => :isolate do
|
203
|
-
ENV["V"] ||=
|
301
|
+
ENV["V"] ||= VERS.last
|
204
302
|
Rake.application[:parser].invoke # this way we can have DEBUG set
|
205
303
|
Rake.application[:lexer].invoke # this way we can have DEBUG set
|
206
304
|
|
@@ -215,7 +313,7 @@ task :debug => :isolate do
|
|
215
313
|
time = (ENV["RP_TIMEOUT"] || 10).to_i
|
216
314
|
|
217
315
|
n = ENV["BUG"]
|
218
|
-
file = (n && "bug#{n}.rb") || ENV["F"] || ENV["FILE"] || "
|
316
|
+
file = (n && "bug#{n}.rb") || ENV["F"] || ENV["FILE"] || "debug.rb"
|
219
317
|
ruby = ENV["R"] || ENV["RUBY"]
|
220
318
|
|
221
319
|
if ruby then
|
@@ -238,19 +336,22 @@ task :debug => :isolate do
|
|
238
336
|
end
|
239
337
|
|
240
338
|
task :debug3 do
|
241
|
-
file = ENV["F"] || "
|
242
|
-
|
339
|
+
file = ENV["F"] || "debug.rb"
|
340
|
+
version = ENV["V"] || ""
|
341
|
+
verbose = ENV["VERBOSE"] ? "-v" : ""
|
243
342
|
munge = "./tools/munge.rb #{verbose}"
|
244
343
|
|
245
344
|
abort "Need a file to parse, via: F=path.rb" unless file
|
246
345
|
|
247
346
|
ENV.delete "V"
|
248
347
|
|
249
|
-
|
250
|
-
|
251
|
-
sh "
|
348
|
+
ruby = "ruby#{version}"
|
349
|
+
|
350
|
+
sh "#{ruby} -v"
|
351
|
+
sh "#{ruby} -y #{file} 2>&1 | #{munge} > tmp/ruby"
|
352
|
+
sh "#{ruby} ./tools/ripper.rb -d #{file} | #{munge} > tmp/rip"
|
252
353
|
sh "rake debug F=#{file} DEBUG=1 2>&1 | #{munge} > tmp/rp"
|
253
|
-
sh "diff -U 999 -d tmp/{
|
354
|
+
sh "diff -U 999 -d tmp/{ruby,rp}"
|
254
355
|
end
|
255
356
|
|
256
357
|
task :cmp do
|
@@ -262,16 +363,25 @@ task :cmp3 do
|
|
262
363
|
end
|
263
364
|
|
264
365
|
task :extract => :isolate do
|
265
|
-
ENV["V"] ||=
|
366
|
+
ENV["V"] ||= VERS.last
|
266
367
|
Rake.application[:parser].invoke # this way we can have DEBUG set
|
267
368
|
|
268
|
-
file = ENV["F"] || ENV["FILE"]
|
369
|
+
file = ENV["F"] || ENV["FILE"] || abort("Need to provide F=<path>")
|
269
370
|
|
270
371
|
ruby "-Ilib", "bin/ruby_parse_extract_error", file
|
271
372
|
end
|
272
373
|
|
374
|
+
task :parse => :isolate do
|
375
|
+
ENV["V"] ||= VERS.last
|
376
|
+
Rake.application[:parser].invoke # this way we can have DEBUG set
|
377
|
+
|
378
|
+
file = ENV["F"] || ENV["FILE"] || abort("Need to provide F=<path>")
|
379
|
+
|
380
|
+
ruby "-Ilib", "bin/ruby_parse", file
|
381
|
+
end
|
382
|
+
|
273
383
|
task :bugs do
|
274
|
-
sh "for f in bug*.rb ; do #{Gem.ruby} -S rake debug F=$f && rm $f ; done"
|
384
|
+
sh "for f in bug*.rb bad*.rb ; do #{Gem.ruby} -S rake debug F=$f && rm $f ; done"
|
275
385
|
end
|
276
386
|
|
277
387
|
# vim: syntax=Ruby
|
data/compare/normalize.rb
CHANGED
@@ -84,6 +84,7 @@ def munge s
|
|
84
84
|
|
85
85
|
"' '", "tSPACE", # needs to be later to avoid bad hits
|
86
86
|
|
87
|
+
"%empty", "none", # newer bison
|
87
88
|
"/* empty */", "none",
|
88
89
|
/^\s*$/, "none",
|
89
90
|
|
@@ -140,6 +141,7 @@ def munge s
|
|
140
141
|
'"do for block"', "kDO_BLOCK",
|
141
142
|
'"do for condition"', "kDO_COND",
|
142
143
|
'"do for lambda"', "kDO_LAMBDA",
|
144
|
+
"tLABEL", "kLABEL",
|
143
145
|
|
144
146
|
# UGH
|
145
147
|
"k_LINE__", "k__LINE__",
|
@@ -155,7 +157,10 @@ def munge s
|
|
155
157
|
/\"(\w+) \(?modifier\)?\"/, proc { |x| "k#{$1.upcase}_MOD" },
|
156
158
|
/\"(\w+)\"/, proc { |x| "k#{$1.upcase}" },
|
157
159
|
|
158
|
-
|
160
|
+
/\$?@(\d+)(\s+|$)/, "", # newer bison
|
161
|
+
|
162
|
+
# TODO: remove for 3.0 work:
|
163
|
+
"lex_ctxt ", "" # 3.0 production that's mostly noise right now
|
159
164
|
]
|
160
165
|
|
161
166
|
renames.each_slice(2) do |(a, b)|
|
@@ -174,7 +179,7 @@ ARGF.each_line do |line|
|
|
174
179
|
|
175
180
|
case line.strip
|
176
181
|
when /^$/ then
|
177
|
-
when /^(\d+) (
|
182
|
+
when /^(\d+) (\$?[@\w]+): (.*)/ then # yacc
|
178
183
|
rule = $2
|
179
184
|
order << rule unless rules.has_key? rule
|
180
185
|
rules[rule] << munge($3)
|
@@ -199,7 +204,7 @@ ARGF.each_line do |line|
|
|
199
204
|
when /^\cL/ then # byacc
|
200
205
|
break
|
201
206
|
else
|
202
|
-
warn "unparsed: #{$.}: #{line.
|
207
|
+
warn "unparsed: #{$.}: #{line.strip.inspect}"
|
203
208
|
end
|
204
209
|
end
|
205
210
|
|