redparse 0.8.4 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -0
- data/COPYING.LGPL +503 -158
- data/History.txt +192 -0
- data/Makefile +9 -0
- data/README.txt +72 -39
- data/bin/redparse +108 -14
- data/lib/miniredparse.rb +1543 -0
- data/lib/redparse.rb +971 -105
- data/lib/redparse/ReduceWithsFor_RedParse_1_8.rb +17412 -0
- data/lib/redparse/ReduceWithsFor_RedParse_1_9.rb +17633 -0
- data/lib/redparse/babynodes.rb +17 -0
- data/lib/redparse/babyparser.rb +17 -0
- data/lib/redparse/cache.rb +290 -6
- data/lib/redparse/compile.rb +6 -97
- data/lib/redparse/decisiontree.rb +1 -1
- data/lib/redparse/float_accurate_to_s.rb +30 -6
- data/lib/redparse/generate.rb +18 -0
- data/lib/redparse/node.rb +415 -124
- data/lib/redparse/parse_tree_server.rb +20 -2
- data/lib/redparse/problemfiles.rb +1 -1
- data/lib/redparse/pthelper.rb +17 -31
- data/lib/redparse/reg_more_sugar.rb +1 -1
- data/lib/redparse/replacing/parse_tree.rb +30 -0
- data/lib/redparse/replacing/ripper.rb +20 -0
- data/lib/redparse/replacing/ruby_parser.rb +28 -0
- data/lib/redparse/ripper.rb +393 -0
- data/lib/redparse/ripper_sexp.rb +153 -0
- data/lib/redparse/stackableclasses.rb +113 -0
- data/lib/redparse/version.rb +18 -1
- data/redparse.gemspec +29 -9
- data/rplt.txt +31 -0
- data/test/data/hd_with_blank_string.rb +3 -0
- data/test/data/pt_known_output.rb +13273 -0
- data/test/data/wp.pp +0 -0
- data/test/generate_parse_tree_server_rc.rb +17 -0
- data/test/rp-locatetest.rb +2 -2
- data/test/test_1.9.rb +338 -35
- data/test/test_all.rb +22 -3
- data/test/test_part.rb +32 -0
- data/test/test_redparse.rb +396 -74
- data/test/test_xform_tree.rb +18 -0
- data/test/unparse_1.9_exceptions.txt +85 -0
- data/test/unparse_1.9_exceptions.txt.old +81 -0
- metadata +71 -46
- data/Rakefile +0 -35
data/History.txt
CHANGED
@@ -1,3 +1,195 @@
|
|
1
|
+
=== v1.0.0 26oct2011
|
2
|
+
* performance improvements:
|
3
|
+
* coalescer:
|
4
|
+
* a new backend, the 'coalescer' is 10x faster and actually works!
|
5
|
+
* canned coalesce output (which takes forever to generate otherwise)
|
6
|
+
* cache:
|
7
|
+
* cached outputs are stored near to the inputs if that's writeable
|
8
|
+
* otherwise, they're in the first writeable directory of:
|
9
|
+
* Etc.getpwuid if any, $HOME if any, root dir
|
10
|
+
* if none are writeable, nothing is cached
|
11
|
+
* defined a parser signature that will change when:
|
12
|
+
* parser source code changes, or
|
13
|
+
* parser type (including modules parser is extended by) changes, or
|
14
|
+
* lexer type changes
|
15
|
+
* many improvements to #unparse (which turns Node trees back into ruby code):
|
16
|
+
* slightly tweaked the unparsing of defined? so it'll reparse correctly
|
17
|
+
* in AssignNode#unparse, always put spaces around assign op
|
18
|
+
* only elide spaces right before [ on unparse of BracketsGetNode
|
19
|
+
* unparse of for node should not start with newlines...
|
20
|
+
* speed up unparse_nl when no token provided
|
21
|
+
* 3rd param to unparse_nl (alt) should never be nil
|
22
|
+
* parens and the like and when to omit them from unparse output:
|
23
|
+
* need to call to_s on body of ParenedNode#unparse
|
24
|
+
* emit extra () on unparse in hash and array lits and method call params.
|
25
|
+
* better detection of when to leave out curly braces in hash literal
|
26
|
+
* unparse of MethodNode and CallSiteNode now omits parens if original did
|
27
|
+
* hardcode unparse of NopNode to "()" (was empty string)
|
28
|
+
* NopNode in str inclusion unparses to #{} (was: #{()})
|
29
|
+
* string literal unparsing:
|
30
|
+
* fix unparse of string when delimiter is \r
|
31
|
+
* better detection of backslash (and friends) in strings
|
32
|
+
* for splitting word array or requoting here docs
|
33
|
+
* exception machinery and unparsing:
|
34
|
+
* tolerate missing #rescues in nodes which allow rescue
|
35
|
+
* oops, #unparse was inserting extraneous empty elses
|
36
|
+
* unparsing float literals:
|
37
|
+
* unparse to original string rep of a float literal if available
|
38
|
+
* (to reduce rounding errors)
|
39
|
+
* unparse of -0.0 preserves its sign
|
40
|
+
* more forgiving about parsing floats
|
41
|
+
* better error message on float coercion problem
|
42
|
+
* ruby 1.9:
|
43
|
+
* generally improving ability to run under mri 1.9 (fixing warnings, mostly)
|
44
|
+
all the new 1.9 syntax should now work
|
45
|
+
* notably improvements to -> and .()
|
46
|
+
avoiding warnings about undef'd ivars
|
47
|
+
avoiding duplication in char classes
|
48
|
+
* encoding:
|
49
|
+
* pass encoding on to the lexer
|
50
|
+
* encoding names should always be represented as symbols
|
51
|
+
* 1.9+extensibility:
|
52
|
+
* made @funclikes and @varlikes: ivars instead of constants
|
53
|
+
* @funclikes and @varlikes vary depending on whether in 1.9 mode
|
54
|
+
* -> is now considered a funclike keyword
|
55
|
+
* VALUELIKE_LA converted from a constant to a method
|
56
|
+
* FUNCLIKE_KEYWORD is now a method rather than a constant
|
57
|
+
* 1.9 syntax:
|
58
|
+
* look for block locals in method params only in -> args
|
59
|
+
* -> now has no method params but does have block params (and block locals)
|
60
|
+
* add locals slot for BlockNode
|
61
|
+
* and fill from vars after ; in block params
|
62
|
+
* special rule for -> in 1.9 not needed now (treated like KWCallNode)
|
63
|
+
* new exception in 1.9 for !@ method
|
64
|
+
* improved support for .() type calls in 1.9 mode
|
65
|
+
* programmer's interface:
|
66
|
+
* ast creation:
|
67
|
+
* Node.new should optionally allow ivars to be set using named params
|
68
|
+
* on StringNode.[], don't overwrite @modifiers if one is provided
|
69
|
+
* now parser uses LiteralNode.create to create literals
|
70
|
+
* (so client code can use LiteralNode.new)
|
71
|
+
* sugary constructor for ListInNode
|
72
|
+
* make RawOpNode#new arg parsing more robust
|
73
|
+
* ast reading/searching/reflecting:
|
74
|
+
* some utilities for finding nodes in node trees
|
75
|
+
* to_ruby is an alias for unparse in RedParse::Node and descendants
|
76
|
+
* ListInNode should compare == only if both sides are ListInNode
|
77
|
+
* explicit, public accessors for things that were private:
|
78
|
+
* added #body #reversed and #test_first to while and until nodes
|
79
|
+
* make the #string representation of literal nodes publicly accessible
|
80
|
+
* MethodNode#has_parens? now returns whether formal args had parens
|
81
|
+
* new names for things that were already available:
|
82
|
+
* give RescueOpNode an #op method (always returns "rescue")
|
83
|
+
* alias RescueOpNode#right to #rescue_with (for the fallback expression)
|
84
|
+
* alias CallSiteNode#has_parens? to #real_parens
|
85
|
+
* alias RescueNode#name to its varname
|
86
|
+
* alias UnOpNode#arg to #val
|
87
|
+
* alias UnaryOpNode to UnOpNode
|
88
|
+
* changes in structure, shouldn't impact users:
|
89
|
+
* empty parens should make a ParenedNode, not a SequenceNode
|
90
|
+
* NopNode is now a descendant of SequenceNode
|
91
|
+
* created UnAmpNode for unary ampersand
|
92
|
+
* location:
|
93
|
+
* print error location when parse or lex error occurs
|
94
|
+
* line numbers in Node trees are present/correct in more cases
|
95
|
+
* inspect:
|
96
|
+
* Node#inspect now omits some unimportant or duplicate subfields
|
97
|
+
* remove some duplication in pp output
|
98
|
+
* put + in front of ListInNode on inspect... so it looks like other nodes
|
99
|
+
* head of a constant is now inspected correctly if it happened to be a node
|
100
|
+
* make #inspect more robust in the presence of weird labels
|
101
|
+
* use shorter inspect in error string for misparsed nodes
|
102
|
+
* allow Node subclasses to exclude certain ivars from #inspect
|
103
|
+
* leave HashNode's @no_braces undef'd if not set... makes inspect view prettier
|
104
|
+
* misc api:
|
105
|
+
* added RedParse.parse, for parsing without a RedParse object
|
106
|
+
* all RedParse.new options after the 1st can now be omitted
|
107
|
+
* allow cache mode to be given in a env var, if not otherwise specified
|
108
|
+
* always put error messages on stderr!
|
109
|
+
* added stack attr to RedParse to allow access to the stack from outside
|
110
|
+
* created a #pretty_stack method for printing the stack out
|
111
|
+
* use pretty_stack to print stack
|
112
|
+
* display more fields of @input on inspect
|
113
|
+
* RedParse#inspect cleaner, while preserving old functionality via #dump
|
114
|
+
* have RedParse#to_s emit a more accurate image of the actual type
|
115
|
+
* improve RedParse#to_s slightly for faux @lexers
|
116
|
+
* made #rules an alias for #expanded_RULES
|
117
|
+
* rubyversion attr is no longer writable
|
118
|
+
* misc
|
119
|
+
* ripper emulation, still pretty raw/incomplete
|
120
|
+
* beginnings of drop-in replacements for 3 other parsers
|
121
|
+
* find stuff in gemspec relative to __FILE__ instead of .
|
122
|
+
* don't need this trashy rakefile anymore
|
123
|
+
* re-enabled the compiler
|
124
|
+
* support octal literals starting with +
|
125
|
+
* make sure there's a #blame in nodes that have #error? [api,AST]
|
126
|
+
* EXCLUDED_IVARS no longer needs to keep symbol and string forms of each elem
|
127
|
+
* command line:
|
128
|
+
* added --coalesce cmdline option to use new 'coalescer' parser compiler
|
129
|
+
* made an option for omitting the output of the parse tree
|
130
|
+
* added --cache cmdline option to control caching
|
131
|
+
* avoid printing initial garbage tokens when --print-tokens enabled
|
132
|
+
* added long names for -7 and -9 flags (version selection)
|
133
|
+
* don't require the compiler unless --compile given on cmdline
|
134
|
+
* don't print ugly minor node details unless -v set
|
135
|
+
* parsetree:
|
136
|
+
* improved compatibility w/ parsetree generally
|
137
|
+
* in HasRescue#parsetree should not modify reciever now
|
138
|
+
* in quirks mode:
|
139
|
+
* always force a :block to wrap #parsetree of begin..end nodes
|
140
|
+
* if the sole expression in the body is an undef
|
141
|
+
* (sigh...improves compatibility with cranky ref impl)
|
142
|
+
* only search for literals to nopify at the start
|
143
|
+
* of a sequence, instead of all but the last element
|
144
|
+
* paren node with no body looks like nil to #parsetree
|
145
|
+
* added several to known parsetree/mri boo-boos
|
146
|
+
* tests for ruby 1.9 variant:
|
147
|
+
* disable caching in 1.9 tests
|
148
|
+
* put each 1.9 test case in its own test method
|
149
|
+
* let's do all 1.9 testing in utf-8
|
150
|
+
* expect no sequences except in block bodies
|
151
|
+
* expect no multiassigns except if top-level is an assignment
|
152
|
+
* print code that failed if exception in 1.9 tests
|
153
|
+
* made test for invalid 1.9 expressions
|
154
|
+
* adding test for 1.9 constructs equivalent to other 1.9 constructs
|
155
|
+
* add test case for block locals in .() call
|
156
|
+
* tried to improve parse tree 'blurring' code
|
157
|
+
* (which ignores bs_handler in strings)
|
158
|
+
* include 1_9 in names of 1.9 tests
|
159
|
+
* tests:
|
160
|
+
* added extra_summary utility method to Test::Unit,
|
161
|
+
* which can print additional messages when the test is over
|
162
|
+
* used with known ruby bugs, keep a them separate
|
163
|
+
* since those aren't bugs in redparse
|
164
|
+
* cut down on the rate of the differed-by-begin warning
|
165
|
+
* a system of 'interactive' testing
|
166
|
+
* to allow user to confirm whether unparse is valid
|
167
|
+
* when not exactly equal to original
|
168
|
+
* only wank about parse errors if an env var is set
|
169
|
+
* an option to divide tests into multiple processes
|
170
|
+
* oops, error in mangle mode's detection of globals
|
171
|
+
* fixed a couple test cases that needed to have newlines
|
172
|
+
* a buncha new good (=exhibits errors) test data
|
173
|
+
* new wrappers and injectables expand the reach of mangle mode tests
|
174
|
+
* correcting or moving bad test cases
|
175
|
+
* eliminate duplicate tests
|
176
|
+
* rules:
|
177
|
+
* simplify operator rule by removing KeywordToken match
|
178
|
+
* but now => has to be parsed separately
|
179
|
+
* herebodytoken delete rule eliminated... now a lexer hack
|
180
|
+
* moved rules for parsing parenthesized expressions down to be lower prec than KWCallNode [rules]
|
181
|
+
* expand duties of expanded_RULES to wrapping string/regexp matchers in KW() [rules]
|
182
|
+
* rewrote #lower_op to return a proc [rules]
|
183
|
+
* cacheing of parse table intermediate structures
|
184
|
+
* @compiled_rules cache not needed anymore
|
185
|
+
* (slightly) improve error handling when no block to item_that
|
186
|
+
* initial stab at DanglingStarNode.create [rules]
|
187
|
+
* mini version of ruleset, just a few rules, for debugging [rules]
|
188
|
+
* improving readability in a char class
|
189
|
+
* moved lexer-specific code into the lexer
|
190
|
+
* __FILE__/__LINE__ value setting
|
191
|
+
* token linenums determination
|
192
|
+
|
1
193
|
=== 0.8.4 / 21dec2009
|
2
194
|
* 5 Major Enhancements:
|
3
195
|
* OpNode and related modules are now classes
|
data/Makefile
CHANGED
@@ -2,6 +2,15 @@ name=RedParse
|
|
2
2
|
lname=redparse
|
3
3
|
gemname=redparse
|
4
4
|
|
5
|
+
lib/redparse/ReduceWithsFor_RedParse_1_8.rb: lib/redparse.rb
|
6
|
+
redparse --cache=no -c
|
7
|
+
|
8
|
+
lib/redparse/ReduceWithsFor_RedParse_1_9.rb: lib/redparse.rb
|
9
|
+
redparse --cache=no -c '1.9'
|
10
|
+
|
11
|
+
parser: lib/redparse/ReduceWithsFor_RedParse_1_8.rb lib/redparse/ReduceWithsFor_RedParse_1_9.rb
|
12
|
+
.PHONY: parser
|
13
|
+
|
5
14
|
#everything after this line is generic
|
6
15
|
|
7
16
|
version=$(shell ruby -r ./lib/$(lname)/version.rb -e "puts $(name)::VERSION")
|
data/README.txt
CHANGED
@@ -4,31 +4,20 @@
|
|
4
4
|
|
5
5
|
== DESCRIPTION:
|
6
6
|
|
7
|
-
RedParse is a ruby parser written in pure ruby.
|
8
|
-
ANTLR, it's parse tool is a home-brewed
|
9
|
-
tool is LALR(1)-equivalent and the 'parse language' is
|
10
|
-
even in it's current
|
7
|
+
RedParse is a ruby parser (and parser-compiler) written in pure ruby.
|
8
|
+
Instead of YACC or ANTLR, it's parse tool is a home-brewed language. (The
|
9
|
+
tool is (at least) LALR(1)-equivalent and the 'parse language' is
|
10
|
+
pretty nice, even in it's current form.)
|
11
11
|
|
12
12
|
My intent is to have a completely correct parser for ruby, in 100%
|
13
|
-
ruby.
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
== REQUIREMENTS:
|
20
|
-
* RedParse requires RubyLexer, my hand-coded lexer for ruby. It also
|
21
|
-
uses Reg, (a pattern-matcher). RubyLexer depends on Sequence,
|
22
|
-
(external iterators). Reg depends on Sequence's predecessor, Cursor,
|
23
|
-
altho Cursor isn't used at all in RedParse. The (long-delayed) next
|
24
|
-
version of Reg will use Sequence. To summarize:
|
25
|
-
* RedParse 0.8.2 requires RubyLexer>=0.7.4 and Reg>=0.4.7
|
26
|
-
* RubyLexer 0.7.4 requires Sequence>=0.2.0
|
27
|
-
* Reg 0.4.7 requires Cursor (not really needed here)
|
28
|
-
* All are available as gems. (Or tarballs on rubyforge, if you must.)
|
13
|
+
ruby. And I think I've more or less succeeded. Aside from some fairly
|
14
|
+
minor quibbles (see below), RedParse can parse all known ruby 1.8 and 1.9
|
15
|
+
constructions correctly. Input text may be encoded in ascii, binary,
|
16
|
+
utf-8, iso-8859-1, and the euc-* family of encodings. Sjis is not yet
|
17
|
+
supported.
|
29
18
|
|
30
19
|
== INSTALL:
|
31
|
-
* gem install redparse #(
|
20
|
+
* gem install redparse #(as root if necessary)
|
32
21
|
|
33
22
|
== LICENSE:
|
34
23
|
|
@@ -40,8 +29,7 @@ Please see COPYING.LGPL for details.
|
|
40
29
|
* Pure ruby, through and through. No part is written in C, YACC,
|
41
30
|
ANTLR, lisp, assembly, intercal, befunge or any other language
|
42
31
|
except ruby.
|
43
|
-
* Pretty AST trees (at least, I think so).
|
44
|
-
necessarily to look at.)
|
32
|
+
* Pretty AST trees (at least, I think so).
|
45
33
|
* AST trees closely mirror the actual structure of source code.
|
46
34
|
* unparser is built in
|
47
35
|
* ParseTree format output too, if you want that.
|
@@ -56,20 +44,20 @@ Please see COPYING.LGPL for details.
|
|
56
44
|
actions (which occupy somewhere under 3100 lines in RedParse).
|
57
45
|
Also, what is a rule? I counted most things which required a
|
58
46
|
separate action in MRI's parser, I'm not sure if that's fair.
|
59
|
-
|
47
|
+
On the other hand, RedParse rules require no separate actions
|
48
|
+
anywhere.In the end, I still think RedParse is still much easier to
|
60
49
|
understand than MRI's parse.y.)
|
61
50
|
* "loosey-goosey" parser happily parses many expressions which normal
|
62
51
|
ruby considers errors.
|
63
52
|
|
64
53
|
== Drawbacks:
|
65
54
|
|
66
|
-
*
|
67
|
-
* Error
|
55
|
+
* Slow. Not as bad as it used to be, tho.
|
56
|
+
* Error coverage is sketchy at best
|
68
57
|
* No warnings at all.
|
69
|
-
* Unit test takes a fairly long time.
|
58
|
+
* Unit test takes a fairly long time (much better now, tho! down to 15min).
|
70
59
|
* Lots of warnings printed during unit test.
|
71
60
|
* Debugging parse rules is not straightforward.
|
72
|
-
* Incomplete support for ruby 1.9.
|
73
61
|
* "loosey-goosey" parser happily parses many expressions which normal
|
74
62
|
ruby considers errors.
|
75
63
|
|
@@ -248,22 +236,47 @@ existing format in the future, but no incompatibility-creating changes.
|
|
248
236
|
ErrorNode #mixed in to nodes with a syntax error
|
249
237
|
+-MisparsedNode #mismatched braces or begin..end or the like
|
250
238
|
|
239
|
+
== REQUIREMENTS:
|
240
|
+
* RedParse requires RubyLexer, my hand-coded lexer for ruby. It also
|
241
|
+
uses Reg, (a pattern-matcher). RubyLexer depends on Sequence,
|
242
|
+
(external iterators). Reg depends on Sequence as well. To summarize:
|
243
|
+
* RedParse 1.0.0 requires RubyLexer 0.8.0 and Reg>=0.4.8
|
244
|
+
* RubyLexer 0.8.0 requires Sequence>=0.2.4
|
245
|
+
* Reg 0.4.8 requires Sequence>=0.2.3
|
246
|
+
* All are available as gems. (Or tarballs on rubyforge, if you must.)
|
247
|
+
|
251
248
|
== Known problems with the parser:
|
252
249
|
* Encoding of the input is not stored anywhere in resulting parse tree.
|
253
250
|
* Ascii, binary, utf-8, and euc encodings are supported, but sjis is not.
|
251
|
+
* offsets of some nodes are incorrect
|
252
|
+
* multi-assign nested in multi-assign as receiver of an attribute lhs won't parse right
|
253
|
+
* these expressions won't parse:
|
254
|
+
<<x.
|
255
|
+
1111
|
256
|
+
x
|
257
|
+
a0 rescue b0()
|
258
|
+
|
259
|
+
?\
|
260
|
+
__END__
|
261
|
+
-?
|
262
|
+
|
263
|
+
__END__ #with a comment
|
264
|
+
|
265
|
+
* =f,g rescue b and c
|
254
266
|
|
255
267
|
== Known problems with the unparser:
|
256
|
-
*
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
268
|
+
* Major:
|
269
|
+
* unparsing of trees whose input was in a character set other than ascii may
|
270
|
+
not work.
|
271
|
+
* Minor:
|
272
|
+
* On unparse, here documents are converted into regular strings. For the
|
273
|
+
most part, these are exactly equivalent to the original. However,
|
274
|
+
whatever tokens appeared between the here document header and body will
|
275
|
+
now show up on a different line. If one of those tokens was __LINE__,
|
276
|
+
it will have a different value in the unparsed code than it had
|
277
|
+
originally.
|
278
|
+
|
279
|
+
== Known problems with ParseTree creator
|
267
280
|
* Major:
|
268
281
|
* converting non-ascii encoded parsetrees to ParseTree format doesn't work
|
269
282
|
* Minor:
|
@@ -273,6 +286,9 @@ existing format in the future, but no incompatibility-creating changes.
|
|
273
286
|
but what I emit is equivalent.
|
274
287
|
* %W"is #{"Slim #{2?"W":"S"}"}#{xx}."
|
275
288
|
* silly empty case nodes aren't always optimized to nop like in ParseTree.
|
289
|
+
* begin..end blocks nested in other begin..end blocks aren't combined
|
290
|
+
* literal strings,numbers,symbols,etc as their own stmts should be optimized away
|
291
|
+
* BEGIN expressions not in their own statement may not be treated right
|
276
292
|
|
277
293
|
== Bugs in ruby
|
278
294
|
* These expressions don't parse the same as in MRI because of bug(s) in MRI:
|
@@ -291,3 +307,20 @@ existing format in the future, but no incompatibility-creating changes.
|
|
291
307
|
* def sum(options = {:weights => weights = Hash.new(1)}); opt; end
|
292
308
|
* def foo(a = 1) end; def foo(a=b=c={}) end; def bar(a=b=c=1,d=2) end
|
293
309
|
* yield [a_i, *p]
|
310
|
+
|
311
|
+
== Copyright
|
312
|
+
redparse - a ruby parser written in ruby
|
313
|
+
Copyright (C) 2008,2009, 2012, 2016 Caleb Clausen
|
314
|
+
|
315
|
+
This program is free software: you can redistribute it and/or modify
|
316
|
+
it under the terms of the GNU Lesser General Public License as published by
|
317
|
+
the Free Software Foundation, either version 3 of the License, or
|
318
|
+
(at your option) any later version.
|
319
|
+
|
320
|
+
This program is distributed in the hope that it will be useful,
|
321
|
+
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
322
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
323
|
+
GNU Lesser General Public License for more details.
|
324
|
+
|
325
|
+
You should have received a copy of the GNU Lesser General Public License
|
326
|
+
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
data/bin/redparse
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
=begin
|
3
3
|
redparse - a ruby parser written in ruby
|
4
|
-
Copyright (C) 2008 Caleb Clausen
|
4
|
+
Copyright (C) 2008, 2012, 2016 Caleb Clausen
|
5
5
|
|
6
6
|
This library is free software; you can redistribute it and/or
|
7
7
|
modify it under the terms of the GNU Lesser General Public
|
@@ -96,27 +96,71 @@ while /^-/===ARGV.first
|
|
96
96
|
when "--pp"; output=:pp
|
97
97
|
when "--lisp"; output=:lisp
|
98
98
|
when "--parsetree"; output=:parsetree
|
99
|
+
when "--ripper"; output=:ripper; ruby19=true
|
100
|
+
when "--no-parsetree"; output=:no_parsetree
|
99
101
|
when "--vsparsetree"; output=:vsparsetree
|
100
102
|
when "--vsparsetree2"; output=:vsparsetree2
|
103
|
+
when "--vsripper"; output=:vsripper; ruby19=true
|
104
|
+
when "--vsripper2"; output=:vsripper2; ruby19=true
|
101
105
|
when "--update-problemfiles"; problemfiles=ProblemFiles.new
|
102
106
|
when "--ignore-silly-begins"; ignorebegins=true
|
103
|
-
when "--tokens"; ENV['PRINT_TOKENS']='1'
|
104
|
-
when "--rawtokens"; ENV['RAW_PRINT_TOKENS']='1'
|
105
|
-
when "--stack"; ENV['PRINT_STACK']='1'
|
107
|
+
when "--tokens"; tokens=ENV['PRINT_TOKENS']='1'
|
108
|
+
when "--rawtokens"; rawtokens=ENV['RAW_PRINT_TOKENS']='1'
|
109
|
+
when "--stack"; stack=ENV['PRINT_STACK']='1'
|
106
110
|
when "--unparse"; unparse=true
|
107
|
-
when
|
108
|
-
|
111
|
+
when /^(?:--output|-o)=?(.*)$/
|
112
|
+
outputf=($1.size>0 ? $1 : (/^[^-]/===ARGV.first && ARGV.shift ))
|
113
|
+
when /^(?:--require|-r)=?(.*)$/
|
114
|
+
reqs||=[]
|
115
|
+
reqs<<($1.size>0 ? $1 : (/^[^-]/===ARGV.first && ARGV.shift ))
|
116
|
+
when /^(?:--coalesce|-c)=?(.*)$/;
|
117
|
+
coalesce=true
|
118
|
+
variant=($1.size>0 ? $1 : (/^[^-]/===ARGV.first && ARGV.shift or ''))
|
119
|
+
when "--compile"; require 'redparse/compile'; compile=true
|
120
|
+
when "--macros", "--macro", "-m"
|
109
121
|
require 'rubygems'
|
110
122
|
require 'macro'
|
111
123
|
parserclass=Macro::RedParseWithMacros
|
124
|
+
when /^--cache(?:=(.*))?$/
|
125
|
+
cache_mode=
|
126
|
+
case mode=$1||ARGV.shift
|
127
|
+
when 'r','ro'; :read_only
|
128
|
+
when 'w','wo'; :write_only
|
129
|
+
when 'rw'; :read_write #default
|
130
|
+
when 'n','no','none'; :none
|
131
|
+
else mode.to_sym
|
132
|
+
end
|
112
133
|
when "-q"; quiet=true
|
113
134
|
when "-v"; verbose=true
|
114
135
|
when "-e"; inputs=[ARGV.join(" ")]; names=["-e"]; break
|
115
|
-
when "-7"; ruby187=true
|
116
|
-
when "-9"; ruby19=true
|
136
|
+
when "-7", "--1.8.7"; ruby187=true
|
137
|
+
when "-9", "--1.9"; ruby19=true
|
117
138
|
else fail "unknown option: #{opt}"
|
118
139
|
end
|
119
140
|
end
|
141
|
+
|
142
|
+
if coalesce
|
143
|
+
variant=variant.split(' ')
|
144
|
+
|
145
|
+
if reqs
|
146
|
+
$:.push '.'
|
147
|
+
reqs.each{|req| require req }
|
148
|
+
$:.delete '.'
|
149
|
+
end
|
150
|
+
versions,modules=variant.partition{|v| /^\d/===v }
|
151
|
+
modules.map!{|m| RedParse.const_get m }
|
152
|
+
fail if versions.size>1
|
153
|
+
version=versions.first||(ruby19 ? 1.9 : 1.8)
|
154
|
+
version=version.to_f
|
155
|
+
classes=modules.grep Class
|
156
|
+
modules-=classes
|
157
|
+
assert classes.size<=1
|
158
|
+
k=classes.first||RedParse
|
159
|
+
rp=k.new("",:rubyversion=>version,:cache_mode=>:none)
|
160
|
+
modules.each{|m| rp.extend m }
|
161
|
+
rp.write_reduce_withs outputf
|
162
|
+
exit
|
163
|
+
end
|
120
164
|
|
121
165
|
unless inputs
|
122
166
|
if ARGV.empty?
|
@@ -142,6 +186,8 @@ inputs.each_index{|i|
|
|
142
186
|
input=inputs[i] or next
|
143
187
|
name=names[i]
|
144
188
|
|
189
|
+
orig_input=input
|
190
|
+
|
145
191
|
if /\A=begin\s/===input
|
146
192
|
#combine 1st 2 lines of input
|
147
193
|
if /\A(=begin(.*)\n=end\s(.*))\n/===input
|
@@ -164,11 +210,22 @@ inputs.each_index{|i|
|
|
164
210
|
else
|
165
211
|
options={:rubyversion=>1.8}
|
166
212
|
end
|
213
|
+
options[:cache_mode]=cache_mode if cache_mode
|
214
|
+
parser=(parserclass||RedParse).new(input,name,1,[],options)
|
215
|
+
seen_ends=0
|
216
|
+
parser.print_filter=proc{|t|
|
217
|
+
if RedParse::KeywordToken===t and /^(\}|end)$/===t.ident
|
218
|
+
seen_ends+=1
|
219
|
+
seen_ends>=3
|
220
|
+
else
|
221
|
+
seen_ends>=2
|
222
|
+
end
|
223
|
+
}
|
167
224
|
tree=if compile
|
168
|
-
huh
|
225
|
+
huh parser.compile
|
169
226
|
huh parse
|
170
227
|
else
|
171
|
-
|
228
|
+
parser.parse
|
172
229
|
end
|
173
230
|
nil
|
174
231
|
} #raise NeverExecThis
|
@@ -198,16 +255,53 @@ inputs.each_index{|i|
|
|
198
255
|
require 'pp'
|
199
256
|
pp tree
|
200
257
|
when :p
|
201
|
-
|
258
|
+
puts tree.inspect(nil,0,verbose)
|
202
259
|
when :lisp
|
203
260
|
puts tree.to_lisp
|
204
261
|
when :parsetree
|
205
262
|
tree=tree.to_parsetree
|
206
263
|
hack=tree.dup
|
207
|
-
p hack
|
208
|
-
|
209
|
-
|
264
|
+
#p hack
|
265
|
+
=begin
|
266
|
+
hack.first[1..2]=[] #get rid of BEGIN blocks inserted into beginning of input
|
267
|
+
hack=if hack.first.size<=2; hack.first[1] else hack end
|
268
|
+
rescue Exception
|
269
|
+
=end
|
210
270
|
pp(hack||[])
|
271
|
+
when :vsripper,:vsripper2,:ripper
|
272
|
+
warn "disabling positioning info in ripper emulation output for now"
|
273
|
+
begin
|
274
|
+
require 'ripper'
|
275
|
+
require 'ripper/sexp'
|
276
|
+
class ::Ripper::SexpBuilder
|
277
|
+
def column; 0; end
|
278
|
+
def lineno; 0; end
|
279
|
+
end
|
280
|
+
rescue LoadError
|
281
|
+
end
|
282
|
+
require 'redparse/ripper'
|
283
|
+
require 'redparse/ripper_sexp'
|
284
|
+
RedParse::Ripper.instrumentSexpBuilder(::Ripper::SexpBuilder)if verbose and defined? ::Ripper
|
285
|
+
RedParse::Ripper.instrumentSexpBuilder(::Ripper::SexpBuilderPP)if verbose and defined? ::Ripper
|
286
|
+
RedParse::Ripper.instrumentSexpBuilder(::RedParse::Ripper::SexpBuilder)if verbose
|
287
|
+
RedParse::Ripper.instrumentSexpBuilder(::RedParse::Ripper::SexpBuilderPP)if verbose
|
288
|
+
rip=Ripper.sexp_raw(orig_input,name) if output!=:ripper and defined? ::Ripper
|
289
|
+
rp=RedParse::Ripper::SexpBuilder.new(input,name)
|
290
|
+
rp.parser=tree
|
291
|
+
begin
|
292
|
+
rp= rp.parse :quirks=>true
|
293
|
+
rescue Exception=>e
|
294
|
+
warn e.backtrace.unshift(e.inspect).join("\n ")
|
295
|
+
rp=[:failed]
|
296
|
+
end
|
297
|
+
if rip and rip==rp
|
298
|
+
puts "no differences"
|
299
|
+
else
|
300
|
+
puts "mine:"
|
301
|
+
pp rp
|
302
|
+
puts "minero's:"
|
303
|
+
pp rip
|
304
|
+
end
|
211
305
|
when :vsparsetree,:vsparsetree2
|
212
306
|
begin
|
213
307
|
require 'rubygems'
|