redparse 0.8.4 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -0
  2. data/COPYING.LGPL +503 -158
  3. data/History.txt +192 -0
  4. data/Makefile +9 -0
  5. data/README.txt +72 -39
  6. data/bin/redparse +108 -14
  7. data/lib/miniredparse.rb +1543 -0
  8. data/lib/redparse.rb +971 -105
  9. data/lib/redparse/ReduceWithsFor_RedParse_1_8.rb +17412 -0
  10. data/lib/redparse/ReduceWithsFor_RedParse_1_9.rb +17633 -0
  11. data/lib/redparse/babynodes.rb +17 -0
  12. data/lib/redparse/babyparser.rb +17 -0
  13. data/lib/redparse/cache.rb +290 -6
  14. data/lib/redparse/compile.rb +6 -97
  15. data/lib/redparse/decisiontree.rb +1 -1
  16. data/lib/redparse/float_accurate_to_s.rb +30 -6
  17. data/lib/redparse/generate.rb +18 -0
  18. data/lib/redparse/node.rb +415 -124
  19. data/lib/redparse/parse_tree_server.rb +20 -2
  20. data/lib/redparse/problemfiles.rb +1 -1
  21. data/lib/redparse/pthelper.rb +17 -31
  22. data/lib/redparse/reg_more_sugar.rb +1 -1
  23. data/lib/redparse/replacing/parse_tree.rb +30 -0
  24. data/lib/redparse/replacing/ripper.rb +20 -0
  25. data/lib/redparse/replacing/ruby_parser.rb +28 -0
  26. data/lib/redparse/ripper.rb +393 -0
  27. data/lib/redparse/ripper_sexp.rb +153 -0
  28. data/lib/redparse/stackableclasses.rb +113 -0
  29. data/lib/redparse/version.rb +18 -1
  30. data/redparse.gemspec +29 -9
  31. data/rplt.txt +31 -0
  32. data/test/data/hd_with_blank_string.rb +3 -0
  33. data/test/data/pt_known_output.rb +13273 -0
  34. data/test/data/wp.pp +0 -0
  35. data/test/generate_parse_tree_server_rc.rb +17 -0
  36. data/test/rp-locatetest.rb +2 -2
  37. data/test/test_1.9.rb +338 -35
  38. data/test/test_all.rb +22 -3
  39. data/test/test_part.rb +32 -0
  40. data/test/test_redparse.rb +396 -74
  41. data/test/test_xform_tree.rb +18 -0
  42. data/test/unparse_1.9_exceptions.txt +85 -0
  43. data/test/unparse_1.9_exceptions.txt.old +81 -0
  44. metadata +71 -46
  45. data/Rakefile +0 -35
@@ -1,3 +1,195 @@
1
+ === v1.0.0 26oct2011
2
+ * performance improvements:
3
+ * coalescer:
4
+ * a new backend, the 'coalescer' is 10x faster and actually works!
5
+ * canned coalesce output (which takes forever to generate otherwise)
6
+ * cache:
7
+ * cached outputs are stored near to the inputs if that's writeable
8
+ * otherwise, they're in the first writeable directory of:
9
+ * Etc.getpwuid if any, $HOME if any, root dir
10
+ * if none are writeable, nothing is cached
11
+ * defined a parser signature that will change when:
12
+ * parser source code changes, or
13
+ * parser type (including modules parser is extended by) changes, or
14
+ * lexer type changes
15
+ * many improvements to #unparse (which turns Node trees back into ruby code):
16
+ * slightly tweaked the unparsing of defined? so it'll reparse correctly
17
+ * in AssignNode#unparse, always put spaces around assign op
18
+ * only elide spaces right before [ on unparse of BracketsGetNode
19
+ * unparse of for node should not start with newlines...
20
+ * speed up unparse_nl when no token provided
21
+ * 3rd param to unparse_nl (alt) should never be nil
22
+ * parens and the like and when to omit them from unparse output:
23
+ * need to call to_s on body of ParenedNode#unparse
24
+ * emit extra () on unparse in hash and array lits and method call params.
25
+ * better detection of when to leave out curly braces in hash literal
26
+ * unparse of MethodNode and CallSiteNode now omits parens if original did
27
+ * hardcode unparse of NopNode to "()" (was empty string)
28
+ * NopNode in str inclusion unparses to #{} (was: #{()})
29
+ * string literal unparsing:
30
+ * fix unparse of string when delimiter is \r
31
+ * better detection of backslash (and friends) in strings
32
+ * for splitting word array or requoting here docs
33
+ * exception machinery and unparsing:
34
+ * tolerate missing #rescues in nodes which allow rescue
35
+ * oops, #unparse was inserting extraneous empty elses
36
+ * unparsing float literals:
37
+ * unparse to original string rep of a float literal if available
38
+ * (to reduce rounding errors)
39
+ * unparse of -0.0 preserves its sign
40
+ * more forgiving about parsing floats
41
+ * better error message on float coercion problem
42
+ * ruby 1.9:
43
+ * generally improving ability to run under mri 1.9 (fixing warnings, mostly)
44
+ all the new 1.9 syntax should now work
45
+ * notably improvements to -> and .()
46
+ avoiding warnings about undef'd ivars
47
+ avoiding duplication in char classes
48
+ * encoding:
49
+ * pass encoding on to the lexer
50
+ * encoding names should always be represented as symbols
51
+ * 1.9+extensibility:
52
+ * made @funclikes and @varlikes: ivars instead of constants
53
+ * @funclikes and @varlikes vary depending on whether in 1.9 mode
54
+ * -> is now considered a funclike keyword
55
+ * VALUELIKE_LA converted from a constant to a method
56
+ * FUNCLIKE_KEYWORD is now a method rather than a constant
57
+ * 1.9 syntax:
58
+ * look for block locals in method params only in -> args
59
+ * -> now has no method params but does have block params (and block locals)
60
+ * add locals slot for BlockNode
61
+ * and fill from vars after ; in block params
62
+ * special rule for -> in 1.9 not needed now (treated like KWCallNode)
63
+ * new exception in 1.9 for !@ method
64
+ * improved support for .() type calls in 1.9 mode
65
+ * programmer's interface:
66
+ * ast creation:
67
+ * Node.new should optionally allow ivars to be set using named params
68
+ * on StringNode.[], don't overwrite @modifiers if one is provided
69
+ * now parser uses LiteralNode.create to create literals
70
+ * (so client code can use LiteralNode.new)
71
+ * sugary constructor for ListInNode
72
+ * make RawOpNode#new arg parsing more robust
73
+ * ast reading/searching/reflecting:
74
+ * some utilities for finding nodes in node trees
75
+ * to_ruby is an alias for unparse in RedParse::Node and descendants
76
+ * ListInNode should compare == only if both sides are ListInNode
77
+ * explicit, public accessors for things that were private:
78
+ * added #body #reversed and #test_first to while and until nodes
79
+ * make the #string representation of literal nodes publicly accessible
80
+ * MethodNode#has_parens? now returns whether formal args had parens
81
+ * new names for things that were already available:
82
+ * give RescueOpNode an #op method (always returns "rescue")
83
+ * alias RescueOpNode#right to #rescue_with (for the fallback expression)
84
+ * alias CallSiteNode#has_parens? to #real_parens
85
+ * alias RescueNode#name to its varname
86
+ * alias UnOpNode#arg to #val
87
+ * alias UnaryOpNode to UnOpNode
88
+ * changes in structure, shouldn't impact users:
89
+ * empty parens should make a ParenedNode, not a SequenceNode
90
+ * NopNode is now a descendant of SequenceNode
91
+ * created UnAmpNode for unary ampersand
92
+ * location:
93
+ * print error location when parse or lex error occurs
94
+ * line numbers in Node trees are present/correct in more cases
95
+ * inspect:
96
+ * Node#inspect now omits some unimportant or duplicate subfields
97
+ * remove some duplication in pp output
98
+ * put + in front of ListInNode on inspect... so it looks like other nodes
99
+ * head of a constant is now inspected correctly if it happened to be a node
100
+ * make #inspect more robust in the presence of weird labels
101
+ * use shorter inspect in error string for misparsed nodes
102
+ * allow Node subclasses to exclude certain ivars from #inspect
103
+ * leave HashNode's @no_braces undef'd if not set... makes inspect view prettier
104
+ * misc api:
105
+ * added RedParse.parse, for parsing without a RedParse object
106
+ * all RedParse.new options after the 1st can now be omitted
107
+ * allow cache mode to be given in a env var, if not otherwise specified
108
+ * always put error messages on stderr!
109
+ * added stack attr to RedParse to allow access to the stack from outside
110
+ * created a #pretty_stack method for printing the stack out
111
+ * use pretty_stack to print stack
112
+ * display more fields of @input on inspect
113
+ * RedParse#inspect cleaner, while preserving old functionality via #dump
114
+ * have RedParse#to_s emit a more accurate image of the actual type
115
+ * improve RedParse#to_s slightly for faux @lexers
116
+ * made #rules an alias for #expanded_RULES
117
+ * rubyversion attr is no longer writable
118
+ * misc
119
+ * ripper emulation, still pretty raw/incomplete
120
+ * beginnings of drop-in replacements for 3 other parsers
121
+ * find stuff in gemspec relative to __FILE__ instead of .
122
+ * don't need this trashy rakefile anymore
123
+ * re-enabled the compiler
124
+ * support octal literals starting with +
125
+ * make sure there's a #blame in nodes that have #error? [api,AST]
126
+ * EXCLUDED_IVARS no longer needs to keep symbol and string forms of each elem
127
+ * command line:
128
+ * added --coalesce cmdline option to use new 'coalescer' parser compiler
129
+ * made an option for omitting the output of the parse tree
130
+ * added --cache cmdline option to control caching
131
+ * avoid printing initial garbage tokens when --print-tokens enabled
132
+ * added long names for -7 and -9 flags (version selection)
133
+ * don't require the compiler unless --compile given on cmdline
134
+ * don't print ugly minor node details unless -v set
135
+ * parsetree:
136
+ * improved compatibility w/ parsetree generally
137
+ * in HasRescue#parsetree should not modify reciever now
138
+ * in quirks mode:
139
+ * always force a :block to wrap #parsetree of begin..end nodes
140
+ * if the sole expression in the body is an undef
141
+ * (sigh...improves compatibility with cranky ref impl)
142
+ * only search for literals to nopify at the start
143
+ * of a sequence, instead of all but the last element
144
+ * paren node with no body looks like nil to #parsetree
145
+ * added several to known parsetree/mri boo-boos
146
+ * tests for ruby 1.9 variant:
147
+ * disable caching in 1.9 tests
148
+ * put each 1.9 test case in its own test method
149
+ * let's do all 1.9 testing in utf-8
150
+ * expect no sequences except in block bodies
151
+ * expect no multiassigns except if top-level is an assignment
152
+ * print code that failed if exception in 1.9 tests
153
+ * made test for invalid 1.9 expressions
154
+ * adding test for 1.9 constructs equivalent to other 1.9 constructs
155
+ * add test case for block locals in .() call
156
+ * tried to improve parse tree 'blurring' code
157
+ * (which ignores bs_handler in strings)
158
+ * include 1_9 in names of 1.9 tests
159
+ * tests:
160
+ * added extra_summary utility method to Test::Unit,
161
+ * which can print additional messages when the test is over
162
+ * used with known ruby bugs, keep a them separate
163
+ * since those aren't bugs in redparse
164
+ * cut down on the rate of the differed-by-begin warning
165
+ * a system of 'interactive' testing
166
+ * to allow user to confirm whether unparse is valid
167
+ * when not exactly equal to original
168
+ * only wank about parse errors if an env var is set
169
+ * an option to divide tests into multiple processes
170
+ * oops, error in mangle mode's detection of globals
171
+ * fixed a couple test cases that needed to have newlines
172
+ * a buncha new good (=exhibits errors) test data
173
+ * new wrappers and injectables expand the reach of mangle mode tests
174
+ * correcting or moving bad test cases
175
+ * eliminate duplicate tests
176
+ * rules:
177
+ * simplify operator rule by removing KeywordToken match
178
+ * but now => has to be parsed separately
179
+ * herebodytoken delete rule eliminated... now a lexer hack
180
+ * moved rules for parsing parenthesized expressions down to be lower prec than KWCallNode [rules]
181
+ * expand duties of expanded_RULES to wrapping string/regexp matchers in KW() [rules]
182
+ * rewrote #lower_op to return a proc [rules]
183
+ * cacheing of parse table intermediate structures
184
+ * @compiled_rules cache not needed anymore
185
+ * (slightly) improve error handling when no block to item_that
186
+ * initial stab at DanglingStarNode.create [rules]
187
+ * mini version of ruleset, just a few rules, for debugging [rules]
188
+ * improving readability in a char class
189
+ * moved lexer-specific code into the lexer
190
+ * __FILE__/__LINE__ value setting
191
+ * token linenums determination
192
+
1
193
  === 0.8.4 / 21dec2009
2
194
  * 5 Major Enhancements:
3
195
  * OpNode and related modules are now classes
data/Makefile CHANGED
@@ -2,6 +2,15 @@ name=RedParse
2
2
  lname=redparse
3
3
  gemname=redparse
4
4
 
5
+ lib/redparse/ReduceWithsFor_RedParse_1_8.rb: lib/redparse.rb
6
+ redparse --cache=no -c
7
+
8
+ lib/redparse/ReduceWithsFor_RedParse_1_9.rb: lib/redparse.rb
9
+ redparse --cache=no -c '1.9'
10
+
11
+ parser: lib/redparse/ReduceWithsFor_RedParse_1_8.rb lib/redparse/ReduceWithsFor_RedParse_1_9.rb
12
+ .PHONY: parser
13
+
5
14
  #everything after this line is generic
6
15
 
7
16
  version=$(shell ruby -r ./lib/$(lname)/version.rb -e "puts $(name)::VERSION")
data/README.txt CHANGED
@@ -4,31 +4,20 @@
4
4
 
5
5
  == DESCRIPTION:
6
6
 
7
- RedParse is a ruby parser written in pure ruby. Instead of YACC or
8
- ANTLR, it's parse tool is a home-brewed "compiler-interpreter". (The
9
- tool is LALR(1)-equivalent and the 'parse language' is pretty nice,
10
- even in it's current crude form.)
7
+ RedParse is a ruby parser (and parser-compiler) written in pure ruby.
8
+ Instead of YACC or ANTLR, it's parse tool is a home-brewed language. (The
9
+ tool is (at least) LALR(1)-equivalent and the 'parse language' is
10
+ pretty nice, even in it's current form.)
11
11
 
12
12
  My intent is to have a completely correct parser for ruby, in 100%
13
- ruby. Currently, RedParse can parse all known ruby 1.8 constructions
14
- correctly. There might be some problems with unparsing or otherwise
15
- working with texts in a character set other than ascii. Some of the
16
- new ruby 1.9 constructions are supported in 1.9 mode. For more
17
- details on known problems, see below.
18
-
19
- == REQUIREMENTS:
20
- * RedParse requires RubyLexer, my hand-coded lexer for ruby. It also
21
- uses Reg, (a pattern-matcher). RubyLexer depends on Sequence,
22
- (external iterators). Reg depends on Sequence's predecessor, Cursor,
23
- altho Cursor isn't used at all in RedParse. The (long-delayed) next
24
- version of Reg will use Sequence. To summarize:
25
- * RedParse 0.8.2 requires RubyLexer>=0.7.4 and Reg>=0.4.7
26
- * RubyLexer 0.7.4 requires Sequence>=0.2.0
27
- * Reg 0.4.7 requires Cursor (not really needed here)
28
- * All are available as gems. (Or tarballs on rubyforge, if you must.)
13
+ ruby. And I think I've more or less succeeded. Aside from some fairly
14
+ minor quibbles (see below), RedParse can parse all known ruby 1.8 and 1.9
15
+ constructions correctly. Input text may be encoded in ascii, binary,
16
+ utf-8, iso-8859-1, and the euc-* family of encodings. Sjis is not yet
17
+ supported.
29
18
 
30
19
  == INSTALL:
31
- * gem install redparse #(if root as necssary)
20
+ * gem install redparse #(as root if necessary)
32
21
 
33
22
  == LICENSE:
34
23
 
@@ -40,8 +29,7 @@ Please see COPYING.LGPL for details.
40
29
  * Pure ruby, through and through. No part is written in C, YACC,
41
30
  ANTLR, lisp, assembly, intercal, befunge or any other language
42
31
  except ruby.
43
- * Pretty AST trees (at least, I think so). (To program for, not
44
- necessarily to look at.)
32
+ * Pretty AST trees (at least, I think so).
45
33
  * AST trees closely mirror the actual structure of source code.
46
34
  * unparser is built in
47
35
  * ParseTree format output too, if you want that.
@@ -56,20 +44,20 @@ Please see COPYING.LGPL for details.
56
44
  actions (which occupy somewhere under 3100 lines in RedParse).
57
45
  Also, what is a rule? I counted most things which required a
58
46
  separate action in MRI's parser, I'm not sure if that's fair.
59
- But in the end, I still think RedParse is still much easier to
47
+ On the other hand, RedParse rules require no separate actions
48
+ anywhere.In the end, I still think RedParse is still much easier to
60
49
  understand than MRI's parse.y.)
61
50
  * "loosey-goosey" parser happily parses many expressions which normal
62
51
  ruby considers errors.
63
52
 
64
53
  == Drawbacks:
65
54
 
66
- * Pathetically, ridiculously slow (ok, compiler-compilers are hard...)
67
- * Error handling is very minimal right now.
55
+ * Slow. Not as bad as it used to be, tho.
56
+ * Error coverage is sketchy at best
68
57
  * No warnings at all.
69
- * Unit test takes a fairly long time.
58
+ * Unit test takes a fairly long time (much better now, tho! down to 15min).
70
59
  * Lots of warnings printed during unit test.
71
60
  * Debugging parse rules is not straightforward.
72
- * Incomplete support for ruby 1.9.
73
61
  * "loosey-goosey" parser happily parses many expressions which normal
74
62
  ruby considers errors.
75
63
 
@@ -248,22 +236,47 @@ existing format in the future, but no incompatibility-creating changes.
248
236
  ErrorNode #mixed in to nodes with a syntax error
249
237
  +-MisparsedNode #mismatched braces or begin..end or the like
250
238
 
239
+ == REQUIREMENTS:
240
+ * RedParse requires RubyLexer, my hand-coded lexer for ruby. It also
241
+ uses Reg, (a pattern-matcher). RubyLexer depends on Sequence,
242
+ (external iterators). Reg depends on Sequence as well. To summarize:
243
+ * RedParse 1.0.0 requires RubyLexer 0.8.0 and Reg>=0.4.8
244
+ * RubyLexer 0.8.0 requires Sequence>=0.2.4
245
+ * Reg 0.4.8 requires Sequence>=0.2.3
246
+ * All are available as gems. (Or tarballs on rubyforge, if you must.)
247
+
251
248
  == Known problems with the parser:
252
249
  * Encoding of the input is not stored anywhere in resulting parse tree.
253
250
  * Ascii, binary, utf-8, and euc encodings are supported, but sjis is not.
251
+ * offsets of some nodes are incorrect
252
+ * multi-assign nested in multi-assign as receiver of an attribute lhs won't parse right
253
+ * these expressions won't parse:
254
+ <<x.
255
+ 1111
256
+ x
257
+ a0 rescue b0()
258
+
259
+ ?\
260
+ __END__
261
+ -?
262
+
263
+ __END__ #with a comment
264
+
265
+ * =f,g rescue b and c
254
266
 
255
267
  == Known problems with the unparser:
256
- * On unparse, here documents are converted into regular strings. For the most
257
- part, these are exactly equivalent to the original. However, whatever tokens
258
- appeared between the here document header and body will now show up on a
259
- different line. If one of those tokens was __LINE__, it will have a
260
- different value in the unparsed code than it had originally.
261
- * some floating-point literals don't survive parse/unparse roundtrip intact,
262
- due to bugs in MRI 1.8's Float#to_s/String#to_f.
263
- * unparsing of trees whose input was in a character set other than ascii may
264
- not work.
265
-
266
- == Known problems with ParseTree compatibility
268
+ * Major:
269
+ * unparsing of trees whose input was in a character set other than ascii may
270
+ not work.
271
+ * Minor:
272
+ * On unparse, here documents are converted into regular strings. For the
273
+ most part, these are exactly equivalent to the original. However,
274
+ whatever tokens appeared between the here document header and body will
275
+ now show up on a different line. If one of those tokens was __LINE__,
276
+ it will have a different value in the unparsed code than it had
277
+ originally.
278
+
279
+ == Known problems with ParseTree creator
267
280
  * Major:
268
281
  * converting non-ascii encoded parsetrees to ParseTree format doesn't work
269
282
  * Minor:
@@ -273,6 +286,9 @@ existing format in the future, but no incompatibility-creating changes.
273
286
  but what I emit is equivalent.
274
287
  * %W"is #{"Slim #{2?"W":"S"}"}#{xx}."
275
288
  * silly empty case nodes aren't always optimized to nop like in ParseTree.
289
+ * begin..end blocks nested in other begin..end blocks aren't combined
290
+ * literal strings,numbers,symbols,etc as their own stmts should be optimized away
291
+ * BEGIN expressions not in their own statement may not be treated right
276
292
 
277
293
  == Bugs in ruby
278
294
  * These expressions don't parse the same as in MRI because of bug(s) in MRI:
@@ -291,3 +307,20 @@ existing format in the future, but no incompatibility-creating changes.
291
307
  * def sum(options = {:weights => weights = Hash.new(1)}); opt; end
292
308
  * def foo(a = 1) end; def foo(a=b=c={}) end; def bar(a=b=c=1,d=2) end
293
309
  * yield [a_i, *p]
310
+
311
+ == Copyright
312
+ redparse - a ruby parser written in ruby
313
+ Copyright (C) 2008,2009, 2012, 2016 Caleb Clausen
314
+
315
+ This program is free software: you can redistribute it and/or modify
316
+ it under the terms of the GNU Lesser General Public License as published by
317
+ the Free Software Foundation, either version 3 of the License, or
318
+ (at your option) any later version.
319
+
320
+ This program is distributed in the hope that it will be useful,
321
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
322
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
323
+ GNU Lesser General Public License for more details.
324
+
325
+ You should have received a copy of the GNU Lesser General Public License
326
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
@@ -1,7 +1,7 @@
1
1
  #!/usr/bin/env ruby
2
2
  =begin
3
3
  redparse - a ruby parser written in ruby
4
- Copyright (C) 2008 Caleb Clausen
4
+ Copyright (C) 2008, 2012, 2016 Caleb Clausen
5
5
 
6
6
  This library is free software; you can redistribute it and/or
7
7
  modify it under the terms of the GNU Lesser General Public
@@ -96,27 +96,71 @@ while /^-/===ARGV.first
96
96
  when "--pp"; output=:pp
97
97
  when "--lisp"; output=:lisp
98
98
  when "--parsetree"; output=:parsetree
99
+ when "--ripper"; output=:ripper; ruby19=true
100
+ when "--no-parsetree"; output=:no_parsetree
99
101
  when "--vsparsetree"; output=:vsparsetree
100
102
  when "--vsparsetree2"; output=:vsparsetree2
103
+ when "--vsripper"; output=:vsripper; ruby19=true
104
+ when "--vsripper2"; output=:vsripper2; ruby19=true
101
105
  when "--update-problemfiles"; problemfiles=ProblemFiles.new
102
106
  when "--ignore-silly-begins"; ignorebegins=true
103
- when "--tokens"; ENV['PRINT_TOKENS']='1'
104
- when "--rawtokens"; ENV['RAW_PRINT_TOKENS']='1'
105
- when "--stack"; ENV['PRINT_STACK']='1'
107
+ when "--tokens"; tokens=ENV['PRINT_TOKENS']='1'
108
+ when "--rawtokens"; rawtokens=ENV['RAW_PRINT_TOKENS']='1'
109
+ when "--stack"; stack=ENV['PRINT_STACK']='1'
106
110
  when "--unparse"; unparse=true
107
- when "--compile", "-c"; compile=true
108
- when "--macros", "--macro", "-m";
111
+ when /^(?:--output|-o)=?(.*)$/
112
+ outputf=($1.size>0 ? $1 : (/^[^-]/===ARGV.first && ARGV.shift ))
113
+ when /^(?:--require|-r)=?(.*)$/
114
+ reqs||=[]
115
+ reqs<<($1.size>0 ? $1 : (/^[^-]/===ARGV.first && ARGV.shift ))
116
+ when /^(?:--coalesce|-c)=?(.*)$/;
117
+ coalesce=true
118
+ variant=($1.size>0 ? $1 : (/^[^-]/===ARGV.first && ARGV.shift or ''))
119
+ when "--compile"; require 'redparse/compile'; compile=true
120
+ when "--macros", "--macro", "-m"
109
121
  require 'rubygems'
110
122
  require 'macro'
111
123
  parserclass=Macro::RedParseWithMacros
124
+ when /^--cache(?:=(.*))?$/
125
+ cache_mode=
126
+ case mode=$1||ARGV.shift
127
+ when 'r','ro'; :read_only
128
+ when 'w','wo'; :write_only
129
+ when 'rw'; :read_write #default
130
+ when 'n','no','none'; :none
131
+ else mode.to_sym
132
+ end
112
133
  when "-q"; quiet=true
113
134
  when "-v"; verbose=true
114
135
  when "-e"; inputs=[ARGV.join(" ")]; names=["-e"]; break
115
- when "-7"; ruby187=true
116
- when "-9"; ruby19=true
136
+ when "-7", "--1.8.7"; ruby187=true
137
+ when "-9", "--1.9"; ruby19=true
117
138
  else fail "unknown option: #{opt}"
118
139
  end
119
140
  end
141
+
142
+ if coalesce
143
+ variant=variant.split(' ')
144
+
145
+ if reqs
146
+ $:.push '.'
147
+ reqs.each{|req| require req }
148
+ $:.delete '.'
149
+ end
150
+ versions,modules=variant.partition{|v| /^\d/===v }
151
+ modules.map!{|m| RedParse.const_get m }
152
+ fail if versions.size>1
153
+ version=versions.first||(ruby19 ? 1.9 : 1.8)
154
+ version=version.to_f
155
+ classes=modules.grep Class
156
+ modules-=classes
157
+ assert classes.size<=1
158
+ k=classes.first||RedParse
159
+ rp=k.new("",:rubyversion=>version,:cache_mode=>:none)
160
+ modules.each{|m| rp.extend m }
161
+ rp.write_reduce_withs outputf
162
+ exit
163
+ end
120
164
 
121
165
  unless inputs
122
166
  if ARGV.empty?
@@ -142,6 +186,8 @@ inputs.each_index{|i|
142
186
  input=inputs[i] or next
143
187
  name=names[i]
144
188
 
189
+ orig_input=input
190
+
145
191
  if /\A=begin\s/===input
146
192
  #combine 1st 2 lines of input
147
193
  if /\A(=begin(.*)\n=end\s(.*))\n/===input
@@ -164,11 +210,22 @@ inputs.each_index{|i|
164
210
  else
165
211
  options={:rubyversion=>1.8}
166
212
  end
213
+ options[:cache_mode]=cache_mode if cache_mode
214
+ parser=(parserclass||RedParse).new(input,name,1,[],options)
215
+ seen_ends=0
216
+ parser.print_filter=proc{|t|
217
+ if RedParse::KeywordToken===t and /^(\}|end)$/===t.ident
218
+ seen_ends+=1
219
+ seen_ends>=3
220
+ else
221
+ seen_ends>=2
222
+ end
223
+ }
167
224
  tree=if compile
168
- huh (parserclass||RedParse).new(input,name,1,[],options).compile
225
+ huh parser.compile
169
226
  huh parse
170
227
  else
171
- (parserclass||RedParse).new(input,name,1,[],options).parse
228
+ parser.parse
172
229
  end
173
230
  nil
174
231
  } #raise NeverExecThis
@@ -198,16 +255,53 @@ inputs.each_index{|i|
198
255
  require 'pp'
199
256
  pp tree
200
257
  when :p
201
- p tree
258
+ puts tree.inspect(nil,0,verbose)
202
259
  when :lisp
203
260
  puts tree.to_lisp
204
261
  when :parsetree
205
262
  tree=tree.to_parsetree
206
263
  hack=tree.dup
207
- p hack
208
- hack.first[1..2]=[] #get rid of BEGIN blocks inserted into beginning of input
209
- hack=if hack.first.size<=2; hack.first[1] else hack end
264
+ #p hack
265
+ =begin
266
+ hack.first[1..2]=[] #get rid of BEGIN blocks inserted into beginning of input
267
+ hack=if hack.first.size<=2; hack.first[1] else hack end
268
+ rescue Exception
269
+ =end
210
270
  pp(hack||[])
271
+ when :vsripper,:vsripper2,:ripper
272
+ warn "disabling positioning info in ripper emulation output for now"
273
+ begin
274
+ require 'ripper'
275
+ require 'ripper/sexp'
276
+ class ::Ripper::SexpBuilder
277
+ def column; 0; end
278
+ def lineno; 0; end
279
+ end
280
+ rescue LoadError
281
+ end
282
+ require 'redparse/ripper'
283
+ require 'redparse/ripper_sexp'
284
+ RedParse::Ripper.instrumentSexpBuilder(::Ripper::SexpBuilder)if verbose and defined? ::Ripper
285
+ RedParse::Ripper.instrumentSexpBuilder(::Ripper::SexpBuilderPP)if verbose and defined? ::Ripper
286
+ RedParse::Ripper.instrumentSexpBuilder(::RedParse::Ripper::SexpBuilder)if verbose
287
+ RedParse::Ripper.instrumentSexpBuilder(::RedParse::Ripper::SexpBuilderPP)if verbose
288
+ rip=Ripper.sexp_raw(orig_input,name) if output!=:ripper and defined? ::Ripper
289
+ rp=RedParse::Ripper::SexpBuilder.new(input,name)
290
+ rp.parser=tree
291
+ begin
292
+ rp= rp.parse :quirks=>true
293
+ rescue Exception=>e
294
+ warn e.backtrace.unshift(e.inspect).join("\n ")
295
+ rp=[:failed]
296
+ end
297
+ if rip and rip==rp
298
+ puts "no differences"
299
+ else
300
+ puts "mine:"
301
+ pp rp
302
+ puts "minero's:"
303
+ pp rip
304
+ end
211
305
  when :vsparsetree,:vsparsetree2
212
306
  begin
213
307
  require 'rubygems'