redparse 0.8.4 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -0
- data/COPYING.LGPL +503 -158
- data/History.txt +192 -0
- data/Makefile +9 -0
- data/README.txt +72 -39
- data/bin/redparse +108 -14
- data/lib/miniredparse.rb +1543 -0
- data/lib/redparse.rb +971 -105
- data/lib/redparse/ReduceWithsFor_RedParse_1_8.rb +17412 -0
- data/lib/redparse/ReduceWithsFor_RedParse_1_9.rb +17633 -0
- data/lib/redparse/babynodes.rb +17 -0
- data/lib/redparse/babyparser.rb +17 -0
- data/lib/redparse/cache.rb +290 -6
- data/lib/redparse/compile.rb +6 -97
- data/lib/redparse/decisiontree.rb +1 -1
- data/lib/redparse/float_accurate_to_s.rb +30 -6
- data/lib/redparse/generate.rb +18 -0
- data/lib/redparse/node.rb +415 -124
- data/lib/redparse/parse_tree_server.rb +20 -2
- data/lib/redparse/problemfiles.rb +1 -1
- data/lib/redparse/pthelper.rb +17 -31
- data/lib/redparse/reg_more_sugar.rb +1 -1
- data/lib/redparse/replacing/parse_tree.rb +30 -0
- data/lib/redparse/replacing/ripper.rb +20 -0
- data/lib/redparse/replacing/ruby_parser.rb +28 -0
- data/lib/redparse/ripper.rb +393 -0
- data/lib/redparse/ripper_sexp.rb +153 -0
- data/lib/redparse/stackableclasses.rb +113 -0
- data/lib/redparse/version.rb +18 -1
- data/redparse.gemspec +29 -9
- data/rplt.txt +31 -0
- data/test/data/hd_with_blank_string.rb +3 -0
- data/test/data/pt_known_output.rb +13273 -0
- data/test/data/wp.pp +0 -0
- data/test/generate_parse_tree_server_rc.rb +17 -0
- data/test/rp-locatetest.rb +2 -2
- data/test/test_1.9.rb +338 -35
- data/test/test_all.rb +22 -3
- data/test/test_part.rb +32 -0
- data/test/test_redparse.rb +396 -74
- data/test/test_xform_tree.rb +18 -0
- data/test/unparse_1.9_exceptions.txt +85 -0
- data/test/unparse_1.9_exceptions.txt.old +81 -0
- metadata +71 -46
- data/Rakefile +0 -35
data/History.txt
CHANGED
@@ -1,3 +1,195 @@
|
|
1
|
+
=== v1.0.0 26oct2011
|
2
|
+
* performance improvements:
|
3
|
+
* coalescer:
|
4
|
+
* a new backend, the 'coalescer' is 10x faster and actually works!
|
5
|
+
* canned coalesce output (which takes forever to generate otherwise)
|
6
|
+
* cache:
|
7
|
+
* cached outputs are stored near to the inputs if that's writeable
|
8
|
+
* otherwise, they're in the first writeable directory of:
|
9
|
+
* Etc.getpwuid if any, $HOME if any, root dir
|
10
|
+
* if none are writeable, nothing is cached
|
11
|
+
* defined a parser signature that will change when:
|
12
|
+
* parser source code changes, or
|
13
|
+
* parser type (including modules parser is extended by) changes, or
|
14
|
+
* lexer type changes
|
15
|
+
* many improvements to #unparse (which turns Node trees back into ruby code):
|
16
|
+
* slightly tweaked the unparsing of defined? so it'll reparse correctly
|
17
|
+
* in AssignNode#unparse, always put spaces around assign op
|
18
|
+
* only elide spaces right before [ on unparse of BracketsGetNode
|
19
|
+
* unparse of for node should not start with newlines...
|
20
|
+
* speed up unparse_nl when no token provided
|
21
|
+
* 3rd param to unparse_nl (alt) should never be nil
|
22
|
+
* parens and the like and when to omit them from unparse output:
|
23
|
+
* need to call to_s on body of ParenedNode#unparse
|
24
|
+
* emit extra () on unparse in hash and array lits and method call params.
|
25
|
+
* better detection of when to leave out curly braces in hash literal
|
26
|
+
* unparse of MethodNode and CallSiteNode now omits parens if original did
|
27
|
+
* hardcode unparse of NopNode to "()" (was empty string)
|
28
|
+
* NopNode in str inclusion unparses to #{} (was: #{()})
|
29
|
+
* string literal unparsing:
|
30
|
+
* fix unparse of string when delimiter is \r
|
31
|
+
* better detection of backslash (and friends) in strings
|
32
|
+
* for splitting word array or requoting here docs
|
33
|
+
* exception machinery and unparsing:
|
34
|
+
* tolerate missing #rescues in nodes which allow rescue
|
35
|
+
* oops, #unparse was inserting extraneous empty elses
|
36
|
+
* unparsing float literals:
|
37
|
+
* unparse to original string rep of a float literal if available
|
38
|
+
* (to reduce rounding errors)
|
39
|
+
* unparse of -0.0 preserves its sign
|
40
|
+
* more forgiving about parsing floats
|
41
|
+
* better error message on float coercion problem
|
42
|
+
* ruby 1.9:
|
43
|
+
* generally improving ability to run under mri 1.9 (fixing warnings, mostly)
|
44
|
+
all the new 1.9 syntax should now work
|
45
|
+
* notably improvements to -> and .()
|
46
|
+
avoiding warnings about undef'd ivars
|
47
|
+
avoiding duplication in char classes
|
48
|
+
* encoding:
|
49
|
+
* pass encoding on to the lexer
|
50
|
+
* encoding names should always be represented as symbols
|
51
|
+
* 1.9+extensibility:
|
52
|
+
* made @funclikes and @varlikes: ivars instead of constants
|
53
|
+
* @funclikes and @varlikes vary depending on whether in 1.9 mode
|
54
|
+
* -> is now considered a funclike keyword
|
55
|
+
* VALUELIKE_LA converted from a constant to a method
|
56
|
+
* FUNCLIKE_KEYWORD is now a method rather than a constant
|
57
|
+
* 1.9 syntax:
|
58
|
+
* look for block locals in method params only in -> args
|
59
|
+
* -> now has no method params but does have block params (and block locals)
|
60
|
+
* add locals slot for BlockNode
|
61
|
+
* and fill from vars after ; in block params
|
62
|
+
* special rule for -> in 1.9 not needed now (treated like KWCallNode)
|
63
|
+
* new exception in 1.9 for !@ method
|
64
|
+
* improved support for .() type calls in 1.9 mode
|
65
|
+
* programmer's interface:
|
66
|
+
* ast creation:
|
67
|
+
* Node.new should optionally allow ivars to be set using named params
|
68
|
+
* on StringNode.[], don't overwrite @modifiers if one is provided
|
69
|
+
* now parser uses LiteralNode.create to create literals
|
70
|
+
* (so client code can use LiteralNode.new)
|
71
|
+
* sugary constructor for ListInNode
|
72
|
+
* make RawOpNode#new arg parsing more robust
|
73
|
+
* ast reading/searching/reflecting:
|
74
|
+
* some utilities for finding nodes in node trees
|
75
|
+
* to_ruby is an alias for unparse in RedParse::Node and descendants
|
76
|
+
* ListInNode should compare == only if both sides are ListInNode
|
77
|
+
* explicit, public accessors for things that were private:
|
78
|
+
* added #body #reversed and #test_first to while and until nodes
|
79
|
+
* make the #string representation of literal nodes publicly accessible
|
80
|
+
* MethodNode#has_parens? now returns whether formal args had parens
|
81
|
+
* new names for things that were already available:
|
82
|
+
* give RescueOpNode an #op method (always returns "rescue")
|
83
|
+
* alias RescueOpNode#right to #rescue_with (for the fallback expression)
|
84
|
+
* alias CallSiteNode#has_parens? to #real_parens
|
85
|
+
* alias RescueNode#name to its varname
|
86
|
+
* alias UnOpNode#arg to #val
|
87
|
+
* alias UnaryOpNode to UnOpNode
|
88
|
+
* changes in structure, shouldn't impact users:
|
89
|
+
* empty parens should make a ParenedNode, not a SequenceNode
|
90
|
+
* NopNode is now a descendant of SequenceNode
|
91
|
+
* created UnAmpNode for unary ampersand
|
92
|
+
* location:
|
93
|
+
* print error location when parse or lex error occurs
|
94
|
+
* line numbers in Node trees are present/correct in more cases
|
95
|
+
* inspect:
|
96
|
+
* Node#inspect now omits some unimportant or duplicate subfields
|
97
|
+
* remove some duplication in pp output
|
98
|
+
* put + in front of ListInNode on inspect... so it looks like other nodes
|
99
|
+
* head of a constant is now inspected correctly if it happened to be a node
|
100
|
+
* make #inspect more robust in the presence of weird labels
|
101
|
+
* use shorter inspect in error string for misparsed nodes
|
102
|
+
* allow Node subclasses to exclude certain ivars from #inspect
|
103
|
+
* leave HashNode's @no_braces undef'd if not set... makes inspect view prettier
|
104
|
+
* misc api:
|
105
|
+
* added RedParse.parse, for parsing without a RedParse object
|
106
|
+
* all RedParse.new options after the 1st can now be omitted
|
107
|
+
* allow cache mode to be given in a env var, if not otherwise specified
|
108
|
+
* always put error messages on stderr!
|
109
|
+
* added stack attr to RedParse to allow access to the stack from outside
|
110
|
+
* created a #pretty_stack method for printing the stack out
|
111
|
+
* use pretty_stack to print stack
|
112
|
+
* display more fields of @input on inspect
|
113
|
+
* RedParse#inspect cleaner, while preserving old functionality via #dump
|
114
|
+
* have RedParse#to_s emit a more accurate image of the actual type
|
115
|
+
* improve RedParse#to_s slightly for faux @lexers
|
116
|
+
* made #rules an alias for #expanded_RULES
|
117
|
+
* rubyversion attr is no longer writable
|
118
|
+
* misc
|
119
|
+
* ripper emulation, still pretty raw/incomplete
|
120
|
+
* beginnings of drop-in replacements for 3 other parsers
|
121
|
+
* find stuff in gemspec relative to __FILE__ instead of .
|
122
|
+
* don't need this trashy rakefile anymore
|
123
|
+
* re-enabled the compiler
|
124
|
+
* support octal literals starting with +
|
125
|
+
* make sure there's a #blame in nodes that have #error? [api,AST]
|
126
|
+
* EXCLUDED_IVARS no longer needs to keep symbol and string forms of each elem
|
127
|
+
* command line:
|
128
|
+
* added --coalesce cmdline option to use new 'coalescer' parser compiler
|
129
|
+
* made an option for omitting the output of the parse tree
|
130
|
+
* added --cache cmdline option to control caching
|
131
|
+
* avoid printing initial garbage tokens when --print-tokens enabled
|
132
|
+
* added long names for -7 and -9 flags (version selection)
|
133
|
+
* don't require the compiler unless --compile given on cmdline
|
134
|
+
* don't print ugly minor node details unless -v set
|
135
|
+
* parsetree:
|
136
|
+
* improved compatibility w/ parsetree generally
|
137
|
+
* in HasRescue#parsetree should not modify reciever now
|
138
|
+
* in quirks mode:
|
139
|
+
* always force a :block to wrap #parsetree of begin..end nodes
|
140
|
+
* if the sole expression in the body is an undef
|
141
|
+
* (sigh...improves compatibility with cranky ref impl)
|
142
|
+
* only search for literals to nopify at the start
|
143
|
+
* of a sequence, instead of all but the last element
|
144
|
+
* paren node with no body looks like nil to #parsetree
|
145
|
+
* added several to known parsetree/mri boo-boos
|
146
|
+
* tests for ruby 1.9 variant:
|
147
|
+
* disable caching in 1.9 tests
|
148
|
+
* put each 1.9 test case in its own test method
|
149
|
+
* let's do all 1.9 testing in utf-8
|
150
|
+
* expect no sequences except in block bodies
|
151
|
+
* expect no multiassigns except if top-level is an assignment
|
152
|
+
* print code that failed if exception in 1.9 tests
|
153
|
+
* made test for invalid 1.9 expressions
|
154
|
+
* adding test for 1.9 constructs equivalent to other 1.9 constructs
|
155
|
+
* add test case for block locals in .() call
|
156
|
+
* tried to improve parse tree 'blurring' code
|
157
|
+
* (which ignores bs_handler in strings)
|
158
|
+
* include 1_9 in names of 1.9 tests
|
159
|
+
* tests:
|
160
|
+
* added extra_summary utility method to Test::Unit,
|
161
|
+
* which can print additional messages when the test is over
|
162
|
+
* used with known ruby bugs, keep a them separate
|
163
|
+
* since those aren't bugs in redparse
|
164
|
+
* cut down on the rate of the differed-by-begin warning
|
165
|
+
* a system of 'interactive' testing
|
166
|
+
* to allow user to confirm whether unparse is valid
|
167
|
+
* when not exactly equal to original
|
168
|
+
* only wank about parse errors if an env var is set
|
169
|
+
* an option to divide tests into multiple processes
|
170
|
+
* oops, error in mangle mode's detection of globals
|
171
|
+
* fixed a couple test cases that needed to have newlines
|
172
|
+
* a buncha new good (=exhibits errors) test data
|
173
|
+
* new wrappers and injectables expand the reach of mangle mode tests
|
174
|
+
* correcting or moving bad test cases
|
175
|
+
* eliminate duplicate tests
|
176
|
+
* rules:
|
177
|
+
* simplify operator rule by removing KeywordToken match
|
178
|
+
* but now => has to be parsed separately
|
179
|
+
* herebodytoken delete rule eliminated... now a lexer hack
|
180
|
+
* moved rules for parsing parenthesized expressions down to be lower prec than KWCallNode [rules]
|
181
|
+
* expand duties of expanded_RULES to wrapping string/regexp matchers in KW() [rules]
|
182
|
+
* rewrote #lower_op to return a proc [rules]
|
183
|
+
* cacheing of parse table intermediate structures
|
184
|
+
* @compiled_rules cache not needed anymore
|
185
|
+
* (slightly) improve error handling when no block to item_that
|
186
|
+
* initial stab at DanglingStarNode.create [rules]
|
187
|
+
* mini version of ruleset, just a few rules, for debugging [rules]
|
188
|
+
* improving readability in a char class
|
189
|
+
* moved lexer-specific code into the lexer
|
190
|
+
* __FILE__/__LINE__ value setting
|
191
|
+
* token linenums determination
|
192
|
+
|
1
193
|
=== 0.8.4 / 21dec2009
|
2
194
|
* 5 Major Enhancements:
|
3
195
|
* OpNode and related modules are now classes
|
data/Makefile
CHANGED
@@ -2,6 +2,15 @@ name=RedParse
|
|
2
2
|
lname=redparse
|
3
3
|
gemname=redparse
|
4
4
|
|
5
|
+
lib/redparse/ReduceWithsFor_RedParse_1_8.rb: lib/redparse.rb
|
6
|
+
redparse --cache=no -c
|
7
|
+
|
8
|
+
lib/redparse/ReduceWithsFor_RedParse_1_9.rb: lib/redparse.rb
|
9
|
+
redparse --cache=no -c '1.9'
|
10
|
+
|
11
|
+
parser: lib/redparse/ReduceWithsFor_RedParse_1_8.rb lib/redparse/ReduceWithsFor_RedParse_1_9.rb
|
12
|
+
.PHONY: parser
|
13
|
+
|
5
14
|
#everything after this line is generic
|
6
15
|
|
7
16
|
version=$(shell ruby -r ./lib/$(lname)/version.rb -e "puts $(name)::VERSION")
|
data/README.txt
CHANGED
@@ -4,31 +4,20 @@
|
|
4
4
|
|
5
5
|
== DESCRIPTION:
|
6
6
|
|
7
|
-
RedParse is a ruby parser written in pure ruby.
|
8
|
-
ANTLR, it's parse tool is a home-brewed
|
9
|
-
tool is LALR(1)-equivalent and the 'parse language' is
|
10
|
-
even in it's current
|
7
|
+
RedParse is a ruby parser (and parser-compiler) written in pure ruby.
|
8
|
+
Instead of YACC or ANTLR, it's parse tool is a home-brewed language. (The
|
9
|
+
tool is (at least) LALR(1)-equivalent and the 'parse language' is
|
10
|
+
pretty nice, even in it's current form.)
|
11
11
|
|
12
12
|
My intent is to have a completely correct parser for ruby, in 100%
|
13
|
-
ruby.
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
== REQUIREMENTS:
|
20
|
-
* RedParse requires RubyLexer, my hand-coded lexer for ruby. It also
|
21
|
-
uses Reg, (a pattern-matcher). RubyLexer depends on Sequence,
|
22
|
-
(external iterators). Reg depends on Sequence's predecessor, Cursor,
|
23
|
-
altho Cursor isn't used at all in RedParse. The (long-delayed) next
|
24
|
-
version of Reg will use Sequence. To summarize:
|
25
|
-
* RedParse 0.8.2 requires RubyLexer>=0.7.4 and Reg>=0.4.7
|
26
|
-
* RubyLexer 0.7.4 requires Sequence>=0.2.0
|
27
|
-
* Reg 0.4.7 requires Cursor (not really needed here)
|
28
|
-
* All are available as gems. (Or tarballs on rubyforge, if you must.)
|
13
|
+
ruby. And I think I've more or less succeeded. Aside from some fairly
|
14
|
+
minor quibbles (see below), RedParse can parse all known ruby 1.8 and 1.9
|
15
|
+
constructions correctly. Input text may be encoded in ascii, binary,
|
16
|
+
utf-8, iso-8859-1, and the euc-* family of encodings. Sjis is not yet
|
17
|
+
supported.
|
29
18
|
|
30
19
|
== INSTALL:
|
31
|
-
* gem install redparse #(
|
20
|
+
* gem install redparse #(as root if necessary)
|
32
21
|
|
33
22
|
== LICENSE:
|
34
23
|
|
@@ -40,8 +29,7 @@ Please see COPYING.LGPL for details.
|
|
40
29
|
* Pure ruby, through and through. No part is written in C, YACC,
|
41
30
|
ANTLR, lisp, assembly, intercal, befunge or any other language
|
42
31
|
except ruby.
|
43
|
-
* Pretty AST trees (at least, I think so).
|
44
|
-
necessarily to look at.)
|
32
|
+
* Pretty AST trees (at least, I think so).
|
45
33
|
* AST trees closely mirror the actual structure of source code.
|
46
34
|
* unparser is built in
|
47
35
|
* ParseTree format output too, if you want that.
|
@@ -56,20 +44,20 @@ Please see COPYING.LGPL for details.
|
|
56
44
|
actions (which occupy somewhere under 3100 lines in RedParse).
|
57
45
|
Also, what is a rule? I counted most things which required a
|
58
46
|
separate action in MRI's parser, I'm not sure if that's fair.
|
59
|
-
|
47
|
+
On the other hand, RedParse rules require no separate actions
|
48
|
+
anywhere.In the end, I still think RedParse is still much easier to
|
60
49
|
understand than MRI's parse.y.)
|
61
50
|
* "loosey-goosey" parser happily parses many expressions which normal
|
62
51
|
ruby considers errors.
|
63
52
|
|
64
53
|
== Drawbacks:
|
65
54
|
|
66
|
-
*
|
67
|
-
* Error
|
55
|
+
* Slow. Not as bad as it used to be, tho.
|
56
|
+
* Error coverage is sketchy at best
|
68
57
|
* No warnings at all.
|
69
|
-
* Unit test takes a fairly long time.
|
58
|
+
* Unit test takes a fairly long time (much better now, tho! down to 15min).
|
70
59
|
* Lots of warnings printed during unit test.
|
71
60
|
* Debugging parse rules is not straightforward.
|
72
|
-
* Incomplete support for ruby 1.9.
|
73
61
|
* "loosey-goosey" parser happily parses many expressions which normal
|
74
62
|
ruby considers errors.
|
75
63
|
|
@@ -248,22 +236,47 @@ existing format in the future, but no incompatibility-creating changes.
|
|
248
236
|
ErrorNode #mixed in to nodes with a syntax error
|
249
237
|
+-MisparsedNode #mismatched braces or begin..end or the like
|
250
238
|
|
239
|
+
== REQUIREMENTS:
|
240
|
+
* RedParse requires RubyLexer, my hand-coded lexer for ruby. It also
|
241
|
+
uses Reg, (a pattern-matcher). RubyLexer depends on Sequence,
|
242
|
+
(external iterators). Reg depends on Sequence as well. To summarize:
|
243
|
+
* RedParse 1.0.0 requires RubyLexer 0.8.0 and Reg>=0.4.8
|
244
|
+
* RubyLexer 0.8.0 requires Sequence>=0.2.4
|
245
|
+
* Reg 0.4.8 requires Sequence>=0.2.3
|
246
|
+
* All are available as gems. (Or tarballs on rubyforge, if you must.)
|
247
|
+
|
251
248
|
== Known problems with the parser:
|
252
249
|
* Encoding of the input is not stored anywhere in resulting parse tree.
|
253
250
|
* Ascii, binary, utf-8, and euc encodings are supported, but sjis is not.
|
251
|
+
* offsets of some nodes are incorrect
|
252
|
+
* multi-assign nested in multi-assign as receiver of an attribute lhs won't parse right
|
253
|
+
* these expressions won't parse:
|
254
|
+
<<x.
|
255
|
+
1111
|
256
|
+
x
|
257
|
+
a0 rescue b0()
|
258
|
+
|
259
|
+
?\
|
260
|
+
__END__
|
261
|
+
-?
|
262
|
+
|
263
|
+
__END__ #with a comment
|
264
|
+
|
265
|
+
* =f,g rescue b and c
|
254
266
|
|
255
267
|
== Known problems with the unparser:
|
256
|
-
*
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
268
|
+
* Major:
|
269
|
+
* unparsing of trees whose input was in a character set other than ascii may
|
270
|
+
not work.
|
271
|
+
* Minor:
|
272
|
+
* On unparse, here documents are converted into regular strings. For the
|
273
|
+
most part, these are exactly equivalent to the original. However,
|
274
|
+
whatever tokens appeared between the here document header and body will
|
275
|
+
now show up on a different line. If one of those tokens was __LINE__,
|
276
|
+
it will have a different value in the unparsed code than it had
|
277
|
+
originally.
|
278
|
+
|
279
|
+
== Known problems with ParseTree creator
|
267
280
|
* Major:
|
268
281
|
* converting non-ascii encoded parsetrees to ParseTree format doesn't work
|
269
282
|
* Minor:
|
@@ -273,6 +286,9 @@ existing format in the future, but no incompatibility-creating changes.
|
|
273
286
|
but what I emit is equivalent.
|
274
287
|
* %W"is #{"Slim #{2?"W":"S"}"}#{xx}."
|
275
288
|
* silly empty case nodes aren't always optimized to nop like in ParseTree.
|
289
|
+
* begin..end blocks nested in other begin..end blocks aren't combined
|
290
|
+
* literal strings,numbers,symbols,etc as their own stmts should be optimized away
|
291
|
+
* BEGIN expressions not in their own statement may not be treated right
|
276
292
|
|
277
293
|
== Bugs in ruby
|
278
294
|
* These expressions don't parse the same as in MRI because of bug(s) in MRI:
|
@@ -291,3 +307,20 @@ existing format in the future, but no incompatibility-creating changes.
|
|
291
307
|
* def sum(options = {:weights => weights = Hash.new(1)}); opt; end
|
292
308
|
* def foo(a = 1) end; def foo(a=b=c={}) end; def bar(a=b=c=1,d=2) end
|
293
309
|
* yield [a_i, *p]
|
310
|
+
|
311
|
+
== Copyright
|
312
|
+
redparse - a ruby parser written in ruby
|
313
|
+
Copyright (C) 2008,2009, 2012, 2016 Caleb Clausen
|
314
|
+
|
315
|
+
This program is free software: you can redistribute it and/or modify
|
316
|
+
it under the terms of the GNU Lesser General Public License as published by
|
317
|
+
the Free Software Foundation, either version 3 of the License, or
|
318
|
+
(at your option) any later version.
|
319
|
+
|
320
|
+
This program is distributed in the hope that it will be useful,
|
321
|
+
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
322
|
+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
323
|
+
GNU Lesser General Public License for more details.
|
324
|
+
|
325
|
+
You should have received a copy of the GNU Lesser General Public License
|
326
|
+
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
data/bin/redparse
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
=begin
|
3
3
|
redparse - a ruby parser written in ruby
|
4
|
-
Copyright (C) 2008 Caleb Clausen
|
4
|
+
Copyright (C) 2008, 2012, 2016 Caleb Clausen
|
5
5
|
|
6
6
|
This library is free software; you can redistribute it and/or
|
7
7
|
modify it under the terms of the GNU Lesser General Public
|
@@ -96,27 +96,71 @@ while /^-/===ARGV.first
|
|
96
96
|
when "--pp"; output=:pp
|
97
97
|
when "--lisp"; output=:lisp
|
98
98
|
when "--parsetree"; output=:parsetree
|
99
|
+
when "--ripper"; output=:ripper; ruby19=true
|
100
|
+
when "--no-parsetree"; output=:no_parsetree
|
99
101
|
when "--vsparsetree"; output=:vsparsetree
|
100
102
|
when "--vsparsetree2"; output=:vsparsetree2
|
103
|
+
when "--vsripper"; output=:vsripper; ruby19=true
|
104
|
+
when "--vsripper2"; output=:vsripper2; ruby19=true
|
101
105
|
when "--update-problemfiles"; problemfiles=ProblemFiles.new
|
102
106
|
when "--ignore-silly-begins"; ignorebegins=true
|
103
|
-
when "--tokens"; ENV['PRINT_TOKENS']='1'
|
104
|
-
when "--rawtokens"; ENV['RAW_PRINT_TOKENS']='1'
|
105
|
-
when "--stack"; ENV['PRINT_STACK']='1'
|
107
|
+
when "--tokens"; tokens=ENV['PRINT_TOKENS']='1'
|
108
|
+
when "--rawtokens"; rawtokens=ENV['RAW_PRINT_TOKENS']='1'
|
109
|
+
when "--stack"; stack=ENV['PRINT_STACK']='1'
|
106
110
|
when "--unparse"; unparse=true
|
107
|
-
when
|
108
|
-
|
111
|
+
when /^(?:--output|-o)=?(.*)$/
|
112
|
+
outputf=($1.size>0 ? $1 : (/^[^-]/===ARGV.first && ARGV.shift ))
|
113
|
+
when /^(?:--require|-r)=?(.*)$/
|
114
|
+
reqs||=[]
|
115
|
+
reqs<<($1.size>0 ? $1 : (/^[^-]/===ARGV.first && ARGV.shift ))
|
116
|
+
when /^(?:--coalesce|-c)=?(.*)$/;
|
117
|
+
coalesce=true
|
118
|
+
variant=($1.size>0 ? $1 : (/^[^-]/===ARGV.first && ARGV.shift or ''))
|
119
|
+
when "--compile"; require 'redparse/compile'; compile=true
|
120
|
+
when "--macros", "--macro", "-m"
|
109
121
|
require 'rubygems'
|
110
122
|
require 'macro'
|
111
123
|
parserclass=Macro::RedParseWithMacros
|
124
|
+
when /^--cache(?:=(.*))?$/
|
125
|
+
cache_mode=
|
126
|
+
case mode=$1||ARGV.shift
|
127
|
+
when 'r','ro'; :read_only
|
128
|
+
when 'w','wo'; :write_only
|
129
|
+
when 'rw'; :read_write #default
|
130
|
+
when 'n','no','none'; :none
|
131
|
+
else mode.to_sym
|
132
|
+
end
|
112
133
|
when "-q"; quiet=true
|
113
134
|
when "-v"; verbose=true
|
114
135
|
when "-e"; inputs=[ARGV.join(" ")]; names=["-e"]; break
|
115
|
-
when "-7"; ruby187=true
|
116
|
-
when "-9"; ruby19=true
|
136
|
+
when "-7", "--1.8.7"; ruby187=true
|
137
|
+
when "-9", "--1.9"; ruby19=true
|
117
138
|
else fail "unknown option: #{opt}"
|
118
139
|
end
|
119
140
|
end
|
141
|
+
|
142
|
+
if coalesce
|
143
|
+
variant=variant.split(' ')
|
144
|
+
|
145
|
+
if reqs
|
146
|
+
$:.push '.'
|
147
|
+
reqs.each{|req| require req }
|
148
|
+
$:.delete '.'
|
149
|
+
end
|
150
|
+
versions,modules=variant.partition{|v| /^\d/===v }
|
151
|
+
modules.map!{|m| RedParse.const_get m }
|
152
|
+
fail if versions.size>1
|
153
|
+
version=versions.first||(ruby19 ? 1.9 : 1.8)
|
154
|
+
version=version.to_f
|
155
|
+
classes=modules.grep Class
|
156
|
+
modules-=classes
|
157
|
+
assert classes.size<=1
|
158
|
+
k=classes.first||RedParse
|
159
|
+
rp=k.new("",:rubyversion=>version,:cache_mode=>:none)
|
160
|
+
modules.each{|m| rp.extend m }
|
161
|
+
rp.write_reduce_withs outputf
|
162
|
+
exit
|
163
|
+
end
|
120
164
|
|
121
165
|
unless inputs
|
122
166
|
if ARGV.empty?
|
@@ -142,6 +186,8 @@ inputs.each_index{|i|
|
|
142
186
|
input=inputs[i] or next
|
143
187
|
name=names[i]
|
144
188
|
|
189
|
+
orig_input=input
|
190
|
+
|
145
191
|
if /\A=begin\s/===input
|
146
192
|
#combine 1st 2 lines of input
|
147
193
|
if /\A(=begin(.*)\n=end\s(.*))\n/===input
|
@@ -164,11 +210,22 @@ inputs.each_index{|i|
|
|
164
210
|
else
|
165
211
|
options={:rubyversion=>1.8}
|
166
212
|
end
|
213
|
+
options[:cache_mode]=cache_mode if cache_mode
|
214
|
+
parser=(parserclass||RedParse).new(input,name,1,[],options)
|
215
|
+
seen_ends=0
|
216
|
+
parser.print_filter=proc{|t|
|
217
|
+
if RedParse::KeywordToken===t and /^(\}|end)$/===t.ident
|
218
|
+
seen_ends+=1
|
219
|
+
seen_ends>=3
|
220
|
+
else
|
221
|
+
seen_ends>=2
|
222
|
+
end
|
223
|
+
}
|
167
224
|
tree=if compile
|
168
|
-
huh
|
225
|
+
huh parser.compile
|
169
226
|
huh parse
|
170
227
|
else
|
171
|
-
|
228
|
+
parser.parse
|
172
229
|
end
|
173
230
|
nil
|
174
231
|
} #raise NeverExecThis
|
@@ -198,16 +255,53 @@ inputs.each_index{|i|
|
|
198
255
|
require 'pp'
|
199
256
|
pp tree
|
200
257
|
when :p
|
201
|
-
|
258
|
+
puts tree.inspect(nil,0,verbose)
|
202
259
|
when :lisp
|
203
260
|
puts tree.to_lisp
|
204
261
|
when :parsetree
|
205
262
|
tree=tree.to_parsetree
|
206
263
|
hack=tree.dup
|
207
|
-
p hack
|
208
|
-
|
209
|
-
|
264
|
+
#p hack
|
265
|
+
=begin
|
266
|
+
hack.first[1..2]=[] #get rid of BEGIN blocks inserted into beginning of input
|
267
|
+
hack=if hack.first.size<=2; hack.first[1] else hack end
|
268
|
+
rescue Exception
|
269
|
+
=end
|
210
270
|
pp(hack||[])
|
271
|
+
when :vsripper,:vsripper2,:ripper
|
272
|
+
warn "disabling positioning info in ripper emulation output for now"
|
273
|
+
begin
|
274
|
+
require 'ripper'
|
275
|
+
require 'ripper/sexp'
|
276
|
+
class ::Ripper::SexpBuilder
|
277
|
+
def column; 0; end
|
278
|
+
def lineno; 0; end
|
279
|
+
end
|
280
|
+
rescue LoadError
|
281
|
+
end
|
282
|
+
require 'redparse/ripper'
|
283
|
+
require 'redparse/ripper_sexp'
|
284
|
+
RedParse::Ripper.instrumentSexpBuilder(::Ripper::SexpBuilder)if verbose and defined? ::Ripper
|
285
|
+
RedParse::Ripper.instrumentSexpBuilder(::Ripper::SexpBuilderPP)if verbose and defined? ::Ripper
|
286
|
+
RedParse::Ripper.instrumentSexpBuilder(::RedParse::Ripper::SexpBuilder)if verbose
|
287
|
+
RedParse::Ripper.instrumentSexpBuilder(::RedParse::Ripper::SexpBuilderPP)if verbose
|
288
|
+
rip=Ripper.sexp_raw(orig_input,name) if output!=:ripper and defined? ::Ripper
|
289
|
+
rp=RedParse::Ripper::SexpBuilder.new(input,name)
|
290
|
+
rp.parser=tree
|
291
|
+
begin
|
292
|
+
rp= rp.parse :quirks=>true
|
293
|
+
rescue Exception=>e
|
294
|
+
warn e.backtrace.unshift(e.inspect).join("\n ")
|
295
|
+
rp=[:failed]
|
296
|
+
end
|
297
|
+
if rip and rip==rp
|
298
|
+
puts "no differences"
|
299
|
+
else
|
300
|
+
puts "mine:"
|
301
|
+
pp rp
|
302
|
+
puts "minero's:"
|
303
|
+
pp rip
|
304
|
+
end
|
211
305
|
when :vsparsetree,:vsparsetree2
|
212
306
|
begin
|
213
307
|
require 'rubygems'
|