ripper 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +31 -0
- data/Rakefile +22 -0
- data/ext/Makefile +195 -0
- data/ext/backports/backports.h +107 -0
- data/ext/backports/regenc.h +1 -0
- data/ext/backports/ruby/encoding.h +68 -0
- data/ext/depend +36 -0
- data/ext/eventids2.c +279 -0
- data/ext/extconf.rb +27 -0
- data/ext/extra/node.h +483 -0
- data/ext/extra/regenc.h +2 -0
- data/ext/id.c +50 -0
- data/ext/id.h +170 -0
- data/ext/lex.c +219 -0
- data/ext/parse.h +187 -0
- data/ext/parse.y +10637 -0
- data/ext/tools/generate-param-macros.rb +14 -0
- data/ext/tools/generate.rb +152 -0
- data/ext/tools/preproc.rb +91 -0
- data/ext/tools/strip.rb +12 -0
- data/ext/tools/ytab.sed +31 -0
- data/lib/ripper.rb +4 -0
- data/lib/ripper/core.rb +70 -0
- data/lib/ripper/filter.rb +70 -0
- data/lib/ripper/lexer.rb +179 -0
- data/lib/ripper/sexp.rb +99 -0
- metadata +94 -0
@@ -0,0 +1,14 @@
|
|
1
|
+
off = true
|
2
|
+
ARGF.each do |line|
|
3
|
+
case line
|
4
|
+
when /RIPPER_PARAMS_DECL_BEGIN/
|
5
|
+
off = false
|
6
|
+
when /RIPPER_PARAMS_DECL_END/
|
7
|
+
exit
|
8
|
+
when /ripper/
|
9
|
+
next if off
|
10
|
+
var = line.scan(/\w+/).last or next
|
11
|
+
base = var.sub(/ripper_/, '')
|
12
|
+
puts %"\#define #{base}\t\t(parser->ripper_#{base})"
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,152 @@
|
|
1
|
+
# $Id: generate.rb 25189 2009-10-02 12:04:37Z akr $
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
|
5
|
+
def main
|
6
|
+
mode = nil
|
7
|
+
ids1src = nil
|
8
|
+
ids2src = nil
|
9
|
+
template = nil
|
10
|
+
output = nil
|
11
|
+
|
12
|
+
parser = @parser = OptionParser.new
|
13
|
+
parser.banner = "Usage: #{File.basename($0)} --mode=MODE [--ids1src=PATH] [--ids2src=PATH] [--output=PATH]"
|
14
|
+
parser.on('--mode=MODE', 'check, eventids1, or eventids2table.') {|m|
|
15
|
+
mode = m
|
16
|
+
}
|
17
|
+
parser.on('--ids1src=PATH', 'A source file of event-IDs 1 (parse.y).') {|path|
|
18
|
+
ids1src = path
|
19
|
+
}
|
20
|
+
parser.on('--ids2src=PATH', 'A source file of event-IDs 2 (eventids2.c).') {|path|
|
21
|
+
ids2src = path
|
22
|
+
}
|
23
|
+
parser.on('--output=PATH', 'An output file.') {|path|
|
24
|
+
output = path
|
25
|
+
}
|
26
|
+
parser.on('--help', 'Prints this message and quit.') {
|
27
|
+
puts parser.help
|
28
|
+
exit true
|
29
|
+
}
|
30
|
+
begin
|
31
|
+
parser.parse!
|
32
|
+
rescue OptionParser::ParseError => err
|
33
|
+
usage err.message
|
34
|
+
end
|
35
|
+
usage 'no mode given' unless mode
|
36
|
+
case mode
|
37
|
+
when 'check'
|
38
|
+
usage 'no --ids1src' unless ids1src
|
39
|
+
usage 'no --ids2src' unless ids2src
|
40
|
+
h = read_ids1_with_locations(ids1src)
|
41
|
+
check_arity h
|
42
|
+
ids2 = read_ids2(ids2src)
|
43
|
+
common = h.keys & ids2
|
44
|
+
unless common.empty?
|
45
|
+
abort "event crash: #{common.join(' ')}"
|
46
|
+
end
|
47
|
+
exit 0
|
48
|
+
when 'eventids1'
|
49
|
+
usage 'no --ids1src' unless ids1src
|
50
|
+
result = generate_eventids1(read_ids1(ids1src))
|
51
|
+
when 'eventids2table'
|
52
|
+
usage 'no --ids2src' unless ids2src
|
53
|
+
result = generate_eventids2_table(read_ids2(ids2src))
|
54
|
+
end
|
55
|
+
if output
|
56
|
+
File.open(output, 'w') {|f|
|
57
|
+
f.write result
|
58
|
+
}
|
59
|
+
else
|
60
|
+
puts result
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def usage(msg)
|
65
|
+
$stderr.puts msg
|
66
|
+
$stderr.puts @parser.help
|
67
|
+
exit false
|
68
|
+
end
|
69
|
+
|
70
|
+
def generate_eventids1(ids)
|
71
|
+
buf = ""
|
72
|
+
ids.each do |id, arity|
|
73
|
+
buf << %Q[static ID ripper_id_#{id};\n]
|
74
|
+
end
|
75
|
+
buf << %Q[\n]
|
76
|
+
buf << %Q[static void\n]
|
77
|
+
buf << %Q[ripper_init_eventids1(VALUE self)\n]
|
78
|
+
buf << %Q[{\n]
|
79
|
+
buf << %Q[ VALUE h;\n]
|
80
|
+
buf << %Q[ ID id;\n]
|
81
|
+
ids.each do |id, arity|
|
82
|
+
buf << %Q[ ripper_id_#{id} = rb_intern_const("on_#{id}");\n]
|
83
|
+
end
|
84
|
+
buf << %Q[\n]
|
85
|
+
buf << %Q[ h = rb_hash_new();\n]
|
86
|
+
buf << %Q[ rb_define_const(self, "PARSER_EVENT_TABLE", h);\n]
|
87
|
+
ids.each do |id, arity|
|
88
|
+
buf << %Q[ id = rb_intern_const("#{id}");\n]
|
89
|
+
buf << %Q[ rb_hash_aset(h, ID2SYM(id), INT2NUM(#{arity}));\n]
|
90
|
+
end
|
91
|
+
buf << %Q[}\n]
|
92
|
+
buf
|
93
|
+
end
|
94
|
+
|
95
|
+
def generate_eventids2_table(ids)
|
96
|
+
buf = ""
|
97
|
+
buf << %Q[static void\n]
|
98
|
+
buf << %Q[ripper_init_eventids2_table(VALUE self)\n]
|
99
|
+
buf << %Q[{\n]
|
100
|
+
buf << %Q[ VALUE h = rb_hash_new();\n]
|
101
|
+
buf << %Q[ ID id;\n]
|
102
|
+
buf << %Q[ rb_define_const(self, "SCANNER_EVENT_TABLE", h);\n]
|
103
|
+
ids.each do |id|
|
104
|
+
buf << %Q[ id = rb_intern_const("#{id}");\n]
|
105
|
+
buf << %Q[ rb_hash_aset(h, ID2SYM(id), INT2NUM(1));\n]
|
106
|
+
end
|
107
|
+
buf << %Q[}\n]
|
108
|
+
buf
|
109
|
+
end
|
110
|
+
|
111
|
+
def read_ids1(path)
|
112
|
+
strip_locations(read_ids1_with_locations(path))
|
113
|
+
end
|
114
|
+
|
115
|
+
def strip_locations(h)
|
116
|
+
h.map {|event, list| [event, list.first[1]] }\
|
117
|
+
.sort_by {|event, arity| event.to_s }
|
118
|
+
end
|
119
|
+
|
120
|
+
def check_arity(h)
|
121
|
+
invalid = false
|
122
|
+
h.each do |event, list|
|
123
|
+
unless list.map {|line, arity| arity }.uniq.size == 1
|
124
|
+
invalid = true
|
125
|
+
locations = list.map {|line, a| "#{line}:#{a}" }.join(', ')
|
126
|
+
$stderr.puts "arity crash [event=#{event}]: #{locations}"
|
127
|
+
end
|
128
|
+
end
|
129
|
+
abort if invalid
|
130
|
+
end
|
131
|
+
|
132
|
+
def read_ids1_with_locations(path)
|
133
|
+
h = {}
|
134
|
+
File.open(path) {|f|
|
135
|
+
f.each do |line|
|
136
|
+
next if /\A\#\s*define\s+s?dispatch/ =~ line
|
137
|
+
next if /ripper_dispatch/ =~ line
|
138
|
+
line.scan(/dispatch(\d)\((\w+)/) do |arity, event|
|
139
|
+
(h[event] ||= []).push [f.lineno, arity.to_i]
|
140
|
+
end
|
141
|
+
end
|
142
|
+
}
|
143
|
+
h
|
144
|
+
end
|
145
|
+
|
146
|
+
def read_ids2(path)
|
147
|
+
File.open(path) {|f|
|
148
|
+
return f.read.scan(/ripper_id_(\w+)/).flatten.uniq.sort
|
149
|
+
}
|
150
|
+
end
|
151
|
+
|
152
|
+
main
|
@@ -0,0 +1,91 @@
|
|
1
|
+
# $Id: preproc.rb 25189 2009-10-02 12:04:37Z akr $
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
|
5
|
+
def main
|
6
|
+
output = nil
|
7
|
+
parser = OptionParser.new
|
8
|
+
parser.banner = "Usage: #{File.basename($0)} [--output=PATH] <parse.y>"
|
9
|
+
parser.on('--output=PATH', 'An output file.') {|path|
|
10
|
+
output = path
|
11
|
+
}
|
12
|
+
parser.on('--help', 'Prints this message and quit.') {
|
13
|
+
puts parser.help
|
14
|
+
exit true
|
15
|
+
}
|
16
|
+
begin
|
17
|
+
parser.parse!
|
18
|
+
rescue OptionParser::ParseError => err
|
19
|
+
$stderr.puts err.message
|
20
|
+
$stderr.puts parser.help
|
21
|
+
exit false
|
22
|
+
end
|
23
|
+
unless ARGV.size == 1
|
24
|
+
abort "wrong number of arguments (#{ARGV.size} for 1)"
|
25
|
+
end
|
26
|
+
out = ""
|
27
|
+
File.open(ARGV[0]) {|f|
|
28
|
+
prelude f, out
|
29
|
+
grammar f, out
|
30
|
+
usercode f, out
|
31
|
+
}
|
32
|
+
if output
|
33
|
+
File.open(output, 'w') {|f|
|
34
|
+
f.write out
|
35
|
+
}
|
36
|
+
else
|
37
|
+
print out
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def prelude(f, out)
|
42
|
+
while line = f.gets
|
43
|
+
case line
|
44
|
+
when %r</\*%%%\*/>
|
45
|
+
out << '/*' << $/
|
46
|
+
when %r</\*%>
|
47
|
+
out << '*/' << $/
|
48
|
+
when %r<%\*/>
|
49
|
+
out << $/
|
50
|
+
when /\A%%/
|
51
|
+
out << '%%' << $/
|
52
|
+
return
|
53
|
+
when /\A%token/
|
54
|
+
out << line.sub(/<\w+>/, '<val>')
|
55
|
+
when /\A%type/
|
56
|
+
out << line.sub(/<\w+>/, '<val>')
|
57
|
+
else
|
58
|
+
out << line
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def grammar(f, out)
|
64
|
+
while line = f.gets
|
65
|
+
case line
|
66
|
+
when %r</\*%%%\*/>
|
67
|
+
out << '#if 0' << $/
|
68
|
+
when %r</\*%c%\*/>
|
69
|
+
out << '/*' << $/
|
70
|
+
when %r</\*%c>
|
71
|
+
out << '*/' << $/
|
72
|
+
when %r</\*%>
|
73
|
+
out << '#endif' << $/
|
74
|
+
when %r<%\*/>
|
75
|
+
out << $/
|
76
|
+
when /\A%%/
|
77
|
+
out << '%%' << $/
|
78
|
+
return
|
79
|
+
else
|
80
|
+
out << line
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def usercode(f, out)
|
86
|
+
while line = f.gets
|
87
|
+
out << line
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
main
|
data/ext/tools/strip.rb
ADDED
data/ext/tools/ytab.sed
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
#!/bin/sed -f
|
2
|
+
/^int yydebug;/{
|
3
|
+
i\
|
4
|
+
#ifndef yydebug
|
5
|
+
a\
|
6
|
+
#endif
|
7
|
+
}
|
8
|
+
/^yydestruct.*yymsg/,/#endif/{
|
9
|
+
/^yydestruct/{
|
10
|
+
/parser/!{
|
11
|
+
h
|
12
|
+
s/^/ruby_parser_&/
|
13
|
+
s/)$/, parser)/
|
14
|
+
/\*/s/parser)$/struct parser_params *&/
|
15
|
+
}
|
16
|
+
}
|
17
|
+
/^#endif/{
|
18
|
+
x
|
19
|
+
/^./{
|
20
|
+
i\
|
21
|
+
struct parser_params *parser;
|
22
|
+
a\
|
23
|
+
#define yydestruct(m, t, v) ruby_parser_yydestruct(m, t, v, parser)
|
24
|
+
}
|
25
|
+
x
|
26
|
+
}
|
27
|
+
}
|
28
|
+
s/^\([ ]*\)\(yyerror[ ]*([ ]*parser,\)/\1parser_\2/
|
29
|
+
s!^ *extern char \*getenv();!/* & */!
|
30
|
+
s/^\(#.*\)".*\.tab\.c"/\1"ripper.c"/
|
31
|
+
/^\(#.*\)".*\.y"/s:\\\\:/:g
|
data/lib/ripper.rb
ADDED
data/lib/ripper/core.rb
ADDED
@@ -0,0 +1,70 @@
|
|
1
|
+
#
|
2
|
+
# $Id: core.rb 25189 2009-10-02 12:04:37Z akr $
|
3
|
+
#
|
4
|
+
# Copyright (c) 2003-2005 Minero Aoki
|
5
|
+
#
|
6
|
+
# This program is free software.
|
7
|
+
# You can distribute and/or modify this program under the Ruby License.
|
8
|
+
# For details of Ruby License, see ruby/COPYING.
|
9
|
+
#
|
10
|
+
|
11
|
+
require File.dirname(__FILE__) + '/../../ext/ripper.so'
|
12
|
+
|
13
|
+
class Ripper
|
14
|
+
|
15
|
+
# Parses Ruby program read from _src_.
|
16
|
+
# _src_ must be a String or a IO or a object which has #gets method.
|
17
|
+
def Ripper.parse(src, filename = '(ripper)', lineno = 1)
|
18
|
+
new(src, filename, lineno).parse
|
19
|
+
end
|
20
|
+
|
21
|
+
# This array contains name of parser events.
|
22
|
+
PARSER_EVENTS = PARSER_EVENT_TABLE.keys
|
23
|
+
|
24
|
+
# This array contains name of scanner events.
|
25
|
+
SCANNER_EVENTS = SCANNER_EVENT_TABLE.keys
|
26
|
+
|
27
|
+
# This array contains name of all ripper events.
|
28
|
+
EVENTS = PARSER_EVENTS + SCANNER_EVENTS
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
#
|
33
|
+
# Parser Events
|
34
|
+
#
|
35
|
+
|
36
|
+
PARSER_EVENT_TABLE.each do |id, arity|
|
37
|
+
module_eval(<<-End, __FILE__, __LINE__ + 1)
|
38
|
+
def on_#{id}(#{ ('a'..'z').to_a[0, arity].join(', ') })
|
39
|
+
#{arity == 0 ? 'nil' : 'a'}
|
40
|
+
end
|
41
|
+
End
|
42
|
+
end
|
43
|
+
|
44
|
+
# This method is called when weak warning is produced by the parser.
|
45
|
+
# _fmt_ and _args_ is printf style.
|
46
|
+
def warn(fmt, *args)
|
47
|
+
end
|
48
|
+
|
49
|
+
# This method is called when strong warning is produced by the parser.
|
50
|
+
# _fmt_ and _args_ is printf style.
|
51
|
+
def warning(fmt, *args)
|
52
|
+
end
|
53
|
+
|
54
|
+
# This method is called when the parser found syntax error.
|
55
|
+
def compile_error(msg)
|
56
|
+
end
|
57
|
+
|
58
|
+
#
|
59
|
+
# Scanner Events
|
60
|
+
#
|
61
|
+
|
62
|
+
SCANNER_EVENTS.each do |id|
|
63
|
+
module_eval(<<-End, __FILE__, __LINE__ + 1)
|
64
|
+
def on_#{id}(token)
|
65
|
+
token
|
66
|
+
end
|
67
|
+
End
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
#
|
2
|
+
# $Id: filter.rb 25189 2009-10-02 12:04:37Z akr $
|
3
|
+
#
|
4
|
+
# Copyright (c) 2004,2005 Minero Aoki
|
5
|
+
#
|
6
|
+
# This program is free software.
|
7
|
+
# You can distribute and/or modify this program under the Ruby License.
|
8
|
+
# For details of Ruby License, see ruby/COPYING.
|
9
|
+
#
|
10
|
+
|
11
|
+
require 'ripper/lexer'
|
12
|
+
|
13
|
+
class Ripper
|
14
|
+
|
15
|
+
# This class handles only scanner events,
|
16
|
+
# and they are dispatched in the `right' order (same with input).
|
17
|
+
class Filter
|
18
|
+
|
19
|
+
def initialize(src, filename = '-', lineno = 1)
|
20
|
+
@__lexer = Lexer.new(src, filename, lineno)
|
21
|
+
@__line = nil
|
22
|
+
@__col = nil
|
23
|
+
end
|
24
|
+
|
25
|
+
# The file name of the input.
|
26
|
+
def filename
|
27
|
+
@__lexer.filename
|
28
|
+
end
|
29
|
+
|
30
|
+
# The line number of the current token.
|
31
|
+
# This value starts from 1.
|
32
|
+
# This method is valid only in event handlers.
|
33
|
+
def lineno
|
34
|
+
@__line
|
35
|
+
end
|
36
|
+
|
37
|
+
# The column number of the current token.
|
38
|
+
# This value starts from 0.
|
39
|
+
# This method is valid only in event handlers.
|
40
|
+
def column
|
41
|
+
@__col
|
42
|
+
end
|
43
|
+
|
44
|
+
# Starts parsing. _init_ is a data accumulator.
|
45
|
+
# It is passed to the next event handler (as of Enumerable#inject).
|
46
|
+
def parse(init = nil)
|
47
|
+
data = init
|
48
|
+
@__lexer.lex.each do |pos, event, tok|
|
49
|
+
@__line, @__col = *pos
|
50
|
+
data = if respond_to?(event, true)
|
51
|
+
then __send__(event, tok, data)
|
52
|
+
else on_default(event, tok, data)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
data
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
|
60
|
+
# This method is called when some event handler have not defined.
|
61
|
+
# _event_ is :on_XXX, _token_ is scanned token, _data_ is a data
|
62
|
+
# accumulator. The return value of this method is passed to the
|
63
|
+
# next event handler (as of Enumerable#inject).
|
64
|
+
def on_default(event, token, data)
|
65
|
+
data
|
66
|
+
end
|
67
|
+
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|