ripper 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,14 @@
1
+ off = true
2
+ ARGF.each do |line|
3
+ case line
4
+ when /RIPPER_PARAMS_DECL_BEGIN/
5
+ off = false
6
+ when /RIPPER_PARAMS_DECL_END/
7
+ exit
8
+ when /ripper/
9
+ next if off
10
+ var = line.scan(/\w+/).last or next
11
+ base = var.sub(/ripper_/, '')
12
+ puts %"\#define #{base}\t\t(parser->ripper_#{base})"
13
+ end
14
+ end
@@ -0,0 +1,152 @@
1
+ # $Id: generate.rb 25189 2009-10-02 12:04:37Z akr $
2
+
3
+ require 'optparse'
4
+
5
+ def main
6
+ mode = nil
7
+ ids1src = nil
8
+ ids2src = nil
9
+ template = nil
10
+ output = nil
11
+
12
+ parser = @parser = OptionParser.new
13
+ parser.banner = "Usage: #{File.basename($0)} --mode=MODE [--ids1src=PATH] [--ids2src=PATH] [--output=PATH]"
14
+ parser.on('--mode=MODE', 'check, eventids1, or eventids2table.') {|m|
15
+ mode = m
16
+ }
17
+ parser.on('--ids1src=PATH', 'A source file of event-IDs 1 (parse.y).') {|path|
18
+ ids1src = path
19
+ }
20
+ parser.on('--ids2src=PATH', 'A source file of event-IDs 2 (eventids2.c).') {|path|
21
+ ids2src = path
22
+ }
23
+ parser.on('--output=PATH', 'An output file.') {|path|
24
+ output = path
25
+ }
26
+ parser.on('--help', 'Prints this message and quit.') {
27
+ puts parser.help
28
+ exit true
29
+ }
30
+ begin
31
+ parser.parse!
32
+ rescue OptionParser::ParseError => err
33
+ usage err.message
34
+ end
35
+ usage 'no mode given' unless mode
36
+ case mode
37
+ when 'check'
38
+ usage 'no --ids1src' unless ids1src
39
+ usage 'no --ids2src' unless ids2src
40
+ h = read_ids1_with_locations(ids1src)
41
+ check_arity h
42
+ ids2 = read_ids2(ids2src)
43
+ common = h.keys & ids2
44
+ unless common.empty?
45
+ abort "event crash: #{common.join(' ')}"
46
+ end
47
+ exit 0
48
+ when 'eventids1'
49
+ usage 'no --ids1src' unless ids1src
50
+ result = generate_eventids1(read_ids1(ids1src))
51
+ when 'eventids2table'
52
+ usage 'no --ids2src' unless ids2src
53
+ result = generate_eventids2_table(read_ids2(ids2src))
54
+ end
55
+ if output
56
+ File.open(output, 'w') {|f|
57
+ f.write result
58
+ }
59
+ else
60
+ puts result
61
+ end
62
+ end
63
+
64
+ def usage(msg)
65
+ $stderr.puts msg
66
+ $stderr.puts @parser.help
67
+ exit false
68
+ end
69
+
70
+ def generate_eventids1(ids)
71
+ buf = ""
72
+ ids.each do |id, arity|
73
+ buf << %Q[static ID ripper_id_#{id};\n]
74
+ end
75
+ buf << %Q[\n]
76
+ buf << %Q[static void\n]
77
+ buf << %Q[ripper_init_eventids1(VALUE self)\n]
78
+ buf << %Q[{\n]
79
+ buf << %Q[ VALUE h;\n]
80
+ buf << %Q[ ID id;\n]
81
+ ids.each do |id, arity|
82
+ buf << %Q[ ripper_id_#{id} = rb_intern_const("on_#{id}");\n]
83
+ end
84
+ buf << %Q[\n]
85
+ buf << %Q[ h = rb_hash_new();\n]
86
+ buf << %Q[ rb_define_const(self, "PARSER_EVENT_TABLE", h);\n]
87
+ ids.each do |id, arity|
88
+ buf << %Q[ id = rb_intern_const("#{id}");\n]
89
+ buf << %Q[ rb_hash_aset(h, ID2SYM(id), INT2NUM(#{arity}));\n]
90
+ end
91
+ buf << %Q[}\n]
92
+ buf
93
+ end
94
+
95
+ def generate_eventids2_table(ids)
96
+ buf = ""
97
+ buf << %Q[static void\n]
98
+ buf << %Q[ripper_init_eventids2_table(VALUE self)\n]
99
+ buf << %Q[{\n]
100
+ buf << %Q[ VALUE h = rb_hash_new();\n]
101
+ buf << %Q[ ID id;\n]
102
+ buf << %Q[ rb_define_const(self, "SCANNER_EVENT_TABLE", h);\n]
103
+ ids.each do |id|
104
+ buf << %Q[ id = rb_intern_const("#{id}");\n]
105
+ buf << %Q[ rb_hash_aset(h, ID2SYM(id), INT2NUM(1));\n]
106
+ end
107
+ buf << %Q[}\n]
108
+ buf
109
+ end
110
+
111
+ def read_ids1(path)
112
+ strip_locations(read_ids1_with_locations(path))
113
+ end
114
+
115
+ def strip_locations(h)
116
+ h.map {|event, list| [event, list.first[1]] }\
117
+ .sort_by {|event, arity| event.to_s }
118
+ end
119
+
120
+ def check_arity(h)
121
+ invalid = false
122
+ h.each do |event, list|
123
+ unless list.map {|line, arity| arity }.uniq.size == 1
124
+ invalid = true
125
+ locations = list.map {|line, a| "#{line}:#{a}" }.join(', ')
126
+ $stderr.puts "arity crash [event=#{event}]: #{locations}"
127
+ end
128
+ end
129
+ abort if invalid
130
+ end
131
+
132
+ def read_ids1_with_locations(path)
133
+ h = {}
134
+ File.open(path) {|f|
135
+ f.each do |line|
136
+ next if /\A\#\s*define\s+s?dispatch/ =~ line
137
+ next if /ripper_dispatch/ =~ line
138
+ line.scan(/dispatch(\d)\((\w+)/) do |arity, event|
139
+ (h[event] ||= []).push [f.lineno, arity.to_i]
140
+ end
141
+ end
142
+ }
143
+ h
144
+ end
145
+
146
+ def read_ids2(path)
147
+ File.open(path) {|f|
148
+ return f.read.scan(/ripper_id_(\w+)/).flatten.uniq.sort
149
+ }
150
+ end
151
+
152
+ main
@@ -0,0 +1,91 @@
1
+ # $Id: preproc.rb 25189 2009-10-02 12:04:37Z akr $
2
+
3
+ require 'optparse'
4
+
5
+ def main
6
+ output = nil
7
+ parser = OptionParser.new
8
+ parser.banner = "Usage: #{File.basename($0)} [--output=PATH] <parse.y>"
9
+ parser.on('--output=PATH', 'An output file.') {|path|
10
+ output = path
11
+ }
12
+ parser.on('--help', 'Prints this message and quit.') {
13
+ puts parser.help
14
+ exit true
15
+ }
16
+ begin
17
+ parser.parse!
18
+ rescue OptionParser::ParseError => err
19
+ $stderr.puts err.message
20
+ $stderr.puts parser.help
21
+ exit false
22
+ end
23
+ unless ARGV.size == 1
24
+ abort "wrong number of arguments (#{ARGV.size} for 1)"
25
+ end
26
+ out = ""
27
+ File.open(ARGV[0]) {|f|
28
+ prelude f, out
29
+ grammar f, out
30
+ usercode f, out
31
+ }
32
+ if output
33
+ File.open(output, 'w') {|f|
34
+ f.write out
35
+ }
36
+ else
37
+ print out
38
+ end
39
+ end
40
+
41
+ def prelude(f, out)
42
+ while line = f.gets
43
+ case line
44
+ when %r</\*%%%\*/>
45
+ out << '/*' << $/
46
+ when %r</\*%>
47
+ out << '*/' << $/
48
+ when %r<%\*/>
49
+ out << $/
50
+ when /\A%%/
51
+ out << '%%' << $/
52
+ return
53
+ when /\A%token/
54
+ out << line.sub(/<\w+>/, '<val>')
55
+ when /\A%type/
56
+ out << line.sub(/<\w+>/, '<val>')
57
+ else
58
+ out << line
59
+ end
60
+ end
61
+ end
62
+
63
+ def grammar(f, out)
64
+ while line = f.gets
65
+ case line
66
+ when %r</\*%%%\*/>
67
+ out << '#if 0' << $/
68
+ when %r</\*%c%\*/>
69
+ out << '/*' << $/
70
+ when %r</\*%c>
71
+ out << '*/' << $/
72
+ when %r</\*%>
73
+ out << '#endif' << $/
74
+ when %r<%\*/>
75
+ out << $/
76
+ when /\A%%/
77
+ out << '%%' << $/
78
+ return
79
+ else
80
+ out << line
81
+ end
82
+ end
83
+ end
84
+
85
+ def usercode(f, out)
86
+ while line = f.gets
87
+ out << line
88
+ end
89
+ end
90
+
91
+ main
@@ -0,0 +1,12 @@
1
+ last_is_void = false
2
+ ARGF.each do |line|
3
+ if line.strip.empty?
4
+ #puts() unless last_is_void
5
+ last_is_void = true
6
+ elsif /\A\#/ === line
7
+ ;
8
+ else
9
+ print line
10
+ last_is_void = false
11
+ end
12
+ end
@@ -0,0 +1,31 @@
1
+ #!/bin/sed -f
2
+ /^int yydebug;/{
3
+ i\
4
+ #ifndef yydebug
5
+ a\
6
+ #endif
7
+ }
8
+ /^yydestruct.*yymsg/,/#endif/{
9
+ /^yydestruct/{
10
+ /parser/!{
11
+ h
12
+ s/^/ruby_parser_&/
13
+ s/)$/, parser)/
14
+ /\*/s/parser)$/struct parser_params *&/
15
+ }
16
+ }
17
+ /^#endif/{
18
+ x
19
+ /^./{
20
+ i\
21
+ struct parser_params *parser;
22
+ a\
23
+ #define yydestruct(m, t, v) ruby_parser_yydestruct(m, t, v, parser)
24
+ }
25
+ x
26
+ }
27
+ }
28
+ s/^\([ ]*\)\(yyerror[ ]*([ ]*parser,\)/\1parser_\2/
29
+ s!^ *extern char \*getenv();!/* & */!
30
+ s/^\(#.*\)".*\.tab\.c"/\1"ripper.c"/
31
+ /^\(#.*\)".*\.y"/s:\\\\:/:g
@@ -0,0 +1,4 @@
1
+ require 'ripper/core'
2
+ require 'ripper/lexer'
3
+ require 'ripper/filter'
4
+ require 'ripper/sexp'
@@ -0,0 +1,70 @@
1
+ #
2
+ # $Id: core.rb 25189 2009-10-02 12:04:37Z akr $
3
+ #
4
+ # Copyright (c) 2003-2005 Minero Aoki
5
+ #
6
+ # This program is free software.
7
+ # You can distribute and/or modify this program under the Ruby License.
8
+ # For details of Ruby License, see ruby/COPYING.
9
+ #
10
+
11
+ require File.dirname(__FILE__) + '/../../ext/ripper.so'
12
+
13
+ class Ripper
14
+
15
+ # Parses Ruby program read from _src_.
16
+ # _src_ must be a String or a IO or a object which has #gets method.
17
+ def Ripper.parse(src, filename = '(ripper)', lineno = 1)
18
+ new(src, filename, lineno).parse
19
+ end
20
+
21
+ # This array contains name of parser events.
22
+ PARSER_EVENTS = PARSER_EVENT_TABLE.keys
23
+
24
+ # This array contains name of scanner events.
25
+ SCANNER_EVENTS = SCANNER_EVENT_TABLE.keys
26
+
27
+ # This array contains name of all ripper events.
28
+ EVENTS = PARSER_EVENTS + SCANNER_EVENTS
29
+
30
+ private
31
+
32
+ #
33
+ # Parser Events
34
+ #
35
+
36
+ PARSER_EVENT_TABLE.each do |id, arity|
37
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
38
+ def on_#{id}(#{ ('a'..'z').to_a[0, arity].join(', ') })
39
+ #{arity == 0 ? 'nil' : 'a'}
40
+ end
41
+ End
42
+ end
43
+
44
+ # This method is called when weak warning is produced by the parser.
45
+ # _fmt_ and _args_ is printf style.
46
+ def warn(fmt, *args)
47
+ end
48
+
49
+ # This method is called when strong warning is produced by the parser.
50
+ # _fmt_ and _args_ is printf style.
51
+ def warning(fmt, *args)
52
+ end
53
+
54
+ # This method is called when the parser found syntax error.
55
+ def compile_error(msg)
56
+ end
57
+
58
+ #
59
+ # Scanner Events
60
+ #
61
+
62
+ SCANNER_EVENTS.each do |id|
63
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
64
+ def on_#{id}(token)
65
+ token
66
+ end
67
+ End
68
+ end
69
+
70
+ end
@@ -0,0 +1,70 @@
1
+ #
2
+ # $Id: filter.rb 25189 2009-10-02 12:04:37Z akr $
3
+ #
4
+ # Copyright (c) 2004,2005 Minero Aoki
5
+ #
6
+ # This program is free software.
7
+ # You can distribute and/or modify this program under the Ruby License.
8
+ # For details of Ruby License, see ruby/COPYING.
9
+ #
10
+
11
+ require 'ripper/lexer'
12
+
13
+ class Ripper
14
+
15
+ # This class handles only scanner events,
16
+ # and they are dispatched in the `right' order (same with input).
17
+ class Filter
18
+
19
+ def initialize(src, filename = '-', lineno = 1)
20
+ @__lexer = Lexer.new(src, filename, lineno)
21
+ @__line = nil
22
+ @__col = nil
23
+ end
24
+
25
+ # The file name of the input.
26
+ def filename
27
+ @__lexer.filename
28
+ end
29
+
30
+ # The line number of the current token.
31
+ # This value starts from 1.
32
+ # This method is valid only in event handlers.
33
+ def lineno
34
+ @__line
35
+ end
36
+
37
+ # The column number of the current token.
38
+ # This value starts from 0.
39
+ # This method is valid only in event handlers.
40
+ def column
41
+ @__col
42
+ end
43
+
44
+ # Starts parsing. _init_ is a data accumulator.
45
+ # It is passed to the next event handler (as of Enumerable#inject).
46
+ def parse(init = nil)
47
+ data = init
48
+ @__lexer.lex.each do |pos, event, tok|
49
+ @__line, @__col = *pos
50
+ data = if respond_to?(event, true)
51
+ then __send__(event, tok, data)
52
+ else on_default(event, tok, data)
53
+ end
54
+ end
55
+ data
56
+ end
57
+
58
+ private
59
+
60
+ # This method is called when some event handler have not defined.
61
+ # _event_ is :on_XXX, _token_ is scanned token, _data_ is a data
62
+ # accumulator. The return value of this method is passed to the
63
+ # next event handler (as of Enumerable#inject).
64
+ def on_default(event, token, data)
65
+ data
66
+ end
67
+
68
+ end
69
+
70
+ end