ripper 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,179 @@
1
+ #
2
+ # $Id: lexer.rb 25189 2009-10-02 12:04:37Z akr $
3
+ #
4
+ # Copyright (c) 2004,2005 Minero Aoki
5
+ #
6
+ # This program is free software.
7
+ # You can distribute and/or modify this program under the Ruby License.
8
+ # For details of Ruby License, see ruby/COPYING.
9
+ #
10
+
11
+ require 'ripper/core'
12
+
13
+ class Ripper
14
+
15
+ # Tokenizes Ruby program and returns an Array of String.
16
+ def Ripper.tokenize(src, filename = '-', lineno = 1)
17
+ Lexer.new(src, filename, lineno).tokenize
18
+ end
19
+
20
+ # Tokenizes Ruby program and returns an Array of Array,
21
+ # which is formatted like [[lineno, column], type, token].
22
+ #
23
+ # require 'ripper'
24
+ # require 'pp'
25
+ #
26
+ # p Ripper.lex("def m(a) nil end")
27
+ # #=> [[[1, 0], :on_kw, "def"],
28
+ # [[1, 3], :on_sp, " " ],
29
+ # [[1, 4], :on_ident, "m" ],
30
+ # [[1, 5], :on_lparen, "(" ],
31
+ # [[1, 6], :on_ident, "a" ],
32
+ # [[1, 7], :on_rparen, ")" ],
33
+ # [[1, 8], :on_sp, " " ],
34
+ # [[1, 9], :on_kw, "nil"],
35
+ # [[1, 12], :on_sp, " " ],
36
+ # [[1, 13], :on_kw, "end"]]
37
+ #
38
+ def Ripper.lex(src, filename = '-', lineno = 1)
39
+ Lexer.new(src, filename, lineno).lex
40
+ end
41
+
42
+ class Lexer < ::Ripper #:nodoc: internal use only
43
+ def tokenize
44
+ lex().map {|pos, event, tok| tok }
45
+ end
46
+
47
+ def lex
48
+ parse().sort_by {|pos, event, tok| pos }
49
+ end
50
+
51
+ def parse
52
+ @buf = []
53
+ super
54
+ @buf
55
+ end
56
+
57
+ private
58
+
59
+ SCANNER_EVENTS.each do |event|
60
+ module_eval(<<-End, __FILE__+'/module_eval', __LINE__ + 1)
61
+ def on_#{event}(tok)
62
+ @buf.push [[lineno(), column()], :on_#{event}, tok]
63
+ end
64
+ End
65
+ end
66
+ end
67
+
68
+ # [EXPERIMENTAL]
69
+ # Parses +src+ and return a string which was matched to +pattern+.
70
+ # +pattern+ should be described as Regexp.
71
+ #
72
+ # require 'ripper'
73
+ #
74
+ # p Ripper.slice('def m(a) nil end', 'ident') #=> "m"
75
+ # p Ripper.slice('def m(a) nil end', '[ident lparen rparen]+') #=> "m(a)"
76
+ # p Ripper.slice("<<EOS\nstring\nEOS",
77
+ # 'heredoc_beg nl $(tstring_content*) heredoc_end', 1)
78
+ # #=> "string\n"
79
+ #
80
+ def Ripper.slice(src, pattern, n = 0)
81
+ if m = token_match(src, pattern)
82
+ then m.string(n)
83
+ else nil
84
+ end
85
+ end
86
+
87
+ def Ripper.token_match(src, pattern) #:nodoc:
88
+ TokenPattern.compile(pattern).match(src)
89
+ end
90
+
91
+ class TokenPattern #:nodoc:
92
+
93
+ class Error < ::StandardError; end
94
+ class CompileError < Error; end
95
+ class MatchError < Error; end
96
+
97
+ class << self
98
+ alias compile new
99
+ end
100
+
101
+ def initialize(pattern)
102
+ @source = pattern
103
+ @re = compile(pattern)
104
+ end
105
+
106
+ def match(str)
107
+ match_list(::Ripper.lex(str))
108
+ end
109
+
110
+ def match_list(tokens)
111
+ if m = @re.match(map_tokens(tokens))
112
+ then MatchData.new(tokens, m)
113
+ else nil
114
+ end
115
+ end
116
+
117
+ private
118
+
119
+ def compile(pattern)
120
+ if m = /[^\w\s$()\[\]{}?*+\.]/.match(pattern)
121
+ raise CompileError, "invalid char in pattern: #{m[0].inspect}"
122
+ end
123
+ buf = ''
124
+ pattern.scan(/(?:\w+|\$\(|[()\[\]\{\}?*+\.]+)/) do |tok|
125
+ case tok
126
+ when /\w/
127
+ buf.concat map_token(tok)
128
+ when '$('
129
+ buf.concat '('
130
+ when '('
131
+ buf.concat '(?:'
132
+ when /[?*\[\])\.]/
133
+ buf.concat tok
134
+ else
135
+ raise 'must not happen'
136
+ end
137
+ end
138
+ Regexp.compile(buf)
139
+ rescue RegexpError => err
140
+ raise CompileError, err.message
141
+ end
142
+
143
+ def map_tokens(tokens)
144
+ tokens.map {|pos,type,str| map_token(type.to_s.sub(/\Aon_/,'')) }.join
145
+ end
146
+
147
+ MAP = {}
148
+ seed = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a
149
+ SCANNER_EVENT_TABLE.each do |ev, |
150
+ raise CompileError, "[RIPPER FATAL] too many system token" if seed.empty?
151
+ MAP[ev.to_s.sub(/\Aon_/,'')] = seed.shift
152
+ end
153
+
154
+ def map_token(tok)
155
+ MAP[tok] or raise CompileError, "unknown token: #{tok}"
156
+ end
157
+
158
+ class MatchData
159
+ def initialize(tokens, match)
160
+ @tokens = tokens
161
+ @match = match
162
+ end
163
+
164
+ def string(n = 0)
165
+ return nil unless @match
166
+ match(n).join
167
+ end
168
+
169
+ private
170
+
171
+ def match(n = 0)
172
+ return [] unless @match
173
+ @tokens[@match.begin(n)...@match.end(n)].map {|pos,type,str| str }
174
+ end
175
+ end
176
+
177
+ end
178
+
179
+ end
@@ -0,0 +1,99 @@
1
+ #
2
+ # $Id: sexp.rb 25189 2009-10-02 12:04:37Z akr $
3
+ #
4
+ # Copyright (c) 2004,2005 Minero Aoki
5
+ #
6
+ # This program is free software.
7
+ # You can distribute and/or modify this program under the Ruby License.
8
+ # For details of Ruby License, see ruby/COPYING.
9
+ #
10
+
11
+ require 'ripper/core'
12
+
13
+ class Ripper
14
+
15
+ # [EXPERIMENTAL]
16
+ # Parses +src+ and create S-exp tree.
17
+ # This method is for mainly developper use.
18
+ #
19
+ # require 'ripper'
20
+ # require 'pp
21
+ #
22
+ # pp Ripper.sexp("def m(a) nil end")
23
+ # #=> [:program,
24
+ # [:stmts_add,
25
+ # [:stmts_new],
26
+ # [:def,
27
+ # [:@ident, "m", [1, 4]],
28
+ # [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil]],
29
+ # [:bodystmt,
30
+ # [:stmts_add, [:stmts_new], [:var_ref, [:@kw, "nil", [1, 9]]]],
31
+ # nil,
32
+ # nil,
33
+ # nil]]]]
34
+ #
35
+ def Ripper.sexp(src, filename = '-', lineno = 1)
36
+ SexpBuilderPP.new(src, filename, lineno).parse
37
+ end
38
+
39
+ def Ripper.sexp_raw(src, filename = '-', lineno = 1)
40
+ SexpBuilder.new(src, filename, lineno).parse
41
+ end
42
+
43
+ class SexpBuilderPP < ::Ripper #:nodoc:
44
+ private
45
+
46
+ PARSER_EVENT_TABLE.each do |event, arity|
47
+ if /_new\z/ =~ event.to_s and arity == 0
48
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
49
+ def on_#{event}
50
+ []
51
+ end
52
+ End
53
+ elsif /_add\z/ =~ event.to_s
54
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
55
+ def on_#{event}(list, item)
56
+ list.push item
57
+ list
58
+ end
59
+ End
60
+ else
61
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
62
+ def on_#{event}(*args)
63
+ [:#{event}, *args]
64
+ end
65
+ End
66
+ end
67
+ end
68
+
69
+ SCANNER_EVENTS.each do |event|
70
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
71
+ def on_#{event}(tok)
72
+ [:@#{event}, tok, [lineno(), column()]]
73
+ end
74
+ End
75
+ end
76
+ end
77
+
78
+ class SexpBuilder < ::Ripper #:nodoc:
79
+ private
80
+
81
+ PARSER_EVENTS.each do |event|
82
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
83
+ def on_#{event}(*args)
84
+ args.unshift :#{event}
85
+ args
86
+ end
87
+ End
88
+ end
89
+
90
+ SCANNER_EVENTS.each do |event|
91
+ module_eval(<<-End, __FILE__, __LINE__ + 1)
92
+ def on_#{event}(tok)
93
+ [:@#{event}, tok, [lineno(), column()]]
94
+ end
95
+ End
96
+ end
97
+ end
98
+
99
+ end
metadata ADDED
@@ -0,0 +1,94 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: ripper
3
+ version: !ruby/object:Gem::Version
4
+ hash: 23
5
+ prerelease:
6
+ segments:
7
+ - 1
8
+ - 0
9
+ - 0
10
+ version: 1.0.0
11
+ platform: ruby
12
+ authors:
13
+ - Loren Segal
14
+ - Minero Aoki
15
+ autorequire:
16
+ bindir: bin
17
+ cert_chain: []
18
+
19
+ date: 2011-01-11 00:00:00 -05:00
20
+ default_executable:
21
+ dependencies: []
22
+
23
+ description: " This gem is meant for Ruby 1.8.x builds. Ruby 1.9.x comes with a functioning\n ripper implementation out of the box. This gem is a port of the Ripper 1.9\n for Ruby 1.8.x. Ripper was written by Minero Aoki and ported by Loren Segal.\n"
24
+ email: lsegal@soen.ca
25
+ executables: []
26
+
27
+ extensions:
28
+ - ext/extconf.rb
29
+ extra_rdoc_files: []
30
+
31
+ files:
32
+ - lib/ripper/core.rb
33
+ - lib/ripper/filter.rb
34
+ - lib/ripper/lexer.rb
35
+ - lib/ripper/sexp.rb
36
+ - lib/ripper.rb
37
+ - ext/backports/backports.h
38
+ - ext/backports/regenc.h
39
+ - ext/backports/ruby/encoding.h
40
+ - ext/depend
41
+ - ext/eventids2.c
42
+ - ext/extconf.rb
43
+ - ext/extra/node.h
44
+ - ext/extra/regenc.h
45
+ - ext/id.c
46
+ - ext/id.h
47
+ - ext/lex.c
48
+ - ext/Makefile
49
+ - ext/parse.h
50
+ - ext/parse.y
51
+ - ext/tools/generate-param-macros.rb
52
+ - ext/tools/generate.rb
53
+ - ext/tools/preproc.rb
54
+ - ext/tools/strip.rb
55
+ - ext/tools/ytab.sed
56
+ - Rakefile
57
+ - README
58
+ has_rdoc: true
59
+ homepage: http://github.com/lsegal/ripper
60
+ licenses: []
61
+
62
+ post_install_message:
63
+ rdoc_options: []
64
+
65
+ require_paths:
66
+ - lib
67
+ - ext
68
+ required_ruby_version: !ruby/object:Gem::Requirement
69
+ none: false
70
+ requirements:
71
+ - - ">="
72
+ - !ruby/object:Gem::Version
73
+ hash: 3
74
+ segments:
75
+ - 0
76
+ version: "0"
77
+ required_rubygems_version: !ruby/object:Gem::Requirement
78
+ none: false
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ hash: 3
83
+ segments:
84
+ - 0
85
+ version: "0"
86
+ requirements: []
87
+
88
+ rubyforge_project:
89
+ rubygems_version: 1.4.0
90
+ signing_key:
91
+ specification_version: 3
92
+ summary: Ripper parses Ruby source and tokenizes or builds an AST
93
+ test_files: []
94
+