ripper 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +31 -0
- data/Rakefile +22 -0
- data/ext/Makefile +195 -0
- data/ext/backports/backports.h +107 -0
- data/ext/backports/regenc.h +1 -0
- data/ext/backports/ruby/encoding.h +68 -0
- data/ext/depend +36 -0
- data/ext/eventids2.c +279 -0
- data/ext/extconf.rb +27 -0
- data/ext/extra/node.h +483 -0
- data/ext/extra/regenc.h +2 -0
- data/ext/id.c +50 -0
- data/ext/id.h +170 -0
- data/ext/lex.c +219 -0
- data/ext/parse.h +187 -0
- data/ext/parse.y +10637 -0
- data/ext/tools/generate-param-macros.rb +14 -0
- data/ext/tools/generate.rb +152 -0
- data/ext/tools/preproc.rb +91 -0
- data/ext/tools/strip.rb +12 -0
- data/ext/tools/ytab.sed +31 -0
- data/lib/ripper.rb +4 -0
- data/lib/ripper/core.rb +70 -0
- data/lib/ripper/filter.rb +70 -0
- data/lib/ripper/lexer.rb +179 -0
- data/lib/ripper/sexp.rb +99 -0
- metadata +94 -0
data/lib/ripper/lexer.rb
ADDED
@@ -0,0 +1,179 @@
|
|
1
|
+
#
|
2
|
+
# $Id: lexer.rb 25189 2009-10-02 12:04:37Z akr $
|
3
|
+
#
|
4
|
+
# Copyright (c) 2004,2005 Minero Aoki
|
5
|
+
#
|
6
|
+
# This program is free software.
|
7
|
+
# You can distribute and/or modify this program under the Ruby License.
|
8
|
+
# For details of Ruby License, see ruby/COPYING.
|
9
|
+
#
|
10
|
+
|
11
|
+
require 'ripper/core'
|
12
|
+
|
13
|
+
class Ripper
|
14
|
+
|
15
|
+
# Tokenizes Ruby program and returns an Array of String.
|
16
|
+
def Ripper.tokenize(src, filename = '-', lineno = 1)
|
17
|
+
Lexer.new(src, filename, lineno).tokenize
|
18
|
+
end
|
19
|
+
|
20
|
+
# Tokenizes Ruby program and returns an Array of Array,
|
21
|
+
# which is formatted like [[lineno, column], type, token].
|
22
|
+
#
|
23
|
+
# require 'ripper'
|
24
|
+
# require 'pp'
|
25
|
+
#
|
26
|
+
# p Ripper.lex("def m(a) nil end")
|
27
|
+
# #=> [[[1, 0], :on_kw, "def"],
|
28
|
+
# [[1, 3], :on_sp, " " ],
|
29
|
+
# [[1, 4], :on_ident, "m" ],
|
30
|
+
# [[1, 5], :on_lparen, "(" ],
|
31
|
+
# [[1, 6], :on_ident, "a" ],
|
32
|
+
# [[1, 7], :on_rparen, ")" ],
|
33
|
+
# [[1, 8], :on_sp, " " ],
|
34
|
+
# [[1, 9], :on_kw, "nil"],
|
35
|
+
# [[1, 12], :on_sp, " " ],
|
36
|
+
# [[1, 13], :on_kw, "end"]]
|
37
|
+
#
|
38
|
+
def Ripper.lex(src, filename = '-', lineno = 1)
|
39
|
+
Lexer.new(src, filename, lineno).lex
|
40
|
+
end
|
41
|
+
|
42
|
+
class Lexer < ::Ripper #:nodoc: internal use only
|
43
|
+
def tokenize
|
44
|
+
lex().map {|pos, event, tok| tok }
|
45
|
+
end
|
46
|
+
|
47
|
+
def lex
|
48
|
+
parse().sort_by {|pos, event, tok| pos }
|
49
|
+
end
|
50
|
+
|
51
|
+
def parse
|
52
|
+
@buf = []
|
53
|
+
super
|
54
|
+
@buf
|
55
|
+
end
|
56
|
+
|
57
|
+
private
|
58
|
+
|
59
|
+
SCANNER_EVENTS.each do |event|
|
60
|
+
module_eval(<<-End, __FILE__+'/module_eval', __LINE__ + 1)
|
61
|
+
def on_#{event}(tok)
|
62
|
+
@buf.push [[lineno(), column()], :on_#{event}, tok]
|
63
|
+
end
|
64
|
+
End
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
# [EXPERIMENTAL]
|
69
|
+
# Parses +src+ and return a string which was matched to +pattern+.
|
70
|
+
# +pattern+ should be described as Regexp.
|
71
|
+
#
|
72
|
+
# require 'ripper'
|
73
|
+
#
|
74
|
+
# p Ripper.slice('def m(a) nil end', 'ident') #=> "m"
|
75
|
+
# p Ripper.slice('def m(a) nil end', '[ident lparen rparen]+') #=> "m(a)"
|
76
|
+
# p Ripper.slice("<<EOS\nstring\nEOS",
|
77
|
+
# 'heredoc_beg nl $(tstring_content*) heredoc_end', 1)
|
78
|
+
# #=> "string\n"
|
79
|
+
#
|
80
|
+
def Ripper.slice(src, pattern, n = 0)
|
81
|
+
if m = token_match(src, pattern)
|
82
|
+
then m.string(n)
|
83
|
+
else nil
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def Ripper.token_match(src, pattern) #:nodoc:
|
88
|
+
TokenPattern.compile(pattern).match(src)
|
89
|
+
end
|
90
|
+
|
91
|
+
class TokenPattern #:nodoc:
|
92
|
+
|
93
|
+
class Error < ::StandardError; end
|
94
|
+
class CompileError < Error; end
|
95
|
+
class MatchError < Error; end
|
96
|
+
|
97
|
+
class << self
|
98
|
+
alias compile new
|
99
|
+
end
|
100
|
+
|
101
|
+
def initialize(pattern)
|
102
|
+
@source = pattern
|
103
|
+
@re = compile(pattern)
|
104
|
+
end
|
105
|
+
|
106
|
+
def match(str)
|
107
|
+
match_list(::Ripper.lex(str))
|
108
|
+
end
|
109
|
+
|
110
|
+
def match_list(tokens)
|
111
|
+
if m = @re.match(map_tokens(tokens))
|
112
|
+
then MatchData.new(tokens, m)
|
113
|
+
else nil
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
private
|
118
|
+
|
119
|
+
def compile(pattern)
|
120
|
+
if m = /[^\w\s$()\[\]{}?*+\.]/.match(pattern)
|
121
|
+
raise CompileError, "invalid char in pattern: #{m[0].inspect}"
|
122
|
+
end
|
123
|
+
buf = ''
|
124
|
+
pattern.scan(/(?:\w+|\$\(|[()\[\]\{\}?*+\.]+)/) do |tok|
|
125
|
+
case tok
|
126
|
+
when /\w/
|
127
|
+
buf.concat map_token(tok)
|
128
|
+
when '$('
|
129
|
+
buf.concat '('
|
130
|
+
when '('
|
131
|
+
buf.concat '(?:'
|
132
|
+
when /[?*\[\])\.]/
|
133
|
+
buf.concat tok
|
134
|
+
else
|
135
|
+
raise 'must not happen'
|
136
|
+
end
|
137
|
+
end
|
138
|
+
Regexp.compile(buf)
|
139
|
+
rescue RegexpError => err
|
140
|
+
raise CompileError, err.message
|
141
|
+
end
|
142
|
+
|
143
|
+
def map_tokens(tokens)
|
144
|
+
tokens.map {|pos,type,str| map_token(type.to_s.sub(/\Aon_/,'')) }.join
|
145
|
+
end
|
146
|
+
|
147
|
+
MAP = {}
|
148
|
+
seed = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a
|
149
|
+
SCANNER_EVENT_TABLE.each do |ev, |
|
150
|
+
raise CompileError, "[RIPPER FATAL] too many system token" if seed.empty?
|
151
|
+
MAP[ev.to_s.sub(/\Aon_/,'')] = seed.shift
|
152
|
+
end
|
153
|
+
|
154
|
+
def map_token(tok)
|
155
|
+
MAP[tok] or raise CompileError, "unknown token: #{tok}"
|
156
|
+
end
|
157
|
+
|
158
|
+
class MatchData
|
159
|
+
def initialize(tokens, match)
|
160
|
+
@tokens = tokens
|
161
|
+
@match = match
|
162
|
+
end
|
163
|
+
|
164
|
+
def string(n = 0)
|
165
|
+
return nil unless @match
|
166
|
+
match(n).join
|
167
|
+
end
|
168
|
+
|
169
|
+
private
|
170
|
+
|
171
|
+
def match(n = 0)
|
172
|
+
return [] unless @match
|
173
|
+
@tokens[@match.begin(n)...@match.end(n)].map {|pos,type,str| str }
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
end
|
178
|
+
|
179
|
+
end
|
data/lib/ripper/sexp.rb
ADDED
@@ -0,0 +1,99 @@
|
|
1
|
+
#
|
2
|
+
# $Id: sexp.rb 25189 2009-10-02 12:04:37Z akr $
|
3
|
+
#
|
4
|
+
# Copyright (c) 2004,2005 Minero Aoki
|
5
|
+
#
|
6
|
+
# This program is free software.
|
7
|
+
# You can distribute and/or modify this program under the Ruby License.
|
8
|
+
# For details of Ruby License, see ruby/COPYING.
|
9
|
+
#
|
10
|
+
|
11
|
+
require 'ripper/core'
|
12
|
+
|
13
|
+
class Ripper
|
14
|
+
|
15
|
+
# [EXPERIMENTAL]
|
16
|
+
# Parses +src+ and create S-exp tree.
|
17
|
+
# This method is for mainly developper use.
|
18
|
+
#
|
19
|
+
# require 'ripper'
|
20
|
+
# require 'pp
|
21
|
+
#
|
22
|
+
# pp Ripper.sexp("def m(a) nil end")
|
23
|
+
# #=> [:program,
|
24
|
+
# [:stmts_add,
|
25
|
+
# [:stmts_new],
|
26
|
+
# [:def,
|
27
|
+
# [:@ident, "m", [1, 4]],
|
28
|
+
# [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil]],
|
29
|
+
# [:bodystmt,
|
30
|
+
# [:stmts_add, [:stmts_new], [:var_ref, [:@kw, "nil", [1, 9]]]],
|
31
|
+
# nil,
|
32
|
+
# nil,
|
33
|
+
# nil]]]]
|
34
|
+
#
|
35
|
+
def Ripper.sexp(src, filename = '-', lineno = 1)
|
36
|
+
SexpBuilderPP.new(src, filename, lineno).parse
|
37
|
+
end
|
38
|
+
|
39
|
+
def Ripper.sexp_raw(src, filename = '-', lineno = 1)
|
40
|
+
SexpBuilder.new(src, filename, lineno).parse
|
41
|
+
end
|
42
|
+
|
43
|
+
class SexpBuilderPP < ::Ripper #:nodoc:
|
44
|
+
private
|
45
|
+
|
46
|
+
PARSER_EVENT_TABLE.each do |event, arity|
|
47
|
+
if /_new\z/ =~ event.to_s and arity == 0
|
48
|
+
module_eval(<<-End, __FILE__, __LINE__ + 1)
|
49
|
+
def on_#{event}
|
50
|
+
[]
|
51
|
+
end
|
52
|
+
End
|
53
|
+
elsif /_add\z/ =~ event.to_s
|
54
|
+
module_eval(<<-End, __FILE__, __LINE__ + 1)
|
55
|
+
def on_#{event}(list, item)
|
56
|
+
list.push item
|
57
|
+
list
|
58
|
+
end
|
59
|
+
End
|
60
|
+
else
|
61
|
+
module_eval(<<-End, __FILE__, __LINE__ + 1)
|
62
|
+
def on_#{event}(*args)
|
63
|
+
[:#{event}, *args]
|
64
|
+
end
|
65
|
+
End
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
SCANNER_EVENTS.each do |event|
|
70
|
+
module_eval(<<-End, __FILE__, __LINE__ + 1)
|
71
|
+
def on_#{event}(tok)
|
72
|
+
[:@#{event}, tok, [lineno(), column()]]
|
73
|
+
end
|
74
|
+
End
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
class SexpBuilder < ::Ripper #:nodoc:
|
79
|
+
private
|
80
|
+
|
81
|
+
PARSER_EVENTS.each do |event|
|
82
|
+
module_eval(<<-End, __FILE__, __LINE__ + 1)
|
83
|
+
def on_#{event}(*args)
|
84
|
+
args.unshift :#{event}
|
85
|
+
args
|
86
|
+
end
|
87
|
+
End
|
88
|
+
end
|
89
|
+
|
90
|
+
SCANNER_EVENTS.each do |event|
|
91
|
+
module_eval(<<-End, __FILE__, __LINE__ + 1)
|
92
|
+
def on_#{event}(tok)
|
93
|
+
[:@#{event}, tok, [lineno(), column()]]
|
94
|
+
end
|
95
|
+
End
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
end
|
metadata
ADDED
@@ -0,0 +1,94 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ripper
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 23
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 1
|
8
|
+
- 0
|
9
|
+
- 0
|
10
|
+
version: 1.0.0
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Loren Segal
|
14
|
+
- Minero Aoki
|
15
|
+
autorequire:
|
16
|
+
bindir: bin
|
17
|
+
cert_chain: []
|
18
|
+
|
19
|
+
date: 2011-01-11 00:00:00 -05:00
|
20
|
+
default_executable:
|
21
|
+
dependencies: []
|
22
|
+
|
23
|
+
description: " This gem is meant for Ruby 1.8.x builds. Ruby 1.9.x comes with a functioning\n ripper implementation out of the box. This gem is a port of the Ripper 1.9\n for Ruby 1.8.x. Ripper was written by Minero Aoki and ported by Loren Segal.\n"
|
24
|
+
email: lsegal@soen.ca
|
25
|
+
executables: []
|
26
|
+
|
27
|
+
extensions:
|
28
|
+
- ext/extconf.rb
|
29
|
+
extra_rdoc_files: []
|
30
|
+
|
31
|
+
files:
|
32
|
+
- lib/ripper/core.rb
|
33
|
+
- lib/ripper/filter.rb
|
34
|
+
- lib/ripper/lexer.rb
|
35
|
+
- lib/ripper/sexp.rb
|
36
|
+
- lib/ripper.rb
|
37
|
+
- ext/backports/backports.h
|
38
|
+
- ext/backports/regenc.h
|
39
|
+
- ext/backports/ruby/encoding.h
|
40
|
+
- ext/depend
|
41
|
+
- ext/eventids2.c
|
42
|
+
- ext/extconf.rb
|
43
|
+
- ext/extra/node.h
|
44
|
+
- ext/extra/regenc.h
|
45
|
+
- ext/id.c
|
46
|
+
- ext/id.h
|
47
|
+
- ext/lex.c
|
48
|
+
- ext/Makefile
|
49
|
+
- ext/parse.h
|
50
|
+
- ext/parse.y
|
51
|
+
- ext/tools/generate-param-macros.rb
|
52
|
+
- ext/tools/generate.rb
|
53
|
+
- ext/tools/preproc.rb
|
54
|
+
- ext/tools/strip.rb
|
55
|
+
- ext/tools/ytab.sed
|
56
|
+
- Rakefile
|
57
|
+
- README
|
58
|
+
has_rdoc: true
|
59
|
+
homepage: http://github.com/lsegal/ripper
|
60
|
+
licenses: []
|
61
|
+
|
62
|
+
post_install_message:
|
63
|
+
rdoc_options: []
|
64
|
+
|
65
|
+
require_paths:
|
66
|
+
- lib
|
67
|
+
- ext
|
68
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
69
|
+
none: false
|
70
|
+
requirements:
|
71
|
+
- - ">="
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
hash: 3
|
74
|
+
segments:
|
75
|
+
- 0
|
76
|
+
version: "0"
|
77
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
78
|
+
none: false
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
hash: 3
|
83
|
+
segments:
|
84
|
+
- 0
|
85
|
+
version: "0"
|
86
|
+
requirements: []
|
87
|
+
|
88
|
+
rubyforge_project:
|
89
|
+
rubygems_version: 1.4.0
|
90
|
+
signing_key:
|
91
|
+
specification_version: 3
|
92
|
+
summary: Ripper parses Ruby source and tokenizes or builds an AST
|
93
|
+
test_files: []
|
94
|
+
|