rdoc 6.0.0.beta1 → 6.0.0.beta2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of rdoc might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.travis.yml +2 -12
- data/appveyor.yml +18 -0
- data/lib/rdoc.rb +2 -2
- data/lib/rdoc/constant.rb +2 -2
- data/lib/rdoc/context.rb +12 -0
- data/lib/rdoc/markup/to_html.rb +5 -3
- data/lib/rdoc/parser/c.rb +2 -4
- data/lib/rdoc/parser/ripper_state_lex.rb +587 -0
- data/lib/rdoc/parser/ruby.rb +441 -355
- data/lib/rdoc/parser/ruby_tools.rb +22 -10
- data/lib/rdoc/stats/normal.rb +2 -2
- data/lib/rdoc/token_stream.rb +31 -18
- data/rdoc.gemspec +5 -5
- metadata +18 -4
- data/lib/gauntlet_rdoc.rb +0 -82
- data/lib/rdoc/ruby_lex.rb +0 -1521
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 79f073ce205804f7cb64ae64f2918b47aa208b80
|
4
|
+
data.tar.gz: 3c5c07e3985797e49ec9d43e3f68ac2b0246061a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cdecae483f5d749fed6339751c6ba2f28cd5c608024ead2d24ca20635548ecb9df5dc44c8a04f2052b3c09e9ee9534381960281ed8275042bd4ff9a14fc6aa88
|
7
|
+
data.tar.gz: 2bbe546d5745c41bce196a253e708e6dca4ed21cd9a0695ee7c61ef1163b6750b6a8648149b3afcfdadde2873af9ba326d1b3c62d9ad29495c92d72100dd1665
|
data/.travis.yml
CHANGED
@@ -2,23 +2,13 @@
|
|
2
2
|
before_install:
|
3
3
|
- gem install bundler --no-document
|
4
4
|
language: ruby
|
5
|
-
notifications:
|
6
|
-
email:
|
7
|
-
- mail@zzak.io
|
8
5
|
rvm:
|
9
6
|
- 2.2.7
|
10
7
|
- 2.3.4
|
11
8
|
- 2.4.1
|
12
9
|
- ruby-head
|
13
|
-
-
|
14
|
-
- jruby-1.7.26
|
15
|
-
- jruby-9.1.9.0
|
10
|
+
- jruby-9.1.13.0
|
16
11
|
env:
|
17
12
|
global:
|
18
13
|
NOBENCHMARK=1
|
19
|
-
script: rake
|
20
|
-
matrix:
|
21
|
-
allow_failures:
|
22
|
-
- rvm: jruby-1.7.26
|
23
|
-
- rvm: jruby-9.1.9.0
|
24
|
-
- rvm: rbx-2
|
14
|
+
script: bundle exec rake
|
data/appveyor.yml
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
---
|
2
|
+
install:
|
3
|
+
- SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
|
4
|
+
- gem install bundler --no-document
|
5
|
+
- bundle install
|
6
|
+
build: off
|
7
|
+
test_script:
|
8
|
+
- rake
|
9
|
+
deploy: off
|
10
|
+
environment:
|
11
|
+
NOBENCHMARK: 1
|
12
|
+
matrix:
|
13
|
+
- ruby_version: "22"
|
14
|
+
- ruby_version: "22-x64"
|
15
|
+
- ruby_version: "23"
|
16
|
+
- ruby_version: "23-x64"
|
17
|
+
- ruby_version: "24"
|
18
|
+
- ruby_version: "24-x64"
|
data/lib/rdoc.rb
CHANGED
@@ -65,7 +65,7 @@ module RDoc
|
|
65
65
|
##
|
66
66
|
# RDoc version you are using
|
67
67
|
|
68
|
-
VERSION = '6.0.0.
|
68
|
+
VERSION = '6.0.0.beta2'
|
69
69
|
|
70
70
|
##
|
71
71
|
# Method visibilities
|
@@ -148,7 +148,7 @@ module RDoc
|
|
148
148
|
|
149
149
|
autoload :KNOWN_CLASSES, 'rdoc/known_classes'
|
150
150
|
|
151
|
-
autoload :
|
151
|
+
autoload :RipperStateLex, 'rdoc/parser/ripper_state_lex'
|
152
152
|
autoload :RubyToken, 'rdoc/ruby_token'
|
153
153
|
autoload :TokenStream, 'rdoc/token_stream'
|
154
154
|
|
data/lib/rdoc/constant.rb
CHANGED
@@ -36,7 +36,7 @@ class RDoc::Constant < RDoc::CodeObject
|
|
36
36
|
@value = value
|
37
37
|
|
38
38
|
@is_alias_for = nil
|
39
|
-
@visibility =
|
39
|
+
@visibility = :public
|
40
40
|
|
41
41
|
self.comment = comment
|
42
42
|
end
|
@@ -136,7 +136,7 @@ class RDoc::Constant < RDoc::CodeObject
|
|
136
136
|
initialize array[1], nil, array[5]
|
137
137
|
|
138
138
|
@full_name = array[2]
|
139
|
-
@visibility = array[3]
|
139
|
+
@visibility = array[3] || :public
|
140
140
|
@is_alias_for = array[4]
|
141
141
|
# 5 handled above
|
142
142
|
# 6 handled below
|
data/lib/rdoc/context.rb
CHANGED
@@ -1079,6 +1079,7 @@ class RDoc::Context < RDoc::CodeObject
|
|
1079
1079
|
return if [:private, :nodoc].include? min_visibility
|
1080
1080
|
remove_invisible_in @method_list, min_visibility
|
1081
1081
|
remove_invisible_in @attributes, min_visibility
|
1082
|
+
remove_invisible_in @constants, min_visibility
|
1082
1083
|
end
|
1083
1084
|
|
1084
1085
|
##
|
@@ -1165,6 +1166,17 @@ class RDoc::Context < RDoc::CodeObject
|
|
1165
1166
|
end
|
1166
1167
|
end
|
1167
1168
|
|
1169
|
+
##
|
1170
|
+
# Given an array +names+ of constants, set the visibility of each constant to
|
1171
|
+
# +visibility+
|
1172
|
+
|
1173
|
+
def set_constant_visibility_for(names, visibility)
|
1174
|
+
names.each do |name|
|
1175
|
+
constant = @constants_hash[name] or next
|
1176
|
+
constant.visibility = visibility
|
1177
|
+
end
|
1178
|
+
end
|
1179
|
+
|
1168
1180
|
##
|
1169
1181
|
# Sorts sections alphabetically (default) or in TomDoc fashion (none,
|
1170
1182
|
# Public, Internal, Deprecated)
|
data/lib/rdoc/markup/to_html.rb
CHANGED
@@ -200,10 +200,12 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
|
|
200
200
|
|
201
201
|
content = if verbatim.ruby? or parseable? text then
|
202
202
|
begin
|
203
|
-
tokens = RDoc::
|
203
|
+
tokens = RDoc::RipperStateLex.parse text
|
204
204
|
klass = ' class="ruby"'
|
205
205
|
|
206
|
-
RDoc::TokenStream.to_html tokens
|
206
|
+
result = RDoc::TokenStream.to_html tokens
|
207
|
+
result = result + "\n" unless "\n" == result[-1]
|
208
|
+
result
|
207
209
|
rescue RDoc::RubyLex::Error
|
208
210
|
CGI.escapeHTML text
|
209
211
|
end
|
@@ -212,7 +214,7 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
|
|
212
214
|
end
|
213
215
|
|
214
216
|
if @options.pipe then
|
215
|
-
@res << "\n<pre><code>#{CGI.escapeHTML text}</code></pre>\n"
|
217
|
+
@res << "\n<pre><code>#{CGI.escapeHTML text}\n</code></pre>\n"
|
216
218
|
else
|
217
219
|
@res << "\n<pre#{klass}>#{content}</pre>\n"
|
218
220
|
end
|
data/lib/rdoc/parser/c.rb
CHANGED
@@ -666,8 +666,7 @@ class RDoc::Parser::C < RDoc::Parser
|
|
666
666
|
|
667
667
|
#meth_obj.params = params
|
668
668
|
meth_obj.start_collecting_tokens
|
669
|
-
tk =
|
670
|
-
tk.set_text body
|
669
|
+
tk = { :line_no => 1, :char_no => 1, :text => body }
|
671
670
|
meth_obj.add_token tk
|
672
671
|
meth_obj.comment = comment
|
673
672
|
meth_obj.line = file_content[0, offset].count("\n") + 1
|
@@ -684,8 +683,7 @@ class RDoc::Parser::C < RDoc::Parser
|
|
684
683
|
find_modifiers comment, meth_obj
|
685
684
|
|
686
685
|
meth_obj.start_collecting_tokens
|
687
|
-
tk =
|
688
|
-
tk.set_text body
|
686
|
+
tk = { :line_no => 1, :char_no => 1, :text => body }
|
689
687
|
meth_obj.add_token tk
|
690
688
|
meth_obj.comment = comment
|
691
689
|
meth_obj.line = file_content[0, offset].count("\n") + 1
|
@@ -0,0 +1,587 @@
|
|
1
|
+
require 'ripper'
|
2
|
+
|
3
|
+
class RDoc::RipperStateLex
|
4
|
+
EXPR_NONE = 0
|
5
|
+
EXPR_BEG = 1
|
6
|
+
EXPR_END = 2
|
7
|
+
EXPR_ENDARG = 4
|
8
|
+
EXPR_ENDFN = 8
|
9
|
+
EXPR_ARG = 16
|
10
|
+
EXPR_CMDARG = 32
|
11
|
+
EXPR_MID = 64
|
12
|
+
EXPR_FNAME = 128
|
13
|
+
EXPR_DOT = 256
|
14
|
+
EXPR_CLASS = 512
|
15
|
+
EXPR_LABEL = 1024
|
16
|
+
EXPR_LABELED = 2048
|
17
|
+
EXPR_FITEM = 4096
|
18
|
+
EXPR_VALUE = EXPR_BEG
|
19
|
+
EXPR_BEG_ANY = (EXPR_BEG | EXPR_MID | EXPR_CLASS)
|
20
|
+
EXPR_ARG_ANY = (EXPR_ARG | EXPR_CMDARG)
|
21
|
+
EXPR_END_ANY = (EXPR_END | EXPR_ENDARG | EXPR_ENDFN)
|
22
|
+
|
23
|
+
class InnerStateLex < Ripper::Filter
|
24
|
+
attr_accessor :lex_state
|
25
|
+
|
26
|
+
def initialize(code)
|
27
|
+
@lex_state = EXPR_BEG
|
28
|
+
@in_fname = false
|
29
|
+
@continue = false
|
30
|
+
reset
|
31
|
+
super(code)
|
32
|
+
end
|
33
|
+
|
34
|
+
def reset
|
35
|
+
@command_start = false
|
36
|
+
@cmd_state = @command_start
|
37
|
+
end
|
38
|
+
|
39
|
+
def on_nl(tok, data)
|
40
|
+
case @lex_state
|
41
|
+
when EXPR_FNAME, EXPR_DOT
|
42
|
+
@continue = true
|
43
|
+
else
|
44
|
+
@continue = false
|
45
|
+
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
|
46
|
+
end
|
47
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
48
|
+
end
|
49
|
+
|
50
|
+
def on_ignored_nl(tok, data)
|
51
|
+
case @lex_state
|
52
|
+
when EXPR_FNAME, EXPR_DOT
|
53
|
+
@continue = true
|
54
|
+
else
|
55
|
+
@continue = false
|
56
|
+
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
|
57
|
+
end
|
58
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
59
|
+
end
|
60
|
+
|
61
|
+
def on_op(tok, data)
|
62
|
+
case tok
|
63
|
+
when '&', '|', '!', '!=', '!~'
|
64
|
+
case @lex_state
|
65
|
+
when EXPR_FNAME, EXPR_DOT
|
66
|
+
@lex_state = EXPR_ARG
|
67
|
+
else
|
68
|
+
@lex_state = EXPR_BEG
|
69
|
+
end
|
70
|
+
when '<<'
|
71
|
+
# TODO next token?
|
72
|
+
case @lex_state
|
73
|
+
when EXPR_FNAME, EXPR_DOT
|
74
|
+
@lex_state = EXPR_ARG
|
75
|
+
else
|
76
|
+
@lex_state = EXPR_BEG
|
77
|
+
end
|
78
|
+
when '?'
|
79
|
+
@lex_state = EXPR_BEG
|
80
|
+
when '&&', '||', '+=', '-=', '*=', '**=',
|
81
|
+
'&=', '|=', '^=', '<<=', '>>=', '||=', '&&='
|
82
|
+
@lex_state = EXPR_BEG
|
83
|
+
else
|
84
|
+
case @lex_state
|
85
|
+
when EXPR_FNAME, EXPR_DOT
|
86
|
+
@lex_state = EXPR_ARG
|
87
|
+
else
|
88
|
+
@lex_state = EXPR_BEG
|
89
|
+
end
|
90
|
+
end
|
91
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
92
|
+
end
|
93
|
+
|
94
|
+
def on_kw(tok, data)
|
95
|
+
case tok
|
96
|
+
when 'class'
|
97
|
+
@lex_state = EXPR_CLASS
|
98
|
+
@in_fname = true
|
99
|
+
when 'def'
|
100
|
+
@lex_state = EXPR_FNAME
|
101
|
+
@continue = true
|
102
|
+
@in_fname = true
|
103
|
+
when 'if', 'unless', 'while', 'until'
|
104
|
+
if ((EXPR_END | EXPR_ENDARG | EXPR_ENDFN | EXPR_ARG | EXPR_CMDARG) & @lex_state) != 0 # postfix if
|
105
|
+
@lex_state = EXPR_BEG | EXPR_LABEL
|
106
|
+
else
|
107
|
+
@lex_state = EXPR_BEG
|
108
|
+
end
|
109
|
+
when 'begin'
|
110
|
+
@lex_state = EXPR_BEG
|
111
|
+
else
|
112
|
+
if @lex_state == EXPR_FNAME
|
113
|
+
@lex_state = EXPR_END
|
114
|
+
else
|
115
|
+
@lex_state = EXPR_END
|
116
|
+
end
|
117
|
+
end
|
118
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
119
|
+
end
|
120
|
+
|
121
|
+
def on_tstring_beg(tok, data)
|
122
|
+
@lex_state = EXPR_BEG
|
123
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
124
|
+
end
|
125
|
+
|
126
|
+
def on_tstring_end(tok, data)
|
127
|
+
@lex_state = EXPR_END | EXPR_ENDARG
|
128
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
129
|
+
end
|
130
|
+
|
131
|
+
def on_CHAR(tok, data)
|
132
|
+
@lex_state = EXPR_END
|
133
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
134
|
+
end
|
135
|
+
|
136
|
+
def on_period(tok, data)
|
137
|
+
@lex_state = EXPR_DOT
|
138
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
139
|
+
end
|
140
|
+
|
141
|
+
def on_int(tok, data)
|
142
|
+
@lex_state = EXPR_END | EXPR_ENDARG
|
143
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
144
|
+
end
|
145
|
+
|
146
|
+
def on_float(tok, data)
|
147
|
+
@lex_state = EXPR_END | EXPR_ENDARG
|
148
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
149
|
+
end
|
150
|
+
|
151
|
+
def on_rational(tok, data)
|
152
|
+
@lex_state = EXPR_END | EXPR_ENDARG
|
153
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
154
|
+
end
|
155
|
+
|
156
|
+
def on_imaginary(tok, data)
|
157
|
+
@lex_state = EXPR_END | EXPR_ENDARG
|
158
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
159
|
+
end
|
160
|
+
|
161
|
+
def on_symbeg(tok, data)
|
162
|
+
@lex_state = EXPR_FNAME
|
163
|
+
@continue = true
|
164
|
+
@in_fname = true
|
165
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
166
|
+
end
|
167
|
+
|
168
|
+
private def on_variables(event, tok, data)
|
169
|
+
if @in_fname
|
170
|
+
@lex_state = EXPR_ENDFN
|
171
|
+
@in_fname = false
|
172
|
+
@continue = false
|
173
|
+
elsif @continue
|
174
|
+
case @lex_state
|
175
|
+
when EXPR_DOT
|
176
|
+
@lex_state = EXPR_ARG
|
177
|
+
else
|
178
|
+
@lex_state = EXPR_ENDFN
|
179
|
+
@continue = false
|
180
|
+
end
|
181
|
+
else
|
182
|
+
@lex_state = EXPR_CMDARG
|
183
|
+
end
|
184
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state})
|
185
|
+
end
|
186
|
+
|
187
|
+
def on_ident(tok, data)
|
188
|
+
on_variables(__method__, tok, data)
|
189
|
+
end
|
190
|
+
|
191
|
+
def on_ivar(tok, data)
|
192
|
+
@lex_state = EXPR_END
|
193
|
+
on_variables(__method__, tok, data)
|
194
|
+
end
|
195
|
+
|
196
|
+
def on_cvar(tok, data)
|
197
|
+
@lex_state = EXPR_END
|
198
|
+
on_variables(__method__, tok, data)
|
199
|
+
end
|
200
|
+
|
201
|
+
def on_gvar(tok, data)
|
202
|
+
@lex_state = EXPR_END
|
203
|
+
on_variables(__method__, tok, data)
|
204
|
+
end
|
205
|
+
|
206
|
+
def on_backref(tok, data)
|
207
|
+
@lex_state = EXPR_END
|
208
|
+
on_variables(__method__, tok, data)
|
209
|
+
end
|
210
|
+
|
211
|
+
def on_lparen(tok, data)
|
212
|
+
@lex_state = EXPR_LABEL | EXPR_BEG
|
213
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
214
|
+
end
|
215
|
+
|
216
|
+
def on_rparen(tok, data)
|
217
|
+
@lex_state = EXPR_ENDFN
|
218
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
219
|
+
end
|
220
|
+
|
221
|
+
def on_lbrace(tok, data)
|
222
|
+
@lex_state = EXPR_LABEL | EXPR_BEG
|
223
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
224
|
+
end
|
225
|
+
|
226
|
+
def on_rbrace(tok, data)
|
227
|
+
@lex_state = EXPR_ENDARG
|
228
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
229
|
+
end
|
230
|
+
|
231
|
+
def on_lbracket(tok, data)
|
232
|
+
@lex_state = EXPR_LABEL | EXPR_BEG
|
233
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
234
|
+
end
|
235
|
+
|
236
|
+
def on_rbracket(tok, data)
|
237
|
+
@lex_state = EXPR_ENDARG
|
238
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
239
|
+
end
|
240
|
+
|
241
|
+
def on_const(tok, data)
|
242
|
+
case @lex_state
|
243
|
+
when EXPR_FNAME
|
244
|
+
@lex_state = EXPR_ENDFN
|
245
|
+
when EXPR_CLASS
|
246
|
+
@lex_state = EXPR_ARG
|
247
|
+
else
|
248
|
+
@lex_state = EXPR_CMDARG
|
249
|
+
end
|
250
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
251
|
+
end
|
252
|
+
|
253
|
+
def on_sp(tok, data)
|
254
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
255
|
+
end
|
256
|
+
|
257
|
+
def on_comma(tok, data)
|
258
|
+
@lex_state = EXPR_BEG | EXPR_LABEL if (EXPR_ARG_ANY & @lex_state) != 0
|
259
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
260
|
+
end
|
261
|
+
|
262
|
+
def on_comment(tok, data)
|
263
|
+
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
|
264
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
265
|
+
end
|
266
|
+
|
267
|
+
def on_ignored_sp(tok, data)
|
268
|
+
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
|
269
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
270
|
+
end
|
271
|
+
|
272
|
+
def on_heredoc_end(tok, data)
|
273
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
|
274
|
+
@lex_state = EXPR_BEG
|
275
|
+
end
|
276
|
+
|
277
|
+
def on_default(event, tok, data)
|
278
|
+
reset
|
279
|
+
@callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state})
|
280
|
+
end
|
281
|
+
|
282
|
+
def each(&block)
|
283
|
+
@callback = block
|
284
|
+
parse
|
285
|
+
end
|
286
|
+
end
|
287
|
+
|
288
|
+
def get_squashed_tk
|
289
|
+
if @buf.empty?
|
290
|
+
tk = @inner_lex_enumerator.next
|
291
|
+
else
|
292
|
+
tk = @buf.shift
|
293
|
+
end
|
294
|
+
case tk[:kind]
|
295
|
+
when :on_symbeg then
|
296
|
+
tk = get_symbol_tk(tk)
|
297
|
+
when :on_tstring_beg then
|
298
|
+
tk = get_string_tk(tk)
|
299
|
+
when :on_backtick then
|
300
|
+
if (EXPR_FNAME & tk[:state]) != 0
|
301
|
+
@inner_lex.lex_state = EXPR_ARG
|
302
|
+
tk[:kind] = :on_ident
|
303
|
+
tk[:state] = @inner_lex.lex_state
|
304
|
+
else
|
305
|
+
tk = get_string_tk(tk)
|
306
|
+
end
|
307
|
+
when :on_regexp_beg then
|
308
|
+
tk = get_regexp_tk(tk)
|
309
|
+
when :on_embdoc_beg then
|
310
|
+
tk = get_embdoc_tk(tk)
|
311
|
+
when :on_heredoc_beg then
|
312
|
+
@heredoc_queue << retrieve_heredoc_info(tk)
|
313
|
+
@inner_lex.lex_state = EXPR_END
|
314
|
+
when :on_nl, :on_ignored_nl, :on_comment, :on_heredoc_end then
|
315
|
+
unless @heredoc_queue.empty?
|
316
|
+
get_heredoc_tk(*@heredoc_queue.shift)
|
317
|
+
end
|
318
|
+
when :on_words_beg then
|
319
|
+
tk = get_words_tk(tk)
|
320
|
+
when :on_qwords_beg then
|
321
|
+
tk = get_words_tk(tk)
|
322
|
+
when :on_symbols_beg then
|
323
|
+
tk = get_words_tk(tk)
|
324
|
+
when :on_qsymbols_beg then
|
325
|
+
tk = get_words_tk(tk)
|
326
|
+
when :on_op then
|
327
|
+
if '&.' == tk[:text]
|
328
|
+
tk[:kind] = :on_period
|
329
|
+
else
|
330
|
+
tk = get_op_tk(tk)
|
331
|
+
end
|
332
|
+
end
|
333
|
+
tk
|
334
|
+
end
|
335
|
+
|
336
|
+
private def get_symbol_tk(tk)
|
337
|
+
is_symbol = true
|
338
|
+
symbol_tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_symbol }
|
339
|
+
if ":'" == tk[:text] or ':"' == tk[:text]
|
340
|
+
tk1 = get_string_tk(tk)
|
341
|
+
symbol_tk[:text] = tk1[:text]
|
342
|
+
symbol_tk[:state] = tk1[:state]
|
343
|
+
else
|
344
|
+
case (tk1 = get_squashed_tk)[:kind]
|
345
|
+
when :on_ident
|
346
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
347
|
+
symbol_tk[:state] = tk1[:state]
|
348
|
+
when :on_tstring_content
|
349
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
350
|
+
symbol_tk[:state] = get_squashed_tk[:state] # skip :on_tstring_end
|
351
|
+
when :on_tstring_end
|
352
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
353
|
+
symbol_tk[:state] = tk1[:state]
|
354
|
+
when :on_op
|
355
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
356
|
+
symbol_tk[:state] = tk1[:state]
|
357
|
+
when :on_ivar
|
358
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
359
|
+
symbol_tk[:state] = tk1[:state]
|
360
|
+
when :on_cvar
|
361
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
362
|
+
symbol_tk[:state] = tk1[:state]
|
363
|
+
when :on_gvar
|
364
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
365
|
+
symbol_tk[:state] = tk1[:state]
|
366
|
+
when :on_const
|
367
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
368
|
+
symbol_tk[:state] = tk1[:state]
|
369
|
+
when :on_kw
|
370
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
371
|
+
symbol_tk[:state] = tk1[:state]
|
372
|
+
else
|
373
|
+
is_symbol = false
|
374
|
+
tk = tk1
|
375
|
+
end
|
376
|
+
end
|
377
|
+
if is_symbol
|
378
|
+
tk = symbol_tk
|
379
|
+
end
|
380
|
+
tk
|
381
|
+
end
|
382
|
+
|
383
|
+
private def get_string_tk(tk)
|
384
|
+
string = tk[:text]
|
385
|
+
state = nil
|
386
|
+
kind = :on_tstring
|
387
|
+
loop do
|
388
|
+
inner_str_tk = get_squashed_tk
|
389
|
+
if inner_str_tk.nil?
|
390
|
+
break
|
391
|
+
elsif :on_tstring_end == inner_str_tk[:kind]
|
392
|
+
string = string + inner_str_tk[:text]
|
393
|
+
state = inner_str_tk[:state]
|
394
|
+
break
|
395
|
+
elsif :on_label_end == inner_str_tk[:kind]
|
396
|
+
string = string + inner_str_tk[:text]
|
397
|
+
state = inner_str_tk[:state]
|
398
|
+
kind = :on_symbol
|
399
|
+
break
|
400
|
+
else
|
401
|
+
string = string + inner_str_tk[:text]
|
402
|
+
if :on_embexpr_beg == inner_str_tk[:kind] then
|
403
|
+
kind = :on_dstring if :on_tstring == kind
|
404
|
+
end
|
405
|
+
end
|
406
|
+
end
|
407
|
+
{
|
408
|
+
:line_no => tk[:line_no],
|
409
|
+
:char_no => tk[:char_no],
|
410
|
+
:kind => kind,
|
411
|
+
:text => string,
|
412
|
+
:state => state
|
413
|
+
}
|
414
|
+
end
|
415
|
+
|
416
|
+
private def get_regexp_tk(tk)
|
417
|
+
string = tk[:text]
|
418
|
+
state = nil
|
419
|
+
loop do
|
420
|
+
inner_str_tk = get_squashed_tk
|
421
|
+
if inner_str_tk.nil?
|
422
|
+
break
|
423
|
+
elsif :on_regexp_end == inner_str_tk[:kind]
|
424
|
+
string = string + inner_str_tk[:text]
|
425
|
+
state = inner_str_tk[:state]
|
426
|
+
break
|
427
|
+
else
|
428
|
+
string = string + inner_str_tk[:text]
|
429
|
+
end
|
430
|
+
end
|
431
|
+
{
|
432
|
+
:line_no => tk[:line_no],
|
433
|
+
:char_no => tk[:char_no],
|
434
|
+
:kind => :on_regexp,
|
435
|
+
:text => string,
|
436
|
+
:state => state
|
437
|
+
}
|
438
|
+
end
|
439
|
+
|
440
|
+
private def get_embdoc_tk(tk)
|
441
|
+
string = tk[:text]
|
442
|
+
until :on_embdoc_end == (embdoc_tk = get_squashed_tk)[:kind] do
|
443
|
+
string = string + embdoc_tk[:text]
|
444
|
+
end
|
445
|
+
string = string + embdoc_tk[:text]
|
446
|
+
{
|
447
|
+
:line_no => tk[:line_no],
|
448
|
+
:char_no => tk[:char_no],
|
449
|
+
:kind => :on_embdoc,
|
450
|
+
:text => string,
|
451
|
+
:state => embdoc_tk[:state]
|
452
|
+
}
|
453
|
+
end
|
454
|
+
|
455
|
+
private def get_heredoc_tk(heredoc_name, indent)
|
456
|
+
string = ''
|
457
|
+
start_tk = nil
|
458
|
+
prev_tk = nil
|
459
|
+
until heredoc_end?(heredoc_name, indent, tk = @inner_lex_enumerator.next) do
|
460
|
+
start_tk = tk unless start_tk
|
461
|
+
if (prev_tk.nil? or "\n" == prev_tk[:text][-1]) and 0 != tk[:char_no]
|
462
|
+
string = string + (' ' * tk[:char_no])
|
463
|
+
end
|
464
|
+
string = string + tk[:text]
|
465
|
+
prev_tk = tk
|
466
|
+
end
|
467
|
+
start_tk = tk unless start_tk
|
468
|
+
prev_tk = tk unless prev_tk
|
469
|
+
@buf.unshift tk # closing heredoc
|
470
|
+
heredoc_tk = {
|
471
|
+
:line_no => start_tk[:line_no],
|
472
|
+
:char_no => start_tk[:char_no],
|
473
|
+
:kind => :on_heredoc,
|
474
|
+
:text => string,
|
475
|
+
:state => prev_tk[:state]
|
476
|
+
}
|
477
|
+
@buf.unshift heredoc_tk
|
478
|
+
end
|
479
|
+
|
480
|
+
private def retrieve_heredoc_info(tk)
|
481
|
+
name = tk[:text].gsub(/\A<<[-~]?(['"`]?)(.+)\1\z/, '\2')
|
482
|
+
indent = tk[:text] =~ /\A<<[-~]/
|
483
|
+
[name, indent]
|
484
|
+
end
|
485
|
+
|
486
|
+
private def heredoc_end?(name, indent, tk)
|
487
|
+
result = false
|
488
|
+
if :on_heredoc_end == tk[:kind] then
|
489
|
+
tk_name = (indent ? tk[:text].gsub(/^ *(.+)\n?$/, '\1') : tk[:text].gsub(/\n\z/, ''))
|
490
|
+
if name == tk_name
|
491
|
+
result = true
|
492
|
+
end
|
493
|
+
end
|
494
|
+
result
|
495
|
+
end
|
496
|
+
|
497
|
+
private def get_words_tk(tk)
|
498
|
+
string = ''
|
499
|
+
start_token = tk[:text]
|
500
|
+
start_quote = tk[:text].rstrip[-1]
|
501
|
+
line_no = tk[:line_no]
|
502
|
+
char_no = tk[:char_no]
|
503
|
+
state = tk[:state]
|
504
|
+
end_quote =
|
505
|
+
case start_quote
|
506
|
+
when ?( then ?)
|
507
|
+
when ?[ then ?]
|
508
|
+
when ?{ then ?}
|
509
|
+
when ?< then ?>
|
510
|
+
else start_quote
|
511
|
+
end
|
512
|
+
end_token = nil
|
513
|
+
loop do
|
514
|
+
tk = get_squashed_tk
|
515
|
+
if tk.nil?
|
516
|
+
end_token = end_quote
|
517
|
+
break
|
518
|
+
elsif :on_tstring_content == tk[:kind] then
|
519
|
+
string += tk[:text]
|
520
|
+
elsif :on_words_sep == tk[:kind] or :on_tstring_end == tk[:kind] then
|
521
|
+
if end_quote == tk[:text].strip then
|
522
|
+
end_token = tk[:text]
|
523
|
+
break
|
524
|
+
else
|
525
|
+
string += tk[:text]
|
526
|
+
end
|
527
|
+
else
|
528
|
+
string += tk[:text]
|
529
|
+
end
|
530
|
+
end
|
531
|
+
text = "#{start_token}#{string}#{end_token}"
|
532
|
+
{
|
533
|
+
:line_no => line_no,
|
534
|
+
:char_no => char_no,
|
535
|
+
:kind => :on_dstring,
|
536
|
+
:text => text,
|
537
|
+
:state => state
|
538
|
+
}
|
539
|
+
end
|
540
|
+
|
541
|
+
private def get_op_tk(tk)
|
542
|
+
redefinable_operators = %w[! != !~ % & * ** + +@ - -@ / < << <= <=> == === =~ > >= >> [] []= ^ ` | ~]
|
543
|
+
if redefinable_operators.include?(tk[:text]) and EXPR_ARG == tk[:state] then
|
544
|
+
@inner_lex.lex_state = EXPR_ARG
|
545
|
+
tk[:kind] = :on_ident
|
546
|
+
tk[:state] = @inner_lex.lex_state
|
547
|
+
elsif tk[:text] =~ /^[-+]$/ then
|
548
|
+
tk_ahead = get_squashed_tk
|
549
|
+
case tk_ahead[:kind]
|
550
|
+
when :on_int, :on_float, :on_rational, :on_imaginary then
|
551
|
+
tk[:text] += tk_ahead[:text]
|
552
|
+
tk[:kind] = tk_ahead[:kind]
|
553
|
+
tk[:state] = tk_ahead[:state]
|
554
|
+
else
|
555
|
+
@buf.unshift tk_ahead
|
556
|
+
end
|
557
|
+
end
|
558
|
+
tk
|
559
|
+
end
|
560
|
+
|
561
|
+
def initialize(code)
|
562
|
+
@buf = []
|
563
|
+
@heredoc_queue = []
|
564
|
+
@inner_lex = InnerStateLex.new(code)
|
565
|
+
@inner_lex_enumerator = Enumerator.new do |y|
|
566
|
+
@inner_lex.each do |tk|
|
567
|
+
y << tk
|
568
|
+
end
|
569
|
+
end
|
570
|
+
end
|
571
|
+
|
572
|
+
def self.parse(code)
|
573
|
+
lex = self.new(code)
|
574
|
+
tokens = []
|
575
|
+
begin
|
576
|
+
while tk = lex.get_squashed_tk
|
577
|
+
tokens.push tk
|
578
|
+
end
|
579
|
+
rescue StopIteration
|
580
|
+
end
|
581
|
+
tokens
|
582
|
+
end
|
583
|
+
|
584
|
+
def self.end?(token)
|
585
|
+
(token[:state] & EXPR_END)
|
586
|
+
end
|
587
|
+
end
|