rdoc 5.0.0 → 6.3.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of rdoc might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/CONTRIBUTING.rdoc +4 -4
- data/Gemfile +9 -0
- data/History.rdoc +12 -2
- data/README.rdoc +5 -6
- data/Rakefile +35 -65
- data/lib/rdoc/alias.rb +1 -1
- data/lib/rdoc/anon_class.rb +1 -1
- data/lib/rdoc/any_method.rb +59 -15
- data/lib/rdoc/attr.rb +1 -1
- data/lib/rdoc/class_module.rb +5 -3
- data/lib/rdoc/code_object.rb +2 -9
- data/lib/rdoc/code_objects.rb +1 -1
- data/lib/rdoc/comment.rb +32 -11
- data/lib/rdoc/constant.rb +3 -3
- data/lib/rdoc/context/section.rb +1 -14
- data/lib/rdoc/context.rb +74 -21
- data/lib/rdoc/cross_reference.rb +33 -15
- data/lib/rdoc/encoding.rb +58 -30
- data/lib/rdoc/erb_partial.rb +2 -2
- data/lib/rdoc/erbio.rb +8 -4
- data/lib/rdoc/extend.rb +1 -1
- data/lib/rdoc/generator/darkfish.rb +60 -29
- data/lib/rdoc/generator/json_index.rb +7 -4
- data/lib/rdoc/generator/markup.rb +3 -13
- data/lib/rdoc/generator/pot/message_extractor.rb +1 -1
- data/lib/rdoc/generator/pot/po.rb +3 -3
- data/lib/rdoc/generator/pot/po_entry.rb +11 -11
- data/lib/rdoc/generator/pot.rb +4 -4
- data/lib/rdoc/generator/ri.rb +1 -1
- data/lib/rdoc/generator/template/darkfish/_footer.rhtml +2 -2
- data/lib/rdoc/generator/template/darkfish/_head.rhtml +9 -7
- data/lib/rdoc/generator/template/darkfish/_sidebar_VCS_info.rhtml +2 -2
- data/lib/rdoc/generator/template/darkfish/_sidebar_classes.rhtml +2 -2
- data/lib/rdoc/generator/template/darkfish/_sidebar_extends.rhtml +7 -7
- data/lib/rdoc/generator/template/darkfish/_sidebar_in_files.rhtml +2 -2
- data/lib/rdoc/generator/template/darkfish/_sidebar_includes.rhtml +7 -7
- data/lib/rdoc/generator/template/darkfish/_sidebar_installed.rhtml +6 -6
- data/lib/rdoc/generator/template/darkfish/_sidebar_methods.rhtml +5 -5
- data/lib/rdoc/generator/template/darkfish/_sidebar_pages.rhtml +5 -5
- data/lib/rdoc/generator/template/darkfish/_sidebar_parent.rhtml +5 -5
- data/lib/rdoc/generator/template/darkfish/_sidebar_sections.rhtml +4 -4
- data/lib/rdoc/generator/template/darkfish/_sidebar_table_of_contents.rhtml +4 -4
- data/lib/rdoc/generator/template/darkfish/class.rhtml +45 -47
- data/lib/rdoc/generator/template/darkfish/css/rdoc.css +55 -6
- data/lib/rdoc/generator/template/darkfish/index.rhtml +3 -4
- data/lib/rdoc/generator/template/darkfish/js/darkfish.js +22 -99
- data/lib/rdoc/generator/template/darkfish/js/search.js +32 -31
- data/lib/rdoc/generator/template/darkfish/servlet_root.rhtml +15 -16
- data/lib/rdoc/generator/template/darkfish/table_of_contents.rhtml +16 -16
- data/lib/rdoc/generator/template/json_index/js/navigation.js +4 -41
- data/lib/rdoc/generator/template/json_index/js/searcher.js +6 -6
- data/lib/rdoc/generator.rb +1 -1
- data/lib/rdoc/ghost_method.rb +1 -1
- data/lib/rdoc/i18n/locale.rb +2 -2
- data/lib/rdoc/i18n/text.rb +5 -5
- data/lib/rdoc/i18n.rb +3 -3
- data/lib/rdoc/include.rb +1 -1
- data/lib/rdoc/known_classes.rb +1 -1
- data/lib/rdoc/markdown/entities.rb +1 -1
- data/lib/rdoc/markdown/literals.kpeg +1 -0
- data/lib/rdoc/markdown/literals.rb +19 -7
- data/lib/rdoc/markdown.kpeg +92 -44
- data/lib/rdoc/markdown.rb +1171 -610
- data/lib/rdoc/markup/attr_changer.rb +1 -1
- data/lib/rdoc/markup/attr_span.rb +9 -3
- data/lib/rdoc/markup/attribute_manager.rb +115 -50
- data/lib/rdoc/markup/attributes.rb +7 -7
- data/lib/rdoc/markup/blank_line.rb +1 -1
- data/lib/rdoc/markup/block_quote.rb +1 -1
- data/lib/rdoc/markup/document.rb +1 -1
- data/lib/rdoc/markup/formatter.rb +25 -24
- data/lib/rdoc/markup/hard_break.rb +1 -1
- data/lib/rdoc/markup/heading.rb +4 -4
- data/lib/rdoc/markup/include.rb +1 -1
- data/lib/rdoc/markup/indented_paragraph.rb +1 -1
- data/lib/rdoc/markup/list.rb +1 -1
- data/lib/rdoc/markup/list_item.rb +1 -1
- data/lib/rdoc/markup/paragraph.rb +1 -1
- data/lib/rdoc/markup/parser.rb +79 -47
- data/lib/rdoc/markup/pre_process.rb +11 -6
- data/lib/rdoc/markup/raw.rb +1 -1
- data/lib/rdoc/markup/regexp_handling.rb +41 -0
- data/lib/rdoc/markup/rule.rb +1 -1
- data/lib/rdoc/markup/to_ansi.rb +1 -1
- data/lib/rdoc/markup/to_bs.rb +4 -4
- data/lib/rdoc/markup/to_html.rb +71 -26
- data/lib/rdoc/markup/to_html_crossref.rb +41 -26
- data/lib/rdoc/markup/to_html_snippet.rb +10 -10
- data/lib/rdoc/markup/to_joined_paragraph.rb +7 -32
- data/lib/rdoc/markup/to_label.rb +10 -10
- data/lib/rdoc/markup/to_markdown.rb +9 -9
- data/lib/rdoc/markup/to_rdoc.rb +35 -7
- data/lib/rdoc/markup/to_table_of_contents.rb +2 -1
- data/lib/rdoc/markup/to_test.rb +1 -1
- data/lib/rdoc/markup/to_tt_only.rb +3 -3
- data/lib/rdoc/markup/verbatim.rb +1 -1
- data/lib/rdoc/markup.rb +14 -17
- data/lib/rdoc/meta_method.rb +1 -1
- data/lib/rdoc/method_attr.rb +2 -2
- data/lib/rdoc/mixin.rb +1 -1
- data/lib/rdoc/normal_class.rb +3 -3
- data/lib/rdoc/normal_module.rb +1 -1
- data/lib/rdoc/options.rb +79 -21
- data/lib/rdoc/parser/c.rb +147 -194
- data/lib/rdoc/parser/changelog.rb +150 -19
- data/lib/rdoc/parser/markdown.rb +1 -1
- data/lib/rdoc/parser/rd.rb +1 -1
- data/lib/rdoc/parser/ripper_state_lex.rb +590 -0
- data/lib/rdoc/parser/ruby.rb +634 -465
- data/lib/rdoc/parser/ruby_tools.rb +33 -34
- data/lib/rdoc/parser/simple.rb +3 -3
- data/lib/rdoc/parser/text.rb +1 -1
- data/lib/rdoc/parser.rb +12 -35
- data/lib/rdoc/rd/block_parser.rb +109 -108
- data/lib/rdoc/rd/block_parser.ry +3 -3
- data/lib/rdoc/rd/inline.rb +5 -5
- data/lib/rdoc/rd/inline_parser.rb +186 -185
- data/lib/rdoc/rd/inline_parser.ry +1 -1
- data/lib/rdoc/rd.rb +1 -1
- data/lib/rdoc/rdoc.rb +54 -41
- data/lib/rdoc/require.rb +1 -1
- data/lib/rdoc/ri/driver.rb +132 -42
- data/lib/rdoc/ri/formatter.rb +1 -1
- data/lib/rdoc/ri/paths.rb +4 -18
- data/lib/rdoc/ri/store.rb +1 -1
- data/lib/rdoc/ri/task.rb +2 -2
- data/lib/rdoc/ri.rb +1 -1
- data/lib/rdoc/rubygems_hook.rb +3 -3
- data/lib/rdoc/servlet.rb +21 -12
- data/lib/rdoc/single_class.rb +1 -1
- data/lib/rdoc/stats/normal.rb +24 -18
- data/lib/rdoc/stats/quiet.rb +1 -1
- data/lib/rdoc/stats/verbose.rb +1 -1
- data/lib/rdoc/stats.rb +1 -1
- data/lib/rdoc/store.rb +38 -27
- data/lib/rdoc/task.rb +2 -2
- data/lib/rdoc/text.rb +16 -21
- data/lib/rdoc/token_stream.rb +56 -33
- data/lib/rdoc/tom_doc.rb +17 -12
- data/lib/rdoc/top_level.rb +9 -3
- data/lib/rdoc/version.rb +8 -0
- data/lib/rdoc.rb +24 -10
- data/man/ri.1 +247 -0
- data/rdoc.gemspec +206 -15
- metadata +15 -64
- data/.document +0 -5
- data/.gitignore +0 -13
- data/.travis.yml +0 -23
- data/lib/gauntlet_rdoc.rb +0 -82
- data/lib/rdoc/generator/template/darkfish/js/jquery.js +0 -4
- data/lib/rdoc/markup/formatter_test_case.rb +0 -764
- data/lib/rdoc/markup/inline.rb +0 -2
- data/lib/rdoc/markup/special.rb +0 -41
- data/lib/rdoc/markup/text_formatter_test_case.rb +0 -115
- data/lib/rdoc/ruby_lex.rb +0 -1367
- data/lib/rdoc/ruby_token.rb +0 -461
- data/lib/rdoc/test_case.rb +0 -204
@@ -1,5 +1,4 @@
|
|
1
|
-
# frozen_string_literal:
|
2
|
-
require 'time'
|
1
|
+
# frozen_string_literal: true
|
3
2
|
|
4
3
|
##
|
5
4
|
# A ChangeLog file parser.
|
@@ -29,13 +28,13 @@ class RDoc::Parser::ChangeLog < RDoc::Parser
|
|
29
28
|
|
30
29
|
if last =~ /\)\s*\z/ and continuation =~ /\A\(/ then
|
31
30
|
last.sub!(/\)\s*\z/, ',')
|
32
|
-
continuation.sub
|
31
|
+
continuation = continuation.sub(/\A\(/, '')
|
33
32
|
end
|
34
33
|
|
35
34
|
if last =~ /\s\z/ then
|
36
35
|
last << continuation
|
37
36
|
else
|
38
|
-
last << ' '
|
37
|
+
last << ' ' + continuation
|
39
38
|
end
|
40
39
|
end
|
41
40
|
|
@@ -106,14 +105,32 @@ class RDoc::Parser::ChangeLog < RDoc::Parser
|
|
106
105
|
entries.group_by do |title, _|
|
107
106
|
begin
|
108
107
|
time = @time_cache[title]
|
109
|
-
(time ||
|
108
|
+
(time || parse_date(title)).strftime '%Y-%m-%d'
|
110
109
|
rescue NoMethodError, ArgumentError
|
111
110
|
time, = title.split ' ', 2
|
112
|
-
|
111
|
+
parse_date(time).strftime '%Y-%m-%d'
|
113
112
|
end
|
114
113
|
end
|
115
114
|
end
|
116
115
|
|
116
|
+
##
|
117
|
+
# Parse date in ISO-8601, RFC-2822, or default of Git
|
118
|
+
|
119
|
+
def parse_date(date)
|
120
|
+
case date
|
121
|
+
when /\A\s*(\d+)-(\d+)-(\d+)(?:[ T](\d+):(\d+):(\d+) *([-+]\d\d):?(\d\d))?\b/
|
122
|
+
Time.new($1, $2, $3, $4, $5, $6, ("#{$7}:#{$8}" if $7))
|
123
|
+
when /\A\s*\w{3}, +(\d+) (\w{3}) (\d+) (\d+):(\d+):(\d+) *(?:([-+]\d\d):?(\d\d))\b/
|
124
|
+
Time.new($3, $2, $1, $4, $5, $6, ("#{$7}:#{$8}" if $7))
|
125
|
+
when /\A\s*\w{3} (\w{3}) +(\d+) (\d+) (\d+):(\d+):(\d+) *(?:([-+]\d\d):?(\d\d))\b/
|
126
|
+
Time.new($3, $1, $2, $4, $5, $6, ("#{$7}:#{$8}" if $7))
|
127
|
+
when /\A\s*\w{3} (\w{3}) +(\d+) (\d+):(\d+):(\d+) (\d+)\b/
|
128
|
+
Time.new($6, $1, $2, $3, $4, $5)
|
129
|
+
else
|
130
|
+
raise ArgumentError, "bad date: #{date}"
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
117
134
|
##
|
118
135
|
# Parses the entries in the ChangeLog.
|
119
136
|
#
|
@@ -131,6 +148,13 @@ class RDoc::Parser::ChangeLog < RDoc::Parser
|
|
131
148
|
|
132
149
|
def parse_entries
|
133
150
|
@time_cache ||= {}
|
151
|
+
|
152
|
+
if /\A((?:.*\n){,3})commit\s/ =~ @content
|
153
|
+
class << self; prepend Git; end
|
154
|
+
parse_info($1)
|
155
|
+
return parse_entries
|
156
|
+
end
|
157
|
+
|
134
158
|
entries = []
|
135
159
|
entry_name = nil
|
136
160
|
entry_body = []
|
@@ -145,29 +169,20 @@ class RDoc::Parser::ChangeLog < RDoc::Parser
|
|
145
169
|
entry_name = $&
|
146
170
|
|
147
171
|
begin
|
148
|
-
time =
|
172
|
+
time = parse_date entry_name
|
149
173
|
@time_cache[entry_name] = time
|
150
|
-
# HACK Ruby 1.8 does not raise ArgumentError for Time.parse "Other"
|
151
|
-
entry_name = nil unless entry_name =~ /#{time.year}/
|
152
|
-
rescue NoMethodError
|
153
|
-
# HACK Ruby 2.1.2 and earlier raises NoMethodError if time part is absent
|
154
|
-
entry_name.split ' ', 2
|
155
174
|
rescue ArgumentError
|
156
|
-
|
157
|
-
Time.parse(entry_name.split(' ', 2)[0]) rescue entry_name = nil
|
158
|
-
else
|
159
|
-
entry_name = nil
|
160
|
-
end
|
175
|
+
entry_name = nil
|
161
176
|
end
|
162
177
|
|
163
178
|
entry_body = []
|
164
179
|
when /^(\t| {8})?\*\s*(.*)/ then # "\t* file.c (func): ..."
|
165
|
-
entry_body << $2
|
180
|
+
entry_body << $2.dup
|
166
181
|
when /^(\t| {8})?\s*(\(.*)/ then # "\t(func): ..."
|
167
182
|
entry = $2
|
168
183
|
|
169
184
|
if entry_body.last =~ /:/ then
|
170
|
-
entry_body << entry
|
185
|
+
entry_body << entry.dup
|
171
186
|
else
|
172
187
|
continue_entry_body entry_body, entry
|
173
188
|
end
|
@@ -190,6 +205,7 @@ class RDoc::Parser::ChangeLog < RDoc::Parser
|
|
190
205
|
|
191
206
|
def scan
|
192
207
|
@time_cache = {}
|
208
|
+
|
193
209
|
entries = parse_entries
|
194
210
|
grouped_entries = group_entries entries
|
195
211
|
|
@@ -200,5 +216,120 @@ class RDoc::Parser::ChangeLog < RDoc::Parser
|
|
200
216
|
@top_level
|
201
217
|
end
|
202
218
|
|
219
|
+
module Git
|
220
|
+
def parse_info(info)
|
221
|
+
/^\s*base-url\s*=\s*(.*\S)/ =~ info
|
222
|
+
@base_url = $1
|
223
|
+
end
|
224
|
+
|
225
|
+
def parse_entries
|
226
|
+
entries = []
|
227
|
+
|
228
|
+
@content.scan(/^commit\s+(\h{20})\h*\n((?:.+\n)*)\n((?: {4}.*\n+)*)/) do
|
229
|
+
entry_name, header, entry_body = $1, $2, $3.gsub(/^ {4}/, '')
|
230
|
+
# header = header.scan(/^ *(\S+?): +(.*)/).to_h
|
231
|
+
# date = header["CommitDate"] || header["Date"]
|
232
|
+
date = header[/^ *(?:Author)?Date: +(.*)/, 1]
|
233
|
+
author = header[/^ *Author: +(.*)/, 1]
|
234
|
+
begin
|
235
|
+
time = parse_date(header[/^ *CommitDate: +(.*)/, 1] || date)
|
236
|
+
@time_cache[entry_name] = time
|
237
|
+
author.sub!(/\s*<(.*)>/, '')
|
238
|
+
email = $1
|
239
|
+
entries << [entry_name, [author, email, date, entry_body]]
|
240
|
+
rescue ArgumentError
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
entries
|
245
|
+
end
|
246
|
+
|
247
|
+
def create_entries entries
|
248
|
+
# git log entries have no strictly itemized style like the old
|
249
|
+
# style, just assume Markdown.
|
250
|
+
entries.map do |commit, entry|
|
251
|
+
LogEntry.new(@base_url, commit, *entry)
|
252
|
+
end
|
253
|
+
end
|
254
|
+
|
255
|
+
LogEntry = Struct.new(:base, :commit, :author, :email, :date, :contents) do
|
256
|
+
HEADING_LEVEL = 3
|
257
|
+
|
258
|
+
def initialize(base, commit, author, email, date, contents)
|
259
|
+
case contents
|
260
|
+
when String
|
261
|
+
contents = RDoc::Markdown.parse(contents).parts.each do |body|
|
262
|
+
case body
|
263
|
+
when RDoc::Markup::Heading
|
264
|
+
body.level += HEADING_LEVEL + 1
|
265
|
+
end
|
266
|
+
end
|
267
|
+
case first = contents[0]
|
268
|
+
when RDoc::Markup::Paragraph
|
269
|
+
contents[0] = RDoc::Markup::Heading.new(HEADING_LEVEL + 1, first.text)
|
270
|
+
end
|
271
|
+
end
|
272
|
+
super
|
273
|
+
end
|
274
|
+
|
275
|
+
def level
|
276
|
+
HEADING_LEVEL
|
277
|
+
end
|
278
|
+
|
279
|
+
def aref
|
280
|
+
"label-#{commit}"
|
281
|
+
end
|
282
|
+
|
283
|
+
def label context = nil
|
284
|
+
aref
|
285
|
+
end
|
286
|
+
|
287
|
+
def text
|
288
|
+
case base
|
289
|
+
when nil
|
290
|
+
"#{date}"
|
291
|
+
when /%s/
|
292
|
+
"{#{date}}[#{base % commit}]"
|
293
|
+
else
|
294
|
+
"{#{date}}[#{base}#{commit}]"
|
295
|
+
end + " {#{author}}[mailto:#{email}]"
|
296
|
+
end
|
297
|
+
|
298
|
+
def accept visitor
|
299
|
+
visitor.accept_heading self
|
300
|
+
begin
|
301
|
+
if visitor.respond_to?(:code_object=)
|
302
|
+
code_object = visitor.code_object
|
303
|
+
visitor.code_object = self
|
304
|
+
end
|
305
|
+
contents.each do |body|
|
306
|
+
body.accept visitor
|
307
|
+
end
|
308
|
+
ensure
|
309
|
+
if visitor.respond_to?(:code_object)
|
310
|
+
visitor.code_object = code_object
|
311
|
+
end
|
312
|
+
end
|
313
|
+
end
|
314
|
+
|
315
|
+
def pretty_print q # :nodoc:
|
316
|
+
q.group(2, '[log_entry: ', ']') do
|
317
|
+
q.text commit
|
318
|
+
q.text ','
|
319
|
+
q.breakable
|
320
|
+
q.group(2, '[date: ', ']') { q.text date }
|
321
|
+
q.text ','
|
322
|
+
q.breakable
|
323
|
+
q.group(2, '[author: ', ']') { q.text author }
|
324
|
+
q.text ','
|
325
|
+
q.breakable
|
326
|
+
q.group(2, '[email: ', ']') { q.text email }
|
327
|
+
q.text ','
|
328
|
+
q.breakable
|
329
|
+
q.pp contents
|
330
|
+
end
|
331
|
+
end
|
332
|
+
end
|
333
|
+
end
|
203
334
|
end
|
204
335
|
|
data/lib/rdoc/parser/markdown.rb
CHANGED
data/lib/rdoc/parser/rd.rb
CHANGED
@@ -0,0 +1,590 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'ripper'
|
3
|
+
|
4
|
+
class RDoc::Parser::RipperStateLex
|
5
|
+
# TODO: Remove this constants after Ruby 2.4 EOL
|
6
|
+
RIPPER_HAS_LEX_STATE = Ripper::Filter.method_defined?(:state)
|
7
|
+
|
8
|
+
Token = Struct.new(:line_no, :char_no, :kind, :text, :state)
|
9
|
+
|
10
|
+
EXPR_NONE = 0
|
11
|
+
EXPR_BEG = 1
|
12
|
+
EXPR_END = 2
|
13
|
+
EXPR_ENDARG = 4
|
14
|
+
EXPR_ENDFN = 8
|
15
|
+
EXPR_ARG = 16
|
16
|
+
EXPR_CMDARG = 32
|
17
|
+
EXPR_MID = 64
|
18
|
+
EXPR_FNAME = 128
|
19
|
+
EXPR_DOT = 256
|
20
|
+
EXPR_CLASS = 512
|
21
|
+
EXPR_LABEL = 1024
|
22
|
+
EXPR_LABELED = 2048
|
23
|
+
EXPR_FITEM = 4096
|
24
|
+
EXPR_VALUE = EXPR_BEG
|
25
|
+
EXPR_BEG_ANY = (EXPR_BEG | EXPR_MID | EXPR_CLASS)
|
26
|
+
EXPR_ARG_ANY = (EXPR_ARG | EXPR_CMDARG)
|
27
|
+
EXPR_END_ANY = (EXPR_END | EXPR_ENDARG | EXPR_ENDFN)
|
28
|
+
|
29
|
+
class InnerStateLex < Ripper::Filter
|
30
|
+
attr_accessor :lex_state
|
31
|
+
|
32
|
+
def initialize(code)
|
33
|
+
@lex_state = EXPR_BEG
|
34
|
+
@in_fname = false
|
35
|
+
@continue = false
|
36
|
+
reset
|
37
|
+
super(code)
|
38
|
+
end
|
39
|
+
|
40
|
+
def reset
|
41
|
+
@command_start = false
|
42
|
+
@cmd_state = @command_start
|
43
|
+
end
|
44
|
+
|
45
|
+
def on_nl(tok, data)
|
46
|
+
case @lex_state
|
47
|
+
when EXPR_FNAME, EXPR_DOT
|
48
|
+
@continue = true
|
49
|
+
else
|
50
|
+
@continue = false
|
51
|
+
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
|
52
|
+
end
|
53
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
54
|
+
end
|
55
|
+
|
56
|
+
def on_ignored_nl(tok, data)
|
57
|
+
case @lex_state
|
58
|
+
when EXPR_FNAME, EXPR_DOT
|
59
|
+
@continue = true
|
60
|
+
else
|
61
|
+
@continue = false
|
62
|
+
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
|
63
|
+
end
|
64
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
65
|
+
end
|
66
|
+
|
67
|
+
def on_op(tok, data)
|
68
|
+
case tok
|
69
|
+
when '&', '|', '!', '!=', '!~'
|
70
|
+
case @lex_state
|
71
|
+
when EXPR_FNAME, EXPR_DOT
|
72
|
+
@lex_state = EXPR_ARG
|
73
|
+
else
|
74
|
+
@lex_state = EXPR_BEG
|
75
|
+
end
|
76
|
+
when '<<'
|
77
|
+
# TODO next token?
|
78
|
+
case @lex_state
|
79
|
+
when EXPR_FNAME, EXPR_DOT
|
80
|
+
@lex_state = EXPR_ARG
|
81
|
+
else
|
82
|
+
@lex_state = EXPR_BEG
|
83
|
+
end
|
84
|
+
when '?'
|
85
|
+
@lex_state = EXPR_BEG
|
86
|
+
when '&&', '||', '+=', '-=', '*=', '**=',
|
87
|
+
'&=', '|=', '^=', '<<=', '>>=', '||=', '&&='
|
88
|
+
@lex_state = EXPR_BEG
|
89
|
+
when '::'
|
90
|
+
case @lex_state
|
91
|
+
when EXPR_ARG, EXPR_CMDARG
|
92
|
+
@lex_state = EXPR_DOT
|
93
|
+
when EXPR_FNAME, EXPR_DOT
|
94
|
+
@lex_state = EXPR_ARG
|
95
|
+
else
|
96
|
+
@lex_state = EXPR_BEG
|
97
|
+
end
|
98
|
+
else
|
99
|
+
case @lex_state
|
100
|
+
when EXPR_FNAME, EXPR_DOT
|
101
|
+
@lex_state = EXPR_ARG
|
102
|
+
else
|
103
|
+
@lex_state = EXPR_BEG
|
104
|
+
end
|
105
|
+
end
|
106
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
107
|
+
end
|
108
|
+
|
109
|
+
def on_kw(tok, data)
|
110
|
+
case tok
|
111
|
+
when 'class'
|
112
|
+
@lex_state = EXPR_CLASS
|
113
|
+
@in_fname = true
|
114
|
+
when 'def'
|
115
|
+
@lex_state = EXPR_FNAME
|
116
|
+
@continue = true
|
117
|
+
@in_fname = true
|
118
|
+
when 'if', 'unless', 'while', 'until'
|
119
|
+
if ((EXPR_MID | EXPR_END | EXPR_ENDARG | EXPR_ENDFN | EXPR_ARG | EXPR_CMDARG) & @lex_state) != 0 # postfix if
|
120
|
+
@lex_state = EXPR_BEG | EXPR_LABEL
|
121
|
+
else
|
122
|
+
@lex_state = EXPR_BEG
|
123
|
+
end
|
124
|
+
when 'begin', 'case', 'when'
|
125
|
+
@lex_state = EXPR_BEG
|
126
|
+
when 'return', 'break'
|
127
|
+
@lex_state = EXPR_MID
|
128
|
+
else
|
129
|
+
if @lex_state == EXPR_FNAME
|
130
|
+
@lex_state = EXPR_END
|
131
|
+
else
|
132
|
+
@lex_state = EXPR_END
|
133
|
+
end
|
134
|
+
end
|
135
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
136
|
+
end
|
137
|
+
|
138
|
+
def on_tstring_beg(tok, data)
|
139
|
+
@lex_state = EXPR_BEG
|
140
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
141
|
+
end
|
142
|
+
|
143
|
+
def on_tstring_end(tok, data)
|
144
|
+
@lex_state = EXPR_END | EXPR_ENDARG
|
145
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
146
|
+
end
|
147
|
+
|
148
|
+
def on_CHAR(tok, data)
|
149
|
+
@lex_state = EXPR_END
|
150
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
151
|
+
end
|
152
|
+
|
153
|
+
def on_period(tok, data)
|
154
|
+
@lex_state = EXPR_DOT
|
155
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
156
|
+
end
|
157
|
+
|
158
|
+
def on_int(tok, data)
|
159
|
+
@lex_state = EXPR_END | EXPR_ENDARG
|
160
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
161
|
+
end
|
162
|
+
|
163
|
+
def on_float(tok, data)
|
164
|
+
@lex_state = EXPR_END | EXPR_ENDARG
|
165
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
166
|
+
end
|
167
|
+
|
168
|
+
def on_rational(tok, data)
|
169
|
+
@lex_state = EXPR_END | EXPR_ENDARG
|
170
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
171
|
+
end
|
172
|
+
|
173
|
+
def on_imaginary(tok, data)
|
174
|
+
@lex_state = EXPR_END | EXPR_ENDARG
|
175
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
176
|
+
end
|
177
|
+
|
178
|
+
def on_symbeg(tok, data)
|
179
|
+
@lex_state = EXPR_FNAME
|
180
|
+
@continue = true
|
181
|
+
@in_fname = true
|
182
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
183
|
+
end
|
184
|
+
|
185
|
+
private def on_variables(event, tok, data)
|
186
|
+
if @in_fname
|
187
|
+
@lex_state = EXPR_ENDFN
|
188
|
+
@in_fname = false
|
189
|
+
@continue = false
|
190
|
+
elsif @continue
|
191
|
+
case @lex_state
|
192
|
+
when EXPR_DOT
|
193
|
+
@lex_state = EXPR_ARG
|
194
|
+
else
|
195
|
+
@lex_state = EXPR_ENDFN
|
196
|
+
@continue = false
|
197
|
+
end
|
198
|
+
else
|
199
|
+
@lex_state = EXPR_CMDARG
|
200
|
+
end
|
201
|
+
data << Token.new(lineno, column, event, tok, @lex_state)
|
202
|
+
end
|
203
|
+
|
204
|
+
def on_ident(tok, data)
|
205
|
+
on_variables(__method__, tok, data)
|
206
|
+
end
|
207
|
+
|
208
|
+
def on_ivar(tok, data)
|
209
|
+
@lex_state = EXPR_END
|
210
|
+
on_variables(__method__, tok, data)
|
211
|
+
end
|
212
|
+
|
213
|
+
def on_cvar(tok, data)
|
214
|
+
@lex_state = EXPR_END
|
215
|
+
on_variables(__method__, tok, data)
|
216
|
+
end
|
217
|
+
|
218
|
+
def on_gvar(tok, data)
|
219
|
+
@lex_state = EXPR_END
|
220
|
+
on_variables(__method__, tok, data)
|
221
|
+
end
|
222
|
+
|
223
|
+
def on_backref(tok, data)
|
224
|
+
@lex_state = EXPR_END
|
225
|
+
on_variables(__method__, tok, data)
|
226
|
+
end
|
227
|
+
|
228
|
+
def on_lparen(tok, data)
|
229
|
+
@lex_state = EXPR_LABEL | EXPR_BEG
|
230
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
231
|
+
end
|
232
|
+
|
233
|
+
def on_rparen(tok, data)
|
234
|
+
@lex_state = EXPR_ENDFN
|
235
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
236
|
+
end
|
237
|
+
|
238
|
+
def on_lbrace(tok, data)
|
239
|
+
@lex_state = EXPR_LABEL | EXPR_BEG
|
240
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
241
|
+
end
|
242
|
+
|
243
|
+
def on_rbrace(tok, data)
|
244
|
+
@lex_state = EXPR_ENDARG
|
245
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
246
|
+
end
|
247
|
+
|
248
|
+
def on_lbracket(tok, data)
|
249
|
+
@lex_state = EXPR_LABEL | EXPR_BEG
|
250
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
251
|
+
end
|
252
|
+
|
253
|
+
def on_rbracket(tok, data)
|
254
|
+
@lex_state = EXPR_ENDARG
|
255
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
256
|
+
end
|
257
|
+
|
258
|
+
def on_const(tok, data)
|
259
|
+
case @lex_state
|
260
|
+
when EXPR_FNAME
|
261
|
+
@lex_state = EXPR_ENDFN
|
262
|
+
when EXPR_CLASS, EXPR_CMDARG, EXPR_MID
|
263
|
+
@lex_state = EXPR_ARG
|
264
|
+
else
|
265
|
+
@lex_state = EXPR_CMDARG
|
266
|
+
end
|
267
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
268
|
+
end
|
269
|
+
|
270
|
+
def on_sp(tok, data)
|
271
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
272
|
+
end
|
273
|
+
|
274
|
+
def on_comma(tok, data)
|
275
|
+
@lex_state = EXPR_BEG | EXPR_LABEL if (EXPR_ARG_ANY & @lex_state) != 0
|
276
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
277
|
+
end
|
278
|
+
|
279
|
+
def on_comment(tok, data)
|
280
|
+
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
|
281
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
282
|
+
end
|
283
|
+
|
284
|
+
def on_ignored_sp(tok, data)
|
285
|
+
@lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
|
286
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
287
|
+
end
|
288
|
+
|
289
|
+
def on_heredoc_beg(tok, data)
|
290
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
291
|
+
@lex_state = EXPR_END
|
292
|
+
data
|
293
|
+
end
|
294
|
+
|
295
|
+
def on_heredoc_end(tok, data)
|
296
|
+
data << Token.new(lineno, column, __method__, tok, @lex_state)
|
297
|
+
@lex_state = EXPR_BEG
|
298
|
+
data
|
299
|
+
end
|
300
|
+
|
301
|
+
def on_default(event, tok, data)
|
302
|
+
reset
|
303
|
+
data << Token.new(lineno, column, event, tok, @lex_state)
|
304
|
+
end
|
305
|
+
end unless RIPPER_HAS_LEX_STATE
|
306
|
+
|
307
|
+
class InnerStateLex < Ripper::Filter
|
308
|
+
def initialize(code)
|
309
|
+
super(code)
|
310
|
+
end
|
311
|
+
|
312
|
+
def on_default(event, tok, data)
|
313
|
+
data << Token.new(lineno, column, event, tok, state)
|
314
|
+
end
|
315
|
+
end if RIPPER_HAS_LEX_STATE
|
316
|
+
|
317
|
+
def get_squashed_tk
|
318
|
+
if @buf.empty?
|
319
|
+
tk = @tokens.shift
|
320
|
+
else
|
321
|
+
tk = @buf.shift
|
322
|
+
end
|
323
|
+
return nil if tk.nil?
|
324
|
+
case tk[:kind]
|
325
|
+
when :on_symbeg then
|
326
|
+
tk = get_symbol_tk(tk)
|
327
|
+
when :on_tstring_beg then
|
328
|
+
tk = get_string_tk(tk)
|
329
|
+
when :on_backtick then
|
330
|
+
if (tk[:state] & (EXPR_FNAME | EXPR_ENDFN)) != 0
|
331
|
+
@inner_lex.lex_state = EXPR_ARG unless RIPPER_HAS_LEX_STATE
|
332
|
+
tk[:kind] = :on_ident
|
333
|
+
tk[:state] = Ripper::Lexer.const_defined?(:State) ? Ripper::Lexer::State.new(EXPR_ARG) : EXPR_ARG
|
334
|
+
else
|
335
|
+
tk = get_string_tk(tk)
|
336
|
+
end
|
337
|
+
when :on_regexp_beg then
|
338
|
+
tk = get_regexp_tk(tk)
|
339
|
+
when :on_embdoc_beg then
|
340
|
+
tk = get_embdoc_tk(tk)
|
341
|
+
when :on_heredoc_beg then
|
342
|
+
@heredoc_queue << retrieve_heredoc_info(tk)
|
343
|
+
@inner_lex.lex_state = EXPR_END unless RIPPER_HAS_LEX_STATE
|
344
|
+
when :on_nl, :on_ignored_nl, :on_comment, :on_heredoc_end then
|
345
|
+
if !@heredoc_queue.empty?
|
346
|
+
get_heredoc_tk(*@heredoc_queue.shift)
|
347
|
+
elsif tk[:text].nil? # :on_ignored_nl sometimes gives nil
|
348
|
+
tk[:text] = ''
|
349
|
+
end
|
350
|
+
when :on_words_beg then
|
351
|
+
tk = get_words_tk(tk)
|
352
|
+
when :on_qwords_beg then
|
353
|
+
tk = get_words_tk(tk)
|
354
|
+
when :on_symbols_beg then
|
355
|
+
tk = get_words_tk(tk)
|
356
|
+
when :on_qsymbols_beg then
|
357
|
+
tk = get_words_tk(tk)
|
358
|
+
when :on_op then
|
359
|
+
if '&.' == tk[:text]
|
360
|
+
tk[:kind] = :on_period
|
361
|
+
else
|
362
|
+
tk = get_op_tk(tk)
|
363
|
+
end
|
364
|
+
end
|
365
|
+
tk
|
366
|
+
end
|
367
|
+
|
368
|
+
private def get_symbol_tk(tk)
|
369
|
+
is_symbol = true
|
370
|
+
symbol_tk = Token.new(tk.line_no, tk.char_no, :on_symbol)
|
371
|
+
if ":'" == tk[:text] or ':"' == tk[:text]
|
372
|
+
tk1 = get_string_tk(tk)
|
373
|
+
symbol_tk[:text] = tk1[:text]
|
374
|
+
symbol_tk[:state] = tk1[:state]
|
375
|
+
else
|
376
|
+
case (tk1 = get_squashed_tk)[:kind]
|
377
|
+
when :on_ident
|
378
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
379
|
+
symbol_tk[:state] = tk1[:state]
|
380
|
+
when :on_tstring_content
|
381
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
382
|
+
symbol_tk[:state] = get_squashed_tk[:state] # skip :on_tstring_end
|
383
|
+
when :on_tstring_end
|
384
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
385
|
+
symbol_tk[:state] = tk1[:state]
|
386
|
+
when :on_op
|
387
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
388
|
+
symbol_tk[:state] = tk1[:state]
|
389
|
+
when :on_ivar
|
390
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
391
|
+
symbol_tk[:state] = tk1[:state]
|
392
|
+
when :on_cvar
|
393
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
394
|
+
symbol_tk[:state] = tk1[:state]
|
395
|
+
when :on_gvar
|
396
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
397
|
+
symbol_tk[:state] = tk1[:state]
|
398
|
+
when :on_const
|
399
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
400
|
+
symbol_tk[:state] = tk1[:state]
|
401
|
+
when :on_kw
|
402
|
+
symbol_tk[:text] = ":#{tk1[:text]}"
|
403
|
+
symbol_tk[:state] = tk1[:state]
|
404
|
+
else
|
405
|
+
is_symbol = false
|
406
|
+
tk = tk1
|
407
|
+
end
|
408
|
+
end
|
409
|
+
if is_symbol
|
410
|
+
tk = symbol_tk
|
411
|
+
end
|
412
|
+
tk
|
413
|
+
end
|
414
|
+
|
415
|
+
private def get_string_tk(tk)
|
416
|
+
string = tk[:text]
|
417
|
+
state = nil
|
418
|
+
kind = :on_tstring
|
419
|
+
loop do
|
420
|
+
inner_str_tk = get_squashed_tk
|
421
|
+
if inner_str_tk.nil?
|
422
|
+
break
|
423
|
+
elsif :on_tstring_end == inner_str_tk[:kind]
|
424
|
+
string = string + inner_str_tk[:text]
|
425
|
+
state = inner_str_tk[:state]
|
426
|
+
break
|
427
|
+
elsif :on_label_end == inner_str_tk[:kind]
|
428
|
+
string = string + inner_str_tk[:text]
|
429
|
+
state = inner_str_tk[:state]
|
430
|
+
kind = :on_symbol
|
431
|
+
break
|
432
|
+
else
|
433
|
+
string = string + inner_str_tk[:text]
|
434
|
+
if :on_embexpr_beg == inner_str_tk[:kind] then
|
435
|
+
kind = :on_dstring if :on_tstring == kind
|
436
|
+
end
|
437
|
+
end
|
438
|
+
end
|
439
|
+
Token.new(tk.line_no, tk.char_no, kind, string, state)
|
440
|
+
end
|
441
|
+
|
442
|
+
private def get_regexp_tk(tk)
|
443
|
+
string = tk[:text]
|
444
|
+
state = nil
|
445
|
+
loop do
|
446
|
+
inner_str_tk = get_squashed_tk
|
447
|
+
if inner_str_tk.nil?
|
448
|
+
break
|
449
|
+
elsif :on_regexp_end == inner_str_tk[:kind]
|
450
|
+
string = string + inner_str_tk[:text]
|
451
|
+
state = inner_str_tk[:state]
|
452
|
+
break
|
453
|
+
else
|
454
|
+
string = string + inner_str_tk[:text]
|
455
|
+
end
|
456
|
+
end
|
457
|
+
Token.new(tk.line_no, tk.char_no, :on_regexp, string, state)
|
458
|
+
end
|
459
|
+
|
460
|
+
private def get_embdoc_tk(tk)
|
461
|
+
string = tk[:text]
|
462
|
+
until :on_embdoc_end == (embdoc_tk = get_squashed_tk)[:kind] do
|
463
|
+
string = string + embdoc_tk[:text]
|
464
|
+
end
|
465
|
+
string = string + embdoc_tk[:text]
|
466
|
+
Token.new(tk.line_no, tk.char_no, :on_embdoc, string, embdoc_tk.state)
|
467
|
+
end
|
468
|
+
|
469
|
+
private def get_heredoc_tk(heredoc_name, indent)
|
470
|
+
string = ''
|
471
|
+
start_tk = nil
|
472
|
+
prev_tk = nil
|
473
|
+
until heredoc_end?(heredoc_name, indent, tk = @tokens.shift) do
|
474
|
+
start_tk = tk unless start_tk
|
475
|
+
if (prev_tk.nil? or "\n" == prev_tk[:text][-1]) and 0 != tk[:char_no]
|
476
|
+
string = string + (' ' * tk[:char_no])
|
477
|
+
end
|
478
|
+
string = string + tk[:text]
|
479
|
+
prev_tk = tk
|
480
|
+
end
|
481
|
+
start_tk = tk unless start_tk
|
482
|
+
prev_tk = tk unless prev_tk
|
483
|
+
@buf.unshift tk # closing heredoc
|
484
|
+
heredoc_tk = Token.new(start_tk.line_no, start_tk.char_no, :on_heredoc, string, prev_tk.state)
|
485
|
+
@buf.unshift heredoc_tk
|
486
|
+
end
|
487
|
+
|
488
|
+
private def retrieve_heredoc_info(tk)
|
489
|
+
name = tk[:text].gsub(/\A<<[-~]?(['"`]?)(.+)\1\z/, '\2')
|
490
|
+
indent = tk[:text] =~ /\A<<[-~]/
|
491
|
+
[name, indent]
|
492
|
+
end
|
493
|
+
|
494
|
+
private def heredoc_end?(name, indent, tk)
|
495
|
+
result = false
|
496
|
+
if :on_heredoc_end == tk[:kind] then
|
497
|
+
tk_name = tk[:text].chomp
|
498
|
+
tk_name.lstrip! if indent
|
499
|
+
if name == tk_name
|
500
|
+
result = true
|
501
|
+
end
|
502
|
+
end
|
503
|
+
result
|
504
|
+
end
|
505
|
+
|
506
|
+
private def get_words_tk(tk)
|
507
|
+
string = ''
|
508
|
+
start_token = tk[:text]
|
509
|
+
start_quote = tk[:text].rstrip[-1]
|
510
|
+
line_no = tk[:line_no]
|
511
|
+
char_no = tk[:char_no]
|
512
|
+
state = tk[:state]
|
513
|
+
end_quote =
|
514
|
+
case start_quote
|
515
|
+
when ?( then ?)
|
516
|
+
when ?[ then ?]
|
517
|
+
when ?{ then ?}
|
518
|
+
when ?< then ?>
|
519
|
+
else start_quote
|
520
|
+
end
|
521
|
+
end_token = nil
|
522
|
+
loop do
|
523
|
+
tk = get_squashed_tk
|
524
|
+
if tk.nil?
|
525
|
+
end_token = end_quote
|
526
|
+
break
|
527
|
+
elsif :on_tstring_content == tk[:kind] then
|
528
|
+
string += tk[:text]
|
529
|
+
elsif :on_words_sep == tk[:kind] or :on_tstring_end == tk[:kind] then
|
530
|
+
if end_quote == tk[:text].strip then
|
531
|
+
end_token = tk[:text]
|
532
|
+
break
|
533
|
+
else
|
534
|
+
string += tk[:text]
|
535
|
+
end
|
536
|
+
else
|
537
|
+
string += tk[:text]
|
538
|
+
end
|
539
|
+
end
|
540
|
+
text = "#{start_token}#{string}#{end_token}"
|
541
|
+
Token.new(line_no, char_no, :on_dstring, text, state)
|
542
|
+
end
|
543
|
+
|
544
|
+
private def get_op_tk(tk)
|
545
|
+
redefinable_operators = %w[! != !~ % & * ** + +@ - -@ / < << <= <=> == === =~ > >= >> [] []= ^ ` | ~]
|
546
|
+
if redefinable_operators.include?(tk[:text]) and tk[:state] == EXPR_ARG then
|
547
|
+
@inner_lex.lex_state = EXPR_ARG unless RIPPER_HAS_LEX_STATE
|
548
|
+
tk[:state] = Ripper::Lexer.const_defined?(:State) ? Ripper::Lexer::State.new(EXPR_ARG) : EXPR_ARG
|
549
|
+
tk[:kind] = :on_ident
|
550
|
+
elsif tk[:text] =~ /^[-+]$/ then
|
551
|
+
tk_ahead = get_squashed_tk
|
552
|
+
case tk_ahead[:kind]
|
553
|
+
when :on_int, :on_float, :on_rational, :on_imaginary then
|
554
|
+
tk[:text] += tk_ahead[:text]
|
555
|
+
tk[:kind] = tk_ahead[:kind]
|
556
|
+
tk[:state] = tk_ahead[:state]
|
557
|
+
when :on_heredoc_beg, :on_tstring, :on_dstring # frozen/non-frozen string literal
|
558
|
+
tk[:text] += tk_ahead[:text]
|
559
|
+
tk[:kind] = tk_ahead[:kind]
|
560
|
+
tk[:state] = tk_ahead[:state]
|
561
|
+
else
|
562
|
+
@buf.unshift tk_ahead
|
563
|
+
end
|
564
|
+
end
|
565
|
+
tk
|
566
|
+
end
|
567
|
+
|
568
|
+
def initialize(code)
|
569
|
+
@buf = []
|
570
|
+
@heredoc_queue = []
|
571
|
+
@inner_lex = InnerStateLex.new(code)
|
572
|
+
@tokens = @inner_lex.parse([])
|
573
|
+
end
|
574
|
+
|
575
|
+
def self.parse(code)
|
576
|
+
lex = self.new(code)
|
577
|
+
tokens = []
|
578
|
+
begin
|
579
|
+
while tk = lex.get_squashed_tk
|
580
|
+
tokens.push tk
|
581
|
+
end
|
582
|
+
rescue StopIteration
|
583
|
+
end
|
584
|
+
tokens
|
585
|
+
end
|
586
|
+
|
587
|
+
def self.end?(token)
|
588
|
+
(token[:state] & EXPR_END)
|
589
|
+
end
|
590
|
+
end
|