coderay 0.5.0.121 → 0.7.1.147

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. data/FOLDERS +53 -0
  2. data/README +21 -13
  3. data/bin/coderay +79 -0
  4. data/demo/demo_cache.rb +12 -0
  5. data/demo/demo_html_list.rb +12 -0
  6. data/lib/coderay.rb +11 -2
  7. data/lib/coderay/duo.rb +29 -0
  8. data/lib/coderay/encoder.rb +4 -4
  9. data/lib/coderay/encoders/_map.rb +6 -6
  10. data/lib/coderay/encoders/count.rb +3 -3
  11. data/lib/coderay/encoders/debug.rb +38 -30
  12. data/lib/coderay/encoders/div.rb +4 -2
  13. data/lib/coderay/encoders/html.rb +9 -19
  14. data/lib/coderay/encoders/html/classes.rb +5 -2
  15. data/lib/coderay/encoders/html/css.rb +5 -6
  16. data/lib/coderay/encoders/html/numerization.rb +28 -18
  17. data/lib/coderay/encoders/html/output.rb +4 -4
  18. data/lib/coderay/encoders/null.rb +16 -16
  19. data/lib/coderay/encoders/page.rb +21 -0
  20. data/lib/coderay/encoders/span.rb +6 -3
  21. data/lib/coderay/encoders/statistic.rb +4 -2
  22. data/lib/coderay/encoders/tokens.rb +35 -35
  23. data/lib/coderay/encoders/xml.rb +53 -52
  24. data/lib/coderay/encoders/yaml.rb +13 -10
  25. data/lib/coderay/helpers/filetype.rb +41 -5
  26. data/lib/coderay/helpers/gzip_simple.rb +1 -1
  27. data/lib/coderay/helpers/plugin.rb +33 -17
  28. data/lib/coderay/scanner.rb +60 -19
  29. data/lib/coderay/scanners/_map.rb +12 -8
  30. data/lib/coderay/scanners/c.rb +16 -8
  31. data/lib/coderay/scanners/delphi.rb +9 -3
  32. data/lib/coderay/scanners/html.rb +167 -0
  33. data/lib/coderay/scanners/nitro_html.rb +125 -0
  34. data/lib/coderay/scanners/plaintext.rb +4 -2
  35. data/lib/coderay/scanners/rhtml.rb +65 -0
  36. data/lib/coderay/scanners/ruby.rb +51 -39
  37. data/lib/coderay/scanners/ruby/patterns.rb +12 -9
  38. data/lib/coderay/scanners/xml.rb +18 -0
  39. data/lib/coderay/scanners/yaml.rb +85 -0
  40. data/lib/coderay/styles/_map.rb +7 -0
  41. data/lib/coderay/styles/cycnus.rb +105 -99
  42. data/lib/coderay/styles/murphy.rb +18 -18
  43. metadata +19 -6
@@ -1,4 +1,5 @@
1
- module CodeRay module Scanners
1
+ module CodeRay
2
+ module Scanners
2
3
 
3
4
  class Delphi < Scanner
4
5
 
@@ -109,7 +110,11 @@ module CodeRay module Scanners
109
110
  end
110
111
 
111
112
  match ||= matched
112
- raise [match, kind], tokens if kind == :error
113
+ if $DEBUG and (not kind or kind == :error)
114
+ raise_inspect 'Error token %p in line %d' %
115
+ [[match, kind], line], tokens
116
+ end
117
+ raise_inspect 'Empty token', tokens unless match
113
118
 
114
119
  tokens << [match, kind]
115
120
 
@@ -120,4 +125,5 @@ module CodeRay module Scanners
120
125
 
121
126
  end
122
127
 
123
- end end
128
+ end
129
+ end
@@ -0,0 +1,167 @@
1
+ module CodeRay
2
+ module Scanners
3
+
4
+ # HTML Scanner
5
+ #
6
+ # $Id$
7
+ class HTML < Scanner
8
+
9
+ include Streamable
10
+ register_for :html
11
+
12
+ ATTR_NAME = /[\w.:-]+/
13
+ ATTR_VALUE_UNQUOTED = ATTR_NAME
14
+ TAG_END = /\/?>/
15
+ HEX = /[0-9a-fA-F]/
16
+ ENTITY = /
17
+ &
18
+ (?:
19
+ \w+
20
+ |
21
+ \#
22
+ (?:
23
+ \d+
24
+ |
25
+ x#{HEX}+
26
+ )
27
+ )
28
+ ;
29
+ /ox
30
+
31
+ PLAIN_STRING_CONTENT = {
32
+ "'" => /[^&'>\n]+/,
33
+ '"' => /[^&">\n]+/,
34
+ }
35
+
36
+ private
37
+ def setup
38
+ @state = :initial
39
+ @plain_string_content = nil
40
+ end
41
+
42
+ def scan_tokens tokens, options
43
+
44
+ state = @state
45
+ plain_string_content = @plain_string_content
46
+
47
+ until eos?
48
+
49
+ kind = :error
50
+ match = nil
51
+
52
+ if scan(/\s+/m)
53
+ kind = :space
54
+
55
+ else
56
+
57
+ case state
58
+
59
+ when :initial
60
+ if scan(/<!--.*?-->/m)
61
+ kind = :comment
62
+ elsif scan(/<!DOCTYPE.*?>/m)
63
+ kind = :preprocessor
64
+ elsif scan(/<\?xml.*?\?>/m)
65
+ kind = :preprocessor
66
+ elsif scan(/<\?.*?\?>|<%.*?%>/m)
67
+ kind = :comment
68
+ elsif scan(/<\/[-\w_.:]*>/m)
69
+ kind = :tag
70
+ elsif match = scan(/<[-\w_.:]*>?/m)
71
+ kind = :tag
72
+ state = :attribute unless match[-1] == ?>
73
+ elsif scan(/[^<>&]+/)
74
+ kind = :plain
75
+ elsif scan(/#{ENTITY}/ox)
76
+ kind = :entity
77
+ elsif scan(/[>&]/)
78
+ kind = :error
79
+ else
80
+ raise_inspect '[BUG] else-case reached with state %p' % [state], tokens
81
+ end
82
+
83
+ when :attribute
84
+ if scan(/#{TAG_END}/)
85
+ kind = :tag
86
+ state = :initial
87
+ elsif scan(/#{ATTR_NAME}/o)
88
+ kind = :attribute_name
89
+ state = :attribute_equal
90
+ else
91
+ getch
92
+ end
93
+
94
+ when :attribute_equal
95
+ if scan(/=/)
96
+ kind = :operator
97
+ state = :attribute_value
98
+ elsif scan(/#{ATTR_NAME}/o)
99
+ kind = :attribute_name
100
+ elsif scan(/#{TAG_END}/o)
101
+ kind = :tag
102
+ state = :initial
103
+ elsif scan(/./)
104
+ state = :attribute
105
+ end
106
+
107
+ when :attribute_value
108
+ if scan(/#{ATTR_VALUE_UNQUOTED}/o)
109
+ kind = :attribute_value
110
+ state = :attribute
111
+ elsif match = scan(/["']/)
112
+ tokens << [:open, :string]
113
+ state = :attribute_value_string
114
+ plain_string_content = PLAIN_STRING_CONTENT[match]
115
+ kind = :delimiter
116
+ elsif scan(/#{TAG_END}/o)
117
+ kind = :tag
118
+ state = :initial
119
+ else
120
+ getch
121
+ end
122
+
123
+ when :attribute_value_string
124
+ if scan(plain_string_content)
125
+ kind = :content
126
+ elsif scan(/['"]/)
127
+ tokens << [matched, :delimiter]
128
+ tokens << [:close, :string]
129
+ state = :attribute
130
+ next
131
+ elsif scan(/#{ENTITY}/ox)
132
+ kind = :entity
133
+ elsif scan(/[\n>]/)
134
+ tokens << [:close, :string]
135
+ kind = :error
136
+ state = :initial
137
+ end
138
+
139
+ else
140
+ raise_inspect 'Unknown state: %p' % [state], tokens
141
+
142
+ end
143
+
144
+ end
145
+
146
+ match ||= matched
147
+ if $DEBUG and (not kind or kind == :error)
148
+ raise_inspect 'Error token %p in line %d' %
149
+ [[match, kind], line], tokens
150
+ end
151
+ raise_inspect 'Empty token', tokens unless match
152
+
153
+ tokens << [match, kind]
154
+ end
155
+
156
+ if options[:keep_state]
157
+ @state = state
158
+ @plain_string_content = plain_string_content
159
+ end
160
+
161
+ tokens
162
+ end
163
+
164
+ end
165
+
166
+ end
167
+ end
@@ -0,0 +1,125 @@
1
+ module CodeRay
2
+ module Scanners
3
+
4
+ load :html
5
+ load :ruby
6
+
7
+ # RHTML Scanner
8
+ #
9
+ # $Id$
10
+ class NitroHTML < Scanner
11
+
12
+ include Streamable
13
+ register_for :nitro_html
14
+
15
+ NITRO_RUBY_BLOCK = /
16
+ <\?r
17
+ (?>
18
+ [^\?]*
19
+ (?> \?(?!>) [^\?]* )*
20
+ )
21
+ (?: \?> )?
22
+ |
23
+ <ruby>
24
+ (?>
25
+ [^<]*
26
+ (?> <(?!\/ruby>) [^<]* )*
27
+ )
28
+ (?: <\/ruby> )?
29
+ |
30
+ <%
31
+ (?>
32
+ [^%]*
33
+ (?> %(?!>) [^%]* )*
34
+ )
35
+ (?: %> )?
36
+ /mx
37
+
38
+ NITRO_VALUE_BLOCK = /
39
+ \#
40
+ (?:
41
+ \{
42
+ [^{}]*
43
+ (?>
44
+ \{ [^}]* \}
45
+ (?> [^{}]* )
46
+ )*
47
+ \}?
48
+ | \| [^|]* \|?
49
+ | \( [^)]* \)?
50
+ | \[ [^\]]* \]?
51
+ | \\ [^\\]* \\?
52
+ )
53
+ /x
54
+
55
+ NITRO_ENTITY = /
56
+ % (?: \#\d+ | \w+ ) ;
57
+ /
58
+
59
+ START_OF_RUBY = /
60
+ (?=[<\#%])
61
+ < (?: \?r | % | ruby> )
62
+ | \# [{(|]
63
+ | % (?: \#\d+ | \w+ ) ;
64
+ /x
65
+
66
+ CLOSING_PAREN = Hash.new do |h, p|
67
+ h[p] = p
68
+ end.update( {
69
+ '(' => ')',
70
+ '[' => ']',
71
+ '{' => '}',
72
+ } )
73
+
74
+ private
75
+
76
+ def setup
77
+ @ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true
78
+ @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
79
+ end
80
+
81
+ def scan_tokens tokens, options
82
+
83
+ until eos?
84
+
85
+ if (match = scan_until(/(?=#{START_OF_RUBY})/o) || scan_until(/\z/)) and not match.empty?
86
+ @html_scanner.tokenize match
87
+
88
+ elsif match = scan(/#{NITRO_VALUE_BLOCK}/o)
89
+ start_tag = match[0,2]
90
+ delimiter = CLOSING_PAREN[start_tag[1,1]]
91
+ end_tag = match[-1,1] == delimiter ? delimiter : ''
92
+ tokens << [:open, :inline]
93
+ tokens << [start_tag, :delimiter]
94
+ code = match[start_tag.size .. -1 - end_tag.size]
95
+ @ruby_scanner.tokenize code
96
+ tokens << [end_tag, :delimiter] unless end_tag.empty?
97
+ tokens << [:close, :inline]
98
+
99
+ elsif match = scan(/#{NITRO_RUBY_BLOCK}/o)
100
+ start_tag = '<?r'
101
+ end_tag = match[-2,2] == '?>' ? '?>' : ''
102
+ tokens << [:open, :inline]
103
+ tokens << [start_tag, :delimiter]
104
+ code = match[start_tag.size .. -(end_tag.size)-1]
105
+ @ruby_scanner.tokenize code
106
+ tokens << [end_tag, :delimiter] unless end_tag.empty?
107
+ tokens << [:close, :inline]
108
+
109
+ elsif entity = scan(/#{NITRO_ENTITY}/o)
110
+ tokens << [entity, :entity]
111
+
112
+ else
113
+ raise_inspect 'else-case reached!', tokens
114
+ end
115
+
116
+ end
117
+
118
+ tokens
119
+
120
+ end
121
+
122
+ end
123
+
124
+ end
125
+ end
@@ -1,4 +1,5 @@
1
- module CodeRay module Scanners
1
+ module CodeRay
2
+ module Scanners
2
3
 
3
4
  class Plaintext < Scanner
4
5
 
@@ -10,4 +11,5 @@ module CodeRay module Scanners
10
11
 
11
12
  end
12
13
 
13
- end end
14
+ end
15
+ end
@@ -0,0 +1,65 @@
1
+ module CodeRay
2
+ module Scanners
3
+
4
+ load :html
5
+ load :ruby
6
+
7
+ # RHTML Scanner
8
+ #
9
+ # $Id$
10
+ class RHTML < Scanner
11
+
12
+ include Streamable
13
+ register_for :rhtml
14
+
15
+ ERB_RUBY_BLOCK = /
16
+ <%(?!%)[=-]?
17
+ (?>
18
+ [^%]*
19
+ (?> %(?!>) [^%]* )*
20
+ )
21
+ (?: %> )?
22
+ /x
23
+
24
+ START_OF_ERB = /
25
+ <%(?!%)
26
+ /x
27
+
28
+ private
29
+
30
+ def setup
31
+ @ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true
32
+ @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
33
+ end
34
+
35
+ def scan_tokens tokens, options
36
+
37
+ until eos?
38
+
39
+ if (match = scan_until(/(?=#{START_OF_ERB})/o) || scan_until(/\z/)) and not match.empty?
40
+ @html_scanner.tokenize match
41
+
42
+ elsif match = scan(/#{ERB_RUBY_BLOCK}/o)
43
+ start_tag = match[/\A<%[-=]?/]
44
+ end_tag = match[/%?>?\z/]
45
+ tokens << [:open, :inline]
46
+ tokens << [start_tag, :delimiter]
47
+ code = match[start_tag.size .. -1 - end_tag.size]
48
+ @ruby_scanner.tokenize code
49
+ tokens << [end_tag, :delimiter] unless end_tag.empty?
50
+ tokens << [:close, :inline]
51
+
52
+ else
53
+ raise_inspect 'else-case reached!', tokens
54
+ end
55
+
56
+ end
57
+
58
+ tokens
59
+
60
+ end
61
+
62
+ end
63
+
64
+ end
65
+ end
@@ -1,4 +1,5 @@
1
- module CodeRay module Scanners
1
+ module CodeRay
2
+ module Scanners
2
3
 
3
4
  # This scanner is really complex, since Ruby _is_ a complex language!
4
5
  #
@@ -36,12 +37,14 @@ module CodeRay module Scanners
36
37
  depth = nil
37
38
  states = []
38
39
 
40
+ patterns = Patterns # avoid constant lookup
41
+
39
42
  until eos?
40
43
  type = :error
41
44
  match = nil
42
45
  kind = nil
43
46
 
44
- if state.instance_of? StringState
47
+ if state.instance_of? patterns::StringState
45
48
  # {{{
46
49
  match = scan_until(state.pattern) || scan_until(/\z/)
47
50
  tokens << [match, :content] unless match.empty?
@@ -67,14 +70,14 @@ module CodeRay module Scanners
67
70
  end
68
71
  tokens << [match, :delimiter]
69
72
  if state.type == :regexp and not eos?
70
- modifiers = scan(/#{REGEXP_MODIFIERS}/ox)
73
+ modifiers = scan(/#{patterns::REGEXP_MODIFIERS}/ox)
71
74
  tokens << [modifiers, :modifier] unless modifiers.empty?
72
75
  if parse_regexp
73
76
  extended = modifiers.index ?x
74
77
  tokens = saved_tokens
75
78
  regexp = tokens
76
79
  for text, type in regexp
77
- if text.is_a? String
80
+ if text.is_a? ::String
78
81
  case type
79
82
  when :content
80
83
  text.scan(/([^#]+)|(#.*)/) do |plain, comment|
@@ -106,7 +109,7 @@ module CodeRay module Scanners
106
109
 
107
110
  when '\\'
108
111
  if state.interpreted
109
- if esc = scan(/ #{ESCAPE} /ox)
112
+ if esc = scan(/ #{patterns::ESCAPE} /ox)
110
113
  tokens << [match + esc, :char]
111
114
  else
112
115
  tokens << [match, :error]
@@ -141,7 +144,7 @@ module CodeRay module Scanners
141
144
  state.paren_depth += 1
142
145
  tokens << [match, :nesting_delimiter]
143
146
 
144
- when REGEXP_SYMBOLS
147
+ when /#{patterns::REGEXP_SYMBOLS}/ox
145
148
  tokens << [match, :function]
146
149
 
147
150
  else
@@ -153,7 +156,7 @@ module CodeRay module Scanners
153
156
  else
154
157
  # {{{
155
158
  if match = scan(/ [ \t\f]+ | \\? \n | \# .* /x) or
156
- ( bol? and match = scan(/#{RUBYDOC_OR_DATA}/o) )
159
+ ( bol? and match = scan(/#{patterns::RUBYDOC_OR_DATA}/o) )
157
160
  fancy_allowed = true
158
161
  case m = match[0]
159
162
  when ?\s, ?\t, ?\f
@@ -186,23 +189,23 @@ module CodeRay module Scanners
186
189
  elsif state == :initial
187
190
 
188
191
  # IDENTS #
189
- if match = scan(/#{METHOD_NAME}/o)
192
+ if match = scan(/#{patterns::METHOD_NAME}/o)
190
193
  if last_token_dot
191
- type = if match[/^[A-Z]/] then :constant else :ident end
194
+ type = if match[/^[A-Z]/] and not match?(/\(/) then :constant else :ident end
192
195
  else
193
- type = IDENT_KIND[match]
194
- if type == :ident and match[/^[A-Z]/] and not match[/[!?]$/]
196
+ type = patterns::IDENT_KIND[match]
197
+ if type == :ident and match[/^[A-Z]/] and not match[/[!?]$/] and not match?(/\(/)
195
198
  type = :constant
196
199
  elsif type == :reserved
197
- state = DEF_NEW_STATE[match]
200
+ state = patterns::DEF_NEW_STATE[match]
198
201
  end
199
202
  end
200
203
  ## experimental!
201
- fancy_allowed = regexp_allowed = :set if REGEXP_ALLOWED[match] or check(/\s+(?:%\S|\/\S)/)
204
+ fancy_allowed = regexp_allowed = :set if patterns::REGEXP_ALLOWED[match] or check(/\s+(?:%\S|\/\S)/)
202
205
 
203
206
  # OPERATORS #
204
207
  elsif (not last_token_dot and match = scan(/ ==?=? | \.\.?\.? | [\(\)\[\]\{\}] | :: | , /x)) or
205
- (last_token_dot and match = scan(/#{METHOD_NAME_OPERATOR}/o))
208
+ (last_token_dot and match = scan(/#{patterns::METHOD_NAME_OPERATOR}/o))
206
209
  if match !~ / [.\)\]\}] /x or match =~ /\.\.\.?/
207
210
  regexp_allowed = fancy_allowed = :set
208
211
  end
@@ -226,32 +229,32 @@ module CodeRay module Scanners
226
229
  elsif match = scan(/ ['"] /mx)
227
230
  tokens << [:open, :string]
228
231
  type = :delimiter
229
- state = StringState.new :string, match == '"', match # important for streaming
232
+ state = patterns::StringState.new :string, match == '"', match # important for streaming
230
233
 
231
- elsif match = scan(/#{INSTANCE_VARIABLE}/o)
234
+ elsif match = scan(/#{patterns::INSTANCE_VARIABLE}/o)
232
235
  type = :instance_variable
233
236
 
234
237
  elsif regexp_allowed and match = scan(/\//)
235
238
  tokens << [:open, :regexp]
236
239
  type = :delimiter
237
240
  interpreted = true
238
- state = StringState.new :regexp, interpreted, match
241
+ state = patterns::StringState.new :regexp, interpreted, match
239
242
  if parse_regexp
240
243
  tokens = []
241
244
  saved_tokens = tokens
242
245
  end
243
246
 
244
- elsif match = scan(/#{NUMERIC}/o)
247
+ elsif match = scan(/#{patterns::NUMERIC}/o)
245
248
  type = if self[1] then :float else :integer end
246
249
 
247
- elsif match = scan(/#{SYMBOL}/o)
250
+ elsif match = scan(/#{patterns::SYMBOL}/o)
248
251
  case delim = match[1]
249
252
  when ?', ?"
250
253
  tokens << [:open, :symbol]
251
254
  tokens << [':', :symbol]
252
255
  match = delim.chr
253
256
  type = :delimiter
254
- state = StringState.new :symbol, delim == ?", match
257
+ state = patterns::StringState.new :symbol, delim == ?", match
255
258
  else
256
259
  type = :symbol
257
260
  end
@@ -260,27 +263,27 @@ module CodeRay module Scanners
260
263
  regexp_allowed = fancy_allowed = :set
261
264
  type = :operator
262
265
 
263
- elsif fancy_allowed and match = scan(/#{HEREDOC_OPEN}/o)
266
+ elsif fancy_allowed and match = scan(/#{patterns::HEREDOC_OPEN}/o)
264
267
  indented = self[1] == '-'
265
268
  quote = self[3]
266
269
  delim = self[quote ? 4 : 2]
267
- type = QUOTE_TO_TYPE[quote]
270
+ type = patterns::QUOTE_TO_TYPE[quote]
268
271
  tokens << [:open, type]
269
272
  tokens << [match, :delimiter]
270
273
  match = :close
271
- heredoc = StringState.new type, quote != '\'', delim, (indented ? :indented : :linestart )
274
+ heredoc = patterns::StringState.new type, quote != '\'', delim, (indented ? :indented : :linestart )
272
275
  heredocs ||= [] # create heredocs if empty
273
276
  heredocs << heredoc
274
277
 
275
- elsif fancy_allowed and match = scan(/#{FANCY_START}/o)
276
- type, interpreted = *FancyStringType.fetch(self[1]) do
278
+ elsif fancy_allowed and match = scan(/#{patterns::FANCY_START_SAVE}/o)
279
+ type, interpreted = *patterns::FancyStringType.fetch(self[1]) do
277
280
  raise_inspect 'Unknown fancy string: %%%p' % k, tokens
278
281
  end
279
282
  tokens << [:open, type]
280
- state = StringState.new type, interpreted, self[2]
283
+ state = patterns::StringState.new type, interpreted, self[2]
281
284
  type = :delimiter
282
285
 
283
- elsif fancy_allowed and match = scan(/#{CHARACTER}/o)
286
+ elsif fancy_allowed and match = scan(/#{patterns::CHARACTER}/o)
284
287
  type = :integer
285
288
 
286
289
  elsif match = scan(/ [\/%]=? | <(?:<|=>?)? | [?:;] /x)
@@ -293,13 +296,13 @@ module CodeRay module Scanners
293
296
  else
294
297
  tokens << [:open, :shell]
295
298
  type = :delimiter
296
- state = StringState.new :shell, true, match
299
+ state = patterns::StringState.new :shell, true, match
297
300
  end
298
301
 
299
- elsif match = scan(/#{GLOBAL_VARIABLE}/o)
302
+ elsif match = scan(/#{patterns::GLOBAL_VARIABLE}/o)
300
303
  type = :global_variable
301
304
 
302
- elsif match = scan(/#{CLASS_VARIABLE}/o)
305
+ elsif match = scan(/#{patterns::CLASS_VARIABLE}/o)
303
306
  type = :class_variable
304
307
 
305
308
  else
@@ -309,7 +312,7 @@ module CodeRay module Scanners
309
312
 
310
313
  elsif state == :def_expected
311
314
  state = :initial
312
- if match = scan(/(?>#{METHOD_NAME_EX})(?!\.|::)/o)
315
+ if match = scan(/(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/o)
313
316
  type = :method
314
317
  else
315
318
  next
@@ -317,16 +320,16 @@ module CodeRay module Scanners
317
320
 
318
321
  elsif state == :undef_expected
319
322
  state = :undef_comma_expected
320
- if match = scan(/#{METHOD_NAME_EX}/o)
323
+ if match = scan(/#{patterns::METHOD_NAME_EX}/o)
321
324
  type = :method
322
- elsif match = scan(/#{SYMBOL}/o)
325
+ elsif match = scan(/#{patterns::SYMBOL}/o)
323
326
  case delim = match[1]
324
327
  when ?', ?"
325
328
  tokens << [:open, :symbol]
326
329
  tokens << [':', :symbol]
327
330
  match = delim.chr
328
331
  type = :delimiter
329
- state = StringState.new :symbol, delim == ?", match
332
+ state = patterns::StringState.new :symbol, delim == ?", match
330
333
  state.next_state = :undef_comma_expected
331
334
  else
332
335
  type = :symbol
@@ -350,7 +353,7 @@ module CodeRay module Scanners
350
353
  type = :operator
351
354
  else
352
355
  state = :initial
353
- if match = scan(/ (?:#{IDENT}::)* #{IDENT} /ox)
356
+ if match = scan(/ (?:#{patterns::IDENT}::)* #{patterns::IDENT} /ox)
354
357
  type = :class
355
358
  else
356
359
  next
@@ -358,14 +361,17 @@ module CodeRay module Scanners
358
361
  end
359
362
 
360
363
  end
364
+ # }}}
361
365
 
362
366
  regexp_allowed = regexp_allowed == :set
363
367
  fancy_allowed = fancy_allowed == :set
364
368
  last_token_dot = last_token_dot == :set
365
369
 
366
- if $DEBUG
367
- raise_inspect 'error token %p in line %d' % [[match, type], line], tokens if not type or type == :error
370
+ if $DEBUG and (not kind or kind == :error)
371
+ raise_inspect 'Error token %p in line %d' %
372
+ [[match, kind], line], tokens
368
373
  end
374
+ raise_inspect 'Empty token', tokens unless match
369
375
 
370
376
  tokens << [match, type]
371
377
 
@@ -373,13 +379,19 @@ module CodeRay module Scanners
373
379
  state = last_state
374
380
  last_state = nil
375
381
  end
376
- # }}}
377
382
  end
378
383
  end
379
384
 
385
+ states << state if state.is_a? patterns::StringState
386
+ until states.empty?
387
+ tokens << [:close, states.pop.type]
388
+ end
389
+
380
390
  tokens
381
391
  end
382
392
  end
383
393
 
384
- end end
394
+ end
395
+ end
396
+
385
397
  # vim:fdm=marker