coderay 0.5.0.121 → 0.7.1.147
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/FOLDERS +53 -0
- data/README +21 -13
- data/bin/coderay +79 -0
- data/demo/demo_cache.rb +12 -0
- data/demo/demo_html_list.rb +12 -0
- data/lib/coderay.rb +11 -2
- data/lib/coderay/duo.rb +29 -0
- data/lib/coderay/encoder.rb +4 -4
- data/lib/coderay/encoders/_map.rb +6 -6
- data/lib/coderay/encoders/count.rb +3 -3
- data/lib/coderay/encoders/debug.rb +38 -30
- data/lib/coderay/encoders/div.rb +4 -2
- data/lib/coderay/encoders/html.rb +9 -19
- data/lib/coderay/encoders/html/classes.rb +5 -2
- data/lib/coderay/encoders/html/css.rb +5 -6
- data/lib/coderay/encoders/html/numerization.rb +28 -18
- data/lib/coderay/encoders/html/output.rb +4 -4
- data/lib/coderay/encoders/null.rb +16 -16
- data/lib/coderay/encoders/page.rb +21 -0
- data/lib/coderay/encoders/span.rb +6 -3
- data/lib/coderay/encoders/statistic.rb +4 -2
- data/lib/coderay/encoders/tokens.rb +35 -35
- data/lib/coderay/encoders/xml.rb +53 -52
- data/lib/coderay/encoders/yaml.rb +13 -10
- data/lib/coderay/helpers/filetype.rb +41 -5
- data/lib/coderay/helpers/gzip_simple.rb +1 -1
- data/lib/coderay/helpers/plugin.rb +33 -17
- data/lib/coderay/scanner.rb +60 -19
- data/lib/coderay/scanners/_map.rb +12 -8
- data/lib/coderay/scanners/c.rb +16 -8
- data/lib/coderay/scanners/delphi.rb +9 -3
- data/lib/coderay/scanners/html.rb +167 -0
- data/lib/coderay/scanners/nitro_html.rb +125 -0
- data/lib/coderay/scanners/plaintext.rb +4 -2
- data/lib/coderay/scanners/rhtml.rb +65 -0
- data/lib/coderay/scanners/ruby.rb +51 -39
- data/lib/coderay/scanners/ruby/patterns.rb +12 -9
- data/lib/coderay/scanners/xml.rb +18 -0
- data/lib/coderay/scanners/yaml.rb +85 -0
- data/lib/coderay/styles/_map.rb +7 -0
- data/lib/coderay/styles/cycnus.rb +105 -99
- data/lib/coderay/styles/murphy.rb +18 -18
- metadata +19 -6
@@ -1,4 +1,5 @@
|
|
1
|
-
module CodeRay
|
1
|
+
module CodeRay
|
2
|
+
module Scanners
|
2
3
|
|
3
4
|
class Delphi < Scanner
|
4
5
|
|
@@ -109,7 +110,11 @@ module CodeRay module Scanners
|
|
109
110
|
end
|
110
111
|
|
111
112
|
match ||= matched
|
112
|
-
|
113
|
+
if $DEBUG and (not kind or kind == :error)
|
114
|
+
raise_inspect 'Error token %p in line %d' %
|
115
|
+
[[match, kind], line], tokens
|
116
|
+
end
|
117
|
+
raise_inspect 'Empty token', tokens unless match
|
113
118
|
|
114
119
|
tokens << [match, kind]
|
115
120
|
|
@@ -120,4 +125,5 @@ module CodeRay module Scanners
|
|
120
125
|
|
121
126
|
end
|
122
127
|
|
123
|
-
end
|
128
|
+
end
|
129
|
+
end
|
@@ -0,0 +1,167 @@
|
|
1
|
+
module CodeRay
|
2
|
+
module Scanners
|
3
|
+
|
4
|
+
# HTML Scanner
|
5
|
+
#
|
6
|
+
# $Id$
|
7
|
+
class HTML < Scanner
|
8
|
+
|
9
|
+
include Streamable
|
10
|
+
register_for :html
|
11
|
+
|
12
|
+
ATTR_NAME = /[\w.:-]+/
|
13
|
+
ATTR_VALUE_UNQUOTED = ATTR_NAME
|
14
|
+
TAG_END = /\/?>/
|
15
|
+
HEX = /[0-9a-fA-F]/
|
16
|
+
ENTITY = /
|
17
|
+
&
|
18
|
+
(?:
|
19
|
+
\w+
|
20
|
+
|
|
21
|
+
\#
|
22
|
+
(?:
|
23
|
+
\d+
|
24
|
+
|
|
25
|
+
x#{HEX}+
|
26
|
+
)
|
27
|
+
)
|
28
|
+
;
|
29
|
+
/ox
|
30
|
+
|
31
|
+
PLAIN_STRING_CONTENT = {
|
32
|
+
"'" => /[^&'>\n]+/,
|
33
|
+
'"' => /[^&">\n]+/,
|
34
|
+
}
|
35
|
+
|
36
|
+
private
|
37
|
+
def setup
|
38
|
+
@state = :initial
|
39
|
+
@plain_string_content = nil
|
40
|
+
end
|
41
|
+
|
42
|
+
def scan_tokens tokens, options
|
43
|
+
|
44
|
+
state = @state
|
45
|
+
plain_string_content = @plain_string_content
|
46
|
+
|
47
|
+
until eos?
|
48
|
+
|
49
|
+
kind = :error
|
50
|
+
match = nil
|
51
|
+
|
52
|
+
if scan(/\s+/m)
|
53
|
+
kind = :space
|
54
|
+
|
55
|
+
else
|
56
|
+
|
57
|
+
case state
|
58
|
+
|
59
|
+
when :initial
|
60
|
+
if scan(/<!--.*?-->/m)
|
61
|
+
kind = :comment
|
62
|
+
elsif scan(/<!DOCTYPE.*?>/m)
|
63
|
+
kind = :preprocessor
|
64
|
+
elsif scan(/<\?xml.*?\?>/m)
|
65
|
+
kind = :preprocessor
|
66
|
+
elsif scan(/<\?.*?\?>|<%.*?%>/m)
|
67
|
+
kind = :comment
|
68
|
+
elsif scan(/<\/[-\w_.:]*>/m)
|
69
|
+
kind = :tag
|
70
|
+
elsif match = scan(/<[-\w_.:]*>?/m)
|
71
|
+
kind = :tag
|
72
|
+
state = :attribute unless match[-1] == ?>
|
73
|
+
elsif scan(/[^<>&]+/)
|
74
|
+
kind = :plain
|
75
|
+
elsif scan(/#{ENTITY}/ox)
|
76
|
+
kind = :entity
|
77
|
+
elsif scan(/[>&]/)
|
78
|
+
kind = :error
|
79
|
+
else
|
80
|
+
raise_inspect '[BUG] else-case reached with state %p' % [state], tokens
|
81
|
+
end
|
82
|
+
|
83
|
+
when :attribute
|
84
|
+
if scan(/#{TAG_END}/)
|
85
|
+
kind = :tag
|
86
|
+
state = :initial
|
87
|
+
elsif scan(/#{ATTR_NAME}/o)
|
88
|
+
kind = :attribute_name
|
89
|
+
state = :attribute_equal
|
90
|
+
else
|
91
|
+
getch
|
92
|
+
end
|
93
|
+
|
94
|
+
when :attribute_equal
|
95
|
+
if scan(/=/)
|
96
|
+
kind = :operator
|
97
|
+
state = :attribute_value
|
98
|
+
elsif scan(/#{ATTR_NAME}/o)
|
99
|
+
kind = :attribute_name
|
100
|
+
elsif scan(/#{TAG_END}/o)
|
101
|
+
kind = :tag
|
102
|
+
state = :initial
|
103
|
+
elsif scan(/./)
|
104
|
+
state = :attribute
|
105
|
+
end
|
106
|
+
|
107
|
+
when :attribute_value
|
108
|
+
if scan(/#{ATTR_VALUE_UNQUOTED}/o)
|
109
|
+
kind = :attribute_value
|
110
|
+
state = :attribute
|
111
|
+
elsif match = scan(/["']/)
|
112
|
+
tokens << [:open, :string]
|
113
|
+
state = :attribute_value_string
|
114
|
+
plain_string_content = PLAIN_STRING_CONTENT[match]
|
115
|
+
kind = :delimiter
|
116
|
+
elsif scan(/#{TAG_END}/o)
|
117
|
+
kind = :tag
|
118
|
+
state = :initial
|
119
|
+
else
|
120
|
+
getch
|
121
|
+
end
|
122
|
+
|
123
|
+
when :attribute_value_string
|
124
|
+
if scan(plain_string_content)
|
125
|
+
kind = :content
|
126
|
+
elsif scan(/['"]/)
|
127
|
+
tokens << [matched, :delimiter]
|
128
|
+
tokens << [:close, :string]
|
129
|
+
state = :attribute
|
130
|
+
next
|
131
|
+
elsif scan(/#{ENTITY}/ox)
|
132
|
+
kind = :entity
|
133
|
+
elsif scan(/[\n>]/)
|
134
|
+
tokens << [:close, :string]
|
135
|
+
kind = :error
|
136
|
+
state = :initial
|
137
|
+
end
|
138
|
+
|
139
|
+
else
|
140
|
+
raise_inspect 'Unknown state: %p' % [state], tokens
|
141
|
+
|
142
|
+
end
|
143
|
+
|
144
|
+
end
|
145
|
+
|
146
|
+
match ||= matched
|
147
|
+
if $DEBUG and (not kind or kind == :error)
|
148
|
+
raise_inspect 'Error token %p in line %d' %
|
149
|
+
[[match, kind], line], tokens
|
150
|
+
end
|
151
|
+
raise_inspect 'Empty token', tokens unless match
|
152
|
+
|
153
|
+
tokens << [match, kind]
|
154
|
+
end
|
155
|
+
|
156
|
+
if options[:keep_state]
|
157
|
+
@state = state
|
158
|
+
@plain_string_content = plain_string_content
|
159
|
+
end
|
160
|
+
|
161
|
+
tokens
|
162
|
+
end
|
163
|
+
|
164
|
+
end
|
165
|
+
|
166
|
+
end
|
167
|
+
end
|
@@ -0,0 +1,125 @@
|
|
1
|
+
module CodeRay
|
2
|
+
module Scanners
|
3
|
+
|
4
|
+
load :html
|
5
|
+
load :ruby
|
6
|
+
|
7
|
+
# RHTML Scanner
|
8
|
+
#
|
9
|
+
# $Id$
|
10
|
+
class NitroHTML < Scanner
|
11
|
+
|
12
|
+
include Streamable
|
13
|
+
register_for :nitro_html
|
14
|
+
|
15
|
+
NITRO_RUBY_BLOCK = /
|
16
|
+
<\?r
|
17
|
+
(?>
|
18
|
+
[^\?]*
|
19
|
+
(?> \?(?!>) [^\?]* )*
|
20
|
+
)
|
21
|
+
(?: \?> )?
|
22
|
+
|
|
23
|
+
<ruby>
|
24
|
+
(?>
|
25
|
+
[^<]*
|
26
|
+
(?> <(?!\/ruby>) [^<]* )*
|
27
|
+
)
|
28
|
+
(?: <\/ruby> )?
|
29
|
+
|
|
30
|
+
<%
|
31
|
+
(?>
|
32
|
+
[^%]*
|
33
|
+
(?> %(?!>) [^%]* )*
|
34
|
+
)
|
35
|
+
(?: %> )?
|
36
|
+
/mx
|
37
|
+
|
38
|
+
NITRO_VALUE_BLOCK = /
|
39
|
+
\#
|
40
|
+
(?:
|
41
|
+
\{
|
42
|
+
[^{}]*
|
43
|
+
(?>
|
44
|
+
\{ [^}]* \}
|
45
|
+
(?> [^{}]* )
|
46
|
+
)*
|
47
|
+
\}?
|
48
|
+
| \| [^|]* \|?
|
49
|
+
| \( [^)]* \)?
|
50
|
+
| \[ [^\]]* \]?
|
51
|
+
| \\ [^\\]* \\?
|
52
|
+
)
|
53
|
+
/x
|
54
|
+
|
55
|
+
NITRO_ENTITY = /
|
56
|
+
% (?: \#\d+ | \w+ ) ;
|
57
|
+
/
|
58
|
+
|
59
|
+
START_OF_RUBY = /
|
60
|
+
(?=[<\#%])
|
61
|
+
< (?: \?r | % | ruby> )
|
62
|
+
| \# [{(|]
|
63
|
+
| % (?: \#\d+ | \w+ ) ;
|
64
|
+
/x
|
65
|
+
|
66
|
+
CLOSING_PAREN = Hash.new do |h, p|
|
67
|
+
h[p] = p
|
68
|
+
end.update( {
|
69
|
+
'(' => ')',
|
70
|
+
'[' => ']',
|
71
|
+
'{' => '}',
|
72
|
+
} )
|
73
|
+
|
74
|
+
private
|
75
|
+
|
76
|
+
def setup
|
77
|
+
@ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true
|
78
|
+
@html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
|
79
|
+
end
|
80
|
+
|
81
|
+
def scan_tokens tokens, options
|
82
|
+
|
83
|
+
until eos?
|
84
|
+
|
85
|
+
if (match = scan_until(/(?=#{START_OF_RUBY})/o) || scan_until(/\z/)) and not match.empty?
|
86
|
+
@html_scanner.tokenize match
|
87
|
+
|
88
|
+
elsif match = scan(/#{NITRO_VALUE_BLOCK}/o)
|
89
|
+
start_tag = match[0,2]
|
90
|
+
delimiter = CLOSING_PAREN[start_tag[1,1]]
|
91
|
+
end_tag = match[-1,1] == delimiter ? delimiter : ''
|
92
|
+
tokens << [:open, :inline]
|
93
|
+
tokens << [start_tag, :delimiter]
|
94
|
+
code = match[start_tag.size .. -1 - end_tag.size]
|
95
|
+
@ruby_scanner.tokenize code
|
96
|
+
tokens << [end_tag, :delimiter] unless end_tag.empty?
|
97
|
+
tokens << [:close, :inline]
|
98
|
+
|
99
|
+
elsif match = scan(/#{NITRO_RUBY_BLOCK}/o)
|
100
|
+
start_tag = '<?r'
|
101
|
+
end_tag = match[-2,2] == '?>' ? '?>' : ''
|
102
|
+
tokens << [:open, :inline]
|
103
|
+
tokens << [start_tag, :delimiter]
|
104
|
+
code = match[start_tag.size .. -(end_tag.size)-1]
|
105
|
+
@ruby_scanner.tokenize code
|
106
|
+
tokens << [end_tag, :delimiter] unless end_tag.empty?
|
107
|
+
tokens << [:close, :inline]
|
108
|
+
|
109
|
+
elsif entity = scan(/#{NITRO_ENTITY}/o)
|
110
|
+
tokens << [entity, :entity]
|
111
|
+
|
112
|
+
else
|
113
|
+
raise_inspect 'else-case reached!', tokens
|
114
|
+
end
|
115
|
+
|
116
|
+
end
|
117
|
+
|
118
|
+
tokens
|
119
|
+
|
120
|
+
end
|
121
|
+
|
122
|
+
end
|
123
|
+
|
124
|
+
end
|
125
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
module CodeRay
|
2
|
+
module Scanners
|
3
|
+
|
4
|
+
load :html
|
5
|
+
load :ruby
|
6
|
+
|
7
|
+
# RHTML Scanner
|
8
|
+
#
|
9
|
+
# $Id$
|
10
|
+
class RHTML < Scanner
|
11
|
+
|
12
|
+
include Streamable
|
13
|
+
register_for :rhtml
|
14
|
+
|
15
|
+
ERB_RUBY_BLOCK = /
|
16
|
+
<%(?!%)[=-]?
|
17
|
+
(?>
|
18
|
+
[^%]*
|
19
|
+
(?> %(?!>) [^%]* )*
|
20
|
+
)
|
21
|
+
(?: %> )?
|
22
|
+
/x
|
23
|
+
|
24
|
+
START_OF_ERB = /
|
25
|
+
<%(?!%)
|
26
|
+
/x
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def setup
|
31
|
+
@ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true
|
32
|
+
@html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
|
33
|
+
end
|
34
|
+
|
35
|
+
def scan_tokens tokens, options
|
36
|
+
|
37
|
+
until eos?
|
38
|
+
|
39
|
+
if (match = scan_until(/(?=#{START_OF_ERB})/o) || scan_until(/\z/)) and not match.empty?
|
40
|
+
@html_scanner.tokenize match
|
41
|
+
|
42
|
+
elsif match = scan(/#{ERB_RUBY_BLOCK}/o)
|
43
|
+
start_tag = match[/\A<%[-=]?/]
|
44
|
+
end_tag = match[/%?>?\z/]
|
45
|
+
tokens << [:open, :inline]
|
46
|
+
tokens << [start_tag, :delimiter]
|
47
|
+
code = match[start_tag.size .. -1 - end_tag.size]
|
48
|
+
@ruby_scanner.tokenize code
|
49
|
+
tokens << [end_tag, :delimiter] unless end_tag.empty?
|
50
|
+
tokens << [:close, :inline]
|
51
|
+
|
52
|
+
else
|
53
|
+
raise_inspect 'else-case reached!', tokens
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
tokens
|
59
|
+
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
end
|
@@ -1,4 +1,5 @@
|
|
1
|
-
module CodeRay
|
1
|
+
module CodeRay
|
2
|
+
module Scanners
|
2
3
|
|
3
4
|
# This scanner is really complex, since Ruby _is_ a complex language!
|
4
5
|
#
|
@@ -36,12 +37,14 @@ module CodeRay module Scanners
|
|
36
37
|
depth = nil
|
37
38
|
states = []
|
38
39
|
|
40
|
+
patterns = Patterns # avoid constant lookup
|
41
|
+
|
39
42
|
until eos?
|
40
43
|
type = :error
|
41
44
|
match = nil
|
42
45
|
kind = nil
|
43
46
|
|
44
|
-
if state.instance_of? StringState
|
47
|
+
if state.instance_of? patterns::StringState
|
45
48
|
# {{{
|
46
49
|
match = scan_until(state.pattern) || scan_until(/\z/)
|
47
50
|
tokens << [match, :content] unless match.empty?
|
@@ -67,14 +70,14 @@ module CodeRay module Scanners
|
|
67
70
|
end
|
68
71
|
tokens << [match, :delimiter]
|
69
72
|
if state.type == :regexp and not eos?
|
70
|
-
modifiers = scan(/#{REGEXP_MODIFIERS}/ox)
|
73
|
+
modifiers = scan(/#{patterns::REGEXP_MODIFIERS}/ox)
|
71
74
|
tokens << [modifiers, :modifier] unless modifiers.empty?
|
72
75
|
if parse_regexp
|
73
76
|
extended = modifiers.index ?x
|
74
77
|
tokens = saved_tokens
|
75
78
|
regexp = tokens
|
76
79
|
for text, type in regexp
|
77
|
-
if text.is_a? String
|
80
|
+
if text.is_a? ::String
|
78
81
|
case type
|
79
82
|
when :content
|
80
83
|
text.scan(/([^#]+)|(#.*)/) do |plain, comment|
|
@@ -106,7 +109,7 @@ module CodeRay module Scanners
|
|
106
109
|
|
107
110
|
when '\\'
|
108
111
|
if state.interpreted
|
109
|
-
if esc = scan(/ #{ESCAPE} /ox)
|
112
|
+
if esc = scan(/ #{patterns::ESCAPE} /ox)
|
110
113
|
tokens << [match + esc, :char]
|
111
114
|
else
|
112
115
|
tokens << [match, :error]
|
@@ -141,7 +144,7 @@ module CodeRay module Scanners
|
|
141
144
|
state.paren_depth += 1
|
142
145
|
tokens << [match, :nesting_delimiter]
|
143
146
|
|
144
|
-
when REGEXP_SYMBOLS
|
147
|
+
when /#{patterns::REGEXP_SYMBOLS}/ox
|
145
148
|
tokens << [match, :function]
|
146
149
|
|
147
150
|
else
|
@@ -153,7 +156,7 @@ module CodeRay module Scanners
|
|
153
156
|
else
|
154
157
|
# {{{
|
155
158
|
if match = scan(/ [ \t\f]+ | \\? \n | \# .* /x) or
|
156
|
-
( bol? and match = scan(/#{RUBYDOC_OR_DATA}/o) )
|
159
|
+
( bol? and match = scan(/#{patterns::RUBYDOC_OR_DATA}/o) )
|
157
160
|
fancy_allowed = true
|
158
161
|
case m = match[0]
|
159
162
|
when ?\s, ?\t, ?\f
|
@@ -186,23 +189,23 @@ module CodeRay module Scanners
|
|
186
189
|
elsif state == :initial
|
187
190
|
|
188
191
|
# IDENTS #
|
189
|
-
if match = scan(/#{METHOD_NAME}/o)
|
192
|
+
if match = scan(/#{patterns::METHOD_NAME}/o)
|
190
193
|
if last_token_dot
|
191
|
-
type = if match[/^[A-Z]/] then :constant else :ident end
|
194
|
+
type = if match[/^[A-Z]/] and not match?(/\(/) then :constant else :ident end
|
192
195
|
else
|
193
|
-
type = IDENT_KIND[match]
|
194
|
-
if type == :ident and match[/^[A-Z]/] and not match[/[!?]$/]
|
196
|
+
type = patterns::IDENT_KIND[match]
|
197
|
+
if type == :ident and match[/^[A-Z]/] and not match[/[!?]$/] and not match?(/\(/)
|
195
198
|
type = :constant
|
196
199
|
elsif type == :reserved
|
197
|
-
state = DEF_NEW_STATE[match]
|
200
|
+
state = patterns::DEF_NEW_STATE[match]
|
198
201
|
end
|
199
202
|
end
|
200
203
|
## experimental!
|
201
|
-
fancy_allowed = regexp_allowed = :set if REGEXP_ALLOWED[match] or check(/\s+(?:%\S|\/\S)/)
|
204
|
+
fancy_allowed = regexp_allowed = :set if patterns::REGEXP_ALLOWED[match] or check(/\s+(?:%\S|\/\S)/)
|
202
205
|
|
203
206
|
# OPERATORS #
|
204
207
|
elsif (not last_token_dot and match = scan(/ ==?=? | \.\.?\.? | [\(\)\[\]\{\}] | :: | , /x)) or
|
205
|
-
(last_token_dot and match = scan(/#{METHOD_NAME_OPERATOR}/o))
|
208
|
+
(last_token_dot and match = scan(/#{patterns::METHOD_NAME_OPERATOR}/o))
|
206
209
|
if match !~ / [.\)\]\}] /x or match =~ /\.\.\.?/
|
207
210
|
regexp_allowed = fancy_allowed = :set
|
208
211
|
end
|
@@ -226,32 +229,32 @@ module CodeRay module Scanners
|
|
226
229
|
elsif match = scan(/ ['"] /mx)
|
227
230
|
tokens << [:open, :string]
|
228
231
|
type = :delimiter
|
229
|
-
state = StringState.new :string, match == '"', match # important for streaming
|
232
|
+
state = patterns::StringState.new :string, match == '"', match # important for streaming
|
230
233
|
|
231
|
-
elsif match = scan(/#{INSTANCE_VARIABLE}/o)
|
234
|
+
elsif match = scan(/#{patterns::INSTANCE_VARIABLE}/o)
|
232
235
|
type = :instance_variable
|
233
236
|
|
234
237
|
elsif regexp_allowed and match = scan(/\//)
|
235
238
|
tokens << [:open, :regexp]
|
236
239
|
type = :delimiter
|
237
240
|
interpreted = true
|
238
|
-
state = StringState.new :regexp, interpreted, match
|
241
|
+
state = patterns::StringState.new :regexp, interpreted, match
|
239
242
|
if parse_regexp
|
240
243
|
tokens = []
|
241
244
|
saved_tokens = tokens
|
242
245
|
end
|
243
246
|
|
244
|
-
elsif match = scan(/#{NUMERIC}/o)
|
247
|
+
elsif match = scan(/#{patterns::NUMERIC}/o)
|
245
248
|
type = if self[1] then :float else :integer end
|
246
249
|
|
247
|
-
elsif match = scan(/#{SYMBOL}/o)
|
250
|
+
elsif match = scan(/#{patterns::SYMBOL}/o)
|
248
251
|
case delim = match[1]
|
249
252
|
when ?', ?"
|
250
253
|
tokens << [:open, :symbol]
|
251
254
|
tokens << [':', :symbol]
|
252
255
|
match = delim.chr
|
253
256
|
type = :delimiter
|
254
|
-
state = StringState.new :symbol, delim == ?", match
|
257
|
+
state = patterns::StringState.new :symbol, delim == ?", match
|
255
258
|
else
|
256
259
|
type = :symbol
|
257
260
|
end
|
@@ -260,27 +263,27 @@ module CodeRay module Scanners
|
|
260
263
|
regexp_allowed = fancy_allowed = :set
|
261
264
|
type = :operator
|
262
265
|
|
263
|
-
elsif fancy_allowed and match = scan(/#{HEREDOC_OPEN}/o)
|
266
|
+
elsif fancy_allowed and match = scan(/#{patterns::HEREDOC_OPEN}/o)
|
264
267
|
indented = self[1] == '-'
|
265
268
|
quote = self[3]
|
266
269
|
delim = self[quote ? 4 : 2]
|
267
|
-
type = QUOTE_TO_TYPE[quote]
|
270
|
+
type = patterns::QUOTE_TO_TYPE[quote]
|
268
271
|
tokens << [:open, type]
|
269
272
|
tokens << [match, :delimiter]
|
270
273
|
match = :close
|
271
|
-
heredoc = StringState.new type, quote != '\'', delim, (indented ? :indented : :linestart )
|
274
|
+
heredoc = patterns::StringState.new type, quote != '\'', delim, (indented ? :indented : :linestart )
|
272
275
|
heredocs ||= [] # create heredocs if empty
|
273
276
|
heredocs << heredoc
|
274
277
|
|
275
|
-
elsif fancy_allowed and match = scan(/#{
|
276
|
-
type, interpreted = *FancyStringType.fetch(self[1]) do
|
278
|
+
elsif fancy_allowed and match = scan(/#{patterns::FANCY_START_SAVE}/o)
|
279
|
+
type, interpreted = *patterns::FancyStringType.fetch(self[1]) do
|
277
280
|
raise_inspect 'Unknown fancy string: %%%p' % k, tokens
|
278
281
|
end
|
279
282
|
tokens << [:open, type]
|
280
|
-
state = StringState.new type, interpreted, self[2]
|
283
|
+
state = patterns::StringState.new type, interpreted, self[2]
|
281
284
|
type = :delimiter
|
282
285
|
|
283
|
-
elsif fancy_allowed and match = scan(/#{CHARACTER}/o)
|
286
|
+
elsif fancy_allowed and match = scan(/#{patterns::CHARACTER}/o)
|
284
287
|
type = :integer
|
285
288
|
|
286
289
|
elsif match = scan(/ [\/%]=? | <(?:<|=>?)? | [?:;] /x)
|
@@ -293,13 +296,13 @@ module CodeRay module Scanners
|
|
293
296
|
else
|
294
297
|
tokens << [:open, :shell]
|
295
298
|
type = :delimiter
|
296
|
-
state = StringState.new :shell, true, match
|
299
|
+
state = patterns::StringState.new :shell, true, match
|
297
300
|
end
|
298
301
|
|
299
|
-
elsif match = scan(/#{GLOBAL_VARIABLE}/o)
|
302
|
+
elsif match = scan(/#{patterns::GLOBAL_VARIABLE}/o)
|
300
303
|
type = :global_variable
|
301
304
|
|
302
|
-
elsif match = scan(/#{CLASS_VARIABLE}/o)
|
305
|
+
elsif match = scan(/#{patterns::CLASS_VARIABLE}/o)
|
303
306
|
type = :class_variable
|
304
307
|
|
305
308
|
else
|
@@ -309,7 +312,7 @@ module CodeRay module Scanners
|
|
309
312
|
|
310
313
|
elsif state == :def_expected
|
311
314
|
state = :initial
|
312
|
-
if match = scan(/(?>#{METHOD_NAME_EX})(?!\.|::)/o)
|
315
|
+
if match = scan(/(?>#{patterns::METHOD_NAME_EX})(?!\.|::)/o)
|
313
316
|
type = :method
|
314
317
|
else
|
315
318
|
next
|
@@ -317,16 +320,16 @@ module CodeRay module Scanners
|
|
317
320
|
|
318
321
|
elsif state == :undef_expected
|
319
322
|
state = :undef_comma_expected
|
320
|
-
if match = scan(/#{METHOD_NAME_EX}/o)
|
323
|
+
if match = scan(/#{patterns::METHOD_NAME_EX}/o)
|
321
324
|
type = :method
|
322
|
-
elsif match = scan(/#{SYMBOL}/o)
|
325
|
+
elsif match = scan(/#{patterns::SYMBOL}/o)
|
323
326
|
case delim = match[1]
|
324
327
|
when ?', ?"
|
325
328
|
tokens << [:open, :symbol]
|
326
329
|
tokens << [':', :symbol]
|
327
330
|
match = delim.chr
|
328
331
|
type = :delimiter
|
329
|
-
state = StringState.new :symbol, delim == ?", match
|
332
|
+
state = patterns::StringState.new :symbol, delim == ?", match
|
330
333
|
state.next_state = :undef_comma_expected
|
331
334
|
else
|
332
335
|
type = :symbol
|
@@ -350,7 +353,7 @@ module CodeRay module Scanners
|
|
350
353
|
type = :operator
|
351
354
|
else
|
352
355
|
state = :initial
|
353
|
-
if match = scan(/ (?:#{IDENT}::)* #{IDENT} /ox)
|
356
|
+
if match = scan(/ (?:#{patterns::IDENT}::)* #{patterns::IDENT} /ox)
|
354
357
|
type = :class
|
355
358
|
else
|
356
359
|
next
|
@@ -358,14 +361,17 @@ module CodeRay module Scanners
|
|
358
361
|
end
|
359
362
|
|
360
363
|
end
|
364
|
+
# }}}
|
361
365
|
|
362
366
|
regexp_allowed = regexp_allowed == :set
|
363
367
|
fancy_allowed = fancy_allowed == :set
|
364
368
|
last_token_dot = last_token_dot == :set
|
365
369
|
|
366
|
-
if $DEBUG
|
367
|
-
raise_inspect '
|
370
|
+
if $DEBUG and (not kind or kind == :error)
|
371
|
+
raise_inspect 'Error token %p in line %d' %
|
372
|
+
[[match, kind], line], tokens
|
368
373
|
end
|
374
|
+
raise_inspect 'Empty token', tokens unless match
|
369
375
|
|
370
376
|
tokens << [match, type]
|
371
377
|
|
@@ -373,13 +379,19 @@ module CodeRay module Scanners
|
|
373
379
|
state = last_state
|
374
380
|
last_state = nil
|
375
381
|
end
|
376
|
-
# }}}
|
377
382
|
end
|
378
383
|
end
|
379
384
|
|
385
|
+
states << state if state.is_a? patterns::StringState
|
386
|
+
until states.empty?
|
387
|
+
tokens << [:close, states.pop.type]
|
388
|
+
end
|
389
|
+
|
380
390
|
tokens
|
381
391
|
end
|
382
392
|
end
|
383
393
|
|
384
|
-
end
|
394
|
+
end
|
395
|
+
end
|
396
|
+
|
385
397
|
# vim:fdm=marker
|