coderay-beta 0.9.1
Sign up to get free protection for your applications and to get access to all the features.
- data/FOLDERS +53 -0
- data/LICENSE +504 -0
- data/bin/coderay +82 -0
- data/bin/coderay_stylesheet +4 -0
- data/lib/README +129 -0
- data/lib/coderay.rb +320 -0
- data/lib/coderay/duo.rb +85 -0
- data/lib/coderay/encoder.rb +213 -0
- data/lib/coderay/encoders/_map.rb +11 -0
- data/lib/coderay/encoders/comment_filter.rb +43 -0
- data/lib/coderay/encoders/count.rb +21 -0
- data/lib/coderay/encoders/debug.rb +49 -0
- data/lib/coderay/encoders/div.rb +19 -0
- data/lib/coderay/encoders/filter.rb +75 -0
- data/lib/coderay/encoders/html.rb +305 -0
- data/lib/coderay/encoders/html/css.rb +70 -0
- data/lib/coderay/encoders/html/numerization.rb +133 -0
- data/lib/coderay/encoders/html/output.rb +206 -0
- data/lib/coderay/encoders/json.rb +69 -0
- data/lib/coderay/encoders/lines_of_code.rb +90 -0
- data/lib/coderay/encoders/null.rb +26 -0
- data/lib/coderay/encoders/page.rb +20 -0
- data/lib/coderay/encoders/span.rb +19 -0
- data/lib/coderay/encoders/statistic.rb +77 -0
- data/lib/coderay/encoders/term.rb +137 -0
- data/lib/coderay/encoders/text.rb +32 -0
- data/lib/coderay/encoders/token_class_filter.rb +84 -0
- data/lib/coderay/encoders/xml.rb +71 -0
- data/lib/coderay/encoders/yaml.rb +22 -0
- data/lib/coderay/for_redcloth.rb +85 -0
- data/lib/coderay/helpers/file_type.rb +240 -0
- data/lib/coderay/helpers/gzip_simple.rb +123 -0
- data/lib/coderay/helpers/plugin.rb +349 -0
- data/lib/coderay/helpers/word_list.rb +138 -0
- data/lib/coderay/scanner.rb +284 -0
- data/lib/coderay/scanners/_map.rb +23 -0
- data/lib/coderay/scanners/c.rb +203 -0
- data/lib/coderay/scanners/cpp.rb +228 -0
- data/lib/coderay/scanners/css.rb +210 -0
- data/lib/coderay/scanners/debug.rb +62 -0
- data/lib/coderay/scanners/delphi.rb +150 -0
- data/lib/coderay/scanners/diff.rb +105 -0
- data/lib/coderay/scanners/groovy.rb +263 -0
- data/lib/coderay/scanners/html.rb +182 -0
- data/lib/coderay/scanners/java.rb +176 -0
- data/lib/coderay/scanners/java/builtin_types.rb +419 -0
- data/lib/coderay/scanners/java_script.rb +224 -0
- data/lib/coderay/scanners/json.rb +112 -0
- data/lib/coderay/scanners/nitro_xhtml.rb +136 -0
- data/lib/coderay/scanners/php.rb +526 -0
- data/lib/coderay/scanners/plaintext.rb +21 -0
- data/lib/coderay/scanners/python.rb +285 -0
- data/lib/coderay/scanners/rhtml.rb +74 -0
- data/lib/coderay/scanners/ruby.rb +404 -0
- data/lib/coderay/scanners/ruby/patterns.rb +238 -0
- data/lib/coderay/scanners/scheme.rb +145 -0
- data/lib/coderay/scanners/sql.rb +162 -0
- data/lib/coderay/scanners/xml.rb +17 -0
- data/lib/coderay/scanners/yaml.rb +144 -0
- data/lib/coderay/style.rb +20 -0
- data/lib/coderay/styles/_map.rb +7 -0
- data/lib/coderay/styles/cycnus.rb +151 -0
- data/lib/coderay/styles/murphy.rb +132 -0
- data/lib/coderay/token_classes.rb +86 -0
- data/lib/coderay/tokens.rb +391 -0
- data/lib/term/ansicolor.rb +220 -0
- metadata +123 -0
@@ -0,0 +1,224 @@
|
|
1
|
+
module CodeRay
|
2
|
+
module Scanners
|
3
|
+
|
4
|
+
class JavaScript < Scanner
|
5
|
+
|
6
|
+
include Streamable
|
7
|
+
|
8
|
+
register_for :java_script
|
9
|
+
file_extension 'js'
|
10
|
+
|
11
|
+
# The actual JavaScript keywords.
|
12
|
+
KEYWORDS = %w[
|
13
|
+
break case catch continue default delete do else
|
14
|
+
finally for function if in instanceof new
|
15
|
+
return switch throw try typeof var void while with
|
16
|
+
]
|
17
|
+
PREDEFINED_CONSTANTS = %w[
|
18
|
+
false null true undefined
|
19
|
+
]
|
20
|
+
|
21
|
+
MAGIC_VARIABLES = %w[ this arguments ] # arguments was introduced in JavaScript 1.4
|
22
|
+
|
23
|
+
KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
|
24
|
+
case delete in instanceof new return throw typeof with
|
25
|
+
]
|
26
|
+
|
27
|
+
# Reserved for future use.
|
28
|
+
RESERVED_WORDS = %w[
|
29
|
+
abstract boolean byte char class debugger double enum export extends
|
30
|
+
final float goto implements import int interface long native package
|
31
|
+
private protected public short static super synchronized throws transient
|
32
|
+
volatile
|
33
|
+
]
|
34
|
+
|
35
|
+
IDENT_KIND = WordList.new(:ident).
|
36
|
+
add(RESERVED_WORDS, :reserved).
|
37
|
+
add(PREDEFINED_CONSTANTS, :pre_constant).
|
38
|
+
add(MAGIC_VARIABLES, :local_variable).
|
39
|
+
add(KEYWORDS, :keyword)
|
40
|
+
|
41
|
+
ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
|
42
|
+
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
|
43
|
+
REGEXP_ESCAPE = / [bBdDsSwW] /x
|
44
|
+
STRING_CONTENT_PATTERN = {
|
45
|
+
"'" => /[^\\']+/,
|
46
|
+
'"' => /[^\\"]+/,
|
47
|
+
'/' => /[^\\\/]+/,
|
48
|
+
}
|
49
|
+
KEY_CHECK_PATTERN = {
|
50
|
+
"'" => / [^\\']* (?: \\.? [^\\']* )* '? \s* : /x,
|
51
|
+
'"' => / [^\\"]* (?: \\.? [^\\"]* )* "? \s* : /x,
|
52
|
+
}
|
53
|
+
|
54
|
+
def scan_tokens tokens, options
|
55
|
+
|
56
|
+
state = :initial
|
57
|
+
string_delimiter = nil
|
58
|
+
value_expected = true
|
59
|
+
key_expected = false
|
60
|
+
function_expected = false
|
61
|
+
|
62
|
+
until eos?
|
63
|
+
|
64
|
+
kind = nil
|
65
|
+
match = nil
|
66
|
+
|
67
|
+
case state
|
68
|
+
|
69
|
+
when :initial
|
70
|
+
|
71
|
+
if match = scan(/ \s+ | \\\n /x)
|
72
|
+
value_expected = true if !value_expected && match.index(?\n)
|
73
|
+
tokens << [match, :space]
|
74
|
+
next
|
75
|
+
|
76
|
+
elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
|
77
|
+
value_expected = true
|
78
|
+
kind = :comment
|
79
|
+
|
80
|
+
elsif check(/\.?\d/)
|
81
|
+
key_expected = value_expected = false
|
82
|
+
if scan(/0[xX][0-9A-Fa-f]+/)
|
83
|
+
kind = :hex
|
84
|
+
elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
|
85
|
+
kind = :oct
|
86
|
+
elsif scan(/\d+[fF]|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
|
87
|
+
kind = :float
|
88
|
+
elsif scan(/\d+/)
|
89
|
+
kind = :integer
|
90
|
+
end
|
91
|
+
|
92
|
+
elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\1>)/xim)
|
93
|
+
# FIXME: scan over nested tags
|
94
|
+
xml_scanner.tokenize match
|
95
|
+
value_expected = false
|
96
|
+
next
|
97
|
+
|
98
|
+
elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /x)
|
99
|
+
value_expected = true
|
100
|
+
last_operator = match[-1]
|
101
|
+
key_expected = (last_operator == ?{) || (last_operator == ?,)
|
102
|
+
function_expected = false
|
103
|
+
kind = :operator
|
104
|
+
|
105
|
+
elsif scan(/ [)\]}]+ /x)
|
106
|
+
function_expected = key_expected = value_expected = false
|
107
|
+
kind = :operator
|
108
|
+
|
109
|
+
elsif match = scan(/ [$a-zA-Z_][A-Za-z_0-9$]* /x)
|
110
|
+
kind = IDENT_KIND[match]
|
111
|
+
value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
|
112
|
+
# TODO: labels
|
113
|
+
if kind == :ident
|
114
|
+
if match.index(?$) # $ allowed inside an identifier
|
115
|
+
kind = :predefined
|
116
|
+
elsif function_expected
|
117
|
+
kind = :function
|
118
|
+
elsif check(/\s*[=:]\s*function\b/)
|
119
|
+
kind = :function
|
120
|
+
elsif key_expected && check(/\s*:/)
|
121
|
+
kind = :key
|
122
|
+
end
|
123
|
+
end
|
124
|
+
function_expected = (kind == :keyword) && (match == 'function')
|
125
|
+
key_expected = false
|
126
|
+
|
127
|
+
elsif match = scan(/["']/)
|
128
|
+
if key_expected && check(KEY_CHECK_PATTERN[match])
|
129
|
+
state = :key
|
130
|
+
else
|
131
|
+
state = :string
|
132
|
+
end
|
133
|
+
tokens << [:open, state]
|
134
|
+
string_delimiter = match
|
135
|
+
kind = :delimiter
|
136
|
+
|
137
|
+
elsif value_expected && (match = scan(/\/(?=\S)/))
|
138
|
+
tokens << [:open, :regexp]
|
139
|
+
state = :regexp
|
140
|
+
string_delimiter = '/'
|
141
|
+
kind = :delimiter
|
142
|
+
|
143
|
+
elsif scan(/ \/ /x)
|
144
|
+
value_expected = true
|
145
|
+
key_expected = false
|
146
|
+
kind = :operator
|
147
|
+
|
148
|
+
else
|
149
|
+
getch
|
150
|
+
kind = :error
|
151
|
+
|
152
|
+
end
|
153
|
+
|
154
|
+
when :string, :regexp, :key
|
155
|
+
if scan(STRING_CONTENT_PATTERN[string_delimiter])
|
156
|
+
kind = :content
|
157
|
+
elsif match = scan(/["'\/]/)
|
158
|
+
tokens << [match, :delimiter]
|
159
|
+
if state == :regexp
|
160
|
+
modifiers = scan(/[gim]+/)
|
161
|
+
tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
|
162
|
+
end
|
163
|
+
tokens << [:close, state]
|
164
|
+
string_delimiter = nil
|
165
|
+
key_expected = value_expected = false
|
166
|
+
state = :initial
|
167
|
+
next
|
168
|
+
elsif state != :regexp && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
|
169
|
+
if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
|
170
|
+
kind = :content
|
171
|
+
else
|
172
|
+
kind = :char
|
173
|
+
end
|
174
|
+
elsif state == :regexp && scan(/ \\ (?: #{ESCAPE} | #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
|
175
|
+
kind = :char
|
176
|
+
elsif scan(/\\./m)
|
177
|
+
kind = :content
|
178
|
+
elsif scan(/ \\ | $ /x)
|
179
|
+
tokens << [:close, state]
|
180
|
+
kind = :error
|
181
|
+
key_expected = value_expected = false
|
182
|
+
state = :initial
|
183
|
+
else
|
184
|
+
raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
|
185
|
+
end
|
186
|
+
|
187
|
+
else
|
188
|
+
raise_inspect 'Unknown state', tokens
|
189
|
+
|
190
|
+
end
|
191
|
+
|
192
|
+
match ||= matched
|
193
|
+
if $DEBUG and not kind
|
194
|
+
raise_inspect 'Error token %p in line %d' %
|
195
|
+
[[match, kind], line], tokens
|
196
|
+
end
|
197
|
+
raise_inspect 'Empty token', tokens unless match
|
198
|
+
|
199
|
+
tokens << [match, kind]
|
200
|
+
|
201
|
+
end
|
202
|
+
|
203
|
+
if [:string, :regexp].include? state
|
204
|
+
tokens << [:close, state]
|
205
|
+
end
|
206
|
+
|
207
|
+
tokens
|
208
|
+
end
|
209
|
+
|
210
|
+
protected
|
211
|
+
|
212
|
+
def reset_instance
|
213
|
+
super
|
214
|
+
@xml_scanner.reset if defined? @xml_scanner
|
215
|
+
end
|
216
|
+
|
217
|
+
def xml_scanner
|
218
|
+
@xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => true
|
219
|
+
end
|
220
|
+
|
221
|
+
end
|
222
|
+
|
223
|
+
end
|
224
|
+
end
|
@@ -0,0 +1,112 @@
|
|
1
|
+
module CodeRay
|
2
|
+
module Scanners
|
3
|
+
|
4
|
+
class JSON < Scanner
|
5
|
+
|
6
|
+
include Streamable
|
7
|
+
|
8
|
+
register_for :json
|
9
|
+
file_extension 'json'
|
10
|
+
|
11
|
+
KINDS_NOT_LOC = [
|
12
|
+
:float, :char, :content, :delimiter,
|
13
|
+
:error, :integer, :operator, :value,
|
14
|
+
]
|
15
|
+
|
16
|
+
CONSTANTS = %w( true false null )
|
17
|
+
IDENT_KIND = WordList.new(:key).add(CONSTANTS, :value)
|
18
|
+
|
19
|
+
ESCAPE = / [bfnrt\\"\/] /x
|
20
|
+
UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x
|
21
|
+
|
22
|
+
def scan_tokens tokens, options
|
23
|
+
|
24
|
+
state = :initial
|
25
|
+
stack = []
|
26
|
+
string_delimiter = nil
|
27
|
+
key_expected = false
|
28
|
+
|
29
|
+
until eos?
|
30
|
+
|
31
|
+
kind = nil
|
32
|
+
match = nil
|
33
|
+
|
34
|
+
case state
|
35
|
+
|
36
|
+
when :initial
|
37
|
+
if match = scan(/ \s+ | \\\n /x)
|
38
|
+
tokens << [match, :space]
|
39
|
+
next
|
40
|
+
elsif match = scan(/ [:,\[{\]}] /x)
|
41
|
+
kind = :operator
|
42
|
+
case match
|
43
|
+
when '{' then stack << :object; key_expected = true
|
44
|
+
when '[' then stack << :array
|
45
|
+
when ':' then key_expected = false
|
46
|
+
when ',' then key_expected = true if stack.last == :object
|
47
|
+
when '}', ']' then stack.pop # no error recovery, but works for valid JSON
|
48
|
+
end
|
49
|
+
elsif match = scan(/ true | false | null /x)
|
50
|
+
kind = IDENT_KIND[match]
|
51
|
+
elsif match = scan(/-?(?:0|[1-9]\d*)/)
|
52
|
+
kind = :integer
|
53
|
+
if scan(/\.\d+(?:[eE][-+]?\d+)?|[eE][-+]?\d+/)
|
54
|
+
match << matched
|
55
|
+
kind = :float
|
56
|
+
end
|
57
|
+
elsif match = scan(/"/)
|
58
|
+
state = key_expected ? :key : :string
|
59
|
+
tokens << [:open, state]
|
60
|
+
kind = :delimiter
|
61
|
+
else
|
62
|
+
getch
|
63
|
+
kind = :error
|
64
|
+
end
|
65
|
+
|
66
|
+
when :string, :key
|
67
|
+
if scan(/[^\\"]+/)
|
68
|
+
kind = :content
|
69
|
+
elsif scan(/"/)
|
70
|
+
tokens << ['"', :delimiter]
|
71
|
+
tokens << [:close, state]
|
72
|
+
state = :initial
|
73
|
+
next
|
74
|
+
elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
|
75
|
+
kind = :char
|
76
|
+
elsif scan(/\\./m)
|
77
|
+
kind = :content
|
78
|
+
elsif scan(/ \\ | $ /x)
|
79
|
+
tokens << [:close, :delimiter]
|
80
|
+
kind = :error
|
81
|
+
state = :initial
|
82
|
+
else
|
83
|
+
raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
|
84
|
+
end
|
85
|
+
|
86
|
+
else
|
87
|
+
raise_inspect 'Unknown state', tokens
|
88
|
+
|
89
|
+
end
|
90
|
+
|
91
|
+
match ||= matched
|
92
|
+
if $DEBUG and not kind
|
93
|
+
raise_inspect 'Error token %p in line %d' %
|
94
|
+
[[match, kind], line], tokens
|
95
|
+
end
|
96
|
+
raise_inspect 'Empty token', tokens unless match
|
97
|
+
|
98
|
+
tokens << [match, kind]
|
99
|
+
|
100
|
+
end
|
101
|
+
|
102
|
+
if [:string, :key].include? state
|
103
|
+
tokens << [:close, state]
|
104
|
+
end
|
105
|
+
|
106
|
+
tokens
|
107
|
+
end
|
108
|
+
|
109
|
+
end
|
110
|
+
|
111
|
+
end
|
112
|
+
end
|
@@ -0,0 +1,136 @@
|
|
1
|
+
module CodeRay
|
2
|
+
module Scanners
|
3
|
+
|
4
|
+
load :html
|
5
|
+
load :ruby
|
6
|
+
|
7
|
+
# Nitro XHTML Scanner
|
8
|
+
class NitroXHTML < Scanner
|
9
|
+
|
10
|
+
include Streamable
|
11
|
+
register_for :nitro_xhtml
|
12
|
+
file_extension :xhtml
|
13
|
+
title 'Nitro XHTML'
|
14
|
+
|
15
|
+
KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
|
16
|
+
|
17
|
+
NITRO_RUBY_BLOCK = /
|
18
|
+
<\?r
|
19
|
+
(?>
|
20
|
+
[^\?]*
|
21
|
+
(?> \?(?!>) [^\?]* )*
|
22
|
+
)
|
23
|
+
(?: \?> )?
|
24
|
+
|
|
25
|
+
<ruby>
|
26
|
+
(?>
|
27
|
+
[^<]*
|
28
|
+
(?> <(?!\/ruby>) [^<]* )*
|
29
|
+
)
|
30
|
+
(?: <\/ruby> )?
|
31
|
+
|
|
32
|
+
<%
|
33
|
+
(?>
|
34
|
+
[^%]*
|
35
|
+
(?> %(?!>) [^%]* )*
|
36
|
+
)
|
37
|
+
(?: %> )?
|
38
|
+
/mx
|
39
|
+
|
40
|
+
NITRO_VALUE_BLOCK = /
|
41
|
+
\#
|
42
|
+
(?:
|
43
|
+
\{
|
44
|
+
[^{}]*
|
45
|
+
(?>
|
46
|
+
\{ [^}]* \}
|
47
|
+
(?> [^{}]* )
|
48
|
+
)*
|
49
|
+
\}?
|
50
|
+
| \| [^|]* \|?
|
51
|
+
| \( [^)]* \)?
|
52
|
+
| \[ [^\]]* \]?
|
53
|
+
| \\ [^\\]* \\?
|
54
|
+
)
|
55
|
+
/x
|
56
|
+
|
57
|
+
NITRO_ENTITY = /
|
58
|
+
% (?: \#\d+ | \w+ ) ;
|
59
|
+
/
|
60
|
+
|
61
|
+
START_OF_RUBY = /
|
62
|
+
(?=[<\#%])
|
63
|
+
< (?: \?r | % | ruby> )
|
64
|
+
| \# [{(|]
|
65
|
+
| % (?: \#\d+ | \w+ ) ;
|
66
|
+
/x
|
67
|
+
|
68
|
+
CLOSING_PAREN = Hash.new do |h, p|
|
69
|
+
h[p] = p
|
70
|
+
end.update( {
|
71
|
+
'(' => ')',
|
72
|
+
'[' => ']',
|
73
|
+
'{' => '}',
|
74
|
+
} )
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
def setup
|
79
|
+
@ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true
|
80
|
+
@html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
|
81
|
+
end
|
82
|
+
|
83
|
+
def reset_instance
|
84
|
+
super
|
85
|
+
@html_scanner.reset
|
86
|
+
end
|
87
|
+
|
88
|
+
def scan_tokens tokens, options
|
89
|
+
|
90
|
+
until eos?
|
91
|
+
|
92
|
+
if (match = scan_until(/(?=#{START_OF_RUBY})/o) || scan_until(/\z/)) and not match.empty?
|
93
|
+
@html_scanner.tokenize match
|
94
|
+
|
95
|
+
elsif match = scan(/#{NITRO_VALUE_BLOCK}/o)
|
96
|
+
start_tag = match[0,2]
|
97
|
+
delimiter = CLOSING_PAREN[start_tag[1,1]]
|
98
|
+
end_tag = match[-1,1] == delimiter ? delimiter : ''
|
99
|
+
tokens << [:open, :inline]
|
100
|
+
tokens << [start_tag, :inline_delimiter]
|
101
|
+
code = match[start_tag.size .. -1 - end_tag.size]
|
102
|
+
@ruby_scanner.tokenize code
|
103
|
+
tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
|
104
|
+
tokens << [:close, :inline]
|
105
|
+
|
106
|
+
elsif match = scan(/#{NITRO_RUBY_BLOCK}/o)
|
107
|
+
start_tag = '<?r'
|
108
|
+
end_tag = match[-2,2] == '?>' ? '?>' : ''
|
109
|
+
tokens << [:open, :inline]
|
110
|
+
tokens << [start_tag, :inline_delimiter]
|
111
|
+
code = match[start_tag.size .. -(end_tag.size)-1]
|
112
|
+
@ruby_scanner.tokenize code
|
113
|
+
tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
|
114
|
+
tokens << [:close, :inline]
|
115
|
+
|
116
|
+
elsif entity = scan(/#{NITRO_ENTITY}/o)
|
117
|
+
tokens << [entity, :entity]
|
118
|
+
|
119
|
+
elsif scan(/%/)
|
120
|
+
tokens << [matched, :error]
|
121
|
+
|
122
|
+
else
|
123
|
+
raise_inspect 'else-case reached!', tokens
|
124
|
+
|
125
|
+
end
|
126
|
+
|
127
|
+
end
|
128
|
+
|
129
|
+
tokens
|
130
|
+
|
131
|
+
end
|
132
|
+
|
133
|
+
end
|
134
|
+
|
135
|
+
end
|
136
|
+
end
|