coderay 1.0.0 → 1.0.0.598.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/FOLDERS +49 -0
- data/Rakefile +6 -5
- data/bin/coderay +74 -190
- data/bin/coderay_stylesheet +4 -0
- data/{README_INDEX.rdoc → lib/README} +20 -10
- data/lib/coderay.rb +60 -62
- data/lib/coderay/duo.rb +55 -2
- data/lib/coderay/encoder.rb +39 -52
- data/lib/coderay/encoders/_map.rb +7 -11
- data/lib/coderay/encoders/comment_filter.rb +61 -0
- data/lib/coderay/encoders/count.rb +26 -11
- data/lib/coderay/encoders/debug.rb +60 -11
- data/lib/coderay/encoders/div.rb +8 -9
- data/lib/coderay/encoders/filter.rb +52 -12
- data/lib/coderay/encoders/html.rb +113 -106
- data/lib/coderay/encoders/html/css.rb +7 -2
- data/lib/coderay/encoders/html/numbering.rb +27 -24
- data/lib/coderay/encoders/html/output.rb +58 -15
- data/lib/coderay/encoders/json.rb +44 -37
- data/lib/coderay/encoders/lines_of_code.rb +56 -9
- data/lib/coderay/encoders/null.rb +13 -6
- data/lib/coderay/encoders/page.rb +8 -8
- data/lib/coderay/encoders/span.rb +9 -10
- data/lib/coderay/encoders/statistic.rb +114 -51
- data/lib/coderay/encoders/terminal.rb +10 -7
- data/lib/coderay/encoders/text.rb +36 -17
- data/lib/coderay/encoders/token_kind_filter.rb +58 -1
- data/lib/coderay/encoders/xml.rb +11 -13
- data/lib/coderay/encoders/yaml.rb +14 -16
- data/lib/coderay/for_redcloth.rb +1 -1
- data/lib/coderay/helpers/file_type.rb +240 -125
- data/lib/coderay/helpers/gzip_simple.rb +123 -0
- data/lib/coderay/helpers/plugin.rb +307 -241
- data/lib/coderay/helpers/word_list.rb +126 -65
- data/lib/coderay/scanner.rb +103 -153
- data/lib/coderay/scanners/_map.rb +16 -18
- data/lib/coderay/scanners/c.rb +13 -13
- data/lib/coderay/scanners/cpp.rb +6 -6
- data/lib/coderay/scanners/css.rb +48 -47
- data/lib/coderay/scanners/debug.rb +55 -9
- data/lib/coderay/scanners/delphi.rb +4 -4
- data/lib/coderay/scanners/diff.rb +25 -43
- data/lib/coderay/scanners/groovy.rb +2 -2
- data/lib/coderay/scanners/html.rb +30 -107
- data/lib/coderay/scanners/java.rb +5 -6
- data/lib/coderay/scanners/java/builtin_types.rb +0 -2
- data/lib/coderay/scanners/java_script.rb +6 -6
- data/lib/coderay/scanners/json.rb +6 -7
- data/lib/coderay/scanners/nitro_xhtml.rb +136 -0
- data/lib/coderay/scanners/php.rb +12 -13
- data/lib/coderay/scanners/plaintext.rb +26 -0
- data/lib/coderay/scanners/python.rb +4 -4
- data/lib/coderay/scanners/{erb.rb → rhtml.rb} +11 -19
- data/lib/coderay/scanners/ruby.rb +208 -219
- data/lib/coderay/scanners/ruby/patterns.rb +85 -18
- data/lib/coderay/scanners/scheme.rb +136 -0
- data/lib/coderay/scanners/sql.rb +22 -29
- data/lib/coderay/scanners/yaml.rb +10 -11
- data/lib/coderay/styles/_map.rb +2 -2
- data/lib/coderay/styles/alpha.rb +104 -102
- data/lib/coderay/styles/cycnus.rb +143 -0
- data/lib/coderay/styles/murphy.rb +123 -0
- data/lib/coderay/token_kinds.rb +86 -87
- data/lib/coderay/tokens.rb +169 -26
- data/test/functional/basic.rb +14 -200
- data/test/functional/examples.rb +14 -20
- data/test/functional/for_redcloth.rb +8 -15
- data/test/functional/load_plugin_scanner.rb +11 -0
- data/test/functional/suite.rb +6 -9
- data/test/functional/vhdl.rb +126 -0
- data/test/functional/word_list.rb +79 -0
- metadata +129 -107
- data/lib/coderay/helpers/gzip.rb +0 -41
- data/lib/coderay/scanners/clojure.rb +0 -217
- data/lib/coderay/scanners/haml.rb +0 -168
- data/lib/coderay/scanners/ruby/string_state.rb +0 -71
- data/lib/coderay/scanners/text.rb +0 -26
- data/lib/coderay/tokens_proxy.rb +0 -55
- data/lib/coderay/version.rb +0 -3
data/lib/coderay/token_kinds.rb
CHANGED
@@ -1,90 +1,89 @@
|
|
1
1
|
module CodeRay
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
2
|
+
class Tokens
|
3
|
+
AbbreviationForKind = Hash.new do |h, k| # :nodoc:
|
4
|
+
if $CODERAY_DEBUG
|
5
|
+
raise 'Undefined Token kind: %p' % [k] # :nodoc:
|
6
|
+
else
|
7
|
+
:NO_HIGHLIGHT
|
8
|
+
end
|
9
|
+
end
|
10
|
+
AbbreviationForKind.update with = { # :nodoc:
|
11
|
+
:annotation => 'at',
|
12
|
+
:attribute_name => 'an',
|
13
|
+
:attribute_value => 'av',
|
14
|
+
:bin => 'bi',
|
15
|
+
:char => 'ch',
|
16
|
+
:class => 'cl',
|
17
|
+
:class_variable => 'cv',
|
18
|
+
:color => 'cr',
|
19
|
+
:comment => 'c',
|
20
|
+
:complex => 'cm',
|
21
|
+
:constant => 'co',
|
22
|
+
:content => 'k',
|
23
|
+
:decorator => 'de',
|
24
|
+
:definition => 'df',
|
25
|
+
:delimiter => 'dl',
|
26
|
+
:directive => 'di',
|
27
|
+
:doc => 'do',
|
28
|
+
:doctype => 'dt',
|
29
|
+
:doc_string => 'ds',
|
30
|
+
:entity => 'en',
|
31
|
+
:error => 'er',
|
32
|
+
:escape => 'e',
|
33
|
+
:exception => 'ex',
|
34
|
+
:filename => 'filename',
|
35
|
+
:float => 'fl',
|
36
|
+
:function => 'fu',
|
37
|
+
:global_variable => 'gv',
|
38
|
+
:hex => 'hx',
|
39
|
+
:imaginary => 'cm',
|
40
|
+
:important => 'im',
|
41
|
+
:include => 'ic',
|
42
|
+
:inline => 'il',
|
43
|
+
:inline_delimiter => 'idl',
|
44
|
+
:instance_variable => 'iv',
|
45
|
+
:integer => 'i',
|
46
|
+
:interpreted => 'in',
|
47
|
+
:key => 'ke',
|
48
|
+
:keyword => 'kw',
|
49
|
+
:label => 'la',
|
50
|
+
:local_variable => 'lv',
|
51
|
+
:modifier => 'mod',
|
52
|
+
:namespace => 'ns',
|
53
|
+
:oct => 'oc',
|
54
|
+
:predefined => 'pd',
|
55
|
+
:preprocessor => 'pp',
|
56
|
+
:pre_constant => 'pc',
|
57
|
+
:pre_type => 'pt',
|
58
|
+
:pseudo_class => 'ps',
|
59
|
+
:regexp => 'rx',
|
60
|
+
:reserved => 'r',
|
61
|
+
:shell => 'sh',
|
62
|
+
:string => 's',
|
63
|
+
:symbol => 'sy',
|
64
|
+
:tag => 'ta',
|
65
|
+
:tag_special => 'ts',
|
66
|
+
:type => 'ty',
|
67
|
+
:value => 'vl',
|
68
|
+
:variable => 'v',
|
69
|
+
|
70
|
+
:insert => 'ins',
|
71
|
+
:delete => 'del',
|
72
|
+
:change => 'chg',
|
73
|
+
:head => 'head',
|
74
|
+
|
75
|
+
:eyecatcher => 'eye',
|
76
|
+
|
77
|
+
:ident => :NO_HIGHLIGHT, # 'id'
|
78
|
+
#:operator => 'op',
|
79
|
+
:operator => :NO_HIGHLIGHT, # 'op'
|
80
|
+
:space => :NO_HIGHLIGHT, # 'sp'
|
81
|
+
:plain => :NO_HIGHLIGHT,
|
82
|
+
}
|
83
|
+
AbbreviationForKind[:method] = AbbreviationForKind[:function]
|
84
|
+
AbbreviationForKind[:nesting_delimiter] = AbbreviationForKind[:delimiter]
|
85
|
+
AbbreviationForKind[:escape] = AbbreviationForKind[:delimiter]
|
86
|
+
AbbreviationForKind[:docstring] = AbbreviationForKind[:comment]
|
87
|
+
#AbbreviationForKind.default = AbbreviationForKind[:error] or raise 'no class found for :error!'
|
7
88
|
end
|
8
|
-
|
9
|
-
# speedup
|
10
|
-
TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity
|
11
|
-
|
12
|
-
TokenKinds.update( # :nodoc:
|
13
|
-
:annotation => 'annotation',
|
14
|
-
:attribute_name => 'attribute-name',
|
15
|
-
:attribute_value => 'attribute-value',
|
16
|
-
:binary => 'bin',
|
17
|
-
:char => 'char',
|
18
|
-
:class => 'class',
|
19
|
-
:class_variable => 'class-variable',
|
20
|
-
:color => 'color',
|
21
|
-
:comment => 'comment',
|
22
|
-
:complex => 'complex',
|
23
|
-
:constant => 'constant',
|
24
|
-
:content => 'content',
|
25
|
-
:debug => 'debug',
|
26
|
-
:decorator => 'decorator',
|
27
|
-
:definition => 'definition',
|
28
|
-
:delimiter => 'delimiter',
|
29
|
-
:directive => 'directive',
|
30
|
-
:doc => 'doc',
|
31
|
-
:doctype => 'doctype',
|
32
|
-
:doc_string => 'doc-string',
|
33
|
-
:entity => 'entity',
|
34
|
-
:error => 'error',
|
35
|
-
:escape => 'escape',
|
36
|
-
:exception => 'exception',
|
37
|
-
:filename => 'filename',
|
38
|
-
:float => 'float',
|
39
|
-
:function => 'function',
|
40
|
-
:global_variable => 'global-variable',
|
41
|
-
:hex => 'hex',
|
42
|
-
:imaginary => 'imaginary',
|
43
|
-
:important => 'important',
|
44
|
-
:include => 'include',
|
45
|
-
:inline => 'inline',
|
46
|
-
:inline_delimiter => 'inline-delimiter',
|
47
|
-
:instance_variable => 'instance-variable',
|
48
|
-
:integer => 'integer',
|
49
|
-
:key => 'key',
|
50
|
-
:keyword => 'keyword',
|
51
|
-
:label => 'label',
|
52
|
-
:local_variable => 'local-variable',
|
53
|
-
:modifier => 'modifier',
|
54
|
-
:namespace => 'namespace',
|
55
|
-
:octal => 'octal',
|
56
|
-
:predefined => 'predefined',
|
57
|
-
:predefined_constant => 'predefined-constant',
|
58
|
-
:predefined_type => 'predefined-type',
|
59
|
-
:preprocessor => 'preprocessor',
|
60
|
-
:pseudo_class => 'pseudo-class',
|
61
|
-
:regexp => 'regexp',
|
62
|
-
:reserved => 'reserved',
|
63
|
-
:shell => 'shell',
|
64
|
-
:string => 'string',
|
65
|
-
:symbol => 'symbol',
|
66
|
-
:tag => 'tag',
|
67
|
-
:type => 'type',
|
68
|
-
:value => 'value',
|
69
|
-
:variable => 'variable',
|
70
|
-
|
71
|
-
:change => 'change',
|
72
|
-
:delete => 'delete',
|
73
|
-
:head => 'head',
|
74
|
-
:insert => 'insert',
|
75
|
-
|
76
|
-
:eyecatcher => 'eyecatcher',
|
77
|
-
|
78
|
-
:ident => false,
|
79
|
-
:operator => false,
|
80
|
-
|
81
|
-
:space => false,
|
82
|
-
:plain => false
|
83
|
-
)
|
84
|
-
|
85
|
-
TokenKinds[:method] = TokenKinds[:function]
|
86
|
-
TokenKinds[:escape] = TokenKinds[:delimiter]
|
87
|
-
TokenKinds[:docstring] = TokenKinds[:comment]
|
88
|
-
|
89
|
-
TokenKinds.freeze
|
90
89
|
end
|
data/lib/coderay/tokens.rb
CHANGED
@@ -1,8 +1,5 @@
|
|
1
1
|
module CodeRay
|
2
|
-
|
3
|
-
# GZip library for writing and reading token dumps.
|
4
|
-
autoload :GZip, 'coderay/helpers/gzip'
|
5
|
-
|
2
|
+
|
6
3
|
# = Tokens TODO: Rewrite!
|
7
4
|
#
|
8
5
|
# The Tokens class represents a list of tokens returnd from
|
@@ -64,25 +61,127 @@ module CodeRay
|
|
64
61
|
#
|
65
62
|
# options are passed to the encoder.
|
66
63
|
def encode encoder, options = {}
|
67
|
-
|
64
|
+
unless encoder.is_a? Encoders::Encoder
|
65
|
+
unless encoder.is_a? Class
|
66
|
+
encoder_class = Encoders[encoder]
|
67
|
+
end
|
68
|
+
encoder = encoder_class.new options
|
69
|
+
end
|
68
70
|
encoder.encode_tokens self, options
|
69
71
|
end
|
70
|
-
|
71
|
-
# Turn
|
72
|
-
|
73
|
-
|
72
|
+
|
73
|
+
# Turn into a string using Encoders::Text.
|
74
|
+
#
|
75
|
+
# +options+ are passed to the encoder if given.
|
76
|
+
def to_s options = {}
|
77
|
+
encode :text, options
|
74
78
|
end
|
75
|
-
|
79
|
+
|
76
80
|
# Redirects unknown methods to encoder calls.
|
77
81
|
#
|
78
82
|
# For example, if you call +tokens.html+, the HTML encoder
|
79
83
|
# is used to highlight the tokens.
|
80
84
|
def method_missing meth, options = {}
|
81
|
-
|
85
|
+
encode_with meth, options
|
82
86
|
rescue PluginHost::PluginNotFound
|
83
87
|
super
|
84
88
|
end
|
85
89
|
|
90
|
+
def encode_with encoder, options = {}
|
91
|
+
Encoders[encoder].new(options).encode_tokens self
|
92
|
+
end
|
93
|
+
|
94
|
+
# Returns the tokens compressed by joining consecutive
|
95
|
+
# tokens of the same kind.
|
96
|
+
#
|
97
|
+
# This can not be undone, but should yield the same output
|
98
|
+
# in most Encoders. It basically makes the output smaller.
|
99
|
+
#
|
100
|
+
# Combined with dump, it saves space for the cost of time.
|
101
|
+
#
|
102
|
+
# If the scanner is written carefully, this is not required -
|
103
|
+
# for example, consecutive //-comment lines could already be
|
104
|
+
# joined in one comment token by the Scanner.
|
105
|
+
def optimize
|
106
|
+
raise NotImplementedError, 'Tokens#optimize needs to be rewritten.'
|
107
|
+
last_kind = last_text = nil
|
108
|
+
new = self.class.new
|
109
|
+
for text, kind in self
|
110
|
+
if text.is_a? String
|
111
|
+
if kind == last_kind
|
112
|
+
last_text << text
|
113
|
+
else
|
114
|
+
new << [last_text, last_kind] if last_kind
|
115
|
+
last_text = text
|
116
|
+
last_kind = kind
|
117
|
+
end
|
118
|
+
else
|
119
|
+
new << [last_text, last_kind] if last_kind
|
120
|
+
last_kind = last_text = nil
|
121
|
+
new << [text, kind]
|
122
|
+
end
|
123
|
+
end
|
124
|
+
new << [last_text, last_kind] if last_kind
|
125
|
+
new
|
126
|
+
end
|
127
|
+
|
128
|
+
# Compact the object itself; see optimize.
|
129
|
+
def optimize!
|
130
|
+
replace optimize
|
131
|
+
end
|
132
|
+
|
133
|
+
# Ensure that all begin_group tokens have a correspondent end_group.
|
134
|
+
#
|
135
|
+
# TODO: Test this!
|
136
|
+
def fix
|
137
|
+
raise NotImplementedError, 'Tokens#fix needs to be rewritten.'
|
138
|
+
tokens = self.class.new
|
139
|
+
# Check token nesting using a stack of kinds.
|
140
|
+
opened = []
|
141
|
+
for type, kind in self
|
142
|
+
case type
|
143
|
+
when :begin_group
|
144
|
+
opened.push [:begin_group, kind]
|
145
|
+
when :begin_line
|
146
|
+
opened.push [:end_line, kind]
|
147
|
+
when :end_group, :end_line
|
148
|
+
expected = opened.pop
|
149
|
+
if [type, kind] != expected
|
150
|
+
# Unexpected end; decide what to do based on the kind:
|
151
|
+
# - token was never opened: delete the end (just skip it)
|
152
|
+
next unless opened.rindex expected
|
153
|
+
# - token was opened earlier: also close tokens in between
|
154
|
+
tokens << token until (token = opened.pop) == expected
|
155
|
+
end
|
156
|
+
end
|
157
|
+
tokens << [type, kind]
|
158
|
+
end
|
159
|
+
# Close remaining opened tokens
|
160
|
+
tokens << token while token = opened.pop
|
161
|
+
tokens
|
162
|
+
end
|
163
|
+
|
164
|
+
def fix!
|
165
|
+
replace fix
|
166
|
+
end
|
167
|
+
|
168
|
+
# TODO: Scanner#split_into_lines
|
169
|
+
#
|
170
|
+
# Makes sure that:
|
171
|
+
# - newlines are single tokens
|
172
|
+
# (which means all other token are single-line)
|
173
|
+
# - there are no open tokens at the end the line
|
174
|
+
#
|
175
|
+
# This makes it simple for encoders that work line-oriented,
|
176
|
+
# like HTML with list-style numeration.
|
177
|
+
def split_into_lines
|
178
|
+
raise NotImplementedError
|
179
|
+
end
|
180
|
+
|
181
|
+
def split_into_lines!
|
182
|
+
replace split_into_lines
|
183
|
+
end
|
184
|
+
|
86
185
|
# Split the tokens into parts of the given +sizes+.
|
87
186
|
#
|
88
187
|
# The result will be an Array of Tokens objects. The parts have
|
@@ -121,15 +220,12 @@ module CodeRay
|
|
121
220
|
content_or_kind
|
122
221
|
end
|
123
222
|
end
|
124
|
-
part.concat
|
125
|
-
|
126
|
-
parts << part
|
127
|
-
part = Tokens.new
|
128
|
-
size = sizes[i += 1]
|
129
|
-
end until size.nil? || size > 0
|
223
|
+
parts << part.concat(closing)
|
224
|
+
part = Tokens.new
|
130
225
|
# ...and open them again.
|
131
226
|
part.concat opened.flatten
|
132
227
|
part_size = 0
|
228
|
+
size = sizes[i += 1]
|
133
229
|
redo unless content.empty?
|
134
230
|
else
|
135
231
|
part << content << item
|
@@ -143,19 +239,19 @@ module CodeRay
|
|
143
239
|
when :end_group, :end_line
|
144
240
|
opened.pop
|
145
241
|
else
|
146
|
-
raise
|
242
|
+
raise 'Unknown token action: %p, kind = %p' % [content, item]
|
147
243
|
end
|
148
244
|
part << content << item
|
149
245
|
content = nil
|
150
246
|
else
|
151
|
-
raise
|
247
|
+
raise 'else case reached'
|
152
248
|
end
|
153
249
|
end
|
154
250
|
parts << part
|
155
251
|
parts << Tokens.new while parts.size < sizes.size
|
156
252
|
parts
|
157
253
|
end
|
158
|
-
|
254
|
+
|
159
255
|
# Dumps the object into a String that can be saved
|
160
256
|
# in files or databases.
|
161
257
|
#
|
@@ -172,8 +268,9 @@ module CodeRay
|
|
172
268
|
#
|
173
269
|
# See GZip module.
|
174
270
|
def dump gzip_level = 7
|
271
|
+
require 'coderay/helpers/gzip_simple'
|
175
272
|
dump = Marshal.dump self
|
176
|
-
dump =
|
273
|
+
dump = dump.gzip gzip_level
|
177
274
|
dump.extend Undumping
|
178
275
|
end
|
179
276
|
|
@@ -181,7 +278,7 @@ module CodeRay
|
|
181
278
|
def count
|
182
279
|
size / 2
|
183
280
|
end
|
184
|
-
|
281
|
+
|
185
282
|
# Include this module to give an object an #undump
|
186
283
|
# method.
|
187
284
|
#
|
@@ -192,17 +289,18 @@ module CodeRay
|
|
192
289
|
Tokens.load self
|
193
290
|
end
|
194
291
|
end
|
195
|
-
|
292
|
+
|
196
293
|
# Undump the object using Marshal.load, then
|
197
294
|
# unzip it using GZip.gunzip.
|
198
295
|
#
|
199
296
|
# The result is commonly a Tokens object, but
|
200
297
|
# this is not guaranteed.
|
201
298
|
def Tokens.load dump
|
202
|
-
|
299
|
+
require 'coderay/helpers/gzip_simple'
|
300
|
+
dump = dump.gunzip
|
203
301
|
@dump = Marshal.load dump
|
204
302
|
end
|
205
|
-
|
303
|
+
|
206
304
|
alias text_token push
|
207
305
|
def begin_group kind; push :begin_group, kind end
|
208
306
|
def end_group kind; push :end_group, kind end
|
@@ -211,5 +309,50 @@ module CodeRay
|
|
211
309
|
alias tokens concat
|
212
310
|
|
213
311
|
end
|
214
|
-
|
312
|
+
|
215
313
|
end
|
314
|
+
|
315
|
+
if $0 == __FILE__
|
316
|
+
$VERBOSE = true
|
317
|
+
$: << File.join(File.dirname(__FILE__), '..')
|
318
|
+
eval DATA.read, nil, $0, __LINE__ + 4
|
319
|
+
end
|
320
|
+
|
321
|
+
__END__
|
322
|
+
require 'test/unit'
|
323
|
+
|
324
|
+
class TokensTest < Test::Unit::TestCase
|
325
|
+
|
326
|
+
def test_creation
|
327
|
+
assert CodeRay::Tokens < Array
|
328
|
+
tokens = nil
|
329
|
+
assert_nothing_raised do
|
330
|
+
tokens = CodeRay::Tokens.new
|
331
|
+
end
|
332
|
+
assert_kind_of Array, tokens
|
333
|
+
end
|
334
|
+
|
335
|
+
def test_adding_tokens
|
336
|
+
tokens = CodeRay::Tokens.new
|
337
|
+
assert_nothing_raised do
|
338
|
+
tokens.text_token 'string', :type
|
339
|
+
tokens.text_token '()', :operator
|
340
|
+
end
|
341
|
+
assert_equal tokens.size, 4
|
342
|
+
assert_equal tokens.count, 2
|
343
|
+
end
|
344
|
+
|
345
|
+
def test_dump_undump
|
346
|
+
tokens = CodeRay::Tokens.new
|
347
|
+
assert_nothing_raised do
|
348
|
+
tokens.text_token 'string', :type
|
349
|
+
tokens.text_token '()', :operator
|
350
|
+
end
|
351
|
+
tokens2 = nil
|
352
|
+
assert_nothing_raised do
|
353
|
+
tokens2 = tokens.dump.undump
|
354
|
+
end
|
355
|
+
assert_equal tokens, tokens2
|
356
|
+
end
|
357
|
+
|
358
|
+
end
|