coderay 1.0.0 → 1.0.0.598.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. data/FOLDERS +49 -0
  2. data/Rakefile +6 -5
  3. data/bin/coderay +74 -190
  4. data/bin/coderay_stylesheet +4 -0
  5. data/{README_INDEX.rdoc → lib/README} +20 -10
  6. data/lib/coderay.rb +60 -62
  7. data/lib/coderay/duo.rb +55 -2
  8. data/lib/coderay/encoder.rb +39 -52
  9. data/lib/coderay/encoders/_map.rb +7 -11
  10. data/lib/coderay/encoders/comment_filter.rb +61 -0
  11. data/lib/coderay/encoders/count.rb +26 -11
  12. data/lib/coderay/encoders/debug.rb +60 -11
  13. data/lib/coderay/encoders/div.rb +8 -9
  14. data/lib/coderay/encoders/filter.rb +52 -12
  15. data/lib/coderay/encoders/html.rb +113 -106
  16. data/lib/coderay/encoders/html/css.rb +7 -2
  17. data/lib/coderay/encoders/html/numbering.rb +27 -24
  18. data/lib/coderay/encoders/html/output.rb +58 -15
  19. data/lib/coderay/encoders/json.rb +44 -37
  20. data/lib/coderay/encoders/lines_of_code.rb +56 -9
  21. data/lib/coderay/encoders/null.rb +13 -6
  22. data/lib/coderay/encoders/page.rb +8 -8
  23. data/lib/coderay/encoders/span.rb +9 -10
  24. data/lib/coderay/encoders/statistic.rb +114 -51
  25. data/lib/coderay/encoders/terminal.rb +10 -7
  26. data/lib/coderay/encoders/text.rb +36 -17
  27. data/lib/coderay/encoders/token_kind_filter.rb +58 -1
  28. data/lib/coderay/encoders/xml.rb +11 -13
  29. data/lib/coderay/encoders/yaml.rb +14 -16
  30. data/lib/coderay/for_redcloth.rb +1 -1
  31. data/lib/coderay/helpers/file_type.rb +240 -125
  32. data/lib/coderay/helpers/gzip_simple.rb +123 -0
  33. data/lib/coderay/helpers/plugin.rb +307 -241
  34. data/lib/coderay/helpers/word_list.rb +126 -65
  35. data/lib/coderay/scanner.rb +103 -153
  36. data/lib/coderay/scanners/_map.rb +16 -18
  37. data/lib/coderay/scanners/c.rb +13 -13
  38. data/lib/coderay/scanners/cpp.rb +6 -6
  39. data/lib/coderay/scanners/css.rb +48 -47
  40. data/lib/coderay/scanners/debug.rb +55 -9
  41. data/lib/coderay/scanners/delphi.rb +4 -4
  42. data/lib/coderay/scanners/diff.rb +25 -43
  43. data/lib/coderay/scanners/groovy.rb +2 -2
  44. data/lib/coderay/scanners/html.rb +30 -107
  45. data/lib/coderay/scanners/java.rb +5 -6
  46. data/lib/coderay/scanners/java/builtin_types.rb +0 -2
  47. data/lib/coderay/scanners/java_script.rb +6 -6
  48. data/lib/coderay/scanners/json.rb +6 -7
  49. data/lib/coderay/scanners/nitro_xhtml.rb +136 -0
  50. data/lib/coderay/scanners/php.rb +12 -13
  51. data/lib/coderay/scanners/plaintext.rb +26 -0
  52. data/lib/coderay/scanners/python.rb +4 -4
  53. data/lib/coderay/scanners/{erb.rb → rhtml.rb} +11 -19
  54. data/lib/coderay/scanners/ruby.rb +208 -219
  55. data/lib/coderay/scanners/ruby/patterns.rb +85 -18
  56. data/lib/coderay/scanners/scheme.rb +136 -0
  57. data/lib/coderay/scanners/sql.rb +22 -29
  58. data/lib/coderay/scanners/yaml.rb +10 -11
  59. data/lib/coderay/styles/_map.rb +2 -2
  60. data/lib/coderay/styles/alpha.rb +104 -102
  61. data/lib/coderay/styles/cycnus.rb +143 -0
  62. data/lib/coderay/styles/murphy.rb +123 -0
  63. data/lib/coderay/token_kinds.rb +86 -87
  64. data/lib/coderay/tokens.rb +169 -26
  65. data/test/functional/basic.rb +14 -200
  66. data/test/functional/examples.rb +14 -20
  67. data/test/functional/for_redcloth.rb +8 -15
  68. data/test/functional/load_plugin_scanner.rb +11 -0
  69. data/test/functional/suite.rb +6 -9
  70. data/test/functional/vhdl.rb +126 -0
  71. data/test/functional/word_list.rb +79 -0
  72. metadata +129 -107
  73. data/lib/coderay/helpers/gzip.rb +0 -41
  74. data/lib/coderay/scanners/clojure.rb +0 -217
  75. data/lib/coderay/scanners/haml.rb +0 -168
  76. data/lib/coderay/scanners/ruby/string_state.rb +0 -71
  77. data/lib/coderay/scanners/text.rb +0 -26
  78. data/lib/coderay/tokens_proxy.rb +0 -55
  79. data/lib/coderay/version.rb +0 -3
@@ -1,90 +1,89 @@
1
1
  module CodeRay
2
-
3
- # A Hash of all known token kinds and their associated CSS classes.
4
- TokenKinds = Hash.new do |h, k|
5
- warn 'Undefined Token kind: %p' % [k] if $CODERAY_DEBUG
6
- false
2
+ class Tokens
3
+ AbbreviationForKind = Hash.new do |h, k| # :nodoc:
4
+ if $CODERAY_DEBUG
5
+ raise 'Undefined Token kind: %p' % [k] # :nodoc:
6
+ else
7
+ :NO_HIGHLIGHT
8
+ end
9
+ end
10
+ AbbreviationForKind.update with = { # :nodoc:
11
+ :annotation => 'at',
12
+ :attribute_name => 'an',
13
+ :attribute_value => 'av',
14
+ :bin => 'bi',
15
+ :char => 'ch',
16
+ :class => 'cl',
17
+ :class_variable => 'cv',
18
+ :color => 'cr',
19
+ :comment => 'c',
20
+ :complex => 'cm',
21
+ :constant => 'co',
22
+ :content => 'k',
23
+ :decorator => 'de',
24
+ :definition => 'df',
25
+ :delimiter => 'dl',
26
+ :directive => 'di',
27
+ :doc => 'do',
28
+ :doctype => 'dt',
29
+ :doc_string => 'ds',
30
+ :entity => 'en',
31
+ :error => 'er',
32
+ :escape => 'e',
33
+ :exception => 'ex',
34
+ :filename => 'filename',
35
+ :float => 'fl',
36
+ :function => 'fu',
37
+ :global_variable => 'gv',
38
+ :hex => 'hx',
39
+ :imaginary => 'cm',
40
+ :important => 'im',
41
+ :include => 'ic',
42
+ :inline => 'il',
43
+ :inline_delimiter => 'idl',
44
+ :instance_variable => 'iv',
45
+ :integer => 'i',
46
+ :interpreted => 'in',
47
+ :key => 'ke',
48
+ :keyword => 'kw',
49
+ :label => 'la',
50
+ :local_variable => 'lv',
51
+ :modifier => 'mod',
52
+ :namespace => 'ns',
53
+ :oct => 'oc',
54
+ :predefined => 'pd',
55
+ :preprocessor => 'pp',
56
+ :pre_constant => 'pc',
57
+ :pre_type => 'pt',
58
+ :pseudo_class => 'ps',
59
+ :regexp => 'rx',
60
+ :reserved => 'r',
61
+ :shell => 'sh',
62
+ :string => 's',
63
+ :symbol => 'sy',
64
+ :tag => 'ta',
65
+ :tag_special => 'ts',
66
+ :type => 'ty',
67
+ :value => 'vl',
68
+ :variable => 'v',
69
+
70
+ :insert => 'ins',
71
+ :delete => 'del',
72
+ :change => 'chg',
73
+ :head => 'head',
74
+
75
+ :eyecatcher => 'eye',
76
+
77
+ :ident => :NO_HIGHLIGHT, # 'id'
78
+ #:operator => 'op',
79
+ :operator => :NO_HIGHLIGHT, # 'op'
80
+ :space => :NO_HIGHLIGHT, # 'sp'
81
+ :plain => :NO_HIGHLIGHT,
82
+ }
83
+ AbbreviationForKind[:method] = AbbreviationForKind[:function]
84
+ AbbreviationForKind[:nesting_delimiter] = AbbreviationForKind[:delimiter]
85
+ AbbreviationForKind[:escape] = AbbreviationForKind[:delimiter]
86
+ AbbreviationForKind[:docstring] = AbbreviationForKind[:comment]
87
+ #AbbreviationForKind.default = AbbreviationForKind[:error] or raise 'no class found for :error!'
7
88
  end
8
-
9
- # speedup
10
- TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity
11
-
12
- TokenKinds.update( # :nodoc:
13
- :annotation => 'annotation',
14
- :attribute_name => 'attribute-name',
15
- :attribute_value => 'attribute-value',
16
- :binary => 'bin',
17
- :char => 'char',
18
- :class => 'class',
19
- :class_variable => 'class-variable',
20
- :color => 'color',
21
- :comment => 'comment',
22
- :complex => 'complex',
23
- :constant => 'constant',
24
- :content => 'content',
25
- :debug => 'debug',
26
- :decorator => 'decorator',
27
- :definition => 'definition',
28
- :delimiter => 'delimiter',
29
- :directive => 'directive',
30
- :doc => 'doc',
31
- :doctype => 'doctype',
32
- :doc_string => 'doc-string',
33
- :entity => 'entity',
34
- :error => 'error',
35
- :escape => 'escape',
36
- :exception => 'exception',
37
- :filename => 'filename',
38
- :float => 'float',
39
- :function => 'function',
40
- :global_variable => 'global-variable',
41
- :hex => 'hex',
42
- :imaginary => 'imaginary',
43
- :important => 'important',
44
- :include => 'include',
45
- :inline => 'inline',
46
- :inline_delimiter => 'inline-delimiter',
47
- :instance_variable => 'instance-variable',
48
- :integer => 'integer',
49
- :key => 'key',
50
- :keyword => 'keyword',
51
- :label => 'label',
52
- :local_variable => 'local-variable',
53
- :modifier => 'modifier',
54
- :namespace => 'namespace',
55
- :octal => 'octal',
56
- :predefined => 'predefined',
57
- :predefined_constant => 'predefined-constant',
58
- :predefined_type => 'predefined-type',
59
- :preprocessor => 'preprocessor',
60
- :pseudo_class => 'pseudo-class',
61
- :regexp => 'regexp',
62
- :reserved => 'reserved',
63
- :shell => 'shell',
64
- :string => 'string',
65
- :symbol => 'symbol',
66
- :tag => 'tag',
67
- :type => 'type',
68
- :value => 'value',
69
- :variable => 'variable',
70
-
71
- :change => 'change',
72
- :delete => 'delete',
73
- :head => 'head',
74
- :insert => 'insert',
75
-
76
- :eyecatcher => 'eyecatcher',
77
-
78
- :ident => false,
79
- :operator => false,
80
-
81
- :space => false,
82
- :plain => false
83
- )
84
-
85
- TokenKinds[:method] = TokenKinds[:function]
86
- TokenKinds[:escape] = TokenKinds[:delimiter]
87
- TokenKinds[:docstring] = TokenKinds[:comment]
88
-
89
- TokenKinds.freeze
90
89
  end
@@ -1,8 +1,5 @@
1
1
  module CodeRay
2
-
3
- # GZip library for writing and reading token dumps.
4
- autoload :GZip, 'coderay/helpers/gzip'
5
-
2
+
6
3
  # = Tokens TODO: Rewrite!
7
4
  #
8
5
  # The Tokens class represents a list of tokens returnd from
@@ -64,25 +61,127 @@ module CodeRay
64
61
  #
65
62
  # options are passed to the encoder.
66
63
  def encode encoder, options = {}
67
- encoder = Encoders[encoder].new options if encoder.respond_to? :to_sym
64
+ unless encoder.is_a? Encoders::Encoder
65
+ unless encoder.is_a? Class
66
+ encoder_class = Encoders[encoder]
67
+ end
68
+ encoder = encoder_class.new options
69
+ end
68
70
  encoder.encode_tokens self, options
69
71
  end
70
-
71
- # Turn tokens into a string by concatenating them.
72
- def to_s
73
- encode CodeRay::Encoders::Encoder.new
72
+
73
+ # Turn into a string using Encoders::Text.
74
+ #
75
+ # +options+ are passed to the encoder if given.
76
+ def to_s options = {}
77
+ encode :text, options
74
78
  end
75
-
79
+
76
80
  # Redirects unknown methods to encoder calls.
77
81
  #
78
82
  # For example, if you call +tokens.html+, the HTML encoder
79
83
  # is used to highlight the tokens.
80
84
  def method_missing meth, options = {}
81
- encode meth, options
85
+ encode_with meth, options
82
86
  rescue PluginHost::PluginNotFound
83
87
  super
84
88
  end
85
89
 
90
+ def encode_with encoder, options = {}
91
+ Encoders[encoder].new(options).encode_tokens self
92
+ end
93
+
94
+ # Returns the tokens compressed by joining consecutive
95
+ # tokens of the same kind.
96
+ #
97
+ # This can not be undone, but should yield the same output
98
+ # in most Encoders. It basically makes the output smaller.
99
+ #
100
+ # Combined with dump, it saves space for the cost of time.
101
+ #
102
+ # If the scanner is written carefully, this is not required -
103
+ # for example, consecutive //-comment lines could already be
104
+ # joined in one comment token by the Scanner.
105
+ def optimize
106
+ raise NotImplementedError, 'Tokens#optimize needs to be rewritten.'
107
+ last_kind = last_text = nil
108
+ new = self.class.new
109
+ for text, kind in self
110
+ if text.is_a? String
111
+ if kind == last_kind
112
+ last_text << text
113
+ else
114
+ new << [last_text, last_kind] if last_kind
115
+ last_text = text
116
+ last_kind = kind
117
+ end
118
+ else
119
+ new << [last_text, last_kind] if last_kind
120
+ last_kind = last_text = nil
121
+ new << [text, kind]
122
+ end
123
+ end
124
+ new << [last_text, last_kind] if last_kind
125
+ new
126
+ end
127
+
128
+ # Compact the object itself; see optimize.
129
+ def optimize!
130
+ replace optimize
131
+ end
132
+
133
+ # Ensure that all begin_group tokens have a correspondent end_group.
134
+ #
135
+ # TODO: Test this!
136
+ def fix
137
+ raise NotImplementedError, 'Tokens#fix needs to be rewritten.'
138
+ tokens = self.class.new
139
+ # Check token nesting using a stack of kinds.
140
+ opened = []
141
+ for type, kind in self
142
+ case type
143
+ when :begin_group
144
+ opened.push [:begin_group, kind]
145
+ when :begin_line
146
+ opened.push [:end_line, kind]
147
+ when :end_group, :end_line
148
+ expected = opened.pop
149
+ if [type, kind] != expected
150
+ # Unexpected end; decide what to do based on the kind:
151
+ # - token was never opened: delete the end (just skip it)
152
+ next unless opened.rindex expected
153
+ # - token was opened earlier: also close tokens in between
154
+ tokens << token until (token = opened.pop) == expected
155
+ end
156
+ end
157
+ tokens << [type, kind]
158
+ end
159
+ # Close remaining opened tokens
160
+ tokens << token while token = opened.pop
161
+ tokens
162
+ end
163
+
164
+ def fix!
165
+ replace fix
166
+ end
167
+
168
+ # TODO: Scanner#split_into_lines
169
+ #
170
+ # Makes sure that:
171
+ # - newlines are single tokens
172
+ # (which means all other token are single-line)
173
+ # - there are no open tokens at the end the line
174
+ #
175
+ # This makes it simple for encoders that work line-oriented,
176
+ # like HTML with list-style numeration.
177
+ def split_into_lines
178
+ raise NotImplementedError
179
+ end
180
+
181
+ def split_into_lines!
182
+ replace split_into_lines
183
+ end
184
+
86
185
  # Split the tokens into parts of the given +sizes+.
87
186
  #
88
187
  # The result will be an Array of Tokens objects. The parts have
@@ -121,15 +220,12 @@ module CodeRay
121
220
  content_or_kind
122
221
  end
123
222
  end
124
- part.concat closing
125
- begin
126
- parts << part
127
- part = Tokens.new
128
- size = sizes[i += 1]
129
- end until size.nil? || size > 0
223
+ parts << part.concat(closing)
224
+ part = Tokens.new
130
225
  # ...and open them again.
131
226
  part.concat opened.flatten
132
227
  part_size = 0
228
+ size = sizes[i += 1]
133
229
  redo unless content.empty?
134
230
  else
135
231
  part << content << item
@@ -143,19 +239,19 @@ module CodeRay
143
239
  when :end_group, :end_line
144
240
  opened.pop
145
241
  else
146
- raise ArgumentError, 'Unknown token action: %p, kind = %p' % [content, item]
242
+ raise 'Unknown token action: %p, kind = %p' % [content, item]
147
243
  end
148
244
  part << content << item
149
245
  content = nil
150
246
  else
151
- raise ArgumentError, 'Token input junk: %p, kind = %p' % [content, item]
247
+ raise 'else case reached'
152
248
  end
153
249
  end
154
250
  parts << part
155
251
  parts << Tokens.new while parts.size < sizes.size
156
252
  parts
157
253
  end
158
-
254
+
159
255
  # Dumps the object into a String that can be saved
160
256
  # in files or databases.
161
257
  #
@@ -172,8 +268,9 @@ module CodeRay
172
268
  #
173
269
  # See GZip module.
174
270
  def dump gzip_level = 7
271
+ require 'coderay/helpers/gzip_simple'
175
272
  dump = Marshal.dump self
176
- dump = GZip.gzip dump, gzip_level
273
+ dump = dump.gzip gzip_level
177
274
  dump.extend Undumping
178
275
  end
179
276
 
@@ -181,7 +278,7 @@ module CodeRay
181
278
  def count
182
279
  size / 2
183
280
  end
184
-
281
+
185
282
  # Include this module to give an object an #undump
186
283
  # method.
187
284
  #
@@ -192,17 +289,18 @@ module CodeRay
192
289
  Tokens.load self
193
290
  end
194
291
  end
195
-
292
+
196
293
  # Undump the object using Marshal.load, then
197
294
  # unzip it using GZip.gunzip.
198
295
  #
199
296
  # The result is commonly a Tokens object, but
200
297
  # this is not guaranteed.
201
298
  def Tokens.load dump
202
- dump = GZip.gunzip dump
299
+ require 'coderay/helpers/gzip_simple'
300
+ dump = dump.gunzip
203
301
  @dump = Marshal.load dump
204
302
  end
205
-
303
+
206
304
  alias text_token push
207
305
  def begin_group kind; push :begin_group, kind end
208
306
  def end_group kind; push :end_group, kind end
@@ -211,5 +309,50 @@ module CodeRay
211
309
  alias tokens concat
212
310
 
213
311
  end
214
-
312
+
215
313
  end
314
+
315
+ if $0 == __FILE__
316
+ $VERBOSE = true
317
+ $: << File.join(File.dirname(__FILE__), '..')
318
+ eval DATA.read, nil, $0, __LINE__ + 4
319
+ end
320
+
321
+ __END__
322
+ require 'test/unit'
323
+
324
+ class TokensTest < Test::Unit::TestCase
325
+
326
+ def test_creation
327
+ assert CodeRay::Tokens < Array
328
+ tokens = nil
329
+ assert_nothing_raised do
330
+ tokens = CodeRay::Tokens.new
331
+ end
332
+ assert_kind_of Array, tokens
333
+ end
334
+
335
+ def test_adding_tokens
336
+ tokens = CodeRay::Tokens.new
337
+ assert_nothing_raised do
338
+ tokens.text_token 'string', :type
339
+ tokens.text_token '()', :operator
340
+ end
341
+ assert_equal tokens.size, 4
342
+ assert_equal tokens.count, 2
343
+ end
344
+
345
+ def test_dump_undump
346
+ tokens = CodeRay::Tokens.new
347
+ assert_nothing_raised do
348
+ tokens.text_token 'string', :type
349
+ tokens.text_token '()', :operator
350
+ end
351
+ tokens2 = nil
352
+ assert_nothing_raised do
353
+ tokens2 = tokens.dump.undump
354
+ end
355
+ assert_equal tokens, tokens2
356
+ end
357
+
358
+ end