coderay 0.4.3.48

Sign up to get free protection for your applications and to get access to all the features.
Files changed (45) hide show
  1. data/LICENSE +340 -0
  2. data/README +103 -0
  3. data/demo/demo_count.rb +10 -0
  4. data/demo/demo_css.rb +4 -0
  5. data/demo/demo_div.rb +19 -0
  6. data/demo/demo_dump.rb +15 -0
  7. data/demo/demo_encoder.rb +39 -0
  8. data/demo/demo_global_vars.rb +13 -0
  9. data/demo/demo_global_vars2.rb +28 -0
  10. data/demo/demo_html.rb +394 -0
  11. data/demo/demo_html2.rb +11 -0
  12. data/demo/demo_load_encoder.rb +17 -0
  13. data/demo/demo_more.rb +204 -0
  14. data/demo/demo_scanner.rb +36 -0
  15. data/demo/demo_server.rb +92 -0
  16. data/demo/demo_simple.rb +10 -0
  17. data/demo/demo_stream.rb +25 -0
  18. data/demo/demo_stream2.rb +8 -0
  19. data/demo/demo_tokens.rb +3 -0
  20. data/lib/coderay.rb +284 -0
  21. data/lib/coderay/encoder.rb +151 -0
  22. data/lib/coderay/encoders/count.rb +21 -0
  23. data/lib/coderay/encoders/div.rb +16 -0
  24. data/lib/coderay/encoders/helpers/html_css.rb +155 -0
  25. data/lib/coderay/encoders/helpers/html_helper.rb +68 -0
  26. data/lib/coderay/encoders/helpers/html_output.rb +237 -0
  27. data/lib/coderay/encoders/html.rb +169 -0
  28. data/lib/coderay/encoders/null.rb +20 -0
  29. data/lib/coderay/encoders/span.rb +16 -0
  30. data/lib/coderay/encoders/statistic.rb +74 -0
  31. data/lib/coderay/encoders/text.rb +33 -0
  32. data/lib/coderay/encoders/tokens.rb +44 -0
  33. data/lib/coderay/encoders/yaml.rb +19 -0
  34. data/lib/coderay/helpers/filetype.rb +145 -0
  35. data/lib/coderay/helpers/gzip_simple.rb +123 -0
  36. data/lib/coderay/helpers/plugin.rb +286 -0
  37. data/lib/coderay/helpers/scanner_helper.rb +63 -0
  38. data/lib/coderay/scanner.rb +197 -0
  39. data/lib/coderay/scanners/c.rb +147 -0
  40. data/lib/coderay/scanners/delphi.rb +123 -0
  41. data/lib/coderay/scanners/helpers/ruby_helper.rb +212 -0
  42. data/lib/coderay/scanners/plaintext.rb +13 -0
  43. data/lib/coderay/scanners/ruby.rb +337 -0
  44. data/lib/coderay/tokens.rb +324 -0
  45. metadata +89 -0
data/demo/demo_css.rb ADDED
@@ -0,0 +1,4 @@
1
+ require 'coderay'
2
+
3
+ # print the default stylesheet for HTML codes
4
+ puts CodeRay::Encoders[:html]::CSS::DEFAULT_STYLESHEET
data/demo/demo_div.rb ADDED
@@ -0,0 +1,19 @@
1
+ require 'coderay'
2
+
3
+ puts CodeRay.scan(DATA.read, :ruby).div
4
+
5
+ __END__
6
+ for a in 0..255
7
+ a = a.chr
8
+ begin
9
+ x = eval("?\\#{a}")
10
+ if x == a[0]
11
+ next
12
+ else
13
+ print "#{a}: #{x}"
14
+ end
15
+ rescue SyntaxError => boom
16
+ print "#{a}: error"
17
+ end
18
+ puts
19
+ end
data/demo/demo_dump.rb ADDED
@@ -0,0 +1,15 @@
1
+ require 'coderay'
2
+
3
+ # scan some code
4
+ tokens = CodeRay.scan(File.read($0), :ruby)
5
+
6
+ # dump using YAML
7
+ yaml = tokens.yaml
8
+ puts 'YAML: %4d bytes' % yaml.size
9
+
10
+ # dump using Marshal
11
+ dump = tokens.dump(0)
12
+ puts 'Dump: %4d bytes' % dump.size
13
+
14
+ # undump and encode
15
+ puts 'undumped:', dump.undump.div(:css => :class)
data/demo/demo_encoder.rb ADDED
@@ -0,0 +1,39 @@
1
+ require 'coderay'
2
+
3
+ SAMPLE = "puts 17 + 4\n"
4
+ puts 'Encoders Demo: ' + SAMPLE
5
+ scanner = CodeRay::Scanners[:ruby].new SAMPLE
6
+ encoder = CodeRay::Encoders[:statistic].new
7
+
8
+ tokens = scanner.tokenize
9
+ stats = encoder.encode_tokens tokens
10
+
11
+ puts
12
+ puts 'Statistic:'
13
+ puts stats
14
+
15
+ # alternative 1
16
+ tokens = CodeRay.scan SAMPLE, :ruby
17
+ encoder = CodeRay.encoder(:tokens)
18
+ textual = encoder.encode_tokens tokens
19
+ puts
20
+ puts 'Original text:'
21
+ puts textual
22
+
23
+ # alternative 2
24
+ yaml = CodeRay.encoder(:yaml).encode SAMPLE, :ruby
25
+ puts
26
+ puts 'YAML:'
27
+ puts yaml
28
+
29
+ # alternative 3
30
+ BIGSAMPLE = SAMPLE * 100
31
+ dump = CodeRay.scan(BIGSAMPLE, :ruby).dump
32
+ puts
33
+ puts 'Dump:'
34
+ p dump
35
+ puts 'compressed: %d byte < %d byte' % [dump.size, BIGSAMPLE.size]
36
+
37
+ puts
38
+ puts 'Undump:'
39
+ puts dump.undump.statistic
data/demo/demo_global_vars.rb ADDED
@@ -0,0 +1,13 @@
1
+ code = <<'CODE'
2
+ $ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel
3
+ oder
4
+ text = $bla.test(...)
5
+ CODE
6
+
7
+ require 'coderay'
8
+
9
+ tokens = CodeRay.scan code, :ruby
10
+ tokens.each_text_token { |text, kind| text.upcase! }
11
+ tokens.each(:global_variable) { |text, kind| text.replace '<--%s-->' % text }
12
+
13
+ print tokens
data/demo/demo_global_vars2.rb ADDED
@@ -0,0 +1,28 @@
1
+ require 'coderay'
2
+ require 'erb'
3
+ include ERB::Util
4
+
5
+ code = <<'CODE'
6
+ $ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel
7
+ oder
8
+ text = $bla.test(...)
9
+ CODE
10
+ puts <<HTML
11
+ <html>
12
+ <head>
13
+ <style>span.glob-var { color: green; font-weight: bold; }</style>
14
+ </head>
15
+ <body>
16
+ HTML
17
+
18
+ CodeRay.scan_stream code, :ruby do |text, kind|
19
+ next if text.is_a? Symbol
20
+ text = h(text)
21
+ text = '<span class="glob-var">%s</span>' % text if kind == :global_variable
22
+ print text
23
+ end
24
+
25
+ puts <<HTML
26
+ </body>
27
+ </html>
28
+ HTML
data/demo/demo_html.rb ADDED
@@ -0,0 +1,394 @@
1
+ $: << '..'
2
+ require 'coderay'
3
+
4
+ tokens = CodeRay.scan DATA.read, :ruby
5
+ html = tokens.html(:tab_width => 2, :line_numbers => :table)
6
+
7
+ puts html.page
8
+
9
+ __END__
10
+ require 'scanner'
11
+
12
+ module CodeRay
13
+
14
+ class RubyScanner < Scanner
15
+
16
+ RESERVED_WORDS = [
17
+ 'and', 'def', 'end', 'in', 'or', 'unless', 'begin',
18
+ 'defined?', 'ensure', 'module', 'redo', 'super', 'until',
19
+ 'BEGIN', 'break', 'do', 'next', 'rescue', 'then',
20
+ 'when', 'END', 'case', 'else', 'for', 'retry',
21
+ 'while', 'alias', 'class', 'elsif', 'if', 'not', 'return',
22
+ 'undef', 'yield',
23
+ ]
24
+
25
+ DEF_KEYWORDS = ['def']
26
+ MODULE_KEYWORDS = ['class', 'module']
27
+ DEF_NEW_STATE = WordList.new(:initial).
28
+ add(DEF_KEYWORDS, :def_expected).
29
+ add(MODULE_KEYWORDS, :module_expected)
30
+
31
+ WORDS_ALLOWING_REGEXP = [
32
+ 'and', 'or', 'not', 'while', 'until', 'unless', 'if', 'elsif', 'when'
33
+ ]
34
+ REGEXP_ALLOWED = WordList.new(false).
35
+ add(WORDS_ALLOWING_REGEXP, :set)
36
+
37
+ PREDEFINED_CONSTANTS = [
38
+ 'nil', 'true', 'false', 'self',
39
+ 'DATA', 'ARGV', 'ARGF', '__FILE__', '__LINE__',
40
+ ]
41
+
42
+ IDENT_KIND = WordList.new(:ident).
43
+ add(RESERVED_WORDS, :reserved).
44
+ add(PREDEFINED_CONSTANTS, :pre_constant)
45
+
46
+ METHOD_NAME = / #{IDENT} [?!]? /xo
47
+ METHOD_NAME_EX = /
48
+ #{METHOD_NAME} # common methods: split, foo=, empty?, gsub!
49
+ | \*\*? # multiplication and power
50
+ | [-+~]@? # plus, minus
51
+ | [\/%&|^`] # division, modulo or format strings, &and, |or, ^xor, `system`
52
+ | \[\]=? # array getter and setter
53
+ | <=?>? | >=? # comparison, rocket operator
54
+ | << | >> # append or shift left, shift right
55
+ | ===? # simple equality and case equality
56
+ /ox
57
+ GLOBAL_VARIABLE = / \$ (?: #{IDENT} | \d+ | [~&+`'=\/,;_.<>!@0$?*":F\\] | -[a-zA-Z_0-9] ) /ox
58
+
59
+ DOUBLEQ = / " [^"\#\\]* (?: (?: \#\{.*?\} | \#(?:$")? | \\. ) [^"\#\\]* )* "? /ox
60
+ SINGLEQ = / ' [^'\\]* (?: \\. [^'\\]* )* '? /ox
61
+ STRING = / #{SINGLEQ} | #{DOUBLEQ} /ox
62
+ SHELL = / ` [^`\#\\]* (?: (?: \#\{.*?\} | \#(?:$`)? | \\. ) [^`\#\\]* )* `? /ox
63
+ REGEXP = / \/ [^\/\#\\]* (?: (?: \#\{.*?\} | \#(?:$\/)? | \\. ) [^\/\#\\]* )* \/? /ox
64
+
65
+ DECIMAL = /\d+(?:_\d+)*/ # doesn't recognize 09 as octal error
66
+ OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
67
+ HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/
68
+ BINARY = /0b[01]+(?:_[01]+)*/
69
+
70
+ EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
71
+ FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) /
72
+ INTEGER = /#{OCTAL}|#{HEXADECIMAL}|#{BINARY}|#{DECIMAL}/
73
+
74
+ def reset
75
+ super
76
+ @regexp_allowed = false
77
+ end
78
+
79
+ def next_token
80
+ return if @scanner.eos?
81
+
82
+ kind = :error
83
+ if @scanner.scan(/\s+/) # in every state
84
+ kind = :space
85
+ @regexp_allowed = :set if @regexp_allowed or @scanner.matched.index(?\n) # delayed flag setting
86
+
87
+ elsif @state == :def_expected
88
+ if @scanner.scan(/ (?: (?:#{IDENT}(?:\.|::))* | (?:@@?|$)? #{IDENT}(?:\.|::) ) #{METHOD_NAME_EX} /ox)
89
+ kind = :method
90
+ @state = :initial
91
+ else
92
+ @scanner.scan(/./)
93
+ kind = :error
94
+ end
95
+ @state = :initial
96
+
97
+ elsif @state == :module_expected
98
+ if @scanner.scan(/<</)
99
+ kind = :operator
100
+ else
101
+ if @scanner.scan(/ (?: #{IDENT} (?:\.|::))* #{IDENT} /ox)
102
+ kind = :method
103
+ else
104
+ @scanner.scan(/./)
105
+ kind = :error
106
+ end
107
+ @state = :initial
108
+ end
109
+
110
+ elsif # state == :initial
111
+ # IDENTIFIERS, KEYWORDS
112
+ if @scanner.scan(GLOBAL_VARIABLE)
113
+ kind = :global_variable
114
+ elsif @scanner.scan(/ @@ #{IDENT} /ox)
115
+ kind = :class_variable
116
+ elsif @scanner.scan(/ @ #{IDENT} /ox)
117
+ kind = :instance_variable
118
+ elsif @scanner.scan(/ __END__\n ( (?!\#CODE\#) .* )? | \#[^\n]* | =begin(?=\s).*? \n=end(?=\s|\z)(?:[^\n]*)? /x)
119
+ kind = :comment
120
+ elsif @scanner.scan(METHOD_NAME)
121
+ if @last_token_dot
122
+ kind = :ident
123
+ else
124
+ matched = @scanner.matched
125
+ kind = IDENT_KIND[matched]
126
+ if kind == :ident and matched =~ /^[A-Z]/
127
+ kind = :constant
128
+ elsif kind == :reserved
129
+ @state = DEF_NEW_STATE[matched]
130
+ @regexp_allowed = REGEXP_ALLOWED[matched]
131
+ end
132
+ end
133
+
134
+ elsif @scanner.scan(STRING)
135
+ kind = :string
136
+ elsif @scanner.scan(SHELL)
137
+ kind = :shell
138
+ ## HEREDOCS
139
+ elsif @scanner.scan(/\//) and @regexp_allowed
140
+ @scanner.unscan
141
+ @scanner.scan(REGEXP)
142
+ kind = :regexp
143
+ ## %strings
144
+ elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
145
+ kind = :global_variable
146
+ elsif @scanner.scan(/
147
+ \? (?:
148
+ [^\s\\]
149
+ |
150
+ \\ (?:M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-))? (?: \\ (?: . | [0-7]{3} | x[0-9A-Fa-f][0-9A-Fa-f] )
151
+ )
152
+ /ox)
153
+ kind = :integer
154
+
155
+ elsif @scanner.scan(/ [-+*\/%=<>;,|&!()\[\]{}~?] | \.\.?\.? | ::? /x)
156
+ kind = :operator
157
+ @regexp_allowed = :set if @scanner.matched[-1,1] =~ /[~=!<>|&^,\(\[+\-\/\*%]\z/
158
+ elsif @scanner.scan(FLOAT)
159
+ kind = :float
160
+ elsif @scanner.scan(INTEGER)
161
+ kind = :integer
162
+ elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
163
+ kind = :global_variable
164
+ else
165
+ @scanner.scan(/./m)
166
+ end
167
+ end
168
+
169
+ token = Token.new @scanner.matched, kind
170
+
171
+ if kind == :regexp
172
+ token.text << @scanner.scan(/[eimnosux]*/)
173
+ end
174
+
175
+ @regexp_allowed = (@regexp_allowed == :set) # delayed flag setting
176
+
177
+ token
178
+ end
179
+ end
180
+
181
+ ScannerList.register RubyScanner, 'ruby'
182
+
183
+ end
184
+
185
+ module CodeRay
186
+ require 'scanner'
187
+
188
+ class Highlighter
189
+
190
+ def initialize lang
191
+ @scanner = Scanner[lang].new
192
+ end
193
+
194
+ def highlight code
195
+ @scanner.feed code
196
+ @scanner.all_tokens.map { |t| t.inspect }.join "\n"
197
+ end
198
+
199
+ end
200
+
201
+ class HTMLHighlighter < Highlighter
202
+
203
+ ClassOfKind = {
204
+ :attribute_name => 'an',
205
+ :attribute_name_fat => 'af',
206
+ :attribute_value => 'av',
207
+ :attribute_value_fat => 'aw',
208
+ :bin => 'bi',
209
+ :char => 'ch',
210
+ :class => 'cl',
211
+ :class_variable => 'cv',
212
+ :color => 'cr',
213
+ :comment => 'c',
214
+ :constant => 'co',
215
+ :definition => 'df',
216
+ :directive => 'di',
217
+ :doc => 'do',
218
+ :doc_string => 'ds',
219
+ :exception => 'ex',
220
+ :error => 'er',
221
+ :float => 'fl',
222
+ :function => 'fu',
223
+ :global_variable => 'gv',
224
+ :hex => 'hx',
225
+ :include => 'ic',
226
+ :instance_variable => 'iv',
227
+ :integer => 'i',
228
+ :interpreted => 'in',
229
+ :label => 'la',
230
+ :local_variable => 'lv',
231
+ :oct => 'oc',
232
+ :operator_name => 'on',
233
+ :pre_constant => 'pc',
234
+ :pre_type => 'pt',
235
+ :predefined => 'pd',
236
+ :preprocessor => 'pp',
237
+ :regexp => 'rx',
238
+ :reserved => 'r',
239
+ :shell => 'sh',
240
+ :string => 's',
241
+ :symbol => 'sy',
242
+ :tag => 'ta',
243
+ :tag_fat => 'tf',
244
+ :tag_special => 'ts',
245
+ :type => 'ty',
246
+ :variable => 'v',
247
+ :xml_text => 'xt',
248
+
249
+ :ident => :NO_HIGHLIGHT,
250
+ :operator => :NO_HIGHLIGHT,
251
+ :space => :NO_HIGHLIGHT,
252
+ }
253
+ ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
254
+ ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
255
+
256
+ def initialize lang, options = {}
257
+ super lang
258
+
259
+ @HTML_TAB = ' ' * options.fetch(:tabs2space, 8)
260
+ case level = options.fetch(:level, 'xhtml')
261
+ when 'html'
262
+ @HTML_BR = "<BR>\n"
263
+ when 'xhtml'
264
+ @HTML_BR = "<br />\n"
265
+ else
266
+ raise "Unknown HTML level: #{level}"
267
+ end
268
+ end
269
+
270
+ def highlight code
271
+ @scanner.feed code
272
+
273
+ out = ''
274
+ while t = @scanner.next_token
275
+ warn t.inspect if t.text.nil?
276
+ out << to_html(t)
277
+ end
278
+ TEMPLATE =~ /<%CONTENT%>/
279
+ $` + out + $'
280
+ end
281
+
282
+ private
283
+ def to_html token
284
+ css_class = ClassOfKind[token.kind]
285
+ if defined? ::DEBUG and not ClassOfKind.has_key? token.kind
286
+ warn "no token class found for :#{token.kind}"
287
+ end
288
+
289
+ text = text_to_html token.text
290
+ if css_class == :NO_HIGHLIGHT
291
+ text
292
+ else
293
+ "<span class=\"#{css_class}\">#{text}</span>"
294
+ end
295
+ end
296
+
297
+ def text_to_html text
298
+ return '' if text.empty?
299
+ text = text.dup # important
300
+ if text.index(/["><&]/)
301
+ text.gsub!('&', '&amp;')
302
+ text.gsub!('"', '&quot;')
303
+ text.gsub!('>', '&gt;')
304
+ text.gsub!('<', '&lt;')
305
+ end
306
+ if text.index(/\s/)
307
+ text.gsub!("\n", @HTML_BR)
308
+ text.gsub!("\t", @HTML_TAB)
309
+ text.gsub!(/^ /, '&nbsp;')
310
+ text.gsub!(' ', ' &nbsp;')
311
+ end
312
+ text
313
+ end
314
+
315
+ TEMPLATE = <<-'TEMPLATE'
316
+ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
317
+ <html dir="ltr">
318
+ <head>
319
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
320
+ <meta http-equiv="Content-Style-Type" content="text/css">
321
+
322
+ <title>RubyBB BBCode</title>
323
+ <style type="text/css">
324
+ .code {
325
+ width: 100%;
326
+ background-color: #FAFAFA;
327
+ border: 1px solid #D1D7DC;
328
+ font-family: 'Courier New', 'Terminal', monospace;
329
+ font-size: 10pt;
330
+ color: black;
331
+ vertical-align: top;
332
+ text-align: left;
333
+ }
334
+ .code .af { color:#00C; }
335
+ .code .an { color:#007; }
336
+ .code .av { color:#700; }
337
+ .code .aw { color:#C00; }
338
+ .code .bi { color:#509; font-weight:bold; }
339
+ .code .c { color:#888; }
340
+ .code .ch { color:#C28; font-weight:bold; }
341
+ .code .cl { color:#B06; font-weight:bold; }
342
+ .code .co { color:#036; font-weight:bold; }
343
+ .code .cr { color:#0A0; }
344
+ .code .cv { color:#369; }
345
+ .code .df { color:#099; font-weight:bold; }
346
+ .code .di { color:#088; font-weight:bold; }
347
+ .code .do { color:#970; }
348
+ .code .ds { color:#D42; font-weight:bold; }
349
+ .code .er { color:#F00; background-color:#FAA; }
350
+ .code .ex { color:#F00; font-weight:bold; }
351
+ .code .fl { color:#60E; font-weight:bold; }
352
+ .code .fu { color:#06B; font-weight:bold; }
353
+ .code .gv { color:#800; font-weight:bold; }
354
+ .code .hx { color:#058; font-weight:bold; }
355
+ .code .i { color:#00D; font-weight:bold; }
356
+ .code .ic { color:#B44; font-weight:bold; }
357
+ .code .in { color:#B2B; font-weight:bold; }
358
+ .code .iv { color:#33B; }
359
+ .code .la { color:#970; font-weight:bold; }
360
+ .code .lv { color:#963; }
361
+ .code .oc { color:#40E; font-weight:bold; }
362
+ .code .on { color:#000; font-weight:bold; }
363
+ .code .pc { color:#038; font-weight:bold; }
364
+ .code .pd { color:#369; font-weight:bold; }
365
+ .code .pp { color:#579; }
366
+ .code .pt { color:#339; font-weight:bold; }
367
+ .code .r { color:#080; font-weight:bold; }
368
+ .code .rx { color:#927; font-weight:bold; }
369
+ .code .s { color:#D42; font-weight:bold; }
370
+ .code .sh { color:#B2B; font-weight:bold; }
371
+ .code .sy { color:#A60; }
372
+ .code .ta { color:#070; }
373
+ .code .tf { color:#070; font-weight:bold; }
374
+ .code .ts { color:#D70; font-weight:bold; }
375
+ .code .ty { color:#339; font-weight:bold; }
376
+ .code .v { color:#036; }
377
+ .code .xt { color:#444; }
378
+ </style>
379
+ </head>
380
+ <body>
381
+ <div class="code">
382
+ <%CONTENT%>
383
+ </div>
384
+ <div class="validators">
385
+ <a href="http://validator.w3.org/check?uri=referer"><img src="http://www.w3.org/Icons/valid-html401" alt="Valid HTML 4.01!" height="31" width="88" style="border:none;"></a>
386
+ <img style="border:0" src="http://jigsaw.w3.org/css-validator/images/vcss" alt="Valid CSS!" >
387
+ </div>
388
+ </body>
389
+ </html>
390
+ TEMPLATE
391
+
392
+ end
393
+
394
+ end