coderay 0.4.3.48

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. data/LICENSE +340 -0
  2. data/README +103 -0
  3. data/demo/demo_count.rb +10 -0
  4. data/demo/demo_css.rb +4 -0
  5. data/demo/demo_div.rb +19 -0
  6. data/demo/demo_dump.rb +15 -0
  7. data/demo/demo_encoder.rb +39 -0
  8. data/demo/demo_global_vars.rb +13 -0
  9. data/demo/demo_global_vars2.rb +28 -0
  10. data/demo/demo_html.rb +394 -0
  11. data/demo/demo_html2.rb +11 -0
  12. data/demo/demo_load_encoder.rb +17 -0
  13. data/demo/demo_more.rb +204 -0
  14. data/demo/demo_scanner.rb +36 -0
  15. data/demo/demo_server.rb +92 -0
  16. data/demo/demo_simple.rb +10 -0
  17. data/demo/demo_stream.rb +25 -0
  18. data/demo/demo_stream2.rb +8 -0
  19. data/demo/demo_tokens.rb +3 -0
  20. data/lib/coderay.rb +284 -0
  21. data/lib/coderay/encoder.rb +151 -0
  22. data/lib/coderay/encoders/count.rb +21 -0
  23. data/lib/coderay/encoders/div.rb +16 -0
  24. data/lib/coderay/encoders/helpers/html_css.rb +155 -0
  25. data/lib/coderay/encoders/helpers/html_helper.rb +68 -0
  26. data/lib/coderay/encoders/helpers/html_output.rb +237 -0
  27. data/lib/coderay/encoders/html.rb +169 -0
  28. data/lib/coderay/encoders/null.rb +20 -0
  29. data/lib/coderay/encoders/span.rb +16 -0
  30. data/lib/coderay/encoders/statistic.rb +74 -0
  31. data/lib/coderay/encoders/text.rb +33 -0
  32. data/lib/coderay/encoders/tokens.rb +44 -0
  33. data/lib/coderay/encoders/yaml.rb +19 -0
  34. data/lib/coderay/helpers/filetype.rb +145 -0
  35. data/lib/coderay/helpers/gzip_simple.rb +123 -0
  36. data/lib/coderay/helpers/plugin.rb +286 -0
  37. data/lib/coderay/helpers/scanner_helper.rb +63 -0
  38. data/lib/coderay/scanner.rb +197 -0
  39. data/lib/coderay/scanners/c.rb +147 -0
  40. data/lib/coderay/scanners/delphi.rb +123 -0
  41. data/lib/coderay/scanners/helpers/ruby_helper.rb +212 -0
  42. data/lib/coderay/scanners/plaintext.rb +13 -0
  43. data/lib/coderay/scanners/ruby.rb +337 -0
  44. data/lib/coderay/tokens.rb +324 -0
  45. metadata +89 -0
@@ -0,0 +1,169 @@
1
+ module CodeRay
2
+ module Encoders
3
+
4
+ class HTML < Encoder
5
+
6
+ include Streamable
7
+ register_for :html
8
+
9
+ FILE_EXTENSION = 'html'
10
+
11
+ DEFAULT_OPTIONS = {
12
+ :tab_width => 8,
13
+
14
+ :level => :xhtml,
15
+ :css => :class,
16
+
17
+ :wrap => :page,
18
+
19
+ :line_numbers => nil,
20
+ :line_number_start => 1,
21
+ :bold_every => 10,
22
+ }
23
+ NUMERIZABLE_WRAPPINGS = [:div, :page]
24
+
25
+ require 'coderay/encoders/helpers/html_helper'
26
+ require 'coderay/encoders/helpers/html_output'
27
+ require 'coderay/encoders/helpers/html_css'
28
+
29
+ def initialize(*)
30
+ super
31
+ @last_options = nil
32
+ end
33
+
34
+ protected
35
+
36
+ HTML_ESCAPE = { #:nodoc:
37
+ '&' => '&amp;',
38
+ '"' => '&quot;',
39
+ '>' => '&gt;',
40
+ '<' => '&lt;',
41
+ }
42
+
43
+ # This is to prevent illegal HTML.
44
+ # Strange chars should still be avoided in codes.
45
+ evil_chars = Array(0x00...0x20) - [?n, ?t]
46
+ evil_chars.each { |i| HTML_ESCAPE[i.chr] = ' ' }
47
+ ansi_chars = Array(0x7f..0xff)
48
+ ansi_chars.each { |i| HTML_ESCAPE[i.chr] = '&#%d;' % i }
49
+ # \x9 (\t) and \xA (\n) not included
50
+ HTML_ESCAPE_PATTERN = /[&"><\0-\x8\xB-\x1f\x7f-\xff]/
51
+
52
+ def setup options
53
+ if options[:line_numbers] and not NUMERIZABLE_WRAPPINGS.include? options[:wrap]
54
+ warn ':line_numbers wanted, but :wrap is %p' % options[:wrap]
55
+ end
56
+ super
57
+ return if options == @last_options
58
+ @last_options = options
59
+
60
+ @HTML_ESCAPE = HTML_ESCAPE.dup
61
+ @HTML_ESCAPE["\t"] = ' ' * options[:tab_width]
62
+
63
+ @opened = [nil]
64
+ @css = CSS.new
65
+
66
+ case options[:css]
67
+
68
+ when :class
69
+ @css_style = Hash.new do |h, k|
70
+ if k.is_a? Array
71
+ type = k.first
72
+ else
73
+ type = k
74
+ end
75
+ c = ClassOfKind[type]
76
+ if c == :NO_HIGHLIGHT
77
+ h[k] = false
78
+ else
79
+ if options[:debug]
80
+ debug_info = ' title="%p"' % [ k ]
81
+ else
82
+ debug_info = ''
83
+ end
84
+ h[k] = '<span%s class="%s">' % [debug_info, c]
85
+ end
86
+ end
87
+
88
+ when :style
89
+ @css_style = Hash.new do |h, k|
90
+ if k.is_a? Array
91
+ styles = k.dup
92
+ else
93
+ styles = [k]
94
+ end
95
+ styles.map! { |c| ClassOfKind[c] }
96
+ if styles.first == :NO_HIGHLIGHT
97
+ h[k] = false
98
+ else
99
+ if options[:debug]
100
+ debug_info = ' title="%s"' % [ styles.inspect.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] } ]
101
+ else
102
+ debug_info = ''
103
+ end
104
+ style = @css[*styles]
105
+ h[k] =
106
+ if style
107
+ '<span%s style="%s">' % [debug_info, style]
108
+ else
109
+ false
110
+ end
111
+ end
112
+ end
113
+
114
+ else
115
+ raise "Unknown value %p for :css." % options[:css]
116
+
117
+ end
118
+ end
119
+
120
+ def finish options
121
+ not_needed = @opened.shift
122
+ @out << '</span>' * @opened.size
123
+
124
+ @out.extend Output
125
+ @out.numerize! options[:line_numbers], options # if options[:line_numbers]
126
+ @out.wrap! options[:wrap] # if options[:wrap]
127
+
128
+ #require 'pp'
129
+ #pp @css_style, @css_style.size
130
+
131
+ super
132
+ end
133
+
134
+ def token text, type
135
+ if text.is_a? String
136
+ if text =~ /#{HTML_ESCAPE_PATTERN}/o
137
+ text = text.gsub(/#{HTML_ESCAPE_PATTERN}/o) { |m| @HTML_ESCAPE[m] }
138
+ end
139
+ @opened[0] = type
140
+ style = @css_style[@opened]
141
+ if style
142
+ @out << style << text << '</span>'
143
+ else
144
+ @out << text
145
+ end
146
+ else
147
+ case text
148
+ when :open
149
+ @opened[0] = type
150
+ @out << @css_style[@opened]
151
+ @opened << type
152
+ when :close
153
+ unless @opened.empty?
154
+ raise 'Not Token to be closed.' unless @opened.size > 1
155
+ @out << '</span>'
156
+ @opened.pop
157
+ end
158
+ when nil
159
+ raise 'Token with nil as text was given: %p' % [[text, type]]
160
+ else
161
+ raise 'unknown token kind: %p' % text
162
+ end
163
+ end
164
+ end
165
+
166
+ end
167
+
168
+ end
169
+ end
@@ -0,0 +1,20 @@
1
+ module CodeRay
2
+ module Encoders
3
+
4
+ class Null < Encoder
5
+
6
+ include Streamable
7
+ register_for :null
8
+
9
+ protected
10
+
11
+ def token(*)
12
+ # do nothing
13
+ end
14
+
15
+ end
16
+
17
+ end
18
+ end
19
+
20
+
@@ -0,0 +1,16 @@
1
+ module CodeRay module Encoders
2
+
3
+ require 'coderay/encoders/html'
4
+ class Span < HTML
5
+
6
+ FILE_EXTENSION = 'span.html'
7
+
8
+ register_for :span
9
+
10
+ DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge({
11
+ :css => :style,
12
+ :wrap => :span,
13
+ })
14
+ end
15
+
16
+ end end
@@ -0,0 +1,74 @@
1
+ module CodeRay module Encoders
2
+
3
+ # Makes a statistic for the given tokens.
4
+ class Statistic < Encoder
5
+
6
+ include Streamable
7
+ register_for :stats, :statistic
8
+
9
+ attr_reader :type_stats, :real_token_count
10
+
11
+ protected
12
+
13
+ TypeStats = Struct.new :count, :size
14
+
15
+ def setup options
16
+ @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }
17
+ @real_token_count = 0
18
+ end
19
+
20
+ def generate tokens, options
21
+ @tokens = tokens
22
+ super
23
+ end
24
+
25
+ def token text, type
26
+ @type_stats['TOTAL'].count += 1
27
+ if text.is_a? String
28
+ @real_token_count += 1 unless type == :space
29
+ @type_stats[type].count += 1
30
+ @type_stats[type].size += text.size
31
+ @type_stats['TOTAL'].size += text.size
32
+ else
33
+ @content_type = type
34
+ @type_stats['open/close'].count += 1
35
+ end
36
+ end
37
+
38
+ STATS = <<-STATS
39
+
40
+ Code Statistics
41
+
42
+ Tokens %8d
43
+ Non-Whitespace %8d
44
+ Bytes Total %8d
45
+
46
+ Token Types (%d):
47
+ type count ratio size (average)
48
+ -------------------------------------------------------------
49
+ %s
50
+ STATS
51
+ # space 12007 33.81 % 1.7
52
+ TOKEN_TYPES_ROW = <<-TKR
53
+ %-20s %8d %6.2f %% %5.1f
54
+ TKR
55
+
56
+ def finish options
57
+ all = @type_stats['TOTAL']
58
+ all_count, all_size = all.count, all.size
59
+ @type_stats.each do |type, stat|
60
+ stat.size /= stat.count.to_f
61
+ end
62
+ types_stats = @type_stats.sort_by { |k, v| -v.count }.map do |k, v|
63
+ TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size]
64
+ end.join
65
+ STATS % [
66
+ all_count, @real_token_count, all_size,
67
+ @type_stats.delete_if { |k, v| k.is_a? String }.size,
68
+ types_stats
69
+ ]
70
+ end
71
+
72
+ end
73
+
74
+ end end
@@ -0,0 +1,33 @@
1
+ module CodeRay
2
+ module Encoders
3
+
4
+ class Text < Encoder
5
+
6
+ include Streamable
7
+ register_for :text
8
+
9
+ FILE_EXTENSION = 'txt'
10
+
11
+ DEFAULT_OPTIONS = {
12
+ :separator => ''
13
+ }
14
+
15
+ protected
16
+ def setup options
17
+ super
18
+ @sep = options[:separator]
19
+ end
20
+
21
+ def token text, kind
22
+ return unless text.respond_to? :to_str
23
+ @out << text + @sep
24
+ end
25
+
26
+ def finish options
27
+ @out.chomp @sep
28
+ end
29
+
30
+ end
31
+
32
+ end
33
+ end
@@ -0,0 +1,44 @@
1
+ module CodeRay
2
+ module Encoders
3
+
4
+ # The Tokens encoder converts the tokens to a simple
5
+ # readable format. It doesn't use colors and is mainly
6
+ # intended for console output.
7
+ #
8
+ # The tokens are converted with Tokens.write_token.
9
+ #
10
+ # The format is:
11
+ #
12
+ # <token-kind> \t <escaped token-text> \n
13
+ #
14
+ # Example:
15
+ #
16
+ # require 'coderay'
17
+ # puts CodeRay.scan("puts 3 + 4", :ruby).tokens
18
+ #
19
+ # prints:
20
+ #
21
+ # ident puts
22
+ # space
23
+ # integer 3
24
+ # space
25
+ # operator +
26
+ # space
27
+ # integer 4
28
+ #
29
+ class Tokens < Encoder
30
+
31
+ include Streamable
32
+ register_for :tokens
33
+
34
+ FILE_EXTENSION = 'tok'
35
+
36
+ protected
37
+ def token *args
38
+ @out << CodeRay::Tokens.write_token(*args)
39
+ end
40
+
41
+ end
42
+
43
+ end
44
+ end
@@ -0,0 +1,19 @@
1
+ module CodeRay
2
+ module Encoders
3
+
4
+ class YAML < Encoder
5
+
6
+ register_for :yaml
7
+
8
+ FILE_EXTENSION = 'yaml'
9
+
10
+ protected
11
+ def compile tokens, options
12
+ require 'yaml'
13
+ @out = tokens.to_a.to_yaml
14
+ end
15
+
16
+ end
17
+
18
+ end
19
+ end
@@ -0,0 +1,145 @@
1
+ # =FileType
2
+ #
3
+ # A simple filetype recognizer
4
+ #
5
+ # Author: murphy (mail to murphy cYcnus de)
6
+ #
7
+ # Version: 0.1 (2005.september.1)
8
+ #
9
+ # ==Documentation
10
+ #
11
+ # TODO
12
+ #
13
+ module FileType
14
+
15
+ UnknownFileType = Class.new Exception
16
+
17
+ class << self
18
+
19
+ def [] filename, read_shebang = false
20
+ name = File.basename filename
21
+ ext = File.extname name
22
+ ext.sub!(/^\./, '') # delete the leading dot
23
+
24
+ type =
25
+ TypeFromExt[ext] ||
26
+ TypeFromExt[ext.downcase] ||
27
+ TypeFromName[name] ||
28
+ TypeFromName[name.downcase]
29
+ type ||= shebang(filename) if read_shebang
30
+
31
+ type
32
+ end
33
+
34
+ def shebang filename
35
+ begin
36
+ File.open filename, 'r' do |f|
37
+ first_line = f.gets
38
+ first_line[TypeFromShebang]
39
+ end
40
+ rescue IOError
41
+ nil
42
+ end
43
+ end
44
+
45
+ # This works like Hash#fetch.
46
+ def fetch filename, default = nil, read_shebang = false
47
+ if default and block_given?
48
+ warn 'block supersedes default value argument'
49
+ end
50
+
51
+ unless type = self[filename, read_shebang]
52
+ return yield if block_given?
53
+ return default if default
54
+ raise UnknownFileType, 'Could not determine type of %p.' % filename
55
+ end
56
+ type
57
+ end
58
+
59
+ end
60
+
61
+ TypeFromExt = {
62
+ 'rb' => :ruby,
63
+ 'rbw' => :ruby,
64
+ 'cpp' => :cpp,
65
+ 'c' => :c,
66
+ 'h' => :c,
67
+ 'xml' => :xml,
68
+ 'htm' => :html,
69
+ 'html' => :html,
70
+ }
71
+
72
+ TypeFromShebang = /\b(?:ruby|perl|python|sh)\b/
73
+
74
+ TypeFromName = {
75
+ 'Rakefile' => :ruby,
76
+ 'Rantfile' => :ruby,
77
+ }
78
+
79
+ end
80
+
81
+ if $0 == __FILE__
82
+ $VERBOSE = true
83
+ eval DATA.read, nil, $0, __LINE__+4
84
+ end
85
+
86
+ __END__
87
+
88
+ require 'test/unit'
89
+
90
+ class TC_FileType < Test::Unit::TestCase
91
+
92
+ def test_fetch
93
+ assert_raise FileType::UnknownFileType do
94
+ FileType.fetch ''
95
+ end
96
+
97
+ assert_throws :not_found do
98
+ FileType.fetch '.' do
99
+ throw :not_found
100
+ end
101
+ end
102
+
103
+ assert_equal :default, FileType.fetch('c', :default)
104
+
105
+ stderr, fake_stderr = $stderr, Object.new
106
+ $err = ''
107
+ def fake_stderr.write x
108
+ $err << x
109
+ end
110
+ $stderr = fake_stderr
111
+ FileType.fetch('c', :default) { }
112
+ assert_equal "block supersedes default value argument\n", $err
113
+ $stderr = stderr
114
+ end
115
+
116
+ def test_ruby
117
+ assert_equal :ruby, FileType['test.rb']
118
+ assert_equal :ruby, FileType['C:\\Program Files\\x\\y\\c\\test.rbw']
119
+ assert_equal :ruby, FileType['/usr/bin/something/Rakefile']
120
+ assert_equal :ruby, FileType['~/myapp/gem/Rantfile']
121
+ assert_not_equal :ruby, FileType['test_rb']
122
+ assert_not_equal :ruby, FileType['Makefile']
123
+ assert_not_equal :ruby, FileType['set.rb/set']
124
+ assert_not_equal :ruby, FileType['~/projects/blabla/rb']
125
+ end
126
+
127
+ def test_c
128
+ assert_equal :c, FileType['test.c']
129
+ assert_equal :c, FileType['C:\\Program Files\\x\\y\\c\\test.h']
130
+ assert_not_equal :c, FileType['test_c']
131
+ assert_not_equal :c, FileType['Makefile']
132
+ assert_not_equal :c, FileType['set.h/set']
133
+ assert_not_equal :c, FileType['~/projects/blabla/c']
134
+ end
135
+
136
+ def test_shebang
137
+ dir = './test'
138
+ if File.directory? dir
139
+ Dir.chdir dir do
140
+ assert_equal :c, FileType['test.c']
141
+ end
142
+ end
143
+ end
144
+
145
+ end