coderay-beta 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. data/FOLDERS +53 -0
  2. data/LICENSE +504 -0
  3. data/bin/coderay +82 -0
  4. data/bin/coderay_stylesheet +4 -0
  5. data/lib/README +129 -0
  6. data/lib/coderay.rb +320 -0
  7. data/lib/coderay/duo.rb +85 -0
  8. data/lib/coderay/encoder.rb +213 -0
  9. data/lib/coderay/encoders/_map.rb +11 -0
  10. data/lib/coderay/encoders/comment_filter.rb +43 -0
  11. data/lib/coderay/encoders/count.rb +21 -0
  12. data/lib/coderay/encoders/debug.rb +49 -0
  13. data/lib/coderay/encoders/div.rb +19 -0
  14. data/lib/coderay/encoders/filter.rb +75 -0
  15. data/lib/coderay/encoders/html.rb +305 -0
  16. data/lib/coderay/encoders/html/css.rb +70 -0
  17. data/lib/coderay/encoders/html/numerization.rb +133 -0
  18. data/lib/coderay/encoders/html/output.rb +206 -0
  19. data/lib/coderay/encoders/json.rb +69 -0
  20. data/lib/coderay/encoders/lines_of_code.rb +90 -0
  21. data/lib/coderay/encoders/null.rb +26 -0
  22. data/lib/coderay/encoders/page.rb +20 -0
  23. data/lib/coderay/encoders/span.rb +19 -0
  24. data/lib/coderay/encoders/statistic.rb +77 -0
  25. data/lib/coderay/encoders/term.rb +137 -0
  26. data/lib/coderay/encoders/text.rb +32 -0
  27. data/lib/coderay/encoders/token_class_filter.rb +84 -0
  28. data/lib/coderay/encoders/xml.rb +71 -0
  29. data/lib/coderay/encoders/yaml.rb +22 -0
  30. data/lib/coderay/for_redcloth.rb +85 -0
  31. data/lib/coderay/helpers/file_type.rb +240 -0
  32. data/lib/coderay/helpers/gzip_simple.rb +123 -0
  33. data/lib/coderay/helpers/plugin.rb +349 -0
  34. data/lib/coderay/helpers/word_list.rb +138 -0
  35. data/lib/coderay/scanner.rb +284 -0
  36. data/lib/coderay/scanners/_map.rb +23 -0
  37. data/lib/coderay/scanners/c.rb +203 -0
  38. data/lib/coderay/scanners/cpp.rb +228 -0
  39. data/lib/coderay/scanners/css.rb +210 -0
  40. data/lib/coderay/scanners/debug.rb +62 -0
  41. data/lib/coderay/scanners/delphi.rb +150 -0
  42. data/lib/coderay/scanners/diff.rb +105 -0
  43. data/lib/coderay/scanners/groovy.rb +263 -0
  44. data/lib/coderay/scanners/html.rb +182 -0
  45. data/lib/coderay/scanners/java.rb +176 -0
  46. data/lib/coderay/scanners/java/builtin_types.rb +419 -0
  47. data/lib/coderay/scanners/java_script.rb +224 -0
  48. data/lib/coderay/scanners/json.rb +112 -0
  49. data/lib/coderay/scanners/nitro_xhtml.rb +136 -0
  50. data/lib/coderay/scanners/php.rb +526 -0
  51. data/lib/coderay/scanners/plaintext.rb +21 -0
  52. data/lib/coderay/scanners/python.rb +285 -0
  53. data/lib/coderay/scanners/rhtml.rb +74 -0
  54. data/lib/coderay/scanners/ruby.rb +404 -0
  55. data/lib/coderay/scanners/ruby/patterns.rb +238 -0
  56. data/lib/coderay/scanners/scheme.rb +145 -0
  57. data/lib/coderay/scanners/sql.rb +162 -0
  58. data/lib/coderay/scanners/xml.rb +17 -0
  59. data/lib/coderay/scanners/yaml.rb +144 -0
  60. data/lib/coderay/style.rb +20 -0
  61. data/lib/coderay/styles/_map.rb +7 -0
  62. data/lib/coderay/styles/cycnus.rb +151 -0
  63. data/lib/coderay/styles/murphy.rb +132 -0
  64. data/lib/coderay/token_classes.rb +86 -0
  65. data/lib/coderay/tokens.rb +391 -0
  66. data/lib/term/ansicolor.rb +220 -0
  67. metadata +123 -0
@@ -0,0 +1,26 @@
1
+ module CodeRay
2
+ module Encoders
3
+
4
+ # = Null Encoder
5
+ #
6
+ # Does nothing and returns an empty string.
7
+ class Null < Encoder
8
+
9
+ include Streamable
10
+ register_for :null
11
+
12
+ # Defined for faster processing
13
+ def to_proc
14
+ proc {}
15
+ end
16
+
17
+ protected
18
+
19
+ def token(*)
20
+ # do nothing
21
+ end
22
+
23
+ end
24
+
25
+ end
26
+ end
@@ -0,0 +1,20 @@
1
+ module CodeRay
2
+ module Encoders
3
+
4
+ load :html
5
+
6
+ class Page < HTML
7
+
8
+ FILE_EXTENSION = 'html'
9
+
10
+ register_for :page
11
+
12
+ DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
13
+ :css => :class,
14
+ :wrap => :page,
15
+ :line_numbers => :table
16
+
17
+ end
18
+
19
+ end
20
+ end
@@ -0,0 +1,19 @@
1
+ module CodeRay
2
+ module Encoders
3
+
4
+ load :html
5
+
6
+ class Span < HTML
7
+
8
+ FILE_EXTENSION = 'span.html'
9
+
10
+ register_for :span
11
+
12
+ DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
13
+ :css => :style,
14
+ :wrap => :span
15
+
16
+ end
17
+
18
+ end
19
+ end
@@ -0,0 +1,77 @@
1
+ module CodeRay
2
+ module Encoders
3
+
4
+ # Makes a statistic for the given tokens.
5
+ class Statistic < Encoder
6
+
7
+ include Streamable
8
+ register_for :stats, :statistic
9
+
10
+ attr_reader :type_stats, :real_token_count
11
+
12
+ protected
13
+
14
+ TypeStats = Struct.new :count, :size
15
+
16
+ def setup options
17
+ @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }
18
+ @real_token_count = 0
19
+ end
20
+
21
+ def generate tokens, options
22
+ @tokens = tokens
23
+ super
24
+ end
25
+
26
+ def text_token text, kind
27
+ @real_token_count += 1 unless kind == :space
28
+ @type_stats[kind].count += 1
29
+ @type_stats[kind].size += text.size
30
+ @type_stats['TOTAL'].size += text.size
31
+ @type_stats['TOTAL'].count += 1
32
+ end
33
+
34
+ # TODO Hierarchy handling
35
+ def block_token action, kind
36
+ @type_stats['TOTAL'].count += 1
37
+ @type_stats['open/close'].count += 1
38
+ end
39
+
40
+ STATS = <<-STATS
41
+
42
+ Code Statistics
43
+
44
+ Tokens %8d
45
+ Non-Whitespace %8d
46
+ Bytes Total %8d
47
+
48
+ Token Types (%d):
49
+ type count ratio size (average)
50
+ -------------------------------------------------------------
51
+ %s
52
+ STATS
53
+ # space 12007 33.81 % 1.7
54
+ TOKEN_TYPES_ROW = <<-TKR
55
+ %-20s %8d %6.2f %% %5.1f
56
+ TKR
57
+
58
+ def finish options
59
+ all = @type_stats['TOTAL']
60
+ all_count, all_size = all.count, all.size
61
+ @type_stats.each do |type, stat|
62
+ stat.size /= stat.count.to_f
63
+ end
64
+ types_stats = @type_stats.sort_by { |k, v| [-v.count, k.to_s] }.map do |k, v|
65
+ TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size]
66
+ end.join
67
+ STATS % [
68
+ all_count, @real_token_count, all_size,
69
+ @type_stats.delete_if { |k, v| k.is_a? String }.size,
70
+ types_stats
71
+ ]
72
+ end
73
+
74
+ end
75
+
76
+ end
77
+ end
@@ -0,0 +1,137 @@
1
+ # encoders/term.rb
2
+ # By Rob Aldred (http://robaldred.co.uk)
3
+ # Based on idea by Nathan Weizenbaum (http://nex-3.com)
4
+ # MIT License (http://www.opensource.org/licenses/mit-license.php)
5
+ #
6
+ # A CodeRay encoder that outputs code highlighted for a color terminal.
7
+ # Check out http://robaldred.co.uk
8
+
9
+ module CodeRay
10
+ module Encoders
11
+ class Term < Encoder
12
+ register_for :term
13
+
14
+ TOKEN_COLORS = {
15
+ :attribute_name => '33',
16
+ :attribute_name_fat => '33',
17
+ :attribute_value => '31',
18
+ :attribute_value_fat => '31',
19
+ :bin => '1;35',
20
+ :char => {:self => '36', :delimiter => '34'},
21
+ :class => '1;35',
22
+ :class_variable => '36',
23
+ :color => '32',
24
+ :comment => '37',
25
+ :constant => ['34', '4'],
26
+ :definition => '1;32',
27
+ :directive => ['32', '4'],
28
+ :doc => '46',
29
+ :doc_string => ['31', '4'],
30
+ :entity => '33',
31
+ :error => ['1;33', '41'],
32
+ :exception => '1;31',
33
+ :float => '1;35',
34
+ :function => '1;34',
35
+ :global_variable => '42',
36
+ :hex => '1;36',
37
+ :include => '33',
38
+ :integer => '1;34',
39
+ :interpreted => '1;35',
40
+ :label => '1;4',
41
+ :local_variable => '33',
42
+ :oct => '1;35',
43
+ :operator_name => '1;29',
44
+ :pre_constant => '1;36',
45
+ :pre_type => '1;30',
46
+ :predefined => ['4', '1;34'],
47
+ :preprocessor => '36',
48
+ :regexp => {
49
+ :content => '31',
50
+ :delimiter => '1;29',
51
+ :modifier => '35',
52
+ :function => '1;29'
53
+ },
54
+ :reserved => '1;31',
55
+ :shell => {:self => '42', :content => '1;29'},
56
+ :string => '32',
57
+ :symbol => '1;32',
58
+ :tag => '34',
59
+ :tag_fat => '1;34',
60
+ :tag_special => ['34', '4'],
61
+ :type => '1;34',
62
+ :variable => '34'
63
+ }
64
+ TOKEN_COLORS[:procedure] = TOKEN_COLORS[:method] = TOKEN_COLORS[:function]
65
+ TOKEN_COLORS[:open] = TOKEN_COLORS[:close] = TOKEN_COLORS[:nesting_delimiter] = TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter]
66
+
67
+ protected
68
+
69
+ def setup(options)
70
+ @out = ''
71
+ @opened = [nil]
72
+ @subcolors = nil
73
+ end
74
+
75
+ def finish(options)
76
+ super
77
+ end
78
+
79
+ def token text, type = :plain
80
+ case text
81
+
82
+ when nil
83
+ # raise 'Token with nil as text was given: %p' % [[text, type]]
84
+
85
+ when String
86
+
87
+ if color = (@subcolors || TOKEN_COLORS)[type]
88
+ color = color[:self] || return if Hash === color
89
+
90
+ @out << col(color) + text.gsub("\n", col(0) + "\n" + col(color)) + col(0)
91
+ @out << col(@subcolors[:self]) if @subcolors && @subcolors[:self]
92
+ else
93
+ @out << text
94
+ end
95
+
96
+ # token groups, eg. strings
97
+ when :open
98
+ @opened[0] = type
99
+ if color = TOKEN_COLORS[type]
100
+ if Hash === color
101
+ @subcolors = color
102
+ @out << col(color[:self]) if color[:self]
103
+ else
104
+ @subcolors = {}
105
+ @out << col(color)
106
+ end
107
+ end
108
+ @opened << type
109
+ when :close
110
+ if @opened.empty?
111
+ # nothing to close
112
+ else
113
+ if (@subcolors || {})[:self]
114
+ @out << col(0)
115
+ end
116
+ @subcolors = nil
117
+ @opened.pop
118
+ end
119
+
120
+ # whole lines to be highlighted, eg. a added/modified/deleted lines in a diff
121
+ when :begin_line
122
+
123
+ when :end_line
124
+
125
+ else
126
+ raise 'unknown token kind: %p' % [text]
127
+ end
128
+ end
129
+
130
+ private
131
+
132
+ def col(color)
133
+ Array(color).map { |c| "\e[#{c}m" }.join
134
+ end
135
+ end
136
+ end
137
+ end
@@ -0,0 +1,32 @@
1
+ module CodeRay
2
+ module Encoders
3
+
4
+ class Text < Encoder
5
+
6
+ include Streamable
7
+ register_for :text
8
+
9
+ FILE_EXTENSION = 'txt'
10
+
11
+ DEFAULT_OPTIONS = {
12
+ :separator => ''
13
+ }
14
+
15
+ protected
16
+ def setup options
17
+ super
18
+ @sep = options[:separator]
19
+ end
20
+
21
+ def text_token text, kind
22
+ text + @sep
23
+ end
24
+
25
+ def finish options
26
+ super.chomp @sep
27
+ end
28
+
29
+ end
30
+
31
+ end
32
+ end
@@ -0,0 +1,84 @@
1
+ ($:.unshift '../..'; require 'coderay') unless defined? CodeRay
2
+ module CodeRay
3
+ module Encoders
4
+
5
+ load :filter
6
+
7
+ class TokenClassFilter < Filter
8
+
9
+ include Streamable
10
+ register_for :token_class_filter
11
+
12
+ DEFAULT_OPTIONS = {
13
+ :exclude => [],
14
+ :include => :all
15
+ }
16
+
17
+ protected
18
+ def setup options
19
+ super
20
+ @exclude = options[:exclude]
21
+ @exclude = Array(@exclude) unless @exclude == :all
22
+ @include = options[:include]
23
+ @include = Array(@include) unless @include == :all
24
+ end
25
+
26
+ def include_text_token? text, kind
27
+ (@include == :all || @include.include?(kind)) &&
28
+ !(@exclude == :all || @exclude.include?(kind))
29
+ end
30
+
31
+ end
32
+
33
+ end
34
+ end
35
+
36
+ if $0 == __FILE__
37
+ $VERBOSE = true
38
+ $: << File.join(File.dirname(__FILE__), '..')
39
+ eval DATA.read, nil, $0, __LINE__ + 4
40
+ end
41
+
42
+ __END__
43
+ require 'test/unit'
44
+
45
+ class TokenClassFilterTest < Test::Unit::TestCase
46
+
47
+ def test_creation
48
+ assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Encoder
49
+ assert CodeRay::Encoders::TokenClassFilter < CodeRay::Encoders::Filter
50
+ filter = nil
51
+ assert_nothing_raised do
52
+ filter = CodeRay.encoder :token_class_filter
53
+ end
54
+ assert_instance_of CodeRay::Encoders::TokenClassFilter, filter
55
+ end
56
+
57
+ def test_filtering_text_tokens
58
+ tokens = CodeRay::Tokens.new
59
+ for i in 1..10
60
+ tokens << [i.to_s, :index]
61
+ tokens << [' ', :space] if i < 10
62
+ end
63
+ assert_equal 10, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :space).size
64
+ assert_equal 10, tokens.token_class_filter(:exclude => :space).size
65
+ assert_equal 9, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :space).size
66
+ assert_equal 9, tokens.token_class_filter(:include => :space).size
67
+ assert_equal 0, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :all).size
68
+ assert_equal 0, tokens.token_class_filter(:exclude => :all).size
69
+ end
70
+
71
+ def test_filtering_block_tokens
72
+ tokens = CodeRay::Tokens.new
73
+ 10.times do |i|
74
+ tokens << [:open, :index]
75
+ tokens << [i.to_s, :content]
76
+ tokens << [:close, :index]
77
+ end
78
+ assert_equal 20, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :include => :blubb).size
79
+ assert_equal 20, tokens.token_class_filter(:include => :blubb).size
80
+ assert_equal 30, CodeRay::Encoders::TokenClassFilter.new.encode_tokens(tokens, :exclude => :index).size
81
+ assert_equal 30, tokens.token_class_filter(:exclude => :index).size
82
+ end
83
+
84
+ end
@@ -0,0 +1,71 @@
1
+ module CodeRay
2
+ module Encoders
3
+
4
+ # = XML Encoder
5
+ #
6
+ # Uses REXML. Very slow.
7
+ class XML < Encoder
8
+
9
+ include Streamable
10
+ register_for :xml
11
+
12
+ FILE_EXTENSION = 'xml'
13
+
14
+ require 'rexml/document'
15
+
16
+ DEFAULT_OPTIONS = {
17
+ :tab_width => 8,
18
+ :pretty => -1,
19
+ :transitive => false,
20
+ }
21
+
22
+ protected
23
+
24
+ def setup options
25
+ @doc = REXML::Document.new
26
+ @doc << REXML::XMLDecl.new
27
+ @tab_width = options[:tab_width]
28
+ @root = @node = @doc.add_element('coderay-tokens')
29
+ end
30
+
31
+ def finish options
32
+ @out = ''
33
+ @doc.write @out, options[:pretty], options[:transitive], true
34
+ @out
35
+ end
36
+
37
+ def text_token text, kind
38
+ if kind == :space
39
+ token = @node
40
+ else
41
+ token = @node.add_element kind.to_s
42
+ end
43
+ text.scan(/(\x20+)|(\t+)|(\n)|[^\x20\t\n]+/) do |space, tab, nl|
44
+ case
45
+ when space
46
+ token << REXML::Text.new(space, true)
47
+ when tab
48
+ token << REXML::Text.new(tab, true)
49
+ when nl
50
+ token << REXML::Text.new(nl, true)
51
+ else
52
+ token << REXML::Text.new($&)
53
+ end
54
+ end
55
+ end
56
+
57
+ def open_token kind
58
+ @node = @node.add_element kind.to_s
59
+ end
60
+
61
+ def close_token kind
62
+ if @node == @root
63
+ raise 'no token to close!'
64
+ end
65
+ @node = @node.parent
66
+ end
67
+
68
+ end
69
+
70
+ end
71
+ end