coderay 1.0.0 → 1.0.0.598.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. data/FOLDERS +49 -0
  2. data/Rakefile +6 -5
  3. data/bin/coderay +74 -190
  4. data/bin/coderay_stylesheet +4 -0
  5. data/{README_INDEX.rdoc → lib/README} +20 -10
  6. data/lib/coderay.rb +60 -62
  7. data/lib/coderay/duo.rb +55 -2
  8. data/lib/coderay/encoder.rb +39 -52
  9. data/lib/coderay/encoders/_map.rb +7 -11
  10. data/lib/coderay/encoders/comment_filter.rb +61 -0
  11. data/lib/coderay/encoders/count.rb +26 -11
  12. data/lib/coderay/encoders/debug.rb +60 -11
  13. data/lib/coderay/encoders/div.rb +8 -9
  14. data/lib/coderay/encoders/filter.rb +52 -12
  15. data/lib/coderay/encoders/html.rb +113 -106
  16. data/lib/coderay/encoders/html/css.rb +7 -2
  17. data/lib/coderay/encoders/html/numbering.rb +27 -24
  18. data/lib/coderay/encoders/html/output.rb +58 -15
  19. data/lib/coderay/encoders/json.rb +44 -37
  20. data/lib/coderay/encoders/lines_of_code.rb +56 -9
  21. data/lib/coderay/encoders/null.rb +13 -6
  22. data/lib/coderay/encoders/page.rb +8 -8
  23. data/lib/coderay/encoders/span.rb +9 -10
  24. data/lib/coderay/encoders/statistic.rb +114 -51
  25. data/lib/coderay/encoders/terminal.rb +10 -7
  26. data/lib/coderay/encoders/text.rb +36 -17
  27. data/lib/coderay/encoders/token_kind_filter.rb +58 -1
  28. data/lib/coderay/encoders/xml.rb +11 -13
  29. data/lib/coderay/encoders/yaml.rb +14 -16
  30. data/lib/coderay/for_redcloth.rb +1 -1
  31. data/lib/coderay/helpers/file_type.rb +240 -125
  32. data/lib/coderay/helpers/gzip_simple.rb +123 -0
  33. data/lib/coderay/helpers/plugin.rb +307 -241
  34. data/lib/coderay/helpers/word_list.rb +126 -65
  35. data/lib/coderay/scanner.rb +103 -153
  36. data/lib/coderay/scanners/_map.rb +16 -18
  37. data/lib/coderay/scanners/c.rb +13 -13
  38. data/lib/coderay/scanners/cpp.rb +6 -6
  39. data/lib/coderay/scanners/css.rb +48 -47
  40. data/lib/coderay/scanners/debug.rb +55 -9
  41. data/lib/coderay/scanners/delphi.rb +4 -4
  42. data/lib/coderay/scanners/diff.rb +25 -43
  43. data/lib/coderay/scanners/groovy.rb +2 -2
  44. data/lib/coderay/scanners/html.rb +30 -107
  45. data/lib/coderay/scanners/java.rb +5 -6
  46. data/lib/coderay/scanners/java/builtin_types.rb +0 -2
  47. data/lib/coderay/scanners/java_script.rb +6 -6
  48. data/lib/coderay/scanners/json.rb +6 -7
  49. data/lib/coderay/scanners/nitro_xhtml.rb +136 -0
  50. data/lib/coderay/scanners/php.rb +12 -13
  51. data/lib/coderay/scanners/plaintext.rb +26 -0
  52. data/lib/coderay/scanners/python.rb +4 -4
  53. data/lib/coderay/scanners/{erb.rb → rhtml.rb} +11 -19
  54. data/lib/coderay/scanners/ruby.rb +208 -219
  55. data/lib/coderay/scanners/ruby/patterns.rb +85 -18
  56. data/lib/coderay/scanners/scheme.rb +136 -0
  57. data/lib/coderay/scanners/sql.rb +22 -29
  58. data/lib/coderay/scanners/yaml.rb +10 -11
  59. data/lib/coderay/styles/_map.rb +2 -2
  60. data/lib/coderay/styles/alpha.rb +104 -102
  61. data/lib/coderay/styles/cycnus.rb +143 -0
  62. data/lib/coderay/styles/murphy.rb +123 -0
  63. data/lib/coderay/token_kinds.rb +86 -87
  64. data/lib/coderay/tokens.rb +169 -26
  65. data/test/functional/basic.rb +14 -200
  66. data/test/functional/examples.rb +14 -20
  67. data/test/functional/for_redcloth.rb +8 -15
  68. data/test/functional/load_plugin_scanner.rb +11 -0
  69. data/test/functional/suite.rb +6 -9
  70. data/test/functional/vhdl.rb +126 -0
  71. data/test/functional/word_list.rb +79 -0
  72. metadata +129 -107
  73. data/lib/coderay/helpers/gzip.rb +0 -41
  74. data/lib/coderay/scanners/clojure.rb +0 -217
  75. data/lib/coderay/scanners/haml.rb +0 -168
  76. data/lib/coderay/scanners/ruby/string_state.rb +0 -71
  77. data/lib/coderay/scanners/text.rb +0 -26
  78. data/lib/coderay/tokens_proxy.rb +0 -55
  79. data/lib/coderay/version.rb +0 -3
@@ -1,18 +1,25 @@
1
1
  module CodeRay
2
2
  module Encoders
3
-
3
+
4
4
  # = Null Encoder
5
5
  #
6
6
  # Does nothing and returns an empty string.
7
7
  class Null < Encoder
8
-
8
+
9
9
  register_for :null
10
-
11
- def text_token text, kind
10
+
11
+ # Defined for faster processing.
12
+ def to_proc
13
+ proc {}
14
+ end
15
+
16
+ protected
17
+
18
+ def token(*)
12
19
  # do nothing
13
20
  end
14
-
21
+
15
22
  end
16
-
23
+
17
24
  end
18
25
  end
@@ -1,6 +1,6 @@
1
1
  module CodeRay
2
2
  module Encoders
3
-
3
+
4
4
  load :html
5
5
 
6
6
  # Wraps the output into a HTML page, using CSS classes and
@@ -8,17 +8,17 @@ module Encoders
8
8
  #
9
9
  # See Encoders::HTML for available options.
10
10
  class Page < HTML
11
-
11
+
12
12
  FILE_EXTENSION = 'html'
13
-
13
+
14
14
  register_for :page
15
-
15
+
16
16
  DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
17
- :css => :class,
18
- :wrap => :page,
17
+ :css => :class,
18
+ :wrap => :page,
19
19
  :line_numbers => :table
20
-
20
+
21
21
  end
22
-
22
+
23
23
  end
24
24
  end
@@ -1,23 +1,22 @@
1
1
  module CodeRay
2
2
  module Encoders
3
-
3
+
4
4
  load :html
5
-
5
+
6
6
  # Wraps HTML output into a SPAN element, using inline styles by default.
7
7
  #
8
8
  # See Encoders::HTML for available options.
9
9
  class Span < HTML
10
-
10
+
11
11
  FILE_EXTENSION = 'span.html'
12
-
12
+
13
13
  register_for :span
14
-
14
+
15
15
  DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
16
- :css => :style,
17
- :wrap => :span,
18
- :line_numbers => false
19
-
16
+ :css => :style,
17
+ :wrap => :span
18
+
20
19
  end
21
-
20
+
22
21
  end
23
22
  end
@@ -1,26 +1,60 @@
1
+ ($:.unshift '../..'; require 'coderay') unless defined? CodeRay
1
2
  module CodeRay
2
3
  module Encoders
3
-
4
+
4
5
  # Makes a statistic for the given tokens.
5
6
  #
6
7
  # Alias: +stats+
7
8
  class Statistic < Encoder
8
-
9
- register_for :statistic
10
-
9
+
10
+ register_for :stats, :statistic
11
+
11
12
  attr_reader :type_stats, :real_token_count # :nodoc:
12
-
13
+
13
14
  TypeStats = Struct.new :count, :size # :nodoc:
14
-
15
+
15
16
  protected
16
-
17
+
17
18
  def setup options
18
- super
19
-
20
19
  @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }
21
20
  @real_token_count = 0
22
21
  end
22
+
23
+ def generate tokens, options
24
+ @tokens = tokens
25
+ super
26
+ end
27
+
28
+ def text_token text, kind
29
+ @real_token_count += 1 unless kind == :space
30
+ @type_stats[kind].count += 1
31
+ @type_stats[kind].size += text.size
32
+ @type_stats['TOTAL'].size += text.size
33
+ @type_stats['TOTAL'].count += 1
34
+ end
35
+
36
+ # TODO Hierarchy handling
37
+ def begin_group kind
38
+ block_token 'begin_group'
39
+ end
40
+
41
+ def end_group kind
42
+ block_token 'end_group'
43
+ end
44
+
45
+ def begin_line kind
46
+ block_token 'begin_line'
47
+ end
48
+
49
+ def end_line kind
50
+ block_token 'end_line'
51
+ end
23
52
 
53
+ def block_token action
54
+ @type_stats['TOTAL'].count += 1
55
+ @type_stats[action].count += 1
56
+ end
57
+
24
58
  STATS = <<-STATS # :nodoc:
25
59
 
26
60
  Code Statistics
@@ -33,12 +67,12 @@ Token Types (%d):
33
67
  type count ratio size (average)
34
68
  -------------------------------------------------------------
35
69
  %s
36
- STATS
37
-
70
+ STATS
71
+ # space 12007 33.81 % 1.7
38
72
  TOKEN_TYPES_ROW = <<-TKR # :nodoc:
39
73
  %-20s %8d %6.2f %% %5.1f
40
- TKR
41
-
74
+ TKR
75
+
42
76
  def finish options
43
77
  all = @type_stats['TOTAL']
44
78
  all_count, all_size = all.count, all.size
@@ -48,49 +82,78 @@ Token Types (%d):
48
82
  types_stats = @type_stats.sort_by { |k, v| [-v.count, k.to_s] }.map do |k, v|
49
83
  TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size]
50
84
  end.join
51
- @out << STATS % [
85
+ STATS % [
52
86
  all_count, @real_token_count, all_size,
53
87
  @type_stats.delete_if { |k, v| k.is_a? String }.size,
54
88
  types_stats
55
89
  ]
56
-
57
- super
58
- end
59
-
60
- public
61
-
62
- def text_token text, kind
63
- @real_token_count += 1 unless kind == :space
64
- @type_stats[kind].count += 1
65
- @type_stats[kind].size += text.size
66
- @type_stats['TOTAL'].size += text.size
67
- @type_stats['TOTAL'].count += 1
68
- end
69
-
70
- # TODO Hierarchy handling
71
- def begin_group kind
72
- block_token ':begin_group', kind
73
- end
74
-
75
- def end_group kind
76
- block_token ':end_group', kind
77
- end
78
-
79
- def begin_line kind
80
- block_token ':begin_line', kind
81
- end
82
-
83
- def end_line kind
84
- block_token ':end_line', kind
85
90
  end
86
-
87
- def block_token action, kind
88
- @type_stats['TOTAL'].count += 1
89
- @type_stats[action].count += 1
90
- @type_stats[kind].count += 1
91
- end
92
-
91
+
93
92
  end
94
-
93
+
94
+ end
95
95
  end
96
+
97
+ if $0 == __FILE__
98
+ $VERBOSE = true
99
+ $: << File.join(File.dirname(__FILE__), '..')
100
+ eval DATA.read, nil, $0, __LINE__ + 4
96
101
  end
102
+
103
+ __END__
104
+ require 'test/unit'
105
+
106
+ class StatisticEncoderTest < Test::Unit::TestCase
107
+
108
+ def test_creation
109
+ assert CodeRay::Encoders::Statistic < CodeRay::Encoders::Encoder
110
+ stats = nil
111
+ assert_nothing_raised do
112
+ stats = CodeRay.encoder :statistic
113
+ end
114
+ assert_kind_of CodeRay::Encoders::Encoder, stats
115
+ end
116
+
117
+ TEST_INPUT = CodeRay::Tokens[
118
+ ['10', :integer],
119
+ ['(\\)', :operator],
120
+ [:begin_group, :string],
121
+ ['test', :content],
122
+ [:end_group, :string],
123
+ [:begin_line, :test],
124
+ ["\n", :space],
125
+ ["\n \t", :space],
126
+ [" \n", :space],
127
+ ["[]", :method],
128
+ [:end_line, :test],
129
+ ].flatten
130
+ TEST_OUTPUT = <<-'DEBUG'
131
+
132
+ Code Statistics
133
+
134
+ Tokens 11
135
+ Non-Whitespace 4
136
+ Bytes Total 20
137
+
138
+ Token Types (5):
139
+ type count ratio size (average)
140
+ -------------------------------------------------------------
141
+ TOTAL 11 100.00 % 1.8
142
+ space 3 27.27 % 3.0
143
+ begin_group 1 9.09 % 0.0
144
+ begin_line 1 9.09 % 0.0
145
+ content 1 9.09 % 4.0
146
+ end_group 1 9.09 % 0.0
147
+ end_line 1 9.09 % 0.0
148
+ integer 1 9.09 % 2.0
149
+ method 1 9.09 % 2.0
150
+ operator 1 9.09 % 3.0
151
+
152
+ DEBUG
153
+
154
+ def test_filtering_text_tokens
155
+ assert_equal TEST_OUTPUT, CodeRay::Encoders::Statistic.new.encode_tokens(TEST_INPUT)
156
+ assert_equal TEST_OUTPUT, TEST_INPUT.statistic
157
+ end
158
+
159
+ end
@@ -22,7 +22,7 @@ module CodeRay
22
22
  :annotation => '35',
23
23
  :attribute_name => '33',
24
24
  :attribute_value => '31',
25
- :binary => '1;35',
25
+ :bin => '1;35',
26
26
  :char => {
27
27
  :self => '36', :delimiter => '34'
28
28
  },
@@ -47,13 +47,14 @@ module CodeRay
47
47
  :hex => '1;36',
48
48
  :include => '33',
49
49
  :integer => '1;34',
50
+ :interpreted => '1;35',
50
51
  :key => '35',
51
52
  :label => '1;15',
52
53
  :local_variable => '33',
53
- :octal => '1;35',
54
+ :oct => '1;35',
54
55
  :operator_name => '1;29',
55
- :predefined_constant => '1;36',
56
- :predefined_type => '1;30',
56
+ :pre_constant => '1;36',
57
+ :pre_type => '1;30',
57
58
  :predefined => ['4', '1;34'],
58
59
  :preprocessor => '36',
59
60
  :pseudo_class => '34',
@@ -78,6 +79,7 @@ module CodeRay
78
79
  },
79
80
  :symbol => '1;32',
80
81
  :tag => '34',
82
+ :tag_special => ['34', '4'],
81
83
  :type => '1;34',
82
84
  :value => '36',
83
85
  :variable => '34',
@@ -91,10 +93,11 @@ module CodeRay
91
93
  TOKEN_COLORS[:method] = TOKEN_COLORS[:function]
92
94
  TOKEN_COLORS[:imaginary] = TOKEN_COLORS[:complex]
93
95
  TOKEN_COLORS[:begin_group] = TOKEN_COLORS[:end_group] =
94
- TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter]
95
-
96
+ TOKEN_COLORS[:nesting_delimiter] = TOKEN_COLORS[:escape] =
97
+ TOKEN_COLORS[:delimiter]
98
+
96
99
  protected
97
-
100
+
98
101
  def setup(options)
99
102
  super
100
103
  @opened = []
@@ -1,10 +1,11 @@
1
+ ($:.unshift '../..'; require 'coderay') unless defined? CodeRay
1
2
  module CodeRay
2
3
  module Encoders
3
4
 
4
5
  # Concats the tokens into a single string, resulting in the original
5
6
  # code string if no tokens were removed.
6
7
  #
7
- # Alias: +plain+, +plaintext+
8
+ # Alias: +plain+
8
9
  #
9
10
  # == Options
10
11
  #
@@ -13,34 +14,52 @@ module Encoders
13
14
  #
14
15
  # Default: empty String
15
16
  class Text < Encoder
16
-
17
+
17
18
  register_for :text
18
-
19
+
19
20
  FILE_EXTENSION = 'txt'
20
-
21
+
21
22
  DEFAULT_OPTIONS = {
22
23
  :separator => nil
23
24
  }
24
-
25
+
25
26
  def text_token text, kind
26
- super
27
-
28
- if @first
29
- @first = false
30
- else
31
- @out << @sep
32
- end if @sep
27
+ @out << text
28
+ @out << @sep if @sep
33
29
  end
34
-
30
+
35
31
  protected
36
32
  def setup options
37
33
  super
38
-
39
- @first = true
40
34
  @sep = options[:separator]
41
35
  end
42
-
36
+
37
+ def finish options
38
+ super.chomp @sep
39
+ end
40
+
43
41
  end
44
-
42
+
43
+ end
45
44
  end
45
+
46
+ if $0 == __FILE__
47
+ $VERBOSE = true
48
+ $: << File.join(File.dirname(__FILE__), '..')
49
+ eval DATA.read, nil, $0, __LINE__ + 4
46
50
  end
51
+
52
+ __END__
53
+ require 'test/unit'
54
+
55
+ class CountTest < Test::Unit::TestCase
56
+
57
+ def test_count
58
+ ruby = <<-RUBY
59
+ puts "Hello world!"
60
+ RUBY
61
+ tokens = CodeRay.scan ruby, :ruby
62
+ assert_equal ruby, tokens.encode_with(:text)
63
+ end
64
+
65
+ end
@@ -1,3 +1,4 @@
1
+ ($:.unshift '../..'; require 'coderay') unless defined? CodeRay
1
2
  module CodeRay
2
3
  module Encoders
3
4
 
@@ -34,7 +35,6 @@ module Encoders
34
35
  protected
35
36
  def setup options
36
37
  super
37
-
38
38
  @group_excluded = false
39
39
  @exclude = options[:exclude]
40
40
  @exclude = Array(@exclude) unless @exclude == :all
@@ -109,3 +109,60 @@ module Encoders
109
109
 
110
110
  end
111
111
  end
112
+
113
+ if $0 == __FILE__
114
+ $VERBOSE = true
115
+ $: << File.join(File.dirname(__FILE__), '..')
116
+ eval DATA.read, nil, $0, __LINE__ + 4
117
+ end
118
+
119
+ __END__
120
+ require 'test/unit'
121
+
122
+ class TokenKindFilterTest < Test::Unit::TestCase
123
+
124
+ def test_creation
125
+ assert CodeRay::Encoders::TokenKindFilter < CodeRay::Encoders::Encoder
126
+ assert CodeRay::Encoders::TokenKindFilter < CodeRay::Encoders::Filter
127
+ filter = nil
128
+ assert_nothing_raised do
129
+ filter = CodeRay.encoder :token_kind_filter
130
+ end
131
+ assert_instance_of CodeRay::Encoders::TokenKindFilter, filter
132
+ end
133
+
134
+ def test_filtering_text_tokens
135
+ tokens = CodeRay::Tokens.new
136
+ for i in 1..10
137
+ tokens.text_token i.to_s, :index
138
+ tokens.text_token ' ', :space if i < 10
139
+ end
140
+ assert_equal 10, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :space).count
141
+ assert_equal 10, tokens.token_kind_filter(:exclude => :space).count
142
+ assert_equal 9, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :space).count
143
+ assert_equal 9, tokens.token_kind_filter(:include => :space).count
144
+ assert_equal 0, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :all).count
145
+ assert_equal 0, tokens.token_kind_filter(:exclude => :all).count
146
+ end
147
+
148
+ def test_filtering_block_tokens
149
+ tokens = CodeRay::Tokens.new
150
+ 10.times do |i|
151
+ tokens.begin_group :index
152
+ tokens.text_token i.to_s, :content
153
+ tokens.end_group :index
154
+ tokens.begin_group :naught if i == 5
155
+ tokens.end_group :naught if i == 7
156
+ tokens.begin_line :blubb
157
+ tokens.text_token i.to_s, :content
158
+ tokens.end_line :blubb
159
+ end
160
+ assert_equal 16, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => :blubb).count
161
+ assert_equal 16, tokens.token_kind_filter(:include => :blubb).count
162
+ assert_equal 24, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :include => [:blubb, :content]).count
163
+ assert_equal 24, tokens.token_kind_filter(:include => [:blubb, :content]).count
164
+ assert_equal 32, CodeRay::Encoders::TokenKindFilter.new.encode_tokens(tokens, :exclude => :index).count
165
+ assert_equal 32, tokens.token_kind_filter(:exclude => :index).count
166
+ end
167
+
168
+ end