coderay 0.9.8 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. data/{lib/README → README_INDEX.rdoc} +10 -21
  2. data/Rakefile +6 -6
  3. data/bin/coderay +193 -64
  4. data/lib/coderay.rb +61 -105
  5. data/lib/coderay/duo.rb +17 -21
  6. data/lib/coderay/encoder.rb +100 -112
  7. data/lib/coderay/encoders/_map.rb +12 -7
  8. data/lib/coderay/encoders/comment_filter.rb +12 -30
  9. data/lib/coderay/encoders/count.rb +29 -11
  10. data/lib/coderay/encoders/debug.rb +32 -20
  11. data/lib/coderay/encoders/div.rb +13 -9
  12. data/lib/coderay/encoders/filter.rb +34 -51
  13. data/lib/coderay/encoders/html.rb +155 -161
  14. data/lib/coderay/encoders/html/css.rb +4 -9
  15. data/lib/coderay/encoders/html/numbering.rb +115 -0
  16. data/lib/coderay/encoders/html/output.rb +22 -70
  17. data/lib/coderay/encoders/json.rb +59 -45
  18. data/lib/coderay/encoders/lines_of_code.rb +12 -57
  19. data/lib/coderay/encoders/null.rb +6 -14
  20. data/lib/coderay/encoders/page.rb +13 -9
  21. data/lib/coderay/encoders/span.rb +13 -9
  22. data/lib/coderay/encoders/statistic.rb +58 -39
  23. data/lib/coderay/encoders/terminal.rb +179 -0
  24. data/lib/coderay/encoders/text.rb +31 -17
  25. data/lib/coderay/encoders/token_kind_filter.rb +111 -0
  26. data/lib/coderay/encoders/xml.rb +19 -18
  27. data/lib/coderay/encoders/yaml.rb +37 -9
  28. data/lib/coderay/for_redcloth.rb +4 -4
  29. data/lib/coderay/helpers/file_type.rb +127 -246
  30. data/lib/coderay/helpers/gzip.rb +41 -0
  31. data/lib/coderay/helpers/plugin.rb +241 -306
  32. data/lib/coderay/helpers/word_list.rb +65 -126
  33. data/lib/coderay/scanner.rb +173 -156
  34. data/lib/coderay/scanners/_map.rb +18 -17
  35. data/lib/coderay/scanners/c.rb +63 -77
  36. data/lib/coderay/scanners/clojure.rb +217 -0
  37. data/lib/coderay/scanners/cpp.rb +71 -84
  38. data/lib/coderay/scanners/css.rb +103 -120
  39. data/lib/coderay/scanners/debug.rb +47 -44
  40. data/lib/coderay/scanners/delphi.rb +70 -76
  41. data/lib/coderay/scanners/diff.rb +141 -50
  42. data/lib/coderay/scanners/erb.rb +81 -0
  43. data/lib/coderay/scanners/groovy.rb +104 -113
  44. data/lib/coderay/scanners/haml.rb +168 -0
  45. data/lib/coderay/scanners/html.rb +181 -110
  46. data/lib/coderay/scanners/java.rb +73 -75
  47. data/lib/coderay/scanners/java/builtin_types.rb +2 -0
  48. data/lib/coderay/scanners/java_script.rb +90 -101
  49. data/lib/coderay/scanners/json.rb +40 -53
  50. data/lib/coderay/scanners/php.rb +123 -147
  51. data/lib/coderay/scanners/python.rb +93 -91
  52. data/lib/coderay/scanners/raydebug.rb +66 -0
  53. data/lib/coderay/scanners/ruby.rb +343 -326
  54. data/lib/coderay/scanners/ruby/patterns.rb +40 -106
  55. data/lib/coderay/scanners/ruby/string_state.rb +71 -0
  56. data/lib/coderay/scanners/sql.rb +80 -66
  57. data/lib/coderay/scanners/text.rb +26 -0
  58. data/lib/coderay/scanners/xml.rb +1 -1
  59. data/lib/coderay/scanners/yaml.rb +74 -73
  60. data/lib/coderay/style.rb +10 -7
  61. data/lib/coderay/styles/_map.rb +3 -3
  62. data/lib/coderay/styles/alpha.rb +143 -0
  63. data/lib/coderay/token_kinds.rb +90 -0
  64. data/lib/coderay/tokens.rb +102 -277
  65. data/lib/coderay/tokens_proxy.rb +55 -0
  66. data/lib/coderay/version.rb +3 -0
  67. data/test/functional/basic.rb +200 -18
  68. data/test/functional/examples.rb +130 -0
  69. data/test/functional/for_redcloth.rb +15 -8
  70. data/test/functional/suite.rb +9 -6
  71. metadata +103 -123
  72. data/FOLDERS +0 -53
  73. data/bin/coderay_stylesheet +0 -4
  74. data/lib/coderay/encoders/html/numerization.rb +0 -133
  75. data/lib/coderay/encoders/term.rb +0 -158
  76. data/lib/coderay/encoders/token_class_filter.rb +0 -84
  77. data/lib/coderay/helpers/gzip_simple.rb +0 -123
  78. data/lib/coderay/scanners/nitro_xhtml.rb +0 -136
  79. data/lib/coderay/scanners/plaintext.rb +0 -20
  80. data/lib/coderay/scanners/rhtml.rb +0 -78
  81. data/lib/coderay/scanners/scheme.rb +0 -145
  82. data/lib/coderay/styles/cycnus.rb +0 -152
  83. data/lib/coderay/styles/murphy.rb +0 -134
  84. data/lib/coderay/token_classes.rb +0 -86
  85. data/test/functional/load_plugin_scanner.rb +0 -11
  86. data/test/functional/vhdl.rb +0 -126
  87. data/test/functional/word_list.rb +0 -79
@@ -21,10 +21,7 @@ module CodeRay
21
21
  # Create a new Duo, holding a lang and a format to highlight code.
22
22
  #
23
23
  # simple:
24
- # CodeRay::Duo[:ruby, :page].highlight 'bla 42'
25
- #
26
- # streaming:
27
- # CodeRay::Duo[:ruby, :page].highlight 'bar 23', :stream => true
24
+ # CodeRay::Duo[:ruby, :html].highlight 'bla 42'
28
25
  #
29
26
  # with options:
30
27
  # CodeRay::Duo[:ruby, :html, :hint => :debug].highlight '????::??'
@@ -38,7 +35,7 @@ module CodeRay
38
35
  # The options are forwarded to scanner and encoder
39
36
  # (see CodeRay.get_scanner_options).
40
37
  def initialize lang = nil, format = nil, options = {}
41
- if format == nil and lang.is_a? Hash and lang.size == 1
38
+ if format.nil? && lang.is_a?(Hash) && lang.size == 1
42
39
  @lang = lang.keys.first
43
40
  @format = lang[@lang]
44
41
  else
@@ -47,12 +44,12 @@ module CodeRay
47
44
  end
48
45
  @options = options
49
46
  end
50
-
47
+
51
48
  class << self
52
49
  # To allow calls like Duo[:ruby, :html].highlight.
53
50
  alias [] new
54
51
  end
55
-
52
+
56
53
  # The scanner of the duo. Only created once.
57
54
  def scanner
58
55
  @scanner ||= CodeRay.scanner @lang, CodeRay.get_scanner_options(@options)
@@ -64,22 +61,21 @@ module CodeRay
64
61
  end
65
62
 
66
63
  # Tokenize and highlight the code using +scanner+ and +encoder+.
67
- #
68
- # If the :stream option is set, the Duo will go into streaming mode,
69
- # saving memory for the cost of time.
70
- def encode code, options = { :stream => false }
71
- stream = options.delete :stream
64
+ def encode code, options = {}
72
65
  options = @options.merge options
73
- if stream
74
- encoder.encode_stream(code, @lang, options)
75
- else
76
- scanner.code = code
77
- encoder.encode_tokens(scanner.tokenize, options)
78
- end
66
+ encoder.encode(code, @lang, options)
79
67
  end
80
68
  alias highlight encode
81
-
69
+
70
+ # Allows to use Duo like a proc object:
71
+ #
72
+ # CodeRay::Duo[:python => :yaml].call(code)
73
+ #
74
+ # or, in Ruby 1.9 and later:
75
+ #
76
+ # CodeRay::Duo[:python => :yaml].(code)
77
+ alias call encode
78
+
82
79
  end
83
-
80
+
84
81
  end
85
-
@@ -1,5 +1,5 @@
1
1
  module CodeRay
2
-
2
+
3
3
  # This module holds the Encoder class and its subclasses.
4
4
  # For example, the HTML encoder is named CodeRay::Encoders::HTML
5
5
  # can be found in coderay/encoders/html.
@@ -8,9 +8,10 @@ module CodeRay
8
8
  # mechanism and the [] method that returns the Encoder class
9
9
  # belonging to the given format.
10
10
  module Encoders
11
+
11
12
  extend PluginHost
12
13
  plugin_path File.dirname(__FILE__), 'encoders'
13
-
14
+
14
15
  # = Encoder
15
16
  #
16
17
  # The Encoder base class. Together with Scanner and
@@ -26,34 +27,32 @@ module CodeRay
26
27
  class Encoder
27
28
  extend Plugin
28
29
  plugin_host Encoders
29
-
30
- attr_reader :token_stream
31
-
30
+
32
31
  class << self
33
-
34
- # Returns if the Encoder can be used in streaming mode.
35
- def streamable?
36
- is_a? Streamable
37
- end
38
-
32
+
39
33
  # If FILE_EXTENSION isn't defined, this method returns the
40
34
  # downcase class name instead.
41
35
  def const_missing sym
42
36
  if sym == :FILE_EXTENSION
43
- plugin_id
37
+ (defined?(@plugin_id) && @plugin_id || name[/\w+$/].downcase).to_s
44
38
  else
45
39
  super
46
40
  end
47
41
  end
48
-
42
+
43
+ # The default file extension for output file of this encoder class.
44
+ def file_extension
45
+ self::FILE_EXTENSION
46
+ end
47
+
49
48
  end
50
-
49
+
51
50
  # Subclasses are to store their default options in this constant.
52
- DEFAULT_OPTIONS = { :stream => false }
53
-
51
+ DEFAULT_OPTIONS = { }
52
+
54
53
  # The options you gave the Encoder at creating.
55
- attr_accessor :options
56
-
54
+ attr_accessor :options, :scanner
55
+
57
56
  # Creates a new Encoder.
58
57
  # +options+ is saved and used for all encode operations, as long
59
58
  # as you don't overwrite it there by passing additional options.
@@ -61,153 +60,142 @@ module CodeRay
61
60
  # Encoder objects provide three encode methods:
62
61
  # - encode simply takes a +code+ string and a +lang+
63
62
  # - encode_tokens expects a +tokens+ object instead
64
- # - encode_stream is like encode, but uses streaming mode.
65
63
  #
66
64
  # Each method has an optional +options+ parameter. These are
67
65
  # added to the options you passed at creation.
68
66
  def initialize options = {}
69
67
  @options = self.class::DEFAULT_OPTIONS.merge options
70
- raise "I am only the basic Encoder class. I can't encode "\
71
- "anything. :( Use my subclasses." if self.class == Encoder
68
+ @@CODERAY_TOKEN_INTERFACE_DEPRECATION_WARNING_GIVEN = false
72
69
  end
73
-
70
+
74
71
  # Encode a Tokens object.
75
72
  def encode_tokens tokens, options = {}
76
73
  options = @options.merge options
74
+ @scanner = tokens.scanner if tokens.respond_to? :scanner
77
75
  setup options
78
76
  compile tokens, options
79
77
  finish options
80
78
  end
81
-
82
- # Encode the given +code+ after tokenizing it using the Scanner
83
- # for +lang+.
79
+
80
+ # Encode the given +code+ using the Scanner for +lang+.
84
81
  def encode code, lang, options = {}
85
82
  options = @options.merge options
86
- scanner_options = CodeRay.get_scanner_options(options)
87
- tokens = CodeRay.scan code, lang, scanner_options
88
- encode_tokens tokens, options
83
+ @scanner = Scanners[lang].new code, CodeRay.get_scanner_options(options).update(:tokens => self)
84
+ setup options
85
+ @scanner.tokenize
86
+ finish options
89
87
  end
90
-
88
+
91
89
  # You can use highlight instead of encode, if that seems
92
90
  # more clear to you.
93
91
  alias highlight encode
94
-
95
- # Encode the given +code+ using the Scanner for +lang+ in
96
- # streaming mode.
97
- def encode_stream code, lang, options = {}
98
- raise NotStreamableError, self unless kind_of? Streamable
99
- options = @options.merge options
100
- setup options
101
- scanner_options = CodeRay.get_scanner_options options
102
- @token_stream =
103
- CodeRay.scan_stream code, lang, scanner_options, &self
104
- finish options
105
- end
106
-
107
- # Behave like a proc. The token method is converted to a proc.
108
- def to_proc
109
- method(:token).to_proc
110
- end
111
-
112
- # Return the default file extension for outputs of this encoder.
92
+
93
+ # The default file extension for this encoder.
113
94
  def file_extension
114
- self.class::FILE_EXTENSION
95
+ self.class.file_extension
115
96
  end
116
-
117
- protected
118
-
119
- # Called with merged options before encoding starts.
120
- # Sets @out to an empty string.
121
- #
122
- # See the HTML Encoder for an example of option caching.
123
- def setup options
124
- @out = ''
97
+
98
+ def << token
99
+ unless @@CODERAY_TOKEN_INTERFACE_DEPRECATION_WARNING_GIVEN
100
+ warn 'Using old Tokens#<< interface.'
101
+ @@CODERAY_TOKEN_INTERFACE_DEPRECATION_WARNING_GIVEN = true
102
+ end
103
+ self.token(*token)
125
104
  end
126
-
105
+
127
106
  # Called with +content+ and +kind+ of the currently scanned token.
128
107
  # For simple scanners, it's enougth to implement this method.
129
108
  #
130
- # By default, it calls text_token or block_token, depending on
131
- # whether +content+ is a String.
109
+ # By default, it calls text_token, begin_group, end_group, begin_line,
110
+ # or end_line, depending on the +content+.
132
111
  def token content, kind
133
- encoded_token =
134
- if content.is_a? ::String
135
- text_token content, kind
136
- elsif content.is_a? ::Symbol
137
- block_token content, kind
138
- else
139
- raise 'Unknown token content type: %p' % [content]
140
- end
141
- append_encoded_token_to_output encoded_token
142
- end
143
-
144
- def append_encoded_token_to_output encoded_token
145
- @out << encoded_token if encoded_token && defined?(@out) && @out
146
- end
147
-
148
- # Called for each text token ([text, kind]), where text is a String.
149
- def text_token text, kind
150
- end
151
-
152
- # Called for each block (non-text) token ([action, kind]),
153
- # where +action+ is a Symbol.
154
- #
155
- # Calls open_token, close_token, begin_line, and end_line according to
156
- # the value of +action+.
157
- def block_token action, kind
158
- case action
159
- when :open
160
- open_token kind
161
- when :close
162
- close_token kind
112
+ case content
113
+ when String
114
+ text_token content, kind
115
+ when :begin_group
116
+ begin_group kind
117
+ when :end_group
118
+ end_group kind
163
119
  when :begin_line
164
120
  begin_line kind
165
121
  when :end_line
166
122
  end_line kind
167
123
  else
168
- raise 'unknown block action: %p' % action
124
+ raise ArgumentError, 'Unknown token content type: %p, kind = %p' % [content, kind]
169
125
  end
170
126
  end
171
127
 
172
- # Called for each block token at the start of the block ([:open, kind]).
173
- def open_token kind
128
+ # Called for each text token ([text, kind]), where text is a String.
129
+ def text_token text, kind
130
+ @out << text
174
131
  end
175
132
 
176
- # Called for each block token end of the block ([:close, kind]).
177
- def close_token kind
133
+ # Starts a token group with the given +kind+.
134
+ def begin_group kind
178
135
  end
179
136
 
180
- # Called for each line token block at the start of the line ([:begin_line, kind]).
137
+ # Ends a token group with the given +kind+.
138
+ def end_group kind
139
+ end
140
+
141
+ # Starts a new line token group with the given +kind+.
181
142
  def begin_line kind
182
143
  end
183
144
 
184
- # Called for each line token block at the end of the line ([:end_line, kind]).
145
+ # Ends a new line token group with the given +kind+.
185
146
  def end_line kind
186
147
  end
187
-
148
+
149
+ protected
150
+
151
+ # Called with merged options before encoding starts.
152
+ # Sets @out to an empty string.
153
+ #
154
+ # See the HTML Encoder for an example of option caching.
155
+ def setup options
156
+ @out = get_output(options)
157
+ end
158
+
159
+ def get_output options
160
+ options[:out] || ''
161
+ end
162
+
163
+ # Append data.to_s to the output. Returns the argument.
164
+ def output data
165
+ @out << data.to_s
166
+ data
167
+ end
168
+
188
169
  # Called with merged options after encoding starts.
189
170
  # The return value is the result of encoding, typically @out.
190
171
  def finish options
191
172
  @out
192
173
  end
193
-
174
+
194
175
  # Do the encoding.
195
176
  #
196
- # The already created +tokens+ object must be used; it can be a
197
- # TokenStream or a Tokens object.
198
- if RUBY_VERSION >= '1.9'
199
- def compile tokens, options
200
- for text, kind in tokens
201
- token text, kind
177
+ # The already created +tokens+ object must be used; it must be a
178
+ # Tokens object.
179
+ def compile tokens, options = {}
180
+ content = nil
181
+ for item in tokens
182
+ if item.is_a? Array
183
+ raise ArgumentError, 'Two-element array tokens are no longer supported.'
184
+ end
185
+ if content
186
+ token content, item
187
+ content = nil
188
+ else
189
+ content = item
202
190
  end
203
191
  end
204
- else
205
- def compile tokens, options
206
- tokens.each(&self)
207
- end
192
+ raise 'odd number list for Tokens' if content
208
193
  end
209
-
194
+
195
+ alias tokens compile
196
+ public :tokens
197
+
210
198
  end
211
-
199
+
212
200
  end
213
201
  end
@@ -1,12 +1,17 @@
1
1
  module CodeRay
2
2
  module Encoders
3
-
3
+
4
4
  map \
5
- :loc => :lines_of_code,
6
- :plain => :text,
7
- :stats => :statistic,
8
- :terminal => :term,
9
- :tex => :latex
10
-
5
+ :loc => :lines_of_code,
6
+ :plain => :text,
7
+ :plaintext => :text,
8
+ :remove_comments => :comment_filter,
9
+ :stats => :statistic,
10
+ :term => :terminal,
11
+ :tty => :terminal,
12
+ :yml => :yaml
13
+
14
+ # No default because Tokens#nonsense should raise NoMethodError.
15
+
11
16
  end
12
17
  end
@@ -1,43 +1,25 @@
1
- ($:.unshift '../..'; require 'coderay') unless defined? CodeRay
2
1
  module CodeRay
3
2
  module Encoders
4
3
 
5
- load :token_class_filter
4
+ load :token_kind_filter
6
5
 
7
- class CommentFilter < TokenClassFilter
6
+ # A simple Filter that removes all tokens of the :comment kind.
7
+ #
8
+ # Alias: +remove_comments+
9
+ #
10
+ # Usage:
11
+ # CodeRay.scan('print # foo', :ruby).comment_filter.text
12
+ # #-> "print "
13
+ #
14
+ # See also: TokenKindFilter, LinesOfCode
15
+ class CommentFilter < TokenKindFilter
8
16
 
9
17
  register_for :comment_filter
10
18
 
11
19
  DEFAULT_OPTIONS = superclass::DEFAULT_OPTIONS.merge \
12
- :exclude => [:comment]
20
+ :exclude => [:comment, :docstring]
13
21
 
14
22
  end
15
23
 
16
24
  end
17
25
  end
18
-
19
- if $0 == __FILE__
20
- $VERBOSE = true
21
- $: << File.join(File.dirname(__FILE__), '..')
22
- eval DATA.read, nil, $0, __LINE__ + 4
23
- end
24
-
25
- __END__
26
- require 'test/unit'
27
-
28
- class CommentFilterTest < Test::Unit::TestCase
29
-
30
- def test_filtering_comments
31
- tokens = CodeRay.scan <<-RUBY, :ruby
32
- #!/usr/bin/env ruby
33
- # a minimal Ruby program
34
- puts "Hello world!"
35
- RUBY
36
- assert_equal <<-RUBY_FILTERED, tokens.comment_filter.text
37
- #!/usr/bin/env ruby
38
-
39
- puts "Hello world!"
40
- RUBY_FILTERED
41
- end
42
-
43
- end