rouge 0.4.0 → 0.5.0

data/Gemfile

@@ -17,3 +17,5 @@ gem 'shotgun'
 # docs
 gem 'yard'
 gem 'github-markup'
+
+gem 'pry'

data/bin/rougify

@@ -9,6 +9,6 @@ load ROOT_DIR.join('lib/rouge/cli.rb')
 begin
   Rouge::CLI.start
 rescue => e
-  $stderr.puts e.message
+  $stderr.puts "rouge: #{e.message}"
   exit 1
 end

data/lib/rouge/formatters/html.rb

@@ -22,9 +22,9 @@ module Rouge
       # Content will be wrapped in a `<pre>` tag with the given
       # `:css_class` unless `:wrap` is set to `false`.
       def initialize(opts={})
-        @css_class = opts[:css_class] || 'highlight'
-        @line_numbers = opts.fetch(:line_numbers) { false }
-        @inline_theme = opts.fetch(:inline_theme) { nil }
+        @css_class = opts.fetch(:css_class, 'highlight')
+        @line_numbers = opts.fetch(:line_numbers, false)
+        @inline_theme = opts.fetch(:inline_theme, nil)
         @wrap = opts.fetch(:wrap, true)
       end
 
@@ -48,15 +48,20 @@ module Rouge
 
       def stream_tableized(tokens, &b)
         num_lines = 0
-        code = ''
+        last_val = ''
+        formatted = ''
 
         tokens.each do |tok, val|
+          last_val = val
           num_lines += val.scan(/\n/).size
-          span(tok, val) { |str| code << str }
+          span(tok, val) { |str| formatted << str }
         end
 
-        # add an extra line number for non-newline-terminated strings
-        num_lines += 1 if code[-1] != "\n"
+        # add an extra line for non-newline-terminated strings
+        if last_val[-1] != "\n"
+          num_lines += 1
+          span(Token::Tokens::Text::Whitespace, "\n") { |str| formatted << str }
+        end
 
         # generate a string of newline-separated line numbers for the gutter
         numbers = num_lines.times.map do |x|
@@ -72,7 +77,7 @@ module Rouge
         yield '</td>'
 
         yield '<td class="code">'
-        yield code
+        yield formatted
         yield '</td>'
 
         yield '</tr></tbody></table>'

data/lib/rouge/formatters/terminal256.rb

@@ -118,7 +118,7 @@ module Rouge
           when 3
             color.chars.map { |c| "#{c}#{c}" }
           when 6
-            color.scan /../
+            color.scan(/../)
           else
             raise "invalid color: #{color}"
           end
@@ -127,19 +127,23 @@ module Rouge
         end
 
         def self.closest_color(r, g, b)
-          distance = 257 * 257 * 3 # (max distance, from #000000 to #ffffff)
+          @@colors_cache ||= {}
+          key = (r << 16) + (g << 8) + b
+          @@colors_cache.fetch(key) do
+            distance = 257 * 257 * 3 # (max distance, from #000000 to #ffffff)
 
-          match = 0
+            match = 0
 
-          xterm_colors.each_with_index do |(cr, cg, cb), i|
-            d = (r - cr)**2 + (g - cg)**2 + (b - cb)**2
-            next if d >= distance
+            xterm_colors.each_with_index do |(cr, cg, cb), i|
+              d = (r - cr)**2 + (g - cg)**2 + (b - cb)**2
+              next if d >= distance
 
-            match = i
-            distance = d
-          end
+              match = i
+              distance = d
+            end
 
-          match
+            match
+          end
         end
       end
 
@@ -151,7 +155,7 @@ module Rouge
       end
 
       def get_style(token)
-        return text_style if token.name == 'Text'
+        return text_style if token == Token::Tokens::Text
 
         theme.get_own_style(token) || text_style
       end

data/lib/rouge/lexer.rb

@@ -6,6 +6,8 @@ module Rouge
   # @abstract
   # A lexer transforms text into a stream of `[token, chunk]` pairs.
   class Lexer
+    include Token::Tokens
+
     class << self
       # Lexes `stream` with the given options.  The lex is delegated to a
       # new instance.
@@ -141,7 +143,7 @@ module Rouge
       def guess(info={})
         lexers = guesses(info)
 
-        return Lexers::Text if lexers.empty?
+        return Lexers::PlainText if lexers.empty?
         return lexers[0] if lexers.size == 1
 
         raise AmbiguousGuess.new(lexers)

data/lib/rouge/lexers/c.rb

@@ -34,53 +34,63 @@ module Rouge
         )
       end
 
-      state :whitespace do
-        rule /^#if\s+0\b/, 'Comment.Preproc', :if_0
-        rule /^#/, 'Comment.Preproc', :macro
-        rule /^#{ws}#if\s+0\b/, 'Comment.Preproc', :if_0
-        rule /^#{ws}#/, 'Comment.Preproc', :macro
-        rule /^(\s*)(#{id}:(?!:))/ do
-          group 'Text'
-          group 'Name.Label'
-        end
+      start { push :bol }
+
+      state :bol do
+        mixin :inline_whitespace
+
+        rule /#if\s0/, Comment::Preproc, :if_0
+
+        rule /#/, Comment::Preproc, :macro
+
+        rule /#{id}:(?!:)/, Name::Label
 
-        rule /\s+/m, 'Text'
-        rule /\\\n/, 'Text' # line continuation
-        rule %r(//(\n|(.|\n)*?[^\\]\n)), 'Comment.Single'
-        rule %r(/(\\\n)?[*](.|\n)*?[*](\\\n)?/), 'Comment.Multiline'
+        rule(//) { pop! }
+      end
+
+      state :inline_whitespace do
+        rule /[ \t\r]+/, Text
+        rule /\\\n/, Text # line continuation
+        rule %r(/(\\\n)?[*].*?[*](\\\n)?/)m, Comment::Multiline
+      end
+
+      state :whitespace do
+        rule /\n+/, Text, :bol
+        rule %r(//(\\.|.)*?\n), Comment::Single, :bol
+        mixin :inline_whitespace
       end
 
       state :statements do
-        rule /\s+/m, 'Text'
-
-        rule /L?"/, 'Literal.String', :string
-        rule %r(L?'(\\.|\\[0-7]{1,3}|\\x[a-f0-9]{1,2}|[^\\'\n])')i, 'Literal.String.Char'
-        rule %r((\d+\.\d*|\.\d+|\d+)[e][+-]?\d+[lu]*)i, 'Literal.Number.Float'
-        rule /0x[0-9a-f]+[lu]*/i, 'Literal.Number.Hex'
-        rule /0[0-7]+[lu]*/i, 'Literal.Number.Oct'
-        rule /\d+[lu]*/i, 'Literal.Number.Integer'
-        rule %r(\*/), 'Error'
-        rule %r([~!%^&*+=\|?:<>/-]), 'Operator'
-        rule /[()\[\],.]/, 'Punctuation'
-        rule /\bcase\b/, 'Keyword', :case
-        rule /(?:true|false|NULL)\b/, 'Name.Builtin'
+        rule /\s+/m, Text
+
+        rule /L?"/, Str, :string
+        rule %r(L?'(\\.|\\[0-7]{1,3}|\\x[a-f0-9]{1,2}|[^\\'\n])')i, Str::Char
+        rule %r((\d+\.\d*|\.\d+|\d+)[e][+-]?\d+[lu]*)i, Num::Float
+        rule /0x[0-9a-f]+[lu]*/i, Num::Hex
+        rule /0[0-7]+[lu]*/i, Num::Oct
+        rule /\d+[lu]*/i, Num::Integer
+        rule %r(\*/), Error
+        rule %r([~!%^&*+=\|?:<>/-]), Operator
+        rule /[()\[\],.]/, Punctuation
+        rule /\bcase\b/, Keyword, :case
+        rule /(?:true|false|NULL)\b/, Name::Builtin
         rule id do |m|
           name = m[0]
 
           if self.class.keywords.include? name
-            token 'Keyword'
+            token Keyword
           elsif self.class.keywords_type.include? name
-            token 'Keyword.Type'
+            token Keyword::Type
           elsif self.class.reserved.include? name
-            token 'Keyword.Reserved'
+            token Keyword::Reserved
           else
-            token 'Name'
+            token Name
           end
         end
       end
 
       state :case do
-        rule /:/, 'Punctuation', :pop!
+        rule /:/, Punctuation, :pop!
         mixin :statements
       end
 
@@ -96,10 +106,10 @@ module Rouge
         )mx do |m|
           # TODO: do this better.
           delegate C, m[1]
-          token 'Name.Function', m[2]
+          token Name::Function, m[2]
           delegate C, m[3]
           delegate C, m[4]
-          token 'Punctuation', m[5]
+          token Punctuation, m[5]
           push :function
         end
 
@@ -112,10 +122,10 @@ module Rouge
         )mx do |m|
           # TODO: do this better.
           delegate C, m[1]
-          token 'Name.Function'
+          token Name::Function
           delegate C, m[3]
           delegate C, m[4]
-          token 'Punctuation'
+          token Punctuation
           push :statement
         end
 
@@ -123,42 +133,42 @@ module Rouge
       end
 
       state :statement do
-        rule /;/, 'Punctuation', :pop!
+        rule /;/, Punctuation, :pop!
         mixin :whitespace
         mixin :statements
-        rule /[{}]/, 'Punctuation'
+        rule /[{}]/, Punctuation
       end
 
       state :function do
         mixin :whitespace
         mixin :statements
-        rule /;/, 'Punctuation'
-        rule /{/, 'Punctuation', :function
-        rule /}/, 'Punctuation', :pop!
+        rule /;/, Punctuation
+        rule /{/, Punctuation, :function
+        rule /}/, Punctuation, :pop!
       end
 
       state :string do
-        rule /"/, 'Literal.String', :pop!
-        rule /\\([\\abfnrtv"']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})/, 'Literal.String.Escape'
-        rule /[^\\"\n]+/, 'Literal.String'
-        rule /\\\n/, 'Literal.String'
-        rule /\\/, 'Literal.String' # stray backslash
+        rule /"/, Str, :pop!
+        rule /\\([\\abfnrtv"']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})/, Str::Escape
+        rule /[^\\"\n]+/, Str
+        rule /\\\n/, Str
+        rule /\\/, Str # stray backslash
       end
 
       state :macro do
-        rule %r([^/\n]+), 'Comment.Preproc'
-        rule %r(/[*].*?[*]/)m, 'Comment.Multiliine'
-        rule %r(//.*$), 'Comment.Single'
-        rule %r(/), 'Comment.Preproc'
-        rule /(?<=\\)\n/, 'Comment.Preproc'
-        rule /\n/, 'Comment.Preproc', :pop!
+        rule %r([^/\n]+), Comment::Preproc
+        rule %r(/[*].*?[*]/)m, Comment::Multiline
+        rule %r(//.*$), Comment::Single
+        rule %r(/), Comment::Preproc
+        rule /(?<=\\)\n/, Comment::Preproc
+        rule /\n/, Comment::Preproc, :pop!
       end
 
       state :if_0 do
-        rule /^\s*#if.*?(?<!\\)\n/, 'Comment.Preproc', :if_0
-        rule /^\s*#el(?:se|if).*\n/, 'Comment.Preproc', :pop!
-        rule /^\s*#endif.*?(?<!\\)\n/, 'Comment.Preproc', :pop!
-        rule /.*?\n/, 'Comment'
+        rule /^\s*#if.*?(?<!\\)\n/, Comment, :if_0
+        rule /^\s*#el(?:se|if).*\n/, Comment::Preproc, :pop!
+        rule /^\s*#endif.*?(?<!\\)\n/, Comment, :pop!
+        rule /.*?\n/, Comment
       end
     end
   end

data/lib/rouge/lexers/clojure.rb

@@ -64,45 +64,45 @@ module Rouge
       keyword = %r([\w!\#$%*+,<=>?/.-]+)
 
       def name_token(name)
-        return 'Keyword' if self.class.keywords.include?(name)
-        return 'Name.Builtin' if self.class.builtins.include?(name)
+        return Keyword if self.class.keywords.include?(name)
+        return Name::Builtin if self.class.builtins.include?(name)
         nil
       end
 
       state :root do
-        rule /;.*?\n/, 'Comment.Single'
-        rule /\s+/m, 'Text.Whitespace'
+        rule /;.*?\n/, Comment::Single
+        rule /\s+/m, Text::Whitespace
 
-        rule /-?\d+\.\d+/, 'Literal.Number.Float'
-        rule /-?\d+/, 'Literal.Number.Integer'
-        rule /0x-?[0-9a-fA-F]+/, 'Literal.Number.Hex'
+        rule /-?\d+\.\d+/, Num::Float
+        rule /-?\d+/, Num::Integer
+        rule /0x-?[0-9a-fA-F]+/, Num::Hex
 
-        rule /"(\\.|[^"])*"/, 'Literal.String'
-        rule /'#{keyword}/, 'Literal.String.Symbol'
-        rule /:#{keyword}/, 'Name.Constant'
-        rule /\\(.|[a-z]+)/i, 'Literal.String.Char'
+        rule /"(\\.|[^"])*"/, Str
+        rule /'#{keyword}/, Str::Symbol
+        rule /:#{keyword}/, Name::Constant
+        rule /\\(.|[a-z]+)/i, Str::Char
 
 
-        rule /~@|[`\'#^~&]/, 'Operator'
+        rule /~@|[`\'#^~&]/, Operator
 
         rule /(\()(\s*)(#{identifier})/m do |m|
-          token 'Punctuation', m[1]
-          token 'Text.Whitespace', m[2]
-          token(name_token(m[3]) || 'Name.Function', m[3])
+          token Punctuation, m[1]
+          token Text::Whitespace, m[2]
+          token(name_token(m[3]) || Name::Function, m[3])
         end
 
         rule identifier do |m|
-          token name_token(m[0]) || 'Name'
+          token name_token(m[0]) || Name
         end
 
         # vectors
-        rule /[\[\]]/, 'Punctuation'
+        rule /[\[\]]/, Punctuation
 
         # maps
-        rule /[{}]/, 'Punctuation'
+        rule /[{}]/, Punctuation
 
         # parentheses
-        rule /[()]/, 'Punctuation'
+        rule /[()]/, Punctuation
       end
     end
   end

data/lib/rouge/lexers/coffeescript.rb

@@ -38,9 +38,9 @@ module Rouge
       id = /[$a-zA-Z_][a-zA-Z0-9_]*/
 
       state :comments_and_whitespace do
-        rule /\s+/m, 'Text'
-        rule /###.*?###/m, 'Comment.Multiline'
-        rule /#.*?\n/, 'Comment.Single'
+        rule /\s+/m, Text
+        rule /###.*?###/m, Comment::Multiline
+        rule /#.*?\n/, Comment::Single
       end
 
       state :multiline_regex do
@@ -49,22 +49,22 @@ module Rouge
         mixin :has_interpolation
         mixin :comments_and_whitespace
 
-        rule %r(///([gim]+\b|\B)), 'Literal.String.Regex', :pop!
-        rule %r(/), 'Literal.String.Regex'
-        rule %r([^/#]+), 'Literal.String.Regex'
+        rule %r(///([gim]+\b|\B)), Str::Regex, :pop!
+        rule %r(/), Str::Regex
+        rule %r([^/#]+), Str::Regex
       end
 
       state :slash_starts_regex do
         mixin :comments_and_whitespace
         rule %r(///) do
-          token 'Literal.String.Regex'
-          pop!; push :multiline_regex
+          token Str::Regex
+          goto :multiline_regex
         end
 
         rule %r(
           /(\\.|[^\[/\\\n]|\[(\\.|[^\]\\\n])*\])+/ # a regex
           ([gim]+\b|\B)
-        )x, 'Literal.String.Regex', :pop!
+        )x, Str::Regex, :pop!
 
         rule(//) { pop! }
       end
@@ -75,97 +75,97 @@ module Rouge
         rule %r(
           [+][+]|--|~|&&|\band\b|\bor\b|\bis\b|\bisnt\b|\bnot\b|[?]|:|=|
           [|][|]|\\(?=\n)|(<<|>>>?|==?|!=?|[-<>+*`%&|^/])=?
-        )x, 'Operator', :slash_starts_regex
+        )x, Operator, :slash_starts_regex
 
-        rule /[-=]>/, 'Name.Function'
+        rule /[-=]>/, Name::Function
 
         rule /(@)([ \t]*)(#{id})/ do
-          group 'Name.Variable.Instance'; group 'Text'
-          group 'Name.Attribute'
+          group Name::Variable::Instance; group Text
+          group Name::Attribute
           push :slash_starts_regex
         end
 
         rule /([.])([ \t]*)(#{id})/ do
-          group 'Punctuation'; group 'Text'
-          group 'Name.Attribute'
+          group Punctuation; group Text
+          group Name::Attribute
           push :slash_starts_regex
         end
 
-        rule /#{id}(?=\s*:)/, 'Name.Attribute', :slash_starts_regex
+        rule /#{id}(?=\s*:)/, Name::Attribute, :slash_starts_regex
 
         rule /#{id}/ do |m|
           if self.class.keywords.include? m[0]
-            token 'Keyword'
+            token Keyword
           elsif self.class.constants.include? m[0]
-            token 'Name.Constant'
+            token Name::Constant
           elsif self.class.builtins.include? m[0]
-            token 'Name.Builtin'
+            token Name::Builtin
           else
-            token 'Name.Other'
+            token Name::Other
           end
 
           push :slash_starts_regex
         end
 
-        rule /[{(\[;,]/, 'Punctuation', :slash_starts_regex
-        rule /[})\].]/, 'Punctuation'
+        rule /[{(\[;,]/, Punctuation, :slash_starts_regex
+        rule /[})\].]/, Punctuation
 
-        rule /\d+[.]\d+([eE]\d+)?[fd]?/, 'Literal.Number.Float'
-        rule /0x[0-9a-fA-F]+/, 'Literal.Number.Hex'
-        rule /\d+/, 'Literal.Number.Integer'
-        rule /"""/, 'Literal.String', :tdqs
-        rule /'''/, 'Literal.String', :tsqs
-        rule /"/, 'Literal.String', :dqs
-        rule /'/, 'Literal.String', :sqs
+        rule /\d+[.]\d+([eE]\d+)?[fd]?/, Num::Float
+        rule /0x[0-9a-fA-F]+/, Num::Hex
+        rule /\d+/, Num::Integer
+        rule /"""/, Str, :tdqs
+        rule /'''/, Str, :tsqs
+        rule /"/, Str, :dqs
+        rule /'/, Str, :sqs
       end
 
       state :strings do
         # all coffeescript strings are multi-line
-        rule /[^#\\'"]+/m, 'Literal.String'
+        rule /[^#\\'"]+/m, Str
 
-        rule /\\./, 'Literal.String.Escape'
-        rule /#/, 'Literal.String'
+        rule /\\./, Str::Escape
+        rule /#/, Str
       end
 
       state :double_strings do
-        rule /'/, 'Literal.String'
+        rule /'/, Str
         mixin :has_interpolation
         mixin :strings
       end
 
       state :single_strings do
-        rule /"/, 'Literal.String'
+        rule /"/, Str
         mixin :strings
       end
 
       state :interpolation do
-        rule /}/, 'Literal.String.Interpol', :pop!
+        rule /}/, Str::Interpol, :pop!
         mixin :root
       end
 
       state :has_interpolation do
-        rule /[#][{]/, 'Literal.String.Interpol', :interpolation
+        rule /[#][{]/, Str::Interpol, :interpolation
       end
 
       state :dqs do
-        rule /"/, 'Literal.String', :pop!
+        rule /"/, Str, :pop!
         mixin :double_strings
       end
 
       state :tdqs do
-        rule /"""/, 'Literal.String', :pop!
-        rule /"/, 'Literal.String'
+        rule /"""/, Str, :pop!
+        rule /"/, Str
         mixin :double_strings
       end
 
       state :sqs do
-        rule /'/, 'Literal.String', :pop!
+        rule /'/, Str, :pop!
         mixin :single_strings
       end
 
       state :tsqs do
-        rule /'''/, 'Literal.String', :pop!
-        rule /'/, 'Literal.String'
+        rule /'''/, Str, :pop!
+        rule /'/, Str
         mixin :single_strings
       end
     end