rouge 0.0.13 → 0.0.14

data/lib/rouge.rb

@@ -18,19 +18,26 @@ load load_dir.join('rouge/version.rb')
 load load_dir.join('rouge/text_analyzer.rb')
 load load_dir.join('rouge/token.rb')
 load load_dir.join('rouge/lexer.rb')
+
 load load_dir.join('rouge/lexers/text.rb')
 load load_dir.join('rouge/lexers/diff.rb')
+load load_dir.join('rouge/lexers/tex.rb')
 
+load load_dir.join('rouge/lexers/make.rb')
 load load_dir.join('rouge/lexers/shell.rb')
 
 load load_dir.join('rouge/lexers/javascript.rb')
 load load_dir.join('rouge/lexers/css.rb')
 load load_dir.join('rouge/lexers/html.rb')
 load load_dir.join('rouge/lexers/xml.rb')
+load load_dir.join('rouge/lexers/php.rb')
+
+load load_dir.join('rouge/lexers/erb.rb')
 
 load load_dir.join('rouge/lexers/tcl.rb')
 load load_dir.join('rouge/lexers/python.rb')
 load load_dir.join('rouge/lexers/ruby.rb')
+load load_dir.join('rouge/lexers/perl.rb')
 
 load load_dir.join('rouge/lexers/haskell.rb')
 load load_dir.join('rouge/lexers/scheme.rb')
@@ -42,6 +49,7 @@ load load_dir.join('rouge/lexers/java.rb')
 
 load load_dir.join('rouge/formatter.rb')
 load load_dir.join('rouge/formatters/html.rb')
+load load_dir.join('rouge/formatters/terminal256.rb')
 
 load load_dir.join('rouge/theme.rb')
 load load_dir.join('rouge/themes/thankful_eyes.rb')

data/lib/rouge/cli.rb

@@ -25,11 +25,13 @@ module Rouge
     end
 
     desc 'highlight [FILE]', 'highlight some code'
-    option :file, :aliases => '-f',  :desc => 'the file to operate on'
+    option :input_file, :aliases => '-i',  :desc => 'the file to operate on'
     option :lexer, :aliases => '-l',
       :desc => ('Which lexer to use.  If not provided, rougify will try to ' +
                 'guess based on --mimetype, the filename, and the file ' +
                 'contents.')
+    option :formatter, :aliases => '-f', :default => 'html',
+      :desc => ('Which formatter to use.')
     option :mimetype, :aliases => '-m',
       :desc => ('a mimetype that Rouge will use to guess the correct lexer. ' +
                 'This is ignored if --lexer is specified.')
@@ -52,8 +54,10 @@ module Rouge
         raise "unknown lexer: #{options[:lexer]}" unless lexer_class
       end
 
+      formatter_class = Formatter.find(options[:formatter])
+
       # only HTML is supported for now
-      formatter = Formatters::HTML.new(normalize_hash_keys(options[:formatter_opts]))
+      formatter = formatter_class.new(normalize_hash_keys(options[:formatter_opts]))
       lexer = lexer_class.new(normalize_hash_keys(options[:lexer_opts]))
 
       puts Rouge.highlight(source, lexer, formatter)

data/lib/rouge/formatter.rb

@@ -1,5 +1,18 @@
 module Rouge
   class Formatter
+    REGISTRY = {}
+
+    def self.tag(tag=nil)
+      return @tag unless tag
+      REGISTRY[tag] = self
+
+      @tag = tag
+    end
+
+    def self.find(tag)
+      REGISTRY[tag]
+    end
+
     def render(tokens)
       enum_for(:stream, tokens).to_a.join
     end

data/lib/rouge/formatters/html.rb

@@ -4,6 +4,8 @@ require 'cgi'
 module Rouge
   module Formatters
     class HTML < Formatter
+      tag 'html'
+
       def initialize(opts={})
         @css_class = opts[:css_class] || 'highlight'
       end

data/lib/rouge/formatters/terminal256.rb

@@ -0,0 +1,152 @@
+module Rouge
+  module Formatters
+    class Terminal256 < Formatter
+      tag 'terminal256'
+
+      attr_reader :theme
+      def initialize(opts={})
+        @theme = opts[:theme] || Themes::ThankfulEyes
+        @theme = Theme.find(@theme) if @theme.is_a? String
+      end
+
+      def stream(tokens, &b)
+        tokens.each do |tok, val|
+          escape = escape_sequence(tok)
+          yield escape.style_string
+          yield val
+          yield escape.reset_string
+        end
+      end
+
+      class EscapeSequence
+        attr_reader :style
+        def initialize(style)
+          @style = style
+        end
+
+        def self.xterm_colors
+          @xterm_colors ||= [].tap do |out|
+            # colors 0..15: 16 basic colors
+            out << [0x00, 0x00, 0x00] # 0
+            out << [0xcd, 0x00, 0x00] # 1
+            out << [0x00, 0xcd, 0x00] # 2
+            out << [0xcd, 0xcd, 0x00] # 3
+            out << [0x00, 0x00, 0xee] # 4
+            out << [0xcd, 0x00, 0xcd] # 5
+            out << [0x00, 0xcd, 0xcd] # 6
+            out << [0xe5, 0xe5, 0xe5] # 7
+            out << [0x7f, 0x7f, 0x7f] # 8
+            out << [0xff, 0x00, 0x00] # 9
+            out << [0x00, 0xff, 0x00] # 10
+            out << [0xff, 0xff, 0x00] # 11
+            out << [0x5c, 0x5c, 0xff] # 12
+            out << [0xff, 0x00, 0xff] # 13
+            out << [0x00, 0xff, 0xff] # 14
+            out << [0xff, 0xff, 0xff] # 15
+
+            # colors 16..232: the 6x6x6 color cube
+            valuerange = [0x00, 0x5f, 0x87, 0xaf, 0xd7, 0xff]
+
+            217.times do |i|
+              r = valuerange[(i / 36) % 6]
+              g = valuerange[(i / 6) % 6]
+              b = valuerange[i % 6]
+              out << [r, g, b]
+            end
+
+            # colors 233..253: grayscale
+            1.upto 22 do |i|
+              v = 8 + i * 10
+              out << [v, v, v]
+            end
+          end
+        end
+
+        def fg
+          return @fg if instance_variable_defined? :@fg
+          @fg = style.fg && self.class.color_index(style.fg)
+        end
+
+        def bg
+          return @bg if instance_variable_defined? :@bg
+          @bg = style.bg && self.class.color_index(style.bg)
+        end
+
+        def style_string
+          @style_string ||= begin
+            attrs = []
+
+            attrs << ['38', '5', fg.to_s] if fg
+            attrs << ['45', '5', bg.to_s] if bg
+            attrs << '01' if style[:bold]
+            attrs << '04' if style[:italic] # underline, but hey, whatevs
+            escape(attrs)
+          end
+        end
+
+        def reset_string
+          @reset_string ||= begin
+            attrs = []
+            attrs << '39' if fg # fg reset
+            attrs << '49' if bg # bg reset
+            attrs << '00' if style[:bold] || style[:italic]
+
+            escape(attrs)
+          end
+        end
+
+      # private
+        def escape(attrs)
+          return '' if attrs.empty?
+          "\e[#{attrs.join(';')}m"
+        end
+
+        def self.color_index(color)
+          @color_index_cache ||= {}
+          @color_index_cache[color] ||= closest_color(*get_rgb(color))
+        end
+
+        def self.get_rgb(color)
+          color = $1 if color =~ /#([0-9a-f]+)/i
+          hexes = case color.size
+          when 3
+            color.chars.map { |c| "#{c}#{c}" }
+          when 6
+            color.scan /../
+          else
+            raise "invalid color: #{color}"
+          end
+
+          hexes.map { |h| h.to_i(16) }
+        end
+
+        def self.closest_color(r, g, b)
+          distance = 257 * 257 * 3 # (max distance, from #000000 to #ffffff)
+
+          match = 0
+
+          xterm_colors.each_with_index do |(cr, cg, cb), i|
+            d = (r - cr)**2 + (g - cg)**2 + (b - cb)**2
+            next if d >= distance
+
+            match = i
+            distance = d
+          end
+
+          match
+        end
+      end
+
+    # private
+      def escape_sequence(token)
+        @escape_sequences ||= {}
+        @escape_sequences[token.name] ||= begin
+          esc = EscapeSequence.new(theme.get_style(token))
+          # don't highlight text backgrounds
+          esc.style.delete(:bg) if token.name == 'Text'
+          esc
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexer.rb

@@ -4,35 +4,10 @@ require 'strscan'
 module Rouge
   class Lexer
     class << self
-      def make(opts={}, &b)
-        _sup = self
-
-        Class.new(self) do
-          @lazy_load_proc = b
-          @default_options = _sup.default_options.merge(opts)
-          @parent = _sup
-        end
-      end
-
       def lex(stream, opts={}, &b)
         new(opts).lex(stream, &b)
       end
 
-    protected
-      def force_load!
-        return self if @force_load
-        @force_load = true
-        @lazy_load_proc && instance_eval(&@lazy_load_proc)
-
-        self
-      end
-    public
-
-      def new(*a, &b)
-        force_load!
-        super(*a, &b)
-      end
-
       def default_options
         @default_options ||= {}
       end
@@ -121,7 +96,6 @@ module Rouge
 
     def initialize(opts={}, &b)
       options(opts)
-      @lazy_load_proc = b
     end
 
     def options(o={})
@@ -146,9 +120,15 @@ module Rouge
       lex(stream).to_a
     end
 
-    def lex(string, &b)
+    def reset!
+      # noop, called after each lex is finished
+    end
+
+    def lex(string, opts={}, &b)
       return enum_for(:lex, string) unless block_given?
 
+      reset! unless opts[:continue]
+
       last_token = nil
       last_val = nil
       stream_tokens(StringScanner.new(string)) do |tok, val|
@@ -244,7 +224,7 @@ module Rouge
       attr_accessor :scanner
       attr_accessor :stack
       attr_accessor :lexer
-      def initialize(lexer, scanner, stack=nil)
+      def initialize(lexer, scanner=nil, stack=nil)
         @lexer = lexer
         @scanner = scanner
         @stack = stack || [lexer.get_state(:root)]
@@ -258,8 +238,15 @@ module Rouge
       end
 
       def push(state_name=nil, &b)
-        debug { "    pushing #{state_name || 'anonymous state'}" }
-        stack.push(state.relative_state(state_name, &b))
+        # use the top of the stack by default
+        if state_name || b
+          push_state = state.relative_state(state_name, &b)
+        else
+          push_state = self.state
+        end
+
+        debug { "    pushing #{push_state.name}" }
+        stack.push(push_state)
       end
 
       def in_state?(state_name)
@@ -300,10 +287,10 @@ module Rouge
       end
 
       def delegate(lexer, text=nil)
-        debug { "    delegating to #{lexer.name}" }
+        debug { "    delegating to #{lexer.inspect}" }
         text ||= scanner[0]
 
-        lexer.lex(text) do |tok, val|
+        lexer.lex(text, :continue => true) do |tok, val|
           debug { "    delegated token: #{tok.inspect}, #{val.inspect}" }
           token(tok, val)
         end
@@ -352,8 +339,8 @@ module Rouge
         else
           tok = Token[tok]
 
-          callback = proc do |ss|
-            token tok, ss[0]
+          callback = proc do
+            token tok
             case next_state
             when :pop!
               pop!
@@ -388,16 +375,6 @@ module Rouge
       states[name] = State.new(self, name, &b)
     end
 
-    def initialize(parent=nil, opts={}, &defn)
-      if parent.is_a? Hash
-        opts = parent
-        parent = nil
-      end
-
-      @parent = parent
-      super(opts, &defn)
-    end
-
     def self.get_state(name)
       return name if name.is_a? State
 
@@ -414,18 +391,23 @@ module Rouge
       self.class.get_state(name)
     end
 
-    def stream_tokens(stream, &b)
-      scan_state = ScanState.new(self, stream)
+    def scan_state
+      @scan_state ||= ScanState.new(self)
+    end
+
+    def reset!
+      @scan_state = nil
 
       self.class.start_procs.each do |pr|
         scan_state.instance_eval(&pr)
       end
-
-      stream_with_state(scan_state, &b)
     end
 
-    def stream_with_state(scan_state, &b)
+    def stream_tokens(stream, &b)
+      scan_state.scanner = stream
+
       until scan_state.eos?
+        debug { "lexer: #{self.class.tag}" }
         debug { "stack: #{scan_state.stack.map(&:name).inspect}" }
         debug { "stream: #{scan_state.scanner.peek(20).inspect}" }
         success = step(get_state(scan_state.state), scan_state, &b)

data/lib/rouge/lexers/erb.rb

@@ -0,0 +1,54 @@
+module Rouge
+  module Lexers
+    class ERB < RegexLexer
+      tag 'erb'
+      aliases 'eruby', 'rhtml'
+
+      filenames '*.erb', '*.erubis', '*.rhtml', '*.eruby'
+
+      def self.analyze_text(text)
+        return 0.4 if text =~ /<%.*%>/
+      end
+
+      attr_reader :parent
+      attr_reader :ruby_lexer
+      def initialize(opts={})
+        @parent = opts.delete(:parent) || 'html'
+        if @parent.is_a? String
+          lexer_class = Lexer.find(@parent)
+          @parent = lexer_class.new(opts)
+        end
+
+        @ruby_lexer = Ruby.new(opts)
+
+        super(opts)
+      end
+
+      open  = /<%%|<%=|<%#|<%-|<%/
+      close = /%%>|-%>|%>/
+
+      state :root do
+        rule /<%#/, 'Comment', :comment
+
+        rule open, 'Comment.Preproc', :ruby
+
+        rule /.+?(?=#{open})|.+/m do
+          delegate lexer.parent
+        end
+      end
+
+      state :comment do
+        rule close, 'Comment', :pop!
+        rule /.+(?=#{close})|.+/m, 'Comment'
+      end
+
+      state :ruby do
+        rule close, 'Comment.Preproc', :pop!
+
+        rule /.+?(?=#{close})|.+/m do
+          delegate lexer.ruby_lexer
+        end
+      end
+    end
+  end
+end