rouge 2.0.7 → 2.1.0

data/lib/rouge/formatters/html_pygments.rb

@@ -7,9 +7,9 @@ module Rouge
       end
 
       def stream(tokens, &b)
-        yield %(<pre class="#{@css_class}"><code>)
+        yield %(<div class="highlight"><pre class="#{@css_class}"><code>)
         @inner.stream(tokens, &b)
-        yield "</code></pre>"
+        yield "</code></pre></div>"
       end
     end
   end

data/lib/rouge/formatters/html_table.rb

@@ -54,7 +54,7 @@ module Rouge
         yield formatted
         yield '</pre></td>'
 
-        yield "</tr></tbody></table>\n"
+        yield "</tr></tbody></table>"
       end
     end
   end

data/lib/rouge/formatters/terminal256.rb

@@ -17,7 +17,7 @@ module Rouge
         tokens.each do |tok, val|
           escape = escape_sequence(tok)
           yield escape.style_string
-          yield val.gsub("\n", "\n#{escape.style_string}")
+          yield val.gsub("\n", "#{escape.reset_string}\n#{escape.style_string}")
           yield escape.reset_string
         end
       end

data/lib/rouge/lexer.rb

@@ -11,6 +11,8 @@ module Rouge
   class Lexer
     include Token::Tokens
 
+    @option_docs = {}
+
     class << self
       # Lexes `stream` with the given options.  The lex is delegated to a
       # new instance.
@@ -20,12 +22,6 @@ module Rouge
         new(opts).lex(stream, &b)
       end
 
-      def default_options(o={})
-        @default_options ||= {}
-        @default_options.merge!(o)
-        @default_options
-      end
-
       # Given a string, return the correct lexer class.
       def find(name)
         registry[name.to_s]
@@ -45,19 +41,30 @@ module Rouge
       # This is used in the Redcarpet plugin as well as Rouge's own
       # markdown lexer for highlighting internal code blocks.
       #
-      def find_fancy(str, code=nil)
+      def find_fancy(str, code=nil, additional_options={})
+        if str && !str.include?('?') && str != 'guess'
+          lexer_class = find(str)
+          return lexer_class && lexer_class.new(additional_options)
+        end
+
         name, opts = str ? str.split('?', 2) : [nil, '']
 
         # parse the options hash from a cgi-style string
         opts = CGI.parse(opts || '').map do |k, vals|
-          [ k.to_sym, vals.empty? ? true : vals[0] ]
+          val = case vals.size
+          when 0 then true
+          when 1 then vals[0]
+          else vals
+          end
+
+          [ k.to_s, val ]
         end
 
-        opts = Hash[opts]
+        opts = additional_options.merge(Hash[opts])
 
         lexer_class = case name
         when 'guess', nil
-          self.guess(:source => code, :mimetype => opts[:mimetype])
+          self.guess(:source => code, :mimetype => opts['mimetype'])
         when String
           self.find(name)
         end
@@ -82,6 +89,14 @@ module Rouge
         end
       end
 
+      def option_docs
+        @option_docs ||= InheritableHash.new(superclass.option_docs)
+      end
+
+      def option(name, desc)
+        option_docs[name.to_s] = desc
+      end
+
       # Specify or get the path name containing a small demo for
       # this lexer (can be overriden by {demo}).
       def demo_file(arg=:absent)
@@ -156,7 +171,17 @@ module Rouge
         guess :source => source
       end
 
-    private
+      def enable_debug!
+        @debug_enabled = true
+      end
+
+      def disable_debug!
+        @debug_enabled = false
+      end
+
+      def debug_enabled?
+        !!@debug_enabled
+      end
 
     protected
       # @private
@@ -234,6 +259,7 @@ module Rouge
 
     # -*- instance methods -*- #
 
+    attr_reader :options
     # Create a new lexer with the given options.  Individual lexers may
     # specify extra options.  The only current globally accepted option
     # is `:debug`.
@@ -244,42 +270,103 @@ module Rouge
     #   state stack at the beginning of each step, along with each regex
     #   tried and each stream consumed.  Try it, it's pretty useful.
     def initialize(opts={})
-      options(opts)
+      @options = {}
+      opts.each { |k, v| @options[k.to_s] = v }
+
+      @debug = Lexer.debug_enabled? && bool_option(:debug)
+    end
 
-      @debug = option(:debug)
+    def as_bool(val)
+      case val
+      when nil, false, 0, '0', 'off'
+        false
+      when Array
+        val.empty? ? true : as_bool(val.last)
+      else
+        true
+      end
     end
 
-    # get and/or specify the options for this lexer.
-    def options(o={})
-      (@options ||= {}).merge!(o)
+    def as_string(val)
+      return as_string(val.last) if val.is_a?(Array)
 
-      self.class.default_options.merge(@options)
+      val ? val.to_s : nil
     end
 
-    # get or specify one option for this lexer
-    def option(k, v=:absent)
-      if v == :absent
-        options[k]
+    def as_list(val)
+      case val
+      when Array
+        val.flat_map { |v| as_list(v) }
+      when String
+        val.split(',')
       else
-        options({ k => v })
+        []
       end
     end
 
-    # @deprecated
-    # Instead of `debug { "foo" }`, simply `puts "foo" if @debug`.
-    #
-    # Leave a debug message if the `:debug` option is set.  The message
-    # is given as a block because some debug messages contain calculated
-    # information that is unnecessary for lexing in the real world.
-    #
-    # Calls to this method should be guarded with "if @debug" for best
-    # performance when debugging is turned off.
-    #
-    # @example
-    #   debug { "hello, world!" } if @debug
-    def debug
-      warn "Lexer#debug is deprecated.  Simply puts if @debug instead."
-      puts yield if @debug
+    def as_lexer(val)
+      return as_lexer(val.last) if val.is_a?(Array)
+      return val.new(@options) if val.is_a?(Class) && val < Lexer
+
+      case val
+      when Lexer
+        val
+      when String
+        lexer_class = Lexer.find(val)
+        lexer_class && lexer_class.new(@options)
+      end
+    end
+
+    def as_token(val)
+      return as_token(val.last) if val.is_a?(Array)
+      case val
+      when Token
+        val
+      else
+        Token[val]
+      end
+    end
+
+    def bool_option(name, &default)
+      if @options.key?(name.to_s)
+        as_bool(@options[name.to_s])
+      else
+        default ? default.call : false
+      end
+    end
+
+    def string_option(name, &default)
+      as_string(@options.delete(name.to_s, &default))
+    end
+
+    def lexer_option(name, &default)
+      as_lexer(@options.delete(name.to_s, &default))
+    end
+
+    def list_option(name, &default)
+      as_list(@options.delete(name.to_s, &default))
+    end
+
+    def token_option(name, &default)
+      as_token(@options.delete(name.to_s, &default))
+    end
+
+    def hash_option(name, defaults, &val_cast)
+      name = name.to_s
+      out = defaults.dup
+
+      base = @options.delete(name.to_s)
+      base = {} unless base.is_a?(Hash)
+      base.each { |k, v| out[k.to_s] = val_cast ? val_cast.call(v) : v }
+
+      @options.keys.each do |key|
+        next unless key =~ /(\w+)\[(\w+)\]/ and $1 == name
+        value = @options.delete(key)
+
+        out[$2] = val_cast ? val_cast.call(value) : value
+      end
+
+      out
     end
 
     # @abstract

data/lib/rouge/lexers/abap.rb

@@ -217,13 +217,13 @@ module Rouge
 
         # operators
         rule %r((->|->>|=>)), Operator
-        rule %r([-\*\+%/~=&\?<>!#@\^]+), Operator
+        rule %r([-\*\+%/~=&\?<>!#\@\^]+), Operator
 
       end
 
       state :operators do
         rule %r((->|->>|=>)), Operator
-        rule %r([-\*\+%/~=&\?<>!#@\^]+), Operator
+        rule %r([-\*\+%/~=&\?<>!#\@\^]+), Operator
       end
 
       state :single_string do

data/lib/rouge/lexers/apache.rb

@@ -13,7 +13,7 @@ module Rouge
         attr_reader :keywords
       end
       # Load Apache keywords from separate YML file
-      @keywords = ::YAML.load(File.open(Pathname.new(__FILE__).dirname.join('apache/keywords.yml'))).tap do |h|
+      @keywords = ::YAML.load_file(Pathname.new(__FILE__).dirname.join('apache/keywords.yml')).tap do |h|
         h.each do |k,v|
           h[k] = Set.new v
         end

data/lib/rouge/lexers/awk.rb

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*- #
+
+module Rouge
+  module Lexers
+    class Awk < RegexLexer
+      title "Awk"
+      desc "pattern-directed scanning and processing language"
+
+      tag 'awk'
+      filenames '*.awk'
+      mimetypes 'application/x-awk'
+
+      def self.analyze_text(text)
+        return 1 if text.shebang?('awk')
+      end
+
+      id = /[$a-zA-Z_][a-zA-Z0-9_]*/
+
+      def self.keywords
+        @keywords ||= Set.new %w(
+          if else while for do break continue return next nextfile delete
+          exit print printf getline
+        )
+      end
+
+      def self.declarations
+        @declarations ||= Set.new %w(function)
+      end
+
+      def self.reserved
+        @reserved ||= Set.new %w(BEGIN END)
+      end
+
+      def self.constants
+        @constants ||= Set.new %w(
+          CONVFMT FS NF NR FNR FILENAME RS OFS ORS OFMT SUBSEP ARGC ARGV
+          ENVIRON
+        )
+      end
+
+      def self.builtins
+        @builtins ||= %w(
+          exp log sqrt sin cos atan2 length rand srand int substr index match
+          split sub gsub sprintf system tolower toupper
+        )
+      end
+
+      state :comments_and_whitespace do
+        rule /\s+/, Text
+        rule %r(#.*?$), Comment::Single
+      end
+
+      state :expr_start do
+        mixin :comments_and_whitespace
+        rule %r(/) do
+          token Str::Regex
+          goto :regex
+        end
+        rule //, Text, :pop!
+      end
+
+      state :regex do
+        rule %r(/) do
+          token Str::Regex
+          goto :regex_end
+        end
+
+        rule %r([^/]\n), Error, :pop!
+
+        rule /\n/, Error, :pop!
+        rule /\[\^/, Str::Escape, :regex_group
+        rule /\[/, Str::Escape, :regex_group
+        rule /\\./, Str::Escape
+        rule %r{[(][?][:=<!]}, Str::Escape
+        rule /[{][\d,]+[}]/, Str::Escape
+        rule /[()?]/, Str::Escape
+        rule /./, Str::Regex
+      end
+
+      state :regex_end do
+        rule(//) { pop! }
+      end
+
+      state :regex_group do
+        # specially highlight / in a group to indicate that it doesn't
+        # close the regex
+        rule /\//, Str::Escape
+
+        rule %r([^/]\n) do
+          token Error
+          pop! 2
+        end
+
+        rule /\]/, Str::Escape, :pop!
+        rule /\\./, Str::Escape
+        rule /./, Str::Regex
+      end
+
+      state :bad_regex do
+        rule /[^\n]+/, Error, :pop!
+      end
+
+      state :root do
+        mixin :comments_and_whitespace
+        rule %r((?<=\n)(?=\s|/)), Text, :expr_start
+        rule %r([-<>+*/%\^!=]=?|in\b|\+\+|--|\|), Operator, :expr_start
+        rule %r(&&|\|\||~!?), Operator, :expr_start
+        rule /[(\[,]/, Punctuation, :expr_start
+        rule /;/, Punctuation, :statement
+        rule /[)\].]/, Punctuation
+
+        rule /[?]/ do
+          token Punctuation
+          push :ternary
+          push :expr_start
+        end
+
+        rule /[{}]/, Punctuation, :statement
+
+        rule id do |m|
+          if self.class.keywords.include? m[0]
+            token Keyword
+            push :expr_start
+          elsif self.class.declarations.include? m[0]
+            token Keyword::Declaration
+            push :expr_start
+          elsif self.class.reserved.include? m[0]
+            token Keyword::Reserved
+          elsif self.class.constants.include? m[0]
+            token Keyword::Constant
+          elsif self.class.builtins.include? m[0]
+            token Name::Builtin
+          elsif m[0] =~ /^\$/
+            token Name::Variable
+          else
+            token Name::Other
+          end
+        end
+
+        rule /[0-9]+\.[0-9]+/, Num::Float
+        rule /[0-9]+/, Num::Integer
+        rule /"(\\[\\"]|[^"])*"/, Str::Double
+        rule /:/, Punctuation
+      end
+
+      state :statement do
+        rule /[{}]/, Punctuation
+        mixin :expr_start
+      end
+
+      state :ternary do
+        rule /:/ do
+          token Punctuation
+          goto :expr_start
+        end
+
+        mixin :root
+      end
+    end
+  end
+end

data/lib/rouge/lexers/clojure.rb

@@ -9,7 +9,7 @@ module Rouge
       tag 'clojure'
       aliases 'clj', 'cljs'
 
-      filenames '*.clj', '*.cljs', '*.cljc', 'build.boot'
+      filenames '*.clj', '*.cljs', '*.cljc', 'build.boot', '*.edn'
 
       mimetypes 'text/x-clojure', 'application/x-clojure'
 
@@ -86,7 +86,7 @@ module Rouge
         rule /\\(.|[a-z]+)/i, Str::Char
 
 
-        rule /~@|[`\'#^~&]/, Operator
+        rule /~@|[`\'#^~&@]/, Operator
 
         rule /(\()(\s*)(#{identifier})/m do |m|
           token Punctuation, m[1]