rouge 0.2.0 → 0.2.1

data/Gemfile

@@ -15,3 +15,4 @@ gem 'sinatra'
 
 # docs
 gem 'yard'
+gem 'github-markup'

data/lib/rouge.rb

@@ -30,8 +30,11 @@ load load_dir.join('rouge/lexers/tex.rb')
 load load_dir.join('rouge/lexers/markdown.rb')
 load load_dir.join('rouge/lexers/yaml.rb')
 
+load load_dir.join('rouge/lexers/sql.rb')
+
 load load_dir.join('rouge/lexers/make.rb')
 load load_dir.join('rouge/lexers/shell.rb')
+load load_dir.join('rouge/lexers/viml.rb')
 
 load load_dir.join('rouge/lexers/javascript.rb')
 load load_dir.join('rouge/lexers/css.rb')

data/lib/rouge/cli.rb

@@ -17,7 +17,7 @@ module Rouge
         exit 0
       end
 
-      unless %w(highlight style).include?(argv.first)
+      unless %w(highlight style list --help -h help).include?(argv.first)
         argv.unshift 'highlight'
       end
 
@@ -30,7 +30,7 @@ module Rouge
       :desc => ('Which lexer to use.  If not provided, rougify will try to ' +
                 'guess based on --mimetype, the filename, and the file ' +
                 'contents.')
-    option :formatter, :aliases => '-f', :default => 'html',
+    option :formatter, :aliases => '-f', :default => 'terminal256',
       :desc => ('Which formatter to use.')
     option :mimetype, :aliases => '-m',
       :desc => ('a mimetype that Rouge will use to guess the correct lexer. ' +
@@ -71,6 +71,22 @@ module Rouge
       puts theme.new(options).render
     end
 
+    desc 'list', 'list the available lexers, formatters, and styles'
+    def list
+      puts "== Available Lexers =="
+      all_lexers = Lexer.all
+      max_len = all_lexers.map { |l| l.tag.size }.max
+
+      Lexer.all.each do |lexer|
+        desc = "#{lexer.desc}"
+        if lexer.aliases.any?
+          desc << " [aliases: #{lexer.aliases.join(',')}]"
+        end
+        puts "%s: %s" % [lexer.tag, desc]
+        puts
+      end
+    end
+
   private
     # TODO: does Thor do this for me?
     def normalize_hash_keys(hash)

data/lib/rouge/formatter.rb

@@ -1,7 +1,10 @@
 module Rouge
+  # A Formatter takes a token stream and formats it for human viewing.
   class Formatter
     REGISTRY = {}
 
+    # Specify or get the unique tag for this formatter.  This is used
+    # for specifying a formatter in `rougify`.
     def self.tag(tag=nil)
       return @tag unless tag
       REGISTRY[tag] = self
@@ -9,14 +12,18 @@ module Rouge
       @tag = tag
     end
 
+    # Find a formatter class given a unique tag.
     def self.find(tag)
       REGISTRY[tag]
     end
 
+    # Format a token stream.
     def render(tokens)
       enum_for(:stream, tokens).to_a.join
     end
 
+    # @abstract
+    # yield strings that, when concatenated, form the formatted output
     def stream(tokens, &b)
       raise 'abstract'
     end

data/lib/rouge/formatters/html.rb

@@ -3,13 +3,17 @@ require 'cgi'
 
 module Rouge
   module Formatters
+    # Transforms a token stream into HTML output.
     class HTML < Formatter
       tag 'html'
 
+      # @option opts :css_class
+      # A css class to be used for the generated <pre> tag.
       def initialize(opts={})
         @css_class = opts[:css_class] || 'highlight'
       end
 
+      # @yield the html output.
       def stream(tokens, &b)
         yield "<pre class=#{@css_class.inspect}>"
         tokens.each do |tok, val|

data/lib/rouge/formatters/terminal256.rb

@@ -1,11 +1,17 @@
 module Rouge
   module Formatters
+    # A formatter for 256-color terminals
     class Terminal256 < Formatter
       tag 'terminal256'
 
+      # @private
       attr_reader :theme
+
+
+      # @option opts :theme
+      #   (default is thankful_eyes) the theme to render with.
       def initialize(opts={})
-        @theme = opts[:theme] || Themes::ThankfulEyes
+        @theme = opts[:theme] || 'thankful_eyes'
         @theme = Theme.find(@theme) if @theme.is_a? String
       end
 
@@ -95,7 +101,7 @@ module Rouge
           end
         end
 
-      # private
+      private
         def escape(attrs)
           return '' if attrs.empty?
           "\e[#{attrs.join(';')}m"

data/lib/rouge/lexer.rb

@@ -2,6 +2,8 @@
 require 'strscan'
 
 module Rouge
+  # @abstract
+  # A lexer transforms text into a stream of `[token, chunk]` pairs.
   class Lexer
     class << self
       # Lexes `stream` with the given options.  The lex is delegated to a
@@ -23,6 +25,20 @@ module Rouge
         registry[name.to_s]
       end
 
+      # Specify or get this lexer's description.
+      def desc(arg=:absent)
+        if arg == :absent
+          @desc
+        else
+          @desc = arg
+        end
+      end
+
+      # @return a list of all lexers.
+      def all
+        registry.values.uniq
+      end
+
       # Guess which lexer to use based on a hash of info.
       #
       # @option info :mimetype
@@ -82,6 +98,7 @@ module Rouge
         best_match
       end
 
+      # @private
       def register(name, lexer)
         registry[name.to_s] = lexer
       end
@@ -100,7 +117,7 @@ module Rouge
         return @tag if t.nil?
 
         @tag = t.to_s
-        aliases @tag
+        Lexer.register(@tag, self)
       end
 
       # Used to specify alternate names this lexer class may be found by.
@@ -113,10 +130,12 @@ module Rouge
       #
       #   Lexer.find('eruby') # => Erb
       def aliases(*args)
+        args.map!(&:to_s)
         args.each { |arg| Lexer.register(arg, self) }
+        (@aliases ||= []).concat(args)
       end
 
-      # Specify a list of filename globs associated with this lexer
+      # Specify a list of filename globs associated with this lexer.
       #
       # @example
       #   class Ruby < Lexer
@@ -144,16 +163,27 @@ module Rouge
 
     # -*- instance methods -*- #
 
+    # Create a new lexer with the given options.  Individual lexers may
+    # specify extra options.  The only current globally accepted option
+    # is `:debug`.
+    #
+    # @option opts :debug
+    #   Prints debug information to stdout.  The particular info depends
+    #   on the lexer in question.  In regex lexers, this will log the
+    #   state stack at the beginning of each step, along with each regex
+    #   tried and each stream consumed.  Try it, it's pretty useful.
     def initialize(opts={})
       options(opts)
     end
 
+    # get and/or specify the options for this lexer.
     def options(o={})
       (@options ||= {}).merge!(o)
 
       self.class.default_options.merge(@options)
     end
 
+    # get or specify one option for this lexer
     def option(k, v=:absent)
       if v == :absent
         options[k]
@@ -209,7 +239,7 @@ module Rouge
 
     # @abstract
     #
-    # Yield [token, chunk] pairs, given a prepared input stream.  This
+    # Yield `[token, chunk]` pairs, given a prepared input stream.  This
     # must be implemented.
     #
     # @param [StringScanner] stream
@@ -220,7 +250,7 @@ module Rouge
 
     # @abstract
     #
-    # return a number between 0 and 1 indicating the likelihood that
+    # Return a number between 0 and 1 indicating the likelihood that
     # the text given should be lexed with this lexer.  The default
     # implementation returns 0.
     #

data/lib/rouge/lexers/c.rb

@@ -5,6 +5,8 @@ module Rouge
       filenames '*.c', '*.h', '*.idc'
       mimetypes 'text/x-chdr', 'text/x-csrc'
 
+      desc "The C programming language"
+
       # optional comment or whitespace
       ws = %r((?:\s|//.*?\n|/[*].*?[*]/)+)
       id = /[a-zA-Z_][a-zA-Z0-9_]*/
@@ -55,6 +57,7 @@ module Rouge
         rule /__(?:#{__reserved.join('|')})\b/, 'Keyword.Reserved'
         rule /(?:true|false|NULL)\b/, 'Name.Builtin'
         rule id, 'Name'
+        rule /\s+/m, 'Text'
       end
 
       state :case do

data/lib/rouge/lexers/common_lisp.rb

@@ -4,6 +4,7 @@ require 'set'
 module Rouge
   module Lexers
     class CommonLisp < RegexLexer
+      desc "The Common Lisp variant of Lisp (common-lisp.net)"
       tag 'common_lisp'
       aliases 'cl', 'common-lisp'
 

data/lib/rouge/lexers/cpp.rb

@@ -1,6 +1,8 @@
 module Rouge
   module Lexers
     class Cpp < RegexLexer
+      desc "The C++ programming language"
+
       tag 'cpp'
       aliases 'c++'
       # the many varied filenames of c++ source files...

data/lib/rouge/lexers/css.rb

@@ -1,6 +1,8 @@
 module Rouge
   module Lexers
     class CSS < RegexLexer
+      desc "Cascading Style Sheets, used to style web pages"
+
       tag 'css'
       filenames '*.css'
       mimetypes 'text/css'

data/lib/rouge/lexers/diff.rb

@@ -1,6 +1,8 @@
 module Rouge
   module Lexers
     class Diff < RegexLexer
+      desc "Lexes unified diffs or patches"
+
       tag 'diff'
       aliases 'patch', 'udiff'
       filenames '*.diff', '*.patch'

data/lib/rouge/lexers/erb.rb

@@ -1,6 +1,8 @@
 module Rouge
   module Lexers
     class ERB < TemplateLexer
+      desc "Embedded ruby template files"
+
       tag 'erb'
       aliases 'eruby', 'rhtml'
 

data/lib/rouge/lexers/factor.rb

@@ -1,6 +1,7 @@
 module Rouge
   module Lexers
     class Factor < RegexLexer
+      desc "Factor, the practical stack language (factorcode.org)"
       tag 'factor'
       filenames '*.factor'
       mimetypes 'text/x-factor'

data/lib/rouge/lexers/haml.rb

@@ -1,6 +1,10 @@
 module Rouge
   module Lexers
+    # A lexer for the Haml templating system for Ruby.
+    # @see http://haml.info
     class Haml < RegexLexer
+      desc "The Haml templating system for Ruby (haml.info)"
+
       tag 'haml'
       aliases 'HAML'
 
@@ -11,9 +15,10 @@ module Rouge
         return 0.1 if text.start_with? '!!!'
       end
 
-      # option :filters is a hash of filter name to lexer of how
-      # various filters should be highlighted.  By default, :javascript
-      # and :stylesheet are supported.
+      # @option opts :filters
+      #   A hash of filter name to lexer of how various filters should be
+      #   highlighted.  By default, :javascript, :css, :ruby, and :erb
+      #   are supported.
       def initialize(opts={})
         (opts.delete(:filters) || {}).each do |name, lexer|
           unless lexer.respond_to? :lex
@@ -41,10 +46,10 @@ module Rouge
           'css' => CSS.new(options),
           'ruby' => ruby,
           'erb' => ERB.new(options),
+          'markdown' => Markdown.new(options),
           # TODO
           # 'sass' => Sass.new(options),
           # 'textile' => Textile.new(options),
-          # 'markdown' => Markdown.new(options),
           # 'maruku' => Maruku.new(options),
         }
       end

data/lib/rouge/lexers/haskell.rb

@@ -1,6 +1,8 @@
 module Rouge
   module Lexers
     class Haskell < RegexLexer
+      desc "The Haskell programming language (haskell.org)"
+
       tag 'haskell'
       aliases 'hs'
       filenames '*.hs'

data/lib/rouge/lexers/html.rb

@@ -1,6 +1,7 @@
 module Rouge
   module Lexers
     class HTML < RegexLexer
+      desc "HTML, the markup language of the web"
       tag 'html'
       filenames '*.htm', '*.html', '*.xhtml', '*.xslt'
       mimetypes 'text/html', 'application/xhtml+xml'

data/lib/rouge/lexers/java.rb

@@ -1,6 +1,8 @@
 module Rouge
   module Lexers
     class Java < RegexLexer
+      desc "The Java programming language (java.com)"
+
       tag 'java'
       filenames '*.java'
       mimetypes 'text/x-java'

data/lib/rouge/lexers/javascript.rb

@@ -1,6 +1,8 @@
 module Rouge
   module Lexers
     class Javascript < RegexLexer
+      desc "JavaScript, the browser scripting language"
+
       tag 'javascript'
       aliases 'js'
       filenames '*.js'
@@ -94,6 +96,7 @@ module Rouge
     end
 
     class JSON < RegexLexer
+      desc "JavaScript Object Notation (json.org)"
       tag 'json'
       filenames '*.json'
       mimetypes 'application/json'
@@ -102,7 +105,7 @@ module Rouge
       # so I'd think this wouldn't be too bad, but for large documents this
       # could mean doing two full lexes.
       def self.analyze_text(text)
-        text.lexes_cleanly?(self) ? 0.8 : 0
+        return 0.8 if text =~ /\A\s*{/m && text.lexes_cleanly?(self)
       end
 
       state :root do

data/lib/rouge/lexers/make.rb

@@ -1,6 +1,7 @@
 module Rouge
   module Lexers
     class Make < RegexLexer
+      desc "Makefile syntax"
       tag 'make'
       aliases 'makefile', 'mf', 'gnumake', 'bsdmake'
       filenames '*.make', 'Makefile', 'makefile', 'Makefile.*', 'GNUmakefile'

data/lib/rouge/lexers/markdown.rb

@@ -1,6 +1,8 @@
 module Rouge
   module Lexers
     class Markdown < RegexLexer
+      desc "Markdown, a light-weight markup language for authors"
+
       tag 'markdown'
       aliases 'md', 'mkd'
       filenames '*.markdown', '*.md', '*.mkd'

data/lib/rouge/lexers/perl.rb

@@ -1,6 +1,8 @@
 module Rouge
   module Lexers
     class Perl < RegexLexer
+      desc "The Perl scripting language (perl.org)"
+
       tag 'perl'
       aliases 'pl'
 

data/lib/rouge/lexers/php.rb

@@ -1,6 +1,7 @@
 module Rouge
   module Lexers
     class PHP < TemplateLexer
+      desc "The PHP scripting language (php.net)"
       tag 'php'
       aliases 'php', 'php3', 'php4', 'php5'
       filenames '*.php', '*.php[345]'
@@ -18,12 +19,15 @@ module Rouge
         super(opts)
       end
 
+      def self.builtins
+        load Pathname.new(__FILE__).dirname.join('php/builtins.rb')
+        self.builtins
+      end
+
       def builtins
         return [] unless @funcnamehighlighting
 
         @builtins ||= Set.new.tap do |builtins|
-          require Pathname.new(__FILE__).dirname.join('php/builtins.rb')
-
           self.class.builtins.each do |mod, fns|
             next if @disabledmodules.include? mod
             builtins.merge(fns)