rouge 0.0.6 → 0.0.7

data/bin/rougify

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require 'pathname'
+ROOT_DIR = Pathname.new(__FILE__).dirname.parent
+load ROOT_DIR.join('lib/rouge.rb')
+load ROOT_DIR.join('lib/rouge/cli.rb')
+
+begin
+  Rouge::CLI.start
+rescue => e
+  $stderr.puts e.message
+  exit 1
+end

data/lib/rouge.rb

@@ -13,6 +13,7 @@ module Rouge
 end
 
 load_dir = Pathname.new(__FILE__).dirname
+load load_dir.join('rouge/text_analyzer.rb')
 load load_dir.join('rouge/token.rb')
 load load_dir.join('rouge/lexer.rb')
 load load_dir.join('rouge/lexers/text.rb')

data/lib/rouge/cli.rb

@@ -0,0 +1,77 @@
+# not required by the main lib.
+# to use this module, require 'rouge/cli'.
+
+# stdlib
+require 'optparse'
+
+# gems
+require 'thor'
+
+module Rouge
+  class CLI < Thor
+    default_task :highlight
+
+    def self.start(argv=ARGV, *a)
+      unless %w(highlight style).include?(argv.first)
+        argv.unshift 'highlight'
+      end
+
+      super(argv, *a)
+    end
+
+    desc 'highlight [FILE]', 'highlight some code'
+    option :file, :aliases => '-f',  :desc => 'the file to operate on'
+    option :lexer, :aliases => '-l',
+      :desc => ('Which lexer to use.  If not provided, rougify will try to ' +
+                'guess based on --mimetype, the filename, and the file ' +
+                'contents.')
+    option :mimetype, :aliases => '-m',
+      :desc => ('a mimetype that Rouge will use to guess the correct lexer. ' +
+                'This is ignored if --lexer is specified.')
+    option :lexer_opts, :aliases => '-L', :type => :hash, :default => {},
+      :desc => ('a hash of options to pass to the lexer.')
+    option :formatter_opts, :aliases => '-F', :type => :hash, :default => {},
+      :desc => ('a hash of options to pass to the formatter.')
+    def highlight(file=nil)
+      filename = options[:file] || file
+      source = filename ? File.read(filename) : $stdin.read
+
+      if options[:lexer].nil?
+        lexer_class = Lexer.guess(
+          :filename => filename,
+          :mimetype => options[:mimetype],
+          :source   => source,
+        )
+      else
+        lexer_class = Lexer.find(options[:lexer])
+        raise "unknown lexer: #{options[:lexer]}" unless lexer_class
+      end
+
+      # only HTML is supported for now
+      formatter = Formatters::HTML.new(normalize_hash_keys(options[:formatter_opts]))
+      lexer = lexer_class.new(normalize_hash_keys(options[:lexer_opts]))
+
+      puts Rouge.highlight(source, lexer, formatter)
+    end
+
+    desc 'style THEME', 'render THEME as css'
+    def style(theme_name='thankful_eyes')
+      theme = Theme.find(theme_name)
+      raise "unknown theme: #{theme_name}" unless theme
+
+      puts theme.new(options).render
+    end
+
+  private
+    # TODO: does Thor do this for me?
+    def normalize_hash_keys(hash)
+      out = {}
+      hash.each do |k, v|
+        new_key = k.tr('-', '_').to_sym
+        out[new_key] = v
+      end
+
+      out
+    end
+  end
+end

data/lib/rouge/lexer.rb

@@ -41,6 +41,53 @@ module Rouge
         registry[name.to_s]
       end
 
+      def guess(info={})
+        by_mimetype = guess_by_mimetype(info[:mimetype]) if info[:mimetype]
+        return by_mimetype if by_mimetype
+
+        by_filename = guess_by_filename(info[:filename]) if info[:filename]
+        return by_filename if by_filename
+
+        by_source = guess_by_source(info[:source]) if info[:source]
+        return by_source if by_source
+
+        # guessing failed, just parse it as text
+        return Lexers::Text
+      end
+
+      def guess_by_mimetype(mt)
+        registry.values.detect do |lexer|
+          lexer.mimetypes.include? mt
+        end
+      end
+
+      def guess_by_filename(fname)
+        fname = File.basename(fname)
+        registry.values.detect do |lexer|
+          lexer.filenames.any? do |pattern|
+            File.fnmatch?(pattern, fname)
+          end
+        end
+      end
+
+      def guess_by_source(source)
+        source = TextAnalyzer.new(source)
+
+        best_result = 0
+        best_match = nil
+        registry.values.each do |lexer|
+          result = lexer.analyze_text(source) || 0
+          return lexer if result == 1
+
+          if result > best_result
+            best_match = lexer
+            best_result = result
+          end
+        end
+
+        best_match
+      end
+
       def register(name, lexer)
         registry[name.to_s] = lexer
       end
@@ -56,14 +103,12 @@ module Rouge
         args.each { |arg| Lexer.register(arg, self) }
       end
 
-      def extensions(*exts)
-        exts.each do |ext|
-          Lexer.extension_registry[ext] = self
-        end
+      def filenames(*fnames)
+        (@filenames ||= []).concat(fnames)
       end
 
-      def extension_registry
-        @extension_registry ||= {}
+      def mimetypes(*mts)
+        (@mimetypes ||= []).concat(mts)
       end
 
     private
@@ -125,6 +170,13 @@ module Rouge
     def stream_tokens(stream, &b)
       raise 'abstract'
     end
+
+    # return a number between 0 and 1 indicating the
+    # likelihood that the text given should be lexed
+    # with this lexer.
+    def self.analyze_text(text)
+      0
+    end
   end
 
   class RegexLexer < Lexer

data/lib/rouge/lexers/css.rb

@@ -1,8 +1,9 @@
 module Rouge
   module Lexers
-    class CSSLexer < RegexLexer
+    class CSS < RegexLexer
       tag 'css'
-      extensions 'css'
+      filenames '*.css'
+      mimetypes 'text/css'
 
       identifier = /[a-zA-Z0-9_-]+/
       number = /-?(?:[0-9]+(\.[0-9]+)?|\.[0-9]+)/

data/lib/rouge/lexers/diff.rb

@@ -2,8 +2,16 @@ module Rouge
   module Lexers
     class Diff < RegexLexer
       tag 'diff'
-      aliases 'patch'
-      extensions 'diff', 'patch'
+      aliases 'patch', 'udiff'
+      filenames '*.diff', '*.patch'
+      mimetypes 'text/x-diff', 'text/x-patch'
+
+      def self.analyze_text(text)
+        return 1   if text.start_with?('Index: ')
+        return 1   if text.start_with?('diff ')
+
+        return 0.9 if text =~ /\A---.*?\n\+\+\+/m
+      end
 
       state :header do
         rule /^diff .*?\n(?=---|\+\+\+)/m, 'Generic.Heading'

data/lib/rouge/lexers/html.rb

@@ -2,7 +2,13 @@ module Rouge
   module Lexers
     class HTML < RegexLexer
       tag 'html'
-      extensions 'htm', 'html'
+      filenames '*.htm', '*.html', '*.xhtml', '*.xslt'
+      mimetypes 'text/html', 'application/xhtml+xml'
+
+      def self.analyze_text(text)
+        return 1 if text.doctype?(/\bhtml\b/i)
+        return 1 if text =~ /<\s*html\b/
+      end
 
       state :root do
         rule /[^<&]+/m, 'Text'
@@ -50,14 +56,14 @@ module Rouge
       state :script_content do
         rule %r(<\s*/\s*script\s*>)m, 'Name.Tag', :pop!
         rule %r(.*?(?=<\s*/\s*script\s*>))m do
-          delegate JavascriptLexer
+          delegate Javascript
         end
       end
 
       state :style_content do
         rule %r(<\s*/\s*style\s*>)m, 'Name.Tag', :pop!
         rule %r(.*(?=<\s*/\s*style\s*>))m do
-          delegate CSSLexer
+          delegate CSS
         end
       end
     end

data/lib/rouge/lexers/javascript.rb

@@ -1,9 +1,17 @@
 module Rouge
   module Lexers
-    class JavascriptLexer < RegexLexer
+    class Javascript < RegexLexer
       tag 'javascript'
       aliases 'js'
-      extensions 'js'
+      filenames '*.js'
+      mimetypes 'application/javascript', 'application/x-javascript',
+                'text/javascript', 'text/x-javascript'
+
+      def self.analyze_text(text)
+        return 1 if text.shebang?('node')
+        return 1 if text.shebang?('jsc')
+        # TODO: rhino, spidermonkey, etc
+      end
 
       state :comments_and_whitespace do
         rule /\s+/, 'Text'
@@ -61,6 +69,7 @@ module Rouge
       ).join('|')
 
       state :root do
+        rule /\A\s*#!.*?\n/m, 'Comment.Preproc'
         rule %r(^(?=\s|/|<!--)), 'Text', :slash_starts_regex
         mixin :comments_and_whitespace
         rule %r(\+\+ | -- | ~ | && | \|\| | \\(?=\n) | << | >>>? | ===
@@ -83,5 +92,60 @@ module Rouge
         rule /'(\\\\|\\'|[^'])*'/, 'Literal.String.Single'
       end
     end
+
+    class JSON < RegexLexer
+      tag 'json'
+      filenames '*.json'
+      mimetypes 'application/json'
+
+      # TODO: is this too much of a performance hit?  JSON is quite simple,
+      # so I'd think this wouldn't be too bad, but for large documents this
+      # could mean doing two full lexes.
+      def self.analyze_text(text)
+        text.lexes_cleanly?(self) ? 0.8 : 0
+      end
+
+      state :root do
+        mixin :whitespace
+        # special case for empty objects
+        rule /(\{)(\s*)(\})/ do
+          group 'Punctuation'
+          group 'Text.Whitespace'
+          group 'Punctuation'
+        end
+        rule /{/,  'Punctuation', :object_key
+        rule /\[/, 'Punctuation', :array
+        rule /-?(?:0|[1-9]\d*)\.\d+(?:e[+-]\d+)?/i, 'Literal.Number.Float'
+        rule /-?(?:0|[1-9]\d*)(?:e[+-]\d+)?/i, 'Literal.Number.Integer'
+        mixin :has_string
+      end
+
+      state :whitespace do
+        rule /\s+/m, 'Text.Whitespace'
+      end
+
+      state :has_string do
+        rule /"(\\.|[^"])*"/, 'Literal.String.Double'
+      end
+
+      state :object_key do
+        mixin :whitespace
+        rule /:/, 'Punctuation', :object_val
+        rule /}/, 'Error', :pop!
+        mixin :has_string
+      end
+
+      state :object_val do
+        rule /,/, 'Punctuation', :pop!
+        rule(/}/) { token 'Punctuation'; pop!; pop! }
+        mixin :root
+      end
+
+      state :array do
+        rule /\]/, 'Punctuation', :pop!
+        rule /,/, 'Punctuation'
+        mixin :root
+      end
+    end
   end
 end

data/lib/rouge/lexers/python.rb

@@ -3,7 +3,12 @@ module Rouge
     class Python < RegexLexer
       tag 'python'
       aliases 'py'
-      extensions 'py'
+      filenames '*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac'
+      mimetypes 'text/x-python', 'application/x-python'
+
+      def self.analyze_text(text)
+        return 1 if text.shebang?(/pythonw?(3|2(\.\d)?)?/)
+      end
 
       keywords = %w(
         assert break continue del elif else except exec
@@ -45,7 +50,8 @@ module Rouge
       dotted_identifier = /[a-z_.][a-z0-9_.]*/i
       state :root do
         rule /\n+/m, 'Text'
-        rule /^(\s*)([rRuU]{,2}""".*?""")/m do
+        rule /^(:)(\s*)([ru]{,2}""".*?""")/mi do
+          group 'Punctuation'
           group 'Text'
           group 'Literal.String.Doc'
         end

data/lib/rouge/lexers/shell.rb

@@ -1,9 +1,14 @@
 module Rouge
   module Lexers
-    class ShellLexer < RegexLexer
+    class Shell < RegexLexer
       tag 'shell'
       aliases 'bash', 'zsh', 'ksh', 'sh'
-      extensions 'sh', 'bash', 'zsh', 'ksh'
+      filenames '*.sh', '*.bash', '*.zsh', '*.ksh'
+      mimetypes 'application/x-sh', 'application/x-shellscript'
+
+      def self.analyze_text(text)
+        text.shebang?(/(ba|z|k)?sh/) ? 1 : 0
+      end
 
       KEYWORDS = %w(
         if fi else while do done for then return function

data/lib/rouge/lexers/tcl.rb

@@ -1,7 +1,15 @@
 module Rouge
   module Lexers
-    class TCLLexer < RegexLexer
+    class TCL < RegexLexer
       tag 'tcl'
+      filenames '*.tcl'
+      mimetypes 'text/x-tcl', 'text/x-script.tcl', 'application/x-tcl'
+
+      def self.analyze_text(text)
+        return 1 if text.shebang? 'tclsh'
+        return 1 if text.shebang? 'wish'
+        return 1 if text.shebang? 'jimsh'
+      end
 
       KEYWORDS = %w(
         after apply array break catch continue elseif else error

data/lib/rouge/lexers/text.rb

@@ -2,6 +2,8 @@ module Rouge
   module Lexers
     class Text < Lexer
       tag 'text'
+      filenames '*.txt'
+      mimetypes 'text/plain'
 
       def stream_tokens(stream, &b)
         yield Token['Text'], stream.string

data/lib/rouge/text_analyzer.rb

@@ -0,0 +1,37 @@
+module Rouge
+  class TextAnalyzer < String
+    def shebang
+      return @shebang if instance_variable_defined? :@shebang
+
+      self =~ /\A\s*#!(.*)$/
+      @shebang = $1
+    end
+
+    def shebang?(match)
+      match = /\b#{match}(\s|$)/
+      match === shebang
+    end
+
+    def doctype
+      return @doctype if instance_variable_defined? :@doctype
+
+      self =~ %r(\A\s*
+        (?:<\?.*?\?>\s*)? # possible <?xml...?> tag
+        <!DOCTYPE\s+(.+?)>
+      )xm
+      @doctype = $1
+    end
+
+    def doctype?(type)
+      type === doctype
+    end
+
+    def lexes_cleanly?(lexer)
+      lexer.lex(self) do |(tok, _)|
+        return false if tok.name == 'Error'
+      end
+
+      true
+    end
+  end
+end

data/lib/rouge/theme.rb

@@ -176,7 +176,7 @@ module Rouge
       return enum_for(:inflate_token, tok) unless block_given?
 
       yield tok
-      tok.sub_tokens.each_value do |st|
+      tok.sub_tokens.each do |(_, st)|
         next if styles[st.name]
 
         inflate_token(st, &b)

data/lib/rouge/themes/thankful_eyes.rb

@@ -22,16 +22,12 @@ module Rouge
 
       style 'Text', :fg => :unicorn, :bg => :krasna
 
-      style 'Comment', :fg => :cool_as_ice
+      style 'Comment', :fg => :cool_as_ice, :italic => true
       style 'Error',
             'Generic.Error', :fg => :aluminum1, :bg => :scarletred2
       style 'Keyword', :fg => :sandy, :bold => true
       style 'Operator',
             'Punctuation', :fg => :backlit, :bold => true
-      style 'Comment.Preproc',
-            'Comment.Multiline',
-            'Comment.Single',
-            'Comment.Special', :fg => :cool_as_ice, :italic => true
       style 'Generic.Deleted', :fg => :scarletred2
       style 'Generic.Inserted', :fg => :go_get_it
       style 'Generic.Emph', :italic => true

data/lib/rouge/version.rb

@@ -1,5 +1,5 @@
 module Rouge
   def self.version
-    "0.0.6"
+    "0.0.7"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rouge
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,29 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-06 00:00:00.000000000 Z
-dependencies: []
+date: 2012-09-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Rouge aims to a be a simple, easy-to-extend drop-in replacement for pygments.
 email:
 - jjmadkisson@gmail.com
-executables: []
+executables:
+- rougify
 extensions: []
 extra_rdoc_files: []
 files:
@@ -33,11 +50,14 @@ files:
 - lib/rouge/themes/base16.rb
 - lib/rouge/token.rb
 - lib/rouge/formatters/html.rb
+- lib/rouge/text_analyzer.rb
 - lib/rouge/version.rb
 - lib/rouge/formatter.rb
+- lib/rouge/cli.rb
 - lib/rouge/lexer.rb
 - lib/rouge/theme.rb
 - lib/rouge.rb
+- bin/rougify
 homepage: http://github.com/jayferd/rouge
 licenses: []
 post_install_message: