rouge 0.1.2 → 0.2.0

data/Gemfile

@@ -12,3 +12,6 @@ gem 'redcarpet'
 
 # for visual tests
 gem 'sinatra'
+
+# docs
+gem 'yard'

data/lib/rouge.rb

@@ -15,13 +15,20 @@ end
 load_dir = Pathname.new(__FILE__).dirname
 load load_dir.join('rouge/version.rb')
 
+load load_dir.join('rouge/util.rb')
+
 load load_dir.join('rouge/text_analyzer.rb')
 load load_dir.join('rouge/token.rb')
+
 load load_dir.join('rouge/lexer.rb')
+load load_dir.join('rouge/regex_lexer.rb')
+load load_dir.join('rouge/template_lexer.rb')
 
 load load_dir.join('rouge/lexers/text.rb')
 load load_dir.join('rouge/lexers/diff.rb')
 load load_dir.join('rouge/lexers/tex.rb')
+load load_dir.join('rouge/lexers/markdown.rb')
+load load_dir.join('rouge/lexers/yaml.rb')
 
 load load_dir.join('rouge/lexers/make.rb')
 load load_dir.join('rouge/lexers/shell.rb')
@@ -29,6 +36,7 @@ load load_dir.join('rouge/lexers/shell.rb')
 load load_dir.join('rouge/lexers/javascript.rb')
 load load_dir.join('rouge/lexers/css.rb')
 load load_dir.join('rouge/lexers/html.rb')
+load load_dir.join('rouge/lexers/haml.rb')
 load load_dir.join('rouge/lexers/xml.rb')
 load load_dir.join('rouge/lexers/php.rb')
 
@@ -38,6 +46,7 @@ load load_dir.join('rouge/lexers/tcl.rb')
 load load_dir.join('rouge/lexers/python.rb')
 load load_dir.join('rouge/lexers/ruby.rb')
 load load_dir.join('rouge/lexers/perl.rb')
+load load_dir.join('rouge/lexers/factor.rb')
 
 load load_dir.join('rouge/lexers/haskell.rb')
 load load_dir.join('rouge/lexers/scheme.rb')

data/lib/rouge/lexer.rb

@@ -4,18 +4,37 @@ require 'strscan'
 module Rouge
   class Lexer
     class << self
+      # Lexes `stream` with the given options.  The lex is delegated to a
+      # new instance.
+      #
+      # @see #lex
       def lex(stream, opts={}, &b)
         new(opts).lex(stream, &b)
       end
 
-      def default_options
+      def default_options(o={})
         @default_options ||= {}
+        @default_options.merge!(o)
+        @default_options
       end
 
+      # Given a string, return the correct lexer class.
       def find(name)
         registry[name.to_s]
       end
 
+      # Guess which lexer to use based on a hash of info.
+      #
+      # @option info :mimetype
+      #   A mimetype to guess by
+      # @option info :filename
+      #   A filename to guess by
+      # @option info :source
+      #   The source itself, which, if guessing by mimetype or filename
+      #   fails, will be searched for shebangs, <!DOCTYPE ...> tags, and
+      #   other hints.
+      #
+      # @see Lexer.analyze_text
       def guess(info={})
         by_mimetype = guess_by_mimetype(info[:mimetype]) if info[:mimetype]
         return by_mimetype if by_mimetype
@@ -67,6 +86,16 @@ module Rouge
         registry[name.to_s] = lexer
       end
 
+      # Used to specify or get the canonical name of this lexer class.
+      #
+      # @example
+      #   class MyLexer < Lexer
+      #     tag 'foo'
+      #   end
+      #
+      #   MyLexer.tag # => 'foo'
+      #
+      #   Lexer.find('foo') # => MyLexer
       def tag(t=nil)
         return @tag if t.nil?
 
@@ -74,14 +103,35 @@ module Rouge
         aliases @tag
       end
 
+      # Used to specify alternate names this lexer class may be found by.
+      #
+      # @example
+      #   class Erb < Lexer
+      #     tag 'erb'
+      #     aliases 'eruby', 'rhtml'
+      #   end
+      #
+      #   Lexer.find('eruby') # => Erb
       def aliases(*args)
         args.each { |arg| Lexer.register(arg, self) }
       end
 
+      # Specify a list of filename globs associated with this lexer
+      #
+      # @example
+      #   class Ruby < Lexer
+      #     filenames '*.rb', '*.ruby', 'Gemfile', 'Rakefile'
+      #   end
       def filenames(*fnames)
         (@filenames ||= []).concat(fnames)
       end
 
+      # Specify a list of mimetypes associated with this lexer.
+      #
+      # @example
+      #   class Html < Lexer
+      #     mimetypes 'text/html', 'application/xhtml+xml'
+      #   end
       def mimetypes(*mts)
         (@mimetypes ||= []).concat(mts)
       end
@@ -94,7 +144,7 @@ module Rouge
 
     # -*- instance methods -*- #
 
-    def initialize(opts={}, &b)
+    def initialize(opts={})
       options(opts)
     end
 
@@ -112,18 +162,28 @@ module Rouge
       end
     end
 
+    # Leave a debug message if the `:debug` option is set.  The message
+    # is given as a block because some debug messages contain calculated
+    # information that is unnecessary for lexing in the real world.
+    #
+    # @example
+    #   debug { "hello, world!" }
     def debug(&b)
       puts(b.call) if option :debug
     end
 
-    def get_tokens(stream)
-      lex(stream).to_a
-    end
-
+    # @abstract
+    #
+    # Called after each lex is finished.  The default implementation
+    # is a noop.
     def reset!
-      # noop, called after each lex is finished
     end
 
+    # Given a string, yield [token, chunk] pairs.  If no block is given,
+    # an enumerator is returned.
+    #
+    # @option opts :continue
+    #   Continue the lex from the previous state (i.e. don't call #reset!)
     def lex(string, opts={}, &b)
       return enum_for(:lex, string) unless block_given?
 
@@ -147,280 +207,28 @@ module Rouge
       b.call(last_token, last_val) if last_token
     end
 
+    # @abstract
+    #
+    # Yield [token, chunk] pairs, given a prepared input stream.  This
+    # must be implemented.
+    #
+    # @param [StringScanner] stream
+    #   the stream
     def stream_tokens(stream, &b)
       raise 'abstract'
     end
 
-    # return a number between 0 and 1 indicating the
-    # likelihood that the text given should be lexed
-    # with this lexer.
+    # @abstract
+    #
+    # return a number between 0 and 1 indicating the likelihood that
+    # the text given should be lexed with this lexer.  The default
+    # implementation returns 0.
+    #
+    # @param [TextAnalyzer] text
+    #   the text to be analyzed, with a couple of handy methods on it,
+    #   like {TextAnalyzer#shebang?} and {TextAnalyzer#doctype?}
     def self.analyze_text(text)
       0
     end
   end
-
-  class RegexLexer < Lexer
-    class Rule
-      attr_reader :callback
-      attr_reader :next_state
-      attr_reader :re
-      def initialize(re, callback, next_state)
-        @re = re
-        @callback = callback
-        @next_state = next_state
-      end
-
-      def inspect
-        "#<Rule #{@re.inspect}>"
-      end
-
-      def consume(stream, &b)
-        stream.scan(@re)
-
-        if stream.matched?
-          yield stream
-          return true
-        end
-
-        false
-      end
-    end
-
-    class State
-      attr_reader :name
-      def initialize(lexer_class, name, &defn)
-        @lexer_class = lexer_class
-        @name = name
-        @defn = defn
-      end
-
-      def relative_state(state_name=nil, &b)
-        if state_name
-          @lexer_class.get_state(state_name)
-        else
-          State.new(@lexer_class, b.inspect, &b).load!
-        end
-      end
-
-      def rules
-        @rules ||= []
-      end
-
-      def load!
-        return self if @loaded
-        @loaded = true
-        StateDSL.new(rules).instance_eval(&@defn)
-        self
-      end
-    end
-
-    class StateDSL
-      attr_reader :rules
-      def initialize(rules)
-        @rules = rules
-      end
-
-      def rule(re, tok=nil, next_state=nil, &callback)
-        if block_given?
-          next_state = tok
-        else
-          tok = Token[tok]
-
-          callback = proc do
-            token tok
-            case next_state
-            when :pop!
-              pop!
-            when Symbol
-              push next_state
-            end # else pass
-          end
-        end
-
-        rules << Rule.new(re, callback, next_state)
-      end
-
-      def mixin(lexer_name)
-        rules << lexer_name.to_s
-      end
-    end
-
-    def self.states
-      @states ||= {}
-    end
-
-    def self.start_procs
-      @start_procs ||= []
-    end
-
-    def self.start(&b)
-      start_procs << b
-    end
-
-    def self.state(name, &b)
-      name = name.to_s
-      states[name] = State.new(self, name, &b)
-    end
-
-    def self.get_state(name)
-      return name if name.is_a? State
-
-      state = states[name.to_s]
-      raise "unknown state: #{name}" unless state
-      state.load!
-    end
-
-    def self.[](name)
-      get_state(name)
-    end
-
-    def get_state(name)
-      self.class.get_state(name)
-    end
-
-    def stack
-      @stack ||= [get_state(:root)]
-    end
-
-    def state
-      stack.last or raise 'empty stack!'
-    end
-
-    def reset!
-      @scan_state = nil
-
-      self.class.start_procs.each do |pr|
-        instance_eval(&pr)
-      end
-    end
-
-    def stream_tokens(stream, &b)
-      until stream.eos?
-        debug { "lexer: #{self.class.tag}" }
-        debug { "stack: #{stack.map(&:name).inspect}" }
-        debug { "stream: #{stream.peek(20).inspect}" }
-        success = step(get_state(state), stream, &b)
-
-        if !success
-          debug { "    no match, yielding Error" }
-          b.call(Token['Error'], stream.getch)
-        end
-      end
-    end
-
-    def step(state, stream, &b)
-      state.rules.each do |rule|
-        return true if run_rule(rule, stream, &b)
-      end
-
-      false
-    end
-
-    def run_rule(rule, stream, &b)
-      case rule
-      when String
-        debug { "  entering mixin #{rule}" }
-        res = step(get_state(rule), stream, &b)
-        debug { "  exiting  mixin #{rule}" }
-        res
-      when Rule
-        debug { "  trying #{rule.inspect}" }
-        scan(stream, rule.re) do
-          debug { "    got #{stream[0].inspect}" }
-
-          run_callback(stream, &rule.callback).each do |tok, res|
-            debug { "    yielding #{tok.to_s.inspect}, #{res.inspect}" }
-            b.call(Token[tok], res)
-          end
-        end
-      end
-    end
-
-    def run_callback(stream, &callback)
-      Enumerator.new do |y|
-        @output_stream = y
-        @group_count = 0
-        @last_matches = stream
-        instance_exec(stream, &callback)
-        @last_matches = nil
-        @output_stream = nil
-      end
-    end
-
-    MAX_NULL_STEPS = 5
-    def scan(scanner, re, &b)
-      @null_steps ||= 0
-
-      if @null_steps >= MAX_NULL_STEPS
-        debug { "    too many scans without consuming the string!" }
-        return false
-      end
-
-      scanner.scan(re)
-
-      if scanner.matched?
-        if scanner.matched_size == 0
-          @null_steps += 1
-        else
-          @null_steps = 0
-        end
-
-        yield self
-        return true
-      end
-
-      return false
-    end
-
-    def token(tok, val=:__absent__)
-      val = @last_matches[0] if val == :__absent__
-      val ||= ''
-
-      raise 'no output stream' unless @output_stream
-
-      @output_stream << [Token[tok], val]
-    end
-
-    def group(tok)
-      token(tok, @last_matches[@group_count += 1])
-    end
-
-    def delegate(lexer, text=nil)
-      debug { "    delegating to #{lexer.inspect}" }
-      text ||= @last_matches[0]
-
-      lexer.lex(text, :continue => true) do |tok, val|
-        debug { "    delegated token: #{tok.inspect}, #{val.inspect}" }
-        token(tok, val)
-      end
-    end
-
-    def push(state_name=nil, &b)
-      # use the top of the stack by default
-      if state_name || b
-        push_state = state.relative_state(state_name, &b)
-      else
-        push_state = self.state
-      end
-
-      debug { "    pushing #{push_state.name}" }
-      stack.push(push_state)
-    end
-
-    def pop!
-      raise 'empty stack!' if stack.empty?
-
-      debug { "    popping stack" }
-      stack.pop
-    end
-
-    def in_state?(state_name)
-      stack.map(&:name).include? state_name.to_s
-    end
-
-    def state?(state_name)
-      state_name.to_s == state.name
-    end
-
-  end
 end