rouge 0.4.0 → 0.5.0

data/lib/rouge/lexers/xml.rb

@@ -21,35 +21,35 @@ module Rouge
       end
 
       state :root do
-        rule /[^<&]+/, 'Text'
-        rule /&\S*?;/, 'Name.Entity'
-        rule /<!\[CDATA\[.*?\]\]\>/, 'Comment.Preproc'
-        rule /<!--/, 'Comment', :comment
-        rule /<\?.*?\?>/, 'Comment.Preproc'
-        rule /<![^>]*>/, 'Comment.Preproc'
+        rule /[^<&]+/, Text
+        rule /&\S*?;/, Name::Entity
+        rule /<!\[CDATA\[.*?\]\]\>/, Comment::Preproc
+        rule /<!--/, Comment, :comment
+        rule /<\?.*?\?>/, Comment::Preproc
+        rule /<![^>]*>/, Comment::Preproc
 
         # open tags
-        rule %r(<\s*[\w:.-]+)m, 'Name.Tag', :tag
+        rule %r(<\s*[\w:.-]+)m, Name::Tag, :tag
 
         # self-closing tags
-        rule %r(<\s*/\s*[\w:.-]+\s*>)m, 'Name.Tag'
+        rule %r(<\s*/\s*[\w:.-]+\s*>)m, Name::Tag
       end
 
       state :comment do
-        rule /[^-]+/m, 'Comment'
-        rule /-->/, 'Comment', :pop!
-        rule /-/, 'Comment'
+        rule /[^-]+/m, Comment
+        rule /-->/, Comment, :pop!
+        rule /-/, Comment
       end
 
       state :tag do
-        rule /\s+/m, 'Text'
-        rule /[\w.:-]+\s*=/m, 'Name.Attribute', :attr
-        rule %r(/?\s*>), 'Name.Tag', :pop!
+        rule /\s+/m, Text
+        rule /[\w.:-]+\s*=/m, Name::Attribute, :attr
+        rule %r(/?\s*>), Name::Tag, :pop!
       end
 
       state :attr do
-        rule /\s+/m, 'Text'
-        rule /".*?"|'.*?'|[^\s>]+/, 'Literal.String', :pop!
+        rule /\s+/m, Text
+        rule /".*?"|'.*?'|[^\s>]+/, Str, :pop!
       end
     end
   end

data/lib/rouge/lexers/yaml.rb

@@ -37,9 +37,7 @@ module Rouge
       end
 
       # Save a possible indentation level
-      def save_indent(opts={})
-        debug { "    yaml: save_indent" }
-        match = @last_match[0]
+      def save_indent(match)
         @next_indent = match.size
         debug { "    yaml: indent: #{self.indent}/#@next_indent" }
         debug { "    yaml: popping indent stack - before: #@indent_stack" }
@@ -55,9 +53,9 @@ module Rouge
         end
       end
 
-      def continue_indent
+      def continue_indent(match)
         debug { "    yaml: continue_indent" }
-        @next_indent += @last_match[0].size
+        @next_indent += match.size
       end
 
       def set_indent(opts={})
@@ -73,65 +71,65 @@ module Rouge
       start { reset_indent }
 
       state :basic do
-        rule /#.*$/, 'Comment.Single'
+        rule /#.*$/, Comment::Single
       end
 
       state :root do
         mixin :basic
 
-        rule /\n+/, 'Text'
+        rule /\n+/, Text
 
         # trailing or pre-comment whitespace
-        rule /[ ]+(?=#|$)/, 'Text'
+        rule /[ ]+(?=#|$)/, Text
 
         rule /^%YAML\b/ do
-          token 'Name.Tag'
+          token Name::Tag
           reset_indent
           push :yaml_directive
         end
 
         rule /^%TAG\b/ do
-          token 'Name.Tag'
+          token Name::Tag
           reset_indent
           push :tag_directive
         end
 
         # doc-start and doc-end indicators
         rule /^(?:---|\.\.\.)(?= |$)/ do
-          token 'Name.Namespace'
+          token Name::Namespace
           reset_indent
           push :block_line
         end
 
         # indentation spaces
-        rule /[ ]*(?!\s|$)/ do
-          text, err = save_indent
-          token 'Text', text
-          token 'Error', err
+        rule /[ ]*(?!\s|$)/ do |m|
+          text, err = save_indent(m[0])
+          token Text, text
+          token Error, err
           push :block_line; push :indentation
         end
       end
 
       state :indentation do
-        rule(/\s*?\n/) { token 'Text'; pop! 2 }
+        rule(/\s*?\n/) { token Text; pop! 2 }
         # whitespace preceding block collection indicators
-        rule /[ ]+(?=[-:?](?:[ ]|$))/ do
-          token 'Text'
-          continue_indent
+        rule /[ ]+(?=[-:?](?:[ ]|$))/ do |m|
+          token Text
+          continue_indent(m[0])
         end
 
         # block collection indicators
-        rule(/[?:-](?=[ ]|$)/) { token 'Punctuation.Indicator'; set_indent }
+        rule(/[?:-](?=[ ]|$)/) { token Punctuation::Indicator; set_indent }
 
         # the beginning of a block line
-        rule(/[ ]*/) { token 'Text'; continue_indent; pop! }
+        rule(/[ ]*/) { |m| token Text; continue_indent(m[0]); pop! }
       end
 
       # indented line in the block context
       state :block_line do
         # line end
-        rule /[ ]*(?=#|$)/, 'Text', :pop!
-        rule /[ ]+/, 'Text'
+        rule /[ ]*(?=#|$)/, Text, :pop!
+        rule /[ ]+/, Text
         # tags, anchors, and aliases
         mixin :descriptors
         # block collections and scalars
@@ -141,54 +139,54 @@ module Rouge
 
         # a plain scalar
         rule /(?=#{plain_scalar_start}|[?:-][^ \t\n\r\f\v])/ do
-          token 'Name.Variable'
+          token Name::Variable
           push :plain_scalar_in_block_context
         end
       end
 
       state :descriptors do
         # a full-form tag
-        rule /!<[0-9A-Za-z;\/?:@&=+$,_.!~*'()\[\]%-]+>/, 'Keyword.Type'
+        rule /!<[0-9A-Za-z;\/?:@&=+$,_.!~*'()\[\]%-]+>/, Keyword::Type
 
         # a tag in the form '!', '!suffix' or '!handle!suffix'
         rule %r(
           (?:![\w-]+)? # handle
           !(?:[\w;/?:@&=+$,.!~*\'()\[\]%-]*) # suffix
-        )x, 'Keyword.Type'
+        )x, Keyword::Type
 
         # an anchor
-        rule /&[\w-]+/, 'Name.Label'
+        rule /&[\w-]+/, Name::Label
 
         # an alias
-        rule /\*[\w-]+/, 'Name.Variable'
+        rule /\*[\w-]+/, Name::Variable
       end
 
       state :block_nodes do
         # implicit key
         rule /:(?=\s|$)/ do
-          token 'Punctuation.Indicator'
+          token Punctuation::Indicator
           set_indent :implicit => true
         end
 
         # literal and folded scalars
         rule /[\|>]/ do
-          token 'Punctuation.Indicator'
+          token Punctuation::Indicator
           push :block_scalar_content
           push :block_scalar_header
         end
       end
 
       state :flow_nodes do
-        rule /\[/, 'Punctuation.Indicator', :flow_sequence
-        rule /\{/, 'Punctuation.Indicator', :flow_mapping
-        rule /'/, 'Literal.String.Single', :single_quoted_scalar
-        rule /"/, 'Literal.String.Double', :double_quoted_scalar
+        rule /\[/, Punctuation::Indicator, :flow_sequence
+        rule /\{/, Punctuation::Indicator, :flow_mapping
+        rule /'/, Str::Single, :single_quoted_scalar
+        rule /"/, Str::Double, :double_quoted_scalar
       end
 
       state :flow_collection do
-        rule /\s+/m, 'Text'
+        rule /\s+/m, Text
         mixin :basic
-        rule /[?:,]/, 'Punctuation.Indicator'
+        rule /[?:,]/, Punctuation::Indicator
         mixin :descriptors
         mixin :flow_nodes
 
@@ -198,17 +196,17 @@ module Rouge
       end
 
       state :flow_sequence do
-        rule /\]/, 'Punctuation.Indicator', :pop!
+        rule /\]/, Punctuation::Indicator, :pop!
         mixin :flow_collection
       end
 
       state :flow_mapping do
-        rule /\}/, 'Punctuation.Indicator', :pop!
+        rule /\}/, Punctuation::Indicator, :pop!
         mixin :flow_collection
       end
 
       state :block_scalar_content do
-        rule /\n+/, 'Text'
+        rule /\n+/, Text
 
         # empty lines never dedent, but they might be part of the scalar.
         rule /^[ ]+$/ do |m|
@@ -217,14 +215,14 @@ module Rouge
 
           indent_mark = @block_scalar_indent || indent_size
 
-          token 'Text', text[0...indent_mark]
-          token 'Name.Constant', text[indent_mark..-1]
+          token Text, text[0...indent_mark]
+          token Name::Constant, text[indent_mark..-1]
         end
 
         # TODO: ^ doesn't actually seem to affect the match at all.
         # Find a way to work around this limitation.
         rule /^[ ]*/ do |m|
-          token 'Text'
+          token Text
 
           indent_size = m[0].size
 
@@ -236,7 +234,7 @@ module Rouge
           end
         end
 
-        rule /[^\n\r\f\v]+/, 'Name.Constant'
+        rule /[^\n\r\f\v]+/, Name::Constant
       end
 
       state :block_scalar_header do
@@ -247,7 +245,7 @@ module Rouge
           )(?=[ ]|$)
         )x do |m|
           @block_scalar_indent = nil
-          pop!; push :ignored_line
+          goto :ignored_line
           next if m[0].empty?
 
           increment = m[1] || m[2]
@@ -255,62 +253,61 @@ module Rouge
             @block_scalar_indent = indent + increment.to_i
           end
 
-          token 'Punctuation.Indicator'
+          token Punctuation::Indicator
         end
       end
 
       state :ignored_line do
         mixin :basic
-        rule /[ ]+/, 'Text'
-        rule /\n/, 'Text', :pop!
+        rule /[ ]+/, Text
+        rule /\n/, Text, :pop!
       end
 
       state :quoted_scalar_whitespaces do
         # leading and trailing whitespace is ignored
-        rule /^[ ]+/, 'Text'
-        rule /[ ]+$/, 'Text'
+        rule /^[ ]+/, Text
+        rule /[ ]+$/, Text
 
-        rule /\n+/m, 'Text'
+        rule /\n+/m, Text
 
-        rule /[ ]+/, 'Name.Variable'
+        rule /[ ]+/, Name::Variable
       end
 
       state :single_quoted_scalar do
         mixin :quoted_scalar_whitespaces
-        rule /\\'/, 'Literal.String.Escape'
-        rule /'/, 'Literal.String', :pop!
-        rule /[^\s']+/, 'Literal.String'
+        rule /\\'/, Str::Escape
+        rule /'/, Str, :pop!
+        rule /[^\s']+/, Str
       end
 
       state :double_quoted_scalar do
-        rule /"/, 'Literal.String', :pop!
+        rule /"/, Str, :pop!
         mixin :quoted_scalar_whitespaces
         # escapes
-        rule /\\[0abt\tn\nvfre "\\N_LP]/, 'Literal.String.Escape'
+        rule /\\[0abt\tn\nvfre "\\N_LP]/, Str::Escape
         rule /\\(?:x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
-          'Literal.String.Escape'
-        rule /[^ \t\n\r\f\v"\\]+/, 'Literal.String'
+          Str::Escape
+        rule /[^ \t\n\r\f\v"\\]+/, Str
       end
 
       state :plain_scalar_in_block_context_new_line do
-        rule /^[ ]+\n/, 'Text'
-        rule /\n+/m, 'Text'
+        rule /^[ ]+\n/, Text
+        rule /\n+/m, Text
         rule /^(?=---|\.\.\.)/ do
           pop! 3
         end
 
         # dedent detection
         rule /^[ ]*/ do |m|
-          token 'Text'
+          token Text
           pop!
 
           indent_size = m[0].size
 
           # dedent = end of scalar
-          if m[0].size <= self.indent
+          if indent_size <= self.indent
             pop!
-            val, err = save_indent
-            # push :block_line
+            save_indent(m[0])
             push :indentation
           end
         end
@@ -318,34 +315,34 @@ module Rouge
 
       state :plain_scalar_in_block_context do
         # the : indicator ends a scalar
-        rule /[ ]*(?=:[ \n]|:$)/, 'Text', :pop!
-        rule /[ ]*:/, 'Literal.String'
-        rule /[ ]+(?=#)/, 'Text', :pop!
-        rule /[ ]+$/, 'Text'
+        rule /[ ]*(?=:[ \n]|:$)/, Text, :pop!
+        rule /[ ]*:/, Str
+        rule /[ ]+(?=#)/, Text, :pop!
+        rule /[ ]+$/, Text
         # check for new documents or dedents at the new line
         rule /\n+/ do
-          token 'Text'
+          token Text
           push :plain_scalar_in_block_context_new_line
         end
 
-        rule /[ ]+/, 'Literal.String'
+        rule /[ ]+/, Str
         # regular non-whitespace characters
-        rule /[^\s:]+/, 'Literal.String'
+        rule /[^\s:]+/, Str
       end
 
       state :plain_scalar_in_flow_context do
-        rule /[ ]*(?=[,:?\[\]{}])/, 'Text', :pop!
-        rule /[ ]+(?=#)/, 'Text', :pop!
-        rule /^[ ]+/, 'Text'
-        rule /[ ]+$/, 'Text'
-        rule /\n+/, 'Text'
-        rule /[ ]+/, 'Name.Variable'
-        rule /[^\s,:?\[\]{}]+/, 'Name.Variable'
+        rule /[ ]*(?=[,:?\[\]{}])/, Text, :pop!
+        rule /[ ]+(?=#)/, Text, :pop!
+        rule /^[ ]+/, Text
+        rule /[ ]+$/, Text
+        rule /\n+/, Text
+        rule /[ ]+/, Name::Variable
+        rule /[^\s,:?\[\]{}]+/, Name::Variable
       end
 
       state :yaml_directive do
         rule /([ ]+)(\d+\.\d+)/ do
-          group 'Text'; group 'Number'
+          group Text; group Num
           pop!; push :ignored_line
         end
       end
@@ -355,8 +352,8 @@ module Rouge
           ([ ]+)(!|![\w-]*!) # prefix
           ([ ]+)(!|!?[\w;/?:@&=+$,.!~*'()\[\]%-]+) # tag handle
         )x do
-          group 'Text'; group 'Keyword.Type'
-          group 'Text'; group 'Keyword.Type'
+          group Text; group Keyword::Type
+          group Text; group Keyword::Type
           pop!; push :ignored_line
         end
       end

data/lib/rouge/plugins/redcarpet.rb

@@ -6,7 +6,7 @@ module Rouge
   module Plugins
     module Redcarpet
       def block_code(code, language)
-        lexer = Lexer.find_fancy(language, code) || Lexers::Text
+        lexer = Lexer.find_fancy(language, code) || Lexers::PlainText
 
         # XXX HACK: Redcarpet strips hard tabs out of code blocks,
         # so we assume you're not using leading spaces that aren't tabs,

data/lib/rouge/regex_lexer.rb

@@ -165,6 +165,7 @@ module Rouge
     # start_procs.
     def reset!
       @stack = nil
+      @current_stream = nil
 
       self.class.start_procs.each do |pr|
         instance_eval(&pr)
@@ -186,6 +187,8 @@ module Rouge
     def stream_tokens(str, &b)
       stream = StringScanner.new(str)
 
+      @current_stream = stream
+
       until stream.eos?
         debug { "lexer: #{self.class.tag}" }
         debug { "stack: #{stack.map(&:name).inspect}" }
@@ -194,7 +197,7 @@ module Rouge
 
         if !success
           debug { "    no match, yielding Error" }
-          b.call(Token['Error'], stream.getch)
+          b.call(Token::Tokens::Error, stream.getch)
         end
       end
     end
@@ -231,9 +234,7 @@ module Rouge
     def run_callback(stream, callback, &output_stream)
       with_output_stream(output_stream) do
         @group_count = 0
-        @last_match = stream
         instance_exec(stream, &callback)
-        @last_match = nil
       end
     end
 
@@ -274,18 +275,20 @@ module Rouge
     #   (optional) the string value to yield.  If absent, this defaults
     #   to the entire last match.
     def token(tok, val=:__absent__)
-      val = @last_match[0] if val == :__absent__
-      val ||= ''
-
-      raise 'no output stream' unless @output_stream
-
-      @output_stream << [Token[tok], val] unless val.empty?
+      val = @current_stream[0] if val == :__absent__
+      yield_token(tok, val)
     end
 
     # Yield a token with the next matched group.  Subsequent calls
     # to this method will yield subsequent groups.
     def group(tok)
-      token(tok, @last_match[@group_count += 1])
+      yield_token(tok, @current_stream[@group_count += 1])
+    end
+
+    def groups(*tokens)
+      tokens.each_with_index do |tok, i|
+        yield_token(tok, @current_stream[i+1])
+      end
     end
 
     # Delegate the lex to another lexer.  The #lex method will be called
@@ -299,7 +302,7 @@ module Rouge
     #   The text to delegate.  This defaults to the last matched string.
     def delegate(lexer, text=nil)
       debug { "    delegating to #{lexer.inspect}" }
-      text ||= @last_match[0]
+      text ||= @current_stream[0]
 
       lexer.lex(text, :continue => true) do |tok, val|
         debug { "    delegated token: #{tok.inspect}, #{val.inspect}" }
@@ -336,6 +339,12 @@ module Rouge
       nil
     end
 
+    # replace the head of the stack with the given state
+    def goto(state_name)
+      raise 'empty stack!' if stack.empty?
+      stack[-1] = get_state(state_name)
+    end
+
     # reset the stack back to `[:root]`.
     def reset_stack
       debug { '    resetting stack' }
@@ -345,7 +354,10 @@ module Rouge
 
     # Check if `state_name` is in the state stack.
     def in_state?(state_name)
-      stack.map(&:name).include? state_name.to_s
+      state_name = state_name.to_s
+      stack.any? do |state|
+        state.name == state_name.to_s
+      end
     end
 
     # Check if `state_name` is the state on top of the state stack.
@@ -357,8 +369,8 @@ module Rouge
     def with_output_stream(output_stream, &b)
       old_output_stream = @output_stream
       @output_stream = Enumerator::Yielder.new do |tok, val|
-        debug { "    yielding #{tok.to_s.inspect}, #{val.inspect}" }
-        output_stream.call(Token[tok], val)
+        debug { "    yielding #{tok.qualname}, #{val.inspect}" }
+        output_stream.call(tok, val)
       end
 
       yield
@@ -366,5 +378,10 @@ module Rouge
     ensure
       @output_stream = old_output_stream
     end
+
+    def yield_token(tok, val)
+      return if val.nil? || val.empty?
+      @output_stream.yield(tok, val)
+    end
   end
 end