rouge 1.1.0 → 1.2.0

data/lib/rouge/lexers/tex.rb

@@ -30,10 +30,7 @@ module Rouge
         rule /\\(begin|end)\{.*?\}/, Name::Tag
 
         rule /(\\verb)\b(\S)(.*?)(\2)/ do |m|
-          group Name::Builtin
-          group Keyword::Pseudo
-          group Str::Other
-          group Keyword::Pseudo
+          groups Name::Builtin, Keyword::Pseudo, Str::Other, Keyword::Pseudo
         end
 
         rule command, Keyword, :command

data/lib/rouge/lexers/toml.rb

@@ -30,8 +30,7 @@ module Rouge
         mixin :basic
 
         rule /(#{identifier})(\s*)(=)/ do
-          group Name::Property; group Text
-          group Punctuation
+          groups Name::Property, Text, Punctuation
           push :value
         end
 

data/lib/rouge/lexers/vb.rb

@@ -0,0 +1,162 @@
+module Rouge
+  module Lexers
+    class VisualBasic < RegexLexer
+      desc "Visual Basic"
+      tag 'vb'
+      aliases 'visualbasic'
+      filenames '*.vbs'
+      mimetypes 'text/x-visualbasic', 'application/x-visualbasic'
+
+      def self.keywords
+        @keywords ||= Set.new %w(
+          AddHandler Alias ByRef ByVal CBool CByte CChar CDate CDbl CDec
+          CInt CLng CObj CSByte CShort CSng CStr CType CUInt CULng CUShort
+          Call Case Catch Class Const Continue Declare Default Delegate
+          Dim DirectCast Do Each Else ElseIf End EndIf Enum Erase Error
+          Event Exit False Finally For Friend Function Get Global GoSub
+          GoTo Handles If Implements Imports Inherits Interface Let
+          Lib Loop Me Module MustInherit MustOverride MyBase MyClass
+          Namespace Narrowing New Next Not NotInheritable NotOverridable
+          Nothing Of On Operator Option Optional Overloads Overridable
+          Overrides ParamArray Partial Private Property Protected Public
+          RaiseEvent ReDim ReadOnly RemoveHandler Resume Return Select Set
+          Shadows Shared Single Static Step Stop Structure Sub SyncLock
+          Then Throw To True Try TryCast Using Wend When While Widening
+          With WithEvents WriteOnly
+        )
+      end
+
+      def self.keywords_type
+        @keywords_type ||= Set.new %w(
+          Boolean Byte Char Date Decimal Double Integer Long Object
+          SByte Short Single String Variant UInteger ULong UShort
+        )
+      end
+
+      def self.operator_words
+        @operator_words ||= Set.new %w(
+          AddressOf And AndAlso As GetType In Is IsNot Like Mod Or OrElse
+          TypeOf Xor
+        )
+      end
+
+      def self.builtins
+        @builtins ||= Set.new %w(
+          Console ConsoleColor
+        )
+      end
+
+      id = /[a-z_]\w*/i
+      upper_id = /[A-Z]\w*/
+
+      state :whitespace do
+        rule /\s+/, Text
+        rule /\n/, Text, :bol
+        rule /rem\b.*?$/i, Comment::Single
+        rule %r(%\{.*?%\})m, Comment::Multiline
+        rule /'.*$/, Comment::Single
+      end
+
+      state :bol do
+        rule /\s+/, Text
+        rule /<.*?>/, Name::Attribute
+        rule(//) { :pop! }
+      end
+
+      state :root do
+        mixin :whitespace
+        rule %r(
+            [#]If\b .*? \bThen
+          | [#]ElseIf\b .*? \bThen
+          | [#]End \s+ If
+          | [#]Const
+          | [#]ExternalSource .*? \n
+          | [#]End \s+ ExternalSource
+          | [#]Region .*? \n
+          | [#]End \s+ Region
+          | [#]ExternalChecksum
+        )x, Comment::Preproc
+        rule /[.]/, Punctuation, :dotted
+        rule /[(){}!#,:]/, Punctuation
+        rule /Option\s+(Strict|Explicit|Compare)\s+(On|Off|Binary|Text)/,
+          Keyword::Declaration
+        rule /End\b/, Keyword, :end
+        rule /(Dim|Const)\b/, Keyword, :dim
+        rule /(Function|Sub|Property)\b/, Keyword, :funcname
+        rule /(Class|Structure|Enum)\b/, Keyword, :classname
+        rule /(Module|Namespace|Imports)\b/, Keyword, :namespace
+
+        rule upper_id do |m|
+          match = m[0]
+          if self.class.keywords.include? match
+            token Keyword
+          elsif self.class.keywords_type.include? match
+            token Keyword::Type
+          elsif self.class.operator_words.include? match
+            token Operator::Word
+          elsif self.class.builtins.include? match
+            token Name::Builtin
+          else
+            token Name
+          end
+        end
+
+        rule(
+          %r(&=|[*]=|/=|\\=|\^=|\+=|-=|<<=|>>=|<<|>>|:=|<=|>=|<>|[-&*/\\^+=<>.]),
+          Operator
+        )
+
+        rule /"/, Str, :string
+        rule /#{id}[%&@!#\$]?/, Name
+        rule /#.*?#/, Literal::Date
+
+        rule /(\d+\.\d*|\d*\.\d+)(f[+-]?\d+)?/i, Num::Float
+        rule /\d+([SILDFR]|US|UI|UL)?/, Num::Integer
+        rule /&H[0-9a-f]+([SILDFR]|US|UI|UL)?/, Num::Integer
+        rule /&O[0-7]+([SILDFR]|US|UI|UL)?/, Num::Integer
+
+        rule /_\n/, Keyword
+      end
+
+      state :dotted do
+        mixin :whitespace
+        rule id, Name, :pop!
+      end
+
+      state :string do
+        rule /""/, Str::Escape
+        rule /"C?/, Str, :pop!
+        rule /[^"]+/, Str
+      end
+
+      state :dim do
+        mixin :whitespace
+        rule id, Name::Variable, :pop!
+        rule(//) { pop! }
+      end
+
+      state :funcname do
+        mixin :whitespace
+        rule id, Name::Function, :pop!
+      end
+
+      state :classname do
+        mixin :whitespace
+        rule id, Name::Class, :pop!
+      end
+
+      state :namespace do
+        mixin :whitespace
+        rule /#{id}([.]#{id})*/, Name::Namespace, :pop!
+      end
+
+      state :end do
+        mixin :whitespace
+        rule /(Function|Sub|Property|Class|Structure|Enum|Module|Namespace)\b/,
+          Keyword, :pop!
+        rule(//) { pop! }
+      end
+    end
+  end
+end
+

data/lib/rouge/lexers/viml.rb

@@ -16,7 +16,7 @@ module Rouge
 
       state :root do
         rule /^(\s*)(".*?)$/ do
-          group Text; group Comment
+          groups Text, Comment
         end
 
         rule /^\s*\\/, Str::Escape

data/lib/rouge/lexers/yaml.rb

@@ -17,7 +17,7 @@ module Rouge
 
       # reset the indentation levels
       def reset_indent
-        debug { "    yaml: reset_indent" }
+        puts "    yaml: reset_indent" if @debug
         @indent_stack = [0]
         @next_indent = 0
         @block_scalar_indent = nil
@@ -39,12 +39,12 @@ module Rouge
       # Save a possible indentation level
       def save_indent(match)
         @next_indent = match.size
-        debug { "    yaml: indent: #{self.indent}/#@next_indent" }
-        debug { "    yaml: popping indent stack - before: #@indent_stack" }
+        puts "    yaml: indent: #{self.indent}/#@next_indent" if @debug
+        puts "    yaml: popping indent stack - before: #@indent_stack" if @debug
         if dedent?(@next_indent)
           @indent_stack.pop while dedent?(@next_indent)
-          debug { "    yaml: popping indent stack - after: #@indent_stack" }
-          debug { "    yaml: indent: #{self.indent}/#@next_indent" }
+          puts "    yaml: popping indent stack - after: #@indent_stack" if @debug
+          puts "    yaml: indent: #{self.indent}/#@next_indent" if @debug
 
           # dedenting to a state not previously indented to is an error
           [match[0...self.indent], match[self.indent..-1]]
@@ -54,7 +54,7 @@ module Rouge
       end
 
       def continue_indent(match)
-        debug { "    yaml: continue_indent" }
+        puts "    yaml: continue_indent" if @debug
         @next_indent += match.size
       end
 
@@ -342,8 +342,8 @@ module Rouge
 
       state :yaml_directive do
         rule /([ ]+)(\d+\.\d+)/ do
-          group Text; group Num
-          pop!; push :ignored_line
+          groups Text, Num
+          goto :ignored_line
         end
       end
 
@@ -352,9 +352,8 @@ module Rouge
           ([ ]+)(!|![\w-]*!) # prefix
           ([ ]+)(!|!?[\w;/?:@&=+$,.!~*'()\[\]%-]+) # tag handle
         )x do
-          group Text; group Keyword::Type
-          group Text; group Keyword::Type
-          pop!; push :ignored_line
+          groups Text, Keyword::Type, Text, Keyword::Type
+          goto :ignored_line
         end
       end
     end

data/lib/rouge/regex_lexer.rb

@@ -10,18 +10,10 @@ module Rouge
     class Rule
       attr_reader :callback
       attr_reader :re
+      attr_reader :beginning_of_line
       def initialize(re, callback)
         @re = re
         @callback = callback
-      end
-
-      # Does the regex start with a ^?
-      #
-      # Since Regexps are immuntable, this is cached to avoid
-      # calling Regexp#source more than once.
-      def beginning_of_line?
-        return @beginning_of_line if instance_variable_defined?(:@beginning_of_line)
-
         @beginning_of_line = re.source[0] == ?^
       end
 
@@ -101,11 +93,25 @@ module Rouge
       def rule(re, tok=nil, next_state=nil, &callback)
         callback ||= case next_state
         when :pop!
-          proc { token tok; pop! }
+          proc do |stream|
+            puts "    yielding #{tok.qualname}, #{stream[0].inspect}" if @debug
+            @output_stream.call(tok, stream[0])
+            puts "    popping stack: #{1}" if @debug
+            @stack.pop or raise 'empty stack!'
+          end
         when Symbol
-          proc { token tok; push next_state }
+          proc do |stream|
+            puts "    yielding #{tok.qualname}, #{stream[0].inspect}" if @debug
+            @output_stream.call(tok, stream[0])
+            state = @states[next_state] || self.class.get_state(next_state)
+            puts "    pushing #{state.name}" if @debug
+            @stack.push(state)
+          end
         else
-          proc { token tok }
+          proc do |stream|
+            puts "    yielding #{tok.qualname}, #{stream[0].inspect}" if @debug
+            @output_stream.call(tok, stream[0])
+          end
         end
 
         rules << Rule.new(re, callback)
@@ -180,10 +186,8 @@ module Rouge
     def self.get_state(name)
       return name if name.is_a? State
 
-      name = name.to_s
-
-      states[name] ||= begin
-        defn = state_definitions[name] or raise "unknown state: #{name.inspect}"
+      states[name.to_sym] ||= begin
+        defn = state_definitions[name.to_s] or raise "unknown state: #{name.inspect}"
         defn.to_state(self)
       end
     end
@@ -235,39 +239,65 @@ module Rouge
       stream = StringScanner.new(str)
 
       @current_stream = stream
+      @output_stream  = b
+      @states         = self.class.states
+      @null_steps     = 0
 
       until stream.eos?
-        debug { "lexer: #{self.class.tag}" }
-        debug { "stack: #{stack.map(&:name).inspect}" }
-        debug { "stream: #{stream.peek(20).inspect}" }
-        success = step(get_state(state), stream, &b)
+        if @debug
+          puts "lexer: #{self.class.tag}"
+          puts "stack: #{stack.map(&:name).inspect}"
+          puts "stream: #{stream.peek(20).inspect}"
+        end
+
+        success = step(state, stream)
 
         if !success
-          debug { "    no match, yielding Error" }
+          puts "    no match, yielding Error" if @debug
           b.call(Token::Tokens::Error, stream.getch)
         end
       end
     end
 
+    # The number of successive scans permitted without consuming
+    # the input stream.  If this is exceeded, the match fails.
+    MAX_NULL_SCANS = 5
+
     # Runs one step of the lex.  Rules in the current state are tried
     # until one matches, at which point its callback is called.
     #
     # @return true if a rule was tried successfully
     # @return false otherwise.
-    def step(state, stream, &b)
+    def step(state, stream)
       state.rules.each do |rule|
-        case rule
-        when State
-          debug { "  entering mixin #{rule.name}" }
-          return true if step(rule, stream, &b)
-          debug { "  exiting  mixin #{rule.name}" }
-        when Rule
-          debug { "  trying #{rule.inspect}" }
-
-          if run_rule(rule, stream)
-            debug { "    got #{stream[0].inspect}" }
-
-            run_callback(stream, rule.callback, &b)
+        if rule.is_a?(State)
+          puts "  entering mixin #{rule.name}" if @debug
+          return true if step(rule, stream)
+          puts "  exiting  mixin #{rule.name}" if @debug
+        else
+          puts "  trying #{rule.inspect}" if @debug
+
+          # XXX HACK XXX
+          # StringScanner's implementation of ^ is b0rken.
+          # see http://bugs.ruby-lang.org/issues/7092
+          # TODO: this doesn't cover cases like /(a|^b)/, but it's
+          # the most common, for now...
+          next if rule.beginning_of_line && !stream.beginning_of_line?
+
+          if size = stream.skip(rule.re)
+            puts "    got #{stream[0].inspect}" if @debug
+
+            instance_exec(stream, &rule.callback)
+
+            if size.zero?
+              @null_steps += 1
+              if @null_steps > MAX_NULL_SCANS
+                puts "    too many scans without consuming the string!" if @debug
+                return false
+              end
+            else
+              @null_steps = 0
+            end
 
             return true
           end
@@ -277,43 +307,6 @@ module Rouge
       false
     end
 
-    # @private
-    def run_callback(stream, callback, &output_stream)
-      with_output_stream(output_stream) do
-        @group_count = 0
-        instance_exec(stream, &callback)
-      end
-    end
-
-    # The number of successive scans permitted without consuming
-    # the input stream.  If this is exceeded, the match fails.
-    MAX_NULL_SCANS = 5
-
-    # @private
-    def run_rule(rule, scanner)
-      # XXX HACK XXX
-      # StringScanner's implementation of ^ is b0rken.
-      # see http://bugs.ruby-lang.org/issues/7092
-      # TODO: this doesn't cover cases like /(a|^b)/, but it's
-      # the most common, for now...
-      return false if rule.beginning_of_line? && !scanner.beginning_of_line?
-
-      if (@null_steps ||= 0) >= MAX_NULL_SCANS
-        debug { "    too many scans without consuming the string!" }
-        return false
-      end
-
-      scanner.scan(rule.re) or return false
-
-      if scanner.matched_size.zero?
-        @null_steps += 1
-      else
-        @null_steps = 0
-      end
-
-      true
-    end
-
     # Yield a token.
     #
     # @param tok
@@ -321,17 +314,20 @@ module Rouge
     # @param val
     #   (optional) the string value to yield.  If absent, this defaults
     #   to the entire last match.
-    def token(tok, val=:__absent__)
-      val = @current_stream[0] if val == :__absent__
+    def token(tok, val=@current_stream[0])
       yield_token(tok, val)
     end
 
+    # @deprecated
+    #
     # Yield a token with the next matched group.  Subsequent calls
     # to this method will yield subsequent groups.
     def group(tok)
-      yield_token(tok, @current_stream[@group_count += 1])
+      raise "RegexLexer#group is deprecated: use #groups instead"
     end
 
+    # Yield tokens corresponding to the matched groups of the current
+    # match.
     def groups(*tokens)
       tokens.each_with_index do |tok, i|
         yield_token(tok, @current_stream[i+1])
@@ -348,11 +344,11 @@ module Rouge
     # @param [String] text
     #   The text to delegate.  This defaults to the last matched string.
     def delegate(lexer, text=nil)
-      debug { "    delegating to #{lexer.inspect}" }
+      puts "    delegating to #{lexer.inspect}" if @debug
       text ||= @current_stream[0]
 
       lexer.lex(text, :continue => true) do |tok, val|
-        debug { "    delegated token: #{tok.inspect}, #{val.inspect}" }
+        puts "    delegated token: #{tok.inspect}, #{val.inspect}" if @debug
         yield_token(tok, val)
       end
     end
@@ -374,7 +370,7 @@ module Rouge
         self.state
       end
 
-      debug { "    pushing #{push_state.name}" }
+      puts "    pushing #{push_state.name}" if @debug
       stack.push(push_state)
     end
 
@@ -383,7 +379,7 @@ module Rouge
     def pop!(times=1)
       raise 'empty stack!' if stack.empty?
 
-      debug { "    popping stack: #{times}" }
+      puts "    popping stack: #{times}" if @debug
 
       stack.pop(times)
 
@@ -393,12 +389,14 @@ module Rouge
     # replace the head of the stack with the given state
     def goto(state_name)
       raise 'empty stack!' if stack.empty?
+
+      puts "    going to state #{state_name} " if @debug
       stack[-1] = get_state(state_name)
     end
 
     # reset the stack back to `[:root]`.
     def reset_stack
-      debug { '    resetting stack' }
+      puts '    resetting stack' if @debug
       stack.clear
       stack.push get_state(:root)
     end
@@ -417,19 +415,6 @@ module Rouge
     end
 
   private
-    def with_output_stream(output_stream, &b)
-      old_output_stream = @output_stream
-      @output_stream = Enumerator::Yielder.new do |tok, val|
-        debug { "    yielding #{tok.qualname}, #{val.inspect}" }
-        output_stream.call(tok, val)
-      end
-
-      yield
-
-    ensure
-      @output_stream = old_output_stream
-    end
-
     def yield_token(tok, val)
       return if val.nil? || val.empty?
       @output_stream.yield(tok, val)