rouge 0.2.6 → 0.2.7

data/lib/rouge/lexer.rb

@@ -259,9 +259,22 @@ module Rouge
     # @example
     #   debug { "hello, world!" }
     def debug(&b)
-      @debug = option(:debug) unless instance_variable_defined?(:@debug)
+      # This method is a hotspot, unfortunately.
+      #
+      # For performance reasons, the "debug" option of a lexer cannot
+      # be changed once it has begun lexing.  This method will redefine
+      # itself on the first call to a noop if "debug" is not set.
+      if option(:debug)
+        class << self
+          def debug; puts yield; end
+        end
+      else
+        class << self
+          def debug; end
+        end
+      end
 
-      puts(b.call) if @debug
+      debug(&b)
     end
 
     # @abstract

data/lib/rouge/lexers/c.rb

@@ -11,18 +11,28 @@ module Rouge
       ws = %r((?:\s|//.*?\n|/[*].*?[*]/)+)
       id = /[a-zA-Z_][a-zA-Z0-9_]*/
 
-      keywords = %w(
-        auto break case const continue default do else enum extern
-        for goto if register restricted return sizeof static struct
-        switch typedef union volatile virtual while
-      )
+      def self.keywords
+        @keywords ||= Set.new %w(
+          auto break case const continue default do else enum extern
+          for goto if register restricted return sizeof static struct
+          switch typedef union volatile virtual while
+        )
+      end
 
-      keywords_type = %w(int long float short double char unsigned signed void)
+      def self.keywords_type
+        @keywords_type ||= Set.new %w(
+          int long float short double char unsigned signed void
+        )
+      end
 
-      __reserved = %w(
-        asm int8 based except int16 stdcall cdecl fastcall int32
-        declspec finally int61 try leave
-      )
+      def self.reserved
+        @reserved ||= Set.new %w(
+          __asm __int8 __based __except __int16 __stdcall __cdecl
+          __fastcall __int32 __declspec __finally __int61 __try __leave
+          inline _inline __inline naked _naked __naked restrict _restrict
+          __restrict thread _thread __thread typename _typename __typename
+        )
+      end
 
       state :whitespace do
         rule /^#if\s+0\b/, 'Comment.Preproc', :if_0
@@ -41,6 +51,8 @@ module Rouge
       end
 
       state :statements do
+        rule /\s+/m, 'Text'
+
         rule /L?"/, 'Literal.String', :string
         rule %r(L?'(\\.|\\[0-7]{1,3}|\\x[a-f0-9]{1,2}|[^\\'\n])')i, 'Literal.String.Char'
         rule %r((\d+\.\d*|\.\d+|\d+)[e][+-]?\d+[lu]*)i, 'Literal.Number.Float'
@@ -51,13 +63,20 @@ module Rouge
         rule %r([~!%^&*+=\|?:<>/-]), 'Operator'
         rule /[()\[\],.]/, 'Punctuation'
         rule /\bcase\b/, 'Keyword', :case
-        rule /(?:#{keywords.join('|')})\b/, 'Keyword'
-        rule /(?:#{keywords_type.join('|')})\b/, 'Keyword.Type'
-        rule /(?:_{0,2}inline|naked|restrict|thread|typename)\b/, 'Keyword.Reserved'
-        rule /__(?:#{__reserved.join('|')})\b/, 'Keyword.Reserved'
         rule /(?:true|false|NULL)\b/, 'Name.Builtin'
-        rule id, 'Name'
-        rule /\s+/m, 'Text'
+        rule id do |m|
+          name = m[0]
+
+          if self.class.keywords.include? name
+            token 'Keyword'
+          elsif self.class.keywords_type.include? name
+            token 'Keyword.Type'
+          elsif self.class.reserved.include? name
+            token 'Keyword.Reserved'
+          else
+            token 'Name'
+          end
+        end
       end
 
       state :case do

data/lib/rouge/lexers/common_lisp.rb

@@ -208,28 +208,6 @@ module Rouge
         structure-object symbol synonym-stream t two-way-stream vector
       )
 
-      postprocess 'Name.Variable' do |tok, val|
-        tok = if BUILTIN_FUNCTIONS.include? val
-          'Name.Builtin'
-        elsif SPECIAL_FORMS.include? val
-          'Keyword'
-        elsif MACROS.include? val
-          'Name.Builtin'
-        elsif LAMBDA_LIST_KEYWORDS.include? val
-          'Keyword'
-        elsif DECLARATIONS.include? val
-          'Keyword'
-        elsif BUILTIN_TYPES.include? val
-          'Keyword.Type'
-        elsif BUILTIN_CLASSES.include? val
-          'Name.Class'
-        else
-          'Name.Variable'
-        end
-
-        token tok, val
-      end
-
       nonmacro = /\\.|[a-zA-Z0-9!$%&*+-\/<=>?@\[\]^_{}~]/
       constituent = /#{nonmacro}|[#.:]/
       terminated = /(?=[ "'()\n,;`])/ # whitespace or terminating macro chars
@@ -319,7 +297,27 @@ module Rouge
         # functions and variables
         # note that these get filtered through in stream_tokens
         rule /\*#{symbol}\*/, 'Name.Variable.Global'
-        rule symbol, 'Name.Variable'
+        rule symbol do |m|
+          sym = m[0]
+
+          if BUILTIN_FUNCTIONS.include? sym
+            token 'Name.Builtin'
+          elsif SPECIAL_FORMS.include? sym
+            token 'Keyword'
+          elsif MACROS.include? sym
+            token 'Name.Builtin'
+          elsif LAMBDA_LIST_KEYWORDS.include? sym
+            token 'Keyword'
+          elsif DECLARATIONS.include? sym
+            token 'Keyword'
+          elsif BUILTIN_TYPES.include? sym
+            token 'Keyword.Type'
+          elsif BUILTIN_CLASSES.include? sym
+            token 'Name.Class'
+          else
+            token 'Name.Variable'
+          end
+        end
 
         rule /\(/, 'Punctuation', :root
         rule /\)/, 'Punctuation' do

data/lib/rouge/lexers/cpp.rb

@@ -12,26 +12,37 @@ module Rouge
                 '*.cxx', '*.hxx'
       mimetypes 'text/x-c++hdr', 'text/x-c++src'
 
-      keywords = %w(
-        asm auto break case catch const const_cast continue
-        default delete do dynamic_cast else enum explicit export
-        extern for friend goto if mutable namespace new operator
-        private protected public register reinterpret_cast return
-        restrict sizeof static static_cast struct switch template
-        this throw throws try typedef typeid typename union using
-        volatile virtual while
-      )
-
-      keywords_type = %w(
-        bool int long float short double char unsigned signed void wchar_t
-      )
-
-      __reserved = %w(
-        asm int8 based except int16 stdcall cdecl fastcall int32 declspec
-        finally int64 try leave wchar_t w64 virtual_inheritance uuidof
-        unaligned super single_inheritance raise noop multiple_inheritance
-        m128i m128d m128 m64 interface identifier forceinline event assume
-      )
+      def self.keywords
+        @keywords ||= Set.new %w(
+          asm auto break case catch const const_cast continue
+          default delete do dynamic_cast else enum explicit export
+          extern for friend goto if mutable namespace new operator
+          private protected public register reinterpret_cast return
+          restrict sizeof static static_cast struct switch template
+          this throw throws try typedef typeid typename union using
+          volatile virtual while
+        )
+      end
+
+      def self.keywords_type
+        @keywords_type ||= Set.new %w(
+          bool int long float short double char unsigned signed void wchar_t
+        )
+      end
+
+      def self.reserved
+        @reserved ||= Set.new %w(
+          __asm __int8 __based __except __int16 __stdcall __cdecl
+          __fastcall __int32 __declspec __finally __int64 __try
+          __leave __wchar_t __w64 __virtual_inheritance __uuidof
+          __unaligned __super __single_inheritance __raise __noop
+          __multiple_inheritance __m128i __m128d __m128 __m64 __interface
+          __identifier __forceinline __event __assume
+          inline _inline __inline
+          naked _naked __naked
+          thread _thread __thread
+        )
+      end
 
       # optional comments or whitespace
       ws = %r((?:\s|//.*?\n|/[*].*?[*]/)+)
@@ -61,18 +72,27 @@ module Rouge
         rule %r([~!%^&*+=\|?:<>/-]), 'Operator'
         rule /[()\[\],.;{}]/, 'Punctuation'
 
-        rule /(?:#{keywords.join('|')})\b/, 'Keyword'
         rule /class\b/, 'Keyword', :classname
-        rule /(?:#{keywords_type.join('|')})\b/, 'Keyword.Type'
-        rule /(?:_{0,2}inline|naked|thread)\b/, 'Keyword.Reserved'
-        rule /__(?:#{__reserved.join('|')})\b/, 'Keyoword.Reserved'
+
         # Offload C++ extensions, http://offload.codeplay.com/
         rule /(?:__offload|__blockingoffload|__outer)\b/, 'Keyword.Pseudo'
 
         rule /(true|false)\b/, 'Keyword.Constant'
         rule /NULL\b/, 'Name.Builtin'
         rule /#{id}:(?!:)/, 'Name.Label'
-        rule id, 'Name'
+        rule id do |m|
+          name = m[0]
+
+          if self.class.keywords.include? name
+            token 'Keyword'
+          elsif self.class.keywords_type.include? name
+            token 'Keyword.Type'
+          elsif self.class.reserved.include? name
+            token 'Keyword.Reserved'
+          else
+            token 'Name'
+          end
+        end
       end
 
       state :classname do

data/lib/rouge/lexers/factor.rb

@@ -262,8 +262,15 @@ module Rouge
         rule /(?:deprecated|final|foldable|flushable|inline|recursive)(?=\s)/,
           'Keyword'
 
-        # words, to be postprocessed for builtins and things
-        rule /\S+/, 'Postprocess.Word'
+        rule /\S+/ do |m|
+          name = m[0]
+
+          if self.class.builtins.values.any? { |b| b.include? name }
+            token 'Name.Builtin'
+          else
+            token 'Name'
+          end
+        end
       end
 
       state :stack_effect do
@@ -286,16 +293,6 @@ module Rouge
         rule /\s+/, 'Text'
         rule /\S+/, 'Name.Namespace'
       end
-
-      postprocess 'Postprocess.Word' do |tok, val|
-        tok = if self.class.builtins.values.any? { |b| b.include? val }
-          'Name.Builtin'
-        else
-          'Name'
-        end
-
-        token tok, val
-      end
     end
   end
 end

data/lib/rouge/lexers/php.rb

@@ -43,17 +43,19 @@ module Rouge
         push :php if start_inline?
       end
 
-      keywords = %w(
-        and E_PARSE old_function E_ERROR or as E_WARNING parent eval
-        PHP_OS break exit case extends PHP_VERSION cfunction FALSE
-        print for require continue foreach require_once declare return
-        default static do switch die stdClass echo else TRUE elseif
-        var empty if xor enddeclare include virtual endfor include_once
-        while endforeach global __FILE__ endif list __LINE__ endswitch
-        new __sleep endwhile not array __wakeup E_ALL NULL final
-        php_user_filter interface implements public private protected
-        abstract clone try catch throw this use namespace
-      )
+      def self.keywords
+        @keywords ||= Set.new %w(
+          and E_PARSE old_function E_ERROR or as E_WARNING parent eval
+          PHP_OS break exit case extends PHP_VERSION cfunction FALSE
+          print for require continue foreach require_once declare return
+          default static do switch die stdClass echo else TRUE elseif
+          var empty if xor enddeclare include virtual endfor include_once
+          while endforeach global __FILE__ endif list __LINE__ endswitch
+          new __sleep endwhile not array __wakeup E_ALL NULL final
+          php_user_filter interface implements public private protected
+          abstract clone try catch throw this use namespace
+        )
+      end
 
       state :root do
         rule /<\?(php|=)?/, 'Comment.Preproc', :php
@@ -93,13 +95,22 @@ module Rouge
           group 'Keyword'; group 'Text'; group 'Name.Constant'
         end
 
-        rule /(?:#{keywords.join('|')})\b/, 'Keyword'
         rule /(true|false|null)\b/, 'Keyword.Constant'
         rule /\$\{\$+[a-z_]\w*\}/i, 'Name.Variable'
         rule /\$+[a-z_]\w*/i, 'Name.Variable'
 
         # may be intercepted for builtin highlighting
-        rule /[\\a-z_][\\\w]*/i, 'Name.Other'
+        rule /[\\a-z_][\\\w]*/i do |m|
+          name = m[0]
+
+          if self.class.keywords.include? name
+            token 'Keyword'
+          elsif self.builtins.include? name
+            token 'Name.Builtin'
+          else
+            token 'Name.Other'
+          end
+        end
 
         rule /(\d+\.\d*|\d*\.\d+)(e[+-]?\d+)?/i, 'Literal.Number.Float'
         rule /\d+e[+-]?\d+/i, 'Literal.Number.Float'
@@ -146,12 +157,6 @@ module Rouge
         rule /\}/, 'Literal.String.Interpol', :pop!
         mixin :php
       end
-
-      postprocess 'Name.Other' do |tok, val|
-        tok = 'Name.Builtin' if builtins.include? val
-
-        token tok, val
-      end
     end
   end
 end

data/lib/rouge/lexers/shell.rb

@@ -5,7 +5,9 @@ module Rouge
 
       tag 'shell'
       aliases 'bash', 'zsh', 'ksh', 'sh'
-      filenames '*.sh', '*.bash', '*.zsh', '*.ksh'
+      filenames '*.sh', '*.bash', '*.zsh', '*.ksh',
+                '.bashrc', '.zshrc', '.kshrc', '.profile'
+
       mimetypes 'application/x-sh', 'application/x-shellscript'
 
       def self.analyze_text(text)

data/lib/rouge/lexers/viml.rb

@@ -41,26 +41,25 @@ module Rouge
         rule /\b(NONE|bold|italic|underline|dark|light)\b/, 'Name.Builtin'
 
         rule /[absg]:\w+\b/, 'Name.Variable'
-        rule /\b\w+\b/, 'Postprocess.Name'
+        rule /\b\w+\b/ do |m|
+          name = m[0]
+          keywords = self.class.keywords
+
+          if mapping_contains?(keywords[:command], name)
+            token 'Keyword'
+          elsif mapping_contains?(keywords[:option], name)
+            token 'Name.Builtin'
+          elsif mapping_contains?(keywords[:auto], name)
+            token 'Name.Builtin'
+          else
+            token 'Text'
+          end
+        end
 
         # no errors in VimL!
         rule /./m, 'Text'
       end
 
-      postprocess 'Postprocess.Name' do |tok, name|
-        keywords = self.class.keywords
-
-        if mapping_contains?(keywords[:command], name)
-          token 'Keyword', name
-        elsif mapping_contains?(keywords[:option], name)
-          token 'Name.Builtin', name
-        elsif mapping_contains?(keywords[:auto], name)
-          token 'Name.Builtin', name
-        else
-          token 'Text', name
-        end
-      end
-
       def mapping_contains?(mapping, word)
         shortest, longest = find_likely_mapping(mapping, word)
 

data/lib/rouge/regex_lexer.rb

@@ -45,10 +45,15 @@ module Rouge
         @rules ||= []
       end
 
-      def load!
+      def load!(lexer_class)
         return self if @loaded
         @loaded = true
         StateDSL.new(rules).instance_eval(&@defn)
+
+        rules.map! do |rule|
+          rule.is_a?(String) ? lexer_class.get_state(rule) : rule
+        end
+
         self
       end
     end
@@ -79,20 +84,13 @@ module Rouge
       #   {RegexLexer#token}, and {RegexLexer#delegate}.  The first
       #   argument can be used to access the match groups.
       def rule(re, tok=nil, next_state=nil, &callback)
-        if block_given?
-          next_state = tok
+        callback ||= case next_state
+        when :pop!
+          proc { token tok; pop! }
+        when Symbol
+          proc { token tok; push next_state }
         else
-          tok = Token[tok]
-
-          callback = proc do
-            token tok
-            case next_state
-            when :pop!
-              pop!
-            when Symbol
-              push next_state
-            end # else pass
-          end
+          proc { token tok }
         end
 
         rules << Rule.new(re, callback)
@@ -127,25 +125,6 @@ module Rouge
       start_procs << b
     end
 
-    # Specify a filter to be applied as the lexer yields tokens.
-    #
-    # @param toktype
-    #   The token type to postprocess
-    # @yield [tok, val]
-    #   The token and the matched value.  The block will be evaluated in
-    #   the context of the lexer, and it must yield an equivalent
-    #   token/value pair, usually by calling #token.
-    def self.postprocess(toktype, &b)
-      postprocesses << [Token[toktype], b]
-    end
-
-    # where the postprocess blocks are stored.
-    # @see postprocess
-    def self.postprocesses
-      @postprocesses ||= InheritableList.new(superclass.postprocesses)
-    end
-    @postprocesses = []
-
     # Define a new state for this lexer with the given name.
     # The block will be evaluated in the context of a {StateDSL}.
     def self.state(name, &b)
@@ -159,7 +138,7 @@ module Rouge
 
       state = states[name.to_s]
       raise "unknown state: #{name}" unless state
-      state.load!
+      state.load!(self)
     end
 
     # @private
@@ -205,21 +184,6 @@ module Rouge
     #
     # @see #step #step (where (2.) is implemented)
     def stream_tokens(stream, &b)
-      stream_without_postprocessing(stream) do |tok, val|
-        _, processor = self.class.postprocesses.find { |t, _| t === tok }
-
-        if processor
-          with_output_stream(b) do
-            instance_exec(tok, val, &processor)
-          end
-        else
-          yield tok, val
-        end
-      end
-    end
-
-    # @private
-    def stream_without_postprocessing(stream, &b)
       until stream.eos?
         debug { "lexer: #{self.class.tag}" }
         debug { "stack: #{stack.map(&:name).inspect}" }
@@ -240,35 +204,25 @@ module Rouge
     # @return false otherwise.
     def step(state, stream, &b)
       state.rules.each do |rule|
-        return true if run_rule(rule, stream, &b)
-      end
+        case rule
+        when State
+          debug { "  entering mixin #{rule.name}" }
+          return true if step(rule, stream, &b)
+          debug { "  exiting  mixin #{rule.name}" }
+        when Rule
+          debug { "  trying #{rule.inspect}" }
 
-      false
-    end
+          if run_rule(rule, stream)
+            debug { "    got #{stream[0].inspect}" }
 
-    # @private
-    def run_rule(rule, stream, &b)
-      case rule
-      when String
-        debug { "  entering mixin #{rule}" }
-        res = step(get_state(rule), stream, &b)
-        debug { "  exiting  mixin #{rule}" }
-        res
-      when Rule
-        debug { "  trying #{rule.inspect}" }
-        # XXX HACK XXX
-        # StringScanner's implementation of ^ is b0rken.
-        # see http://bugs.ruby-lang.org/issues/7092
-        # TODO: this doesn't cover cases like /(a|^b)/, but it's
-        # the most common, for now...
-        return false if rule.beginning_of_line? && !stream.beginning_of_line?
-
-        scan(stream, rule.re) do
-          debug { "    got #{stream[0].inspect}" }
-
-          run_callback(stream, rule.callback, &b)
+            run_callback(stream, rule.callback, &b)
+
+            return true
+          end
         end
       end
+
+      false
     end
 
     # @private
@@ -286,28 +240,28 @@ module Rouge
     MAX_NULL_SCANS = 5
 
     # @private
-    def scan(scanner, re, &b)
-      @null_steps ||= 0
-
-      if @null_steps >= MAX_NULL_SCANS
+    def run_rule(rule, scanner, &b)
+      # XXX HACK XXX
+      # StringScanner's implementation of ^ is b0rken.
+      # see http://bugs.ruby-lang.org/issues/7092
+      # TODO: this doesn't cover cases like /(a|^b)/, but it's
+      # the most common, for now...
+      return false if rule.beginning_of_line? && !scanner.beginning_of_line?
+
+      if (@null_steps ||= 0) >= MAX_NULL_SCANS
         debug { "    too many scans without consuming the string!" }
         return false
       end
 
-      scanner.scan(re)
+      scanner.scan(rule.re) or return false
 
-      if scanner.matched?
-        if scanner.matched_size == 0
-          @null_steps += 1
-        else
-          @null_steps = 0
-        end
-
-        yield self
-        return true
+      if scanner.matched_size.zero?
+        @null_steps += 1
+      else
+        @null_steps = 0
       end
 
-      return false
+      true
     end
 
     # Yield a token.
@@ -358,7 +312,7 @@ module Rouge
       push_state = if state_name
         get_state(state_name)
       elsif block_given?
-        State.new(b.inspect, &b).load!
+        State.new(b.inspect, &b).load!(self.class)
       else
         # use the top of the stack by default
         self.state

data/lib/rouge/version.rb

@@ -1,5 +1,5 @@
 module Rouge
   def self.version
-    "0.2.6"
+    "0.2.7"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rouge
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-21 00:00:00.000000000 Z
+date: 2012-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor