rouge 0.5.1 → 0.5.2

data/lib/rouge/demos/objective_c

@@ -0,0 +1,12 @@
+@interface Person : NSObject {
+  @public
+  NSString *name;
+  @private
+  int age;
+}
+
+@property(copy) NSString *name;
+@property(readonly) int age;
+
+-(id)initWithAge:(int)age;
+@end

data/lib/rouge/lexer.rb

@@ -205,7 +205,7 @@ module Rouge
 
         best_result = threshold
         best_match = nil
-        registry.values.each do |lexer|
+        lexers.each do |lexer|
           result = lexer.analyze_text(source) || 0
           return lexer if result == 1
 

data/lib/rouge/lexers/c.rb

@@ -22,6 +22,19 @@ module Rouge
       def self.keywords_type
         @keywords_type ||= Set.new %w(
           int long float short double char unsigned signed void
+
+          jmp_buf FILE DIR div_t ldiv_t mbstate_t sig_atomic_t fpos_t
+          clock_t time_t va_list size_t ssize_t off_t wchar_t ptrdiff_t
+          wctrans_t wint_t wctype_t
+
+          _Bool _Complex int8_t int16_t int32_t int64_t
+          uint8_t uint16_t uint32_t uint64_t int_least8_t
+          int_least16_t int_least32_t int_least64_t
+          uint_least8_t uint_least16_t uint_least32_t
+          uint_least64_t int_fast8_t int_fast16_t int_fast32_t
+          int_fast64_t uint_fast8_t uint_fast16_t uint_fast32_t
+          uint_fast64_t intptr_t uintptr_t intmax_t
+          uintmax_t
         )
       end
 
@@ -34,20 +47,31 @@ module Rouge
         )
       end
 
+      # high priority for filename matches
+      def self.analyze_text(*)
+        0.3
+      end
+
+      def self.builtins
+        @builtins ||= []
+      end
+
       start { push :bol }
 
-      state :bol do
+      state :expr_bol do
         mixin :inline_whitespace
 
-        rule /#if\s0/, Comment::Preproc, :if_0
-
+        rule /#if\s0/, Comment, :if_0
         rule /#/, Comment::Preproc, :macro
 
-        rule /#{id}:(?!:)/, Name::Label
-
         rule(//) { pop! }
       end
 
+      state :bol do
+        rule /#{id}:(?!:)/, Name::Label
+        mixin :expr_bol
+      end
+
       state :inline_whitespace do
         rule /[ \t\r]+/, Text
         rule /\\\n/, Text # line continuation
@@ -55,14 +79,18 @@ module Rouge
       end
 
       state :whitespace do
-        rule /\n+/, Text, :bol
+        rule /\n+/m, Text, :bol
         rule %r(//(\\.|.)*?\n), Comment::Single, :bol
         mixin :inline_whitespace
       end
 
-      state :statements do
-        rule /\s+/m, Text
+      state :expr_whitespace do
+        rule /\n+/m, Text, :expr_bol
+        mixin :whitespace
+      end
 
+      state :statements do
+        mixin :whitespace
         rule /L?"/, Str, :string
         rule %r(L?'(\\.|\\[0-7]{1,3}|\\x[a-f0-9]{1,2}|[^\\'\n])')i, Str::Char
         rule %r((\d+\.\d*|\.\d+|\d+)[e][+-]?\d+[lu]*)i, Num::Float
@@ -83,6 +111,8 @@ module Rouge
             token Keyword::Type
           elsif self.class.reserved.include? name
             token Keyword::Reserved
+          elsif self.class.builtins.include? name
+            token Name::Builtin
           else
             token Name
           end
@@ -95,7 +125,7 @@ module Rouge
       end
 
       state :root do
-        mixin :whitespace
+        mixin :expr_whitespace
 
         # functions
         rule %r(
@@ -105,10 +135,10 @@ module Rouge
           (#{ws})({)         # open brace
         )mx do |m|
           # TODO: do this better.
-          delegate C, m[1]
+          recurse m[1]
           token Name::Function, m[2]
-          delegate C, m[3]
-          delegate C, m[4]
+          recurse m[3]
+          recurse m[4]
           token Punctuation, m[5]
           push :function
         end
@@ -121,10 +151,10 @@ module Rouge
           (#{ws})(;)       # semicolon
         )mx do |m|
           # TODO: do this better.
-          delegate C, m[1]
+          recurse m[1]
           token Name::Function
-          delegate C, m[3]
-          delegate C, m[4]
+          recurse m[3]
+          recurse m[4]
           token Punctuation
           push :statement
         end
@@ -134,7 +164,7 @@ module Rouge
 
       state :statement do
         rule /;/, Punctuation, :pop!
-        mixin :whitespace
+        mixin :expr_whitespace
         mixin :statements
         rule /[{}]/, Punctuation
       end
@@ -156,18 +186,19 @@ module Rouge
       end
 
       state :macro do
-        rule %r([^/\n]+), Comment::Preproc
-        rule %r(/[*].*?[*]/)m, Comment::Multiline
-        rule %r(//.*$), Comment::Single
-        rule %r(/), Comment::Preproc
-        rule /(?<=\\)\n/, Comment::Preproc
+        # NB: pop! goes back to :bol
         rule /\n/, Comment::Preproc, :pop!
+        rule %r([^/\n\\]+), Comment::Preproc
+        rule /\\./m, Comment::Preproc
+        mixin :inline_whitespace
+        rule %r(/), Comment::Preproc
       end
 
       state :if_0 do
-        rule /^\s*#if.*?(?<!\\)\n/, Comment, :if_0
-        rule /^\s*#el(?:se|if).*\n/, Comment::Preproc, :pop!
-        rule /^\s*#endif.*?(?<!\\)\n/, Comment, :pop!
+        # NB: no \b here, to cover #ifdef and #ifndef
+        rule /^\s*#if/, Comment, :if_0
+        rule /^\s*#\s*el(?:se|if)/, Comment, :pop!
+        rule /^\s*#\s*endif\b.*?(?<!\\)\n/m, Comment, :pop!
         rule /.*?\n/, Comment
       end
     end

data/lib/rouge/lexers/cpp.rb

@@ -1,6 +1,8 @@
+require Pathname.new(__FILE__).dirname.join('c.rb')
+
 module Rouge
   module Lexers
-    class Cpp < RegexLexer
+    class Cpp < C
       desc "The C++ programming language"
 
       tag 'cpp'
@@ -13,110 +15,30 @@ module Rouge
       mimetypes 'text/x-c++hdr', 'text/x-c++src'
 
       def self.keywords
-        @keywords ||= Set.new %w(
-          asm auto break case catch const const_cast continue
-          default delete do dynamic_cast else enum explicit export
-          extern for friend goto if mutable namespace new operator
-          private protected public register reinterpret_cast return
-          restrict sizeof static static_cast struct switch template
-          this throw throws try typedef typeid typename union using
-          volatile virtual while
-        )
-      end
-
-      def self.keywords_type
-        @keywords_type ||= Set.new %w(
-          bool int long float short double char unsigned signed void wchar_t
-        )
+        @keywords ||= super + Set.new(%w(
+          asm catch const_cast delete dynamic_cast explicit export
+          friend mutable namespace new operator private protected public
+          reinterpret_cast restrict static_cast template this throw
+          throws typeid typename using virtual
+        ))
       end
 
       def self.reserved
-        @reserved ||= Set.new %w(
-          __asm __int8 __based __except __int16 __stdcall __cdecl
-          __fastcall __int32 __declspec __finally __int64 __try
-          __leave __wchar_t __w64 __virtual_inheritance __uuidof
-          __unaligned __super __single_inheritance __raise __noop
-          __multiple_inheritance __m128i __m128d __m128 __m64 __interface
-          __identifier __forceinline __event __assume
-          inline _inline __inline
-          naked _naked __naked
-          thread _thread __thread
-        )
+        @reserved ||= super + Set.new(%w(
+          __virtual_inheritance __uuidof __super __single_inheritance
+          __multiple_inheritance __interface __event
+        ))
       end
 
-      # optional comments or whitespace
-      ws = %r((?:\s|//.*?\n|/[*].*?[*]/)+)
       id = /[a-zA-Z_][a-zA-Z0-9]*/
 
-      start { push :bol }
-
-      state :bol do
-        mixin :inline_whitespace
-        rule /#if\s+0/ do
-          token Comment::Preproc
-          goto :if_0
-        end
-
-        rule /#/ do
-          token Comment::Preproc
-          goto :macro
-        end
-
-        rule(//) { pop! }
-      end
-
-      state :inline_whitespace do
-        rule /[ \t\r]+/, Text
-        rule /\\\n/, Text
-        rule %r(/(\\\n)?[*].*?[*](\\\n)?/)m, Comment::Multiline
-      end
-
-      state :whitespace do
-        mixin :inline_whitespace
-        rule %r(/(\\\n)?/(\n|(.|\n)*?[^\\]\n)), Comment::Single, :bol
-        rule /\n/, Text, :bol
-      end
-
-      state :multiline_comment do
-        rule %r([*](\\\n)?/), Comment::Multiline, :pop!
-        rule %r([*]), Comment::Multiline
-        rule %r([^*]+), Comment::Multiline
-      end
-
-      state :root do
-        mixin :whitespace
-
-        rule /L?"/, Str, :string
-        rule %r(L?'(\\.|\\[0-7]{1,3}|\\x[a-f0-9]{1,2}|[^\\'\n])')i, Str::Char
-        rule %r((\d+\.\d*|\.\d+|\d+)[e][+-]?\d+[lu]*)i, Num::Float
-        rule /0x[0-9a-f]+[lu]*/i, Num::Hex
-        rule /0[0-7]+[lu]*/i, Num::Oct
-        rule /\d+[lu]*/i, Num::Integer
-        rule %r(\*/), Error
-        rule %r([~!%^&*+=\|?:<>/-]), Operator
-        rule /[()\[\],.;{}]/, Punctuation
-
-        rule /class\b/, Keyword, :classname
-
+      prepend :root do
         # Offload C++ extensions, http://offload.codeplay.com/
         rule /(?:__offload|__blockingoffload|__outer)\b/, Keyword::Pseudo
+      end
 
-        rule /(true|false)\b/, Keyword::Constant
-        rule /NULL\b/, Name::Builtin
-        rule /#{id}:(?!:)/, Name::Label
-        rule id do |m|
-          name = m[0]
-
-          if self.class.keywords.include? name
-            token Keyword
-          elsif self.class.keywords_type.include? name
-            token Keyword::Type
-          elsif self.class.reserved.include? name
-            token Keyword::Reserved
-          else
-            token Name
-          end
-        end
+      prepend :statements do
+        rule /class\b/, Keyword, :classname
       end
 
       state :classname do
@@ -126,33 +48,6 @@ module Rouge
         rule /\s*(?=>)/m, Text, :pop!
         mixin :whitespace
       end
-
-      state :string do
-        rule /"/, Str, :pop!
-        rule /\\([\\abfnrtv"']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})/, Str::Escape
-        rule /[^\\"\n]+/, Str
-        rule /\\\n/, Str
-        rule /\\/, Str # stray backslash
-      end
-
-      state :macro do
-        rule %r([^/\n]+), Comment::Preproc
-        rule %r(/[*].*?[*]/)m, Comment::Multiline
-        rule %r(//.*$), Comment::Single
-        rule %r(/), Comment::Preproc
-        rule /(?<=\\)\n/, Comment::Preproc
-        rule /\n/ do
-          token Comment::Preproc
-          goto :bol
-        end
-      end
-
-      state :if_0 do
-        rule /^\s*#if.*?(?<!\\)\n/, Comment::Preproc, :if_0
-        rule /^\s*#el(?:se|if).*\n/, Comment::Preproc, :pop!
-        rule /^\s*#endif.*?(?<!\\)\n/, Comment::Preproc, :pop!
-        rule /.*?\n/, Comment
-      end
     end
   end
 end

data/lib/rouge/lexers/objective_c.rb

@@ -0,0 +1,186 @@
+require Pathname.new(__FILE__).dirname.join('c.rb')
+
+module Rouge
+  module Lexers
+    class ObjectiveC < C
+      desc 'objective_c'
+      tag 'objective_c'
+      aliases 'objc'
+      filenames '*.m', '*.h'
+
+      mimetypes 'text/x-objective_c', 'application/x-objective_c'
+
+      def self.at_keywords
+        @at_keywords ||= %w(
+          selector private protected public encode synchronized try
+          throw catch finally end property synthesize dynamic optional
+          interface implementation
+        )
+      end
+
+      def self.at_builtins
+        @at_builtins ||= %w(true false YES NO)
+      end
+
+      def self.builtins
+        @builtins ||= %w(YES NO nil)
+      end
+
+      def self.analyze_text(text)
+        return 1 if text =~ /@(end|implementation|protocol)\b/
+
+        id = /[a-z$_][a-z0-9$_]*/i
+        return 0.4 if text =~ %r(
+          \[ \s* #{id} \s+
+          (?:
+            #{id} \s* \]
+            | #{id}? :
+          )
+        )x
+        return 0.4 if text.include? '@"'
+      end
+
+      id = /[a-z$_][a-z0-9$_]*/i
+
+      prepend :statements do
+        rule /@"/, Str, :string
+        rule /@'(\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|\\.|[^\\'\n]')/
+          Str::Char
+        rule /@(\d+[.]\d*|[.]\d+|\d+)e[+-]?\d+l?/i,
+          Num::Float
+        rule /@(\d+[.]\d*|[.]\d+|\d+f)f?/i, Num::Float
+        rule /@0x\h+[lL]?/, Num::Hex
+        rule /@0[0-7]+l?/i, Num::Oct
+        rule /@\d+l?/, Num::Integer
+        rule /\bin\b/, Keyword
+
+        rule /@(?:interface|implementation)\b/ do
+          token Keyword
+          goto :classname
+        end
+
+        rule /@(?:class|protocol)\b/ do
+          token Keyword
+          goto :forward_classname
+        end
+
+        rule /@([[:alnum:]]+)/ do |m|
+          if self.class.at_keywords.include? m[1]
+            token Keyword
+          elsif self.class.at_builtins.include? m[1]
+            token Name::Builtin
+          else
+            token Error
+          end
+        end
+
+        rule /[?]/, Punctuation, :ternary
+        rule /\[/,  Punctuation, :message
+      end
+
+      state :ternary do
+        rule /:/, Punctuation, :pop!
+        mixin :statements
+      end
+
+      state :message_shared do
+        rule /\]/, Punctuation, :pop!
+        rule /;/, Error
+
+        mixin :statement
+      end
+
+      state :message do
+        rule /(#{id})(\s*)(:)/ do
+          groups(Name::Function, Text, Punctuation)
+          goto :message_with_args
+        end
+
+        rule /(#{id})(\s*)(\])/ do
+          groups(Name::Function, Text, Punctuation)
+          pop!
+        end
+
+        mixin :message_shared
+      end
+
+      state :message_with_args do
+        rule /(#{id})(\s*)(:)/ do
+          groups(Name::Function, Text, Punctuation)
+        end
+
+        mixin :message_shared
+      end
+
+      state :classname do
+        mixin :whitespace
+
+        rule /(#{id})(\s*)(:)(\s*)(#{id})/ do
+          groups(Name::Class, Text,
+                 Punctuation, Text,
+                 Name::Class)
+          pop!
+        end
+
+        rule /(#{id})(\s*)([(])(\s*)(#{id})(\s*)([)])/ do
+          groups(Name::Class, Text,
+                 Punctuation, Text,
+                 Name::Label, Text,
+                 Punctuation)
+        end
+
+        rule id, Name::Class, :pop!
+      end
+
+      state :forward_classname do
+        rule /(#{id})(\s*)(,)(\s*)/ do
+          groups(Name::Class, Text, Punctuation, Text)
+          push
+        end
+
+        rule /(#{id})(\s*)(;?)/ do
+          groups(Name::Class, Text, Punctuation)
+          pop!
+        end
+      end
+
+      prepend :root do
+        rule %r(
+          ([-+])(\s*)
+          ([(].*?[)])?(\s*)
+          (?=#{id}:?)
+        )ix do |m|
+          token Keyword, m[1]; token Text, m[2]
+          recurse m[3]; token Text, m[4]
+          push :method_definition
+        end
+      end
+
+      state :method_definition do
+        rule /,/, Punctuation
+        rule /[.][.][.]/, Punctuation
+        rule /([(].*?[)])(#{id})/ do |m|
+          recurse m[1]; token Name::Variable, m[2]
+        end
+
+        rule /(#{id})(\s*)(:)/m do
+          groups(Name::Function, Text, Punctuation)
+        end
+
+        rule /;/, Punctuation, :pop!
+
+        rule /{/ do
+          token Punctuation
+          goto :function
+        end
+
+        mixin :inline_whitespace
+        rule %r(//.*?\n), Comment::Single
+        rule /\s+/m, Text
+
+        rule(//) { pop! }
+      end
+    end
+  end
+end
+

data/lib/rouge/lexers/sass.rb

@@ -2,7 +2,7 @@ require 'rouge/lexers/sass/common'
 
 module Rouge
   module Lexers
-    class Sass < RegexLexer
+    class Sass < SassCommon
       include Indentation
 
       desc 'The Sass stylesheet language language (sass-lang.com)'
@@ -66,8 +66,6 @@ module Rouge
       state :end_section do
         rule(/\n/) { token Text; reset_stack }
       end
-
-      include SassCommon
     end
   end
 end

data/lib/rouge/lexers/sass/common.rb

@@ -1,180 +1,171 @@
 module Rouge
   module Lexers
     # shared states with SCSS
-    # TODO: make this less nasty to do
-    module SassCommon
-      include Token::Tokens
+    class SassCommon < RegexLexer
+      id = /[\w-]+/
 
-      def self.included(klass)
-        klass.class_eval(&INCLUDED_PROC)
-      end
-
-      INCLUDED_PROC = proc do
-        id = /[\w-]+/
-
-        state :content_common do
-          rule /@for\b/, Keyword, :for
-          rule /@(debug|warn|if|while)/, Keyword, :value
-          rule /(@mixin)(\s+)(#{id})/ do
-            group Keyword; group Text; group Name::Function
-            push :value
-          end
+      state :content_common do
+        rule /@for\b/, Keyword, :for
+        rule /@(debug|warn|if|while)/, Keyword, :value
+        rule /(@mixin)(\s+)(#{id})/ do
+          group Keyword; group Text; group Name::Function
+          push :value
+        end
 
-          rule /@extend\b/, Keyword, :selector
+        rule /@extend\b/, Keyword, :selector
 
-          rule /(@include)(\s+)(#{id})/ do
-            group Keyword; group Text; group Name::Decorator
-            push :value
-          end
+        rule /(@include)(\s+)(#{id})/ do
+          group Keyword; group Text; group Name::Decorator
+          push :value
+        end
 
-          rule /@#{id}/, Keyword, :selector
+        rule /@#{id}/, Keyword, :selector
 
-          # $variable: assignment
-          rule /([$]#{id})([ \t]*)(:)/ do
-            group Name::Variable; group Text; group Punctuation
-            push :value
-          end
+        # $variable: assignment
+        rule /([$]#{id})([ \t]*)(:)/ do
+          group Name::Variable; group Text; group Punctuation
+          push :value
         end
+      end
 
-        state :value do
-          mixin :end_section
-          rule /[ \t]+/, Text
-          rule /[$]#{id}/, Name::Variable
-          rule /url[(]/, Str::Other, :string_url
-          rule /#{id}(?=\s*[(])/, Name::Function
-          rule /%#{id}/, Name::Decorator
-
-          # named literals
-          rule /(true|false)\b/, Name::Builtin::Pseudo
-          rule /(and|or|not)\b/, Operator::Word
-
-          # colors and numbers
-          rule /#[a-z0-9]{1,6}/i, Num::Hex
-          rule /-?\d+(%|[a-z]+)?/, Num
-          rule /-?\d*\.\d+(%|[a-z]+)?/, Num::Integer
-
-          mixin :has_strings
-          mixin :has_interp
-
-          rule /[~^*!&%<>\|+=@:,.\/?-]+/, Operator
-          rule /[\[\]()]+/, Punctuation
-          rule %r(/[*]), Comment::Multiline, :inline_comment
-          rule %r(//[^\n]*), Comment::Single
-
-          # identifiers
-          rule(id) do |m|
-            if CSS.builtins.include? m[0]
-              token Name::Builtin
-            elsif CSS.constants.include? m[0]
-              token Name::Constant
-            else
-              token Name
-            end
+      state :value do
+        mixin :end_section
+        rule /[ \t]+/, Text
+        rule /[$]#{id}/, Name::Variable
+        rule /url[(]/, Str::Other, :string_url
+        rule /#{id}(?=\s*[(])/, Name::Function
+        rule /%#{id}/, Name::Decorator
+
+        # named literals
+        rule /(true|false)\b/, Name::Builtin::Pseudo
+        rule /(and|or|not)\b/, Operator::Word
+
+        # colors and numbers
+        rule /#[a-z0-9]{1,6}/i, Num::Hex
+        rule /-?\d+(%|[a-z]+)?/, Num
+        rule /-?\d*\.\d+(%|[a-z]+)?/, Num::Integer
+
+        mixin :has_strings
+        mixin :has_interp
+
+        rule /[~^*!&%<>\|+=@:,.\/?-]+/, Operator
+        rule /[\[\]()]+/, Punctuation
+        rule %r(/[*]), Comment::Multiline, :inline_comment
+        rule %r(//[^\n]*), Comment::Single
+
+        # identifiers
+        rule(id) do |m|
+          if CSS.builtins.include? m[0]
+            token Name::Builtin
+          elsif CSS.constants.include? m[0]
+            token Name::Constant
+          else
+            token Name
           end
         end
+      end
 
-        state :has_interp do
-          rule /[#][{]/, Str::Interpol, :interpolation
-        end
+      state :has_interp do
+        rule /[#][{]/, Str::Interpol, :interpolation
+      end
 
-        state :has_strings do
-          rule /"/, Str::Double, :dq
-          rule /'/, Str::Single, :sq
-        end
+      state :has_strings do
+        rule /"/, Str::Double, :dq
+        rule /'/, Str::Single, :sq
+      end
 
-        state :interpolation do
-          rule /}/, Str::Interpol, :pop!
-          mixin :value
-        end
+      state :interpolation do
+        rule /}/, Str::Interpol, :pop!
+        mixin :value
+      end
 
-        state :selector do
-          mixin :end_section
-
-          mixin :has_strings
-          mixin :has_interp
-          rule /[ \t]+/, Text
-          rule /:/, Name::Decorator, :pseudo_class
-          rule /[.]/, Name::Class, :class
-          rule /#/, Name::Namespace, :id
-          rule /%/, Name::Variable, :placeholder
-          rule id, Name::Tag
-          rule /&/, Keyword
-          rule /[~^*!&\[\]()<>\|+=@:;,.\/?-]/, Operator
-        end
+      state :selector do
+        mixin :end_section
+
+        mixin :has_strings
+        mixin :has_interp
+        rule /[ \t]+/, Text
+        rule /:/, Name::Decorator, :pseudo_class
+        rule /[.]/, Name::Class, :class
+        rule /#/, Name::Namespace, :id
+        rule /%/, Name::Variable, :placeholder
+        rule id, Name::Tag
+        rule /&/, Keyword
+        rule /[~^*!&\[\]()<>\|+=@:;,.\/?-]/, Operator
+      end
 
-        state :dq do
-          rule /"/, Str::Double, :pop!
-          mixin :has_interp
-          rule /(\\.|#(?![{])|[^\n"#])+/, Str::Double
-        end
+      state :dq do
+        rule /"/, Str::Double, :pop!
+        mixin :has_interp
+        rule /(\\.|#(?![{])|[^\n"#])+/, Str::Double
+      end
 
-        state :sq do
-          rule /'/, Str::Single, :pop!
-          mixin :has_interp
-          rule /(\\.|#(?![{])|[^\n'#])+/, Str::Single
-        end
+      state :sq do
+        rule /'/, Str::Single, :pop!
+        mixin :has_interp
+        rule /(\\.|#(?![{])|[^\n'#])+/, Str::Single
+      end
 
-        state :string_url do
-          rule /[)]/, Str::Other, :pop!
-          rule /(\\.|#(?![{])|[^\n)#])+/, Str::Other
-          mixin :has_interp
-        end
+      state :string_url do
+        rule /[)]/, Str::Other, :pop!
+        rule /(\\.|#(?![{])|[^\n)#])+/, Str::Other
+        mixin :has_interp
+      end
 
-        state :selector_piece do
-          mixin :has_interp
-          rule(//) { pop! }
-        end
+      state :selector_piece do
+        mixin :has_interp
+        rule(//) { pop! }
+      end
 
-        state :pseudo_class do
-          rule id, Name::Decorator
-          mixin :selector_piece
-        end
+      state :pseudo_class do
+        rule id, Name::Decorator
+        mixin :selector_piece
+      end
 
-        state :class do
-          rule id, Name::Class
-          mixin :selector_piece
-        end
+      state :class do
+        rule id, Name::Class
+        mixin :selector_piece
+      end
 
-        state :id do
-          rule id, Name::Namespace
-          mixin :selector_piece
-        end
+      state :id do
+        rule id, Name::Namespace
+        mixin :selector_piece
+      end
 
-        state :placeholder do
-          rule id, Name::Variable
-          mixin :selector_piece
-        end
+      state :placeholder do
+        rule id, Name::Variable
+        mixin :selector_piece
+      end
 
-        state :for do
-          rule /(from|to|through)/, Operator::Word
-          mixin :value
-        end
+      state :for do
+        rule /(from|to|through)/, Operator::Word
+        mixin :value
+      end
 
-        state :attr_common do
-          mixin :has_interp
-          rule id do |m|
-            if CSS.attributes.include? m[0]
-              token Name::Label
-            else
-              token Name::Attribute
-            end
+      state :attr_common do
+        mixin :has_interp
+        rule id do |m|
+          if CSS.attributes.include? m[0]
+            token Name::Label
+          else
+            token Name::Attribute
           end
         end
+      end
 
-        state :attribute do
-          mixin :attr_common
+      state :attribute do
+        mixin :attr_common
 
-          rule /([ \t]*)(:)/ do
-            group Text; group Punctuation
-            push :value
-          end
+        rule /([ \t]*)(:)/ do
+          group Text; group Punctuation
+          push :value
         end
+      end
 
-        state :inline_comment do
-          rule /(\\#|#(?=[^\n{])|\*(?=[^\n\/])|[^\n#*])+/, Comment::Multiline
-          mixin :has_interp
-          rule %r([*]/), Comment::Multiline, :pop!
-        end
+      state :inline_comment do
+        rule /(\\#|#(?=[^\n{])|\*(?=[^\n\/])|[^\n#*])+/, Comment::Multiline
+        mixin :has_interp
+        rule %r([*]/), Comment::Multiline, :pop!
       end
     end
   end

data/lib/rouge/lexers/scss.rb

@@ -2,7 +2,7 @@ require 'rouge/lexers/sass/common'
 
 module Rouge
   module Lexers
-    class Scss < RegexLexer
+    class Scss < SassCommon
       desc "SCSS stylesheets (sass-lang.com)"
       tag 'scss'
       filenames '*.scss'
@@ -26,8 +26,6 @@ module Rouge
         rule /\n/, Text
         rule(/[;{}]/) { token Punctuation; reset_stack }
       end
-
-      include SassCommon
     end
   end
 end

data/lib/rouge/lexers/yaml.rb

@@ -58,12 +58,12 @@ module Rouge
         @next_indent += match.size
       end
 
-      def set_indent(opts={})
+      def set_indent(match, opts={})
         if indent < @next_indent
           @indent_stack << @next_indent
         end
 
-        @next_indent += @last_match[0].size unless opts[:implicit]
+        @next_indent += match.size unless opts[:implicit]
       end
 
       plain_scalar_start = /[^ \t\n\r\f\v?:,\[\]{}#&*!\|>'"%@`]/
@@ -119,7 +119,7 @@ module Rouge
         end
 
         # block collection indicators
-        rule(/[?:-](?=[ ]|$)/) { token Punctuation::Indicator; set_indent }
+        rule(/[?:-](?=[ ]|$)/) { |m| token Punctuation::Indicator; set_indent m[0] }
 
         # the beginning of a block line
         rule(/[ ]*/) { |m| token Text; continue_indent(m[0]); pop! }
@@ -163,9 +163,9 @@ module Rouge
 
       state :block_nodes do
         # implicit key
-        rule /:(?=\s|$)/ do
+        rule /:(?=\s|$)/ do |m|
           token Punctuation::Indicator
-          set_indent :implicit => true
+          set_indent m[0], :implicit => true
         end
 
         # literal and folded scalars

data/lib/rouge/regex_lexer.rb

@@ -35,35 +35,50 @@ module Rouge
     #
     # @see RegexLexer.state
     class State
-      attr_reader :name
-      def initialize(name, &defn)
+      attr_reader :name, :rules
+      def initialize(name, rules)
         @name = name
-        @defn = defn
+        @rules = rules
       end
 
-      def rules
-        @rules ||= []
+      def inspect
+        "#<#{self.class.name} #{@name.inspect}>"
       end
+    end
 
-      def load!(lexer_class)
-        return self if @loaded
-        @loaded = true
-        StateDSL.new(rules).instance_eval(&@defn)
+    class StateDSL
+      attr_reader :rules
+      def initialize(name, &defn)
+        @name = name
+        @defn = defn
+        @rules = []
+      end
 
-        rules.map! do |rule|
+      def to_state(lexer_class)
+        load!
+        rules = @rules.map do |rule|
           rule.is_a?(String) ? lexer_class.get_state(rule) : rule
         end
+        State.new(@name, rules)
+      end
 
-        self
+      def prepended(&defn)
+        parent_defn = @defn
+        StateDSL.new(@name) do
+          instance_eval(&defn)
+          instance_eval(&parent_defn)
+        end
       end
-    end
 
-    class StateDSL
-      attr_reader :rules
-      def initialize(rules)
-        @rules = rules
+      def appended(&defn)
+        parent_defn = @defn
+        StateDSL.new(@name) do
+          instance_eval(&parent_defn)
+          instance_eval(&defn)
+        end
       end
 
+    protected
       # Define a new rule for this state.
       #
       # @overload rule(re, token, next_state=nil)
@@ -99,8 +114,15 @@ module Rouge
       # Mix in the rules from another state into this state.  The rules
       # from the mixed-in state will be tried in order before moving on
       # to the rest of the rules in this state.
-      def mixin(lexer_name)
-        rules << lexer_name.to_s
+      def mixin(state)
+        rules << state.to_s
+      end
+
+    private
+      def load!
+        return if @loaded
+        @loaded = true
+        instance_eval(&@defn)
       end
     end
 
@@ -110,6 +132,16 @@ module Rouge
       @states ||= {}
     end
 
+    def self.state_definitions
+      @state_definitions ||= InheritableHash.new(superclass.state_definitions)
+    end
+    @state_definitions = {}
+
+    def self.replace_state(name, new_defn)
+      states[name] = nil
+      state_definitions[name] = new_defn
+    end
+
     # The routines to run at the beginning of a fresh lex.
     # @see start
     def self.start_procs
@@ -129,16 +161,31 @@ module Rouge
     # The block will be evaluated in the context of a {StateDSL}.
     def self.state(name, &b)
       name = name.to_s
-      states[name] = State.new(name, &b)
+      state_definitions[name] = StateDSL.new(name, &b)
+    end
+
+    def self.prepend(name, &b)
+      name = name.to_s
+      dsl = state_definitions[name] or raise "no such state #{name.inspect}"
+      replace_state(name, dsl.prepended(&b))
+    end
+
+    def self.append(state, &b)
+      name = name.to_s
+      dsl = state_definitions[name] or raise "no such state #{name.inspect}"
+      replace_state(name, dsl.appended(&b))
     end
 
     # @private
     def self.get_state(name)
       return name if name.is_a? State
 
-      state = states[name.to_s]
-      raise "unknown state: #{name}" unless state
-      state.load!(self)
+      name = name.to_s
+
+      states[name] ||= begin
+        defn = state_definitions[name] or raise "unknown state: #{name.inspect}"
+        defn.to_state(self)
+      end
     end
 
     # @private
@@ -306,10 +353,14 @@ module Rouge
 
       lexer.lex(text, :continue => true) do |tok, val|
         debug { "    delegated token: #{tok.inspect}, #{val.inspect}" }
-        token(tok, val)
+        yield_token(tok, val)
       end
     end
 
+    def recurse(text=nil)
+      delegate(self.class, text)
+    end
+
     # Push a state onto the stack.  If no state name is given and you've
     # passed a block, a state will be dynamically created using the
     # {StateDSL}.

data/lib/rouge/version.rb

@@ -1,5 +1,5 @@
 module Rouge
   def self.version
-    "0.5.1"
+    "0.5.2"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rouge
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-15 00:00:00.000000000 Z
+date: 2013-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   type: :runtime
@@ -93,6 +93,7 @@ files:
 - lib/rouge/lexers/erlang.rb
 - lib/rouge/lexers/yaml.rb
 - lib/rouge/lexers/make.rb
+- lib/rouge/lexers/objective_c.rb
 - lib/rouge/lexers/lua.rb
 - lib/rouge/lexers/viml.rb
 - lib/rouge/lexers/java.rb
@@ -168,6 +169,7 @@ files:
 - lib/rouge/demos/erb
 - lib/rouge/demos/cpp
 - lib/rouge/demos/coffeescript
+- lib/rouge/demos/objective_c
 - lib/rouge/demos/sql
 - lib/rouge/demos/puppet
 - lib/rouge/demos/ini