rouge 0.4.0 → 0.5.0

data/lib/rouge/lexers/common_lisp.rb

@@ -214,65 +214,65 @@ module Rouge
       symbol = /(\|[^\|]+\||#{nonmacro}#{constituent}*)/
 
       state :root do
-        rule /\s+/m, 'Text'
-        rule /;.*$/, 'Comment.Single'
-        rule /#\|/, 'Comment.Multiline', :multiline_comment
+        rule /\s+/m, Text
+        rule /;.*$/, Comment::Single
+        rule /#\|/, Comment::Multiline, :multiline_comment
 
         # encoding comment
-        rule /#\d*Y.*$/, 'Comment.Special'
-        rule /"(\\.|[^"\\])*"/, 'Literal.String'
+        rule /#\d*Y.*$/, Comment::Special
+        rule /"(\\.|[^"\\])*"/, Str
 
-        rule /[:']#{symbol}/, 'Literal.String.Symbol'
-        rule /['`]/, 'Operator'
+        rule /[:']#{symbol}/, Str::Symbol
+        rule /['`]/, Operator
 
         # numbers
-        rule /[-+]?\d+\.?#{terminated}/, 'Literal.Number.Integer'
-        rule %r([-+]?\d+/\d+#{terminated}), 'Literal.Number.Integer'
+        rule /[-+]?\d+\.?#{terminated}/, Num::Integer
+        rule %r([-+]?\d+/\d+#{terminated}), Num::Integer
         rule %r(
           [-+]?
           (\d*\.\d+([defls][-+]?\d+)?
           |\d+(\.\d*)?[defls][-+]?\d+)
           #{terminated}
-        )x, 'Literal.Number.Float'
+        )x, Num::Float
 
         # sharpsign strings and characters
-        rule /#\\.#{terminated}/, 'Literal.String.Char'
-        rule /#\\#{symbol}/, 'Literal.String.Char'
+        rule /#\\.#{terminated}/, Str::Char
+        rule /#\\#{symbol}/, Str::Char
 
-        rule /#\(/, 'Operator', :root
+        rule /#\(/, Operator, :root
 
         # bitstring
-        rule /#\d*\*[01]*/, 'Literal.Other'
+        rule /#\d*\*[01]*/, Other
 
         # uninterned symbol
-        rule /#:#{symbol}/, 'Literal.String.Symbol'
+        rule /#:#{symbol}/, Str::Symbol
 
         # read-time and load-time evaluation
-        rule /#[.,]/, 'Operator'
+        rule /#[.,]/, Operator
 
         # function shorthand
-        rule /#'/, 'Name.Function'
+        rule /#'/, Name::Function
 
         # binary rational
-        rule /#b[+-]?[01]+(\/[01]+)?/i, 'Literal.Number'
+        rule /#b[+-]?[01]+(\/[01]+)?/i, Num
 
         # octal rational
-        rule /#o[+-]?[0-7]+(\/[0-7]+)?/i, 'Literal.Number.Oct'
+        rule /#o[+-]?[0-7]+(\/[0-7]+)?/i, Num::Oct
 
         # hex rational
-        rule /#x[+-]?[0-9a-f]+(\/[0-9a-f]+)?/i, 'Literal.Number'
+        rule /#x[+-]?[0-9a-f]+(\/[0-9a-f]+)?/i, Num
 
         # complex
         rule /(#c)(\()/i do
-          group 'Literal.Number'
-          group 'Punctuation'
+          group Num
+          group Punctuation
           push :root
         end
 
         # arrays and structures
         rule /(#(?:\d+a|s))(\()/i do
-          group 'Literal.Other'
-          group 'Punctuation'
+          group Literal::Other
+          group Punctuation
           push :root
         end
 
@@ -280,66 +280,67 @@ module Rouge
         rule /#p?"(\\.|[^"])*"/i
 
         # reference
-        rule /#\d+[=#]/, 'Operator'
+        rule /#\d+[=#]/, Operator
 
         # read-time comment
-        rule /#+nil#{terminated}\s*\(/, 'Comment.Preproc', :commented_form
+        rule /#+nil#{terminated}\s*\(/, Comment, :commented_form
 
         # read-time conditional
-        rule /#[+-]/, 'Operator'
+        rule /#[+-]/, Operator
 
         # special operators that should have been parsed already
-        rule /(,@|,|\.)/, 'Operator'
+        rule /(,@|,|\.)/, Operator
 
         # special constants
-        rule /(t|nil)#{terminated}/, 'Name.Constant'
+        rule /(t|nil)#{terminated}/, Name::Constant
 
         # functions and variables
         # note that these get filtered through in stream_tokens
-        rule /\*#{symbol}\*/, 'Name.Variable.Global'
+        rule /\*#{symbol}\*/, Name::Variable::Global
         rule symbol do |m|
           sym = m[0]
 
           if BUILTIN_FUNCTIONS.include? sym
-            token 'Name.Builtin'
+            token Name::Builtin
           elsif SPECIAL_FORMS.include? sym
-            token 'Keyword'
+            token Keyword
           elsif MACROS.include? sym
-            token 'Name.Builtin'
+            token Name::Builtin
           elsif LAMBDA_LIST_KEYWORDS.include? sym
-            token 'Keyword'
+            token Keyword
           elsif DECLARATIONS.include? sym
-            token 'Keyword'
+            token Keyword
           elsif BUILTIN_TYPES.include? sym
-            token 'Keyword.Type'
+            token Keyword::Type
           elsif BUILTIN_CLASSES.include? sym
-            token 'Name.Class'
+            token Name::Class
           else
-            token 'Name.Variable'
+            token Name::Variable
           end
         end
 
-        rule /\(/, 'Punctuation', :root
-        rule /\)/, 'Punctuation' do
+        rule /\(/, Punctuation, :root
+        rule /\)/, Punctuation do
           if stack.empty?
-            token 'Error'
+            token Error
           else
-            token 'Punctuation'
+            token Punctuation
             pop!
           end
         end
       end
 
       state :multiline_comment do
-        rule /#\|/, 'Comment.Multiline', :multiline_comment
-        rule /\|#/, 'Comment.Multiline', :pop!
-        rule /[^\|#]+/, 'Comment.Multiline'
-        rule /[\|#]/, 'Comment.Multiline'
+        rule /#\|/, Comment::Multiline, :multiline_comment
+        rule /\|#/, Comment::Multiline, :pop!
+        rule /[^\|#]+/, Comment::Multiline
+        rule /[\|#]/, Comment::Multiline
       end
 
       state :commented_form do
-        rule /\(/, 'Comment.Preproc', :commented_form
-        rule /\)/, 'Comment.Preproc', :pop!
+        rule /\(/, Comment, :commented_form
+        rule /\)/, Comment, :pop!
+        rule /[^()]+/, Comment
       end
     end
   end

data/lib/rouge/lexers/conf.rb

@@ -9,12 +9,12 @@ module Rouge
 
       # short and sweet
       state :root do
-        rule /#.*?\n/, 'Comment'
-        rule /".*?"/, 'Literal.String.Double'
-        rule /'.*?'/, 'Literal.String.Single'
-        rule /[a-z]\w*/i, 'Name'
-        rule /\d+/, 'Literal.Number'
-        rule /[^\d\w#"']+/, 'Text'
+        rule /#.*?\n/, Comment
+        rule /".*?"/, Str::Double
+        rule /'.*?'/, Str::Single
+        rule /[a-z]\w*/i, Name
+        rule /\d+/, Num
+        rule /[^\d\w#"']+/, Text
       end
     end
   end

data/lib/rouge/lexers/cpp.rb

@@ -53,105 +53,105 @@ module Rouge
       state :bol do
         mixin :inline_whitespace
         rule /#if\s+0/ do
-          token 'Comment.Preproc'
-          pop!; push :if_0
+          token Comment::Preproc
+          goto :if_0
         end
 
         rule /#/ do
-          token 'Comment.Preproc'
-          pop!; push :macro
+          token Comment::Preproc
+          goto :macro
         end
 
         rule(//) { pop! }
       end
 
       state :inline_whitespace do
-        rule /[ \t\r]+/, 'Text'
-        rule /\\\n/, 'Text'
-        rule %r(/(\\\n)?[*].*?[*](\\\n)?/)m, 'Comment.Multiline'
+        rule /[ \t\r]+/, Text
+        rule /\\\n/, Text
+        rule %r(/(\\\n)?[*].*?[*](\\\n)?/)m, Comment::Multiline
       end
 
       state :whitespace do
         mixin :inline_whitespace
-        rule %r(/(\\\n)?/(\n|(.|\n)*?[^\\]\n)), 'Comment.Single', :bol
-        rule /\n/, 'Text', :bol
+        rule %r(/(\\\n)?/(\n|(.|\n)*?[^\\]\n)), Comment::Single, :bol
+        rule /\n/, Text, :bol
       end
 
       state :multiline_comment do
-        rule %r([*](\\\n)?/), 'Comment.Multiline', :pop!
-        rule %r([*]), 'Comment.Multiline'
-        rule %r([^*]+), 'Comment.Multiline'
+        rule %r([*](\\\n)?/), Comment::Multiline, :pop!
+        rule %r([*]), Comment::Multiline
+        rule %r([^*]+), Comment::Multiline
       end
 
       state :root do
         mixin :whitespace
 
-        rule /L?"/, 'Literal.String', :string
-        rule %r(L?'(\\.|\\[0-7]{1,3}|\\x[a-f0-9]{1,2}|[^\\'\n])')i, 'Literal.String.Char'
-        rule %r((\d+\.\d*|\.\d+|\d+)[e][+-]?\d+[lu]*)i, 'Literal.Number.Float'
-        rule /0x[0-9a-f]+[lu]*/i, 'Literal.Number.Hex'
-        rule /0[0-7]+[lu]*/i, 'Literal.Number.Oct'
-        rule /\d+[lu]*/i, 'Literal.Number.Integer'
-        rule %r(\*/), 'Error'
-        rule %r([~!%^&*+=\|?:<>/-]), 'Operator'
-        rule /[()\[\],.;{}]/, 'Punctuation'
+        rule /L?"/, Str, :string
+        rule %r(L?'(\\.|\\[0-7]{1,3}|\\x[a-f0-9]{1,2}|[^\\'\n])')i, Str::Char
+        rule %r((\d+\.\d*|\.\d+|\d+)[e][+-]?\d+[lu]*)i, Num::Float
+        rule /0x[0-9a-f]+[lu]*/i, Num::Hex
+        rule /0[0-7]+[lu]*/i, Num::Oct
+        rule /\d+[lu]*/i, Num::Integer
+        rule %r(\*/), Error
+        rule %r([~!%^&*+=\|?:<>/-]), Operator
+        rule /[()\[\],.;{}]/, Punctuation
 
-        rule /class\b/, 'Keyword', :classname
+        rule /class\b/, Keyword, :classname
 
         # Offload C++ extensions, http://offload.codeplay.com/
-        rule /(?:__offload|__blockingoffload|__outer)\b/, 'Keyword.Pseudo'
+        rule /(?:__offload|__blockingoffload|__outer)\b/, Keyword::Pseudo
 
-        rule /(true|false)\b/, 'Keyword.Constant'
-        rule /NULL\b/, 'Name.Builtin'
-        rule /#{id}:(?!:)/, 'Name.Label'
+        rule /(true|false)\b/, Keyword::Constant
+        rule /NULL\b/, Name::Builtin
+        rule /#{id}:(?!:)/, Name::Label
         rule id do |m|
           name = m[0]
 
           if self.class.keywords.include? name
-            token 'Keyword'
+            token Keyword
           elsif self.class.keywords_type.include? name
-            token 'Keyword.Type'
+            token Keyword::Type
           elsif self.class.reserved.include? name
-            token 'Keyword.Reserved'
+            token Keyword::Reserved
           else
-            token 'Name'
+            token Name
           end
         end
       end
 
       state :classname do
-        rule id, 'Name.Class', :pop!
+        rule id, Name::Class, :pop!
 
         # template specification
-        rule /\s*(?=>)/m, 'Text', :pop!
+        rule /\s*(?=>)/m, Text, :pop!
         mixin :whitespace
       end
 
       state :string do
-        rule /"/, 'Literal.String', :pop!
-        rule /\\([\\abfnrtv"']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})/, 'Literal.String.Escape'
-        rule /[^\\"\n]+/, 'Literal.String'
-        rule /\\\n/, 'Literal.String'
-        rule /\\/, 'Literal.String' # stray backslash
+        rule /"/, Str, :pop!
+        rule /\\([\\abfnrtv"']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})/, Str::Escape
+        rule /[^\\"\n]+/, Str
+        rule /\\\n/, Str
+        rule /\\/, Str # stray backslash
       end
 
       state :macro do
-        rule %r([^/\n]+), 'Comment.Preproc'
-        rule %r(/[*].*?[*]/)m, 'Comment.Multiliine'
-        rule %r(//.*$), 'Comment.Single'
-        rule %r(/), 'Comment.Preproc'
-        rule /(?<=\\)\n/, 'Comment.Preproc'
+        rule %r([^/\n]+), Comment::Preproc
+        rule %r(/[*].*?[*]/)m, Comment::Multiline
+        rule %r(//.*$), Comment::Single
+        rule %r(/), Comment::Preproc
+        rule /(?<=\\)\n/, Comment::Preproc
         rule /\n/ do
-          token 'Comment.Preproc'
-          pop!; push :bol
+          token Comment::Preproc
+          goto :bol
         end
       end
 
       state :if_0 do
-        rule /^\s*#if.*?(?<!\\)\n/, 'Comment.Preproc', :if_0
-        rule /^\s*#el(?:se|if).*\n/, 'Comment.Preproc', :pop!
-        rule /^\s*#endif.*?(?<!\\)\n/, 'Comment.Preproc', :pop!
-        rule /.*?\n/, 'Comment'
+        rule /^\s*#if.*?(?<!\\)\n/, Comment::Preproc, :if_0
+        rule /^\s*#el(?:se|if).*\n/, Comment::Preproc, :pop!
+        rule /^\s*#endif.*?(?<!\\)\n/, Comment::Preproc, :pop!
+        rule /.*?\n/, Comment
       end
     end
   end

data/lib/rouge/lexers/csharp.rb

@@ -30,52 +30,52 @@ module Rouge
       )
 
       state :whitespace do
-        rule /\s+/m, 'Text'
-        rule %r(//.*?\n), 'Comment.Single'
-        rule %r(/[*].*?[*]/)m, 'Comment.Multiline'
+        rule /\s+/m, Text
+        rule %r(//.*?\n), Comment::Single
+        rule %r(/[*].*?[*]/)m, Comment::Multiline
       end
 
       state :root do
         mixin :whitespace
 
-        rule /^\s*\[.*?\]/, 'Name.Attribute'
-        rule /[$]\s*"/, 'Literal.String', :splice_string
-        rule /[$]\s*<#/, 'Literal.String', :splice_recstring
-        rule /<#/, 'Literal.String', :recstring
+        rule /^\s*\[.*?\]/, Name::Attribute
+        rule /[$]\s*"/, Str, :splice_string
+        rule /[$]\s*<#/, Str, :splice_recstring
+        rule /<#/, Str, :recstring
 
-        rule /(<\[)\s*(#{id}:)?/, 'Keyword'
-        rule /\]>/, 'Keyword'
+        rule /(<\[)\s*(#{id}:)?/, Keyword
+        rule /\]>/, Keyword
 
-        rule /[~!%^&*()+=|\[\]{}:;,.<>\/?-]/, 'Punctuation'
-        rule /@"(\\.|.)*?"/, 'Literal.String'
-        rule /"(\\.|.)*?["\n]/, 'Literal.String'
-        rule /'(\\.|.)'/, 'Literal.String.Char'
-        rule /0x[0-9a-f]+[lu]?/i, 'Literal.Number'
+        rule /[~!%^&*()+=|\[\]{}:;,.<>\/?-]/, Punctuation
+        rule /@"(\\.|.)*?"/, Str
+        rule /"(\\.|.)*?["\n]/, Str
+        rule /'(\\.|.)'/, Str::Char
+        rule /0x[0-9a-f]+[lu]?/i, Num
         rule %r(
           [0-9]
           ([.][0-9]*)? # decimal
           (e[+-][0-9]+)? # exponent
           [fldu]? # type
-        )ix, 'Literal.Number'
+        )ix, Num
         rule /^#[ \t]*(#{cpp_keywords.join('|')})\b.*?\n/,
-          'Comment.Preproc'
-        rule /\b(#{keywords.join('|')})\b/, 'Keyword'
-        rule /\b(#{keywords_type.join('|')})\b/, 'Keyword.Type'
-        rule /class|struct/, 'Keyword', :class
-        rule /namespace|using/, 'Keyword', :namespace
-        rule /#{id}(?=\s*[(])/, 'Name.Function'
-        rule id, 'Name'
+          Comment::Preproc
+        rule /\b(#{keywords.join('|')})\b/, Keyword
+        rule /\b(#{keywords_type.join('|')})\b/, Keyword::Type
+        rule /class|struct/, Keyword, :class
+        rule /namespace|using/, Keyword, :namespace
+        rule /#{id}(?=\s*[(])/, Name::Function
+        rule id, Name
       end
 
       state :class do
         mixin :whitespace
-        rule id, 'Name.Class', :pop!
+        rule id, Name::Class, :pop!
       end
 
       state :namespace do
         mixin :whitespace
-        rule /(?=[(])/, 'Text', :pop!
-        rule /(#{id}|[.])+/, 'Name.Namespace', :pop!
+        rule /(?=[(])/, Text, :pop!
+        rule /(#{id}|[.])+/, Name::Namespace, :pop!
       end
 
     end

data/lib/rouge/lexers/css.rb

@@ -181,38 +181,38 @@ module Rouge
 
       state :root do
         mixin :basics
-        rule /{/, 'Punctuation', :stanza
-        rule /:#{identifier}/, 'Name.Decorator'
-        rule /\.#{identifier}/, 'Name.Class'
-        rule /##{identifier}/, 'Name.Function'
-        rule /@#{identifier}/, 'Keyword', :at_rule
-        rule identifier, 'Name.Tag'
-        rule %r([~^*!%&\[\]()<>|+=@:;,./?-]), 'Operator'
+        rule /{/, Punctuation, :stanza
+        rule /:#{identifier}/, Name::Decorator
+        rule /\.#{identifier}/, Name::Class
+        rule /##{identifier}/, Name::Function
+        rule /@#{identifier}/, Keyword, :at_rule
+        rule identifier, Name::Tag
+        rule %r([~^*!%&\[\]()<>|+=@:;,./?-]), Operator
       end
 
       state :value do
         mixin :basics
-        rule /url\(.*?\)/, 'Literal.String.Other'
-        rule /#[0-9a-f]{1,6}/i, 'Literal.Number' # colors
-        rule /#{number}(?:em|px|%|pt|pc|in|mm|m|ex|s)?\b/, 'Literal.Number'
-        rule /[\[\]():\/.,]/, 'Punctuation'
-        rule /"(\\\\|\\"|[^"])*"/, 'Literal.String.Single'
-        rule /'(\\\\|\\'|[^'])*'/, 'Literal.String.Double'
+        rule /url\(.*?\)/, Str::Other
+        rule /#[0-9a-f]{1,6}/i, Num # colors
+        rule /#{number}(?:em|px|%|pt|pc|in|mm|m|ex|s)?\b/, Num
+        rule /[\[\]():\/.,]/, Punctuation
+        rule /"(\\\\|\\"|[^"])*"/, Str::Single
+        rule /'(\\\\|\\'|[^'])*'/, Str::Double
         rule(identifier) do |m|
           if self.class.constants.include? m[0]
-            token 'Name.Constant'
+            token Name::Constant
           elsif self.class.builtins.include? m[0]
-            token 'Name.Builtin'
+            token Name::Builtin
           else
-            token 'Name'
+            token Name
           end
         end
       end
 
       state :at_rule do
-        rule /{(?=\s*#{identifier}\s*:)/m, 'Punctuation', :at_stanza
-        rule /{/, 'Punctuation', :at_body
-        rule /;/, 'Punctuation', :pop!
+        rule /{(?=\s*#{identifier}\s*:)/m, Punctuation, :at_stanza
+        rule /{/, Punctuation, :at_body
+        rule /;/, Punctuation, :pop!
         mixin :value
       end
 
@@ -228,40 +228,39 @@ module Rouge
 
       state :at_content do
         rule /}/ do
-          token 'Punctuation'
-          pop!; pop!
+          token Punctuation
+          pop! 2
         end
       end
 
       state :basics do
-        rule /\s+/m, 'Text'
-        rule %r(/\*(?:.*?)\*/)m, 'Comment'
+        rule /\s+/m, Text
+        rule %r(/\*(?:.*?)\*/)m, Comment
       end
 
       state :stanza do
         mixin :basics
-        rule /}/, 'Punctuation', :pop!
+        rule /}/, Punctuation, :pop!
         rule /(#{identifier})(\s*)(:)/m do |m|
-          if self.class.attributes.include? m[1]
-            group 'Name.Label'
+          name_tok = if self.class.attributes.include? m[1]
+            Name::Label
           elsif self.class.vendor_prefixes.any? { |p| m[1].start_with?(p) }
-            group 'Name.Label'
+            Name::Label
           else
-            group 'Name.Property'
+            Name::Property
           end
 
-          group 'Text'
-          group 'Punctuation'
+          groups name_tok, Text, Punctuation
 
           push :stanza_value
         end
       end
 
       state :stanza_value do
-        rule /;/, 'Punctuation', :pop!
+        rule /;/, Punctuation, :pop!
         rule(/(?=})/) { pop! }
-        rule /!important\b/, 'Comment.Preproc'
-        rule /^@.*?$/, 'Comment.Preproc'
+        rule /!important\b/, Comment::Preproc
+        rule /^@.*?$/, Comment::Preproc
         mixin :value
       end
     end