rouge 0.0.2 → 0.0.3

data/lib/rouge/lexers/css.rb

@@ -0,0 +1,165 @@
+module Rouge
+  module Lexers
+    class CSSLexer < RegexLexer
+      tag 'css'
+      extensions 'css'
+
+      identifier = /[a-zA-Z0-9_-]+/
+      number = /-?(?:[0-9]+(\.[0-9]+)?|\.[0-9]+)/
+      keywords = %w(
+        azimuth background-attachment background-color
+        background-image background-position background-repeat
+        background border-bottom-color border-bottom-style
+        border-bottom-width border-left-color border-left-style
+        border-left-width border-right border-right-color
+        border-right-style border-right-width border-top-color
+        border-top-style border-top-width border-bottom
+        border-collapse border-left border-width border-color
+        border-spacing border-style border-top border caption-side
+        clear clip color content counter-increment counter-reset
+        cue-after cue-before cue cursor direction display
+        elevation empty-cells float font-family font-size
+        font-size-adjust font-stretch font-style font-variant
+        font-weight font height letter-spacing line-height
+        list-style-type list-style-image list-style-position
+        list-style margin-bottom margin-left margin-right
+        margin-top margin marker-offset marks max-height max-width
+        min-height min-width opacity orphans outline outline-color
+        outline-style outline-width overflow(?:-x -y ) padding-bottom
+        padding-left padding-right padding-top padding page
+        page-break-after page-break-before page-break-inside
+        pause-after pause-before pause pitch pitch-range
+        play-during position quotes richness right size
+        speak-header speak-numeral speak-punctuation speak
+        speech-rate stress table-layout text-align text-decoration
+        text-indent text-shadow text-transform top unicode-bidi
+        vertical-align visibility voice-family volume white-space
+        widows width word-spacing z-index bottom left
+        above absolute always armenian aural auto avoid baseline
+        behind below bidi-override blink block bold bolder both
+        capitalize center-left center-right center circle
+        cjk-ideographic close-quote collapse condensed continuous
+        crop crosshair cross cursive dashed decimal-leading-zero
+        decimal default digits disc dotted double e-resize embed
+        extra-condensed extra-expanded expanded fantasy far-left
+        far-right faster fast fixed georgian groove hebrew help
+        hidden hide higher high hiragana-iroha hiragana icon
+        inherit inline-table inline inset inside invert italic
+        justify katakana-iroha katakana landscape larger large
+        left-side leftwards level lighter line-through list-item
+        loud lower-alpha lower-greek lower-roman lowercase ltr
+        lower low medium message-box middle mix monospace
+        n-resize narrower ne-resize no-close-quote no-open-quote
+        no-repeat none normal nowrap nw-resize oblique once
+        open-quote outset outside overline pointer portrait px
+        relative repeat-x repeat-y repeat rgb ridge right-side
+        rightwards s-resize sans-serif scroll se-resize
+        semi-condensed semi-expanded separate serif show silent
+        slow slower small-caps small-caption smaller soft solid
+        spell-out square static status-bar super sw-resize
+        table-caption table-cell table-column table-column-group
+        table-footer-group table-header-group table-row
+        table-row-group text text-bottom text-top thick thin
+        transparent ultra-condensed ultra-expanded underline
+        upper-alpha upper-latin upper-roman uppercase url
+        visible w-resize wait wider x-fast x-high x-large x-loud
+        x-low x-small x-soft xx-large xx-small yes
+      ).join('|')
+
+      builtins = %w(
+        indigo gold firebrick indianred yellow darkolivegreen
+        darkseagreen mediumvioletred mediumorchid chartreuse
+        mediumslateblue black springgreen crimson lightsalmon brown
+        turquoise olivedrab cyan silver skyblue gray darkturquoise
+        goldenrod darkgreen darkviolet darkgray lightpink teal
+        darkmagenta lightgoldenrodyellow lavender yellowgreen thistle
+        violet navy orchid blue ghostwhite honeydew cornflowerblue
+        darkblue darkkhaki mediumpurple cornsilk red bisque slategray
+        darkcyan khaki wheat deepskyblue darkred steelblue aliceblue
+        gainsboro mediumturquoise floralwhite coral purple lightgrey
+        lightcyan darksalmon beige azure lightsteelblue oldlace
+        greenyellow royalblue lightseagreen mistyrose sienna
+        lightcoral orangered navajowhite lime palegreen burlywood
+        seashell mediumspringgreen fuchsia papayawhip blanchedalmond
+        peru aquamarine white darkslategray ivory dodgerblue
+        lemonchiffon chocolate orange forestgreen slateblue olive
+        mintcream antiquewhite darkorange cadetblue moccasin
+        limegreen saddlebrown darkslateblue lightskyblue deeppink
+        plum aqua darkgoldenrod maroon sandybrown magenta tan
+        rosybrown pink lightblue palevioletred mediumseagreen
+        dimgray powderblue seagreen snow mediumblue midnightblue
+        paleturquoise palegoldenrod whitesmoke darkorchid salmon
+        lightslategray lawngreen lightgreen tomato hotpink
+        lightyellow lavenderblush linen mediumaquamarine green
+        blueviolet peachpuff
+      ).join('|')
+
+      state :root do
+        mixin :basics
+        rule /{/, 'Punctuation', :stanza
+        rule /:#{identifier}/, 'Name.Decorator'
+        rule /\.#{identifier}/, 'Name.Class'
+        rule /##{identifier}/, 'Name.Function'
+        rule /@#{identifier}/, 'Keyword', :at_rule
+        rule identifier, 'Name.Tag'
+        rule %r([~^*!%&\[\]()<>|+=@:;,./?-]), 'Operator'
+      end
+
+      state :value do
+        mixin :basics
+        rule /url\(.*?\)/, 'Literal.String.Other'
+        rule /#[0-9a-f]{1,6}/i, 'Literal.Number' # colors
+        rule /#{number}(?:em|px|%|pt|pc|in|mm|m|ex|s)?\b/, 'Literal.Number'
+        rule /[\[\]():\/.]/, 'Punctuation'
+        rule /"(\\\\|\\"|[^"])*"/, 'Literal.String.Single'
+        rule /'(\\\\|\\'|[^'])*'/, 'Literal.String.Double'
+        rule identifier, 'Name'
+      end
+
+      state :at_rule do
+        rule /{(?=\s*#{identifier}\s*:)/m, 'Punctuation', :at_stanza
+        rule /{/, 'Punctuation', :at_body
+        rule /;/, 'Punctuation', :pop!
+        mixin :value
+      end
+
+      state :at_body do
+        mixin :at_content
+        mixin :root
+      end
+
+      state :at_stanza do
+        mixin :at_content
+        mixin :stanza
+      end
+
+      state :at_content do
+        rule /}/ do
+          token 'Punctuation'
+          pop!; pop!
+        end
+      end
+
+      state :basics do
+        rule /\s+/m, 'Text'
+        rule %r(/\*(?:.*?)\*/)m, 'Comment'
+      end
+
+      state :stanza do
+        mixin :basics
+        rule /}/, 'Punctuation', :pop!
+        rule /(?:#{keywords})\s*:/m, 'Keyword', :stanza_value
+        rule /(?:#{builtins})\s*:/m, 'Name.Builtin', :stanza_value
+        rule /#{identifier}\s*:/m, 'Name', :stanza_value
+      end
+
+      state :stanza_value do
+        rule /;/, 'Punctuation', :pop!
+        rule(/(?=})/) { pop! }
+        rule /!important\b/, 'Comment.Preproc'
+        rule /^@.*?$/, 'Comment.Preproc'
+        mixin :value
+      end
+    end
+  end
+end

data/lib/rouge/lexers/diff.rb

@@ -0,0 +1,28 @@
+module Rouge
+  module Lexers
+    class Diff < RegexLexer
+      tag 'diff'
+      aliases 'patch'
+      extensions 'diff', 'patch'
+
+      state :header do
+        rule /^diff .*?\n(?=---|\+\+\+)/m, 'Generic.Heading'
+        rule /^--- .*?\n/, 'Generic.Deleted'
+        rule /^\+\+\+ .*?\n/, 'Generic.Inserted'
+      end
+
+      state :diff do
+        rule /@@ -\d+,\d+ \+\d+,\d+ @@.*?\n/, 'Generic.Heading'
+        rule /^\+.*?\n/, 'Generic.Inserted'
+        rule /^-.*?\n/,  'Generic.Deleted'
+        rule /^ .*?\n/,  'Text'
+        rule /^.*?\n/,   'Error'
+      end
+
+      state :root do
+        mixin :header
+        mixin :diff
+      end
+    end
+  end
+end

data/lib/rouge/lexers/html.rb

@@ -0,0 +1,65 @@
+module Rouge
+  module Lexers
+    class HTML < RegexLexer
+      tag 'html'
+      extensions 'htm', 'html'
+
+      state :root do
+        rule /[^<&]+/m, 'Text'
+        rule /&\S*?;/, 'Name.Entity'
+        rule /<!\[CDATA\[.*?\]\]>/m, 'Comment.Preproc'
+        rule /<!--/, 'Comment', :comment
+        rule /<\?.*?\?>/m, 'Comment.Preproc' # php? really?
+
+        rule /<\s*script\s*/m do
+          token 'Name.Tag'
+          push :script_content
+          push :tag
+        end
+
+        rule /<\s*style\s*/m do
+          token 'Name.Tag'
+          push :style_content
+          push :tag
+        end
+
+        rule %r(<\s*[a-zA-Z0-9:]+), 'Name.Tag', :tag # opening tags
+        rule %r(<\s*/\s*[a-zA-Z0-9:]+\s*>), 'Name.Tag' # closing tags
+      end
+
+      state :comment do
+        rule /[^-]+/, 'Comment'
+        rule /-->/, 'Comment', :pop!
+        rule /-/, 'Comment'
+      end
+
+      state :tag do
+        rule /\s+/m, 'Text'
+        rule /[a-zA-Z0-9_:-]+\s*=/m, 'Name.Attribute', :attr
+        rule /[a-zA-Z0-9_:-]+/, 'Name.Attribute'
+        rule %r(/?\s*>)m, 'Name.Tag', :pop!
+      end
+
+      state :attr do
+        # TODO: are backslash escapes valid here?
+        rule /".*?"/,   'Literal.String', :pop!
+        rule /'.*?'/,   'Literal.String', :pop!
+        rule /[^\s>]+/, 'Literal.String', :pop!
+      end
+
+      state :script_content do
+        rule %r(<\s*/\s*script\s*>)m, 'Name.Tag', :pop!
+        rule %r(.*?(?=<\s*/\s*script\s*>))m do
+          delegate JavascriptLexer
+        end
+      end
+
+      state :style_content do
+        rule %r(<\s*/\s*style\s*>)m, 'Name.Tag', :pop!
+        rule %r(.*(?=<\s*/\s*style\s*>))m do
+          delegate CSSLexer
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/javascript.rb

@@ -1,23 +1,28 @@
 module Rouge
   module Lexers
-    JavascriptLexer = RegexLexer.create do
-      option :debug, true
-
-      name 'javascript'
+    class JavascriptLexer < RegexLexer
+      tag 'javascript'
       aliases 'js'
+      extensions 'js'
 
-      lexer :comments_and_whitespace do
+      state :comments_and_whitespace do
         rule /\s+/, 'Text'
         rule /<!--/, 'Comment' # really...?
         rule %r(//.*?\n), 'Comment.Single'
         rule %r(/\*.*?\*/), 'Comment.Multiline'
       end
 
-      lexer :slash_starts_regex do
+      state :slash_starts_regex do
         mixin :comments_and_whitespace
+
         rule %r(
-          /(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/ # a nonempty regex
-          (?:[gim]+\b|\B) # regex flags
+          / # opening slash
+          ( \\. # escape sequences
+          | [^/\\\n] # regular characters
+          | \[ (\\. | [^\]\\\n])* \] # character classes
+          )+
+          / # closing slash
+          (?:[gim]+\b|\B) # flags
         )x, 'Literal.String.Regex'
 
         # if it's not matched by the above r.e., it's not
@@ -25,10 +30,10 @@ module Rouge
         # end of the line.
         rule %r(/), 'Literal.String.Regex', :bad_regex
         rule //, 'Text', :pop!
+      end
 
-        lexer :bad_regex do
-          rule /[^\n]+/, 'Error', :pop!
-        end
+      state :bad_regex do
+        rule /[^\n]+/, 'Error', :pop!
       end
 
       keywords = %w(
@@ -55,10 +60,11 @@ module Rouge
         window
       ).join('|')
 
-      lexer :root do
+      state :root do
         rule %r(^(?=\s|/|<!--)), 'Text', :slash_starts_regex
         mixin :comments_and_whitespace
-        rule %r(\+\+|--|~|&&|\?|\|\||\\(?=\n)|<<|>>>?|===|!==),
+        rule %r(\+\+ | -- | ~ | && | \|\| | \\(?=\n) | << | >>>? | ===
+               | !== | \? | : )x,
           'Operator', :slash_starts_regex
         rule %r([-<>+*%&|\^/!=]=?), 'Operator', :slash_starts_regex
         rule /[{(\[;,]/, 'Punctuation', :slash_starts_regex
@@ -70,14 +76,12 @@ module Rouge
         rule /(?:#{builtins})\b/, 'Name.Builtin'
         rule /[$a-zA-Z_][a-zA-Z0-9_]*/, 'Name.Other'
 
-        rule /[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?/, 'Number.Float'
-        rule /0x[0-9a-fA-F]+/, 'Number.Hex'
-        rule /[0-9]+/, 'Number.Integer'
+        rule /[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?/, 'Literal.Number.Float'
+        rule /0x[0-9a-fA-F]+/, 'Literal.Number.Hex'
+        rule /[0-9]+/, 'Literal.Number.Integer'
         rule /"(\\\\|\\"|[^"])*"/, 'Literal.String.Double'
         rule /'(\\\\|\\'|[^'])*'/, 'Literal.String.Single'
       end
-
-      mixin :root
     end
   end
 end

data/lib/rouge/lexers/shell.rb

@@ -1,12 +1,13 @@
 module Rouge
   module Lexers
-    ShellLexer = RegexLexer.create do
-      name 'shell'
+    class ShellLexer < RegexLexer
+      tag 'shell'
       aliases 'bash', 'zsh', 'ksh', 'sh'
+      extensions 'sh', 'bash', 'zsh', 'ksh'
 
       KEYWORDS = %w(
-        if fi else while do done for then return function case
-        select continue until esac elif
+        if fi else while do done for then return function
+        select continue until esac elif in
       ).join('|')
 
       BUILTINS = %w(
@@ -18,16 +19,17 @@ module Rouge
         ulimit umask unalias unset wait
       ).join('|')
 
-      lexer :basic do
+      state :basic do
         rule /#.*\n/, 'Comment'
 
         rule /\b(#{KEYWORDS})\s*\b/, 'Keyword'
+        rule /\bcase\b/, 'Keyword', :case
 
         rule /\b(#{BUILTINS})\s*\b(?!\.)/, 'Name.Builtin'
 
-        rule /(\b\w+)(=)/ do |_, var, eq, &out|
-          out.call 'Name.Variable', var
-          out.call 'Operator', eq
+        rule /(\b\w+)(=)/ do |m|
+          group 'Name.Variable'
+          group 'Operator'
         end
 
         rule /[\[\]{}()=]/, 'Operator'
@@ -38,17 +40,17 @@ module Rouge
         rule /<<-?\s*(\'?)\\?(\w+)[\w\W]+?\2/, 'Literal.String'
       end
 
-      lexer :double_quotes do
-        rule /"/, 'Literal.String.Double', :pop!
-        rule /\\./, 'Literal.String.Escape'
+      state :double_quotes do
+        # NB: "abc$" is literally the string abc$.
+        # Here we prevent :interp from interpreting $" as a variable.
+        rule /(?:\$#?)?"/, 'Literal.String.Double', :pop!
         mixin :interp
         rule /[^"`\\$]+/, 'Literal.String.Double'
       end
 
-      lexer :data do
-        # TODO: this should be its own sublexer so we can capture
-        # interpolation and such
-        rule /$?"/, 'Literal.String.Double', :double_quotes
+      state :data do
+        rule /\\./, 'Literal.String.Escape'
+        rule /\$?"/, 'Literal.String.Double', :double_quotes
 
         # single quotes are much easier than double quotes - we can
         # literally just scan until the next single quote.
@@ -57,15 +59,17 @@ module Rouge
         # single-quotes. A single-quote cannot occur within single-quotes.
         rule /$?'[^']*'/, 'Literal.String.Single'
 
+        rule /\*/, 'Keyword'
+
         rule /;/, 'Text'
         rule /\s+/, 'Text'
-        rule /[^=\s\[\]{}()$"\'`\\<]+/, 'Text'
+        rule /[^=\s{}()$"\'`\\<]+/, 'Text'
         rule /\d+(?= |\Z)/, 'Number'
         rule /</, 'Text'
         mixin :interp
       end
 
-      lexer :curly do
+      state :curly do
         rule /}/, 'Keyword', :pop!
         rule /:-/, 'Keyword'
         rule /[a-zA-Z0-9_]+/, 'Name.Variable'
@@ -73,24 +77,38 @@ module Rouge
         mixin :root
       end
 
-      lexer :paren do
+      state :paren do
         rule /\)/, 'Keyword', :pop!
         mixin :root
       end
 
-      lexer :math do
+      state :math do
         rule /\)\)/, 'Keyword', :pop!
         rule %r([-+*/%^|&]|\*\*|\|\|), 'Operator'
         rule /\d+/, 'Number'
         mixin :root
       end
 
-      lexer :backticks do
+      state :case do
+        rule /\besac\b/, 'Keyword', :pop!
+        rule /\|/, 'Punctuation'
+        rule /\)/, 'Punctuation', :case_stanza
+        mixin :root
+      end
+
+      state :case_stanza do
+        rule /;;/, 'Punctuation', :pop!
+        mixin :root
+      end
+
+      state :backticks do
         rule /`/, 'Literal.String.Backtick', :pop!
         mixin :root
       end
 
-      lexer :interp do
+      state :interp do
+        rule /\\$/, 'Literal.String.Escape' # line continuation
+        rule /\\./, 'Literal.String.Escape'
         rule /\$\(\(/, 'Keyword', :math
         rule /\$\(/, 'Keyword', :paren
         rule /\${#?/, 'Keyword', :curly
@@ -98,12 +116,10 @@ module Rouge
         rule /\$#?(\w+|.)/, 'Name.Variable'
       end
 
-      lexer :root do
+      state :root do
         mixin :basic
         mixin :data
       end
-
-      mixin :root
     end
   end
 end