rouge 2.0.7 → 2.1.0

data/lib/rouge/lexers/r.rb

@@ -48,8 +48,8 @@ module Rouge
       end
 
       state :root do
-        rule /#'.*?\n/, Comment::Doc
-        rule /#.*?\n/, Comment::Single
+        rule /#'.*?$/, Comment::Doc
+        rule /#.*?$/, Comment::Single
         rule /\s+/m, Text::Whitespace
 
         rule /`[^`]+?`/, Name

data/lib/rouge/lexers/ruby.rb

@@ -108,12 +108,14 @@ module Rouge
       )
 
       keywords_pseudo = %w(
-        initialize new loop include extend raise attr_reader attr_writer
-        attr_accessor alias_method attr catch throw private module_function
+        loop include extend raise
+        alias_method attr catch throw private module_function
         public protected true false nil __FILE__ __LINE__
       )
 
       builtins_g = %w(
+        attr_reader attr_writer attr_accessor
+
         __id__ __send__ abort ancestors at_exit autoload binding callcc
         caller catch chomp chop class_eval class_variables clone
         const_defined\? const_get const_missing const_set constants
@@ -150,7 +152,7 @@ module Rouge
         rule /\n\s*/m, Text, :expr_start
         rule /#.*$/, Comment::Single
 
-        rule %r(=begin\b.*?end\b)m, Comment::Multiline
+        rule %r(=begin\b.*?\n=end\b)m, Comment::Multiline
       end
 
       state :inline_whitespace do
@@ -164,6 +166,7 @@ module Rouge
         rule /0_?[0-7]+(?:_[0-7]+)*/, Num::Oct
         rule /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/, Num::Hex
         rule /0b[01]+(?:_[01]+)*/, Num::Bin
+        rule /\d+\.\d+(e[\+\-]?\d+)?/, Num::Float
         rule /[\d]+(?:_\d+)*/, Num::Integer
 
         # names
@@ -204,6 +207,10 @@ module Rouge
 
         mixin :has_heredocs
 
+        # `..` and `...` for ranges must have higher priority than `.`
+        # Otherwise, they will be parsed as :method_call
+        rule /\.{2,3}/, Operator, :expr_start
+
         rule /[A-Z][a-zA-Z0-9_]*/, Name::Constant, :method_call
         rule /(\.|::)(\s*)([a-z_]\w*[!?]?|[*%&^`~+-\/\[<>=])/ do
           groups Punctuation, Text, Name::Function
@@ -212,7 +219,7 @@ module Rouge
 
         rule /[a-zA-Z_]\w*[?!]/, Name, :expr_start
         rule /[a-zA-Z_]\w*/, Name, :method_call
-        rule /\*\*|<<?|>>?|>=|<=|<=>|=~|={3}|!~|&&?|\|\||\.{1,3}/,
+        rule /\*\*|<<?|>>?|>=|<=|<=>|=~|={3}|!~|&&?|\|\||\./,
           Operator, :expr_start
         rule /[-+\/*%=<>&!^|~]=?/, Operator, :expr_start
         rule(/[?]/) { token Punctuation; push :ternary; push :expr_start }
@@ -247,25 +254,31 @@ module Rouge
       state :resolve_heredocs do
         mixin :string_intp_escaped
 
-        rule /(\n)([^#\\\n]*)$/ do |m|
+        rule /\n/, Str::Heredoc, :test_heredoc
+        rule /[#\\\n]/, Str::Heredoc
+        rule /[^#\\\n]+/, Str::Heredoc
+      end
+
+      state :test_heredoc do
+        rule /[^#\\\n]*$/ do |m|
           tolerant, heredoc_name = @heredoc_queue.first
-          check = tolerant ? m[2].strip : m[2].rstrip
+          check = tolerant ? m[0].strip : m[0].rstrip
 
           # check if we found the end of the heredoc
-          line_tok = if check == heredoc_name
+          puts "    end heredoc check #{check.inspect} = #{heredoc_name.inspect}" if @debug
+          if check == heredoc_name
             @heredoc_queue.shift
             # if there's no more, we're done looking.
             pop! if @heredoc_queue.empty?
-            Name::Constant
+            token Name::Constant
           else
-            Str::Heredoc
+            token Str::Heredoc
           end
 
-          groups(Str::Heredoc, line_tok)
+          pop!
         end
 
-        rule /[#\\\n]/, Str::Heredoc
-        rule /[^#\\\n]+/, Str::Heredoc
+        rule(//) { pop! }
       end
 
       state :funcname do
@@ -301,7 +314,7 @@ module Rouge
           goto :expr_start
         end
 
-        rule /[A-Z_]\w*/, Name::Class
+        rule /[A-Z_]\w*/, Name::Class, :pop!
 
         rule(//) { pop! }
       end

data/lib/rouge/lexers/rust.rb

@@ -58,17 +58,18 @@ module Rouge
         mixin :whitespace
         rule /\s+/, Text
         rule /#\[/ do
-          token Comment::Preproc; push :attribute
+          token Name::Decorator; push :attribute
         end
         rule(//) { pop! }
+        rule /#\s[^\n]*/, Comment::Preproc
       end
 
       state :attribute do
         mixin :whitespace
         mixin :has_literals
-        rule /[(,)=]/, Comment::Preproc
-        rule /\]/, Comment::Preproc, :pop!
-        rule id, Comment::Preproc
+        rule /[(,)=]/, Name::Decorator
+        rule /\]/, Name::Decorator, :pop!
+        rule id, Name::Decorator
       end
 
       state :whitespace do
@@ -86,7 +87,7 @@ module Rouge
         rule %r([=-]>), Keyword
         rule %r(<->), Keyword
         rule /[()\[\]{}|,:;]/, Punctuation
-        rule /[*!@~&+%^<>=-\?]/, Operator
+        rule /[*!@~&+%^<>=-\?]|\.{2,3}/, Operator
 
         rule /([.]\s*)?#{id}(?=\s*[(])/m, Name::Function
         rule /[.]\s*#{id}/, Name::Property
@@ -98,6 +99,7 @@ module Rouge
         rule /\bmacro_rules!/, Name::Decorator, :macro_rules
         rule /#{id}!/, Name::Decorator, :macro
 
+        rule /'#{id}/, Name::Variable
         rule /#{id}/ do |m|
           name = m[0]
           if self.class.builtins.include? name

data/lib/rouge/lexers/sed.rb

@@ -24,10 +24,12 @@ module Rouge
         end
 
         state :brackets do
-          rule /\^?/ do
+          rule /\^/ do
             token Punctuation
             goto :brackets_int
           end
+
+          rule(//) { goto :brackets_int }
         end
 
         state :brackets_int do
@@ -133,7 +135,7 @@ module Rouge
         rule /\d+/, addr_tok
         rule /[$,~+!]/, addr_tok
 
-        rule %r((/)(\\.|.)*?(/)) do |m|
+        rule %r((/)((?:\\.|.)*?)(/)) do |m|
           token addr_tok, m[1]; delegate regex, m[2]; token addr_tok, m[3]
         end
 

data/lib/rouge/lexers/shell.rb

@@ -29,6 +29,8 @@ module Rouge
         local logout popd printf pushd pwd read readonly set shift
         shopt source suspend test time times trap true type typeset
         ulimit umask unalias unset wait
+
+        ls tar cat grep sudo
       ).join('|')
 
       state :basic do
@@ -37,30 +39,47 @@ module Rouge
         rule /\b(#{KEYWORDS})\s*\b/, Keyword
         rule /\bcase\b/, Keyword, :case
 
-        rule /\b(#{BUILTINS})\s*\b(?!\.)/, Name::Builtin
-
-        rule /^\S*[\$%>#] +/, Generic::Prompt
+        rule /\b(#{BUILTINS})\s*\b(?!(\.|-))/, Name::Builtin
+        rule /[.](?=\s)/, Name::Builtin
 
         rule /(\b\w+)(=)/ do |m|
           groups Name::Variable, Operator
         end
 
-        rule /[\[\]{}()=]/, Operator
+        rule /[\[\]{}()!=>]/, Operator
         rule /&&|\|\|/, Operator
 
-        rule /<<</, Operator # here-string
-        rule /<<-?\s*(\'?)\\?(\w+)\1/ do |m|
-          lsh = Str::Heredoc
-          token lsh
-          heredocstr = Regexp.escape(m[2])
+        # here-string
+        rule /<<</, Operator
+
+        rule /(<<-?)(\s*)(\'?)(\\?)(\w+)(\3)/ do |m|
+          groups Operator, Text, Str::Heredoc, Str::Heredoc, Name::Constant, Str::Heredoc
+          @heredocstr = Regexp.escape(m[5])
+          push :heredoc
+        end
+      end
 
-          push do
-            rule /\s*#{heredocstr}\s*\n/, lsh, :pop!
-            rule /.*?\n/, lsh
+      state :heredoc do
+        rule /\n/, Str::Heredoc, :heredoc_nl
+        rule /[^$\n]+/, Str::Heredoc
+        mixin :interp
+        rule /[$]/, Str::Heredoc
+      end
+
+      state :heredoc_nl do
+        rule /\s*(\w+)\s*\n/ do |m|
+          if m[1] == @heredocstr
+            token Name::Constant
+            pop! 2
+          else
+            token Str::Heredoc
           end
         end
+
+        rule(//) { pop! }
       end
 
+
       state :double_quotes do
         # NB: "abc$" is literally the string abc$.
         # Here we prevent :interp from interpreting $" as a variable.
@@ -95,8 +114,10 @@ module Rouge
 
         rule /\*/, Keyword
 
-        rule /;/, Text
-        rule /[^=\*\s{}()$"'`\\<]+/, Text
+        rule /;/, Punctuation
+
+        rule /--?[\w-]+/, Name::Tag
+        rule /[^=\*\s{}()$"'`;\\<]+/, Text
         rule /\d+(?= |\Z)/, Num
         rule /</, Text
         mixin :interp
@@ -117,8 +138,8 @@ module Rouge
 
       state :math do
         rule /\)\)/, Keyword, :pop!
-        rule %r([-+*/%^|&]|\*\*|\|\|), Operator
-        rule /\d+/, Num
+        rule %r([-+*/%^|&!]|\*\*|\|\|), Operator
+        rule /\d+(#\w+)?/, Num
         mixin :root
       end
 
@@ -147,6 +168,7 @@ module Rouge
         rule /\${#?/, Keyword, :curly
         rule /`/, Str::Backtick, :backticks
         rule /\$#?(\w+|.)/, Name::Variable
+        rule /\$[*@]/, Name::Variable
       end
 
       state :root do

data/lib/rouge/lexers/sieve.rb

@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*- #
+
+module Rouge
+  module Lexers
+    class Sieve < RegexLexer
+      title "Sieve"
+      desc "mail filtering language"
+
+      tag 'sieve'
+      filenames '*.sieve'
+
+      id = /:?[a-zA-Z_][a-zA-Z0-9_]*/
+
+      # control commands (rfc5228 § 3)
+      def self.controls
+        @controls ||= %w(if elsif else require stop)
+      end
+
+      def self.actions
+        @actions ||= Set.new(
+          # action commands (rfc5228 § 2.9)
+          %w(keep fileinto redirect discard) +
+          # Editheader Extension (rfc5293)
+          %w(addheader deleteheader) +
+          # Reject and Extended Reject Extensions (rfc5429)
+          %w(reject ereject) +
+          # Extension for Notifications (rfc5435)
+          %w(notify) +
+          # Imap4flags Extension (rfc5232)
+          %w(setflag addflag removeflag) +
+          # Vacation Extension (rfc5230)
+          %w(vacation) +
+          # MIME Part Tests, Iteration, Extraction, Replacement, and Enclosure (rfc5703)
+          %w(replace enclose extracttext)
+        )
+      end
+
+      def self.tests
+        @tests ||= Set.new(
+          # test commands (rfc5228 § 5)
+          %w(address allof anyof exists false header not size true) +
+          # Body Extension (rfc5173)
+          %w(body) +
+          # Imap4flags Extension (rfc5232)
+          %w(hasflag) +
+          # Spamtest and Virustest Extensions (rfc5235)
+          %w(spamtest virustest) +
+          # Date and Index Extensions (rfc5260)
+          %w(date currentdate) +
+          # Extension for Notifications (rfc5435)
+          %w(valid_notify_method notify_method_capability) +
+          # Extensions for Checking Mailbox Status and Accessing Mailbox
+          # Metadata (rfc5490)
+          %w(mailboxexists metadata metadataexists servermetadata servermetadataexists)
+        )
+      end
+
+      state :comments_and_whitespace do
+        rule /\s+/, Text
+        rule %r(#.*?\n), Comment::Single
+        rule %r(/(\\\n)?[*].*?[*](\\\n)?/)m, Comment::Multiline
+      end
+
+      state :string do
+        rule /\\./, Str::Escape
+        rule /"/, Str::Double, :pop!
+        # Variables Extension (rfc5229)
+        rule /\${(?:[0-9][.0-9]*|[a-zA-Z_][.a-zA-Z0-9_]*)}/, Str::Interpol
+        rule /./, Str::Double
+      end
+
+      state :root do
+        mixin :comments_and_whitespace
+
+        rule /[\[\](),;{}]/, Punctuation
+
+        rule id do |m|
+          if self.class.controls.include? m[0]
+            token Keyword
+          elsif self.class.tests.include? m[0]
+            token Name::Variable
+          elsif self.class.actions.include? m[0]
+            token Name::Function
+          elsif m[0] =~ /^:/ # tags like :contains, :matches etc.
+            token Operator
+          else
+            token Name::Other
+          end
+        end
+
+        rule /"/, Str::Double, :string
+        rule /[0-9]+[KMG]/, Num::Integer
+      end
+    end
+  end
+end

data/lib/rouge/lexers/sml.rb

@@ -41,7 +41,8 @@ module Rouge
         rule /[(\[{]/, Punctuation, :main
         rule /[)\]}]/, Punctuation, :pop!
 
-        rule /\b(let|if|local)\b(?!')/, Keyword::Reserved do
+        rule /\b(let|if|local)\b(?!')/ do
+          token Keyword::Reserved
           push; push
         end
 
@@ -190,7 +191,7 @@ module Rouge
       end
 
       state :breakout do
-        rule /(?=\w+\b(#{SML.keywords.to_a.join('|')})\b(?!'))/ do
+        rule /(?=\b(#{SML.keywords.to_a.join('|')})\b(?!'))/ do
           pop!
         end
       end

data/lib/rouge/lexers/tsx.rb

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*- #
+
+module Rouge
+  module Lexers
+    load_lexer 'jsx.rb'
+    load_lexer 'typescript/common.rb'
+
+    class TSX < JSX
+      include TypescriptCommon
+
+      title 'TypeScript'
+      desc 'tsx'
+
+      tag 'tsx'
+      filenames '*.tsx'
+    end
+  end
+end
+

data/lib/rouge/lexers/typescript.rb

@@ -3,8 +3,11 @@
 module Rouge
   module Lexers
     load_lexer 'javascript.rb'
+    load_lexer 'typescript/common.rb'
 
     class Typescript < Javascript
+      include TypescriptCommon
+
       title "TypeScript"
       desc "TypeScript, a superset of JavaScript"
 
@@ -14,33 +17,6 @@ module Rouge
       filenames '*.ts', '*.d.ts'
 
       mimetypes 'text/typescript'
-
-      def self.keywords
-        @keywords ||= super + Set.new(%w(
-          import export from as is
-          namespace new static private protected public
-          super async await extends implements readonly
-        ))
-      end
-
-      def self.declarations
-        @declarations ||= super + Set.new(%w(
-          const type constructor abstract
-        ))
-      end
-
-      def self.reserved
-        @reserved ||= super + Set.new(%w(
-          string any number namespace module
-          declare default interface
-        ))
-      end
-
-      def self.builtins
-        @builtins ||= super + %w(
-          Promise Set Map WeakSet WeakMap Symbol
-        )
-      end
     end
   end
 end