rouge 3.19.0 → 3.21.0

data/lib/rouge/lexers/javascript.rb

@@ -19,6 +19,8 @@ module Rouge
       mimetypes 'application/javascript', 'application/x-javascript',
                 'text/javascript', 'text/x-javascript'
 
+      # Pseudo-documentation: https://stackoverflow.com/questions/1661197/what-characters-are-valid-for-javascript-variable-names
+
       def self.detect?(text)
         return 1 if text.shebang?('node')
         return 1 if text.shebang?('jsc')
@@ -138,7 +140,7 @@ module Rouge
       end
 
       def self.id_regex
-        /[$a-z_][a-z0-9_]*/io
+        /[\p{L}\p{Nl}$_][\p{Word}]*/io
       end
 
       id = self.id_regex
@@ -225,8 +227,6 @@ module Rouge
           groups Name::Label, Text, Punctuation
         end
 
-        rule %r/[{}]/, Punctuation
-
         mixin :expr_start
       end
 

data/lib/rouge/lexers/jinja.rb

@@ -40,6 +40,17 @@ module Rouge
         rule %r/{#/, Comment, :comment
         rule %r/##.*/, Comment
 
+        # Raw and verbatim
+        rule %r/({%-?)(\s*)(raw|verbatim)(\s*)(-?%})/ do |m|
+          groups Comment::Preproc, Text, Keyword, Text, Comment::Preproc
+          case m[3]
+          when "raw"
+            push :raw
+          when "verbatim"
+            push :verbatim
+          end
+        end
+
         # Statements
         rule %r/\{\%/ do
           token Comment::Preproc
@@ -114,11 +125,6 @@ module Rouge
       end
 
       state :statement do
-        rule %r/(raw|verbatim)(\s+)(\%\})/ do
-          groups Keyword, Text, Comment::Preproc
-          goto :raw
-        end
-
         rule %r/(\w+\.?)/ do |m|
           if self.class.keywords.include?(m[0])
             groups Keyword
@@ -142,12 +148,21 @@ module Rouge
       end
 
       state :raw do
-        rule %r{(\{\%)(\s+)(endverbatim|endraw)(\s+)(\%\})} do
+        rule %r/({%-?)(\s*)(endraw)(\s*)(-?%})/ do
           groups Comment::Preproc, Text, Keyword, Text, Comment::Preproc
           pop!
         end
+        rule %r/[^{]+/, Text
+        rule %r/{/, Text
+      end
 
-        rule %r/(.+?)/m, Text
+      state :verbatim do
+        rule %r/({%-?)(\s*)(endverbatim)(\s*)(-?%})/ do
+          groups Comment::Preproc, Text, Keyword, Text, Comment::Preproc
+          pop!
+        end
+        rule %r/[^{]+/, Text
+        rule %r/{/, Text
       end
     end
   end

data/lib/rouge/lexers/jsx.rb

@@ -6,97 +6,85 @@ module Rouge
 
     class JSX < Javascript
       title 'JSX'
-      desc 'React JSX (https://facebook.github.io/react/)'
+      desc 'An XML-like syntax extension to JavaScript (facebook.github.io/jsx/)'
       tag 'jsx'
       aliases 'jsx', 'react'
       filenames '*.jsx'
 
       mimetypes 'text/x-jsx', 'application/x-jsx'
 
-      id = Javascript.id_regex
+      start { @html = HTML.new(options); push :expr_start }
 
-      def start_embed!
-        @embed ||= JSX.new(options)
-        @embed.reset!
-        @embed.push(:expr_start)
-        push :jsx_embed_root
-      end
-
-      def tag_token(name)
-        name[0] =~ /\p{Lower}/ ? Name::Tag : Name::Class
+      prepend :expr_start do
+        mixin :tag
       end
 
-      start { @html = HTML.new(options) }
-
-      state :jsx_tags do
-        rule %r/</, Punctuation, :jsx_element
+      state :tag do
+        rule %r/</ do
+          token Punctuation
+          push :tag_opening
+          push :element
+          push :element_name
+        end
       end
 
-      state :jsx_internal do
-        rule %r(</) do
+      state :tag_opening do
+        rule %r/<\// do
           token Punctuation
-          goto :jsx_end_tag
+          goto :element
+          push :element_name
         end
-
+        mixin :tag
         rule %r/{/ do
           token Str::Interpol
-          start_embed!
+          push :interpol
+          push :expr_start
         end
-
-        rule %r/[^<>{]+/ do
+        rule %r/[^<{]+/ do
           delegate @html
         end
-
-        mixin :jsx_tags
-      end
-
-      prepend :expr_start do
-        mixin :jsx_tags
       end
 
-      state :jsx_tag do
+      state :element do
         mixin :comments_and_whitespace
-        rule %r/#{id}/ do |m|
-          token tag_token(m[0])
+        rule %r/\/>/ do
+          token Punctuation
+          pop! 2
         end
-
-        rule %r/[.]/, Punctuation
-      end
-
-      state :jsx_end_tag do
-        mixin :jsx_tag
         rule %r/>/, Punctuation, :pop!
-      end
-
-      state :jsx_element do
-        rule %r/#{id}=/, Name::Attribute, :jsx_attribute
-        mixin :jsx_tag
-        rule %r/>/ do token Punctuation; goto :jsx_internal end
-        rule %r(/>), Punctuation, :pop!
-      end
-
-      state :jsx_attribute do
-        rule %r/"(\\[\\"]|[^"])*"/, Str::Double, :pop!
-        rule %r/'(\\[\\']|[^'])*'/, Str::Single, :pop!
         rule %r/{/ do
           token Str::Interpol
-          pop!
-          start_embed!
+          push :interpol
+          push :expr_start
         end
+        rule %r/\w+/, Name::Attribute
+        rule %r/=/, Punctuation
+        rule %r/(["']).*?(\1)/, Str
       end
 
-      state :jsx_embed_root do
-        rule %r/[.][.][.]/, Punctuation
+      state :element_name do
+        rule %r/[A-Z]\w*/, Name::Class
+        rule %r/\w+/, Name::Tag
+        rule %r/\./, Punctuation
+        rule(//) { pop! }
+      end
+
+      state :interpol do
         rule %r/}/, Str::Interpol, :pop!
-        mixin :jsx_embed
+        rule %r/{/ do
+          token Punctuation
+          push :interpol_inner
+          push :statement
+        end
+        mixin :root
       end
 
-      state :jsx_embed do
-        rule %r/{/ do delegate @embed; push :jsx_embed end
-        rule %r/}/ do delegate @embed; pop! end
-        rule %r/[^{}]+/ do
-          delegate @embed
+      state :interpol_inner do
+        rule %r/}/ do
+          token Punctuation
+          goto :statement
         end
+        mixin :root
       end
     end
   end

data/lib/rouge/lexers/julia.rb

@@ -11,6 +11,8 @@ module Rouge
       filenames '*.jl'
       mimetypes 'text/x-julia', 'application/x-julia'
 
+      # Documentation: https://docs.julialang.org/en/v1/manual/variables/#Allowed-Variable-Names-1
+
       def self.detect?(text)
         return true if text.shebang? 'julia'
       end
@@ -252,13 +254,13 @@ module Rouge
 
 
       state :funcname do
-        rule %r/[a-zA-Z_]\w*/, Name::Function, :pop!
+        rule %r/[\p{L}\p{Nl}\p{S}_][\p{Word}\p{S}\p{Po}!]*/, Name::Function, :pop!
         rule %r/\([^\s\w{]{1,2}\)/, Operator, :pop!
         rule %r/[^\s\w{]{1,2}/, Operator, :pop!
       end
 
       state :typename do
-        rule %r/[a-zA-Z_]\w*/, Name::Class, :pop!
+        rule %r/[\p{L}\p{Nl}\p{S}_][\p{Word}\p{S}\p{Po}!]*/, Name::Class, :pop!
       end
 
       state :stringescape do

data/lib/rouge/lexers/livescript.rb

@@ -0,0 +1,310 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Livescript < RegexLexer
+      tag 'livescript'
+      aliases 'ls'
+      filenames '*.ls'
+      mimetypes 'text/livescript'
+
+      title 'LiveScript'
+      desc 'LiveScript, a language which compiles to JavaScript (livescript.net)'
+
+      def self.detect?(text)
+        return text.shebang? 'lsc'
+      end
+
+      def self.declarations
+        @declarations ||= Set.new %w(const let var function class extends implements)
+      end
+
+      def self.keywords
+        @keywords ||= Set.new %w(
+          loop until for in of while break return continue switch case
+          fallthrough default otherwise when then if unless else throw try
+          catch finally new delete typeof instanceof super by from to til
+          with require do debugger import export yield
+        )
+      end
+
+      def self.constants
+        @constants ||= Javascript.constants + %w(yes no on off void)
+      end
+
+      def self.builtins
+        @builtins ||= Javascript.builtins + %w(this it that arguments)
+      end
+
+      def self.loop_control_keywords
+        @loop_control_keywords ||= Set.new %w(break continue)
+      end
+
+      id = /[$a-z_]((-(?=[a-z]))?[a-z0-9_])*/i
+      int_number = /\d[\d_]*/
+      int = /#{int_number}(e[+-]?#{int_number})?[$\w]*/ # the last class matches units
+
+      state :root do
+        rule(%r(^(?=\s|/))) { push :slash_starts_regex }
+        mixin :comments
+        mixin :whitespace
+
+        # list of words
+        rule %r/(<\[)(.*?)(\]>)/m do
+          groups Punctuation, Str, Punctuation
+        end
+
+        # function declarations
+        rule %r/!\s*function\b/, Keyword::Declaration
+        rule %r/!?[-~]>|<[-~]!?/, Keyword::Declaration
+
+        # switch arrow
+        rule %r/(=>)/, Keyword
+
+        # prototype attributes
+        rule %r/(::)(#{id})/ do
+          groups Punctuation, Name::Attribute
+          push :id
+        end
+        rule %r/(::)(#{int})/ do
+          groups Punctuation, Num::Integer
+          push :id
+        end
+
+        # instance attributes
+        rule %r/(@)(#{id})/ do
+          groups Name::Variable::Instance, Name::Attribute
+          push :id
+        end
+        rule %r/([.])(#{id})/ do
+          groups Punctuation, Name::Attribute
+          push :id
+        end
+        rule %r/([.])(\d+)/ do
+          groups Punctuation, Num::Integer
+          push :id
+        end
+        rule %r/#{id}(?=\s*:[^:=])/, Name::Attribute
+
+        # operators
+        rule %r(
+          [+][+]|--|&&|\b(and|x?or|is(nt)?|not)\b(?!-[a-zA-Z]|_)|[|][|]|
+          [.]([|&^]|<<|>>>?)[.]|\\(?=\n)|[.:]=|<<<<?|<[|]|[|]>|
+          (<<|>>|==?|!=?|[-<>+*%^/~?])=?
+        )x, Operator, :slash_starts_regex
+
+        # arguments shorthand
+        rule %r/(&)(#{id})?/ do
+          groups Name::Builtin, Name::Attribute
+        end
+
+        # switch case
+        rule %r/[|]|\bcase(?=\s)/, Keyword, :switch_underscore
+
+        rule %r/@/, Name::Variable::Instance
+        rule %r/[.]{3}/, Punctuation
+        rule %r/:/, Punctuation
+
+        # keywords
+        rule %r/#{id}/ do |m|
+          if self.class.loop_control_keywords.include? m[0]
+            token Keyword
+            push :loop_control
+            next
+          elsif self.class.keywords.include? m[0]
+            token Keyword
+          elsif self.class.constants.include? m[0]
+            token Name::Constant
+          elsif self.class.builtins.include? m[0]
+            token Name::Builtin
+          elsif self.class.declarations.include? m[0]
+            token Keyword::Declaration
+          elsif /^[A-Z]/.match(m[0]) && /[^-][a-z]/.match(m[0])
+            token Name::Class
+          else
+            token Name::Variable
+          end
+          push :id
+        end
+
+        # punctuation and brackets
+        rule %r/\](?=[!?.]|#{id})/, Punctuation, :id
+        rule %r/[{(\[;,]/, Punctuation, :slash_starts_regex
+        rule %r/[})\].]/, Punctuation
+
+        # literals
+        rule %r/#{int_number}[.]#{int}/, Num::Float
+        rule %r/0x[0-9A-Fa-f]+/, Num::Hex
+        rule %r/#{int}/, Num::Integer
+
+        # strings
+        rule %r/"""/ do
+          token Str
+          push do
+            rule %r/"""/, Str, :pop!
+            rule %r/"/, Str
+            mixin :double_strings
+          end
+        end
+
+        rule %r/'''/ do
+          token Str
+          push do
+            rule %r/'''/, Str, :pop!
+            rule %r/'/, Str
+            mixin :single_strings
+          end
+        end
+
+        rule %r/"/ do
+          token Str
+          push do
+            rule %r/"/, Str, :pop!
+            mixin :double_strings
+          end
+        end
+
+        rule %r/'/ do
+          token Str
+          push do
+            rule %r/'/, Str, :pop!
+            mixin :single_strings
+          end
+        end
+
+        # words
+        rule %r/\\\S[^\s,;\])}]*/, Str
+      end
+
+      state :code_escape do
+        rule %r(\\(
+          c[A-Z]|
+          x[0-9a-fA-F]{2}|
+          u[0-9a-fA-F]{4}|
+          u\{[0-9a-fA-F]{4}\}
+        ))x, Str::Escape
+      end
+
+      state :interpolated_expression do
+        rule %r/}/, Str::Interpol, :pop!
+        mixin :root
+      end
+
+      state :interpolation do
+        # with curly braces
+        rule %r/[#][{]/, Str::Interpol, :interpolated_expression
+        # without curly braces
+        rule %r/(#)(#{id})/ do |m|
+          groups Str::Interpol, (self.class.builtins.include? m[2]) ? Name::Builtin : Name::Variable
+        end
+      end
+
+      state :whitespace do
+        # white space and loop labels
+        rule %r/(\s+?)(?:^([^\S\n]*)(:#{id}))?/m do
+          groups Text, Text, Name::Label
+        end
+      end
+
+      state :whitespace_single_line do
+        rule %r([^\S\n]+), Text
+      end
+
+      state :slash_starts_regex do
+        mixin :comments
+        mixin :whitespace
+        mixin :multiline_regex_begin
+
+        rule %r(
+          /(\\.|[^\[/\\\n]|\[(\\.|[^\]\\\n])*\])+/ # a regex
+          ([gimy]+\b|\B)
+        )x, Str::Regex, :pop!
+
+        rule(//) { pop! }
+      end
+
+      state :multiline_regex_begin do
+        rule %r(//) do
+          token Str::Regex
+          goto :multiline_regex
+        end
+      end
+
+      state :multiline_regex_end do
+        rule %r(//([gimy]+\b|\B)), Str::Regex, :pop!
+      end
+
+      state :multiline_regex do
+        mixin :multiline_regex_end
+        mixin :regex_comment
+        mixin :interpolation
+        mixin :code_escape
+        rule %r/\\\D/, Str::Escape
+        rule %r/\\\d+/, Name::Variable
+        rule %r/./m, Str::Regex
+      end
+
+      state :regex_comment do
+        rule %r/^#(\s+.*)?$/, Comment::Single
+        rule %r/(\s+)(#)(\s+.*)?$/ do
+          groups Text, Comment::Single, Comment::Single
+        end
+      end
+
+      state :comments do
+        rule %r(/\*.*?\*/)m, Comment::Multiline
+        rule %r/#.*$/, Comment::Single
+      end
+
+      state :switch_underscore do
+        mixin :whitespace_single_line
+        rule %r/_(?=\s*=>|\s+then\b)/, Keyword
+        rule(//) { pop! }
+      end
+
+      state :loop_control do
+        mixin :whitespace_single_line
+        rule %r/#{id}(?=[);\n])/, Name::Label
+        rule(//) { pop! }
+      end
+
+      state :id do
+        rule %r/[!?]|[.](?!=)/, Punctuation
+        rule %r/[{]/ do
+          # destructuring
+          token Punctuation
+          push do
+            rule %r/[,;]/, Punctuation
+            rule %r/#{id}/, Name::Attribute
+            rule %r/#{int}/, Num::Integer
+            mixin :whitespace
+            rule %r/[}]/, Punctuation, :pop!
+          end
+        end
+        rule %r/#{id}/, Name::Attribute
+        rule %r/#{int}/, Num::Integer
+        rule(//) { goto :slash_starts_regex }
+      end
+
+      state :strings do
+        # all strings are multi-line
+        rule %r/[^#\\'"]+/m, Str
+        mixin :code_escape
+        rule %r/\\./, Str::Escape
+        rule %r/#/, Str
+      end
+
+      state :double_strings do
+        rule %r/'/, Str
+        mixin :interpolation
+        mixin :strings
+      end
+
+      state :single_strings do
+        rule %r/"/, Str
+        mixin :strings
+      end
+    end
+  end
+end