rouge 3.20.0 → 3.25.0

data/lib/rouge/lexers/janet.rb

@@ -0,0 +1,218 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Janet < RegexLexer
+      title "Janet"
+      desc "The Janet programming language (janet-lang.org)"
+
+      tag 'janet'
+      aliases 'jdn'
+
+      filenames '*.janet', '*.jdn'
+
+      mimetypes 'text/x-janet', 'application/x-janet'
+
+      def self.specials
+        @specials ||= Set.new %w(
+          break def do fn if quote quasiquote splice set unquote var while
+        )
+      end
+
+      def self.bundled
+        @bundled ||= Set.new %w(
+          % %= * *= + ++ += - -- -= -> ->> -?> -?>> / /= < <= = > >=
+          abstract? accumulate accumulate2 all all-bindings
+          all-dynamics and apply array array/concat array/ensure
+          array/fill array/insert array/new array/new-filled
+          array/peek array/pop array/push array/remove array/slice
+          array? as-> as?-> asm assert bad-compile bad-parse band
+          blshift bnot boolean? bor brshift brushift buffer buffer/bit
+          buffer/bit-clear buffer/bit-set buffer/bit-toggle
+          buffer/blit buffer/clear buffer/fill buffer/format
+          buffer/new buffer/new-filled buffer/popn buffer/push-byte
+          buffer/push-string buffer/push-word buffer/slice buffer?
+          bxor bytes? case cfunction? chr cli-main comment comp
+          compare compare= compare< compare<= compare> compare>=
+          compile complement comptime cond coro count debug
+          debug/arg-stack debug/break debug/fbreak debug/lineage
+          debug/stack debug/stacktrace debug/step debug/unbreak
+          debug/unfbreak debugger-env dec deep-not= deep= default
+          default-peg-grammar def- defer defmacro defmacro- defn defn-
+          defglobal describe dictionary? disasm distinct doc doc*
+          doc-format dofile drop drop-until drop-while dyn each eachk
+          eachp eachy edefer eflush empty? env-lookup eprin eprinf
+          eprint eprintf error errorf eval eval-string even? every?
+          extreme false? fiber/can-resume? fiber/current fiber/getenv
+          fiber/maxstack fiber/new fiber/root fiber/setenv
+          fiber/setmaxstack fiber/status fiber? file/close file/flush
+          file/open file/popen file/read file/seek file/temp
+          file/write filter find find-index first flatten flatten-into
+          flush for forv freeze frequencies function? gccollect
+          gcinterval gcsetinterval generate gensym get get-in getline
+          hash idempotent? identity import import* if-let if-not
+          if-with in inc indexed? int/s64 int/u64 int? interleave
+          interpose invert janet/build janet/config-bits janet/version
+          juxt juxt* keep keys keyword keyword? kvs label last length
+          let load-image load-image-dict loop macex macex1 make-env
+          make-image make-image-dict map mapcat marshal math/-inf
+          math/abs math/acos math/acosh math/asin math/asinh math/atan
+          math/atan2 math/atanh math/cbrt math/ceil math/cos math/cosh
+          math/e math/erf math/erfc math/exp math/exp2 math/expm1
+          math/floor math/gamma math/hypot math/inf math/log
+          math/log10 math/log1p math/log2 math/next math/pi math/pow
+          math/random math/rng math/rng-buffer math/rng-int
+          math/rng-uniform math/round math/seedrandom math/sin
+          math/sinh math/sqrt math/tan math/tanh math/trunc match max
+          mean merge merge-into min mod module/add-paths module/cache
+          module/expand-path module/find module/loaders module/loading
+          module/paths nan? nat? native neg? net/chunk net/close
+          net/connect net/read net/server net/write next nil? not not=
+          number? odd? one? or os/arch os/cd os/chmod os/clock
+          os/cryptorand os/cwd os/date os/dir os/environ os/execute
+          os/exit os/getenv os/link os/lstat os/mkdir os/mktime
+          os/perm-int os/perm-string os/readlink os/realpath os/rename
+          os/rm os/rmdir os/setenv os/shell os/sleep os/stat
+          os/symlink os/time os/touch os/umask os/which pairs parse
+          parser/byte parser/clone parser/consume parser/eof
+          parser/error parser/flush parser/has-more parser/insert
+          parser/new parser/produce parser/state parser/status
+          parser/where partial partition peg/compile peg/match pos?
+          postwalk pp prewalk prin prinf print printf product prompt
+          propagate protect put put-in quit range reduce reduce2
+          repeat repl require resume return reverse reversed root-env
+          run-context scan-number seq setdyn shortfn signal slice
+          slurp some sort sort-by sorted sorted-by spit stderr stdin
+          stdout string string/ascii-lower string/ascii-upper
+          string/bytes string/check-set string/find string/find-all
+          string/format string/from-bytes string/has-prefix?
+          string/has-suffix? string/join string/repeat string/replace
+          string/replace-all string/reverse string/slice string/split
+          string/trim string/triml string/trimr string? struct struct?
+          sum symbol symbol? table table/clone table/getproto
+          table/new table/rawget table/setproto table/to-struct table?
+          take take-until take-while tarray/buffer tarray/copy-bytes
+          tarray/length tarray/new tarray/properties tarray/slice
+          tarray/swap-bytes thread/close thread/current thread/new
+          thread/receive thread/send trace tracev true? truthy? try
+          tuple tuple/brackets tuple/setmap tuple/slice
+          tuple/sourcemap tuple/type tuple? type unless unmarshal
+          untrace update update-in use values var- varfn varglobal
+          walk walk-ind walk-dict when when-let when-with with
+          with-dyns with-syms with-vars yield zero? zipcoll
+        )
+      end
+
+      def name_token(name)
+        if self.class.specials.include? name
+          Keyword
+        elsif self.class.bundled.include? name
+          Keyword::Reserved
+        else
+          Name::Function
+        end
+      end
+
+      punctuation = %r/[_!$%^&*+=~<>.?\/-]/o
+      symbol = %r/([[:alpha:]]|#{punctuation})([[:word:]]|#{punctuation}|:)*/o
+
+      state :root do
+        rule %r/#.*?$/, Comment::Single
+        rule %r/\s+/m, Text::Whitespace
+
+        rule %r/(true|false|nil)\b/, Name::Constant
+        rule %r/(['~])(#{symbol})/ do
+          groups Operator, Str::Symbol
+        end
+        rule %r/:([[:word:]]|#{punctuation}|:)*/, Keyword::Constant
+
+        # radix-specified numbers
+        rule %r/[+-]?\d{1,2}r[\w.]+(&[+-]?\w+)?/, Num::Float
+
+        # hex numbers
+        rule %r/[+-]?0x\h[\h_]*(\.\h[\h_]*)?/, Num::Hex
+        rule %r/[+-]?0x\.\h[\h_]*/, Num::Hex
+
+        # decimal numbers (Janet treats all decimals as floats)
+        rule %r/[+-]?\d[\d_]*(\.\d[\d_]*)?([e][+-]?\d+)?/i, Num::Float
+        rule %r/[+-]?\.\d[\d_]*([e][+-]?\d+)?/i, Num::Float
+
+        rule %r/@?"/, Str::Double, :string
+        rule %r/@?(`+).*?\1/m, Str::Heredoc
+
+        rule %r/\(/, Punctuation, :function
+
+        rule %r/(')(@?[(\[{])/ do
+          groups Operator, Punctuation
+          push :quote
+        end
+
+        rule %r/(~)(@?[(\[{])/ do
+          groups Operator, Punctuation
+          push :quasiquote
+        end
+
+        rule %r/[\#~,';\|]/, Operator
+
+        rule %r/@?[(){}\[\]]/, Punctuation
+
+        rule symbol, Name
+      end
+
+      state :string do
+        rule %r/"/, Str::Double, :pop!
+        rule %r/\\(u\h{4}|U\h{6})/, Str::Escape
+        rule %r/\\./, Str::Escape
+        rule %r/[^"\\]+/, Str::Double
+      end
+
+      state :function do
+        rule %r/[\)]/, Punctuation, :pop!
+
+        rule symbol do |m|
+          case m[0]
+          when "quote"
+            token Keyword
+            goto :quote
+          when "quasiquote"
+            token Keyword
+            goto :quasiquote
+          else
+            token name_token(m[0])
+            goto :root
+          end
+        end
+
+        mixin :root
+      end
+
+      state :quote do
+        rule %r/[(\[{]/, Punctuation, :push
+        rule %r/[)\]}]/, Punctuation, :pop!
+        rule symbol, Str::Escape
+        mixin :root
+      end
+
+      state :quasiquote do
+        rule %r/(,)(\()/ do
+          groups Operator, Punctuation
+          push :function
+        end
+        rule %r/(\()(\s*)(unquote)(\s+)(\()/ do
+          groups Punctuation, Text, Keyword, Text, Punctuation
+          push :function
+        end
+
+        rule %r/(,)(#{symbol})/ do
+          groups Operator, Name
+        end
+        rule %r/(\()(\s*)(unquote)(\s+)(#{symbol})/ do
+          groups Punctuation, Text, Keyword, Text, Name
+        end
+
+        mixin :quote
+      end
+    end
+  end
+end

data/lib/rouge/lexers/javascript.rb

@@ -19,6 +19,8 @@ module Rouge
       mimetypes 'application/javascript', 'application/x-javascript',
                 'text/javascript', 'text/x-javascript'
 
+      # Pseudo-documentation: https://stackoverflow.com/questions/1661197/what-characters-are-valid-for-javascript-variable-names
+
       def self.detect?(text)
         return 1 if text.shebang?('node')
         return 1 if text.shebang?('jsc')
@@ -138,7 +140,7 @@ module Rouge
       end
 
       def self.id_regex
-        /[$a-z_][a-z0-9_]*/io
+        /[\p{L}\p{Nl}$_][\p{Word}]*/io
       end
 
       id = self.id_regex
@@ -160,6 +162,10 @@ module Rouge
           push :template_string
         end
 
+        # special case for the safe navigation operator ?.
+        # so that we don't start detecting a ternary expr
+        rule %r/[?][.]/, Punctuation
+
         rule %r/[?]/ do
           token Punctuation
           push :ternary
@@ -263,9 +269,11 @@ module Rouge
 
       # template strings
       state :template_string do
-        rule %r/\${/, Punctuation, :template_string_expr
+        rule %r/[$]{/, Punctuation, :template_string_expr
         rule %r/`/, Str::Double, :pop!
-        rule %r/(\\\\|\\[\$`]|[^\$`]|\$(?!{))*/, Str::Double
+        rule %r/\\[$`]/, Str::Escape
+        rule %r/[^$`\\]+/, Str::Double
+        rule %r/[\\$]/, Str::Double
       end
 
       state :template_string_expr do

data/lib/rouge/lexers/jinja.rb

@@ -40,6 +40,17 @@ module Rouge
         rule %r/{#/, Comment, :comment
         rule %r/##.*/, Comment
 
+        # Raw and verbatim
+        rule %r/({%-?)(\s*)(raw|verbatim)(\s*)(-?%})/ do |m|
+          groups Comment::Preproc, Text, Keyword, Text, Comment::Preproc
+          case m[3]
+          when "raw"
+            push :raw
+          when "verbatim"
+            push :verbatim
+          end
+        end
+
         # Statements
         rule %r/\{\%/ do
           token Comment::Preproc
@@ -114,11 +125,6 @@ module Rouge
       end
 
       state :statement do
-        rule %r/(raw|verbatim)(\s+)(\%\})/ do
-          groups Keyword, Text, Comment::Preproc
-          goto :raw
-        end
-
         rule %r/(\w+\.?)/ do |m|
           if self.class.keywords.include?(m[0])
             groups Keyword
@@ -142,12 +148,21 @@ module Rouge
       end
 
       state :raw do
-        rule %r{(\{\%)(\s+)(endverbatim|endraw)(\s+)(\%\})} do
+        rule %r/({%-?)(\s*)(endraw)(\s*)(-?%})/ do
           groups Comment::Preproc, Text, Keyword, Text, Comment::Preproc
           pop!
         end
+        rule %r/[^{]+/, Text
+        rule %r/{/, Text
+      end
 
-        rule %r/(.+?)/m, Text
+      state :verbatim do
+        rule %r/({%-?)(\s*)(endverbatim)(\s*)(-?%})/ do
+          groups Comment::Preproc, Text, Keyword, Text, Comment::Preproc
+          pop!
+        end
+        rule %r/[^{]+/, Text
+        rule %r/{/, Text
       end
     end
   end

data/lib/rouge/lexers/jsl.rb

@@ -48,7 +48,7 @@ module Rouge
         rule %r/\\![btrnNf0\\"]/, Str::Escape
         rule %r/\\/, Str::Double
         rule %r/"/, Str::Double, :pop!
-        rule %r/[^\\"]*/m, Str::Double
+        rule %r/[^\\"]+/m, Str::Double
       end
     end
   end

data/lib/rouge/lexers/jsonnet.rb

@@ -126,6 +126,7 @@ module Rouge
 
       state :string do
         rule %r/\\([\\\/bfnrt]|(u[0-9a-fA-F]{4}))/, Str::Escape
+        rule %r/\\./, Str::Escape
       end
 
       state :string_double do
@@ -137,15 +138,15 @@ module Rouge
 
       state :string_single do
         mixin :string
-        rule %r/\\'/, Str::Escape
         rule %r/'/, Str, :pop!
         rule %r/[^\\']+/, Str
       end
 
       state :string_block do
         mixin :string
-        rule %r/\|\|\|/, Str, :pop!
-        rule %r/.*/, Str
+        rule %r/[|][|][|]/, Str, :pop!
+        rule %r/[^|\\]+/, Str
+        rule %r/[|]/, Str
       end
     end
   end

data/lib/rouge/lexers/jsp.rb

@@ -112,9 +112,8 @@ module Rouge
 
       state :jsp_interp_literal_start do
         rule %r/'/, Literal, :pop!
-        rule %r/[^']*/, Literal
+        rule %r/[^']+/, Literal
       end
-
     end
   end
-end
+end

data/lib/rouge/lexers/julia.rb

@@ -11,6 +11,8 @@ module Rouge
       filenames '*.jl'
       mimetypes 'text/x-julia', 'application/x-julia'
 
+      # Documentation: https://docs.julialang.org/en/v1/manual/variables/#Allowed-Variable-Names-1
+
       def self.detect?(text)
         return true if text.shebang? 'julia'
       end
@@ -252,13 +254,13 @@ module Rouge
 
 
       state :funcname do
-        rule %r/[a-zA-Z_]\w*/, Name::Function, :pop!
+        rule %r/[\p{L}\p{Nl}\p{S}_][\p{Word}\p{S}\p{Po}!]*/, Name::Function, :pop!
         rule %r/\([^\s\w{]{1,2}\)/, Operator, :pop!
         rule %r/[^\s\w{]{1,2}/, Operator, :pop!
       end
 
       state :typename do
-        rule %r/[a-zA-Z_]\w*/, Name::Class, :pop!
+        rule %r/[\p{L}\p{Nl}\p{S}_][\p{Word}\p{S}\p{Po}!]*/, Name::Class, :pop!
       end
 
       state :stringescape do

data/lib/rouge/lexers/kotlin.rb

@@ -53,7 +53,7 @@ module Rouge
           groups Keyword::Declaration, Text
           push :property
         end
-        rule %r'(return|continue|break|this|super)(@#{name})?' do
+        rule %r'(return|continue|break|this|super)(@#{name})?\b' do
           groups Keyword, Name::Decorator
         end
         rule %r'\bfun\b', Keyword
@@ -65,6 +65,9 @@ module Rouge
         rule %r'/[*].*[*]/', Comment::Multiline # single line block comment
         rule %r'/[*].*', Comment::Multiline, :comment # multiline block comment
         rule %r'\n', Text
+        rule %r'(::)(class)' do
+          groups Operator, Keyword
+        end
         rule %r'::|!!|\?[:.]', Operator
         rule %r"(\.\.)", Operator
         # Number literals
@@ -130,9 +133,10 @@ module Rouge
       end
 
       state :comment do
-        rule %r'\s*/[*].*', Comment::Multiline, :comment
-        rule %r'.*[*]/', Comment::Multiline, :pop!
-        rule %r'.*', Comment::Multiline
+        rule %r'/[*]', Comment::Multiline, :comment
+        rule %r'[*]/', Comment::Multiline, :pop!
+        rule %r'[^/*]+', Comment::Multiline
+        rule %r'[/*]', Comment::Multiline
       end
     end
   end

data/lib/rouge/lexers/opentype_feature_file.rb

@@ -91,7 +91,6 @@ module Rouge
       state :strings do
         rule %r/"/, Str, :pop!
         rule %r/[^"%\n]+/, Str
-        rule %r/(\([a-z0-9_]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?/i, Str
       end
     end
   end

data/lib/rouge/lexers/perl.rb

@@ -65,7 +65,7 @@ module Rouge
       end
 
       state :root do
-        rule %r/#.*?$/, Comment::Single
+        rule %r/#.*/, Comment::Single
         rule %r/^=[a-zA-Z0-9]+\s+.*?\n=cut/m, Comment::Multiline
         rule %r/(?:#{keywords.join('|')})\b/, Keyword
 
@@ -99,6 +99,9 @@ module Rouge
           re_tok, :balanced_regex
 
         rule %r/\s+/, Text
+
+        rule(/(?=[a-z_]\w*(\s*#.*\n)*\s*=>)/i) { push :fat_comma }
+
         rule %r/(?:#{builtins.join('|')})\b/, Name::Builtin
         rule %r/((__(DIE|WARN)__)|(DATA|STD(IN|OUT|ERR)))\b/,
           Name::Builtin::Pseudo
@@ -107,9 +110,10 @@ module Rouge
 
         rule %r/(__(END|DATA)__)\b/, Comment::Preproc, :end_part
         rule %r/\$\^[ADEFHILMOPSTWX]/, Name::Variable::Global
-        rule %r/\$[\\"'\[\]&`+*.,;=%~?@$!<>(^\|\/-](?!\w)/, Name::Variable::Global
+        rule %r/\$[\\"'\[\]&`+*.,;=%~?@$!<>(^\|\/_-](?!\w)/, Name::Variable::Global
+        rule %r/[$@%&*][$@%&*#_]*(?=[a-z{\[;])/i, Name::Variable, :varname
+
         rule %r/[-+\/*%=<>&^\|!\\~]=?/, Operator
-        rule %r/[$@%#]+/, Name::Variable, :varname
 
         rule %r/0_?[0-7]+(_[0-7]+)*/, Num::Oct
         rule %r/0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*/, Num::Hex
@@ -142,6 +146,13 @@ module Rouge
         rule %r/.*?\n/, Str::Interpol
       end
 
+      state :fat_comma do
+        rule %r/#.*/, Comment::Single
+        rule %r/\w+/, Str
+        rule %r/\s+/, Text
+        rule %r/=>/, Operator, :pop!
+      end
+
       state :name_common do
         rule %r/\w+::/, Name::Namespace
         rule %r/[\w:]+/, Name::Variable, :pop!
@@ -149,8 +160,9 @@ module Rouge
 
       state :varname do
         rule %r/\s+/, Text
-        rule %r/\{/, Punctuation, :pop! # hash syntax
-        rule %r/\)|,/, Punctuation, :pop! # arg specifier
+        rule %r/[{\[]/, Punctuation, :pop! # hash syntax
+        rule %r/[),]/, Punctuation, :pop! # arg specifier
+        rule %r/[;]/, Punctuation, :pop! # postfix
         mixin :name_common
       end
 
@@ -178,16 +190,24 @@ module Rouge
       end
 
       state :sq do
-        rule %r/\\[']/, Str::Escape
+        rule %r/\\[\\']/, Str::Escape
         rule %r/[^\\']+/, Str::Single
         rule %r/'/, Punctuation, :pop!
+        rule %r/\\/, Str::Single
       end
 
       state :dq do
         mixin :string_intp
-        rule %r/\\[\\tnr"]/, Str::Escape
+        rule %r/\\[\\tnrabefluLUE"$@]/, Str::Escape
+        rule %r/\\0\d{2}/, Str::Escape
+        rule %r/\\o\{\d+\}/, Str::Escape
+        rule %r/\\x\h{2}/, Str::Escape
+        rule %r/\\x\{\h+\}/, Str::Escape
+        rule %r/\\c./, Str::Escape
+        rule %r/\\N\{[^\}]+\}/, Str::Escape
         rule %r/[^\\"]+?/, Str::Double
         rule %r/"/, Punctuation, :pop!
+        rule %r/\\/, Str::Escape
       end
 
       state :bq do