rouge 3.20.0 → 3.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a0ad085dddf353cc95e795e5f556b4f2d9821068cceeeaab4cb34cb11f3ad776
-  data.tar.gz: f70271022198b1d7e95df35d856274234720d54d4ef3f1a42c782246e2cf4c47
+  metadata.gz: 8ef1072cc2523dba637a191d195865b0b2fb9f38c4f6d9961fb58748306f6cd1
+  data.tar.gz: 4cd1b0d5262fa8a1d4ddb564cf4e9a185d6e8043bee0261f119e7a8ccb81e1fd
 SHA512:
-  metadata.gz: 473cfdc4ce0c458d0a7b0c3b1e6684723ceb7c15dea4d1080baf201786ecd149e9d21cd25f65735edbbf4b4f763a5d7d1854a4d909bef6902b7d609d30ebcc2b
-  data.tar.gz: 579d31241b2fe8d4f2b9471dd6bb97b5621051811260d02b21a012e9e35ccf34d83e0e998981e959aa0ce6a1b00932c06f7ed8b8b093fde95d53c7a8ebbb67ce
+  metadata.gz: 04a39cda7513ce8084b6cdddd3016e7ca679218688dfb2dcb6f6fd8a0e627ec189f38a31ba2c44bd2a8c5b16191e24d0f343f860db5dfbcbb7ca49b92aaff197
+  data.tar.gz: 50172d87860fa9fb4b187dee66c72954ce926e0d73f3e78e1a15a7f3437e749d48b2f935fa6ad08be457a6dd317a3c5c20340a7f7cc1621825b97887e82822c1

data/lib/rouge/demos/brightscript

@@ -0,0 +1,6 @@
+function main(args as dynamic) as void
+    screen = CreateObject("roSGScreen")
+    'Create a scene and load /components/helloworld.xml'
+    scene = screen.CreateScene("HelloWorld")
+    screen.show()
+end function

data/lib/rouge/demos/janet

@@ -0,0 +1,3 @@
+(defn a-fun
+  "A function"
+  (do-something))

data/lib/rouge/demos/ssh

@@ -0,0 +1,4 @@
+Host example
+    Hostname example.com
+    User user
+    Port 1234

data/lib/rouge/lexers/batchfile.rb

@@ -78,7 +78,7 @@ module Rouge
 
       state :basic do
         # Comments
-        rule %r/\brem\b.*$/i, Comment
+        rule %r/@?\brem\b.*$/i, Comment
         # Empty Labels
         rule %r/^::.*$/, Comment
 

data/lib/rouge/lexers/brightscript.rb

@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Brightscript < RegexLexer
+      title "BrightScript"
+      desc "BrightScript Programming Language (https://developer.roku.com/en-ca/docs/references/brightscript/language/brightscript-language-reference.md)"
+      tag 'brightscript'
+      aliases 'bs', 'brs'
+      filenames '*.brs'
+
+      # https://developer.roku.com/en-ca/docs/references/brightscript/language/global-utility-functions.md
+      # https://developer.roku.com/en-ca/docs/references/brightscript/language/global-string-functions.md
+      # https://developer.roku.com/en-ca/docs/references/brightscript/language/global-math-functions.md
+      def self.name_builtin
+        @name_builtin ||= Set.new %w(
+          ABS ASC ATN CDBL CHR CINT CONTROL COPYFILE COS CREATEDIRECTORY CSNG
+          DELETEDIRECTORY DELETEFILE EXP FINDMEMBERFUNCTION FINDNODE FIX
+          FORMATDRIVEFORMATJSON GETINTERFACE INSTR INT LCASE LEFT LEN LISTDIR
+          LOG MATCHFILES MID MOVEFILE OBSERVEFIELD PARSEJSON PARSEXML
+          READASCIIFILE REBOOTSYSTEM RIGHT RND RUNGARBAGECOLLECTOR SGN SIN
+          SLEEP SQR STR STRI STRING STRINGI STRTOI SUBSTITUTE TANTEXTTOP TEXT
+          TRUCASE UPTIME VALVISIBLE VISIBLE WAIT
+        )
+      end
+
+      # https://developer.roku.com/en-ca/docs/references/brightscript/language/reserved-words.md
+      def self.keyword_reserved
+        @keyword_reserved ||= Set.new %w(
+          BOX CREATEOBJECT DIM EACH ELSE ELSEIF END ENDFUNCTION ENDIF ENDSUB
+          ENDWHILE EVAL EXIT EXITWHILE FALSE FOR FUNCTION GETGLOBALAA
+          GETLASTRUNCOMPILEERROR GETLASTRUNRUNTIMEERROR GOTO IF IN INVALID LET
+          LINE_NUM M NEXT OBJFUN POS PRINT REM RETURN RUN STEP STOP SUB TAB TO
+          TRUE TYPE WHILE
+        )
+      end
+
+      # These keywords are present in BrightScript, but not supported in standard .brs files
+      def self.keyword_reserved_unsupported
+        @keyword_reserved_unsupported ||= Set.new %w(
+          CLASS CONST IMPORT LIBRARY NAMESPACE PRIVATE PROTECTED PUBLIC
+        )
+      end
+
+      # https://developer.roku.com/en-ca/docs/references/brightscript/language/expressions-variables-types.md
+      def self.keyword_type
+        @keyword_type ||= Set.new %w(
+          BOOLEAN DIM DOUBLE DYNAMIC FLOAT FUNCTION INTEGER INTERFACE INVALID
+          LONGINTEGER OBJECT STRING VOID
+        )
+      end
+
+      # https://developer.roku.com/en-ca/docs/references/brightscript/language/expressions-variables-types.md#operators
+      def self.operator_word
+        @operator_word ||= Set.new %w(
+          AND AS MOD NOT OR THEN
+        )
+      end
+
+      # Scene graph components configured as builtins. See BrightScript component documentation e.g.
+      # https://developer.roku.com/en-ca/docs/references/brightscript/components/roappinfo.md
+      def self.builtins
+        @builtins ||= Set.new %w(
+          roAppendFile roAppInfo roAppManager roArray roAssociativeArray
+          roAudioGuide roAudioMetadata roAudioPlayer roAudioPlayerEvent
+          roAudioResourceroBitmap roBoolean roBoolean roBrightPackage roBrSub
+          roButton roByteArray roCaptionRenderer roCaptionRendererEvent
+          roCecInterface roCECStatusEvent roChannelStore roChannelStoreEvent
+          roClockWidget roCodeRegistrationScreen
+          roCodeRegistrationScreenEventroCompositor roControlDown roControlPort
+          roControlPort roControlUp roCreateFile roDatagramReceiver
+          roDatagramSender roDataGramSocket roDateTime roDeviceInfo
+          roDeviceInfoEvent roDoubleroEVPCipher roEVPDigest roFileSystem
+          roFileSystemEvent roFloat roFont roFontMetrics roFontRegistry
+          roFunction roGlobal roGpio roGridScreen roGridScreenEvent
+          roHdmiHotPlugEventroHdmiStatus roHdmiStatusEvent roHMAC roHttpAgent
+          roImageCanvas roImageCanvasEvent roImageMetadata roImagePlayer
+          roImageWidgetroInput roInputEvent roInt roInt roInvalid roInvalid
+          roIRRemote roKeyboard roKeyboardPress roKeyboardScreen
+          roKeyboardScreenEventroList roListScreen roListScreenEvent
+          roLocalization roLongInteger roMessageDialog roMessageDialogEvent
+          roMessagePort roMicrophone roMicrophoneEvent roNetworkConfiguration
+          roOneLineDialog roOneLineDialogEventroParagraphScreen
+          roParagraphScreenEvent roPath roPinEntryDialog roPinEntryDialogEvent
+          roPinentryScreen roPosterScreen roPosterScreenEventroProgramGuide
+          roQuadravoxButton roReadFile roRectangleroRegexroRegion roRegistry
+          roRegistrySection roResourceManager roRSA roRssArticle roRssParser
+          roScreen roSearchHistory roSearchScreen roSearchScreenEvent
+          roSerialPort roSGNode roSGNodeEvent roSGScreenroSGScreenEvent
+          roSlideShowroSlideShowEvent roSNS5 roSocketAddress roSocketEvent
+          roSpringboardScreen roSpringboardScreenEventroSprite roStorageInfo
+          roStreamSocket roStringroSystemLogroSystemLogEvent roSystemTime
+          roTextFieldroTextScreen roTextScreenEvent roTextToSpeech
+          roTextToSpeechEvent roTextureManager roTextureRequest
+          roTextureRequestEventroTextWidget roTimer roTimespan roTouchScreen
+          roTunerroTunerEvent roUniversalControlEvent roUrlEvent roUrlTransfer
+          roVideoEvent roVideoInput roVideoMode roVideoPlayer roVideoPlayerEvent
+          roVideoScreen roVideoScreenEventroWriteFile roXMLElement roXMLList
+        )
+      end
+
+      id = /[$a-z_][a-z0-9_]*/io
+
+      state :root do
+        rule %r/\s+/m, Text::Whitespace
+
+        # https://developer.roku.com/en-ca/docs/references/brightscript/language/expressions-variables-types.md#comments
+        rule %r/\'.*/, Comment::Single
+        rule %r/REM.*/i, Comment::Single
+
+        # https://developer.roku.com/en-ca/docs/references/brightscript/language/expressions-variables-types.md#operators
+        rule %r([~!%^&*+=\|?:<>/-]), Operator
+
+        rule %r/\d*\.\d+(e-?\d+)?/i, Num::Float
+        rule %r/\d+[lu]*/i, Num::Integer
+
+        rule %r/".*?"/, Str::Double
+
+        rule %r/#{id}(?=\s*[(])/, Name::Function
+
+        rule %r/[()\[\],.;{}]/, Punctuation
+
+        rule id do |m|
+          caseSensitiveChunk = m[0]
+          caseInsensitiveChunk = m[0].upcase
+
+          if self.class.builtins.include?(caseSensitiveChunk)
+            token Keyword::Reserved
+          elsif self.class.keyword_reserved.include?(caseInsensitiveChunk)
+            token Keyword::Reserved
+          elsif self.class.keyword_reserved_unsupported.include?(caseInsensitiveChunk)
+            token Keyword::Reserved
+          elsif self.class.keyword_type.include?(caseInsensitiveChunk)
+            token Keyword::Type
+          elsif self.class.name_builtin.include?(caseInsensitiveChunk)
+            token Name::Builtin
+          elsif self.class.operator_word.include?(caseInsensitiveChunk)
+            token Operator::Word
+          else
+            token Name
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/cpp.rb

@@ -75,15 +75,16 @@ module Rouge
         rule id, Name::Class, :pop!
 
         # template specification
-        rule %r/\s*(?=>)/m, Text, :pop!
-        rule %r/[.]{3}/, Operator
         mixin :whitespace
+        rule %r/[.]{3}/, Operator
+        rule %r/,/, Punctuation, :pop!
+        rule(//) { pop! }
       end
 
       state :template do
-        rule %r/>/, Punctuation, :pop!
+        rule %r/[>;]/, Punctuation, :pop!
         rule %r/typename\b/, Keyword, :classname
-        mixin :root
+        mixin :statements
       end
 
       state :case do

data/lib/rouge/lexers/css.rb

@@ -11,7 +11,9 @@ module Rouge
       filenames '*.css'
       mimetypes 'text/css'
 
-      identifier = /[a-zA-Z0-9_-]+/
+      # Documentation: https://www.w3.org/TR/CSS21/syndata.html#characters
+
+      identifier = /[\p{L}_-][\p{Word}\p{Cf}-]*/
       number = /-?(?:[0-9]+(\.[0-9]+)?|\.[0-9]+)/
 
       def self.attributes

data/lib/rouge/lexers/docker.rb

@@ -8,7 +8,7 @@ module Rouge
       desc "Dockerfile syntax"
       tag 'docker'
       aliases 'dockerfile'
-      filenames 'Dockerfile', '*.Dockerfile', '*.docker', 'Dockerfile_*'
+      filenames 'Dockerfile', '*.Dockerfile', '*.docker'
       mimetypes 'text/x-dockerfile-config'
 
       KEYWORDS = %w(

data/lib/rouge/lexers/html.rb

@@ -47,13 +47,13 @@ module Rouge
         rule %r(</), Name::Tag, :tag_end
         rule %r/</, Name::Tag, :tag_start
 
-        rule %r(<\s*[a-zA-Z0-9:-]+), Name::Tag, :tag # opening tags
-        rule %r(<\s*/\s*[a-zA-Z0-9:-]+\s*>), Name::Tag # closing tags
+        rule %r(<\s*[\p{L}:_-][\p{Word}\p{Cf}:.·-]*), Name::Tag, :tag # opening tags
+        rule %r(<\s*/\s*[\p{L}:_-][\p{Word}\p{Cf}:.·-]*\s*>), Name::Tag # closing tags
       end
 
       state :tag_end do
         mixin :tag_end_end
-        rule %r/[a-zA-Z0-9:-]+/ do
+        rule %r/[\p{L}:_-][\p{Word}\p{Cf}:.·-]*/ do
           token Name::Tag
           goto :tag_end_end
         end
@@ -67,7 +67,7 @@ module Rouge
       state :tag_start do
         rule %r/\s+/, Text
 
-        rule %r/[a-zA-Z0-9:-]+/ do
+        rule %r/[\p{L}:_-][\p{Word}\p{Cf}:.·-]*/ do
           token Name::Tag
           goto :tag
         end
@@ -83,8 +83,8 @@ module Rouge
 
       state :tag do
         rule %r/\s+/m, Text
-        rule %r/[a-zA-Z0-9_:\[\]()*.-]+\s*=\s*/m, Name::Attribute, :attr
-        rule %r/[a-zA-Z0-9_:#*-]+/, Name::Attribute
+        rule %r/[\p{L}:_\[\]()*.-][\p{Word}\p{Cf}:.·\[\]()*-]*\s*=\s*/m, Name::Attribute, :attr
+        rule %r/[\p{L}:_*#-][\p{Word}\p{Cf}:.·*#-]*/, Name::Attribute
         rule %r(/?\s*>)m, Name::Tag, :pop!
       end
 

data/lib/rouge/lexers/janet.rb

@@ -0,0 +1,217 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Janet < RegexLexer
+      title "Janet"
+      desc "The Janet programming language (janet-lang.org)"
+
+      tag 'janet'
+      aliases 'jdn'
+
+      filenames '*.janet', '*.jdn'
+
+      mimetypes 'text/x-janet', 'application/x-janet'
+
+      def self.specials
+        @specials ||= Set.new %w(
+          break def do fn if quote quasiquote splice set unquote var while
+        )
+      end
+
+      def self.bundled
+        @bundled ||= Set.new %w(
+          % %= * *= + ++ += - -- -= -> ->> -?> -?>> / /= < <= = > >=
+          abstract? accumulate accumulate2 all all-bindings
+          all-dynamics and apply array array/concat array/ensure
+          array/fill array/insert array/new array/new-filled
+          array/peek array/pop array/push array/remove array/slice
+          array? as-> as?-> asm assert bad-compile bad-parse band
+          blshift bnot boolean? bor brshift brushift buffer buffer/bit
+          buffer/bit-clear buffer/bit-set buffer/bit-toggle
+          buffer/blit buffer/clear buffer/fill buffer/format
+          buffer/new buffer/new-filled buffer/popn buffer/push-byte
+          buffer/push-string buffer/push-word buffer/slice buffer?
+          bxor bytes? case cfunction? chr cli-main comment comp
+          compare compare= compare< compare<= compare> compare>=
+          compile complement comptime cond coro count debug
+          debug/arg-stack debug/break debug/fbreak debug/lineage
+          debug/stack debug/stacktrace debug/step debug/unbreak
+          debug/unfbreak debugger-env dec deep-not= deep= default
+          default-peg-grammar def- defer defmacro defmacro- defn defn-
+          defglobal describe dictionary? disasm distinct doc doc*
+          doc-format dofile drop drop-until drop-while dyn each eachk
+          eachp eachy edefer eflush empty? env-lookup eprin eprinf
+          eprint eprintf error errorf eval eval-string even? every?
+          extreme false? fiber/can-resume? fiber/current fiber/getenv
+          fiber/maxstack fiber/new fiber/root fiber/setenv
+          fiber/setmaxstack fiber/status fiber? file/close file/flush
+          file/open file/popen file/read file/seek file/temp
+          file/write filter find find-index first flatten flatten-into
+          flush for forv freeze frequencies function? gccollect
+          gcinterval gcsetinterval generate gensym get get-in getline
+          hash idempotent? identity import import* if-let if-not
+          if-with in inc indexed? int/s64 int/u64 int? interleave
+          interpose invert janet/build janet/config-bits janet/version
+          juxt juxt* keep keys keyword keyword? kvs label last length
+          let load-image load-image-dict loop macex macex1 make-env
+          make-image make-image-dict map mapcat marshal math/-inf
+          math/abs math/acos math/acosh math/asin math/asinh math/atan
+          math/atan2 math/atanh math/cbrt math/ceil math/cos math/cosh
+          math/e math/erf math/erfc math/exp math/exp2 math/expm1
+          math/floor math/gamma math/hypot math/inf math/log
+          math/log10 math/log1p math/log2 math/next math/pi math/pow
+          math/random math/rng math/rng-buffer math/rng-int
+          math/rng-uniform math/round math/seedrandom math/sin
+          math/sinh math/sqrt math/tan math/tanh math/trunc match max
+          mean merge merge-into min mod module/add-paths module/cache
+          module/expand-path module/find module/loaders module/loading
+          module/paths nan? nat? native neg? net/chunk net/close
+          net/connect net/read net/server net/write next nil? not not=
+          number? odd? one? or os/arch os/cd os/chmod os/clock
+          os/cryptorand os/cwd os/date os/dir os/environ os/execute
+          os/exit os/getenv os/link os/lstat os/mkdir os/mktime
+          os/perm-int os/perm-string os/readlink os/realpath os/rename
+          os/rm os/rmdir os/setenv os/shell os/sleep os/stat
+          os/symlink os/time os/touch os/umask os/which pairs parse
+          parser/byte parser/clone parser/consume parser/eof
+          parser/error parser/flush parser/has-more parser/insert
+          parser/new parser/produce parser/state parser/status
+          parser/where partial partition peg/compile peg/match pos?
+          postwalk pp prewalk prin prinf print printf product prompt
+          propagate protect put put-in quit range reduce reduce2
+          repeat repl require resume return reverse reversed root-env
+          run-context scan-number seq setdyn shortfn signal slice
+          slurp some sort sort-by sorted sorted-by spit stderr stdin
+          stdout string string/ascii-lower string/ascii-upper
+          string/bytes string/check-set string/find string/find-all
+          string/format string/from-bytes string/has-prefix?
+          string/has-suffix? string/join string/repeat string/replace
+          string/replace-all string/reverse string/slice string/split
+          string/trim string/triml string/trimr string? struct struct?
+          sum symbol symbol? table table/clone table/getproto
+          table/new table/rawget table/setproto table/to-struct table?
+          take take-until take-while tarray/buffer tarray/copy-bytes
+          tarray/length tarray/new tarray/properties tarray/slice
+          tarray/swap-bytes thread/close thread/current thread/new
+          thread/receive thread/send trace tracev true? truthy? try
+          tuple tuple/brackets tuple/setmap tuple/slice
+          tuple/sourcemap tuple/type tuple? type unless unmarshal
+          untrace update update-in use values var- varfn varglobal
+          walk walk-ind walk-dict when when-let when-with with
+          with-dyns with-syms with-vars yield zero? zipcoll
+        )
+      end
+
+      def name_token(name)
+        if self.class.specials.include? name
+          Keyword
+        elsif self.class.bundled.include? name
+          Keyword::Reserved
+        else
+          Name::Function
+        end
+      end
+
+      punctuation = %r/[_!@$%^&*+=~<>.?\/-]/o
+      symbol = %r/([[:alpha:]]|#{punctuation})([[:word:]]|#{punctuation}|:)*/o
+
+      state :root do
+        rule %r/#.*?$/, Comment::Single
+        rule %r/\s+/m, Text::Whitespace
+
+        rule %r/(true|false|nil)\b/, Name::Constant
+        rule %r/(['~])(#{symbol})/ do
+          groups Operator, Str::Symbol
+        end
+        rule %r/:([[:word:]]|#{punctuation}|:)*/, Keyword::Constant
+
+        # radix-specified numbers
+        rule %r/[+-]?\d{1,2}r[\w.]+(&[+-]?\w+)?/, Num::Float
+
+        # hex numbers
+        rule %r/[+-]?0x\h[\h_]*(\.\h[\h_]*)?/, Num::Hex
+        rule %r/[+-]?0x\.\h[\h_]*/, Num::Hex
+
+        # decimal numbers (Janet treats all decimals as floats)
+        rule %r/[+-]?\d[\d_]*(\.\d[\d_]*)?([e][+-]?\d+)?/i, Num::Float
+        rule %r/[+-]?\.\d[\d_]*([e][+-]?\d+)?/i, Num::Float
+
+        rule %r/@?"/, Str::Double, :string
+        rule %r/@?(`+).*?\1/m, Str::Heredoc
+
+        rule %r/\(/, Punctuation, :function
+
+        rule %r/(')([\(\[])/ do
+          groups Operator, Punctuation
+          push :quote
+        end
+
+        rule %r/(~)([\(\[])/ do
+          groups Operator, Punctuation
+          push :quasiquote
+        end
+
+        rule %r/[\#~,';\|]/, Operator
+
+        rule %r/@?[({\[]/, Punctuation, :push
+        rule %r/[)}\]]/, Punctuation, :pop!
+
+        rule symbol, Name
+      end
+
+      state :string do
+        rule %r/"/, Str::Double, :pop!
+        rule %r/\\(u\h{4}|U\h{6})/, Str::Escape
+        rule %r/\\./, Str::Escape
+        rule %r/[^"\\]+/, Str::Double
+      end
+
+      state :function do
+        rule symbol do |m|
+          case m[0]
+          when "quote"
+            token Keyword
+            goto :quote
+          when "quasiquote"
+            token Keyword
+            goto :quasiquote
+          else
+            token name_token(m[0])
+            goto :root
+          end
+        end
+
+        mixin :root
+      end
+
+      state :quote do
+        rule %r/[\(\[]/, Punctuation, :push
+        rule symbol, Str::Symbol
+        mixin :root
+      end
+
+      state :quasiquote do
+        rule %r/(,)(\()/ do
+          groups Operator, Punctuation
+          push :function
+        end
+        rule %r/(\()(\s*)(unquote)(\s+)(\()/ do
+          groups Punctuation, Text, Keyword, Text, Punctuation
+          push :root
+          push :function
+        end
+
+        rule %r/(,)(#{symbol})/ do
+          groups Operator, Name
+        end
+        rule %r/(\()(\s*)(unquote)(\s+)(#{symbol})/ do
+          groups Punctuation, Text, Keyword, Text, Name
+        end
+
+        mixin :quote
+      end
+    end
+  end
+end

data/lib/rouge/lexers/javascript.rb

@@ -19,6 +19,8 @@ module Rouge
       mimetypes 'application/javascript', 'application/x-javascript',
                 'text/javascript', 'text/x-javascript'
 
+      # Pseudo-documentation: https://stackoverflow.com/questions/1661197/what-characters-are-valid-for-javascript-variable-names
+
       def self.detect?(text)
         return 1 if text.shebang?('node')
         return 1 if text.shebang?('jsc')
@@ -138,7 +140,7 @@ module Rouge
       end
 
       def self.id_regex
-        /[$a-z_][a-z0-9_]*/io
+        /[\p{L}\p{Nl}$_][\p{Word}]*/io
       end
 
       id = self.id_regex

data/lib/rouge/lexers/jinja.rb

@@ -40,6 +40,17 @@ module Rouge
         rule %r/{#/, Comment, :comment
         rule %r/##.*/, Comment
 
+        # Raw and verbatim
+        rule %r/({%-?)(\s*)(raw|verbatim)(\s*)(-?%})/ do |m|
+          groups Comment::Preproc, Text, Keyword, Text, Comment::Preproc
+          case m[3]
+          when "raw"
+            push :raw
+          when "verbatim"
+            push :verbatim
+          end
+        end
+
         # Statements
         rule %r/\{\%/ do
           token Comment::Preproc
@@ -114,11 +125,6 @@ module Rouge
       end
 
       state :statement do
-        rule %r/(raw|verbatim)(\s+)(\%\})/ do
-          groups Keyword, Text, Comment::Preproc
-          goto :raw
-        end
-
         rule %r/(\w+\.?)/ do |m|
           if self.class.keywords.include?(m[0])
             groups Keyword
@@ -142,12 +148,21 @@ module Rouge
       end
 
       state :raw do
-        rule %r{(\{\%)(\s+)(endverbatim|endraw)(\s+)(\%\})} do
+        rule %r/({%-?)(\s*)(endraw)(\s*)(-?%})/ do
           groups Comment::Preproc, Text, Keyword, Text, Comment::Preproc
           pop!
         end
+        rule %r/[^{]+/, Text
+        rule %r/{/, Text
+      end
 
-        rule %r/(.+?)/m, Text
+      state :verbatim do
+        rule %r/({%-?)(\s*)(endverbatim)(\s*)(-?%})/ do
+          groups Comment::Preproc, Text, Keyword, Text, Comment::Preproc
+          pop!
+        end
+        rule %r/[^{]+/, Text
+        rule %r/{/, Text
       end
     end
   end

data/lib/rouge/lexers/julia.rb

@@ -11,6 +11,8 @@ module Rouge
       filenames '*.jl'
       mimetypes 'text/x-julia', 'application/x-julia'
 
+      # Documentation: https://docs.julialang.org/en/v1/manual/variables/#Allowed-Variable-Names-1
+
       def self.detect?(text)
         return true if text.shebang? 'julia'
       end
@@ -252,13 +254,13 @@ module Rouge
 
 
       state :funcname do
-        rule %r/[a-zA-Z_]\w*/, Name::Function, :pop!
+        rule %r/[\p{L}\p{Nl}\p{S}_][\p{Word}\p{S}\p{Po}!]*/, Name::Function, :pop!
         rule %r/\([^\s\w{]{1,2}\)/, Operator, :pop!
         rule %r/[^\s\w{]{1,2}/, Operator, :pop!
       end
 
       state :typename do
-        rule %r/[a-zA-Z_]\w*/, Name::Class, :pop!
+        rule %r/[\p{L}\p{Nl}\p{S}_][\p{Word}\p{S}\p{Po}!]*/, Name::Class, :pop!
       end
 
       state :stringescape do

data/lib/rouge/lexers/perl.rb

@@ -65,7 +65,7 @@ module Rouge
       end
 
       state :root do
-        rule %r/#.*?$/, Comment::Single
+        rule %r/#.*/, Comment::Single
         rule %r/^=[a-zA-Z0-9]+\s+.*?\n=cut/m, Comment::Multiline
         rule %r/(?:#{keywords.join('|')})\b/, Keyword
 
@@ -99,6 +99,9 @@ module Rouge
           re_tok, :balanced_regex
 
         rule %r/\s+/, Text
+
+        rule(/(?=[a-z_]\w*(\s*#.*\n)*\s*=>)/i) { push :fat_comma }
+
         rule %r/(?:#{builtins.join('|')})\b/, Name::Builtin
         rule %r/((__(DIE|WARN)__)|(DATA|STD(IN|OUT|ERR)))\b/,
           Name::Builtin::Pseudo
@@ -142,6 +145,13 @@ module Rouge
         rule %r/.*?\n/, Str::Interpol
       end
 
+      state :fat_comma do
+        rule %r/#.*/, Comment::Single
+        rule %r/\w+/, Str
+        rule %r/\s+/, Text
+        rule %r/=>/, Operator, :pop!
+      end
+
       state :name_common do
         rule %r/\w+::/, Name::Namespace
         rule %r/[\w:]+/, Name::Variable, :pop!
@@ -178,16 +188,24 @@ module Rouge
       end
 
       state :sq do
-        rule %r/\\[']/, Str::Escape
+        rule %r/\\[\\']/, Str::Escape
         rule %r/[^\\']+/, Str::Single
         rule %r/'/, Punctuation, :pop!
+        rule %r/\\/, Str::Single
       end
 
       state :dq do
         mixin :string_intp
-        rule %r/\\[\\tnr"]/, Str::Escape
+        rule %r/\\[\\tnrabefluLUE"$@]/, Str::Escape
+        rule %r/\\0\d{2}/, Str::Escape
+        rule %r/\\o\{\d+\}/, Str::Escape
+        rule %r/\\x\h{2}/, Str::Escape
+        rule %r/\\x\{\h+\}/, Str::Escape
+        rule %r/\\c./, Str::Escape
+        rule %r/\\N\{[^\}]+\}/, Str::Escape
         rule %r/[^\\"]+?/, Str::Double
         rule %r/"/, Punctuation, :pop!
+        rule %r/\\/, Str::Escape
       end
 
       state :bq do

data/lib/rouge/lexers/powershell.rb

@@ -165,6 +165,7 @@ module Rouge
       end
 
       state :parameters do
+        rule %r/`./m, Str::Escape
         rule %r/\s*?\n/, Text::Whitespace, :pop!
         rule %r/[;(){}\]]/, Punctuation, :pop!
         rule %r/[|=]/, Operator, :pop!
@@ -217,6 +218,7 @@ module Rouge
           push :parameters unless m[3].nil?
         end
 
+        rule %r/\?/, Name::Function, :parameters
         rule %r/[-+*\/%=!.&|]/, Operator
         rule %r/@\{/, Punctuation, :hasht
         rule %r/@\(/, Punctuation, :array

data/lib/rouge/lexers/rego.rb

@@ -9,36 +9,51 @@ module Rouge
             tag 'rego'
             filenames '*.rego'
 
+            def self.constants
+              @constants ||= Set.new %w(
+                true false null
+              )
+            end
+
+            def self.operators
+              @operators ||= Set.new %w(
+                as default else import not package some with
+              )
+            end
+
             state :basic do
               rule %r/\s+/, Text
               rule %r/#.*/, Comment::Single
-      
+
               rule %r/[\[\](){}|.,;!]/, Punctuation
-      
+
               rule %r/"[^"]*"/, Str::Double
-      
+
               rule %r/-?\d+\.\d+([eE][+-]?\d+)?/, Num::Float
               rule %r/-?\d+([eE][+-]?\d+)?/, Num
 
               rule %r/\\u[0-9a-fA-F]{4}/, Num::Hex
               rule %r/\\["\/bfnrt]/, Str::Escape
             end
-      
-            state :atoms do
-              rule %r/(true|false|null)/, Keyword::Constant
-              rule %r/[[:word:]]*/, Str::Symbol
-            end
-      
+
             state :operators do
               rule %r/(=|!=|>=|<=|>|<|\+|-|\*|%|\/|\||&|:=)/, Operator
-              rule %r/(default|not|package|import|as|with|else|some)/, Operator
               rule %r/[\/:?@^~]+/, Operator
             end
-      
+
             state :root do
               mixin :basic
               mixin :operators
-              mixin :atoms
+
+              rule %r/[[:word:]]+/ do |m|
+                if self.class.constants.include? m[0]
+                  token Keyword::Constant
+                elsif self.class.operators.include? m[0]
+                  token Operator::Word
+                else
+                  token Name
+                end
+              end
             end
         end
     end

data/lib/rouge/lexers/sass/common.rb

@@ -29,6 +29,7 @@ module Rouge
         end
 
         rule %r/@#{id}/, Keyword, :selector
+        rule %r/&/, Keyword, :selector
 
         # $variable: assignment
         rule %r/([$]#{id})([ \t]*)(:)/ do

data/lib/rouge/lexers/ssh.rb

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class SSH < RegexLexer
+      tag 'ssh'
+
+      title "SSH Config File"
+      desc 'A lexer for SSH configuration files'
+      filenames 'ssh_config'
+
+      state :root do
+        rule %r/[a-z0-9]+/i, Keyword, :statement
+        mixin :base
+      end
+
+      state :statement do
+        rule %r/\n/, Text, :pop!
+        rule %r/(?:yes|no|confirm|ask|always|auto|none|force)\b/, Name::Constant
+
+        rule %r/\d+/, Num
+        rule %r/[^#\s;{}$\\]+/, Text
+        mixin :base
+      end
+
+      state :base do
+        rule %r/\s+/, Text
+        rule %r/#.*/, Comment::Single
+      end
+    end
+  end
+end

data/lib/rouge/lexers/twig.rb

@@ -17,10 +17,10 @@ module Rouge
 
       def self.keywords
         @@keywords ||= %w(as do extends flush from import include use else starts
-                          ends with without autoescape endautoescape block endblock
-                          embed endembed filter endfilter for endfor if endif
-                          macro endmacro sandbox endsandbox set endset
-                          spaceless endspaceless verbatim endverbatim)
+                          ends with without autoescape endautoescape block
+                          endblock embed endembed filter endfilter for endfor
+                          if endif macro endmacro sandbox endsandbox set endset
+                          spaceless endspaceless)
       end
 
       def self.tests

data/lib/rouge/lexers/xml.rb

@@ -12,6 +12,8 @@ module Rouge
       mimetypes 'text/xml', 'application/xml', 'image/svg+xml',
                 'application/rss+xml', 'application/atom+xml'
 
+      # Documentation: https://www.w3.org/TR/xml11/#charsets and https://www.w3.org/TR/xml11/#sec-suggested-names
+
       def self.detect?(text)
         return false if text.doctype?(/html/)
         return true if text =~ /\A<\?xml\b/
@@ -27,10 +29,10 @@ module Rouge
         rule %r/<![^>]*>/, Comment::Preproc
 
         # open tags
-        rule %r(<\s*[\w:.-]+)m, Name::Tag, :tag
+        rule %r(<\s*[\p{L}:_][\p{Word}\p{Cf}:.·-]*)m, Name::Tag, :tag
 
         # self-closing tags
-        rule %r(<\s*/\s*[\w:.-]+\s*>)m, Name::Tag
+        rule %r(<\s*/\s*[\p{L}:_][\p{Word}\p{Cf}:.·-]*\s*>)m, Name::Tag
       end
 
       state :comment do
@@ -41,7 +43,7 @@ module Rouge
 
       state :tag do
         rule %r/\s+/m, Text
-        rule %r/[\w.:-]+\s*=/m, Name::Attribute, :attr
+        rule %r/[\p{L}:_][\p{Word}\p{Cf}:.·-]*\s*=/m, Name::Attribute, :attr
         rule %r(/?\s*>), Name::Tag, :pop!
       end
 

data/lib/rouge/lexers/yaml.rb

@@ -11,6 +11,8 @@ module Rouge
       aliases 'yml'
       filenames '*.yaml', '*.yml'
 
+      # Documentation: https://yaml.org/spec/1.2/spec.html
+
       def self.detect?(text)
         # look for the %YAML directive
         return true if text =~ /\A\s*%YAML/m
@@ -165,15 +167,15 @@ module Rouge
         )x, Keyword::Type
 
         # an anchor
-        rule %r/&[\w-]+/, Name::Label
+        rule %r/&[\p{L}\p{Nl}\p{Nd}_-]+/, Name::Label
 
         # an alias
-        rule %r/\*[\w-]+/, Name::Variable
+        rule %r/\*[\p{L}\p{Nl}\p{Nd}_-]+/, Name::Variable
       end
 
       state :block_nodes do
         # implicit key
-        rule %r/((?:\w[\w -]*)?)(:)(?=\s|$)/ do |m|
+        rule %r/((?:[\p{L}\p{Nl}\p{Nd}_][\p{L}\p{Nl}\p{Nd}\p{Blank}_-]*)?)(:)(?=\s|$)/ do |m|
           groups Name::Attribute, Punctuation::Indicator
           set_indent m[0], :implicit => true
         end

data/lib/rouge/version.rb

@@ -3,6 +3,6 @@
 
 module Rouge
   def self.version
-    "3.20.0"
+    "3.21.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rouge
 version: !ruby/object:Gem::Version
-  version: 3.20.0
+  version: 3.21.0
 platform: ruby
 authors:
 - Jeanine Adkisson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-09 00:00:00.000000000 Z
+date: 2020-07-14 00:00:00.000000000 Z
 dependencies: []
 description: Rouge aims to a be a simple, easy-to-extend drop-in replacement for pygments.
 email:
@@ -39,6 +39,7 @@ files:
 - lib/rouge/demos/biml
 - lib/rouge/demos/bpf
 - lib/rouge/demos/brainfuck
+- lib/rouge/demos/brightscript
 - lib/rouge/demos/bsl
 - lib/rouge/demos/c
 - lib/rouge/demos/ceylon
@@ -108,6 +109,7 @@ files:
 - lib/rouge/demos/irb
 - lib/rouge/demos/irb_output
 - lib/rouge/demos/isbl
+- lib/rouge/demos/janet
 - lib/rouge/demos/java
 - lib/rouge/demos/javascript
 - lib/rouge/demos/jinja
@@ -190,6 +192,7 @@ files:
 - lib/rouge/demos/sparql
 - lib/rouge/demos/sqf
 - lib/rouge/demos/sql
+- lib/rouge/demos/ssh
 - lib/rouge/demos/supercollider
 - lib/rouge/demos/swift
 - lib/rouge/demos/tap
@@ -257,6 +260,7 @@ files:
 - lib/rouge/lexers/biml.rb
 - lib/rouge/lexers/bpf.rb
 - lib/rouge/lexers/brainfuck.rb
+- lib/rouge/lexers/brightscript.rb
 - lib/rouge/lexers/bsl.rb
 - lib/rouge/lexers/c.rb
 - lib/rouge/lexers/ceylon.rb
@@ -327,6 +331,7 @@ files:
 - lib/rouge/lexers/irb.rb
 - lib/rouge/lexers/isbl.rb
 - lib/rouge/lexers/isbl/builtins.rb
+- lib/rouge/lexers/janet.rb
 - lib/rouge/lexers/java.rb
 - lib/rouge/lexers/javascript.rb
 - lib/rouge/lexers/jinja.rb
@@ -420,6 +425,7 @@ files:
 - lib/rouge/lexers/sqf.rb
 - lib/rouge/lexers/sqf/keywords.rb
 - lib/rouge/lexers/sql.rb
+- lib/rouge/lexers/ssh.rb
 - lib/rouge/lexers/supercollider.rb
 - lib/rouge/lexers/swift.rb
 - lib/rouge/lexers/tap.rb