rouge 4.1.3 → 4.7.0

data/lib/rouge/lexers/bicep.rb

@@ -0,0 +1,111 @@
+module Rouge
+  module Lexers
+    class Bicep < Rouge::RegexLexer
+      tag 'bicep'
+      filenames '*.bicep'
+
+      title "Bicep"
+      desc 'Bicep is a domain-specific language (DSL) that uses declarative syntax to deploy Azure resources.'
+
+      def self.keywords
+        @keywords ||= Set.new %w(
+          as assert existing extends extension false for from func if import in metadata module 
+          none null output param provider resource targetScope test true type using var void with
+        )
+      end
+
+      def self.datatypes
+        @datatypes ||= Set.new %w(array bool int object string)
+      end
+
+      def self.functions
+        @functions ||= Set.new %w(
+          array base64 base64ToJson base64ToString bool cidrHost cidrSubnet concat contains dataUri
+          dataUriToString dateTimeAdd dateTimeFromEpoch dateTimeToEpoch deployer deployment empty endsWith
+          environment extensionResourceId fail filter first flatten format getSecret groupBy guid indexOf int
+          intersection items join json last lastIndexOf length list* listAccountSas listKeys listSecrets loadFileAsBase64
+          loadJsonContent loadTextContent loadYamlContent managementGroup managementGroupResourceId map mapValue max min
+          newGuid objectKeys padLeft parseCidr pickZones range readEnvironmentVariable reduce reference replace resourceGroup
+          resourceId shallowMerge skip sort split startsWith string subscription subscriptionResourceId substring take tenant
+          tenantResourceId toLogicalZone toLower toObject toPhysicalZone toUpper trim union uniqueString uri uriComponent
+          uriComponentToString utcNow
+        )
+      end
+
+      operators = %w(+ - * / % < <= > >= == != =~ !~ && || ! ?? ... .?)
+
+      punctuations = %w(( ) { } [ ] , : ; = .)
+
+      state :root do
+        mixin :comments
+
+        # Match strings
+        rule %r/'/, Str::Single, :string
+
+        # Match numbers
+        rule %r/\b\d+\b/, Num
+
+        # Rules for sets of reserved keywords
+        rule %r/\b\w+\b/ do |m|
+          if self.class.keywords.include? m[0]
+            token Keyword
+          elsif self.class.datatypes.include? m[0]
+            token Keyword::Type
+          elsif self.class.functions.include? m[0]
+            token Name::Function
+          else
+            token Name
+          end
+        end
+
+        # Match operators
+        rule %r/#{operators.map { |o| Regexp.escape(o) }.join('|')}/, Operator
+
+        # Enter a state when encountering an opening curly bracket
+        rule %r/{/, Punctuation::Indicator, :block
+
+        # Match punctuation
+        rule %r/#{punctuations.map { |p| Regexp.escape(p) }.join('|')}/, Punctuation
+
+        # Match identifiers
+        rule %r/[a-zA-Z_]\w*/, Name
+
+        # Match decorators
+        rule %r/@[a-zA-Z_]\w*/, Name::Decorator
+
+        # Ignore whitespace
+        rule %r/\s+/, Text
+      end
+      
+      state :comments do
+        rule %r(//[^\n\r]+), Comment::Single
+        rule %r(/\*.*?\*/)m, Comment::Multiline
+      end
+
+      state :string do
+        rule %r/[^'$}]+/, Str::Single
+        rule %r/\$(?!\{)/, Str::Single
+        rule %r/\$[\{]/, Str::Interpol, :interp
+        rule %r/\'/, Str::Single, :pop!
+        rule %r/\$+/, Str::Single
+      end
+
+      state :interp do
+        rule %r/\}/, Str::Interpol, :pop!
+        mixin :root
+      end
+
+      # State for matching code blocks between curly brackets
+      state :block do
+        # Match property names
+        rule %r/\b([a-zA-Z_]\w*)\b(?=\s*:)/, Name::Property
+
+        # Match closing curly brackets
+        rule %r/}/, Punctuation::Indicator, :pop!
+
+        # Include the root state for nested tokens
+        mixin :root
+      end
+    end
+  end
+end

data/lib/rouge/lexers/bpf.rb

@@ -9,11 +9,11 @@ module Rouge
       tag 'bpf'
 
       TYPE_KEYWORDS = %w(
-        u8 u16 u32 u64 ll
+        u8 u16 u32 u64 s8 s16 s32 s64 ll
       ).join('|')
 
       MISC_KEYWORDS = %w(
-        be16 be32 be64 exit lock map
+        be16 be32 be64 le16 le32 le64 bswap16 bswap32 bswap64 exit lock map
       ).join('|')
 
       state :root do
@@ -32,21 +32,21 @@ module Rouge
         rule %r/(call)(\s+)(\d+)/i do
           groups Keyword, Text::Whitespace, Literal::Number::Integer
         end
-        rule %r/(call)(\s+)(\w+)(#)(\d+)/i do
+        rule %r/(call)(\s+)(\w+)(?:(#)(\d+))?/i do
           groups Keyword, Text::Whitespace, Name::Builtin, Punctuation, Literal::Number::Integer
         end
 
         # Unconditional jumps
-        rule %r/(goto)(\s*)(\+\d+)?(\s*)(<?\w+>?)/i do
-          groups Keyword, Text::Whitespace, Literal::Number::Integer, Text::Whitespace, Name::Label
+        rule %r/(gotol?)(\s*)([-+]0x\w+)?([-+]\d+)?(\s*)(<?\w+>?)/i do
+          groups Keyword, Text::Whitespace, Literal::Number::Hex, Literal::Number::Integer, Text::Whitespace, Name::Label
         end
 
         # Conditional jumps
-        rule %r/(if)(\s+)([rw]\d+)(\s*)([s!=<>]+)(\s*)(0x\h+|[-]?\d+)(\s*)(goto)(\s*)(\+\d+)?(\s*)(<?\w+>?)/i do
-          groups Keyword, Text::Whitespace, Name, Text::Whitespace, Operator, Text::Whitespace, Literal::Number, Text::Whitespace, Keyword, Text::Whitespace, Literal::Number::Integer, Text::Whitespace, Name::Label
+        rule %r/(if)(\s+)([rw]\d+)(\s*)([s!=<>]+)(\s*)(0x\h+|[-]?\d+)(\s*)(gotol?)(\s*)([-+]0x\w+)?([-+]\d+)?(\s*)(<?\w+>?)/i do
+          groups Keyword, Text::Whitespace, Name, Text::Whitespace, Operator, Text::Whitespace, Literal::Number, Text::Whitespace, Keyword, Text::Whitespace, Literal::Number::Hex, Literal::Number::Integer, Text::Whitespace, Name::Label
         end
-        rule %r/(if)(\s+)([rw]\d+)(\s*)([s!=<>]+)(\s*)([rw]\d+)(\s*)(goto)(\s*)(\+\d+)?(\s*)(<?\w+>?)/i do
-          groups Keyword, Text::Whitespace, Name, Text::Whitespace, Operator, Text::Whitespace, Name, Text::Whitespace, Keyword, Text::Whitespace, Literal::Number::Integer, Text::Whitespace, Name::Label
+        rule %r/(if)(\s+)([rw]\d+)(\s*)([s!=<>]+)(\s*)([rw]\d+)(\s*)(gotol?)(\s*)([-+]0x\w+)?([-+]\d+)?(\s*)(<?\w+>?)/i do
+          groups Keyword, Text::Whitespace, Name, Text::Whitespace, Operator, Text::Whitespace, Name, Text::Whitespace, Keyword, Text::Whitespace, Literal::Number::Hex, Literal::Number::Integer, Text::Whitespace, Name::Label
         end
 
         # Dereferences
@@ -70,7 +70,7 @@ module Rouge
         rule %r/#{MISC_KEYWORDS}/i, Keyword
 
         # Literals and global objects (maps) refered by name
-        rule %r/(0x\h+|[-]?\d+)(\s*)(ll)?/i do
+        rule %r/([-]?0x\h+|[-]?\d+)(\s*)(ll)?/i do
           groups Literal::Number, Text::Whitespace, Keyword::Type
         end
         rule %r/(\w+)(\s*)(ll)/i do

data/lib/rouge/lexers/brainfuck.rb

@@ -9,6 +9,7 @@ module Rouge
       mimetypes 'text/x-brainfuck'
 
       title "Brainfuck"
+      aliases "bf"
       desc "The Brainfuck programming language"
 
       start { push :bol }

data/lib/rouge/lexers/clean.rb

@@ -79,7 +79,7 @@ module Rouge
 
         rule %r/code(\s+inline)?\s*{/, Comment::Preproc, :abc
 
-        rule %r/_*[a-z][\w_`]*/ do |m|
+        rule %r/_*[a-z][\w`]*/ do |m|
           if self.class.keywords.include?(m[0])
             token Keyword
           else
@@ -87,7 +87,7 @@ module Rouge
           end
         end
 
-        rule %r/_*[A-Z][\w_`]*/ do |m|
+        rule %r/_*[A-Z][\w`]*/ do |m|
           if m[0]=='True' || m[0]=='False'
             token Keyword::Constant
           else
@@ -95,7 +95,7 @@ module Rouge
           end
         end
 
-        rule %r/[^\w_\s`]/, Punctuation
+        rule %r/[^\w\s`]/, Punctuation
         rule %r/_\b/, Punctuation
       end
 
@@ -136,7 +136,7 @@ module Rouge
 
         rule %r/}/, Comment::Preproc, :pop!
         rule %r/\.\w*/, Keyword, :abc_rest_of_line
-        rule %r/[\w_]+/, Name::Builtin, :abc_rest_of_line
+        rule %r/[\w]+/, Name::Builtin, :abc_rest_of_line
       end
 
       state :abc_rest_of_line do

data/lib/rouge/lexers/cobol.rb

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class COBOL < RegexLexer
+      title 'COBOL'
+      desc 'COBOL (Common Business-Oriented Language) programming language'
+      tag 'cobol'
+      filenames '*.cob', '*.cbl', '*.cpy', '*.cpb'
+      mimetypes 'text/x-cobol'
+
+      identifier = /\p{Alpha}[\p{Alnum}-]*/
+
+      def self.divisions
+        @divisions ||= %w(
+          IDENTIFICATION ENVIRONMENT DATA PROCEDURE DIVISION
+        )
+      end
+
+      def self.sections
+        @sections ||= %w(
+          CONFIGURATION INPUT-OUTPUT FILE WORKING-STORAGE LOCAL-STORAGE LINKAGE SECTION
+        )
+      end
+
+      # List of COBOL keywords
+      # sourced from https://www.ibm.com/docs/en/cobol-zos/6.4?topic=appendixes-reserved-words
+      def self.keywords
+        @keywords ||= Set.new(%w(
+          ACCEPT ACCESS ACTIVE-CLASS ADD ADDRESS ADVANCING AFTER ALIGNED ALL ALLOCATE ALPHABET ALPHABETIC ALPHABETIC-LOWER
+          ALPHABETIC-UPPER ALPHANUMERIC ALPHANUMERIC-EDITED ALSO ALTER ALTERNATE AND ANYCASE ANY APPLY ARE AREA AREAS
+          ASCENDING ASSIGN AT AUTHOR B-AND B-NOT B-OR B-XOR BASED BASIS BEFORE BEGINNING BINARY BINARY-CHAR BINARY-DOUBLE
+          BINARY-LONG BINARY-SHORT BIT BLANK BLOCK BOOLEAN BOTTOM BY BYTE-LENGTH CALL CANCEL CBL CD CF CH CHARACTER
+          CHARACTERS CLASS CLASS-ID CLOCK-UNITS CLOSE COBOL CODE CODE-SET COL COLLATING COLS COLUMN COLUMNS COM-REG COMMA
+          COMMON COMMUNICATION COMP-1 COMP-2 COMP-3 COMP-4 COMP-5 COMP COMPUTATIONAL-1 COMPUTATIONAL-2
+          COMPUTATIONAL-3 COMPUTATIONAL-4 COMPUTATIONAL-5 COMPUTATIONAL COMPUTE CONDITION CONSTANT CONTAINS CONTENT
+          CONTINUE CONTROL CONTROLS CONVERTING COPY CORR CORRESPONDING COUNT CRT CURRENCY CURSOR DATA-POINTER DATE
+          DATE-COMPILED DATE-WRITTEN DAY DAY-OF-WEEK DBCS DE DEBUG-CONTENTS DEBUG-ITEM DEBUG-LINE DEBUG-NAME DEBUG-SUB-1
+          DEBUG-SUB-2 DEBUG-SUB-3 DEBUGGING DECIMAL-POINT DECLARATIVES DEFAULT DELETE DELIMITED DELIMITER DEPENDING
+          DESCENDING DESTINATION DETAIL DISABLE DISPLAY-1 DISPLAY DIVIDE DOWN DUPLICATES DYNAMIC EC EGCS EGI
+          EJECT ELSE EMI ENABLE END-ACCEPT END-ADD END-CALL END-COMPUTE END-DELETE END-DISPLAY END-DIVIDE END-EVALUATE
+          END-EXEC END-IF END-INVOKE END-JSON END-MULTIPLY END-OF-PAGE END-PERFORM END-READ END-RECEIVE END-RETURN
+          END-REWRITE END-SEARCH END-START END-STRING END-SUBTRACT END-UNSTRING END-WRITE END-XML ENDING END ENTER ENTRY
+          EO EOP EQUAL ERROR ESI EVALUATE EVERY EXCEPTION EXCEPTION-OBJECT EXEC EXECUTE EXIT EXTEND EXTERNAL
+          FACTORY FALSE FD FILE-CONTROL FILLER FINAL FIRST FLOAT-EXTENDED FLOAT-LONG FLOAT-SHORT FOOTING FOR FORMAT
+          FREE FROM FUNCTION FUNCTION-ID FUNCTION-POINTER GENERATE GET GIVING GLOBAL GO GOBACK GREATER GROUP GROUP-USAGE
+          HEADING HIGH-VALUE HIGH-VALUES I-O-CONTROL I-O ID IF IN INDEX INDEXED INDICATE INHERITS INITIAL
+          INITIALIZE INITIATE INPUT INSERT INSPECT INSTALLATION INTERFACE INTERFACE-ID INTO INVALID INVOKE
+          IS JAVA JNIENVPTR JSON JSON-CODE JSON-STATUS JUST JUSTIFIED KANJI KEY LABEL LAST LEADING LEFT LENGTH LESS LIMIT
+          LIMITS LINAGE-COUNTER LINAGE LINE-COUNTER LINES LINE LOCALE LOCK LOW-VALUE LOW-VALUES
+          MEMORY MERGE MESSAGE METHOD METHOD-ID MINUS MODE MODULES MORE-LABELS MOVE MULTIPLE MULTIPLY NATIONAL
+          NATIONAL-EDITED NATIVE NEGATIVE NESTED NEXT NO NOT NULL NULLS NUMBER NUMERIC NUMERIC-EDITED OBJECT
+          OBJECT-COMPUTER OBJECT-REFERENCE OCCURS OF OFF OMITTED ON OPEN OPTIONAL OPTIONS OR ORDER ORGANIZATION
+          OTHER OUTPUT OVERFLOW OVERRIDE PACKED-DECIMAL PADDING PAGE PAGE-COUNTER PASSWORD PERFORM PF PH PIC PICTURE
+          PLUS POINTER- POINTER-31 POINTER-32 POINTER-64 POINTER POSITION POSITIVE PRESENT PRINTING
+          PROCEDURE-POINTER PROCEDURES PROCEED PROCESSING PROGRAM-ID PROGRAM-POINTER PROGRAM PROPERTY PROTOTYPE
+          PURGE QUEUE QUOTE QUOTES RAISE RAISING RANDOM RD READ READY RECEIVE RECORD RECORDING RECORDS RECURSIVE REDEFINES
+          REEL REFERENCE REFERENCES RELATIVE RELEASE RELOAD REMAINDER REMOVAL RENAMES REPLACE REPLACING REPORT REPORTING
+          REPORTS REPOSITORY RERUN RESERVE RESET RESUME RETRY RETURN RETURN-CODE RETURNING REVERSED REWIND REWRITE RF RH
+          RIGHT ROUNDED RUN SAME SCREEN SD SEARCH SECTION SECURITY SEGMENT SEGMENT-LIMIT SELECT SELF SEND SENTENCE
+          SEPARATE SEQUENCE SEQUENTIAL SERVICE SET SHARING SHIFT-IN SHIFT-OUT SIGN SIZE SKIP1 SKIP2 SKIP3
+          SORT-CONTROL SORT-CORE-SIZE SORT-FILE-SIZE SORT-MERGE SORT-MESSAGE SORT-MODE-SIZE SORT-RETURN SORT
+          SOURCE-COMPUTER SOURCES SOURCE SPACE SPACES SPECIAL-NAMES SQL SQLIMS STANDARD-1 STANDARD-2 STANDARD START STATUS STOP
+          STRING SUB-QUEUE-1 SUB-QUEUE-2 SUB-QUEUE-3 SUBTRACT SUM SUPER SUPPRESS SYMBOLIC SYNC SYNCHRONIZED SYSTEM-DEFAULT
+          TABLE TALLY TALLYING TAPE TERMINAL TERMINATE TEST TEXT THAN THEN THROUGH THRU TIME TIMES TITLE TO TOP TRACE
+          TRAILING TRUE TYPE TYPEDEF UNIT UNIVERSAL UNLOCK UNSTRING UNTIL UP UPON USAGE USE USER-DEFAULT USING UTF-8
+          VAL-STATUS VALID VALIDATE VALIDATE-STATUS VALUE VALUES VARYING VOLATILE WHEN WHEN-COMPILED WITH WORDS
+          WRITE WRITE-ONLY XML-CODE XML-EVENT XML-INFORMATION XML-NAMESPACE XML-NAMESPACE-PREFIX
+          XML-NNAMESPACE XML-NNAMESPACE-PREFIX XML-NTEXT XML-SCHEMA XML-TEXT XML ZERO ZEROES ZEROS
+        ))
+      end
+
+      state :root do
+        # First detect the comments
+        rule %r/^(      \*).*|^(^Debug \*).*/, Comment::Special
+
+        # Strings
+        rule %r/"/, Str::Double, :string_double
+        rule %r/'/, Str::Single, :string_single
+
+        # Keywords and divisions
+        rule  %r/(?<![\w-])#{identifier}(?![\w-])/i do |m|
+          if self.class.divisions.include?(m[0].upcase)
+            token Keyword::Declaration
+          elsif self.class.sections.include?(m[0].upcase)
+            token Keyword::Namespace
+          elsif self.class.keywords.include?(m[0].upcase)
+            token Keyword
+          else
+            token Name
+          end
+        end
+
+        # Numbers
+        rule %r/[-+]?\b\d+(\.\d+)?\b/, Num
+
+        # Punctuation
+        rule %r/[.,;:()]/, Punctuation
+
+        # Comments
+        rule %r/\*>.*/, Comment::Single
+
+        # Operators
+        rule %r/[+\-*\/><=]/, Operator
+
+        # Whitespace remaining
+        rule %r/\s/, Text::Whitespace
+
+        # Anything else remaining
+        rule %r/.+/, Text
+      end
+
+      # TODO double check string escaping in COBOL
+      # TODO Fix that a string opened by " can't be closed by '
+      # TODO Fix that strings can't be multi-line
+
+      # Handle strings where " opens a string and must be closed by "
+      state :string_double do
+        # Ensure strings can't span multiple lines
+        rule %r/[^"\\\n]+/, Str
+        rule %r/\\./, Str::Escape
+        rule %r/"/, Str::Double, :pop!
+        rule %r/\n/, Error # Flag an error if a string goes to the next line
+      end
+
+      # Handle strings where ' opens a string and must be closed by '
+      state :string_single do
+        # Ensure strings can't span multiple lines
+        rule %r/[^'\\\n]+/, Str
+        rule %r/\\./, Str::Escape
+        rule %r/'/, Str::Single, :pop!
+        rule %r/\n/, Error # Flag an error if a string goes to the next line
+      end
+    end
+  end
+end

data/lib/rouge/lexers/codeowners.rb

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Codeowners < RegexLexer
+      title 'CODEOWNERS'
+      desc 'Code Owners syntax (https://docs.gitlab.com/ee/user/project/codeowners/reference.html)'
+      tag 'codeowners'
+      filenames 'CODEOWNERS'
+
+      state :root do
+        rule %r/[ \t\r\n]+/, Text::Whitespace
+        rule %r/^\s*#.*$/, Comment::Single
+
+        rule %r(
+          (\^?\[(?!\d+\])[^\]]+\])
+          (\[\d+\])?
+        )x do
+          groups Name::Namespace, Literal::Number
+        end
+
+        rule %r/\S*@\S+/, Name::Function
+
+        rule %r/[\p{Word}\.\/\-\*]+/, Name
+        rule %r/.*\\[\#\s]/, Name
+      end
+    end
+  end
+end

data/lib/rouge/lexers/cpp.rb

@@ -27,7 +27,7 @@ module Rouge
           delete dynamic_cast explicit export friend
           mutable namespace new operator private protected public
           reinterpret_cast requires restrict size_of static_cast this throw throws
-          typeid typename using virtual final override
+          typeid typename using virtual final override import module
 
           alignas alignof decltype noexcept static_assert
           thread_local try
@@ -62,9 +62,10 @@ module Rouge
         rule %r/(class|struct)\b/, Keyword, :classname
         rule %r/template\b/, Keyword, :template
         rule %r/#{dq}(\.#{dq})?(?:y|d|h|(?:min)|s|(?:ms)|(?:us)|(?:ns)|i|(?:if)|(?:il))\b/, Num::Other
-        rule %r((#{dq}[.]#{dq}?|[.]#{dq})(e[+-]?#{dq}[lu]*)?)i, Num::Float
-        rule %r(#{dq}e[+-]?#{dq}[lu]*)i, Num::Float
-        rule %r/0x\h('?\h)*[lu]*/i, Num::Hex
+        rule %r((#{dq}[.]#{dq}?|[.]#{dq})([ep][+-]?#{dq})?[luf]*)i, Num::Float
+        rule %r(#{dq}[ep][+-]?#{dq}[luf]*)i, Num::Float
+        rule %r/0x\h('?\h)*([ep][+-]?#{dq})?[lu]*/i, Num::Hex
+        rule %r/0x\h('?\h)*[.]\h+([ep][+-]?#{dq})[luf]*/i, Num::Hex
         rule %r/0b[01]+('[01]+)*/, Num::Bin
         rule %r/0[0-7]('?[0-7])*[lu]*/i, Num::Oct
         rule %r/#{dq}[lu]*/i, Num::Integer

data/lib/rouge/lexers/crystal.rb

@@ -21,7 +21,7 @@ module Rouge
         rule %r(
           :  # initial :
           @{0,2} # optional ivar, for :@foo and :@@foo
-          [a-z_]\w*[!?]? # the symbol
+          [\p{Ll}_]\p{Word}*[!?]? # the symbol
         )xi, Str::Symbol
 
         # special symbols
@@ -35,7 +35,7 @@ module Rouge
         # %-sigiled strings
         # %(abc), %[abc], %<abc>, %.abc., %r.abc., etc
         delimiter_map = { '{' => '}', '[' => ']', '(' => ')', '<' => '>' }
-        rule %r/%([rqswQWxiI])?([^\w\s}])/ do |m|
+        rule %r/%([rqswQWxiI])?([^\p{Word}\s}])/ do |m|
           open = Regexp.escape(m[2])
           close = Regexp.escape(delimiter_map[m[2]] || m[2])
           interp = /[rQWxI]/ === m[1]
@@ -77,7 +77,7 @@ module Rouge
 
       state :strings do
         mixin :symbols
-        rule %r/\b[a-z_]\w*?[?!]?:\s+/, Str::Symbol, :expr_start
+        rule %r/\b[\p{Ll}_]\p{Word}*?[?!]?:\s+/, Str::Symbol, :expr_start
         rule %r/"/, Str::Double, :simple_string
         rule %r/(?<!\.)`/, Str::Backtick, :simple_backtick
         rule %r/(')(\\u[a-fA-F0-9]{4}|\\u\{[a-fA-F0-9]{1,6}\}|\\[abefnrtv])?(\\\\|\\'|[^'])*(')/ do
@@ -166,9 +166,9 @@ module Rouge
         rule %r/@\[([^\]]+)\]/, Name::Decorator
 
         # names
-        rule %r/@@[a-z_]\w*/i, Name::Variable::Class
-        rule %r/@[a-z_]\w*/i, Name::Variable::Instance
-        rule %r/\$\w+/, Name::Variable::Global
+        rule %r/@@[\p{Ll}_]\p{Word}*/i, Name::Variable::Class
+        rule %r/@[\p{Ll}_]\p{Word}*/i, Name::Variable::Instance
+        rule %r/\$\p{Word}+/, Name::Variable::Global
         rule %r(\$[!@&`'+~=/\\,;.<>_*\$?:"]), Name::Variable::Global
         rule %r/\$-[0adFiIlpvw]/, Name::Variable::Global
         rule %r/::/, Operator
@@ -181,7 +181,7 @@ module Rouge
         rule %r(
           (module)
           (\s+)
-          ([a-zA-Z_][a-zA-Z0-9_]*(::[a-zA-Z_][a-zA-Z0-9_]*)*)
+          ([\p{L}_][\p{L}0-9_]*(::[\p{L}_][\p{L}0-9_]*)*)
         )x do
           groups Keyword, Text, Name::Namespace
         end
@@ -207,14 +207,14 @@ module Rouge
         # Otherwise, they will be parsed as :method_call
         rule %r/\.{2,3}/, Operator, :expr_start
 
-        rule %r/[A-Z][a-zA-Z0-9_]*/, Name::Constant, :method_call
-        rule %r/(\.|::)(\s*)([a-z_]\w*[!?]?|[*%&^`~+-\/\[<>=])/ do
+        rule %r/[\p{Lu}][\p{L}0-9_]*/, Name::Constant, :method_call
+        rule %r/(\.|::)(\s*)([\p{Ll}_]\p{Word}*[!?]?|[*%&^`~+-\/\[<>=])/ do
           groups Punctuation, Text, Name::Function
           push :method_call
         end
 
-        rule %r/[a-zA-Z_]\w*[?!]/, Name, :expr_start
-        rule %r/[a-zA-Z_]\w*/, Name, :method_call
+        rule %r/[\p{L}_]\p{Word}*[?!]/, Name, :expr_start
+        rule %r/[\p{L}_]\p{Word}*/, Name, :method_call
         rule %r/\*\*|\/\/|>=|<=|<=>|<<?|>>?|=~|={3}|!~|&&?|\|\||\./,
           Operator, :expr_start
         rule %r/{%|%}/, Punctuation
@@ -225,7 +225,7 @@ module Rouge
       end
 
       state :has_heredocs do
-        rule %r/(?<!\w)(<<[-~]?)(["`']?)([a-zA-Z_]\w*)(\2)/ do |m|
+        rule %r/(?<!\p{Word})(<<[-~]?)(["`']?)([\p{L}_]\p{Word}*)(\2)/ do |m|
           token Operator, m[1]
           token Name::Constant, "#{m[2]}#{m[3]}#{m[4]}"
           @heredoc_queue << [['<<-', '<<~'].include?(m[1]), m[3]]
@@ -282,9 +282,9 @@ module Rouge
         rule %r/\s+/, Text
         rule %r/\(/, Punctuation, :defexpr
         rule %r(
-          (?:([a-zA-Z_]\w*)(\.))?
+          (?:([\p{L}_]\p{Word}*)(\.))?
           (
-            [a-zA-Z_]\w*[!?]? |
+            [\p{L}_]\p{Word}*[!?]? |
             \*\*? | [-+]@? | [/%&\|^`~] | \[\]=? |
             <<? | >>? | <=>? | >= | ===?
           )
@@ -311,7 +311,7 @@ module Rouge
           goto :expr_start
         end
 
-        rule %r/[A-Z_]\w*/, Name::Class, :pop!
+        rule %r/[\p{Lu}_]\p{Word}*/, Name::Class, :pop!
 
         rule(//) { pop! }
       end
@@ -343,7 +343,7 @@ module Rouge
 
       state :string_intp do
         rule %r/[#][{]/, Str::Interpol, :in_interp
-        rule %r/#(@@?|\$)[a-z_]\w*/i, Str::Interpol
+        rule %r/#(@@?|\$)[\p{Ll}_]\p{Word}*/i, Str::Interpol
       end
 
       state :string_intp_escaped do
@@ -399,7 +399,7 @@ module Rouge
         rule %r(
           [?](\\[MC]-)*     # modifiers
           (\\([\\abefnrstv\#"']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})|\S)
-          (?!\w)
+          (?!\p{Word})
         )x, Str::Char, :pop!
 
         # special case for using a single space.  Ruby demands that

data/lib/rouge/lexers/csharp.rb

@@ -12,35 +12,40 @@ module Rouge
       title "C#"
       desc 'a multi-paradigm language targeting .NET'
 
-      # TODO: support more of unicode
-      id = /@?[_a-z]\w*/i
-
-      #Reserved Identifiers
-      #Contextual Keywords
-      #LINQ Query Expressions
-      keywords = %w(
-        abstract as base break case catch checked const continue
-        default delegate do else enum event explicit extern false
-        finally fixed for foreach goto if implicit in interface
-        internal is lock new null operator out override params private
-        protected public readonly ref return sealed sizeof stackalloc
-        static switch this throw true try typeof unchecked unsafe
-        virtual void volatile while
-        add alias async await get global partial remove set value where
-        yield nameof notnull
-        ascending by descending equals from group in init into join let
-        on orderby select unmanaged when and not or with
-      )
-
-      keywords_type = %w(
-        bool byte char decimal double dynamic float int long nint nuint
-        object sbyte short string uint ulong ushort var
-      )
-
-      cpp_keywords = %w(
-        if endif else elif define undef line error warning region
-        endregion pragma nullable
-      )
+      id = /@?[_\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Nl}][\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Nl}\p{Nd}\p{Pc}\p{Cf}\p{Mn}\p{Mc}]*/
+
+      # Reserved Identifiers
+      # Contextual Keywords
+      # LINQ Query Expressions
+      def self.keywords
+        @keywords ||= %w(
+          abstract add alias and as ascending async await base
+          break by case catch checked const continue default delegate
+          descending do else enum equals event explicit extern false
+          finally fixed for foreach from get global goto group
+          if implicit in init interface internal into is join
+          let lock nameof new notnull null on operator orderby
+          out override params partial private protected public readonly
+          ref remove return sealed set sizeof stackalloc static
+          switch this throw true try typeof unchecked unsafe
+          unmanaged value virtual void volatile when where while
+          with yield
+        )
+      end
+
+      def self.keywords_type
+        @keywords_type ||= %w(
+          bool byte char decimal double dynamic float int long nint nuint
+          object sbyte short string uint ulong ushort var
+        )
+      end
+
+      def self.cpp_keywords
+        @cpp_keywords ||= %w(
+          if endif else elif define undef line error warning region
+          endregion pragma nullable
+        )
+      end
 
       state :whitespace do
         rule %r/\s+/m, Text
@@ -91,10 +96,9 @@ module Rouge
         )ix, Num
         rule %r/\b(?:class|record|struct|interface)\b/, Keyword, :class
         rule %r/\b(?:namespace|using)\b/, Keyword, :namespace
-        rule %r/^#[ \t]*(#{cpp_keywords.join('|')})\b.*?\n/,
-          Comment::Preproc
-        rule %r/\b(#{keywords.join('|')})\b/, Keyword
-        rule %r/\b(#{keywords_type.join('|')})\b/, Keyword::Type
+        rule %r/^#[ \t]*(#{CSharp.cpp_keywords.join('|')})\b.*?\n/, Comment::Preproc
+        rule %r/\b(#{CSharp.keywords.join('|')})\b/, Keyword
+        rule %r/\b(#{CSharp.keywords_type.join('|')})\b/, Keyword::Type
         rule %r/#{id}(?=\s*[(])/, Name::Function
         rule id, Name
       end
@@ -109,7 +113,6 @@ module Rouge
         rule %r/(?=[(])/, Text, :pop!
         rule %r/(#{id}|[.])+/, Name::Namespace, :pop!
       end
-
     end
   end
 end