rouge 3.1.0 → 3.18.0

data/lib/rouge/lexers/jsx.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rouge
   module Lexers
     load_lexer 'javascript.rb'
@@ -27,7 +29,7 @@ module Rouge
       start { @html = HTML.new(options) }
 
       state :jsx_tags do
-        rule /</, Punctuation, :jsx_element
+        rule %r/</, Punctuation, :jsx_element
       end
 
       state :jsx_internal do
@@ -36,12 +38,12 @@ module Rouge
           goto :jsx_end_tag
         end
 
-        rule /{/ do
+        rule %r/{/ do
           token Str::Interpol
           start_embed!
         end
 
-        rule /[^<>{]+/ do
+        rule %r/[^<>{]+/ do
           delegate @html
         end
 
@@ -54,29 +56,29 @@ module Rouge
 
       state :jsx_tag do
         mixin :comments_and_whitespace
-        rule /#{id}/ do |m|
+        rule %r/#{id}/ do |m|
           token tag_token(m[0])
         end
 
-        rule /[.]/, Punctuation
+        rule %r/[.]/, Punctuation
       end
 
       state :jsx_end_tag do
         mixin :jsx_tag
-        rule />/, Punctuation, :pop!
+        rule %r/>/, Punctuation, :pop!
       end
 
       state :jsx_element do
-        rule /#{id}=/, Name::Attribute, :jsx_attribute
+        rule %r/#{id}=/, Name::Attribute, :jsx_attribute
         mixin :jsx_tag
-        rule />/ do token Punctuation; goto :jsx_internal end
+        rule %r/>/ do token Punctuation; goto :jsx_internal end
         rule %r(/>), Punctuation, :pop!
       end
 
       state :jsx_attribute do
-        rule /"(\\[\\"]|[^"])*"/, Str::Double, :pop!
-        rule /'(\\[\\']|[^'])*'/, Str::Single, :pop!
-        rule /{/ do
+        rule %r/"(\\[\\"]|[^"])*"/, Str::Double, :pop!
+        rule %r/'(\\[\\']|[^'])*'/, Str::Single, :pop!
+        rule %r/{/ do
           token Str::Interpol
           pop!
           start_embed!
@@ -84,15 +86,15 @@ module Rouge
       end
 
       state :jsx_embed_root do
-        rule /[.][.][.]/, Punctuation
-        rule /}/, Str::Interpol, :pop!
+        rule %r/[.][.][.]/, Punctuation
+        rule %r/}/, Str::Interpol, :pop!
         mixin :jsx_embed
       end
 
       state :jsx_embed do
-        rule /{/ do delegate @embed; push :jsx_embed end
-        rule /}/ do delegate @embed; pop! end
-        rule /[^{}]+/ do
+        rule %r/{/ do delegate @embed; push :jsx_embed end
+        rule %r/}/ do delegate @embed; pop! end
+        rule %r/[^{}]+/ do
           delegate @embed
         end
       end

data/lib/rouge/lexers/julia.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
 
 module Rouge
   module Lexers
@@ -15,42 +16,153 @@ module Rouge
       end
 
       BUILTINS            = /\b(?:
-                              applicable      | assert    | convert
-                            | dlopen          | dlsym     | edit
-                            | eps             | error     | exit
-                            | finalizer       | hash      | im
-                            | Inf             | invoke    | is
-                            | isa             | isequal   | load
-                            | method_exists   | Nan       | new
-                            | ntuple          | pi        | promote
-                            | promote_type    | realmax   | realmin
-                            | sizeof          | subtype   | system
-                            | throw           | tuple     | typemax
-                            | typemin         | typeof    | uid
-                            | whos
+                              true      | false    | missing | nothing
+                            | Inf       | Inf16    | Inf32   | Inf64
+                            | NaN       | NaN16    | NaN32   | NaN64
+                            | stdout    | stderr   | stdin   | devnull
+                            | pi        | π        | ℯ       | im
+                            | ARGS      | C_NULL   | ENV     | ENDIAN_BOM
+                            | VERSION   | undef    | (LOAD|DEPOT)_PATH
                             )\b/x
 
       KEYWORDS            = /\b(?:
                               function | return | module | import | export
                             | if       | else   | elseif | end    | for
-                            | in       | while  | try    | catch  | super
-                            | const
+                            | in       | isa    | while  | try    | catch
+                            | const    | local  | global | using  | struct
+                            | mutable struct    | abstract type   | finally
+                            | begin    | do     | quote  | macro  | for outer
+                            | where
                             )\b/x
 
+      # NOTE: The list of types was generated automatically using the following script:
+      # using Pkg, InteractiveUtils
+      #
+      # allnames = [names(Core); names(Base, imported=true)]
+      #
+      # for stdlib in readdir(Pkg.Types.stdlib_dir())
+      #     mod = Symbol(basename(stdlib))
+      #     @eval begin
+      #         using $mod
+      #         append!(allnames, names($mod))
+      #     end
+      # end
+      #
+      # sort!(unique!(allnames))
+      #
+      # i = 1
+      # for sym in allnames
+      #     global i # needed at the top level, e.g. in the REPL
+      #     isdefined(Main, sym) || continue
+      #     getfield(which(Main, sym), sym) isa Type || continue
+      #     sym === :(=>) && continue # Actually an alias for Pair
+      #     print("| ", sym)
+      #     i % 3 == 0 ? println() : print(" ") # print 3 to a line
+      #     i += 1
+      # end
       TYPES               = /\b(?:
-                              Int         | UInt           | Int8
-                            | UInt8       | Int16          | UInt16
-                            | Int32       | UInt32         | Int64
-                            | UInt64      | Int128         | UInt128
-                            | Float16     | Float32        | Float64
-                            | Bool        | Inf            | Inf16
-                            | Inf32       | NaN            | NaN16
-                            | NaN32       | BigInt         | BigFloat
-                            | Char        | ASCIIString    | UTF8String
-                            | UTF16String | UTF32String    | AbstractString
-                            | WString     | String         | Regex
-                            | RegexMatch  | Complex64      | Complex128
-                            | Any         | Nothing        | None
+                              ARPACKException | AbstractArray | AbstractChannel
+                            | AbstractChar | AbstractDict | AbstractDisplay
+                            | AbstractFloat | AbstractIrrational | AbstractLogger
+                            | AbstractMatrix | AbstractREPL | AbstractRNG
+                            | AbstractRange | AbstractSerializer | AbstractSet
+                            | AbstractSparseArray | AbstractSparseMatrix | AbstractSparseVector
+                            | AbstractString | AbstractUnitRange | AbstractVecOrMat
+                            | AbstractVector | AbstractWorkerPool | Adjoint
+                            | Any | ArgumentError | Array
+                            | AssertionError | Base64DecodePipe | Base64EncodePipe
+                            | BasicREPL | Bidiagonal | BigFloat
+                            | BigInt | BitArray | BitMatrix
+                            | BitSet | BitVector | Bool
+                            | BoundsError | BunchKaufman | CachingPool
+                            | CapturedException | CartesianIndex | CartesianIndices
+                            | Cchar | Cdouble | Cfloat
+                            | Channel | Char | Cholesky
+                            | CholeskyPivoted | Cint | Cintmax_t
+                            | Clong | Clonglong | ClusterManager
+                            | Cmd | Colon | Complex
+                            | ComplexF16 | ComplexF32 | ComplexF64
+                            | CompositeException | Condition | ConsoleLogger
+                            | Cptrdiff_t | Cshort | Csize_t
+                            | Cssize_t | Cstring | Cuchar
+                            | Cuint | Cuintmax_t | Culong
+                            | Culonglong | Cushort | Cvoid
+                            | Cwchar_t | Cwstring | DataType
+                            | Date | DateFormat | DatePeriod
+                            | DateTime | Day | DenseArray
+                            | DenseMatrix | DenseVecOrMat | DenseVector
+                            | Diagonal | Dict | DimensionMismatch
+                            | Dims | DivideError | DomainError
+                            | EOFError | Eigen | Enum
+                            | ErrorException | Exception | ExponentialBackOff
+                            | Expr | FDWatcher | Factorization
+                            | FileMonitor | Float16 | Float32
+                            | Float64 | FolderMonitor | Function
+                            | GeneralizedEigen | GeneralizedSVD | GeneralizedSchur
+                            | GenericArray | GenericDict | GenericSet
+                            | GenericString | GitConfig | GitRepo
+                            | GlobalRef | HMAC_CTX | HTML
+                            | Hermitian | Hessenberg | Hour
+                            | IO | IOBuffer | IOContext
+                            | IOStream | IPAddr | IPv4
+                            | IPv6 | IdDict | IndexCartesian
+                            | IndexLinear | IndexStyle | InexactError
+                            | InitError | Int | Int128
+                            | Int16 | Int32 | Int64
+                            | Int8 | Integer | InterruptException
+                            | InvalidStateException | Irrational | KeyError
+                            | LAPACKException | LDLt | LQ
+                            | LU | LinRange | LineEditREPL
+                            | LineNumberNode | LinearIndices | LoadError
+                            | LogLevel | LowerTriangular | MIME
+                            | Matrix | MersenneTwister | Method
+                            | MethodError | Microsecond | Millisecond
+                            | Minute | Missing | MissingException
+                            | Module | Month | NTuple
+                            | NamedTuple | Nanosecond | Nothing
+                            | NullLogger | Number | OrdinalRange
+                            | OutOfMemoryError | OverflowError | PackageMode
+                            | PackageSpec | Pair | PartialQuickSort
+                            | Period | PermutedDimsArray | Pipe
+                            | PollingFileWatcher | PosDefException | ProcessExitedException
+                            | Ptr | QR | QRPivoted
+                            | QuoteNode | RandomDevice | RankDeficientException
+                            | Rational | RawFD | ReadOnlyMemoryError
+                            | Real | ReentrantLock | Ref
+                            | Regex | RegexMatch | RemoteChannel
+                            | RemoteException | RoundingMode | SHA1_CTX
+                            | SHA224_CTX | SHA256_CTX | SHA2_224_CTX
+                            | SHA2_256_CTX | SHA2_384_CTX | SHA2_512_CTX
+                            | SHA384_CTX | SHA3_224_CTX | SHA3_256_CTX
+                            | SHA3_384_CTX | SHA3_512_CTX | SHA512_CTX
+                            | SVD | Schur | Second
+                            | SegmentationFault | Serializer | Set
+                            | SharedArray | SharedMatrix | SharedVector
+                            | Signed | SimpleLogger | SingularException
+                            | Some | SparseMatrixCSC | SparseVector
+                            | StackOverflowError | StepRange | StepRangeLen
+                            | StreamREPL | StridedArray | StridedMatrix
+                            | StridedVecOrMat | StridedVector | String
+                            | StringIndexError | SubArray | SubString
+                            | SubstitutionString | SymTridiagonal | Symbol
+                            | Symmetric | SystemError | TCPSocket
+                            | Task | TestSetException | Text
+                            | TextDisplay | Time | TimePeriod
+                            | TimeType | TimeZone | Timer
+                            | Transpose | Tridiagonal | Tuple
+                            | Type | TypeError | TypeVar
+                            | UDPSocket | UInt | UInt128
+                            | UInt16 | UInt32 | UInt64
+                            | UInt8 | UTC | UUID
+                            | UndefInitializer | UndefKeywordError | UndefRefError
+                            | UndefVarError | UniformScaling | Union
+                            | UnionAll | UnitLowerTriangular | UnitRange
+                            | UnitUpperTriangular | Unsigned | UpgradeLevel
+                            | UpperTriangular | Val | Vararg
+                            | VecElement | VecOrMat | Vector
+                            | VersionNumber | WeakKeyDict | WeakRef
+                            | Week | WorkerConfig | WorkerPool
+                            | Year
                             )\b/x
 
       OPERATORS           = / \+      | =        | -     | \*   | \/
@@ -63,109 +175,115 @@ module Rouge
                               | <=    | ≤        | >=    | ≥    | \.
                               | ::    | <:       | ->    | \?   | \.\*
                               | \.\^  | \.\\     | \.\/  | \\   | <
-                              | >
+                              | >     | ÷        | >:    | :    | ===
+                              | !==   | =>
                             /x
 
-      PUNCTUATION         = / [ \[ \] { } : \( \) , ; @ ] /x
+      PUNCTUATION         = /[\[\]{}\(\),;]/
 
 
       state :root do
-        rule /\n/, Text
-        rule /[^\S\n]+/, Text
-        rule /#=/, Comment::Multiline, :blockcomment
-        rule /#.*$/, Comment
+        rule %r/\n/, Text
+        rule %r/[^\S\n]+/, Text
+        rule %r/#=/, Comment::Multiline, :blockcomment
+        rule %r/#.*$/, Comment
         rule OPERATORS, Operator
-        rule /\\\n/, Text
-        rule /\\/, Text
+        rule %r/\\\n/, Text
+        rule %r/\\/, Text
 
 
-        # functions
-        rule /(function)((?:\s|\\\s)+)/ do
+        # functions and macros
+        rule %r/(function|macro)((?:\s|\\\s)+)/ do
           groups Keyword, Name::Function
           push :funcname
         end
 
         # types
-        rule /(type|typealias|abstract)((?:\s|\\\s)+)/ do
+        rule %r/((?:mutable )?struct|(?:abstract|primitive) type)((?:\s|\\\s)+)/ do
           groups Keyword, Name::Class
           push :typename
         end
         rule TYPES, Keyword::Type
 
         # keywords
-        rule /(local|global|const)\b/, Keyword::Declaration
+        rule %r/(local|global|const)\b/, Keyword::Declaration
         rule KEYWORDS, Keyword
 
+        # TODO: end is a builtin when inside of an indexing expression
         rule BUILTINS, Name::Builtin
 
+        # TODO: symbols
+
         # backticks
-        rule /`.*?`/, Literal::String::Backtick
+        rule %r/`.*?`/, Literal::String::Backtick
 
         # chars
-        rule /'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,3}|\\u[a-fA-F0-9]{1,4}|\\U[a-fA-F0-9]{1,6}|[^\\\'\n])'/, Literal::String::Char
+        rule %r/'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,3}|\\u[a-fA-F0-9]{1,4}|\\U[a-fA-F0-9]{1,6}|[^\\\'\n])'/, Literal::String::Char
 
         # try to match trailing transpose
-        rule /(?<=[.\w)\]])\'+/, Operator
+        rule %r/(?<=[.\w)\]])\'+/, Operator
 
         # strings
-        rule /(?:[IL])"/, Literal::String, :string
-        rule /[E]?"/, Literal::String, :string
+        # TODO: triple quoted string literals
+        # TODO: Detect string interpolation
+        rule %r/(?:[IL])"/, Literal::String, :string
+        rule %r/[E]?"/, Literal::String, :string
 
         # names
-        rule /@[\w.]+/, Name::Decorator
-        rule /(?:[a-zA-Z_\u00A1-\uffff]|[\u1000-\u10ff])(?:[a-zA-Z_0-9\u00A1-\uffff]|[\u1000-\u10ff])*!*/, Name
+        rule %r/@[\w.]+/, Name::Decorator
+        rule %r/(?:[a-zA-Z_\u00A1-\uffff]|[\u1000-\u10ff])(?:[a-zA-Z_0-9\u00A1-\uffff]|[\u1000-\u10ff])*!*/, Name
 
         rule PUNCTUATION, Other
 
         # numbers
-        rule /(\d+(_\d+)+\.\d*|\d*\.\d+(_\d+)+)([eEf][+-]?[0-9]+)?/, Literal::Number::Float
-        rule /(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?/, Literal::Number::Float
-        rule /\d+(_\d+)+[eEf][+-]?[0-9]+/, Literal::Number::Float
-        rule /\d+[eEf][+-]?[0-9]+/, Literal::Number::Float
-        rule /0b[01]+(_[01]+)+/, Literal::Number::Bin
-        rule /0b[01]+/, Literal::Number::Bin
-        rule /0o[0-7]+(_[0-7]+)+/, Literal::Number::Oct
-        rule /0o[0-7]+/, Literal::Number::Oct
-        rule /0x[a-fA-F0-9]+(_[a-fA-F0-9]+)+/, Literal::Number::Hex
-        rule /0x[a-fA-F0-9]+/, Literal::Number::Hex
-        rule /\d+(_\d+)+/, Literal::Number::Integer
-        rule /\d+/, Literal::Number::Integer
+        rule %r/(\d+(_\d+)+\.\d*|\d*\.\d+(_\d+)+)([eEf][+-]?[0-9]+)?/, Literal::Number::Float
+        rule %r/(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?/, Literal::Number::Float
+        rule %r/\d+(_\d+)+[eEf][+-]?[0-9]+/, Literal::Number::Float
+        rule %r/\d+[eEf][+-]?[0-9]+/, Literal::Number::Float
+        rule %r/0b[01]+(_[01]+)+/, Literal::Number::Bin
+        rule %r/0b[01]+/, Literal::Number::Bin
+        rule %r/0o[0-7]+(_[0-7]+)+/, Literal::Number::Oct
+        rule %r/0o[0-7]+/, Literal::Number::Oct
+        rule %r/0x[a-fA-F0-9]+(_[a-fA-F0-9]+)+/, Literal::Number::Hex
+        rule %r/0x[a-fA-F0-9]+/, Literal::Number::Hex
+        rule %r/\d+(_\d+)+/, Literal::Number::Integer
+        rule %r/\d+/, Literal::Number::Integer
       end
 
 
       state :funcname do
-        rule /[a-zA-Z_]\w*/, Name::Function, :pop!
-        rule /\([^\s\w{]{1,2}\)/, Operator, :pop!
-        rule /[^\s\w{]{1,2}/, Operator, :pop!
+        rule %r/[a-zA-Z_]\w*/, Name::Function, :pop!
+        rule %r/\([^\s\w{]{1,2}\)/, Operator, :pop!
+        rule %r/[^\s\w{]{1,2}/, Operator, :pop!
       end
 
       state :typename do
-        rule /[a-zA-Z_]\w*/, Name::Class, :pop!
+        rule %r/[a-zA-Z_]\w*/, Name::Class, :pop!
       end
 
       state :stringescape do
-        rule /\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})/,
+        rule %r/\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})/,
           Literal::String::Escape
       end
 
       state :blockcomment do
-        rule /[^=#]/, Comment::Multiline
-        rule /#=/, Comment::Multiline, :blockcomment
-        rule /\=#/, Comment::Multiline, :pop!
-        rule /[=#]/, Comment::Multiline
+        rule %r/[^=#]/, Comment::Multiline
+        rule %r/#=/, Comment::Multiline, :blockcomment
+        rule %r/\=#/, Comment::Multiline, :pop!
+        rule %r/[=#]/, Comment::Multiline
       end
 
       state :string do
         mixin :stringescape
 
-        rule /"/, Literal::String, :pop!
-        rule /\\\\|\\"|\\\n/, Literal::String::Escape  # included here for raw strings
-        rule /\$(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?/, Literal::String::Interpol
-        rule /[^\\"$]+/, Literal::String
+        rule %r/"/, Literal::String, :pop!
+        rule %r/\\\\|\\"|\\\n/, Literal::String::Escape  # included here for raw strings
+        rule %r/\$(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?/, Literal::String::Interpol
+        rule %r/[^\\"$]+/, Literal::String
         # quotes, dollar signs, and backslashes must be parsed one at a time
-        rule /["\\]/, Literal::String
+        rule %r/["\\]/, Literal::String
         # unhandled string formatting sign
-        rule /\$/, Literal::String
+        rule %r/\$/, Literal::String
       end
     end
   end

data/lib/rouge/lexers/kotlin.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
 
 module Rouge
   module Lexers
@@ -9,7 +10,7 @@ module Rouge
       desc "Kotlin Programming Language (http://kotlinlang.org)"
 
       tag 'kotlin'
-      filenames '*.kt'
+      filenames '*.kt', '*.kts'
       mimetypes 'text/x-kotlin'
 
       keywords = %w(
@@ -18,31 +19,17 @@ module Rouge
         external false final finally for fun get if import in infix
         inline inner interface internal is lateinit noinline null
         object open operator out override package private protected
-        public reified return sealed set super tailrec this throw
-        true try typealias typeof val var vararg when where while
-        yield
+        public reified return sealed set super suspend tailrec this
+        throw true try typealias typeof val var vararg when where
+        while yield
       )
 
-      name = %r'@?[_\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Nl}][\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Nl}\p{Nd}\p{Pc}\p{Cf}\p{Mn}\p{Mc}]*'
+      name_chars = %r'[-_\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Nl}\p{Nd}\p{Pc}\p{Cf}\p{Mn}\p{Mc}]*'
 
-      id = %r'(#{name}|`#{name}`)'
+      class_name = %r'`?[\p{Lu}]#{name_chars}`?'
+      name = %r'`?[_\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Nl}]#{name_chars}`?'
 
       state :root do
-        rule %r'^\s*\[.*?\]', Name::Attribute
-        rule %r'[^\S\n]+', Text
-        rule %r'\\\n', Text # line continuation
-        rule %r'//.*?$', Comment::Single
-        rule %r'/[*].*?[*]/'m, Comment::Multiline
-        rule %r'\n', Text
-        rule %r'::|!!|\?[:.]', Operator
-        rule %r"(\.\.)", Operator
-        rule %r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation
-        rule %r'[{}]', Punctuation
-        rule %r'@"(""|[^"])*"'m, Str
-        rule %r'""".*?"""'m, Str
-        rule %r'"(\\\\|\\"|[^"\n])*["\n]'m, Str
-        rule %r"'\\.'|'[^\\]'", Str::Char
-        rule %r"[0-9](\.[0-9]+)?([eE][+-][0-9]+)?[flFL]?|0[xX][0-9a-fA-F]+[Ll]?", Num
         rule %r'\b(companion)(\s+)(object)\b' do
           groups Keyword, Text, Keyword
         end
@@ -50,29 +37,94 @@ module Rouge
           groups Keyword::Declaration, Text
           push :class
         end
+        rule %r'\b(fun)(\s+)' do
+          groups Keyword, Text
+          push :function
+        end
         rule %r'\b(package|import)(\s+)' do
           groups Keyword, Text
           push :package
         end
+        rule %r'\b(val|var)(\s+)(\()' do
+          groups Keyword::Declaration, Text, Punctuation
+          push :destructure
+        end
         rule %r'\b(val|var)(\s+)' do
           groups Keyword::Declaration, Text
           push :property
         end
-        rule %r/\bfun\b/, Keyword
-        rule /\b(?:#{keywords.join('|')})\b/, Keyword
-        rule id, Name
+        rule %r'(return|continue|break|this|super)(@#{name})?' do
+          groups Keyword, Name::Decorator
+        end
+        rule %r'\bfun\b', Keyword
+        rule %r'\b(?:#{keywords.join('|')})\b', Keyword
+        rule %r'^\s*\[.*?\]', Name::Attribute
+        rule %r'[^\S\n]+', Text
+        rule %r'\\\n', Text # line continuation
+        rule %r'//.*?$', Comment::Single
+        rule %r'/[*].*[*]/', Comment::Multiline # single line block comment
+        rule %r'/[*].*', Comment::Multiline, :comment # multiline block comment
+        rule %r'\n', Text
+        rule %r'::|!!|\?[:.]', Operator
+        rule %r"(\.\.)", Operator
+        rule %r'[~!%^&*()+=|\[\]:;,.<>/?-]', Punctuation
+        rule %r'[{}]', Punctuation
+        rule %r'@"(""|[^"])*"'m, Str
+        rule %r'""".*?"""'m, Str
+        rule %r'"(\\\\|\\"|[^"\n])*["\n]'m, Str
+        rule %r"'\\.'|'[^\\]'", Str::Char
+        rule %r"[0-9](\.[0-9]+)?([eE][+-][0-9]+)?[flFL]?|0[xX][0-9a-fA-F]+[Ll]?", Num
+        rule %r'(@#{class_name})', Name::Decorator
+        rule %r'(#{class_name})(<)' do
+          groups Name::Class, Punctuation
+          push :generic_parameters
+        end
+        rule class_name, Name::Class
+        rule %r'(#{name})(?=\s*[({])', Name::Function
+        rule %r'(#{name})@', Name::Decorator # label
+        rule name, Name
       end
 
       state :package do
-        rule /\S+/, Name::Namespace, :pop!
+        rule %r'\S+', Name::Namespace, :pop!
       end
 
       state :class do
-        rule id, Name::Class, :pop!
+        rule class_name, Name::Class, :pop!
+      end
+
+      state :function do
+        rule %r'(<)', Punctuation, :generic_parameters
+        rule %r'(\s+)', Text
+        rule %r'(#{class_name})(\.)' do
+          groups Name::Class, Punctuation
+        end
+        rule name, Name::Function, :pop!
+      end
+
+      state :generic_parameters do
+        rule class_name, Name::Class
+        rule %r'(<)', Punctuation, :generic_parameters
+        rule %r'(,)', Punctuation
+        rule %r'(\s+)', Text
+        rule %r'(>)', Punctuation, :pop!
       end
 
       state :property do
-        rule id, Name::Property, :pop!
+        rule name, Name::Property, :pop!
+      end
+
+      state :destructure do
+        rule %r'(,)', Punctuation
+        rule %r'(\))', Punctuation, :pop!
+        rule %r'(\s+)', Text
+        rule name, Name::Property
+      end
+
+      state :comment do
+        rule %r'\s*/[*].*', Comment::Multiline, :comment
+        rule %r'.*[*]/', Comment::Multiline, :pop!
+        rule %r'.*', Comment::Multiline
       end
     end
   end