rouge 3.3.0 → 3.4.0

data/lib/rouge/lexers/c.rb

@@ -106,7 +106,7 @@ module Rouge
         rule /\d+[lu]*/i, Num::Integer
         rule %r(\*/), Error
         rule %r([~!%^&*+=\|?:<>/-]), Operator
-        rule /[()\[\],.]/, Punctuation
+        rule /[()\[\],.;]/, Punctuation
         rule /\bcase\b/, Keyword, :case
         rule /(?:true|false|NULL)\b/, Name::Builtin
         rule id do |m|
@@ -133,13 +133,11 @@ module Rouge
 
       state :root do
         mixin :expr_whitespace
-
-        # functions
         rule %r(
           ([\w*\s]+?[\s*]) # return arguments
           (#{id})          # function name
           (\s*\([^;]*?\))  # signature
-          (#{ws})({)         # open brace
+          (#{ws}?)({|;)    # open brace or semicolon
         )mx do |m|
           # TODO: do this better.
           recurse m[1]
@@ -147,33 +145,12 @@ module Rouge
           recurse m[3]
           recurse m[4]
           token Punctuation, m[5]
-          push :function
-        end
-
-        # function declarations
-        rule %r(
-          ([\w*\s]+?[\s*]) # return arguments
-          (#{id})          # function name
-          (\s*\([^;]*?\))  # signature
-          (#{ws})(;)       # semicolon
-        )mx do |m|
-          # TODO: do this better.
-          recurse m[1]
-          token Name::Function, m[2]
-          recurse m[3]
-          recurse m[4]
-          token Punctuation, m[5]
-          push :statement
+          if m[5] == ?{
+            push :function
+          end
         end
-
-        rule(//) { push :statement }
-      end
-
-      state :statement do
-        rule /;/, Punctuation, :pop!
-        mixin :expr_whitespace
+        rule /\{/, Punctuation, :function
         mixin :statements
-        rule /[{}]/, Punctuation
       end
 
       state :function do

data/lib/rouge/lexers/coffeescript.rb

@@ -18,9 +18,17 @@ module Rouge
 
       def self.keywords
         @keywords ||= Set.new %w(
-          for in of while break return continue switch when then if else
-          throw try catch finally new delete typeof instanceof super
-          extends this class by
+          for by while until loop break continue return
+          switch when then if else do yield throw try catch finally await
+          new delete typeof instanceof super extends this class
+          import export debugger
+        )
+      end
+
+      def self.reserved
+        @reserved ||= Set.new %w(
+          case function var void with const let enum
+          native implements interface package private protected public static
         )
       end
 
@@ -43,7 +51,7 @@ module Rouge
 
       state :comments_and_whitespace do
         rule /\s+/m, Text
-        rule /###\s*\n.*?###/m, Comment::Multiline
+        rule /###[^#].*?###/m, Comment::Multiline
         rule /#.*$/, Comment::Single
       end
 
@@ -77,8 +85,8 @@ module Rouge
         rule(%r(^(?=\s|/|<!--))) { push :slash_starts_regex }
         mixin :comments_and_whitespace
         rule %r(
-          [+][+]|--|~|&&|\band\b|\bor\b|\bis\b|\bisnt\b|\bnot\b|[?]|:|=|
-          [|][|]|\\(?=\n)|(<<|>>>?|==?|!=?|[-<>+*`%&|^/])=?
+          [+][+]|--|~|&&|\band\b|\bor\b|\bis\b|\bisnt\b|\bnot\b|\bin\b|\bof\b|
+          [?]|:|=|[|][|]|\\(?=\n)|(<<|>>>?|==?|!=?|[-<>+*`%&|^/])=?
         )x, Operator, :slash_starts_regex
 
         rule /[-=]>/, Name::Function

data/lib/rouge/lexers/common_lisp.rb

@@ -320,7 +320,7 @@ module Rouge
 
         rule /\(/, Punctuation, :root
         rule /\)/, Punctuation do
-          if stack.empty?
+          if stack.size == 1
             token Error
           else
             token Punctuation

data/lib/rouge/lexers/console.rb

@@ -118,7 +118,7 @@ module Rouge
           $' =~ /\A\s*/
           yield Text, $& unless $&.empty?
 
-          lang_lexer.lex($', continue: true, &output)
+          lang_lexer.continue_lex($', &output)
         elsif comment_regex =~ input[0].strip
           puts "console: matched comment #{input[0].inspect}" if @debug
           output_lexer.reset!
@@ -129,7 +129,7 @@ module Rouge
           puts "console: matched output #{input[0].inspect}" if @debug
           lang_lexer.reset!
 
-          output_lexer.lex(input[0], continue: true, &output)
+          output_lexer.continue_lex(input[0], &output)
         end
       end
     end

data/lib/rouge/lexers/coq.rb

@@ -141,7 +141,7 @@ module Rouge
       end
 
       state :string do
-        rule /[^\\"]+/, Str::Double
+        rule /(?:\\")+|[^"]/, Str::Double
         mixin :escape_sequence
         rule /\\\n/, Str::Double
         rule /"/, Str::Double, :pop!

data/lib/rouge/lexers/csharp.rb

@@ -71,7 +71,6 @@ module Rouge
       state :root do
         mixin :whitespace
 
-        rule /^\s*\[.*?\]/, Name::Attribute
         rule /[$]\s*"/, Str, :splice_string
         rule /[$]@\s*"/, Str, :splice_literal
 

data/lib/rouge/lexers/diff.rb

@@ -19,14 +19,18 @@ module Rouge
 
       state :root do
         rule(/^ .*$\n?/, Text)
-        rule(/^---$\n?/, Text)
+        rule(/^---$\n?/, Punctuation)
+        rule(/^[+>]+.*$\n?/, Generic::Inserted)
         rule(/^\+.*$\n?/, Generic::Inserted)
-        rule(/^-+.*$\n?/, Generic::Deleted)
+        rule(/^[-<]+.*$\n?/, Generic::Deleted)
         rule(/^!.*$\n?/, Generic::Strong)
-        rule(/^@.*$\n?/, Generic::Subheading)
         rule(/^([Ii]ndex|diff).*$\n?/, Generic::Heading)
+        rule(/^(@@[^@]*@@)([^\n]*\n)/) do
+          groups Punctuation, Text
+        end
+        rule(/^\w.*$\n?/, Punctuation)
         rule(/^=.*$\n?/, Generic::Heading)
-        rule(/.*$\n?/, Text)
+        rule(/\s.*$\n?/, Text)
       end
     end
   end

data/lib/rouge/lexers/docker.rb

@@ -8,7 +8,7 @@ module Rouge
       desc "Dockerfile syntax"
       tag 'docker'
       aliases 'dockerfile'
-      filenames 'Dockerfile', '*.docker'
+      filenames 'Dockerfile', '*.Dockerfile', '*.docker', 'Dockerfile_*'
       mimetypes 'text/x-dockerfile-config'
 
       KEYWORDS = %w(

data/lib/rouge/lexers/escape.rb

@@ -0,0 +1,55 @@
+module Rouge
+  module Lexers
+    class Escape < Lexer
+      tag 'escape'
+      aliases 'esc'
+
+      desc 'A generic lexer for including escaped content - see Formatter.enable_escape!'
+
+      option :start, 'the beginning of the escaped section, default "<!"'
+      option :end, 'the end of the escaped section, e.g. "!>"'
+      option :lang, 'the language to lex in unescaped sections'
+
+      attr_reader :start
+      attr_reader :end
+      attr_reader :lang
+
+      def initialize(*)
+        super
+        @start = string_option(:start) { '<!' }
+        @end = string_option(:end) { '!>' }
+        @lang = lexer_option(:lang) { PlainText.new }
+      end
+
+      def to_start_regex
+        @to_start_regex ||= /(.*?)(#{Regexp.escape(@start)})/m
+      end
+
+      def to_end_regex
+        @to_end_regex ||= /(.*?)(#{Regexp.escape(@end)})/m
+      end
+
+      def stream_tokens(str, &b)
+        stream = StringScanner.new(str)
+
+        loop do
+          if stream.scan(to_start_regex)
+            puts "pre-escape: #{stream[1].inspect}" if @debug
+            @lang.continue_lex(stream[1], &b)
+          else
+            # no more start delimiters, scan til the end
+            @lang.continue_lex(stream.rest, &b)
+            return
+          end
+
+          if stream.scan(to_end_regex)
+            yield Token::Tokens::Escape, stream[1]
+          else
+            yield Token::Tokens::Escape, stream.rest
+            return
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/go.rb

@@ -149,7 +149,7 @@ module Rouge
         rule(OPERATOR,              Operator)
         rule(SEPARATOR,             Punctuation)
         rule(IDENTIFIER,            Name)
-        rule(WHITE_SPACE,           Other)
+        rule(WHITE_SPACE,           Text)
       end
 
       state :root do

data/lib/rouge/lexers/graphql.rb

@@ -27,6 +27,13 @@ module Rouge
         rule /\bunion\b/, Keyword, :union_definition
 
         mixin :basic
+
+        # Markdown descriptions
+        rule /(""")(.*?)(""")/m do |m|
+          token Str::Double, m[1]
+          delegate Markdown, m[2]
+          token Str::Double, m[3]
+        end
       end
 
       state :basic do
@@ -227,6 +234,9 @@ module Rouge
           }
         }
 
+        # Multiline strings
+        rule /""".*?"""/m, Str::Double
+
         rule /\$#{name}\b/, &pop_unless_list[Name::Variable]
         rule /\b(?:true|false|null)\b/, &pop_unless_list[Keyword::Constant]
         rule /[+-]?[0-9]+\.[0-9]+(?:[eE][+-]?[0-9]+)?/, &pop_unless_list[Num::Float]

data/lib/rouge/lexers/html.rb

@@ -12,6 +12,7 @@ module Rouge
 
       def self.detect?(text)
         return true if text.doctype?(/\bhtml\b/i)
+        return false if text =~ /\A<\?xml\b/
         return true if text =~ /<\s*html\b/
       end
 

data/lib/rouge/lexers/java.rb

@@ -24,6 +24,8 @@ module Rouge
       types = %w(boolean byte char double float int long short var void)
 
       id = /[a-zA-Z_][a-zA-Z0-9_]*/
+      const_name = /[A-Z][A-Z0-9_]*\b/
+      class_name = /[A-Z][a-zA-Z0-9]*\b/
 
       state :root do
         rule /[^\S\n]+/, Text
@@ -59,6 +61,8 @@ module Rouge
         end
 
         rule /#{id}:/, Name::Label
+        rule const_name, Name::Constant
+        rule class_name, Name::Class
         rule /\$?#{id}/, Name
         rule /[~^*!%&\[\](){}<>\|+=:;,.\/?-]/, Operator
 

data/lib/rouge/lexers/javascript.rb

@@ -98,26 +98,23 @@ module Rouge
 
       def self.keywords
         @keywords ||= Set.new %w(
-          for in of while do break return continue switch case default
-          if else throw try catch finally new delete typeof instanceof
-          void this yield import export from as async super this
+          as async await break case catch continue debugger default delete
+          do else export finally from for if import in instanceof new of
+          return super switch this throw try typeof void while yield
         )
       end
 
       def self.declarations
         @declarations ||= Set.new %w(
           var let const with function class
-          extends constructor get set
+          extends constructor get set static
         )
       end
 
       def self.reserved
         @reserved ||= Set.new %w(
-          abstract boolean byte char debugger double enum
-          final float goto implements int interface
-          long native package private protected public short static
-          synchronized throws transient volatile
-          eval arguments await
+          enum implements interface
+          package private protected public
         )
       end
 
@@ -200,22 +197,21 @@ module Rouge
         rule /0b[01][01_]*/i, Num::Bin
         rule /[0-9]+/, Num::Integer
 
-        rule /"/, Str::Double, :dq
-        rule /'/, Str::Single, :sq
+        rule /"/, Str::Delimiter, :dq
+        rule /'/, Str::Delimiter, :sq
         rule /:/, Punctuation
       end
 
       state :dq do
+        rule /\\[\\nrt"]?/, Str::Escape
         rule /[^\\"]+/, Str::Double
-        rule /\\n/, Str::Escape
-        rule /\\"/, Str::Escape
-        rule /"/, Str::Double, :pop!
+        rule /"/, Str::Delimiter, :pop!
       end
 
       state :sq do
+        rule /\\[\\nrt']?/, Str::Escape
         rule /[^\\']+/, Str::Single
-        rule /\\'/, Str::Escape
-        rule /'/, Str::Single, :pop!
+        rule /'/, Str::Delimiter, :pop!
       end
 
       # braced parts that aren't object literals

data/lib/rouge/lexers/jinja.rb

@@ -82,7 +82,7 @@ module Rouge
 
         # Arithmetic operators (+, -, *, **, //, /)
         # TODO : implement modulo (%)
-        rule /(\+|\-|\*|\/\/?|\*\*?)/, Operator
+        rule /(\+|\-|\*|\/\/?|\*\*?|=)/, Operator
 
         # Comparisons operators (<=, <, >=, >, ==, ===, !=)
         rule /(<=?|>=?|===?|!=)/, Operator
@@ -112,6 +112,11 @@ module Rouge
       end
 
       state :statement do
+        rule /(raw|verbatim)(\s+)(\%\})/ do
+          groups Keyword, Text, Comment::Preproc
+          goto :raw
+        end
+
         rule /(\w+\.?)/ do |m|
           if self.class.keywords.include?(m[0])
             groups Keyword
@@ -133,6 +138,15 @@ module Rouge
 
         rule /\%\}/, Comment::Preproc, :pop!
       end
+
+      state :raw do
+        rule %r{(\{\%)(\s+)(endverbatim|endraw)(\s+)(\%\})} do
+          groups Comment::Preproc, Text, Keyword, Text, Comment::Preproc
+          pop!
+        end
+
+        rule /(.+?)/m, Text
+      end
     end
   end
 end

data/lib/rouge/lexers/julia.rb

@@ -21,6 +21,8 @@ module Rouge
                             | NaN       | NaN16    | NaN32   | NaN64
                             | stdout    | stderr   | stdin   | devnull
                             | pi        | π        | ℯ       | im
+                            | ARGS      | C_NULL   | ENV     | ENDIAN_BOM
+                            | VERSION   | undef    | (LOAD|DEPOT)_PATH
                             )\b/x
 
       KEYWORDS            = /\b(?:
@@ -30,21 +32,137 @@ module Rouge
                             | const    | local  | global | using  | struct
                             | mutable struct    | abstract type   | finally
                             | begin    | do     | quote  | macro  | for outer
+                            | where
                             )\b/x
 
+      # NOTE: The list of types was generated automatically using the following script:
+      # using Pkg, InteractiveUtils
+      #
+      # allnames = [names(Core); names(Base, imported=true)]
+      #
+      # for stdlib in readdir(Pkg.Types.stdlib_dir())
+      #     mod = Symbol(basename(stdlib))
+      #     @eval begin
+      #         using $mod
+      #         append!(allnames, names($mod))
+      #     end
+      # end
+      #
+      # sort!(unique!(allnames))
+      #
+      # i = 1
+      # for sym in allnames
+      #     global i # needed at the top level, e.g. in the REPL
+      #     isdefined(Main, sym) || continue
+      #     getfield(which(Main, sym), sym) isa Type || continue
+      #     sym === :(=>) && continue # Actually an alias for Pair
+      #     print("| ", sym)
+      #     i % 3 == 0 ? println() : print(" ") # print 3 to a line
+      #     i += 1
+      # end
       TYPES               = /\b(?:
-                              Int         | UInt           | Int8
-                            | UInt8       | Int16          | UInt16
-                            | Int32       | UInt32         | Int64
-                            | UInt64      | Int128         | UInt128
-                            | Float16     | Float32        | Float64
-                            | Bool        | BigInt         | BigFloat
-                            | Complex     | ComplexF16     | ComplexF32
-                            | ComplexF64  | Missing        | Nothing
-                            | Char        | String         | SubString
-                            | Regex       | RegexMatch     | Any
-                            | Type        | DataType       | UnionAll
-                            | (Abstract)?(Array|Vector|Matrix|VecOrMat)
+                              ARPACKException | AbstractArray | AbstractChannel
+                            | AbstractChar | AbstractDict | AbstractDisplay
+                            | AbstractFloat | AbstractIrrational | AbstractLogger
+                            | AbstractMatrix | AbstractREPL | AbstractRNG
+                            | AbstractRange | AbstractSerializer | AbstractSet
+                            | AbstractSparseArray | AbstractSparseMatrix | AbstractSparseVector
+                            | AbstractString | AbstractUnitRange | AbstractVecOrMat
+                            | AbstractVector | AbstractWorkerPool | Adjoint
+                            | Any | ArgumentError | Array
+                            | AssertionError | Base64DecodePipe | Base64EncodePipe
+                            | BasicREPL | Bidiagonal | BigFloat
+                            | BigInt | BitArray | BitMatrix
+                            | BitSet | BitVector | Bool
+                            | BoundsError | BunchKaufman | CachingPool
+                            | CapturedException | CartesianIndex | CartesianIndices
+                            | Cchar | Cdouble | Cfloat
+                            | Channel | Char | Cholesky
+                            | CholeskyPivoted | Cint | Cintmax_t
+                            | Clong | Clonglong | ClusterManager
+                            | Cmd | Colon | Complex
+                            | ComplexF16 | ComplexF32 | ComplexF64
+                            | CompositeException | Condition | ConsoleLogger
+                            | Cptrdiff_t | Cshort | Csize_t
+                            | Cssize_t | Cstring | Cuchar
+                            | Cuint | Cuintmax_t | Culong
+                            | Culonglong | Cushort | Cvoid
+                            | Cwchar_t | Cwstring | DataType
+                            | Date | DateFormat | DatePeriod
+                            | DateTime | Day | DenseArray
+                            | DenseMatrix | DenseVecOrMat | DenseVector
+                            | Diagonal | Dict | DimensionMismatch
+                            | Dims | DivideError | DomainError
+                            | EOFError | Eigen | Enum
+                            | ErrorException | Exception | ExponentialBackOff
+                            | Expr | FDWatcher | Factorization
+                            | FileMonitor | Float16 | Float32
+                            | Float64 | FolderMonitor | Function
+                            | GeneralizedEigen | GeneralizedSVD | GeneralizedSchur
+                            | GenericArray | GenericDict | GenericSet
+                            | GenericString | GitConfig | GitRepo
+                            | GlobalRef | HMAC_CTX | HTML
+                            | Hermitian | Hessenberg | Hour
+                            | IO | IOBuffer | IOContext
+                            | IOStream | IPAddr | IPv4
+                            | IPv6 | IdDict | IndexCartesian
+                            | IndexLinear | IndexStyle | InexactError
+                            | InitError | Int | Int128
+                            | Int16 | Int32 | Int64
+                            | Int8 | Integer | InterruptException
+                            | InvalidStateException | Irrational | KeyError
+                            | LAPACKException | LDLt | LQ
+                            | LU | LinRange | LineEditREPL
+                            | LineNumberNode | LinearIndices | LoadError
+                            | LogLevel | LowerTriangular | MIME
+                            | Matrix | MersenneTwister | Method
+                            | MethodError | Microsecond | Millisecond
+                            | Minute | Missing | MissingException
+                            | Module | Month | NTuple
+                            | NamedTuple | Nanosecond | Nothing
+                            | NullLogger | Number | OrdinalRange
+                            | OutOfMemoryError | OverflowError | PackageMode
+                            | PackageSpec | Pair | PartialQuickSort
+                            | Period | PermutedDimsArray | Pipe
+                            | PollingFileWatcher | PosDefException | ProcessExitedException
+                            | Ptr | QR | QRPivoted
+                            | QuoteNode | RandomDevice | RankDeficientException
+                            | Rational | RawFD | ReadOnlyMemoryError
+                            | Real | ReentrantLock | Ref
+                            | Regex | RegexMatch | RemoteChannel
+                            | RemoteException | RoundingMode | SHA1_CTX
+                            | SHA224_CTX | SHA256_CTX | SHA2_224_CTX
+                            | SHA2_256_CTX | SHA2_384_CTX | SHA2_512_CTX
+                            | SHA384_CTX | SHA3_224_CTX | SHA3_256_CTX
+                            | SHA3_384_CTX | SHA3_512_CTX | SHA512_CTX
+                            | SVD | Schur | Second
+                            | SegmentationFault | Serializer | Set
+                            | SharedArray | SharedMatrix | SharedVector
+                            | Signed | SimpleLogger | SingularException
+                            | Some | SparseMatrixCSC | SparseVector
+                            | StackOverflowError | StepRange | StepRangeLen
+                            | StreamREPL | StridedArray | StridedMatrix
+                            | StridedVecOrMat | StridedVector | String
+                            | StringIndexError | SubArray | SubString
+                            | SubstitutionString | SymTridiagonal | Symbol
+                            | Symmetric | SystemError | TCPSocket
+                            | Task | TestSetException | Text
+                            | TextDisplay | Time | TimePeriod
+                            | TimeType | TimeZone | Timer
+                            | Transpose | Tridiagonal | Tuple
+                            | Type | TypeError | TypeVar
+                            | UDPSocket | UInt | UInt128
+                            | UInt16 | UInt32 | UInt64
+                            | UInt8 | UTC | UUID
+                            | UndefInitializer | UndefKeywordError | UndefRefError
+                            | UndefVarError | UniformScaling | Union
+                            | UnionAll | UnitLowerTriangular | UnitRange
+                            | UnitUpperTriangular | Unsigned | UpgradeLevel
+                            | UpperTriangular | Val | Vararg
+                            | VecElement | VecOrMat | Vector
+                            | VersionNumber | WeakKeyDict | WeakRef
+                            | Week | WorkerConfig | WorkerPool
+                            | Year
                             )\b/x
 
       OPERATORS           = / \+      | =        | -     | \*   | \/
@@ -58,10 +176,10 @@ module Rouge
                               | ::    | <:       | ->    | \?   | \.\*
                               | \.\^  | \.\\     | \.\/  | \\   | <
                               | >     | ÷        | >:    | :    | ===
-                              | !==
+                              | !==   | =>
                             /x
 
-      PUNCTUATION         = / [ \[ \] { } \( \) , ; @ ] /x
+      PUNCTUATION         = / [ \[ \] { } \( \) , ; ] /x
 
 
       state :root do
@@ -74,14 +192,14 @@ module Rouge
         rule /\\/, Text
 
 
-        # functions
-        rule /(function)((?:\s|\\\s)+)/ do
+        # functions and macros
+        rule /(function|macro)((?:\s|\\\s)+)/ do
           groups Keyword, Name::Function
           push :funcname
         end
 
         # types
-        rule /(type|typealias|abstract)((?:\s|\\\s)+)/ do
+        rule /((mutable )?struct|(abstract|primitive) type)((?:\s|\\\s)+)/ do
           groups Keyword, Name::Class
           push :typename
         end
@@ -91,8 +209,11 @@ module Rouge
         rule /(local|global|const)\b/, Keyword::Declaration
         rule KEYWORDS, Keyword
 
+        # TODO: end is a builtin when inside of an indexing expression
         rule BUILTINS, Name::Builtin
 
+        # TODO: symbols
+
         # backticks
         rule /`.*?`/, Literal::String::Backtick
 
@@ -103,6 +224,8 @@ module Rouge
         rule /(?<=[.\w)\]])\'+/, Operator
 
         # strings
+        # TODO: triple quoted string literals
+        # TODO: Detect string interpolation
         rule /(?:[IL])"/, Literal::String, :string
         rule /[E]?"/, Literal::String, :string