rouge 3.6.0 → 3.11.0

data/lib/rouge/lexers/hocon.rb

@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    load_lexer 'json.rb'
+
+    class HOCON < JSON
+      title 'HOCON'
+      desc "Human-Optimized Config Object Notation (https://github.com/lightbend/config)"
+      tag 'hocon'
+      filenames '*.hocon'
+
+      state :comments do
+        # Comments
+        rule %r(//.*?$), Comment::Single
+        rule %r(#.*?$), Comment::Single
+      end
+
+      prepend :root do
+        mixin :comments
+      end
+
+      prepend :object do
+        # Keywords
+        rule %r/\b(?:include|url|file|classpath)\b/, Keyword
+      end
+
+      state :name do
+        rule %r/("(?:\"|[^"\n])*?")(\s*)([:=]|(?={))/ do
+          groups Name::Label, Text::Whitespace, Punctuation
+        end
+
+        rule %r/([-\w.]+)(\s*)([:=]|(?={))/ do
+          groups Name::Label, Text::Whitespace, Punctuation
+        end
+      end
+
+      state :value do
+        mixin :comments
+
+        rule %r/\n/, Text::Whitespace
+        rule %r/\s+/, Text::Whitespace
+
+        mixin :constants
+
+        # Interpolation
+        rule %r/[$][{][?]?/, Literal::String::Interpol, :interpolation
+
+        # Strings
+        rule %r/"""/, Literal::String::Double, :multiline_string
+        rule %r/"/, Str::Double, :string
+
+        rule %r/\[/, Punctuation, :array
+        rule %r/{/, Punctuation, :object
+
+        # Symbols (only those not handled by JSON)
+        rule %r/[()=]/, Punctuation
+
+        # Values
+        rule %r/[^$"{}\[\]:=,\+#`^?!@*&]+?/, Literal
+      end
+
+      state :interpolation do
+        rule %r/[\w\-\.]+?/, Name::Variable
+        rule %r/}/, Literal::String::Interpol, :pop!
+      end
+
+      prepend :string do
+        rule %r/[$][{][?]?/, Literal::String::Interpol, :interpolation
+        rule %r/[^\\"\${]+/, Literal::String::Double
+      end
+
+      state :multiline_string do
+        rule %r/"[^"]{1,2}/, Literal::String::Double
+        mixin :string
+        rule %r/"""/, Literal::String::Double, :pop!
+      end
+
+      prepend :constants do
+        # Numbers (handle the case where we have multiple periods, ie. IP addresses)
+        rule %r/\d+\.(\d+\.?){3,}/, Literal
+      end
+    end
+  end
+end

data/lib/rouge/lexers/hql.rb

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*- #
+
+module Rouge
+  module Lexers
+    load_lexer 'sql.rb'
+
+    class HQL < SQL
+      title "HQL"
+      desc "Hive Query Language SQL dialect"
+      tag 'hql'
+      filenames '*.hql'
+
+      def self.keywords
+        # sources:
+        # https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL
+        # https://cwiki.apache.org/confluence/display/Hive/LanguageManual+UDF
+        @keywords ||= Set.new(%w(
+          ADD ADMIN AFTER ANALYZE ARCHIVE ASC BEFORE BUCKET BUCKETS CASCADE
+          CHANGE CLUSTER CLUSTERED CLUSTERSTATUS COLLECTION COLUMNS COMMENT
+          COMPACT COMPACTIONS COMPUTE CONCATENATE CONTINUE DATA DATABASES
+          DATETIME DAY DBPROPERTIES DEFERRED DEFINED DELIMITED DEPENDENCY DESC
+          DIRECTORIES DIRECTORY DISABLE DISTRIBUTE ELEM_TYPE ENABLE ESCAPED
+          EXCLUSIVE EXPLAIN EXPORT FIELDS FILE FILEFORMAT FIRST FORMAT FORMATTED
+          FUNCTIONS HOLD_DDLTIME HOUR IDXPROPERTIES IGNORE INDEX INDEXES INPATH
+          INPUTDRIVER INPUTFORMAT ITEMS JAR KEYS KEY_TYPE LIMIT LINES LOAD
+          LOCATION LOCK LOCKS LOGICAL LONG MAPJOIN MATERIALIZED METADATA MINUS
+          MINUTE MONTH MSCK NOSCAN NO_DROP OFFLINE OPTION OUTPUTDRIVER
+          OUTPUTFORMAT OVERWRITE OWNER PARTITIONED PARTITIONS PLUS PRETTY
+          PRINCIPALS PROTECTION PURGE READ READONLY REBUILD RECORDREADER
+          RECORDWRITER REGEXP RELOAD RENAME REPAIR REPLACE REPLICATION RESTRICT
+          REWRITE RLIKE ROLE ROLES SCHEMA SCHEMAS SECOND SEMI SERDE
+          SERDEPROPERTIES SERVER SETS SHARED SHOW SHOW_DATABASE SKEWED SORT
+          SORTED SSL STATISTICS STORED STREAMTABLE STRING STRUCT TABLES
+          TBLPROPERTIES TEMPORARY TERMINATED TINYINT TOUCH TRANSACTIONS UNARCHIVE
+          UNDO UNIONTYPE UNLOCK UNSET UNSIGNED URI USE UTC UTCTIMESTAMP
+          VALUE_TYPE VIEW WHILE YEAR IF
+
+          ALL ALTER AND ARRAY AS AUTHORIZATION BETWEEN BIGINT BINARY BOOLEAN
+          BOTH BY CASE CAST CHAR COLUMN CONF CREATE CROSS CUBE CURRENT
+          CURRENT_DATE CURRENT_TIMESTAMP CURSOR DATABASE DATE DECIMAL DELETE
+          DESCRIBE DISTINCT DOUBLE DROP ELSE END EXCHANGE EXISTS EXTENDED
+          EXTERNAL FALSE FETCH FLOAT FOLLOWING FOR FROM FULL FUNCTION GRANT
+          GROUP GROUPING HAVING IF IMPORT IN INNER INSERT INT INTERSECT
+          INTERVAL INTO IS JOIN LATERAL LEFT LESS LIKE LOCAL MACRO MAP MORE
+          NONE NOT NULL OF ON OR ORDER OUT OUTER OVER PARTIALSCAN PARTITION
+          PERCENT PRECEDING PRESERVE PROCEDURE RANGE READS REDUCE REVOKE RIGHT
+          ROLLUP ROW ROWS SELECT SET SMALLINT TABLE TABLESAMPLE THEN TIMESTAMP
+          TO TRANSFORM TRIGGER TRUE TRUNCATE UNBOUNDED UNION UNIQUEJOIN UPDATE
+          USER USING UTC_TMESTAMP VALUES VARCHAR WHEN WHERE WINDOW WITH
+
+          AUTOCOMMIT ISOLATION LEVEL OFFSET SNAPSHOT TRANSACTION WORK WRITE
+
+          COMMIT ONLY REGEXP RLIKE ROLLBACK START
+
+          ABORT KEY LAST NORELY NOVALIDATE NULLS RELY VALIDATE
+
+          CACHE CONSTRAINT FOREIGN PRIMARY REFERENCES
+
+          DETAIL DOW EXPRESSION OPERATOR QUARTER SUMMARY VECTORIZATION WEEK YEARS MONTHS WEEKS DAYS HOURS MINUTES SECONDS
+
+          DAYOFWEEK EXTRACT FLOOR INTEGER PRECISION VIEWS
+
+          TIMESTAMPTZ ZONE
+
+          TIME NUMERIC
+
+          NAMED_STRUCT CREATE_UNION
+
+          ROUND BROUND FLOOR CEIL CEILING RAND EXP LN LOG10 LOG2 LOG POW POWER SQRT BIN
+          HEX UNHEX CONV ABS PMOD SIN ASIN COS ACOS TAN ATAN DEGREES RADIANS POSITIVE
+          NEGATIVE SIGN E PI FACTORIAL CBRT SHIFTLEFT SHIFTRIGHT SHIFTRIGHTUNSIGNED
+          GREATEST LEAST WIDTH_BUCKET SIZE SIZE MAP_KEYS MAP_VALUES ARRAY_CONTAINS
+          SORT_ARRAY BINARY CAST FROM_UNIXTIME UNIX_TIMESTAMP UNIX_TIMESTAMP
+          UNIX_TIMESTAMP TO_DATE YEAR QUARTER MONTH DAY DAYOFMONTH HOUR MINUTE SECOND
+          WEEKOFYEAR EXTRACT DATEDIFF DATE_ADD DATE_SUB FROM_UTC_TIMESTAMP
+          TO_UTC_TIMESTAMP CURRENT_DATE CURRENT_TIMESTAMP ADD_MONTHS LAST_DAY NEXT_DAY
+          TRUNC MONTHS_BETWEEN DATE_FORMAT IF ISNULL ISNOTNULL  NVL COALESCE CASE WHEN
+          then else end NULLIF ASSERT_TRUE ASCII BASE64 CHARACTER_LENGTH CHR CONCAT
+          CONTEXT_NGRAMS CONCAT_WS CONCAT_WS DECODE ELT ENCODE FIELD  FIND_IN_SET
+          FORMAT_NUMBER GET_JSON_OBJECT IN_FILE INSTR LENGTH LOCATE LOWER LCASE LPAD LTRIM
+          NGRAMS OCTET_LENGTH PARSE_URL PRINTF REGEXP_EXTRACT REGEXP_REPLACE REPEAT
+          REPLACE REVERSE RPAD RTRIM SENTENCES SPACE SPLIT STR_TO_MAP SUBSTR SUBSTRING
+          SUBSTRING_INDEX TRANSLATE TRIM UNBASE64 UPPER UCASE INITCAP LEVENSHTEIN SOUNDEX
+          MASK MASK_FIRST_N MASK_LAST_N MASK_SHOW_FIRST_N MASK_SHOW_LAST_N MASK_HASH
+          JAVA_METHOD REFLECT HASH CURRENT_USER LOGGED_IN_USER CURRENT_DATABASE MD5 SHA1
+          SHA CRC32 SHA2 AES_ENCRYPT AES_DECRYPT VERSION COUNT SUM AVG MIN MAX VARIANCE
+          VAR_POP VAR_SAMP STDDEV_POP STDDEV_SAMP COVAR_POP COVAR_SAMP CORR PERCENTILE
+          PERCENTILE_APPROX PERCENTILE_APPROX REGR_AVGX REGR_AVGY REGR_COUNT
+          REGR_INTERCEPT REGR_R2 REGR_SLOPE REGR_SXX REGR_SXY REGR_SYY HISTOGRAM_NUMERIC
+          COLLECT_SET COLLECT_LIST NTILE EXPLODE EXPLODE POSEXPLODE INLINE STACK
+
+          JSON_TUPLE PARSE_URL_TUPLE
+
+          XPATH XPATH_SHORT XPATH_INT XPATH_LONG XPATH_FLOAT XPATH_DOUBLE
+          XPATH_NUMBER XPATH_STRING GET_JSON_OBJECT JSON_TUPLE
+
+          PARSE_URL_TUPLE
+        ))
+      end
+
+      def self.keywords_type
+        # source: https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types
+        @keywords_type ||= Set.new(%w(
+            TINYINT SMALLINT INT INTEGER BIGINT FLOAT DOUBLE PRECISION DECIMAL NUMERIC
+            TIMESTAMP DATE INTERVAL
+            STRING VARCHAR CHAR
+            BOOLEAN BINARY
+            ARRAY MAP STRUCT UNIONTYPE
+        ))
+      end
+
+      prepend :root do
+        # a double-quoted string is a string literal in Hive QL.
+        rule %r/"/, Str::Double, :double_string
+
+        # interpolation of variables through ${...}
+        rule %r/\$\{/, Name::Variable, :hive_variable
+      end
+
+      prepend :single_string do
+        rule %r/\$\{/, Name::Variable, :hive_variable
+        rule %r/[^\\'\$]+/, Str::Single
+      end
+
+      prepend :double_string do
+        rule %r/\$\{/, Name::Variable, :hive_variable
+        # double-quoted strings are string literals so need to change token
+        rule %r/"/, Str::Double, :pop!
+        rule %r/[^\\"\$]+/, Str::Double
+      end
+
+      state :hive_variable do
+        rule %r/\}/, Name::Variable, :pop!
+        rule %r/[^\}]+/, Name::Variable
+      end
+
+    end
+  end
+end

data/lib/rouge/lexers/html.rb

@@ -83,8 +83,8 @@ module Rouge
 
       state :tag do
         rule %r/\s+/m, Text
-        rule %r/[a-zA-Z0-9_:-]+\s*=\s*/m, Name::Attribute, :attr
-        rule %r/[a-zA-Z0-9_:-]+/, Name::Attribute
+        rule %r/[a-zA-Z0-9_:\[\]()*.-]+\s*=\s*/m, Name::Attribute, :attr
+        rule %r/[a-zA-Z0-9_:#*-]+/, Name::Attribute
         rule %r(/?\s*>)m, Name::Tag, :pop!
       end
 

data/lib/rouge/lexers/http.rb

@@ -26,8 +26,8 @@ module Rouge
         # request
         rule %r(
           (#{HTTP.http_methods.join('|')})([ ]+) # method
-          ([^ ]+)([ ]+)                     # path
-          (HTTPS?)(/)(1[.][01])(\r?\n|$)  # http version
+          ([^ ]+)([ ]+)                          # path
+          (HTTPS?)(/)(\d(?:\.\d)?)(\r?\n|$)      # http version
         )ox do
           groups(
             Name::Function, Text,
@@ -40,9 +40,9 @@ module Rouge
 
         # response
         rule %r(
-          (HTTPS?)(/)(1[.][01])([ ]+) # http version
-          (\d{3})([ ]+)               # status
-          ([^\r\n]+)(\r?\n|$)       # status message
+          (HTTPS?)(/)(\d(?:\.\d))([ ]+)  # http version
+          (\d{3})([ ]+)?                 # status
+          ([^\r\n]*)?(\r?\n|$)           # status message
         )x do
           groups(
             Keyword, Operator, Num, Text,

data/lib/rouge/lexers/igorpro.rb

@@ -20,7 +20,7 @@ module Rouge
           break return continue
           for endfor do while
           case default
-	  try catch endtry
+          try catch endtry
           abortonrte
         )
       end

data/lib/rouge/lexers/javascript.rb

@@ -73,7 +73,7 @@ module Rouge
       end
 
       state :regex_end do
-        rule %r/[gim]+/, Str::Regex, :pop!
+        rule %r/[gimuy]+/, Str::Regex, :pop!
         rule(//) { pop! }
       end
 

data/lib/rouge/lexers/jsl.rb

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class JSL < RegexLexer
+      title "JSL"
+      desc "The JMP Scripting Language (JSL) (jmp.com)"
+
+      tag 'jsl'
+      filenames '*.jsl'
+
+      state :root do
+        rule %r/\s+/m, Text::Whitespace
+
+        rule %r(//.*?$), Comment::Single
+        rule %r(/\*.*?\*/)m, Comment::Multiline
+
+        # messages
+        rule %r/(<<)(.*?)(\(|;)/ do |m|
+          groups Operator, Name::Function, Punctuation
+        end
+
+        # covers built-in and custom functions
+        rule %r/([a-z_][\w\s'%.\\]*)(\()/i do |m|
+          groups Keyword, Punctuation
+        end
+
+        rule %r/\b[+-]?(?:[0-9]+(?:\.[0-9]+)?|\.[0-9]+|\.)(?:e[+-]?[0-9]+)?i?\b/i, Num
+
+        rule %r/\d{2}(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\d{2}(\d{2})?(:\d{2}:\d{2}(:\d{2}(\.\d*)?)?)?/i, Literal::Date
+
+        rule %r/::[a-z_][\w\s'%.\\]*/i, Name::Variable
+        rule %r/:\w+/, Name
+        rule %r/[a-z_][\w\s'%.\\]*/i, Name::Variable
+        rule %r/"(?:\\!"|[^"])*?"n/m, Name::Variable
+
+        rule %r/(")(\\\[)(.*?)(\]\\)(")/m do
+          groups Str::Double, Str::Escape, Str::Double, Str::Escape, Str::Double  # escaped string
+        end
+        rule %r/"/, Str::Double, :dq
+
+        rule %r/[-+*\/!%&<>\|=:]/, Operator
+        rule %r/[\[\](){},;]/, Punctuation
+      end
+
+      state :dq do
+        rule %r/\\![btrnNf0\\"]/, Str::Escape
+        rule %r/\\/, Str::Double
+        rule %r/"/, Str::Double, :pop!
+        rule %r/[^\\"]*/m, Str::Double
+      end
+    end
+  end
+end

data/lib/rouge/lexers/json.rb

@@ -12,13 +12,39 @@ module Rouge
                 'application/hal+json', 'application/problem+json',
                 'application/schema+json'
 
+      state :whitespace do
+        rule %r/\s+/, Text::Whitespace
+      end
+
       state :root do
-        rule %r/\s+/m, Text::Whitespace
+        mixin :whitespace
+        rule %r/{/, Punctuation, :object
+        rule %r/\[/, Punctuation, :array
+
+        mixin :name
+        mixin :value
+      end
+
+      state :object do
+        mixin :whitespace
+        mixin :name
+        mixin :value
+        rule %r/}/, Punctuation, :pop!
+        rule %r/,/, Punctuation
+      end
+
+      state :name do
+        rule %r/("(?:\\.|[^"\\\n])*?")(\s*)(:)/ do
+          groups Name::Label, Text::Whitespace, Punctuation
+        end
+      end
+
+      state :value do
+        mixin :whitespace
+        mixin :constants
         rule %r/"/, Str::Double, :string
-        rule %r/(?:true|false|null)\b/, Keyword::Constant
-        rule %r/[{},:\[\]]/, Punctuation
-        rule %r/-?(?:0|[1-9]\d*)\.\d+(?:e[+-]?\d+)?/i, Num::Float
-        rule %r/-?(?:0|[1-9]\d*)(?:e[+-]?\d+)?/i, Num::Integer
+        rule %r/\[/, Punctuation, :array
+        rule %r/{/, Punctuation, :object
       end
 
       state :string do
@@ -26,6 +52,18 @@ module Rouge
         rule %r/\\./, Str::Escape
         rule %r/"/, Str::Double, :pop!
       end
+
+      state :array do
+        mixin :value
+        rule %r/\]/, Punctuation, :pop!
+        rule %r/,/, Punctuation
+      end
+
+      state :constants do
+        rule %r/(?:true|false|null)/, Keyword::Constant
+        rule %r/-?(?:0|[1-9]\d*)\.\d+(?:e[+-]?\d+)?/i, Num::Float
+        rule %r/-?(?:0|[1-9]\d*)(?:e[+-]?\d+)?/i, Num::Integer
+      end
     end
   end
 end

data/lib/rouge/lexers/julia.rb

@@ -199,7 +199,7 @@ module Rouge
         end
 
         # types
-        rule %r/((mutable )?struct|(abstract|primitive) type)((?:\s|\\\s)+)/ do
+        rule %r/((?:mutable )?struct|(?:abstract|primitive) type)((?:\s|\\\s)+)/ do
           groups Keyword, Name::Class
           push :typename
         end

data/lib/rouge/lexers/kotlin.rb

@@ -24,19 +24,12 @@ module Rouge
         while yield
       )
 
-      name = %r'@?[_\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Nl}][\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Nl}\p{Nd}\p{Pc}\p{Cf}\p{Mn}\p{Mc}]*'
-      name_backtick = %r'#{name}|`#{name}`'
+      name_chars = %r'[-_\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Nl}\p{Nd}\p{Pc}\p{Cf}\p{Mn}\p{Mc}]*'
 
-      id = %r'(#{name_backtick})'
+      class_name = %r'`?[\p{Lu}]#{name_chars}`?'
+      name = %r'`?[_\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Nl}]#{name_chars}`?'
 
       state :root do
-        rule %r'(\))(\s*)(:)(\s+)(#{name_backtick})(<)' do
-          groups Punctuation, Text, Punctuation, Text, Name::Class, Punctuation
-          push :generic_parameters
-        end
-        rule %r'(\))(\s*)(:)(\s+)(#{name_backtick})' do
-          groups Punctuation, Text, Punctuation, Text, Name::Class
-        end
         rule %r'\b(companion)(\s+)(object)\b' do
           groups Keyword, Text, Keyword
         end
@@ -48,13 +41,6 @@ module Rouge
           groups Keyword, Text
           push :function
         end
-        rule %r'(#{name_backtick})(:)(\s+)(#{name_backtick})(<)' do
-          groups Name::Variable, Punctuation, Text, Name::Class, Punctuation
-          push :generic_parameters
-        end
-        rule %r'(#{name_backtick})(:)(\s+)(#{name_backtick})' do
-          groups Name::Variable, Punctuation, Text, Name::Class
-        end
         rule %r'\b(package|import)(\s+)' do
           groups Keyword, Text
           push :package
@@ -67,8 +53,8 @@ module Rouge
           groups Keyword::Declaration, Text
           push :property
         end
-        rule %r/\bfun\b/, Keyword
-        rule %r/\b(?:#{keywords.join('|')})\b/, Keyword
+        rule %r'\bfun\b', Keyword
+        rule %r'\b(?:#{keywords.join('|')})\b', Keyword
         rule %r'^\s*\[.*?\]', Name::Attribute
         rule %r'[^\S\n]+', Text
         rule %r'\\\n', Text # line continuation
@@ -85,43 +71,50 @@ module Rouge
         rule %r'"(\\\\|\\"|[^"\n])*["\n]'m, Str
         rule %r"'\\.'|'[^\\]'", Str::Char
         rule %r"[0-9](\.[0-9]+)?([eE][+-][0-9]+)?[flFL]?|0[xX][0-9a-fA-F]+[Ll]?", Num
-        rule %r/@#{id}/, Name::Decorator
-        rule id, Name
+        rule %r'(@#{class_name})', Name::Decorator
+        rule %r'(#{class_name})(<)' do
+          groups Name::Class, Punctuation
+          push :generic_parameters
+        end
+        rule class_name, Name::Class
+        rule %r'(#{name})(?=\s*[({])', Name::Function
+        rule name, Name
       end
 
       state :package do
-        rule %r/\S+/, Name::Namespace, :pop!
+        rule %r'\S+', Name::Namespace, :pop!
       end
 
       state :class do
-        rule id, Name::Class, :pop!
+        rule class_name, Name::Class, :pop!
       end
 
       state :function do
         rule %r'(<)', Punctuation, :generic_parameters
         rule %r'(\s+)', Text
-        rule %r'(#{name_backtick})(\.)' do
+        rule %r'(#{class_name})(\.)' do
           groups Name::Class, Punctuation
         end
-        rule id, Name::Function, :pop!
+        rule name, Name::Function, :pop!
       end
 
       state :generic_parameters do
-        rule id, Name::Class
+        rule class_name, Name::Class
+        rule %r'(<)', Punctuation, :generic_parameters
         rule %r'(,)', Punctuation
         rule %r'(\s+)', Text
         rule %r'(>)', Punctuation, :pop!
       end
 
       state :property do
-        rule id, Name::Property, :pop!
+        rule name, Name::Property, :pop!
       end
 
       state :destructure do
         rule %r'(,)', Punctuation
         rule %r'(\))', Punctuation, :pop!
         rule %r'(\s+)', Text
-        rule id, Name::Property
+        rule name, Name::Property
       end
 
       state :comment do