rouge 3.18.0

data/lib/rouge/lexers/hql.rb

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*- #
+
+module Rouge
+  module Lexers
+    load_lexer 'sql.rb'
+
+    class HQL < SQL
+      title "HQL"
+      desc "Hive Query Language SQL dialect"
+      tag 'hql'
+      filenames '*.hql'
+
+      def self.keywords
+        # sources:
+        # https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL
+        # https://cwiki.apache.org/confluence/display/Hive/LanguageManual+UDF
+        @keywords ||= Set.new(%w(
+          ADD ADMIN AFTER ANALYZE ARCHIVE ASC BEFORE BUCKET BUCKETS CASCADE
+          CHANGE CLUSTER CLUSTERED CLUSTERSTATUS COLLECTION COLUMNS COMMENT
+          COMPACT COMPACTIONS COMPUTE CONCATENATE CONTINUE DATA DATABASES
+          DATETIME DAY DBPROPERTIES DEFERRED DEFINED DELIMITED DEPENDENCY DESC
+          DIRECTORIES DIRECTORY DISABLE DISTRIBUTE ELEM_TYPE ENABLE ESCAPED
+          EXCLUSIVE EXPLAIN EXPORT FIELDS FILE FILEFORMAT FIRST FORMAT FORMATTED
+          FUNCTIONS HOLD_DDLTIME HOUR IDXPROPERTIES IGNORE INDEX INDEXES INPATH
+          INPUTDRIVER INPUTFORMAT ITEMS JAR KEYS KEY_TYPE LIMIT LINES LOAD
+          LOCATION LOCK LOCKS LOGICAL LONG MAPJOIN MATERIALIZED METADATA MINUS
+          MINUTE MONTH MSCK NOSCAN NO_DROP OFFLINE OPTION OUTPUTDRIVER
+          OUTPUTFORMAT OVERWRITE OWNER PARTITIONED PARTITIONS PLUS PRETTY
+          PRINCIPALS PROTECTION PURGE READ READONLY REBUILD RECORDREADER
+          RECORDWRITER REGEXP RELOAD RENAME REPAIR REPLACE REPLICATION RESTRICT
+          REWRITE RLIKE ROLE ROLES SCHEMA SCHEMAS SECOND SEMI SERDE
+          SERDEPROPERTIES SERVER SETS SHARED SHOW SHOW_DATABASE SKEWED SORT
+          SORTED SSL STATISTICS STORED STREAMTABLE STRING STRUCT TABLES
+          TBLPROPERTIES TEMPORARY TERMINATED TINYINT TOUCH TRANSACTIONS UNARCHIVE
+          UNDO UNIONTYPE UNLOCK UNSET UNSIGNED URI USE UTC UTCTIMESTAMP
+          VALUE_TYPE VIEW WHILE YEAR IF
+
+          ALL ALTER AND ARRAY AS AUTHORIZATION BETWEEN BIGINT BINARY BOOLEAN
+          BOTH BY CASE CAST CHAR COLUMN CONF CREATE CROSS CUBE CURRENT
+          CURRENT_DATE CURRENT_TIMESTAMP CURSOR DATABASE DATE DECIMAL DELETE
+          DESCRIBE DISTINCT DOUBLE DROP ELSE END EXCHANGE EXISTS EXTENDED
+          EXTERNAL FALSE FETCH FLOAT FOLLOWING FOR FROM FULL FUNCTION GRANT
+          GROUP GROUPING HAVING IF IMPORT IN INNER INSERT INT INTERSECT
+          INTERVAL INTO IS JOIN LATERAL LEFT LESS LIKE LOCAL MACRO MAP MORE
+          NONE NOT NULL OF ON OR ORDER OUT OUTER OVER PARTIALSCAN PARTITION
+          PERCENT PRECEDING PRESERVE PROCEDURE RANGE READS REDUCE REVOKE RIGHT
+          ROLLUP ROW ROWS SELECT SET SMALLINT TABLE TABLESAMPLE THEN TIMESTAMP
+          TO TRANSFORM TRIGGER TRUE TRUNCATE UNBOUNDED UNION UNIQUEJOIN UPDATE
+          USER USING UTC_TMESTAMP VALUES VARCHAR WHEN WHERE WINDOW WITH
+
+          AUTOCOMMIT ISOLATION LEVEL OFFSET SNAPSHOT TRANSACTION WORK WRITE
+
+          COMMIT ONLY REGEXP RLIKE ROLLBACK START
+
+          ABORT KEY LAST NORELY NOVALIDATE NULLS RELY VALIDATE
+
+          CACHE CONSTRAINT FOREIGN PRIMARY REFERENCES
+
+          DETAIL DOW EXPRESSION OPERATOR QUARTER SUMMARY VECTORIZATION WEEK YEARS MONTHS WEEKS DAYS HOURS MINUTES SECONDS
+
+          DAYOFWEEK EXTRACT FLOOR INTEGER PRECISION VIEWS
+
+          TIMESTAMPTZ ZONE
+
+          TIME NUMERIC
+
+          NAMED_STRUCT CREATE_UNION
+
+          ROUND BROUND FLOOR CEIL CEILING RAND EXP LN LOG10 LOG2 LOG POW POWER SQRT BIN
+          HEX UNHEX CONV ABS PMOD SIN ASIN COS ACOS TAN ATAN DEGREES RADIANS POSITIVE
+          NEGATIVE SIGN E PI FACTORIAL CBRT SHIFTLEFT SHIFTRIGHT SHIFTRIGHTUNSIGNED
+          GREATEST LEAST WIDTH_BUCKET SIZE SIZE MAP_KEYS MAP_VALUES ARRAY_CONTAINS
+          SORT_ARRAY BINARY CAST FROM_UNIXTIME UNIX_TIMESTAMP UNIX_TIMESTAMP
+          UNIX_TIMESTAMP TO_DATE YEAR QUARTER MONTH DAY DAYOFMONTH HOUR MINUTE SECOND
+          WEEKOFYEAR EXTRACT DATEDIFF DATE_ADD DATE_SUB FROM_UTC_TIMESTAMP
+          TO_UTC_TIMESTAMP CURRENT_DATE CURRENT_TIMESTAMP ADD_MONTHS LAST_DAY NEXT_DAY
+          TRUNC MONTHS_BETWEEN DATE_FORMAT IF ISNULL ISNOTNULL  NVL COALESCE CASE WHEN
+          then else end NULLIF ASSERT_TRUE ASCII BASE64 CHARACTER_LENGTH CHR CONCAT
+          CONTEXT_NGRAMS CONCAT_WS CONCAT_WS DECODE ELT ENCODE FIELD  FIND_IN_SET
+          FORMAT_NUMBER GET_JSON_OBJECT IN_FILE INSTR LENGTH LOCATE LOWER LCASE LPAD LTRIM
+          NGRAMS OCTET_LENGTH PARSE_URL PRINTF REGEXP_EXTRACT REGEXP_REPLACE REPEAT
+          REPLACE REVERSE RPAD RTRIM SENTENCES SPACE SPLIT STR_TO_MAP SUBSTR SUBSTRING
+          SUBSTRING_INDEX TRANSLATE TRIM UNBASE64 UPPER UCASE INITCAP LEVENSHTEIN SOUNDEX
+          MASK MASK_FIRST_N MASK_LAST_N MASK_SHOW_FIRST_N MASK_SHOW_LAST_N MASK_HASH
+          JAVA_METHOD REFLECT HASH CURRENT_USER LOGGED_IN_USER CURRENT_DATABASE MD5 SHA1
+          SHA CRC32 SHA2 AES_ENCRYPT AES_DECRYPT VERSION COUNT SUM AVG MIN MAX VARIANCE
+          VAR_POP VAR_SAMP STDDEV_POP STDDEV_SAMP COVAR_POP COVAR_SAMP CORR PERCENTILE
+          PERCENTILE_APPROX PERCENTILE_APPROX REGR_AVGX REGR_AVGY REGR_COUNT
+          REGR_INTERCEPT REGR_R2 REGR_SLOPE REGR_SXX REGR_SXY REGR_SYY HISTOGRAM_NUMERIC
+          COLLECT_SET COLLECT_LIST NTILE EXPLODE EXPLODE POSEXPLODE INLINE STACK
+
+          JSON_TUPLE PARSE_URL_TUPLE
+
+          XPATH XPATH_SHORT XPATH_INT XPATH_LONG XPATH_FLOAT XPATH_DOUBLE
+          XPATH_NUMBER XPATH_STRING GET_JSON_OBJECT JSON_TUPLE
+
+          PARSE_URL_TUPLE
+        ))
+      end
+
+      def self.keywords_type
+        # source: https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types
+        @keywords_type ||= Set.new(%w(
+            TINYINT SMALLINT INT INTEGER BIGINT FLOAT DOUBLE PRECISION DECIMAL NUMERIC
+            TIMESTAMP DATE INTERVAL
+            STRING VARCHAR CHAR
+            BOOLEAN BINARY
+            ARRAY MAP STRUCT UNIONTYPE
+        ))
+      end
+
+      prepend :root do
+        # a double-quoted string is a string literal in Hive QL.
+        rule %r/"/, Str::Double, :double_string
+
+        # interpolation of variables through ${...}
+        rule %r/\$\{/, Name::Variable, :hive_variable
+      end
+
+      prepend :single_string do
+        rule %r/\$\{/, Name::Variable, :hive_variable
+        rule %r/[^\\'\$]+/, Str::Single
+      end
+
+      prepend :double_string do
+        rule %r/\$\{/, Name::Variable, :hive_variable
+        # double-quoted strings are string literals so need to change token
+        rule %r/"/, Str::Double, :pop!
+        rule %r/[^\\"\$]+/, Str::Double
+      end
+
+      state :hive_variable do
+        rule %r/\}/, Name::Variable, :pop!
+        rule %r/[^\}]+/, Name::Variable
+      end
+
+    end
+  end
+end

data/lib/rouge/lexers/html.rb

@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class HTML < RegexLexer
+      title "HTML"
+      desc "HTML, the markup language of the web"
+      tag 'html'
+      filenames '*.htm', '*.html', '*.xhtml'
+      mimetypes 'text/html', 'application/xhtml+xml'
+
+      def self.detect?(text)
+        return true if text.doctype?(/\bhtml\b/i)
+        return false if text =~ /\A<\?xml\b/
+        return true if text =~ /<\s*html\b/
+      end
+
+      start do
+        @javascript = Javascript.new(options)
+        @css = CSS.new(options)
+      end
+
+      state :root do
+        rule %r/[^<&]+/m, Text
+        rule %r/&\S*?;/, Name::Entity
+        rule %r/<!DOCTYPE .*?>/im, Comment::Preproc
+        rule %r/<!\[CDATA\[.*?\]\]>/m, Comment::Preproc
+        rule %r/<!--/, Comment, :comment
+        rule %r/<\?.*?\?>/m, Comment::Preproc # php? really?
+
+        rule %r/<\s*script\s*/m do
+          token Name::Tag
+          @javascript.reset!
+          push :script_content
+          push :tag
+        end
+
+        rule %r/<\s*style\s*/m do
+          token Name::Tag
+          @css.reset!
+          @lang = @css
+          push :style_content
+          push :tag
+        end
+
+        rule %r(</), Name::Tag, :tag_end
+        rule %r/</, Name::Tag, :tag_start
+
+        rule %r(<\s*[a-zA-Z0-9:-]+), Name::Tag, :tag # opening tags
+        rule %r(<\s*/\s*[a-zA-Z0-9:-]+\s*>), Name::Tag # closing tags
+      end
+
+      state :tag_end do
+        mixin :tag_end_end
+        rule %r/[a-zA-Z0-9:-]+/ do
+          token Name::Tag
+          goto :tag_end_end
+        end
+      end
+
+      state :tag_end_end do
+        rule %r/\s+/, Text
+        rule %r/>/, Name::Tag, :pop!
+      end
+
+      state :tag_start do
+        rule %r/\s+/, Text
+
+        rule %r/[a-zA-Z0-9:-]+/ do
+          token Name::Tag
+          goto :tag
+        end
+
+        rule(//) { goto :tag }
+      end
+
+      state :comment do
+        rule %r/[^-]+/, Comment
+        rule %r/-->/, Comment, :pop!
+        rule %r/-/, Comment
+      end
+
+      state :tag do
+        rule %r/\s+/m, Text
+        rule %r/[a-zA-Z0-9_:\[\]()*.-]+\s*=\s*/m, Name::Attribute, :attr
+        rule %r/[a-zA-Z0-9_:#*-]+/, Name::Attribute
+        rule %r(/?\s*>)m, Name::Tag, :pop!
+      end
+
+      state :attr do
+        # TODO: are backslash escapes valid here?
+        rule %r/"/ do
+          token Str
+          goto :dq
+        end
+
+        rule %r/'/ do
+          token Str
+          goto :sq
+        end
+
+        rule %r/[^\s>]+/, Str, :pop!
+      end
+
+      state :dq do
+        rule %r/"/, Str, :pop!
+        rule %r/[^"]+/, Str
+      end
+
+      state :sq do
+        rule %r/'/, Str, :pop!
+        rule %r/[^']+/, Str
+      end
+
+      state :script_content do
+        rule %r([^<]+) do
+          delegate @javascript
+        end
+
+        rule %r(<\s*/\s*script\s*>)m, Name::Tag, :pop!
+
+        rule %r(<) do
+          delegate @javascript
+        end
+      end
+
+      state :style_content do
+        rule %r/[^<]+/ do
+          delegate @lang
+        end
+
+        rule %r(<\s*/\s*style\s*>)m, Name::Tag, :pop!
+
+        rule %r/</ do
+          delegate @lang
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/http.rb

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class HTTP < RegexLexer
+      tag 'http'
+      title "HTTP"
+      desc 'http requests and responses'
+
+      def self.http_methods
+        @http_methods ||= %w(GET POST PUT DELETE HEAD OPTIONS TRACE PATCH)
+      end
+
+      def content_lexer
+        return Lexers::PlainText unless @content_type
+
+        @content_lexer ||= Lexer.guess_by_mimetype(@content_type)
+      rescue Lexer::AmbiguousGuess
+        @content_lexer = Lexers::PlainText
+      end
+
+      start { @content_type = 'text/plain' }
+
+      state :root do
+        # request
+        rule %r(
+          (#{HTTP.http_methods.join('|')})([ ]+) # method
+          ([^ ]+)([ ]+)                          # path
+          (HTTPS?)(/)(\d(?:\.\d)?)(\r?\n|$)      # http version
+        )ox do
+          groups(
+            Name::Function, Text,
+            Name::Namespace, Text,
+            Keyword, Operator, Num, Text
+          )
+
+          push :headers
+        end
+
+        # response
+        rule %r(
+          (HTTPS?)(/)(\d(?:\.\d))([ ]+)  # http version
+          (\d{3})([ ]+)?                 # status
+          ([^\r\n]*)?(\r?\n|$)           # status message
+        )x do
+          groups(
+            Keyword, Operator, Num, Text,
+            Num, Text,
+            Name::Exception, Text
+          )
+          push :headers
+        end
+      end
+
+      state :headers do
+        rule %r/([^\s:]+)( *)(:)( *)([^\r\n]+)(\r?\n|$)/ do |m|
+          key = m[1]
+          value = m[5]
+          if key.strip.casecmp('content-type').zero?
+            @content_type = value.split(';')[0].downcase
+          end
+
+          groups Name::Attribute, Text, Punctuation, Text, Str, Text
+        end
+
+        rule %r/([^\r\n]+)(\r?\n|$)/ do
+          groups Str, Text
+        end
+
+        rule %r/\r?\n/, Text, :content
+      end
+
+      state :content do
+        rule %r/.+/m do
+          delegate(content_lexer)
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/hylang.rb

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class HyLang < RegexLexer
+      title "HyLang"
+      desc "The HyLang programming language (hylang.org)"
+
+      tag 'hylang'
+      aliases 'hy'
+
+      filenames '*.hy'
+
+      mimetypes 'text/x-hy', 'application/x-hy'
+
+      def self.keywords
+        @keywords ||= Set.new %w(
+          False None True and as assert break class continue def
+          del elif else except finally for from global if import
+          in is lambda nonlocal not or pass raise return try
+        )
+      end
+
+      def self.builtins
+        @builtins ||= Set.new %w(
+          != % %= & &= * ** **= *= *map
+          + += , - -= -> ->> . / //
+          //= /= < << <<= <= = > >= >>
+          >>= @ @= ^ ^= accumulate apply as-> assoc butlast
+          calling-module-name car cdr chain coll? combinations comp complement compress cond
+          cons cons? constantly count cut cycle dec defclass defmacro defmacro!
+          defmacro/g! defmain defn defreader dict-comp disassemble dispatch-reader-macro distinct do doto
+          drop drop-last drop-while empty? eval eval-and-compile eval-when-compile even? every? filter
+          first flatten float? fn for* fraction genexpr gensym get group-by
+          identity if* if-not if-python2 inc input instance? integer integer-char? integer?
+          interleave interpose islice iterable? iterate iterator? juxt keyword keyword? last
+          let lif lif-not list* list-comp macro-error macroexpand macroexpand-1 map merge-with
+          multicombinations name neg? none? not-in not? nth numeric? odd? partition
+          permutations pos? product quasiquote quote range read read-str reduce remove
+          repeat repeatedly require rest second set-comp setv some string string?
+          symbol? take take-nth take-while tee unless unquote unquote-splicing when with*
+          with-decorator with-gensyms xor yield-from zero? zip zip-longest | |= ~
+        )
+      end
+
+      identifier = %r([\w!$%*+,<=>?/.-]+)
+      keyword = %r([\w!\#$%*+,<=>?/.-]+)
+
+      def name_token(name)
+        return Keyword if self.class.keywords.include?(name)
+        return Name::Builtin if self.class.builtins.include?(name)
+        nil
+      end
+
+      state :root do
+        rule %r/;.*?$/, Comment::Single
+        rule %r/\s+/m, Text::Whitespace
+
+        rule %r/-?\d+\.\d+/, Num::Float
+        rule %r/-?\d+/, Num::Integer
+        rule %r/0x-?[0-9a-fA-F]+/, Num::Hex
+
+        rule %r/"(\\.|[^"])*"/, Str
+        rule %r/'#{keyword}/, Str::Symbol
+        rule %r/::?#{keyword}/, Name::Constant
+        rule %r/\\(.|[a-z]+)/i, Str::Char
+
+
+        rule %r/~@|[`\'#^~&@]/, Operator
+
+        rule %r/(\()(\s*)(#{identifier})/m do |m|
+          token Punctuation, m[1]
+          token Text::Whitespace, m[2]
+          token(name_token(m[3]) || Name::Function, m[3])
+        end
+
+        rule identifier do |m|
+          token name_token(m[0]) || Name
+        end
+
+        # vectors
+        rule %r/[\[\]]/, Punctuation
+
+        # maps
+        rule %r/[{}]/, Punctuation
+
+        # parentheses
+        rule %r/[()]/, Punctuation
+      end
+    end
+  end
+end
+