rouge 3.16.0 → 3.21.0

data/lib/rouge/lexers/console.rb

@@ -9,12 +9,14 @@ module Rouge
     # line before passing the remainder of the line to the language lexer for
     # the shell (by default, the {Shell} lexer).
     #
-    # The {ConsoleLexer} class accepts four options:
+    # The {ConsoleLexer} class accepts five options:
     # 1. **lang**: the shell language to lex (default: `shell`);
     # 2. **output**: the output language (default: `plaintext?token=Generic.Output`);
     # 3. **prompt**: comma-separated list of strings that indicate the end of a
     #    prompt (default: `$,#,>,;`);
     # 4. **comments**: whether to enable comments.
+    # 5. **error**: comma-separated list of strings that indicate the start of an
+    #    error message
     #
     # The comments option, if enabled, will lex lines that begin with a `#` as a
     # comment. Please note that this option will only work if the prompt is
@@ -39,12 +41,13 @@ module Rouge
       tag 'console'
       aliases 'terminal', 'shell_session', 'shell-session'
       filenames '*.cap'
-      desc 'A generic lexer for shell sessions. Accepts ?lang and ?output lexer options, a ?prompt option, and ?comments to enable # comments.'
+      desc 'A generic lexer for shell sessions. Accepts ?lang and ?output lexer options, a ?prompt option, ?comments to enable # comments, and ?error to handle error messages.'
 
       option :lang, 'the shell language to lex (default: shell)'
       option :output, 'the output language (default: plaintext?token=Generic.Output)'
       option :prompt, 'comma-separated list of strings that indicate the end of a prompt. (default: $,#,>,;)'
       option :comments, 'enable hash-comments at the start of a line - otherwise interpreted as a prompt. (default: false, implied by ?prompt not containing `#`)'
+      option :error, 'comma-separated list of strings that indicate the start of an error message'
 
       def initialize(*)
         super
@@ -52,14 +55,24 @@ module Rouge
         @lang = lexer_option(:lang) { 'shell' }
         @output = lexer_option(:output) { PlainText.new(token: Generic::Output) }
         @comments = bool_option(:comments) { :guess }
+        @error = list_option(:error) { nil }
       end
 
-      def prompt_regex
-        @prompt_regex ||= begin
-          /^#{prompt_prefix_regex}(?:#{end_chars.map(&Regexp.method(:escape)).join('|')})/
+      # whether to allow comments. if manually specifying a prompt that isn't
+      # simply "#", we flag this to on
+      def allow_comments?
+        case @comments
+        when :guess
+          @prompt && !@prompt.empty? && !end_chars.include?('#')
+        else
+          @comments
         end
       end
 
+      def comment_regex
+        /\A\s*?#/
+      end
+
       def end_chars
         @end_chars ||= if @prompt.any?
           @prompt.reject { |c| c.empty? }
@@ -70,22 +83,9 @@ module Rouge
         end
       end
 
-      # whether to allow comments. if manually specifying a prompt that isn't
-      # simply "#", we flag this to on
-      def allow_comments?
-        case @comments
-        when :guess
-          @prompt && !@prompt.empty? && !end_chars.include?('#')
-        else
-          @comments
-        end
-      end
-
-      def prompt_prefix_regex
-        if allow_comments?
-          /[^<#]*?/m
-        else
-          /.*?/m
+      def error_regex
+        @error_regex ||= if @error.any?
+          /^(?:#{@error.map(&Regexp.method(:escape)).join('|')})/
         end
       end
 
@@ -102,6 +102,10 @@ module Rouge
         end
       end
 
+      def line_regex
+        /(\\.|[^\\])*?(\n|$)/m
+      end
+
       def output_lexer
         @output_lexer ||= case @output
         when nil
@@ -115,22 +119,6 @@ module Rouge
         end
       end
 
-      def line_regex
-        /(\\.|[^\\])*?(\n|$)/m
-      end
-
-      def comment_regex
-        /\A\s*?#/
-      end
-
-      def stream_tokens(input, &output)
-        input = StringScanner.new(input)
-        lang_lexer.reset!
-        output_lexer.reset!
-
-        process_line(input, &output) while !input.eos?
-      end
-
       def process_line(input, &output)
         input.scan(line_regex)
 
@@ -162,6 +150,12 @@ module Rouge
           lang_lexer.reset!
 
           yield Comment, input[0]
+        elsif error_regex =~ input[0]
+          puts "console: matched error #{input[0].inspect}" if @debug
+          output_lexer.reset!
+          lang_lexer.reset!
+
+          yield Generic::Error, input[0]
         else
           puts "console: matched output #{input[0].inspect}" if @debug
           lang_lexer.reset!
@@ -169,6 +163,28 @@ module Rouge
           output_lexer.continue_lex(input[0], &output)
         end
       end
+
+      def prompt_prefix_regex
+        if allow_comments?
+          /[^<#]*?/m
+        else
+          /.*?/m
+        end
+      end
+
+      def prompt_regex
+        @prompt_regex ||= begin
+          /^#{prompt_prefix_regex}(?:#{end_chars.map(&Regexp.method(:escape)).join('|')})/
+        end
+      end
+
+      def stream_tokens(input, &output)
+        input = StringScanner.new(input)
+        lang_lexer.reset!
+        output_lexer.reset!
+
+        process_line(input, &output) while !input.eos?
+      end
     end
   end
 end

data/lib/rouge/lexers/cpp.rb

@@ -22,10 +22,10 @@ module Rouge
 
       def self.keywords
         @keywords ||= super + Set.new(%w(
-          asm auto catch const_cast delete dynamic_cast explicit export
-          friend mutable namespace new operator private protected public
-          reinterpret_cast restrict size_of static_cast template this throw
-          throws typeid typename using virtual final override
+          asm auto catch const_cast delete dynamic_cast explicit export friend
+          mutable namespace new operator private protected public
+          reinterpret_cast restrict size_of static_cast this throw throws
+          typeid typename using virtual final override
 
           alignas alignof constexpr decltype noexcept static_assert
           thread_local try
@@ -57,7 +57,8 @@ module Rouge
       dq = /\d('?\d)*/
 
       prepend :statements do
-        rule %r/class\b/, Keyword, :classname
+        rule %r/(class|struct)\b/, Keyword, :classname
+        rule %r/template\b/, Keyword, :template
         rule %r/\d+(\.\d+)?(?:h|(?:min)|s|(?:ms)|(?:us)|(?:ns))/, Num::Other
         rule %r((#{dq}[.]#{dq}?|[.]#{dq})(e[+-]?#{dq}[lu]*)?)i, Num::Float
         rule %r(#{dq}e[+-]?#{dq}[lu]*)i, Num::Float
@@ -67,15 +68,28 @@ module Rouge
         rule %r/#{dq}[lu]*/i, Num::Integer
         rule %r/\bnullptr\b/, Name::Builtin
         rule %r/(?:u8|u|U|L)?R"([a-zA-Z0-9_{}\[\]#<>%:;.?*\+\-\/\^&|~!=,"']{,16})\(.*?\)\1"/m, Str
+        rule %r/::/, Operator
       end
 
       state :classname do
         rule id, Name::Class, :pop!
 
         # template specification
-        rule %r/\s*(?=>)/m, Text, :pop!
-        rule %r/[.]{3}/, Operator
         mixin :whitespace
+        rule %r/[.]{3}/, Operator
+        rule %r/,/, Punctuation, :pop!
+        rule(//) { pop! }
+      end
+
+      state :template do
+        rule %r/[>;]/, Punctuation, :pop!
+        rule %r/typename\b/, Keyword, :classname
+        mixin :statements
+      end
+
+      state :case do
+        rule %r/:(?!:)/, Punctuation, :pop!
+        mixin :statements
       end
     end
   end

data/lib/rouge/lexers/css.rb

@@ -11,7 +11,9 @@ module Rouge
       filenames '*.css'
       mimetypes 'text/css'
 
-      identifier = /[a-zA-Z0-9_-]+/
+      # Documentation: https://www.w3.org/TR/CSS21/syndata.html#characters
+
+      identifier = /[\p{L}_-][\p{Word}\p{Cf}-]*/
       number = /-?(?:[0-9]+(\.[0-9]+)?|\.[0-9]+)/
 
       def self.attributes

data/lib/rouge/lexers/cypher.rb

@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Cypher < RegexLexer
+      tag 'cypher'
+      aliases 'cypher'
+      filenames '*.cypher'
+      mimetypes 'application/x-cypher-query'
+
+      title "Cypher"
+      desc 'The Cypher query language (neo4j.com/docs/cypher-manual)'
+
+      def self.functions
+        @functions ||= Set.new %w(
+          ABS ACOS ALLSHORTESTPATHS ASIN ATAN ATAN2 AVG CEIL COALESCE COLLECT
+          COS COT COUNT DATE DEGREES E ENDNODE EXP EXTRACT FILTER FLOOR
+          HAVERSIN HEAD ID KEYS LABELS LAST LEFT LENGTH LOG LOG10 LOWER LTRIM
+          MAX MIN NODE NODES PERCENTILECONT PERCENTILEDISC PI RADIANS RAND
+          RANGE REDUCE REL RELATIONSHIP RELATIONSHIPS REPLACE REVERSE RIGHT
+          ROUND RTRIM SHORTESTPATH SIGN SIN SIZE SPLIT SQRT STARTNODE STDEV
+          STDEVP STR SUBSTRING SUM TAIL TAN TIMESTAMP TOFLOAT TOINT TOINTEGER
+          TOSTRING TRIM TYPE UPPER
+        )
+      end
+
+      def self.predicates
+        @predicates ||= Set.new %w(
+          ALL AND ANY CONTAINS EXISTS HAS IN NONE NOT OR SINGLE XOR
+        )
+      end
+
+      def self.keywords
+        @keywords ||= Set.new %w(
+          AS ASC ASCENDING ASSERT BY CASE COMMIT CONSTRAINT CREATE CSV CYPHER
+          DELETE DESC DESCENDING DETACH DISTINCT DROP ELSE END ENDS EXPLAIN
+          FALSE FIELDTERMINATOR FOREACH FROM HEADERS IN INDEX IS JOIN LIMIT
+          LOAD MATCH MERGE NULL ON OPTIONAL ORDER PERIODIC PROFILE REMOVE
+          RETURN SCAN SET SKIP START STARTS THEN TRUE UNION UNIQUE UNWIND USING
+          WHEN WHERE WITH CALL YIELD
+        )
+      end
+
+      state :root do
+        rule %r/[\s]+/, Text
+        rule %r(//.*?$), Comment::Single
+
+        rule %r([*+\-<>=&|~%^]), Operator
+        rule %r/[{}),;\[\]]/, Str::Symbol
+
+        # literal number
+        rule %r/(\w+)(:)(\s*)(-?[._\d]+)/ do
+          groups Name::Label, Str::Delimiter, Text::Whitespace, Num
+        end
+
+        # function-like
+        # - "name("
+        # - "name  ("
+        # - "name ("
+        rule %r/(\w+)(\s*)(\()/ do |m|
+          name = m[1].upcase
+          if self.class.functions.include? name
+            groups Name::Function, Text::Whitespace, Str::Symbol
+          elsif self.class.keywords.include? name
+            groups Keyword, Text::Whitespace, Str::Symbol
+          else
+            groups Name, Text::Whitespace, Str::Symbol
+          end
+        end
+
+        rule %r/:\w+/, Name::Class
+
+        # number range
+        rule %r/(-?\d+)(\.\.)(-?\d+)/ do
+          groups Num, Operator, Num
+        end
+
+        # numbers
+        rule %r/(\d+\.\d*|\d*\.\d+)(e[+-]?\d+)?/i, Num::Float
+        rule %r/\d+e[+-]?\d+/i, Num::Float
+        rule %r/0[0-7]+/, Num::Oct
+        rule %r/0x[a-f0-9]+/i, Num::Hex
+        rule %r/\d+/, Num::Integer
+
+        rule %r([.\w]+:), Name::Property
+
+        # remaining "("
+        rule %r/\(/, Str::Symbol
+
+        rule %r/[.\w$]+/ do |m|
+          match = m[0].upcase
+          if self.class.predicates.include? match
+            token Operator::Word
+          elsif self.class.keywords.include? match
+            token Keyword
+          else
+            token Name
+          end
+        end
+
+        rule %r/"(\\\\|\\"|[^"])*"/, Str::Double
+        rule %r/'(\\\\|\\'|[^'])*'/, Str::Single
+        rule %r/`(\\\\|\\`|[^`])*`/, Str::Backtick
+      end
+    end
+  end
+end

data/lib/rouge/lexers/datastudio.rb

@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Datastudio < RegexLexer
+      tag 'datastudio'
+      filenames '*.job'
+      mimetypes 'text/x-datastudio'
+
+      title "Datastudio"
+      desc 'Datastudio scripting language'
+
+      id = /@?[_a-z]\w*/i
+
+      def self.sql_keywords
+        @sql_keywords ||= %w(
+          ABORT ABS ABSOLUTE ACCESS ADA ADD ADMIN AFTER AGGREGATE ALIAS
+          ALL ALLOCATE ALTER ANALYSE ANALYZE AND ANY ARE AS ASC ASENSITIVE
+          ASSERTION ASSIGNMENT ASYMMETRIC AT ATOMIC AUTHORIZATION
+          AVG BACKWARD BEFORE BEGIN BETWEEN BITVAR BIT_LENGTH BOTH
+          BREADTH BY C CACHE CALL CALLED CARDINALITY CASCADE CASCADED
+          CASE CAST CATALOG CATALOG_NAME CHAIN CHARACTERISTICS
+          CHARACTER_LENGTH CHARACTER_SET_CATALOG CHARACTER_SET_NAME
+          CHARACTER_SET_SCHEMA CHAR_LENGTH CHECK CHECKED CHECKPOINT
+          CLASS CLASS_ORIGIN CLOB CLOSE CLUSTER COALSECE COBOL COLLATE
+          COLLATION COLLATION_CATALOG COLLATION_NAME COLLATION_SCHEMA
+          COLUMN COLUMN_NAME COMMAND_FUNCTION COMMAND_FUNCTION_CODE
+          COMMENT COMMIT COMMITTED COMPLETION CONDITION_NUMBER
+          CONNECT CONNECTION CONNECTION_NAME CONSTRAINT CONSTRAINTS
+          CONSTRAINT_CATALOG CONSTRAINT_NAME CONSTRAINT_SCHEMA
+          CONSTRUCTOR CONTAINS CONTINUE CONVERSION CONVERT COPY
+          CORRESPONTING COUNT CREATE CREATEDB CREATEUSER CROSS CUBE
+          CURRENT CURRENT_DATE CURRENT_PATH CURRENT_ROLE CURRENT_TIME
+          CURRENT_TIMESTAMP CURRENT_USER CURSOR CURSOR_NAME CYCLE DATA
+          DATABASE DATETIME_INTERVAL_CODE DATETIME_INTERVAL_PRECISION
+          DAY DEALLOCATE DECLARE DEFAULT DEFAULTS DEFERRABLE DEFERRED
+          DEFINED DEFINER DELETE DELIMITER DELIMITERS DEREF DESC DESCRIBE
+          DESCRIPTOR DESTROY DESTRUCTOR DETERMINISTIC DIAGNOSTICS
+          DICTIONARY DISCONNECT DISPATCH DISTINCT DO DOMAIN DROP
+          DYNAMIC DYNAMIC_FUNCTION DYNAMIC_FUNCTION_CODE EACH ELSE
+          ENCODING ENCRYPTED END END-EXEC EQUALS ESCAPE EVERY EXCEPT
+          ESCEPTION EXCLUDING EXCLUSIVE EXEC EXECUTE EXISTING EXISTS
+          EXPLAIN EXTERNAL EXTRACT FALSE FETCH FINAL FIRST FOR FORCE
+          FOREIGN FORTRAN FORWARD FOUND FREE FREEZE FROM FULL FUNCTION
+          G GENERAL GENERATED GET GLOBAL GO GOTO GRANT GRANTED GROUP
+          GROUPING HANDLER HAVING HIERARCHY HOLD HOST IDENTITY IGNORE
+          ILIKE IMMEDIATE IMMUTABLE IMPLEMENTATION IMPLICIT IN INCLUDING
+          INCREMENT INDEX INDITCATOR INFIX INHERITS INITIALIZE INITIALLY
+          INNER INOUT INPUT INSENSITIVE INSERT INSTANTIABLE INSTEAD
+          INTERSECT INTO INVOKER IS ISNULL ISOLATION ITERATE JOIN KEY
+          KEY_MEMBER KEY_TYPE LANCOMPILER LANGUAGE LARGE LAST LATERAL
+          LEADING LEFT LENGTH LESS LEVEL LIKE LIMIT LISTEN LOAD LOCAL
+          LOCALTIME LOCALTIMESTAMP LOCATION LOCATOR LOCK LOWER MAP MATCH
+          MAX MAXVALUE MESSAGE_LENGTH MESSAGE_OCTET_LENGTH MESSAGE_TEXT
+          METHOD MIN MINUTE MINVALUE MOD MODE MODIFIES MODIFY MONTH
+          MORE MOVE MUMPS NAMES NATURAL NCLOB NEW NEXT
+          NO NOCREATEDB NOCREATEUSER NONE NOT NOTHING NOTIFY NOTNULL
+          NULL NULLABLE NULLIF OBJECT OCTET_LENGTH OF OFF OFFSET OIDS
+          OLD ON ONLY OPEN OPERATION OPERATOR OPTION OPTIONS OR ORDER
+          ORDINALITY OUT OUTER OUTPUT OVERLAPS OVERLAY OVERRIDING
+          OWNER PAD PARAMETER PARAMETERS PARAMETER_MODE PARAMATER_NAME
+          PARAMATER_ORDINAL_POSITION PARAMETER_SPECIFIC_CATALOG
+          PARAMETER_SPECIFIC_NAME PARAMATER_SPECIFIC_SCHEMA PARTIAL PASCAL
+          PENDANT PLACING PLI POSITION POSTFIX PREFIX PREORDER
+          PREPARE PRESERVE PRIMARY PRIOR PRIVILEGES PROCEDURAL PROCEDURE
+          PUBLIC READ READS RECHECK RECURSIVE REF REFERENCES REFERENCING
+          REINDEX RELATIVE RENAME REPEATABLE REPLACE RESET RESTART
+          RESTRICT RESULT RETURN RETURNED_LENGTH RETURNED_OCTET_LENGTH
+          RETURNED_SQLSTATE RETURNS REVOKE RIGHT ROLE ROLLBACK ROLLUP
+          ROUTINE ROUTINE_CATALOG ROUTINE_NAME ROUTINE_SCHEMA ROW ROWS
+          ROW_COUNT RULE SAVE_POINT SCALE SCHEMA SCHEMA_NAME SCOPE SCROLL
+          SEARCH SECOND SECURITY SELECT SELF SENSITIVE SERIALIZABLE
+          SERVER_NAME SESSION SESSION_USER SET SETOF SETS SHARE SHOW
+          SIMILAR SIMPLE SIZE SOME SOURCE SPACE SPECIFIC SPECIFICTYPE
+          SPECIFIC_NAME SQL SQLCODE SQLERROR SQLEXCEPTION SQLSTATE
+          SQLWARNINIG STABLE START STATE STATEMENT STATIC STATISTICS
+          STDIN STDOUT STORAGE STRICT STRUCTURE STYPE SUBCLASS_ORIGIN
+          SUBLIST SUBSTRING SUM SYMMETRIC SYSID SYSTEM SYSTEM_USER
+          TABLE TABLE_NAME  TEMP TEMPLATE TEMPORARY TERMINATE THAN THEN
+          TIMEZONE_HOUR TIMEZONE_MINUTE TO TOAST TRAILING
+          TRANSATION TRANSACTIONS_COMMITTED TRANSACTIONS_ROLLED_BACK
+          TRANSATION_ACTIVE TRANSFORM TRANSFORMS TRANSLATE TRANSLATION
+          TREAT TRIGGER TRIGGER_CATALOG TRIGGER_NAME TRIGGER_SCHEMA TRIM
+          TRUE TRUNCATE TRUSTED TYPE UNCOMMITTED UNDER UNENCRYPTED UNION
+          UNIQUE UNKNOWN UNLISTEN UNNAMED UNNEST UNTIL UPDATE UPPER
+          USAGE USER USER_DEFINED_TYPE_CATALOG USER_DEFINED_TYPE_NAME
+          USER_DEFINED_TYPE_SCHEMA USING VACUUM VALID VALIDATOR VALUES
+          VARIABLE VERBOSE VERSION VIEW VOLATILE WHEN WHENEVER WHERE
+          WITH WITHOUT WORK WRITE ZONE
+        )
+      end
+
+      state :whitespace do
+        rule %r/\s+/m, Text
+        rule %r(//.*?$), Comment::Single
+        rule %r(/[*].*?[*]/)m, Comment::Multiline
+      end
+
+      state :string do
+        rule %r/%(\\.|.)+?%/, Str::Escape
+        rule %r/\\"/, Str::Double
+        rule %r/"/, Str::Double, :pop!
+        rule %r/./m, Str::Double
+      end
+
+      state :string_s do
+        rule %r/%(\\.|.)+?%/, Str::Escape
+        rule %r/\\'/, Str::Single
+        rule %r/'/, Str::Single, :pop!
+        rule %r/./m, Str::Single
+      end
+
+      state :root do
+        mixin :whitespace
+
+        rule %r/^:#{id}/, Name::Label
+        rule %r/@#{id}(\.#{id})?/m, Name::Entity
+        rule %r/%(\\.|.)+?%/, Name::Variable
+        rule %r/[~!%^&*()+=|\[\]{}:;,.<>\/?-]/, Punctuation
+        rule %r/"/, Str::Double, :string
+        rule %r/'/, Str::Single, :string_s
+        rule %r/\d(\.\d*)?/i, Num
+        rule %r/#{id}(?=\s*[(])/, Name::Function
+        rule id do |m|
+          name = m[0].upcase
+          
+          if self.class.sql_keywords.include? name
+            token Keyword
+          else
+            token Name
+          end
+        end
+      end
+
+    end
+  end
+end