rouge 4.7.0 → 5.0.0

data/lib/rouge/guessers/disambiguation.rb

@@ -90,7 +90,7 @@ module Rouge
       disambiguate '*.m' do
         next ObjectiveC if matches?(/@(end|implementation|protocol|property)\b/)
         next ObjectiveC if contains?('@"')
-        
+
         # Objective-C dereferenced pointers and Mathematica comments are similar.
         # Disambiguate for Mathematica by looking for any amount of whitespace (or no whitespace)
         # followed by "(*" (e.g. `(* comment *)`).
@@ -107,7 +107,7 @@ module Rouge
         next Matlab if matches?(/^\s*?%/)
         # Matlab cell array creation: data = {
         next Matlab if matches?(/^\s*[a-zA-Z]\w*\s*=\s*\{/)
-        
+
         next Mason if matches? %r!(</?%|<&)!
       end
 

data/lib/rouge/guessers/glob_mapping.rb

@@ -32,12 +32,12 @@ module Rouge
         basename = File.basename(filename)
 
         collect_best(lexers) do |lexer|
-          (@glob_map[lexer.name] || []).map do |pattern|
+          (@glob_map[lexer.name] || []).filter_map do |pattern|
             if test_glob(pattern, basename)
               # specificity is better the fewer wildcards there are
               -pattern.scan(/[*?\[]/).size
             end
-          end.compact.min
+          end.min
         end
       end
     end

data/lib/rouge/guessers/modeline.rb

@@ -31,11 +31,11 @@ module Rouge
 
         source_text = get_source(@source)
 
-        lines = source_text.split(/\n/)
+        lines = source_text.split("\n")
 
         search_space = (lines.first(@lines) + lines.last(@lines)).join("\n")
 
-        matches = MODELINES.map { |re| re.match(search_space) }.compact
+        matches = MODELINES.filter_map { |re| re.match(search_space) }
         return lexers unless matches.any?
 
         match_set = Set.new(matches.map { |m| m[1] })

data/lib/rouge/lexer.rb

@@ -1,11 +1,6 @@
 # -*- coding: utf-8 -*- #
 # frozen_string_literal: true
 
-# stdlib
-require 'strscan'
-require 'set'
-require 'uri'
-
 module Rouge
   # @abstract
   # A lexer transforms text into a stream of `[token, chunk]` pairs.
@@ -129,14 +124,14 @@ module Rouge
       def demo_file(arg=:absent)
         return @demo_file = Pathname.new(arg) unless arg == :absent
 
-        @demo_file = Pathname.new(File.join(__dir__, 'demos', tag))
+        @demo_file ||= Pathname.new(File.join(__dir__, 'demos', tag))
       end
 
       # Specify or get a small demo string for this lexer
       def demo(arg=:absent)
         return @demo = arg unless arg == :absent
 
-        @demo = File.read(demo_file, mode: 'rt:bom|utf-8')
+        @demo ||= File.read(demo_file, mode: 'rt:bom|utf-8')
       end
 
       # @return a list of all lexers.
@@ -188,7 +183,7 @@ module Rouge
         return lexers[0] if lexers.size == 1
 
         if fallback
-          fallback.call(lexers)
+          yield(lexers)
         else
           raise Guesser::Ambiguous.new(lexers)
         end
@@ -233,7 +228,33 @@ module Rouge
         registry[name.to_s] = lexer
       end
 
+      def lazy_procs
+        @lazy_procs ||= []
+      end
+
     public
+      def eager_load!
+        return if @_loaded
+        @_loaded = true
+
+        lazy_procs.each { |b| instance_eval(&b) }
+
+        superclass.eager_load! unless superclass == Lexer
+
+        self
+      end
+
+      def lazy(auto: true, &block)
+        @skip_auto_load = true unless auto
+        lazy_procs << block
+      end
+
+      def skip_auto_load?
+        return true if @skip_auto_load
+        return superclass.skip_auto_load? unless superclass == Lexer
+        false
+      end
+
       # Used to specify or get the canonical name of this lexer class.
       #
       # @example
@@ -325,10 +346,15 @@ module Rouge
     def initialize(opts={})
       @options = {}
       opts.each { |k, v| @options[k.to_s] = v }
+      eager_load! unless self.class.skip_auto_load?
 
       @debug = Lexer.debug_enabled? && bool_option('debug')
     end
 
+    def eager_load!
+      self.class.eager_load!
+    end
+
     # Returns a new lexer with the given options set. Useful for e.g. setting
     # debug flags post hoc, or providing global overrides for certain options
     def with(opts={})
@@ -394,7 +420,7 @@ module Rouge
       if @options.key?(name_str)
         as_bool(@options[name_str])
       else
-        default ? default.call : false
+        default ? yield : false
       end
     end
 
@@ -420,13 +446,13 @@ module Rouge
 
       base = @options.delete(name.to_s)
       base = {} unless base.is_a?(Hash)
-      base.each { |k, v| out[k.to_s] = val_cast ? val_cast.call(v) : v }
+      base.each { |k, v| out[k.to_s] = val_cast ? yield(v) : v }
 
       @options.keys.each do |key|
         next unless key =~ /(\w+)\[(\w+)\]/ and $1 == name
         value = @options.delete(key)
 
-        out[$2] = val_cast ? val_cast.call(value) : value
+        out[$2] = val_cast ? yield(value) : value
       end
 
       out
@@ -487,12 +513,12 @@ module Rouge
           next
         end
 
-        b.call(last_token, last_val) if last_token
+        yield(last_token, last_val) if last_token
         last_token = tok
         last_val = val
       end
 
-      b.call(last_token, last_val) if last_token
+      yield(last_token, last_val) if last_token
     end
 
     # delegated to {Lexer.tag}
@@ -524,14 +550,7 @@ module Rouge
     end
   end
 
+  # container module for built-in lexers
   module Lexers
-    BASE_DIR = "#{__dir__}/lexers".freeze
-    @_loaded_lexers = {}
-
-    def self.load_lexer(relpath)
-      return if @_loaded_lexers.key?(relpath)
-      @_loaded_lexers[relpath] = true
-      Kernel::load File.join(BASE_DIR, relpath)
-    end
   end
 end

data/lib/rouge/lexers/abap/builtins.rb

@@ -0,0 +1,187 @@
+# ABAP keywords, builtins, RAP syntax and CDS syntax taken from 
+#
+# https://help.sap.com/doc/abapdocu_758_index_htm/7.58/en-US/index.htm?file=abenabap_shortref.htm
+# https://help.sap.com/doc/abapdocu_758_index_htm/7.58/en-us/abenbuilt_in_functions_overview.htm
+# https://help.sap.com/doc/abapdocu_758_index_htm/7.58/en-US/abencds_language_elements.htm
+# https://help.sap.com/doc/abapdocu_758_index_htm/7.58/en-us/abenabap_words.htm
+#
+# Updated with modern ABAP 7.40+ and CDS keywords from ABAP 7.58 documentation
+#
+# This file is manually maintained by Manuel Laggner <manuel@laggner.info>
+#
+# Please contact them with any requests for updates.
+
+module Rouge
+  module Lexers
+    class ABAP
+      KEYWORDS = Set.new %w(
+        *-INPUT ?TO ABAP-SOURCE ABBREVIATED ABS ABSTRACT ACCEPT ACCEPTING
+        ACCESS ACCORDING ACCP ACTION ACTIVATION ACTUAL ADD
+        ADD-CORRESPONDING ADJACENT ADJUST_NUMBERS AFTER ALIAS ALIASES ALIGN
+        ALL ALLOCATE ALLOW_PRECISION_LOSS ALLOWED ALPHA ANALYSIS ANALYZER
+        AMDP AND ANNOTATION ANNOTATIONS ANY APPEND APPENDAGE APPENDING
+        APPLICATION ARCHIVE AREA ARITHMETIC AS ASCENDING ASPECT ASSERT ASSIGN
+        ASSIGNED ASSIGNING ASSOCIATION ASSOCIATIONS ASYNCHRONOUS AT
+        ATTRIBUTES AUTHORITY AUTHORITY-CHECK AUTHORIZATION AUTO AVG BACK
+        BACKGROUND BACKUP BACKWARD BADI BASE BEFORE BEGIN BETWEEN BIG BINARY
+        BINTOHEX BIT BIT-AND BIT-NOT BIT-OR BIT-XOR BITFIELD BLACK BLANK
+        BLANKS BLOB BLOCK BLOCKED BLOCKS BLUE BOUND BOUNDARIES BOUNDS BOXED
+        BREAK-POINT BREAKUP BT BUFFER BULK BY BYPASSING BYTE BYTE-CA BYTE-CN
+        BYTE-CO BYTE-CS BYTE-NA BYTE-NS BYTE-ORDER CA CALL CALLING CASE CAST
+        CASTING CATCH CEIL CENTER CENTERED CHAIN CHAIN-INPUT CHAIN-REQUEST
+        CHANGE CHANGING CHANNELS CHAR CHAR-TO-HEX CHARACTER CHECK CHECKBOX
+        CHECK_BEFORE_SAVE CHILD CI_ CID CIRCULAR CLASS CLASS-CODING
+        CLASS-DATA CLASS-EVENTS CLASS-METHODS CLASS-POOL CDS CLEANUP
+        CLEANUP_FINALIZE CLEAR CLIENT CLIENTS CLNT CLOB CLOCK CLOSE CN CO
+        COALESCE CODE CODING COLLECT COLOR COLUMN COLUMNS COL_BACKGROUND
+        COL_GROUP COL_HEADING COL_KEY COL_NEGATIVE COL_NORMAL COL_POSITIVE
+        COL_TOTAL COMMENT COMMENTS COMMIT COMMON COMMUNICATION COMPARING
+        COMPONENT COMPONENTS COMPOSITION COMPRESSION COMPUTE CONCAT
+        CONCATENATE CONCAT_WITH_SPACE COND CONDENSE CONDITION CONNECT
+        CONNECTION CONSTANTS CONTEXT CONTEXTS CONTINUE CONTROL CONTROLS CONV
+        CONVERSION CONVERT COPIES COPY CORR CORR_SPEARMAN CORRESPONDING
+        CORRSPEARMAN COUNT COUNTRY COVER CP CPI CREATE CREATING CRITICAL CS
+        CUKY CURR CURRENCY CURRENCY_CONVERSION CURRENT CURSOR
+        CURSOR-SELECTION CUSTOMER CUSTOMER-FUNCTION DANGEROUS DATA DATABASE
+        DATAINFO DATASET DATE DATN DATN_ADD_DAYS DATN_ADD_MONTHS
+        DATN_DAYS_BETWEEN DATS DATS_ADD_DAYS DATS_ADD_MONTHS
+        DATS_DAYS_BETWEEN DATS_FROM_DATN DATS_IS_VALID DATS_TIMS_TO_TSTMP
+        DATS_TO_DATN DAYLIGHT DAYNAME DB2 DB6 DD/MM/YY DD/MM/YYYY DDMMYY
+        DEALLOCATE DEC DECFLOAT16 DECFLOAT16_DEC DECFLOAT34 DECFLOAT34_DEC
+        DECIMALS DECIMAL_SHIFT DECLARATIONS DEEP DEFAULT DEFERRED DEFINE
+        DEFINING DEFINITION DELETE DELETING DEMAND DENSE_RANK DEPARTMENT
+        DEPTH DESCENDING DESCRIBE DESTINATION DETAIL DETERMINATION DETERMINE
+        DETERMINISTIC DF16_DEC DF16_RAW DF16_SCL DF34_DEC DF34_RAW DF34_SCL
+        DIALOG DIRECTORY DISABLE DISCARDING DISCONNECT DISPLAY DISPLAY-MODE
+        DISTANCE DISTINCT DIV DIVIDE DIVIDE-CORRESPONDING DIVISION DO DUMMY
+        DUPLICATE DUPLICATES DURATION DURING DYNAMIC DYNPRO E EARLY EDIT
+        EDITOR-CALL ELSE ELSEIF EMPTY ENABLED ENABLING ENCODING END
+        END-ENHANCEMENT-SECTION END-LINES END-OF-DEFINITION END-OF-FILE
+        END-OF-EDITING END-OF-PAGE END-OF-SELECTION END-TEST-INJECTION
+        END-TEST-SEAM ENDAT ENDCASE ENDCATCH ENDCHAIN ENDCLASS ENDDO
+        ENDENHANCEMENT ENDEXEC ENDFORM ENDFUNCTION ENDIAN ENDIF ENDING
+        ENDINTERFACE ENDLOOP ENDMETHOD ENDMODULE ENDON ENDPROVIDE ENDSELECT
+        ENDTRY ENDWHILE ENDWITH ENGINEERING ENHANCEMENT ENHANCEMENT-POINT
+        ENHANCEMENT-SECTION ENHANCEMENTS ENTITIES ENTITY ENTRIES ENTRY ENUM
+        ENVIRONMENT EQ EQUIV ERROR ERRORMESSAGE ERRORS ESCAPE ESCAPING EVENT
+        EVENTS EXACT EXCEPT EXCEPTION EXCEPTION-TABLE EXCEPTIONS EXCLUDE
+        EXCLUDING EXEC EXECUTE EXISTS EXIT EXIT-COMMAND EXPAND EXPANDING
+        EXPIRATION EXPLICIT EXPONENT EXPORT EXPORTING EXTEND EXTENDED
+        EXTENSION EXTRACT EXTRACT_DAY EXTRACT_HOUR EXTRACT_MINUTE
+        EXTRACT_MONTH EXTRACT_SECOND EXTRACT_YEAR FAIL FETCH FIELD
+        FIELD-GROUPS FIELD-SYMBOL FIELD-SYMBOLS FIELDS FILE FILTER
+        FILTER-TABLE FILTERS FINAL FIND FIRST FIRST-LINE FIRST_VALUE
+        FIXED-POINT FKEQ FKGE FLOOR FLTP FLTP_TO_DEC FLUSH FOLLOWING FONT
+        FOR FORM FORMAT FORWARD FOUND FRAME FRAMES FREE FRIENDS FROM FULL
+        FUNCTION FUNCTION-POOL FUNCTIONALITY FURTHER GAPS GE GENERATE GET
+        GGM1 GEOM_EWKB GIVING GKEQ GKGE GLOBAL GRANT GRAPH GREEN GROUP
+        GROUPING GROUPS GT HANDLE HANDLER HARMLESS HASHED HAVING HDB
+        HEAD-LINES HEADER HEADERS HEADING HELP-ID HELP-REQUEST HEXTOBIN HIDE
+        HIERARCHY HIERARCHY_ANCESTORS HIERARCHY_ANCESTORS_AGGREGATE
+        HIERARCHY_DESCENDANTS HIERARCHY_DESCENDANTS_AGGREGATE
+        HIERARCHY_SIBLINGS HIGH HINT HOLD HOTSPOT I ICON ID IDENTIFICATION
+        IDENTIFIER IDS IF IGNORE IGNORING IMMEDIATELY IMPLEMENTATION
+        IMPLEMENTATIONS IMPLEMENTED IMPLICIT IMPORT IMPORTING IN INACTIVE
+        INCL INCLUDE INCLUDES INCLUDING INCREMENT INDEX INDEX-LINE INFOTYPES
+        INHERITING INIT INITIAL INITIALIZATION INNER INOUT INPUT INSERT
+        INSTANCE INSTANCES INSTR INT1 INT2 INT4 INT8 INTENSIFIED INTERFACE
+        INTERFACE-POOL INTERFACES INTERNAL INTERVALS INTO INVERSE
+        INVERTED-DATE IS ISO IS_VALID ITNO JOB JOIN KEEP KEEPING KERNEL KEY
+        KEYS KEYWORDS KIND LAG LANG LANGUAGE LAST LAST_VALUE LATE LAYOUT
+        LCHR LDB_PROCESS LE LEAD LEADING LEAVE LEFT LEFT-JUSTIFIED LEFTPLUS
+        LEFTSPACE LEGACY LENGTH LET LEVEL LEVELS LIKE LIKE_REGEXPR LINE
+        LINE-COUNT LINE-SELECTION LINE-SIZE LINEFEED LINES LIST
+        LIST-PROCESSING LISTBOX LITTLE LLANG LOAD LOAD-OF-PROGRAM LOB LOCAL
+        LOCALE LOCATE LOCATE_REGEXPR LOCATE_REGEXPR_AFTER LOCATOR LOCK
+        LOCKS LOG-POINT LOGFILE LOGICAL LONG LOOP LOW LOWER LPAD LPI LRAW
+        LT LTRIM M MAIL MAIN MAJOR-ID MANY MAP_MESSAGES MAPPING MARGIN MARK
+        MASK MATCH MATCH_RESULT MATCH_RESULT_TAB MATCHCODE MATCHED MAX
+        MAXIMUM MEASURES MEDIAN MEDIUM MEMBERS MEMORY MESH MESSAGE
+        MESSAGE-ID MESSAGES MESSAGING METHOD METHODS MIN MINIMUM MINOR-ID
+        MM/DD/YY MM/DD/YYYY MMDDYY MOD MODE MODIF MODIFIER MODIFY MODULE
+        MONTHNAME MOVE MOVE-CORRESPONDING MSSQLNT MULTIPLE MULTIPLY
+        MULTIPLY-CORRESPONDING NA NAME NAMETAB NATIVE NB NE NESTED NESTING
+        NEW NEW-LINE NEW-PAGE NEW-SECTION NEXT NO NO-DISPLAY NO-EXTENSION
+        NO-GAP NO-GAPS NO-GROUPING NO-HEADING NO-SCROLLING NO-SIGN NO-TITLE
+        NO-TOPOFPAGE NO-ZERO NODE NODES NON-UNICODE NON-UNIQUE NOT NP NS
+        NTILE NULL NULLS NUMBER NUMBERING NUMC O OBJECT OBJECTS OBLIGATORY
+        OCCURRENCE OCCURRENCES OCCURS OF OFF OFFSET ON ONE ONLY OPEN OPTION
+        OPTIONAL OPTIONS OR ORACLE ORDER ORPHANS OTHER OTHERS OUT OUTER
+        OUTPUT OUTPUT-LENGTH OVERFLOW OVERLAY PACK PACKAGE PAD PADDING PAGE
+        PAGES PARAMETER PARAMETER-TABLE PARAMETERS PARENT PARENTS PART
+        PARTIALLY PARTITION PATTERN PCRE PERCENTAGE PERFORM PERFORMING
+        PERSON PF PF-STATUS PINK PLACES POOL POSITION POS_HIGH POS_LOW
+        PRAGMAS PREC PRECEDING PRECOMPILED PREFERRED PRESERVING PRIMARY
+        PRINT PRINT-CONTROL PRIORITY PRIVATE PROCEDURE PROCESS PROGRAM
+        PROPERTY PROTECTED PROVIDE PUBLIC PUSH PUSHBUTTON PUT QUAN
+        QUEUE-ONLY QUICKINFO RADIOBUTTON RAISE RAISING RANGE RANGES RAW
+        RAWSTRING READ READ-ONLY READER RECEIVE RECEIVED RECEIVER RECEIVING
+        RED REDEFINITION REDUCE REDUCED REF REFERENCE REFRESH REGEX REJECT
+        RELATING REMOTE RENAMING REPLACE REPLACEMENT REPLACING REPORT
+        REPORTED REQUEST REQUESTED REQUIRED RESERVE RESET RESOLUTION
+        RESPECTING RESPONSE RESPONSES RESPONSIBLE RESULT RESULTS RESUMABLE
+        RESUME RETRY RETURN RETURNCODE RETURNING RETURNS RIGHT
+        RIGHT-JUSTIFIED RIGHTPLUS RIGHTSPACE RISK RMC_COMMUNICATION_FAILURE
+        RMC_INVALID_STATUS RMC_SYSTEM_FAILURE ROLE ROLLBACK ROUND ROW
+        ROW_NUMBER ROWS RPAD RTRIM RUN SAP SAP-SPOOL SAVING SCALAR
+        SCALE_PRESERVING SCALE_PRESERVING_SCIENTIFIC SCAN SCIENTIFIC
+        SCIENTIFIC_WITH_LEADING_ZERO SCHEMA SCREEN SCROLL SCROLL-BOUNDARY
+        SCROLLING SEARCH SECONDARY SECONDS SECTION SELECT SELECT-OPTIONS
+        SELECTION SELECTION-SCREEN SELECTION-SET SELECTION-SETS
+        SELECTION-TABLE SELECTIONS SEND SEPARATE SEPARATED SESSION SET SETS
+        SHARED SHIFT SHORT SHORTDUMP SHORTDUMP-ID SIBLINGS SIGN
+        SIGN_AS_POSTFIX SIMPLE SINGLE SIZE SKIP SKIPPING SMART SOME SORT
+        SORTABLE SORTED SOURCE SPACE SPANTREE SPECIFIED SPLIT SPOOL SPOTS
+        SQL SQLSCRIPT SSTRING STABLE STAMP STANDARD START START-OF-EDITING
+        START-OF-SELECTION STARTING STATE STATEMENT STATEMENTS STATIC
+        STATICS STATUSINFO STDDEV STEP-LOOP STEP STOP STRING STRING_AGG
+        STRUCTURE STRUCTURES STYLE SUBKEY SUBMATCH_RESULT
+        SUBMATCH_RESULT_TAB SUBMATCHES SUBMIT SUBROUTINE SUBSCREEN SUBSTRING
+        SUBSTRING_REGEXPR SUBTOTAL SUBTRACT SUBTRACT-CORRESPONDING SUFFIX
+        SUM SUMMARY SUMMING SUPPLIED SUPPLY SUPPRESS SWITCH SWITCHSTATES
+        SYMBOL SYBASE SYNCPOINTS SYNTAX SYNTAX-CHECK SYNTAX-TRACE SYST
+        SYSTEM-CALL SYSTEM-EXCEPTIONS SYSTEM-EXIT T000 TAB TABBED TABLE
+        TABLES TABLEVIEW TABSTRIP TARGET TASK TASKS TEST TEST-INJECTION
+        TEST-SEAM TESTING TEXT TEXTPOOL THEN THROW TIME TIMES TIMESTAMP
+        TIMEZONE TIMN TIMS TIMS_FROM_TIMN TIMS_IS_VALID TIMS_TO_TIMN TITLE
+        TITLE-LINES TITLEBAR TO TOKENIZATION TOKENS TOP-LINES TOP-OF-PAGE
+        TOTAL TRACE-ENTRY TRACE-FILE TRACE-TABLE TRAILING TRANSACTION
+        TRANSFER TRANSFORMATION TRANSLATE TRANSPORTING TRDIR TRMAC TRUNCATE
+        TRUNCATION TRY TSTMP_ADD_SECONDS TSTMP_CURRENT_UTCTIMESTAMP
+        TSTMP_IS_VALID TSTMP_SECONDS_BETWEEN TYPE TYPE-POOL TYPE-POOLS
+        TYPES ULINE UNASSIGN UNDER UNICODE UNION UNIQUE UNIT UNIT_CONVERSION
+        UNIX UNBLOCKED UNBOUNDED UNPACK UNTIL UNWIND UP UPDATE UPPER USER
+        USER-COMMAND USING UTF-8 VALID VALIDATE VALIDATION VALUE
+        VALUE-REQUEST VALUES VAR VARC VARY VARYING VERBATIM
+        VERIFICATION-MESSAGE VERSION VIA VIEW VISIBLE WAIT WARNING WEEKDAY
+        WHEN WHENEVER WHERE WHILE WIDTH WINDOW WINDOWS WITH WITH-HEADING
+        WITH-TITLE WITHOUT WORD WORK WORKSPACE WRITE WRITER XML XSD YELLOW
+        YES YYMMDD Z ZERO ZONE
+      )
+
+      BUILTINS = Set.new %w(
+        abs acos add_days add_months add_seconds asin atan attribute
+        bintohex bit-set boolc boolx cast ceil cfunc charlen char_off
+        class_constructor cmax cmin cnt coalesce concat concat_lines_of cond
+        cond-var condense constructor contains contains_any_not_of
+        contains_any_of conv corresponding cos cosh count count_any_not_of
+        count_any_of curr_to_decfloat_amount currency_conversion
+        current_utctimestamp datn_add_days datn_add_months datn_days_between
+        dats_from_datn dats_to_datn days_between dbmaxlen dbtab deserialize
+        distance div division escape exact exp filter find find_any_not_of
+        find_any_of find_end floor fltp_to_dec frac get_numeric_value
+        hextobin idx index insert ipow is_valid lines line_exists
+        line_index log log10 match matches mod new nmax nmin numofchar reduce
+        ref repeat replace replace_regexpr rescale reverse round
+        seconds_between segment serialize shift_left shift_right sign sin
+        sinh sqrt strlen substring substring_after substring_before
+        substring_from substring_to sum switch switch-var table_line tan tanh
+        tims_from_timn tims_to_timn to_lower to_mixed to_upper translate
+        trunc tstmp_to_dats tstmp_to_dst tstmp_to_tims tstmpl_from_utcl
+        tstmpl_to_utcl unit_conversion utcl_add_seconds utcl_current
+        utcl_seconds_between utclong_add utclong_current utclong_diff value
+        xsdbool xstrlen
+      )
+    end
+  end
+end

data/lib/rouge/lexers/abap.rb

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*- #
 # frozen_string_literal: true
 
-# ABAP elements taken from http://help.sap.com/abapdocu_750/en/index.htm?file=abapdo.htm
-
 module Rouge
   module Lexers
     class ABAP < RegexLexer
@@ -12,205 +10,93 @@ module Rouge
       filenames '*.abap'
       mimetypes 'text/x-abap'
 
-      def self.keywords
-        @keywords = Set.new %w(
-          *-INPUT ?TO ABAP-SOURCE ABBREVIATED ABS ABSTRACT ACCEPT ACCEPTING
-          ACCORDING ACCP ACTIVATION ACTUAL ADD ADD-CORRESPONDING ADJACENT
-          AFTER ALIAS ALIASES ALIGN ALL ALLOCATE ALPHA ANALYSIS ANALYZER AND
-          ANY APPEND APPENDAGE APPENDING APPLICATION ARCHIVE AREA ARITHMETIC
-          AS ASCENDING ASPECT ASSERT ASSIGN ASSIGNED ASSIGNING ASSOCIATION
-          ASYNCHRONOUS AT ATTRIBUTES AUTHORITY AUTHORITY-CHECK AVG BACK
-          BACKGROUND BACKUP BACKWARD BADI BASE BEFORE BEGIN BETWEEN BIG BINARY
-          BINTOHEX BIT BIT-AND BIT-NOT BIT-OR BIT-XOR BLACK BLANK BLANKS BLOB
-          BLOCK BLOCKS BLUE BOUND BOUNDARIES BOUNDS BOXED BREAK-POINT BT
-          BUFFER BY BYPASSING BYTE BYTE-CA BYTE-CN BYTE-CO BYTE-CS BYTE-NA
-          BYTE-NS BYTE-ORDER CA CALL CALLING CASE CAST CASTING CATCH CEIL
-          CENTER CENTERED CHAIN CHAIN-INPUT CHAIN-REQUEST CHANGE CHANGING
-          CHANNELS CHAR CHAR-TO-HEX CHARACTER CHECK CHECKBOX CIRCULAR CLASS
-          CLASS-CODING CLASS-DATA CLASS-EVENTS CLASS-METHODS CLASS-POOL
-          CLEANUP CLEAR CLIENT CLNT CLOB CLOCK CLOSE CN CO COALESCE CODE
-          CODING COLLECT COLOR COLUMN COLUMNS COL_BACKGROUND COL_GROUP
-          COL_HEADING COL_KEY COL_NEGATIVE COL_NORMAL COL_POSITIVE COL_TOTAL
-          COMMENT COMMENTS COMMIT COMMON COMMUNICATION COMPARING COMPONENT
-          COMPONENTS COMPRESSION COMPUTE CONCAT CONCATENATE CONCAT_WITH_SPACE
-          COND CONDENSE CONDITION CONNECT CONNECTION CONSTANTS CONTEXT
-          CONTEXTS CONTINUE CONTROL CONTROLS CONV CONVERSION CONVERT COPIES
-          COPY CORRESPONDING COUNT COUNTRY COVER CP CPI CREATE CREATING
-          CRITICAL CS CUKY CURR CURRENCY CURRENCY_CONVERSION CURRENT CURSOR
-          CURSOR-SELECTION CUSTOMER CUSTOMER-FUNCTION CX_DYNAMIC_CHECK
-          CX_NO_CHECK CX_ROOT CX_SQL_EXCEPTION CX_STATIC_CHECK DANGEROUS DATA
-          DATABASE DATAINFO DATASET DATE DATS DATS_ADD_DAYS DATS_ADD_MONTHS
-          DATS_DAYS_BETWEEN DATS_IS_VALID DAYLIGHT DD/MM/YY DD/MM/YYYY DDMMYY
-          DEALLOCATE DEC DECIMALS DECIMAL_SHIFT DECLARATIONS DEEP DEFAULT
-          DEFERRED DEFINE DEFINING DEFINITION DELETE DELETING DEMAND
-          DEPARTMENT DESCENDING DESCRIBE DESTINATION DETAIL DF16_DEC DF16_RAW
-          DF16_SCL DF34_DEC DF34_RAW DF34_SCL DIALOG DIRECTORY DISCONNECT
-          DISPLAY DISPLAY-MODE DISTANCE DISTINCT DIV DIVIDE
-          DIVIDE-CORRESPONDING DIVISION DO DUMMY DUPLICATE DUPLICATES DURATION
-          DURING DYNAMIC DYNPRO E EDIT EDITOR-CALL ELSE ELSEIF EMPTY ENABLED
-          ENABLING ENCODING END END-ENHANCEMENT-SECTION END-LINES
-          END-OF-DEFINITION END-OF-FILE END-OF-PAGE END-OF-SELECTION
-          END-TEST-INJECTION END-TEST-SEAM ENDAT ENDCASE ENDCATCH ENDCHAIN
-          ENDCLASS ENDDO ENDENHANCEMENT ENDEXEC ENDFORM ENDFUNCTION ENDIAN
-          ENDIF ENDING ENDINTERFACE ENDLOOP ENDMETHOD ENDMODULE ENDON
-          ENDPROVIDE ENDSELECT ENDTRY ENDWHILE ENDWITH ENGINEERING ENHANCEMENT
-          ENHANCEMENT-POINT ENHANCEMENT-SECTION ENHANCEMENTS ENTRIES ENTRY
-          ENVIRONMENT EQ EQUIV ERRORMESSAGE ERRORS ESCAPE ESCAPING EVENT
-          EVENTS EXACT EXCEPT EXCEPTION EXCEPTION-TABLE EXCEPTIONS EXCLUDE
-          EXCLUDING EXEC EXECUTE EXISTS EXIT EXIT-COMMAND EXPAND EXPANDING
-          EXPIRATION EXPLICIT EXPONENT EXPORT EXPORTING EXTEND EXTENDED
-          EXTENSION EXTRACT FAIL FETCH FIELD FIELD-GROUPS FIELD-SYMBOL
-          FIELD-SYMBOLS FIELDS FILE FILTER FILTER-TABLE FILTERS FINAL FIND
-          FIRST FIRST-LINE FIXED-POINT FKEQ FKGE FLOOR FLTP FLUSH FONT FOR
-          FORM FORMAT FORWARD FOUND FRAME FRAMES FREE FRIENDS FROM FUNCTION
-          FUNCTION-POOL FUNCTIONALITY FURTHER GAPS GE GENERATE GET
-          GET_PRINT_PARAMETERS GIVING GKEQ GKGE GLOBAL GRANT GREEN GROUP
-          GROUPS GT HANDLE HANDLER HARMLESS HASHED HAVING HDB HEAD-LINES
-          HEADER HEADERS HEADING HELP-ID HELP-REQUEST HEXTOBIN HIDE HIGH HINT
-          HOLD HOTSPOT I ICON ID IDENTIFICATION IDENTIFIER IDS IF
-          IF_ABAP_CLOSE_RESOURCE IF_ABAP_CODEPAGE IF_ABAP_DB_BLOB_HANDLE
-          IF_ABAP_DB_CLOB_HANDLE IF_ABAP_DB_LOB_HANDLE IF_ABAP_DB_READER
-          IF_ABAP_DB_WRITER IF_ABAP_READER IF_ABAP_WRITER IF_MESSAGE
-          IF_OS_CA_INSTANCE IF_OS_CA_PERSISTENCY IF_OS_FACTORY IF_OS_QUERY
-          IF_OS_QUERY_MANAGER IF_OS_QUERY_OPTIONS IF_OS_STATE
-          IF_OS_TRANSACTION IF_OS_TRANSACTION_MANAGER IF_SERIALIZABLE_OBJECT
-          IF_SHM_BUILD_INSTANCE IF_SYSTEM_UUID IF_T100_DYN_MSG IF_T100_MESSAGE
-          IGNORE IGNORING IMMEDIATELY IMPLEMENTATION IMPLEMENTATIONS
-          IMPLEMENTED IMPLICIT IMPORT IMPORTING IN INACTIVE INCL INCLUDE
-          INCLUDES INCLUDING INCREMENT INDEX INDEX-LINE INFOTYPES INHERITING
-          INIT INITIAL INITIALIZATION INNER INOUT INPUT INSERT INSTANCE
-          INSTANCES INSTR INT1 INT2 INT4 INT8 INTENSIFIED INTERFACE
-          INTERFACE-POOL INTERFACES INTERNAL INTERVALS INTO INVERSE
-          INVERTED-DATE IS ISO ITNO JOB JOIN KEEP KEEPING KERNEL KEY KEYS
-          KEYWORDS KIND LANG LANGUAGE LAST LATE LAYOUT LCHR LDB_PROCESS LE
-          LEADING LEAVE LEFT LEFT-JUSTIFIED LEFTPLUS LEFTSPACE LEGACY LENGTH
-          LET LEVEL LEVELS LIKE LINE LINE-COUNT LINE-SELECTION LINE-SIZE
-          LINEFEED LINES LIST LIST-PROCESSING LISTBOX LITTLE LLANG LOAD
-          LOAD-OF-PROGRAM LOB LOCAL LOCALE LOCATOR LOG-POINT LOGFILE LOGICAL
-          LONG LOOP LOW LOWER LPAD LPI LRAW LT LTRIM M MAIL MAIN MAJOR-ID
-          MAPPING MARGIN MARK MASK MATCH MATCHCODE MAX MAXIMUM MEDIUM MEMBERS
-          MEMORY MESH MESSAGE MESSAGE-ID MESSAGES MESSAGING METHOD METHODS MIN
-          MINIMUM MINOR-ID MM/DD/YY MM/DD/YYYY MMDDYY MOD MODE MODIF MODIFIER
-          MODIFY MODULE MOVE MOVE-CORRESPONDING MULTIPLY
-          MULTIPLY-CORRESPONDING NA NAME NAMETAB NATIVE NB NE NESTED NESTING
-          NEW NEW-LINE NEW-PAGE NEW-SECTION NEXT NO NO-DISPLAY NO-EXTENSION
-          NO-GAP NO-GAPS NO-GROUPING NO-HEADING NO-SCROLLING NO-SIGN NO-TITLE
-          NO-TOPOFPAGE NO-ZERO NODE NODES NON-UNICODE NON-UNIQUE NOT NP NS
-          NULL NUMBER NUMC O OBJECT OBJECTS OBLIGATORY OCCURRENCE OCCURRENCES
-          OCCURS OF OFF OFFSET ON ONLY OPEN OPTION OPTIONAL OPTIONS OR ORDER
-          OTHER OTHERS OUT OUTER OUTPUT OUTPUT-LENGTH OVERFLOW OVERLAY PACK
-          PACKAGE PAD PADDING PAGE PAGES PARAMETER PARAMETER-TABLE PARAMETERS
-          PART PARTIALLY PATTERN PERCENTAGE PERFORM PERFORMING PERSON PF
-          PF-STATUS PINK PLACES POOL POSITION POS_HIGH POS_LOW PRAGMAS PREC
-          PRECOMPILED PREFERRED PRESERVING PRIMARY PRINT PRINT-CONTROL
-          PRIORITY PRIVATE PROCEDURE PROCESS PROGRAM PROPERTY PROTECTED
-          PROVIDE PUBLIC PUSH PUSHBUTTON PUT QUAN QUEUE-ONLY QUICKINFO
-          RADIOBUTTON RAISE RAISING RANGE RANGES RAW RAWSTRING READ READ-ONLY
-          READER RECEIVE RECEIVED RECEIVER RECEIVING RED REDEFINITION REDUCE
-          REDUCED REF REFERENCE REFRESH REGEX REJECT REMOTE RENAMING REPLACE
-          REPLACEMENT REPLACING REPORT REQUEST REQUESTED RESERVE RESET
-          RESOLUTION RESPECTING RESPONSIBLE RESULT RESULTS RESUMABLE RESUME
-          RETRY RETURN RETURNCODE RETURNING RETURNS RIGHT RIGHT-JUSTIFIED
-          RIGHTPLUS RIGHTSPACE RISK RMC_COMMUNICATION_FAILURE
-          RMC_INVALID_STATUS RMC_SYSTEM_FAILURE ROLE ROLLBACK ROUND ROWS RPAD
-          RTRIM RUN SAP SAP-SPOOL SAVING SCALE_PRESERVING
-          SCALE_PRESERVING_SCIENTIFIC SCAN SCIENTIFIC
-          SCIENTIFIC_WITH_LEADING_ZERO SCREEN SCROLL SCROLL-BOUNDARY SCROLLING
-          SEARCH SECONDARY SECONDS SECTION SELECT SELECT-OPTIONS SELECTION
-          SELECTION-SCREEN SELECTION-SET SELECTION-SETS SELECTION-TABLE
-          SELECTIONS SEND SEPARATE SEPARATED SET SHARED SHIFT SHORT
-          SHORTDUMP-ID SIGN SIGN_AS_POSTFIX SIMPLE SINGLE SIZE SKIP SKIPPING
-          SMART SOME SORT SORTABLE SORTED SOURCE SPACE SPECIFIED SPLIT SPOOL
-          SPOTS SQL SQLSCRIPT SSTRING STABLE STAMP STANDARD START-OF-SELECTION
-          STARTING STATE STATEMENT STATEMENTS STATIC STATICS STATUSINFO
-          STEP-LOOP STOP STRING STRUCTURE STRUCTURES STYLE SUBKEY SUBMATCHES
-          SUBMIT SUBROUTINE SUBSCREEN SUBSTRING SUBTRACT
-          SUBTRACT-CORRESPONDING SUFFIX SUM SUMMARY SUMMING SUPPLIED SUPPLY
-          SUPPRESS SWITCH SWITCHSTATES SYMBOL SYNCPOINTS SYNTAX SYNTAX-CHECK
-          SYNTAX-TRACE SYST SYSTEM-CALL SYSTEM-EXCEPTIONS SYSTEM-EXIT TAB
-          TABBED TABLE TABLES TABLEVIEW TABSTRIP TARGET TASK TASKS TEST
-          TEST-INJECTION TEST-SEAM TESTING TEXT TEXTPOOL THEN THROW TIME TIMES
-          TIMESTAMP TIMEZONE TIMS TIMS_IS_VALID TITLE TITLE-LINES TITLEBAR TO
-          TOKENIZATION TOKENS TOP-LINES TOP-OF-PAGE TRACE-FILE TRACE-TABLE
-          TRAILING TRANSACTION TRANSFER TRANSFORMATION TRANSLATE TRANSPORTING
-          TRMAC TRUNCATE TRUNCATION TRY TSTMP_ADD_SECONDS
-          TSTMP_CURRENT_UTCTIMESTAMP TSTMP_IS_VALID TSTMP_SECONDS_BETWEEN TYPE
-          TYPE-POOL TYPE-POOLS TYPES ULINE UNASSIGN UNDER UNICODE UNION UNIQUE
-          UNIT UNIT_CONVERSION UNIX UNPACK UNTIL UNWIND UP UPDATE UPPER USER
-          USER-COMMAND USING UTF-8 VALID VALUE VALUE-REQUEST VALUES VARC VARY
-          VARYING VERIFICATION-MESSAGE VERSION VIA VIEW VISIBLE WAIT WARNING
-          WHEN WHENEVER WHERE WHILE WIDTH WINDOW WINDOWS WITH WITH-HEADING
-          WITH-TITLE WITHOUT WORD WORK WRITE WRITER XML XSD YELLOW YES YYMMDD
-          Z ZERO ZONE
-        )
-      end
-
-      def self.builtins
-        @keywords = Set.new %w(
-          acos apply asin assign atan attribute bit-set boolc boolx call
-          call-method cast ceil cfunc charlen char_off class_constructor clear
-          cluster cmax cmin cnt communication_failure concat_lines_of cond
-          cond-var condense constructor contains contains_any_not_of
-          contains_any_of copy cos cosh count count_any_not_of count_any_of
-          create cursor data dbmaxlen dbtab deserialize destructor distance
-          empty error_message escape exp extensible find find_any_not_of
-          find_any_of find_end floor frac from_mixed group hashed header idx
-          include index insert ipow itab key lax lines line_exists line_index
-          log log10 loop loop_key match matches me mesh_path namespace nmax
-          nmin node numeric numofchar object parameter primary_key read ref
-          repeat replace rescale resource_failure reverse root round segment
-          sender serialize shift_left shift_right sign simple sin sinh skip
-          sorted space sqrt standard strlen substring substring_after
-          substring_before substring_from substring_to sum switch switch-var
-          system_failure table table_line tan tanh template text to_lower
-          to_mixed to_upper transform translate trunc type value variable write
-          xsdbool xsequence xstrlen
-        )
-      end
+      lazy { require_relative 'abap/builtins' }
 
-      def self.types
-        @types = Set.new %w(
-         b c d decfloat16 decfloat34 f i int8 n p s t x
-         clike csequence decfloat string xstring
-        )
-      end
+      TYPES = Set.new %w(
+        b c d decfloat16 decfloat34 f i int8 n p s t x
+        clike csequence decfloat string xstring
+      )
 
-      def self.new_keywords
-        @types = Set.new %w(
-          DATA FIELD-SYMBOL
-        )
-      end
+      NEW_KEYWORDS = Set.new %w(DATA FIELD-SYMBOL)
 
       state :root do
         rule %r/\s+/m, Text
 
-        rule %r/".*/, Comment::Single
-        rule %r(^\*.*), Comment::Multiline
+        rule %r/(".*?$)/, Comment::Single
+        rule %r/(^\*.*?$)/, Comment::Single
         rule %r/\d+/, Num::Integer
+
+        # String templates |...|
+        rule %r/\|/, Str::Interpol, :string_template
+
         rule %r/('|`)/, Str::Single, :single_string
-        rule %r/[\[\]\(\)\{\}\.,:\|]/, Punctuation
+
+        # CDS annotations: @KEYWORD.field
+        rule %r/(@)([A-Z][A-Za-z0-9_]*)(\.)([A-Za-z][A-Za-z0-9_]*)/ do |m|
+          token Operator, m[1]
+          if KEYWORDS.include? m[2].upcase
+            token Keyword, m[2]
+          else
+            token Name, m[2]
+          end
+          token Punctuation, m[3]
+          token Name, m[4]
+        end
+
+        # structure field access with dot (table.field should not highlight field as keyword)
+        # This must come before the general punctuation rule
+        # In dot-access patterns, treat both parts as names (even if they match keywords)
+        rule %r/([A-Za-z][A-Za-z0-9_]*)(\.)([A-Za-z][A-Za-z0-9_]*)/ do |m|
+          token Name, m[1]
+          token Punctuation, m[2]
+          token Name, m[3]
+        end
+        # Parameter names in function/method interfaces (word before =)
+        # Matches word followed by optional whitespace and =
+        # This handles EXPORTING/IMPORTING/CHANGING/TABLES/USING parameter names
+        rule %r/([A-Za-z][A-Za-z0-9_]*)(?=\s*=)/ do |m|
+          token Name, m[1]
+        end
+        rule %r/[\[\]\(\)\{\}\.,:]/, Punctuation
 
         # builtins / new ABAP 7.40 keywords (@DATA(), ...)
         rule %r/(->|=>)?([A-Za-z][A-Za-z0-9_\-]*)(\()/ do |m|
-          if m[1] != ''
-            token Operator, m[1]
-          end
+          token Operator, m[1]
+
+          word = m[2]
 
-          if (self.class.new_keywords.include? m[2].upcase) && m[1].nil?
+          if (NEW_KEYWORDS.include? word.upcase) && m[1].nil?
             token Keyword, m[2]
-          elsif (self.class.builtins.include? m[2].downcase) && m[1].nil?
+          elsif (BUILTINS.include? word.downcase) && m[1].nil?
             token Name::Builtin, m[2]
           else
             token Name, m[2]
           end
+
           token Punctuation, m[3]
         end
 
+        # hyphenated keywords (like NON-UNIQUE)
+        rule %r/[A-Za-z][A-Za-z0-9_]*(-[A-Za-z][A-Za-z0-9_]*)+/ do |m|
+          if KEYWORDS.include? m[0].upcase
+            token Keyword
+          else
+            token Name
+          end
+        end
+
+        # structure component access (variable-component should not be highlighted as keyword)
+        # this rule matches: word-word where the second part is lowercase or starts lowercase
+        rule %r/[A-Za-z][A-Za-z0-9_]*-[a-z][A-Za-z0-9_]*/, Name
+
+        # variable names starting with $ (like $session)
+        rule %r/\$[A-Za-z][A-Za-z0-9_]*/, Name
+
         # keywords, types and normal text
         rule %r/\w\w*/ do |m|
-          if self.class.keywords.include? m[0].upcase
+          if KEYWORDS.include? m[0].upcase
             token Keyword
-          elsif self.class.types.include? m[0].downcase
+          elsif TYPES.include? m[0].downcase
             token Keyword::Type
           else
             token Name
@@ -235,6 +121,12 @@ module Rouge
         rule %r/[^\\'`]+/, Str::Single
       end
 
+      state :string_template do
+        rule %r/\{[^}]*\}/, Str::Interpol  # embedded expressions
+        rule %r/\|/, Str::Interpol, :pop!
+        rule %r/[^|{]+/, Str::Interpol
+      end
+
     end
   end
 end