rouge 4.6.1 → 5.0.0

data/lib/rouge/lexers/moonscript.rb

@@ -1,10 +1,10 @@
 # -*- coding: utf-8 -*- #
 # frozen_string_literal: true
 
+require_relative 'lua'
+
 module Rouge
   module Lexers
-    load_lexer 'lua.rb'
-
     class Moonscript < RegexLexer
       title "MoonScript"
       desc "Moonscript (http://www.moonscript.org)"
@@ -16,6 +16,9 @@ module Rouge
       option :function_highlighting, 'Whether to highlight builtin functions (default: true)'
       option :disabled_modules, 'builtin modules to disable'
 
+      # depends on lua builtins
+      lazy { Lua.eager_load! }
+
       def initialize(*)
         super
 
@@ -31,7 +34,7 @@ module Rouge
         return [] unless @function_highlighting
 
         @builtins ||= Set.new.tap do |builtins|
-          Rouge::Lexers::Lua.builtins.each do |mod, fns|
+          Lua::BUILTINS.each do |mod, fns|
             next if @disabled_modules.include? mod
             builtins.merge(fns)
           end
@@ -44,36 +47,35 @@ module Rouge
       end
 
       state :base do
-        ident = '(?:\w\w*)'
-
-        rule %r((?i)(\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?'), Num::Float
-        rule %r((?i)\d+e[+-]?\d+), Num::Float
-        rule %r((?i)0x[0-9a-f]*), Num::Hex
+        rule %r((\d*\.\d+|\d+\.\d*)(e[+-]?\d+)?')i, Num::Float
+        rule %r(\d+e[+-]?\d+)i, Num::Float
+        rule %r(0x\h*), Num::Hex
         rule %r(\d+), Num::Integer
-        rule %r(@#{ident}*), Name::Variable::Instance
+        rule %r(@\w+), Name::Variable::Instance
         rule %r([A-Z]\w*), Name::Class
         rule %r("?[^"]+":), Literal::String::Symbol
-        rule %r(#{ident}:), Literal::String::Symbol
-        rule %r(:#{ident}), Literal::String::Symbol
+        rule %r(\w+:), Literal::String::Symbol
+        rule %r(:\w+), Literal::String::Symbol
 
         rule %r(\s+), Text::Whitespace
         rule %r((==|~=|!=|<=|>=|\.\.\.|\.\.|->|=>|[=+\-*/%^<>#!\\])), Operator
         rule %r([\[\]\{\}\(\)\.,:;]), Punctuation
         rule %r((and|or|not)\b), Operator::Word
 
-        keywords = %w{
-          break class continue do else elseif end extends for if import in
+        keywords = Set.new %w{
+          break class continue do else elseif end extends for if import in from
           repeat return switch super then unless until using when with while
         }
-        rule %r((#{keywords.join('|')})\b), Keyword
         rule %r((local|export)\b), Keyword::Declaration
         rule %r((true|false|nil)\b), Keyword::Constant
 
-        rule %r([A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?) do |m|
+        rule %r([a-z_]\w*(\.[a-z_]\w*)?)i do |m|
           name = m[0]
-          if self.builtins.include?(name)
+          if keywords.include?(name)
+            token Keyword
+          elsif self.builtins.include?(name)
             token Name::Builtin
-          elsif name =~ /\./
+          elsif name.include?('.')
             a, b = name.split('.', 2)
             token Name, a
             token Punctuation, '.'

data/lib/rouge/lexers/nesasm.rb

@@ -21,7 +21,7 @@ module Rouge
 
       def self.keywords_type
         @keywords_type ||= %w(
-          DB DW BYTE WORD 
+          DB DW BYTE WORD
         )
       end
 
@@ -32,12 +32,12 @@ module Rouge
           ENDIF FAIL INESPRG INESCHR INESMAP INESMIR FUNC
         )
       end
-      
+
       state :root do
         rule %r/\s+/m, Text
         rule %r(;.*), Comment::Single
 
-        rule %r/[\(\)\,\.\[\]]/, Punctuation 
+        rule %r/[\(\)\,\.\[\]]/, Punctuation
         rule %r/\#?\%[0-1]+/, Num::Bin # #%00110011 %00110011
         rule %r/\#?\$\h+/, Num::Hex  # $1f #$1f
         rule %r/\#?\d+/, Num # 10 #10
@@ -45,7 +45,7 @@ module Rouge
 
         rule %r/\#?\w+:?/i do |m|
           name = m[0].upcase
-          
+
           if self.class.keywords.include? name
             token Keyword
           elsif self.class.keywords_type.include? name
@@ -58,13 +58,13 @@ module Rouge
         end
 
         rule %r/\#?(?:LOW|HIGH)\(.*\)/i, Keyword::Reserved # LOW() #HIGH()
-        
+
         rule %r/\#\(/, Punctuation # #()
 
         rule %r/"/, Str, :string
 
         rule %r/'\w'/, Str::Char # 'A' for example
-        
+
         rule %r/\\\??[\d@#]/, Name::Builtin # builtin parameters for use inside macros and functions:   \1-\9 , \?1-\?9 , \# , \@
       end
 

data/lib/rouge/lexers/nial.rb

@@ -8,7 +8,7 @@ module Rouge
       desc 'The Nial programming language (nial-array-language.org)'
       tag 'nial'
       filenames '*.ndf', '*.nlg'
-      
+
       def self.keywords
         @keywords ||= Set.new ["is", "gets", "op", "tr", ";",
                      "if", "then", "elseif", "else",
@@ -40,7 +40,7 @@ module Rouge
         "reduce", "reducecols", "reducerows",
         "sort", "team", "timeit", "twig"]
       end
-      
+
       def self.funcs
         @funcs ||= Set.new ["operation", "expression", "and", "abs",
                   "allbools", "allints", "allchars", "allin",
@@ -56,9 +56,9 @@ module Rouge
                   "cutall", "cut", "display", "deparse",
                   "deepupdate", "descan", "depth", "diagram",
                   "div", "divide", "drop", "dropright", "edit",
-                  "empty", "expression", "exit", "except", "erase",
+                  "empty", "exit", "except", "erase",
                   "equal", "eval", "eraserecord", "execute", "exp",
-                  "external", "exprs", "findall", "find", 
+                  "external", "exprs", "findall", "find",
                   "fault", "falsehood", "filestatus", "filelength",
                   "filepath", "filetally", "floor", "first",
                   "flip", "fuse", "fromraw", "front",
@@ -72,8 +72,7 @@ module Rouge
                   "loaddefs", "nonlocal", "max", "match", "log",
                   "lt", "lower", "lte", "mate", "min", "maxlength",
                   "mod", "mix", "minus", "nialroot", "mold", "not",
-                  "numeric", "no_op", "no_expr", "notin",
-                  "operation", "open", "or", "opposite", "opp",
+                  "numeric", "no_op", "no_expr", "notin", "open", "or", "opposite", "opp",
                   "operators", "plus", "pick", "pack", "pass", "pair", "parse",
                   "paste", "phrase", "place", "picture", "placeall",
                   "power", "positions", "post", "quotient", "putfile",
@@ -137,7 +136,7 @@ module Rouge
             token Text
           end
         end
-        
+
         rule %r/\b[_A-Za-z]\w*\b/ do |m|
           lower = m[0].downcase
           if self.class.keywords.include?(lower)
@@ -155,7 +154,7 @@ module Rouge
 
         rule %r/\s+/, Text
       end
-      
+
       state :str do
         rule %r/''/, Str::Escape
         rule %r/[^']+/, Str::Single

data/lib/rouge/lexers/nim.rb

@@ -134,9 +134,9 @@ module Rouge
         # Note: Have to do this with a block to push multiple states first,
         #       since we can't pass array of states like w/ Pygments.
         rule(/[0-9][0-9_]*(?=([eE.]|'?[fF](32|64)))/) do
-         push :floatsuffix
-         push :floatnumber
-         token Num::Float
+          push :floatsuffix
+          push :floatnumber
+          token Num::Float
         end
 
         rule(/0[xX][a-fA-F0-9][a-fA-F0-9_]*/, Num::Hex,     :intsuffix)

data/lib/rouge/lexers/nix.rb

@@ -10,6 +10,8 @@ module Rouge
       aliases 'nixos'
       filenames '*.nix'
 
+      id_boundary = /(?![a-zA-Z0-9_'-])/
+
       state :whitespaces do
         rule %r/^\s*\n\s*$/m, Text
         rule %r/\s+/, Text
@@ -30,15 +32,15 @@ module Rouge
       end
 
       state :null do
-        rule %r/(null)/, Keyword::Constant
+        rule %r/null#{id_boundary}/, Keyword::Constant
       end
 
       state :boolean do
-        rule %r/(true|false)/, Keyword::Constant
+        rule %r/(?:true|false)#{id_boundary}/, Keyword::Constant
       end
 
       state :binding do
-        rule %r/[a-zA-Z_][a-zA-Z0-9-]*/, Name::Variable
+        rule %r/[a-zA-Z_][a-zA-Z0-9_'-]*/, Name::Variable
       end
 
       state :path do
@@ -161,22 +163,22 @@ module Rouge
 
       state :keywords_namespace do
         keywords = %w(with in inherit)
-        rule %r/(?:#{keywords.join('|')})\b/, Keyword::Namespace
+        rule %r/(?:#{keywords.join('|')})#{id_boundary}/, Keyword::Namespace
       end
 
       state :keywords_declaration do
         keywords = %w(let)
-        rule %r/(?:#{keywords.join('|')})\b/, Keyword::Declaration
+        rule %r/(?:#{keywords.join('|')})#{id_boundary}/, Keyword::Declaration
       end
 
       state :keywords_conditional do
         keywords = %w(if then else)
-        rule %r/(?:#{keywords.join('|')})\b/, Keyword
+        rule %r/(?:#{keywords.join('|')})#{id_boundary}/, Keyword
       end
 
       state :keywords_reserved do
         keywords = %w(rec assert map)
-        rule %r/(?:#{keywords.join('|')})\b/, Keyword::Reserved
+        rule %r/(?:#{keywords.join('|')})#{id_boundary}/, Keyword::Reserved
       end
 
       state :keywords_builtin do
@@ -192,7 +194,7 @@ module Rouge
           throw
           toString
         )
-        rule %r/(?:#{keywords.join('|')})\b/, Keyword::Reserved
+        rule %r/(?:#{keywords.join('|')})#{id_boundary}/, Keyword::Reserved
       end
 
       state :ignore do

data/lib/rouge/lexers/objective_c.rb

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*- #
 # frozen_string_literal: true
 
+require_relative 'c'
+require_relative 'objective_c/common'
+
 module Rouge
   module Lexers
-    load_lexer 'c.rb'
-    load_lexer 'objective_c/common.rb'
-
     class ObjectiveC < C
       extend ObjectiveCCommon
 

data/lib/rouge/lexers/objective_cpp.rb

@@ -1,10 +1,11 @@
 # -*- coding: utf-8 -*- #
 # frozen_string_literal: true
 
+require_relative 'cpp'
+require_relative 'objective_c/common'
+
 module Rouge
   module Lexers
-    load_lexer 'cpp.rb'
-    load_lexer 'objective_c/common.rb'
 
     class ObjectiveCpp < Cpp
       extend ObjectiveCCommon

data/lib/rouge/lexers/ocaml.rb

@@ -1,10 +1,10 @@
 # -*- coding: utf-8 -*- #
 # frozen_string_literal: true
 
+require_relative 'ocaml/common'
+
 module Rouge
   module Lexers
-    load_lexer 'ocaml/common.rb'
-
     class OCaml < OCamlCommon
       title "OCaml"
       desc 'Objective Caml (ocaml.org)'

data/lib/rouge/lexers/openedge.rb

@@ -21,7 +21,7 @@ module Rouge
           ADD-BUFFER ADD-CALC-COL ADD-CALC-COLU ADD-CALC-COLUM ADD-CALC-COLUMN ADD-COLUMNS-FROM
           ADD-EVENTS-PROC ADD-EVENTS-PROCE ADD-EVENTS-PROCED ADD-EVENTS-PROCEDU ADD-EVENTS-PROCEDUR
           ADD-EVENTS-PROCEDURE ADD-FIELDS-FROM ADD-FIRST ADD-HEADER-ENTRY ADD-INDEX-FIELD ADD-INTERVAL
-          ADD-LAST ADD-LIKE-COLUMN ADD-LIKE-FIELD ADD-LIKE-INDEX ADD-NEW-FIELD ADD-NEW-INDEX       
+          ADD-LAST ADD-LIKE-COLUMN ADD-LIKE-FIELD ADD-LIKE-INDEX ADD-NEW-FIELD ADD-NEW-INDEX
           ADD-SCHEMA-LOCATION ADD-SOURCE-BUFFER ADD-SUPER-PROC ADD-SUPER-PROCE ADD-SUPER-PROCED
           ADD-SUPER-PROCEDU ADD-SUPER-PROCEDUR ADD-SUPER-PROCEDURE ADM-DATA ADVISE AFTER-BUFFER AFTER-ROWID
           AFTER-TABLE ALERT-BOX ALIAS ALL ALLOW-COLUMN-SEARCHING ALLOW-PREV-DESERIALIZATION ALLOW-REPLICATION
@@ -486,7 +486,7 @@ module Rouge
           WRITE-JSON WRITE-MESSAGE WRITE-PROCESSING-INSTRUCTION WRITE-STATUS WRITE-XML WRITE-XMLSCHEMA X XCODE
           XCODE-SESSION-KEY X-DOCUMENT XML-DATA-TYPE XML-ENTITY-EXPANSION-LIMIT XML-NODE-NAME XML-NODE-TYPE
           XML-SCHEMA-PAT XML-SCHEMA-PATH XML-STRICT-ENTITY-RESOLUTION XML-SUPPRESS-NAMESPACE-PROCESSING
-          X-NODEREF X-OF XOR XREF XREF-XML Y YEAR YEAR-OFFSET YES YES-NO YES-NO-CANCEL Y-OF          
+          X-NODEREF X-OF XOR XREF XREF-XML Y YEAR YEAR-OFFSET YES YES-NO YES-NO-CANCEL Y-OF
         )
       end
 
@@ -521,7 +521,7 @@ module Rouge
       end
 
       state :whitespace do
-        rule %r/\s+/m, Text 
+        rule %r/\s+/m, Text
       end
 
       state :singlelinecomment do
@@ -560,7 +560,7 @@ module Rouge
         rule %r([~!%^*+=\|?:<>/-]), Operator
         rule %r/[()\[\],.]/, Punctuation
       end
-      
+
       state :doublequotedstring do
         rule %r/\~[~nrt"]?/, Str::Escape
         rule %r/[^\\"]+/, Str::Double
@@ -572,7 +572,7 @@ module Rouge
         rule %r/[^\\']+/, Str::Single
         rule %r/'/, Str::Delimiter, :pop!
       end
-      
+
       state :preproc do
         rule %r/(\&analyze-suspend|\&analyze-resume)/i, Comment::Preproc, :analyze_suspend_resume_content
         rule %r/(\&scoped-define|\&global-define)\s*([\.\w\\\/-]*)/i , Comment::Preproc, :analyze_suspend_resume_content
@@ -584,18 +584,18 @@ module Rouge
         rule %r/.*\n/, Comment::Preproc, :pop!
         rule %r/.*/, Comment::Preproc, :pop!
       end
-      
+
       state :preproc_content do
         rule %r/\n/, Text, :pop!
         rule %r/\s+/, Text
- 
+
         rule %r/({?&)(\S+)/ do
           groups Comment::Preproc, Name::Other
         end
- 
+
         rule %r/"/, Str, :string
         mixin :numeric
- 
+
         rule %r/\S+/, Name
       end
 

data/lib/rouge/lexers/pdf.rb

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+# vim: set ts=2 sw=2 et:
+
+# PDF = Portable Document Format page description language
+# As defined by ISO 32000-2:2020 including resolved errata from https://pdf-issues.pdfa.org/
+#
+# The PDF syntax is also known as "COS" and can be used with FDF (Forms Data Field) files as
+# per ISO 32000-2:2020 clause 12.7.8.
+#
+# This is a token-based parser ONLY! It is intended to syntax highlight full or partial fragments
+# of nicely written hand-written PDF syntax in documentation such as ISO specifications. It is NOT
+# intended to cope with real-world PDFs that will contain arbitrary binary data (that form invalid
+# UTF-8 sequences and generate "ArgumentError: invalid byte sequence in UTF-8" Ruby errors) and
+# other types of malformations or syntax errors.
+#
+# Author: Peter Wyatt, CTO, PDF Association. 2024
+#
+module Rouge
+  module Lexers
+    class Pdf < RegexLexer
+      title "PDF"
+      desc "PDF - Portable Document Format (ISO 32000)"
+      tag 'pdf'
+      aliases "fdf", 'cos'
+      filenames '*.pdf', '*.fdf'
+      mimetypes 'application/pdf', 'application/fdf' # IANA registered media types
+
+      # PDF and FDF files must start with "%PDF-x.y" or "%FDF-x.y"
+      # where x is the single digit major version and y is the single digit minor version.
+      # For simplicity as a syntax highlighter, assumes occurs at start of a line.
+      def self.detect?(text)
+        return true if /\A%(P|F)DF-\d\.\d/.match?(text)
+      end
+
+      # PDF Delimiters (ISO 32000-2:2020, Table 1 and Table 2).
+      # Ruby whitespace "\s" is /[ \t\r\n\f\v]/ which does not include NUL (ISO 32000-2:2020, Table 1).
+      # PDF also support 2 character EOL sequences.
+
+      state :root do
+        # Start-of-file header comment is special (comment is up to EOL)
+        rule %r/^%(P|F)DF-\d\.\d.*$/, Comment::Preproc
+
+        # End-of-file marker comment is special (comment is up to EOL)
+        rule %r/^%%EOF.*$/, Comment::Preproc
+
+        # PDF only has single-line comments: from "%" to EOL
+        rule %r/%.*$/, Comment::Single
+
+        # PDF Boolean and null object keywords
+        rule %r/(false|true|null)/, Keyword::Constant
+
+        # PDF Dictionary and array object start and end tokens
+        rule %r/(<<|>>|\[|\])/, Punctuation
+
+        # PDF Hex string - can contain whitespace and span multiple lines.
+        # This rule must be after "<<"/">>"
+        rule %r/<[0-9A-Fa-f\s]*>/m, Str::Other
+
+        # PDF literal strings are complex (multi-line, escapes, etc.). Use separate state machine.
+        rule %r/\(/, Str, :stringliteral
+
+        # PDF Name objects - can be empty (i.e., nothing after "/").
+        # No special processing required for 2-digit hex codes that start with "#".
+        rule %r/\/[^\(\)<>\[\]\/%\s]*/, Name::Other
+
+        # PDF objects and stream (no checking of object ID)
+        # Note that object number and generation numbers do not have sign.
+        rule %r/\d+\s\d+\sobj/, Keyword::Declaration
+        rule %r/endstream|endobj|stream/, Keyword::Declaration
+
+        # PDF conventional file layout keywords
+        rule %r/startxref|trailer|xref/, Keyword::Declaration
+
+        # PDF cross reference section entries (20 bytes including EOL).
+        # Explicit single SPACE separators.
+        rule %r/^\d{10} \d{5} (n|f)\s*$/, Keyword::Namespace
+
+        # PDF Indirect reference (lax, allows zero as the object number).
+        # Requires terminating delimiter lookahead to disambiguate from "RG" operator
+        rule %r/\d+\s\d+\sR(?=[\(\)<>\[\]\/%\s])/, Name::Decorator
+
+        # PDF Real object
+        rule %r/(\-|\+)?([0-9]+\.?|[0-9]*\.[0-9]+|[0-9]+\.[0-9]*)/, Num::Float
+
+        # PDF Integer object
+        rule %r/(\-|\+)?[0-9]+/, Num::Integer
+
+        # A run of non-delimiters is most likely a PDF content stream
+        # operator (ISO 32000-2:2020, Annex A).
+        rule %r/[^\(\)<>\[\]\/%\s]+/, Operator::Word
+
+        # Whitespace (except inside strings and comments) is ignored = /[ \t\r\n\f\v]/.
+        # Ruby doesn't include NUL as whitespace (vs ISO 32000-2:2020 Table 1)
+        rule %r/\s+/, Text::Whitespace
+      end
+
+      # PDF literal string. See ISO 32000-2:2020 clause 7.3.4.2 and Table 3
+      state :stringliteral do
+        rule %r/\(/, Str, :stringliteral             # recursive for internal bracketed strings
+        rule %r/\\\(/, Str::Escape, :stringliteral   # recursive for internal escaped bracketed strings
+        rule %r/\)/, Str, :pop!
+        rule %r/\\\)/, Str::Escape, :pop!
+        rule %r/\\([0-7]{3}|n|r|t|b|f|\\)/, Str::Escape
+        rule %r/[^\(\)\\]+/, Str
+      end
+    end
+  end
+end