rouge 3.19.0 → 3.24.0

data/lib/rouge/lexers/velocity.rb

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*- #
+
+module Rouge
+  module Lexers
+    class Velocity < TemplateLexer
+      title 'Velocity'
+      desc 'Velocity is a Java-based template engine (velocity.apache.org)'
+      tag 'velocity'
+      filenames '*.vm', '*.velocity', '*.fhtml'
+      mimetypes 'text/html+velocity'
+
+      id = /[a-z_]\w*/i
+
+      state :root do
+        rule %r/[^{#$]+/ do
+          delegate parent
+        end
+
+        rule %r/(#)(\*.*?\*)(#)/m, Comment::Multiline
+        rule %r/(##)(.*?$)/, Comment::Single
+
+        rule %r/(#\{?)(#{id})(\}?)(\s?\()/m do
+          groups Punctuation, Name::Function, Punctuation, Punctuation
+          push :directive_params
+        end
+
+        rule %r/(#\{?)(#{id})(\}|\b)/m do
+          groups Punctuation, Name::Function, Punctuation
+        end
+
+        rule %r/\$\{?/, Punctuation, :variable
+      end
+
+      state :variable do
+        rule %r/#{id}/, Name::Variable
+        rule %r/\(/, Punctuation, :func_params
+        rule %r/(\.)(#{id})/ do
+          groups Punctuation, Name::Variable
+        end
+        rule %r/\}/, Punctuation, :pop!
+        rule(//) { pop! }
+      end
+
+      state :directive_params do
+        rule %r/(&&|\|\||==?|!=?|[-<>+*%&|^\/])|\b(eq|ne|gt|lt|ge|le|not|in)\b/, Operator
+        rule %r/\[/, Operator, :range_operator
+        rule %r/\b#{id}\b/, Name::Function
+        mixin :func_params
+      end
+
+      state :range_operator do
+        rule %r/[.]{2}/, Operator
+        mixin :func_params
+        rule %r/\]/, Operator, :pop!
+      end
+
+      state :func_params do
+        rule %r/\$\{?/, Punctuation, :variable
+        rule %r/\s+/, Text
+        rule %r/,/, Punctuation
+        rule %r/"(\\\\|\\"|[^"])*"/, Str::Double
+        rule %r/'(\\\\|\\'|[^'])*'/, Str::Single
+        rule %r/0[xX][0-9a-fA-F]+[Ll]?/, Num::Hex
+        rule %r/\b[0-9]+\b/, Num::Integer
+        rule %r/(true|false|null)\b/, Keyword::Constant
+        rule %r/[(\[]/, Punctuation, :push!
+        rule %r/[)\]}]/, Punctuation, :pop!
+      end
+    end
+  end
+end

data/lib/rouge/lexers/wollok.rb

@@ -18,7 +18,6 @@ module Rouge
 
       state :whitespaces_and_comments do
         rule %r/\s+/m, Text::Whitespace
-        rule %r/$+/m, Text::Whitespace
         rule %r(//.*$), Comment::Single
         rule %r(/\*(.|\s)*?\*/)m, Comment::Multiline
       end

data/lib/rouge/lexers/xml.rb

@@ -12,6 +12,8 @@ module Rouge
       mimetypes 'text/xml', 'application/xml', 'image/svg+xml',
                 'application/rss+xml', 'application/atom+xml'
 
+      # Documentation: https://www.w3.org/TR/xml11/#charsets and https://www.w3.org/TR/xml11/#sec-suggested-names
+
       def self.detect?(text)
         return false if text.doctype?(/html/)
         return true if text =~ /\A<\?xml\b/
@@ -27,10 +29,10 @@ module Rouge
         rule %r/<![^>]*>/, Comment::Preproc
 
         # open tags
-        rule %r(<\s*[\w:.-]+)m, Name::Tag, :tag
+        rule %r(<\s*[\p{L}:_][\p{Word}\p{Cf}:.·-]*)m, Name::Tag, :tag
 
         # self-closing tags
-        rule %r(<\s*/\s*[\w:.-]+\s*>)m, Name::Tag
+        rule %r(<\s*/\s*[\p{L}:_][\p{Word}\p{Cf}:.·-]*\s*>)m, Name::Tag
       end
 
       state :comment do
@@ -41,7 +43,7 @@ module Rouge
 
       state :tag do
         rule %r/\s+/m, Text
-        rule %r/[\w.:-]+\s*=/m, Name::Attribute, :attr
+        rule %r/[\p{L}:_][\p{Word}\p{Cf}:.·-]*\s*=/m, Name::Attribute, :attr
         rule %r(/?\s*>), Name::Tag, :pop!
       end
 

data/lib/rouge/lexers/yaml.rb

@@ -11,6 +11,8 @@ module Rouge
       aliases 'yml'
       filenames '*.yaml', '*.yml'
 
+      # Documentation: https://yaml.org/spec/1.2/spec.html
+
       def self.detect?(text)
         # look for the %YAML directive
         return true if text =~ /\A\s*%YAML/m
@@ -165,15 +167,15 @@ module Rouge
         )x, Keyword::Type
 
         # an anchor
-        rule %r/&[\w-]+/, Name::Label
+        rule %r/&[\p{L}\p{Nl}\p{Nd}_-]+/, Name::Label
 
         # an alias
-        rule %r/\*[\w-]+/, Name::Variable
+        rule %r/\*[\p{L}\p{Nl}\p{Nd}_-]+/, Name::Variable
       end
 
       state :block_nodes do
         # implicit key
-        rule %r/((?:\w[\w -]*)?)(:)(?=\s|$)/ do |m|
+        rule %r/((?:[\p{L}\p{Nl}\p{Nd}_][\p{L}\p{Nl}\p{Nd}\p{Blank}_-]*)?)(:)(?=\s|$)/ do |m|
           groups Name::Attribute, Punctuation::Indicator
           set_indent m[0], :implicit => true
         end

data/lib/rouge/lexers/zig.rb

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class Zig < RegexLexer
+      tag 'zig'
+      aliases 'zir'
+      filenames '*.zig'
+      mimetypes 'text/x-zig'
+
+      title 'Zig'
+      desc 'The Zig programming language (ziglang.org)'
+
+      def self.keywords
+        @keywords ||= %w(
+          align linksection threadlocal struct enum union error break return
+          anyframe fn c_longlong c_ulonglong c_longdouble c_void comptime_float
+          c_short c_ushort c_int c_uint c_long c_ulong continue asm defer
+          errdefer const var extern packed export pub if else switch and or
+          orelse while for bool unreachable try catch async suspend nosuspend
+          await resume undefined usingnamespace test void noreturn type
+          anyerror usize noalias inline noinline comptime callconv volatile
+          allowzero
+        )
+      end
+
+      def self.builtins
+        @builtins ||= %w(
+          @addWithOverflow @as @atomicLoad @atomicStore @bitCast @breakpoint
+          @alignCast @alignOf @cDefine @cImport @cInclude @bitOffsetOf
+          @atomicRmw @bytesToSlice @byteOffsetOf @OpaqueType @panic @ptrCast
+          @bitReverse @Vector @sin @cUndef @canImplicitCast @clz @cmpxchgWeak
+          @cmpxchgStrong @compileError @compileLog @ctz @popCount @divExact
+          @divFloor @cos @divTrunc @embedFile @export @tagName @TagType
+          @errorName @call @errorReturnTrace @fence @fieldParentPtr @field
+          @unionInit @errorToInt @intToEnum @enumToInt @setAlignStack @frame
+          @Frame @exp @exp2 @log @log2 @log10 @fabs @floor @ceil @trunc @round
+          @floatCast @intToFloat @floatToInt @boolToInt @errSetCast @intToError
+          @frameAddress @import @newStackCall @asyncCall @intToPtr @intCast
+          @frameSize @memcpy @memset @mod @mulWithOverflow @splat @ptrToInt
+          @rem @returnAddress @setCold @Type @shuffle @setGlobalLinkage
+          @setGlobalSection @shlExact @This @hasDecl @hasField
+          @setRuntimeSafety @setEvalBranchQuota @setFloatMode @shlWithOverflow
+          @shrExact @sizeOf @bitSizeOf @sqrt @byteSwap @subWithOverflow
+          @sliceToBytes comptime_int @truncate @typeInfo @typeName @TypeOf
+        )
+      end
+
+      id = /[a-z_]\w*/i
+      escapes = /\\ ([nrt'"\\0] | x\h{2} | u\h{4} | U\h{8})/x
+
+      state :bol do
+        mixin :whitespace
+        rule %r/#\s[^\n]*/, Comment::Special
+        rule(//) { pop! }
+      end
+
+      state :attribute do
+        mixin :whitespace
+        mixin :literals
+        rule %r/[(,)=:]/, Name::Decorator
+        rule %r/\]/, Name::Decorator, :pop!
+        rule id, Name::Decorator
+      end
+
+      state :whitespace do
+        rule %r/\s+/, Text
+        rule %r(//[^\n]*), Comment
+      end
+
+      state :root do
+        rule %r/\n/, Text, :bol
+
+        mixin :whitespace
+
+        rule %r/\b(?:(i|u)[0-9]+)\b/, Keyword::Type
+        rule %r/\b(?:f(16|32|64|128))\b/, Keyword::Type
+        rule %r/\b(?:(isize|usize))\b/, Keyword::Type
+
+        mixin :literals
+
+        rule %r/'#{id}/, Name::Variable
+        rule %r/([.]?)(\s*)(@?#{id})(\s*)([(]?)/ do |m|
+          name = m[3]
+          t = if self.class.keywords.include? name
+                Keyword
+              elsif self.class.builtins.include? name
+                Name::Builtin
+              elsif !m[1].empty? && !m[5].empty?
+                Name::Function
+              elsif !m[1].empty?
+                Name::Property
+              else
+                Name
+              end
+
+          groups Punctuation, Text, t, Text, Punctuation
+        end
+
+        rule %r/[()\[\]{}|,:;]/, Punctuation
+        rule %r/[*\/!@~&+%^<>=\?-]|\.{1,3}/, Operator
+      end
+
+      state :literals do
+        rule %r/\b(?:true|false|null)\b/, Keyword::Constant
+        rule %r(
+        ' (?: #{escapes} | [^\\] ) '
+        )x, Str::Char
+
+        rule %r/"/, Str, :string
+        rule %r/r(#*)".*?"\1/m, Str
+
+        dot = /[.][0-9_]+/
+        exp = /e[-+]?[0-9_]+/
+
+        rule %r(
+          [0-9]+
+          (#{dot}  #{exp}?
+          |#{dot}? #{exp}
+          )
+        )x, Num::Float
+
+        rule %r(
+        ( 0b[10_]+
+         | 0x[0-9a-fA-F_]+
+         | [0-9_]+
+        )
+        )x, Num::Integer
+      end
+
+      state :string do
+        rule %r/"/, Str, :pop!
+        rule escapes, Str::Escape
+        rule %r/[^"\\]+/m, Str
+      end
+    end
+  end
+end

data/lib/rouge/regex_lexer.rb

@@ -6,6 +6,37 @@ module Rouge
   # A stateful lexer that uses sets of regular expressions to
   # tokenize a string.  Most lexers are instances of RegexLexer.
   class RegexLexer < Lexer
+    class InvalidRegex < StandardError
+      def initialize(re)
+        @re = re
+      end
+
+      def to_s
+        "regex #{@re.inspect} matches empty string, but has no predicate!"
+      end
+    end
+
+    class ClosedState < StandardError
+      attr_reader :state
+      def initialize(state)
+        @state = state
+      end
+
+      def rule
+        @state.rules.last
+      end
+
+      def to_s
+        rule = @state.rules.last
+        msg = "State :#{state.name} cannot continue after #{rule.inspect}, which will always match."
+        if rule.re.source.include?('*')
+          msg += " Consider replacing * with +."
+        end
+
+        msg
+      end
+    end
+
     # A rule is a tuple of a regular expression to test, and a callback
     # to perform if the test succeeds.
     #
@@ -42,12 +73,13 @@ module Rouge
     end
 
     class StateDSL
-      attr_reader :rules
+      attr_reader :rules, :name
       def initialize(name, &defn)
         @name = name
         @defn = defn
         @rules = []
         @loaded = false
+        @closed = false
       end
 
       def to_state(lexer_class)
@@ -95,10 +127,14 @@ module Rouge
       #   {RegexLexer#token}, and {RegexLexer#delegate}.  The first
       #   argument can be used to access the match groups.
       def rule(re, tok=nil, next_state=nil, &callback)
+        raise ClosedState.new(self) if @closed
+
         if tok.nil? && callback.nil?
           raise "please pass `rule` a token to yield or a callback"
         end
 
+        matches_empty = re =~ ''
+
         callback ||= case next_state
         when :pop!
           proc do |stream|
@@ -123,6 +159,9 @@ module Rouge
             @stack.push(state)
           end
         when nil
+          # cannot use an empty-matching regexp with no predicate
+          raise InvalidRegex.new(re) if matches_empty
+
           proc do |stream|
             puts "    yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug
             @output_stream.call(tok, stream[0])
@@ -132,6 +171,22 @@ module Rouge
         end
 
         rules << Rule.new(re, callback)
+
+        close! if matches_empty && !context_sensitive?(re)
+      end
+
+      def context_sensitive?(re)
+        source = re.source
+        return true if source =~ /[(][?]<?[!=]/
+
+        # anchors count as lookahead/behind
+        return true if source =~ /[$^]/
+
+        false
+      end
+
+      def close!
+        @closed = true
       end
 
       # Mix in the rules from another state into this state.  The rules

data/lib/rouge/version.rb

@@ -3,6 +3,6 @@
 
 module Rouge
   def self.version
-    "3.19.0"
+    "3.24.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rouge
 version: !ruby/object:Gem::Version
-  version: 3.19.0
+  version: 3.24.0
 platform: ruby
 authors:
 - Jeanine Adkisson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-12 00:00:00.000000000 Z
+date: 2020-10-13 00:00:00.000000000 Z
 dependencies: []
 description: Rouge aims to a be a simple, easy-to-extend drop-in replacement for pygments.
 email:
@@ -31,12 +31,15 @@ files:
 - lib/rouge/demos/apiblueprint
 - lib/rouge/demos/applescript
 - lib/rouge/demos/armasm
+- lib/rouge/demos/augeas
 - lib/rouge/demos/awk
 - lib/rouge/demos/batchfile
 - lib/rouge/demos/bbcbasic
+- lib/rouge/demos/bibtex
 - lib/rouge/demos/biml
 - lib/rouge/demos/bpf
 - lib/rouge/demos/brainfuck
+- lib/rouge/demos/brightscript
 - lib/rouge/demos/bsl
 - lib/rouge/demos/c
 - lib/rouge/demos/ceylon
@@ -70,6 +73,7 @@ files:
 - lib/rouge/demos/eiffel
 - lib/rouge/demos/elixir
 - lib/rouge/demos/elm
+- lib/rouge/demos/email
 - lib/rouge/demos/epp
 - lib/rouge/demos/erb
 - lib/rouge/demos/erlang
@@ -93,6 +97,7 @@ files:
 - lib/rouge/demos/haskell
 - lib/rouge/demos/haxe
 - lib/rouge/demos/hcl
+- lib/rouge/demos/hlsl
 - lib/rouge/demos/hocon
 - lib/rouge/demos/hql
 - lib/rouge/demos/html
@@ -105,6 +110,8 @@ files:
 - lib/rouge/demos/irb
 - lib/rouge/demos/irb_output
 - lib/rouge/demos/isbl
+- lib/rouge/demos/j
+- lib/rouge/demos/janet
 - lib/rouge/demos/java
 - lib/rouge/demos/javascript
 - lib/rouge/demos/jinja
@@ -120,6 +127,7 @@ files:
 - lib/rouge/demos/liquid
 - lib/rouge/demos/literate_coffeescript
 - lib/rouge/demos/literate_haskell
+- lib/rouge/demos/livescript
 - lib/rouge/demos/llvm
 - lib/rouge/demos/lua
 - lib/rouge/demos/lustre
@@ -152,6 +160,7 @@ files:
 - lib/rouge/demos/plaintext
 - lib/rouge/demos/plist
 - lib/rouge/demos/pony
+- lib/rouge/demos/postscript
 - lib/rouge/demos/powershell
 - lib/rouge/demos/praat
 - lib/rouge/demos/prolog
@@ -186,8 +195,10 @@ files:
 - lib/rouge/demos/sparql
 - lib/rouge/demos/sqf
 - lib/rouge/demos/sql
+- lib/rouge/demos/ssh
 - lib/rouge/demos/supercollider
 - lib/rouge/demos/swift
+- lib/rouge/demos/systemd
 - lib/rouge/demos/tap
 - lib/rouge/demos/tcl
 - lib/rouge/demos/terraform
@@ -202,6 +213,7 @@ files:
 - lib/rouge/demos/vala
 - lib/rouge/demos/vb
 - lib/rouge/demos/vcl
+- lib/rouge/demos/velocity
 - lib/rouge/demos/verilog
 - lib/rouge/demos/vhdl
 - lib/rouge/demos/viml
@@ -213,10 +225,12 @@ files:
 - lib/rouge/demos/xquery
 - lib/rouge/demos/yaml
 - lib/rouge/demos/yang
+- lib/rouge/demos/zig
 - lib/rouge/formatter.rb
 - lib/rouge/formatters/html.rb
 - lib/rouge/formatters/html_inline.rb
 - lib/rouge/formatters/html_legacy.rb
+- lib/rouge/formatters/html_line_highlighter.rb
 - lib/rouge/formatters/html_line_table.rb
 - lib/rouge/formatters/html_linewise.rb
 - lib/rouge/formatters/html_pygments.rb
@@ -243,12 +257,15 @@ files:
 - lib/rouge/lexers/apiblueprint.rb
 - lib/rouge/lexers/apple_script.rb
 - lib/rouge/lexers/armasm.rb
+- lib/rouge/lexers/augeas.rb
 - lib/rouge/lexers/awk.rb
 - lib/rouge/lexers/batchfile.rb
 - lib/rouge/lexers/bbcbasic.rb
+- lib/rouge/lexers/bibtex.rb
 - lib/rouge/lexers/biml.rb
 - lib/rouge/lexers/bpf.rb
 - lib/rouge/lexers/brainfuck.rb
+- lib/rouge/lexers/brightscript.rb
 - lib/rouge/lexers/bsl.rb
 - lib/rouge/lexers/c.rb
 - lib/rouge/lexers/ceylon.rb
@@ -282,6 +299,7 @@ files:
 - lib/rouge/lexers/eiffel.rb
 - lib/rouge/lexers/elixir.rb
 - lib/rouge/lexers/elm.rb
+- lib/rouge/lexers/email.rb
 - lib/rouge/lexers/epp.rb
 - lib/rouge/lexers/erb.rb
 - lib/rouge/lexers/erlang.rb
@@ -306,6 +324,7 @@ files:
 - lib/rouge/lexers/haskell.rb
 - lib/rouge/lexers/haxe.rb
 - lib/rouge/lexers/hcl.rb
+- lib/rouge/lexers/hlsl.rb
 - lib/rouge/lexers/hocon.rb
 - lib/rouge/lexers/hql.rb
 - lib/rouge/lexers/html.rb
@@ -318,6 +337,8 @@ files:
 - lib/rouge/lexers/irb.rb
 - lib/rouge/lexers/isbl.rb
 - lib/rouge/lexers/isbl/builtins.rb
+- lib/rouge/lexers/j.rb
+- lib/rouge/lexers/janet.rb
 - lib/rouge/lexers/java.rb
 - lib/rouge/lexers/javascript.rb
 - lib/rouge/lexers/jinja.rb
@@ -334,6 +355,7 @@ files:
 - lib/rouge/lexers/liquid.rb
 - lib/rouge/lexers/literate_coffeescript.rb
 - lib/rouge/lexers/literate_haskell.rb
+- lib/rouge/lexers/livescript.rb
 - lib/rouge/lexers/llvm.rb
 - lib/rouge/lexers/llvm/keywords.rb
 - lib/rouge/lexers/lua.rb
@@ -374,6 +396,7 @@ files:
 - lib/rouge/lexers/plain_text.rb
 - lib/rouge/lexers/plist.rb
 - lib/rouge/lexers/pony.rb
+- lib/rouge/lexers/postscript.rb
 - lib/rouge/lexers/powershell.rb
 - lib/rouge/lexers/praat.rb
 - lib/rouge/lexers/prolog.rb
@@ -410,8 +433,10 @@ files:
 - lib/rouge/lexers/sqf.rb
 - lib/rouge/lexers/sqf/keywords.rb
 - lib/rouge/lexers/sql.rb
+- lib/rouge/lexers/ssh.rb
 - lib/rouge/lexers/supercollider.rb
 - lib/rouge/lexers/swift.rb
+- lib/rouge/lexers/systemd.rb
 - lib/rouge/lexers/tap.rb
 - lib/rouge/lexers/tcl.rb
 - lib/rouge/lexers/terraform.rb
@@ -427,6 +452,7 @@ files:
 - lib/rouge/lexers/vala.rb
 - lib/rouge/lexers/varnish.rb
 - lib/rouge/lexers/vb.rb
+- lib/rouge/lexers/velocity.rb
 - lib/rouge/lexers/verilog.rb
 - lib/rouge/lexers/vhdl.rb
 - lib/rouge/lexers/viml.rb
@@ -439,6 +465,7 @@ files:
 - lib/rouge/lexers/xquery.rb
 - lib/rouge/lexers/yaml.rb
 - lib/rouge/lexers/yang.rb
+- lib/rouge/lexers/zig.rb
 - lib/rouge/plugins/redcarpet.rb
 - lib/rouge/regex_lexer.rb
 - lib/rouge/template_lexer.rb