rouge 3.1.0 → 3.18.0

data/lib/rouge/lexers/xquery.rb

@@ -0,0 +1,145 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    load_lexer 'xpath.rb'
+    class XQuery < XPath
+      title 'XQuery'
+      desc 'XQuery 3.1: An XML Query Language'
+      tag 'xquery'
+      filenames '*.xquery', '*.xq'
+      mimetypes 'application/xquery'
+
+      def self.keywords
+        @keywords ||= Regexp.union super, Regexp.union(%w(
+          xquery encoding version declare module
+          namespace copy-namespaces boundary-space construction
+          default collation base-uri preserve strip
+          ordering ordered unordered order empty greatest least
+          preserve no-preserve inherit no-inherit
+          decimal-format decimal-separator grouping-separator
+          infinity minus-sign NaN percent per-mille
+          zero-digit digit pattern-separator exponent-separator
+          import schema at element option
+          function external context item
+          typeswitch switch case
+          try catch
+          validate lax strict type
+          document element attribute text comment processing-instruction
+          for let where order group by return
+          allowing tumbling stable sliding window
+          start end only when previous next count collation
+          ascending descending
+        ))
+      end
+
+      # Mixin states:
+
+      state :tags do
+        rule %r/<#{XPath.qName}/, Name::Tag, :start_tag
+        rule %r/<!--/, Comment, :xml_comment
+        rule %r/<\?.*?\?>/, Comment::Preproc
+        rule %r/<!\[CDATA\[.*?\]\]>/, Comment::Preproc
+        rule %r/&\S*?;/, Name::Entity
+      end
+
+      # Lexical states:
+
+      prepend :root do
+        mixin :tags
+
+        rule %r/\{/, Punctuation
+        rule %r/\}`?/ do
+          token Punctuation
+          if stack.length > 1
+            pop!
+          end
+        end
+
+        rule %r/(namespace)(\s+)(#{XPath.ncName})/ do
+          groups Keyword, Text::Whitespace, Name::Namespace
+        end
+
+        rule %r/(#{XQuery.keywords})\b/, Keyword
+        rule %r/;/, Punctuation
+        rule %r/%/, Keyword::Declaration, :annotation
+
+        rule %r/(\(#)(\s*)(#{XPath.eqName})/ do
+          groups Comment::Preproc, Text::Whitespace, Name::Tag
+          push :pragma
+        end
+
+        rule %r/``\[/, Str, :str_constructor
+      end
+
+      state :annotation do
+        mixin :commentsAndWhitespace
+        rule XPath.eqName, Keyword::Declaration, :pop!
+      end
+
+      state :pragma do
+        mixin :commentsAndWhitespace
+        rule %r/#\)/, Comment::Preproc, :pop!
+        rule %r/./, Comment::Preproc
+      end
+
+      # https://www.w3.org/TR/xquery-31/#id-string-constructors
+      state :str_constructor do
+        rule %r/`\{/, Punctuation, :root
+        rule %r/\]``/, Str, :pop!
+        rule %r/[^`\]]+/m, Str
+        rule %r/[`\]]/, Str
+      end
+
+      state :xml_comment do
+        rule %r/[^-]+/m, Comment
+        rule %r/-->/, Comment, :pop!
+        rule %r/-/, Comment
+      end
+
+      state :start_tag do
+        rule %r/\s+/m, Text::Whitespace
+        rule %r/([\w.:-]+\s*=)(")/m do
+          groups Name::Attribute, Str
+          push :quot_attr
+        end
+        rule %r/([\w.:-]+\s*=)(')/m do
+          groups Name::Attribute, Str
+          push :apos_attr
+        end
+        rule %r/>/, Name::Tag, :tag_content
+        rule %r(/>), Name::Tag, :pop!
+      end
+
+      state :quot_attr do
+        rule %r/"/, Str, :pop!
+        rule %r/\{\{/, Str
+        rule %r/\{/, Punctuation, :root
+        rule %r/[^"{>]+/m, Str
+      end
+
+      state :apos_attr do
+        rule %r/'/, Str, :pop!
+        rule %r/\{\{/, Str
+        rule %r/\{/, Punctuation, :root
+        rule %r/[^'{>]+/m, Str
+      end
+
+      state :tag_content do
+        rule %r/\s+/m, Text::Whitespace
+        mixin :tags
+
+        rule %r/(\{\{|\}\})/, Text
+        rule %r/\{/, Punctuation, :root
+
+        rule %r/[^{}<&]/, Text
+
+        rule %r(</#{XPath.qName}(\s*)>) do
+          token Name::Tag
+          pop! 2 # pop self and tag_start
+        end
+      end
+    end
+  end
+end

data/lib/rouge/lexers/yaml.rb

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*- #
+# frozen_string_literal: true
 
 module Rouge
   module Lexers
@@ -77,38 +78,38 @@ module Rouge
       start { reset_indent }
 
       state :basic do
-        rule /#.*$/, Comment::Single
+        rule %r/#.*$/, Comment::Single
       end
 
       state :root do
         mixin :basic
 
-        rule /\n+/, Text
+        rule %r/\n+/, Text
 
         # trailing or pre-comment whitespace
-        rule /[ ]+(?=#|$)/, Text
+        rule %r/[ ]+(?=#|$)/, Text
 
-        rule /^%YAML\b/ do
+        rule %r/^%YAML\b/ do
           token Name::Tag
           reset_indent
           push :yaml_directive
         end
 
-        rule /^%TAG\b/ do
+        rule %r/^%TAG\b/ do
           token Name::Tag
           reset_indent
           push :tag_directive
         end
 
         # doc-start and doc-end indicators
-        rule /^(?:---|\.\.\.)(?= |$)/ do
+        rule %r/^(?:---|\.\.\.)(?= |$)/ do
           token Name::Namespace
           reset_indent
           push :block_line
         end
 
         # indentation spaces
-        rule /[ ]*(?!\s|$)/ do |m|
+        rule %r/[ ]*(?!\s|$)/ do |m|
           text, err = save_indent(m[0])
           token Text, text
           token Error, err
@@ -119,7 +120,7 @@ module Rouge
       state :indentation do
         rule(/\s*?\n/) { token Text; pop! 2 }
         # whitespace preceding block collection indicators
-        rule /[ ]+(?=[-:?](?:[ ]|$))/ do |m|
+        rule %r/[ ]+(?=[-:?](?:[ ]|$))/ do |m|
           token Text
           continue_indent(m[0])
         end
@@ -137,8 +138,8 @@ module Rouge
       # indented line in the block context
       state :block_line do
         # line end
-        rule /[ ]*(?=#|$)/, Text, :pop!
-        rule /[ ]+/, Text
+        rule %r/[ ]*(?=#|$)/, Text, :pop!
+        rule %r/[ ]+/, Text
         # tags, anchors, and aliases
         mixin :descriptors
         # block collections and scalars
@@ -147,7 +148,7 @@ module Rouge
         mixin :flow_nodes
 
         # a plain scalar
-        rule /(?=#{plain_scalar_start}|[?:-][^ \t\n\r\f\v])/ do
+        rule %r/(?=#{plain_scalar_start}|[?:-][^ \t\n\r\f\v])/ do
           token Name::Variable
           push :plain_scalar_in_block_context
         end
@@ -155,7 +156,7 @@ module Rouge
 
       state :descriptors do
         # a full-form tag
-        rule /!<[0-9A-Za-z;\/?:@&=+$,_.!~*'()\[\]%-]+>/, Keyword::Type
+        rule %r/!<[0-9A-Za-z;\/?:@&=+$,_.!~*'()\[\]%-]+>/, Keyword::Type
 
         # a tag in the form '!', '!suffix' or '!handle!suffix'
         rule %r(
@@ -164,21 +165,21 @@ module Rouge
         )x, Keyword::Type
 
         # an anchor
-        rule /&[\w-]+/, Name::Label
+        rule %r/&[\w-]+/, Name::Label
 
         # an alias
-        rule /\*[\w-]+/, Name::Variable
+        rule %r/\*[\w-]+/, Name::Variable
       end
 
       state :block_nodes do
         # implicit key
-        rule /((?:\w[\w -]*)?)(:)(?=\s|$)/ do |m|
+        rule %r/((?:\w[\w -]*)?)(:)(?=\s|$)/ do |m|
           groups Name::Attribute, Punctuation::Indicator
           set_indent m[0], :implicit => true
         end
 
         # literal and folded scalars
-        rule /[\|>]/ do
+        rule %r/[\|>][+-]?/ do
           token Punctuation::Indicator
           push :block_scalar_content
           push :block_scalar_header
@@ -186,39 +187,39 @@ module Rouge
       end
 
       state :flow_nodes do
-        rule /\[/, Punctuation::Indicator, :flow_sequence
-        rule /\{/, Punctuation::Indicator, :flow_mapping
-        rule /'/, Str::Single, :single_quoted_scalar
-        rule /"/, Str::Double, :double_quoted_scalar
+        rule %r/\[/, Punctuation::Indicator, :flow_sequence
+        rule %r/\{/, Punctuation::Indicator, :flow_mapping
+        rule %r/'/, Str::Single, :single_quoted_scalar
+        rule %r/"/, Str::Double, :double_quoted_scalar
       end
 
       state :flow_collection do
-        rule /\s+/m, Text
+        rule %r/\s+/m, Text
         mixin :basic
-        rule /[?:,]/, Punctuation::Indicator
+        rule %r/[?:,]/, Punctuation::Indicator
         mixin :descriptors
         mixin :flow_nodes
 
-        rule /(?=#{plain_scalar_start})/ do
+        rule %r/(?=#{plain_scalar_start})/ do
           push :plain_scalar_in_flow_context
         end
       end
 
       state :flow_sequence do
-        rule /\]/, Punctuation::Indicator, :pop!
+        rule %r/\]/, Punctuation::Indicator, :pop!
         mixin :flow_collection
       end
 
       state :flow_mapping do
-        rule /\}/, Punctuation::Indicator, :pop!
+        rule %r/\}/, Punctuation::Indicator, :pop!
         mixin :flow_collection
       end
 
       state :block_scalar_content do
-        rule /\n+/, Text
+        rule %r/\n+/, Text
 
         # empty lines never dedent, but they might be part of the scalar.
-        rule /^[ ]+$/ do |m|
+        rule %r/^[ ]+$/ do |m|
           text = m[0]
           indent_size = text.size
 
@@ -230,7 +231,7 @@ module Rouge
 
         # TODO: ^ doesn't actually seem to affect the match at all.
         # Find a way to work around this limitation.
-        rule /^[ ]*/ do |m|
+        rule %r/^[ ]*/ do |m|
           token Text
 
           indent_size = m[0].size
@@ -245,7 +246,7 @@ module Rouge
           end
         end
 
-        rule /[^\n\r\f\v]+/, Name::Constant
+        rule %r/[^\n\r\f\v]+/, Str
       end
 
       state :block_scalar_header do
@@ -270,46 +271,46 @@ module Rouge
 
       state :ignored_line do
         mixin :basic
-        rule /[ ]+/, Text
-        rule /\n/, Text, :pop!
+        rule %r/[ ]+/, Text
+        rule %r/\n/, Text, :pop!
       end
 
       state :quoted_scalar_whitespaces do
         # leading and trailing whitespace is ignored
-        rule /^[ ]+/, Text
-        rule /[ ]+$/, Text
+        rule %r/^[ ]+/, Text
+        rule %r/[ ]+$/, Text
 
-        rule /\n+/m, Text
+        rule %r/\n+/m, Text
 
-        rule /[ ]+/, Name::Variable
+        rule %r/[ ]+/, Name::Variable
       end
 
       state :single_quoted_scalar do
         mixin :quoted_scalar_whitespaces
-        rule /\\'/, Str::Escape
-        rule /'/, Str, :pop!
-        rule /[^\s']+/, Str
+        rule %r/\\'/, Str::Escape
+        rule %r/'/, Str, :pop!
+        rule %r/[^\s']+/, Str
       end
 
       state :double_quoted_scalar do
-        rule /"/, Str, :pop!
+        rule %r/"/, Str, :pop!
         mixin :quoted_scalar_whitespaces
         # escapes
-        rule /\\[0abt\tn\nvfre "\\N_LP]/, Str::Escape
-        rule /\\(?:x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
+        rule %r/\\[0abt\tn\nvfre "\\N_LP]/, Str::Escape
+        rule %r/\\(?:x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
           Str::Escape
-        rule /[^ \t\n\r\f\v"\\]+/, Str
+        rule %r/[^ \t\n\r\f\v"\\]+/, Str
       end
 
       state :plain_scalar_in_block_context_new_line do
-        rule /^[ ]+\n/, Text
-        rule /\n+/m, Text
-        rule /^(?=---|\.\.\.)/ do
+        rule %r/^[ ]+\n/, Text
+        rule %r/\n+/m, Text
+        rule %r/^(?=---|\.\.\.)/ do
           pop! 3
         end
 
         # dedent detection
-        rule /^[ ]*/ do |m|
+        rule %r/^[ ]*/ do |m|
           token Text
           pop!
 
@@ -326,34 +327,36 @@ module Rouge
 
       state :plain_scalar_in_block_context do
         # the : indicator ends a scalar
-        rule /[ ]*(?=:[ \n]|:$)/, Text, :pop!
-        rule /[ ]*:/, Str
-        rule /[ ]+(?=#)/, Text, :pop!
-        rule /[ ]+$/, Text
+        rule %r/[ ]*(?=:[ \n]|:$)/, Text, :pop!
+        rule %r/[ ]*:\S+/, Str
+        rule %r/[ ]+(?=#)/, Text, :pop!
+        rule %r/[ ]+$/, Text
         # check for new documents or dedents at the new line
-        rule /\n+/ do
+        rule %r/\n+/ do
           token Text
           push :plain_scalar_in_block_context_new_line
         end
 
-        rule /[ ]+/, Str
+        rule %r/[ ]+/, Str
         rule SPECIAL_VALUES, Name::Constant
+        rule %r/\d+(?:\.\d+)?(?=(\r?\n)| +#)/, Literal::Number, :pop!
+
         # regular non-whitespace characters
-        rule /[^\s:]+/, Str
+        rule %r/[^\s:]+/, Str
       end
 
       state :plain_scalar_in_flow_context do
-        rule /[ ]*(?=[,:?\[\]{}])/, Text, :pop!
-        rule /[ ]+(?=#)/, Text, :pop!
-        rule /^[ ]+/, Text
-        rule /[ ]+$/, Text
-        rule /\n+/, Text
-        rule /[ ]+/, Name::Variable
-        rule /[^\s,:?\[\]{}]+/, Name::Variable
+        rule %r/[ ]*(?=[,:?\[\]{}])/, Text, :pop!
+        rule %r/[ ]+(?=#)/, Text, :pop!
+        rule %r/^[ ]+/, Text
+        rule %r/[ ]+$/, Text
+        rule %r/\n+/, Text
+        rule %r/[ ]+/, Name::Variable
+        rule %r/[^\s,:?\[\]{}]+/, Name::Variable
       end
 
       state :yaml_directive do
-        rule /([ ]+)(\d+\.\d+)/ do
+        rule %r/([ ]+)(\d+\.\d+)/ do
           groups Text, Num
           goto :ignored_line
         end

data/lib/rouge/lexers/yang.rb

@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*- #
+# frozen_string_literal: true
+
+module Rouge
+  module Lexers
+    class YANG < RegexLexer
+      title 'YANG'
+      desc "Lexer for the YANG 1.1 modeling language (RFC7950)"
+      tag 'yang'
+      filenames '*.yang'
+      mimetypes 'application/yang'
+
+      id = /[\w_-]+(?=[^\w\-\:])\b/
+
+      #Keywords from RFC7950 ; oriented at BNF style
+      def self.top_stmts_keywords
+        @top_stms_keywords ||= Set.new %w(
+          module submodule
+        )
+      end
+
+      def self.module_header_stmts_keywords
+        @module_header_stmts_keywords ||= Set.new %w(
+          belongs-to namespace prefix yang-version
+        )
+      end
+
+      def self.meta_stmts_keywords
+        @meta_stmts_keywords ||= Set.new %w(
+          contact description organization reference revision
+        )
+      end
+
+      def self.linkage_stmts_keywords
+        @linkage_stmts_keywords ||= Set.new %w(
+          import include revision-date
+        )
+      end
+
+      def self.body_stmts_keywords
+        @body_stms_keywords ||= Set.new %w(
+          action argument augment deviation extension feature grouping identity
+          if-feature input notification output rpc typedef
+        )
+      end
+
+      def self.data_def_stmts_keywords
+        @data_def_stms_keywords ||= Set.new %w(
+          anydata anyxml case choice config container deviate leaf leaf-list
+          list must presence refine uses when
+        )
+      end
+
+      def self.type_stmts_keywords
+        @type_stmts_keywords ||= Set.new %w(
+          base bit default enum error-app-tag error-message fraction-digits
+          length max-elements min-elements modifier ordered-by path pattern
+          position range require-instance status type units value yin-element
+        )
+      end
+
+      def self.list_stmts_keywords
+        @list_stmts_keywords ||= Set.new %w(
+          key mandatory unique
+        )
+      end
+
+      #RFC7950 other keywords
+      def self.constants_keywords
+        @constants_keywords ||= Set.new %w(
+          add current delete deprecated false invert-match max min
+          not-supported obsolete replace true unbounded user
+        )
+      end
+
+      #RFC7950 Built-In Types
+      def self.types
+        @types ||= Set.new %w(
+          binary bits boolean decimal64 empty enumeration identityref
+          instance-identifier int16 int32 int64 int8 leafref string uint16
+          uint32 uint64 uint8 union
+        )
+      end
+
+      state :comment do
+        rule %r/[^*\/]/, Comment
+        rule %r/\/\*/, Comment, :comment
+        rule %r/\*\//, Comment, :pop!
+        rule %r/[*\/]/, Comment
+      end
+
+      #Keyword::Reserved
+      #groups Name::Tag, Text::Whitespace
+      state :root do
+        rule %r/\s+/, Text::Whitespace
+        rule %r/[\{\}\;]+/, Punctuation
+        rule %r/(?<![\-\w])(and|or|not|\+|\.)(?![\-\w])/, Operator
+
+        rule %r/"(?:\\"|[^"])*?"/, Str::Double #for double quotes
+        rule %r/'(?:\\'|[^'])*?'/, Str::Single #for single quotes
+
+        rule %r/\/\*/, Comment, :comment
+        rule %r/\/\/.*?$/, Comment
+
+        #match BNF stmt for `node-identifier` with [ prefix ":"]
+        rule %r/(?:^|(?<=[\s{};]))([\w.-]+)(:)([\w.-]+)(?=[\s{};])/ do
+          groups Name::Namespace, Punctuation, Name
+        end
+
+        #match BNF stmt `date-arg-str`
+        rule %r/([0-9]{4}\-[0-9]{2}\-[0-9]{2})(?=[\s\{\}\;])/, Name::Label
+        rule %r/([0-9]+\.[0-9]+)(?=[\s\{\}\;])/, Num::Float
+        rule %r/([0-9]+)(?=[\s\{\}\;])/, Num::Integer
+
+        rule id do |m|
+          name = m[0].downcase
+
+          if self.class.top_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.module_header_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.meta_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.linkage_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.body_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.data_def_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.type_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.list_stmts_keywords.include? name
+            token Keyword::Declaration
+          elsif self.class.types.include? name
+            token Keyword::Type
+          elsif self.class.constants_keywords.include? name
+            token Name::Constant
+          else
+            token Name
+          end
+        end
+
+        rule %r/[^;{}\s'"]+/, Name
+      end
+    end
+  end
+end