pygments.rb 1.2.1 → 2.0.0

data/vendor/pygments-main/pygments/lexers/webidl.py

@@ -0,0 +1,299 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.webidl
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for Web IDL, including some extensions.
+
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, default, include, words
+from pygments.token import Comment, Keyword, Name, Number, Punctuation, \
+    String, Text
+
+__all__ = ['WebIDLLexer']
+
+_builtin_types = (
+    # primitive types
+    'byte', 'octet', 'boolean',
+    r'(?:unsigned\s+)?(?:short|long(?:\s+long)?)',
+    r'(?:unrestricted\s+)?(?:float|double)',
+    # string types
+    'DOMString', 'ByteString', 'USVString',
+    # exception types
+    'Error', 'DOMException',
+    # typed array types
+    'Uint8Array', 'Uint16Array', 'Uint32Array', 'Uint8ClampedArray',
+    'Float32Array', 'Float64Array',
+    # buffer source types
+    'ArrayBuffer', 'DataView', 'Int8Array', 'Int16Array', 'Int32Array',
+    # other
+    'any', 'void', 'object', 'RegExp',
+)
+_identifier = r'_?[A-Za-z][a-zA-Z0-9_-]*'
+_keyword_suffix = r'(?![\w-])'
+_string = r'"[^"]*"'
+
+
+class WebIDLLexer(RegexLexer):
+    """
+    For Web IDL.
+
+    .. versionadded:: 2.6
+    """
+
+    name = 'Web IDL'
+    aliases = ['webidl']
+    filenames = ['*.webidl']
+
+    tokens = {
+        'common': [
+            (r'\s+', Text),
+            (r'(?s)/\*.*?\*/', Comment.Multiline),
+            (r'//.*', Comment.Single),
+            (r'^#.*', Comment.Preproc),
+        ],
+        'root': [
+            include('common'),
+            (r'\[', Punctuation, 'extended_attributes'),
+            (r'partial' + _keyword_suffix, Keyword),
+            (r'typedef' + _keyword_suffix, Keyword, ('typedef', 'type')),
+            (r'interface' + _keyword_suffix, Keyword, 'interface_rest'),
+            (r'enum' + _keyword_suffix, Keyword, 'enum_rest'),
+            (r'callback' + _keyword_suffix, Keyword, 'callback_rest'),
+            (r'dictionary' + _keyword_suffix, Keyword, 'dictionary_rest'),
+            (r'namespace' + _keyword_suffix, Keyword, 'namespace_rest'),
+            (_identifier, Name.Class, 'implements_rest'),
+        ],
+        'extended_attributes': [
+            include('common'),
+            (r',', Punctuation),
+            (_identifier, Name.Decorator),
+            (r'=', Punctuation, 'extended_attribute_rest'),
+            (r'\(', Punctuation, 'argument_list'),
+            (r'\]', Punctuation, '#pop'),
+        ],
+        'extended_attribute_rest': [
+            include('common'),
+            (_identifier, Name, 'extended_attribute_named_rest'),
+            (_string, String),
+            (r'\(', Punctuation, 'identifier_list'),
+            default('#pop'),
+        ],
+        'extended_attribute_named_rest': [
+            include('common'),
+            (r'\(', Punctuation, 'argument_list'),
+            default('#pop'),
+        ],
+        'argument_list': [
+            include('common'),
+            (r'\)', Punctuation, '#pop'),
+            default('argument'),
+        ],
+        'argument': [
+            include('common'),
+            (r'optional' + _keyword_suffix, Keyword),
+            (r'\[', Punctuation, 'extended_attributes'),
+            (r',', Punctuation, '#pop'),
+            (r'\)', Punctuation, '#pop:2'),
+            default(('argument_rest', 'type'))
+        ],
+        'argument_rest': [
+            include('common'),
+            (_identifier, Name.Variable),
+            (r'\.\.\.', Punctuation),
+            (r'=', Punctuation, 'default_value'),
+            default('#pop'),
+        ],
+        'identifier_list': [
+            include('common'),
+            (_identifier, Name.Class),
+            (r',', Punctuation),
+            (r'\)', Punctuation, '#pop'),
+        ],
+        'type': [
+            include('common'),
+            (r'(?:' + r'|'.join(_builtin_types) + r')' + _keyword_suffix,
+             Keyword.Type, 'type_null'),
+            (words(('sequence', 'Promise', 'FrozenArray'),
+                   suffix=_keyword_suffix), Keyword.Type, 'type_identifier'),
+            (_identifier, Name.Class, 'type_identifier'),
+            (r'\(', Punctuation, 'union_type'),
+        ],
+        'union_type': [
+            include('common'),
+            (r'or' + _keyword_suffix, Keyword),
+            (r'\)', Punctuation, ('#pop', 'type_null')),
+            default('type'),
+        ],
+        'type_identifier': [
+            (r'<', Punctuation, 'type_list'),
+            default(('#pop', 'type_null'))
+        ],
+        'type_null': [
+            (r'\?', Punctuation),
+            default('#pop:2'),
+        ],
+        'default_value': [
+            include('common'),
+            include('const_value'),
+            (_string, String, '#pop'),
+            (r'\[\s*\]', Punctuation, '#pop'),
+        ],
+        'const_value': [
+            include('common'),
+            (words(('true', 'false', '-Infinity', 'Infinity', 'NaN', 'null'),
+                   suffix=_keyword_suffix), Keyword.Constant, '#pop'),
+            (r'-?(?:(?:[0-9]+\.[0-9]*|[0-9]*\.[0-9]+)(?:[Ee][+-]?[0-9]+)?' +
+             r'|[0-9]+[Ee][+-]?[0-9]+)', Number.Float, '#pop'),
+            (r'-?[1-9][0-9]*', Number.Integer, '#pop'),
+            (r'-?0[Xx][0-9A-Fa-f]+', Number.Hex, '#pop'),
+            (r'-?0[0-7]*', Number.Oct, '#pop'),
+        ],
+        'typedef': [
+            include('common'),
+            (_identifier, Name.Class),
+            (r';', Punctuation, '#pop'),
+        ],
+        'namespace_rest': [
+            include('common'),
+            (_identifier, Name.Namespace),
+            (r'\{', Punctuation, 'namespace_body'),
+            (r';', Punctuation, '#pop'),
+        ],
+        'namespace_body': [
+            include('common'),
+            (r'\[', Punctuation, 'extended_attributes'),
+            (r'readonly' + _keyword_suffix, Keyword),
+            (r'attribute' + _keyword_suffix,
+             Keyword, ('attribute_rest', 'type')),
+            (r'const' + _keyword_suffix, Keyword, ('const_rest', 'type')),
+            (r'\}', Punctuation, '#pop'),
+            default(('operation_rest', 'type')),
+        ],
+        'interface_rest': [
+            include('common'),
+            (_identifier, Name.Class),
+            (r':', Punctuation),
+            (r'\{', Punctuation, 'interface_body'),
+            (r';', Punctuation, '#pop'),
+        ],
+        'interface_body': [
+            (words(('iterable', 'maplike', 'setlike'), suffix=_keyword_suffix),
+             Keyword, 'iterable_maplike_setlike_rest'),
+            (words(('setter', 'getter', 'creator', 'deleter', 'legacycaller',
+                    'inherit', 'static', 'stringifier', 'jsonifier'),
+                   suffix=_keyword_suffix), Keyword),
+            (r'serializer' + _keyword_suffix, Keyword, 'serializer_rest'),
+            (r';', Punctuation),
+            include('namespace_body'),
+        ],
+        'attribute_rest': [
+            include('common'),
+            (_identifier, Name.Variable),
+            (r';', Punctuation, '#pop'),
+        ],
+        'const_rest': [
+            include('common'),
+            (_identifier, Name.Constant),
+            (r'=', Punctuation, 'const_value'),
+            (r';', Punctuation, '#pop'),
+        ],
+        'operation_rest': [
+            include('common'),
+            (r';', Punctuation, '#pop'),
+            default('operation'),
+        ],
+        'operation': [
+            include('common'),
+            (_identifier, Name.Function),
+            (r'\(', Punctuation, 'argument_list'),
+            (r';', Punctuation, '#pop:2'),
+        ],
+        'iterable_maplike_setlike_rest': [
+            include('common'),
+            (r'<', Punctuation, 'type_list'),
+            (r';', Punctuation, '#pop'),
+        ],
+        'type_list': [
+            include('common'),
+            (r',', Punctuation),
+            (r'>', Punctuation, '#pop'),
+            default('type'),
+        ],
+        'serializer_rest': [
+            include('common'),
+            (r'=', Punctuation, 'serialization_pattern'),
+            (r';', Punctuation, '#pop'),
+            default('operation'),
+        ],
+        'serialization_pattern': [
+            include('common'),
+            (_identifier, Name.Variable, '#pop'),
+            (r'\{', Punctuation, 'serialization_pattern_map'),
+            (r'\[', Punctuation, 'serialization_pattern_list'),
+        ],
+        'serialization_pattern_map': [
+            include('common'),
+            (words(('getter', 'inherit', 'attribute'),
+                   suffix=_keyword_suffix), Keyword),
+            (r',', Punctuation),
+            (_identifier, Name.Variable),
+            (r'\}', Punctuation, '#pop:2'),
+        ],
+        'serialization_pattern_list': [
+            include('common'),
+            (words(('getter', 'attribute'), suffix=_keyword_suffix), Keyword),
+            (r',', Punctuation),
+            (_identifier, Name.Variable),
+            (r']', Punctuation, '#pop:2'),
+        ],
+        'enum_rest': [
+            include('common'),
+            (_identifier, Name.Class),
+            (r'\{', Punctuation, 'enum_body'),
+            (r';', Punctuation, '#pop'),
+        ],
+        'enum_body': [
+            include('common'),
+            (_string, String),
+            (r',', Punctuation),
+            (r'\}', Punctuation, '#pop'),
+        ],
+        'callback_rest': [
+            include('common'),
+            (r'interface' + _keyword_suffix,
+             Keyword, ('#pop', 'interface_rest')),
+            (_identifier, Name.Class),
+            (r'=', Punctuation, ('operation', 'type')),
+            (r';', Punctuation, '#pop'),
+        ],
+        'dictionary_rest': [
+            include('common'),
+            (_identifier, Name.Class),
+            (r':', Punctuation),
+            (r'\{', Punctuation, 'dictionary_body'),
+            (r';', Punctuation, '#pop'),
+        ],
+        'dictionary_body': [
+            include('common'),
+            (r'\[', Punctuation, 'extended_attributes'),
+            (r'required' + _keyword_suffix, Keyword),
+            (r'\}', Punctuation, '#pop'),
+            default(('dictionary_item', 'type')),
+        ],
+        'dictionary_item': [
+            include('common'),
+            (_identifier, Name.Variable),
+            (r'=', Punctuation, 'default_value'),
+            (r';', Punctuation, '#pop'),
+        ],
+        'implements_rest': [
+            include('common'),
+            (r'implements' + _keyword_suffix, Keyword),
+            (_identifier, Name.Class),
+            (r';', Punctuation, '#pop'),
+        ],
+    }

data/vendor/pygments-main/pygments/lexers/webmisc.py

@@ -5,7 +5,7 @@
 
     Lexers for misc. web stuff.
 
-    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -15,7 +15,6 @@ from pygments.lexer import RegexLexer, ExtendedRegexLexer, include, bygroups, \
     default, using
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
     Number, Punctuation, Literal
-from pygments.util import unirange
 
 from pygments.lexers.css import _indentation, _starts_block
 from pygments.lexers.html import HtmlLexer
@@ -74,15 +73,15 @@ class XQueryLexer(ExtendedRegexLexer):
 
     # FIX UNICODE LATER
     # ncnamestartchar = (
-    #    ur"[A-Z]|_|[a-z]|[\u00C0-\u00D6]|[\u00D8-\u00F6]|[\u00F8-\u02FF]|"
-    #    ur"[\u0370-\u037D]|[\u037F-\u1FFF]|[\u200C-\u200D]|[\u2070-\u218F]|"
-    #    ur"[\u2C00-\u2FEF]|[\u3001-\uD7FF]|[\uF900-\uFDCF]|[\uFDF0-\uFFFD]|"
-    #    ur"[\u10000-\uEFFFF]"
+    #    r"[A-Z]|_|[a-z]|[\u00C0-\u00D6]|[\u00D8-\u00F6]|[\u00F8-\u02FF]|"
+    #    r"[\u0370-\u037D]|[\u037F-\u1FFF]|[\u200C-\u200D]|[\u2070-\u218F]|"
+    #    r"[\u2C00-\u2FEF]|[\u3001-\uD7FF]|[\uF900-\uFDCF]|[\uFDF0-\uFFFD]|"
+    #    r"[\u10000-\uEFFFF]"
     # )
     ncnamestartchar = r"(?:[A-Z]|_|[a-z])"
     # FIX UNICODE LATER
-    # ncnamechar = ncnamestartchar + (ur"|-|\.|[0-9]|\u00B7|[\u0300-\u036F]|"
-    #                                 ur"[\u203F-\u2040]")
+    # ncnamechar = ncnamestartchar + (r"|-|\.|[0-9]|\u00B7|[\u0300-\u036F]|"
+    #                                 r"[\u203F-\u2040]")
     ncnamechar = r"(?:" + ncnamestartchar + r"|-|\.|[0-9])"
     ncname = "(?:%s+%s*)" % (ncnamestartchar, ncnamechar)
     pitarget_namestartchar = r"(?:[A-KN-WYZ]|_|:|[a-kn-wyz])"
@@ -99,14 +98,14 @@ class XQueryLexer(ExtendedRegexLexer):
     stringsingle = r"(?:'(?:" + entityref + r"|" + charref + r"|''|[^&'])*')"
 
     # FIX UNICODE LATER
-    # elementcontentchar = (ur'\t|\r|\n|[\u0020-\u0025]|[\u0028-\u003b]|'
-    #                       ur'[\u003d-\u007a]|\u007c|[\u007e-\u007F]')
+    # elementcontentchar = (r'\t|\r|\n|[\u0020-\u0025]|[\u0028-\u003b]|'
+    #                       r'[\u003d-\u007a]|\u007c|[\u007e-\u007F]')
     elementcontentchar = r'[A-Za-z]|\s|\d|[!"#$%()*+,\-./:;=?@\[\\\]^_\'`|~]'
-    # quotattrcontentchar = (ur'\t|\r|\n|[\u0020-\u0021]|[\u0023-\u0025]|'
-    #                        ur'[\u0027-\u003b]|[\u003d-\u007a]|\u007c|[\u007e-\u007F]')
+    # quotattrcontentchar = (r'\t|\r|\n|[\u0020-\u0021]|[\u0023-\u0025]|'
+    #                        r'[\u0027-\u003b]|[\u003d-\u007a]|\u007c|[\u007e-\u007F]')
     quotattrcontentchar = r'[A-Za-z]|\s|\d|[!#$%()*+,\-./:;=?@\[\\\]^_\'`|~]'
-    # aposattrcontentchar = (ur'\t|\r|\n|[\u0020-\u0025]|[\u0028-\u003b]|'
-    #                        ur'[\u003d-\u007a]|\u007c|[\u007e-\u007F]')
+    # aposattrcontentchar = (r'\t|\r|\n|[\u0020-\u0025]|[\u0028-\u003b]|'
+    #                        r'[\u003d-\u007a]|\u007c|[\u007e-\u007F]')
     aposattrcontentchar = r'[A-Za-z]|\s|\d|[!"#$%()*+,\-./:;=?@\[\\\]^_`|~]'
 
     # CHAR elements - fix the above elementcontentchar, quotattrcontentchar,
@@ -158,6 +157,9 @@ class XQueryLexer(ExtendedRegexLexer):
         # state stack
         if len(lexer.xquery_parse_state) == 0:
             ctx.stack.pop()
+            if not ctx.stack:
+                # make sure we have at least the root state on invalid inputs
+                ctx.stack = ['root']
         elif len(ctx.stack) > 1:
             ctx.stack.append(lexer.xquery_parse_state.pop())
         else:
@@ -438,7 +440,7 @@ class XQueryLexer(ExtendedRegexLexer):
         ],
         'varname': [
             (r'\(:', Comment, 'comment'),
-            (r'(' + qname + ')(\()?', bygroups(Name, Punctuation), 'operator'),
+            (r'(' + qname + r')(\()?', bygroups(Name, Punctuation), 'operator'),
         ],
         'singletype': [
             include('whitespace'),
@@ -515,8 +517,8 @@ class XQueryLexer(ExtendedRegexLexer):
         'xml_comment': [
             (r'(-->)', popstate_xmlcomment_callback),
             (r'[^-]{1,2}', Literal),
-            (u'\\t|\\r|\\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' +
-             unirange(0x10000, 0x10ffff), Literal),
+            (r'\t|\r|\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|[\U00010000-\U0010FFFF]',
+             Literal),
         ],
         'processing_instruction': [
             (r'\s+', Text, 'processing_instruction_content'),
@@ -525,13 +527,13 @@ class XQueryLexer(ExtendedRegexLexer):
         ],
         'processing_instruction_content': [
             (r'\?>', String.Doc, '#pop'),
-            (u'\\t|\\r|\\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' +
-             unirange(0x10000, 0x10ffff), Literal),
+            (r'\t|\r|\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|[\U00010000-\U0010FFFF]',
+             Literal),
         ],
         'cdata_section': [
             (r']]>', String.Doc, '#pop'),
-            (u'\\t|\\r|\\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' +
-             unirange(0x10000, 0x10ffff), Literal),
+            (r'\t|\r|\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|[\U00010000-\U0010FFFF]',
+             Literal),
         ],
         'start_tag': [
             include('whitespace'),
@@ -600,8 +602,8 @@ class XQueryLexer(ExtendedRegexLexer):
         ],
         'pragmacontents': [
             (r'#\)', Punctuation, 'operator'),
-            (u'\\t|\\r|\\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|' +
-             unirange(0x10000, 0x10ffff), Literal),
+            (r'\t|\r|\n|[\u0020-\uD7FF]|[\uE000-\uFFFD]|[\U00010000-\U0010FFFF]',
+             Literal),
             (r'(\s+)', Text),
         ],
         'occurrenceindicator': [
@@ -643,9 +645,9 @@ class XQueryLexer(ExtendedRegexLexer):
              bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'operator'),
             (r'(declare)(\s+)(context)(\s+)(item)',
              bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'operator'),
-            (ncname + ':\*', Name, 'operator'),
-            ('\*:'+ncname, Name.Tag, 'operator'),
-            ('\*', Name.Tag, 'operator'),
+            (ncname + r':\*', Name, 'operator'),
+            (r'\*:'+ncname, Name.Tag, 'operator'),
+            (r'\*', Name.Tag, 'operator'),
             (stringdouble, String.Double, 'operator'),
             (stringsingle, String.Single, 'operator'),
 
@@ -661,7 +663,8 @@ class XQueryLexer(ExtendedRegexLexer):
 
             # NAMESPACE KEYWORD
             (r'(declare)(\s+)(default)(\s+)(element|function)',
-             bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration), 'namespacekeyword'),
+             bygroups(Keyword.Declaration, Text, Keyword.Declaration, Text, Keyword.Declaration),
+             'namespacekeyword'),
             (r'(import)(\s+)(schema|module)',
              bygroups(Keyword.Pseudo, Text, Keyword.Pseudo), 'namespacekeyword'),
             (r'(declare)(\s+)(copy-namespaces)',
@@ -861,7 +864,7 @@ class QmlLexer(RegexLexer):
 
 
 class CirruLexer(RegexLexer):
-    """
+    r"""
     Syntax rules of Cirru can be found at:
     http://cirru.org/
 

data/vendor/pygments-main/pygments/lexers/whiley.py

@@ -5,7 +5,7 @@
 
     Lexers for the Whiley language.
 
-    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -97,11 +97,11 @@ class WhileyLexer(RegexLexer):
 
             # operators and punctuation
             (r'[{}()\[\],.;]', Punctuation),
-            (u'[+\\-*/%&|<>^!~@=:?'
+            (r'[+\-*/%&|<>^!~@=:?'
              # unicode operators
-             u'\u2200\u2203\u2205\u2282\u2286\u2283\u2287'
-             u'\u222A\u2229\u2264\u2265\u2208\u2227\u2228'
-             u']', Operator),
+             r'\u2200\u2203\u2205\u2282\u2286\u2283\u2287'
+             r'\u222A\u2229\u2264\u2265\u2208\u2227\u2228'
+             r']', Operator),
 
             # identifier
             (r'[a-zA-Z_]\w*', Name),

data/vendor/pygments-main/pygments/lexers/x10.py

@@ -5,7 +5,7 @@
 
     Lexers for the X10 programming language.
 
-    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexers/xorg.py

@@ -5,7 +5,7 @@
 
     Lexers for Xorg configs.
 
-    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -16,6 +16,7 @@ __all__ = ['XorgLexer']
 
 
 class XorgLexer(RegexLexer):
+    """Lexer for xorg.conf file."""
     name = 'Xorg'
     aliases = ['xorg.conf']
     filenames = ['xorg.conf']
@@ -26,9 +27,9 @@ class XorgLexer(RegexLexer):
             (r'\s+', Text),
             (r'#.*$', Comment),
 
-            (r'((|Sub)Section)(\s+)("\w+")',
-             bygroups(String.Escape, String.Escape, Text, String.Escape)),
-            (r'(End(|Sub)Section)', String.Escape),
+            (r'((?:Sub)?Section)(\s+)("\w+")',
+             bygroups(String.Escape, Text, String.Escape)),
+            (r'(End(?:Sub)?Section)', String.Escape),
 
             (r'(\w+)(\s+)([^\n#]+)',
              bygroups(Name.Builtin, Text, Name.Constant)),

data/vendor/pygments-main/pygments/lexers/yang.py

@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.yang
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for the YANG 1.1 modeling language. See :rfc:`7950`.
+
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import (RegexLexer, bygroups, words)
+from pygments.token import (Text, Token, Name, String, Comment,
+                            Number)
+
+__all__ = ['YangLexer']
+
+class YangLexer(RegexLexer):
+    """
+    Lexer for `YANG <https://tools.ietf.org/html/rfc7950/>`_, based on RFC7950
+
+    .. versionadded:: 2.7
+    """
+    name = 'YANG'
+    aliases = ['yang']
+    filenames = ['*.yang']
+    mimetypes = ['application/yang']
+
+    #Keywords from RFC7950 ; oriented at BNF style
+    TOP_STMTS_KEYWORDS = ("module", "submodule")
+    MODULE_HEADER_STMT_KEYWORDS = ("belongs-to", "namespace", "prefix", "yang-version")
+    META_STMT_KEYWORDS = ("contact", "description", "organization",
+                          "reference", "revision")
+    LINKAGE_STMTS_KEYWORDS = ("import", "include", "revision-date")
+    BODY_STMT_KEYWORDS = ("action", "argument", "augment", "deviation",
+                          "extension", "feature", "grouping", "identity",
+                          "if-feature", "input", "notification", "output",
+                          "rpc", "typedef")
+    DATA_DEF_STMT_KEYWORDS = ("anydata", "anyxml", "case", "choice",
+                              "config", "container", "deviate", "leaf",
+                              "leaf-list", "list", "must", "presence",
+                              "refine", "uses", "when")
+    TYPE_STMT_KEYWORDS = ("base", "bit", "default", "enum", "error-app-tag",
+                          "error-message", "fraction-digits", "length",
+                          "max-elements", "min-elements", "modifier",
+                          "ordered-by", "path", "pattern", "position",
+                          "range", "require-instance", "status", "type",
+                          "units", "value", "yin-element")
+    LIST_STMT_KEYWORDS = ("key", "mandatory", "unique")
+
+    #RFC7950 other keywords
+    CONSTANTS_KEYWORDS = ("add", "current", "delete", "deprecated", "false",
+                          "invert-match", "max", "min", "not-supported",
+                          "obsolete", "replace", "true", "unbounded", "user")
+
+    #RFC7950 Built-In Types
+    TYPES = ("binary", "bits", "boolean", "decimal64", "empty", "enumeration",
+             "identityref", "instance-identifier", "int16", "int32", "int64",
+             "int8", "leafref", "string", "uint16", "uint32", "uint64",
+             "uint8", "union")
+
+    suffix_re_pattern = r'(?=[^\w\-:])'
+
+    tokens = {
+        'comments': [
+            (r'[^*/]', Comment),
+            (r'/\*', Comment, '#push'),
+            (r'\*/', Comment, '#pop'),
+            (r'[*/]', Comment),
+        ],
+        "root": [
+            (r'\s+', Text.Whitespace),
+            (r'[{};]+', Token.Punctuation),
+            (r'(?<![\-\w])(and|or|not|\+|\.)(?![\-\w])', Token.Operator),
+
+            (r'"(?:\\"|[^"])*?"', String.Double),
+            (r"'(?:\\'|[^'])*?'", String.Single),
+
+            (r'/\*', Comment, 'comments'),
+            (r'//.*?$', Comment),
+
+            #match BNF stmt for `node-identifier` with [ prefix ":"]
+            (r'(?:^|(?<=[\s{};]))([\w.-]+)(:)([\w.-]+)(?=[\s{};])',
+             bygroups(Name.Namespace, Token.Punctuation, Name.Variable)),
+
+            #match BNF stmt `date-arg-str`
+            (r'([0-9]{4}\-[0-9]{2}\-[0-9]{2})(?=[\s{};])', Name.Label),
+            (r'([0-9]+\.[0-9]+)(?=[\s{};])', Number.Float),
+            (r'([0-9]+)(?=[\s{};])', Number.Integer),
+
+            (words(TOP_STMTS_KEYWORDS, suffix=suffix_re_pattern), Token.Keyword),
+            (words(MODULE_HEADER_STMT_KEYWORDS, suffix=suffix_re_pattern), Token.Keyword),
+            (words(META_STMT_KEYWORDS, suffix=suffix_re_pattern), Token.Keyword),
+            (words(LINKAGE_STMTS_KEYWORDS, suffix=suffix_re_pattern), Token.Keyword),
+            (words(BODY_STMT_KEYWORDS, suffix=suffix_re_pattern), Token.Keyword),
+            (words(DATA_DEF_STMT_KEYWORDS, suffix=suffix_re_pattern), Token.Keyword),
+            (words(TYPE_STMT_KEYWORDS, suffix=suffix_re_pattern), Token.Keyword),
+            (words(LIST_STMT_KEYWORDS, suffix=suffix_re_pattern), Token.Keyword),
+            (words(TYPES, suffix=suffix_re_pattern), Name.Class),
+            (words(CONSTANTS_KEYWORDS, suffix=suffix_re_pattern), Name.Class),
+
+            (r'[^;{}\s\'"]+', Name.Variable),
+        ]
+    }