pygments.rb 1.1.0 → 2.0.0.rc2

data/vendor/pygments-main/pygments/lexers/css.py

@@ -5,7 +5,7 @@
 
     Lexers for CSS and related stylesheet formats.
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -16,7 +16,6 @@ from pygments.lexer import ExtendedRegexLexer, RegexLexer, include, bygroups, \
     default, words, inherit
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
     Number, Punctuation
-from pygments.util import iteritems
 
 __all__ = ['CssLexer', 'SassLexer', 'ScssLexer', 'LessCssLexer']
 
@@ -125,7 +124,7 @@ _css_properties = (
     'wrap-flow', 'wrap-inside', 'wrap-through', 'writing-mode', 'z-index',
 )
 
-# List of keyword values obtained from: 
+# List of keyword values obtained from:
 # http://cssvalues.com/
 _keyword_values = (
     'absolute', 'alias', 'all', 'all-petite-caps', 'all-scroll',
@@ -263,7 +262,7 @@ _time_units = (
     's', 'ms',
 )
 _all_units = _angle_units + _frequency_units + _length_units + \
-             _resolution_units + _time_units
+    _resolution_units + _time_units
 
 
 class CssLexer(RegexLexer):
@@ -322,16 +321,18 @@ class CssLexer(RegexLexer):
             include('urls'),
             (r'('+r'|'.join(_functional_notation_keyword_values)+r')(\()',
              bygroups(Name.Builtin, Punctuation), 'function-start'),
-            (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'),
+            (r'([a-zA-Z_][\w-]+)(\()',
+             bygroups(Name.Function, Punctuation), 'function-start'),
             (words(_keyword_values, suffix=r'\b'), Keyword.Constant),
             (words(_other_keyword_values, suffix=r'\b'), Keyword.Constant),
             (words(_color_keywords, suffix=r'\b'), Keyword.Constant),
-            (words(_css_properties, suffix=r'\b'), Keyword), # for transition-property etc.
+            # for transition-property etc.
+            (words(_css_properties, suffix=r'\b'), Keyword),
             (r'\!important', Comment.Preproc),
             (r'/\*(?:.|\n)*?\*/', Comment),
 
             include('numeric-values'),
-            
+
             (r'[~^*!%&<>|+=@:./?-]+', Operator),
             (r'[\[\](),]+', Punctuation),
             (r'"(\\\\|\\"|[^"])*"', String.Double),
@@ -351,12 +352,13 @@ class CssLexer(RegexLexer):
             # function-start may be entered recursively
             (r'(' + r'|'.join(_functional_notation_keyword_values) + r')(\()',
              bygroups(Name.Builtin, Punctuation), 'function-start'),
-            (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'),
+            (r'([a-zA-Z_][\w-]+)(\()',
+             bygroups(Name.Function, Punctuation), 'function-start'),
 
             (r'/\*(?:.|\n)*?\*/', Comment),
             include('numeric-values'),
             (r'[*+/-]', Operator),
-            (r'[,]', Punctuation),
+            (r',', Punctuation),
             (r'"(\\\\|\\"|[^"])*"', String.Double),
             (r"'(\\\\|\\'|[^'])*'", String.Single),
             (r'[a-zA-Z_-]\w*', Name),
@@ -373,8 +375,8 @@ class CssLexer(RegexLexer):
         'numeric-values': [
             (r'\#[a-zA-Z0-9]{1,6}', Number.Hex),
             (r'[+\-]?[0-9]*[.][0-9]+', Number.Float, 'numeric-end'),
-            (r'[+\-]?[0-9]+', Number.Integer, 'numeric-end'),            
-        ],        
+            (r'[+\-]?[0-9]+', Number.Integer, 'numeric-end'),
+        ],
         'numeric-end': [
             (words(_all_units, suffix=r'\b'), Keyword.Type),
             (r'%', Keyword.Type),
@@ -394,7 +396,7 @@ common_sass_tokens = {
             'behind', 'below', 'bidi-override', 'blink', 'block', 'bold', 'bolder', 'both',
             'capitalize', 'center-left', 'center-right', 'center', 'circle',
             'cjk-ideographic', 'close-quote', 'collapse', 'condensed', 'continuous',
-            'crop', 'crosshair', 'cross', 'cursive', 'dashed', 'decimal-leading-zero',
+            'crosshair', 'cross', 'cursive', 'dashed', 'decimal-leading-zero',
             'decimal', 'default', 'digits', 'disc', 'dotted', 'double', 'e-resize', 'embed',
             'extra-condensed', 'extra-expanded', 'expanded', 'fantasy', 'far-left',
             'far-right', 'faster', 'fast', 'fixed', 'georgian', 'groove', 'hebrew', 'help',
@@ -466,9 +468,9 @@ common_sass_tokens = {
     ],
 
     'string-single': [
-        (r"(\\.|#(?=[^\n{])|[^\n'#])+", String.Double),
+        (r"(\\.|#(?=[^\n{])|[^\n'#])+", String.Single),
         (r'#\{', String.Interpol, 'interpolation'),
-        (r"'", String.Double, '#pop'),
+        (r"'", String.Single, '#pop'),
     ],
 
     'string-url': [
@@ -609,7 +611,7 @@ class SassLexer(ExtendedRegexLexer):
             (r"\*/", Comment, '#pop'),
         ],
     }
-    for group, common in iteritems(common_sass_tokens):
+    for group, common in common_sass_tokens.items():
         tokens[group] = copy.copy(common)
     tokens['value'].append((r'\n', Text, 'root'))
     tokens['selector'].append((r'\n', Text, 'root'))
@@ -659,7 +661,7 @@ class ScssLexer(RegexLexer):
             (r"\*/", Comment, '#pop'),
         ],
     }
-    for group, common in iteritems(common_sass_tokens):
+    for group, common in common_sass_tokens.items():
         tokens[group] = copy.copy(common)
     tokens['value'].extend([(r'\n', Text), (r'[;{}]', Punctuation, '#pop')])
     tokens['selector'].extend([(r'\n', Text), (r'[;{}]', Punctuation, '#pop')])

data/vendor/pygments-main/pygments/lexers/d.py

@@ -5,7 +5,7 @@
 
     Lexers for D languages.
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -47,11 +47,16 @@ class DLexer(RegexLexer):
                 'module', 'new', 'nothrow', 'out', 'override', 'package', 'pragma',
                 'private', 'protected', 'public', 'pure', 'ref', 'return', 'scope',
                 'shared', 'static', 'struct', 'super', 'switch', 'synchronized',
-                'template', 'this', 'throw', 'try', 'typedef', 'typeid', 'typeof',
+                'template', 'this', 'throw', 'try', 'typeid', 'typeof',
                 'union', 'unittest', 'version', 'volatile', 'while', 'with',
                 '__gshared', '__traits', '__vector', '__parameters'),
                 suffix=r'\b'),
              Keyword),
+            (words((
+                # Removed in 2.072
+                'typedef', ),
+                suffix=r'\b'),
+             Keyword.Removed),
             (words((
                 'bool', 'byte', 'cdouble', 'cent', 'cfloat', 'char', 'creal',
                 'dchar', 'double', 'float', 'idouble', 'ifloat', 'int', 'ireal',
@@ -60,9 +65,9 @@ class DLexer(RegexLexer):
              Keyword.Type),
             (r'(false|true|null)\b', Keyword.Constant),
             (words((
-                '__FILE__', '__MODULE__', '__LINE__', '__FUNCTION__', '__PRETTY_FUNCTION__'
-                '', '__DATE__', '__EOF__', '__TIME__', '__TIMESTAMP__', '__VENDOR__',
-                '__VERSION__'), suffix=r'\b'),
+                '__FILE__', '__FILE_FULL_PATH__', '__MODULE__', '__LINE__', '__FUNCTION__',
+                '__PRETTY_FUNCTION__', '__DATE__', '__EOF__', '__TIME__', '__TIMESTAMP__',
+                '__VENDOR__', '__VERSION__'), suffix=r'\b'),
              Keyword.Pseudo),
             (r'macro\b', Keyword.Reserved),
             (r'(string|wstring|dstring|size_t|ptrdiff_t)\b', Name.Builtin),

data/vendor/pygments-main/pygments/lexers/dalvik.py

@@ -5,7 +5,7 @@
 
     Pygments lexers for Dalvik VM-related languages.
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexers/data.py

@@ -5,13 +5,13 @@
 
     Lexers for data file format.
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import re
 
-from pygments.lexer import RegexLexer, ExtendedRegexLexer, LexerContext, \
+from pygments.lexer import Lexer, RegexLexer, ExtendedRegexLexer, LexerContext, \
     include, bygroups, inherit
 from pygments.token import Text, Comment, Keyword, Name, String, Number, \
     Punctuation, Literal, Error
@@ -23,7 +23,7 @@ class YamlLexerContext(LexerContext):
     """Indentation context for the YAML lexer."""
 
     def __init__(self, *args, **kwds):
-        super(YamlLexerContext, self).__init__(*args, **kwds)
+        super().__init__(*args, **kwds)
         self.indent_stack = []
         self.indent = -1
         self.next_indent = 0
@@ -205,7 +205,7 @@ class YamlLexer(ExtendedRegexLexer):
              bygroups(Text, Number), 'ignored-line'),
         ],
 
-        # the %YAG directive
+        # the %TAG directive
         'tag-directive': [
             # a tag handle and the corresponding prefix
             (r'([ ]+)(!|![\w-]*!)'
@@ -218,7 +218,7 @@ class YamlLexer(ExtendedRegexLexer):
         'indentation': [
             # trailing whitespaces are ignored
             (r'[ ]*$', something(Text), '#pop:2'),
-            # whitespaces preceeding block collection indicators
+            # whitespaces preceding block collection indicators
             (r'[ ]+(?=[?:-](?:[ ]|$))', save_indent(Text)),
             # block collection indicators
             (r'[?:-](?=[ ]|$)', set_indent(Punctuation.Indicator)),
@@ -232,6 +232,9 @@ class YamlLexer(ExtendedRegexLexer):
             (r'[ ]*(?=#|$)', something(Text), '#pop'),
             # whitespaces separating tokens
             (r'[ ]+', Text),
+            # key with colon
+            (r'''([^#,:?\[\]{}"'\n]+)(:)(?=[ ]|$)''',
+             bygroups(Name.Tag, set_indent(Punctuation, implicit=True))),
             # tags, anchors and aliases,
             include('descriptors'),
             # block collections and scalars
@@ -250,7 +253,7 @@ class YamlLexer(ExtendedRegexLexer):
             (r'!<[\w#;/?:@&=+$,.!~*\'()\[\]%-]+>', Keyword.Type),
             # a tag in the form '!', '!suffix' or '!handle!suffix'
             (r'!(?:[\w-]+!)?'
-             r'[\w#;/?:@&=+$,.!~*\'()\[\]%-]+', Keyword.Type),
+             r'[\w#;/?:@&=+$,.!~*\'()\[\]%-]*', Keyword.Type),
             # an anchor
             (r'&[\w-]+', Name.Label),
             # an alias
@@ -308,6 +311,9 @@ class YamlLexer(ExtendedRegexLexer):
 
         # a flow mapping indicated by '{' and '}'
         'flow-mapping': [
+            # key with colon
+            (r'''([^,:?\[\]{}"'\n]+)(:)(?=[ ]|$)''',
+             bygroups(Name.Tag, Punctuation)),
             # include flow collection rules
             include('flow-collection'),
             # the closing indicator
@@ -427,10 +433,10 @@ class YamlLexer(ExtendedRegexLexer):
     def get_tokens_unprocessed(self, text=None, context=None):
         if context is None:
             context = YamlLexerContext(text, 0)
-        return super(YamlLexer, self).get_tokens_unprocessed(text, context)
+        return super().get_tokens_unprocessed(text, context)
 
 
-class JsonLexer(RegexLexer):
+class JsonLexer(Lexer):
     """
     For JSON data structures.
 
@@ -438,75 +444,192 @@ class JsonLexer(RegexLexer):
     """
 
     name = 'JSON'
-    aliases = ['json']
-    filenames = ['*.json']
-    mimetypes = ['application/json']
+    aliases = ['json', 'json-object']
+    filenames = ['*.json', 'Pipfile.lock']
+    mimetypes = ['application/json', 'application/json-object']
+
+    # No validation of integers, floats, or constants is done.
+    # As long as the characters are members of the following
+    # sets, the token will be considered valid. For example,
+    #
+    #     "--1--" is parsed as an integer
+    #     "1...eee" is parsed as a float
+    #     "trustful" is parsed as a constant
+    #
+    integers = set('-0123456789')
+    floats = set('.eE+')
+    constants = set('truefalsenull')  # true|false|null
+    hexadecimals = set('0123456789abcdefABCDEF')
+    punctuations = set('{}[],')
+    whitespaces = {'\u0020', '\u000a', '\u000d', '\u0009'}
+
+    def get_tokens_unprocessed(self, text):
+        """Parse JSON data."""
+
+        in_string = False
+        in_escape = False
+        in_unicode_escape = 0
+        in_whitespace = False
+        in_constant = False
+        in_number = False
+        in_float = False
+        in_punctuation = False
+
+        start = 0
+
+        # The queue is used to store data that may need to be tokenized
+        # differently based on what follows. In particular, JSON object
+        # keys are tokenized differently than string values, but cannot
+        # be distinguished until punctuation is encountered outside the
+        # string.
+        #
+        # A ":" character after the string indicates that the string is
+        # an object key; any other character indicates the string is a
+        # regular string value.
+        #
+        # The queue holds tuples that contain the following data:
+        #
+        #     (start_index, token_type, text)
+        #
+        # By default the token type of text in double quotes is
+        # String.Double. The token type will be replaced if a colon
+        # is encountered after the string closes.
+        #
+        queue = []
+
+        for stop, character in enumerate(text):
+            if in_string:
+                if in_unicode_escape:
+                    if character in self.hexadecimals:
+                        in_unicode_escape -= 1
+                        if not in_unicode_escape:
+                            in_escape = False
+                    else:
+                        in_unicode_escape = 0
+                        in_escape = False
+
+                elif in_escape:
+                    if character == 'u':
+                        in_unicode_escape = 4
+                    else:
+                        in_escape = False
+
+                elif character == '\\':
+                    in_escape = True
+
+                elif character == '"':
+                    queue.append((start, String.Double, text[start:stop + 1]))
+                    in_string = False
+                    in_escape = False
+                    in_unicode_escape = 0
+
+                continue
+
+            elif in_whitespace:
+                if character in self.whitespaces:
+                    continue
+
+                if queue:
+                    queue.append((start, Text, text[start:stop]))
+                else:
+                    yield start, Text, text[start:stop]
+                in_whitespace = False
+                # Fall through so the new character can be evaluated.
+
+            elif in_constant:
+                if character in self.constants:
+                    continue
+
+                yield start, Keyword.Constant, text[start:stop]
+                in_constant = False
+                # Fall through so the new character can be evaluated.
+
+            elif in_number:
+                if character in self.integers:
+                    continue
+                elif character in self.floats:
+                    in_float = True
+                    continue
+
+                if in_float:
+                    yield start, Number.Float, text[start:stop]
+                else:
+                    yield start, Number.Integer, text[start:stop]
+                in_number = False
+                in_float = False
+                # Fall through so the new character can be evaluated.
+
+            elif in_punctuation:
+                if character in self.punctuations:
+                    continue
+
+                yield start, Punctuation, text[start:stop]
+                in_punctuation = False
+                # Fall through so the new character can be evaluated.
+
+            start = stop
+
+            if character == '"':
+                in_string = True
+
+            elif character in self.whitespaces:
+                in_whitespace = True
+
+            elif character in {'f', 'n', 't'}:  # The first letters of true|false|null
+                # Exhaust the queue. Accept the existing token types.
+                yield from queue
+                queue.clear()
+
+                in_constant = True
+
+            elif character in self.integers:
+                # Exhaust the queue. Accept the existing token types.
+                yield from queue
+                queue.clear()
+
+                in_number = True
+
+            elif character == ':':
+                # Yield from the queue. Replace string token types.
+                for _start, _token, _text in queue:
+                    if _token is Text:
+                        yield _start, _token, _text
+                    elif _token is String.Double:
+                        yield _start, Name.Tag, _text
+                    else:
+                        yield _start, Error, _text
+                queue.clear()
+
+                in_punctuation = True
+
+            elif character in self.punctuations:
+                # Exhaust the queue. Accept the existing token types.
+                yield from queue
+                queue.clear()
+
+                in_punctuation = True
 
-    flags = re.DOTALL
-
-    # integer part of a number
-    int_part = r'-?(0|[1-9]\d*)'
-
-    # fractional part of a number
-    frac_part = r'\.\d+'
-
-    # exponential part of a number
-    exp_part = r'[eE](\+|-)?\d+'
-
-    tokens = {
-        'whitespace': [
-            (r'\s+', Text),
-        ],
-
-        # represents a simple terminal value
-        'simplevalue': [
-            (r'(true|false|null)\b', Keyword.Constant),
-            (('%(int_part)s(%(frac_part)s%(exp_part)s|'
-              '%(exp_part)s|%(frac_part)s)') % vars(),
-             Number.Float),
-            (int_part, Number.Integer),
-            (r'"(\\\\|\\"|[^"])*"', String.Double),
-        ],
-
-
-        # the right hand side of an object, after the attribute name
-        'objectattribute': [
-            include('value'),
-            (r':', Punctuation),
-            # comma terminates the attribute but expects more
-            (r',', Punctuation, '#pop'),
-            # a closing bracket terminates the entire object, so pop twice
-            (r'\}', Punctuation, '#pop:2'),
-        ],
-
-        # a json object - { attr, attr, ... }
-        'objectvalue': [
-            include('whitespace'),
-            (r'"(\\\\|\\"|[^"])*"', Name.Tag, 'objectattribute'),
-            (r'\}', Punctuation, '#pop'),
-        ],
-
-        # json array - [ value, value, ... }
-        'arrayvalue': [
-            include('whitespace'),
-            include('value'),
-            (r',', Punctuation),
-            (r'\]', Punctuation, '#pop'),
-        ],
-
-        # a json value - either a simple value or a complex value (object or array)
-        'value': [
-            include('whitespace'),
-            include('simplevalue'),
-            (r'\{', Punctuation, 'objectvalue'),
-            (r'\[', Punctuation, 'arrayvalue'),
-        ],
-
-        # the root of a json document whould be a value
-        'root': [
-            include('value'),
-        ],
-    }
+            else:
+                # Exhaust the queue. Accept the existing token types.
+                yield from queue
+                queue.clear()
+
+                yield start, Error, character
+
+        # Yield any remaining text.
+        yield from queue
+        if in_string:
+            yield start, Error, text[start:]
+        elif in_float:
+            yield start, Number.Float, text[start:]
+        elif in_number:
+            yield start, Number.Integer, text[start:]
+        elif in_constant:
+            yield start, Keyword.Constant, text[start:]
+        elif in_whitespace:
+            yield start, Text, text[start:]
+        elif in_punctuation:
+            yield start, Punctuation, text[start:]
 
 
 class JsonBareObjectLexer(JsonLexer):
@@ -514,28 +637,21 @@ class JsonBareObjectLexer(JsonLexer):
     For JSON data structures (with missing object curly braces).
 
     .. versionadded:: 2.2
+
+    .. deprecated:: 2.8.0
+
+       Behaves the same as `JsonLexer` now.
     """
 
     name = 'JSONBareObject'
-    aliases = ['json-object']
+    aliases = []
     filenames = []
-    mimetypes = ['application/json-object']
-
-    tokens = {
-        'root': [
-            (r'\}', Error),
-            include('objectvalue'),
-        ],
-        'objectattribute': [
-            (r'\}', Error),
-            inherit,
-        ],
-    }
+    mimetypes = []
 
 
 class JsonLdLexer(JsonLexer):
     """
-    For `JSON-LD <http://json-ld.org/>`_ linked data.
+    For `JSON-LD <https://json-ld.org/>`_ linked data.
 
     .. versionadded:: 2.0
     """
@@ -545,11 +661,38 @@ class JsonLdLexer(JsonLexer):
     filenames = ['*.jsonld']
     mimetypes = ['application/ld+json']
 
-    tokens = {
-        'objectvalue': [
-            (r'"@(context|id|value|language|type|container|list|set|'
-             r'reverse|index|base|vocab|graph)"', Name.Decorator,
-             'objectattribute'),
-            inherit,
-        ],
+    json_ld_keywords = {
+        '"@%s"' % keyword
+        for keyword in (
+            'base',
+            'container',
+            'context',
+            'direction',
+            'graph',
+            'id',
+            'import',
+            'included',
+            'index',
+            'json',
+            'language',
+            'list',
+            'nest',
+            'none',
+            'prefix',
+            'propagate',
+            'protected',
+            'reverse',
+            'set',
+            'type',
+            'value',
+            'version',
+            'vocab',
+        )
     }
+
+    def get_tokens_unprocessed(self, text):
+        for start, token, value in super(JsonLdLexer, self).get_tokens_unprocessed(text):
+            if token is Name.Tag and value in self.json_ld_keywords:
+                yield start, Name.Decorator, value
+            else:
+                yield start, token, value