pygments.rb 1.2.0 → 2.1.0

data/vendor/pygments-main/pygments/lexer.py

@@ -1,16 +1,13 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexer
     ~~~~~~~~~~~~~~
 
     Base lexer classes.
 
-    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-from __future__ import print_function
-
 import re
 import sys
 import time
@@ -19,7 +16,7 @@ from pygments.filter import apply_filters, Filter
 from pygments.filters import get_filter_by_name
 from pygments.token import Error, Text, Other, _TokenType
 from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
-    make_analysator, text_type, add_metaclass, iteritems, Future, guess_decode
+    make_analysator, Future, guess_decode
 from pygments.regexopt import regex_opt
 
 __all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer',
@@ -48,8 +45,7 @@ class LexerMeta(type):
         return type.__new__(mcs, name, bases, d)
 
 
-@add_metaclass(LexerMeta)
-class Lexer(object):
+class Lexer(metaclass=LexerMeta):
     """
     Lexer for a specific language.
 
@@ -145,16 +141,16 @@ class Lexer(object):
         Also preprocess the text, i.e. expand tabs and strip it if
         wanted and applies registered filters.
         """
-        if not isinstance(text, text_type):
+        if not isinstance(text, str):
             if self.encoding == 'guess':
                 text, _ = guess_decode(text)
             elif self.encoding == 'chardet':
                 try:
                     import chardet
-                except ImportError:
+                except ImportError as e:
                     raise ImportError('To enable chardet encoding guessing, '
                                       'please install the chardet library '
-                                      'from http://chardet.feedparser.org/')
+                                      'from http://chardet.feedparser.org/') from e
                 # check for BOM first
                 decoded = None
                 for bom, encoding in _encoding_map:
@@ -169,11 +165,11 @@ class Lexer(object):
                 text = decoded
             else:
                 text = text.decode(self.encoding)
-                if text.startswith(u'\ufeff'):
-                    text = text[len(u'\ufeff'):]
+                if text.startswith('\ufeff'):
+                    text = text[len('\ufeff'):]
         else:
-            if text.startswith(u'\ufeff'):
-                text = text[len(u'\ufeff'):]
+            if text.startswith('\ufeff'):
+                text = text[len('\ufeff'):]
 
         # text now *is* a unicode string
         text = text.replace('\r\n', '\n')
@@ -252,7 +248,7 @@ class include(str):  # pylint: disable=invalid-name
     pass
 
 
-class _inherit(object):
+class _inherit:
     """
     Indicates the a state should inherit from its superclass.
     """
@@ -275,7 +271,7 @@ class combined(tuple):  # pylint: disable=invalid-name
         pass
 
 
-class _PseudoMatch(object):
+class _PseudoMatch:
     """
     A pseudo match object constructed from a string.
     """
@@ -328,11 +324,12 @@ def bygroups(*args):
     return callback
 
 
-class _This(object):
+class _This:
     """
     Special singleton used for indicating the caller class.
     Used by ``using``.
     """
+
 this = _This()
 
 
@@ -498,7 +495,7 @@ class RegexLexerMeta(LexerMeta):
                 rex = cls._process_regex(tdef[0], rflags, state)
             except Exception as err:
                 raise ValueError("uncompilable regex %r in state %r of %r: %s" %
-                                 (tdef[0], state, cls, err))
+                                 (tdef[0], state, cls, err)) from err
 
             token = cls._process_token(tdef[1])
 
@@ -536,7 +533,7 @@ class RegexLexerMeta(LexerMeta):
         for c in cls.__mro__:
             toks = c.__dict__.get('tokens', {})
 
-            for state, items in iteritems(toks):
+            for state, items in toks.items():
                 curitems = tokens.get(state)
                 if curitems is None:
                     # N.b. because this is assigned by reference, sufficiently
@@ -582,8 +579,7 @@ class RegexLexerMeta(LexerMeta):
         return type.__call__(cls, *args, **kwds)
 
 
-@add_metaclass(RegexLexerMeta)
-class RegexLexer(Lexer):
+class RegexLexer(Lexer, metaclass=RegexLexerMeta):
     """
     Base for simple stateful regular expression-based lexers.
     Simplifies the lexing process so that you need only
@@ -631,22 +627,27 @@ class RegexLexer(Lexer):
                         if type(action) is _TokenType:
                             yield pos, action, m.group()
                         else:
-                            for item in action(self, m):
-                                yield item
+                            yield from action(self, m)
                     pos = m.end()
                     if new_state is not None:
                         # state transition
                         if isinstance(new_state, tuple):
                             for state in new_state:
                                 if state == '#pop':
-                                    statestack.pop()
+                                    if len(statestack) > 1:
+                                        statestack.pop()
                                 elif state == '#push':
                                     statestack.append(statestack[-1])
                                 else:
                                     statestack.append(state)
                         elif isinstance(new_state, int):
-                            # pop
-                            del statestack[new_state:]
+                            # pop, but keep at least one state on the stack
+                            # (random code leading to unexpected pops should
+                            # not allow exceptions)
+                            if abs(new_state) >= len(statestack):
+                                del statestack[1:]
+                            else:
+                                del statestack[new_state:]
                         elif new_state == '#push':
                             statestack.append(statestack[-1])
                         else:
@@ -661,7 +662,7 @@ class RegexLexer(Lexer):
                         # at EOL, reset state to "root"
                         statestack = ['root']
                         statetokens = tokendefs['root']
-                        yield pos, Text, u'\n'
+                        yield pos, Text, '\n'
                         pos += 1
                         continue
                     yield pos, Error, text[pos]
@@ -670,7 +671,7 @@ class RegexLexer(Lexer):
                     break
 
 
-class LexerContext(object):
+class LexerContext:
     """
     A helper object that holds lexer position data.
     """
@@ -713,8 +714,7 @@ class ExtendedRegexLexer(RegexLexer):
                             yield ctx.pos, action, m.group()
                             ctx.pos = m.end()
                         else:
-                            for item in action(self, m, ctx):
-                                yield item
+                            yield from action(self, m, ctx)
                             if not new_state:
                                 # altered the state stack?
                                 statetokens = tokendefs[ctx.stack[-1]]
@@ -724,14 +724,18 @@ class ExtendedRegexLexer(RegexLexer):
                         if isinstance(new_state, tuple):
                             for state in new_state:
                                 if state == '#pop':
-                                    ctx.stack.pop()
+                                    if len(ctx.stack) > 1:
+                                        ctx.stack.pop()
                                 elif state == '#push':
                                     ctx.stack.append(ctx.stack[-1])
                                 else:
                                     ctx.stack.append(state)
                         elif isinstance(new_state, int):
-                            # pop
-                            del ctx.stack[new_state:]
+                            # see RegexLexer for why this check is made
+                            if abs(new_state) >= len(ctx.stack):
+                                del ctx.state[1:]
+                            else:
+                                del ctx.stack[new_state:]
                         elif new_state == '#push':
                             ctx.stack.append(ctx.stack[-1])
                         else:
@@ -746,7 +750,7 @@ class ExtendedRegexLexer(RegexLexer):
                         # at EOL, reset state to "root"
                         ctx.stack = ['root']
                         statetokens = tokendefs['root']
-                        yield ctx.pos, Text, u'\n'
+                        yield ctx.pos, Text, '\n'
                         ctx.pos += 1
                         continue
                     yield ctx.pos, Error, text[ctx.pos]
@@ -774,8 +778,7 @@ def do_insertions(insertions, tokens):
         index, itokens = next(insertions)
     except StopIteration:
         # no insertions
-        for item in tokens:
-            yield item
+        yield from tokens
         return
 
     realpos = None
@@ -790,8 +793,9 @@ def do_insertions(insertions, tokens):
         oldi = 0
         while insleft and i + len(v) >= index:
             tmpval = v[oldi:index - i]
-            yield realpos, t, tmpval
-            realpos += len(tmpval)
+            if tmpval:
+                yield realpos, t, tmpval
+                realpos += len(tmpval)
             for it_index, it_token, it_value in itokens:
                 yield realpos, it_token, it_value
                 realpos += len(it_value)
@@ -801,8 +805,9 @@ def do_insertions(insertions, tokens):
             except StopIteration:
                 insleft = False
                 break  # not strictly necessary
-        yield realpos, t, v[oldi:]
-        realpos += len(v) - oldi
+        if oldi < len(v):
+            yield realpos, t, v[oldi:]
+            realpos += len(v) - oldi
 
     # leftover tokens
     while insleft:
@@ -840,8 +845,7 @@ class ProfilingRegexLexerMeta(RegexLexerMeta):
         return match_func
 
 
-@add_metaclass(ProfilingRegexLexerMeta)
-class ProfilingRegexLexer(RegexLexer):
+class ProfilingRegexLexer(RegexLexer, metaclass=ProfilingRegexLexerMeta):
     """Drop-in replacement for RegexLexer that does profiling of its regexes."""
 
     _prof_data = []
@@ -850,8 +854,7 @@ class ProfilingRegexLexer(RegexLexer):
     def get_tokens_unprocessed(self, text, stack=('root',)):
         # this needs to be a stack, since using(this) will produce nested calls
         self.__class__._prof_data.append({})
-        for tok in RegexLexer.get_tokens_unprocessed(self, text, stack):
-            yield tok
+        yield from RegexLexer.get_tokens_unprocessed(self, text, stack)
         rawdata = self.__class__._prof_data.pop()
         data = sorted(((s, repr(r).strip('u\'').replace('\\\\', '\\')[:65],
                         n, 1000 * t, 1000 * t / n)

data/vendor/pygments-main/pygments/lexers/__init__.py

@@ -1,11 +1,10 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexers
     ~~~~~~~~~~~~~~~
 
     Pygments lexers.
 
-    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -18,11 +17,15 @@ from os.path import basename
 from pygments.lexers._mapping import LEXERS
 from pygments.modeline import get_filetype_from_buffer
 from pygments.plugin import find_plugin_lexers
-from pygments.util import ClassNotFound, itervalues, guess_decode
+from pygments.util import ClassNotFound, guess_decode
 
+COMPAT = {
+    'Python3Lexer': 'PythonLexer',
+    'Python3TracebackLexer': 'PythonTracebackLexer',
+}
 
 __all__ = ['get_lexer_by_name', 'get_lexer_for_filename', 'find_lexer_class',
-           'guess_lexer', 'load_lexer_from_file'] + list(LEXERS)
+           'guess_lexer', 'load_lexer_from_file'] + list(LEXERS) + list(COMPAT)
 
 _lexer_cache = {}
 _pattern_cache = {}
@@ -48,7 +51,7 @@ def get_all_lexers():
     """Return a generator of tuples in the form ``(name, aliases,
     filenames, mimetypes)`` of all know lexers.
     """
-    for item in itervalues(LEXERS):
+    for item in LEXERS.values():
         yield item[1:]
     for lexer in find_plugin_lexers():
         yield lexer.name, lexer.aliases, lexer.filenames, lexer.mimetypes
@@ -62,7 +65,7 @@ def find_lexer_class(name):
     if name in _lexer_cache:
         return _lexer_cache[name]
     # lookup builtin lexers
-    for module_name, lname, aliases, _, _ in itervalues(LEXERS):
+    for module_name, lname, aliases, _, _ in LEXERS.values():
         if name == lname:
             _load_lexers(module_name)
             return _lexer_cache[name]
@@ -82,7 +85,7 @@ def find_lexer_class_by_name(_alias):
     if not _alias:
         raise ClassNotFound('no lexer for alias %r found' % _alias)
     # lookup builtin lexers
-    for module_name, name, aliases, _, _ in itervalues(LEXERS):
+    for module_name, name, aliases, _, _ in LEXERS.values():
         if _alias.lower() in aliases:
             if name not in _lexer_cache:
                 _load_lexers(module_name)
@@ -103,7 +106,7 @@ def get_lexer_by_name(_alias, **options):
         raise ClassNotFound('no lexer for alias %r found' % _alias)
 
     # lookup builtin lexers
-    for module_name, name, aliases, _, _ in itervalues(LEXERS):
+    for module_name, name, aliases, _, _ in LEXERS.values():
         if _alias.lower() in aliases:
             if name not in _lexer_cache:
                 _load_lexers(module_name)
@@ -133,7 +136,8 @@ def load_lexer_from_file(filename, lexername="CustomLexer", **options):
     try:
         # This empty dict will contain the namespace for the exec'd file
         custom_namespace = {}
-        exec(open(filename, 'rb').read(), custom_namespace)
+        with open(filename, 'rb') as f:
+            exec(f.read(), custom_namespace)
         # Retrieve the class `lexername` from that namespace
         if lexername not in custom_namespace:
             raise ClassNotFound('no valid %s class found in %s' %
@@ -141,9 +145,9 @@ def load_lexer_from_file(filename, lexername="CustomLexer", **options):
         lexer_class = custom_namespace[lexername]
         # And finally instantiate it with the options
         return lexer_class(**options)
-    except IOError as err:
-        raise ClassNotFound('cannot read %s' % filename)
-    except ClassNotFound as err:
+    except OSError as err:
+        raise ClassNotFound('cannot read %s: %s' % (filename, err))
+    except ClassNotFound:
         raise
     except Exception as err:
         raise ClassNotFound('error when loading custom lexer: %s' % err)
@@ -159,7 +163,7 @@ def find_lexer_class_for_filename(_fn, code=None):
     """
     matches = []
     fn = basename(_fn)
-    for modname, name, _, filenames, _ in itervalues(LEXERS):
+    for modname, name, _, filenames, _ in LEXERS.values():
         for filename in filenames:
             if _fn_matches(fn, filename):
                 if name not in _lexer_cache:
@@ -170,7 +174,7 @@ def find_lexer_class_for_filename(_fn, code=None):
             if _fn_matches(fn, filename):
                 matches.append((cls, filename))
 
-    if sys.version_info > (3,) and isinstance(code, bytes):
+    if isinstance(code, bytes):
         # decode it, since all analyse_text functions expect unicode
         code = guess_decode(code)
 
@@ -211,7 +215,7 @@ def get_lexer_for_mimetype(_mime, **options):
 
     Raises ClassNotFound if not found.
     """
-    for modname, name, _, _, mimetypes in itervalues(LEXERS):
+    for modname, name, _, _, mimetypes in LEXERS.values():
         if _mime in mimetypes:
             if name not in _lexer_cache:
                 _load_lexers(modname)
@@ -230,8 +234,7 @@ def _iter_lexerclasses(plugins=True):
             _load_lexers(module_name)
         yield _lexer_cache[name]
     if plugins:
-        for lexer in find_plugin_lexers():
-            yield lexer
+        yield from find_plugin_lexers()
 
 
 def guess_lexer_for_filename(_fn, _text, **options):
@@ -288,6 +291,13 @@ def guess_lexer_for_filename(_fn, _text, **options):
 def guess_lexer(_text, **options):
     """Guess a lexer by strong distinctions in the text (eg, shebang)."""
 
+    if not isinstance(_text, str):
+        inencoding = options.get('inencoding', options.get('encoding'))
+        if inencoding:
+            _text = _text.decode(inencoding or 'utf8')
+        else:
+            _text, _ = guess_decode(_text)
+
     # try to get a vim modeline first
     ft = get_filetype_from_buffer(_text)
 
@@ -319,6 +329,8 @@ class _automodule(types.ModuleType):
             cls = _lexer_cache[info[1]]
             setattr(self, name, cls)
             return cls
+        if name in COMPAT:
+            return getattr(self, COMPAT[name])
         raise AttributeError(name)
 
 

data/vendor/pygments-main/pygments/lexers/_asy_builtins.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexers._asy_builtins
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -10,11 +9,11 @@
     TODO: perl/python script in Asymptote SVN similar to asy-list.pl but only
     for function and variable names.
 
-    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-ASYFUNCNAME = set((
+ASYFUNCNAME = {
     'AND',
     'Arc',
     'ArcArrow',
@@ -1038,9 +1037,9 @@ ASYFUNCNAME = set((
     'ztick',
     'ztick3',
     'ztrans'
-))
+}
 
-ASYVARNAME = set((
+ASYVARNAME = {
     'AliceBlue',
     'Align',
     'Allow',
@@ -1642,4 +1641,4 @@ ASYVARNAME = set((
     'ylabelwidth',
     'zerotickfuzz',
     'zerowinding'
-))
+}

data/vendor/pygments-main/pygments/lexers/_cl_builtins.py

@@ -1,15 +1,14 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexers._cl_builtins
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     ANSI Common Lisp builtins.
 
-    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-BUILTIN_FUNCTIONS = set((  # 638 functions
+BUILTIN_FUNCTIONS = {  # 638 functions
     '<', '<=', '=', '>', '>=', '-', '/', '/=', '*', '+', '1-', '1+',
     'abort', 'abs', 'acons', 'acos', 'acosh', 'add-method', 'adjoin',
     'adjustable-array-p', 'adjust-array', 'allocate-instance',
@@ -157,17 +156,17 @@ BUILTIN_FUNCTIONS = set((  # 638 functions
     'wild-pathname-p', 'write', 'write-byte', 'write-char', 'write-line',
     'write-sequence', 'write-string', 'write-to-string', 'yes-or-no-p',
     'y-or-n-p', 'zerop',
-))
+}
 
-SPECIAL_FORMS = set((
+SPECIAL_FORMS = {
     'block', 'catch', 'declare', 'eval-when', 'flet', 'function', 'go', 'if',
     'labels', 'lambda', 'let', 'let*', 'load-time-value', 'locally', 'macrolet',
     'multiple-value-call', 'multiple-value-prog1', 'progn', 'progv', 'quote',
     'return-from', 'setq', 'symbol-macrolet', 'tagbody', 'the', 'throw',
     'unwind-protect',
-))
+}
 
-MACROS = set((
+MACROS = {
     'and', 'assert', 'call-method', 'case', 'ccase', 'check-type', 'cond',
     'ctypecase', 'decf', 'declaim', 'defclass', 'defconstant', 'defgeneric',
     'define-compiler-macro', 'define-condition', 'define-method-combination',
@@ -188,19 +187,19 @@ MACROS = set((
     'with-input-from-string', 'with-open-file', 'with-open-stream',
     'with-output-to-string', 'with-package-iterator', 'with-simple-restart',
     'with-slots', 'with-standard-io-syntax',
-))
+}
 
-LAMBDA_LIST_KEYWORDS = set((
+LAMBDA_LIST_KEYWORDS = {
     '&allow-other-keys', '&aux', '&body', '&environment', '&key', '&optional',
     '&rest', '&whole',
-))
+}
 
-DECLARATIONS = set((
+DECLARATIONS = {
     'dynamic-extent', 'ignore', 'optimize', 'ftype', 'inline', 'special',
     'ignorable', 'notinline', 'type',
-))
+}
 
-BUILTIN_TYPES = set((
+BUILTIN_TYPES = {
     'atom', 'boolean', 'base-char', 'base-string', 'bignum', 'bit',
     'compiled-function', 'extended-char', 'fixnum', 'keyword', 'nil',
     'signed-byte', 'short-float', 'single-float', 'double-float', 'long-float',
@@ -217,9 +216,9 @@ BUILTIN_TYPES = set((
     'simple-type-error', 'simple-warning', 'stream-error', 'storage-condition',
     'style-warning', 'type-error', 'unbound-variable', 'unbound-slot',
     'undefined-function', 'warning',
-))
+}
 
-BUILTIN_CLASSES = set((
+BUILTIN_CLASSES = {
     'array', 'broadcast-stream', 'bit-vector', 'built-in-class', 'character',
     'class', 'complex', 'concatenated-stream', 'cons', 'echo-stream',
     'file-stream', 'float', 'function', 'generic-function', 'hash-table',
@@ -229,4 +228,4 @@ BUILTIN_CLASSES = set((
     'standard-generic-function', 'standard-method', 'standard-object',
     'string-stream', 'stream', 'string', 'structure-class', 'structure-object',
     'symbol', 'synonym-stream', 't', 'two-way-stream', 'vector',
-))
+}