pygments.rb 1.2.0 → 2.1.0

data/vendor/pygments-main/requirements.txt

@@ -1,5 +0,0 @@
-coverage
-nose
-pyflakes
-pylint
-tox

data/vendor/pygments-main/scripts/check_sources.py

@@ -1,211 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-    Checker for file headers
-    ~~~~~~~~~~~~~~~~~~~~~~~~
-
-    Make sure each Python file has a correct file header
-    including copyright and license information.
-
-    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-from __future__ import print_function
-
-import io
-import os
-import re
-import sys
-import getopt
-from os.path import join, splitext, abspath
-
-
-checkers = {}
-
-
-def checker(*suffixes, **kwds):
-    only_pkg = kwds.pop('only_pkg', False)
-
-    def deco(func):
-        for suffix in suffixes:
-            checkers.setdefault(suffix, []).append(func)
-        func.only_pkg = only_pkg
-        return func
-    return deco
-
-
-name_mail_re = r'[\w ]+(<.*?>)?'
-copyright_re = re.compile(r'^    :copyright: Copyright 2006-2017 by '
-                          r'the Pygments team, see AUTHORS\.$', re.UNICODE)
-copyright_2_re = re.compile(r'^                %s(, %s)*[,.]$' %
-                            (name_mail_re, name_mail_re), re.UNICODE)
-is_const_re  = re.compile(r'if.*?==\s+(None|False|True)\b')
-
-misspellings = ["developement", "adress", "verificate",  # ALLOW-MISSPELLING
-                "informations", "unlexer"]               # ALLOW-MISSPELLING
-
-
-@checker('.py')
-def check_syntax(fn, lines):
-    if '#!/' in lines[0]:
-        lines = lines[1:]
-    if 'coding:' in lines[0]:
-        lines = lines[1:]
-    try:
-        compile('\n'.join(lines), fn, "exec")
-    except SyntaxError as err:
-        yield 0, "not compilable: %s" % err
-
-
-@checker('.py')
-def check_style_and_encoding(fn, lines):
-    for lno, line in enumerate(lines):
-        if len(line) > 110:
-            yield lno+1, "line too long"
-        if is_const_re.search(line):
-            yield lno+1, 'using == None/True/False'
-
-
-@checker('.py', only_pkg=True)
-def check_fileheader(fn, lines):
-    # line number correction
-    c = 1
-    if lines[0:1] == ['#!/usr/bin/env python']:
-        lines = lines[1:]
-        c = 2
-
-    llist = []
-    docopen = False
-    for lno, l in enumerate(lines):
-        llist.append(l)
-        if lno == 0:
-            if l != '# -*- coding: utf-8 -*-':
-                yield 1, "missing coding declaration"
-        elif lno == 1:
-            if l != '"""' and l != 'r"""':
-                yield 2, 'missing docstring begin (""")'
-            else:
-                docopen = True
-        elif docopen:
-            if l == '"""':
-                # end of docstring
-                if lno <= 4:
-                    yield lno+c, "missing module name in docstring"
-                break
-
-            if l != "" and l[:4] != '    ' and docopen:
-                yield lno+c, "missing correct docstring indentation"
-
-            if lno == 2:
-                # if not in package, don't check the module name
-                modname = fn[:-3].replace('/', '.').replace('.__init__', '')
-                while modname:
-                    if l.lower()[4:] == modname:
-                        break
-                    modname = '.'.join(modname.split('.')[1:])
-                else:
-                    yield 3, "wrong module name in docstring heading"
-                modnamelen = len(l.strip())
-            elif lno == 3:
-                if l.strip() != modnamelen * "~":
-                    yield 4, "wrong module name underline, should be ~~~...~"
-
-    else:
-        yield 0, "missing end and/or start of docstring..."
-
-    # check for copyright and license fields
-    license = llist[-2:-1]
-    if license != ["    :license: BSD, see LICENSE for details."]:
-        yield 0, "no correct license info"
-
-    ci = -3
-    copyright = llist[ci:ci+1]
-    while copyright and copyright_2_re.match(copyright[0]):
-        ci -= 1
-        copyright = llist[ci:ci+1]
-    if not copyright or not copyright_re.match(copyright[0]):
-        yield 0, "no correct copyright info"
-
-
-def main(argv):
-    try:
-        gopts, args = getopt.getopt(argv[1:], "vi:")
-    except getopt.GetoptError:
-        print("Usage: %s [-v] [-i ignorepath]* [path]" % argv[0])
-        return 2
-    opts = {}
-    for opt, val in gopts:
-        if opt == '-i':
-            val = abspath(val)
-        opts.setdefault(opt, []).append(val)
-
-    if len(args) == 0:
-        path = '.'
-    elif len(args) == 1:
-        path = args[0]
-    else:
-        print("Usage: %s [-v] [-i ignorepath]* [path]" % argv[0])
-        return 2
-
-    verbose = '-v' in opts
-
-    num = 0
-    out = io.StringIO()
-
-    # TODO: replace os.walk run with iteration over output of
-    #       `svn list -R`.
-
-    for root, dirs, files in os.walk(path):
-        if '.hg' in dirs:
-            dirs.remove('.hg')
-        if 'examplefiles' in dirs:
-            dirs.remove('examplefiles')
-        if '-i' in opts and abspath(root) in opts['-i']:
-            del dirs[:]
-            continue
-        # XXX: awkward: for the Makefile call: don't check non-package
-        #      files for file headers
-        in_pygments_pkg = root.startswith('./pygments')
-        for fn in files:
-
-            fn = join(root, fn)
-            if fn[:2] == './':
-                fn = fn[2:]
-
-            if '-i' in opts and abspath(fn) in opts['-i']:
-                continue
-
-            ext = splitext(fn)[1]
-            checkerlist = checkers.get(ext, None)
-            if not checkerlist:
-                continue
-
-            if verbose:
-                print("Checking %s..." % fn)
-
-            try:
-                lines = open(fn, 'rb').read().decode('utf-8').splitlines()
-            except (IOError, OSError) as err:
-                print("%s: cannot open: %s" % (fn, err))
-                num += 1
-                continue
-
-            for checker in checkerlist:
-                if not in_pygments_pkg and checker.only_pkg:
-                    continue
-                for lno, msg in checker(fn, lines):
-                    print(u"%s:%d: %s" % (fn, lno, msg), file=out)
-                    num += 1
-    if verbose:
-        print()
-    if num == 0:
-        print("No errors found.")
-    else:
-        print(out.getvalue().rstrip('\n'))
-        print("%d error%s found." % (num, num > 1 and "s" or ""))
-    return int(num > 0)
-
-
-if __name__ == '__main__':
-    sys.exit(main(sys.argv))

data/vendor/pygments-main/scripts/debug_lexer.py

@@ -1,246 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-"""
-    Lexing error finder
-    ~~~~~~~~~~~~~~~~~~~
-
-    For the source files given on the command line, display
-    the text where Error tokens are being generated, along
-    with some context.
-
-    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-from __future__ import print_function
-
-import os
-import sys
-
-# always prefer Pygments from source if exists
-srcpath = os.path.join(os.path.dirname(__file__), '..')
-if os.path.isdir(os.path.join(srcpath, 'pygments')):
-    sys.path.insert(0, srcpath)
-
-
-from pygments.lexer import RegexLexer, ExtendedRegexLexer, LexerContext, \
-    ProfilingRegexLexer, ProfilingRegexLexerMeta
-from pygments.lexers import get_lexer_by_name, find_lexer_class, \
-    find_lexer_class_for_filename
-from pygments.token import Error, Text, _TokenType
-from pygments.cmdline import _parse_options
-
-
-class DebuggingRegexLexer(ExtendedRegexLexer):
-    """Make the state stack, position and current match instance attributes."""
-
-    def get_tokens_unprocessed(self, text, stack=('root',)):
-        """
-        Split ``text`` into (tokentype, text) pairs.
-
-        ``stack`` is the inital stack (default: ``['root']``)
-        """
-        tokendefs = self._tokens
-        self.ctx = ctx = LexerContext(text, 0)
-        ctx.stack = list(stack)
-        statetokens = tokendefs[ctx.stack[-1]]
-        while 1:
-            for rexmatch, action, new_state in statetokens:
-                self.m = m = rexmatch(text, ctx.pos, ctx.end)
-                if m:
-                    if action is not None:
-                        if type(action) is _TokenType:
-                            yield ctx.pos, action, m.group()
-                            ctx.pos = m.end()
-                        else:
-                            if not isinstance(self, ExtendedRegexLexer):
-                                for item in action(self, m):
-                                    yield item
-                                ctx.pos = m.end()
-                            else:
-                                for item in action(self, m, ctx):
-                                    yield item
-                                if not new_state:
-                                    # altered the state stack?
-                                    statetokens = tokendefs[ctx.stack[-1]]
-                    if new_state is not None:
-                        # state transition
-                        if isinstance(new_state, tuple):
-                            for state in new_state:
-                                if state == '#pop':
-                                    ctx.stack.pop()
-                                elif state == '#push':
-                                    ctx.stack.append(ctx.stack[-1])
-                                else:
-                                    ctx.stack.append(state)
-                        elif isinstance(new_state, int):
-                            # pop
-                            del ctx.stack[new_state:]
-                        elif new_state == '#push':
-                            ctx.stack.append(ctx.stack[-1])
-                        else:
-                            assert False, 'wrong state def: %r' % new_state
-                        statetokens = tokendefs[ctx.stack[-1]]
-                    break
-            else:
-                try:
-                    if ctx.pos >= ctx.end:
-                        break
-                    if text[ctx.pos] == '\n':
-                        # at EOL, reset state to 'root'
-                        ctx.stack = ['root']
-                        statetokens = tokendefs['root']
-                        yield ctx.pos, Text, u'\n'
-                        ctx.pos += 1
-                        continue
-                    yield ctx.pos, Error, text[ctx.pos]
-                    ctx.pos += 1
-                except IndexError:
-                    break
-
-
-def main(fn, lexer=None, options={}):
-    if lexer is not None:
-        lxcls = get_lexer_by_name(lexer).__class__
-    else:
-        lxcls = find_lexer_class_for_filename(os.path.basename(fn))
-        if lxcls is None:
-            name, rest = fn.split('_', 1)
-            lxcls = find_lexer_class(name)
-            if lxcls is None:
-                raise AssertionError('no lexer found for file %r' % fn)
-        print('Using lexer: %s (%s.%s)' % (lxcls.name, lxcls.__module__,
-                                           lxcls.__name__))
-    debug_lexer = False
-    # if profile:
-    #     # does not work for e.g. ExtendedRegexLexers
-    #     if lxcls.__bases__ == (RegexLexer,):
-    #         # yes we can!  (change the metaclass)
-    #         lxcls.__class__ = ProfilingRegexLexerMeta
-    #         lxcls.__bases__ = (ProfilingRegexLexer,)
-    #         lxcls._prof_sort_index = profsort
-    # else:
-    #     if lxcls.__bases__ == (RegexLexer,):
-    #         lxcls.__bases__ = (DebuggingRegexLexer,)
-    #         debug_lexer = True
-    #     elif lxcls.__bases__ == (DebuggingRegexLexer,):
-    #         # already debugged before
-    #         debug_lexer = True
-    #     else:
-    #         # HACK: ExtendedRegexLexer subclasses will only partially work here.
-    #         lxcls.__bases__ = (DebuggingRegexLexer,)
-    #         debug_lexer = True
-
-    lx = lxcls(**options)
-    lno = 1
-    if fn == '-':
-        text = sys.stdin.read()
-    else:
-        with open(fn, 'rb') as fp:
-            text = fp.read().decode('utf-8')
-    text = text.strip('\n') + '\n'
-    tokens = []
-    states = []
-
-    def show_token(tok, state):
-        reprs = list(map(repr, tok))
-        print('   ' + reprs[1] + ' ' + ' ' * (29-len(reprs[1])) + reprs[0], end=' ')
-        if debug_lexer:
-            print(' ' + ' ' * (29-len(reprs[0])) + ' : '.join(state) if state else '', end=' ')
-        print()
-
-    for type, val in lx.get_tokens(text):
-        lno += val.count('\n')
-        if type == Error and not ignerror:
-            print('Error parsing', fn, 'on line', lno)
-            if not showall:
-                print('Previous tokens' + (debug_lexer and ' and states' or '') + ':')
-                for i in range(max(len(tokens) - num, 0), len(tokens)):
-                    if debug_lexer:
-                        show_token(tokens[i], states[i])
-                    else:
-                        show_token(tokens[i], None)
-            print('Error token:')
-            l = len(repr(val))
-            print('   ' + repr(val), end=' ')
-            if debug_lexer and hasattr(lx, 'ctx'):
-                print(' ' * (60-l) + ' : '.join(lx.ctx.stack), end=' ')
-            print()
-            print()
-            return 1
-        tokens.append((type, val))
-        if debug_lexer:
-            if hasattr(lx, 'ctx'):
-                states.append(lx.ctx.stack[:])
-            else:
-                states.append(None)
-        if showall:
-            show_token((type, val), states[-1] if debug_lexer else None)
-    return 0
-
-
-def print_help():
-    print('''\
-Pygments development helper to quickly debug lexers.
-
-    scripts/debug_lexer.py [options] file ...
-
-Give one or more filenames to lex them and display possible error tokens
-and/or profiling info.  Files are assumed to be encoded in UTF-8.
-
-Selecting lexer and options:
-
-    -l NAME         use lexer named NAME (default is to guess from
-                    the given filenames)
-    -O OPTIONSTR    use lexer options parsed from OPTIONSTR
-
-Debugging lexing errors:
-
-    -n N            show the last N tokens on error
-    -a              always show all lexed tokens (default is only
-                    to show them when an error occurs)
-    -e              do not stop on error tokens
-
-Profiling:
-
-    -p              use the ProfilingRegexLexer to profile regexes
-                    instead of the debugging lexer
-    -s N            sort profiling output by column N (default is
-                    column 4, the time per call)
-''')
-
-num = 10
-showall = False
-ignerror = False
-lexer = None
-options = {}
-profile = False
-profsort = 4
-
-if __name__ == '__main__':
-    import getopt
-    opts, args = getopt.getopt(sys.argv[1:], 'n:l:aepO:s:h')
-    for opt, val in opts:
-        if opt == '-n':
-            num = int(val)
-        elif opt == '-a':
-            showall = True
-        elif opt == '-e':
-            ignerror = True
-        elif opt == '-l':
-            lexer = val
-        elif opt == '-p':
-            profile = True
-        elif opt == '-s':
-            profsort = int(val)
-        elif opt == '-O':
-            options = _parse_options([val])
-        elif opt == '-h':
-            print_help()
-            sys.exit(0)
-    ret = 0
-    if not args:
-        print_help()
-    for f in args:
-        ret += main(f, lexer, options)
-    sys.exit(bool(ret))