gitlab-pygments.rb 0.3.2

data/vendor/pygments-main/tests/old_run.py

@@ -0,0 +1,138 @@
+# -*- coding: utf-8 -*-
+"""
+    Pygments unit tests
+    ~~~~~~~~~~~~~~~~~~
+
+    Usage::
+
+        python run.py [testfile ...]
+
+
+    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import sys, os
+import unittest
+
+from os.path import dirname, basename, join, abspath
+
+import pygments
+
+try:
+    import coverage
+except ImportError:
+    coverage = None
+
+testdir = abspath(dirname(__file__))
+
+failed = []
+total_test_count = 0
+error_test_count = 0
+
+
+def err(file, what, exc):
+    print >>sys.stderr, file, 'failed %s:' % what,
+    print >>sys.stderr, exc
+    failed.append(file[:-3])
+
+
+class QuietTestRunner(object):
+    """Customized test runner for relatively quiet output"""
+
+    def __init__(self, testname, stream=sys.stderr):
+        self.testname = testname
+        self.stream = unittest._WritelnDecorator(stream)
+
+    def run(self, test):
+        global total_test_count
+        global error_test_count
+        result = unittest._TextTestResult(self.stream, True, 1)
+        test(result)
+        if not result.wasSuccessful():
+            self.stream.write(' FAIL:')
+            result.printErrors()
+            failed.append(self.testname)
+        else:
+            self.stream.write(' ok\n')
+        total_test_count += result.testsRun
+        error_test_count += len(result.errors) + len(result.failures)
+        return result
+
+
+def run_tests(with_coverage=False):
+    # needed to avoid confusion involving atexit handlers
+    import logging
+
+    if sys.argv[1:]:
+        # test only files given on cmdline
+        files = [entry + '.py' for entry in sys.argv[1:] if entry.startswith('test_')]
+    else:
+        files = [entry for entry in os.listdir(testdir)
+                 if (entry.startswith('test_') and entry.endswith('.py'))]
+        files.sort()
+
+    WIDTH = 85
+
+    print >>sys.stderr, \
+        ('Pygments %s Test Suite running%s, stand by...' %
+         (pygments.__version__,
+          with_coverage and " with coverage analysis" or "")).center(WIDTH)
+    print >>sys.stderr, ('(using Python %s)' % sys.version.split()[0]).center(WIDTH)
+    print >>sys.stderr, '='*WIDTH
+
+    if with_coverage:
+        coverage.erase()
+        coverage.start()
+
+    for testfile in files:
+        globs = {'__file__': join(testdir, testfile)}
+        try:
+            execfile(join(testdir, testfile), globs)
+        except Exception, exc:
+            raise
+            err(testfile, 'execfile', exc)
+            continue
+        sys.stderr.write(testfile[:-3] + ': ')
+        try:
+            runner = QuietTestRunner(testfile[:-3])
+            # make a test suite of all TestCases in the file
+            tests = []
+            for name, thing in globs.iteritems():
+                if name.endswith('Test'):
+                    tests.append((name, unittest.makeSuite(thing)))
+            tests.sort()
+            suite = unittest.TestSuite()
+            suite.addTests([x[1] for x in tests])
+            runner.run(suite)
+        except Exception, exc:
+            err(testfile, 'running test', exc)
+
+    print >>sys.stderr, '='*WIDTH
+    if failed:
+        print >>sys.stderr, '%d of %d tests failed.' % \
+              (error_test_count, total_test_count)
+        print >>sys.stderr, 'Tests failed in:', ', '.join(failed)
+        ret = 1
+    else:
+        if total_test_count == 1:
+            print >>sys.stderr, '1 test happy.'
+        else:
+            print >>sys.stderr, 'All %d tests happy.' % total_test_count
+        ret = 0
+
+    if with_coverage:
+        coverage.stop()
+        modules = [mod for name, mod in sys.modules.iteritems()
+                   if name.startswith('pygments.') and mod]
+        coverage.report(modules)
+
+    return ret
+
+
+if __name__ == '__main__':
+    with_coverage = False
+    if sys.argv[1:2] == ['-C']:
+        with_coverage = bool(coverage)
+        del sys.argv[1]
+    sys.exit(run_tests(with_coverage))

data/vendor/pygments-main/tests/run.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+"""
+    Pygments unit tests
+    ~~~~~~~~~~~~~~~~~~
+
+    Usage::
+
+        python run.py [testfile ...]
+
+
+    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import sys, os
+
+if sys.version_info >= (3,):
+    # copy test suite over to "build/lib" and convert it
+    print ('Copying and converting sources to build/lib/test...')
+    from distutils.util import copydir_run_2to3
+    testroot = os.path.dirname(__file__)
+    newroot = os.path.join(testroot, '..', 'build/lib/test')
+    copydir_run_2to3(testroot, newroot)
+    # make nose believe that we run from the converted dir
+    os.chdir(newroot)
+else:
+    # only find tests in this directory
+    os.chdir(os.path.dirname(__file__))
+
+
+try:
+    import nose
+except ImportError:
+    print ('nose is required to run the Pygments test suite')
+    sys.exit(1)
+
+try:
+    # make sure the current source is first on sys.path
+    sys.path.insert(0, '..')
+    import pygments
+except ImportError:
+    print ('Cannot find Pygments to test: %s' % sys.exc_info()[1])
+    sys.exit(1)
+else:
+    print ('Pygments %s test suite running (Python %s)...' %
+           (pygments.__version__, sys.version.split()[0]))
+
+nose.main()

data/vendor/pygments-main/tests/support.py

@@ -0,0 +1,15 @@
+# coding: utf-8
+"""
+Support for Pygments tests
+"""
+
+import os
+
+
+def location(mod_name):
+    """
+    Return the file and directory that the code for *mod_name* is in.
+    """
+    source = mod_name.endswith("pyc") and mod_name[:-1] or mod_name
+    source = os.path.abspath(source)
+    return source, os.path.dirname(source)

data/vendor/pygments-main/tests/test_basic_api.py

@@ -0,0 +1,294 @@
+# -*- coding: utf-8 -*-
+"""
+    Pygments basic API tests
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import os
+import random
+import unittest
+
+from pygments import lexers, formatters, filters, format
+from pygments.token import _TokenType, Text
+from pygments.lexer import RegexLexer
+from pygments.formatters.img import FontNotFound
+from pygments.util import BytesIO, StringIO, bytes, b
+
+import support
+
+TESTFILE, TESTDIR = support.location(__file__)
+
+test_content = [chr(i) for i in xrange(33, 128)] * 5
+random.shuffle(test_content)
+test_content = ''.join(test_content) + '\n'
+
+
+def test_lexer_import_all():
+    # instantiate every lexer, to see if the token type defs are correct
+    for x in lexers.LEXERS.keys():
+        c = getattr(lexers, x)()
+
+
+def test_lexer_classes():
+    # test that every lexer class has the correct public API
+    def verify(cls):
+        assert type(cls.name) is str
+        for attr in 'aliases', 'filenames', 'alias_filenames', 'mimetypes':
+            assert hasattr(cls, attr)
+            assert type(getattr(cls, attr)) is list, \
+                   "%s: %s attribute wrong" % (cls, attr)
+        result = cls.analyse_text("abc")
+        assert isinstance(result, float) and 0.0 <= result <= 1.0
+        result = cls.analyse_text(".abc")
+        assert isinstance(result, float) and 0.0 <= result <= 1.0
+
+        inst = cls(opt1="val1", opt2="val2")
+        if issubclass(cls, RegexLexer):
+            if not hasattr(cls, '_tokens'):
+                # if there's no "_tokens", the lexer has to be one with
+                # multiple tokendef variants
+                assert cls.token_variants
+                for variant in cls.tokens:
+                    assert 'root' in cls.tokens[variant]
+            else:
+                assert 'root' in cls._tokens, \
+                       '%s has no root state' % cls
+
+        if cls.name in ['XQuery', 'Opa']:   # XXX temporary
+            return
+
+        tokens = list(inst.get_tokens(test_content))
+        txt = ""
+        for token in tokens:
+            assert isinstance(token, tuple)
+            assert isinstance(token[0], _TokenType)
+            if isinstance(token[1], str):
+                print repr(token[1])
+            assert isinstance(token[1], unicode)
+            txt += token[1]
+        assert txt == test_content, "%s lexer roundtrip failed: %r != %r" % \
+               (cls.name, test_content, txt)
+
+    for lexer in lexers._iter_lexerclasses():
+        yield verify, lexer
+
+
+def test_lexer_options():
+    # test that the basic options work
+    def ensure(tokens, output):
+        concatenated = ''.join(token[1] for token in tokens)
+        assert concatenated == output, \
+               '%s: %r != %r' % (lexer, concatenated, output)
+    def verify(cls):
+        inst = cls(stripnl=False)
+        ensure(inst.get_tokens('a\nb'), 'a\nb\n')
+        ensure(inst.get_tokens('\n\n\n'), '\n\n\n')
+        inst = cls(stripall=True)
+        ensure(inst.get_tokens('   \n  b\n\n\n'), 'b\n')
+        # some lexers require full lines in input
+        if cls.__name__ not in (
+            'PythonConsoleLexer', 'RConsoleLexer', 'RubyConsoleLexer',
+            'SqliteConsoleLexer', 'MatlabSessionLexer', 'ErlangShellLexer',
+            'BashSessionLexer', 'LiterateHaskellLexer', 'PostgresConsoleLexer',
+            'ElixirConsoleLexer', 'JuliaConsoleLexer'):
+            inst = cls(ensurenl=False)
+            ensure(inst.get_tokens('a\nb'), 'a\nb')
+            inst = cls(ensurenl=False, stripall=True)
+            ensure(inst.get_tokens('a\nb\n\n'), 'a\nb')
+
+    for lexer in lexers._iter_lexerclasses():
+        if lexer.__name__ == 'RawTokenLexer':
+            # this one is special
+            continue
+        yield verify, lexer
+
+
+def test_get_lexers():
+    # test that the lexers functions work
+    def verify(func, args):
+        x = func(opt='val', *args)
+        assert isinstance(x, lexers.PythonLexer)
+        assert x.options["opt"] == "val"
+
+    for func, args in [(lexers.get_lexer_by_name, ("python",)),
+                       (lexers.get_lexer_for_filename, ("test.py",)),
+                       (lexers.get_lexer_for_mimetype, ("text/x-python",)),
+                       (lexers.guess_lexer, ("#!/usr/bin/python -O\nprint",)),
+                       (lexers.guess_lexer_for_filename, ("a.py", "<%= @foo %>"))
+                       ]:
+        yield verify, func, args
+
+    for cls, (_, lname, aliases, _, mimetypes) in lexers.LEXERS.iteritems():
+        assert cls == lexers.find_lexer_class(lname).__name__
+
+        for alias in aliases:
+            assert cls == lexers.get_lexer_by_name(alias).__class__.__name__
+
+        for mimetype in mimetypes:
+            assert cls == lexers.get_lexer_for_mimetype(mimetype).__class__.__name__
+
+
+def test_formatter_public_api():
+    ts = list(lexers.PythonLexer().get_tokens("def f(): pass"))
+    out = StringIO()
+    # test that every formatter class has the correct public API
+    def verify(formatter, info):
+        assert len(info) == 4
+        assert info[0], "missing formatter name"
+        assert info[1], "missing formatter aliases"
+        assert info[3], "missing formatter docstring"
+
+        if formatter.name == 'Raw tokens':
+            # will not work with Unicode output file
+            return
+
+        try:
+            inst = formatter(opt1="val1")
+        except (ImportError, FontNotFound):
+            return
+        try:
+            inst.get_style_defs()
+        except NotImplementedError:
+            # may be raised by formatters for which it doesn't make sense
+            pass
+        inst.format(ts, out)
+
+    for formatter, info in formatters.FORMATTERS.iteritems():
+        yield verify, formatter, info
+
+def test_formatter_encodings():
+    from pygments.formatters import HtmlFormatter
+
+    # unicode output
+    fmt = HtmlFormatter()
+    tokens = [(Text, u"ä")]
+    out = format(tokens, fmt)
+    assert type(out) is unicode
+    assert u"ä" in out
+
+    # encoding option
+    fmt = HtmlFormatter(encoding="latin1")
+    tokens = [(Text, u"ä")]
+    assert u"ä".encode("latin1") in format(tokens, fmt)
+
+    # encoding and outencoding option
+    fmt = HtmlFormatter(encoding="latin1", outencoding="utf8")
+    tokens = [(Text, u"ä")]
+    assert u"ä".encode("utf8") in format(tokens, fmt)
+
+
+def test_formatter_unicode_handling():
+    # test that the formatter supports encoding and Unicode
+    tokens = list(lexers.PythonLexer(encoding='utf-8').
+                  get_tokens("def f(): 'ä'"))
+
+    def verify(formatter):
+        try:
+            inst = formatter(encoding=None)
+        except (ImportError, FontNotFound):
+            # some dependency or font not installed
+            return
+
+        if formatter.name != 'Raw tokens':
+            out = format(tokens, inst)
+            if formatter.unicodeoutput:
+                assert type(out) is unicode
+
+            inst = formatter(encoding='utf-8')
+            out = format(tokens, inst)
+            assert type(out) is bytes, '%s: %r' % (formatter, out)
+            # Cannot test for encoding, since formatters may have to escape
+            # non-ASCII characters.
+        else:
+            inst = formatter()
+            out = format(tokens, inst)
+            assert type(out) is bytes, '%s: %r' % (formatter, out)
+
+    for formatter, info in formatters.FORMATTERS.iteritems():
+        yield verify, formatter
+
+
+def test_get_formatters():
+    # test that the formatters functions work
+    x = formatters.get_formatter_by_name("html", opt="val")
+    assert isinstance(x, formatters.HtmlFormatter)
+    assert x.options["opt"] == "val"
+
+    x = formatters.get_formatter_for_filename("a.html", opt="val")
+    assert isinstance(x, formatters.HtmlFormatter)
+    assert x.options["opt"] == "val"
+
+
+def test_styles():
+    # minimal style test
+    from pygments.formatters import HtmlFormatter
+    fmt = HtmlFormatter(style="pastie")
+
+
+class FiltersTest(unittest.TestCase):
+
+    def test_basic(self):
+        filter_args = {
+            'whitespace': {'spaces': True, 'tabs': True, 'newlines': True},
+            'highlight': {'names': ['isinstance', 'lexers', 'x']},
+        }
+        for x in filters.FILTERS.keys():
+            lx = lexers.PythonLexer()
+            lx.add_filter(x, **filter_args.get(x, {}))
+            fp = open(TESTFILE, 'rb')
+            try:
+                text = fp.read().decode('utf-8')
+            finally:
+                fp.close()
+            tokens = list(lx.get_tokens(text))
+            roundtext = ''.join([t[1] for t in tokens])
+            if x not in ('whitespace', 'keywordcase'):
+                # these filters change the text
+                self.assertEqual(roundtext, text,
+                                 "lexer roundtrip with %s filter failed" % x)
+
+    def test_raiseonerror(self):
+        lx = lexers.PythonLexer()
+        lx.add_filter('raiseonerror', excclass=RuntimeError)
+        self.assertRaises(RuntimeError, list, lx.get_tokens('$'))
+
+    def test_whitespace(self):
+        lx = lexers.PythonLexer()
+        lx.add_filter('whitespace', spaces='%')
+        fp = open(TESTFILE, 'rb')
+        try:
+            text = fp.read().decode('utf-8')
+        finally:
+            fp.close()
+        lxtext = ''.join([t[1] for t in list(lx.get_tokens(text))])
+        self.assertFalse(' ' in lxtext)
+
+    def test_keywordcase(self):
+        lx = lexers.PythonLexer()
+        lx.add_filter('keywordcase', case='capitalize')
+        fp = open(TESTFILE, 'rb')
+        try:
+            text = fp.read().decode('utf-8')
+        finally:
+            fp.close()
+        lxtext = ''.join([t[1] for t in list(lx.get_tokens(text))])
+        self.assertTrue('Def' in lxtext and 'Class' in lxtext)
+
+    def test_codetag(self):
+        lx = lexers.PythonLexer()
+        lx.add_filter('codetagify')
+        text = u'# BUG: text'
+        tokens = list(lx.get_tokens(text))
+        self.assertEqual('# ', tokens[0][1])
+        self.assertEqual('BUG', tokens[1][1])
+
+    def test_codetag_boundary(self):
+        # ticket #368
+        lx = lexers.PythonLexer()
+        lx.add_filter('codetagify')
+        text = u'# DEBUG: text'
+        tokens = list(lx.get_tokens(text))
+        self.assertEqual('# DEBUG: text', tokens[0][1])