gitlab-pygments.rb 0.3.2

data/vendor/pygments-main/tests/test_clexer.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+"""
+    Basic CLexer Test
+    ~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+import os
+
+from pygments.token import Text, Number
+from pygments.lexers import CLexer
+
+
+class CLexerTest(unittest.TestCase):
+
+    def setUp(self):
+        self.lexer = CLexer()
+
+    def testNumbers(self):
+        code = '42 23.42 23. .42 023 0xdeadbeef 23e+42 42e-23'
+        wanted = []
+        for item in zip([Number.Integer, Number.Float, Number.Float,
+                         Number.Float, Number.Oct, Number.Hex,
+                         Number.Float, Number.Float], code.split()):
+            wanted.append(item)
+            wanted.append((Text, ' '))
+        wanted = [(Text, '')] + wanted[:-1] + [(Text, '\n')]
+        self.assertEqual(list(self.lexer.get_tokens(code)), wanted)

data/vendor/pygments-main/tests/test_cmdline.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+"""
+    Command line test
+    ~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+# Test the command line interface
+
+import sys, os
+import unittest
+import StringIO
+
+from pygments import highlight
+from pygments.cmdline import main as cmdline_main
+
+import support
+
+TESTFILE, TESTDIR = support.location(__file__)
+
+
+def run_cmdline(*args):
+    saved_stdout = sys.stdout
+    saved_stderr = sys.stderr
+    new_stdout = sys.stdout = StringIO.StringIO()
+    new_stderr = sys.stderr = StringIO.StringIO()
+    try:
+        ret = cmdline_main(["pygmentize"] + list(args))
+    finally:
+        sys.stdout = saved_stdout
+        sys.stderr = saved_stderr
+    return (ret, new_stdout.getvalue(), new_stderr.getvalue())
+
+
+class CmdLineTest(unittest.TestCase):
+
+    def test_L_opt(self):
+        c, o, e = run_cmdline("-L")
+        self.assertEqual(c, 0)
+        self.assertTrue("Lexers" in o and "Formatters" in o and
+                        "Filters" in o and "Styles" in o)
+        c, o, e = run_cmdline("-L", "lexer")
+        self.assertEqual(c, 0)
+        self.assertTrue("Lexers" in o and "Formatters" not in o)
+        c, o, e = run_cmdline("-L", "lexers")
+        self.assertEqual(c, 0)
+
+    def test_O_opt(self):
+        filename = TESTFILE
+        c, o, e = run_cmdline("-Ofull=1,linenos=true,foo=bar",
+                              "-fhtml", filename)
+        self.assertEqual(c, 0)
+        self.assertTrue("<html" in o)
+        self.assertTrue('class="linenos"' in o)
+
+    def test_P_opt(self):
+        filename = TESTFILE
+        c, o, e = run_cmdline("-Pfull", "-Ptitle=foo, bar=baz=,",
+                              "-fhtml", filename)
+        self.assertEqual(c, 0)
+        self.assertTrue("<title>foo, bar=baz=,</title>" in o)
+
+    def test_F_opt(self):
+        filename = TESTFILE
+        c, o, e = run_cmdline("-Fhighlight:tokentype=Name.Blubb,"
+                              "names=TESTFILE filename",
+                              "-fhtml", filename)
+        self.assertEqual(c, 0)
+        self.assertTrue('<span class="n-Blubb' in o)
+
+    def test_H_opt(self):
+        c, o, e = run_cmdline("-H", "formatter", "html")
+        self.assertEqual(c, 0)
+        self.assertTrue('HTML' in o)
+
+    def test_S_opt(self):
+        c, o, e = run_cmdline("-S", "default", "-f", "html", "-O", "linenos=1")
+        self.assertEqual(c, 0)
+
+    def test_invalid_opts(self):
+        for opts in [("-L", "-lpy"), ("-L", "-fhtml"), ("-L", "-Ox"),
+                     ("-a",), ("-Sst", "-lpy"), ("-H",),
+                     ("-H", "formatter"),]:
+            self.assertTrue(run_cmdline(*opts)[0] == 2)
+
+    def test_normal(self):
+        # test that cmdline gives the same output as library api
+        from pygments.lexers import PythonLexer
+        from pygments.formatters import HtmlFormatter
+        filename = TESTFILE
+        fp = open(filename, 'rb')
+        try:
+            code = fp.read()
+        finally:
+            fp.close()
+
+        output = highlight(code, PythonLexer(), HtmlFormatter())
+
+        c, o, e = run_cmdline("-lpython", "-fhtml", filename)
+
+        self.assertEqual(o, output)
+        self.assertEqual(e, "")
+        self.assertEqual(c, 0)

data/vendor/pygments-main/tests/test_examplefiles.py

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+"""
+    Pygments tests with example files
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import os
+import pprint
+import difflib
+import cPickle as pickle
+
+from pygments.lexers import get_lexer_for_filename, get_lexer_by_name
+from pygments.token import Error
+from pygments.util import ClassNotFound, b
+
+STORE_OUTPUT = False
+
+# generate methods
+def test_example_files():
+    testdir = os.path.dirname(__file__)
+    outdir = os.path.join(testdir, 'examplefiles', 'output')
+    if STORE_OUTPUT and not os.path.isdir(outdir):
+        os.makedirs(outdir)
+    for fn in os.listdir(os.path.join(testdir, 'examplefiles')):
+        if fn.startswith('.') or fn.endswith('#'):
+            continue
+
+        absfn = os.path.join(testdir, 'examplefiles', fn)
+        if not os.path.isfile(absfn):
+            continue
+        outfn = os.path.join(outdir, fn)
+
+        try:
+            lx = get_lexer_for_filename(absfn)
+        except ClassNotFound:
+            if "_" not in fn:
+                raise AssertionError('file %r has no registered extension, '
+                                     'nor is of the form <lexer>_filename '
+                                     'for overriding, thus no lexer found.'
+                                    % fn)
+            try:
+                name, rest = fn.split("_", 1)
+                lx = get_lexer_by_name(name)
+            except ClassNotFound:
+                raise AssertionError('no lexer found for file %r' % fn)
+        yield check_lexer, lx, absfn, outfn
+
+def check_lexer(lx, absfn, outfn):
+    fp = open(absfn, 'rb')
+    try:
+        text = fp.read()
+    finally:
+        fp.close()
+    text = text.replace(b('\r\n'), b('\n'))
+    text = text.strip(b('\n')) + b('\n')
+    try:
+        text = text.decode('utf-8')
+    except UnicodeError:
+        text = text.decode('latin1')
+    ntext = []
+    tokens = []
+    for type, val in lx.get_tokens(text):
+        ntext.append(val)
+        assert type != Error, \
+            'lexer %s generated error token for %s: %r at position %d' % \
+            (lx, absfn, val, len(u''.join(ntext)))
+        tokens.append((type, val))
+    if u''.join(ntext) != text:
+        print '\n'.join(difflib.unified_diff(u''.join(ntext).splitlines(),
+                                             text.splitlines()))
+        raise AssertionError('round trip failed for ' + absfn)
+
+    # check output against previous run if enabled
+    if STORE_OUTPUT:
+        # no previous output -- store it
+        if not os.path.isfile(outfn):
+            fp = open(outfn, 'wb')
+            try:
+                pickle.dump(tokens, fp)
+            finally:
+                fp.close()
+            return
+        # otherwise load it and compare
+        fp = open(outfn, 'rb')
+        try:
+            stored_tokens = pickle.load(fp)
+        finally:
+            fp.close()
+        if stored_tokens != tokens:
+            f1 = pprint.pformat(stored_tokens)
+            f2 = pprint.pformat(tokens)
+            print '\n'.join(difflib.unified_diff(f1.splitlines(),
+                                                 f2.splitlines()))
+            assert False, absfn

data/vendor/pygments-main/tests/test_html_formatter.py

@@ -0,0 +1,162 @@
+# -*- coding: utf-8 -*-
+"""
+    Pygments HTML formatter tests
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import os
+import re
+import unittest
+import StringIO
+import tempfile
+from os.path import join, dirname, isfile
+
+from pygments.lexers import PythonLexer
+from pygments.formatters import HtmlFormatter, NullFormatter
+from pygments.formatters.html import escape_html
+from pygments.util import uni_open
+
+import support
+
+TESTFILE, TESTDIR = support.location(__file__)
+
+fp = uni_open(TESTFILE, encoding='utf-8')
+try:
+    tokensource = list(PythonLexer().get_tokens(fp.read()))
+finally:
+    fp.close()
+
+
+class HtmlFormatterTest(unittest.TestCase):
+    def test_correct_output(self):
+        hfmt = HtmlFormatter(nowrap=True)
+        houtfile = StringIO.StringIO()
+        hfmt.format(tokensource, houtfile)
+
+        nfmt = NullFormatter()
+        noutfile = StringIO.StringIO()
+        nfmt.format(tokensource, noutfile)
+
+        stripped_html = re.sub('<.*?>', '', houtfile.getvalue())
+        escaped_text = escape_html(noutfile.getvalue())
+        self.assertEqual(stripped_html, escaped_text)
+
+    def test_external_css(self):
+        # test correct behavior
+        # CSS should be in /tmp directory
+        fmt1 = HtmlFormatter(full=True, cssfile='fmt1.css', outencoding='utf-8')
+        # CSS should be in TESTDIR (TESTDIR is absolute)
+        fmt2 = HtmlFormatter(full=True, cssfile=join(TESTDIR, 'fmt2.css'),
+                             outencoding='utf-8')
+        tfile = tempfile.NamedTemporaryFile(suffix='.html')
+        fmt1.format(tokensource, tfile)
+        try:
+            fmt2.format(tokensource, tfile)
+            self.assertTrue(isfile(join(TESTDIR, 'fmt2.css')))
+        except IOError:
+            # test directory not writable
+            pass
+        tfile.close()
+
+        self.assertTrue(isfile(join(dirname(tfile.name), 'fmt1.css')))
+        os.unlink(join(dirname(tfile.name), 'fmt1.css'))
+        try:
+            os.unlink(join(TESTDIR, 'fmt2.css'))
+        except OSError:
+            pass
+
+    def test_all_options(self):
+        for optdict in [dict(nowrap=True),
+                        dict(linenos=True),
+                        dict(linenos=True, full=True),
+                        dict(linenos=True, full=True, noclasses=True)]:
+
+            outfile = StringIO.StringIO()
+            fmt = HtmlFormatter(**optdict)
+            fmt.format(tokensource, outfile)
+
+    def test_linenos(self):
+        optdict = dict(linenos=True)
+        outfile = StringIO.StringIO()
+        fmt = HtmlFormatter(**optdict)
+        fmt.format(tokensource, outfile)
+        html = outfile.getvalue()
+        self.assertTrue(re.search("<pre>\s+1\s+2\s+3", html))
+
+    def test_linenos_with_startnum(self):
+        optdict = dict(linenos=True, linenostart=5)
+        outfile = StringIO.StringIO()
+        fmt = HtmlFormatter(**optdict)
+        fmt.format(tokensource, outfile)
+        html = outfile.getvalue()
+        self.assertTrue(re.search("<pre>\s+5\s+6\s+7", html))
+
+    def test_lineanchors(self):
+        optdict = dict(lineanchors="foo")
+        outfile = StringIO.StringIO()
+        fmt = HtmlFormatter(**optdict)
+        fmt.format(tokensource, outfile)
+        html = outfile.getvalue()
+        self.assertTrue(re.search("<pre><a name=\"foo-1\">", html))
+
+    def test_lineanchors_with_startnum(self):
+        optdict = dict(lineanchors="foo", linenostart=5)
+        outfile = StringIO.StringIO()
+        fmt = HtmlFormatter(**optdict)
+        fmt.format(tokensource, outfile)
+        html = outfile.getvalue()
+        self.assertTrue(re.search("<pre><a name=\"foo-5\">", html))
+
+    def test_valid_output(self):
+        # test all available wrappers
+        fmt = HtmlFormatter(full=True, linenos=True, noclasses=True,
+                            outencoding='utf-8')
+
+        handle, pathname = tempfile.mkstemp('.html')
+        tfile = os.fdopen(handle, 'w+b')
+        fmt.format(tokensource, tfile)
+        tfile.close()
+        catname = os.path.join(TESTDIR, 'dtds', 'HTML4.soc')
+        try:
+            import subprocess
+            po = subprocess.Popen(['nsgmls', '-s', '-c', catname, pathname],
+                                  stdout=subprocess.PIPE)
+            ret = po.wait()
+            output = po.stdout.read()
+            po.stdout.close()
+        except OSError:
+            # nsgmls not available
+            pass
+        else:
+            if ret:
+                print output
+            self.assertFalse(ret, 'nsgmls run reported errors')
+
+        os.unlink(pathname)
+
+    def test_get_style_defs(self):
+        fmt = HtmlFormatter()
+        sd = fmt.get_style_defs()
+        self.assertTrue(sd.startswith('.'))
+
+        fmt = HtmlFormatter(cssclass='foo')
+        sd = fmt.get_style_defs()
+        self.assertTrue(sd.startswith('.foo'))
+        sd = fmt.get_style_defs('.bar')
+        self.assertTrue(sd.startswith('.bar'))
+        sd = fmt.get_style_defs(['.bar', '.baz'])
+        fl = sd.splitlines()[0]
+        self.assertTrue('.bar' in fl and '.baz' in fl)
+
+    def test_unicode_options(self):
+        fmt = HtmlFormatter(title=u'Föö',
+                            cssclass=u'bär',
+                            cssstyles=u'div:before { content: \'bäz\' }',
+                            encoding='utf-8')
+        handle, pathname = tempfile.mkstemp('.html')
+        tfile = os.fdopen(handle, 'w+b')
+        fmt.format(tokensource, tfile)
+        tfile.close()

data/vendor/pygments-main/tests/test_latex_formatter.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+"""
+    Pygments LaTeX formatter tests
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import os
+import unittest
+import tempfile
+
+from pygments.formatters import LatexFormatter
+from pygments.lexers import PythonLexer
+
+import support
+
+TESTFILE, TESTDIR = support.location(__file__)
+
+
+class LatexFormatterTest(unittest.TestCase):
+
+    def test_valid_output(self):
+        fp = open(TESTFILE)
+        try:
+            tokensource = list(PythonLexer().get_tokens(fp.read()))
+        finally:
+            fp.close()
+        fmt = LatexFormatter(full=True, encoding='latin1')
+
+        handle, pathname = tempfile.mkstemp('.tex')
+        # place all output files in /tmp too
+        old_wd = os.getcwd()
+        os.chdir(os.path.dirname(pathname))
+        tfile = os.fdopen(handle, 'wb')
+        fmt.format(tokensource, tfile)
+        tfile.close()
+        try:
+            import subprocess
+            po = subprocess.Popen(['latex', '-interaction=nonstopmode',
+                                   pathname], stdout=subprocess.PIPE)
+            ret = po.wait()
+            output = po.stdout.read()
+            po.stdout.close()
+        except OSError:
+            # latex not available
+            pass
+        else:
+            if ret:
+                print output
+            self.assertFalse(ret, 'latex run reported errors')
+
+        os.unlink(pathname)
+        os.chdir(old_wd)