pygments.rb 0.6.3 → 1.0.0

data/vendor/pygments-main/tests/test_clexer.py

@@ -1,236 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-    Basic CLexer Test
-    ~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-import unittest
-import os
-import textwrap
-
-from pygments.token import Text, Number, Token
-from pygments.lexers import CLexer
-
-
-class CLexerTest(unittest.TestCase):
-
-    def setUp(self):
-        self.lexer = CLexer()
-
-    def testNumbers(self):
-        code = '42 23.42 23. .42 023 0xdeadbeef 23e+42 42e-23'
-        wanted = []
-        for item in zip([Number.Integer, Number.Float, Number.Float,
-                         Number.Float, Number.Oct, Number.Hex,
-                         Number.Float, Number.Float], code.split()):
-            wanted.append(item)
-            wanted.append((Text, ' '))
-        wanted = wanted[:-1] + [(Text, '\n')]
-        self.assertEqual(list(self.lexer.get_tokens(code)), wanted)
-
-    def testSwitch(self):
-        fragment = u'''\
-        int main()
-        {
-            switch (0)
-            {
-                case 0:
-                default:
-                    ;
-            }
-        }
-        '''
-        tokens = [
-            (Token.Keyword.Type, u'int'),
-            (Token.Text, u' '),
-            (Token.Name.Function, u'main'),
-            (Token.Punctuation, u'('),
-            (Token.Punctuation, u')'),
-            (Token.Text, u'\n'),
-            (Token.Punctuation, u'{'),
-            (Token.Text, u'\n'),
-            (Token.Text, u'    '),
-            (Token.Keyword, u'switch'),
-            (Token.Text, u' '),
-            (Token.Punctuation, u'('),
-            (Token.Literal.Number.Integer, u'0'),
-            (Token.Punctuation, u')'),
-            (Token.Text, u'\n'),
-            (Token.Text, u'    '),
-            (Token.Punctuation, u'{'),
-            (Token.Text, u'\n'),
-            (Token.Text, u'        '),
-            (Token.Keyword, u'case'),
-            (Token.Text, u' '),
-            (Token.Literal.Number.Integer, u'0'),
-            (Token.Operator, u':'),
-            (Token.Text, u'\n'),
-            (Token.Text, u'        '),
-            (Token.Keyword, u'default'),
-            (Token.Operator, u':'),
-            (Token.Text, u'\n'),
-            (Token.Text, u'            '),
-            (Token.Punctuation, u';'),
-            (Token.Text, u'\n'),
-            (Token.Text, u'    '),
-            (Token.Punctuation, u'}'),
-            (Token.Text, u'\n'),
-            (Token.Punctuation, u'}'),
-            (Token.Text, u'\n'),
-        ]
-        self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment))))
-
-    def testSwitchSpaceBeforeColon(self):
-        fragment = u'''\
-        int main()
-        {
-            switch (0)
-            {
-                case 0 :
-                default :
-                    ;
-            }
-        }
-        '''
-        tokens = [
-            (Token.Keyword.Type, u'int'),
-            (Token.Text, u' '),
-            (Token.Name.Function, u'main'),
-            (Token.Punctuation, u'('),
-            (Token.Punctuation, u')'),
-            (Token.Text, u'\n'),
-            (Token.Punctuation, u'{'),
-            (Token.Text, u'\n'),
-            (Token.Text, u'    '),
-            (Token.Keyword, u'switch'),
-            (Token.Text, u' '),
-            (Token.Punctuation, u'('),
-            (Token.Literal.Number.Integer, u'0'),
-            (Token.Punctuation, u')'),
-            (Token.Text, u'\n'),
-            (Token.Text, u'    '),
-            (Token.Punctuation, u'{'),
-            (Token.Text, u'\n'),
-            (Token.Text, u'        '),
-            (Token.Keyword, u'case'),
-            (Token.Text, u' '),
-            (Token.Literal.Number.Integer, u'0'),
-            (Token.Text, u' '),
-            (Token.Operator, u':'),
-            (Token.Text, u'\n'),
-            (Token.Text, u'        '),
-            (Token.Keyword, u'default'),
-            (Token.Text, u' '),
-            (Token.Operator, u':'),
-            (Token.Text, u'\n'),
-            (Token.Text, u'            '),
-            (Token.Punctuation, u';'),
-            (Token.Text, u'\n'),
-            (Token.Text, u'    '),
-            (Token.Punctuation, u'}'),
-            (Token.Text, u'\n'),
-            (Token.Punctuation, u'}'),
-            (Token.Text, u'\n'),
-        ]
-        self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment))))
-
-    def testLabel(self):
-        fragment = u'''\
-        int main()
-        {
-        foo:
-          goto foo;
-        }
-        '''
-        tokens = [
-            (Token.Keyword.Type, u'int'),
-            (Token.Text, u' '),
-            (Token.Name.Function, u'main'),
-            (Token.Punctuation, u'('),
-            (Token.Punctuation, u')'),
-            (Token.Text, u'\n'),
-            (Token.Punctuation, u'{'),
-            (Token.Text, u'\n'),
-            (Token.Name.Label, u'foo'),
-            (Token.Punctuation, u':'),
-            (Token.Text, u'\n'),
-            (Token.Text, u'  '),
-            (Token.Keyword, u'goto'),
-            (Token.Text, u' '),
-            (Token.Name, u'foo'),
-            (Token.Punctuation, u';'),
-            (Token.Text, u'\n'),
-            (Token.Punctuation, u'}'),
-            (Token.Text, u'\n'),
-        ]
-        self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment))))
-
-    def testLabelSpaceBeforeColon(self):
-        fragment = u'''\
-        int main()
-        {
-        foo :
-          goto foo;
-        }
-        '''
-        tokens = [
-            (Token.Keyword.Type, u'int'),
-            (Token.Text, u' '),
-            (Token.Name.Function, u'main'),
-            (Token.Punctuation, u'('),
-            (Token.Punctuation, u')'),
-            (Token.Text, u'\n'),
-            (Token.Punctuation, u'{'),
-            (Token.Text, u'\n'),
-            (Token.Name.Label, u'foo'),
-            (Token.Text, u' '),
-            (Token.Punctuation, u':'),
-            (Token.Text, u'\n'),
-            (Token.Text, u'  '),
-            (Token.Keyword, u'goto'),
-            (Token.Text, u' '),
-            (Token.Name, u'foo'),
-            (Token.Punctuation, u';'),
-            (Token.Text, u'\n'),
-            (Token.Punctuation, u'}'),
-            (Token.Text, u'\n'),
-        ]
-        self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment))))
-
-    def testLabelFollowedByStatement(self):
-        fragment = u'''\
-        int main()
-        {
-        foo:return 0;
-          goto foo;
-        }
-        '''
-        tokens = [
-            (Token.Keyword.Type, u'int'),
-            (Token.Text, u' '),
-            (Token.Name.Function, u'main'),
-            (Token.Punctuation, u'('),
-            (Token.Punctuation, u')'),
-            (Token.Text, u'\n'),
-            (Token.Punctuation, u'{'),
-            (Token.Text, u'\n'),
-            (Token.Name.Label, u'foo'),
-            (Token.Punctuation, u':'),
-            (Token.Keyword, u'return'),
-            (Token.Text, u' '),
-            (Token.Literal.Number.Integer, u'0'),
-            (Token.Punctuation, u';'),
-            (Token.Text, u'\n'),
-            (Token.Text, u'  '),
-            (Token.Keyword, u'goto'),
-            (Token.Text, u' '),
-            (Token.Name, u'foo'),
-            (Token.Punctuation, u';'),
-            (Token.Text, u'\n'),
-            (Token.Punctuation, u'}'),
-            (Token.Text, u'\n'),
-        ]
-        self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment))))

data/vendor/pygments-main/tests/test_cmdline.py

@@ -1,106 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-    Command line test
-    ~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-# Test the command line interface
-
-import io
-import sys
-import unittest
-
-from pygments import highlight
-from pygments.util import StringIO
-from pygments.cmdline import main as cmdline_main
-
-import support
-
-TESTFILE, TESTDIR = support.location(__file__)
-
-
-def run_cmdline(*args):
-    saved_stdout = sys.stdout
-    saved_stderr = sys.stderr
-    new_stdout = sys.stdout = StringIO()
-    new_stderr = sys.stderr = StringIO()
-    try:
-        ret = cmdline_main(["pygmentize"] + list(args))
-    finally:
-        sys.stdout = saved_stdout
-        sys.stderr = saved_stderr
-    return (ret, new_stdout.getvalue(), new_stderr.getvalue())
-
-
-class CmdLineTest(unittest.TestCase):
-
-    def test_L_opt(self):
-        c, o, e = run_cmdline("-L")
-        self.assertEqual(c, 0)
-        self.assertTrue("Lexers" in o and "Formatters" in o and
-                        "Filters" in o and "Styles" in o)
-        c, o, e = run_cmdline("-L", "lexer")
-        self.assertEqual(c, 0)
-        self.assertTrue("Lexers" in o and "Formatters" not in o)
-        c, o, e = run_cmdline("-L", "lexers")
-        self.assertEqual(c, 0)
-
-    def test_O_opt(self):
-        filename = TESTFILE
-        c, o, e = run_cmdline("-Ofull=1,linenos=true,foo=bar",
-                              "-fhtml", filename)
-        self.assertEqual(c, 0)
-        self.assertTrue("<html" in o)
-        self.assertTrue('class="linenos"' in o)
-
-    def test_P_opt(self):
-        filename = TESTFILE
-        c, o, e = run_cmdline("-Pfull", "-Ptitle=foo, bar=baz=,",
-                              "-fhtml", filename)
-        self.assertEqual(c, 0)
-        self.assertTrue("<title>foo, bar=baz=,</title>" in o)
-
-    def test_F_opt(self):
-        filename = TESTFILE
-        c, o, e = run_cmdline("-Fhighlight:tokentype=Name.Blubb,"
-                              "names=TESTFILE filename",
-                              "-fhtml", filename)
-        self.assertEqual(c, 0)
-        self.assertTrue('<span class="n-Blubb' in o)
-
-    def test_H_opt(self):
-        c, o, e = run_cmdline("-H", "formatter", "html")
-        self.assertEqual(c, 0)
-        self.assertTrue('HTML' in o)
-
-    def test_S_opt(self):
-        c, o, e = run_cmdline("-S", "default", "-f", "html", "-O", "linenos=1")
-        self.assertEqual(c, 0)
-
-    def test_invalid_opts(self):
-        for opts in [("-L", "-lpy"), ("-L", "-fhtml"), ("-L", "-Ox"),
-                     ("-a",), ("-Sst", "-lpy"), ("-H",),
-                     ("-H", "formatter"),]:
-            self.assertTrue(run_cmdline(*opts)[0] == 2)
-
-    def test_normal(self):
-        # test that cmdline gives the same output as library api
-        from pygments.lexers import PythonLexer
-        from pygments.formatters import HtmlFormatter
-        filename = TESTFILE
-        fp = open(filename, 'rb')
-        try:
-            code = fp.read()
-        finally:
-            fp.close()
-
-        output = highlight(code, PythonLexer(), HtmlFormatter())
-
-        c, o, e = run_cmdline("-lpython", "-fhtml", filename)
-
-        self.assertEqual(o, output)
-        self.assertEqual(e, "")
-        self.assertEqual(c, 0)

data/vendor/pygments-main/tests/test_examplefiles.py

@@ -1,110 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
-    Pygments tests with example files
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-from __future__ import print_function
-
-import os
-import pprint
-import difflib
-import pickle
-
-from pygments.lexers import get_lexer_for_filename, get_lexer_by_name
-from pygments.token import Error
-from pygments.util import ClassNotFound
-
-STORE_OUTPUT = False
-
-# generate methods
-def test_example_files():
-    testdir = os.path.dirname(__file__)
-    outdir = os.path.join(testdir, 'examplefiles', 'output')
-    if STORE_OUTPUT and not os.path.isdir(outdir):
-        os.makedirs(outdir)
-    for fn in os.listdir(os.path.join(testdir, 'examplefiles')):
-        if fn.startswith('.') or fn.endswith('#'):
-            continue
-
-        absfn = os.path.join(testdir, 'examplefiles', fn)
-        if not os.path.isfile(absfn):
-            continue
-
-        print(absfn)
-        code = open(absfn, 'rb').read()
-        try:
-            code = code.decode('utf-8')
-        except UnicodeError:
-            code = code.decode('latin1')
-
-        outfn = os.path.join(outdir, fn)
-
-        lx = None
-        if '_' in fn:
-            try:
-                lx = get_lexer_by_name(fn.split('_')[0])
-            except ClassNotFound:
-                pass
-        if lx is None:
-            try:
-                lx = get_lexer_for_filename(absfn, code=code)
-            except ClassNotFound:
-                raise AssertionError('file %r has no registered extension, '
-                                     'nor is of the form <lexer>_filename '
-                                     'for overriding, thus no lexer found.'
-                                     % fn)
-        yield check_lexer, lx, absfn, outfn
-
-def check_lexer(lx, absfn, outfn):
-    fp = open(absfn, 'rb')
-    try:
-        text = fp.read()
-    finally:
-        fp.close()
-    text = text.replace(b'\r\n', b'\n')
-    text = text.strip(b'\n') + b'\n'
-    try:
-        text = text.decode('utf-8')
-        if text.startswith(u'\ufeff'):
-            text = text[len(u'\ufeff'):]
-    except UnicodeError:
-        text = text.decode('latin1')
-    ntext = []
-    tokens = []
-    for type, val in lx.get_tokens(text):
-        ntext.append(val)
-        assert type != Error, \
-            'lexer %s generated error token for %s: %r at position %d' % \
-            (lx, absfn, val, len(u''.join(ntext)))
-        tokens.append((type, val))
-    if u''.join(ntext) != text:
-        print('\n'.join(difflib.unified_diff(u''.join(ntext).splitlines(),
-                                             text.splitlines())))
-        raise AssertionError('round trip failed for ' + absfn)
-
-    # check output against previous run if enabled
-    if STORE_OUTPUT:
-        # no previous output -- store it
-        if not os.path.isfile(outfn):
-            fp = open(outfn, 'wb')
-            try:
-                pickle.dump(tokens, fp)
-            finally:
-                fp.close()
-            return
-        # otherwise load it and compare
-        fp = open(outfn, 'rb')
-        try:
-            stored_tokens = pickle.load(fp)
-        finally:
-            fp.close()
-        if stored_tokens != tokens:
-            f1 = pprint.pformat(stored_tokens)
-            f2 = pprint.pformat(tokens)
-            print('\n'.join(difflib.unified_diff(f1.splitlines(),
-                                                 f2.splitlines())))
-            assert False, absfn