pygments.rb-jruby 0.5.4

data/vendor/pygments-main/tests/test_lexers_other.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+"""
+    Tests for other lexers
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import glob
+import os
+import unittest
+
+from pygments.lexers import guess_lexer
+from pygments.lexers.other import RexxLexer
+
+
+def _exampleFilePath(filename):
+    return os.path.join(os.path.dirname(__file__), 'examplefiles', filename)
+
+
+class AnalyseTextTest(unittest.TestCase):
+    def _testCanRecognizeAndGuessExampleFiles(self, lexer):
+        assert lexer is not None
+
+        for pattern in lexer.filenames:
+            exampleFilesPattern = _exampleFilePath(pattern)
+            for exampleFilePath in glob.glob(exampleFilesPattern):
+                exampleFile = open(exampleFilePath, 'rb')
+                try:
+                    text = exampleFile.read()
+                    probability = lexer.analyse_text(text)
+                    self.assertTrue(probability > 0,
+                        '%s must recognize %r' % (
+                        lexer.name, exampleFilePath))
+                    guessedLexer = guess_lexer(text)
+                    self.assertEqual(guessedLexer.name, lexer.name)
+                finally:
+                    exampleFile.close()
+
+    def testCanRecognizeAndGuessExampleFiles(self):
+        self._testCanRecognizeAndGuessExampleFiles(RexxLexer)
+
+
+class RexxLexerTest(unittest.TestCase):
+    def testCanGuessFromText(self):
+        self.assertAlmostEqual(0.01,
+            RexxLexer.analyse_text('/* */'))
+        self.assertAlmostEqual(1.0,
+            RexxLexer.analyse_text('''/* Rexx */
+                say "hello world"'''))
+        self.assertLess(0.5,
+            RexxLexer.analyse_text('/* */\n'
+                'hello:pRoceduRe\n'
+                '  say "hello world"'))
+        self.assertLess(0.2,
+            RexxLexer.analyse_text('''/* */
+                if 1 > 0 then do
+                    say "ok"
+                end
+                else do
+                    say "huh?"
+                end'''))
+        self.assertLess(0.2,
+            RexxLexer.analyse_text('''/* */
+                greeting = "hello world!"
+                parse value greeting "hello" name "!"
+                say name'''))

data/vendor/pygments-main/tests/test_perllexer.py

@@ -0,0 +1,137 @@
+# -*- coding: utf-8 -*-
+"""
+    Pygments regex lexer tests
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import time
+import unittest
+
+from pygments.token import String
+from pygments.lexers.agile import PerlLexer
+
+
+class RunawayRegexTest(unittest.TestCase):
+    # A previous version of the Perl lexer would spend a great deal of
+    # time backtracking when given particular strings.  These tests show that
+    # the runaway backtracking doesn't happen any more (at least for the given
+    # cases).
+
+    lexer = PerlLexer()
+
+    ### Test helpers.
+
+    def assert_single_token(self, s, token):
+        """Show that a given string generates only one token."""
+        tokens = list(self.lexer.get_tokens_unprocessed(s))
+        self.assertEqual(len(tokens), 1, tokens)
+        self.assertEqual(s, tokens[0][2])
+        self.assertEqual(token, tokens[0][1])
+
+    def assert_tokens(self, strings, expected_tokens):
+        """Show that a given string generates the expected tokens."""
+        tokens = list(self.lexer.get_tokens_unprocessed(''.join(strings)))
+        self.assertEqual(len(tokens), len(expected_tokens), tokens)
+        for index, s in enumerate(strings):
+            self.assertEqual(s, tokens[index][2])
+            self.assertEqual(expected_tokens[index], tokens[index][1])
+
+    def assert_fast_tokenization(self, s):
+        """Show that a given string is tokenized quickly."""
+        start = time.time()
+        tokens = list(self.lexer.get_tokens_unprocessed(s))
+        end = time.time()
+        # Isn't 10 seconds kind of a long time?  Yes, but we don't want false
+        # positives when the tests are starved for CPU time.
+        if end-start > 10:
+            self.fail('tokenization took too long')
+        return tokens
+
+    ### Strings.
+
+    def test_single_quote_strings(self):
+        self.assert_single_token(r"'foo\tbar\\\'baz'", String)
+        self.assert_fast_tokenization("'" + '\\'*999)
+
+    def test_double_quote_strings(self):
+        self.assert_single_token(r'"foo\tbar\\\"baz"', String)
+        self.assert_fast_tokenization('"' + '\\'*999)
+
+    def test_backtick_strings(self):
+        self.assert_single_token(r'`foo\tbar\\\`baz`', String.Backtick)
+        self.assert_fast_tokenization('`' + '\\'*999)
+
+    ### Regex matches with various delimiters.
+
+    def test_match(self):
+        self.assert_single_token(r'/aa\tbb/', String.Regex)
+        self.assert_fast_tokenization('/' + '\\'*999)
+
+    def test_match_with_slash(self):
+        self.assert_tokens(['m', '/\n\\t\\\\/'], [String.Regex, String.Regex])
+        self.assert_fast_tokenization('m/xxx\n' + '\\'*999)
+
+    def test_match_with_bang(self):
+        self.assert_tokens(['m', r'!aa\t\!bb!'], [String.Regex, String.Regex])
+        self.assert_fast_tokenization('m!' + '\\'*999)
+
+    def test_match_with_brace(self):
+        self.assert_tokens(['m', r'{aa\t\}bb}'], [String.Regex, String.Regex])
+        self.assert_fast_tokenization('m{' + '\\'*999)
+
+    def test_match_with_angle_brackets(self):
+        self.assert_tokens(['m', r'<aa\t\>bb>'], [String.Regex, String.Regex])
+        self.assert_fast_tokenization('m<' + '\\'*999)
+
+    def test_match_with_parenthesis(self):
+        self.assert_tokens(['m', r'(aa\t\)bb)'], [String.Regex, String.Regex])
+        self.assert_fast_tokenization('m(' + '\\'*999)
+
+    def test_match_with_at_sign(self):
+        self.assert_tokens(['m', r'@aa\t\@bb@'], [String.Regex, String.Regex])
+        self.assert_fast_tokenization('m@' + '\\'*999)
+
+    def test_match_with_percent_sign(self):
+        self.assert_tokens(['m', r'%aa\t\%bb%'], [String.Regex, String.Regex])
+        self.assert_fast_tokenization('m%' + '\\'*999)
+
+    def test_match_with_dollar_sign(self):
+        self.assert_tokens(['m', r'$aa\t\$bb$'], [String.Regex, String.Regex])
+        self.assert_fast_tokenization('m$' + '\\'*999)
+
+    ### Regex substitutions with various delimeters.
+
+    def test_substitution_with_slash(self):
+        self.assert_single_token('s/aaa/bbb/g', String.Regex)
+        self.assert_fast_tokenization('s/foo/' + '\\'*999)
+
+    def test_substitution_with_at_sign(self):
+        self.assert_single_token(r's@aaa@bbb@g', String.Regex)
+        self.assert_fast_tokenization('s@foo@' + '\\'*999)
+
+    def test_substitution_with_percent_sign(self):
+        self.assert_single_token(r's%aaa%bbb%g', String.Regex)
+        self.assert_fast_tokenization('s%foo%' + '\\'*999)
+
+    def test_substitution_with_brace(self):
+        self.assert_single_token(r's{aaa}', String.Regex)
+        self.assert_fast_tokenization('s{' + '\\'*999)
+
+    def test_substitution_with_angle_bracket(self):
+        self.assert_single_token(r's<aaa>', String.Regex)
+        self.assert_fast_tokenization('s<' + '\\'*999)
+
+    def test_substitution_with_angle_bracket(self):
+        self.assert_single_token(r's<aaa>', String.Regex)
+        self.assert_fast_tokenization('s<' + '\\'*999)
+
+    def test_substitution_with_square_bracket(self):
+        self.assert_single_token(r's[aaa]', String.Regex)
+        self.assert_fast_tokenization('s[' + '\\'*999)
+
+    def test_substitution_with_parenthesis(self):
+        self.assert_single_token(r's(aaa)', String.Regex)
+        self.assert_fast_tokenization('s(' + '\\'*999)

data/vendor/pygments-main/tests/test_regexlexer.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+"""
+    Pygments regex lexer tests
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+
+from pygments.token import Text
+from pygments.lexer import RegexLexer
+from pygments.lexer import bygroups
+
+
+class TestLexer(RegexLexer):
+    """Test tuple state transitions including #pop."""
+    tokens = {
+        'root': [
+            ('a', Text.Root, 'rag'),
+            ('e', Text.Root),
+        ],
+        'beer': [
+            ('d', Text.Beer, ('#pop', '#pop')),
+        ],
+        'rag': [
+            ('b', Text.Rag, '#push'),
+            ('c', Text.Rag, ('#pop', 'beer')),
+        ],
+    }
+
+
+class TupleTransTest(unittest.TestCase):
+    def test(self):
+        lx = TestLexer()
+        toks = list(lx.get_tokens_unprocessed('abcde'))
+        self.assertEqual(toks,
+           [(0, Text.Root, 'a'), (1, Text.Rag, 'b'), (2, Text.Rag, 'c'),
+            (3, Text.Beer, 'd'), (4, Text.Root, 'e')])
+
+    def test_multiline(self):
+        lx = TestLexer()
+        toks = list(lx.get_tokens_unprocessed('a\ne'))
+        self.assertEqual(toks,
+           [(0, Text.Root, 'a'), (1, Text, u'\n'),
+            (2, Text.Root, 'e')])

data/vendor/pygments-main/tests/test_token.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+"""
+    Test suite for the token module
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+
+from pygments import token
+
+
+class TokenTest(unittest.TestCase):
+
+    def test_tokentype(self):
+        e = self.assertEqual
+
+        t = token.String
+
+        e(t.split(), [token.Token, token.Literal, token.String])
+
+        e(t.__class__, token._TokenType)
+
+    def test_functions(self):
+        self.assertTrue(token.is_token_subtype(token.String, token.String))
+        self.assertTrue(token.is_token_subtype(token.String, token.Literal))
+        self.assertFalse(token.is_token_subtype(token.Literal, token.String))
+
+        self.assertTrue(token.string_to_tokentype(token.String) is token.String)
+        self.assertTrue(token.string_to_tokentype('') is token.Token)
+        self.assertTrue(token.string_to_tokentype('String') is token.String)
+
+    def test_sanity_check(self):
+        stp = token.STANDARD_TYPES.copy()
+        stp[token.Token] = '---' # Token and Text do conflict, that is okay
+        t = {}
+        for k, v in stp.iteritems():
+            t.setdefault(v, []).append(k)
+        if len(t) == len(stp):
+            return # Okay
+
+        for k, v in t.iteritems():
+            if len(v) > 1:
+                self.fail("%r has more than one key: %r" % (k, v))

data/vendor/pygments-main/tests/test_using_api.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+"""
+    Pygments tests for using()
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+
+from pygments.lexer import using, bygroups, this, RegexLexer
+from pygments.token import String, Text, Keyword
+
+class TestLexer(RegexLexer):
+    tokens = {
+        'root': [
+            (r'#.*',
+             using(this, state='invalid')),
+            (r'(")(.+?)(")',
+             bygroups(String, using(this, state='string'), String)),
+            (r'[^"]+', Text),
+        ],
+        'string': [
+            (r'.+', Keyword),
+        ],
+    }
+
+
+class UsingStateTest(unittest.TestCase):
+    def test_basic(self):
+        expected = [(Text, 'a'), (String, '"'), (Keyword, 'bcd'),
+                    (String, '"'), (Text, 'e\n')]
+        t = list(TestLexer().get_tokens('a"bcd"e'))
+        self.assertEqual(t, expected)
+
+    def test_error(self):
+        def gen():
+            return list(TestLexer().get_tokens('#a'))
+        self.assertRaises(KeyError, gen)

data/vendor/pygments-main/tests/test_util.py

@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+"""
+    Test suite for the util module
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+import unittest
+
+from pygments import util
+
+
+class FakeLexer(object):
+    def analyse(text):
+        return float(text)
+    analyse = util.make_analysator(analyse)
+
+
+class UtilTest(unittest.TestCase):
+
+    def test_getoptions(self):
+        raises = self.assertRaises
+        equals = self.assertEqual
+
+        equals(util.get_bool_opt({}, 'a', True), True)
+        equals(util.get_bool_opt({}, 'a', 1), True)
+        equals(util.get_bool_opt({}, 'a', 'true'), True)
+        equals(util.get_bool_opt({}, 'a', 'no'), False)
+        raises(util.OptionError, util.get_bool_opt, {}, 'a', [])
+        raises(util.OptionError, util.get_bool_opt, {}, 'a', 'foo')
+
+        equals(util.get_int_opt({}, 'a', 1), 1)
+        raises(util.OptionError, util.get_int_opt, {}, 'a', [])
+        raises(util.OptionError, util.get_int_opt, {}, 'a', 'bar')
+
+        equals(util.get_list_opt({}, 'a', [1]), [1])
+        equals(util.get_list_opt({}, 'a', '1 2'), ['1', '2'])
+        raises(util.OptionError, util.get_list_opt, {}, 'a', 1)
+
+
+    def test_docstring_headline(self):
+        def f1():
+            """
+            docstring headline
+
+            other text
+            """
+        def f2():
+            """
+            docstring
+            headline
+
+            other text
+            """
+
+        self.assertEqual(util.docstring_headline(f1), "docstring headline")
+        self.assertEqual(util.docstring_headline(f2), "docstring headline")
+
+    def test_analysator_returns_float(self):
+        # If an analysator wrapped by make_analysator returns a floating point
+        # number, then that number will be returned by the wrapper.
+        self.assertEqual(FakeLexer.analyse('0.5'), 0.5)
+
+    def test_analysator_returns_boolean(self):
+        # If an analysator wrapped by make_analysator returns a boolean value,
+        # then the wrapper will return 1.0 if the boolean was True or 0.0 if
+        # it was False.
+        self.assertEqual(FakeLexer.analyse(True), 1.0)
+        self.assertEqual(FakeLexer.analyse(False), 0.0)
+
+    def test_analysator_raises_exception(self):
+        # If an analysator wrapped by make_analysator raises an exception,
+        # then the wrapper will return 0.0.
+        class ErrorLexer(object):
+            def analyse(text):
+                raise RuntimeError('something bad happened')
+            analyse = util.make_analysator(analyse)
+        self.assertEqual(ErrorLexer.analyse(''), 0.0)
+
+    def test_analysator_value_error(self):
+        # When converting the analysator's return value to a float a
+        # ValueError may occur.  If that happens 0.0 is returned instead.
+        self.assertEqual(FakeLexer.analyse('bad input'), 0.0)
+
+    def test_analysator_type_error(self):
+        # When converting the analysator's return value to a float a
+        # TypeError may occur.  If that happens 0.0 is returned instead.
+        self.assertEqual(FakeLexer.analyse(None), 0.0)
+
+    def test_shebang_matches(self):
+        self.assertTrue(util.shebang_matches('#!/usr/bin/env python', r'python(2\.\d)?'))
+        self.assertTrue(util.shebang_matches('#!/usr/bin/python2.4', r'python(2\.\d)?'))
+        self.assertTrue(util.shebang_matches('#!/usr/bin/startsomethingwith python',
+                                             r'python(2\.\d)?'))
+        self.assertTrue(util.shebang_matches('#!C:\\Python2.4\\Python.exe',
+                                             r'python(2\.\d)?'))
+
+        self.assertFalse(util.shebang_matches('#!/usr/bin/python-ruby',
+                                              r'python(2\.\d)?'))
+        self.assertFalse(util.shebang_matches('#!/usr/bin/python/ruby',
+                                              r'python(2\.\d)?'))
+        self.assertFalse(util.shebang_matches('#!', r'python'))
+
+    def test_doctype_matches(self):
+        self.assertTrue(util.doctype_matches(
+            '<!DOCTYPE html PUBLIC "a"> <html>', 'html.*'))
+        self.assertFalse(util.doctype_matches(
+            '<?xml ?> <DOCTYPE html PUBLIC "a"> <html>', 'html.*'))
+        self.assertTrue(util.html_doctype_matches(
+            '<?xml ?><!DOCTYPE html PUBLIC  "-//W3C//DTD XHTML 1.0 Strict//EN">'))
+
+    def test_xml(self):
+        self.assertTrue(util.looks_like_xml(
+            '<?xml ?><!DOCTYPE html PUBLIC  "-//W3C//DTD XHTML 1.0 Strict//EN">'))
+        self.assertTrue(util.looks_like_xml('<html xmlns>abc</html>'))
+        self.assertFalse(util.looks_like_xml('<html>'))
+
+    def test_unirange(self):
+        first_non_bmp = u'\U00010000'
+        r = re.compile(util.unirange(0x10000, 0x20000))
+        m = r.match(first_non_bmp)
+        self.assertTrue(m)
+        self.assertEquals(m.end(), len(first_non_bmp))
+        self.assertFalse(r.match(u'\uffff'))
+        self.assertFalse(r.match(u'xxx'))
+        # Tests that end is inclusive
+        r = re.compile(util.unirange(0x10000, 0x10000) + '+')
+        # Tests that the plus works for the entire unicode point, if narrow
+        # build
+        m = r.match(first_non_bmp * 2)
+        self.assertTrue(m)
+        self.assertEquals(m.end(), len(first_non_bmp) * 2)