pygments.rb 0.5.4 → 0.6.0

data/vendor/pygments-main/tests/test_cmdline.py

@@ -3,17 +3,18 @@
     Command line test
     ~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 # Test the command line interface
 
-import sys, os
+import io
+import sys
 import unittest
-import StringIO
 
 from pygments import highlight
+from pygments.util import StringIO
 from pygments.cmdline import main as cmdline_main
 
 import support
@@ -24,8 +25,8 @@ TESTFILE, TESTDIR = support.location(__file__)
 def run_cmdline(*args):
     saved_stdout = sys.stdout
     saved_stderr = sys.stderr
-    new_stdout = sys.stdout = StringIO.StringIO()
-    new_stderr = sys.stderr = StringIO.StringIO()
+    new_stdout = sys.stdout = StringIO()
+    new_stderr = sys.stderr = StringIO()
     try:
         ret = cmdline_main(["pygmentize"] + list(args))
     finally:

data/vendor/pygments-main/tests/test_examplefiles.py

@@ -3,18 +3,20 @@
     Pygments tests with example files
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+from __future__ import print_function
+
 import os
 import pprint
 import difflib
-import cPickle as pickle
+import pickle
 
 from pygments.lexers import get_lexer_for_filename, get_lexer_by_name
 from pygments.token import Error
-from pygments.util import ClassNotFound, b
+from pygments.util import ClassNotFound
 
 STORE_OUTPUT = False
 
@@ -31,21 +33,30 @@ def test_example_files():
         absfn = os.path.join(testdir, 'examplefiles', fn)
         if not os.path.isfile(absfn):
             continue
-        outfn = os.path.join(outdir, fn)
 
+        print(absfn)
+        code = open(absfn, 'rb').read()
         try:
-            lx = get_lexer_for_filename(absfn)
-        except ClassNotFound:
-            if "_" not in fn:
+            code = code.decode('utf-8')
+        except UnicodeError:
+            code = code.decode('latin1')
+
+        outfn = os.path.join(outdir, fn)
+
+        lx = None
+        if '_' in fn:
+            try:
+                lx = get_lexer_by_name(fn.split('_')[0])
+            except ClassNotFound:
+                pass
+        if lx is None:
+            try:
+                lx = get_lexer_for_filename(absfn, code=code)
+            except ClassNotFound:
                 raise AssertionError('file %r has no registered extension, '
                                      'nor is of the form <lexer>_filename '
                                      'for overriding, thus no lexer found.'
-                                    % fn)
-            try:
-                name, rest = fn.split("_", 1)
-                lx = get_lexer_by_name(name)
-            except ClassNotFound:
-                raise AssertionError('no lexer found for file %r' % fn)
+                                     % fn)
         yield check_lexer, lx, absfn, outfn
 
 def check_lexer(lx, absfn, outfn):
@@ -54,8 +65,8 @@ def check_lexer(lx, absfn, outfn):
         text = fp.read()
     finally:
         fp.close()
-    text = text.replace(b('\r\n'), b('\n'))
-    text = text.strip(b('\n')) + b('\n')
+    text = text.replace(b'\r\n', b'\n')
+    text = text.strip(b'\n') + b'\n'
     try:
         text = text.decode('utf-8')
         if text.startswith(u'\ufeff'):
@@ -71,8 +82,8 @@ def check_lexer(lx, absfn, outfn):
             (lx, absfn, val, len(u''.join(ntext)))
         tokens.append((type, val))
     if u''.join(ntext) != text:
-        print '\n'.join(difflib.unified_diff(u''.join(ntext).splitlines(),
-                                             text.splitlines()))
+        print('\n'.join(difflib.unified_diff(u''.join(ntext).splitlines(),
+                                             text.splitlines())))
         raise AssertionError('round trip failed for ' + absfn)
 
     # check output against previous run if enabled
@@ -94,6 +105,6 @@ def check_lexer(lx, absfn, outfn):
         if stored_tokens != tokens:
             f1 = pprint.pformat(stored_tokens)
             f2 = pprint.pformat(tokens)
-            print '\n'.join(difflib.unified_diff(f1.splitlines(),
-                                                 f2.splitlines()))
+            print('\n'.join(difflib.unified_diff(f1.splitlines(),
+                                                 f2.splitlines())))
             assert False, absfn

data/vendor/pygments-main/tests/test_html_formatter.py

@@ -3,27 +3,29 @@
     Pygments HTML formatter tests
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+from __future__ import print_function
+
+import io
 import os
 import re
 import unittest
-import StringIO
 import tempfile
 from os.path import join, dirname, isfile
 
+from pygments.util import StringIO
 from pygments.lexers import PythonLexer
 from pygments.formatters import HtmlFormatter, NullFormatter
 from pygments.formatters.html import escape_html
-from pygments.util import uni_open
 
 import support
 
 TESTFILE, TESTDIR = support.location(__file__)
 
-fp = uni_open(TESTFILE, encoding='utf-8')
+fp = io.open(TESTFILE, encoding='utf-8')
 try:
     tokensource = list(PythonLexer().get_tokens(fp.read()))
 finally:
@@ -33,11 +35,11 @@ finally:
 class HtmlFormatterTest(unittest.TestCase):
     def test_correct_output(self):
         hfmt = HtmlFormatter(nowrap=True)
-        houtfile = StringIO.StringIO()
+        houtfile = StringIO()
         hfmt.format(tokensource, houtfile)
 
         nfmt = NullFormatter()
-        noutfile = StringIO.StringIO()
+        noutfile = StringIO()
         nfmt.format(tokensource, noutfile)
 
         stripped_html = re.sub('<.*?>', '', houtfile.getvalue())
@@ -74,13 +76,13 @@ class HtmlFormatterTest(unittest.TestCase):
                         dict(linenos=True, full=True),
                         dict(linenos=True, full=True, noclasses=True)]:
 
-            outfile = StringIO.StringIO()
+            outfile = StringIO()
             fmt = HtmlFormatter(**optdict)
             fmt.format(tokensource, outfile)
 
     def test_linenos(self):
         optdict = dict(linenos=True)
-        outfile = StringIO.StringIO()
+        outfile = StringIO()
         fmt = HtmlFormatter(**optdict)
         fmt.format(tokensource, outfile)
         html = outfile.getvalue()
@@ -88,7 +90,7 @@ class HtmlFormatterTest(unittest.TestCase):
 
     def test_linenos_with_startnum(self):
         optdict = dict(linenos=True, linenostart=5)
-        outfile = StringIO.StringIO()
+        outfile = StringIO()
         fmt = HtmlFormatter(**optdict)
         fmt.format(tokensource, outfile)
         html = outfile.getvalue()
@@ -96,7 +98,7 @@ class HtmlFormatterTest(unittest.TestCase):
 
     def test_lineanchors(self):
         optdict = dict(lineanchors="foo")
-        outfile = StringIO.StringIO()
+        outfile = StringIO()
         fmt = HtmlFormatter(**optdict)
         fmt.format(tokensource, outfile)
         html = outfile.getvalue()
@@ -104,7 +106,7 @@ class HtmlFormatterTest(unittest.TestCase):
 
     def test_lineanchors_with_startnum(self):
         optdict = dict(lineanchors="foo", linenostart=5)
-        outfile = StringIO.StringIO()
+        outfile = StringIO()
         fmt = HtmlFormatter(**optdict)
         fmt.format(tokensource, outfile)
         html = outfile.getvalue()
@@ -132,7 +134,7 @@ class HtmlFormatterTest(unittest.TestCase):
             pass
         else:
             if ret:
-                print output
+                print(output)
             self.assertFalse(ret, 'nsgmls run reported errors')
 
         os.unlink(pathname)
@@ -172,7 +174,7 @@ class HtmlFormatterTest(unittest.TestCase):
             # anymore in the actual source
             fmt = HtmlFormatter(tagsfile='support/tags', lineanchors='L',
                                 tagurlformat='%(fname)s%(fext)s')
-            outfile = StringIO.StringIO()
+            outfile = StringIO()
             fmt.format(tokensource, outfile)
             self.assertTrue('<a href="test_html_formatter.py#L-165">test_ctags</a>'
                             in outfile.getvalue())

data/vendor/pygments-main/tests/test_latex_formatter.py

@@ -3,10 +3,12 @@
     Pygments LaTeX formatter tests
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+from __future__ import print_function
+
 import os
 import unittest
 import tempfile
@@ -48,7 +50,7 @@ class LatexFormatterTest(unittest.TestCase):
             pass
         else:
             if ret:
-                print output
+                print(output)
             self.assertFalse(ret, 'latex run reported errors')
 
         os.unlink(pathname)

data/vendor/pygments-main/tests/test_lexers_other.py

@@ -3,7 +3,7 @@
     Tests for other lexers
     ~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -28,7 +28,7 @@ class AnalyseTextTest(unittest.TestCase):
             for exampleFilePath in glob.glob(exampleFilesPattern):
                 exampleFile = open(exampleFilePath, 'rb')
                 try:
-                    text = exampleFile.read()
+                    text = exampleFile.read().decode('utf-8')
                     probability = lexer.analyse_text(text)
                     self.assertTrue(probability > 0,
                         '%s must recognize %r' % (
@@ -49,20 +49,20 @@ class RexxLexerTest(unittest.TestCase):
         self.assertAlmostEqual(1.0,
             RexxLexer.analyse_text('''/* Rexx */
                 say "hello world"'''))
-        self.assertLess(0.5,
-            RexxLexer.analyse_text('/* */\n'
+        val = RexxLexer.analyse_text('/* */\n'
                 'hello:pRoceduRe\n'
-                '  say "hello world"'))
-        self.assertLess(0.2,
-            RexxLexer.analyse_text('''/* */
+                '  say "hello world"')
+        self.assertTrue(val > 0.5, val)
+        val = RexxLexer.analyse_text('''/* */
                 if 1 > 0 then do
                     say "ok"
                 end
                 else do
                     say "huh?"
-                end'''))
-        self.assertLess(0.2,
-            RexxLexer.analyse_text('''/* */
+                end''')
+        self.assertTrue(val > 0.2, val)
+        val = RexxLexer.analyse_text('''/* */
                 greeting = "hello world!"
                 parse value greeting "hello" name "!"
-                say name'''))
+                say name''')
+        self.assertTrue(val > 0.2, val)

data/vendor/pygments-main/tests/test_objectiveclexer.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+"""
+    Basic CLexer Test
+    ~~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import unittest
+import os
+
+from pygments.token import Token
+from pygments.lexers import ObjectiveCLexer
+
+
+class ObjectiveCLexerTest(unittest.TestCase):
+
+    def setUp(self):
+        self.lexer = ObjectiveCLexer()
+
+    def testLiteralNumberInt(self):
+        fragment = u'@(1);\n'
+        expected = [
+            (Token.Literal, u'@('),
+            (Token.Literal.Number.Integer, u'1'),
+            (Token.Literal, u')'),
+            (Token.Punctuation, u';'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(expected, list(self.lexer.get_tokens(fragment)))
+
+    def testLiteralNumberExpression(self):
+        fragment = u'@(1+2);\n'
+        expected = [
+            (Token.Literal, u'@('),
+            (Token.Literal.Number.Integer, u'1'),
+            (Token.Operator, u'+'),
+            (Token.Literal.Number.Integer, u'2'),
+            (Token.Literal, u')'),
+            (Token.Punctuation, u';'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(expected, list(self.lexer.get_tokens(fragment)))
+
+    def testLiteralNumberNestedExpression(self):
+        fragment = u'@(1+(2+3));\n'
+        expected = [
+            (Token.Literal, u'@('),
+            (Token.Literal.Number.Integer, u'1'),
+            (Token.Operator, u'+'),
+            (Token.Punctuation, u'('),
+            (Token.Literal.Number.Integer, u'2'),
+            (Token.Operator, u'+'),
+            (Token.Literal.Number.Integer, u'3'),
+            (Token.Punctuation, u')'),
+            (Token.Literal, u')'),
+            (Token.Punctuation, u';'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(expected, list(self.lexer.get_tokens(fragment)))
+
+    def testLiteralNumberBool(self):
+        fragment = u'@NO;\n'
+        expected = [
+            (Token.Literal.Number, u'@NO'),
+            (Token.Punctuation, u';'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(expected, list(self.lexer.get_tokens(fragment)))
+
+    def testLieralNumberBoolExpression(self):
+        fragment = u'@(YES);\n'
+        expected = [
+            (Token.Literal, u'@('),
+            (Token.Name.Builtin, u'YES'),
+            (Token.Literal, u')'),
+            (Token.Punctuation, u';'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(expected, list(self.lexer.get_tokens(fragment)))

data/vendor/pygments-main/tests/test_perllexer.py

@@ -3,7 +3,7 @@
     Pygments regex lexer tests
     ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/tests/test_qbasiclexer.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+"""
+    Tests for QBasic
+    ~~~~~~~~~~~~~~~~
+
+    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import glob
+import os
+import unittest
+
+from pygments.token import Token
+from pygments.lexers.qbasic import QBasicLexer
+
+class QBasicTest(unittest.TestCase):
+    def setUp(self):
+        self.lexer = QBasicLexer()
+        self.maxDiff = None
+
+    def testKeywordsWithDollar(self):
+        fragment = u'DIM x\nx = RIGHT$("abc", 1)\n'
+        expected = [
+            (Token.Keyword.Declaration, u'DIM'),
+            (Token.Text.Whitespace, u' '),
+            (Token.Name.Variable.Global, u'x'),
+            (Token.Text, u'\n'),
+            (Token.Name.Variable.Global, u'x'),
+            (Token.Text.Whitespace, u' '),
+            (Token.Operator, u'='),
+            (Token.Text.Whitespace, u' '),
+            (Token.Keyword.Reserved, u'RIGHT$'),
+            (Token.Punctuation, u'('),
+            (Token.Literal.String.Double, u'"abc"'),
+            (Token.Punctuation, u','),
+            (Token.Text.Whitespace, u' '),
+            (Token.Literal.Number.Integer.Long, u'1'),
+            (Token.Punctuation, u')'),
+            (Token.Text, u'\n'),
+        ]
+        self.assertEqual(expected, list(self.lexer.get_tokens(fragment)))
+

data/vendor/pygments-main/tests/test_regexlexer.py

@@ -3,7 +3,7 @@
     Pygments regex lexer tests
     ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -12,6 +12,7 @@ import unittest
 from pygments.token import Text
 from pygments.lexer import RegexLexer
 from pygments.lexer import bygroups
+from pygments.lexer import default
 
 
 class TestLexer(RegexLexer):
@@ -20,6 +21,7 @@ class TestLexer(RegexLexer):
         'root': [
             ('a', Text.Root, 'rag'),
             ('e', Text.Root),
+            default(('beer', 'beer'))
         ],
         'beer': [
             ('d', Text.Beer, ('#pop', '#pop')),
@@ -45,3 +47,8 @@ class TupleTransTest(unittest.TestCase):
         self.assertEqual(toks,
            [(0, Text.Root, 'a'), (1, Text, u'\n'),
             (2, Text.Root, 'e')])
+
+    def test_default(self):
+        lx = TestLexer()
+        toks = list(lx.get_tokens_unprocessed('d'))
+        self.assertEqual(toks, [(0, Text.Beer, 'd')])