pygments.rb 0.6.3 → 1.0.0

data/vendor/pygments-main/pygments/util.py

@@ -5,7 +5,7 @@
 
     Utility functions.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -17,18 +17,19 @@ split_path_re = re.compile(r'[/\\ ]')
 doctype_lookup_re = re.compile(r'''(?smx)
     (<\?.*?\?>)?\s*
     <!DOCTYPE\s+(
+     [a-zA-Z_][a-zA-Z0-9]*
+     (?: \s+      # optional in HTML5
      [a-zA-Z_][a-zA-Z0-9]*\s+
-     [a-zA-Z_][a-zA-Z0-9]*\s+
-     "[^"]*")
+     "[^"]*")?
+     )
      [^>]*>
 ''')
 tag_re = re.compile(r'<(.+?)(\s.*?)?>.*?</.+?>(?uism)')
+xml_decl_re = re.compile(r'\s*<\?xml[^>]*\?>', re.I)
 
 
 class ClassNotFound(ValueError):
-    """
-    If one of the get_*_by_* functions didn't find a matching class.
-    """
+    """Raised if one of the lookup functions didn't find a matching class."""
 
 
 class OptionError(Exception):
@@ -54,7 +55,7 @@ def get_bool_opt(options, optname, default=None):
     elif not isinstance(string, string_types):
         raise OptionError('Invalid type %r for option %s; use '
                           '1/0, yes/no, true/false, on/off' % (
-                          string, optname))
+                              string, optname))
     elif string.lower() in ('1', 'yes', 'true', 'on'):
         return True
     elif string.lower() in ('0', 'no', 'false', 'off'):
@@ -62,7 +63,7 @@ def get_bool_opt(options, optname, default=None):
     else:
         raise OptionError('Invalid value %r for option %s; use '
                           '1/0, yes/no, true/false, on/off' % (
-                          string, optname))
+                              string, optname))
 
 
 def get_int_opt(options, optname, default=None):
@@ -72,11 +73,11 @@ def get_int_opt(options, optname, default=None):
     except TypeError:
         raise OptionError('Invalid type %r for option %s; you '
                           'must give an integer value' % (
-                          string, optname))
+                              string, optname))
     except ValueError:
         raise OptionError('Invalid value %r for option %s; you '
                           'must give an integer value' % (
-                          string, optname))
+                              string, optname))
 
 
 def get_list_opt(options, optname, default=None):
@@ -88,7 +89,7 @@ def get_list_opt(options, optname, default=None):
     else:
         raise OptionError('Invalid type %r for option %s; you '
                           'must give a list value' % (
-                          val, optname))
+                              val, optname))
 
 
 def docstring_headline(obj):
@@ -104,10 +105,7 @@ def docstring_headline(obj):
 
 
 def make_analysator(f):
-    """
-    Return a static text analysation function that
-    returns float values.
-    """
+    """Return a static text analyser function that returns float values."""
     def text_analyse(text):
         try:
             rv = f(text)
@@ -124,8 +122,7 @@ def make_analysator(f):
 
 
 def shebang_matches(text, regex):
-    """
-    Check if the given regular expression matches the last part of the
+    r"""Check if the given regular expression matches the last part of the
     shebang if one exists.
 
         >>> from pygments.util import shebang_matches
@@ -163,15 +160,15 @@ def shebang_matches(text, regex):
                      if x and not x.startswith('-')][-1]
         except IndexError:
             return False
-        regex = re.compile('^%s(\.(exe|cmd|bat|bin))?$' % regex, re.IGNORECASE)
+        regex = re.compile(r'^%s(\.(exe|cmd|bat|bin))?$' % regex, re.IGNORECASE)
         if regex.search(found) is not None:
             return True
     return False
 
 
 def doctype_matches(text, regex):
-    """
-    Check if the doctype matches a regular expression (if present).
+    """Check if the doctype matches a regular expression (if present).
+
     Note that this method only checks the first part of a DOCTYPE.
     eg: 'html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"'
     """
@@ -179,21 +176,21 @@ def doctype_matches(text, regex):
     if m is None:
         return False
     doctype = m.group(2)
-    return re.compile(regex).match(doctype.strip()) is not None
+    return re.compile(regex, re.I).match(doctype.strip()) is not None
 
 
 def html_doctype_matches(text):
-    """
-    Check if the file looks like it has a html doctype.
-    """
-    return doctype_matches(text, r'html\s+PUBLIC\s+"-//W3C//DTD X?HTML.*')
+    """Check if the file looks like it has a html doctype."""
+    return doctype_matches(text, r'html')
 
 
 _looks_like_xml_cache = {}
+
+
 def looks_like_xml(text):
-    """
-    Check if a doctype exists or if we have some tags.
-    """
+    """Check if a doctype exists or if we have some tags."""
+    if xml_decl_re.match(text):
+        return True
     key = hash(text)
     try:
         return _looks_like_xml_cache[key]
@@ -205,6 +202,7 @@ def looks_like_xml(text):
         _looks_like_xml_cache[key] = rv
         return rv
 
+
 # Python narrow build compatibility
 
 def _surrogatepair(c):
@@ -215,10 +213,9 @@ def _surrogatepair(c):
     # http://www.unicode.org/book/ch03.pdf
     return (0xd7c0 + (c >> 10), (0xdc00 + (c & 0x3ff)))
 
+
 def unirange(a, b):
-    """
-    Returns a regular expression string to match the given non-BMP range.
-    """
+    """Returns a regular expression string to match the given non-BMP range."""
     if b < a:
         raise ValueError("Bad character range")
     if a < 0x10000 or b < 0x10000:
@@ -255,6 +252,98 @@ def unirange(a, b):
 
             return u'(?:' + u'|'.join(buf) + u')'
 
+
+def format_lines(var_name, seq, raw=False, indent_level=0):
+    """Formats a sequence of strings for output."""
+    lines = []
+    base_indent = ' ' * indent_level * 4
+    inner_indent = ' ' * (indent_level + 1) * 4
+    lines.append(base_indent + var_name + ' = (')
+    if raw:
+        # These should be preformatted reprs of, say, tuples.
+        for i in seq:
+            lines.append(inner_indent + i + ',')
+    else:
+        for i in seq:
+            # Force use of single quotes
+            r = repr(i + '"')
+            lines.append(inner_indent + r[:-2] + r[-1] + ',')
+    lines.append(base_indent + ')')
+    return '\n'.join(lines)
+
+
+def duplicates_removed(it, already_seen=()):
+    """
+    Returns a list with duplicates removed from the iterable `it`.
+
+    Order is preserved.
+    """
+    lst = []
+    seen = set()
+    for i in it:
+        if i in seen or i in already_seen:
+            continue
+        lst.append(i)
+        seen.add(i)
+    return lst
+
+
+class Future(object):
+    """Generic class to defer some work.
+
+    Handled specially in RegexLexerMeta, to support regex string construction at
+    first use.
+    """
+    def get(self):
+        raise NotImplementedError
+
+
+def guess_decode(text):
+    """Decode *text* with guessed encoding.
+
+    First try UTF-8; this should fail for non-UTF-8 encodings.
+    Then try the preferred locale encoding.
+    Fall back to latin-1, which always works.
+    """
+    try:
+        text = text.decode('utf-8')
+        return text, 'utf-8'
+    except UnicodeDecodeError:
+        try:
+            import locale
+            prefencoding = locale.getpreferredencoding()
+            text = text.decode()
+            return text, prefencoding
+        except (UnicodeDecodeError, LookupError):
+            text = text.decode('latin1')
+            return text, 'latin1'
+
+
+def guess_decode_from_terminal(text, term):
+    """Decode *text* coming from terminal *term*.
+
+    First try the terminal encoding, if given.
+    Then try UTF-8.  Then try the preferred locale encoding.
+    Fall back to latin-1, which always works.
+    """
+    if getattr(term, 'encoding', None):
+        try:
+            text = text.decode(term.encoding)
+        except UnicodeDecodeError:
+            pass
+        else:
+            return text, term.encoding
+    return guess_decode(text)
+
+
+def terminal_encoding(term):
+    """Return our best guess of encoding for the given *term*."""
+    if getattr(term, 'encoding', None):
+        return term.encoding
+    import locale
+    return locale.getpreferredencoding()
+
+
 # Python 2/3 compatibility
 
 if sys.version_info < (3, 0):
@@ -265,7 +354,8 @@ if sys.version_info < (3, 0):
     u_prefix = 'u'
     iteritems = dict.iteritems
     itervalues = dict.itervalues
-    import StringIO, cStringIO
+    import StringIO
+    import cStringIO
     # unfortunately, io.StringIO in Python 2 doesn't accept str at all
     StringIO = StringIO.StringIO
     BytesIO = cStringIO.StringIO
@@ -277,7 +367,13 @@ else:
     u_prefix = ''
     iteritems = dict.items
     itervalues = dict.values
-    from io import StringIO, BytesIO
+    from io import StringIO, BytesIO, TextIOWrapper
+
+    class UnclosingTextIOWrapper(TextIOWrapper):
+        # Don't close underlying buffer on destruction.
+        def close(self):
+            self.flush()
+
 
 def add_metaclass(metaclass):
     """Class decorator for creating a class with a metaclass."""

data/vendor/pygments-main/requirements.txt

@@ -0,0 +1,5 @@
+coverage
+nose
+pyflakes
+pylint
+tox

data/vendor/pygments-main/scripts/check_sources.py

@@ -7,7 +7,7 @@
     Make sure each Python file has a correct file header
     including copyright and license information.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -23,8 +23,10 @@ from os.path import join, splitext, abspath
 
 checkers = {}
 
+
 def checker(*suffixes, **kwds):
     only_pkg = kwds.pop('only_pkg', False)
+
     def deco(func):
         for suffix in suffixes:
             checkers.setdefault(suffix, []).append(func)
@@ -34,58 +36,42 @@ def checker(*suffixes, **kwds):
 
 
 name_mail_re = r'[\w ]+(<.*?>)?'
-copyright_re = re.compile(r'^    :copyright: Copyright 2006-2014 by '
+copyright_re = re.compile(r'^    :copyright: Copyright 2006-2015 by '
                           r'the Pygments team, see AUTHORS\.$', re.UNICODE)
 copyright_2_re = re.compile(r'^                %s(, %s)*[,.]$' %
                             (name_mail_re, name_mail_re), re.UNICODE)
-coding_re    = re.compile(r'coding[:=]\s*([-\w.]+)')
-not_ix_re    = re.compile(r'\bnot\s+\S+?\s+i[sn]\s\S+')
 is_const_re  = re.compile(r'if.*?==\s+(None|False|True)\b')
 
 misspellings = ["developement", "adress", "verificate",  # ALLOW-MISSPELLING
-                "informations"]                          # ALLOW-MISSPELLING
+                "informations", "unlexer"]               # ALLOW-MISSPELLING
 
 
 @checker('.py')
 def check_syntax(fn, lines):
+    if '#!/' in lines[0]:
+        lines = lines[1:]
+    if 'coding:' in lines[0]:
+        lines = lines[1:]
     try:
-        compile(''.join(lines), fn, "exec")
+        compile('\n'.join(lines), fn, "exec")
     except SyntaxError as err:
         yield 0, "not compilable: %s" % err
 
 
 @checker('.py')
 def check_style_and_encoding(fn, lines):
-    encoding = 'ascii'
     for lno, line in enumerate(lines):
-        if len(line) > 90:
+        if len(line) > 110:
             yield lno+1, "line too long"
-        m = not_ix_re.search(line)
-        if m:
-            yield lno+1, '"' + m.group() + '"'
         if is_const_re.search(line):
             yield lno+1, 'using == None/True/False'
-        if lno < 2:
-            co = coding_re.search(line)
-            if co:
-                encoding = co.group(1)
-        try:
-            line.decode(encoding)
-        except AttributeError:
-            # Python 3 - encoding was already checked
-            pass
-        except UnicodeDecodeError as err:
-            yield lno+1, "not decodable: %s\n   Line: %r" % (err, line)
-        except LookupError as err:
-            yield 0, "unknown encoding: %s" % encoding
-            encoding = 'latin1'
 
 
 @checker('.py', only_pkg=True)
 def check_fileheader(fn, lines):
     # line number correction
     c = 1
-    if lines[0:1] == ['#!/usr/bin/env python\n']:
+    if lines[0:1] == ['#!/usr/bin/env python']:
         lines = lines[1:]
         c = 2
 
@@ -94,31 +80,28 @@ def check_fileheader(fn, lines):
     for lno, l in enumerate(lines):
         llist.append(l)
         if lno == 0:
-            if l == '# -*- coding: rot13 -*-\n':
-                # special-case pony package
-                return
-            elif l != '# -*- coding: utf-8 -*-\n':
+            if l != '# -*- coding: utf-8 -*-':
                 yield 1, "missing coding declaration"
         elif lno == 1:
-            if l != '"""\n' and l != 'r"""\n':
+            if l != '"""' and l != 'r"""':
                 yield 2, 'missing docstring begin (""")'
             else:
                 docopen = True
         elif docopen:
-            if l == '"""\n':
+            if l == '"""':
                 # end of docstring
                 if lno <= 4:
                     yield lno+c, "missing module name in docstring"
                 break
 
-            if l != "\n" and l[:4] != '    ' and docopen:
+            if l != "" and l[:4] != '    ' and docopen:
                 yield lno+c, "missing correct docstring indentation"
 
             if lno == 2:
                 # if not in package, don't check the module name
                 modname = fn[:-3].replace('/', '.').replace('.__init__', '')
                 while modname:
-                    if l.lower()[4:-1] == modname:
+                    if l.lower()[4:] == modname:
                         break
                     modname = '.'.join(modname.split('.')[1:])
                 else:
@@ -133,7 +116,7 @@ def check_fileheader(fn, lines):
 
     # check for copyright and license fields
     license = llist[-2:-1]
-    if license != ["    :license: BSD, see LICENSE for details.\n"]:
+    if license != ["    :license: BSD, see LICENSE for details."]:
         yield 0, "no correct license info"
 
     ci = -3
@@ -176,16 +159,19 @@ def main(argv):
     for root, dirs, files in os.walk(path):
         if '.hg' in dirs:
             dirs.remove('.hg')
+        if 'examplefiles' in dirs:
+            dirs.remove('examplefiles')
         if '-i' in opts and abspath(root) in opts['-i']:
             del dirs[:]
             continue
         # XXX: awkward: for the Makefile call: don't check non-package
         #      files for file headers
-        in_pocoo_pkg = root.startswith('./pygments')
+        in_pygments_pkg = root.startswith('./pygments')
         for fn in files:
 
             fn = join(root, fn)
-            if fn[:2] == './': fn = fn[2:]
+            if fn[:2] == './':
+                fn = fn[2:]
 
             if '-i' in opts and abspath(fn) in opts['-i']:
                 continue
@@ -199,15 +185,14 @@ def main(argv):
                 print("Checking %s..." % fn)
 
             try:
-                f = open(fn, 'r')
-                lines = list(f)
+                lines = open(fn, 'rb').read().decode('utf-8').splitlines()
             except (IOError, OSError) as err:
                 print("%s: cannot open: %s" % (fn, err))
                 num += 1
                 continue
 
             for checker in checkerlist:
-                if not in_pocoo_pkg and checker.only_pkg:
+                if not in_pygments_pkg and checker.only_pkg:
                     continue
                 for lno, msg in checker(fn, lines):
                     print(u"%s:%d: %s" % (fn, lno, msg), file=out)