mortar-pygments.rb 0.5.5

data/vendor/pygments-main/pygments/util.py

@@ -0,0 +1,277 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.util
+    ~~~~~~~~~~~~~
+
+    Utility functions.
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+import sys
+import codecs
+
+
+split_path_re = re.compile(r'[/\\ ]')
+doctype_lookup_re = re.compile(r'''(?smx)
+    (<\?.*?\?>)?\s*
+    <!DOCTYPE\s+(
+     [a-zA-Z_][a-zA-Z0-9]*\s+
+     [a-zA-Z_][a-zA-Z0-9]*\s+
+     "[^"]*")
+     [^>]*>
+''')
+tag_re = re.compile(r'<(.+?)(\s.*?)?>.*?</.+?>(?uism)')
+
+
+class ClassNotFound(ValueError):
+    """
+    If one of the get_*_by_* functions didn't find a matching class.
+    """
+
+
+class OptionError(Exception):
+    pass
+
+
+def get_choice_opt(options, optname, allowed, default=None, normcase=False):
+    string = options.get(optname, default)
+    if normcase:
+        string = string.lower()
+    if string not in allowed:
+        raise OptionError('Value for option %s must be one of %s' %
+                          (optname, ', '.join(map(str, allowed))))
+    return string
+
+
+def get_bool_opt(options, optname, default=None):
+    string = options.get(optname, default)
+    if isinstance(string, bool):
+        return string
+    elif isinstance(string, int):
+        return bool(string)
+    elif not isinstance(string, basestring):
+        raise OptionError('Invalid type %r for option %s; use '
+                          '1/0, yes/no, true/false, on/off' % (
+                          string, optname))
+    elif string.lower() in ('1', 'yes', 'true', 'on'):
+        return True
+    elif string.lower() in ('0', 'no', 'false', 'off'):
+        return False
+    else:
+        raise OptionError('Invalid value %r for option %s; use '
+                          '1/0, yes/no, true/false, on/off' % (
+                          string, optname))
+
+
+def get_int_opt(options, optname, default=None):
+    string = options.get(optname, default)
+    try:
+        return int(string)
+    except TypeError:
+        raise OptionError('Invalid type %r for option %s; you '
+                          'must give an integer value' % (
+                          string, optname))
+    except ValueError:
+        raise OptionError('Invalid value %r for option %s; you '
+                          'must give an integer value' % (
+                          string, optname))
+
+
+def get_list_opt(options, optname, default=None):
+    val = options.get(optname, default)
+    if isinstance(val, basestring):
+        return val.split()
+    elif isinstance(val, (list, tuple)):
+        return list(val)
+    else:
+        raise OptionError('Invalid type %r for option %s; you '
+                          'must give a list value' % (
+                          val, optname))
+
+
+def docstring_headline(obj):
+    if not obj.__doc__:
+        return ''
+    res = []
+    for line in obj.__doc__.strip().splitlines():
+        if line.strip():
+            res.append(" " + line.strip())
+        else:
+            break
+    return ''.join(res).lstrip()
+
+
+def make_analysator(f):
+    """
+    Return a static text analysation function that
+    returns float values.
+    """
+    def text_analyse(text):
+        try:
+            rv = f(text)
+        except Exception:
+            return 0.0
+        if not rv:
+            return 0.0
+        try:
+            return min(1.0, max(0.0, float(rv)))
+        except (ValueError, TypeError):
+            return 0.0
+    text_analyse.__doc__ = f.__doc__
+    return staticmethod(text_analyse)
+
+
+def shebang_matches(text, regex):
+    """
+    Check if the given regular expression matches the last part of the
+    shebang if one exists.
+
+        >>> from pygments.util import shebang_matches
+        >>> shebang_matches('#!/usr/bin/env python', r'python(2\.\d)?')
+        True
+        >>> shebang_matches('#!/usr/bin/python2.4', r'python(2\.\d)?')
+        True
+        >>> shebang_matches('#!/usr/bin/python-ruby', r'python(2\.\d)?')
+        False
+        >>> shebang_matches('#!/usr/bin/python/ruby', r'python(2\.\d)?')
+        False
+        >>> shebang_matches('#!/usr/bin/startsomethingwith python',
+        ...                 r'python(2\.\d)?')
+        True
+
+    It also checks for common windows executable file extensions::
+
+        >>> shebang_matches('#!C:\\Python2.4\\Python.exe', r'python(2\.\d)?')
+        True
+
+    Parameters (``'-f'`` or ``'--foo'`` are ignored so ``'perl'`` does
+    the same as ``'perl -e'``)
+
+    Note that this method automatically searches the whole string (eg:
+    the regular expression is wrapped in ``'^$'``)
+    """
+    index = text.find('\n')
+    if index >= 0:
+        first_line = text[:index].lower()
+    else:
+        first_line = text.lower()
+    if first_line.startswith('#!'):
+        try:
+            found = [x for x in split_path_re.split(first_line[2:].strip())
+                     if x and not x.startswith('-')][-1]
+        except IndexError:
+            return False
+        regex = re.compile('^%s(\.(exe|cmd|bat|bin))?$' % regex, re.IGNORECASE)
+        if regex.search(found) is not None:
+            return True
+    return False
+
+
+def doctype_matches(text, regex):
+    """
+    Check if the doctype matches a regular expression (if present).
+    Note that this method only checks the first part of a DOCTYPE.
+    eg: 'html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"'
+    """
+    m = doctype_lookup_re.match(text)
+    if m is None:
+        return False
+    doctype = m.group(2)
+    return re.compile(regex).match(doctype.strip()) is not None
+
+
+def html_doctype_matches(text):
+    """
+    Check if the file looks like it has a html doctype.
+    """
+    return doctype_matches(text, r'html\s+PUBLIC\s+"-//W3C//DTD X?HTML.*')
+
+
+_looks_like_xml_cache = {}
+def looks_like_xml(text):
+    """
+    Check if a doctype exists or if we have some tags.
+    """
+    key = hash(text)
+    try:
+        return _looks_like_xml_cache[key]
+    except KeyError:
+        m = doctype_lookup_re.match(text)
+        if m is not None:
+            return True
+        rv = tag_re.search(text[:1000]) is not None
+        _looks_like_xml_cache[key] = rv
+        return rv
+
+# Python narrow build compatibility
+
+def _surrogatepair(c):
+    return (0xd7c0 + (c >> 10), (0xdc00 + (c & 0x3ff)))
+
+def unirange(a, b):
+    """
+    Returns a regular expression string to match the given non-BMP range.
+    """
+    if b < a:
+        raise ValueError("Bad character range")
+    if a < 0x10000 or b < 0x10000:
+        raise ValueError("unirange is only defined for non-BMP ranges")
+
+    if sys.maxunicode > 0xffff:
+        # wide build
+        return u'[%s-%s]' % (unichr(a), unichr(b))
+    else:
+        # narrow build stores surrogates, and the 're' module handles them
+        # (incorrectly) as characters.  Since there is still ordering among
+        # these characters, expand the range to one that it understands.  Some
+        # background in http://bugs.python.org/issue3665 and
+        # http://bugs.python.org/issue12749
+        #
+        # Additionally, the lower constants are using unichr rather than
+        # literals because jython [which uses the wide path] can't load this
+        # file if they are literals.
+        ah, al = _surrogatepair(a)
+        bh, bl = _surrogatepair(b)
+        if ah == bh:
+            return u'(?:%s[%s-%s])' % (unichr(ah), unichr(al), unichr(bl))
+        else:
+            buf = []
+            buf.append(u'%s[%s-%s]' %
+                       (unichr(ah), unichr(al),
+                        ah == bh and unichr(bl) or unichr(0xdfff)))
+            if ah - bh > 1:
+                buf.append(u'[%s-%s][%s-%s]' %
+                           unichr(ah+1), unichr(bh-1), unichr(0xdc00), unichr(0xdfff))
+            if ah != bh:
+                buf.append(u'%s[%s-%s]' %
+                           (unichr(bh), unichr(0xdc00), unichr(bl)))
+
+            return u'(?:' + u'|'.join(buf) + u')'
+
+# Python 2/3 compatibility
+
+if sys.version_info < (3,0):
+    b = bytes = str
+    u_prefix = 'u'
+    import StringIO, cStringIO
+    BytesIO = cStringIO.StringIO
+    StringIO = StringIO.StringIO
+    uni_open = codecs.open
+else:
+    import builtins
+    bytes = builtins.bytes
+    u_prefix = ''
+    def b(s):
+        if isinstance(s, str):
+            return bytes(map(ord, s))
+        elif isinstance(s, bytes):
+            return s
+        else:
+            raise TypeError("Invalid argument %r for b()" % (s,))
+    import io
+    BytesIO = io.BytesIO
+    StringIO = io.StringIO
+    uni_open = builtins.open

data/vendor/pygments-main/scripts/check_sources.py

@@ -0,0 +1,242 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+    Checker for file headers
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Make sure each Python file has a correct file header
+    including copyright and license information.
+
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import sys, os, re
+import getopt
+import cStringIO
+from os.path import join, splitext, abspath
+
+
+checkers = {}
+
+def checker(*suffixes, **kwds):
+    only_pkg = kwds.pop('only_pkg', False)
+    def deco(func):
+        for suffix in suffixes:
+            checkers.setdefault(suffix, []).append(func)
+        func.only_pkg = only_pkg
+        return func
+    return deco
+
+
+name_mail_re = r'[\w ]+(<.*?>)?'
+copyright_re = re.compile(r'^    :copyright: Copyright 2006-2013 by '
+                          r'the Pygments team, see AUTHORS\.$', re.UNICODE)
+copyright_2_re = re.compile(r'^                %s(, %s)*[,.]$' %
+                            (name_mail_re, name_mail_re), re.UNICODE)
+coding_re    = re.compile(r'coding[:=]\s*([-\w.]+)')
+not_ix_re    = re.compile(r'\bnot\s+\S+?\s+i[sn]\s\S+')
+is_const_re  = re.compile(r'if.*?==\s+(None|False|True)\b')
+
+misspellings = ["developement", "adress", "verificate",  # ALLOW-MISSPELLING
+                "informations"]                          # ALLOW-MISSPELLING
+
+
+@checker('.py')
+def check_syntax(fn, lines):
+    try:
+        compile(''.join(lines), fn, "exec")
+    except SyntaxError, err:
+        yield 0, "not compilable: %s" % err
+
+
+@checker('.py')
+def check_style_and_encoding(fn, lines):
+    encoding = 'ascii'
+    for lno, line in enumerate(lines):
+        if len(line) > 90:
+            yield lno+1, "line too long"
+        m = not_ix_re.search(line)
+        if m:
+            yield lno+1, '"' + m.group() + '"'
+        if is_const_re.search(line):
+            yield lno+1, 'using == None/True/False'
+        if lno < 2:
+            co = coding_re.search(line)
+            if co:
+                encoding = co.group(1)
+        try:
+            line.decode(encoding)
+        except UnicodeDecodeError, err:
+            yield lno+1, "not decodable: %s\n   Line: %r" % (err, line)
+        except LookupError, err:
+            yield 0, "unknown encoding: %s" % encoding
+            encoding = 'latin1'
+
+
+@checker('.py', only_pkg=True)
+def check_fileheader(fn, lines):
+    # line number correction
+    c = 1
+    if lines[0:1] == ['#!/usr/bin/env python\n']:
+        lines = lines[1:]
+        c = 2
+
+    llist = []
+    docopen = False
+    for lno, l in enumerate(lines):
+        llist.append(l)
+        if lno == 0:
+            if l == '# -*- coding: rot13 -*-\n':
+                # special-case pony package
+                return
+            elif l != '# -*- coding: utf-8 -*-\n':
+                yield 1, "missing coding declaration"
+        elif lno == 1:
+            if l != '"""\n' and l != 'r"""\n':
+                yield 2, 'missing docstring begin (""")'
+            else:
+                docopen = True
+        elif docopen:
+            if l == '"""\n':
+                # end of docstring
+                if lno <= 4:
+                    yield lno+c, "missing module name in docstring"
+                break
+
+            if l != "\n" and l[:4] != '    ' and docopen:
+                yield lno+c, "missing correct docstring indentation"
+
+            if lno == 2:
+                # if not in package, don't check the module name
+                modname = fn[:-3].replace('/', '.').replace('.__init__', '')
+                while modname:
+                    if l.lower()[4:-1] == modname:
+                        break
+                    modname = '.'.join(modname.split('.')[1:])
+                else:
+                    yield 3, "wrong module name in docstring heading"
+                modnamelen = len(l.strip())
+            elif lno == 3:
+                if l.strip() != modnamelen * "~":
+                    yield 4, "wrong module name underline, should be ~~~...~"
+
+    else:
+        yield 0, "missing end and/or start of docstring..."
+
+    # check for copyright and license fields
+    license = llist[-2:-1]
+    if license != ["    :license: BSD, see LICENSE for details.\n"]:
+        yield 0, "no correct license info"
+
+    ci = -3
+    copyright = [s.decode('utf-8') for s in llist[ci:ci+1]]
+    while copyright and copyright_2_re.match(copyright[0]):
+        ci -= 1
+        copyright = llist[ci:ci+1]
+    if not copyright or not copyright_re.match(copyright[0]):
+        yield 0, "no correct copyright info"
+
+
+@checker('.py', '.html', '.js')
+def check_whitespace_and_spelling(fn, lines):
+    for lno, line in enumerate(lines):
+        if "\t" in line:
+            yield lno+1, "OMG TABS!!!1 "
+        if line[:-1].rstrip(' \t') != line[:-1]:
+            yield lno+1, "trailing whitespace"
+        for word in misspellings:
+            if word in line and 'ALLOW-MISSPELLING' not in line:
+                yield lno+1, '"%s" used' % word
+
+
+bad_tags = ('<b>', '<i>', '<u>', '<s>', '<strike>'
+            '<center>', '<big>', '<small>', '<font')
+
+@checker('.html')
+def check_xhtml(fn, lines):
+    for lno, line in enumerate(lines):
+        for bad_tag in bad_tags:
+            if bad_tag in line:
+                yield lno+1, "used " + bad_tag
+
+
+def main(argv):
+    try:
+        gopts, args = getopt.getopt(argv[1:], "vi:")
+    except getopt.GetoptError:
+        print "Usage: %s [-v] [-i ignorepath]* [path]" % argv[0]
+        return 2
+    opts = {}
+    for opt, val in gopts:
+        if opt == '-i':
+            val = abspath(val)
+        opts.setdefault(opt, []).append(val)
+
+    if len(args) == 0:
+        path = '.'
+    elif len(args) == 1:
+        path = args[0]
+    else:
+        print "Usage: %s [-v] [-i ignorepath]* [path]" % argv[0]
+        return 2
+
+    verbose = '-v' in opts
+
+    num = 0
+    out = cStringIO.StringIO()
+
+    # TODO: replace os.walk run with iteration over output of
+    #       `svn list -R`.
+
+    for root, dirs, files in os.walk(path):
+        if '.svn' in dirs:
+            dirs.remove('.svn')
+        if '-i' in opts and abspath(root) in opts['-i']:
+            del dirs[:]
+            continue
+        # XXX: awkward: for the Makefile call: don't check non-package
+        #      files for file headers
+        in_pocoo_pkg = root.startswith('./pygments')
+        for fn in files:
+
+            fn = join(root, fn)
+            if fn[:2] == './': fn = fn[2:]
+
+            if '-i' in opts and abspath(fn) in opts['-i']:
+                continue
+
+            ext = splitext(fn)[1]
+            checkerlist = checkers.get(ext, None)
+            if not checkerlist:
+                continue
+
+            if verbose:
+                print "Checking %s..." % fn
+
+            try:
+                f = open(fn, 'r')
+                lines = list(f)
+            except (IOError, OSError), err:
+                print "%s: cannot open: %s" % (fn, err)
+                num += 1
+                continue
+
+            for checker in checkerlist:
+                if not in_pocoo_pkg and checker.only_pkg:
+                    continue
+                for lno, msg in checker(fn, lines):
+                    print >>out, "%s:%d: %s" % (fn, lno, msg)
+                    num += 1
+    if verbose:
+        print
+    if num == 0:
+        print "No errors found."
+    else:
+        print out.getvalue().rstrip('\n')
+        print "%d error%s found." % (num, num > 1 and "s" or "")
+    return int(num > 0)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv))

data/vendor/pygments-main/scripts/detect_missing_analyse_text.py

@@ -0,0 +1,32 @@
+import sys
+
+from pygments.lexers import get_all_lexers, find_lexer_class
+from pygments.lexer import Lexer
+
+def main():
+    uses = {}
+
+    for name, aliases, filenames, mimetypes in get_all_lexers():
+        cls = find_lexer_class(name)
+        if not cls.aliases:
+            print cls, "has no aliases"
+        for f in filenames:
+            if f not in uses:
+                uses[f] = []
+            uses[f].append(cls)
+
+    ret = 0
+    for k, v in uses.iteritems():
+        if len(v) > 1:
+            #print "Multiple for", k, v
+            for i in v:
+                if i.analyse_text is None:
+                    print i, "has a None analyse_text"
+                    ret |= 1
+                elif Lexer.analyse_text.__doc__ == i.analyse_text.__doc__:
+                    print i, "needs analyse_text, multiple lexers for", k
+                    ret |= 2
+    return ret
+
+if __name__ == '__main__':
+    sys.exit(main())