pygments.rb 0.6.3 → 1.0.0

data/vendor/pygments-main/pygments/lexer.py

@@ -5,20 +5,26 @@
 
     Base lexer classes.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
-import re, itertools
+
+from __future__ import print_function
+
+import re
+import sys
+import time
 
 from pygments.filter import apply_filters, Filter
 from pygments.filters import get_filter_by_name
 from pygments.token import Error, Text, Other, _TokenType
 from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
-     make_analysator, text_type, add_metaclass, iteritems
-
+    make_analysator, text_type, add_metaclass, iteritems, Future, guess_decode
+from pygments.regexopt import regex_opt
 
 __all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer',
-           'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this', 'default']
+           'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this',
+           'default', 'words']
 
 
 _encoding_map = [(b'\xef\xbb\xbf', 'utf-8'),
@@ -36,10 +42,10 @@ class LexerMeta(type):
     static methods which always return float values.
     """
 
-    def __new__(cls, name, bases, d):
+    def __new__(mcs, name, bases, d):
         if 'analyse_text' in d:
             d['analyse_text'] = make_analysator(d['analyse_text'])
-        return type.__new__(cls, name, bases, d)
+        return type.__new__(mcs, name, bases, d)
 
 
 @add_metaclass(LexerMeta)
@@ -64,9 +70,11 @@ class Lexer(object):
     ``encoding``
         If given, must be an encoding name. This encoding will be used to
         convert the input string to Unicode, if it is not already a Unicode
-        string (default: ``'latin1'``).
-        Can also be ``'guess'`` to use a simple UTF-8 / Latin1 detection, or
-        ``'chardet'`` to use the chardet library, if it is installed.
+        string (default: ``'guess'``, which uses a simple UTF-8 / Locale /
+        Latin1 detection.  Can also be ``'chardet'`` to use the chardet
+        library, if it is installed.
+    ``inencoding``
+        Overrides the ``encoding`` if given.
     """
 
     #: Name of the lexer
@@ -93,8 +101,8 @@ class Lexer(object):
         self.stripall = get_bool_opt(options, 'stripall', False)
         self.ensurenl = get_bool_opt(options, 'ensurenl', True)
         self.tabsize = get_int_opt(options, 'tabsize', 0)
-        self.encoding = options.get('encoding', 'latin1')
-        # self.encoding = options.get('inencoding', None) or self.encoding
+        self.encoding = options.get('encoding', 'guess')
+        self.encoding = options.get('inencoding') or self.encoding
         self.filters = []
         for filter_ in get_list_opt(options, 'filters', ()):
             self.add_filter(filter_)
@@ -139,12 +147,7 @@ class Lexer(object):
         """
         if not isinstance(text, text_type):
             if self.encoding == 'guess':
-                try:
-                    text = text.decode('utf-8')
-                    if text.startswith(u'\ufeff'):
-                        text = text[len(u'\ufeff'):]
-                except UnicodeDecodeError:
-                    text = text.decode('latin1')
+                text, _ = guess_decode(text)
             elif self.encoding == 'chardet':
                 try:
                     import chardet
@@ -160,7 +163,7 @@ class Lexer(object):
                         break
                 # no BOM found, so use chardet
                 if decoded is None:
-                    enc = chardet.detect(text[:1024]) # Guess using first 1KB
+                    enc = chardet.detect(text[:1024])  # Guess using first 1KB
                     decoded = text.decode(enc.get('encoding') or 'utf-8',
                                           'replace')
                 text = decoded
@@ -185,7 +188,7 @@ class Lexer(object):
             text += '\n'
 
         def streamer():
-            for i, t, v in self.get_tokens_unprocessed(text):
+            for _, t, v in self.get_tokens_unprocessed(text):
                 yield t, v
         stream = streamer()
         if not unfiltered:
@@ -237,12 +240,12 @@ class DelegatingLexer(Lexer):
                              self.root_lexer.get_tokens_unprocessed(buffered))
 
 
-#-------------------------------------------------------------------------------
+# ------------------------------------------------------------------------------
 # RegexLexer and ExtendedRegexLexer
 #
 
 
-class include(str):
+class include(str):  # pylint: disable=invalid-name
     """
     Indicates that a state should include rules from another state.
     """
@@ -256,10 +259,10 @@ class _inherit(object):
     def __repr__(self):
         return 'inherit'
 
-inherit = _inherit()
+inherit = _inherit()  # pylint: disable=invalid-name
 
 
-class combined(tuple):
+class combined(tuple):  # pylint: disable=invalid-name
     """
     Indicates a state combined from multiple states.
     """
@@ -316,8 +319,8 @@ def bygroups(*args):
                 if data is not None:
                     if ctx:
                         ctx.pos = match.start(i + 1)
-                    for item in action(lexer, _PseudoMatch(match.start(i + 1),
-                                       data), ctx):
+                    for item in action(lexer,
+                                       _PseudoMatch(match.start(i + 1), data), ctx):
                         if item:
                             yield item
         if ctx:
@@ -387,26 +390,46 @@ class default:
     """
     Indicates a state or state action (e.g. #pop) to apply.
     For example default('#pop') is equivalent to ('', Token, '#pop')
-    Note that state tuples may be used as well
+    Note that state tuples may be used as well.
+
+    .. versionadded:: 2.0
     """
     def __init__(self, state):
         self.state = state
 
 
+class words(Future):
+    """
+    Indicates a list of literal words that is transformed into an optimized
+    regex that matches any of the words.
+
+    .. versionadded:: 2.0
+    """
+    def __init__(self, words, prefix='', suffix=''):
+        self.words = words
+        self.prefix = prefix
+        self.suffix = suffix
+
+    def get(self):
+        return regex_opt(self.words, prefix=self.prefix, suffix=self.suffix)
+
+
 class RegexLexerMeta(LexerMeta):
     """
     Metaclass for RegexLexer, creates the self._tokens attribute from
     self.tokens on the first instantiation.
     """
 
-    def _process_regex(cls, regex, rflags):
+    def _process_regex(cls, regex, rflags, state):
         """Preprocess the regular expression component of a token definition."""
+        if isinstance(regex, Future):
+            regex = regex.get()
         return re.compile(regex, rflags).match
 
     def _process_token(cls, token):
         """Preprocess the token component of a token definition."""
         assert type(token) is _TokenType or callable(token), \
-               'token type must be simple type or callable, not %r' % (token,)
+            'token type must be simple type or callable, not %r' % (token,)
         return token
 
     def _process_new_state(cls, new_state, unprocessed, processed):
@@ -439,7 +462,7 @@ class RegexLexerMeta(LexerMeta):
             for istate in new_state:
                 assert (istate in unprocessed or
                         istate in ('#pop', '#push')), \
-                       'unknown new state ' + istate
+                    'unknown new state ' + istate
             return new_state
         else:
             assert False, 'unknown new state def %r' % new_state
@@ -460,7 +483,9 @@ class RegexLexerMeta(LexerMeta):
                                                  str(tdef)))
                 continue
             if isinstance(tdef, _inherit):
-                # processed already
+                # should be processed already, but may not in the case of:
+                # 1. the state has no counterpart in any parent
+                # 2. the state includes more than one 'inherit'
                 continue
             if isinstance(tdef, default):
                 new_state = cls._process_new_state(tdef.state, unprocessed, processed)
@@ -470,7 +495,7 @@ class RegexLexerMeta(LexerMeta):
             assert type(tdef) is tuple, "wrong rule def %r" % tdef
 
             try:
-                rex = cls._process_regex(tdef[0], rflags)
+                rex = cls._process_regex(tdef[0], rflags, state)
             except Exception as err:
                 raise ValueError("uncompilable regex %r in state %r of %r: %s" %
                                  (tdef[0], state, cls, err))
@@ -508,12 +533,16 @@ class RegexLexerMeta(LexerMeta):
         """
         tokens = {}
         inheritable = {}
-        for c in itertools.chain((cls,), cls.__mro__):
+        for c in cls.__mro__:
             toks = c.__dict__.get('tokens', {})
 
             for state, items in iteritems(toks):
                 curitems = tokens.get(state)
                 if curitems is None:
+                    # N.b. because this is assigned by reference, sufficiently
+                    # deep hierarchies are processed incrementally (e.g. for
+                    # A(B), B(C), C(RegexLexer), B will be premodified so X(B)
+                    # will not see any inherits in B).
                     tokens[state] = items
                     try:
                         inherit_ndx = items.index(inherit)
@@ -529,6 +558,8 @@ class RegexLexerMeta(LexerMeta):
                 # Replace the "inherit" value with the items
                 curitems[inherit_ndx:inherit_ndx+1] = items
                 try:
+                    # N.b. this is the index in items (that is, the superclass
+                    # copy), so offset required when storing below.
                     new_inh_ndx = items.index(inherit)
                 except ValueError:
                     pass
@@ -623,6 +654,8 @@ class RegexLexer(Lexer):
                         statetokens = tokendefs[statestack[-1]]
                     break
             else:
+                # We are here only if all state tokens have been considered
+                # and there was not a match on any of them.
                 try:
                     if text[pos] == '\n':
                         # at EOL, reset state to "root"
@@ -645,7 +678,7 @@ class LexerContext(object):
     def __init__(self, text, pos, stack=None, end=None):
         self.text = text
         self.pos = pos
-        self.end = end or len(text) # end=0 not supported ;-)
+        self.end = end or len(text)  # end=0 not supported ;-)
         self.stack = stack or ['root']
 
     def __repr__(self):
@@ -783,3 +816,56 @@ def do_insertions(insertions, tokens):
         except StopIteration:
             insleft = False
             break  # not strictly necessary
+
+
+class ProfilingRegexLexerMeta(RegexLexerMeta):
+    """Metaclass for ProfilingRegexLexer, collects regex timing info."""
+
+    def _process_regex(cls, regex, rflags, state):
+        if isinstance(regex, words):
+            rex = regex_opt(regex.words, prefix=regex.prefix,
+                            suffix=regex.suffix)
+        else:
+            rex = regex
+        compiled = re.compile(rex, rflags)
+
+        def match_func(text, pos, endpos=sys.maxsize):
+            info = cls._prof_data[-1].setdefault((state, rex), [0, 0.0])
+            t0 = time.time()
+            res = compiled.match(text, pos, endpos)
+            t1 = time.time()
+            info[0] += 1
+            info[1] += t1 - t0
+            return res
+        return match_func
+
+
+@add_metaclass(ProfilingRegexLexerMeta)
+class ProfilingRegexLexer(RegexLexer):
+    """Drop-in replacement for RegexLexer that does profiling of its regexes."""
+
+    _prof_data = []
+    _prof_sort_index = 4  # defaults to time per call
+
+    def get_tokens_unprocessed(self, text, stack=('root',)):
+        # this needs to be a stack, since using(this) will produce nested calls
+        self.__class__._prof_data.append({})
+        for tok in RegexLexer.get_tokens_unprocessed(self, text, stack):
+            yield tok
+        rawdata = self.__class__._prof_data.pop()
+        data = sorted(((s, repr(r).strip('u\'').replace('\\\\', '\\')[:65],
+                        n, 1000 * t, 1000 * t / n)
+                       for ((s, r), (n, t)) in rawdata.items()),
+                      key=lambda x: x[self._prof_sort_index],
+                      reverse=True)
+        sum_total = sum(x[3] for x in data)
+
+        print()
+        print('Profiling result for %s lexing %d chars in %.3f ms' %
+              (self.__class__.__name__, len(text), sum_total))
+        print('=' * 110)
+        print('%-20s %-64s ncalls  tottime  percall' % ('state', 'regex'))
+        print('-' * 110)
+        for d in data:
+            print('%-20s %-65s %5d %8.4f %8.4f' % d)
+        print('=' * 110)

data/vendor/pygments-main/pygments/lexers/__init__.py

@@ -5,20 +5,20 @@
 
     Pygments lexers.
 
-    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+import re
 import sys
 import types
 import fnmatch
-import re
 from os.path import basename
 
 from pygments.lexers._mapping import LEXERS
 from pygments.modeline import get_filetype_from_buffer
 from pygments.plugin import find_plugin_lexers
-from pygments.util import ClassNotFound, itervalues
+from pygments.util import ClassNotFound, itervalues, guess_decode
 
 
 __all__ = ['get_lexer_by_name', 'get_lexer_for_filename', 'find_lexer_class',
@@ -27,23 +27,17 @@ __all__ = ['get_lexer_by_name', 'get_lexer_for_filename', 'find_lexer_class',
 _lexer_cache = {}
 _pattern_cache = {}
 
+
 def _fn_matches(fn, glob):
-    """
-    Return whether the supplied file name fn matches pattern filename
-    """
+    """Return whether the supplied file name fn matches pattern filename."""
     if glob not in _pattern_cache:
-        pattern = re.compile(fnmatch.translate(glob))
-        _pattern_cache[glob] = pattern
-    else:
-        pattern = _pattern_cache[glob]
-
-    return pattern.match(fn)
+        pattern = _pattern_cache[glob] = re.compile(fnmatch.translate(glob))
+        return pattern.match(fn)
+    return _pattern_cache[glob].match(fn)
 
 
 def _load_lexers(module_name):
-    """
-    Load a lexer (and all others in the module too).
-    """
+    """Load a lexer (and all others in the module too)."""
     mod = __import__(module_name, None, None, ['__all__'])
     for lexer_name in mod.__all__:
         cls = getattr(mod, lexer_name)
@@ -51,8 +45,7 @@ def _load_lexers(module_name):
 
 
 def get_all_lexers():
-    """
-    Return a generator of tuples in the form ``(name, aliases,
+    """Return a generator of tuples in the form ``(name, aliases,
     filenames, mimetypes)`` of all know lexers.
     """
     for item in itervalues(LEXERS):
@@ -62,8 +55,9 @@ def get_all_lexers():
 
 
 def find_lexer_class(name):
-    """
-    Lookup a lexer class by name. Return None if not found.
+    """Lookup a lexer class by name.
+
+    Return None if not found.
     """
     if name in _lexer_cache:
         return _lexer_cache[name]
@@ -78,9 +72,32 @@ def find_lexer_class(name):
             return cls
 
 
-def get_lexer_by_name(_alias, **options):
+def find_lexer_class_by_name(alias):
+    """Lookup a lexer class by alias.
+
+    Like `get_lexer_by_name`, but does not instantiate the class.
+
+    .. versionadded:: 2.2
     """
-    Get a lexer by an alias.
+    if not _alias:
+        raise ClassNotFound('no lexer for alias %r found' % _alias)
+    # lookup builtin lexers
+    for module_name, name, aliases, _, _ in itervalues(LEXERS):
+        if _alias.lower() in aliases:
+            if name not in _lexer_cache:
+                _load_lexers(module_name)
+            return _lexer_cache[name]
+    # continue with lexers from setuptools entrypoints
+    for cls in find_plugin_lexers():
+        if _alias.lower() in cls.aliases:
+            return cls
+    raise ClassNotFound('no lexer for alias %r found' % _alias)
+
+
+def get_lexer_by_name(_alias, **options):
+    """Get a lexer by an alias.
+
+    Raises ClassNotFound if not found.
     """
     if not _alias:
         raise ClassNotFound('no lexer for alias %r found' % _alias)
@@ -93,16 +110,18 @@ def get_lexer_by_name(_alias, **options):
             return _lexer_cache[name](**options)
     # continue with lexers from setuptools entrypoints
     for cls in find_plugin_lexers():
-        if _alias in cls.aliases:
+        if _alias.lower() in cls.aliases:
             return cls(**options)
     raise ClassNotFound('no lexer for alias %r found' % _alias)
 
 
-def get_lexer_for_filename(_fn, code=None, **options):
-    """
-    Get a lexer for a filename.  If multiple lexers match the filename
-    pattern, use ``analyse_text()`` to figure out which one is more
-    appropriate.
+def find_lexer_class_for_filename(_fn, code=None):
+    """Get a lexer for a filename.
+
+    If multiple lexers match the filename pattern, use ``analyse_text()`` to
+    figure out which one is more appropriate.
+
+    Returns None if not found.
     """
     matches = []
     fn = basename(_fn)
@@ -119,7 +138,7 @@ def get_lexer_for_filename(_fn, code=None, **options):
 
     if sys.version_info > (3,) and isinstance(code, bytes):
         # decode it, since all analyse_text functions expect unicode
-        code = code.decode('latin1')
+        code = guess_decode(code)
 
     def get_rating(info):
         cls, filename = info
@@ -135,14 +154,28 @@ def get_lexer_for_filename(_fn, code=None, **options):
 
     if matches:
         matches.sort(key=get_rating)
-        #print "Possible lexers, after sort:", matches
-        return matches[-1][0](**options)
-    raise ClassNotFound('no lexer for filename %r found' % _fn)
+        # print "Possible lexers, after sort:", matches
+        return matches[-1][0]
 
 
-def get_lexer_for_mimetype(_mime, **options):
+def get_lexer_for_filename(_fn, code=None, **options):
+    """Get a lexer for a filename.
+
+    If multiple lexers match the filename pattern, use ``analyse_text()`` to
+    figure out which one is more appropriate.
+
+    Raises ClassNotFound if not found.
     """
-    Get a lexer for a mimetype.
+    res = find_lexer_class_for_filename(_fn, code)
+    if not res:
+        raise ClassNotFound('no lexer for filename %r found' % _fn)
+    return res(**options)
+
+
+def get_lexer_for_mimetype(_mime, **options):
+    """Get a lexer for a mimetype.
+
+    Raises ClassNotFound if not found.
     """
     for modname, name, _, _, mimetypes in itervalues(LEXERS):
         if _mime in mimetypes:
@@ -155,17 +188,16 @@ def get_lexer_for_mimetype(_mime, **options):
     raise ClassNotFound('no lexer for mimetype %r found' % _mime)
 
 
-def _iter_lexerclasses():
-    """
-    Return an iterator over all lexer classes.
-    """
+def _iter_lexerclasses(plugins=True):
+    """Return an iterator over all lexer classes."""
     for key in sorted(LEXERS):
         module_name, name = LEXERS[key][:2]
         if name not in _lexer_cache:
             _load_lexers(module_name)
         yield _lexer_cache[name]
-    for lexer in find_plugin_lexers():
-        yield lexer
+    if plugins:
+        for lexer in find_plugin_lexers():
+            yield lexer
 
 
 def guess_lexer_for_filename(_fn, _text, **options):
@@ -185,16 +217,17 @@ def guess_lexer_for_filename(_fn, _text, **options):
         <pygments.lexers.templates.CssPhpLexer object at 0xb7ba518c>
     """
     fn = basename(_fn)
-    primary = None
+    primary = {}
     matching_lexers = set()
     for lexer in _iter_lexerclasses():
         for filename in lexer.filenames:
             if _fn_matches(fn, filename):
                 matching_lexers.add(lexer)
-                primary = lexer
+                primary[lexer] = True
         for filename in lexer.alias_filenames:
             if _fn_matches(fn, filename):
                 matching_lexers.add(lexer)
+                primary[lexer] = False
     if not matching_lexers:
         raise ClassNotFound('no lexer for filename %r found' % fn)
     if len(matching_lexers) == 1:
@@ -206,21 +239,20 @@ def guess_lexer_for_filename(_fn, _text, **options):
             return lexer(**options)
         result.append((rv, lexer))
 
-    # since py3 can no longer sort by class name by default, here is the
-    # sorting function that works in both
-    def type_sort(type_):
-        return (type_[0], type_[1].__name__)
+    def type_sort(t):
+        # sort by:
+        # - analyse score
+        # - is primary filename pattern?
+        # - priority
+        # - last resort: class name
+        return (t[0], primary[t[1]], t[1].priority, t[1].__name__)
     result.sort(key=type_sort)
 
-    if not result[-1][0] and primary is not None:
-        return primary(**options)
     return result[-1][1](**options)
 
 
 def guess_lexer(_text, **options):
-    """
-    Guess a lexer by strong distinctions in the text (eg, shebang).
-    """
+    """Guess a lexer by strong distinctions in the text (eg, shebang)."""
 
     # try to get a vim modeline first
     ft = get_filetype_from_buffer(_text)
@@ -256,8 +288,8 @@ class _automodule(types.ModuleType):
         raise AttributeError(name)
 
 
-oldmod = sys.modules['pygments.lexers']
-newmod = _automodule('pygments.lexers')
+oldmod = sys.modules[__name__]
+newmod = _automodule(__name__)
 newmod.__dict__.update(oldmod.__dict__)
-sys.modules['pygments.lexers'] = newmod
+sys.modules[__name__] = newmod
 del newmod.newmod, newmod.oldmod, newmod.sys, newmod.types