gitlab-pygments.rb 0.3.2 → 0.5.4

data/vendor/pygments-main/pygments/formatters/html.py

@@ -5,18 +5,23 @@
 
     Formatter for HTML output.
 
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import os
 import sys
+import os.path
 import StringIO
 
 from pygments.formatter import Formatter
 from pygments.token import Token, Text, STANDARD_TYPES
 from pygments.util import get_bool_opt, get_int_opt, get_list_opt, bytes
 
+try:
+    import ctags
+except ImportError:
+    ctags = None
 
 __all__ = ['HtmlFormatter']
 
@@ -141,6 +146,12 @@ class HtmlFormatter(Formatter):
     the style definitions inside a ``<style>`` tag, or in a separate file if
     the `cssfile` option is given.
 
+    When `tagsfile` is set to the path of a ctags index file, it is used to
+    generate hyperlinks from names to their definition.  You must enable
+    `anchorlines` and run ctags with the `-n` option for this to work.  The
+    `python-ctags` module from PyPI must be installed to use this feature;
+    otherwise a `RuntimeError` will be raised.
+
     The `get_style_defs(arg='')` method of a `HtmlFormatter` returns a string
     containing CSS rules for the CSS classes used by the formatter. The
     argument `arg` can be used to specify additional CSS selectors that
@@ -282,10 +293,27 @@ class HtmlFormatter(Formatter):
         output line in an anchor tag with a ``name`` of ``foo-linenumber``.
         This allows easy linking to certain lines. *New in Pygments 0.9.*
 
+    `linespans`
+        If set to a nonempty string, e.g. ``foo``, the formatter will wrap each
+        output line in a span tag with an ``id`` of ``foo-linenumber``.
+        This allows easy access to lines via javascript. *New in Pygments 1.6.*
+
     `anchorlinenos`
         If set to `True`, will wrap line numbers in <a> tags. Used in
         combination with `linenos` and `lineanchors`.
 
+    `tagsfile`
+        If set to the path of a ctags file, wrap names in anchor tags that
+        link to their definitions. `lineanchors` should be used, and the
+        tags file should specify line numbers (see the `-n` option to ctags).
+        *New in Pygments 1.6.*
+
+    `tagurlformat`
+        A string formatting pattern used to generate links to ctags definitions.
+        Available variables are `%(path)s`, `%(fname)s` and `%(fext)s`.
+        Defaults to an empty string, resulting in just `#prefix-number` links.
+        *New in Pygments 1.6.*
+
 
     **Subclassing the HTML formatter**
 
@@ -351,6 +379,14 @@ class HtmlFormatter(Formatter):
         self.prestyles = self._decodeifneeded(options.get('prestyles', ''))
         self.cssfile = self._decodeifneeded(options.get('cssfile', ''))
         self.noclobber_cssfile = get_bool_opt(options, 'noclobber_cssfile', False)
+        self.tagsfile = self._decodeifneeded(options.get('tagsfile', ''))
+        self.tagurlformat = self._decodeifneeded(options.get('tagurlformat', ''))
+
+        if self.tagsfile:
+            if not ctags:
+                raise RuntimeError('The "ctags" package must to be installed '
+                                   'to be able to use the "tagsfile" feature.')
+            self._ctags = ctags.CTags(self.tagsfile)
 
         linenos = options.get('linenos', False)
         if linenos == 'inline':
@@ -366,6 +402,7 @@ class HtmlFormatter(Formatter):
         self.nobackground = get_bool_opt(options, 'nobackground', False)
         self.lineseparator = options.get('lineseparator', '\n')
         self.lineanchors = options.get('lineanchors', '')
+        self.linespans = options.get('linespans', '')
         self.anchorlinenos = options.get('anchorlinenos', False)
         self.hl_lines = set()
         for lineno in get_list_opt(options, 'hl_lines', []):
@@ -596,7 +633,8 @@ class HtmlFormatter(Formatter):
 
     def _wrap_lineanchors(self, inner):
         s = self.lineanchors
-        i = self.linenostart - 1 # subtract 1 since we have to increment i *before* yielding
+        i = self.linenostart - 1 # subtract 1 since we have to increment i
+                                 # *before* yielding
         for t, line in inner:
             if t:
                 i += 1
@@ -604,6 +642,16 @@ class HtmlFormatter(Formatter):
             else:
                 yield 0, line
 
+    def _wrap_linespans(self, inner):
+        s = self.linespans
+        i = self.linenostart - 1
+        for t, line in inner:
+            if t:
+                i += 1
+                yield 1, '<span id="%s-%d">%s</span>' % (s, i, line)
+            else:
+                yield 0, line
+
     def _wrap_div(self, inner):
         style = []
         if (self.noclasses and not self.nobackground and
@@ -643,6 +691,7 @@ class HtmlFormatter(Formatter):
         getcls = self.ttype2class.get
         c2s = self.class2style
         escape_table = _escape_html_table
+        tagsfile = self.tagsfile
 
         lspan = ''
         line = ''
@@ -659,6 +708,19 @@ class HtmlFormatter(Formatter):
 
             parts = value.translate(escape_table).split('\n')
 
+            if tagsfile and ttype in Token.Name:
+                filename, linenumber = self._lookup_ctag(value)
+                if linenumber:
+                    base, filename = os.path.split(filename)
+                    if base:
+                        base += '/'
+                    filename, extension = os.path.splitext(filename)
+                    url = self.tagurlformat % {'path': base, 'fname': filename,
+                                               'fext': extension}
+                    parts[0] = "<a href=\"%s#%s-%d\">%s" % \
+                        (url, self.lineanchors, linenumber, parts[0])
+                    parts[-1] = parts[-1] + "</a>"
+
             # for all but the last line
             for part in parts[:-1]:
                 if line:
@@ -688,6 +750,13 @@ class HtmlFormatter(Formatter):
         if line:
             yield 1, line + (lspan and '</span>') + lsep
 
+    def _lookup_ctag(self, token):
+        entry = ctags.TagEntry()
+        if self._ctags.find(entry, token, 0):
+            return entry['file'], entry['lineNumber']
+        else:
+            return None, None
+
     def _highlight_lines(self, tokensource):
         """
         Highlighted the lines specified in the `hl_lines` option by
@@ -740,6 +809,8 @@ class HtmlFormatter(Formatter):
                 source = self._wrap_inlinelinenos(source)
             if self.lineanchors:
                 source = self._wrap_lineanchors(source)
+            if self.linespans:
+                source = self._wrap_linespans(source)
             source = self.wrap(source, outfile)
             if self.linenos == 1:
                 source = self._wrap_tablelinenos(source)

data/vendor/pygments-main/pygments/formatters/img.py

@@ -5,12 +5,11 @@
 
     Formatter for Pixmap output.
 
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import sys
-from commands import getstatusoutput
 
 from pygments.formatter import Formatter
 from pygments.util import get_bool_opt, get_int_opt, \
@@ -73,6 +72,7 @@ class FontManager(object):
             self._create_nix()
 
     def _get_nix_font_path(self, name, style):
+        from commands import getstatusoutput
         exit, out = getstatusoutput('fc-list "%s:style=%s" file' %
                                     (name, style))
         if not exit:

data/vendor/pygments-main/pygments/formatters/latex.py

@@ -5,7 +5,7 @@
 
     Formatter for LaTeX fancyvrb output.
 
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -306,17 +306,17 @@ class LatexFormatter(Formatter):
             realoutfile = outfile
             outfile = StringIO()
 
-        outfile.write(r'\begin{Verbatim}[commandchars=\\\{\}')
+        outfile.write(ur'\begin{Verbatim}[commandchars=\\\{\}')
         if self.linenos:
             start, step = self.linenostart, self.linenostep
-            outfile.write(',numbers=left' +
-                          (start and ',firstnumber=%d' % start or '') +
-                          (step and ',stepnumber=%d' % step or ''))
+            outfile.write(u',numbers=left' +
+                          (start and u',firstnumber=%d' % start or u'') +
+                          (step and u',stepnumber=%d' % step or u''))
         if self.mathescape or self.texcomments:
-            outfile.write(r',codes={\catcode`\$=3\catcode`\^=7\catcode`\_=8}')
+            outfile.write(ur',codes={\catcode`\$=3\catcode`\^=7\catcode`\_=8}')
         if self.verboptions:
-            outfile.write(',' + self.verboptions)
-        outfile.write(']\n')
+            outfile.write(u',' + self.verboptions)
+        outfile.write(u']\n')
 
         for ttype, value in tokensource:
             if ttype in Token.Comment:
@@ -366,7 +366,7 @@ class LatexFormatter(Formatter):
             else:
                 outfile.write(value)
 
-        outfile.write('\\end{Verbatim}\n')
+        outfile.write(u'\\end{Verbatim}\n')
 
         if self.full:
             realoutfile.write(DOC_TEMPLATE %

data/vendor/pygments-main/pygments/formatters/other.py

@@ -5,7 +5,7 @@
 
     Other formatters: NullFormatter, RawTokenFormatter.
 
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -102,8 +102,6 @@ class RawTokenFormatter(Formatter):
                 outfile.write(text.encode())
             flush = outfile.flush
 
-        lasttype = None
-        lastval = u''
         if self.error_color:
             for ttype, value in tokensource:
                 line = "%s\t%r\n" % (ttype, value)

data/vendor/pygments-main/pygments/formatters/rtf.py

@@ -5,7 +5,7 @@
 
     A formatter that generates RTF files.
 
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/formatters/svg.py

@@ -5,7 +5,7 @@
 
     Formatter for SVG output.
 
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/formatters/terminal.py

@@ -5,7 +5,7 @@
 
     Formatter for terminal output with ANSI sequences.
 
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/formatters/terminal256.py

@@ -11,7 +11,7 @@
 
     Formatter version 1.
 
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexer.py

@@ -5,10 +5,10 @@
 
     Base lexer classes.
 
-    :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
-import re
+import re, itertools
 
 from pygments.filter import apply_filters, Filter
 from pygments.filters import get_filter_by_name
@@ -18,7 +18,7 @@ from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
 
 
 __all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer',
-           'LexerContext', 'include', 'bygroups', 'using', 'this']
+           'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this']
 
 
 _encoding_map = [('\xef\xbb\xbf', 'utf-8'),
@@ -72,15 +72,18 @@ class Lexer(object):
     #: Shortcuts for the lexer
     aliases = []
 
-    #: fn match rules
+    #: File name globs
     filenames = []
 
-    #: fn alias filenames
+    #: Secondary file name globs
     alias_filenames = []
 
-    #: mime types
+    #: MIME types
     mimetypes = []
 
+    #: Priority, should multiple lexers match and no content is provided
+    priority = 0
+
     __metaclass__ = LexerMeta
 
     def __init__(self, **options):
@@ -163,6 +166,10 @@ class Lexer(object):
                 text = decoded
             else:
                 text = text.decode(self.encoding)
+        else:
+            if text.startswith(u'\ufeff'):
+                text = text[len(u'\ufeff'):]
+
         # text now *is* a unicode string
         text = text.replace('\r\n', '\n')
         text = text.replace('\r', '\n')
@@ -238,6 +245,16 @@ class include(str):
     pass
 
 
+class _inherit(object):
+    """
+    Indicates the a state should inherit from its superclass.
+    """
+    def __repr__(self):
+        return 'inherit'
+
+inherit = _inherit()
+
+
 class combined(tuple):
     """
     Indicates a state combined from multiple states.
@@ -428,6 +445,9 @@ class RegexLexerMeta(LexerMeta):
                 tokens.extend(cls._process_state(unprocessed, processed,
                                                  str(tdef)))
                 continue
+            if isinstance(tdef, _inherit):
+                # processed already
+                continue
 
             assert type(tdef) is tuple, "wrong rule def %r" % tdef
 
@@ -456,6 +476,49 @@ class RegexLexerMeta(LexerMeta):
             cls._process_state(tokendefs, processed, state)
         return processed
 
+    def get_tokendefs(cls):
+        """
+        Merge tokens from superclasses in MRO order, returning a single tokendef
+        dictionary.
+
+        Any state that is not defined by a subclass will be inherited
+        automatically.  States that *are* defined by subclasses will, by
+        default, override that state in the superclass.  If a subclass wishes to
+        inherit definitions from a superclass, it can use the special value
+        "inherit", which will cause the superclass' state definition to be
+        included at that point in the state.
+        """
+        tokens = {}
+        inheritable = {}
+        for c in itertools.chain((cls,), cls.__mro__):
+            toks = c.__dict__.get('tokens', {})
+
+            for state, items in toks.iteritems():
+                curitems = tokens.get(state)
+                if curitems is None:
+                    tokens[state] = items
+                    try:
+                        inherit_ndx = items.index(inherit)
+                    except ValueError:
+                        continue
+                    inheritable[state] = inherit_ndx
+                    continue
+
+                inherit_ndx = inheritable.pop(state, None)
+                if inherit_ndx is None:
+                    continue
+
+                # Replace the "inherit" value with the items
+                curitems[inherit_ndx:inherit_ndx+1] = items
+                try:
+                    new_inh_ndx = items.index(inherit)
+                except ValueError:
+                    pass
+                else:
+                    inheritable[state] = inherit_ndx + new_inh_ndx
+
+        return tokens
+
     def __call__(cls, *args, **kwds):
         """Instantiate cls after preprocessing its token definitions."""
         if '_tokens' not in cls.__dict__:
@@ -465,7 +528,7 @@ class RegexLexerMeta(LexerMeta):
                 # don't process yet
                 pass
             else:
-                cls._tokens = cls.process_tokendef('', cls.tokens)
+                cls._tokens = cls.process_tokendef('', cls.get_tokendefs())
 
         return type.__call__(cls, *args, **kwds)
 
@@ -606,7 +669,13 @@ class ExtendedRegexLexer(RegexLexer):
                     if new_state is not None:
                         # state transition
                         if isinstance(new_state, tuple):
-                            ctx.stack.extend(new_state)
+                            for state in new_state:
+                                if state == '#pop':
+                                    ctx.stack.pop()
+                                elif state == '#push':
+                                    ctx.stack.append(statestack[-1])
+                                else:
+                                    ctx.stack.append(state)
                         elif isinstance(new_state, int):
                             # pop
                             del ctx.stack[new_state:]
@@ -622,10 +691,10 @@ class ExtendedRegexLexer(RegexLexer):
                         break
                     if text[ctx.pos] == '\n':
                         # at EOL, reset state to "root"
-                        ctx.pos += 1
                         ctx.stack = ['root']
                         statetokens = tokendefs['root']
                         yield ctx.pos, Text, u'\n'
+                        ctx.pos += 1
                         continue
                     yield ctx.pos, Error, text[ctx.pos]
                     ctx.pos += 1
@@ -694,4 +763,3 @@ def do_insertions(insertions, tokens):
         except StopIteration:
             insleft = False
             break  # not strictly necessary
-