pygments.rb 2.4.0 → 3.0.0

data/vendor/pygments-main/pygments/formatters/rtf.py

@@ -4,12 +4,14 @@
 
     A formatter that generates RTF files.
 
-    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+from collections import OrderedDict
 from pygments.formatter import Formatter
-from pygments.util import get_int_opt, surrogatepair
+from pygments.style import _ansimap
+from pygments.util import get_bool_opt, get_int_opt, get_list_opt, surrogatepair
 
 
 __all__ = ['RtfFormatter']
@@ -42,6 +44,59 @@ class RtfFormatter(Formatter):
         default is 24 half-points, giving a size 12 font.
 
         .. versionadded:: 2.0
+
+    `linenos`
+        Turn on line numbering (default: ``False``).
+
+        .. versionadded:: 2.18
+
+    `lineno_fontsize`
+        Font size for line numbers. Size is specified in half points
+        (default: `fontsize`). 
+
+        .. versionadded:: 2.18
+
+    `lineno_padding`
+        Number of spaces between the (inline) line numbers and the
+        source code (default: ``2``).
+
+        .. versionadded:: 2.18
+
+    `linenostart`
+        The line number for the first line (default: ``1``).
+
+        .. versionadded:: 2.18
+
+    `linenostep`
+        If set to a number n > 1, only every nth line number is printed.
+
+        .. versionadded:: 2.18
+
+    `lineno_color`
+        Color for line numbers specified as a hex triplet, e.g. ``'5e5e5e'``. 
+        Defaults to the style's line number color if it is a hex triplet, 
+        otherwise ansi bright black.
+
+        .. versionadded:: 2.18
+
+    `hl_lines`
+        Specify a list of lines to be highlighted, as line numbers separated by
+        spaces, e.g. ``'3 7 8'``. The line numbers are relative to the input 
+        (i.e. the first line is line 1) unless `hl_linenostart` is set.
+
+        .. versionadded:: 2.18
+
+    `hl_color`
+        Color for highlighting the lines specified in `hl_lines`, specified as 
+        a hex triplet (default: style's `highlight_color`).
+
+        .. versionadded:: 2.18
+
+    `hl_linenostart`
+        If set to ``True`` line numbers in `hl_lines` are specified
+        relative to `linenostart` (default ``False``).
+
+        .. versionadded:: 2.18
     """
     name = 'RTF'
     aliases = ['rtf']
@@ -62,6 +117,40 @@ class RtfFormatter(Formatter):
         Formatter.__init__(self, **options)
         self.fontface = options.get('fontface') or ''
         self.fontsize = get_int_opt(options, 'fontsize', 0)
+        self.linenos = get_bool_opt(options, 'linenos', False)
+        self.lineno_fontsize = get_int_opt(options, 'lineno_fontsize',
+                                           self.fontsize)
+        self.lineno_padding = get_int_opt(options, 'lineno_padding', 2)
+        self.linenostart = abs(get_int_opt(options, 'linenostart', 1))
+        self.linenostep = abs(get_int_opt(options, 'linenostep', 1))
+        self.hl_linenostart = get_bool_opt(options, 'hl_linenostart', False)
+
+        self.hl_color = options.get('hl_color', '')
+        if not self.hl_color:
+            self.hl_color = self.style.highlight_color
+
+        self.hl_lines = []
+        for lineno in get_list_opt(options, 'hl_lines', []):
+            try:
+                lineno = int(lineno)
+                if self.hl_linenostart:
+                    lineno = lineno - self.linenostart + 1
+                self.hl_lines.append(lineno)
+            except ValueError:
+                pass
+
+        self.lineno_color = options.get('lineno_color', '')
+        if not self.lineno_color:
+            if  self.style.line_number_color == 'inherit':
+                # style color is the css value 'inherit'
+                # default to ansi bright-black
+                self.lineno_color = _ansimap['ansibrightblack']
+            else:
+                # style color is assumed to be a hex triplet as other
+                # colors in pygments/style.py
+                self.lineno_color = self.style.line_number_color
+
+        self.color_mapping = self._create_color_mapping()
 
     def _escape(self, text):
         return text.replace('\\', '\\\\') \
@@ -90,43 +179,145 @@ class RtfFormatter(Formatter):
                 # Force surrogate pairs
                 buf.append('{\\u%d}{\\u%d}' % surrogatepair(cn))
 
-        return ''.join(buf).replace('\n', '\\par\n')
+        return ''.join(buf).replace('\n', '\\par')
 
-    def format_unencoded(self, tokensource, outfile):
-        # rtf 1.8 header
-        outfile.write('{\\rtf1\\ansi\\uc0\\deff0'
-                      '{\\fonttbl{\\f0\\fmodern\\fprq1\\fcharset0%s;}}'
-                      '{\\colortbl;' % (self.fontface and
-                                        ' ' + self._escape(self.fontface) or
-                                        ''))
-
-        # convert colors and save them in a mapping to access them later.
-        color_mapping = {}
+    @staticmethod
+    def hex_to_rtf_color(hex_color):
+        if hex_color[0] == "#":
+            hex_color = hex_color[1:]
+
+        return '\\red%d\\green%d\\blue%d;' % (
+                        int(hex_color[0:2], 16),
+                        int(hex_color[2:4], 16),
+                        int(hex_color[4:6], 16)
+                    )
+
+    def _split_tokens_on_newlines(self, tokensource):
+        """
+        Split tokens containing newline characters into multiple token
+        each representing a line of the input file. Needed for numbering
+        lines of e.g. multiline comments.
+        """
+        for ttype, value in tokensource:
+            if value == '\n':
+                yield (ttype, value)
+            elif "\n" in value:
+                lines = value.split("\n")
+                for line in lines[:-1]:
+                    yield (ttype, line+"\n")
+                if lines[-1]:
+                    yield (ttype, lines[-1])
+            else:
+                yield (ttype, value)
+
+    def _create_color_mapping(self):
+        """
+        Create a mapping of style hex colors to index/offset in
+        the RTF color table.
+        """
+        color_mapping = OrderedDict()
         offset = 1
+
+        if self.linenos:
+            color_mapping[self.lineno_color] = offset
+            offset += 1
+
+        if self.hl_lines:
+            color_mapping[self.hl_color] = offset
+            offset += 1
+
         for _, style in self.style:
             for color in style['color'], style['bgcolor'], style['border']:
                 if color and color not in color_mapping:
                     color_mapping[color] = offset
-                    outfile.write('\\red%d\\green%d\\blue%d;' % (
-                        int(color[0:2], 16),
-                        int(color[2:4], 16),
-                        int(color[4:6], 16)
-                    ))
                     offset += 1
-        outfile.write('}\\f0 ')
+
+        return color_mapping
+
+    @property
+    def _lineno_template(self):
+        if self.lineno_fontsize != self.fontsize:
+            return '{{\\fs{} \\cf{} %s{}}}'.format(self.lineno_fontsize,
+                          self.color_mapping[self.lineno_color],
+                          " " * self.lineno_padding)
+
+        return '{{\\cf{} %s{}}}'.format(self.color_mapping[self.lineno_color],
+                      " " * self.lineno_padding)
+
+    @property
+    def _hl_open_str(self):
+        return rf'{{\highlight{self.color_mapping[self.hl_color]} '
+
+    @property
+    def _rtf_header(self):
+        lines = []
+        # rtf 1.8 header
+        lines.append('{\\rtf1\\ansi\\uc0\\deff0'
+                     '{\\fonttbl{\\f0\\fmodern\\fprq1\\fcharset0%s;}}'
+                     % (self.fontface and ' '
+                        + self._escape(self.fontface) or ''))
+
+        # color table
+        lines.append('{\\colortbl;')
+        for color, _ in self.color_mapping.items():
+            lines.append(self.hex_to_rtf_color(color))
+        lines.append('}')
+
+        # font and fontsize
+        lines.append('\\f0\\sa0')
         if self.fontsize:
-            outfile.write('\\fs%d' % self.fontsize)
+            lines.append('\\fs%d' % self.fontsize)
+
+        # ensure Libre Office Writer imports and renders consecutive
+        # space characters the same width, needed for line numbering.
+        # https://bugs.documentfoundation.org/show_bug.cgi?id=144050
+        lines.append('\\dntblnsbdb')
+
+        return lines
+
+    def format_unencoded(self, tokensource, outfile):
+        for line in self._rtf_header:
+            outfile.write(line + "\n")
+
+        tokensource = self._split_tokens_on_newlines(tokensource)
+
+        # first pass of tokens to count lines, needed for line numbering
+        if self.linenos:
+            line_count = 0
+            tokens = [] # for copying the token source generator
+            for ttype, value in tokensource:
+                tokens.append((ttype, value))
+                if value.endswith("\n"):
+                    line_count += 1
+
+            # width of line number strings (for padding with spaces)
+            linenos_width = len(str(line_count+self.linenostart-1))
+
+            tokensource = tokens
 
         # highlight stream
+        lineno = 1
+        start_new_line = True
         for ttype, value in tokensource:
+            if start_new_line and lineno in self.hl_lines:
+                outfile.write(self._hl_open_str)
+
+            if start_new_line and self.linenos:
+                if (lineno-self.linenostart+1)%self.linenostep == 0:
+                    current_lineno = lineno + self.linenostart - 1
+                    lineno_str = str(current_lineno).rjust(linenos_width)
+                else:
+                    lineno_str = "".rjust(linenos_width)
+                outfile.write(self._lineno_template % lineno_str)
+
             while not self.style.styles_token(ttype) and ttype.parent:
                 ttype = ttype.parent
             style = self.style.style_for_token(ttype)
             buf = []
             if style['bgcolor']:
-                buf.append('\\cb%d' % color_mapping[style['bgcolor']])
+                buf.append('\\cb%d' % self.color_mapping[style['bgcolor']])
             if style['color']:
-                buf.append('\\cf%d' % color_mapping[style['color']])
+                buf.append('\\cf%d' % self.color_mapping[style['color']])
             if style['bold']:
                 buf.append('\\b')
             if style['italic']:
@@ -135,12 +326,24 @@ class RtfFormatter(Formatter):
                 buf.append('\\ul')
             if style['border']:
                 buf.append('\\chbrdr\\chcfpat%d' %
-                           color_mapping[style['border']])
+                           self.color_mapping[style['border']])
             start = ''.join(buf)
             if start:
-                outfile.write('{%s ' % start)
+                outfile.write(f'{{{start} ')
             outfile.write(self._escape_text(value))
             if start:
                 outfile.write('}')
+            start_new_line = False
+
+            # complete line of input
+            if value.endswith("\n"):
+                # close line highlighting
+                if lineno in self.hl_lines:
+                    outfile.write('}')
+                # newline in RTF file after closing }
+                outfile.write("\n")
+
+                start_new_line = True
+                lineno += 1
 
-        outfile.write('}')
+        outfile.write('}\n')

data/vendor/pygments-main/pygments/formatters/svg.py

@@ -4,7 +4,7 @@
 
     Formatter for SVG output.
 
-    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -60,11 +60,11 @@ class SvgFormatter(Formatter):
 
     `linenostep`
         If set to a number n > 1, only every nth line number is printed.
-        
+
     `linenowidth`
         Maximum width devoted to line numbers (default: ``3*ystep``, sufficient
-        for up to 4-digit line numbers. Increase width for longer code blocks).  
-        
+        for up to 4-digit line numbers. Increase width for longer code blocks).
+
     `xoffset`
         Starting offset in X direction, defaults to ``0``.
 
@@ -97,10 +97,11 @@ class SvgFormatter(Formatter):
         self.fontsize = options.get('fontsize', '14px')
         self.xoffset = get_int_opt(options, 'xoffset', 0)
         fs = self.fontsize.strip()
-        if fs.endswith('px'): fs = fs[:-2].strip()
+        if fs.endswith('px'):
+            fs = fs[:-2].strip()
         try:
             int_fs = int(fs)
-        except:
+        except ValueError:
             int_fs = 20
         self.yoffset = get_int_opt(options, 'yoffset', int_fs)
         self.ystep = get_int_opt(options, 'ystep', int_fs + 5)
@@ -122,30 +123,27 @@ class SvgFormatter(Formatter):
         y = self.yoffset
         if not self.nowrap:
             if self.encoding:
-                outfile.write('<?xml version="1.0" encoding="%s"?>\n' %
-                              self.encoding)
+                outfile.write(f'<?xml version="1.0" encoding="{self.encoding}"?>\n')
             else:
                 outfile.write('<?xml version="1.0"?>\n')
             outfile.write('<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" '
                           '"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/'
                           'svg10.dtd">\n')
             outfile.write('<svg xmlns="http://www.w3.org/2000/svg">\n')
-            outfile.write('<g font-family="%s" font-size="%s">\n' %
-                          (self.fontfamily, self.fontsize))
-        
-        counter = self.linenostart 
+            outfile.write(f'<g font-family="{self.fontfamily}" font-size="{self.fontsize}">\n')
+
+        counter = self.linenostart
         counter_step = self.linenostep
         counter_style = self._get_style(Comment)
         line_x = x
-        
+
         if self.linenos:
             if counter % counter_step == 0:
-                outfile.write('<text x="%s" y="%s" %s text-anchor="end">%s</text>' %
-                    (x+self.linenowidth,y,counter_style,counter))
+                outfile.write(f'<text x="{x+self.linenowidth}" y="{y}" {counter_style} text-anchor="end">{counter}</text>')
             line_x += self.linenowidth + self.ystep
             counter += 1
 
-        outfile.write('<text x="%s" y="%s" xml:space="preserve">' % (line_x, y))
+        outfile.write(f'<text x="{line_x}" y="{y}" xml:space="preserve">')
         for ttype, value in tokensource:
             style = self._get_style(ttype)
             tspan = style and '<tspan' + style + '>' or ''
@@ -159,11 +157,10 @@ class SvgFormatter(Formatter):
                 y += self.ystep
                 outfile.write('</text>\n')
                 if self.linenos and counter % counter_step == 0:
-                    outfile.write('<text x="%s" y="%s" text-anchor="end" %s>%s</text>' %
-                        (x+self.linenowidth,y,counter_style,counter))
-                
+                    outfile.write(f'<text x="{x+self.linenowidth}" y="{y}" text-anchor="end" {counter_style}>{counter}</text>')
+
                 counter += 1
-                outfile.write('<text x="%s" y="%s" ' 'xml:space="preserve">' % (line_x,y))
+                outfile.write(f'<text x="{line_x}" y="{y}" ' 'xml:space="preserve">')
             outfile.write(tspan + parts[-1] + tspanend)
         outfile.write('</text>')
 

data/vendor/pygments-main/pygments/formatters/terminal.py

@@ -4,7 +4,7 @@
 
     Formatter for terminal output with ANSI sequences.
 
-    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/formatters/terminal256.py

@@ -10,7 +10,7 @@
 
     Formatter version 1.
 
-    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexer.py

@@ -4,7 +4,7 @@
 
     Base lexer classes.
 
-    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -67,10 +67,17 @@ class Lexer(metaclass=LexerMeta):
        :no-value:
     .. autoattribute:: priority
 
-    Lexers included in Pygments should have an additional attribute:
+    Lexers included in Pygments should have two additional attributes:
 
     .. autoattribute:: url
        :no-value:
+    .. autoattribute:: version_added
+       :no-value:
+
+    Lexers included in Pygments may have additional attributes:
+
+    .. autoattribute:: _example
+       :no-value:
 
     You can pass options to the constructor. The basic options recognized
     by all lexers and processed by the base `Lexer` class are:
@@ -125,9 +132,16 @@ class Lexer(metaclass=LexerMeta):
     priority = 0
 
     #: URL of the language specification/definition. Used in the Pygments
-    #: documentation.
+    #: documentation. Set to an empty string to disable.
     url = None
 
+    #: Version of Pygments in which the lexer was added.
+    version_added = None
+
+    #: Example file name. Relative to the ``tests/examplefiles`` directory.
+    #: This is used by the documentation generator to show an example.
+    _example = None
+
     def __init__(self, **options):
         """
         This constructor takes arbitrary options as keyword arguments.
@@ -160,10 +174,9 @@ class Lexer(metaclass=LexerMeta):
 
     def __repr__(self):
         if self.options:
-            return '<pygments.lexers.%s with %r>' % (self.__class__.__name__,
-                                                     self.options)
+            return f'<pygments.lexers.{self.__class__.__name__} with {self.options!r}>'
         else:
-            return '<pygments.lexers.%s>' % self.__class__.__name__
+            return f'<pygments.lexers.{self.__class__.__name__}>'
 
     def add_filter(self, filter_, **options):
         """
@@ -190,20 +203,9 @@ class Lexer(metaclass=LexerMeta):
         it's the same as if the return values was ``0.0``.
         """
 
-    def get_tokens(self, text, unfiltered=False):
-        """
-        This method is the basic interface of a lexer. It is called by
-        the `highlight()` function. It must process the text and return an
-        iterable of ``(tokentype, value)`` pairs from `text`.
-
-        Normally, you don't need to override this method. The default
-        implementation processes the options recognized by all lexers
-        (`stripnl`, `stripall` and so on), and then yields all tokens
-        from `get_tokens_unprocessed()`, with the ``index`` dropped.
+    def _preprocess_lexer_input(self, text):
+        """Apply preprocessing such as decoding the input, removing BOM and normalizing newlines."""
 
-        If `unfiltered` is set to `True`, the filtering mechanism is
-        bypassed even if filters are defined.
-        """
         if not isinstance(text, str):
             if self.encoding == 'guess':
                 text, _ = guess_decode(text)
@@ -246,6 +248,24 @@ class Lexer(metaclass=LexerMeta):
         if self.ensurenl and not text.endswith('\n'):
             text += '\n'
 
+        return text
+
+    def get_tokens(self, text, unfiltered=False):
+        """
+        This method is the basic interface of a lexer. It is called by
+        the `highlight()` function. It must process the text and return an
+        iterable of ``(tokentype, value)`` pairs from `text`.
+
+        Normally, you don't need to override this method. The default
+        implementation processes the options recognized by all lexers
+        (`stripnl`, `stripall` and so on), and then yields all tokens
+        from `get_tokens_unprocessed()`, with the ``index`` dropped.
+
+        If `unfiltered` is set to `True`, the filtering mechanism is
+        bypassed even if filters are defined.
+        """
+        text = self._preprocess_lexer_input(text)
+
         def streamer():
             for _, t, v in self.get_tokens_unprocessed(text):
                 yield t, v
@@ -490,7 +510,7 @@ class RegexLexerMeta(LexerMeta):
     def _process_token(cls, token):
         """Preprocess the token component of a token definition."""
         assert type(token) is _TokenType or callable(token), \
-            'token type must be simple type or callable, not %r' % (token,)
+            f'token type must be simple type or callable, not {token!r}'
         return token
 
     def _process_new_state(cls, new_state, unprocessed, processed):
@@ -506,14 +526,14 @@ class RegexLexerMeta(LexerMeta):
             elif new_state[:5] == '#pop:':
                 return -int(new_state[5:])
             else:
-                assert False, 'unknown new state %r' % new_state
+                assert False, f'unknown new state {new_state!r}'
         elif isinstance(new_state, combined):
             # combine a new state from existing ones
             tmp_state = '_tmp_%d' % cls._tmpname
             cls._tmpname += 1
             itokens = []
             for istate in new_state:
-                assert istate != new_state, 'circular state ref %r' % istate
+                assert istate != new_state, f'circular state ref {istate!r}'
                 itokens.extend(cls._process_state(unprocessed,
                                                   processed, istate))
             processed[tmp_state] = itokens
@@ -526,12 +546,12 @@ class RegexLexerMeta(LexerMeta):
                     'unknown new state ' + istate
             return new_state
         else:
-            assert False, 'unknown new state def %r' % new_state
+            assert False, f'unknown new state def {new_state!r}'
 
     def _process_state(cls, unprocessed, processed, state):
         """Preprocess a single state definition."""
-        assert type(state) is str, "wrong state name %r" % state
-        assert state[0] != '#', "invalid state name %r" % state
+        assert isinstance(state, str), f"wrong state name {state!r}"
+        assert state[0] != '#', f"invalid state name {state!r}"
         if state in processed:
             return processed[state]
         tokens = processed[state] = []
@@ -539,7 +559,7 @@ class RegexLexerMeta(LexerMeta):
         for tdef in unprocessed[state]:
             if isinstance(tdef, include):
                 # it's a state reference
-                assert tdef != state, "circular state reference %r" % state
+                assert tdef != state, f"circular state reference {state!r}"
                 tokens.extend(cls._process_state(unprocessed, processed,
                                                  str(tdef)))
                 continue
@@ -553,13 +573,12 @@ class RegexLexerMeta(LexerMeta):
                 tokens.append((re.compile('').match, None, new_state))
                 continue
 
-            assert type(tdef) is tuple, "wrong rule def %r" % tdef
+            assert type(tdef) is tuple, f"wrong rule def {tdef!r}"
 
             try:
                 rex = cls._process_regex(tdef[0], rflags, state)
             except Exception as err:
-                raise ValueError("uncompilable regex %r in state %r of %r: %s" %
-                                 (tdef[0], state, cls, err)) from err
+                raise ValueError(f"uncompilable regex {tdef[0]!r} in state {state!r} of {cls!r}: {err}") from err
 
             token = cls._process_token(tdef[1])
 
@@ -720,7 +739,7 @@ class RegexLexer(Lexer, metaclass=RegexLexerMeta):
                         elif new_state == '#push':
                             statestack.append(statestack[-1])
                         else:
-                            assert False, "wrong state def: %r" % new_state
+                            assert False, f"wrong state def: {new_state!r}"
                         statetokens = tokendefs[statestack[-1]]
                     break
             else:
@@ -752,8 +771,7 @@ class LexerContext:
         self.stack = stack or ['root']
 
     def __repr__(self):
-        return 'LexerContext(%r, %r, %r)' % (
-            self.text, self.pos, self.stack)
+        return f'LexerContext({self.text!r}, {self.pos!r}, {self.stack!r})'
 
 
 class ExtendedRegexLexer(RegexLexer):
@@ -808,7 +826,7 @@ class ExtendedRegexLexer(RegexLexer):
                         elif new_state == '#push':
                             ctx.stack.append(ctx.stack[-1])
                         else:
-                            assert False, "wrong state def: %r" % new_state
+                            assert False, f"wrong state def: {new_state!r}"
                         statetokens = tokendefs[ctx.stack[-1]]
                     break
             else: