@pyscript/core 0.5.7 → 0.5.9

package/env/lib/python3.12/site-packages/pip/_vendor/pygments/formatters/irc.py

@@ -1,154 +0,0 @@
-"""
-    pygments.formatters.irc
-    ~~~~~~~~~~~~~~~~~~~~~~~
-
-    Formatter for IRC output
-
-    :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-from pip._vendor.pygments.formatter import Formatter
-from pip._vendor.pygments.token import Keyword, Name, Comment, String, Error, \
-    Number, Operator, Generic, Token, Whitespace
-from pip._vendor.pygments.util import get_choice_opt
-
-
-__all__ = ['IRCFormatter']
-
-
-#: Map token types to a tuple of color values for light and dark
-#: backgrounds.
-IRC_COLORS = {
-    Token:              ('',            ''),
-
-    Whitespace:         ('gray',   'brightblack'),
-    Comment:            ('gray',   'brightblack'),
-    Comment.Preproc:    ('cyan',        'brightcyan'),
-    Keyword:            ('blue',    'brightblue'),
-    Keyword.Type:       ('cyan',        'brightcyan'),
-    Operator.Word:      ('magenta',      'brightcyan'),
-    Name.Builtin:       ('cyan',        'brightcyan'),
-    Name.Function:      ('green',   'brightgreen'),
-    Name.Namespace:     ('_cyan_',      '_brightcyan_'),
-    Name.Class:         ('_green_', '_brightgreen_'),
-    Name.Exception:     ('cyan',        'brightcyan'),
-    Name.Decorator:     ('brightblack',    'gray'),
-    Name.Variable:      ('red',     'brightred'),
-    Name.Constant:      ('red',     'brightred'),
-    Name.Attribute:     ('cyan',        'brightcyan'),
-    Name.Tag:           ('brightblue',        'brightblue'),
-    String:             ('yellow',       'yellow'),
-    Number:             ('blue',    'brightblue'),
-
-    Generic.Deleted:    ('brightred',        'brightred'),
-    Generic.Inserted:   ('green',  'brightgreen'),
-    Generic.Heading:    ('**',         '**'),
-    Generic.Subheading: ('*magenta*',   '*brightmagenta*'),
-    Generic.Error:      ('brightred',        'brightred'),
-
-    Error:              ('_brightred_',      '_brightred_'),
-}
-
-
-IRC_COLOR_MAP = {
-    'white': 0,
-    'black': 1,
-    'blue': 2,
-    'brightgreen': 3,
-    'brightred': 4,
-    'yellow': 5,
-    'magenta': 6,
-    'orange': 7,
-    'green': 7, #compat w/ ansi
-    'brightyellow': 8,
-    'lightgreen': 9,
-    'brightcyan': 9, # compat w/ ansi
-    'cyan': 10,
-    'lightblue': 11,
-    'red': 11, # compat w/ ansi
-    'brightblue': 12,
-    'brightmagenta': 13,
-    'brightblack': 14,
-    'gray': 15,
-}
-
-def ircformat(color, text):
-    if len(color) < 1:
-        return text
-    add = sub = ''
-    if '_' in color: # italic
-        add += '\x1D'
-        sub = '\x1D' + sub
-        color = color.strip('_')
-    if '*' in color: # bold
-        add += '\x02'
-        sub = '\x02' + sub
-        color = color.strip('*')
-    # underline (\x1F) not supported
-    # backgrounds (\x03FF,BB) not supported
-    if len(color) > 0: # actual color - may have issues with ircformat("red", "blah")+"10" type stuff
-        add += '\x03' + str(IRC_COLOR_MAP[color]).zfill(2)
-        sub = '\x03' + sub
-    return add + text + sub
-    return '<'+add+'>'+text+'</'+sub+'>'
-
-
-class IRCFormatter(Formatter):
-    r"""
-    Format tokens with IRC color sequences
-
-    The `get_style_defs()` method doesn't do anything special since there is
-    no support for common styles.
-
-    Options accepted:
-
-    `bg`
-        Set to ``"light"`` or ``"dark"`` depending on the terminal's background
-        (default: ``"light"``).
-
-    `colorscheme`
-        A dictionary mapping token types to (lightbg, darkbg) color names or
-        ``None`` (default: ``None`` = use builtin colorscheme).
-
-    `linenos`
-        Set to ``True`` to have line numbers in the output as well
-        (default: ``False`` = no line numbers).
-    """
-    name = 'IRC'
-    aliases = ['irc', 'IRC']
-    filenames = []
-
-    def __init__(self, **options):
-        Formatter.__init__(self, **options)
-        self.darkbg = get_choice_opt(options, 'bg',
-                                     ['light', 'dark'], 'light') == 'dark'
-        self.colorscheme = options.get('colorscheme', None) or IRC_COLORS
-        self.linenos = options.get('linenos', False)
-        self._lineno = 0
-
-    def _write_lineno(self, outfile):
-        if self.linenos:
-            self._lineno += 1
-            outfile.write("%04d: " % self._lineno)
-
-    def format_unencoded(self, tokensource, outfile):
-        self._write_lineno(outfile)
-
-        for ttype, value in tokensource:
-            color = self.colorscheme.get(ttype)
-            while color is None:
-                ttype = ttype[:-1]
-                color = self.colorscheme.get(ttype)
-            if color:
-                color = color[self.darkbg]
-                spl = value.split('\n')
-                for line in spl[:-1]:
-                    if line:
-                        outfile.write(ircformat(color, line))
-                    outfile.write('\n')
-                    self._write_lineno(outfile)
-                if spl[-1]:
-                    outfile.write(ircformat(color, spl[-1]))
-            else:
-                outfile.write(value)

package/env/lib/python3.12/site-packages/pip/_vendor/pygments/formatters/latex.py

@@ -1,518 +0,0 @@
-"""
-    pygments.formatters.latex
-    ~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    Formatter for LaTeX fancyvrb output.
-
-    :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS.
-    :license: BSD, see LICENSE for details.
-"""
-
-from io import StringIO
-
-from pip._vendor.pygments.formatter import Formatter
-from pip._vendor.pygments.lexer import Lexer, do_insertions
-from pip._vendor.pygments.token import Token, STANDARD_TYPES
-from pip._vendor.pygments.util import get_bool_opt, get_int_opt
-
-
-__all__ = ['LatexFormatter']
-
-
-def escape_tex(text, commandprefix):
-    return text.replace('\\', '\x00'). \
-                replace('{', '\x01'). \
-                replace('}', '\x02'). \
-                replace('\x00', rf'\{commandprefix}Zbs{{}}'). \
-                replace('\x01', rf'\{commandprefix}Zob{{}}'). \
-                replace('\x02', rf'\{commandprefix}Zcb{{}}'). \
-                replace('^', rf'\{commandprefix}Zca{{}}'). \
-                replace('_', rf'\{commandprefix}Zus{{}}'). \
-                replace('&', rf'\{commandprefix}Zam{{}}'). \
-                replace('<', rf'\{commandprefix}Zlt{{}}'). \
-                replace('>', rf'\{commandprefix}Zgt{{}}'). \
-                replace('#', rf'\{commandprefix}Zsh{{}}'). \
-                replace('%', rf'\{commandprefix}Zpc{{}}'). \
-                replace('$', rf'\{commandprefix}Zdl{{}}'). \
-                replace('-', rf'\{commandprefix}Zhy{{}}'). \
-                replace("'", rf'\{commandprefix}Zsq{{}}'). \
-                replace('"', rf'\{commandprefix}Zdq{{}}'). \
-                replace('~', rf'\{commandprefix}Zti{{}}')
-
-
-DOC_TEMPLATE = r'''
-\documentclass{%(docclass)s}
-\usepackage{fancyvrb}
-\usepackage{color}
-\usepackage[%(encoding)s]{inputenc}
-%(preamble)s
-
-%(styledefs)s
-
-\begin{document}
-
-\section*{%(title)s}
-
-%(code)s
-\end{document}
-'''
-
-## Small explanation of the mess below :)
-#
-# The previous version of the LaTeX formatter just assigned a command to
-# each token type defined in the current style.  That obviously is
-# problematic if the highlighted code is produced for a different style
-# than the style commands themselves.
-#
-# This version works much like the HTML formatter which assigns multiple
-# CSS classes to each <span> tag, from the most specific to the least
-# specific token type, thus falling back to the parent token type if one
-# is not defined.  Here, the classes are there too and use the same short
-# forms given in token.STANDARD_TYPES.
-#
-# Highlighted code now only uses one custom command, which by default is
-# \PY and selectable by the commandprefix option (and in addition the
-# escapes \PYZat, \PYZlb and \PYZrb which haven't been renamed for
-# backwards compatibility purposes).
-#
-# \PY has two arguments: the classes, separated by +, and the text to
-# render in that style.  The classes are resolved into the respective
-# style commands by magic, which serves to ignore unknown classes.
-#
-# The magic macros are:
-# * \PY@it, \PY@bf, etc. are unconditionally wrapped around the text
-#   to render in \PY@do.  Their definition determines the style.
-# * \PY@reset resets \PY@it etc. to do nothing.
-# * \PY@toks parses the list of classes, using magic inspired by the
-#   keyval package (but modified to use plusses instead of commas
-#   because fancyvrb redefines commas inside its environments).
-# * \PY@tok processes one class, calling the \PY@tok@classname command
-#   if it exists.
-# * \PY@tok@classname sets the \PY@it etc. to reflect the chosen style
-#   for its class.
-# * \PY resets the style, parses the classnames and then calls \PY@do.
-#
-# Tip: to read this code, print it out in substituted form using e.g.
-# >>> print STYLE_TEMPLATE % {'cp': 'PY'}
-
-STYLE_TEMPLATE = r'''
-\makeatletter
-\def\%(cp)s@reset{\let\%(cp)s@it=\relax \let\%(cp)s@bf=\relax%%
-    \let\%(cp)s@ul=\relax \let\%(cp)s@tc=\relax%%
-    \let\%(cp)s@bc=\relax \let\%(cp)s@ff=\relax}
-\def\%(cp)s@tok#1{\csname %(cp)s@tok@#1\endcsname}
-\def\%(cp)s@toks#1+{\ifx\relax#1\empty\else%%
-    \%(cp)s@tok{#1}\expandafter\%(cp)s@toks\fi}
-\def\%(cp)s@do#1{\%(cp)s@bc{\%(cp)s@tc{\%(cp)s@ul{%%
-    \%(cp)s@it{\%(cp)s@bf{\%(cp)s@ff{#1}}}}}}}
-\def\%(cp)s#1#2{\%(cp)s@reset\%(cp)s@toks#1+\relax+\%(cp)s@do{#2}}
-
-%(styles)s
-
-\def\%(cp)sZbs{\char`\\}
-\def\%(cp)sZus{\char`\_}
-\def\%(cp)sZob{\char`\{}
-\def\%(cp)sZcb{\char`\}}
-\def\%(cp)sZca{\char`\^}
-\def\%(cp)sZam{\char`\&}
-\def\%(cp)sZlt{\char`\<}
-\def\%(cp)sZgt{\char`\>}
-\def\%(cp)sZsh{\char`\#}
-\def\%(cp)sZpc{\char`\%%}
-\def\%(cp)sZdl{\char`\$}
-\def\%(cp)sZhy{\char`\-}
-\def\%(cp)sZsq{\char`\'}
-\def\%(cp)sZdq{\char`\"}
-\def\%(cp)sZti{\char`\~}
-%% for compatibility with earlier versions
-\def\%(cp)sZat{@}
-\def\%(cp)sZlb{[}
-\def\%(cp)sZrb{]}
-\makeatother
-'''
-
-
-def _get_ttype_name(ttype):
-    fname = STANDARD_TYPES.get(ttype)
-    if fname:
-        return fname
-    aname = ''
-    while fname is None:
-        aname = ttype[-1] + aname
-        ttype = ttype.parent
-        fname = STANDARD_TYPES.get(ttype)
-    return fname + aname
-
-
-class LatexFormatter(Formatter):
-    r"""
-    Format tokens as LaTeX code. This needs the `fancyvrb` and `color`
-    standard packages.
-
-    Without the `full` option, code is formatted as one ``Verbatim``
-    environment, like this:
-
-    .. sourcecode:: latex
-
-        \begin{Verbatim}[commandchars=\\\{\}]
-        \PY{k}{def }\PY{n+nf}{foo}(\PY{n}{bar}):
-            \PY{k}{pass}
-        \end{Verbatim}
-
-    Wrapping can be disabled using the `nowrap` option.
-
-    The special command used here (``\PY``) and all the other macros it needs
-    are output by the `get_style_defs` method.
-
-    With the `full` option, a complete LaTeX document is output, including
-    the command definitions in the preamble.
-
-    The `get_style_defs()` method of a `LatexFormatter` returns a string
-    containing ``\def`` commands defining the macros needed inside the
-    ``Verbatim`` environments.
-
-    Additional options accepted:
-
-    `nowrap`
-        If set to ``True``, don't wrap the tokens at all, not even inside a
-        ``\begin{Verbatim}`` environment. This disables most other options
-        (default: ``False``).
-
-    `style`
-        The style to use, can be a string or a Style subclass (default:
-        ``'default'``).
-
-    `full`
-        Tells the formatter to output a "full" document, i.e. a complete
-        self-contained document (default: ``False``).
-
-    `title`
-        If `full` is true, the title that should be used to caption the
-        document (default: ``''``).
-
-    `docclass`
-        If the `full` option is enabled, this is the document class to use
-        (default: ``'article'``).
-
-    `preamble`
-        If the `full` option is enabled, this can be further preamble commands,
-        e.g. ``\usepackage`` (default: ``''``).
-
-    `linenos`
-        If set to ``True``, output line numbers (default: ``False``).
-
-    `linenostart`
-        The line number for the first line (default: ``1``).
-
-    `linenostep`
-        If set to a number n > 1, only every nth line number is printed.
-
-    `verboptions`
-        Additional options given to the Verbatim environment (see the *fancyvrb*
-        docs for possible values) (default: ``''``).
-
-    `commandprefix`
-        The LaTeX commands used to produce colored output are constructed
-        using this prefix and some letters (default: ``'PY'``).
-
-        .. versionadded:: 0.7
-        .. versionchanged:: 0.10
-           The default is now ``'PY'`` instead of ``'C'``.
-
-    `texcomments`
-        If set to ``True``, enables LaTeX comment lines.  That is, LaTex markup
-        in comment tokens is not escaped so that LaTeX can render it (default:
-        ``False``).
-
-        .. versionadded:: 1.2
-
-    `mathescape`
-        If set to ``True``, enables LaTeX math mode escape in comments. That
-        is, ``'$...$'`` inside a comment will trigger math mode (default:
-        ``False``).
-
-        .. versionadded:: 1.2
-
-    `escapeinside`
-        If set to a string of length 2, enables escaping to LaTeX. Text
-        delimited by these 2 characters is read as LaTeX code and
-        typeset accordingly. It has no effect in string literals. It has
-        no effect in comments if `texcomments` or `mathescape` is
-        set. (default: ``''``).
-
-        .. versionadded:: 2.0
-
-    `envname`
-        Allows you to pick an alternative environment name replacing Verbatim.
-        The alternate environment still has to support Verbatim's option syntax.
-        (default: ``'Verbatim'``).
-
-        .. versionadded:: 2.0
-    """
-    name = 'LaTeX'
-    aliases = ['latex', 'tex']
-    filenames = ['*.tex']
-
-    def __init__(self, **options):
-        Formatter.__init__(self, **options)
-        self.nowrap = get_bool_opt(options, 'nowrap', False)
-        self.docclass = options.get('docclass', 'article')
-        self.preamble = options.get('preamble', '')
-        self.linenos = get_bool_opt(options, 'linenos', False)
-        self.linenostart = abs(get_int_opt(options, 'linenostart', 1))
-        self.linenostep = abs(get_int_opt(options, 'linenostep', 1))
-        self.verboptions = options.get('verboptions', '')
-        self.nobackground = get_bool_opt(options, 'nobackground', False)
-        self.commandprefix = options.get('commandprefix', 'PY')
-        self.texcomments = get_bool_opt(options, 'texcomments', False)
-        self.mathescape = get_bool_opt(options, 'mathescape', False)
-        self.escapeinside = options.get('escapeinside', '')
-        if len(self.escapeinside) == 2:
-            self.left = self.escapeinside[0]
-            self.right = self.escapeinside[1]
-        else:
-            self.escapeinside = ''
-        self.envname = options.get('envname', 'Verbatim')
-
-        self._create_stylesheet()
-
-    def _create_stylesheet(self):
-        t2n = self.ttype2name = {Token: ''}
-        c2d = self.cmd2def = {}
-        cp = self.commandprefix
-
-        def rgbcolor(col):
-            if col:
-                return ','.join(['%.2f' % (int(col[i] + col[i + 1], 16) / 255.0)
-                                 for i in (0, 2, 4)])
-            else:
-                return '1,1,1'
-
-        for ttype, ndef in self.style:
-            name = _get_ttype_name(ttype)
-            cmndef = ''
-            if ndef['bold']:
-                cmndef += r'\let\$$@bf=\textbf'
-            if ndef['italic']:
-                cmndef += r'\let\$$@it=\textit'
-            if ndef['underline']:
-                cmndef += r'\let\$$@ul=\underline'
-            if ndef['roman']:
-                cmndef += r'\let\$$@ff=\textrm'
-            if ndef['sans']:
-                cmndef += r'\let\$$@ff=\textsf'
-            if ndef['mono']:
-                cmndef += r'\let\$$@ff=\textsf'
-            if ndef['color']:
-                cmndef += (r'\def\$$@tc##1{{\textcolor[rgb]{{{}}}{{##1}}}}'.format(rgbcolor(ndef['color'])))
-            if ndef['border']:
-                cmndef += (r'\def\$$@bc##1{{{{\setlength{{\fboxsep}}{{\string -\fboxrule}}'
-                           r'\fcolorbox[rgb]{{{}}}{{{}}}{{\strut ##1}}}}}}'.format(rgbcolor(ndef['border']),
-                            rgbcolor(ndef['bgcolor'])))
-            elif ndef['bgcolor']:
-                cmndef += (r'\def\$$@bc##1{{{{\setlength{{\fboxsep}}{{0pt}}'
-                           r'\colorbox[rgb]{{{}}}{{\strut ##1}}}}}}'.format(rgbcolor(ndef['bgcolor'])))
-            if cmndef == '':
-                continue
-            cmndef = cmndef.replace('$$', cp)
-            t2n[ttype] = name
-            c2d[name] = cmndef
-
-    def get_style_defs(self, arg=''):
-        """
-        Return the command sequences needed to define the commands
-        used to format text in the verbatim environment. ``arg`` is ignored.
-        """
-        cp = self.commandprefix
-        styles = []
-        for name, definition in self.cmd2def.items():
-            styles.append(rf'\@namedef{{{cp}@tok@{name}}}{{{definition}}}')
-        return STYLE_TEMPLATE % {'cp': self.commandprefix,
-                                 'styles': '\n'.join(styles)}
-
-    def format_unencoded(self, tokensource, outfile):
-        # TODO: add support for background colors
-        t2n = self.ttype2name
-        cp = self.commandprefix
-
-        if self.full:
-            realoutfile = outfile
-            outfile = StringIO()
-
-        if not self.nowrap:
-            outfile.write('\\begin{' + self.envname + '}[commandchars=\\\\\\{\\}')
-            if self.linenos:
-                start, step = self.linenostart, self.linenostep
-                outfile.write(',numbers=left' +
-                              (start and ',firstnumber=%d' % start or '') +
-                              (step and ',stepnumber=%d' % step or ''))
-            if self.mathescape or self.texcomments or self.escapeinside:
-                outfile.write(',codes={\\catcode`\\$=3\\catcode`\\^=7'
-                              '\\catcode`\\_=8\\relax}')
-            if self.verboptions:
-                outfile.write(',' + self.verboptions)
-            outfile.write(']\n')
-
-        for ttype, value in tokensource:
-            if ttype in Token.Comment:
-                if self.texcomments:
-                    # Try to guess comment starting lexeme and escape it ...
-                    start = value[0:1]
-                    for i in range(1, len(value)):
-                        if start[0] != value[i]:
-                            break
-                        start += value[i]
-
-                    value = value[len(start):]
-                    start = escape_tex(start, cp)
-
-                    # ... but do not escape inside comment.
-                    value = start + value
-                elif self.mathescape:
-                    # Only escape parts not inside a math environment.
-                    parts = value.split('$')
-                    in_math = False
-                    for i, part in enumerate(parts):
-                        if not in_math:
-                            parts[i] = escape_tex(part, cp)
-                        in_math = not in_math
-                    value = '$'.join(parts)
-                elif self.escapeinside:
-                    text = value
-                    value = ''
-                    while text:
-                        a, sep1, text = text.partition(self.left)
-                        if sep1:
-                            b, sep2, text = text.partition(self.right)
-                            if sep2:
-                                value += escape_tex(a, cp) + b
-                            else:
-                                value += escape_tex(a + sep1 + b, cp)
-                        else:
-                            value += escape_tex(a, cp)
-                else:
-                    value = escape_tex(value, cp)
-            elif ttype not in Token.Escape:
-                value = escape_tex(value, cp)
-            styles = []
-            while ttype is not Token:
-                try:
-                    styles.append(t2n[ttype])
-                except KeyError:
-                    # not in current style
-                    styles.append(_get_ttype_name(ttype))
-                ttype = ttype.parent
-            styleval = '+'.join(reversed(styles))
-            if styleval:
-                spl = value.split('\n')
-                for line in spl[:-1]:
-                    if line:
-                        outfile.write(f"\\{cp}{{{styleval}}}{{{line}}}")
-                    outfile.write('\n')
-                if spl[-1]:
-                    outfile.write(f"\\{cp}{{{styleval}}}{{{spl[-1]}}}")
-            else:
-                outfile.write(value)
-
-        if not self.nowrap:
-            outfile.write('\\end{' + self.envname + '}\n')
-
-        if self.full:
-            encoding = self.encoding or 'utf8'
-            # map known existings encodings from LaTeX distribution
-            encoding = {
-                'utf_8': 'utf8',
-                'latin_1': 'latin1',
-                'iso_8859_1': 'latin1',
-            }.get(encoding.replace('-', '_'), encoding)
-            realoutfile.write(DOC_TEMPLATE %
-                dict(docclass  = self.docclass,
-                     preamble  = self.preamble,
-                     title     = self.title,
-                     encoding  = encoding,
-                     styledefs = self.get_style_defs(),
-                     code      = outfile.getvalue()))
-
-
-class LatexEmbeddedLexer(Lexer):
-    """
-    This lexer takes one lexer as argument, the lexer for the language
-    being formatted, and the left and right delimiters for escaped text.
-
-    First everything is scanned using the language lexer to obtain
-    strings and comments. All other consecutive tokens are merged and
-    the resulting text is scanned for escaped segments, which are given
-    the Token.Escape type. Finally text that is not escaped is scanned
-    again with the language lexer.
-    """
-    def __init__(self, left, right, lang, **options):
-        self.left = left
-        self.right = right
-        self.lang = lang
-        Lexer.__init__(self, **options)
-
-    def get_tokens_unprocessed(self, text):
-        # find and remove all the escape tokens (replace with an empty string)
-        # this is very similar to DelegatingLexer.get_tokens_unprocessed.
-        buffered = ''
-        insertions = []
-        insertion_buf = []
-        for i, t, v in self._find_safe_escape_tokens(text):
-            if t is None:
-                if insertion_buf:
-                    insertions.append((len(buffered), insertion_buf))
-                    insertion_buf = []
-                buffered += v
-            else:
-                insertion_buf.append((i, t, v))
-        if insertion_buf:
-            insertions.append((len(buffered), insertion_buf))
-        return do_insertions(insertions,
-                             self.lang.get_tokens_unprocessed(buffered))
-
-    def _find_safe_escape_tokens(self, text):
-        """ find escape tokens that are not in strings or comments """
-        for i, t, v in self._filter_to(
-            self.lang.get_tokens_unprocessed(text),
-            lambda t: t in Token.Comment or t in Token.String
-        ):
-            if t is None:
-                for i2, t2, v2 in self._find_escape_tokens(v):
-                    yield i + i2, t2, v2
-            else:
-                yield i, None, v
-
-    def _filter_to(self, it, pred):
-        """ Keep only the tokens that match `pred`, merge the others together """
-        buf = ''
-        idx = 0
-        for i, t, v in it:
-            if pred(t):
-                if buf:
-                    yield idx, None, buf
-                    buf = ''
-                yield i, t, v
-            else:
-                if not buf:
-                    idx = i
-                buf += v
-        if buf:
-            yield idx, None, buf
-
-    def _find_escape_tokens(self, text):
-        """ Find escape tokens within text, give token=None otherwise """
-        index = 0
-        while text:
-            a, sep1, text = text.partition(self.left)
-            if a:
-                yield index, None, a
-                index += len(a)
-            if sep1:
-                b, sep2, text = text.partition(self.right)
-                if sep2:
-                    yield index + len(sep1), Token.Escape, b
-                    index += len(sep1) + len(b) + len(sep2)
-                else:
-                    yield index, Token.Error, sep1
-                    index += len(sep1)
-                    text = b