pygments.rb 2.3.1 → 2.4.0

data/vendor/pygments-main/pygments/lexer.py

@@ -4,7 +4,7 @@
 
     Base lexer classes.
 
-    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -14,15 +14,16 @@ import time
 
 from pygments.filter import apply_filters, Filter
 from pygments.filters import get_filter_by_name
-from pygments.token import Error, Text, Other, _TokenType
+from pygments.token import Error, Text, Other, Whitespace, _TokenType
 from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
     make_analysator, Future, guess_decode
 from pygments.regexopt import regex_opt
 
 __all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer',
            'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this',
-           'default', 'words']
+           'default', 'words', 'line_re']
 
+line_re = re.compile('.*?\n')
 
 _encoding_map = [(b'\xef\xbb\xbf', 'utf-8'),
                  (b'\xff\xfe\0\0', 'utf-32'),
@@ -49,7 +50,31 @@ class Lexer(metaclass=LexerMeta):
     """
     Lexer for a specific language.
 
-    Basic options recognized:
+    See also :doc:`lexerdevelopment`, a high-level guide to writing
+    lexers.
+
+    Lexer classes have attributes used for choosing the most appropriate
+    lexer based on various criteria.
+
+    .. autoattribute:: name
+       :no-value:
+    .. autoattribute:: aliases
+       :no-value:
+    .. autoattribute:: filenames
+       :no-value:
+    .. autoattribute:: alias_filenames
+    .. autoattribute:: mimetypes
+       :no-value:
+    .. autoattribute:: priority
+
+    Lexers included in Pygments should have an additional attribute:
+
+    .. autoattribute:: url
+       :no-value:
+
+    You can pass options to the constructor. The basic options recognized
+    by all lexers and processed by the base `Lexer` class are:
+
     ``stripnl``
         Strip leading and trailing newlines from the input (default: True).
     ``stripall``
@@ -73,25 +98,55 @@ class Lexer(metaclass=LexerMeta):
         Overrides the ``encoding`` if given.
     """
 
-    #: Name of the lexer
+    #: Full name of the lexer, in human-readable form
     name = None
 
-    #: Shortcuts for the lexer
+    #: A list of short, unique identifiers that can be used to look
+    #: up the lexer from a list, e.g., using `get_lexer_by_name()`.
     aliases = []
 
-    #: File name globs
+    #: A list of `fnmatch` patterns that match filenames which contain
+    #: content for this lexer. The patterns in this list should be unique among
+    #: all lexers.
     filenames = []
 
-    #: Secondary file name globs
+    #: A list of `fnmatch` patterns that match filenames which may or may not
+    #: contain content for this lexer. This list is used by the
+    #: :func:`.guess_lexer_for_filename()` function, to determine which lexers
+    #: are then included in guessing the correct one. That means that
+    #: e.g. every lexer for HTML and a template language should include
+    #: ``\*.html`` in this list.
     alias_filenames = []
 
-    #: MIME types
+    #: A list of MIME types for content that can be lexed with this lexer.
     mimetypes = []
 
     #: Priority, should multiple lexers match and no content is provided
     priority = 0
 
+    #: URL of the language specification/definition. Used in the Pygments
+    #: documentation.
+    url = None
+
     def __init__(self, **options):
+        """
+        This constructor takes arbitrary options as keyword arguments.
+        Every subclass must first process its own options and then call
+        the `Lexer` constructor, since it processes the basic
+        options like `stripnl`.
+
+        An example looks like this:
+
+        .. sourcecode:: python
+
+           def __init__(self, **options):
+               self.compress = options.get('compress', '')
+               Lexer.__init__(self, **options)
+
+        As these options must all be specifiable as strings (due to the
+        command line usage), there are various utility functions
+        available to help with that, see `Utilities`_.
+        """
         self.options = options
         self.stripnl = get_bool_opt(options, 'stripnl', True)
         self.stripall = get_bool_opt(options, 'stripall', False)
@@ -120,10 +175,13 @@ class Lexer(metaclass=LexerMeta):
 
     def analyse_text(text):
         """
-        Has to return a float between ``0`` and ``1`` that indicates
-        if a lexer wants to highlight this text. Used by ``guess_lexer``.
-        If this method returns ``0`` it won't highlight it in any case, if
-        it returns ``1`` highlighting with this lexer is guaranteed.
+        A static method which is called for lexer guessing.
+
+        It should analyse the text and return a float in the range
+        from ``0.0`` to ``1.0``.  If it returns ``0.0``, the lexer
+        will not be selected as the most probable one, if it returns
+        ``1.0``, it will be selected immediately.  This is used by
+        `guess_lexer`.
 
         The `LexerMeta` metaclass automatically wraps this function so
         that it works like a static method (no ``self`` or ``cls``
@@ -134,12 +192,17 @@ class Lexer(metaclass=LexerMeta):
 
     def get_tokens(self, text, unfiltered=False):
         """
-        Return an iterable of (tokentype, value) pairs generated from
-        `text`. If `unfiltered` is set to `True`, the filtering mechanism
-        is bypassed even if filters are defined.
+        This method is the basic interface of a lexer. It is called by
+        the `highlight()` function. It must process the text and return an
+        iterable of ``(tokentype, value)`` pairs from `text`.
 
-        Also preprocess the text, i.e. expand tabs and strip it if
-        wanted and applies registered filters.
+        Normally, you don't need to override this method. The default
+        implementation processes the options recognized by all lexers
+        (`stripnl`, `stripall` and so on), and then yields all tokens
+        from `get_tokens_unprocessed()`, with the ``index`` dropped.
+
+        If `unfiltered` is set to `True`, the filtering mechanism is
+        bypassed even if filters are defined.
         """
         if not isinstance(text, str):
             if self.encoding == 'guess':
@@ -193,11 +256,12 @@ class Lexer(metaclass=LexerMeta):
 
     def get_tokens_unprocessed(self, text):
         """
-        Return an iterable of (index, tokentype, value) pairs where "index"
-        is the starting position of the token within the input text.
+        This method should process the text and return an iterable of
+        ``(index, tokentype, value)`` tuples where ``index`` is the starting
+        position of the token within the input text.
 
-        In subclasses, implement this method as a generator to
-        maximize effectiveness.
+        It must be overridden by subclasses. It is recommended to
+        implement it as a generator to maximize effectiveness.
         """
         raise NotImplementedError
 
@@ -590,19 +654,24 @@ class RegexLexer(Lexer, metaclass=RegexLexerMeta):
     #: Defaults to MULTILINE.
     flags = re.MULTILINE
 
+    #: At all time there is a stack of states. Initially, the stack contains
+    #: a single state 'root'. The top of the stack is called "the current state".
+    #:
     #: Dict of ``{'state': [(regex, tokentype, new_state), ...], ...}``
     #:
-    #: The initial state is 'root'.
     #: ``new_state`` can be omitted to signify no state transition.
-    #: If it is a string, the state is pushed on the stack and changed.
-    #: If it is a tuple of strings, all states are pushed on the stack and
-    #: the current state will be the topmost.
-    #: It can also be ``combined('state1', 'state2', ...)``
+    #: If ``new_state`` is a string, it is pushed on the stack. This ensure
+    #: the new current state is ``new_state``.
+    #: If ``new_state`` is a tuple of strings, all of those strings are pushed
+    #: on the stack and the current state will be the last element of the list.
+    #: ``new_state`` can also be ``combined('state1', 'state2', ...)``
     #: to signify a new, anonymous state combined from the rules of two
     #: or more existing ones.
     #: Furthermore, it can be '#pop' to signify going back one step in
     #: the state stack, or '#push' to push the current state on the stack
-    #: again.
+    #: again. Note that if you push while in a combined state, the combined
+    #: state itself is pushed, and not only the state in which the rule is
+    #: defined.
     #:
     #: The tuple can also be replaced with ``include('state')``, in which
     #: case the rules from the state named by the string are included in the
@@ -613,7 +682,7 @@ class RegexLexer(Lexer, metaclass=RegexLexerMeta):
         """
         Split ``text`` into (tokentype, text) pairs.
 
-        ``stack`` is the inital stack (default: ``['root']``)
+        ``stack`` is the initial stack (default: ``['root']``)
         """
         pos = 0
         tokendefs = self._tokens
@@ -662,7 +731,7 @@ class RegexLexer(Lexer, metaclass=RegexLexerMeta):
                         # at EOL, reset state to "root"
                         statestack = ['root']
                         statetokens = tokendefs['root']
-                        yield pos, Text, '\n'
+                        yield pos, Whitespace, '\n'
                         pos += 1
                         continue
                     yield pos, Error, text[pos]
@@ -733,7 +802,7 @@ class ExtendedRegexLexer(RegexLexer):
                         elif isinstance(new_state, int):
                             # see RegexLexer for why this check is made
                             if abs(new_state) >= len(ctx.stack):
-                                del ctx.state[1:]
+                                del ctx.stack[1:]
                             else:
                                 del ctx.stack[new_state:]
                         elif new_state == '#push':
@@ -787,7 +856,7 @@ def do_insertions(insertions, tokens):
     # iterate over the token stream where we want to insert
     # the tokens from the insertion list.
     for i, t, v in tokens:
-        # first iteration. store the postition of first item
+        # first iteration. store the position of first item
         if realpos is None:
             realpos = i
         oldi = 0

data/vendor/pygments-main/pygments/lexers/__init__.py

@@ -4,7 +4,7 @@
 
     Pygments lexers.
 
-    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -47,20 +47,24 @@ def _load_lexers(module_name):
         _lexer_cache[cls.name] = cls
 
 
-def get_all_lexers():
+def get_all_lexers(plugins=True):
     """Return a generator of tuples in the form ``(name, aliases,
     filenames, mimetypes)`` of all know lexers.
+
+    If *plugins* is true (the default), plugin lexers supplied by entrypoints
+    are also returned.  Otherwise, only builtin ones are considered.
     """
     for item in LEXERS.values():
         yield item[1:]
-    for lexer in find_plugin_lexers():
-        yield lexer.name, lexer.aliases, lexer.filenames, lexer.mimetypes
+    if plugins:
+        for lexer in find_plugin_lexers():
+            yield lexer.name, lexer.aliases, lexer.filenames, lexer.mimetypes
 
 
 def find_lexer_class(name):
-    """Lookup a lexer class by name.
-
-    Return None if not found.
+    """
+    Return the `Lexer` subclass that with the *name* attribute as given by
+    the *name* argument.
     """
     if name in _lexer_cache:
         return _lexer_cache[name]
@@ -76,10 +80,15 @@ def find_lexer_class(name):
 
 
 def find_lexer_class_by_name(_alias):
-    """Lookup a lexer class by alias.
+    """
+    Return the `Lexer` subclass that has `alias` in its aliases list, without
+    instantiating it.
 
     Like `get_lexer_by_name`, but does not instantiate the class.
 
+    Will raise :exc:`pygments.util.ClassNotFound` if no lexer with that alias is
+    found.
+
     .. versionadded:: 2.2
     """
     if not _alias:
@@ -98,9 +107,13 @@ def find_lexer_class_by_name(_alias):
 
 
 def get_lexer_by_name(_alias, **options):
-    """Get a lexer by an alias.
+    """
+    Return an instance of a `Lexer` subclass that has `alias` in its
+    aliases list. The lexer is given the `options` at its
+    instantiation.
 
-    Raises ClassNotFound if not found.
+    Will raise :exc:`pygments.util.ClassNotFound` if no lexer with that alias is
+    found.
     """
     if not _alias:
         raise ClassNotFound('no lexer for alias %r found' % _alias)
@@ -199,10 +212,15 @@ def find_lexer_class_for_filename(_fn, code=None):
 def get_lexer_for_filename(_fn, code=None, **options):
     """Get a lexer for a filename.
 
-    If multiple lexers match the filename pattern, use ``analyse_text()`` to
-    figure out which one is more appropriate.
+    Return a `Lexer` subclass instance that has a filename pattern
+    matching `fn`. The lexer is given the `options` at its
+    instantiation.
+
+    Raise :exc:`pygments.util.ClassNotFound` if no lexer for that filename
+    is found.
 
-    Raises ClassNotFound if not found.
+    If multiple lexers match the filename pattern, use their ``analyse_text()``
+    methods to figure out which one is more appropriate.
     """
     res = find_lexer_class_for_filename(_fn, code)
     if not res:
@@ -211,9 +229,12 @@ def get_lexer_for_filename(_fn, code=None, **options):
 
 
 def get_lexer_for_mimetype(_mime, **options):
-    """Get a lexer for a mimetype.
+    """
+    Return a `Lexer` subclass instance that has `mime` in its mimetype
+    list. The lexer is given the `options` at its instantiation.
 
-    Raises ClassNotFound if not found.
+    Will raise :exc:`pygments.util.ClassNotFound` if not lexer for that mimetype
+    is found.
     """
     for modname, name, _, _, mimetypes in LEXERS.values():
         if _mime in mimetypes:
@@ -239,19 +260,11 @@ def _iter_lexerclasses(plugins=True):
 
 def guess_lexer_for_filename(_fn, _text, **options):
     """
-    Lookup all lexers that handle those filenames primary (``filenames``)
-    or secondary (``alias_filenames``). Then run a text analysis for those
-    lexers and choose the best result.
-
-    usage::
-
-        >>> from pygments.lexers import guess_lexer_for_filename
-        >>> guess_lexer_for_filename('hello.html', '<%= @foo %>')
-        <pygments.lexers.templates.RhtmlLexer object at 0xb7d2f32c>
-        >>> guess_lexer_for_filename('hello.html', '<h1>{{ title|e }}</h1>')
-        <pygments.lexers.templates.HtmlDjangoLexer object at 0xb7d2f2ac>
-        >>> guess_lexer_for_filename('style.css', 'a { color: <?= $link ?> }')
-        <pygments.lexers.templates.CssPhpLexer object at 0xb7ba518c>
+    As :func:`guess_lexer()`, but only lexers which have a pattern in `filenames`
+    or `alias_filenames` that matches `filename` are taken into consideration.
+
+    :exc:`pygments.util.ClassNotFound` is raised if no lexer thinks it can
+    handle the content.
     """
     fn = basename(_fn)
     primary = {}
@@ -289,7 +302,15 @@ def guess_lexer_for_filename(_fn, _text, **options):
 
 
 def guess_lexer(_text, **options):
-    """Guess a lexer by strong distinctions in the text (eg, shebang)."""
+    """
+    Return a `Lexer` subclass instance that's guessed from the text in
+    `text`. For that, the :meth:`.analyse_text()` method of every known lexer
+    class is called with the text as argument, and the lexer which returned the
+    highest value will be instantiated and returned.
+
+    :exc:`pygments.util.ClassNotFound` is raised if no lexer thinks it can
+    handle the content.
+    """
 
     if not isinstance(_text, str):
         inencoding = options.get('inencoding', options.get('encoding'))

data/vendor/pygments-main/pygments/lexers/_ada_builtins.py

@@ -0,0 +1,103 @@
+"""
+    pygments.lexers._ada_builtins
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Ada builtins.
+
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+KEYWORD_LIST = (
+    'abort',
+    'abs',
+    'abstract',
+    'accept',
+    'access',
+    'aliased',
+    'all',
+    'array',
+    'at',
+    'begin',
+    'body',
+    'case',
+    'constant',
+    'declare',
+    'delay',
+    'delta',
+    'digits',
+    'do',
+    'else',
+    'elsif',
+    'end',
+    'entry',
+    'exception',
+    'exit',
+    'interface',
+    'for',
+    'goto',
+    'if',
+    'is',
+    'limited',
+    'loop',
+    'new',
+    'null',
+    'of',
+    'or',
+    'others',
+    'out',
+    'overriding',
+    'pragma',
+    'protected',
+    'raise',
+    'range',
+    'record',
+    'renames',
+    'requeue',
+    'return',
+    'reverse',
+    'select',
+    'separate',
+    'some',
+    'subtype',
+    'synchronized',
+    'task',
+    'tagged',
+    'terminate',
+    'then',
+    'type',
+    'until',
+    'when',
+    'while',
+    'xor'
+)
+
+BUILTIN_LIST = (
+    'Address',
+    'Byte',
+    'Boolean',
+    'Character',
+    'Controlled',
+    'Count',
+    'Cursor',
+    'Duration',
+    'File_Mode',
+    'File_Type',
+    'Float',
+    'Generator',
+    'Integer',
+    'Long_Float',
+    'Long_Integer',
+    'Long_Long_Float',
+    'Long_Long_Integer',
+    'Natural',
+    'Positive',
+    'Reference_Type',
+    'Short_Float',
+    'Short_Integer',
+    'Short_Short_Float',
+    'Short_Short_Integer',
+    'String',
+    'Wide_Character',
+    'Wide_String'
+)

data/vendor/pygments-main/pygments/lexers/_asy_builtins.py

@@ -9,7 +9,7 @@
     TODO: perl/python script in Asymptote SVN similar to asy-list.pl but only
     for function and variable names.
 
-    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexers/_cl_builtins.py

@@ -4,7 +4,7 @@
 
     ANSI Common Lisp builtins.
 
-    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexers/_cocoa_builtins.py

@@ -5,9 +5,9 @@
     This file defines a set of types used across Cocoa frameworks from Apple.
     There is a list of @interfaces, @protocols and some other (structs, unions)
 
-    File may be also used as standalone generator for aboves.
+    File may be also used as standalone generator for above.
 
-    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexers/_csound_builtins.py

@@ -2,7 +2,7 @@
     pygments.lexers._csound_builtins
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -34,7 +34,7 @@ tabrowlin
 vbap1move
 '''.split())
 
-# Opcodes in Csound 6.16.0 using:
+# Opcodes in Csound 6.18.0 using:
 #   python3 -c "
 #   import re
 #   from subprocess import Popen, PIPE
@@ -256,6 +256,7 @@ bbcuts
 betarand
 bexprnd
 bformdec1
+bformdec2
 bformenc1
 binit
 biquad
@@ -430,6 +431,8 @@ dumpk4
 duserrnd
 dust
 dust2
+elapsedcycles
+elapsedtime
 envlpx
 envlpxr
 ephasor
@@ -437,6 +440,8 @@ eqfil
 evalstr
 event
 event_i
+eventcycles
+eventtime
 exciter
 exitnow
 exp
@@ -565,6 +570,7 @@ grain
 grain2
 grain3
 granule
+gtadsr
 gtf
 guiro
 harmon
@@ -896,6 +902,7 @@ median
 mediank
 metro
 metro2
+metrobpm
 mfb
 midglobal
 midiarp
@@ -944,6 +951,7 @@ mp3bitrate
 mp3in
 mp3len
 mp3nchnls
+mp3out
 mp3scal
 mp3sr
 mpulse
@@ -1147,6 +1155,7 @@ pvsftr
 pvsftw
 pvsfwrite
 pvsgain
+pvsgendy
 pvshift
 pvsifd
 pvsin
@@ -1313,7 +1322,9 @@ scale
 scale2
 scalearray
 scanhammer
+scanmap
 scans
+scansmap
 scantable
 scanu
 scanu2
@@ -1332,6 +1343,8 @@ sense
 sensekey
 seqtime
 seqtime2
+sequ
+sequstate
 serialBegin
 serialEnd
 serialFlush
@@ -1524,8 +1537,10 @@ trfilter
 trhighest
 trigExpseg
 trigLinseg
+trigexpseg
 trigger
 trighold
+triglinseg
 trigphasor
 trigseq
 trim
@@ -1656,10 +1671,6 @@ wterrain2
 xadsr
 xin
 xout
-xscanmap
-xscans
-xscansmap
-xscanu
 xtratim
 xyscale
 zacl
@@ -1761,5 +1772,9 @@ vbap4
 vbap4move
 vbap8
 vbap8move
+xscanmap
+xscans
+xscansmap
+xscanu
 xyin
 '''.split())