pygments.rb 1.2.1 → 2.0.0

data/vendor/pygments-main/pygments/lexers/matlab.py

@@ -5,13 +5,14 @@
 
     Lexers for Matlab and related languages.
 
-    :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import re
 
-from pygments.lexer import Lexer, RegexLexer, bygroups, words, do_insertions
+from pygments.lexer import Lexer, RegexLexer, bygroups, default, words, \
+    do_insertions
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
     Number, Punctuation, Generic, Whitespace
 
@@ -45,33 +46,35 @@ class MatlabLexer(RegexLexer):
     # specfun: Special Math functions
     # elmat: Elementary matrices and matrix manipulation
     #
-    # taken from Matlab version 7.4.0.336 (R2007a)
+    # taken from Matlab version 9.4 (R2018a)
     #
     elfun = ("sin", "sind", "sinh", "asin", "asind", "asinh", "cos", "cosd", "cosh",
              "acos", "acosd", "acosh", "tan", "tand", "tanh", "atan", "atand", "atan2",
-             "atanh", "sec", "secd", "sech", "asec", "asecd", "asech", "csc", "cscd",
+             "atan2d", "atanh", "sec", "secd", "sech", "asec", "asecd", "asech", "csc", "cscd",
              "csch", "acsc", "acscd", "acsch", "cot", "cotd", "coth", "acot", "acotd",
-             "acoth", "hypot", "exp", "expm1", "log", "log1p", "log10", "log2", "pow2",
+             "acoth", "hypot", "deg2rad", "rad2deg", "exp", "expm1", "log", "log1p", "log10", "log2", "pow2",
              "realpow", "reallog", "realsqrt", "sqrt", "nthroot", "nextpow2", "abs",
              "angle", "complex", "conj", "imag", "real", "unwrap", "isreal", "cplxpair",
              "fix", "floor", "ceil", "round", "mod", "rem", "sign")
     specfun = ("airy", "besselj", "bessely", "besselh", "besseli", "besselk", "beta",
-               "betainc", "betaln", "ellipj", "ellipke", "erf", "erfc", "erfcx",
-               "erfinv", "expint", "gamma", "gammainc", "gammaln", "psi", "legendre",
+               "betainc", "betaincinv", "betaln", "ellipj", "ellipke", "erf", "erfc", "erfcx",
+               "erfinv", "erfcinv", "expint", "gamma", "gammainc", "gammaincinv", "gammaln", "psi", "legendre",
                "cross", "dot", "factor", "isprime", "primes", "gcd", "lcm", "rat",
                "rats", "perms", "nchoosek", "factorial", "cart2sph", "cart2pol",
                "pol2cart", "sph2cart", "hsv2rgb", "rgb2hsv")
-    elmat = ("zeros", "ones", "eye", "repmat", "rand", "randn", "linspace", "logspace",
+    elmat = ("zeros", "ones", "eye", "repmat", "repelem", "linspace", "logspace",
              "freqspace", "meshgrid", "accumarray", "size", "length", "ndims", "numel",
-             "disp", "isempty", "isequal", "isequalwithequalnans", "cat", "reshape",
-             "diag", "blkdiag", "tril", "triu", "fliplr", "flipud", "flipdim", "rot90",
+             "disp", "isempty", "isequal", "isequaln", "cat", "reshape",
+             "diag", "blkdiag", "tril", "triu", "fliplr", "flipud", "flip", "rot90",
              "find", "end", "sub2ind", "ind2sub", "bsxfun", "ndgrid", "permute",
              "ipermute", "shiftdim", "circshift", "squeeze", "isscalar", "isvector",
-             "ans", "eps", "realmax", "realmin", "pi", "i", "inf", "nan", "isnan",
-             "isinf", "isfinite", "j", "why", "compan", "gallery", "hadamard", "hankel",
+             "isrow", "iscolumn", "ismatrix", "eps", "realmax", "realmin", "intmax", "intmin", "flintmax", "pi", "i", "inf", "nan", "isnan",
+             "isinf", "isfinite", "j", "true", "false", "compan", "gallery", "hadamard", "hankel",
              "hilb", "invhilb", "magic", "pascal", "rosser", "toeplitz", "vander",
              "wilkinson")
 
+    _operators = r'-|==|~=|<=|>=|<|>|&&|&|~|\|\|?|\.\*|\*|\+|\.\^|\.\\|\./|/|\\'
+
     tokens = {
         'root': [
             # line starting with '!' is sent as a system command.  not sure what
@@ -79,25 +82,40 @@ class MatlabLexer(RegexLexer):
             (r'^!.*', String.Other),
             (r'%\{\s*\n', Comment.Multiline, 'blockcomment'),
             (r'%.*$', Comment),
-            (r'^\s*function', Keyword, 'deffunc'),
-
-            # from 'iskeyword' on version 7.11 (R2010):
-            (words((
-                'break', 'case', 'catch', 'classdef', 'continue', 'else', 'elseif',
-                'end', 'enumerated', 'events', 'for', 'function', 'global', 'if',
-                'methods', 'otherwise', 'parfor', 'persistent', 'properties',
-                'return', 'spmd', 'switch', 'try', 'while'), suffix=r'\b'),
+            (r'^\s*function\b', Keyword, 'deffunc'),
+
+            # from 'iskeyword' on version 9.4 (R2018a):
+            # Check that there is no preceding dot, as keywords are valid field
+            # names.
+            (words(('break', 'case', 'catch', 'classdef', 'continue', 'else',
+                    'elseif', 'end', 'for', 'function',
+                    'global', 'if', 'otherwise', 'parfor',
+                    'persistent', 'return', 'spmd', 'switch',
+                    'try', 'while'),
+                   prefix=r'(?<!\.)', suffix=r'\b'),
              Keyword),
 
             ("(" + "|".join(elfun + specfun + elmat) + r')\b',  Name.Builtin),
 
             # line continuation with following comment:
-            (r'\.\.\..*$', Comment),
+            (r'(\.\.\.)(.*)$', bygroups(Keyword, Comment)),
+
+            # command form:
+            # "How MATLAB Recognizes Command Syntax" specifies that an operator
+            # is recognized if it is either surrounded by spaces or by no
+            # spaces on both sides; only the former case matters for us.  (This
+            # allows distinguishing `cd ./foo` from `cd ./ foo`.)
+            (r'(?:^|(?<=;))(\s*)(\w+)(\s+)(?!=|\(|(?:%s)\s+)' % _operators,
+             bygroups(Text, Name, Text), 'commandargs'),
 
             # operators:
-            (r'-|==|~=|<|>|<=|>=|&&|&|~|\|\|?', Operator),
-            # operators requiring escape for re:
-            (r'\.\*|\*|\+|\.\^|\.\\|\.\/|\/|\\', Operator),
+            (_operators, Operator),
+
+            # numbers (must come before punctuation to handle `.5`; cannot use
+            # `\b` due to e.g. `5. + .5`).
+            (r'(?<!\w)((\d+\.\d*)|(\d*\.\d+))([eEf][+-]?\d+)?(?!\w)', Number.Float),
+            (r'\b\d+[eEf][+-]?[0-9]+\b', Number.Float),
+            (r'\b\d+\b', Number.Integer),
 
             # punctuation:
             (r'\[|\]|\(|\)|\{|\}|:|@|\.|,', Punctuation),
@@ -107,17 +125,12 @@ class MatlabLexer(RegexLexer):
             # (not great, but handles common cases...)
             (r'(?<=[\w)\].])\'+', Operator),
 
-            (r'(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?', Number.Float),
-            (r'\d+[eEf][+-]?[0-9]+', Number.Float),
-            (r'\d+', Number.Integer),
+            (r'"(""|[^"])*"', String),
 
             (r'(?<![\w)\].])\'', String, 'string'),
             (r'[a-zA-Z_]\w*', Name),
             (r'.', Text),
         ],
-        'string': [
-            (r'[^\']*\'', String, '#pop')
-        ],
         'blockcomment': [
             (r'^\s*%\}', Comment.Multiline, '#pop'),
             (r'^.*\n', Comment.Multiline),
@@ -131,12 +144,36 @@ class MatlabLexer(RegexLexer):
             # function with no args
             (r'(\s*)([a-zA-Z_]\w*)', bygroups(Text, Name.Function), '#pop'),
         ],
+        'string': [
+            (r"[^']*'", String, '#pop'),
+        ],
+        'commandargs': [
+            # If an equal sign or other operator is encountered, this
+            # isn't a command. It might be a variable assignment or
+            # comparison operation with multiple spaces before the
+            # equal sign or operator
+            (r"=", Punctuation, '#pop'),
+            (_operators, Operator, '#pop'),
+            (r"[ \t]+", Text),
+            ("'[^']*'", String),
+            (r"[^';\s]+", String),
+            (";", Punctuation, '#pop'),
+            default('#pop'),
+        ]
     }
 
     def analyse_text(text):
-        if re.match('^\s*%', text, re.M):  # comment
+        # function declaration.
+        first_non_comment = next((line for line in text.splitlines()
+                                  if not re.match(r'^\s*%', text)), '').strip()
+        if (first_non_comment.startswith('function')
+                and '{' not in first_non_comment):
+            return 1.
+        # comment
+        elif re.search(r'^\s*%', text, re.M):
             return 0.2
-        elif re.match('^!\w+', text, re.M):  # system cmd
+        # system cmd
+        elif re.search(r'^!\w+', text, re.M):
             return 0.2
 
 
@@ -158,6 +195,7 @@ class MatlabSessionLexer(Lexer):
 
         curcode = ''
         insertions = []
+        continuation = False
 
         for match in line_re.finditer(text):
             line = match.group()
@@ -180,21 +218,36 @@ class MatlabSessionLexer(Lexer):
                 # line = "\n" + line
                 token = (0, Generic.Traceback, line)
                 insertions.append((idx, [token]))
-
+            elif continuation:
+                # line_start is the length of the most recent prompt symbol
+                line_start = len(insertions[-1][-1][-1])
+                # Set leading spaces with the length of the prompt to be a generic prompt
+                # This keeps code aligned when prompts are removed, say with some Javascript
+                if line.startswith(' '*line_start):
+                    insertions.append((len(curcode),
+                                    [(0, Generic.Prompt, line[:line_start])]))
+                    curcode += line[line_start:]
+                else:
+                    curcode += line
             else:
                 if curcode:
-                    for item in do_insertions(
-                            insertions, mlexer.get_tokens_unprocessed(curcode)):
-                        yield item
+                    yield from do_insertions(
+                        insertions, mlexer.get_tokens_unprocessed(curcode))
                     curcode = ''
                     insertions = []
 
                 yield match.start(), Generic.Output, line
 
+            # Does not allow continuation if a comment is included after the ellipses.
+            # Continues any line that ends with ..., even comments (lines that start with %)
+            if line.strip().endswith('...'):
+                continuation = True
+            else:
+                continuation = False
+
         if curcode:  # or item:
-            for item in do_insertions(
-                    insertions, mlexer.get_tokens_unprocessed(curcode)):
-                yield item
+            yield from do_insertions(
+                insertions, mlexer.get_tokens_unprocessed(curcode))
 
 
 class OctaveLexer(RegexLexer):
@@ -536,7 +589,7 @@ class OctaveLexer(RegexLexer):
         'root': [
             # We should look into multiline comments
             (r'[%#].*$', Comment),
-            (r'^\s*function', Keyword, 'deffunc'),
+            (r'^\s*function\b', Keyword, 'deffunc'),
 
             # from 'iskeyword' on hg changeset 8cc154f45e37
             (words((
@@ -594,6 +647,10 @@ class OctaveLexer(RegexLexer):
         ],
     }
 
+    def analyse_text(text):
+        """Octave is quite hard to spot, and it looks like Matlab as well."""
+        return 0
+
 
 class ScilabLexer(RegexLexer):
     """
@@ -609,7 +666,7 @@ class ScilabLexer(RegexLexer):
     tokens = {
         'root': [
             (r'//.*?$', Comment.Single),
-            (r'^\s*function', Keyword, 'deffunc'),
+            (r'^\s*function\b', Keyword, 'deffunc'),
 
             (words((
                 '__FILE__', '__LINE__', 'break', 'case', 'catch', 'classdef', 'continue', 'do', 'else',

data/vendor/pygments-main/pygments/lexers/mime.py

@@ -0,0 +1,226 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.mime
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Multipurpose Internet Mail Extensions (MIME) data.
+
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include
+from pygments.lexers import get_lexer_for_mimetype
+from pygments.token import Text, Name, String, Operator, Comment, Other
+from pygments.util import get_int_opt, ClassNotFound
+
+__all__ = ["MIMELexer"]
+
+
+class MIMELexer(RegexLexer):
+    """
+    Lexer for Multipurpose Internet Mail Extensions (MIME) data. This lexer is
+    designed to process the nested mulitpart data.
+
+    It assumes that the given data contains both header and body (and is
+    splitted by empty line). If no valid header is found, then the entire data
+    would be treated as body.
+
+    Additional options accepted:
+
+    `MIME-max-level`
+        Max recurssion level for nested MIME structure. Any negative number
+        would treated as unlimited. (default: -1)
+
+    `Content-Type`
+        Treat the data as specific content type. Useful when header is
+        missing, or this lexer would try to parse from header. (default:
+        `text/plain`)
+
+    `Multipart-Boundary`
+        Set the default multipart boundary delimiter. This option is only used
+        when `Content-Type` is `multipart` and header is missing. This lexer
+        would try to parse from header by default. (default: None)
+
+    `Content-Transfer-Encoding`
+        Treat the data as specific encoding. Or this lexer would try to parse
+        from header by default. (default: None)
+
+    .. versionadded:: 2.5
+    """
+
+    name = "MIME"
+    aliases = ["mime"]
+    mimetypes = ["multipart/mixed",
+                 "multipart/related",
+                 "multipart/alternative"]
+
+    def __init__(self, **options):
+        super().__init__(**options)
+        self.boundary = options.get("Multipart-Boundary")
+        self.content_transfer_encoding = options.get("Content_Transfer_Encoding")
+        self.content_type = options.get("Content_Type", "text/plain")
+        self.max_nested_level = get_int_opt(options, "MIME-max-level", -1)
+
+    def analyse_text(text):
+        try:
+            header, body = text.strip().split("\n\n", 1)
+            if not body.strip():
+                return 0.1
+
+            invalid_headers = MIMELexer.tokens["header"].sub("", header)
+            if invalid_headers.strip():
+                return 0.1
+            else:
+                return 1
+
+        except ValueError:
+            return 0.1
+
+    def get_header_tokens(self, match):
+        field = match.group(1)
+
+        if field.lower() in self.attention_headers:
+            yield match.start(1), Name.Tag, field + ":"
+            yield match.start(2), Text.Whitespace, match.group(2)
+
+            pos = match.end(2)
+            body = match.group(3)
+            for i, t, v in self.get_tokens_unprocessed(body, ("root", field.lower())):
+                yield pos + i, t, v
+
+        else:
+            yield match.start(), Comment, match.group()
+
+    def get_body_tokens(self, match):
+        pos_body_start = match.start()
+        entire_body = match.group()
+
+        # skip first newline
+        if entire_body[0] == '\n':
+            yield pos_body_start, Text.Whitespace, '\n'
+            pos_body_start = pos_body_start + 1
+            entire_body = entire_body[1:]
+
+        # if it is not a mulitpart
+        if not self.content_type.startswith("multipart") or not self.boundary:
+            for i, t, v in self.get_bodypart_tokens(entire_body):
+                yield pos_body_start + i, t, v
+            return
+
+        # find boundary
+        bdry_pattern = r"^--%s(--)?\n" % re.escape(self.boundary)
+        bdry_matcher = re.compile(bdry_pattern, re.MULTILINE)
+
+        # some data has prefix text before first boundary
+        m = bdry_matcher.search(entire_body)
+        if m:
+            pos_part_start = pos_body_start + m.end()
+            pos_iter_start = lpos_end = m.end()
+            yield pos_body_start, Text, entire_body[:m.start()]
+            yield pos_body_start + lpos_end, String.Delimiter, m.group()
+        else:
+            pos_part_start = pos_body_start
+            pos_iter_start = 0
+
+        # process tokens of each body part
+        for m in bdry_matcher.finditer(entire_body, pos_iter_start):
+            # bodypart
+            lpos_start = pos_part_start - pos_body_start
+            lpos_end = m.start()
+            part = entire_body[lpos_start:lpos_end]
+            for i, t, v in self.get_bodypart_tokens(part):
+                yield pos_part_start + i, t, v
+
+            # boundary
+            yield pos_body_start + lpos_end, String.Delimiter, m.group()
+            pos_part_start = pos_body_start + m.end()
+
+        # some data has suffix text after last boundary
+        lpos_start = pos_part_start - pos_body_start
+        if lpos_start != len(entire_body):
+            yield pos_part_start, Text, entire_body[lpos_start:]
+
+    def get_bodypart_tokens(self, text):
+        # return if:
+        #  * no content
+        #  * no content type specific
+        #  * content encoding is not readable
+        #  * max recurrsion exceed
+        if not text.strip() or not self.content_type:
+            return [(0, Other, text)]
+
+        cte = self.content_transfer_encoding
+        if cte and cte not in {"8bit", "7bit", "quoted-printable"}:
+            return [(0, Other, text)]
+
+        if self.max_nested_level == 0:
+            return [(0, Other, text)]
+
+        # get lexer
+        try:
+            lexer = get_lexer_for_mimetype(self.content_type)
+        except ClassNotFound:
+            return [(0, Other, text)]
+
+        if isinstance(lexer, type(self)):
+            lexer.max_nested_level = self.max_nested_level - 1
+
+        return lexer.get_tokens_unprocessed(text)
+
+    def store_content_type(self, match):
+        self.content_type = match.group(1)
+
+        prefix_len = match.start(1) - match.start(0)
+        yield match.start(0), Text.Whitespace, match.group(0)[:prefix_len]
+        yield match.start(1), Name.Label, match.group(2)
+        yield match.end(2), String.Delimiter, '/'
+        yield match.start(3), Name.Label, match.group(3)
+
+    def get_content_type_subtokens(self, match):
+        yield match.start(1), Text, match.group(1)
+        yield match.start(2), Text.Whitespace, match.group(2)
+        yield match.start(3), Name.Attribute, match.group(3)
+        yield match.start(4), Operator, match.group(4)
+        yield match.start(5), String, match.group(5)
+
+        if match.group(3).lower() == "boundary":
+            boundary = match.group(5).strip()
+            if boundary[0] == '"' and boundary[-1] == '"':
+                boundary = boundary[1:-1]
+            self.boundary = boundary
+
+    def store_content_transfer_encoding(self, match):
+        self.content_transfer_encoding = match.group(0).lower()
+        yield match.start(0), Name.Constant, match.group(0)
+
+    attention_headers = {"content-type", "content-transfer-encoding"}
+
+    tokens = {
+        "root": [
+            (r"^([\w-]+):( *)([\s\S]*?\n)(?![ \t])", get_header_tokens),
+            (r"^$[\s\S]+", get_body_tokens),
+        ],
+        "header": [
+            # folding
+            (r"\n[ \t]", Text.Whitespace),
+            (r"\n(?![ \t])", Text.Whitespace, "#pop"),
+        ],
+        "content-type": [
+            include("header"),
+            (
+                r"^\s*((multipart|application|audio|font|image|model|text|video"
+                r"|message)/([\w-]+))",
+                store_content_type,
+            ),
+            (r'(;)((?:[ \t]|\n[ \t])*)([\w:-]+)(=)([\s\S]*?)(?=;|\n(?![ \t]))',
+             get_content_type_subtokens),
+            (r';[ \t]*\n(?![ \t])', Text, '#pop'),
+        ],
+        "content-transfer-encoding": [
+            include("header"),
+            (r"([\w-]+)", store_content_transfer_encoding),
+        ],
+    }