pygments.rb 1.1.2 → 2.0.0

data/vendor/pygments-main/pygments/lexers/mime.py

@@ -0,0 +1,226 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.mime
+    ~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Multipurpose Internet Mail Extensions (MIME) data.
+
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include
+from pygments.lexers import get_lexer_for_mimetype
+from pygments.token import Text, Name, String, Operator, Comment, Other
+from pygments.util import get_int_opt, ClassNotFound
+
+__all__ = ["MIMELexer"]
+
+
+class MIMELexer(RegexLexer):
+    """
+    Lexer for Multipurpose Internet Mail Extensions (MIME) data. This lexer is
+    designed to process the nested mulitpart data.
+
+    It assumes that the given data contains both header and body (and is
+    splitted by empty line). If no valid header is found, then the entire data
+    would be treated as body.
+
+    Additional options accepted:
+
+    `MIME-max-level`
+        Max recurssion level for nested MIME structure. Any negative number
+        would treated as unlimited. (default: -1)
+
+    `Content-Type`
+        Treat the data as specific content type. Useful when header is
+        missing, or this lexer would try to parse from header. (default:
+        `text/plain`)
+
+    `Multipart-Boundary`
+        Set the default multipart boundary delimiter. This option is only used
+        when `Content-Type` is `multipart` and header is missing. This lexer
+        would try to parse from header by default. (default: None)
+
+    `Content-Transfer-Encoding`
+        Treat the data as specific encoding. Or this lexer would try to parse
+        from header by default. (default: None)
+
+    .. versionadded:: 2.5
+    """
+
+    name = "MIME"
+    aliases = ["mime"]
+    mimetypes = ["multipart/mixed",
+                 "multipart/related",
+                 "multipart/alternative"]
+
+    def __init__(self, **options):
+        super().__init__(**options)
+        self.boundary = options.get("Multipart-Boundary")
+        self.content_transfer_encoding = options.get("Content_Transfer_Encoding")
+        self.content_type = options.get("Content_Type", "text/plain")
+        self.max_nested_level = get_int_opt(options, "MIME-max-level", -1)
+
+    def analyse_text(text):
+        try:
+            header, body = text.strip().split("\n\n", 1)
+            if not body.strip():
+                return 0.1
+
+            invalid_headers = MIMELexer.tokens["header"].sub("", header)
+            if invalid_headers.strip():
+                return 0.1
+            else:
+                return 1
+
+        except ValueError:
+            return 0.1
+
+    def get_header_tokens(self, match):
+        field = match.group(1)
+
+        if field.lower() in self.attention_headers:
+            yield match.start(1), Name.Tag, field + ":"
+            yield match.start(2), Text.Whitespace, match.group(2)
+
+            pos = match.end(2)
+            body = match.group(3)
+            for i, t, v in self.get_tokens_unprocessed(body, ("root", field.lower())):
+                yield pos + i, t, v
+
+        else:
+            yield match.start(), Comment, match.group()
+
+    def get_body_tokens(self, match):
+        pos_body_start = match.start()
+        entire_body = match.group()
+
+        # skip first newline
+        if entire_body[0] == '\n':
+            yield pos_body_start, Text.Whitespace, '\n'
+            pos_body_start = pos_body_start + 1
+            entire_body = entire_body[1:]
+
+        # if it is not a mulitpart
+        if not self.content_type.startswith("multipart") or not self.boundary:
+            for i, t, v in self.get_bodypart_tokens(entire_body):
+                yield pos_body_start + i, t, v
+            return
+
+        # find boundary
+        bdry_pattern = r"^--%s(--)?\n" % re.escape(self.boundary)
+        bdry_matcher = re.compile(bdry_pattern, re.MULTILINE)
+
+        # some data has prefix text before first boundary
+        m = bdry_matcher.search(entire_body)
+        if m:
+            pos_part_start = pos_body_start + m.end()
+            pos_iter_start = lpos_end = m.end()
+            yield pos_body_start, Text, entire_body[:m.start()]
+            yield pos_body_start + lpos_end, String.Delimiter, m.group()
+        else:
+            pos_part_start = pos_body_start
+            pos_iter_start = 0
+
+        # process tokens of each body part
+        for m in bdry_matcher.finditer(entire_body, pos_iter_start):
+            # bodypart
+            lpos_start = pos_part_start - pos_body_start
+            lpos_end = m.start()
+            part = entire_body[lpos_start:lpos_end]
+            for i, t, v in self.get_bodypart_tokens(part):
+                yield pos_part_start + i, t, v
+
+            # boundary
+            yield pos_body_start + lpos_end, String.Delimiter, m.group()
+            pos_part_start = pos_body_start + m.end()
+
+        # some data has suffix text after last boundary
+        lpos_start = pos_part_start - pos_body_start
+        if lpos_start != len(entire_body):
+            yield pos_part_start, Text, entire_body[lpos_start:]
+
+    def get_bodypart_tokens(self, text):
+        # return if:
+        #  * no content
+        #  * no content type specific
+        #  * content encoding is not readable
+        #  * max recurrsion exceed
+        if not text.strip() or not self.content_type:
+            return [(0, Other, text)]
+
+        cte = self.content_transfer_encoding
+        if cte and cte not in {"8bit", "7bit", "quoted-printable"}:
+            return [(0, Other, text)]
+
+        if self.max_nested_level == 0:
+            return [(0, Other, text)]
+
+        # get lexer
+        try:
+            lexer = get_lexer_for_mimetype(self.content_type)
+        except ClassNotFound:
+            return [(0, Other, text)]
+
+        if isinstance(lexer, type(self)):
+            lexer.max_nested_level = self.max_nested_level - 1
+
+        return lexer.get_tokens_unprocessed(text)
+
+    def store_content_type(self, match):
+        self.content_type = match.group(1)
+
+        prefix_len = match.start(1) - match.start(0)
+        yield match.start(0), Text.Whitespace, match.group(0)[:prefix_len]
+        yield match.start(1), Name.Label, match.group(2)
+        yield match.end(2), String.Delimiter, '/'
+        yield match.start(3), Name.Label, match.group(3)
+
+    def get_content_type_subtokens(self, match):
+        yield match.start(1), Text, match.group(1)
+        yield match.start(2), Text.Whitespace, match.group(2)
+        yield match.start(3), Name.Attribute, match.group(3)
+        yield match.start(4), Operator, match.group(4)
+        yield match.start(5), String, match.group(5)
+
+        if match.group(3).lower() == "boundary":
+            boundary = match.group(5).strip()
+            if boundary[0] == '"' and boundary[-1] == '"':
+                boundary = boundary[1:-1]
+            self.boundary = boundary
+
+    def store_content_transfer_encoding(self, match):
+        self.content_transfer_encoding = match.group(0).lower()
+        yield match.start(0), Name.Constant, match.group(0)
+
+    attention_headers = {"content-type", "content-transfer-encoding"}
+
+    tokens = {
+        "root": [
+            (r"^([\w-]+):( *)([\s\S]*?\n)(?![ \t])", get_header_tokens),
+            (r"^$[\s\S]+", get_body_tokens),
+        ],
+        "header": [
+            # folding
+            (r"\n[ \t]", Text.Whitespace),
+            (r"\n(?![ \t])", Text.Whitespace, "#pop"),
+        ],
+        "content-type": [
+            include("header"),
+            (
+                r"^\s*((multipart|application|audio|font|image|model|text|video"
+                r"|message)/([\w-]+))",
+                store_content_type,
+            ),
+            (r'(;)((?:[ \t]|\n[ \t])*)([\w:-]+)(=)([\s\S]*?)(?=;|\n(?![ \t]))',
+             get_content_type_subtokens),
+            (r';[ \t]*\n(?![ \t])', Text, '#pop'),
+        ],
+        "content-transfer-encoding": [
+            include("header"),
+            (r"([\w-]+)", store_content_transfer_encoding),
+        ],
+    }

data/vendor/pygments-main/pygments/lexers/ml.py

@@ -5,7 +5,7 @@
 
     Lexers for ML family languages.
 
-    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -15,7 +15,7 @@ from pygments.lexer import RegexLexer, include, bygroups, default, words
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
     Number, Punctuation, Error
 
-__all__ = ['SMLLexer', 'OcamlLexer', 'OpaLexer']
+__all__ = ['SMLLexer', 'OcamlLexer', 'OpaLexer', 'ReasonLexer', 'FStarLexer']
 
 
 class SMLLexer(RegexLexer):
@@ -30,7 +30,7 @@ class SMLLexer(RegexLexer):
     filenames = ['*.sml', '*.sig', '*.fun']
     mimetypes = ['text/x-standardml', 'application/x-standardml']
 
-    alphanumid_reserved = set((
+    alphanumid_reserved = {
         # Core
         'abstype', 'and', 'andalso', 'as', 'case', 'datatype', 'do', 'else',
         'end', 'exception', 'fn', 'fun', 'handle', 'if', 'in', 'infix',
@@ -39,16 +39,16 @@ class SMLLexer(RegexLexer):
         # Modules
         'eqtype', 'functor', 'include', 'sharing', 'sig', 'signature',
         'struct', 'structure', 'where',
-    ))
+    }
 
-    symbolicid_reserved = set((
+    symbolicid_reserved = {
         # Core
-        ':', '\|', '=', '=>', '->', '#',
+        ':', r'\|', '=', '=>', '->', '#',
         # Modules
         ':>',
-    ))
+    }
 
-    nonid_reserved = set(('(', ')', '[', ']', '{', '}', ',', ';', '...', '_'))
+    nonid_reserved = {'(', ')', '[', ']', '{', '}', ',', ';', '...', '_'}
 
     alphanumid_re = r"[a-zA-Z][\w']*"
     symbolicid_re = r"[!%&$#+\-/:<=>?@\\~`^|*]+"
@@ -445,7 +445,6 @@ class OcamlLexer(RegexLexer):
         ],
     }
 
-
 class OpaLexer(RegexLexer):
     """
     Lexer for the Opa language (http://opalang.org).
@@ -767,3 +766,193 @@ class OpaLexer(RegexLexer):
             (r'[^\-]+|-', Comment),
         ],
     }
+
+
+class ReasonLexer(RegexLexer):
+    """
+    For the ReasonML language (https://reasonml.github.io/).
+
+    .. versionadded:: 2.6
+    """
+
+    name = 'ReasonML'
+    aliases = ['reason', "reasonml"]
+    filenames = ['*.re', '*.rei']
+    mimetypes = ['text/x-reasonml']
+
+    keywords = (
+        'as', 'assert', 'begin', 'class', 'constraint', 'do', 'done', 'downto',
+        'else', 'end', 'exception', 'external', 'false', 'for', 'fun', 'esfun',
+        'function', 'functor', 'if', 'in', 'include', 'inherit', 'initializer', 'lazy',
+        'let', 'switch', 'module', 'pub', 'mutable', 'new', 'nonrec', 'object', 'of',
+        'open', 'pri', 'rec', 'sig', 'struct', 'then', 'to', 'true', 'try',
+        'type', 'val', 'virtual', 'when', 'while', 'with',
+    )
+    keyopts = (
+        '!=', '#', '&', '&&', r'\(', r'\)', r'\*', r'\+', ',', '-',
+        r'-\.', '=>', r'\.', r'\.\.', r'\.\.\.', ':', '::', ':=', ':>', ';', ';;', '<',
+        '<-', '=', '>', '>]', r'>\}', r'\?', r'\?\?', r'\[', r'\[<', r'\[>',
+        r'\[\|', ']', '_', '`', r'\{', r'\{<', r'\|', r'\|\|', r'\|]', r'\}', '~'
+    )
+
+    operators = r'[!$%&*+\./:<=>?@^|~-]'
+    word_operators = ('and', 'asr', 'land', 'lor', 'lsl', 'lsr', 'lxor', 'mod', 'or')
+    prefix_syms = r'[!?~]'
+    infix_syms = r'[=<>@^|&+\*/$%-]'
+    primitives = ('unit', 'int', 'float', 'bool', 'string', 'char', 'list', 'array')
+
+    tokens = {
+        'escape-sequence': [
+            (r'\\[\\"\'ntbr]', String.Escape),
+            (r'\\[0-9]{3}', String.Escape),
+            (r'\\x[0-9a-fA-F]{2}', String.Escape),
+        ],
+        'root': [
+            (r'\s+', Text),
+            (r'false|true|\(\)|\[\]', Name.Builtin.Pseudo),
+            (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Namespace, 'dotted'),
+            (r'\b([A-Z][\w\']*)', Name.Class),
+            (r'//.*?\n', Comment.Single),
+            (r'\/\*(?!/)', Comment.Multiline, 'comment'),
+            (r'\b(%s)\b' % '|'.join(keywords), Keyword),
+            (r'(%s)' % '|'.join(keyopts[::-1]), Operator.Word),
+            (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator),
+            (r'\b(%s)\b' % '|'.join(word_operators), Operator.Word),
+            (r'\b(%s)\b' % '|'.join(primitives), Keyword.Type),
+
+            (r"[^\W\d][\w']*", Name),
+
+            (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float),
+            (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
+            (r'0[oO][0-7][0-7_]*', Number.Oct),
+            (r'0[bB][01][01_]*', Number.Bin),
+            (r'\d[\d_]*', Number.Integer),
+
+            (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'",
+             String.Char),
+            (r"'.'", String.Char),
+            (r"'", Keyword),
+
+            (r'"', String.Double, 'string'),
+
+            (r'[~?][a-z][\w\']*:', Name.Variable),
+        ],
+        'comment': [
+            (r'[^/*]+', Comment.Multiline),
+            (r'\/\*', Comment.Multiline, '#push'),
+            (r'\*\/', Comment.Multiline, '#pop'),
+            (r'\*', Comment.Multiline),
+        ],
+        'string': [
+            (r'[^\\"]+', String.Double),
+            include('escape-sequence'),
+            (r'\\\n', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'dotted': [
+            (r'\s+', Text),
+            (r'\.', Punctuation),
+            (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace),
+            (r'[A-Z][\w\']*', Name.Class, '#pop'),
+            (r'[a-z_][\w\']*', Name, '#pop'),
+            default('#pop'),
+        ],
+    }
+
+
+class FStarLexer(RegexLexer):
+    """
+    For the F* language (https://www.fstar-lang.org/).
+    .. versionadded:: 2.7
+    """
+
+    name = 'FStar'
+    aliases = ['fstar']
+    filenames = ['*.fst', '*.fsti']
+    mimetypes = ['text/x-fstar']
+
+    keywords = (
+        'abstract', 'attributes', 'noeq', 'unopteq', 'and'
+        'begin', 'by', 'default', 'effect', 'else', 'end', 'ensures',
+        'exception', 'exists', 'false', 'forall', 'fun', 'function', 'if',
+        'in', 'include', 'inline', 'inline_for_extraction', 'irreducible',
+        'logic', 'match', 'module', 'mutable', 'new', 'new_effect', 'noextract',
+        'of', 'open', 'opaque', 'private', 'range_of', 'reifiable',
+        'reify', 'reflectable', 'requires', 'set_range_of', 'sub_effect',
+        'synth', 'then', 'total', 'true', 'try', 'type', 'unfold', 'unfoldable',
+        'val', 'when', 'with', 'not'
+    )
+    decl_keywords = ('let', 'rec')
+    assume_keywords = ('assume', 'admit', 'assert', 'calc')
+    keyopts = (
+        r'~', r'-', r'/\\', r'\\/', r'<:', r'<@', r'\(\|', r'\|\)', r'#', r'u#',
+        r'&', r'\(', r'\)', r'\(\)', r',', r'~>', r'->', r'<-', r'<--', r'<==>',
+        r'==>', r'\.', r'\?', r'\?\.', r'\.\[', r'\.\(', r'\.\(\|', r'\.\[\|',
+        r'\{:pattern', r':', r'::', r':=', r';', r';;', r'=', r'%\[', r'!\{',
+        r'\[', r'\[@', r'\[\|', r'\|>', r'\]', r'\|\]', r'\{', r'\|', r'\}', r'\$'
+    )
+
+    operators = r'[!$%&*+\./:<=>?@^|~-]'
+    prefix_syms = r'[!?~]'
+    infix_syms = r'[=<>@^|&+\*/$%-]'
+    primitives = ('unit', 'int', 'float', 'bool', 'string', 'char', 'list', 'array')
+
+    tokens = {
+        'escape-sequence': [
+            (r'\\[\\"\'ntbr]', String.Escape),
+            (r'\\[0-9]{3}', String.Escape),
+            (r'\\x[0-9a-fA-F]{2}', String.Escape),
+        ],
+        'root': [
+            (r'\s+', Text),
+            (r'false|true|False|True|\(\)|\[\]', Name.Builtin.Pseudo),
+            (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Namespace, 'dotted'),
+            (r'\b([A-Z][\w\']*)', Name.Class),
+            (r'\(\*(?![)])', Comment, 'comment'),
+            (r'^\/\/.+$', Comment),
+            (r'\b(%s)\b' % '|'.join(keywords), Keyword),
+            (r'\b(%s)\b' % '|'.join(assume_keywords), Name.Exception),
+            (r'\b(%s)\b' % '|'.join(decl_keywords), Keyword.Declaration),
+            (r'(%s)' % '|'.join(keyopts[::-1]), Operator),
+            (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator),
+            (r'\b(%s)\b' % '|'.join(primitives), Keyword.Type),
+
+            (r"[^\W\d][\w']*", Name),
+
+            (r'-?\d[\d_]*(.[\d_]*)?([eE][+\-]?\d[\d_]*)', Number.Float),
+            (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
+            (r'0[oO][0-7][0-7_]*', Number.Oct),
+            (r'0[bB][01][01_]*', Number.Bin),
+            (r'\d[\d_]*', Number.Integer),
+
+            (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'",
+             String.Char),
+            (r"'.'", String.Char),
+            (r"'", Keyword),  # a stray quote is another syntax element
+            (r"\`([\w\'.]+)\`", Operator.Word),  # for infix applications
+            (r"\`", Keyword),  # for quoting
+            (r'"', String.Double, 'string'),
+
+            (r'[~?][a-z][\w\']*:', Name.Variable),
+        ],
+        'comment': [
+            (r'[^(*)]+', Comment),
+            (r'\(\*', Comment, '#push'),
+            (r'\*\)', Comment, '#pop'),
+            (r'[(*)]', Comment),
+        ],
+        'string': [
+            (r'[^\\"]+', String.Double),
+            include('escape-sequence'),
+            (r'\\\n', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'dotted': [
+            (r'\s+', Text),
+            (r'\.', Punctuation),
+            (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace),
+            (r'[A-Z][\w\']*', Name.Class, '#pop'),
+            (r'[a-z_][\w\']*', Name, '#pop'),
+            default('#pop'),
+        ],
+    }