pygments.rb 1.2.1 → 2.0.0.rc2

data/vendor/pygments-main/pygments/lexers/tnt.py

@@ -0,0 +1,263 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.tnt
+    ~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Typographic Number Theory.
+
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import Lexer
+from pygments.token import Text, Comment, Operator, Keyword, Name, Number, \
+     Punctuation, Error
+
+__all__ = ['TNTLexer']
+
+
+class TNTLexer(Lexer):
+    """
+    Lexer for Typographic Number Theory, as described in the book
+    Gödel, Escher, Bach, by Douglas R. Hofstadter,
+    or as summarized here:
+    https://github.com/Kenny2github/language-tnt/blob/master/README.md#summary-of-tnt
+
+    .. versionadded:: 2.7
+    """
+
+    name = 'Typographic Number Theory'
+    aliases = ['tnt']
+    filenames = ['*.tnt']
+
+    cur = []
+
+    LOGIC = set('⊃→]&∧^|∨Vv')
+    OPERATORS = set('+.⋅*')
+    VARIABLES = set('abcde')
+    PRIMES = set("'′")
+    NEGATORS = set('~!')
+    QUANTIFIERS = set('AE∀∃')
+    NUMBERS = set('0123456789')
+    WHITESPACE = set('\t \v\n')
+
+    RULES = re.compile('''(?xi)
+        joining | separation | double-tilde | fantasy\\ rule
+        | carry[- ]over(?:\\ of)?(?:\\ line)?\\ ([0-9]+) | detachment
+        | contrapositive | De\\ Morgan | switcheroo
+        | specification | generalization | interchange
+        | existence | symmetry | transitivity
+        | add\\ S | drop\\ S | induction
+        | axiom\\ ([1-5]) | premise | push | pop
+    ''')
+    LINENOS = re.compile(r'(?:[0-9]+)(?:(?:, ?|,? and )(?:[0-9]+))*')
+    COMMENT = re.compile(r'\[[^\n\]]+\]')
+
+    def __init__(self, *args, **kwargs):
+        Lexer.__init__(self, *args, **kwargs)
+        self.cur = []
+
+    def whitespace(self, start, text, required=False):
+        """Tokenize whitespace."""
+        end = start
+        try:
+            while text[end] in self.WHITESPACE:
+                end += 1
+        except IndexError:
+            end = len(text)
+        if required:
+            assert end != start
+        if end != start:
+            self.cur.append((start, Text, text[start:end]))
+        return end
+
+    def variable(self, start, text):
+        """Tokenize a variable."""
+        assert text[start] in self.VARIABLES
+        end = start+1
+        while text[end] in self.PRIMES:
+            end += 1
+        self.cur.append((start, Name.Variable, text[start:end]))
+        return end
+
+    def term(self, start, text):
+        """Tokenize a term."""
+        if text[start] == 'S':  # S...S(...) or S...0
+            end = start+1
+            while text[end] == 'S':
+                end += 1
+            self.cur.append((start, Number.Integer, text[start:end]))
+            return self.term(end, text)
+        if text[start] == '0':  # the singleton 0
+            self.cur.append((start, Number.Integer, text[start]))
+            return start+1
+        if text[start] in self.VARIABLES:  # a''...
+            return self.variable(start, text)
+        if text[start] == '(':  # (...+...)
+            self.cur.append((start, Punctuation, text[start]))
+            start = self.term(start+1, text)
+            assert text[start] in self.OPERATORS
+            self.cur.append((start, Operator, text[start]))
+            start = self.term(start+1, text)
+            assert text[start] == ')'
+            self.cur.append((start, Punctuation, text[start]))
+            return start+1
+        raise AssertionError  # no matches
+
+    def formula(self, start, text):
+        """Tokenize a formula."""
+        if text[start] in self.NEGATORS:  # ~<...>
+            end = start+1
+            while text[end] in self.NEGATORS:
+                end += 1
+            self.cur.append((start, Operator, text[start:end]))
+            return self.formula(end, text)
+        if text[start] in self.QUANTIFIERS:  # Aa:<...>
+            self.cur.append((start, Keyword.Declaration, text[start]))
+            start = self.variable(start+1, text)
+            assert text[start] == ':'
+            self.cur.append((start, Punctuation, text[start]))
+            return self.formula(start+1, text)
+        if text[start] == '<':  # <...&...>
+            self.cur.append((start, Punctuation, text[start]))
+            start = self.formula(start+1, text)
+            assert text[start] in self.LOGIC
+            self.cur.append((start, Operator, text[start]))
+            start = self.formula(start+1, text)
+            assert text[start] == '>'
+            self.cur.append((start, Punctuation, text[start]))
+            return start+1
+        # ...=...
+        start = self.term(start, text)
+        assert text[start] == '='
+        self.cur.append((start, Operator, text[start]))
+        start = self.term(start+1, text)
+        return start
+
+    def rule(self, start, text):
+        """Tokenize a rule."""
+        match = self.RULES.match(text, start)
+        assert match is not None
+        groups = sorted(match.regs[1:])  # exclude whole match
+        for group in groups:
+            if group[0] >= 0:  # this group matched
+                self.cur.append((start, Keyword, text[start:group[0]]))
+                self.cur.append((group[0], Number.Integer,
+                                 text[group[0]:group[1]]))
+                if group[1] != match.end():
+                    self.cur.append((group[1], Keyword,
+                                     text[group[1]:match.end()]))
+                break
+        else:
+            self.cur.append((start, Keyword, text[start:match.end()]))
+        return match.end()
+
+    def lineno(self, start, text):
+        """Tokenize a line referral."""
+        end = start
+        while text[end] not in self.NUMBERS:
+            end += 1
+        self.cur.append((start, Punctuation, text[start]))
+        self.cur.append((start+1, Text, text[start+1:end]))
+        start = end
+        match = self.LINENOS.match(text, start)
+        assert match is not None
+        assert text[match.end()] == ')'
+        self.cur.append((match.start(), Number.Integer, match.group(0)))
+        self.cur.append((match.end(), Punctuation, text[match.end()]))
+        return match.end() + 1
+
+    def error_till_line_end(self, start, text):
+        """Mark everything from ``start`` to the end of the line as Error."""
+        end = start
+        try:
+            while text[end] != '\n':  # there's whitespace in rules
+                end += 1
+        except IndexError:
+            end = len(text)
+        if end != start:
+            self.cur.append((start, Error, text[start:end]))
+        end = self.whitespace(end, text)
+        return end
+
+    def get_tokens_unprocessed(self, text):
+        """Returns a list of TNT tokens."""
+        self.cur = []
+        start = end = self.whitespace(0, text)
+        while start <= end < len(text):
+            try:
+                # try line number
+                while text[end] in self.NUMBERS:
+                    end += 1
+                if end != start:  # actual number present
+                    self.cur.append((start, Number.Integer, text[start:end]))
+                    # whitespace is required after a line number
+                    orig = len(self.cur)
+                    try:
+                        start = end = self.whitespace(end, text, True)
+                    except AssertionError:
+                        del self.cur[orig:]
+                        start = end = self.error_till_line_end(end, text)
+                        continue
+                # at this point it could be a comment
+                match = self.COMMENT.match(text, start)
+                if match is not None:
+                    self.cur.append((start, Comment, text[start:match.end()]))
+                    start = end = match.end()
+                    # anything after the closing bracket is invalid
+                    start = end = self.error_till_line_end(start, text)
+                    # do not attempt to process the rest
+                    continue
+                del match
+                if text[start] in '[]':  # fantasy push or pop
+                    self.cur.append((start, Keyword, text[start]))
+                    start += 1
+                    end += 1
+                else:
+                    # one formula, possibly containing subformulae
+                    orig = len(self.cur)
+                    try:
+                        start = end = self.formula(start, text)
+                    except AssertionError:  # not well-formed
+                        del self.cur[orig:]
+                        while text[end] not in self.WHITESPACE:
+                            end += 1
+                        self.cur.append((start, Error, text[start:end]))
+                        start = end
+                # skip whitespace after formula
+                orig = len(self.cur)
+                try:
+                    start = end = self.whitespace(end, text, True)
+                except AssertionError:
+                    del self.cur[orig:]
+                    start = end = self.error_till_line_end(start, text)
+                    continue
+                # rule proving this formula a theorem
+                orig = len(self.cur)
+                try:
+                    start = end = self.rule(start, text)
+                except AssertionError:
+                    del self.cur[orig:]
+                    start = end = self.error_till_line_end(start, text)
+                    continue
+                # skip whitespace after rule
+                start = end = self.whitespace(end, text)
+                # line marker
+                if text[start] == '(':
+                    orig = len(self.cur)
+                    try:
+                        start = end = self.lineno(start, text)
+                    except AssertionError:
+                        del self.cur[orig:]
+                        start = end = self.error_till_line_end(start, text)
+                        continue
+                    start = end = self.whitespace(start, text)
+            except IndexError:
+                try:
+                    del self.cur[orig:]
+                except NameError:
+                    pass # if orig was never defined, fine
+                self.error_till_line_end(start, text)
+        return self.cur

data/vendor/pygments-main/pygments/lexers/trafficscript.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.trafficscript
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for RiverBed's TrafficScript (RTS) language.
+
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer
+from pygments.token import String, Number, Name, Keyword, Operator, Text, Comment
+
+__all__ = ['RtsLexer']
+
+
+class RtsLexer(RegexLexer):
+    """
+    For `Riverbed Stingray Traffic Manager <http://www.riverbed.com/stingray>`_
+
+    .. versionadded:: 2.1
+    """
+    name = 'TrafficScript'
+    aliases = ['rts','trafficscript']
+    filenames = ['*.rts']
+
+    tokens = {
+        'root' : [
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String),
+            (r'"', String, 'escapable-string'),
+            (r'(0x[0-9a-fA-F]+|\d+)', Number),
+            (r'\d+\.\d+', Number.Float),
+            (r'\$[a-zA-Z](\w|_)*', Name.Variable),
+            (r'(if|else|for(each)?|in|while|do|break|sub|return|import)', Keyword),
+            (r'[a-zA-Z][\w.]*', Name.Function),
+            (r'[-+*/%=,;(){}<>^.!~|&\[\]\?\:]', Operator),
+            (r'(>=|<=|==|!=|'
+             r'&&|\|\||'
+             r'\+=|.=|-=|\*=|/=|%=|<<=|>>=|&=|\|=|\^=|'
+             r'>>|<<|'
+             r'\+\+|--|=>)', Operator),
+            (r'[ \t\r]+', Text),
+            (r'#[^\n]*', Comment),
+        ],
+        'escapable-string' : [
+            (r'\\[tsn]', String.Escape),
+            (r'[^"]', String),
+            (r'"', String, '#pop'),
+        ],
+
+    }

data/vendor/pygments-main/pygments/lexers/typoscript.py

@@ -0,0 +1,219 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.typoscript
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for TypoScript
+
+    `TypoScriptLexer`
+        A TypoScript lexer.
+
+    `TypoScriptCssDataLexer`
+        Lexer that highlights markers, constants and registers within css.
+
+    `TypoScriptHtmlDataLexer`
+        Lexer that highlights markers, constants and registers within html tags.
+
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, include, bygroups, using
+from pygments.token import Text, Comment, Name, String, Number, \
+    Operator, Punctuation
+
+__all__ = ['TypoScriptLexer', 'TypoScriptCssDataLexer', 'TypoScriptHtmlDataLexer']
+
+
+class TypoScriptCssDataLexer(RegexLexer):
+    """
+    Lexer that highlights markers, constants and registers within css blocks.
+
+    .. versionadded:: 2.2
+    """
+
+    name = 'TypoScriptCssData'
+    aliases = ['typoscriptcssdata']
+
+    tokens = {
+        'root': [
+            # marker: ###MARK###
+            (r'(.*)(###\w+###)(.*)', bygroups(String, Name.Constant, String)),
+            # constant: {$some.constant}
+            (r'(\{)(\$)((?:[\w\-]+\.)*)([\w\-]+)(\})',
+             bygroups(String.Symbol, Operator, Name.Constant,
+                      Name.Constant, String.Symbol)),  # constant
+            # constant: {register:somevalue}
+            (r'(.*)(\{)([\w\-]+)(\s*:\s*)([\w\-]+)(\})(.*)',
+             bygroups(String, String.Symbol, Name.Constant, Operator,
+                      Name.Constant, String.Symbol, String)),  # constant
+            # whitespace
+            (r'\s+', Text),
+            # comments
+            (r'/\*(?:(?!\*/).)*\*/', Comment),
+            (r'(?<!(#|\'|"))(?:#(?!(?:[a-fA-F0-9]{6}|[a-fA-F0-9]{3}))[^\n#]+|//[^\n]*)',
+             Comment),
+            # other
+            (r'[<>,:=.*%+|]', String),
+            (r'[\w"\-!/&;(){}]+', String),
+        ]
+    }
+
+
+class TypoScriptHtmlDataLexer(RegexLexer):
+    """
+    Lexer that highlights markers, constants and registers within html tags.
+
+    .. versionadded:: 2.2
+    """
+
+    name = 'TypoScriptHtmlData'
+    aliases = ['typoscripthtmldata']
+
+    tokens = {
+        'root': [
+            # INCLUDE_TYPOSCRIPT
+            (r'(INCLUDE_TYPOSCRIPT)', Name.Class),
+            # Language label or extension resource FILE:... or LLL:... or EXT:...
+            (r'(EXT|FILE|LLL):[^}\n"]*', String),
+            # marker: ###MARK###
+            (r'(.*)(###\w+###)(.*)', bygroups(String, Name.Constant, String)),
+            # constant: {$some.constant}
+            (r'(\{)(\$)((?:[\w\-]+\.)*)([\w\-]+)(\})',
+             bygroups(String.Symbol, Operator, Name.Constant,
+                      Name.Constant, String.Symbol)),  # constant
+            # constant: {register:somevalue}
+            (r'(.*)(\{)([\w\-]+)(\s*:\s*)([\w\-]+)(\})(.*)',
+             bygroups(String, String.Symbol, Name.Constant, Operator,
+                      Name.Constant, String.Symbol, String)),  # constant
+            # whitespace
+            (r'\s+', Text),
+            # other
+            (r'[<>,:=.*%+|]', String),
+            (r'[\w"\-!/&;(){}#]+', String),
+        ]
+    }
+
+
+class TypoScriptLexer(RegexLexer):
+    """
+    Lexer for TypoScript code.
+
+    http://docs.typo3.org/typo3cms/TyposcriptReference/
+
+    .. versionadded:: 2.2
+    """
+
+    name = 'TypoScript'
+    aliases = ['typoscript']
+    filenames = ['*.typoscript']
+    mimetypes = ['text/x-typoscript']
+
+    flags = re.DOTALL | re.MULTILINE
+
+    tokens = {
+        'root': [
+            include('comment'),
+            include('constant'),
+            include('html'),
+            include('label'),
+            include('whitespace'),
+            include('keywords'),
+            include('punctuation'),
+            include('operator'),
+            include('structure'),
+            include('literal'),
+            include('other'),
+        ],
+        'keywords': [
+            # Conditions
+            (r'(?i)(\[)(browser|compatVersion|dayofmonth|dayofweek|dayofyear|'
+             r'device|ELSE|END|GLOBAL|globalString|globalVar|hostname|hour|IP|'
+             r'language|loginUser|loginuser|minute|month|page|PIDinRootline|'
+             r'PIDupinRootline|system|treeLevel|useragent|userFunc|usergroup|'
+             r'version)([^\]]*)(\])',
+             bygroups(String.Symbol, Name.Constant, Text, String.Symbol)),
+            # Functions
+            (r'(?=[\w\-])(HTMLparser|HTMLparser_tags|addParams|cache|encapsLines|'
+             r'filelink|if|imageLinkWrap|imgResource|makelinks|numRows|numberFormat|'
+             r'parseFunc|replacement|round|select|split|stdWrap|strPad|tableStyle|'
+             r'tags|textStyle|typolink)(?![\w\-])', Name.Function),
+            # Toplevel objects and _*
+            (r'(?:(=?\s*<?\s+|^\s*))(cObj|field|config|content|constants|FEData|'
+             r'file|frameset|includeLibs|lib|page|plugin|register|resources|sitemap|'
+             r'sitetitle|styles|temp|tt_[^:.\s]*|types|xmlnews|INCLUDE_TYPOSCRIPT|'
+             r'_CSS_DEFAULT_STYLE|_DEFAULT_PI_VARS|_LOCAL_LANG)(?![\w\-])',
+             bygroups(Operator, Name.Builtin)),
+            # Content objects
+            (r'(?=[\w\-])(CASE|CLEARGIF|COA|COA_INT|COBJ_ARRAY|COLUMNS|CONTENT|'
+             r'CTABLE|EDITPANEL|FILE|FILES|FLUIDTEMPLATE|FORM|HMENU|HRULER|HTML|'
+             r'IMAGE|IMGTEXT|IMG_RESOURCE|LOAD_REGISTER|MEDIA|MULTIMEDIA|OTABLE|'
+             r'PAGE|QTOBJECT|RECORDS|RESTORE_REGISTER|SEARCHRESULT|SVG|SWFOBJECT|'
+             r'TEMPLATE|TEXT|USER|USER_INT)(?![\w\-])', Name.Class),
+            # Menu states
+            (r'(?=[\w\-])(ACTIFSUBRO|ACTIFSUB|ACTRO|ACT|CURIFSUBRO|CURIFSUB|CURRO|'
+             r'CUR|IFSUBRO|IFSUB|NO|SPC|USERDEF1RO|USERDEF1|USERDEF2RO|USERDEF2|'
+             r'USRRO|USR)', Name.Class),
+            # Menu objects
+            (r'(?=[\w\-])(GMENU_FOLDOUT|GMENU_LAYERS|GMENU|IMGMENUITEM|IMGMENU|'
+             r'JSMENUITEM|JSMENU|TMENUITEM|TMENU_LAYERS|TMENU)', Name.Class),
+            # PHP objects
+            (r'(?=[\w\-])(PHP_SCRIPT(_EXT|_INT)?)', Name.Class),
+            (r'(?=[\w\-])(userFunc)(?![\w\-])', Name.Function),
+        ],
+        'whitespace': [
+            (r'\s+', Text),
+        ],
+        'html': [
+            (r'<\S[^\n>]*>', using(TypoScriptHtmlDataLexer)),
+            (r'&[^;\n]*;', String),
+            (r'(?s)(_CSS_DEFAULT_STYLE)(\s*)(\()(.*(?=\n\)))',
+             bygroups(Name.Class, Text, String.Symbol, using(TypoScriptCssDataLexer))),
+        ],
+        'literal': [
+            (r'0x[0-9A-Fa-f]+t?', Number.Hex),
+            # (r'[0-9]*\.[0-9]+([eE][0-9]+)?[fd]?\s*(?:[^=])', Number.Float),
+            (r'[0-9]+', Number.Integer),
+            (r'(###\w+###)', Name.Constant),
+        ],
+        'label': [
+            # Language label or extension resource FILE:... or LLL:... or EXT:...
+            (r'(EXT|FILE|LLL):[^}\n"]*', String),
+            # Path to a resource
+            (r'(?![^\w\-])([\w\-]+(?:/[\w\-]+)+/?)(\S*\n)',
+             bygroups(String, String)),
+        ],
+        'punctuation': [
+            (r'[,.]', Punctuation),
+        ],
+        'operator': [
+            (r'[<>,:=.*%+|]', Operator),
+        ],
+        'structure': [
+            # Brackets and braces
+            (r'[{}()\[\]\\]', String.Symbol),
+        ],
+        'constant': [
+            # Constant: {$some.constant}
+            (r'(\{)(\$)((?:[\w\-]+\.)*)([\w\-]+)(\})',
+                bygroups(String.Symbol, Operator, Name.Constant,
+                         Name.Constant, String.Symbol)),  # constant
+            # Constant: {register:somevalue}
+            (r'(\{)([\w\-]+)(\s*:\s*)([\w\-]+)(\})',
+                bygroups(String.Symbol, Name.Constant, Operator,
+                         Name.Constant, String.Symbol)),  # constant
+            # Hex color: #ff0077
+            (r'(#[a-fA-F0-9]{6}\b|#[a-fA-F0-9]{3}\b)', String.Char)
+        ],
+        'comment': [
+            (r'(?<!(#|\'|"))(?:#(?!(?:[a-fA-F0-9]{6}|[a-fA-F0-9]{3}))[^\n#]+|//[^\n]*)',
+             Comment),
+            (r'/\*(?:(?!\*/).)*\*/', Comment),
+            (r'(\s*#\s*\n)', Comment),
+        ],
+        'other': [
+            (r'[\w"\-!/&;]+', Text),
+        ],
+    }