pygments.rb 1.2.1 → 2.0.0.rc2

data/vendor/pygments-main/pygments/lexers/esoteric.py

@@ -0,0 +1,304 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.esoteric
+    ~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for esoteric languages.
+
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Error
+
+import re
+
+__all__ = ['BrainfuckLexer', 'BefungeLexer', 'RedcodeLexer', 'CAmkESLexer',
+           'CapDLLexer', 'AheuiLexer']
+
+
+class BrainfuckLexer(RegexLexer):
+    """
+    Lexer for the esoteric `BrainFuck <http://www.muppetlabs.com/~breadbox/bf/>`_
+    language.
+    """
+
+    name = 'Brainfuck'
+    aliases = ['brainfuck', 'bf']
+    filenames = ['*.bf', '*.b']
+    mimetypes = ['application/x-brainfuck']
+
+    tokens = {
+        'common': [
+            # use different colors for different instruction types
+            (r'[.,]+', Name.Tag),
+            (r'[+-]+', Name.Builtin),
+            (r'[<>]+', Name.Variable),
+            (r'[^.,+\-<>\[\]]+', Comment),
+        ],
+        'root': [
+            (r'\[', Keyword, 'loop'),
+            (r'\]', Error),
+            include('common'),
+        ],
+        'loop': [
+            (r'\[', Keyword, '#push'),
+            (r'\]', Keyword, '#pop'),
+            include('common'),
+        ]
+    }
+
+    def analyse_text(text):
+        """It's safe to assume that a program which mostly consists of + -
+        and < > is brainfuck."""
+        plus_minus_count = 0
+        greater_less_count = 0
+
+        range_to_check = max(256, len(text))
+
+        for c in text[:range_to_check]:
+            if c == '+' or c == '-':
+                plus_minus_count += 1
+            if c == '<' or c == '>':
+                greater_less_count += 1
+
+        if plus_minus_count > (0.25 * range_to_check):
+            return 1.0
+        if greater_less_count > (0.25 * range_to_check):
+            return 1.0
+
+        result = 0
+        if '[-]' in text:
+            result += 0.5
+
+        return result
+
+
+class BefungeLexer(RegexLexer):
+    """
+    Lexer for the esoteric `Befunge <http://en.wikipedia.org/wiki/Befunge>`_
+    language.
+
+    .. versionadded:: 0.7
+    """
+    name = 'Befunge'
+    aliases = ['befunge']
+    filenames = ['*.befunge']
+    mimetypes = ['application/x-befunge']
+
+    tokens = {
+        'root': [
+            (r'[0-9a-f]', Number),
+            (r'[+*/%!`-]', Operator),             # Traditional math
+            (r'[<>^v?\[\]rxjk]', Name.Variable),  # Move, imperatives
+            (r'[:\\$.,n]', Name.Builtin),         # Stack ops, imperatives
+            (r'[|_mw]', Keyword),
+            (r'[{}]', Name.Tag),                  # Befunge-98 stack ops
+            (r'".*?"', String.Double),            # Strings don't appear to allow escapes
+            (r'\'.', String.Single),              # Single character
+            (r'[#;]', Comment),                   # Trampoline... depends on direction hit
+            (r'[pg&~=@iotsy]', Keyword),          # Misc
+            (r'[()A-Z]', Comment),                # Fingerprints
+            (r'\s+', Text),                       # Whitespace doesn't matter
+        ],
+    }
+
+
+class CAmkESLexer(RegexLexer):
+    """
+    Basic lexer for the input language for the
+    `CAmkES <https://sel4.systems/CAmkES/>`_ component platform.
+
+    .. versionadded:: 2.1
+    """
+    name = 'CAmkES'
+    aliases = ['camkes', 'idl4']
+    filenames = ['*.camkes', '*.idl4']
+
+    tokens = {
+        'root': [
+            # C pre-processor directive
+            (r'^\s*#.*\n', Comment.Preproc),
+
+            # Whitespace, comments
+            (r'\s+', Text),
+            (r'/\*(.|\n)*?\*/', Comment),
+            (r'//.*\n', Comment),
+
+            (r'[\[(){},.;\]]', Punctuation),
+            (r'[~!%^&*+=|?:<>/-]', Operator),
+
+            (words(('assembly', 'attribute', 'component', 'composition',
+                    'configuration', 'connection', 'connector', 'consumes',
+                    'control', 'dataport', 'Dataport', 'Dataports', 'emits',
+                    'event', 'Event', 'Events', 'export', 'from', 'group',
+                    'hardware', 'has', 'interface', 'Interface', 'maybe',
+                    'procedure', 'Procedure', 'Procedures', 'provides',
+                    'template', 'thread', 'threads', 'to', 'uses', 'with'),
+                   suffix=r'\b'), Keyword),
+
+            (words(('bool', 'boolean', 'Buf', 'char', 'character', 'double',
+                    'float', 'in', 'inout', 'int', 'int16_6', 'int32_t',
+                    'int64_t', 'int8_t', 'integer', 'mutex', 'out', 'real',
+                    'refin', 'semaphore', 'signed', 'string', 'struct',
+                    'uint16_t', 'uint32_t', 'uint64_t', 'uint8_t', 'uintptr_t',
+                    'unsigned', 'void'),
+                   suffix=r'\b'), Keyword.Type),
+
+            # Recognised attributes
+            (r'[a-zA-Z_]\w*_(priority|domain|buffer)', Keyword.Reserved),
+            (words(('dma_pool', 'from_access', 'to_access'), suffix=r'\b'),
+                Keyword.Reserved),
+
+            # CAmkES-level include
+            (r'import\s+(<[^>]*>|"[^"]*");', Comment.Preproc),
+
+            # C-level include
+            (r'include\s+(<[^>]*>|"[^"]*");', Comment.Preproc),
+
+            # Literals
+            (r'0[xX][\da-fA-F]+', Number.Hex),
+            (r'-?[\d]+', Number),
+            (r'-?[\d]+\.[\d]+', Number.Float),
+            (r'"[^"]*"', String),
+            (r'[Tt]rue|[Ff]alse', Name.Builtin),
+
+            # Identifiers
+            (r'[a-zA-Z_]\w*', Name),
+        ],
+    }
+
+
+class CapDLLexer(RegexLexer):
+    """
+    Basic lexer for
+    `CapDL <https://ssrg.nicta.com.au/publications/nictaabstracts/Kuz_KLW_10.abstract.pml>`_.
+
+    The source of the primary tool that reads such specifications is available
+    at https://github.com/seL4/capdl/tree/master/capDL-tool. Note that this
+    lexer only supports a subset of the grammar. For example, identifiers can
+    shadow type names, but these instances are currently incorrectly
+    highlighted as types. Supporting this would need a stateful lexer that is
+    considered unnecessarily complex for now.
+
+    .. versionadded:: 2.2
+    """
+    name = 'CapDL'
+    aliases = ['capdl']
+    filenames = ['*.cdl']
+
+    tokens = {
+        'root': [
+            # C pre-processor directive
+            (r'^\s*#.*\n', Comment.Preproc),
+
+            # Whitespace, comments
+            (r'\s+', Text),
+            (r'/\*(.|\n)*?\*/', Comment),
+            (r'(//|--).*\n', Comment),
+
+            (r'[<>\[(){},:;=\]]', Punctuation),
+            (r'\.\.', Punctuation),
+
+            (words(('arch', 'arm11', 'caps', 'child_of', 'ia32', 'irq', 'maps',
+                    'objects'), suffix=r'\b'), Keyword),
+
+            (words(('aep', 'asid_pool', 'cnode', 'ep', 'frame', 'io_device',
+                    'io_ports', 'io_pt', 'notification', 'pd', 'pt', 'tcb',
+                    'ut', 'vcpu'), suffix=r'\b'), Keyword.Type),
+
+            # Properties
+            (words(('asid', 'addr', 'badge', 'cached', 'dom', 'domainID', 'elf',
+                    'fault_ep', 'G', 'guard', 'guard_size', 'init', 'ip',
+                    'prio', 'sp', 'R', 'RG', 'RX', 'RW', 'RWG', 'RWX', 'W',
+                    'WG', 'WX', 'level', 'masked', 'master_reply', 'paddr',
+                    'ports', 'reply', 'uncached'), suffix=r'\b'),
+             Keyword.Reserved),
+
+            # Literals
+            (r'0[xX][\da-fA-F]+', Number.Hex),
+            (r'\d+(\.\d+)?(k|M)?', Number),
+            (words(('bits',), suffix=r'\b'), Number),
+            (words(('cspace', 'vspace', 'reply_slot', 'caller_slot',
+                    'ipc_buffer_slot'), suffix=r'\b'), Number),
+
+            # Identifiers
+            (r'[a-zA-Z_][-@\.\w]*', Name),
+        ],
+    }
+
+
+class RedcodeLexer(RegexLexer):
+    """
+    A simple Redcode lexer based on ICWS'94.
+    Contributed by Adam Blinkinsop <blinks@acm.org>.
+
+    .. versionadded:: 0.8
+    """
+    name = 'Redcode'
+    aliases = ['redcode']
+    filenames = ['*.cw']
+
+    opcodes = ('DAT', 'MOV', 'ADD', 'SUB', 'MUL', 'DIV', 'MOD',
+               'JMP', 'JMZ', 'JMN', 'DJN', 'CMP', 'SLT', 'SPL',
+               'ORG', 'EQU', 'END')
+    modifiers = ('A', 'B', 'AB', 'BA', 'F', 'X', 'I')
+
+    tokens = {
+        'root': [
+            # Whitespace:
+            (r'\s+', Text),
+            (r';.*$', Comment.Single),
+            # Lexemes:
+            #  Identifiers
+            (r'\b(%s)\b' % '|'.join(opcodes), Name.Function),
+            (r'\b(%s)\b' % '|'.join(modifiers), Name.Decorator),
+            (r'[A-Za-z_]\w+', Name),
+            #  Operators
+            (r'[-+*/%]', Operator),
+            (r'[#$@<>]', Operator),  # mode
+            (r'[.,]', Punctuation),  # mode
+            #  Numbers
+            (r'[-+]?\d+', Number.Integer),
+        ],
+    }
+
+
+class AheuiLexer(RegexLexer):
+    """
+    Aheui_ Lexer.
+
+    Aheui_ is esoteric language based on Korean alphabets.
+
+    .. _Aheui: http://aheui.github.io/
+
+    """
+
+    name = 'Aheui'
+    aliases = ['aheui']
+    filenames = ['*.aheui']
+
+    tokens = {
+        'root': [
+            ('['
+             '나-낳냐-냫너-넣녀-녛노-놓뇨-눟뉴-닇'
+             '다-닿댜-댷더-덯뎌-뎧도-돟됴-둫듀-딓'
+             '따-땋땨-떃떠-떻뗘-뗳또-똫뚀-뚷뜌-띟'
+             '라-랗랴-럏러-렇려-렿로-롷료-뤃류-릫'
+             '마-맣먀-먛머-멓며-몋모-뫃묘-뭏뮤-믷'
+             '바-밯뱌-뱧버-벟벼-볗보-봏뵤-붛뷰-빃'
+             '빠-빻뺘-뺳뻐-뻫뼈-뼣뽀-뽛뾰-뿧쀼-삏'
+             '사-샇샤-샿서-섷셔-셯소-솧쇼-숳슈-싛'
+             '싸-쌓쌰-썋써-쎃쎠-쎻쏘-쏳쑈-쑿쓔-씧'
+             '자-잫쟈-쟣저-젛져-졓조-좋죠-줗쥬-즿'
+             '차-챃챠-챻처-첳쳐-쳫초-촣쵸-춯츄-칗'
+             '카-캏캬-컇커-컿켜-켷코-콯쿄-쿻큐-킣'
+             '타-탛탸-턓터-텋텨-톃토-톻툐-퉇튜-틯'
+             '파-팧퍄-퍟퍼-펗펴-폏포-퐇표-풓퓨-픻'
+             '하-핳햐-햫허-헣혀-혛호-홓효-훟휴-힇'
+             ']', Operator),
+            ('.', Comment),
+        ],
+    }

data/vendor/pygments-main/pygments/lexers/ezhil.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.ezhil
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Pygments lexers for Ezhil language.
+
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+from pygments.lexer import RegexLexer, include, words
+from pygments.token import Keyword, Text, Comment, Name
+from pygments.token import String, Number, Punctuation, Operator
+
+__all__ = ['EzhilLexer']
+
+
+class EzhilLexer(RegexLexer):
+    """
+    Lexer for `Ezhil, a Tamil script-based programming language <http://ezhillang.org>`_
+
+    .. versionadded:: 2.1
+    """
+    name = 'Ezhil'
+    aliases = ['ezhil']
+    filenames = ['*.n']
+    mimetypes = ['text/x-ezhil']
+    flags = re.MULTILINE | re.UNICODE
+    # Refer to tamil.utf8.tamil_letters from open-tamil for a stricter version of this.
+    # This much simpler version is close enough, and includes combining marks.
+    _TALETTERS = '[a-zA-Z_]|[\u0b80-\u0bff]'
+    tokens = {
+        'root': [
+            include('keywords'),
+            (r'#.*\n', Comment.Single),
+            (r'[@+/*,^\-%]|[!<>=]=?|&&?|\|\|?', Operator),
+            ('இல்', Operator.Word),
+            (words(('assert', 'max', 'min',
+                    'நீளம்', 'சரம்_இடமாற்று', 'சரம்_கண்டுபிடி',
+                    'பட்டியல்', 'பின்இணை', 'வரிசைப்படுத்து',
+                    'எடு', 'தலைகீழ்', 'நீட்டிக்க', 'நுழைக்க', 'வை',
+                    'கோப்பை_திற', 'கோப்பை_எழுது', 'கோப்பை_மூடு',
+                    'pi', 'sin', 'cos', 'tan', 'sqrt', 'hypot', 'pow',
+                    'exp', 'log', 'log10', 'exit',
+                    ), suffix=r'\b'), Name.Builtin),
+            (r'(True|False)\b', Keyword.Constant),
+            (r'[^\S\n]+', Text),
+            include('identifier'),
+            include('literal'),
+            (r'[(){}\[\]:;.]', Punctuation),
+        ],
+        'keywords': [
+            ('பதிப்பி|தேர்ந்தெடு|தேர்வு|ஏதேனில்|ஆனால்|இல்லைஆனால்|இல்லை|ஆக|ஒவ்வொன்றாக|இல்|வரை|செய்|முடியேனில்|பின்கொடு|முடி|நிரல்பாகம்|தொடர்|நிறுத்து|நிரல்பாகம்', Keyword),
+        ],
+        'identifier': [
+            ('(?:'+_TALETTERS+')(?:[0-9]|'+_TALETTERS+')*', Name),
+        ],
+        'literal': [
+            (r'".*?"', String),
+            (r'(?u)\d+((\.\d*)?[eE][+-]?\d+|\.\d*)', Number.Float),
+            (r'(?u)\d+', Number.Integer),
+        ]
+    }
+
+    def analyse_text(text):
+        """This language uses Tamil-script. We'll assume that if there's a
+        decent amount of Tamil-characters, it's this language. This assumption
+        is obviously horribly off if someone uses string literals in tamil
+        in another language."""
+        if len(re.findall(r'[\u0b80-\u0bff]', text)) > 10:
+            return 0.25
+
+    def __init__(self, **options):
+        super().__init__(**options)
+        self.encoding = options.get('encoding', 'utf-8')

data/vendor/pygments-main/pygments/lexers/factor.py

@@ -0,0 +1,344 @@
+# -*- coding: utf-8 -*-
+"""
+    pygments.lexers.factor
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Factor language.
+
+    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, default, words
+from pygments.token import Text, Comment, Keyword, Name, String, Number
+
+__all__ = ['FactorLexer']
+
+
+class FactorLexer(RegexLexer):
+    """
+    Lexer for the `Factor <http://factorcode.org>`_ language.
+
+    .. versionadded:: 1.4
+    """
+    name = 'Factor'
+    aliases = ['factor']
+    filenames = ['*.factor']
+    mimetypes = ['text/x-factor']
+
+    flags = re.MULTILINE | re.UNICODE
+
+    builtin_kernel = words((
+        '-rot', '2bi', '2bi@', '2bi*', '2curry', '2dip', '2drop', '2dup', '2keep', '2nip',
+        '2over', '2tri', '2tri@', '2tri*', '3bi', '3curry', '3dip', '3drop', '3dup', '3keep',
+        '3tri', '4dip', '4drop', '4dup', '4keep', '<wrapper>', '=', '>boolean', 'clone',
+        '?', '?execute', '?if', 'and', 'assert', 'assert=', 'assert?', 'bi', 'bi-curry',
+        'bi-curry@', 'bi-curry*', 'bi@', 'bi*', 'boa', 'boolean', 'boolean?', 'both?',
+        'build', 'call', 'callstack', 'callstack>array', 'callstack?', 'clear', '(clone)',
+        'compose', 'compose?', 'curry', 'curry?', 'datastack', 'die', 'dip', 'do', 'drop',
+        'dup', 'dupd', 'either?', 'eq?', 'equal?', 'execute', 'hashcode', 'hashcode*',
+        'identity-hashcode', 'identity-tuple', 'identity-tuple?', 'if', 'if*',
+        'keep', 'loop', 'most', 'new', 'nip', 'not', 'null', 'object', 'or', 'over',
+        'pick', 'prepose', 'retainstack', 'rot', 'same?', 'swap', 'swapd', 'throw',
+        'tri', 'tri-curry', 'tri-curry@', 'tri-curry*', 'tri@', 'tri*', 'tuple',
+        'tuple?', 'unless', 'unless*', 'until', 'when', 'when*', 'while', 'with',
+        'wrapper', 'wrapper?', 'xor'), suffix=r'\s')
+
+    builtin_assocs = words((
+        '2cache', '<enum>', '>alist', '?at', '?of', 'assoc', 'assoc-all?',
+        'assoc-any?', 'assoc-clone-like', 'assoc-combine', 'assoc-diff',
+        'assoc-diff!', 'assoc-differ', 'assoc-each', 'assoc-empty?',
+        'assoc-filter', 'assoc-filter!', 'assoc-filter-as', 'assoc-find',
+        'assoc-hashcode', 'assoc-intersect', 'assoc-like', 'assoc-map',
+        'assoc-map-as', 'assoc-partition', 'assoc-refine', 'assoc-size',
+        'assoc-stack', 'assoc-subset?', 'assoc-union', 'assoc-union!',
+        'assoc=', 'assoc>map', 'assoc?', 'at', 'at+', 'at*', 'cache', 'change-at',
+        'clear-assoc', 'delete-at', 'delete-at*', 'enum', 'enum?', 'extract-keys',
+        'inc-at', 'key?', 'keys', 'map>assoc', 'maybe-set-at', 'new-assoc', 'of',
+        'push-at', 'rename-at', 'set-at', 'sift-keys', 'sift-values', 'substitute',
+        'unzip', 'value-at', 'value-at*', 'value?', 'values', 'zip'), suffix=r'\s')
+
+    builtin_combinators = words((
+        '2cleave', '2cleave>quot', '3cleave', '3cleave>quot', '4cleave',
+        '4cleave>quot', 'alist>quot', 'call-effect', 'case', 'case-find',
+        'case>quot', 'cleave', 'cleave>quot', 'cond', 'cond>quot', 'deep-spread>quot',
+        'execute-effect', 'linear-case-quot', 'no-case', 'no-case?', 'no-cond',
+        'no-cond?', 'recursive-hashcode', 'shallow-spread>quot', 'spread',
+        'to-fixed-point', 'wrong-values', 'wrong-values?'), suffix=r'\s')
+
+    builtin_math = words((
+        '-', '/', '/f', '/i', '/mod', '2/', '2^', '<', '<=', '<fp-nan>', '>',
+        '>=', '>bignum', '>fixnum', '>float', '>integer', '(all-integers?)',
+        '(each-integer)', '(find-integer)', '*', '+', '?1+',
+        'abs', 'align', 'all-integers?', 'bignum', 'bignum?', 'bit?', 'bitand',
+        'bitnot', 'bitor', 'bits>double', 'bits>float', 'bitxor', 'complex',
+        'complex?', 'denominator', 'double>bits', 'each-integer', 'even?',
+        'find-integer', 'find-last-integer', 'fixnum', 'fixnum?', 'float',
+        'float>bits', 'float?', 'fp-bitwise=', 'fp-infinity?', 'fp-nan-payload',
+        'fp-nan?', 'fp-qnan?', 'fp-sign', 'fp-snan?', 'fp-special?',
+        'if-zero', 'imaginary-part', 'integer', 'integer>fixnum',
+        'integer>fixnum-strict', 'integer?', 'log2', 'log2-expects-positive',
+        'log2-expects-positive?', 'mod', 'neg', 'neg?', 'next-float',
+        'next-power-of-2', 'number', 'number=', 'number?', 'numerator', 'odd?',
+        'out-of-fixnum-range', 'out-of-fixnum-range?', 'power-of-2?',
+        'prev-float', 'ratio', 'ratio?', 'rational', 'rational?', 'real',
+        'real-part', 'real?', 'recip', 'rem', 'sgn', 'shift', 'sq', 'times',
+        'u<', 'u<=', 'u>', 'u>=', 'unless-zero', 'unordered?', 'when-zero',
+        'zero?'), suffix=r'\s')
+
+    builtin_sequences = words((
+        '1sequence', '2all?', '2each', '2map', '2map-as', '2map-reduce', '2reduce',
+        '2selector', '2sequence', '3append', '3append-as', '3each', '3map', '3map-as',
+        '3sequence', '4sequence', '<repetition>', '<reversed>', '<slice>', '?first',
+        '?last', '?nth', '?second', '?set-nth', 'accumulate', 'accumulate!',
+        'accumulate-as', 'all?', 'any?', 'append', 'append!', 'append-as',
+        'assert-sequence', 'assert-sequence=', 'assert-sequence?',
+        'binary-reduce', 'bounds-check', 'bounds-check?', 'bounds-error',
+        'bounds-error?', 'but-last', 'but-last-slice', 'cartesian-each',
+        'cartesian-map', 'cartesian-product', 'change-nth', 'check-slice',
+        'check-slice-error', 'clone-like', 'collapse-slice', 'collector',
+        'collector-for', 'concat', 'concat-as', 'copy', 'count', 'cut', 'cut-slice',
+        'cut*', 'delete-all', 'delete-slice', 'drop-prefix', 'each', 'each-from',
+        'each-index', 'empty?', 'exchange', 'filter', 'filter!', 'filter-as', 'find',
+        'find-from', 'find-index', 'find-index-from', 'find-last', 'find-last-from',
+        'first', 'first2', 'first3', 'first4', 'flip', 'follow', 'fourth', 'glue', 'halves',
+        'harvest', 'head', 'head-slice', 'head-slice*', 'head*', 'head?',
+        'if-empty', 'immutable', 'immutable-sequence', 'immutable-sequence?',
+        'immutable?', 'index', 'index-from', 'indices', 'infimum', 'infimum-by',
+        'insert-nth', 'interleave', 'iota', 'iota-tuple', 'iota-tuple?', 'join',
+        'join-as', 'last', 'last-index', 'last-index-from', 'length', 'lengthen',
+        'like', 'longer', 'longer?', 'longest', 'map', 'map!', 'map-as', 'map-find',
+        'map-find-last', 'map-index', 'map-integers', 'map-reduce', 'map-sum',
+        'max-length', 'member-eq?', 'member?', 'midpoint@', 'min-length',
+        'mismatch', 'move', 'new-like', 'new-resizable', 'new-sequence',
+        'non-negative-integer-expected', 'non-negative-integer-expected?',
+        'nth', 'nths', 'pad-head', 'pad-tail', 'padding', 'partition', 'pop', 'pop*',
+        'prefix', 'prepend', 'prepend-as', 'produce', 'produce-as', 'product', 'push',
+        'push-all', 'push-either', 'push-if', 'reduce', 'reduce-index', 'remove',
+        'remove!', 'remove-eq', 'remove-eq!', 'remove-nth', 'remove-nth!', 'repetition',
+        'repetition?', 'replace-slice', 'replicate', 'replicate-as', 'rest',
+        'rest-slice', 'reverse', 'reverse!', 'reversed', 'reversed?', 'second',
+        'selector', 'selector-for', 'sequence', 'sequence-hashcode', 'sequence=',
+        'sequence?', 'set-first', 'set-fourth', 'set-last', 'set-length', 'set-nth',
+        'set-second', 'set-third', 'short', 'shorten', 'shorter', 'shorter?',
+        'shortest', 'sift', 'slice', 'slice-error', 'slice-error?', 'slice?',
+        'snip', 'snip-slice', 'start', 'start*', 'subseq', 'subseq?', 'suffix',
+        'suffix!', 'sum', 'sum-lengths', 'supremum', 'supremum-by', 'surround', 'tail',
+        'tail-slice', 'tail-slice*', 'tail*', 'tail?', 'third', 'trim',
+        'trim-head', 'trim-head-slice', 'trim-slice', 'trim-tail', 'trim-tail-slice',
+        'unclip', 'unclip-last', 'unclip-last-slice', 'unclip-slice', 'unless-empty',
+        'virtual-exemplar', 'virtual-sequence', 'virtual-sequence?', 'virtual@',
+        'when-empty'), suffix=r'\s')
+
+    builtin_namespaces = words((
+        '+@', 'change', 'change-global', 'counter', 'dec', 'get', 'get-global',
+        'global', 'inc', 'init-namespaces', 'initialize', 'is-global', 'make-assoc',
+        'namespace', 'namestack', 'off', 'on', 'set', 'set-global', 'set-namestack',
+        'toggle', 'with-global', 'with-scope', 'with-variable', 'with-variables'),
+        suffix=r'\s')
+
+    builtin_arrays = words((
+        '1array', '2array', '3array', '4array', '<array>', '>array', 'array',
+        'array?', 'pair', 'pair?', 'resize-array'), suffix=r'\s')
+
+    builtin_io = words((
+        '(each-stream-block-slice)', '(each-stream-block)',
+        '(stream-contents-by-block)', '(stream-contents-by-element)',
+        '(stream-contents-by-length-or-block)',
+        '(stream-contents-by-length)', '+byte+', '+character+',
+        'bad-seek-type', 'bad-seek-type?', 'bl', 'contents', 'each-block',
+        'each-block-size', 'each-block-slice', 'each-line', 'each-morsel',
+        'each-stream-block', 'each-stream-block-slice', 'each-stream-line',
+        'error-stream', 'flush', 'input-stream', 'input-stream?',
+        'invalid-read-buffer', 'invalid-read-buffer?', 'lines', 'nl',
+        'output-stream', 'output-stream?', 'print', 'read', 'read-into',
+        'read-partial', 'read-partial-into', 'read-until', 'read1', 'readln',
+        'seek-absolute', 'seek-absolute?', 'seek-end', 'seek-end?',
+        'seek-input', 'seek-output', 'seek-relative', 'seek-relative?',
+        'stream-bl', 'stream-contents', 'stream-contents*', 'stream-copy',
+        'stream-copy*', 'stream-element-type', 'stream-flush',
+        'stream-length', 'stream-lines', 'stream-nl', 'stream-print',
+        'stream-read', 'stream-read-into', 'stream-read-partial',
+        'stream-read-partial-into', 'stream-read-partial-unsafe',
+        'stream-read-unsafe', 'stream-read-until', 'stream-read1',
+        'stream-readln', 'stream-seek', 'stream-seekable?', 'stream-tell',
+        'stream-write', 'stream-write1', 'tell-input', 'tell-output',
+        'with-error-stream', 'with-error-stream*', 'with-error>output',
+        'with-input-output+error-streams',
+        'with-input-output+error-streams*', 'with-input-stream',
+        'with-input-stream*', 'with-output-stream', 'with-output-stream*',
+        'with-output>error', 'with-output+error-stream',
+        'with-output+error-stream*', 'with-streams', 'with-streams*',
+        'write', 'write1'), suffix=r'\s')
+
+    builtin_strings = words((
+        '1string', '<string>', '>string', 'resize-string', 'string',
+        'string?'), suffix=r'\s')
+
+    builtin_vectors = words((
+        '1vector', '<vector>', '>vector', '?push', 'vector', 'vector?'),
+        suffix=r'\s')
+
+    builtin_continuations = words((
+        '<condition>', '<continuation>', '<restart>', 'attempt-all',
+        'attempt-all-error', 'attempt-all-error?', 'callback-error-hook',
+        'callcc0', 'callcc1', 'cleanup', 'compute-restarts', 'condition',
+        'condition?', 'continuation', 'continuation?', 'continue',
+        'continue-restart', 'continue-with', 'current-continuation',
+        'error', 'error-continuation', 'error-in-thread', 'error-thread',
+        'ifcc', 'ignore-errors', 'in-callback?', 'original-error', 'recover',
+        'restart', 'restart?', 'restarts', 'rethrow', 'rethrow-restarts',
+        'return', 'return-continuation', 'thread-error-hook', 'throw-continue',
+        'throw-restarts', 'with-datastack', 'with-return'), suffix=r'\s')
+
+    tokens = {
+        'root': [
+            # factor allows a file to start with a shebang
+            (r'#!.*$', Comment.Preproc),
+            default('base'),
+        ],
+        'base': [
+            (r'\s+', Text),
+
+            # defining words
+            (r'((?:MACRO|MEMO|TYPED)?:[:]?)(\s+)(\S+)',
+             bygroups(Keyword, Text, Name.Function)),
+            (r'(M:[:]?)(\s+)(\S+)(\s+)(\S+)',
+             bygroups(Keyword, Text, Name.Class, Text, Name.Function)),
+            (r'(C:)(\s+)(\S+)(\s+)(\S+)',
+             bygroups(Keyword, Text, Name.Function, Text, Name.Class)),
+            (r'(GENERIC:)(\s+)(\S+)',
+             bygroups(Keyword, Text, Name.Function)),
+            (r'(HOOK:|GENERIC#)(\s+)(\S+)(\s+)(\S+)',
+             bygroups(Keyword, Text, Name.Function, Text, Name.Function)),
+            (r'\(\s', Name.Function, 'stackeffect'),
+            (r';\s', Keyword),
+
+            # imports and namespaces
+            (r'(USING:)(\s+)',
+             bygroups(Keyword.Namespace, Text), 'vocabs'),
+            (r'(USE:|UNUSE:|IN:|QUALIFIED:)(\s+)(\S+)',
+             bygroups(Keyword.Namespace, Text, Name.Namespace)),
+            (r'(QUALIFIED-WITH:)(\s+)(\S+)(\s+)(\S+)',
+             bygroups(Keyword.Namespace, Text, Name.Namespace, Text, Name.Namespace)),
+            (r'(FROM:|EXCLUDE:)(\s+)(\S+)(\s+=>\s)',
+             bygroups(Keyword.Namespace, Text, Name.Namespace, Text), 'words'),
+            (r'(RENAME:)(\s+)(\S+)(\s+)(\S+)(\s+=>\s+)(\S+)',
+             bygroups(Keyword.Namespace, Text, Name.Function, Text, Name.Namespace, Text, Name.Function)),
+            (r'(ALIAS:|TYPEDEF:)(\s+)(\S+)(\s+)(\S+)',
+             bygroups(Keyword.Namespace, Text, Name.Function, Text, Name.Function)),
+            (r'(DEFER:|FORGET:|POSTPONE:)(\s+)(\S+)',
+             bygroups(Keyword.Namespace, Text, Name.Function)),
+
+            # tuples and classes
+            (r'(TUPLE:|ERROR:)(\s+)(\S+)(\s+<\s+)(\S+)',
+             bygroups(Keyword, Text, Name.Class, Text, Name.Class), 'slots'),
+            (r'(TUPLE:|ERROR:|BUILTIN:)(\s+)(\S+)',
+             bygroups(Keyword, Text, Name.Class), 'slots'),
+            (r'(MIXIN:|UNION:|INTERSECTION:)(\s+)(\S+)',
+             bygroups(Keyword, Text, Name.Class)),
+            (r'(PREDICATE:)(\s+)(\S+)(\s+<\s+)(\S+)',
+             bygroups(Keyword, Text, Name.Class, Text, Name.Class)),
+            (r'(C:)(\s+)(\S+)(\s+)(\S+)',
+             bygroups(Keyword, Text, Name.Function, Text, Name.Class)),
+            (r'(INSTANCE:)(\s+)(\S+)(\s+)(\S+)',
+             bygroups(Keyword, Text, Name.Class, Text, Name.Class)),
+            (r'(SLOT:)(\s+)(\S+)', bygroups(Keyword, Text, Name.Function)),
+            (r'(SINGLETON:)(\s+)(\S+)', bygroups(Keyword, Text, Name.Class)),
+            (r'SINGLETONS:', Keyword, 'classes'),
+
+            # other syntax
+            (r'(CONSTANT:|SYMBOL:|MAIN:|HELP:)(\s+)(\S+)',
+             bygroups(Keyword, Text, Name.Function)),
+            (r'SYMBOLS:\s', Keyword, 'words'),
+            (r'SYNTAX:\s', Keyword),
+            (r'ALIEN:\s', Keyword),
+            (r'(STRUCT:)(\s+)(\S+)', bygroups(Keyword, Text, Name.Class)),
+            (r'(FUNCTION:)(\s+\S+\s+)(\S+)(\s+\(\s+[^)]+\)\s)',
+             bygroups(Keyword.Namespace, Text, Name.Function, Text)),
+            (r'(FUNCTION-ALIAS:)(\s+)(\S+)(\s+\S+\s+)(\S+)(\s+\(\s+[^)]+\)\s)',
+             bygroups(Keyword.Namespace, Text, Name.Function, Text, Name.Function, Text)),
+
+            # vocab.private
+            (r'(?:<PRIVATE|PRIVATE>)\s', Keyword.Namespace),
+
+            # strings
+            (r'"""\s+(?:.|\n)*?\s+"""', String),
+            (r'"(?:\\\\|\\"|[^"])*"', String),
+            (r'\S+"\s+(?:\\\\|\\"|[^"])*"', String),
+            (r'CHAR:\s+(?:\\[\\abfnrstv]|[^\\]\S*)\s', String.Char),
+
+            # comments
+            (r'!\s+.*$', Comment),
+            (r'#!\s+.*$', Comment),
+            (r'/\*\s+(?:.|\n)*?\s\*/\s', Comment),
+
+            # boolean constants
+            (r'[tf]\s', Name.Constant),
+
+            # symbols and literals
+            (r'[\\$]\s+\S+', Name.Constant),
+            (r'M\\\s+\S+\s+\S+', Name.Constant),
+
+            # numbers
+            (r'[+-]?(?:[\d,]*\d)?\.(?:\d([\d,]*\d)?)?(?:[eE][+-]?\d+)?\s', Number),
+            (r'[+-]?\d(?:[\d,]*\d)?(?:[eE][+-]?\d+)?\s', Number),
+            (r'0x[a-fA-F\d](?:[a-fA-F\d,]*[a-fA-F\d])?(?:p\d([\d,]*\d)?)?\s', Number),
+            (r'NAN:\s+[a-fA-F\d](?:[a-fA-F\d,]*[a-fA-F\d])?(?:p\d([\d,]*\d)?)?\s', Number),
+            (r'0b[01]+\s', Number.Bin),
+            (r'0o[0-7]+\s', Number.Oct),
+            (r'(?:\d([\d,]*\d)?)?\+\d(?:[\d,]*\d)?/\d(?:[\d,]*\d)?\s', Number),
+            (r'(?:\-\d([\d,]*\d)?)?\-\d(?:[\d,]*\d)?/\d(?:[\d,]*\d)?\s', Number),
+
+            # keywords
+            (r'(?:deprecated|final|foldable|flushable|inline|recursive)\s',
+             Keyword),
+
+            # builtins
+            (builtin_kernel, Name.Builtin),
+            (builtin_assocs, Name.Builtin),
+            (builtin_combinators, Name.Builtin),
+            (builtin_math, Name.Builtin),
+            (builtin_sequences, Name.Builtin),
+            (builtin_namespaces, Name.Builtin),
+            (builtin_arrays, Name.Builtin),
+            (builtin_io, Name.Builtin),
+            (builtin_strings, Name.Builtin),
+            (builtin_vectors, Name.Builtin),
+            (builtin_continuations, Name.Builtin),
+
+            # everything else is text
+            (r'\S+', Text),
+        ],
+        'stackeffect': [
+            (r'\s+', Text),
+            (r'\(\s+', Name.Function, 'stackeffect'),
+            (r'\)\s', Name.Function, '#pop'),
+            (r'--\s', Name.Function),
+            (r'\S+', Name.Variable),
+        ],
+        'slots': [
+            (r'\s+', Text),
+            (r';\s', Keyword, '#pop'),
+            (r'(\{\s+)(\S+)(\s+[^}]+\s+\}\s)',
+             bygroups(Text, Name.Variable, Text)),
+            (r'\S+', Name.Variable),
+        ],
+        'vocabs': [
+            (r'\s+', Text),
+            (r';\s', Keyword, '#pop'),
+            (r'\S+', Name.Namespace),
+        ],
+        'classes': [
+            (r'\s+', Text),
+            (r';\s', Keyword, '#pop'),
+            (r'\S+', Name.Class),
+        ],
+        'words': [
+            (r'\s+', Text),
+            (r';\s', Keyword, '#pop'),
+            (r'\S+', Name.Function),
+        ],
+    }