pygments.rb 2.3.1 → 2.4.0

data/vendor/pygments-main/pygments/lexers/shell.py

@@ -4,26 +4,23 @@
 
     Lexers for various shells.
 
-    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import re
 
 from pygments.lexer import Lexer, RegexLexer, do_insertions, bygroups, \
-    include, default, this, using, words
-from pygments.token import Punctuation, \
+    include, default, this, using, words, line_re
+from pygments.token import Punctuation, Whitespace, \
     Text, Comment, Operator, Keyword, Name, String, Number, Generic
 from pygments.util import shebang_matches
 
-
 __all__ = ['BashLexer', 'BashSessionLexer', 'TcshLexer', 'BatchLexer',
            'SlurmBashLexer', 'MSDOSSessionLexer', 'PowerShellLexer',
            'PowerShellSessionLexer', 'TcshSessionLexer', 'FishShellLexer',
            'ExeclineLexer']
 
-line_re = re.compile('.*?\n')
-
 
 class BashLexer(RegexLexer):
     """
@@ -37,6 +34,7 @@ class BashLexer(RegexLexer):
     filenames = ['*.sh', '*.ksh', '*.bash', '*.ebuild', '*.eclass',
                  '*.exheres-0', '*.exlib', '*.zsh',
                  '.bashrc', 'bashrc', '.bash_*', 'bash_*', 'zshrc', '.zshrc',
+                 '.kshrc', 'kshrc',
                  'PKGBUILD']
     mimetypes = ['application/x-sh', 'application/x-shellscript', 'text/x-shellscript']
 
@@ -57,9 +55,9 @@ class BashLexer(RegexLexer):
         ],
         'basic': [
             (r'\b(if|fi|else|while|in|do|done|for|then|return|function|case|'
-             r'select|continue|until|esac|elif)(\s*)\b',
-             bygroups(Keyword, Text)),
-            (r'\b(alias|bg|bind|break|builtin|caller|cd|command|compgen|'
+             r'select|break|continue|until|esac|elif)(\s*)\b',
+             bygroups(Keyword, Whitespace)),
+            (r'\b(alias|bg|bind|builtin|caller|cd|command|compgen|'
              r'complete|declare|dirs|disown|echo|enable|eval|exec|exit|'
              r'export|false|fc|fg|getopts|hash|help|history|jobs|kill|let|'
              r'local|logout|popd|printf|pushd|pwd|read|readonly|set|shift|'
@@ -69,7 +67,7 @@ class BashLexer(RegexLexer):
             (r'\A#!.+\n', Comment.Hashbang),
             (r'#.*\n', Comment.Single),
             (r'\\[\w\W]', String.Escape),
-            (r'(\b\w+)(\s*)(\+?=)', bygroups(Name.Variable, Text, Operator)),
+            (r'(\b\w+)(\s*)(\+?=)', bygroups(Name.Variable, Whitespace, Operator)),
             (r'[\[\]{}()=]', Operator),
             (r'<<<', Operator),  # here-string
             (r'<<-?\s*(\'?)\\?(\w+)[\w\W]+?\2', String),
@@ -83,7 +81,7 @@ class BashLexer(RegexLexer):
             (r';', Punctuation),
             (r'&', Punctuation),
             (r'\|', Punctuation),
-            (r'\s+', Text),
+            (r'\s+', Whitespace),
             (r'\d+\b', Number),
             (r'[^=\s\[\]{}()$"\'`\\<&|;]+', Text),
             (r'<', Text),
@@ -107,10 +105,12 @@ class BashLexer(RegexLexer):
         ],
         'math': [
             (r'\)\)', Keyword, '#pop'),
-            (r'[-+*/%^|&]|\*\*|\|\|', Operator),
-            (r'\d+#\d+', Number),
+            (r'\*\*|\|\||<<|>>|[-+*/%^|&<>]', Operator),
+            (r'\d+#[\da-zA-Z]+', Number),
             (r'\d+#(?! )', Number),
+            (r'0[xX][\da-fA-F]+', Number),
             (r'\d+', Number),
+            (r'[a-zA-Z_]\w*', Name.Variable),  # user variable
             include('root'),
         ],
         'backticks': [
@@ -156,6 +156,7 @@ class ShellSessionBaseLexer(Lexer):
     .. versionadded:: 2.1
     """
 
+    _bare_continuation = False
     _venv = re.compile(r'^(\([^)]*\))(\s*)')
 
     def get_tokens_unprocessed(self, text):
@@ -174,10 +175,10 @@ class ShellSessionBaseLexer(Lexer):
                 venv = venv_match.group(1)
                 venv_whitespace = venv_match.group(2)
                 insertions.append((len(curcode),
-                    [(0, Generic.Prompt.VirtualEnv, venv)]))
+                                   [(0, Generic.Prompt.VirtualEnv, venv)]))
                 if venv_whitespace:
                     insertions.append((len(curcode),
-                        [(0, Text, venv_whitespace)]))
+                                       [(0, Text, venv_whitespace)]))
                 line = line[venv_match.end():]
 
             m = self._ps1rgx.match(line)
@@ -195,11 +196,17 @@ class ShellSessionBaseLexer(Lexer):
             elif backslash_continuation:
                 if line.startswith(self._ps2):
                     insertions.append((len(curcode),
-                                    [(0, Generic.Prompt, line[:len(self._ps2)])]))
+                                       [(0, Generic.Prompt,
+                                         line[:len(self._ps2)])]))
                     curcode += line[len(self._ps2):]
                 else:
                     curcode += line
                 backslash_continuation = curcode.endswith('\\\n')
+            elif self._bare_continuation and line.startswith(self._ps2):
+                insertions.append((len(curcode),
+                                   [(0, Generic.Prompt,
+                                     line[:len(self._ps2)])]))
+                curcode += line[len(self._ps2):]
             else:
                 if insertions:
                     toks = innerlexer.get_tokens_unprocessed(curcode)
@@ -584,9 +591,9 @@ class TcshLexer(RegexLexer):
         ],
         'basic': [
             (r'\b(if|endif|else|while|then|foreach|case|default|'
-             r'continue|goto|breaksw|end|switch|endsw)\s*\b',
+             r'break|continue|goto|breaksw|end|switch|endsw)\s*\b',
              Keyword),
-            (r'\b(alias|alloc|bg|bindkey|break|builtins|bye|caller|cd|chdir|'
+            (r'\b(alias|alloc|bg|bindkey|builtins|bye|caller|cd|chdir|'
              r'complete|dirs|echo|echotc|eval|exec|exit|fg|filetest|getxvers|'
              r'glob|getspath|hashstat|history|hup|inlib|jobs|kill|'
              r'limit|log|login|logout|ls-F|migrate|newgrp|nice|nohup|notify|'
@@ -664,7 +671,7 @@ class PowerShellLexer(RegexLexer):
     keywords = (
         'while validateset validaterange validatepattern validatelength '
         'validatecount until trap switch return ref process param parameter in '
-        'if global: function foreach for finally filter end elseif else '
+        'if global: local: function foreach for finally filter end elseif else '
         'dynamicparam do default continue cmdletbinding break begin alias \\? '
         '% #script #private #local #global mandatory parametersetname position '
         'valuefrompipeline valuefrompipelinebypropertyname '
@@ -776,8 +783,9 @@ class PowerShellSessionLexer(ShellSessionBaseLexer):
     mimetypes = []
 
     _innerLexerCls = PowerShellLexer
+    _bare_continuation = True
     _ps1rgx = re.compile(r'^((?:\[[^]]+\]: )?PS[^>]*> ?)(.*\n?)')
-    _ps2 = '>> '
+    _ps2 = '> '
 
 
 class FishShellLexer(RegexLexer):
@@ -818,7 +826,7 @@ class FishShellLexer(RegexLexer):
              Name.Builtin),
             (r'#.*\n', Comment),
             (r'\\[\w\W]', String.Escape),
-            (r'(\b\w+)(\s*)(=)', bygroups(Name.Variable, Text, Operator)),
+            (r'(\b\w+)(\s*)(=)', bygroups(Name.Variable, Whitespace, Operator)),
             (r'[\[\]()=]', Operator),
             (r'<<-?\s*(\'?)\\?(\w+)[\w\W]+?\2', String),
         ],

data/vendor/pygments-main/pygments/lexers/sieve.py

@@ -13,12 +13,13 @@
     https://tools.ietf.org/html/rfc5429
     https://tools.ietf.org/html/rfc8580
 
-    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 from pygments.lexer import RegexLexer, bygroups
-from pygments.token import Comment, Name, Literal, String, Text, Punctuation, Keyword
+from pygments.token import Comment, Name, Literal, String, Text, Punctuation, \
+    Keyword
 
 __all__ = ["SieveLexer"]
 
@@ -26,6 +27,8 @@ __all__ = ["SieveLexer"]
 class SieveLexer(RegexLexer):
     """
     Lexer for sieve format.
+
+    .. versionadded:: 2.6
     """
     name = 'Sieve'
     filenames = ['*.siv', '*.sieve']
@@ -39,10 +42,17 @@ class SieveLexer(RegexLexer):
             (r'(?i)require',
              Keyword.Namespace),
             # tags:
-            (r'(?i)(:)(addresses|all|contains|content|create|copy|comparator|count|days|detail|domain|fcc|flags|from|handle|importance|is|localpart|length|lowerfirst|lower|matches|message|mime|options|over|percent|quotewildcard|raw|regex|specialuse|subject|text|under|upperfirst|upper|value)',
+            (r'(?i)(:)(addresses|all|contains|content|create|copy|comparator|'
+             r'count|days|detail|domain|fcc|flags|from|handle|importance|is|'
+             r'localpart|length|lowerfirst|lower|matches|message|mime|options|'
+             r'over|percent|quotewildcard|raw|regex|specialuse|subject|text|'
+             r'under|upperfirst|upper|value)',
              bygroups(Name.Tag, Name.Tag)),
             # tokens:
-            (r'(?i)(address|addflag|allof|anyof|body|discard|elsif|else|envelope|ereject|exists|false|fileinto|if|hasflag|header|keep|notify_method_capability|notify|not|redirect|reject|removeflag|setflag|size|spamtest|stop|string|true|vacation|virustest)',
+            (r'(?i)(address|addflag|allof|anyof|body|discard|elsif|else|envelope|'
+             r'ereject|exists|false|fileinto|if|hasflag|header|keep|'
+             r'notify_method_capability|notify|not|redirect|reject|removeflag|'
+             r'setflag|size|spamtest|stop|string|true|vacation|virustest)',
              Name.Builtin),
             (r'(?i)set',
              Keyword.Declaration),

data/vendor/pygments-main/pygments/lexers/slash.py

@@ -5,7 +5,7 @@
     Lexer for the `Slash <https://github.com/arturadib/Slash-A>`_ programming
     language.
 
-    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexers/smalltalk.py

@@ -4,7 +4,7 @@
 
     Lexers for Smalltalk and related languages.
 
-    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -17,13 +17,14 @@ __all__ = ['SmalltalkLexer', 'NewspeakLexer']
 
 class SmalltalkLexer(RegexLexer):
     """
-    For `Smalltalk <http://www.smalltalk.org/>`_ syntax.
+    For Smalltalk syntax.
     Contributed by Stefan Matthias Aust.
     Rewritten by Nils Winter.
 
     .. versionadded:: 0.10
     """
     name = 'Smalltalk'
+    url = 'http://www.smalltalk.org/'
     filenames = ['*.st']
     aliases = ['smalltalk', 'squeak', 'st']
     mimetypes = ['text/x-smalltalk']
@@ -138,11 +139,12 @@ class SmalltalkLexer(RegexLexer):
 
 class NewspeakLexer(RegexLexer):
     """
-    For `Newspeak <http://newspeaklanguage.org/>`_ syntax.
+    For Newspeak syntax.
 
     .. versionadded:: 1.1
     """
     name = 'Newspeak'
+    url = 'http://newspeaklanguage.org/'
     filenames = ['*.ns2']
     aliases = ['newspeak', ]
     mimetypes = ['text/x-newspeak']

data/vendor/pygments-main/pygments/lexers/smithy.py

@@ -4,12 +4,10 @@
 
     Lexers for the Smithy IDL.
 
-    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-import re
-
 from pygments.lexer import RegexLexer, bygroups, words
 from pygments.token import Text, Comment, Keyword, Name, String, \
     Number, Whitespace, Punctuation
@@ -24,10 +22,10 @@ class SmithyLexer(RegexLexer):
     .. versionadded:: 2.10
     """
     name = 'Smithy'
+    url = 'https://awslabs.github.io/smithy/'
     filenames = ['*.smithy']
     aliases = ['smithy']
 
-    flags = re.MULTILINE | re.UNICODE
     unquoted = r'[A-Za-z0-9_\.#$-]+'
     identifier = r"[A-Za-z0-9_\.#$-]+"
 
@@ -58,8 +56,9 @@ class SmithyLexer(RegexLexer):
             (words(aggregate_shapes,
                    prefix=r'^', suffix=r'(\s+' + identifier + r')'),
                 bygroups(Keyword.Declaration, Name.Class)),
-            (r'^(metadata)(\s+.+)(\s*)(=)',
-                bygroups(Keyword.Declaration, Name.Class, Whitespace, Name.Decorator)),
+            (r'^(metadata)(\s+)((?:\S+)|(?:\"[^"]+\"))(\s*)(=)',
+                bygroups(Keyword.Declaration, Whitespace, Name.Class,
+                         Whitespace, Name.Decorator)),
             (r"(true|false|null)", Keyword.Constant),
             (r"(-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?)", Number),
             (identifier + ":", Name.Label),

data/vendor/pygments-main/pygments/lexers/smv.py

@@ -4,7 +4,7 @@
 
     Lexers for the SMV languages.
 
-    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexers/snobol.py

@@ -4,7 +4,7 @@
 
     Lexers for the SNOBOL language.
 
-    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexers/solidity.py

@@ -4,12 +4,10 @@
 
     Lexers for Solidity.
 
-    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
-import re
-
 from pygments.lexer import RegexLexer, bygroups, include, words
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
     Number, Punctuation, Whitespace
@@ -29,8 +27,6 @@ class SolidityLexer(RegexLexer):
     filenames = ['*.sol']
     mimetypes = []
 
-    flags = re.MULTILINE | re.UNICODE
-
     datatype = (
         r'\b(address|bool|(?:(?:bytes|hash|int|string|uint)(?:8|16|24|32|40|48|56|64'
         r'|72|80|88|96|104|112|120|128|136|144|152|160|168|176|184|192|200|208'
@@ -63,8 +59,8 @@ class SolidityLexer(RegexLexer):
             (datatype, Keyword.Type),
             include('constants'),
             (r'[a-zA-Z_]\w*', Text),
-            (r'[!<=>+*/-]', Operator),
-            (r'[.;:{}(),\[\]]', Punctuation)
+            (r'[~!%^&*+=|?:<>/-]', Operator),
+            (r'[.;{}(),\[\]]', Punctuation)
         ],
         'comments': [
             (r'//(\n|[\w\W]*?[^\\]\n)', Comment.Single),

data/vendor/pygments-main/pygments/lexers/sophia.py

@@ -0,0 +1,103 @@
+"""
+    pygments.lexers.sophia
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexer for Sophia.
+
+    Derived from pygments/lexers/reason.py.
+
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, include, default, words
+from pygments.token import Comment, Keyword, Name, Number, Operator, \
+    Punctuation, String, Text
+
+__all__ = ['SophiaLexer']
+
+class SophiaLexer(RegexLexer):
+    """
+    A Sophia lexer.
+
+    .. versionadded:: 2.11
+    """
+
+    name = 'Sophia'
+    aliases = ['sophia']
+    filenames = ['*.aes']
+    mimetypes = []
+
+    keywords = (
+        'contract', 'include', 'let', 'switch', 'type', 'record', 'datatype',
+        'if', 'elif', 'else', 'function', 'stateful', 'payable', 'public',
+        'entrypoint', 'private', 'indexed', 'namespace', 'interface', 'main',
+        'using', 'as', 'for', 'hiding',
+    )
+
+    builtins = ('state', 'put', 'abort', 'require')
+
+    word_operators = ('mod', 'band', 'bor', 'bxor', 'bnot')
+
+    primitive_types = ('int', 'address', 'bool', 'bits', 'bytes', 'string',
+                       'list', 'option', 'char', 'unit', 'map', 'event',
+                       'hash', 'signature', 'oracle', 'oracle_query')
+
+    tokens = {
+        'escape-sequence': [
+            (r'\\[\\"\'ntbr]', String.Escape),
+            (r'\\[0-9]{3}', String.Escape),
+            (r'\\x[0-9a-fA-F]{2}', String.Escape),
+        ],
+        'root': [
+            (r'\s+', Text.Whitespace),
+            (r'(true|false)\b', Keyword.Constant),
+            (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Class, 'dotted'),
+            (r'\b([A-Z][\w\']*)', Name.Function),
+            (r'//.*?\n', Comment.Single),
+            (r'\/\*(?!/)', Comment.Multiline, 'comment'),
+
+            (r'0[xX][\da-fA-F][\da-fA-F_]*', Number.Hex),
+            (r'#[\da-fA-F][\da-fA-F_]*', Name.Label),
+            (r'\d[\d_]*', Number.Integer),
+
+            (words(keywords, suffix=r'\b'), Keyword),
+            (words(builtins, suffix=r'\b'), Name.Builtin),
+            (words(word_operators, prefix=r'\b', suffix=r'\b'), Operator.Word),
+            (words(primitive_types, prefix=r'\b', suffix=r'\b'), Keyword.Type),
+
+            (r'[=!<>+\\*/:&|?~@^-]', Operator.Word),
+            (r'[.;:{}(),\[\]]', Punctuation),
+
+            (r"(ak_|ok_|oq_|ct_)[\w']*", Name.Label),
+            (r"[^\W\d][\w']*", Name),
+
+            (r"'(?:(\\[\\\"'ntbr ])|(\\[0-9]{3})|(\\x[0-9a-fA-F]{2}))'",
+             String.Char),
+            (r"'.'", String.Char),
+            (r"'[a-z][\w]*", Name.Variable),
+
+            (r'"', String.Double, 'string')
+        ],
+        'comment': [
+            (r'[^/*]+', Comment.Multiline),
+            (r'\/\*', Comment.Multiline, '#push'),
+            (r'\*\/', Comment.Multiline, '#pop'),
+            (r'\*', Comment.Multiline),
+        ],
+        'string': [
+            (r'[^\\"]+', String.Double),
+            include('escape-sequence'),
+            (r'\\\n', String.Double),
+            (r'"', String.Double, '#pop'),
+        ],
+        'dotted': [
+            (r'\s+', Text),
+            (r'\.', Punctuation),
+            (r'[A-Z][\w\']*(?=\s*\.)', Name.Function),
+            (r'[A-Z][\w\']*', Name.Function, '#pop'),
+            (r'[a-z_][\w\']*', Name, '#pop'),
+            default('#pop'),
+        ],
+    }
+

data/vendor/pygments-main/pygments/lexers/special.py

@@ -4,14 +4,13 @@
 
     Special lexers.
 
-    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
 import ast
-import re
 
-from pygments.lexer import Lexer
+from pygments.lexer import Lexer, line_re
 from pygments.token import Token, Error, Text, Generic
 from pygments.util import get_choice_opt
 
@@ -51,8 +50,6 @@ class OutputLexer(Lexer):
 
 _ttype_cache = {}
 
-line_re = re.compile('.*?\n')
-
 
 class RawTokenLexer(Lexer):
     """

data/vendor/pygments-main/pygments/lexers/spice.py

@@ -0,0 +1,71 @@
+"""
+    pygments.lexers.spice
+    ~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Spice programming language.
+
+    :copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+    Number, Punctuation, Whitespace
+
+__all__ = ['SpiceLexer']
+
+
+class SpiceLexer(RegexLexer):
+    """
+    For Spice source.
+
+    .. versionadded:: 2.11
+    """
+    name = 'Spice'
+    url = 'https://www.spicelang.com'
+    filenames = ['*.spice']
+    aliases = ['spice', 'spicelang']
+    mimetypes = ['text/x-spice']
+
+    tokens = {
+        'root': [
+            (r'\n', Whitespace),
+            (r'\s+', Whitespace),
+            (r'\\\n', Text),
+            # comments
+            (r'//(.*?)\n', Comment.Single),
+            (r'/(\\\n)?[*]{2}(.|\n)*?[*](\\\n)?/', String.Doc),
+            (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline),
+            # keywords
+            (r'(import|as)\b', Keyword.Namespace),
+            (r'(f|p|type|struct|interface|enum|alias|operator)\b', Keyword.Declaration),
+            (words(('if', 'else', 'for', 'foreach', 'do', 'while', 'break',
+                    'continue', 'return', 'assert', 'thread', 'unsafe', 'ext',
+                    'dll'), suffix=r'\b'), Keyword),
+            (words(('const', 'signed', 'unsigned', 'inline', 'public', 'heap'),
+                   suffix=r'\b'), Keyword.Pseudo),
+            (words(('new', 'switch', 'case', 'yield', 'stash', 'pick', 'sync',
+                    'class'), suffix=r'\b'), Keyword.Reserved),
+            (r'(true|false|nil)\b', Keyword.Constant),
+            (words(('double', 'int', 'short', 'long', 'byte', 'char', 'string',
+                    'bool', 'dyn'), suffix=r'\b'), Keyword.Type),
+            (words(('printf', 'sizeof', 'len', 'tid', 'join'), suffix=r'\b(\()'),
+             bygroups(Name.Builtin, Punctuation)),
+            # numeric literals
+            (r'[0-9]*[.][0-9]+', Number.Double),
+            (r'0[bB][01]+[sl]?', Number.Bin),
+            (r'0[oO][0-7]+[sl]?', Number.Oct),
+            (r'0[xXhH][0-9a-fA-F]+[sl]?', Number.Hex),
+            (r'(0[dD])?[0-9]+[sl]?', Number.Integer),
+            # string literal
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
+            # char literal
+            (r'\'(\\\\|\\[^\\]|[^\'\\])\'', String.Char),
+            # tokens
+            (r'<<=|>>=|<<|>>|<=|>=|\+=|-=|\*=|/=|\%=|\|=|&=|\^=|&&|\|\||&|\||'
+             r'\+\+|--|\%|\^|\~|==|!=|::|[.]{3}|[+\-*/&]', Operator),
+            (r'[|<>=!()\[\]{}.,;:\?]', Punctuation),
+            # identifiers
+            (r'[^\W\d]\w*', Name.Other),
+        ]
+    }