pygments.rb 2.0.0.rc3 → 2.3.0

data/vendor/pygments-main/pygments/lexers/rust.py

@@ -1,11 +1,10 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexers.rust
     ~~~~~~~~~~~~~~~~~~~~
 
     Lexers for the Rust language.
 
-    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -110,7 +109,7 @@ class RustLexer(RegexLexer):
             # Types in positions.
             (r'(?::|->)', Text, 'typename'),
             # Labels
-            (r'(break|continue)(\s*)(\'[A-Za-z_]\w*)?',
+            (r'(break|continue)(\b\s*)(\'[A-Za-z_]\w*)?',
              bygroups(Keyword, Text.Whitespace, Name.Label)),
 
             # Character literals
@@ -136,7 +135,7 @@ class RustLexer(RegexLexer):
             # String literals
             (r'b"', String, 'bytestring'),
             (r'"', String, 'string'),
-            (r'b?r(#*)".*?"\1', String),
+            (r'(?s)b?r(#*)".*?"\1', String),
 
             # Lifetime names
             (r"'", Operator, 'lifetime'),
@@ -153,6 +152,11 @@ class RustLexer(RegexLexer):
 
             # Attributes
             (r'#!?\[', Comment.Preproc, 'attribute['),
+
+            # Misc
+            # Lone hashes: not used in Rust syntax, but allowed in macro
+            # arguments, most famously for quote::quote!()
+            (r'#', Text),
         ],
         'comment': [
             (r'[^*/]+', Comment.Multiline),
@@ -209,16 +213,10 @@ class RustLexer(RegexLexer):
         'attribute_common': [
             (r'"', String, 'string'),
             (r'\[', Comment.Preproc, 'attribute['),
-            (r'\(', Comment.Preproc, 'attribute('),
         ],
         'attribute[': [
             include('attribute_common'),
-            (r'\];?', Comment.Preproc, '#pop'),
-            (r'[^"\]]+', Comment.Preproc),
-        ],
-        'attribute(': [
-            include('attribute_common'),
-            (r'\);?', Comment.Preproc, '#pop'),
-            (r'[^")]+', Comment.Preproc),
+            (r'\]', Comment.Preproc, '#pop'),
+            (r'[^"\]\[]+', Comment.Preproc),
         ],
     }

data/vendor/pygments-main/pygments/lexers/sas.py

@@ -1,11 +1,10 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexers.sas
     ~~~~~~~~~~~~~~~~~~~
 
     Lexer for SAS.
 
-    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexers/scdoc.py

@@ -1,11 +1,10 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexers.scdoc
     ~~~~~~~~~~~~~~~~~~~~~
 
     Lexer for scdoc, a simple man page generator.
 
-    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexers/scripting.py

@@ -1,11 +1,10 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexers.scripting
     ~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Lexer for scripting and embedded languages.
 
-    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -169,9 +168,9 @@ class MoonScriptLexer(LuaLexer):
     .. versionadded:: 1.5
     """
 
-    name = "MoonScript"
-    aliases = ["moon", "moonscript"]
-    filenames = ["*.moon"]
+    name = 'MoonScript'
+    aliases = ['moonscript', 'moon']
+    filenames = ['*.moon']
     mimetypes = ['text/x-moonscript', 'application/x-moonscript']
 
     tokens = {
@@ -238,7 +237,7 @@ class ChaiscriptLexer(RegexLexer):
     """
 
     name = 'ChaiScript'
-    aliases = ['chai', 'chaiscript']
+    aliases = ['chaiscript', 'chai']
     filenames = ['*.chai']
     mimetypes = ['text/x-chaiscript', 'application/x-chaiscript']
 
@@ -283,7 +282,7 @@ class ChaiscriptLexer(RegexLexer):
             (r'0x[0-9a-fA-F]+', Number.Hex),
             (r'[0-9]+', Number.Integer),
             (r'"', String.Double, 'dqstring'),
-            (r"'(\\\\|\\'|[^'])*'", String.Single),
+            (r"'(\\\\|\\[^\\]|[^'\\])*'", String.Single),
         ],
         'dqstring': [
             (r'\$\{[^"}]+?\}', String.Interpol),
@@ -689,7 +688,7 @@ class AppleScriptLexer(RegexLexer):
             (r'\b(%s)s?\b' % '|'.join(StudioClasses), Name.Builtin),
             (r'\b(%s)\b' % '|'.join(StudioCommands), Name.Builtin),
             (r'\b(%s)\b' % '|'.join(References), Name.Builtin),
-            (r'"(\\\\|\\"|[^"])*"', String.Double),
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String.Double),
             (r'\b(%s)\b' % Identifiers, Name.Variable),
             (r'[-+]?(\d+\.\d*|\d*\.\d+)(E[-+][0-9]+)?', Number.Float),
             (r'[-+]?\d+', Number.Integer),
@@ -833,7 +832,7 @@ class MOOCodeLexer(RegexLexer):
             # Numbers
             (r'(0|[1-9][0-9_]*)', Number.Integer),
             # Strings
-            (r'"(\\\\|\\"|[^"])*"', String),
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
             # exceptions
             (r'(E_PERM|E_DIV)', Name.Exception),
             # db-refs
@@ -924,7 +923,7 @@ class HybrisLexer(RegexLexer):
                 'Runnable', 'CGI', 'ClientSocket', 'Socket', 'ServerSocket',
                 'File', 'Console', 'Directory', 'Exception'), suffix=r'\b'),
              Keyword.Type),
-            (r'"(\\\\|\\"|[^"])*"', String),
+            (r'"(\\\\|\\[^\\]|[^"\\])*"', String),
             (r"'\\.'|'[^\\]'|'\\u[0-9a-f]{4}'", String.Char),
             (r'(\.)([a-zA-Z_]\w*)',
              bygroups(Operator, Name.Attribute)),
@@ -1237,9 +1236,9 @@ class MiniScriptLexer(RegexLexer):
     .. versionadded:: 2.6
     """
 
-    name = "MiniScript"
-    aliases = ["ms", "miniscript"]
-    filenames = ["*.ms"]
+    name = 'MiniScript'
+    aliases = ['miniscript', 'ms']
+    filenames = ['*.ms']
     mimetypes = ['text/x-minicript', 'application/x-miniscript']
 
     tokens = {

data/vendor/pygments-main/pygments/lexers/sgf.py

@@ -1,11 +1,10 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexers.sgf
     ~~~~~~~~~~~~~~~~~~~
 
     Lexer for Smart Game Format (sgf) file format.
 
-    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexers/shell.py

@@ -1,11 +1,10 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexers.shell
     ~~~~~~~~~~~~~~~~~~~~~
 
     Lexers for various shells.
 
-    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -57,7 +56,7 @@ class BashLexer(RegexLexer):
             (r'\$', Text),
         ],
         'basic': [
-            (r'\b(if|fi|else|while|do|done|for|then|return|function|case|'
+            (r'\b(if|fi|else|while|in|do|done|for|then|return|function|case|'
              r'select|continue|until|esac|elif)(\s*)\b',
              bygroups(Keyword, Text)),
             (r'\b(alias|bg|bind|break|builtin|caller|cd|command|compgen|'
@@ -169,10 +168,6 @@ class ShellSessionBaseLexer(Lexer):
 
         for match in line_re.finditer(text):
             line = match.group()
-            if backslash_continuation:
-                curcode += line
-                backslash_continuation = curcode.endswith('\\\n')
-                continue
 
             venv_match = self._venv.match(line)
             if venv_match:
@@ -197,10 +192,13 @@ class ShellSessionBaseLexer(Lexer):
                                    [(0, Generic.Prompt, m.group(1))]))
                 curcode += m.group(2)
                 backslash_continuation = curcode.endswith('\\\n')
-            elif line.startswith(self._ps2):
-                insertions.append((len(curcode),
-                                   [(0, Generic.Prompt, line[:len(self._ps2)])]))
-                curcode += line[len(self._ps2):]
+            elif backslash_continuation:
+                if line.startswith(self._ps2):
+                    insertions.append((len(curcode),
+                                    [(0, Generic.Prompt, line[:len(self._ps2)])]))
+                    curcode += line[len(self._ps2):]
+                else:
+                    curcode += line
                 backslash_continuation = curcode.endswith('\\\n')
             else:
                 if insertions:
@@ -232,8 +230,8 @@ class BashSessionLexer(ShellSessionBaseLexer):
     _innerLexerCls = BashLexer
     _ps1rgx = re.compile(
         r'^((?:(?:\[.*?\])|(?:\(\S+\))?(?:| |sh\S*?|\w+\S+[@:]\S+(?:\s+\S+)' \
-        r'?|\[\S+[@:][^\n]+\].+))\s*[$#%])(.*\n?)')
-    _ps2 = '>'
+        r'?|\[\S+[@:][^\n]+\].+))\s*[$#%]\s*)(.*\n?)')
+    _ps2 = '> '
 
 
 class BatchLexer(RegexLexer):
@@ -243,7 +241,7 @@ class BatchLexer(RegexLexer):
     .. versionadded:: 0.7
     """
     name = 'Batchfile'
-    aliases = ['bat', 'batch', 'dosbatch', 'winbatch']
+    aliases = ['batch', 'bat', 'dosbatch', 'winbatch']
     filenames = ['*.bat', '*.cmd']
     mimetypes = ['application/x-dos-batch']
 
@@ -657,7 +655,7 @@ class PowerShellLexer(RegexLexer):
     .. versionadded:: 1.5
     """
     name = 'PowerShell'
-    aliases = ['powershell', 'posh', 'ps1', 'psm1']
+    aliases = ['powershell', 'pwsh', 'posh', 'ps1', 'psm1']
     filenames = ['*.ps1', '*.psm1']
     mimetypes = ['text/x-powershell']
 
@@ -735,7 +733,7 @@ class PowerShellLexer(RegexLexer):
             (r'\[[a-z_\[][\w. `,\[\]]*\]', Name.Constant),  # .net [type]s
             (r'-[a-z_]\w*', Name),
             (r'\w+', Name),
-            (r'[.,;@{}\[\]$()=+*/\\&%!~?^`|<>-]|::', Punctuation),
+            (r'[.,;:@{}\[\]$()=+*/\\&%!~?^`|<>-]', Punctuation),
         ],
         'child': [
             (r'\)', Punctuation, '#pop'),
@@ -773,7 +771,7 @@ class PowerShellSessionLexer(ShellSessionBaseLexer):
     """
 
     name = 'PowerShell Session'
-    aliases = ['ps1con']
+    aliases = ['pwsh-session', 'ps1con']
     filenames = []
     mimetypes = []
 

data/vendor/pygments-main/pygments/lexers/sieve.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexers.sieve
     ~~~~~~~~~~~~~~~~~~~~~
@@ -14,7 +13,7 @@
     https://tools.ietf.org/html/rfc5429
     https://tools.ietf.org/html/rfc8580
 
-    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexers/slash.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexers.slash
     ~~~~~~~~~~~~~~~~~~~~~
@@ -6,7 +5,7 @@
     Lexer for the `Slash <https://github.com/arturadib/Slash-A>`_ programming
     language.
 
-    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexers/smalltalk.py

@@ -1,11 +1,10 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexers.smalltalk
     ~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Lexers for Smalltalk and related languages.
 
-    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexers/smithy.py

@@ -0,0 +1,79 @@
+"""
+    pygments.lexers.smithy
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Lexers for the Smithy IDL.
+
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
+    :license: BSD, see LICENSE for details.
+"""
+
+import re
+
+from pygments.lexer import RegexLexer, bygroups, words
+from pygments.token import Text, Comment, Keyword, Name, String, \
+    Number, Whitespace, Punctuation
+
+__all__ = ['SmithyLexer']
+
+
+class SmithyLexer(RegexLexer):
+    """
+    For Smithy IDL
+
+    .. versionadded:: 2.10
+    """
+    name = 'Smithy'
+    filenames = ['*.smithy']
+    aliases = ['smithy']
+
+    flags = re.MULTILINE | re.UNICODE
+    unquoted = r'[A-Za-z0-9_\.#$-]+'
+    identifier = r"[A-Za-z0-9_\.#$-]+"
+
+    simple_shapes = (
+        'use', 'byte', 'short', 'integer', 'long', 'float', 'document',
+        'double', 'bigInteger', 'bigDecimal', 'boolean', 'blob', 'string',
+        'timestamp',
+    )
+
+    aggregate_shapes = (
+       'apply', 'list', 'map', 'set', 'structure', 'union', 'resource',
+       'operation', 'service', 'trait'
+    )
+
+    tokens = {
+        'root': [
+            (r'///.*$', Comment.Multiline),
+            (r'//.*$', Comment),
+            (r'@[0-9a-zA-Z\.#-]*', Name.Decorator),
+            (r'(=)', Name.Decorator),
+            (r'^(\$version)(:)(.+)',
+                bygroups(Keyword.Declaration, Name.Decorator, Name.Class)),
+            (r'^(namespace)(\s+' + identifier + r')\b',
+                bygroups(Keyword.Declaration, Name.Class)),
+            (words(simple_shapes,
+                   prefix=r'^', suffix=r'(\s+' + identifier + r')\b'),
+                bygroups(Keyword.Declaration, Name.Class)),
+            (words(aggregate_shapes,
+                   prefix=r'^', suffix=r'(\s+' + identifier + r')'),
+                bygroups(Keyword.Declaration, Name.Class)),
+            (r'^(metadata)(\s+.+)(\s*)(=)',
+                bygroups(Keyword.Declaration, Name.Class, Whitespace, Name.Decorator)),
+            (r"(true|false|null)", Keyword.Constant),
+            (r"(-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?)", Number),
+            (identifier + ":", Name.Label),
+            (identifier, Name.Variable.Class),
+            (r'\[', Text, "#push"),
+            (r'\]', Text, "#pop"),
+            (r'\(', Text, "#push"),
+            (r'\)', Text, "#pop"),
+            (r'\{', Text, "#push"),
+            (r'\}', Text, "#pop"),
+            (r'"{3}(\\\\|\n|\\")*"{3}', String.Doc),
+            (r'"(\\\\|\n|\\"|[^"])*"', String.Double),
+            (r"'(\\\\|\n|\\'|[^'])*'", String.Single),
+            (r'[:,]+', Punctuation),
+            (r'\s+', Whitespace),
+        ]
+    }

data/vendor/pygments-main/pygments/lexers/smv.py

@@ -1,11 +1,10 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexers.smv
     ~~~~~~~~~~~~~~~~~~~
 
     Lexers for the SMV languages.
 
-    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -69,7 +68,7 @@ class NuSMVLexer(RegexLexer):
             (r'\-?\d+\b', Number.Integer),
             (r'0[su][bB]\d*_[01_]+', Number.Bin),
             (r'0[su][oO]\d*_[0-7_]+', Number.Oct),
-            (r'0[su][dD]\d*_[\d_]+', Number.Dec),
+            (r'0[su][dD]\d*_[\d_]+', Number.Decimal),
             (r'0[su][hH]\d*_[\da-fA-F_]+', Number.Hex),
 
             # Whitespace, punctuation and the rest

data/vendor/pygments-main/pygments/lexers/snobol.py

@@ -1,11 +1,10 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexers.snobol
     ~~~~~~~~~~~~~~~~~~~~~~
 
     Lexers for the SNOBOL language.
 
-    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexers/solidity.py

@@ -1,11 +1,10 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexers.solidity
     ~~~~~~~~~~~~~~~~~~~~~~~~
 
     Lexers for Solidity.
 
-    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 

data/vendor/pygments-main/pygments/lexers/special.py

@@ -1,23 +1,22 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexers.special
     ~~~~~~~~~~~~~~~~~~~~~~~
 
     Special lexers.
 
-    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
+import ast
 import re
-from io import BytesIO
 
 from pygments.lexer import Lexer
-from pygments.token import Token, Error, Text
+from pygments.token import Token, Error, Text, Generic
 from pygments.util import get_choice_opt
 
 
-__all__ = ['TextLexer', 'RawTokenLexer']
+__all__ = ['TextLexer', 'OutputLexer', 'RawTokenLexer']
 
 
 class TextLexer(Lexer):
@@ -37,16 +36,27 @@ class TextLexer(Lexer):
         return TextLexer.priority
 
 
+class OutputLexer(Lexer):
+    """
+    Simple lexer that highlights everything as ``Token.Generic.Output``.
+
+    .. versionadded:: 2.10
+    """
+    name = 'Text output'
+    aliases = ['output']
+
+    def get_tokens_unprocessed(self, text):
+        yield 0, Generic.Output, text
+
+
 _ttype_cache = {}
 
-line_re = re.compile(b'.*?\n')
+line_re = re.compile('.*?\n')
 
 
 class RawTokenLexer(Lexer):
     """
-    Recreate a token stream formatted with the `RawTokenFormatter`.  This
-    lexer raises exceptions during parsing if the token stream in the
-    file is malformed.
+    Recreate a token stream formatted with the `RawTokenFormatter`.
 
     Additional options accepted:
 
@@ -55,7 +65,7 @@ class RawTokenLexer(Lexer):
         the given compression algorithm before lexing (default: ``""``).
     """
     name = 'Raw token data'
-    aliases = ['raw']
+    aliases = []
     filenames = []
     mimetypes = ['application/x-pygments-tokens']
 
@@ -65,20 +75,23 @@ class RawTokenLexer(Lexer):
         Lexer.__init__(self, **options)
 
     def get_tokens(self, text):
-        if isinstance(text, str):
-            # raw token stream never has any non-ASCII characters
-            text = text.encode('ascii')
-        if self.compress == 'gz':
-            import gzip
-            gzipfile = gzip.GzipFile('', 'rb', 9, BytesIO(text))
-            text = gzipfile.read()
-        elif self.compress == 'bz2':
-            import bz2
-            text = bz2.decompress(text)
-
-        # do not call Lexer.get_tokens() because we do not want Unicode
-        # decoding to occur, and stripping is not optional.
-        text = text.strip(b'\n') + b'\n'
+        if self.compress:
+            if isinstance(text, str):
+                text = text.encode('latin1')
+            try:
+                if self.compress == 'gz':
+                    import gzip
+                    text = gzip.decompress(text)
+                elif self.compress == 'bz2':
+                    import bz2
+                    text = bz2.decompress(text)
+            except OSError:
+                yield Error, text.decode('latin1')
+        if isinstance(text, bytes):
+            text = text.decode('latin1')
+
+        # do not call Lexer.get_tokens() because stripping is not optional.
+        text = text.strip('\n') + '\n'
         for i, t, v in self.get_tokens_unprocessed(text):
             yield t, v
 
@@ -86,11 +99,7 @@ class RawTokenLexer(Lexer):
         length = 0
         for match in line_re.finditer(text):
             try:
-                ttypestr, val = match.group().split(b'\t', 1)
-            except ValueError:
-                val = match.group().decode('ascii', 'replace')
-                ttype = Error
-            else:
+                ttypestr, val = match.group().rstrip().split('\t', 1)
                 ttype = _ttype_cache.get(ttypestr)
                 if not ttype:
                     ttype = Token
@@ -100,6 +109,11 @@ class RawTokenLexer(Lexer):
                             raise ValueError('malformed token name')
                         ttype = getattr(ttype, ttype_)
                     _ttype_cache[ttypestr] = ttype
-                val = val[2:-2].decode('unicode-escape')
+                val = ast.literal_eval(val)
+                if not isinstance(val, str):
+                    raise ValueError('expected str')
+            except (SyntaxError, ValueError):
+                val = match.group()
+                ttype = Error
             yield length, ttype, val
             length += len(val)

data/vendor/pygments-main/pygments/lexers/sql.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """
     pygments.lexers.sql
     ~~~~~~~~~~~~~~~~~~~
@@ -34,7 +33,7 @@
     The ``tests/examplefiles`` contains a few test files with data to be
     parsed by these lexers.
 
-    :copyright: Copyright 2006-2020 by the Pygments team, see AUTHORS.
+    :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS.
     :license: BSD, see LICENSE for details.
 """
 
@@ -102,8 +101,7 @@ def language_callback(lexer, match):
     yield (match.start(3), String, match.group(3))
     # 4 = string contents
     if lx:
-        for x in lx.get_tokens_unprocessed(match.group(4)):
-            yield x
+        yield from lx.get_tokens_unprocessed(match.group(4))
     else:
         yield (match.start(4), String, match.group(4))
     # 5 = $, 6 = delimiter, 7 = $
@@ -397,14 +395,14 @@ class SqlLexer(RegexLexer):
                 'CHARACTERISTICS', 'CHARACTER_LENGTH', 'CHARACTER_SET_CATALOG',
                 'CHARACTER_SET_NAME', 'CHARACTER_SET_SCHEMA', 'CHAR_LENGTH', 'CHECK',
                 'CHECKED', 'CHECKPOINT', 'CLASS', 'CLASS_ORIGIN', 'CLOB', 'CLOSE',
-                'CLUSTER', 'COALSECE', 'COBOL', 'COLLATE', 'COLLATION',
+                'CLUSTER', 'COALESCE', 'COBOL', 'COLLATE', 'COLLATION',
                 'COLLATION_CATALOG', 'COLLATION_NAME', 'COLLATION_SCHEMA', 'COLUMN',
                 'COLUMN_NAME', 'COMMAND_FUNCTION', 'COMMAND_FUNCTION_CODE', 'COMMENT',
                 'COMMIT', 'COMMITTED', 'COMPLETION', 'CONDITION_NUMBER', 'CONNECT',
                 'CONNECTION', 'CONNECTION_NAME', 'CONSTRAINT', 'CONSTRAINTS',
                 'CONSTRAINT_CATALOG', 'CONSTRAINT_NAME', 'CONSTRAINT_SCHEMA',
                 'CONSTRUCTOR', 'CONTAINS', 'CONTINUE', 'CONVERSION', 'CONVERT',
-                'COPY', 'CORRESPONTING', 'COUNT', 'CREATE', 'CREATEDB', 'CREATEUSER',
+                'COPY', 'CORRESPONDING', 'COUNT', 'CREATE', 'CREATEDB', 'CREATEUSER',
                 'CROSS', 'CUBE', 'CURRENT', 'CURRENT_DATE', 'CURRENT_PATH',
                 'CURRENT_ROLE', 'CURRENT_TIME', 'CURRENT_TIMESTAMP', 'CURRENT_USER',
                 'CURSOR', 'CURSOR_NAME', 'CYCLE', 'DATA', 'DATABASE',
@@ -440,9 +438,9 @@ class SqlLexer(RegexLexer):
                 'OPEN', 'OPERATION', 'OPERATOR', 'OPTION', 'OPTIONS', 'OR', 'ORDER',
                 'ORDINALITY', 'OUT', 'OUTER', 'OUTPUT', 'OVERLAPS', 'OVERLAY',
                 'OVERRIDING', 'OWNER', 'PAD', 'PARAMETER', 'PARAMETERS', 'PARAMETER_MODE',
-                'PARAMATER_NAME', 'PARAMATER_ORDINAL_POSITION',
+                'PARAMETER_NAME', 'PARAMETER_ORDINAL_POSITION',
                 'PARAMETER_SPECIFIC_CATALOG', 'PARAMETER_SPECIFIC_NAME',
-                'PARAMATER_SPECIFIC_SCHEMA', 'PARTIAL', 'PASCAL', 'PENDANT', 'PERIOD', 'PLACING',
+                'PARAMETER_SPECIFIC_SCHEMA', 'PARTIAL', 'PASCAL', 'PENDANT', 'PERIOD', 'PLACING',
                 'PLI', 'POSITION', 'POSTFIX', 'PRECEEDS', 'PRECISION', 'PREFIX', 'PREORDER',
                 'PREPARE', 'PRESERVE', 'PRIMARY', 'PRIOR', 'PRIVILEGES', 'PROCEDURAL',
                 'PROCEDURE', 'PUBLIC', 'READ', 'READS', 'RECHECK', 'RECURSIVE', 'REF',
@@ -462,8 +460,8 @@ class SqlLexer(RegexLexer):
                 'SUBLIST', 'SUBSTRING', 'SUCCEEDS', 'SUM', 'SYMMETRIC', 'SYSID', 'SYSTEM',
                 'SYSTEM_USER', 'TABLE', 'TABLE_NAME', ' TEMP', 'TEMPLATE', 'TEMPORARY',
                 'TERMINATE', 'THAN', 'THEN', 'TIME', 'TIMESTAMP', 'TIMEZONE_HOUR',
-                'TIMEZONE_MINUTE', 'TO', 'TOAST', 'TRAILING', 'TRANSATION',
-                'TRANSACTIONS_COMMITTED', 'TRANSACTIONS_ROLLED_BACK', 'TRANSATION_ACTIVE',
+                'TIMEZONE_MINUTE', 'TO', 'TOAST', 'TRAILING', 'TRANSACTION',
+                'TRANSACTIONS_COMMITTED', 'TRANSACTIONS_ROLLED_BACK', 'TRANSACTION_ACTIVE',
                 'TRANSFORM', 'TRANSFORMS', 'TRANSLATE', 'TRANSLATION', 'TREAT', 'TRIGGER',
                 'TRIGGER_CATALOG', 'TRIGGER_NAME', 'TRIGGER_SCHEMA', 'TRIM', 'TRUE',
                 'TRUNCATE', 'TRUSTED', 'TYPE', 'UNCOMMITTED', 'UNDER', 'UNENCRYPTED',
@@ -497,9 +495,6 @@ class SqlLexer(RegexLexer):
         ]
     }
 
-    def analyse_text(text):
-        return 0.01
-
 
 class TransactSqlLexer(RegexLexer):
     """