@scrypted/arlo 0.11.26 → 0.11.28

package/venv/lib/python3.10/site-packages/bs4/dammit.py

@@ -1,1408 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Beautiful Soup bonus library: Unicode, Dammit
-
-This library converts a bytestream to Unicode through any means
-necessary. It is heavily based on code from Mark Pilgrim's `Universal
-Feed Parser <https://pypi.org/project/feedparser/>`_, now maintained
-by Kurt McKee. It does not rewrite the body of an XML or HTML document
-to reflect a new encoding; that's the job of `TreeBuilder`.
-
-"""
-
-# Use of this source code is governed by the MIT license.
-__license__ = "MIT"
-
-from html.entities import codepoint2name
-from collections import defaultdict
-import codecs
-from html.entities import html5
-import re
-from logging import Logger, getLogger
-from types import ModuleType
-from typing import (
-    Dict,
-    Iterator,
-    List,
-    Optional,
-    Pattern,
-    Set,
-    Tuple,
-    Type,
-    Union,
-    cast,
-)
-from typing_extensions import Literal
-from bs4._typing import (
-    _Encoding,
-    _Encodings,
-)
-import warnings
-
-# Import a library to autodetect character encodings. We'll support
-# any of a number of libraries that all support the same API:
-#
-# * cchardet
-# * chardet
-# * charset-normalizer
-chardet_module: Optional[ModuleType] = None
-try:
-    #  PyPI package: cchardet
-    import cchardet
-
-    chardet_module = cchardet
-except ImportError:
-    try:
-        #  Debian package: python-chardet
-        #  PyPI package: chardet
-        import chardet
-
-        chardet_module = chardet
-    except ImportError:
-        try:
-            # PyPI package: charset-normalizer
-            import charset_normalizer
-
-            chardet_module = charset_normalizer
-        except ImportError:
-            # No chardet available.
-            pass
-
-
-def _chardet_dammit(s: bytes) -> Optional[str]:
-    """Try as hard as possible to detect the encoding of a bytestring."""
-    if chardet_module is None or isinstance(s, str):
-        return None
-    module = chardet_module
-    return module.detect(s)["encoding"]
-
-
-# Build bytestring and Unicode versions of regular expressions for finding
-# a declared encoding inside an XML or HTML document.
-xml_encoding: str = "^\\s*<\\?.*encoding=['\"](.*?)['\"].*\\?>"  #: :meta private:
-html_meta: str = (
-    "<\\s*meta[^>]+charset\\s*=\\s*[\"']?([^>]*?)[ /;'\">]"  #: :meta private:
-)
-
-# TODO-TYPING: The Pattern type here could use more refinement, but it's tricky.
-encoding_res: Dict[Type, Dict[str, Pattern]] = dict()
-encoding_res[bytes] = {
-    "html": re.compile(html_meta.encode("ascii"), re.I),
-    "xml": re.compile(xml_encoding.encode("ascii"), re.I),
-}
-encoding_res[str] = {
-    "html": re.compile(html_meta, re.I),
-    "xml": re.compile(xml_encoding, re.I),
-}
-
-
-class EntitySubstitution(object):
-    """The ability to substitute XML or HTML entities for certain characters."""
-
-    #: A map of named HTML entities to the corresponding Unicode string.
-    #:
-    #: :meta hide-value:
-    HTML_ENTITY_TO_CHARACTER: Dict[str, str]
-
-    #: A map of Unicode strings to the corresponding named HTML entities;
-    #: the inverse of HTML_ENTITY_TO_CHARACTER.
-    #:
-    #: :meta hide-value:
-    CHARACTER_TO_HTML_ENTITY: Dict[str, str]
-
-    #: A regular expression that matches any character (or, in rare
-    #: cases, pair of characters) that can be replaced with a named
-    #: HTML entity.
-    #:
-    #: :meta hide-value:
-    CHARACTER_TO_HTML_ENTITY_RE: Pattern[str]
-
-    #: A very similar regular expression to
-    #: CHARACTER_TO_HTML_ENTITY_RE, but which also matches unescaped
-    #: ampersands. This is used by the 'html' formatted to provide
-    #: backwards-compatibility, even though the HTML5 spec allows most
-    #: ampersands to go unescaped.
-    #:
-    #: :meta hide-value:
-    CHARACTER_TO_HTML_ENTITY_WITH_AMPERSAND_RE: Pattern[str]
-
-    @classmethod
-    def _populate_class_variables(cls) -> None:
-        """Initialize variables used by this class to manage the plethora of
-        HTML5 named entities.
-
-        This function sets the following class variables:
-
-        CHARACTER_TO_HTML_ENTITY - A mapping of Unicode strings like "⦨" to
-        entity names like "angmsdaa". When a single Unicode string has
-        multiple entity names, we try to choose the most commonly-used
-        name.
-
-        HTML_ENTITY_TO_CHARACTER: A mapping of entity names like "angmsdaa" to
-        Unicode strings like "⦨".
-
-        CHARACTER_TO_HTML_ENTITY_RE: A regular expression matching (almost) any
-        Unicode string that corresponds to an HTML5 named entity.
-
-        CHARACTER_TO_HTML_ENTITY_WITH_AMPERSAND_RE: A very similar
-        regular expression to CHARACTER_TO_HTML_ENTITY_RE, but which
-        also matches unescaped ampersands. This is used by the 'html'
-        formatted to provide backwards-compatibility, even though the HTML5
-        spec allows most ampersands to go unescaped.
-        """
-        unicode_to_name = {}
-        name_to_unicode = {}
-
-        short_entities = set()
-        long_entities_by_first_character = defaultdict(set)
-
-        for name_with_semicolon, character in sorted(html5.items()):
-            # "It is intentional, for legacy compatibility, that many
-            # code points have multiple character reference names. For
-            # example, some appear both with and without the trailing
-            # semicolon, or with different capitalizations."
-            # - https://html.spec.whatwg.org/multipage/named-characters.html#named-character-references
-            #
-            # The parsers are in charge of handling (or not) character
-            # references with no trailing semicolon, so we remove the
-            # semicolon whenever it appears.
-            if name_with_semicolon.endswith(";"):
-                name = name_with_semicolon[:-1]
-            else:
-                name = name_with_semicolon
-
-            # When parsing HTML, we want to recognize any known named
-            # entity and convert it to a sequence of Unicode
-            # characters.
-            if name not in name_to_unicode:
-                name_to_unicode[name] = character
-
-            # When _generating_ HTML, we want to recognize special
-            # character sequences that _could_ be converted to named
-            # entities.
-            unicode_to_name[character] = name
-
-            # We also need to build a regular expression that lets us
-            # _find_ those characters in output strings so we can
-            # replace them.
-            #
-            # This is tricky, for two reasons.
-
-            if len(character) == 1 and ord(character) < 128 and character not in "<>":
-                # First, it would be annoying to turn single ASCII
-                # characters like | into named entities like
-                # &verbar;. The exceptions are <>, which we _must_
-                # turn into named entities to produce valid HTML.
-                continue
-
-            if len(character) > 1 and all(ord(x) < 128 for x in character):
-                # We also do not want to turn _combinations_ of ASCII
-                # characters like 'fj' into named entities like '&fjlig;',
-                # though that's more debateable.
-                continue
-
-            # Second, some named entities have a Unicode value that's
-            # a subset of the Unicode value for some _other_ named
-            # entity.  As an example, \u2267' is &GreaterFullEqual;,
-            # but '\u2267\u0338' is &NotGreaterFullEqual;. Our regular
-            # expression needs to match the first two characters of
-            # "\u2267\u0338foo", but only the first character of
-            # "\u2267foo".
-            #
-            # In this step, we build two sets of characters that
-            # _eventually_ need to go into the regular expression. But
-            # we won't know exactly what the regular expression needs
-            # to look like until we've gone through the entire list of
-            # named entities.
-            if len(character) == 1 and character != "&":
-                short_entities.add(character)
-            else:
-                long_entities_by_first_character[character[0]].add(character)
-
-        # Now that we've been through the entire list of entities, we
-        # can create a regular expression that matches any of them.
-        particles = set()
-        for short in short_entities:
-            long_versions = long_entities_by_first_character[short]
-            if not long_versions:
-                particles.add(short)
-            else:
-                ignore = "".join([x[1] for x in long_versions])
-                # This finds, e.g. \u2267 but only if it is _not_
-                # followed by \u0338.
-                particles.add("%s(?![%s])" % (short, ignore))
-
-        for long_entities in list(long_entities_by_first_character.values()):
-            for long_entity in long_entities:
-                particles.add(long_entity)
-
-        re_definition = "(%s)" % "|".join(particles)
-
-        particles.add("&")
-        re_definition_with_ampersand = "(%s)" % "|".join(particles)
-
-        # If an entity shows up in both html5 and codepoint2name, it's
-        # likely that HTML5 gives it several different names, such as
-        # 'rsquo' and 'rsquor'. When converting Unicode characters to
-        # named entities, the codepoint2name name should take
-        # precedence where possible, since that's the more easily
-        # recognizable one.
-        for codepoint, name in list(codepoint2name.items()):
-            character = chr(codepoint)
-            unicode_to_name[character] = name
-
-        cls.CHARACTER_TO_HTML_ENTITY = unicode_to_name
-        cls.HTML_ENTITY_TO_CHARACTER = name_to_unicode
-        cls.CHARACTER_TO_HTML_ENTITY_RE = re.compile(re_definition)
-        cls.CHARACTER_TO_HTML_ENTITY_WITH_AMPERSAND_RE = re.compile(
-            re_definition_with_ampersand
-        )
-
-    #: A map of Unicode strings to the corresponding named XML entities.
-    #:
-    #: :meta hide-value:
-    CHARACTER_TO_XML_ENTITY: Dict[str, str] = {
-        "'": "apos",
-        '"': "quot",
-        "&": "amp",
-        "<": "lt",
-        ">": "gt",
-    }
-
-    # Matches any named or numeric HTML entity.
-    ANY_ENTITY_RE = re.compile("&(#\\d+|#x[0-9a-fA-F]+|\\w+);", re.I)
-
-    #: A regular expression matching an angle bracket or an ampersand that
-    #: is not part of an XML or HTML entity.
-    #:
-    #: :meta hide-value:
-    BARE_AMPERSAND_OR_BRACKET: Pattern[str] = re.compile(
-        "([<>]|" "&(?!#\\d+;|#x[0-9a-fA-F]+;|\\w+;)" ")"
-    )
-
-    #: A regular expression matching an angle bracket or an ampersand.
-    #:
-    #: :meta hide-value:
-    AMPERSAND_OR_BRACKET: Pattern[str] = re.compile("([<>&])")
-
-    @classmethod
-    def _substitute_html_entity(cls, matchobj: re.Match) -> str:
-        """Used with a regular expression to substitute the
-        appropriate HTML entity for a special character string."""
-        original_entity = matchobj.group(0)
-        entity = cls.CHARACTER_TO_HTML_ENTITY.get(original_entity)
-        if entity is None:
-            return "&amp;%s;" % original_entity
-        return "&%s;" % entity
-
-    @classmethod
-    def _substitute_xml_entity(cls, matchobj: re.Match) -> str:
-        """Used with a regular expression to substitute the
-        appropriate XML entity for a special character string."""
-        entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)]
-        return "&%s;" % entity
-
-    @classmethod
-    def _escape_entity_name(cls, matchobj: re.Match) -> str:
-        return "&amp;%s;" % matchobj.group(1)
-
-    @classmethod
-    def _escape_unrecognized_entity_name(cls, matchobj: re.Match) -> str:
-        possible_entity = matchobj.group(1)
-        if possible_entity in cls.HTML_ENTITY_TO_CHARACTER:
-            return "&%s;" % possible_entity
-        return "&amp;%s;" % possible_entity
-
-    @classmethod
-    def quoted_attribute_value(cls, value: str) -> str:
-        """Make a value into a quoted XML attribute, possibly escaping it.
-
-         Most strings will be quoted using double quotes.
-
-          Bob's Bar -> "Bob's Bar"
-
-         If a string contains double quotes, it will be quoted using
-         single quotes.
-
-          Welcome to "my bar" -> 'Welcome to "my bar"'
-
-         If a string contains both single and double quotes, the
-         double quotes will be escaped, and the string will be quoted
-         using double quotes.
-
-          Welcome to "Bob's Bar" -> Welcome to &quot;Bob's bar&quot;
-
-        :param value: The XML attribute value to quote
-        :return: The quoted value
-        """
-        quote_with = '"'
-        if '"' in value:
-            if "'" in value:
-                # The string contains both single and double
-                # quotes.  Turn the double quotes into
-                # entities. We quote the double quotes rather than
-                # the single quotes because the entity name is
-                # "&quot;" whether this is HTML or XML.  If we
-                # quoted the single quotes, we'd have to decide
-                # between &apos; and &squot;.
-                replace_with = "&quot;"
-                value = value.replace('"', replace_with)
-            else:
-                # There are double quotes but no single quotes.
-                # We can use single quotes to quote the attribute.
-                quote_with = "'"
-        return quote_with + value + quote_with
-
-    @classmethod
-    def substitute_xml(cls, value: str, make_quoted_attribute: bool = False) -> str:
-        """Replace special XML characters with named XML entities.
-
-        The less-than sign will become &lt;, the greater-than sign
-        will become &gt;, and any ampersands will become &amp;. If you
-        want ampersands that seem to be part of an entity definition
-        to be left alone, use `substitute_xml_containing_entities`
-        instead.
-
-        :param value: A string to be substituted.
-
-        :param make_quoted_attribute: If True, then the string will be
-         quoted, as befits an attribute value.
-
-        :return: A version of ``value`` with special characters replaced
-         with named entities.
-        """
-        # Escape angle brackets and ampersands.
-        value = cls.AMPERSAND_OR_BRACKET.sub(cls._substitute_xml_entity, value)
-
-        if make_quoted_attribute:
-            value = cls.quoted_attribute_value(value)
-        return value
-
-    @classmethod
-    def substitute_xml_containing_entities(
-        cls, value: str, make_quoted_attribute: bool = False
-    ) -> str:
-        """Substitute XML entities for special XML characters.
-
-        :param value: A string to be substituted. The less-than sign will
-          become &lt;, the greater-than sign will become &gt;, and any
-          ampersands that are not part of an entity defition will
-          become &amp;.
-
-        :param make_quoted_attribute: If True, then the string will be
-         quoted, as befits an attribute value.
-        """
-        # Escape angle brackets, and ampersands that aren't part of
-        # entities.
-        value = cls.BARE_AMPERSAND_OR_BRACKET.sub(cls._substitute_xml_entity, value)
-
-        if make_quoted_attribute:
-            value = cls.quoted_attribute_value(value)
-        return value
-
-    @classmethod
-    def substitute_html(cls, s: str) -> str:
-        """Replace certain Unicode characters with named HTML entities.
-
-        This differs from ``data.encode(encoding, 'xmlcharrefreplace')``
-        in that the goal is to make the result more readable (to those
-        with ASCII displays) rather than to recover from
-        errors. There's absolutely nothing wrong with a UTF-8 string
-        containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that
-        character with "&eacute;" will make it more readable to some
-        people.
-
-        :param s: The string to be modified.
-        :return: The string with some Unicode characters replaced with
-           HTML entities.
-        """
-        # Convert any appropriate characters to HTML entities.
-        return cls.CHARACTER_TO_HTML_ENTITY_WITH_AMPERSAND_RE.sub(
-            cls._substitute_html_entity, s
-        )
-
-    @classmethod
-    def substitute_html5(cls, s: str) -> str:
-        """Replace certain Unicode characters with named HTML entities
-        using HTML5 rules.
-
-        Specifically, this method is much less aggressive about
-        escaping ampersands than substitute_html. Only ambiguous
-        ampersands are escaped, per the HTML5 standard:
-
-        "An ambiguous ampersand is a U+0026 AMPERSAND character (&)
-        that is followed by one or more ASCII alphanumerics, followed
-        by a U+003B SEMICOLON character (;), where these characters do
-        not match any of the names given in the named character
-        references section."
-
-        Unlike substitute_html5_raw, this method assumes HTML entities
-        were converted to Unicode characters on the way in, as
-        Beautiful Soup does. By the time Beautiful Soup does its work,
-        the only ambiguous ampersands that need to be escaped are the
-        ones that were escaped in the original markup when mentioning
-        HTML entities.
-
-        :param s: The string to be modified.
-        :return: The string with some Unicode characters replaced with
-           HTML entities.
-        """
-        # First, escape any HTML entities found in the markup.
-        s = cls.ANY_ENTITY_RE.sub(cls._escape_entity_name, s)
-
-        # Next, convert any appropriate characters to unescaped HTML entities.
-        s = cls.CHARACTER_TO_HTML_ENTITY_RE.sub(cls._substitute_html_entity, s)
-
-        return s
-
-    @classmethod
-    def substitute_html5_raw(cls, s: str) -> str:
-        """Replace certain Unicode characters with named HTML entities
-        using HTML5 rules.
-
-        substitute_html5_raw is similar to substitute_html5 but it is
-        designed for standalone use (whereas substitute_html5 is
-        designed for use with Beautiful Soup).
-
-        :param s: The string to be modified.
-        :return: The string with some Unicode characters replaced with
-           HTML entities.
-        """
-        # First, escape the ampersand for anything that looks like an
-        # entity but isn't in the list of recognized entities. All other
-        # ampersands can be left alone.
-        s = cls.ANY_ENTITY_RE.sub(cls._escape_unrecognized_entity_name, s)
-
-        # Then, convert a range of Unicode characters to unescaped
-        # HTML entities.
-        s = cls.CHARACTER_TO_HTML_ENTITY_RE.sub(cls._substitute_html_entity, s)
-
-        return s
-
-
-EntitySubstitution._populate_class_variables()
-
-
-class EncodingDetector:
-    """This class is capable of guessing a number of possible encodings
-    for a bytestring.
-
-    Order of precedence:
-
-    1. Encodings you specifically tell EncodingDetector to try first
-       (the ``known_definite_encodings`` argument to the constructor).
-
-    2. An encoding determined by sniffing the document's byte-order mark.
-
-    3. Encodings you specifically tell EncodingDetector to try if
-       byte-order mark sniffing fails (the ``user_encodings`` argument to the
-       constructor).
-
-    4. An encoding declared within the bytestring itself, either in an
-       XML declaration (if the bytestring is to be interpreted as an XML
-       document), or in a <meta> tag (if the bytestring is to be
-       interpreted as an HTML document.)
-
-    5. An encoding detected through textual analysis by chardet,
-       cchardet, or a similar external library.
-
-    6. UTF-8.
-
-    7. Windows-1252.
-
-    :param markup: Some markup in an unknown encoding.
-
-    :param known_definite_encodings: When determining the encoding
-        of ``markup``, these encodings will be tried first, in
-        order. In HTML terms, this corresponds to the "known
-        definite encoding" step defined in `section 13.2.3.1 of the HTML standard <https://html.spec.whatwg.org/multipage/parsing.html#parsing-with-a-known-character-encoding>`_.
-
-    :param user_encodings: These encodings will be tried after the
-        ``known_definite_encodings`` have been tried and failed, and
-        after an attempt to sniff the encoding by looking at a
-        byte order mark has failed. In HTML terms, this
-        corresponds to the step "user has explicitly instructed
-        the user agent to override the document's character
-        encoding", defined in `section 13.2.3.2 of the HTML standard <https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding>`_.
-
-    :param override_encodings: A **deprecated** alias for
-        ``known_definite_encodings``. Any encodings here will be tried
-        immediately after the encodings in
-        ``known_definite_encodings``.
-
-    :param is_html: If True, this markup is considered to be
-        HTML. Otherwise it's assumed to be XML.
-
-    :param exclude_encodings: These encodings will not be tried,
-        even if they otherwise would be.
-
-    """
-
-    def __init__(
-        self,
-        markup: bytes,
-        known_definite_encodings: Optional[_Encodings] = None,
-        is_html: Optional[bool] = False,
-        exclude_encodings: Optional[_Encodings] = None,
-        user_encodings: Optional[_Encodings] = None,
-        override_encodings: Optional[_Encodings] = None,
-    ):
-        self.known_definite_encodings = list(known_definite_encodings or [])
-        if override_encodings:
-            warnings.warn(
-                "The 'override_encodings' argument was deprecated in 4.10.0. Use 'known_definite_encodings' instead.",
-                DeprecationWarning,
-                stacklevel=3,
-            )
-            self.known_definite_encodings += override_encodings
-        self.user_encodings = user_encodings or []
-        exclude_encodings = exclude_encodings or []
-        self.exclude_encodings = set([x.lower() for x in exclude_encodings])
-        self.chardet_encoding = None
-        self.is_html = False if is_html is None else is_html
-        self.declared_encoding: Optional[str] = None
-
-        # First order of business: strip a byte-order mark.
-        self.markup, self.sniffed_encoding = self.strip_byte_order_mark(markup)
-
-    known_definite_encodings: _Encodings
-    user_encodings: _Encodings
-    exclude_encodings: _Encodings
-    chardet_encoding: Optional[_Encoding]
-    is_html: bool
-    declared_encoding: Optional[_Encoding]
-    markup: bytes
-    sniffed_encoding: Optional[_Encoding]
-
-    def _usable(self, encoding: Optional[_Encoding], tried: Set[_Encoding]) -> bool:
-        """Should we even bother to try this encoding?
-
-        :param encoding: Name of an encoding.
-        :param tried: Encodings that have already been tried. This
-            will be modified as a side effect.
-        """
-        if encoding is None:
-            return False
-        encoding = encoding.lower()
-        if encoding in self.exclude_encodings:
-            return False
-        if encoding not in tried:
-            tried.add(encoding)
-            return True
-        return False
-
-    @property
-    def encodings(self) -> Iterator[_Encoding]:
-        """Yield a number of encodings that might work for this markup.
-
-        :yield: A sequence of strings. Each is the name of an encoding
-           that *might* work to convert a bytestring into Unicode.
-        """
-        tried: Set[_Encoding] = set()
-
-        # First, try the known definite encodings
-        for e in self.known_definite_encodings:
-            if self._usable(e, tried):
-                yield e
-
-        # Did the document originally start with a byte-order mark
-        # that indicated its encoding?
-        if self.sniffed_encoding is not None and self._usable(
-            self.sniffed_encoding, tried
-        ):
-            yield self.sniffed_encoding
-
-        # Sniffing the byte-order mark did nothing; try the user
-        # encodings.
-        for e in self.user_encodings:
-            if self._usable(e, tried):
-                yield e
-
-        # Look within the document for an XML or HTML encoding
-        # declaration.
-        if self.declared_encoding is None:
-            self.declared_encoding = self.find_declared_encoding(
-                self.markup, self.is_html
-            )
-        if self.declared_encoding is not None and self._usable(
-            self.declared_encoding, tried
-        ):
-            yield self.declared_encoding
-
-        # Use third-party character set detection to guess at the
-        # encoding.
-        if self.chardet_encoding is None:
-            self.chardet_encoding = _chardet_dammit(self.markup)
-        if self.chardet_encoding is not None and self._usable(
-            self.chardet_encoding, tried
-        ):
-            yield self.chardet_encoding
-
-        # As a last-ditch effort, try utf-8 and windows-1252.
-        for e in ("utf-8", "windows-1252"):
-            if self._usable(e, tried):
-                yield e
-
-    @classmethod
-    def strip_byte_order_mark(cls, data: bytes) -> Tuple[bytes, Optional[_Encoding]]:
-        """If a byte-order mark is present, strip it and return the encoding it implies.
-
-        :param data: A bytestring that may or may not begin with a
-           byte-order mark.
-
-        :return: A 2-tuple (data stripped of byte-order mark, encoding implied by byte-order mark)
-        """
-        encoding = None
-        if isinstance(data, str):
-            # Unicode data cannot have a byte-order mark.
-            return data, encoding
-        if (
-            (len(data) >= 4)
-            and (data[:2] == b"\xfe\xff")
-            and (data[2:4] != b"\x00\x00")
-        ):
-            encoding = "utf-16be"
-            data = data[2:]
-        elif (
-            (len(data) >= 4)
-            and (data[:2] == b"\xff\xfe")
-            and (data[2:4] != b"\x00\x00")
-        ):
-            encoding = "utf-16le"
-            data = data[2:]
-        elif data[:3] == b"\xef\xbb\xbf":
-            encoding = "utf-8"
-            data = data[3:]
-        elif data[:4] == b"\x00\x00\xfe\xff":
-            encoding = "utf-32be"
-            data = data[4:]
-        elif data[:4] == b"\xff\xfe\x00\x00":
-            encoding = "utf-32le"
-            data = data[4:]
-        return data, encoding
-
-    @classmethod
-    def find_declared_encoding(
-        cls,
-        markup: Union[bytes, str],
-        is_html: bool = False,
-        search_entire_document: bool = False,
-    ) -> Optional[_Encoding]:
-        """Given a document, tries to find an encoding declared within the
-        text of the document itself.
-
-        An XML encoding is declared at the beginning of the document.
-
-        An HTML encoding is declared in a <meta> tag, hopefully near the
-        beginning of the document.
-
-        :param markup: Some markup.
-        :param is_html: If True, this markup is considered to be HTML. Otherwise
-            it's assumed to be XML.
-        :param search_entire_document: Since an encoding is supposed
-            to declared near the beginning of the document, most of
-            the time it's only necessary to search a few kilobytes of
-            data.  Set this to True to force this method to search the
-            entire document.
-        :return: The declared encoding, if one is found.
-        """
-        if search_entire_document:
-            xml_endpos = html_endpos = len(markup)
-        else:
-            xml_endpos = 1024
-            html_endpos = max(2048, int(len(markup) * 0.05))
-
-        if isinstance(markup, bytes):
-            res = encoding_res[bytes]
-        else:
-            res = encoding_res[str]
-
-        xml_re = res["xml"]
-        html_re = res["html"]
-        declared_encoding: Optional[_Encoding] = None
-        declared_encoding_match = xml_re.search(markup, endpos=xml_endpos)
-        if not declared_encoding_match and is_html:
-            declared_encoding_match = html_re.search(markup, endpos=html_endpos)
-        if declared_encoding_match is not None:
-            declared_encoding = declared_encoding_match.groups()[0]
-        if declared_encoding:
-            if isinstance(declared_encoding, bytes):
-                declared_encoding = declared_encoding.decode("ascii", "replace")
-            return declared_encoding.lower()
-        return None
-
-
-class UnicodeDammit:
-    """A class for detecting the encoding of a bytestring containing an
-    HTML or XML document, and decoding it to Unicode. If the source
-    encoding is windows-1252, `UnicodeDammit` can also replace
-    Microsoft smart quotes with their HTML or XML equivalents.
-
-    :param markup: HTML or XML markup in an unknown encoding.
-
-    :param known_definite_encodings: When determining the encoding
-        of ``markup``, these encodings will be tried first, in
-        order. In HTML terms, this corresponds to the "known
-        definite encoding" step defined in `section 13.2.3.1 of the HTML standard <https://html.spec.whatwg.org/multipage/parsing.html#parsing-with-a-known-character-encoding>`_.
-
-    :param user_encodings: These encodings will be tried after the
-        ``known_definite_encodings`` have been tried and failed, and
-        after an attempt to sniff the encoding by looking at a
-        byte order mark has failed. In HTML terms, this
-        corresponds to the step "user has explicitly instructed
-        the user agent to override the document's character
-        encoding", defined in `section 13.2.3.2 of the HTML standard <https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding>`_.
-
-    :param override_encodings: A **deprecated** alias for
-        ``known_definite_encodings``. Any encodings here will be tried
-        immediately after the encodings in
-        ``known_definite_encodings``.
-
-    :param smart_quotes_to: By default, Microsoft smart quotes will,
-       like all other characters, be converted to Unicode
-       characters. Setting this to ``ascii`` will convert them to ASCII
-       quotes instead.  Setting it to ``xml`` will convert them to XML
-       entity references, and setting it to ``html`` will convert them
-       to HTML entity references.
-
-    :param is_html: If True, ``markup`` is treated as an HTML
-       document. Otherwise it's treated as an XML document.
-
-    :param exclude_encodings: These encodings will not be considered,
-       even if the sniffing code thinks they might make sense.
-
-    """
-
-    def __init__(
-        self,
-        markup: bytes,
-        known_definite_encodings: Optional[_Encodings] = [],
-        smart_quotes_to: Optional[Literal["ascii", "xml", "html"]] = None,
-        is_html: bool = False,
-        exclude_encodings: Optional[_Encodings] = [],
-        user_encodings: Optional[_Encodings] = None,
-        override_encodings: Optional[_Encodings] = None,
-    ):
-        self.smart_quotes_to = smart_quotes_to
-        self.tried_encodings = []
-        self.contains_replacement_characters = False
-        self.is_html = is_html
-        self.log = getLogger(__name__)
-        self.detector = EncodingDetector(
-            markup,
-            known_definite_encodings,
-            is_html,
-            exclude_encodings,
-            user_encodings,
-            override_encodings,
-        )
-
-        # Short-circuit if the data is in Unicode to begin with.
-        if isinstance(markup, str) or markup == b"":
-            self.markup = markup
-            self.unicode_markup = str(markup)
-            self.original_encoding = None
-            return
-
-        # The encoding detector may have stripped a byte-order mark.
-        # Use the stripped markup from this point on.
-        self.markup = self.detector.markup
-
-        u = None
-        for encoding in self.detector.encodings:
-            markup = self.detector.markup
-            u = self._convert_from(encoding)
-            if u is not None:
-                break
-
-        if not u:
-            # None of the encodings worked. As an absolute last resort,
-            # try them again with character replacement.
-
-            for encoding in self.detector.encodings:
-                if encoding != "ascii":
-                    u = self._convert_from(encoding, "replace")
-                if u is not None:
-                    self.log.warning(
-                        "Some characters could not be decoded, and were "
-                        "replaced with REPLACEMENT CHARACTER."
-                    )
-
-                    self.contains_replacement_characters = True
-                    break
-
-        # If none of that worked, we could at this point force it to
-        # ASCII, but that would destroy so much data that I think
-        # giving up is better.
-        #
-        # Note that this is extremely unlikely, probably impossible,
-        # because the "replace" strategy is so powerful. Even running
-        # the Python binary through Unicode, Dammit gives you Unicode,
-        # albeit Unicode riddled with REPLACEMENT CHARACTER.
-        if u is None:
-            self.original_encoding = None
-            self.unicode_markup = None
-        else:
-            self.unicode_markup = u
-
-    #: The original markup, before it was converted to Unicode.
-    #: This is not necessarily the same as what was passed in to the
-    #: constructor, since any byte-order mark will be stripped.
-    markup: bytes
-
-    #: The Unicode version of the markup, following conversion. This
-    #: is set to None if there was simply no way to convert the
-    #: bytestring to Unicode (as with binary data).
-    unicode_markup: Optional[str]
-
-    #: This is True if `UnicodeDammit.unicode_markup` contains
-    #: U+FFFD REPLACEMENT_CHARACTER characters which were not present
-    #: in `UnicodeDammit.markup`. These mark character sequences that
-    #: could not be represented in Unicode.
-    contains_replacement_characters: bool
-
-    #: Unicode, Dammit's best guess as to the original character
-    #: encoding of `UnicodeDammit.markup`.
-    original_encoding: Optional[_Encoding]
-
-    #: The strategy used to handle Microsoft smart quotes.
-    smart_quotes_to: Optional[str]
-
-    #: The (encoding, error handling strategy) 2-tuples that were used to
-    #: try and convert the markup to Unicode.
-    tried_encodings: List[Tuple[_Encoding, str]]
-
-    log: Logger  #: :meta private:
-
-    def _sub_ms_char(self, match: re.Match) -> bytes:
-        """Changes a MS smart quote character to an XML or HTML
-        entity, or an ASCII character.
-
-        TODO: Since this is only used to convert smart quotes, it
-        could be simplified, and MS_CHARS_TO_ASCII made much less
-        parochial.
-        """
-        orig: bytes = match.group(1)
-        sub: bytes
-        if self.smart_quotes_to == "ascii":
-            if orig in self.MS_CHARS_TO_ASCII:
-                sub = self.MS_CHARS_TO_ASCII[orig].encode()
-            else:
-                # Shouldn't happen; substitute the character
-                # with itself.
-                sub = orig
-        else:
-            if orig in self.MS_CHARS:
-                substitutions = self.MS_CHARS[orig]
-                if type(substitutions) is tuple:
-                    if self.smart_quotes_to == "xml":
-                        sub = b"&#x" + substitutions[1].encode() + b";"
-                    else:
-                        sub = b"&" + substitutions[0].encode() + b";"
-                else:
-                    substitutions = cast(str, substitutions)
-                    sub = substitutions.encode()
-            else:
-                # Shouldn't happen; substitute the character
-                # for itself.
-                sub = orig
-        return sub
-
-    #: This dictionary maps commonly seen values for "charset" in HTML
-    #: meta tags to the corresponding Python codec names. It only covers
-    #: values that aren't in Python's aliases and can't be determined
-    #: by the heuristics in `find_codec`.
-    #:
-    #: :meta hide-value:
-    CHARSET_ALIASES: Dict[str, _Encoding] = {
-        "macintosh": "mac-roman",
-        "x-sjis": "shift-jis",
-    }
-
-    #: A list of encodings that tend to contain Microsoft smart quotes.
-    #:
-    #: :meta hide-value:
-    ENCODINGS_WITH_SMART_QUOTES: _Encodings = [
-        "windows-1252",
-        "iso-8859-1",
-        "iso-8859-2",
-    ]
-
-    def _convert_from(
-        self, proposed: _Encoding, errors: str = "strict"
-    ) -> Optional[str]:
-        """Attempt to convert the markup to the proposed encoding.
-
-        :param proposed: The name of a character encoding.
-        :param errors: An error handling strategy, used when calling `str`.
-        :return: The converted markup, or `None` if the proposed
-           encoding/error handling strategy didn't work.
-        """
-        lookup_result = self.find_codec(proposed)
-        if lookup_result is None or (lookup_result, errors) in self.tried_encodings:
-            return None
-        proposed = lookup_result
-        self.tried_encodings.append((proposed, errors))
-        markup = self.markup
-        # Convert smart quotes to HTML if coming from an encoding
-        # that might have them.
-        if (
-            self.smart_quotes_to is not None
-            and proposed in self.ENCODINGS_WITH_SMART_QUOTES
-        ):
-            smart_quotes_re = b"([\x80-\x9f])"
-            smart_quotes_compiled = re.compile(smart_quotes_re)
-            markup = smart_quotes_compiled.sub(self._sub_ms_char, markup)
-
-        try:
-            # print("Trying to convert document to %s (errors=%s)" % (
-            #    proposed, errors))
-            u = self._to_unicode(markup, proposed, errors)
-            self.unicode_markup = u
-            self.original_encoding = proposed
-        except Exception:
-            # print("That didn't work!")
-            # print(e)
-            return None
-        # print("Correct encoding: %s" % proposed)
-        return self.unicode_markup
-
-    def _to_unicode(
-        self, data: bytes, encoding: _Encoding, errors: str = "strict"
-    ) -> str:
-        """Given a bytestring and its encoding, decodes the string into Unicode.
-
-        :param encoding: The name of an encoding.
-        :param errors: An error handling strategy, used when calling `str`.
-        """
-        return str(data, encoding, errors)
-
-    @property
-    def declared_html_encoding(self) -> Optional[_Encoding]:
-        """If the markup is an HTML document, returns the encoding, if any,
-        declared *inside* the document.
-        """
-        if not self.is_html:
-            return None
-        return self.detector.declared_encoding
-
-    def find_codec(self, charset: _Encoding) -> Optional[str]:
-        """Look up the Python codec corresponding to a given character set.
-
-        :param charset: The name of a character set.
-        :return: The name of a Python codec.
-        """
-        value = (
-            self._codec(self.CHARSET_ALIASES.get(charset, charset))
-            or (charset and self._codec(charset.replace("-", "")))
-            or (charset and self._codec(charset.replace("-", "_")))
-            or (charset and charset.lower())
-            or charset
-        )
-        if value:
-            return value.lower()
-        return None
-
-    def _codec(self, charset: _Encoding) -> Optional[str]:
-        if not charset:
-            return charset
-        codec = None
-        try:
-            codecs.lookup(charset)
-            codec = charset
-        except (LookupError, ValueError):
-            pass
-        return codec
-
-    #: A partial mapping of ISO-Latin-1 to HTML entities/XML numeric entities.
-    #:
-    #: :meta hide-value:
-    MS_CHARS: Dict[bytes, Union[str, Tuple[str, str]]] = {
-        b"\x80": ("euro", "20AC"),
-        b"\x81": " ",
-        b"\x82": ("sbquo", "201A"),
-        b"\x83": ("fnof", "192"),
-        b"\x84": ("bdquo", "201E"),
-        b"\x85": ("hellip", "2026"),
-        b"\x86": ("dagger", "2020"),
-        b"\x87": ("Dagger", "2021"),
-        b"\x88": ("circ", "2C6"),
-        b"\x89": ("permil", "2030"),
-        b"\x8a": ("Scaron", "160"),
-        b"\x8b": ("lsaquo", "2039"),
-        b"\x8c": ("OElig", "152"),
-        b"\x8d": "?",
-        b"\x8e": ("#x17D", "17D"),
-        b"\x8f": "?",
-        b"\x90": "?",
-        b"\x91": ("lsquo", "2018"),
-        b"\x92": ("rsquo", "2019"),
-        b"\x93": ("ldquo", "201C"),
-        b"\x94": ("rdquo", "201D"),
-        b"\x95": ("bull", "2022"),
-        b"\x96": ("ndash", "2013"),
-        b"\x97": ("mdash", "2014"),
-        b"\x98": ("tilde", "2DC"),
-        b"\x99": ("trade", "2122"),
-        b"\x9a": ("scaron", "161"),
-        b"\x9b": ("rsaquo", "203A"),
-        b"\x9c": ("oelig", "153"),
-        b"\x9d": "?",
-        b"\x9e": ("#x17E", "17E"),
-        b"\x9f": ("Yuml", ""),
-    }
-
-    #: A parochial partial mapping of ISO-Latin-1 to ASCII. Contains
-    #: horrors like stripping diacritical marks to turn á into a, but also
-    #: contains non-horrors like turning “ into ".
-    #:
-    #: Seriously, don't use this for anything other than removing smart
-    #: quotes.
-    #:
-    #: :meta private:
-    MS_CHARS_TO_ASCII: Dict[bytes, str] = {
-        b"\x80": "EUR",
-        b"\x81": " ",
-        b"\x82": ",",
-        b"\x83": "f",
-        b"\x84": ",,",
-        b"\x85": "...",
-        b"\x86": "+",
-        b"\x87": "++",
-        b"\x88": "^",
-        b"\x89": "%",
-        b"\x8a": "S",
-        b"\x8b": "<",
-        b"\x8c": "OE",
-        b"\x8d": "?",
-        b"\x8e": "Z",
-        b"\x8f": "?",
-        b"\x90": "?",
-        b"\x91": "'",
-        b"\x92": "'",
-        b"\x93": '"',
-        b"\x94": '"',
-        b"\x95": "*",
-        b"\x96": "-",
-        b"\x97": "--",
-        b"\x98": "~",
-        b"\x99": "(TM)",
-        b"\x9a": "s",
-        b"\x9b": ">",
-        b"\x9c": "oe",
-        b"\x9d": "?",
-        b"\x9e": "z",
-        b"\x9f": "Y",
-        b"\xa0": " ",
-        b"\xa1": "!",
-        b"\xa2": "c",
-        b"\xa3": "GBP",
-        b"\xa4": "$",  # This approximation is especially parochial--this is the
-        # generic currency symbol.
-        b"\xa5": "YEN",
-        b"\xa6": "|",
-        b"\xa7": "S",
-        b"\xa8": "..",
-        b"\xa9": "",
-        b"\xaa": "(th)",
-        b"\xab": "<<",
-        b"\xac": "!",
-        b"\xad": " ",
-        b"\xae": "(R)",
-        b"\xaf": "-",
-        b"\xb0": "o",
-        b"\xb1": "+-",
-        b"\xb2": "2",
-        b"\xb3": "3",
-        b"\xb4": "'",
-        b"\xb5": "u",
-        b"\xb6": "P",
-        b"\xb7": "*",
-        b"\xb8": ",",
-        b"\xb9": "1",
-        b"\xba": "(th)",
-        b"\xbb": ">>",
-        b"\xbc": "1/4",
-        b"\xbd": "1/2",
-        b"\xbe": "3/4",
-        b"\xbf": "?",
-        b"\xc0": "A",
-        b"\xc1": "A",
-        b"\xc2": "A",
-        b"\xc3": "A",
-        b"\xc4": "A",
-        b"\xc5": "A",
-        b"\xc6": "AE",
-        b"\xc7": "C",
-        b"\xc8": "E",
-        b"\xc9": "E",
-        b"\xca": "E",
-        b"\xcb": "E",
-        b"\xcc": "I",
-        b"\xcd": "I",
-        b"\xce": "I",
-        b"\xcf": "I",
-        b"\xd0": "D",
-        b"\xd1": "N",
-        b"\xd2": "O",
-        b"\xd3": "O",
-        b"\xd4": "O",
-        b"\xd5": "O",
-        b"\xd6": "O",
-        b"\xd7": "*",
-        b"\xd8": "O",
-        b"\xd9": "U",
-        b"\xda": "U",
-        b"\xdb": "U",
-        b"\xdc": "U",
-        b"\xdd": "Y",
-        b"\xde": "b",
-        b"\xdf": "B",
-        b"\xe0": "a",
-        b"\xe1": "a",
-        b"\xe2": "a",
-        b"\xe3": "a",
-        b"\xe4": "a",
-        b"\xe5": "a",
-        b"\xe6": "ae",
-        b"\xe7": "c",
-        b"\xe8": "e",
-        b"\xe9": "e",
-        b"\xea": "e",
-        b"\xeb": "e",
-        b"\xec": "i",
-        b"\xed": "i",
-        b"\xee": "i",
-        b"\xef": "i",
-        b"\xf0": "o",
-        b"\xf1": "n",
-        b"\xf2": "o",
-        b"\xf3": "o",
-        b"\xf4": "o",
-        b"\xf5": "o",
-        b"\xf6": "o",
-        b"\xf7": "/",
-        b"\xf8": "o",
-        b"\xf9": "u",
-        b"\xfa": "u",
-        b"\xfb": "u",
-        b"\xfc": "u",
-        b"\xfd": "y",
-        b"\xfe": "b",
-        b"\xff": "y",
-    }
-
-    #: A map used when removing rogue Windows-1252/ISO-8859-1
-    #: characters in otherwise UTF-8 documents.
-    #:
-    #: Note that \\x81, \\x8d, \\x8f, \\x90, and \\x9d are undefined in
-    #: Windows-1252.
-    #:
-    #: :meta hide-value:
-    WINDOWS_1252_TO_UTF8: Dict[int, bytes] = {
-        0x80: b"\xe2\x82\xac",  # €
-        0x82: b"\xe2\x80\x9a",  # ‚
-        0x83: b"\xc6\x92",  # ƒ
-        0x84: b"\xe2\x80\x9e",  # „
-        0x85: b"\xe2\x80\xa6",  # …
-        0x86: b"\xe2\x80\xa0",  # †
-        0x87: b"\xe2\x80\xa1",  # ‡
-        0x88: b"\xcb\x86",  # ˆ
-        0x89: b"\xe2\x80\xb0",  # ‰
-        0x8A: b"\xc5\xa0",  # Š
-        0x8B: b"\xe2\x80\xb9",  # ‹
-        0x8C: b"\xc5\x92",  # Œ
-        0x8E: b"\xc5\xbd",  # Ž
-        0x91: b"\xe2\x80\x98",  # ‘
-        0x92: b"\xe2\x80\x99",  # ’
-        0x93: b"\xe2\x80\x9c",  # “
-        0x94: b"\xe2\x80\x9d",  # ”
-        0x95: b"\xe2\x80\xa2",  # •
-        0x96: b"\xe2\x80\x93",  # –
-        0x97: b"\xe2\x80\x94",  # —
-        0x98: b"\xcb\x9c",  # ˜
-        0x99: b"\xe2\x84\xa2",  # ™
-        0x9A: b"\xc5\xa1",  # š
-        0x9B: b"\xe2\x80\xba",  # ›
-        0x9C: b"\xc5\x93",  # œ
-        0x9E: b"\xc5\xbe",  # ž
-        0x9F: b"\xc5\xb8",  # Ÿ
-        0xA0: b"\xc2\xa0",  #
-        0xA1: b"\xc2\xa1",  # ¡
-        0xA2: b"\xc2\xa2",  # ¢
-        0xA3: b"\xc2\xa3",  # £
-        0xA4: b"\xc2\xa4",  # ¤
-        0xA5: b"\xc2\xa5",  # ¥
-        0xA6: b"\xc2\xa6",  # ¦
-        0xA7: b"\xc2\xa7",  # §
-        0xA8: b"\xc2\xa8",  # ¨
-        0xA9: b"\xc2\xa9",  # ©
-        0xAA: b"\xc2\xaa",  # ª
-        0xAB: b"\xc2\xab",  # «
-        0xAC: b"\xc2\xac",  # ¬
-        0xAD: b"\xc2\xad",  # ­
-        0xAE: b"\xc2\xae",  # ®
-        0xAF: b"\xc2\xaf",  # ¯
-        0xB0: b"\xc2\xb0",  # °
-        0xB1: b"\xc2\xb1",  # ±
-        0xB2: b"\xc2\xb2",  # ²
-        0xB3: b"\xc2\xb3",  # ³
-        0xB4: b"\xc2\xb4",  # ´
-        0xB5: b"\xc2\xb5",  # µ
-        0xB6: b"\xc2\xb6",  # ¶
-        0xB7: b"\xc2\xb7",  # ·
-        0xB8: b"\xc2\xb8",  # ¸
-        0xB9: b"\xc2\xb9",  # ¹
-        0xBA: b"\xc2\xba",  # º
-        0xBB: b"\xc2\xbb",  # »
-        0xBC: b"\xc2\xbc",  # ¼
-        0xBD: b"\xc2\xbd",  # ½
-        0xBE: b"\xc2\xbe",  # ¾
-        0xBF: b"\xc2\xbf",  # ¿
-        0xC0: b"\xc3\x80",  # À
-        0xC1: b"\xc3\x81",  # Á
-        0xC2: b"\xc3\x82",  # Â
-        0xC3: b"\xc3\x83",  # Ã
-        0xC4: b"\xc3\x84",  # Ä
-        0xC5: b"\xc3\x85",  # Å
-        0xC6: b"\xc3\x86",  # Æ
-        0xC7: b"\xc3\x87",  # Ç
-        0xC8: b"\xc3\x88",  # È
-        0xC9: b"\xc3\x89",  # É
-        0xCA: b"\xc3\x8a",  # Ê
-        0xCB: b"\xc3\x8b",  # Ë
-        0xCC: b"\xc3\x8c",  # Ì
-        0xCD: b"\xc3\x8d",  # Í
-        0xCE: b"\xc3\x8e",  # Î
-        0xCF: b"\xc3\x8f",  # Ï
-        0xD0: b"\xc3\x90",  # Ð
-        0xD1: b"\xc3\x91",  # Ñ
-        0xD2: b"\xc3\x92",  # Ò
-        0xD3: b"\xc3\x93",  # Ó
-        0xD4: b"\xc3\x94",  # Ô
-        0xD5: b"\xc3\x95",  # Õ
-        0xD6: b"\xc3\x96",  # Ö
-        0xD7: b"\xc3\x97",  # ×
-        0xD8: b"\xc3\x98",  # Ø
-        0xD9: b"\xc3\x99",  # Ù
-        0xDA: b"\xc3\x9a",  # Ú
-        0xDB: b"\xc3\x9b",  # Û
-        0xDC: b"\xc3\x9c",  # Ü
-        0xDD: b"\xc3\x9d",  # Ý
-        0xDE: b"\xc3\x9e",  # Þ
-        0xDF: b"\xc3\x9f",  # ß
-        0xE0: b"\xc3\xa0",  # à
-        0xE1: b"\xa1",  # á
-        0xE2: b"\xc3\xa2",  # â
-        0xE3: b"\xc3\xa3",  # ã
-        0xE4: b"\xc3\xa4",  # ä
-        0xE5: b"\xc3\xa5",  # å
-        0xE6: b"\xc3\xa6",  # æ
-        0xE7: b"\xc3\xa7",  # ç
-        0xE8: b"\xc3\xa8",  # è
-        0xE9: b"\xc3\xa9",  # é
-        0xEA: b"\xc3\xaa",  # ê
-        0xEB: b"\xc3\xab",  # ë
-        0xEC: b"\xc3\xac",  # ì
-        0xED: b"\xc3\xad",  # í
-        0xEE: b"\xc3\xae",  # î
-        0xEF: b"\xc3\xaf",  # ï
-        0xF0: b"\xc3\xb0",  # ð
-        0xF1: b"\xc3\xb1",  # ñ
-        0xF2: b"\xc3\xb2",  # ò
-        0xF3: b"\xc3\xb3",  # ó
-        0xF4: b"\xc3\xb4",  # ô
-        0xF5: b"\xc3\xb5",  # õ
-        0xF6: b"\xc3\xb6",  # ö
-        0xF7: b"\xc3\xb7",  # ÷
-        0xF8: b"\xc3\xb8",  # ø
-        0xF9: b"\xc3\xb9",  # ù
-        0xFA: b"\xc3\xba",  # ú
-        0xFB: b"\xc3\xbb",  # û
-        0xFC: b"\xc3\xbc",  # ü
-        0xFD: b"\xc3\xbd",  # ý
-        0xFE: b"\xc3\xbe",  # þ
-    }
-
-    #: :meta private:
-    MULTIBYTE_MARKERS_AND_SIZES: List[Tuple[int, int, int]] = [
-        (0xC2, 0xDF, 2),  # 2-byte characters start with a byte C2-DF
-        (0xE0, 0xEF, 3),  # 3-byte characters start with E0-EF
-        (0xF0, 0xF4, 4),  # 4-byte characters start with F0-F4
-    ]
-
-    #: :meta private:
-    FIRST_MULTIBYTE_MARKER: int = MULTIBYTE_MARKERS_AND_SIZES[0][0]
-
-    #: :meta private:
-    LAST_MULTIBYTE_MARKER: int = MULTIBYTE_MARKERS_AND_SIZES[-1][1]
-
-    @classmethod
-    def detwingle(
-        cls,
-        in_bytes: bytes,
-        main_encoding: _Encoding = "utf8",
-        embedded_encoding: _Encoding = "windows-1252",
-    ) -> bytes:
-        """Fix characters from one encoding embedded in some other encoding.
-
-        Currently the only situation supported is Windows-1252 (or its
-        subset ISO-8859-1), embedded in UTF-8.
-
-        :param in_bytes: A bytestring that you suspect contains
-            characters from multiple encodings. Note that this *must*
-            be a bytestring. If you've already converted the document
-            to Unicode, you're too late.
-        :param main_encoding: The primary encoding of ``in_bytes``.
-        :param embedded_encoding: The encoding that was used to embed characters
-            in the main document.
-        :return: A bytestring similar to ``in_bytes``, in which
-          ``embedded_encoding`` characters have been converted to
-          their ``main_encoding`` equivalents.
-        """
-        if embedded_encoding.replace("_", "-").lower() not in (
-            "windows-1252",
-            "windows_1252",
-        ):
-            raise NotImplementedError(
-                "Windows-1252 and ISO-8859-1 are the only currently supported "
-                "embedded encodings."
-            )
-
-        if main_encoding.lower() not in ("utf8", "utf-8"):
-            raise NotImplementedError(
-                "UTF-8 is the only currently supported main encoding."
-            )
-
-        byte_chunks = []
-
-        chunk_start = 0
-        pos = 0
-        while pos < len(in_bytes):
-            byte = in_bytes[pos]
-            if byte >= cls.FIRST_MULTIBYTE_MARKER and byte <= cls.LAST_MULTIBYTE_MARKER:
-                # This is the start of a UTF-8 multibyte character. Skip
-                # to the end.
-                for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES:
-                    if byte >= start and byte <= end:
-                        pos += size
-                        break
-            elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8:
-                # We found a Windows-1252 character!
-                # Save the string up to this point as a chunk.
-                byte_chunks.append(in_bytes[chunk_start:pos])
-
-                # Now translate the Windows-1252 character into UTF-8
-                # and add it as another, one-byte chunk.
-                byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte])
-                pos += 1
-                chunk_start = pos
-            else:
-                # Go on to the next character.
-                pos += 1
-        if chunk_start == 0:
-            # The string is unchanged.
-            return in_bytes
-        else:
-            # Store the final chunk.
-            byte_chunks.append(in_bytes[chunk_start:])
-        return b"".join(byte_chunks)