@scrypted/server 0.0.141 → 0.0.145

package/.venv/lib/python3.9/site-packages/pip/_vendor/chardet/metadata/languages.py

@@ -1,310 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-Metadata about languages used by our model training code for our
-SingleByteCharSetProbers.  Could be used for other things in the future.
-
-This code is based on the language metadata from the uchardet project.
-"""
-from __future__ import absolute_import, print_function
-
-from string import ascii_letters
-
-
-# TODO: Add Ukranian (KOI8-U)
-
-class Language(object):
-    """Metadata about a language useful for training models
-
-    :ivar name: The human name for the language, in English.
-    :type name: str
-    :ivar iso_code: 2-letter ISO 639-1 if possible, 3-letter ISO code otherwise,
-                    or use another catalog as a last resort.
-    :type iso_code: str
-    :ivar use_ascii: Whether or not ASCII letters should be included in trained
-                     models.
-    :type use_ascii: bool
-    :ivar charsets: The charsets we want to support and create data for.
-    :type charsets: list of str
-    :ivar alphabet: The characters in the language's alphabet. If `use_ascii` is
-                    `True`, you only need to add those not in the ASCII set.
-    :type alphabet: str
-    :ivar wiki_start_pages: The Wikipedia pages to start from if we're crawling
-                            Wikipedia for training data.
-    :type wiki_start_pages: list of str
-    """
-    def __init__(self, name=None, iso_code=None, use_ascii=True, charsets=None,
-                 alphabet=None, wiki_start_pages=None):
-        super(Language, self).__init__()
-        self.name = name
-        self.iso_code = iso_code
-        self.use_ascii = use_ascii
-        self.charsets = charsets
-        if self.use_ascii:
-            if alphabet:
-                alphabet += ascii_letters
-            else:
-                alphabet = ascii_letters
-        elif not alphabet:
-            raise ValueError('Must supply alphabet if use_ascii is False')
-        self.alphabet = ''.join(sorted(set(alphabet))) if alphabet else None
-        self.wiki_start_pages = wiki_start_pages
-
-    def __repr__(self):
-        return '{}({})'.format(self.__class__.__name__,
-                               ', '.join('{}={!r}'.format(k, v)
-                                         for k, v in self.__dict__.items()
-                                         if not k.startswith('_')))
-
-
-LANGUAGES = {'Arabic': Language(name='Arabic',
-                                iso_code='ar',
-                                use_ascii=False,
-                                # We only support encodings that use isolated
-                                # forms, because the current recommendation is
-                                # that the rendering system handles presentation
-                                # forms. This means we purposefully skip IBM864.
-                                charsets=['ISO-8859-6', 'WINDOWS-1256',
-                                          'CP720', 'CP864'],
-                                alphabet=u'ءآأؤإئابةتثجحخدذرزسشصضطظعغػؼؽؾؿـفقكلمنهوىيًٌٍَُِّ',
-                                wiki_start_pages=[u'الصفحة_الرئيسية']),
-             'Belarusian': Language(name='Belarusian',
-                                    iso_code='be',
-                                    use_ascii=False,
-                                    charsets=['ISO-8859-5', 'WINDOWS-1251',
-                                              'IBM866', 'MacCyrillic'],
-                                    alphabet=(u'АБВГДЕЁЖЗІЙКЛМНОПРСТУЎФХЦЧШЫЬЭЮЯ'
-                                              u'абвгдеёжзійклмнопрстуўфхцчшыьэюяʼ'),
-                                    wiki_start_pages=[u'Галоўная_старонка']),
-             'Bulgarian': Language(name='Bulgarian',
-                                   iso_code='bg',
-                                   use_ascii=False,
-                                   charsets=['ISO-8859-5', 'WINDOWS-1251',
-                                             'IBM855'],
-                                   alphabet=(u'АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯ'
-                                             u'абвгдежзийклмнопрстуфхцчшщъьюя'),
-                                   wiki_start_pages=[u'Начална_страница']),
-             'Czech': Language(name='Czech',
-                               iso_code='cz',
-                               use_ascii=True,
-                               charsets=['ISO-8859-2', 'WINDOWS-1250'],
-                               alphabet=u'áčďéěíňóřšťúůýžÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ',
-                               wiki_start_pages=[u'Hlavní_strana']),
-             'Danish': Language(name='Danish',
-                                iso_code='da',
-                                use_ascii=True,
-                                charsets=['ISO-8859-1', 'ISO-8859-15',
-                                          'WINDOWS-1252'],
-                                alphabet=u'æøåÆØÅ',
-                                wiki_start_pages=[u'Forside']),
-             'German': Language(name='German',
-                                iso_code='de',
-                                use_ascii=True,
-                                charsets=['ISO-8859-1', 'WINDOWS-1252'],
-                                alphabet=u'äöüßÄÖÜ',
-                                wiki_start_pages=[u'Wikipedia:Hauptseite']),
-             'Greek': Language(name='Greek',
-                               iso_code='el',
-                               use_ascii=False,
-                               charsets=['ISO-8859-7', 'WINDOWS-1253'],
-                               alphabet=(u'αβγδεζηθικλμνξοπρσςτυφχψωάέήίόύώ'
-                                         u'ΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΣΤΥΦΧΨΩΆΈΉΊΌΎΏ'),
-                               wiki_start_pages=[u'Πύλη:Κύρια']),
-             'English': Language(name='English',
-                                 iso_code='en',
-                                 use_ascii=True,
-                                 charsets=['ISO-8859-1', 'WINDOWS-1252'],
-                                 wiki_start_pages=[u'Main_Page']),
-             'Esperanto': Language(name='Esperanto',
-                                   iso_code='eo',
-                                   # Q, W, X, and Y not used at all
-                                   use_ascii=False,
-                                   charsets=['ISO-8859-3'],
-                                   alphabet=(u'abcĉdefgĝhĥijĵklmnoprsŝtuŭvz'
-                                             u'ABCĈDEFGĜHĤIJĴKLMNOPRSŜTUŬVZ'),
-                                   wiki_start_pages=[u'Vikipedio:Ĉefpaĝo']),
-             'Spanish': Language(name='Spanish',
-                                 iso_code='es',
-                                 use_ascii=True,
-                                 charsets=['ISO-8859-1', 'ISO-8859-15',
-                                           'WINDOWS-1252'],
-                                 alphabet=u'ñáéíóúüÑÁÉÍÓÚÜ',
-                                 wiki_start_pages=[u'Wikipedia:Portada']),
-             'Estonian': Language(name='Estonian',
-                                  iso_code='et',
-                                  use_ascii=False,
-                                  charsets=['ISO-8859-4', 'ISO-8859-13',
-                                            'WINDOWS-1257'],
-                                  # C, F, Š, Q, W, X, Y, Z, Ž are only for
-                                  # loanwords
-                                  alphabet=(u'ABDEGHIJKLMNOPRSTUVÕÄÖÜ'
-                                            u'abdeghijklmnoprstuvõäöü'),
-                                  wiki_start_pages=[u'Esileht']),
-             'Finnish': Language(name='Finnish',
-                                 iso_code='fi',
-                                 use_ascii=True,
-                                 charsets=['ISO-8859-1', 'ISO-8859-15',
-                                           'WINDOWS-1252'],
-                                 alphabet=u'ÅÄÖŠŽåäöšž',
-                                 wiki_start_pages=[u'Wikipedia:Etusivu']),
-             'French': Language(name='French',
-                                iso_code='fr',
-                                use_ascii=True,
-                                charsets=['ISO-8859-1', 'ISO-8859-15',
-                                          'WINDOWS-1252'],
-                                alphabet=u'œàâçèéîïùûêŒÀÂÇÈÉÎÏÙÛÊ',
-                                wiki_start_pages=[u'Wikipédia:Accueil_principal',
-                                                  u'Bœuf (animal)']),
-             'Hebrew': Language(name='Hebrew',
-                                iso_code='he',
-                                use_ascii=False,
-                                charsets=['ISO-8859-8', 'WINDOWS-1255'],
-                                alphabet=u'אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ',
-                                wiki_start_pages=[u'עמוד_ראשי']),
-             'Croatian': Language(name='Croatian',
-                                  iso_code='hr',
-                                  # Q, W, X, Y are only used for foreign words.
-                                  use_ascii=False,
-                                  charsets=['ISO-8859-2', 'WINDOWS-1250'],
-                                  alphabet=(u'abcčćdđefghijklmnoprsštuvzž'
-                                            u'ABCČĆDĐEFGHIJKLMNOPRSŠTUVZŽ'),
-                                  wiki_start_pages=[u'Glavna_stranica']),
-             'Hungarian': Language(name='Hungarian',
-                                   iso_code='hu',
-                                   # Q, W, X, Y are only used for foreign words.
-                                   use_ascii=False,
-                                   charsets=['ISO-8859-2', 'WINDOWS-1250'],
-                                   alphabet=(u'abcdefghijklmnoprstuvzáéíóöőúüű'
-                                             u'ABCDEFGHIJKLMNOPRSTUVZÁÉÍÓÖŐÚÜŰ'),
-                                   wiki_start_pages=[u'Kezdőlap']),
-             'Italian': Language(name='Italian',
-                                 iso_code='it',
-                                 use_ascii=True,
-                                 charsets=['ISO-8859-1', 'ISO-8859-15',
-                                           'WINDOWS-1252'],
-                                 alphabet=u'ÀÈÉÌÒÓÙàèéìòóù',
-                                 wiki_start_pages=[u'Pagina_principale']),
-             'Lithuanian': Language(name='Lithuanian',
-                                    iso_code='lt',
-                                    use_ascii=False,
-                                    charsets=['ISO-8859-13', 'WINDOWS-1257',
-                                              'ISO-8859-4'],
-                                    # Q, W, and X not used at all
-                                    alphabet=(u'AĄBCČDEĘĖFGHIĮYJKLMNOPRSŠTUŲŪVZŽ'
-                                              u'aąbcčdeęėfghiįyjklmnoprsštuųūvzž'),
-                                    wiki_start_pages=[u'Pagrindinis_puslapis']),
-             'Latvian': Language(name='Latvian',
-                                 iso_code='lv',
-                                 use_ascii=False,
-                                 charsets=['ISO-8859-13', 'WINDOWS-1257',
-                                           'ISO-8859-4'],
-                                 # Q, W, X, Y are only for loanwords
-                                 alphabet=(u'AĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽ'
-                                           u'aābcčdeēfgģhiījkķlļmnņoprsštuūvzž'),
-                                 wiki_start_pages=[u'Sākumlapa']),
-             'Macedonian': Language(name='Macedonian',
-                                    iso_code='mk',
-                                    use_ascii=False,
-                                    charsets=['ISO-8859-5', 'WINDOWS-1251',
-                                              'MacCyrillic', 'IBM855'],
-                                    alphabet=(u'АБВГДЃЕЖЗЅИЈКЛЉМНЊОПРСТЌУФХЦЧЏШ'
-                                              u'абвгдѓежзѕијклљмнњопрстќуфхцчџш'),
-                                    wiki_start_pages=[u'Главна_страница']),
-             'Dutch': Language(name='Dutch',
-                               iso_code='nl',
-                               use_ascii=True,
-                               charsets=['ISO-8859-1', 'WINDOWS-1252'],
-                               wiki_start_pages=[u'Hoofdpagina']),
-             'Polish': Language(name='Polish',
-                                iso_code='pl',
-                                # Q and X are only used for foreign words.
-                                use_ascii=False,
-                                charsets=['ISO-8859-2', 'WINDOWS-1250'],
-                                alphabet=(u'AĄBCĆDEĘFGHIJKLŁMNŃOÓPRSŚTUWYZŹŻ'
-                                          u'aąbcćdeęfghijklłmnńoóprsśtuwyzźż'),
-                                wiki_start_pages=[u'Wikipedia:Strona_główna']),
-             'Portuguese': Language(name='Portuguese',
-                                 iso_code='pt',
-                                 use_ascii=True,
-                                 charsets=['ISO-8859-1', 'ISO-8859-15',
-                                           'WINDOWS-1252'],
-                                 alphabet=u'ÁÂÃÀÇÉÊÍÓÔÕÚáâãàçéêíóôõú',
-                                 wiki_start_pages=[u'Wikipédia:Página_principal']),
-             'Romanian': Language(name='Romanian',
-                                  iso_code='ro',
-                                  use_ascii=True,
-                                  charsets=['ISO-8859-2', 'WINDOWS-1250'],
-                                  alphabet=u'ăâîșțĂÂÎȘȚ',
-                                  wiki_start_pages=[u'Pagina_principală']),
-             'Russian': Language(name='Russian',
-                                 iso_code='ru',
-                                 use_ascii=False,
-                                 charsets=['ISO-8859-5', 'WINDOWS-1251',
-                                           'KOI8-R', 'MacCyrillic', 'IBM866',
-                                           'IBM855'],
-                                 alphabet=(u'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'
-                                           u'АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ'),
-                                 wiki_start_pages=[u'Заглавная_страница']),
-             'Slovak': Language(name='Slovak',
-                                iso_code='sk',
-                                use_ascii=True,
-                                charsets=['ISO-8859-2', 'WINDOWS-1250'],
-                                alphabet=u'áäčďéíĺľňóôŕšťúýžÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ',
-                                wiki_start_pages=[u'Hlavná_stránka']),
-             'Slovene': Language(name='Slovene',
-                                 iso_code='sl',
-                                 # Q, W, X, Y are only used for foreign words.
-                                 use_ascii=False,
-                                 charsets=['ISO-8859-2', 'WINDOWS-1250'],
-                                 alphabet=(u'abcčdefghijklmnoprsštuvzž'
-                                           u'ABCČDEFGHIJKLMNOPRSŠTUVZŽ'),
-                                 wiki_start_pages=[u'Glavna_stran']),
-             # Serbian can be written in both Latin and Cyrillic, but there's no
-             # simple way to get the Latin alphabet pages from Wikipedia through
-             # the API, so for now we just support Cyrillic.
-             'Serbian': Language(name='Serbian',
-                                 iso_code='sr',
-                                 alphabet=(u'АБВГДЂЕЖЗИЈКЛЉМНЊОПРСТЋУФХЦЧЏШ'
-                                           u'абвгдђежзијклљмнњопрстћуфхцчџш'),
-                                 charsets=['ISO-8859-5', 'WINDOWS-1251',
-                                           'MacCyrillic', 'IBM855'],
-                                 wiki_start_pages=[u'Главна_страна']),
-             'Thai': Language(name='Thai',
-                              iso_code='th',
-                              use_ascii=False,
-                              charsets=['ISO-8859-11', 'TIS-620', 'CP874'],
-                              alphabet=u'กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛',
-                              wiki_start_pages=[u'หน้าหลัก']),
-             'Turkish': Language(name='Turkish',
-                                 iso_code='tr',
-                                 # Q, W, and X are not used by Turkish
-                                 use_ascii=False,
-                                 charsets=['ISO-8859-3', 'ISO-8859-9',
-                                           'WINDOWS-1254'],
-                                 alphabet=(u'abcçdefgğhıijklmnoöprsştuüvyzâîû'
-                                           u'ABCÇDEFGĞHIİJKLMNOÖPRSŞTUÜVYZÂÎÛ'),
-                                 wiki_start_pages=[u'Ana_Sayfa']),
-             'Vietnamese': Language(name='Vietnamese',
-                                    iso_code='vi',
-                                    use_ascii=False,
-                                    # Windows-1258 is the only common 8-bit
-                                    # Vietnamese encoding supported by Python.
-                                    # From Wikipedia:
-                                    # For systems that lack support for Unicode,
-                                    # dozens of 8-bit Vietnamese code pages are
-                                    # available.[1] The most common are VISCII
-                                    # (TCVN 5712:1993), VPS, and Windows-1258.[3]
-                                    # Where ASCII is required, such as when
-                                    # ensuring readability in plain text e-mail,
-                                    # Vietnamese letters are often encoded
-                                    # according to Vietnamese Quoted-Readable
-                                    # (VIQR) or VSCII Mnemonic (VSCII-MNEM),[4]
-                                    # though usage of either variable-width
-                                    # scheme has declined dramatically following
-                                    # the adoption of Unicode on the World Wide
-                                    # Web.
-                                    charsets=['WINDOWS-1258'],
-                                    alphabet=(u'aăâbcdđeêghiklmnoôơpqrstuưvxy'
-                                              u'AĂÂBCDĐEÊGHIKLMNOÔƠPQRSTUƯVXY'),
-                                    wiki_start_pages=[u'Chữ_Quốc_ngữ']),
-            }

package/.venv/lib/python3.9/site-packages/pip/_vendor/chardet/sbcharsetprober.py

@@ -1,145 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Universal charset detector code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 2001
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Mark Pilgrim - port to Python
-#   Shy Shalom - original C code
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-
-from collections import namedtuple
-
-from .charsetprober import CharSetProber
-from .enums import CharacterCategory, ProbingState, SequenceLikelihood
-
-
-SingleByteCharSetModel = namedtuple('SingleByteCharSetModel',
-                                    ['charset_name',
-                                     'language',
-                                     'char_to_order_map',
-                                     'language_model',
-                                     'typical_positive_ratio',
-                                     'keep_ascii_letters',
-                                     'alphabet'])
-
-
-class SingleByteCharSetProber(CharSetProber):
-    SAMPLE_SIZE = 64
-    SB_ENOUGH_REL_THRESHOLD = 1024  #  0.25 * SAMPLE_SIZE^2
-    POSITIVE_SHORTCUT_THRESHOLD = 0.95
-    NEGATIVE_SHORTCUT_THRESHOLD = 0.05
-
-    def __init__(self, model, reversed=False, name_prober=None):
-        super(SingleByteCharSetProber, self).__init__()
-        self._model = model
-        # TRUE if we need to reverse every pair in the model lookup
-        self._reversed = reversed
-        # Optional auxiliary prober for name decision
-        self._name_prober = name_prober
-        self._last_order = None
-        self._seq_counters = None
-        self._total_seqs = None
-        self._total_char = None
-        self._freq_char = None
-        self.reset()
-
-    def reset(self):
-        super(SingleByteCharSetProber, self).reset()
-        # char order of last character
-        self._last_order = 255
-        self._seq_counters = [0] * SequenceLikelihood.get_num_categories()
-        self._total_seqs = 0
-        self._total_char = 0
-        # characters that fall in our sampling range
-        self._freq_char = 0
-
-    @property
-    def charset_name(self):
-        if self._name_prober:
-            return self._name_prober.charset_name
-        else:
-            return self._model.charset_name
-
-    @property
-    def language(self):
-        if self._name_prober:
-            return self._name_prober.language
-        else:
-            return self._model.language
-
-    def feed(self, byte_str):
-        # TODO: Make filter_international_words keep things in self.alphabet
-        if not self._model.keep_ascii_letters:
-            byte_str = self.filter_international_words(byte_str)
-        if not byte_str:
-            return self.state
-        char_to_order_map = self._model.char_to_order_map
-        language_model = self._model.language_model
-        for char in byte_str:
-            order = char_to_order_map.get(char, CharacterCategory.UNDEFINED)
-            # XXX: This was SYMBOL_CAT_ORDER before, with a value of 250, but
-            #      CharacterCategory.SYMBOL is actually 253, so we use CONTROL
-            #      to make it closer to the original intent. The only difference
-            #      is whether or not we count digits and control characters for
-            #      _total_char purposes.
-            if order < CharacterCategory.CONTROL:
-                self._total_char += 1
-            # TODO: Follow uchardet's lead and discount confidence for frequent
-            #       control characters.
-            #       See https://github.com/BYVoid/uchardet/commit/55b4f23971db61
-            if order < self.SAMPLE_SIZE:
-                self._freq_char += 1
-                if self._last_order < self.SAMPLE_SIZE:
-                    self._total_seqs += 1
-                    if not self._reversed:
-                        lm_cat = language_model[self._last_order][order]
-                    else:
-                        lm_cat = language_model[order][self._last_order]
-                    self._seq_counters[lm_cat] += 1
-            self._last_order = order
-
-        charset_name = self._model.charset_name
-        if self.state == ProbingState.DETECTING:
-            if self._total_seqs > self.SB_ENOUGH_REL_THRESHOLD:
-                confidence = self.get_confidence()
-                if confidence > self.POSITIVE_SHORTCUT_THRESHOLD:
-                    self.logger.debug('%s confidence = %s, we have a winner',
-                                      charset_name, confidence)
-                    self._state = ProbingState.FOUND_IT
-                elif confidence < self.NEGATIVE_SHORTCUT_THRESHOLD:
-                    self.logger.debug('%s confidence = %s, below negative '
-                                      'shortcut threshhold %s', charset_name,
-                                      confidence,
-                                      self.NEGATIVE_SHORTCUT_THRESHOLD)
-                    self._state = ProbingState.NOT_ME
-
-        return self.state
-
-    def get_confidence(self):
-        r = 0.01
-        if self._total_seqs > 0:
-            r = ((1.0 * self._seq_counters[SequenceLikelihood.POSITIVE]) /
-                 self._total_seqs / self._model.typical_positive_ratio)
-            r = r * self._freq_char / self._total_char
-            if r >= 1.0:
-                r = 0.99
-        return r

package/.venv/lib/python3.9/site-packages/pip/_vendor/chardet/sbcsgroupprober.py

@@ -1,83 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is Mozilla Universal charset detector code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 2001
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Mark Pilgrim - port to Python
-#   Shy Shalom - original C code
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-
-from .charsetgroupprober import CharSetGroupProber
-from .hebrewprober import HebrewProber
-from .langbulgarianmodel import (ISO_8859_5_BULGARIAN_MODEL,
-                                 WINDOWS_1251_BULGARIAN_MODEL)
-from .langgreekmodel import ISO_8859_7_GREEK_MODEL, WINDOWS_1253_GREEK_MODEL
-from .langhebrewmodel import WINDOWS_1255_HEBREW_MODEL
-# from .langhungarianmodel import (ISO_8859_2_HUNGARIAN_MODEL,
-#                                  WINDOWS_1250_HUNGARIAN_MODEL)
-from .langrussianmodel import (IBM855_RUSSIAN_MODEL, IBM866_RUSSIAN_MODEL,
-                               ISO_8859_5_RUSSIAN_MODEL, KOI8_R_RUSSIAN_MODEL,
-                               MACCYRILLIC_RUSSIAN_MODEL,
-                               WINDOWS_1251_RUSSIAN_MODEL)
-from .langthaimodel import TIS_620_THAI_MODEL
-from .langturkishmodel import ISO_8859_9_TURKISH_MODEL
-from .sbcharsetprober import SingleByteCharSetProber
-
-
-class SBCSGroupProber(CharSetGroupProber):
-    def __init__(self):
-        super(SBCSGroupProber, self).__init__()
-        hebrew_prober = HebrewProber()
-        logical_hebrew_prober = SingleByteCharSetProber(WINDOWS_1255_HEBREW_MODEL,
-                                                        False, hebrew_prober)
-        # TODO: See if using ISO-8859-8 Hebrew model works better here, since
-        #       it's actually the visual one
-        visual_hebrew_prober = SingleByteCharSetProber(WINDOWS_1255_HEBREW_MODEL,
-                                                       True, hebrew_prober)
-        hebrew_prober.set_model_probers(logical_hebrew_prober,
-                                        visual_hebrew_prober)
-        # TODO: ORDER MATTERS HERE. I changed the order vs what was in master
-        #       and several tests failed that did not before. Some thought
-        #       should be put into the ordering, and we should consider making
-        #       order not matter here, because that is very counter-intuitive.
-        self.probers = [
-            SingleByteCharSetProber(WINDOWS_1251_RUSSIAN_MODEL),
-            SingleByteCharSetProber(KOI8_R_RUSSIAN_MODEL),
-            SingleByteCharSetProber(ISO_8859_5_RUSSIAN_MODEL),
-            SingleByteCharSetProber(MACCYRILLIC_RUSSIAN_MODEL),
-            SingleByteCharSetProber(IBM866_RUSSIAN_MODEL),
-            SingleByteCharSetProber(IBM855_RUSSIAN_MODEL),
-            SingleByteCharSetProber(ISO_8859_7_GREEK_MODEL),
-            SingleByteCharSetProber(WINDOWS_1253_GREEK_MODEL),
-            SingleByteCharSetProber(ISO_8859_5_BULGARIAN_MODEL),
-            SingleByteCharSetProber(WINDOWS_1251_BULGARIAN_MODEL),
-            # TODO: Restore Hungarian encodings (iso-8859-2 and windows-1250)
-            #       after we retrain model.
-            # SingleByteCharSetProber(ISO_8859_2_HUNGARIAN_MODEL),
-            # SingleByteCharSetProber(WINDOWS_1250_HUNGARIAN_MODEL),
-            SingleByteCharSetProber(TIS_620_THAI_MODEL),
-            SingleByteCharSetProber(ISO_8859_9_TURKISH_MODEL),
-            hebrew_prober,
-            logical_hebrew_prober,
-            visual_hebrew_prober,
-        ]
-        self.reset()

package/.venv/lib/python3.9/site-packages/pip/_vendor/chardet/sjisprober.py

@@ -1,92 +0,0 @@
-######################## BEGIN LICENSE BLOCK ########################
-# The Original Code is mozilla.org code.
-#
-# The Initial Developer of the Original Code is
-# Netscape Communications Corporation.
-# Portions created by the Initial Developer are Copyright (C) 1998
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#   Mark Pilgrim - port to Python
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
-# 02110-1301  USA
-######################### END LICENSE BLOCK #########################
-
-from .mbcharsetprober import MultiByteCharSetProber
-from .codingstatemachine import CodingStateMachine
-from .chardistribution import SJISDistributionAnalysis
-from .jpcntx import SJISContextAnalysis
-from .mbcssm import SJIS_SM_MODEL
-from .enums import ProbingState, MachineState
-
-
-class SJISProber(MultiByteCharSetProber):
-    def __init__(self):
-        super(SJISProber, self).__init__()
-        self.coding_sm = CodingStateMachine(SJIS_SM_MODEL)
-        self.distribution_analyzer = SJISDistributionAnalysis()
-        self.context_analyzer = SJISContextAnalysis()
-        self.reset()
-
-    def reset(self):
-        super(SJISProber, self).reset()
-        self.context_analyzer.reset()
-
-    @property
-    def charset_name(self):
-        return self.context_analyzer.charset_name
-
-    @property
-    def language(self):
-        return "Japanese"
-
-    def feed(self, byte_str):
-        for i in range(len(byte_str)):
-            coding_state = self.coding_sm.next_state(byte_str[i])
-            if coding_state == MachineState.ERROR:
-                self.logger.debug('%s %s prober hit error at byte %s',
-                                  self.charset_name, self.language, i)
-                self._state = ProbingState.NOT_ME
-                break
-            elif coding_state == MachineState.ITS_ME:
-                self._state = ProbingState.FOUND_IT
-                break
-            elif coding_state == MachineState.START:
-                char_len = self.coding_sm.get_current_charlen()
-                if i == 0:
-                    self._last_char[1] = byte_str[0]
-                    self.context_analyzer.feed(self._last_char[2 - char_len:],
-                                               char_len)
-                    self.distribution_analyzer.feed(self._last_char, char_len)
-                else:
-                    self.context_analyzer.feed(byte_str[i + 1 - char_len:i + 3
-                                                        - char_len], char_len)
-                    self.distribution_analyzer.feed(byte_str[i - 1:i + 1],
-                                                    char_len)
-
-        self._last_char[0] = byte_str[-1]
-
-        if self.state == ProbingState.DETECTING:
-            if (self.context_analyzer.got_enough_data() and
-               (self.get_confidence() > self.SHORTCUT_THRESHOLD)):
-                self._state = ProbingState.FOUND_IT
-
-        return self.state
-
-    def get_confidence(self):
-        context_conf = self.context_analyzer.get_confidence()
-        distrib_conf = self.distribution_analyzer.get_confidence()
-        return max(context_conf, distrib_conf)