@scrypted/server 0.0.138 → 0.0.139

package/.venv/lib/python3.9/site-packages/pip/_vendor/chardet/metadata/languages.py

@@ -0,0 +1,310 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Metadata about languages used by our model training code for our
+SingleByteCharSetProbers.  Could be used for other things in the future.
+
+This code is based on the language metadata from the uchardet project.
+"""
+from __future__ import absolute_import, print_function
+
+from string import ascii_letters
+
+
+# TODO: Add Ukranian (KOI8-U)
+
+class Language(object):
+    """Metadata about a language useful for training models
+
+    :ivar name: The human name for the language, in English.
+    :type name: str
+    :ivar iso_code: 2-letter ISO 639-1 if possible, 3-letter ISO code otherwise,
+                    or use another catalog as a last resort.
+    :type iso_code: str
+    :ivar use_ascii: Whether or not ASCII letters should be included in trained
+                     models.
+    :type use_ascii: bool
+    :ivar charsets: The charsets we want to support and create data for.
+    :type charsets: list of str
+    :ivar alphabet: The characters in the language's alphabet. If `use_ascii` is
+                    `True`, you only need to add those not in the ASCII set.
+    :type alphabet: str
+    :ivar wiki_start_pages: The Wikipedia pages to start from if we're crawling
+                            Wikipedia for training data.
+    :type wiki_start_pages: list of str
+    """
+    def __init__(self, name=None, iso_code=None, use_ascii=True, charsets=None,
+                 alphabet=None, wiki_start_pages=None):
+        super(Language, self).__init__()
+        self.name = name
+        self.iso_code = iso_code
+        self.use_ascii = use_ascii
+        self.charsets = charsets
+        if self.use_ascii:
+            if alphabet:
+                alphabet += ascii_letters
+            else:
+                alphabet = ascii_letters
+        elif not alphabet:
+            raise ValueError('Must supply alphabet if use_ascii is False')
+        self.alphabet = ''.join(sorted(set(alphabet))) if alphabet else None
+        self.wiki_start_pages = wiki_start_pages
+
+    def __repr__(self):
+        return '{}({})'.format(self.__class__.__name__,
+                               ', '.join('{}={!r}'.format(k, v)
+                                         for k, v in self.__dict__.items()
+                                         if not k.startswith('_')))
+
+
+LANGUAGES = {'Arabic': Language(name='Arabic',
+                                iso_code='ar',
+                                use_ascii=False,
+                                # We only support encodings that use isolated
+                                # forms, because the current recommendation is
+                                # that the rendering system handles presentation
+                                # forms. This means we purposefully skip IBM864.
+                                charsets=['ISO-8859-6', 'WINDOWS-1256',
+                                          'CP720', 'CP864'],
+                                alphabet=u'ءآأؤإئابةتثجحخدذرزسشصضطظعغػؼؽؾؿـفقكلمنهوىيًٌٍَُِّ',
+                                wiki_start_pages=[u'الصفحة_الرئيسية']),
+             'Belarusian': Language(name='Belarusian',
+                                    iso_code='be',
+                                    use_ascii=False,
+                                    charsets=['ISO-8859-5', 'WINDOWS-1251',
+                                              'IBM866', 'MacCyrillic'],
+                                    alphabet=(u'АБВГДЕЁЖЗІЙКЛМНОПРСТУЎФХЦЧШЫЬЭЮЯ'
+                                              u'абвгдеёжзійклмнопрстуўфхцчшыьэюяʼ'),
+                                    wiki_start_pages=[u'Галоўная_старонка']),
+             'Bulgarian': Language(name='Bulgarian',
+                                   iso_code='bg',
+                                   use_ascii=False,
+                                   charsets=['ISO-8859-5', 'WINDOWS-1251',
+                                             'IBM855'],
+                                   alphabet=(u'АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯ'
+                                             u'абвгдежзийклмнопрстуфхцчшщъьюя'),
+                                   wiki_start_pages=[u'Начална_страница']),
+             'Czech': Language(name='Czech',
+                               iso_code='cz',
+                               use_ascii=True,
+                               charsets=['ISO-8859-2', 'WINDOWS-1250'],
+                               alphabet=u'áčďéěíňóřšťúůýžÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ',
+                               wiki_start_pages=[u'Hlavní_strana']),
+             'Danish': Language(name='Danish',
+                                iso_code='da',
+                                use_ascii=True,
+                                charsets=['ISO-8859-1', 'ISO-8859-15',
+                                          'WINDOWS-1252'],
+                                alphabet=u'æøåÆØÅ',
+                                wiki_start_pages=[u'Forside']),
+             'German': Language(name='German',
+                                iso_code='de',
+                                use_ascii=True,
+                                charsets=['ISO-8859-1', 'WINDOWS-1252'],
+                                alphabet=u'äöüßÄÖÜ',
+                                wiki_start_pages=[u'Wikipedia:Hauptseite']),
+             'Greek': Language(name='Greek',
+                               iso_code='el',
+                               use_ascii=False,
+                               charsets=['ISO-8859-7', 'WINDOWS-1253'],
+                               alphabet=(u'αβγδεζηθικλμνξοπρσςτυφχψωάέήίόύώ'
+                                         u'ΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΣΤΥΦΧΨΩΆΈΉΊΌΎΏ'),
+                               wiki_start_pages=[u'Πύλη:Κύρια']),
+             'English': Language(name='English',
+                                 iso_code='en',
+                                 use_ascii=True,
+                                 charsets=['ISO-8859-1', 'WINDOWS-1252'],
+                                 wiki_start_pages=[u'Main_Page']),
+             'Esperanto': Language(name='Esperanto',
+                                   iso_code='eo',
+                                   # Q, W, X, and Y not used at all
+                                   use_ascii=False,
+                                   charsets=['ISO-8859-3'],
+                                   alphabet=(u'abcĉdefgĝhĥijĵklmnoprsŝtuŭvz'
+                                             u'ABCĈDEFGĜHĤIJĴKLMNOPRSŜTUŬVZ'),
+                                   wiki_start_pages=[u'Vikipedio:Ĉefpaĝo']),
+             'Spanish': Language(name='Spanish',
+                                 iso_code='es',
+                                 use_ascii=True,
+                                 charsets=['ISO-8859-1', 'ISO-8859-15',
+                                           'WINDOWS-1252'],
+                                 alphabet=u'ñáéíóúüÑÁÉÍÓÚÜ',
+                                 wiki_start_pages=[u'Wikipedia:Portada']),
+             'Estonian': Language(name='Estonian',
+                                  iso_code='et',
+                                  use_ascii=False,
+                                  charsets=['ISO-8859-4', 'ISO-8859-13',
+                                            'WINDOWS-1257'],
+                                  # C, F, Š, Q, W, X, Y, Z, Ž are only for
+                                  # loanwords
+                                  alphabet=(u'ABDEGHIJKLMNOPRSTUVÕÄÖÜ'
+                                            u'abdeghijklmnoprstuvõäöü'),
+                                  wiki_start_pages=[u'Esileht']),
+             'Finnish': Language(name='Finnish',
+                                 iso_code='fi',
+                                 use_ascii=True,
+                                 charsets=['ISO-8859-1', 'ISO-8859-15',
+                                           'WINDOWS-1252'],
+                                 alphabet=u'ÅÄÖŠŽåäöšž',
+                                 wiki_start_pages=[u'Wikipedia:Etusivu']),
+             'French': Language(name='French',
+                                iso_code='fr',
+                                use_ascii=True,
+                                charsets=['ISO-8859-1', 'ISO-8859-15',
+                                          'WINDOWS-1252'],
+                                alphabet=u'œàâçèéîïùûêŒÀÂÇÈÉÎÏÙÛÊ',
+                                wiki_start_pages=[u'Wikipédia:Accueil_principal',
+                                                  u'Bœuf (animal)']),
+             'Hebrew': Language(name='Hebrew',
+                                iso_code='he',
+                                use_ascii=False,
+                                charsets=['ISO-8859-8', 'WINDOWS-1255'],
+                                alphabet=u'אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ',
+                                wiki_start_pages=[u'עמוד_ראשי']),
+             'Croatian': Language(name='Croatian',
+                                  iso_code='hr',
+                                  # Q, W, X, Y are only used for foreign words.
+                                  use_ascii=False,
+                                  charsets=['ISO-8859-2', 'WINDOWS-1250'],
+                                  alphabet=(u'abcčćdđefghijklmnoprsštuvzž'
+                                            u'ABCČĆDĐEFGHIJKLMNOPRSŠTUVZŽ'),
+                                  wiki_start_pages=[u'Glavna_stranica']),
+             'Hungarian': Language(name='Hungarian',
+                                   iso_code='hu',
+                                   # Q, W, X, Y are only used for foreign words.
+                                   use_ascii=False,
+                                   charsets=['ISO-8859-2', 'WINDOWS-1250'],
+                                   alphabet=(u'abcdefghijklmnoprstuvzáéíóöőúüű'
+                                             u'ABCDEFGHIJKLMNOPRSTUVZÁÉÍÓÖŐÚÜŰ'),
+                                   wiki_start_pages=[u'Kezdőlap']),
+             'Italian': Language(name='Italian',
+                                 iso_code='it',
+                                 use_ascii=True,
+                                 charsets=['ISO-8859-1', 'ISO-8859-15',
+                                           'WINDOWS-1252'],
+                                 alphabet=u'ÀÈÉÌÒÓÙàèéìòóù',
+                                 wiki_start_pages=[u'Pagina_principale']),
+             'Lithuanian': Language(name='Lithuanian',
+                                    iso_code='lt',
+                                    use_ascii=False,
+                                    charsets=['ISO-8859-13', 'WINDOWS-1257',
+                                              'ISO-8859-4'],
+                                    # Q, W, and X not used at all
+                                    alphabet=(u'AĄBCČDEĘĖFGHIĮYJKLMNOPRSŠTUŲŪVZŽ'
+                                              u'aąbcčdeęėfghiįyjklmnoprsštuųūvzž'),
+                                    wiki_start_pages=[u'Pagrindinis_puslapis']),
+             'Latvian': Language(name='Latvian',
+                                 iso_code='lv',
+                                 use_ascii=False,
+                                 charsets=['ISO-8859-13', 'WINDOWS-1257',
+                                           'ISO-8859-4'],
+                                 # Q, W, X, Y are only for loanwords
+                                 alphabet=(u'AĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽ'
+                                           u'aābcčdeēfgģhiījkķlļmnņoprsštuūvzž'),
+                                 wiki_start_pages=[u'Sākumlapa']),
+             'Macedonian': Language(name='Macedonian',
+                                    iso_code='mk',
+                                    use_ascii=False,
+                                    charsets=['ISO-8859-5', 'WINDOWS-1251',
+                                              'MacCyrillic', 'IBM855'],
+                                    alphabet=(u'АБВГДЃЕЖЗЅИЈКЛЉМНЊОПРСТЌУФХЦЧЏШ'
+                                              u'абвгдѓежзѕијклљмнњопрстќуфхцчџш'),
+                                    wiki_start_pages=[u'Главна_страница']),
+             'Dutch': Language(name='Dutch',
+                               iso_code='nl',
+                               use_ascii=True,
+                               charsets=['ISO-8859-1', 'WINDOWS-1252'],
+                               wiki_start_pages=[u'Hoofdpagina']),
+             'Polish': Language(name='Polish',
+                                iso_code='pl',
+                                # Q and X are only used for foreign words.
+                                use_ascii=False,
+                                charsets=['ISO-8859-2', 'WINDOWS-1250'],
+                                alphabet=(u'AĄBCĆDEĘFGHIJKLŁMNŃOÓPRSŚTUWYZŹŻ'
+                                          u'aąbcćdeęfghijklłmnńoóprsśtuwyzźż'),
+                                wiki_start_pages=[u'Wikipedia:Strona_główna']),
+             'Portuguese': Language(name='Portuguese',
+                                 iso_code='pt',
+                                 use_ascii=True,
+                                 charsets=['ISO-8859-1', 'ISO-8859-15',
+                                           'WINDOWS-1252'],
+                                 alphabet=u'ÁÂÃÀÇÉÊÍÓÔÕÚáâãàçéêíóôõú',
+                                 wiki_start_pages=[u'Wikipédia:Página_principal']),
+             'Romanian': Language(name='Romanian',
+                                  iso_code='ro',
+                                  use_ascii=True,
+                                  charsets=['ISO-8859-2', 'WINDOWS-1250'],
+                                  alphabet=u'ăâîșțĂÂÎȘȚ',
+                                  wiki_start_pages=[u'Pagina_principală']),
+             'Russian': Language(name='Russian',
+                                 iso_code='ru',
+                                 use_ascii=False,
+                                 charsets=['ISO-8859-5', 'WINDOWS-1251',
+                                           'KOI8-R', 'MacCyrillic', 'IBM866',
+                                           'IBM855'],
+                                 alphabet=(u'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'
+                                           u'АБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ'),
+                                 wiki_start_pages=[u'Заглавная_страница']),
+             'Slovak': Language(name='Slovak',
+                                iso_code='sk',
+                                use_ascii=True,
+                                charsets=['ISO-8859-2', 'WINDOWS-1250'],
+                                alphabet=u'áäčďéíĺľňóôŕšťúýžÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ',
+                                wiki_start_pages=[u'Hlavná_stránka']),
+             'Slovene': Language(name='Slovene',
+                                 iso_code='sl',
+                                 # Q, W, X, Y are only used for foreign words.
+                                 use_ascii=False,
+                                 charsets=['ISO-8859-2', 'WINDOWS-1250'],
+                                 alphabet=(u'abcčdefghijklmnoprsštuvzž'
+                                           u'ABCČDEFGHIJKLMNOPRSŠTUVZŽ'),
+                                 wiki_start_pages=[u'Glavna_stran']),
+             # Serbian can be written in both Latin and Cyrillic, but there's no
+             # simple way to get the Latin alphabet pages from Wikipedia through
+             # the API, so for now we just support Cyrillic.
+             'Serbian': Language(name='Serbian',
+                                 iso_code='sr',
+                                 alphabet=(u'АБВГДЂЕЖЗИЈКЛЉМНЊОПРСТЋУФХЦЧЏШ'
+                                           u'абвгдђежзијклљмнњопрстћуфхцчџш'),
+                                 charsets=['ISO-8859-5', 'WINDOWS-1251',
+                                           'MacCyrillic', 'IBM855'],
+                                 wiki_start_pages=[u'Главна_страна']),
+             'Thai': Language(name='Thai',
+                              iso_code='th',
+                              use_ascii=False,
+                              charsets=['ISO-8859-11', 'TIS-620', 'CP874'],
+                              alphabet=u'กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛',
+                              wiki_start_pages=[u'หน้าหลัก']),
+             'Turkish': Language(name='Turkish',
+                                 iso_code='tr',
+                                 # Q, W, and X are not used by Turkish
+                                 use_ascii=False,
+                                 charsets=['ISO-8859-3', 'ISO-8859-9',
+                                           'WINDOWS-1254'],
+                                 alphabet=(u'abcçdefgğhıijklmnoöprsştuüvyzâîû'
+                                           u'ABCÇDEFGĞHIİJKLMNOÖPRSŞTUÜVYZÂÎÛ'),
+                                 wiki_start_pages=[u'Ana_Sayfa']),
+             'Vietnamese': Language(name='Vietnamese',
+                                    iso_code='vi',
+                                    use_ascii=False,
+                                    # Windows-1258 is the only common 8-bit
+                                    # Vietnamese encoding supported by Python.
+                                    # From Wikipedia:
+                                    # For systems that lack support for Unicode,
+                                    # dozens of 8-bit Vietnamese code pages are
+                                    # available.[1] The most common are VISCII
+                                    # (TCVN 5712:1993), VPS, and Windows-1258.[3]
+                                    # Where ASCII is required, such as when
+                                    # ensuring readability in plain text e-mail,
+                                    # Vietnamese letters are often encoded
+                                    # according to Vietnamese Quoted-Readable
+                                    # (VIQR) or VSCII Mnemonic (VSCII-MNEM),[4]
+                                    # though usage of either variable-width
+                                    # scheme has declined dramatically following
+                                    # the adoption of Unicode on the World Wide
+                                    # Web.
+                                    charsets=['WINDOWS-1258'],
+                                    alphabet=(u'aăâbcdđeêghiklmnoôơpqrstuưvxy'
+                                              u'AĂÂBCDĐEÊGHIKLMNOÔƠPQRSTUƯVXY'),
+                                    wiki_start_pages=[u'Chữ_Quốc_ngữ']),
+            }

package/.venv/lib/python3.9/site-packages/pip/_vendor/chardet/sbcharsetprober.py

@@ -0,0 +1,145 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from collections import namedtuple
+
+from .charsetprober import CharSetProber
+from .enums import CharacterCategory, ProbingState, SequenceLikelihood
+
+
+SingleByteCharSetModel = namedtuple('SingleByteCharSetModel',
+                                    ['charset_name',
+                                     'language',
+                                     'char_to_order_map',
+                                     'language_model',
+                                     'typical_positive_ratio',
+                                     'keep_ascii_letters',
+                                     'alphabet'])
+
+
+class SingleByteCharSetProber(CharSetProber):
+    SAMPLE_SIZE = 64
+    SB_ENOUGH_REL_THRESHOLD = 1024  #  0.25 * SAMPLE_SIZE^2
+    POSITIVE_SHORTCUT_THRESHOLD = 0.95
+    NEGATIVE_SHORTCUT_THRESHOLD = 0.05
+
+    def __init__(self, model, reversed=False, name_prober=None):
+        super(SingleByteCharSetProber, self).__init__()
+        self._model = model
+        # TRUE if we need to reverse every pair in the model lookup
+        self._reversed = reversed
+        # Optional auxiliary prober for name decision
+        self._name_prober = name_prober
+        self._last_order = None
+        self._seq_counters = None
+        self._total_seqs = None
+        self._total_char = None
+        self._freq_char = None
+        self.reset()
+
+    def reset(self):
+        super(SingleByteCharSetProber, self).reset()
+        # char order of last character
+        self._last_order = 255
+        self._seq_counters = [0] * SequenceLikelihood.get_num_categories()
+        self._total_seqs = 0
+        self._total_char = 0
+        # characters that fall in our sampling range
+        self._freq_char = 0
+
+    @property
+    def charset_name(self):
+        if self._name_prober:
+            return self._name_prober.charset_name
+        else:
+            return self._model.charset_name
+
+    @property
+    def language(self):
+        if self._name_prober:
+            return self._name_prober.language
+        else:
+            return self._model.language
+
+    def feed(self, byte_str):
+        # TODO: Make filter_international_words keep things in self.alphabet
+        if not self._model.keep_ascii_letters:
+            byte_str = self.filter_international_words(byte_str)
+        if not byte_str:
+            return self.state
+        char_to_order_map = self._model.char_to_order_map
+        language_model = self._model.language_model
+        for char in byte_str:
+            order = char_to_order_map.get(char, CharacterCategory.UNDEFINED)
+            # XXX: This was SYMBOL_CAT_ORDER before, with a value of 250, but
+            #      CharacterCategory.SYMBOL is actually 253, so we use CONTROL
+            #      to make it closer to the original intent. The only difference
+            #      is whether or not we count digits and control characters for
+            #      _total_char purposes.
+            if order < CharacterCategory.CONTROL:
+                self._total_char += 1
+            # TODO: Follow uchardet's lead and discount confidence for frequent
+            #       control characters.
+            #       See https://github.com/BYVoid/uchardet/commit/55b4f23971db61
+            if order < self.SAMPLE_SIZE:
+                self._freq_char += 1
+                if self._last_order < self.SAMPLE_SIZE:
+                    self._total_seqs += 1
+                    if not self._reversed:
+                        lm_cat = language_model[self._last_order][order]
+                    else:
+                        lm_cat = language_model[order][self._last_order]
+                    self._seq_counters[lm_cat] += 1
+            self._last_order = order
+
+        charset_name = self._model.charset_name
+        if self.state == ProbingState.DETECTING:
+            if self._total_seqs > self.SB_ENOUGH_REL_THRESHOLD:
+                confidence = self.get_confidence()
+                if confidence > self.POSITIVE_SHORTCUT_THRESHOLD:
+                    self.logger.debug('%s confidence = %s, we have a winner',
+                                      charset_name, confidence)
+                    self._state = ProbingState.FOUND_IT
+                elif confidence < self.NEGATIVE_SHORTCUT_THRESHOLD:
+                    self.logger.debug('%s confidence = %s, below negative '
+                                      'shortcut threshhold %s', charset_name,
+                                      confidence,
+                                      self.NEGATIVE_SHORTCUT_THRESHOLD)
+                    self._state = ProbingState.NOT_ME
+
+        return self.state
+
+    def get_confidence(self):
+        r = 0.01
+        if self._total_seqs > 0:
+            r = ((1.0 * self._seq_counters[SequenceLikelihood.POSITIVE]) /
+                 self._total_seqs / self._model.typical_positive_ratio)
+            r = r * self._freq_char / self._total_char
+            if r >= 1.0:
+                r = 0.99
+        return r

package/.venv/lib/python3.9/site-packages/pip/_vendor/chardet/sbcsgroupprober.py

@@ -0,0 +1,83 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is Mozilla Universal charset detector code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 2001
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#   Shy Shalom - original C code
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from .charsetgroupprober import CharSetGroupProber
+from .hebrewprober import HebrewProber
+from .langbulgarianmodel import (ISO_8859_5_BULGARIAN_MODEL,
+                                 WINDOWS_1251_BULGARIAN_MODEL)
+from .langgreekmodel import ISO_8859_7_GREEK_MODEL, WINDOWS_1253_GREEK_MODEL
+from .langhebrewmodel import WINDOWS_1255_HEBREW_MODEL
+# from .langhungarianmodel import (ISO_8859_2_HUNGARIAN_MODEL,
+#                                  WINDOWS_1250_HUNGARIAN_MODEL)
+from .langrussianmodel import (IBM855_RUSSIAN_MODEL, IBM866_RUSSIAN_MODEL,
+                               ISO_8859_5_RUSSIAN_MODEL, KOI8_R_RUSSIAN_MODEL,
+                               MACCYRILLIC_RUSSIAN_MODEL,
+                               WINDOWS_1251_RUSSIAN_MODEL)
+from .langthaimodel import TIS_620_THAI_MODEL
+from .langturkishmodel import ISO_8859_9_TURKISH_MODEL
+from .sbcharsetprober import SingleByteCharSetProber
+
+
+class SBCSGroupProber(CharSetGroupProber):
+    def __init__(self):
+        super(SBCSGroupProber, self).__init__()
+        hebrew_prober = HebrewProber()
+        logical_hebrew_prober = SingleByteCharSetProber(WINDOWS_1255_HEBREW_MODEL,
+                                                        False, hebrew_prober)
+        # TODO: See if using ISO-8859-8 Hebrew model works better here, since
+        #       it's actually the visual one
+        visual_hebrew_prober = SingleByteCharSetProber(WINDOWS_1255_HEBREW_MODEL,
+                                                       True, hebrew_prober)
+        hebrew_prober.set_model_probers(logical_hebrew_prober,
+                                        visual_hebrew_prober)
+        # TODO: ORDER MATTERS HERE. I changed the order vs what was in master
+        #       and several tests failed that did not before. Some thought
+        #       should be put into the ordering, and we should consider making
+        #       order not matter here, because that is very counter-intuitive.
+        self.probers = [
+            SingleByteCharSetProber(WINDOWS_1251_RUSSIAN_MODEL),
+            SingleByteCharSetProber(KOI8_R_RUSSIAN_MODEL),
+            SingleByteCharSetProber(ISO_8859_5_RUSSIAN_MODEL),
+            SingleByteCharSetProber(MACCYRILLIC_RUSSIAN_MODEL),
+            SingleByteCharSetProber(IBM866_RUSSIAN_MODEL),
+            SingleByteCharSetProber(IBM855_RUSSIAN_MODEL),
+            SingleByteCharSetProber(ISO_8859_7_GREEK_MODEL),
+            SingleByteCharSetProber(WINDOWS_1253_GREEK_MODEL),
+            SingleByteCharSetProber(ISO_8859_5_BULGARIAN_MODEL),
+            SingleByteCharSetProber(WINDOWS_1251_BULGARIAN_MODEL),
+            # TODO: Restore Hungarian encodings (iso-8859-2 and windows-1250)
+            #       after we retrain model.
+            # SingleByteCharSetProber(ISO_8859_2_HUNGARIAN_MODEL),
+            # SingleByteCharSetProber(WINDOWS_1250_HUNGARIAN_MODEL),
+            SingleByteCharSetProber(TIS_620_THAI_MODEL),
+            SingleByteCharSetProber(ISO_8859_9_TURKISH_MODEL),
+            hebrew_prober,
+            logical_hebrew_prober,
+            visual_hebrew_prober,
+        ]
+        self.reset()

package/.venv/lib/python3.9/site-packages/pip/_vendor/chardet/sjisprober.py

@@ -0,0 +1,92 @@
+######################## BEGIN LICENSE BLOCK ########################
+# The Original Code is mozilla.org code.
+#
+# The Initial Developer of the Original Code is
+# Netscape Communications Corporation.
+# Portions created by the Initial Developer are Copyright (C) 1998
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   Mark Pilgrim - port to Python
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301  USA
+######################### END LICENSE BLOCK #########################
+
+from .mbcharsetprober import MultiByteCharSetProber
+from .codingstatemachine import CodingStateMachine
+from .chardistribution import SJISDistributionAnalysis
+from .jpcntx import SJISContextAnalysis
+from .mbcssm import SJIS_SM_MODEL
+from .enums import ProbingState, MachineState
+
+
+class SJISProber(MultiByteCharSetProber):
+    def __init__(self):
+        super(SJISProber, self).__init__()
+        self.coding_sm = CodingStateMachine(SJIS_SM_MODEL)
+        self.distribution_analyzer = SJISDistributionAnalysis()
+        self.context_analyzer = SJISContextAnalysis()
+        self.reset()
+
+    def reset(self):
+        super(SJISProber, self).reset()
+        self.context_analyzer.reset()
+
+    @property
+    def charset_name(self):
+        return self.context_analyzer.charset_name
+
+    @property
+    def language(self):
+        return "Japanese"
+
+    def feed(self, byte_str):
+        for i in range(len(byte_str)):
+            coding_state = self.coding_sm.next_state(byte_str[i])
+            if coding_state == MachineState.ERROR:
+                self.logger.debug('%s %s prober hit error at byte %s',
+                                  self.charset_name, self.language, i)
+                self._state = ProbingState.NOT_ME
+                break
+            elif coding_state == MachineState.ITS_ME:
+                self._state = ProbingState.FOUND_IT
+                break
+            elif coding_state == MachineState.START:
+                char_len = self.coding_sm.get_current_charlen()
+                if i == 0:
+                    self._last_char[1] = byte_str[0]
+                    self.context_analyzer.feed(self._last_char[2 - char_len:],
+                                               char_len)
+                    self.distribution_analyzer.feed(self._last_char, char_len)
+                else:
+                    self.context_analyzer.feed(byte_str[i + 1 - char_len:i + 3
+                                                        - char_len], char_len)
+                    self.distribution_analyzer.feed(byte_str[i - 1:i + 1],
+                                                    char_len)
+
+        self._last_char[0] = byte_str[-1]
+
+        if self.state == ProbingState.DETECTING:
+            if (self.context_analyzer.got_enough_data() and
+               (self.get_confidence() > self.SHORTCUT_THRESHOLD)):
+                self._state = ProbingState.FOUND_IT
+
+        return self.state
+
+    def get_confidence(self):
+        context_conf = self.context_analyzer.get_confidence()
+        distrib_conf = self.distribution_analyzer.get_confidence()
+        return max(context_conf, distrib_conf)