@tushar-br/desktop 1.0.282 → 1.0.283

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/postprocessor/sponsorblock.py

@@ -0,0 +1,104 @@
+import hashlib
+import json
+import re
+import urllib.parse
+
+from .ffmpeg import FFmpegPostProcessor
+
+
+class SponsorBlockPP(FFmpegPostProcessor):
+    # https://wiki.sponsor.ajay.app/w/Types
+    EXTRACTORS = {
+        'Youtube': 'YouTube',
+    }
+    POI_CATEGORIES = {
+        'poi_highlight': 'Highlight',
+    }
+    NON_SKIPPABLE_CATEGORIES = {
+        **POI_CATEGORIES,
+        'chapter': 'Chapter',
+    }
+    CATEGORIES = {
+        'sponsor': 'Sponsor',
+        'intro': 'Intermission/Intro Animation',
+        'outro': 'Endcards/Credits',
+        'selfpromo': 'Unpaid/Self Promotion',
+        'preview': 'Preview/Recap',
+        'filler': 'Filler Tangent',
+        'interaction': 'Interaction Reminder',
+        'music_offtopic': 'Non-Music Section',
+        **NON_SKIPPABLE_CATEGORIES
+    }
+
+    def __init__(self, downloader, categories=None, api='https://sponsor.ajay.app'):
+        FFmpegPostProcessor.__init__(self, downloader)
+        self._categories = tuple(categories or self.CATEGORIES.keys())
+        self._API_URL = api if re.match('^https?://', api) else 'https://' + api
+
+    def run(self, info):
+        extractor = info['extractor_key']
+        if extractor not in self.EXTRACTORS:
+            self.to_screen(f'SponsorBlock is not supported for {extractor}')
+            return [], info
+
+        self.to_screen('Fetching SponsorBlock segments')
+        info['sponsorblock_chapters'] = self._get_sponsor_chapters(info, info.get('duration'))
+        return [], info
+
+    def _get_sponsor_chapters(self, info, duration):
+        segments = self._get_sponsor_segments(info['id'], self.EXTRACTORS[info['extractor_key']])
+
+        def duration_filter(s):
+            start_end = s['segment']
+            # Ignore entire video segments (https://wiki.sponsor.ajay.app/w/Types).
+            if start_end == (0, 0):
+                return False
+            # Ignore milliseconds difference at the start.
+            if start_end[0] <= 1:
+                start_end[0] = 0
+            # Make POI chapters 1 sec so that we can properly mark them
+            if s['category'] in self.POI_CATEGORIES.keys():
+                start_end[1] += 1
+            # Ignore milliseconds difference at the end.
+            # Never allow the segment to exceed the video.
+            if duration and duration - start_end[1] <= 1:
+                start_end[1] = duration
+            # SponsorBlock duration may be absent or it may deviate from the real one.
+            diff = abs(duration - s['videoDuration']) if s['videoDuration'] else 0
+            return diff < 1 or (diff < 5 and diff / (start_end[1] - start_end[0]) < 0.05)
+
+        duration_match = [s for s in segments if duration_filter(s)]
+        if len(duration_match) != len(segments):
+            self.report_warning('Some SponsorBlock segments are from a video of different duration, maybe from an old version of this video')
+
+        def to_chapter(s):
+            (start, end), cat = s['segment'], s['category']
+            title = s['description'] if cat == 'chapter' else self.CATEGORIES[cat]
+            return {
+                'start_time': start,
+                'end_time': end,
+                'category': cat,
+                'title': title,
+                'type': s['actionType'],
+                '_categories': [(cat, start, end, title)],
+            }
+
+        sponsor_chapters = [to_chapter(s) for s in duration_match]
+        if not sponsor_chapters:
+            self.to_screen('No matching segments were found in the SponsorBlock database')
+        else:
+            self.to_screen(f'Found {len(sponsor_chapters)} segments in the SponsorBlock database')
+        return sponsor_chapters
+
+    def _get_sponsor_segments(self, video_id, service):
+        hash = hashlib.sha256(video_id.encode('ascii')).hexdigest()
+        # SponsorBlock API recommends using first 4 hash characters.
+        url = f'{self._API_URL}/api/skipSegments/{hash[:4]}?' + urllib.parse.urlencode({
+            'service': service,
+            'categories': json.dumps(self._categories),
+            'actionTypes': json.dumps(['skip', 'poi', 'chapter'])
+        })
+        for d in self._download_json(url) or []:
+            if d['videoID'] == video_id:
+                return d['segments']
+        return []

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/postprocessor/xattrpp.py

@@ -0,0 +1,63 @@
+import os
+
+from .common import PostProcessor
+from ..compat import compat_os_name
+from ..utils import (
+    PostProcessingError,
+    XAttrMetadataError,
+    XAttrUnavailableError,
+    hyphenate_date,
+    write_xattr,
+)
+
+
+class XAttrMetadataPP(PostProcessor):
+    """Set extended attributes on downloaded file (if xattr support is found)
+
+    More info about extended attributes for media:
+        http://freedesktop.org/wiki/CommonExtendedAttributes/
+        http://www.freedesktop.org/wiki/PhreedomDraft/
+        http://dublincore.org/documents/usageguide/elements.shtml
+
+    TODO:
+        * capture youtube keywords and put them in 'user.dublincore.subject' (comma-separated)
+        * figure out which xattrs can be used for 'duration', 'thumbnail', 'resolution'
+    """
+
+    XATTR_MAPPING = {
+        'user.xdg.referrer.url': 'webpage_url',
+        # 'user.xdg.comment': 'description',
+        'user.dublincore.title': 'title',
+        'user.dublincore.date': 'upload_date',
+        'user.dublincore.description': 'description',
+        'user.dublincore.contributor': 'uploader',
+        'user.dublincore.format': 'format',
+    }
+
+    def run(self, info):
+        mtime = os.stat(info['filepath']).st_mtime
+        self.to_screen('Writing metadata to file\'s xattrs')
+        try:
+            for xattrname, infoname in self.XATTR_MAPPING.items():
+                value = info.get(infoname)
+                if value:
+                    if infoname == 'upload_date':
+                        value = hyphenate_date(value)
+                    write_xattr(info['filepath'], xattrname, value.encode())
+
+        except XAttrUnavailableError as e:
+            raise PostProcessingError(str(e))
+        except XAttrMetadataError as e:
+            if e.reason == 'NO_SPACE':
+                self.report_warning(
+                    'There\'s no disk space left, disk quota exceeded or filesystem xattr limit exceeded. '
+                    'Some extended attributes are not written')
+            elif e.reason == 'VALUE_TOO_LONG':
+                self.report_warning('Unable to write extended attributes due to too long values.')
+            else:
+                tip = ('You need to use NTFS' if compat_os_name == 'nt'
+                       else 'You may have to enable them in your "/etc/fstab"')
+                raise PostProcessingError(f'This filesystem doesn\'t support extended attributes. {tip}')
+
+        self.try_utime(info['filepath'], mtime, mtime)
+        return [], info

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/socks.py

@@ -0,0 +1,274 @@
+# Public Domain SOCKS proxy protocol implementation
+# Adapted from https://gist.github.com/bluec0re/cafd3764412967417fd3
+# References:
+# SOCKS4 protocol http://www.openssh.com/txt/socks4.protocol
+# SOCKS4A protocol http://www.openssh.com/txt/socks4a.protocol
+# SOCKS5 protocol https://tools.ietf.org/html/rfc1928
+# SOCKS5 username/password authentication https://tools.ietf.org/html/rfc1929
+
+import collections
+import socket
+import struct
+
+from .compat import compat_ord
+
+__author__ = 'Timo Schmid <coding@timoschmid.de>'
+
+SOCKS4_VERSION = 4
+SOCKS4_REPLY_VERSION = 0x00
+# Excerpt from SOCKS4A protocol:
+# if the client cannot resolve the destination host's domain name to find its
+# IP address, it should set the first three bytes of DSTIP to NULL and the last
+# byte to a non-zero value.
+SOCKS4_DEFAULT_DSTIP = struct.pack('!BBBB', 0, 0, 0, 0xFF)
+
+SOCKS5_VERSION = 5
+SOCKS5_USER_AUTH_VERSION = 0x01
+SOCKS5_USER_AUTH_SUCCESS = 0x00
+
+
+class Socks4Command:
+    CMD_CONNECT = 0x01
+    CMD_BIND = 0x02
+
+
+class Socks5Command(Socks4Command):
+    CMD_UDP_ASSOCIATE = 0x03
+
+
+class Socks5Auth:
+    AUTH_NONE = 0x00
+    AUTH_GSSAPI = 0x01
+    AUTH_USER_PASS = 0x02
+    AUTH_NO_ACCEPTABLE = 0xFF  # For server response
+
+
+class Socks5AddressType:
+    ATYP_IPV4 = 0x01
+    ATYP_DOMAINNAME = 0x03
+    ATYP_IPV6 = 0x04
+
+
+class ProxyError(OSError):
+    ERR_SUCCESS = 0x00
+
+    def __init__(self, code=None, msg=None):
+        if code is not None and msg is None:
+            msg = self.CODES.get(code) or 'unknown error'
+        super().__init__(code, msg)
+
+
+class InvalidVersionError(ProxyError):
+    def __init__(self, expected_version, got_version):
+        msg = ('Invalid response version from server. Expected {:02x} got '
+               '{:02x}'.format(expected_version, got_version))
+        super().__init__(0, msg)
+
+
+class Socks4Error(ProxyError):
+    ERR_SUCCESS = 90
+
+    CODES = {
+        91: 'request rejected or failed',
+        92: 'request rejected because SOCKS server cannot connect to identd on the client',
+        93: 'request rejected because the client program and identd report different user-ids'
+    }
+
+
+class Socks5Error(ProxyError):
+    ERR_GENERAL_FAILURE = 0x01
+
+    CODES = {
+        0x01: 'general SOCKS server failure',
+        0x02: 'connection not allowed by ruleset',
+        0x03: 'Network unreachable',
+        0x04: 'Host unreachable',
+        0x05: 'Connection refused',
+        0x06: 'TTL expired',
+        0x07: 'Command not supported',
+        0x08: 'Address type not supported',
+        0xFE: 'unknown username or invalid password',
+        0xFF: 'all offered authentication methods were rejected'
+    }
+
+
+class ProxyType:
+    SOCKS4 = 0
+    SOCKS4A = 1
+    SOCKS5 = 2
+
+
+Proxy = collections.namedtuple('Proxy', (
+    'type', 'host', 'port', 'username', 'password', 'remote_dns'))
+
+
+class sockssocket(socket.socket):
+    def __init__(self, *args, **kwargs):
+        self._proxy = None
+        super().__init__(*args, **kwargs)
+
+    def setproxy(self, proxytype, addr, port, rdns=True, username=None, password=None):
+        assert proxytype in (ProxyType.SOCKS4, ProxyType.SOCKS4A, ProxyType.SOCKS5)
+
+        self._proxy = Proxy(proxytype, addr, port, username, password, rdns)
+
+    def recvall(self, cnt):
+        data = b''
+        while len(data) < cnt:
+            cur = self.recv(cnt - len(data))
+            if not cur:
+                raise EOFError(f'{cnt - len(data)} bytes missing')
+            data += cur
+        return data
+
+    def _recv_bytes(self, cnt):
+        data = self.recvall(cnt)
+        return struct.unpack(f'!{cnt}B', data)
+
+    @staticmethod
+    def _len_and_data(data):
+        return struct.pack('!B', len(data)) + data
+
+    def _check_response_version(self, expected_version, got_version):
+        if got_version != expected_version:
+            self.close()
+            raise InvalidVersionError(expected_version, got_version)
+
+    def _resolve_address(self, destaddr, default, use_remote_dns, family=None):
+        for f in (family,) if family else (socket.AF_INET, socket.AF_INET6):
+            try:
+                return f, socket.inet_pton(f, destaddr)
+            except OSError:
+                continue
+
+        if use_remote_dns and self._proxy.remote_dns:
+            return 0, default
+        else:
+            res = socket.getaddrinfo(destaddr, None, family=family or 0)
+            f, _, _, _, ipaddr = res[0]
+            return f, socket.inet_pton(f, ipaddr[0])
+
+    def _setup_socks4(self, address, is_4a=False):
+        destaddr, port = address
+
+        _, ipaddr = self._resolve_address(destaddr, SOCKS4_DEFAULT_DSTIP, use_remote_dns=is_4a, family=socket.AF_INET)
+
+        packet = struct.pack('!BBH', SOCKS4_VERSION, Socks4Command.CMD_CONNECT, port) + ipaddr
+
+        username = (self._proxy.username or '').encode()
+        packet += username + b'\x00'
+
+        if is_4a and self._proxy.remote_dns and ipaddr == SOCKS4_DEFAULT_DSTIP:
+            packet += destaddr.encode() + b'\x00'
+
+        self.sendall(packet)
+
+        version, resp_code, dstport, dsthost = struct.unpack('!BBHI', self.recvall(8))
+
+        self._check_response_version(SOCKS4_REPLY_VERSION, version)
+
+        if resp_code != Socks4Error.ERR_SUCCESS:
+            self.close()
+            raise Socks4Error(resp_code)
+
+        return (dsthost, dstport)
+
+    def _setup_socks4a(self, address):
+        self._setup_socks4(address, is_4a=True)
+
+    def _socks5_auth(self):
+        packet = struct.pack('!B', SOCKS5_VERSION)
+
+        auth_methods = [Socks5Auth.AUTH_NONE]
+        if self._proxy.username and self._proxy.password:
+            auth_methods.append(Socks5Auth.AUTH_USER_PASS)
+
+        packet += struct.pack('!B', len(auth_methods))
+        packet += struct.pack(f'!{len(auth_methods)}B', *auth_methods)
+
+        self.sendall(packet)
+
+        version, method = self._recv_bytes(2)
+
+        self._check_response_version(SOCKS5_VERSION, version)
+
+        if method == Socks5Auth.AUTH_NO_ACCEPTABLE or (
+                method == Socks5Auth.AUTH_USER_PASS and (not self._proxy.username or not self._proxy.password)):
+            self.close()
+            raise Socks5Error(Socks5Auth.AUTH_NO_ACCEPTABLE)
+
+        if method == Socks5Auth.AUTH_USER_PASS:
+            username = self._proxy.username.encode()
+            password = self._proxy.password.encode()
+            packet = struct.pack('!B', SOCKS5_USER_AUTH_VERSION)
+            packet += self._len_and_data(username) + self._len_and_data(password)
+            self.sendall(packet)
+
+            version, status = self._recv_bytes(2)
+
+            self._check_response_version(SOCKS5_USER_AUTH_VERSION, version)
+
+            if status != SOCKS5_USER_AUTH_SUCCESS:
+                self.close()
+                raise Socks5Error(Socks5Error.ERR_GENERAL_FAILURE)
+
+    def _setup_socks5(self, address):
+        destaddr, port = address
+
+        family, ipaddr = self._resolve_address(destaddr, None, use_remote_dns=True)
+
+        self._socks5_auth()
+
+        reserved = 0
+        packet = struct.pack('!BBB', SOCKS5_VERSION, Socks5Command.CMD_CONNECT, reserved)
+        if ipaddr is None:
+            destaddr = destaddr.encode()
+            packet += struct.pack('!B', Socks5AddressType.ATYP_DOMAINNAME)
+            packet += self._len_and_data(destaddr)
+        elif family == socket.AF_INET:
+            packet += struct.pack('!B', Socks5AddressType.ATYP_IPV4) + ipaddr
+        elif family == socket.AF_INET6:
+            packet += struct.pack('!B', Socks5AddressType.ATYP_IPV6) + ipaddr
+        packet += struct.pack('!H', port)
+
+        self.sendall(packet)
+
+        version, status, reserved, atype = self._recv_bytes(4)
+
+        self._check_response_version(SOCKS5_VERSION, version)
+
+        if status != Socks5Error.ERR_SUCCESS:
+            self.close()
+            raise Socks5Error(status)
+
+        if atype == Socks5AddressType.ATYP_IPV4:
+            destaddr = self.recvall(4)
+        elif atype == Socks5AddressType.ATYP_DOMAINNAME:
+            alen = compat_ord(self.recv(1))
+            destaddr = self.recvall(alen)
+        elif atype == Socks5AddressType.ATYP_IPV6:
+            destaddr = self.recvall(16)
+        destport = struct.unpack('!H', self.recvall(2))[0]
+
+        return (destaddr, destport)
+
+    def _make_proxy(self, connect_func, address):
+        if not self._proxy:
+            return connect_func(self, address)
+
+        result = connect_func(self, (self._proxy.host, self._proxy.port))
+        if result != 0 and result is not None:
+            return result
+        setup_funcs = {
+            ProxyType.SOCKS4: self._setup_socks4,
+            ProxyType.SOCKS4A: self._setup_socks4a,
+            ProxyType.SOCKS5: self._setup_socks5,
+        }
+        setup_funcs[self._proxy.type](address)
+        return result
+
+    def connect(self, address):
+        self._make_proxy(socket.socket.connect, address)
+
+    def connect_ex(self, address):
+        return self._make_proxy(socket.socket.connect_ex, address)