@tushar-br/desktop 1.0.282 → 1.0.283

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/fancode.py

@@ -0,0 +1,181 @@
+from .common import InfoExtractor
+
+from ..compat import compat_str
+from ..utils import (
+    parse_iso8601,
+    ExtractorError,
+    try_get,
+    mimetype2ext
+)
+
+
+class FancodeVodIE(InfoExtractor):
+    _WORKING = False
+    IE_NAME = 'fancode:vod'
+
+    _VALID_URL = r'https?://(?:www\.)?fancode\.com/video/(?P<id>[0-9]+)\b'
+
+    _TESTS = [{
+        'url': 'https://fancode.com/video/15043/match-preview-pbks-vs-mi',
+        'params': {
+            'skip_download': True,
+        },
+        'info_dict': {
+            'id': '6249806281001',
+            'ext': 'mp4',
+            'title': 'Match Preview: PBKS vs MI',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            "timestamp": 1619081590,
+            'view_count': int,
+            'like_count': int,
+            'upload_date': '20210422',
+            'uploader_id': '6008340455001'
+        }
+    }, {
+        'url': 'https://fancode.com/video/15043',
+        'only_matching': True,
+    }]
+
+    _ACCESS_TOKEN = None
+    _NETRC_MACHINE = 'fancode'
+
+    _LOGIN_HINT = 'Use "--username refresh --password <refresh_token>" to login using a refresh token'
+
+    headers = {
+        'content-type': 'application/json',
+        'origin': 'https://fancode.com',
+        'referer': 'https://fancode.com',
+    }
+
+    def _perform_login(self, username, password):
+        # Access tokens are shortlived, so get them using the refresh token.
+        if username != 'refresh':
+            self.report_warning(f'Login using username and password is not currently supported. {self._LOGIN_HINT}')
+
+        self.report_login()
+        data = '''{
+            "query":"mutation RefreshToken($refreshToken: String\\u0021) { refreshToken(refreshToken: $refreshToken) { accessToken }}",
+            "variables":{
+                "refreshToken":"%s"
+            },
+            "operationName":"RefreshToken"
+        }''' % password
+
+        token_json = self.download_gql('refresh token', data, "Getting the Access token")
+        self._ACCESS_TOKEN = try_get(token_json, lambda x: x['data']['refreshToken']['accessToken'])
+        if self._ACCESS_TOKEN is None:
+            self.report_warning('Failed to get Access token')
+        else:
+            self.headers.update({'Authorization': 'Bearer %s' % self._ACCESS_TOKEN})
+
+    def _check_login_required(self, is_available, is_premium):
+        msg = None
+        if is_premium and self._ACCESS_TOKEN is None:
+            msg = f'This video is only available for registered users. {self._LOGIN_HINT}'
+        elif not is_available and self._ACCESS_TOKEN is not None:
+            msg = 'This video isn\'t available to the current logged in account'
+        if msg:
+            self.raise_login_required(msg, metadata_available=True, method=None)
+
+    def download_gql(self, variable, data, note, fatal=False, headers=headers):
+        return self._download_json(
+            'https://www.fancode.com/graphql', variable,
+            data=data.encode(), note=note,
+            headers=headers, fatal=fatal)
+
+    def _real_extract(self, url):
+
+        BRIGHTCOVE_URL_TEMPLATE = 'https://players.brightcove.net/%s/default_default/index.html?videoId=%s'
+        video_id = self._match_id(url)
+
+        brightcove_user_id = '6008340455001'
+        data = '''{
+            "query":"query Video($id: Int\\u0021, $filter: SegmentFilter) { media(id: $id, filter: $filter) { id contentId title contentId publishedTime totalViews totalUpvotes provider thumbnail { src } mediaSource {brightcove } duration isPremium isUserEntitled tags duration }}",
+            "variables":{
+                "id":%s,
+                "filter":{
+                    "contentDataType":"DEFAULT"
+                }
+            },
+            "operationName":"Video"
+        }''' % video_id
+
+        metadata_json = self.download_gql(video_id, data, note='Downloading metadata')
+
+        media = try_get(metadata_json, lambda x: x['data']['media'], dict) or {}
+        brightcove_video_id = try_get(media, lambda x: x['mediaSource']['brightcove'], compat_str)
+
+        if brightcove_video_id is None:
+            raise ExtractorError('Unable to extract brightcove Video ID')
+
+        is_premium = media.get('isPremium')
+
+        self._check_login_required(media.get('isUserEntitled'), is_premium)
+
+        return {
+            '_type': 'url_transparent',
+            'url': BRIGHTCOVE_URL_TEMPLATE % (brightcove_user_id, brightcove_video_id),
+            'ie_key': 'BrightcoveNew',
+            'id': video_id,
+            'title': media['title'],
+            'like_count': media.get('totalUpvotes'),
+            'view_count': media.get('totalViews'),
+            'tags': media.get('tags'),
+            'release_timestamp': parse_iso8601(media.get('publishedTime')),
+            'availability': self._availability(needs_premium=is_premium),
+        }
+
+
+class FancodeLiveIE(FancodeVodIE):  # XXX: Do not subclass from concrete IE
+    _WORKING = False
+    IE_NAME = 'fancode:live'
+
+    _VALID_URL = r'https?://(www\.)?fancode\.com/match/(?P<id>[0-9]+).+'
+
+    _TESTS = [{
+        'url': 'https://fancode.com/match/35328/cricket-fancode-ecs-hungary-2021-bub-vs-blb?slug=commentary',
+        'info_dict': {
+            'id': '35328',
+            'ext': 'mp4',
+            'title': 'BUB vs BLB',
+            "timestamp": 1624863600,
+            'is_live': True,
+            'upload_date': '20210628',
+        },
+        'skip': 'Ended'
+    }, {
+        'url': 'https://fancode.com/match/35328/',
+        'only_matching': True,
+    }, {
+        'url': 'https://fancode.com/match/35567?slug=scorecard',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+
+        id = self._match_id(url)
+        data = '''{
+            "query":"query MatchResponse($id: Int\\u0021, $isLoggedIn: Boolean\\u0021) { match: matchWithScores(id: $id) { id matchDesc mediaId videoStreamId videoStreamUrl { ...VideoSource } liveStreams { videoStreamId videoStreamUrl { ...VideoSource } contentId } name startTime streamingStatus isPremium isUserEntitled @include(if: $isLoggedIn) status metaTags bgImage { src } sport { name slug } tour { id name } squads { name shortName } liveStreams { contentId } mediaId }}fragment VideoSource on VideoSource { title description posterUrl url deliveryType playerType}",
+            "variables":{
+                "id":%s,
+                "isLoggedIn":true
+            },
+            "operationName":"MatchResponse"
+        }''' % id
+
+        info_json = self.download_gql(id, data, "Info json")
+
+        match_info = try_get(info_json, lambda x: x['data']['match'])
+
+        if match_info.get('streamingStatus') != "STARTED":
+            raise ExtractorError('The stream can\'t be accessed', expected=True)
+        self._check_login_required(match_info.get('isUserEntitled'), True)  # all live streams are premium only
+
+        return {
+            'id': id,
+            'title': match_info.get('name'),
+            'formats': self._extract_akamai_formats(try_get(match_info, lambda x: x['videoStreamUrl']['url']), id),
+            'ext': mimetype2ext(try_get(match_info, lambda x: x['videoStreamUrl']['deliveryType'])),
+            'is_live': True,
+            'release_timestamp': parse_iso8601(match_info.get('startTime'))
+        }

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/faz.py

@@ -0,0 +1,89 @@
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_etree_fromstring
+from ..utils import (
+    xpath_element,
+    xpath_text,
+    int_or_none,
+)
+
+
+class FazIE(InfoExtractor):
+    IE_NAME = 'faz.net'
+    _VALID_URL = r'https?://(?:www\.)?faz\.net/(?:[^/]+/)*.*?-(?P<id>\d+)\.html'
+
+    _TESTS = [{
+        'url': 'http://www.faz.net/multimedia/videos/stockholm-chemie-nobelpreis-fuer-drei-amerikanische-forscher-12610585.html',
+        'info_dict': {
+            'id': '12610585',
+            'ext': 'mp4',
+            'title': 'Stockholm: Chemie-Nobelpreis für drei amerikanische Forscher',
+            'description': 'md5:1453fbf9a0d041d985a47306192ea253',
+        },
+    }, {
+        'url': 'http://www.faz.net/aktuell/politik/berlin-gabriel-besteht-zerreissprobe-ueber-datenspeicherung-13659345.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.faz.net/berlin-gabriel-besteht-zerreissprobe-ueber-datenspeicherung-13659345.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.faz.net/-13659345.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.faz.net/aktuell/politik/-13659345.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.faz.net/foobarblafasel-13659345.html',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        description = self._og_search_description(webpage)
+        media = self._html_search_regex(
+            r"data-videojs-media='([^']+)",
+            webpage, 'media')
+        if media == 'extern':
+            perform_url = self._search_regex(
+                r"<iframe[^>]+?src='((?:http:)?//player\.performgroup\.com/eplayer/eplayer\.html#/?[0-9a-f]{26}\.[0-9a-z]{26})",
+                webpage, 'perform url')
+            return self.url_result(perform_url)
+        config = compat_etree_fromstring(media)
+
+        encodings = xpath_element(config, 'ENCODINGS', 'encodings', True)
+        formats = []
+        for pref, code in enumerate(['LOW', 'HIGH', 'HQ']):
+            encoding = xpath_element(encodings, code)
+            if encoding is not None:
+                encoding_url = xpath_text(encoding, 'FILENAME')
+                if encoding_url:
+                    tbr = xpath_text(encoding, 'AVERAGEBITRATE', 1000)
+                    if tbr:
+                        tbr = int_or_none(tbr.replace(',', '.'))
+                    f = {
+                        'url': encoding_url,
+                        'format_id': code.lower(),
+                        'quality': pref,
+                        'tbr': tbr,
+                        'vcodec': xpath_text(encoding, 'CODEC'),
+                    }
+                    mobj = re.search(r'(\d+)x(\d+)_(\d+)\.mp4', encoding_url)
+                    if mobj:
+                        f.update({
+                            'width': int(mobj.group(1)),
+                            'height': int(mobj.group(2)),
+                            'tbr': tbr or int(mobj.group(3)),
+                        })
+                    formats.append(f)
+
+        return {
+            'id': video_id,
+            'title': self._og_search_title(webpage),
+            'formats': formats,
+            'description': description.strip() if description else None,
+            'thumbnail': xpath_text(config, 'STILL/STILL_BIG'),
+            'duration': int_or_none(xpath_text(config, 'DURATION')),
+        }

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/fc2.py

@@ -0,0 +1,280 @@
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_parse_qs
+from ..networking import Request
+from ..utils import (
+    ExtractorError,
+    js_to_json,
+    traverse_obj,
+    update_url_query,
+    urlencode_postdata,
+    urljoin,
+)
+
+
+class FC2IE(InfoExtractor):
+    _VALID_URL = r'^(?:https?://video\.fc2\.com/(?:[^/]+/)*content/|fc2:)(?P<id>[^/]+)'
+    IE_NAME = 'fc2'
+    _NETRC_MACHINE = 'fc2'
+    _TESTS = [{
+        'url': 'http://video.fc2.com/en/content/20121103kUan1KHs',
+        'md5': 'a6ebe8ebe0396518689d963774a54eb7',
+        'info_dict': {
+            'id': '20121103kUan1KHs',
+            'ext': 'flv',
+            'title': 'Boxing again with Puff',
+        },
+    }, {
+        'url': 'http://video.fc2.com/en/content/20150125cEva0hDn/',
+        'info_dict': {
+            'id': '20150125cEva0hDn',
+            'ext': 'mp4',
+        },
+        'params': {
+            'username': 'ytdl@yt-dl.org',
+            'password': '(snip)',
+        },
+        'skip': 'requires actual password',
+    }, {
+        'url': 'http://video.fc2.com/en/a/content/20130926eZpARwsF',
+        'only_matching': True,
+    }]
+
+    def _login(self):
+        username, password = self._get_login_info()
+        if username is None or password is None:
+            return False
+
+        # Log in
+        login_form_strs = {
+            'email': username,
+            'password': password,
+            'done': 'video',
+            'Submit': ' Login ',
+        }
+
+        login_data = urlencode_postdata(login_form_strs)
+        request = Request(
+            'https://secure.id.fc2.com/index.php?mode=login&switch_language=en', login_data)
+
+        login_results = self._download_webpage(request, None, note='Logging in', errnote='Unable to log in')
+        if 'mode=redirect&login=done' not in login_results:
+            self.report_warning('unable to log in: bad username or password')
+            return False
+
+        # this is also needed
+        login_redir = Request('http://id.fc2.com/?mode=redirect&login=done')
+        self._download_webpage(
+            login_redir, None, note='Login redirect', errnote='Login redirect failed')
+
+        return True
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        self._login()
+        webpage = None
+        if not url.startswith('fc2:'):
+            webpage = self._download_webpage(url, video_id)
+            self.cookiejar.clear_session_cookies()  # must clear
+            self._login()
+
+        title, thumbnail, description = None, None, None
+        if webpage is not None:
+            title = self._html_search_regex(
+                (r'<h2\s+class="videoCnt_title">([^<]+?)</h2>',
+                 r'\s+href="[^"]+"\s*title="([^"]+?)"\s*rel="nofollow">\s*<img',
+                 # there's two matches in the webpage
+                 r'\s+href="[^"]+"\s*title="([^"]+?)"\s*rel="nofollow">\s*\1'),
+                webpage,
+                'title', fatal=False)
+            thumbnail = self._og_search_thumbnail(webpage)
+            description = self._og_search_description(webpage, default=None)
+
+        vidplaylist = self._download_json(
+            'https://video.fc2.com/api/v3/videoplaylist/%s?sh=1&fs=0' % video_id, video_id,
+            note='Downloading info page')
+        vid_url = traverse_obj(vidplaylist, ('playlist', 'nq'))
+        if not vid_url:
+            raise ExtractorError('Unable to extract video URL')
+        vid_url = urljoin('https://video.fc2.com/', vid_url)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'url': vid_url,
+            'ext': 'mp4',
+            'protocol': 'm3u8_native',
+            'description': description,
+            'thumbnail': thumbnail,
+        }
+
+
+class FC2EmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://video\.fc2\.com/flv2\.swf\?(?P<query>.+)'
+    IE_NAME = 'fc2:embed'
+
+    _TEST = {
+        'url': 'http://video.fc2.com/flv2.swf?t=201404182936758512407645&i=20130316kwishtfitaknmcgd76kjd864hso93htfjcnaogz629mcgfs6rbfk0hsycma7shkf85937cbchfygd74&i=201403223kCqB3Ez&d=2625&sj=11&lang=ja&rel=1&from=11&cmt=1&tk=TlRBM09EQTNNekU9&tl=プリズン･ブレイク%20S1-01%20マイケル%20【吹替】',
+        'md5': 'b8aae5334cb691bdb1193a88a6ab5d5a',
+        'info_dict': {
+            'id': '201403223kCqB3Ez',
+            'ext': 'flv',
+            'title': 'プリズン･ブレイク S1-01 マイケル 【吹替】',
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
+    }
+
+    def _real_extract(self, url):
+        mobj = self._match_valid_url(url)
+        query = compat_parse_qs(mobj.group('query'))
+
+        video_id = query['i'][-1]
+        title = query.get('tl', ['FC2 video %s' % video_id])[0]
+
+        sj = query.get('sj', [None])[0]
+        thumbnail = None
+        if sj:
+            # See thumbnailImagePath() in ServerConst.as of flv2.swf
+            thumbnail = 'http://video%s-thumbnail.fc2.com/up/pic/%s.jpg' % (
+                sj, '/'.join((video_id[:6], video_id[6:8], video_id[-2], video_id[-1], video_id)))
+
+        return {
+            '_type': 'url_transparent',
+            'ie_key': FC2IE.ie_key(),
+            'url': 'fc2:%s' % video_id,
+            'title': title,
+            'thumbnail': thumbnail,
+        }
+
+
+class FC2LiveIE(InfoExtractor):
+    _VALID_URL = r'https?://live\.fc2\.com/(?P<id>\d+)'
+    IE_NAME = 'fc2:live'
+
+    _TESTS = [{
+        'url': 'https://live.fc2.com/57892267/',
+        'info_dict': {
+            'id': '57892267',
+            'title': 'どこまで・・・',
+            'uploader': 'あつあげ',
+            'uploader_id': '57892267',
+            'thumbnail': r're:https?://.+fc2.+',
+        },
+        'skip': 'livestream',
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage('https://live.fc2.com/%s/' % video_id, video_id)
+
+        self._set_cookie('live.fc2.com', 'js-player_size', '1')
+
+        member_api = self._download_json(
+            'https://live.fc2.com/api/memberApi.php', video_id, data=urlencode_postdata({
+                'channel': '1',
+                'profile': '1',
+                'user': '1',
+                'streamid': video_id
+            }), note='Requesting member info')
+
+        control_server = self._download_json(
+            'https://live.fc2.com/api/getControlServer.php', video_id, note='Downloading ControlServer data',
+            data=urlencode_postdata({
+                'channel_id': video_id,
+                'mode': 'play',
+                'orz': '',
+                'channel_version': member_api['data']['channel_data']['version'],
+                'client_version': '2.1.0\n [1]',
+                'client_type': 'pc',
+                'client_app': 'browser_hls',
+                'ipv6': '',
+            }), headers={'X-Requested-With': 'XMLHttpRequest'})
+        self._set_cookie('live.fc2.com', 'l_ortkn', control_server['orz_raw'])
+
+        ws_url = update_url_query(control_server['url'], {'control_token': control_server['control_token']})
+        playlist_data = None
+
+        ws = self._request_webpage(Request(ws_url, headers={
+            'Origin': 'https://live.fc2.com',
+        }), video_id, note='Fetching HLS playlist info via WebSocket')
+
+        self.write_debug('Sending HLS server request')
+
+        while True:
+            recv = ws.recv()
+            if not recv:
+                continue
+            data = self._parse_json(recv, video_id, fatal=False)
+            if not data or not isinstance(data, dict):
+                continue
+
+            if data.get('name') == 'connect_complete':
+                break
+        ws.send(r'{"name":"get_hls_information","arguments":{},"id":1}')
+
+        while True:
+            recv = ws.recv()
+            if not recv:
+                continue
+            data = self._parse_json(recv, video_id, fatal=False)
+            if not data or not isinstance(data, dict):
+                continue
+            if data.get('name') == '_response_' and data.get('id') == 1:
+                self.write_debug('Goodbye')
+                playlist_data = data
+                break
+            self.write_debug('Server said: %s%s' % (recv[:100], '...' if len(recv) > 100 else ''))
+
+        if not playlist_data:
+            raise ExtractorError('Unable to fetch HLS playlist info via WebSocket')
+
+        formats = []
+        for name, playlists in playlist_data['arguments'].items():
+            if not isinstance(playlists, list):
+                continue
+            for pl in playlists:
+                if pl.get('status') == 0 and 'master_playlist' in pl.get('url'):
+                    formats.extend(self._extract_m3u8_formats(
+                        pl['url'], video_id, ext='mp4', m3u8_id=name, live=True,
+                        headers={
+                            'Origin': 'https://live.fc2.com',
+                            'Referer': url,
+                        }))
+
+        for fmt in formats:
+            fmt.update({
+                'protocol': 'fc2_live',
+                'ws': ws,
+            })
+
+        title = self._html_search_meta(('og:title', 'twitter:title'), webpage, 'live title', fatal=False)
+        if not title:
+            title = self._html_extract_title(webpage, 'html title', fatal=False)
+            if title:
+                # remove service name in <title>
+                title = re.sub(r'\s+-\s+.+$', '', title)
+        uploader = None
+        if title:
+            match = self._search_regex(r'^(.+?)\s*\[(.+?)\]$', title, 'title and uploader', default=None, group=(1, 2))
+            if match and all(match):
+                title, uploader = match
+
+        live_info_view = self._search_regex(r'(?s)liveInfoView\s*:\s*({.+?}),\s*premiumStateView', webpage, 'user info', fatal=False) or None
+        if live_info_view:
+            # remove jQuery code from object literal
+            live_info_view = re.sub(r'\$\(.+?\)[^,]+,', '"",', live_info_view)
+            live_info_view = self._parse_json(js_to_json(live_info_view), video_id)
+
+        return {
+            'id': video_id,
+            'title': title or traverse_obj(live_info_view, 'title'),
+            'description': self._html_search_meta(
+                ('og:description', 'twitter:description'),
+                webpage, 'live description', fatal=False) or traverse_obj(live_info_view, 'info'),
+            'formats': formats,
+            'uploader': uploader or traverse_obj(live_info_view, 'name'),
+            'uploader_id': video_id,
+            'thumbnail': traverse_obj(live_info_view, 'thumb'),
+            'is_live': True,
+        }

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/fczenit.py

@@ -0,0 +1,51 @@
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    float_or_none,
+)
+
+
+class FczenitIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?fc-zenit\.ru/video/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'http://fc-zenit.ru/video/41044/',
+        'md5': '0e3fab421b455e970fa1aa3891e57df0',
+        'info_dict': {
+            'id': '41044',
+            'ext': 'mp4',
+            'title': 'Так пишется история: казанский разгром ЦСКА на «Зенит-ТВ»',
+            'timestamp': 1462283735,
+            'upload_date': '20160503',
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        msi_id = self._search_regex(
+            r"(?s)config\s*=\s*{.+?video_id\s*:\s*'([^']+)'", webpage, 'msi id')
+
+        msi_data = self._download_json(
+            'http://player.fc-zenit.ru/msi/video', msi_id, query={
+                'video': msi_id,
+            })['data']
+        title = msi_data['name']
+
+        formats = [{
+            'format_id': q.get('label'),
+            'url': q['url'],
+            'height': int_or_none(q.get('label')),
+        } for q in msi_data['qualities'] if q.get('url')]
+
+        tags = [tag['label'] for tag in msi_data.get('tags', []) if tag.get('label')]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'thumbnail': msi_data.get('preview'),
+            'formats': formats,
+            'duration': float_or_none(msi_data.get('duration')),
+            'timestamp': int_or_none(msi_data.get('date')),
+            'tags': tags,
+        }