@tushar-br/desktop 1.0.282 → 1.0.283

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/urort.py

@@ -0,0 +1,60 @@
+import urllib.parse
+
+from .common import InfoExtractor
+from ..utils import unified_strdate
+
+
+class UrortIE(InfoExtractor):
+    _WORKING = False
+    IE_DESC = 'NRK P3 Urørt'
+    _VALID_URL = r'https?://(?:www\.)?urort\.p3\.no/#!/Band/(?P<id>[^/]+)$'
+
+    _TEST = {
+        'url': 'https://urort.p3.no/#!/Band/Gerilja',
+        'md5': '5ed31a924be8a05e47812678a86e127b',
+        'info_dict': {
+            'id': '33124-24',
+            'ext': 'mp3',
+            'title': 'The Bomb',
+            'thumbnail': r're:^https?://.+\.jpg',
+            'uploader': 'Gerilja',
+            'uploader_id': 'Gerilja',
+            'upload_date': '20100323',
+        },
+        'params': {
+            'matchtitle': '^The Bomb$',  # To test, we want just one video
+        }
+    }
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+
+        fstr = urllib.parse.quote("InternalBandUrl eq '%s'" % playlist_id)
+        json_url = 'http://urort.p3.no/breeze/urort/TrackDTOViews?$filter=%s&$orderby=Released%%20desc&$expand=Tags%%2CFiles' % fstr
+        songs = self._download_json(json_url, playlist_id)
+        entries = []
+        for s in songs:
+            formats = [{
+                'tbr': f.get('Quality'),
+                'ext': f['FileType'],
+                'format_id': '%s-%s' % (f['FileType'], f.get('Quality', '')),
+                'url': 'http://p3urort.blob.core.windows.net/tracks/%s' % f['FileRef'],
+                'quality': 3 if f['FileType'] == 'mp3' else 2,
+            } for f in s['Files']]
+            e = {
+                'id': '%d-%s' % (s['BandId'], s['$id']),
+                'title': s['Title'],
+                'uploader_id': playlist_id,
+                'uploader': s.get('BandName', playlist_id),
+                'thumbnail': 'http://urort.p3.no/cloud/images/%s' % s['Image'],
+                'upload_date': unified_strdate(s.get('Released')),
+                'formats': formats,
+            }
+            entries.append(e)
+
+        return {
+            '_type': 'playlist',
+            'id': playlist_id,
+            'title': playlist_id,
+            'entries': entries,
+        }

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/urplay.py

@@ -0,0 +1,164 @@
+from .common import InfoExtractor
+from ..utils import (
+    dict_get,
+    ExtractorError,
+    int_or_none,
+    ISO639Utils,
+    parse_age_limit,
+    try_get,
+    unified_timestamp,
+)
+
+
+class URPlayIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ur(?:play|skola)\.se/(?:program|Produkter)/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'https://urplay.se/program/203704-ur-samtiden-livet-universum-och-rymdens-markliga-musik-om-vetenskap-kritiskt-tankande-och-motstand',
+        'md5': '5ba36643c77cc3d34ffeadad89937d1e',
+        'info_dict': {
+            'id': '203704',
+            'ext': 'mp4',
+            'title': 'UR Samtiden - Livet, universum och rymdens märkliga musik : Om vetenskap, kritiskt tänkande och motstånd',
+            'description': 'md5:5344508a52aa78c1ced6c1b8b9e44e9a',
+            'thumbnail': r're:^https?://.+\.jpg',
+            'timestamp': 1513292400,
+            'upload_date': '20171214',
+            'series': 'UR Samtiden - Livet, universum och rymdens märkliga musik',
+            'duration': 2269,
+            'categories': ['Vetenskap & teknik'],
+            'tags': ['Kritiskt tänkande', 'Vetenskap', 'Vetenskaplig verksamhet'],
+            'episode': 'Om vetenskap, kritiskt tänkande och motstånd',
+            'age_limit': 15,
+        },
+    }, {
+        'url': 'https://urplay.se/program/222967-en-foralders-dagbok-mitt-barn-skadar-sig-sjalv',
+        'info_dict': {
+            'id': '222967',
+            'ext': 'mp4',
+            'title': 'En förälders dagbok : Mitt barn skadar sig själv',
+            'description': 'md5:9f771eef03a732a213b367b52fe826ca',
+            'thumbnail': r're:^https?://.+\.jpg',
+            'timestamp': 1629676800,
+            'upload_date': '20210823',
+            'series': 'En förälders dagbok',
+            'duration': 1740,
+            'age_limit': 15,
+            'episode_number': 3,
+            'categories': 'count:2',
+            'tags': 'count:7',
+            'episode': 'Mitt barn skadar sig själv',
+        },
+    }, {
+        'url': 'https://urskola.se/Produkter/190031-Tripp-Trapp-Trad-Sovkudde',
+        'info_dict': {
+            'id': '190031',
+            'ext': 'mp4',
+            'title': 'Tripp, Trapp, Träd : Sovkudde',
+            'description': 'md5:b86bffdae04a7e9379d1d7e5947df1d1',
+            'thumbnail': r're:^https?://.+\.jpg',
+            'timestamp': 1440086400,
+            'upload_date': '20150820',
+            'series': 'Tripp, Trapp, Träd',
+            'duration': 865,
+            'age_limit': 1,
+            'episode_number': 1,
+            'categories': [],
+            'tags': ['Sova'],
+            'episode': 'Sovkudde',
+            'season': 'Säsong 1',
+        },
+    }, {
+        'url': 'http://urskola.se/Produkter/155794-Smasagor-meankieli-Grodan-i-vida-varlden',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        url = url.replace('skola.se/Produkter', 'play.se/program')
+        webpage = self._download_webpage(url, video_id)
+        urplayer_data = self._search_nextjs_data(webpage, video_id, fatal=False) or {}
+        if urplayer_data:
+            urplayer_data = try_get(urplayer_data, lambda x: x['props']['pageProps']['program'], dict)
+            if not urplayer_data:
+                raise ExtractorError('Unable to parse __NEXT_DATA__')
+        else:
+            accessible_episodes = self._parse_json(self._html_search_regex(
+                r'data-react-class="routes/Product/components/ProgramContainer/ProgramContainer"[^>]+data-react-props="({.+?})"',
+                webpage, 'urplayer data'), video_id)['accessibleEpisodes']
+            urplayer_data = next(e for e in accessible_episodes if e.get('id') == int_or_none(video_id))
+        episode = urplayer_data['title']
+
+        host = self._download_json('http://streaming-loadbalancer.ur.se/loadbalancer.json', video_id)['redirect']
+        formats = []
+        urplayer_streams = urplayer_data.get('streamingInfo', {})
+
+        for k, v in urplayer_streams.get('raw', {}).items():
+            if not (k in ('sd', 'hd', 'mp3', 'm4a') and isinstance(v, dict)):
+                continue
+            file_http = v.get('location')
+            if file_http:
+                formats.extend(self._extract_wowza_formats(
+                    'http://%s/%splaylist.m3u8' % (host, file_http),
+                    video_id, skip_protocols=['f4m', 'rtmp', 'rtsp']))
+
+        subtitles = {}
+
+        def parse_lang_code(code):
+            "3-character language code or None (utils candidate)"
+            if code is None:
+                return
+            lang = code.lower()
+            if not ISO639Utils.long2short(lang):
+                lang = ISO639Utils.short2long(lang)
+            return lang or None
+
+        for stream in urplayer_data['streamingInfo'].values():
+            for k, v in stream.items():
+                if (k in ('sd', 'hd') or not isinstance(v, dict)):
+                    continue
+                lang, sttl_url = (v.get(kk) for kk in ('language', 'location', ))
+                if not sttl_url:
+                    continue
+                lang = parse_lang_code(lang)
+                if not lang:
+                    continue
+                sttl = subtitles.get(lang) or []
+                sttl.append({'ext': k, 'url': sttl_url, })
+                subtitles[lang] = sttl
+
+        image = urplayer_data.get('image') or {}
+        thumbnails = []
+        for k, v in image.items():
+            t = {
+                'id': k,
+                'url': v,
+            }
+            wh = k.split('x')
+            if len(wh) == 2:
+                t.update({
+                    'width': int_or_none(wh[0]),
+                    'height': int_or_none(wh[1]),
+                })
+            thumbnails.append(t)
+
+        series = urplayer_data.get('series') or {}
+        series_title = dict_get(series, ('seriesTitle', 'title')) or dict_get(urplayer_data, ('seriesTitle', 'mainTitle'))
+
+        return {
+            'id': video_id,
+            'title': '%s : %s' % (series_title, episode) if series_title else episode,
+            'description': urplayer_data.get('description'),
+            'thumbnails': thumbnails,
+            'timestamp': unified_timestamp(urplayer_data.get('publishedAt')),
+            'series': series_title,
+            'formats': formats,
+            'duration': int_or_none(urplayer_data.get('duration')),
+            'categories': urplayer_data.get('categories'),
+            'tags': urplayer_data.get('keywords'),
+            'season': series.get('label'),
+            'episode': episode,
+            'episode_number': int_or_none(urplayer_data.get('episodeNumber')),
+            'age_limit': parse_age_limit(min(try_get(a, lambda x: x['from'], int) or 0
+                                             for a in urplayer_data.get('ageRanges', []))),
+            'subtitles': subtitles,
+        }

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/usanetwork.py

@@ -0,0 +1,21 @@
+from .nbc import NBCIE
+
+
+class USANetworkIE(NBCIE):  # XXX: Do not subclass from concrete IE
+    _VALID_URL = r'https?(?P<permalink>://(?:www\.)?usanetwork\.com/(?:[^/]+/videos?|movies?)/(?:[^/]+/)?(?P<id>\d+))'
+    _TESTS = [{
+        'url': 'https://www.usanetwork.com/peacock-trailers/video/intelligence-trailer/4185302',
+        'info_dict': {
+            'id': '4185302',
+            'ext': 'mp4',
+            'title': 'Intelligence (Trailer)',
+            'description': 'A maverick NSA agent enlists the help of a junior systems analyst in a workplace power grab.',
+            'upload_date': '20200715',
+            'timestamp': 1594785600,
+            'uploader': 'NBCU-MPAT',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }]

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/usatoday.py

@@ -0,0 +1,60 @@
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    get_element_by_attribute,
+    parse_duration,
+    try_get,
+    update_url_query,
+)
+from ..compat import compat_str
+
+
+class USATodayIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?usatoday\.com/(?:[^/]+/)*(?P<id>[^?/#]+)'
+    _TESTS = [{
+        # Brightcove Partner ID = 29906170001
+        'url': 'http://www.usatoday.com/media/cinematic/video/81729424/us-france-warn-syrian-regime-ahead-of-new-peace-talks/',
+        'md5': '033587d2529dc3411a1ab3644c3b8827',
+        'info_dict': {
+            'id': '4799374959001',
+            'ext': 'mp4',
+            'title': 'US, France warn Syrian regime ahead of new peace talks',
+            'timestamp': 1457891045,
+            'description': 'md5:7e50464fdf2126b0f533748d3c78d58f',
+            'uploader_id': '29906170001',
+            'upload_date': '20160313',
+        }
+    }, {
+        # ui-video-data[asset_metadata][items][brightcoveaccount] = 28911775001
+        'url': 'https://www.usatoday.com/story/tech/science/2018/08/21/yellowstone-supervolcano-eruption-stop-worrying-its-blow/973633002/',
+        'info_dict': {
+            'id': '5824495846001',
+            'ext': 'mp4',
+            'title': 'Yellowstone more likely to crack rather than explode',
+            'timestamp': 1534790612,
+            'description': 'md5:3715e7927639a4f16b474e9391687c62',
+            'uploader_id': '28911775001',
+            'upload_date': '20180820',
+        }
+    }]
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s'
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(update_url_query(url, {'ajax': 'true'}), display_id)
+        ui_video_data = get_element_by_attribute('class', 'ui-video-data', webpage)
+        if not ui_video_data:
+            raise ExtractorError('no video on the webpage', expected=True)
+        video_data = self._parse_json(ui_video_data, display_id)
+        item = try_get(video_data, lambda x: x['asset_metadata']['items'], dict) or {}
+
+        return {
+            '_type': 'url_transparent',
+            'url': self.BRIGHTCOVE_URL_TEMPLATE % (item.get('brightcoveaccount', '29906170001'), item.get('brightcoveid') or video_data['brightcove_id']),
+            'id': compat_str(video_data['id']),
+            'title': video_data['title'],
+            'thumbnail': video_data.get('thumbnail'),
+            'description': video_data.get('description'),
+            'duration': parse_duration(video_data.get('length')),
+            'ie_key': 'BrightcoveNew',
+        }

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/ustream.py

@@ -0,0 +1,275 @@
+import random
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_str,
+    compat_urlparse,
+)
+from ..utils import (
+    encode_data_uri,
+    ExtractorError,
+    int_or_none,
+    float_or_none,
+    join_nonempty,
+    mimetype2ext,
+    str_or_none,
+)
+
+
+class UstreamIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?(?:ustream\.tv|video\.ibm\.com)/(?P<type>recorded|embed|embed/recorded)/(?P<id>\d+)'
+    IE_NAME = 'ustream'
+    _EMBED_REGEX = [r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?(?:ustream\.tv|video\.ibm\.com)/embed/.+?)\1']
+    _TESTS = [{
+        'url': 'http://www.ustream.tv/recorded/20274954',
+        'md5': '088f151799e8f572f84eb62f17d73e5c',
+        'info_dict': {
+            'id': '20274954',
+            'ext': 'flv',
+            'title': 'Young Americans for Liberty February 7, 2012 2:28 AM',
+            'description': 'Young Americans for Liberty February 7, 2012 2:28 AM',
+            'timestamp': 1328577035,
+            'upload_date': '20120207',
+            'uploader': 'yaliberty',
+            'uploader_id': '6780869',
+        },
+    }, {
+        # From http://sportscanada.tv/canadagames/index.php/week2/figure-skating/444
+        # Title and uploader available only from params JSON
+        'url': 'http://www.ustream.tv/embed/recorded/59307601?ub=ff0000&lc=ff0000&oc=ffffff&uc=ffffff&v=3&wmode=direct',
+        'md5': '5a2abf40babeac9812ed20ae12d34e10',
+        'info_dict': {
+            'id': '59307601',
+            'ext': 'flv',
+            'title': '-CG11- Canada Games Figure Skating',
+            'uploader': 'sportscanadatv',
+        },
+        'skip': 'This Pro Broadcaster has chosen to remove this video from the ustream.tv site.',
+    }, {
+        'url': 'http://www.ustream.tv/embed/10299409',
+        'info_dict': {
+            'id': '10299409',
+        },
+        'playlist_count': 3,
+    }, {
+        'url': 'http://www.ustream.tv/recorded/91343263',
+        'info_dict': {
+            'id': '91343263',
+            'ext': 'mp4',
+            'title': 'GitHub Universe - General Session - Day 1',
+            'upload_date': '20160914',
+            'description': 'GitHub Universe - General Session - Day 1',
+            'timestamp': 1473872730,
+            'uploader': 'wa0dnskeqkr',
+            'uploader_id': '38977840',
+        },
+        'params': {
+            'skip_download': True,  # m3u8 download
+        },
+    }, {
+        'url': 'https://video.ibm.com/embed/recorded/128240221?&autoplay=true&controls=true&volume=100',
+        'only_matching': True,
+    }]
+
+    def _get_stream_info(self, url, video_id, app_id_ver, extra_note=None):
+        def num_to_hex(n):
+            return hex(n)[2:]
+
+        rnd = random.randrange
+
+        if not extra_note:
+            extra_note = ''
+
+        conn_info = self._download_json(
+            'http://r%d-1-%s-recorded-lp-live.ums.ustream.tv/1/ustream' % (rnd(1e8), video_id),
+            video_id, note='Downloading connection info' + extra_note,
+            query={
+                'type': 'viewer',
+                'appId': app_id_ver[0],
+                'appVersion': app_id_ver[1],
+                'rsid': '%s:%s' % (num_to_hex(rnd(1e8)), num_to_hex(rnd(1e8))),
+                'rpin': '_rpin.%d' % rnd(1e15),
+                'referrer': url,
+                'media': video_id,
+                'application': 'recorded',
+            })
+        host = conn_info[0]['args'][0]['host']
+        connection_id = conn_info[0]['args'][0]['connectionId']
+
+        return self._download_json(
+            'http://%s/1/ustream?connectionId=%s' % (host, connection_id),
+            video_id, note='Downloading stream info' + extra_note)
+
+    def _get_streams(self, url, video_id, app_id_ver):
+        # Sometimes the return dict does not have 'stream'
+        for trial_count in range(3):
+            stream_info = self._get_stream_info(
+                url, video_id, app_id_ver,
+                extra_note=' (try %d)' % (trial_count + 1) if trial_count > 0 else '')
+            if 'stream' in stream_info[0]['args'][0]:
+                return stream_info[0]['args'][0]['stream']
+        return []
+
+    def _parse_segmented_mp4(self, dash_stream_info):
+        def resolve_dash_template(template, idx, chunk_hash):
+            return template.replace('%', compat_str(idx), 1).replace('%', chunk_hash)
+
+        formats = []
+        for stream in dash_stream_info['streams']:
+            # Use only one provider to avoid too many formats
+            provider = dash_stream_info['providers'][0]
+            fragments = [{
+                'url': resolve_dash_template(
+                    provider['url'] + stream['initUrl'], 0, dash_stream_info['hashes']['0'])
+            }]
+            for idx in range(dash_stream_info['videoLength'] // dash_stream_info['chunkTime']):
+                fragments.append({
+                    'url': resolve_dash_template(
+                        provider['url'] + stream['segmentUrl'], idx,
+                        dash_stream_info['hashes'][compat_str(idx // 10 * 10)])
+                })
+            content_type = stream['contentType']
+            kind = content_type.split('/')[0]
+            f = {
+                'format_id': join_nonempty(
+                    'dash', kind, str_or_none(stream.get('bitrate'))),
+                'protocol': 'http_dash_segments',
+                # TODO: generate a MPD doc for external players?
+                'url': encode_data_uri(b'<MPD/>', 'text/xml'),
+                'ext': mimetype2ext(content_type),
+                'height': stream.get('height'),
+                'width': stream.get('width'),
+                'fragments': fragments,
+            }
+            if kind == 'video':
+                f.update({
+                    'vcodec': stream.get('codec'),
+                    'acodec': 'none',
+                    'vbr': stream.get('bitrate'),
+                })
+            else:
+                f.update({
+                    'vcodec': 'none',
+                    'acodec': stream.get('codec'),
+                    'abr': stream.get('bitrate'),
+                })
+            formats.append(f)
+        return formats
+
+    def _real_extract(self, url):
+        m = self._match_valid_url(url)
+        video_id = m.group('id')
+
+        # some sites use this embed format (see: https://github.com/ytdl-org/youtube-dl/issues/2990)
+        if m.group('type') == 'embed/recorded':
+            video_id = m.group('id')
+            desktop_url = 'http://www.ustream.tv/recorded/' + video_id
+            return self.url_result(desktop_url, 'Ustream')
+        if m.group('type') == 'embed':
+            video_id = m.group('id')
+            webpage = self._download_webpage(url, video_id)
+            content_video_ids = self._parse_json(self._search_regex(
+                r'ustream\.vars\.offAirContentVideoIds=([^;]+);', webpage,
+                'content video IDs'), video_id)
+            return self.playlist_result(
+                map(lambda u: self.url_result('http://www.ustream.tv/recorded/' + u, 'Ustream'), content_video_ids),
+                video_id)
+
+        params = self._download_json(
+            'https://api.ustream.tv/videos/%s.json' % video_id, video_id)
+
+        error = params.get('error')
+        if error:
+            raise ExtractorError(
+                '%s returned error: %s' % (self.IE_NAME, error), expected=True)
+
+        video = params['video']
+
+        title = video['title']
+        filesize = float_or_none(video.get('file_size'))
+
+        formats = [{
+            'id': video_id,
+            'url': video_url,
+            'ext': format_id,
+            'filesize': filesize,
+        } for format_id, video_url in video['media_urls'].items() if video_url]
+
+        if not formats:
+            hls_streams = self._get_streams(url, video_id, app_id_ver=(11, 2))
+            if hls_streams:
+                # m3u8_native leads to intermittent ContentTooShortError
+                formats.extend(self._extract_m3u8_formats(
+                    hls_streams[0]['url'], video_id, ext='mp4', m3u8_id='hls'))
+
+            '''
+            # DASH streams handling is incomplete as 'url' is missing
+            dash_streams = self._get_streams(url, video_id, app_id_ver=(3, 1))
+            if dash_streams:
+                formats.extend(self._parse_segmented_mp4(dash_streams))
+            '''
+
+        description = video.get('description')
+        timestamp = int_or_none(video.get('created_at'))
+        duration = float_or_none(video.get('length'))
+        view_count = int_or_none(video.get('views'))
+
+        uploader = video.get('owner', {}).get('username')
+        uploader_id = video.get('owner', {}).get('id')
+
+        thumbnails = [{
+            'id': thumbnail_id,
+            'url': thumbnail_url,
+        } for thumbnail_id, thumbnail_url in video.get('thumbnail', {}).items()]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'thumbnails': thumbnails,
+            'timestamp': timestamp,
+            'duration': duration,
+            'view_count': view_count,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'formats': formats,
+        }
+
+
+class UstreamChannelIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ustream\.tv/channel/(?P<slug>.+)'
+    IE_NAME = 'ustream:channel'
+    _TEST = {
+        'url': 'http://www.ustream.tv/channel/channeljapan',
+        'info_dict': {
+            'id': '10874166',
+        },
+        'playlist_mincount': 17,
+    }
+
+    def _real_extract(self, url):
+        m = self._match_valid_url(url)
+        display_id = m.group('slug')
+        webpage = self._download_webpage(url, display_id)
+        channel_id = self._html_search_meta('ustream:channel_id', webpage)
+
+        BASE = 'http://www.ustream.tv'
+        next_url = '/ajax/socialstream/videos/%s/1.json' % channel_id
+        video_ids = []
+        while next_url:
+            reply = self._download_json(
+                compat_urlparse.urljoin(BASE, next_url), display_id,
+                note='Downloading video information (next: %d)' % (len(video_ids) + 1))
+            video_ids.extend(re.findall(r'data-content-id="(\d.*)"', reply['data']))
+            next_url = reply['nextUrl']
+
+        entries = [
+            self.url_result('http://www.ustream.tv/recorded/' + vid, 'Ustream')
+            for vid in video_ids]
+        return {
+            '_type': 'playlist',
+            'id': channel_id,
+            'display_id': display_id,
+            'entries': entries,
+        }