@tushar-br/desktop 1.0.282 → 1.0.283

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/mgtv.py

@@ -0,0 +1,165 @@
+import base64
+import time
+import uuid
+
+from .common import InfoExtractor
+from ..networking.exceptions import HTTPError
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    parse_resolution,
+    traverse_obj,
+    try_get,
+    url_or_none,
+    urljoin,
+)
+
+
+class MGTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:w(?:ww)?\.)?mgtv\.com/(v|b)/(?:[^/]+/)*(?P<id>\d+)\.html'
+    IE_DESC = '芒果TV'
+    IE_NAME = 'MangoTV'
+
+    _TESTS = [{
+        'url': 'http://www.mgtv.com/v/1/290525/f/3116640.html',
+        'info_dict': {
+            'id': '3116640',
+            'ext': 'mp4',
+            'title': '我是歌手 第四季',
+            'description': '我是歌手第四季双年巅峰会',
+            'duration': 7461,
+            'thumbnail': r're:^https?://.*\.jpg$',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://w.mgtv.com/b/427837/15588271.html',
+        'info_dict': {
+            'id': '15588271',
+            'ext': 'mp4',
+            'title': '春日迟迟再出发 沉浸版第1期：陆莹结婚半年查出肾炎被离婚 吴雅婷把一半票根退给前夫',
+            'description': 'md5:a7a05a05b1aa87bd50cae619b19bbca6',
+            'thumbnail': r're:^https?://.+\.jpg',
+            'duration': 4026,
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://w.mgtv.com/b/333652/7329822.html',
+        'info_dict': {
+            'id': '7329822',
+            'ext': 'mp4',
+            'title': '拜托，请你爱我',
+            'description': 'md5:cd81be6499bafe32e4d143abd822bf9c',
+            'thumbnail': r're:^https?://.+\.jpg',
+            'duration': 2656,
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://w.mgtv.com/b/427837/15591647.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://w.mgtv.com/b/388252/15634192.html?fpa=33318&fpos=4&lastp=ch_home',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.mgtv.com/b/301817/3826653.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://w.mgtv.com/b/301817/3826653.html',
+        'only_matching': True,
+    }]
+
+    _RESOLUTIONS = {
+        '标清': ('480p', '854x480'),
+        '高清': ('540p', '960x540'),
+        '超清': ('720p', '1280x720'),
+        '蓝光': ('1080p', '1920x1080'),
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        tk2 = base64.urlsafe_b64encode(
+            f'did={str(uuid.uuid4())}|pno=1030|ver=0.3.0301|clit={int(time.time())}'.encode())[::-1]
+        try:
+            api_data = self._download_json(
+                'https://pcweb.api.mgtv.com/player/video', video_id, query={
+                    'tk2': tk2,
+                    'video_id': video_id,
+                    'type': 'pch5'
+                }, headers=self.geo_verification_headers())['data']
+        except ExtractorError as e:
+            if isinstance(e.cause, HTTPError) and e.cause.status == 401:
+                error = self._parse_json(e.cause.response.read().decode(), None)
+                if error.get('code') == 40005:
+                    self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
+                raise ExtractorError(error['msg'], expected=True)
+            raise
+
+        stream_data = self._download_json(
+            'https://pcweb.api.mgtv.com/player/getSource', video_id, query={
+                'tk2': tk2,
+                'pm2': api_data['atc']['pm2'],
+                'video_id': video_id,
+                'type': 'pch5',
+                'src': 'intelmgtv',
+            }, headers=self.geo_verification_headers())['data']
+        stream_domain = traverse_obj(stream_data, ('stream_domain', ..., {url_or_none}), get_all=False)
+
+        formats = []
+        for idx, stream in enumerate(traverse_obj(stream_data, ('stream', lambda _, v: v['url']))):
+            stream_name = traverse_obj(stream, 'name', 'standardName', 'barName', expected_type=str)
+            resolution = traverse_obj(
+                self._RESOLUTIONS, (stream_name, 1 if stream.get('scale') == '16:9' else 0))
+            format_url = traverse_obj(self._download_json(
+                urljoin(stream_domain, stream['url']), video_id, fatal=False,
+                note=f'Downloading video info for format {resolution or stream_name}'),
+                ('info', {url_or_none}))
+            if not format_url:
+                continue
+            tbr = int_or_none(stream.get('filebitrate') or self._search_regex(
+                r'_(\d+)_mp4/', format_url, 'tbr', default=None))
+            formats.append({
+                'format_id': str(tbr or idx),
+                'url': format_url,
+                'ext': 'mp4',
+                'tbr': tbr,
+                'vcodec': stream.get('videoFormat'),
+                'acodec': stream.get('audioFormat'),
+                **parse_resolution(resolution),
+                'protocol': 'm3u8_native',
+                'http_headers': {
+                    'Referer': url,
+                },
+                'format_note': stream_name,
+            })
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            **traverse_obj(api_data, ('info', {
+                'title': ('title', {str.strip}),
+                'description': ('desc', {str}),
+                'duration': ('duration', {int_or_none}),
+                'thumbnail': ('thumb', {url_or_none}),
+            })),
+            'subtitles': self.extract_subtitles(video_id, stream_domain),
+        }
+
+    def _get_subtitles(self, video_id, domain):
+        info = self._download_json(f'https://pcweb.api.mgtv.com/video/title?videoId={video_id}',
+                                   video_id, fatal=False) or {}
+        subtitles = {}
+        for sub in try_get(info, lambda x: x['data']['title']) or []:
+            url_sub = sub.get('url')
+            if not url_sub:
+                continue
+            locale = sub.get('captionSimpleName') or 'en'
+            sub = self._download_json(f'{domain}{url_sub}', video_id, fatal=False,
+                                      note=f'Download subtitle for locale {sub.get("name")} ({locale})') or {}
+            sub_url = url_or_none(sub.get('info'))
+            if not sub_url:
+                continue
+            subtitles.setdefault(locale.lower(), []).append({
+                'url': sub_url,
+                'name': sub.get('name'),
+                'ext': 'srt'
+            })
+        return subtitles

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/microsoftembed.py

@@ -0,0 +1,65 @@
+from .common import InfoExtractor
+from ..utils import int_or_none, traverse_obj, unified_timestamp
+
+
+class MicrosoftEmbedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?microsoft\.com/(?:[^/]+/)?videoplayer/embed/(?P<id>[a-z0-9A-Z]+)'
+
+    _TESTS = [{
+        'url': 'https://www.microsoft.com/en-us/videoplayer/embed/RWL07e',
+        'md5': 'eb0ae9007f9b305f9acd0a03e74cb1a9',
+        'info_dict': {
+            'id': 'RWL07e',
+            'title': 'Microsoft for Public Health and Social Services',
+            'ext': 'mp4',
+            'thumbnail': 'http://img-prod-cms-rt-microsoft-com.akamaized.net/cms/api/am/imageFileData/RWL7Ju?ver=cae5',
+            'age_limit': 0,
+            'timestamp': 1631658316,
+            'upload_date': '20210914'
+        }
+    }]
+    _API_URL = 'https://prod-video-cms-rt-microsoft-com.akamaized.net/vhs/api/videos/'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        metadata = self._download_json(self._API_URL + video_id, video_id)
+
+        formats = []
+        for source_type, source in metadata['streams'].items():
+            if source_type == 'smooth_Streaming':
+                formats.extend(self._extract_ism_formats(source['url'], video_id, 'mss'))
+            elif source_type == 'apple_HTTP_Live_Streaming':
+                formats.extend(self._extract_m3u8_formats(source['url'], video_id, 'mp4'))
+            elif source_type == 'mPEG_DASH':
+                formats.extend(self._extract_mpd_formats(source['url'], video_id))
+            else:
+                formats.append({
+                    'format_id': source_type,
+                    'url': source['url'],
+                    'height': source.get('heightPixels'),
+                    'width': source.get('widthPixels'),
+                })
+
+        subtitles = {
+            lang: [{
+                'url': data.get('url'),
+                'ext': 'vtt',
+            }] for lang, data in traverse_obj(metadata, 'captions', default={}).items()
+        }
+
+        thumbnails = [{
+            'url': thumb.get('url'),
+            'width': thumb.get('width') or None,
+            'height': thumb.get('height') or None,
+        } for thumb in traverse_obj(metadata, ('snippet', 'thumbnails', ...))]
+        self._remove_duplicate_formats(thumbnails)
+
+        return {
+            'id': video_id,
+            'title': traverse_obj(metadata, ('snippet', 'title')),
+            'timestamp': unified_timestamp(traverse_obj(metadata, ('snippet', 'activeStartDate'))),
+            'age_limit': int_or_none(traverse_obj(metadata, ('snippet', 'minimumAge'))) or 0,
+            'formats': formats,
+            'subtitles': subtitles,
+            'thumbnails': thumbnails,
+        }

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/microsoftstream.py

@@ -0,0 +1,121 @@
+from base64 import b64decode
+
+from .common import InfoExtractor
+from ..utils import (
+    merge_dicts,
+    parse_iso8601,
+    parse_duration,
+    parse_resolution,
+    try_get,
+    url_basename,
+)
+
+
+class MicrosoftStreamIE(InfoExtractor):
+    IE_NAME = 'microsoftstream'
+    IE_DESC = 'Microsoft Stream'
+    _VALID_URL = r'https?://(?:web|www|msit)\.microsoftstream\.com/video/(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
+
+    _TESTS = [{
+        'url': 'https://web.microsoftstream.com/video/6e51d928-4f46-4f1c-b141-369925e37b62?list=user&userId=f5491e02-e8fe-4e34-b67c-ec2e79a6ecc0',
+        'only_matching': True,
+    }, {
+        'url': 'https://msit.microsoftstream.com/video/b60f5987-aabd-4e1c-a42f-c559d138f2ca',
+        'only_matching': True,
+    }]
+
+    def _get_all_subtitles(self, api_url, video_id, headers):
+        subtitles = {}
+        automatic_captions = {}
+        text_tracks = self._download_json(
+            f'{api_url}/videos/{video_id}/texttracks', video_id,
+            note='Downloading subtitles JSON', fatal=False, headers=headers,
+            query={'api-version': '1.4-private'}).get('value') or []
+        for track in text_tracks:
+            if not track.get('language') or not track.get('url'):
+                continue
+            sub_dict = automatic_captions if track.get('autoGenerated') else subtitles
+            sub_dict.setdefault(track['language'], []).append({
+                'ext': 'vtt',
+                'url': track.get('url')
+            })
+        return {
+            'subtitles': subtitles,
+            'automatic_captions': automatic_captions
+        }
+
+    def extract_all_subtitles(self, *args, **kwargs):
+        if (self.get_param('writesubtitles', False)
+                or self.get_param('writeautomaticsub', False)
+                or self.get_param('listsubtitles')):
+            return self._get_all_subtitles(*args, **kwargs)
+        return {}
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        if '<title>Microsoft Stream</title>' not in webpage:
+            self.raise_login_required(method='cookies')
+
+        access_token = self._html_search_regex(r'"AccessToken":"(.+?)"', webpage, 'access token')
+        api_url = self._html_search_regex(r'"ApiGatewayUri":"(.+?)"', webpage, 'api url')
+
+        headers = {'Authorization': f'Bearer {access_token}'}
+
+        video_data = self._download_json(
+            f'{api_url}/videos/{video_id}', video_id,
+            headers=headers, query={
+                '$expand': 'creator,tokens,status,liveEvent,extensions',
+                'api-version': '1.4-private'
+            })
+        video_id = video_data.get('id') or video_id
+        language = video_data.get('language')
+
+        thumbnails = []
+        for thumbnail_id in ('extraSmall', 'small', 'medium', 'large'):
+            thumbnail_url = try_get(video_data, lambda x: x['posterImage'][thumbnail_id]['url'], str)
+            if not thumbnail_url:
+                continue
+            thumb = {
+                'id': thumbnail_id,
+                'url': thumbnail_url,
+            }
+            thumb_name = url_basename(thumbnail_url)
+            thumb_name = str(b64decode(thumb_name + '=' * (-len(thumb_name) % 4)))
+            thumb.update(parse_resolution(thumb_name))
+            thumbnails.append(thumb)
+
+        formats = []
+        for playlist in video_data['playbackUrls']:
+            if playlist['mimeType'] == 'application/vnd.apple.mpegurl':
+                formats.extend(self._extract_m3u8_formats(
+                    playlist['playbackUrl'], video_id,
+                    ext='mp4', entry_protocol='m3u8_native', m3u8_id='hls',
+                    fatal=False, headers=headers))
+            elif playlist['mimeType'] == 'application/dash+xml':
+                formats.extend(self._extract_mpd_formats(
+                    playlist['playbackUrl'], video_id, mpd_id='dash',
+                    fatal=False, headers=headers))
+            elif playlist['mimeType'] == 'application/vnd.ms-sstr+xml':
+                formats.extend(self._extract_ism_formats(
+                    playlist['playbackUrl'], video_id, ism_id='mss',
+                    fatal=False, headers=headers))
+        formats = [merge_dicts(f, {'language': language}) for f in formats]
+
+        return {
+            'id': video_id,
+            'title': video_data['name'],
+            'description': video_data.get('description'),
+            'uploader': try_get(video_data, lambda x: x['creator']['name'], str),
+            'uploader_id': try_get(video_data, (lambda x: x['creator']['mail'],
+                                                lambda x: x['creator']['id']), str),
+            'thumbnails': thumbnails,
+            **self.extract_all_subtitles(api_url, video_id, headers),
+            'timestamp': parse_iso8601(video_data.get('created')),
+            'duration': parse_duration(try_get(video_data, lambda x: x['media']['duration'])),
+            'webpage_url': f'https://web.microsoftstream.com/video/{video_id}',
+            'view_count': try_get(video_data, lambda x: x['metrics']['views'], int),
+            'like_count': try_get(video_data, lambda x: x['metrics']['likes'], int),
+            'comment_count': try_get(video_data, lambda x: x['metrics']['comments'], int),
+            'formats': formats,
+        }

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/microsoftvirtualacademy.py

@@ -0,0 +1,189 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_duration,
+    smuggle_url,
+    unsmuggle_url,
+    xpath_text,
+)
+
+
+class MicrosoftVirtualAcademyBaseIE(InfoExtractor):
+    def _extract_base_url(self, course_id, display_id):
+        return self._download_json(
+            'https://api-mlxprod.microsoft.com/services/products/anonymous/%s' % course_id,
+            display_id, 'Downloading course base URL')
+
+    def _extract_chapter_and_title(self, title):
+        if not title:
+            return None, None
+        m = re.search(r'(?P<chapter>\d+)\s*\|\s*(?P<title>.+)', title)
+        return (int(m.group('chapter')), m.group('title')) if m else (None, title)
+
+
+class MicrosoftVirtualAcademyIE(MicrosoftVirtualAcademyBaseIE):
+    IE_NAME = 'mva'
+    IE_DESC = 'Microsoft Virtual Academy videos'
+    _VALID_URL = r'(?:%s:|https?://(?:mva\.microsoft|(?:www\.)?microsoftvirtualacademy)\.com/[^/]+/training-courses/[^/?#&]+-)(?P<course_id>\d+)(?::|\?l=)(?P<id>[\da-zA-Z]+_\d+)' % IE_NAME
+
+    _TESTS = [{
+        'url': 'https://mva.microsoft.com/en-US/training-courses/microsoft-azure-fundamentals-virtual-machines-11788?l=gfVXISmEB_6804984382',
+        'md5': '7826c44fc31678b12ad8db11f6b5abb9',
+        'info_dict': {
+            'id': 'gfVXISmEB_6804984382',
+            'ext': 'mp4',
+            'title': 'Course Introduction',
+            'formats': 'mincount:3',
+            'subtitles': {
+                'en': [{
+                    'ext': 'ttml',
+                }],
+            },
+        }
+    }, {
+        'url': 'mva:11788:gfVXISmEB_6804984382',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        url, smuggled_data = unsmuggle_url(url, {})
+
+        mobj = self._match_valid_url(url)
+        course_id = mobj.group('course_id')
+        video_id = mobj.group('id')
+
+        base_url = smuggled_data.get('base_url') or self._extract_base_url(course_id, video_id)
+
+        settings = self._download_xml(
+            '%s/content/content_%s/videosettings.xml?v=1' % (base_url, video_id),
+            video_id, 'Downloading video settings XML')
+
+        _, title = self._extract_chapter_and_title(xpath_text(
+            settings, './/Title', 'title', fatal=True))
+
+        formats = []
+
+        for sources in settings.findall('.//MediaSources'):
+            sources_type = sources.get('videoType')
+            for source in sources.findall('./MediaSource'):
+                video_url = source.text
+                if not video_url or not video_url.startswith('http'):
+                    continue
+                if sources_type == 'smoothstreaming':
+                    formats.extend(self._extract_ism_formats(
+                        video_url, video_id, 'mss', fatal=False))
+                    continue
+                video_mode = source.get('videoMode')
+                height = int_or_none(self._search_regex(
+                    r'^(\d+)[pP]$', video_mode or '', 'height', default=None))
+                codec = source.get('codec')
+                acodec, vcodec = [None] * 2
+                if codec:
+                    codecs = codec.split(',')
+                    if len(codecs) == 2:
+                        acodec, vcodec = codecs
+                    elif len(codecs) == 1:
+                        vcodec = codecs[0]
+                formats.append({
+                    'url': video_url,
+                    'format_id': video_mode,
+                    'height': height,
+                    'acodec': acodec,
+                    'vcodec': vcodec,
+                })
+
+        subtitles = {}
+        for source in settings.findall('.//MarkerResourceSource'):
+            subtitle_url = source.text
+            if not subtitle_url:
+                continue
+            subtitles.setdefault('en', []).append({
+                'url': '%s/%s' % (base_url, subtitle_url),
+                'ext': source.get('type'),
+            })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'subtitles': subtitles,
+            'formats': formats
+        }
+
+
+class MicrosoftVirtualAcademyCourseIE(MicrosoftVirtualAcademyBaseIE):
+    IE_NAME = 'mva:course'
+    IE_DESC = 'Microsoft Virtual Academy courses'
+    _VALID_URL = r'(?:%s:|https?://(?:mva\.microsoft|(?:www\.)?microsoftvirtualacademy)\.com/[^/]+/training-courses/(?P<display_id>[^/?#&]+)-)(?P<id>\d+)' % IE_NAME
+
+    _TESTS = [{
+        'url': 'https://mva.microsoft.com/en-US/training-courses/microsoft-azure-fundamentals-virtual-machines-11788',
+        'info_dict': {
+            'id': '11788',
+            'title': 'Microsoft Azure Fundamentals: Virtual Machines',
+        },
+        'playlist_count': 36,
+    }, {
+        # with emphasized chapters
+        'url': 'https://mva.microsoft.com/en-US/training-courses/developing-windows-10-games-with-construct-2-16335',
+        'info_dict': {
+            'id': '16335',
+            'title': 'Developing Windows 10 Games with Construct 2',
+        },
+        'playlist_count': 10,
+    }, {
+        'url': 'https://www.microsoftvirtualacademy.com/en-US/training-courses/microsoft-azure-fundamentals-virtual-machines-11788',
+        'only_matching': True,
+    }, {
+        'url': 'mva:course:11788',
+        'only_matching': True,
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if MicrosoftVirtualAcademyIE.suitable(url) else super(
+            MicrosoftVirtualAcademyCourseIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        mobj = self._match_valid_url(url)
+        course_id = mobj.group('id')
+        display_id = mobj.group('display_id')
+
+        base_url = self._extract_base_url(course_id, display_id)
+
+        manifest = self._download_json(
+            '%s/imsmanifestlite.json' % base_url,
+            display_id, 'Downloading course manifest JSON')['manifest']
+
+        organization = manifest['organizations']['organization'][0]
+
+        entries = []
+        for chapter in organization['item']:
+            chapter_number, chapter_title = self._extract_chapter_and_title(chapter.get('title'))
+            chapter_id = chapter.get('@identifier')
+            for item in chapter.get('item', []):
+                item_id = item.get('@identifier')
+                if not item_id:
+                    continue
+                metadata = item.get('resource', {}).get('metadata') or {}
+                if metadata.get('learningresourcetype') != 'Video':
+                    continue
+                _, title = self._extract_chapter_and_title(item.get('title'))
+                duration = parse_duration(metadata.get('duration'))
+                description = metadata.get('description')
+                entries.append({
+                    '_type': 'url_transparent',
+                    'url': smuggle_url(
+                        'mva:%s:%s' % (course_id, item_id), {'base_url': base_url}),
+                    'title': title,
+                    'description': description,
+                    'duration': duration,
+                    'chapter': chapter_title,
+                    'chapter_number': chapter_number,
+                    'chapter_id': chapter_id,
+                })
+
+        title = organization.get('title') or manifest.get('metadata', {}).get('title')
+
+        return self.playlist_result(entries, course_id, title)