@tushar-br/desktop 1.0.282 → 1.0.283

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/cspan.py

@@ -0,0 +1,286 @@
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_HTMLParseError
+from ..utils import (
+    determine_ext,
+    ExtractorError,
+    extract_attributes,
+    find_xpath_attr,
+    get_element_by_attribute,
+    get_element_by_class,
+    int_or_none,
+    join_nonempty,
+    js_to_json,
+    merge_dicts,
+    parse_iso8601,
+    parse_qs,
+    smuggle_url,
+    str_to_int,
+    unescapeHTML,
+)
+from .senategov import SenateISVPIE
+from .ustream import UstreamIE
+
+
+class CSpanIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?c-span\.org/video/\?(?P<id>[0-9a-f]+)'
+    IE_DESC = 'C-SPAN'
+    _TESTS = [{
+        'url': 'http://www.c-span.org/video/?313572-1/HolderonV',
+        'md5': '94b29a4f131ff03d23471dd6f60b6a1d',
+        'info_dict': {
+            'id': '315139',
+            'title': 'Attorney General Eric Holder on Voting Rights Act Decision',
+        },
+        'playlist_mincount': 2,
+        'skip': 'Regularly fails on travis, for unknown reasons',
+    }, {
+        'url': 'http://www.c-span.org/video/?c4486943/cspan-international-health-care-models',
+        # md5 is unstable
+        'info_dict': {
+            'id': 'c4486943',
+            'ext': 'mp4',
+            'title': 'CSPAN - International Health Care Models',
+            'description': 'md5:7a985a2d595dba00af3d9c9f0783c967',
+        }
+    }, {
+        'url': 'http://www.c-span.org/video/?318608-1/gm-ignition-switch-recall',
+        'info_dict': {
+            'id': '342759',
+            'title': 'General Motors Ignition Switch Recall',
+        },
+        'playlist_mincount': 6,
+    }, {
+        # Video from senate.gov
+        'url': 'http://www.c-span.org/video/?104517-1/immigration-reforms-needed-protect-skilled-american-workers',
+        'info_dict': {
+            'id': 'judiciary031715',
+            'ext': 'mp4',
+            'title': 'Immigration Reforms Needed to Protect Skilled American Workers',
+        },
+        'params': {
+            'skip_download': True,  # m3u8 downloads
+        }
+    }, {
+        # Ustream embedded video
+        'url': 'https://www.c-span.org/video/?114917-1/armed-services',
+        'info_dict': {
+            'id': '58428542',
+            'ext': 'flv',
+            'title': 'USHR07 Armed Services Committee',
+            'description': 'hsas00-2118-20150204-1000et-07\n\n\nUSHR07 Armed Services Committee',
+            'timestamp': 1423060374,
+            'upload_date': '20150204',
+            'uploader': 'HouseCommittee',
+            'uploader_id': '12987475',
+        },
+    }, {
+        # Audio Only
+        'url': 'https://www.c-span.org/video/?437336-1/judiciary-antitrust-competition-policy-consumer-rights',
+        'only_matching': True,
+    }]
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_%s/index.html?videoId=%s'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_type = None
+        webpage = self._download_webpage(url, video_id)
+
+        ustream_url = UstreamIE._extract_url(webpage)
+        if ustream_url:
+            return self.url_result(ustream_url, UstreamIE.ie_key())
+
+        if '&vod' not in url:
+            bc = self._search_regex(
+                r"(<[^>]+id='brightcove-player-embed'[^>]+>)",
+                webpage, 'brightcove embed', default=None)
+            if bc:
+                bc_attr = extract_attributes(bc)
+                bc_url = self.BRIGHTCOVE_URL_TEMPLATE % (
+                    bc_attr.get('data-bcaccountid', '3162030207001'),
+                    bc_attr.get('data-noprebcplayerid', 'SyGGpuJy3g'),
+                    bc_attr.get('data-newbcplayerid', 'default'),
+                    bc_attr['data-bcid'])
+                return self.url_result(smuggle_url(bc_url, {'source_url': url}))
+
+        def add_referer(formats):
+            for f in formats:
+                f.setdefault('http_headers', {})['Referer'] = url
+
+        # As of 01.12.2020 this path looks to cover all cases making the rest
+        # of the code unnecessary
+        jwsetup = self._parse_json(
+            self._search_regex(
+                r'(?s)jwsetup\s*=\s*({.+?})\s*;', webpage, 'jwsetup',
+                default='{}'),
+            video_id, transform_source=js_to_json, fatal=False)
+        if jwsetup:
+            info = self._parse_jwplayer_data(
+                jwsetup, video_id, require_title=False, m3u8_id='hls',
+                base_url=url)
+            add_referer(info['formats'])
+            for subtitles in info['subtitles'].values():
+                for subtitle in subtitles:
+                    ext = determine_ext(subtitle['url'])
+                    if ext == 'php':
+                        ext = 'vtt'
+                    subtitle['ext'] = ext
+            ld_info = self._search_json_ld(webpage, video_id, default={})
+            try:
+                title = get_element_by_class('video-page-title', webpage)
+            except compat_HTMLParseError:
+                title = None
+            if title is None:
+                title = self._og_search_title(webpage)
+            description = get_element_by_attribute('itemprop', 'description', webpage) or \
+                self._html_search_meta(['og:description', 'description'], webpage)
+            return merge_dicts(info, ld_info, {
+                'title': title,
+                'thumbnail': get_element_by_attribute('itemprop', 'thumbnailUrl', webpage),
+                'description': description,
+                'timestamp': parse_iso8601(get_element_by_attribute('itemprop', 'uploadDate', webpage)),
+                'location': get_element_by_attribute('itemprop', 'contentLocation', webpage),
+                'duration': int_or_none(self._search_regex(
+                    r'jwsetup\.seclength\s*=\s*(\d+);',
+                    webpage, 'duration', fatal=False)),
+                'view_count': str_to_int(self._search_regex(
+                    r"<span[^>]+class='views'[^>]*>([\d,]+)\s+Views</span>",
+                    webpage, 'views', fatal=False)),
+            })
+
+        # Obsolete
+        # We first look for clipid, because clipprog always appears before
+        patterns = [r'id=\'clip(%s)\'\s*value=\'([0-9]+)\'' % t for t in ('id', 'prog')]
+        results = list(filter(None, (re.search(p, webpage) for p in patterns)))
+        if results:
+            matches = results[0]
+            video_type, video_id = matches.groups()
+            video_type = 'clip' if video_type == 'id' else 'program'
+        else:
+            m = re.search(r'data-(?P<type>clip|prog)id=["\'](?P<id>\d+)', webpage)
+            if m:
+                video_id = m.group('id')
+                video_type = 'program' if m.group('type') == 'prog' else 'clip'
+            else:
+                senate_isvp_url = SenateISVPIE._extract_url(webpage)
+                if senate_isvp_url:
+                    title = self._og_search_title(webpage)
+                    surl = smuggle_url(senate_isvp_url, {'force_title': title})
+                    return self.url_result(surl, 'SenateISVP', video_id, title)
+                video_id = self._search_regex(
+                    r'jwsetup\.clipprog\s*=\s*(\d+);',
+                    webpage, 'jwsetup program id', default=None)
+                if video_id:
+                    video_type = 'program'
+        if video_type is None or video_id is None:
+            error_message = get_element_by_class('VLplayer-error-message', webpage)
+            if error_message:
+                raise ExtractorError(error_message)
+            raise ExtractorError('unable to find video id and type')
+
+        def get_text_attr(d, attr):
+            return d.get(attr, {}).get('#text')
+
+        data = self._download_json(
+            'http://www.c-span.org/assets/player/ajax-player.php?os=android&html5=%s&id=%s' % (video_type, video_id),
+            video_id)['video']
+        if data['@status'] != 'Success':
+            raise ExtractorError('%s said: %s' % (self.IE_NAME, get_text_attr(data, 'error')), expected=True)
+
+        doc = self._download_xml(
+            'http://www.c-span.org/common/services/flashXml.php?%sid=%s' % (video_type, video_id),
+            video_id)
+
+        description = self._html_search_meta('description', webpage)
+
+        title = find_xpath_attr(doc, './/string', 'name', 'title').text
+        thumbnail = find_xpath_attr(doc, './/string', 'name', 'poster').text
+
+        files = data['files']
+        capfile = get_text_attr(data, 'capfile')
+
+        entries = []
+        for partnum, f in enumerate(files):
+            formats = []
+            for quality in f.get('qualities', []):
+                formats.append({
+                    'format_id': '%s-%sp' % (get_text_attr(quality, 'bitrate'), get_text_attr(quality, 'height')),
+                    'url': unescapeHTML(get_text_attr(quality, 'file')),
+                    'height': int_or_none(get_text_attr(quality, 'height')),
+                    'tbr': int_or_none(get_text_attr(quality, 'bitrate')),
+                })
+            if not formats:
+                path = unescapeHTML(get_text_attr(f, 'path'))
+                if not path:
+                    continue
+                formats = self._extract_m3u8_formats(
+                    path, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls') if determine_ext(path) == 'm3u8' else [{'url': path, }]
+            add_referer(formats)
+            entries.append({
+                'id': '%s_%d' % (video_id, partnum + 1),
+                'title': (
+                    title if len(files) == 1 else
+                    '%s part %d' % (title, partnum + 1)),
+                'formats': formats,
+                'description': description,
+                'thumbnail': thumbnail,
+                'duration': int_or_none(get_text_attr(f, 'length')),
+                'subtitles': {
+                    'en': [{
+                        'url': capfile,
+                        'ext': determine_ext(capfile, 'dfxp')
+                    }],
+                } if capfile else None,
+            })
+
+        if len(entries) == 1:
+            entry = dict(entries[0])
+            entry['id'] = 'c' + video_id if video_type == 'clip' else video_id
+            return entry
+        else:
+            return {
+                '_type': 'playlist',
+                'entries': entries,
+                'title': title,
+                'id': 'c' + video_id if video_type == 'clip' else video_id,
+            }
+
+
+class CSpanCongressIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?c-span\.org/congress/'
+    _TESTS = [{
+        'url': 'https://www.c-span.org/congress/?chamber=house&date=2017-12-13&t=1513208380',
+        'info_dict': {
+            'id': 'house_2017-12-13',
+            'title': 'Congressional Chronicle - Members of Congress, Hearings and More',
+            'description': 'md5:54c264b7a8f219937987610243305a84',
+            'thumbnail': r're:https://ximage.c-spanvideo.org/.+',
+            'ext': 'mp4'
+        }
+    }]
+
+    def _real_extract(self, url):
+        query = parse_qs(url)
+        video_date = query.get('date', [None])[0]
+        video_id = join_nonempty(query.get('chamber', ['senate'])[0], video_date, delim='_')
+        webpage = self._download_webpage(url, video_id)
+        if not video_date:
+            jwp_date = re.search(r'jwsetup.clipprogdate = \'(?P<date>\d{4}-\d{2}-\d{2})\';', webpage)
+            if jwp_date:
+                video_id = f'{video_id}_{jwp_date.group("date")}'
+        jwplayer_data = self._parse_json(
+            self._search_regex(r'jwsetup\s*=\s*({(?:.|\n)[^;]+});', webpage, 'player config'),
+            video_id, transform_source=js_to_json)
+
+        title = self._generic_title('', webpage)
+        description = (self._og_search_description(webpage, default=None)
+                       or self._html_search_meta('description', webpage, 'description', default=None))
+
+        return {
+            **self._parse_jwplayer_data(jwplayer_data, video_id, False),
+            'title': re.sub(r'\s+', ' ', title.split('|')[0]).strip(),
+            'description': description,
+            'http_headers': {'Referer': 'https://www.c-span.org/'},
+        }

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/ctsnews.py

@@ -0,0 +1,84 @@
+from .common import InfoExtractor
+from ..utils import unified_timestamp
+from .youtube import YoutubeIE
+
+
+class CtsNewsIE(InfoExtractor):
+    IE_DESC = '華視新聞'
+    _VALID_URL = r'https?://news\.cts\.com\.tw/[a-z]+/[a-z]+/\d+/(?P<id>\d+)\.html'
+    _TESTS = [{
+        'url': 'http://news.cts.com.tw/cts/international/201501/201501291578109.html',
+        'md5': 'a9875cb790252b08431186d741beaabe',
+        'info_dict': {
+            'id': '201501291578109',
+            'ext': 'mp4',
+            'title': '以色列.真主黨交火 3人死亡 - 華視新聞網',
+            'description': '以色列和黎巴嫩真主黨，爆發五年最嚴重衝突，雙方砲轟交火，兩名以軍死亡，還有一名西班牙籍的聯合國維和人員也不幸罹難。大陸陝西、河南、安徽、江蘇和湖北五個省份出現大暴雪，嚴重影響陸空交通，不過九華山卻出現...',
+            'timestamp': 1422528540,
+            'upload_date': '20150129',
+        }
+    }, {
+        # News count not appear on page but still available in database
+        'url': 'http://news.cts.com.tw/cts/international/201309/201309031304098.html',
+        'md5': '3aee7e0df7cdff94e43581f54c22619e',
+        'info_dict': {
+            'id': '201309031304098',
+            'ext': 'mp4',
+            'title': '韓國31歲童顏男 貌如十多歲小孩 - 華視新聞網',
+            'description': '越有年紀的人，越希望看起來年輕一點，而南韓卻有一位31歲的男子，看起來像是11、12歲的小孩，身...',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'timestamp': 1378205880,
+            'upload_date': '20130903',
+        }
+    }, {
+        # With Youtube embedded video
+        'url': 'http://news.cts.com.tw/cts/money/201501/201501291578003.html',
+        'md5': 'e4726b2ccd70ba2c319865e28f0a91d1',
+        'info_dict': {
+            'id': 'OVbfO7d0_hQ',
+            'ext': 'mp4',
+            'title': 'iPhone6熱銷 蘋果財報亮眼',
+            'description': 'md5:f395d4f485487bb0f992ed2c4b07aa7d',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'upload_date': '20150128',
+            'uploader_id': 'TBSCTS',
+            'uploader': '中華電視公司',
+        },
+        'add_ie': ['Youtube'],
+    }]
+
+    def _real_extract(self, url):
+        news_id = self._match_id(url)
+        page = self._download_webpage(url, news_id)
+
+        news_id = self._hidden_inputs(page).get('get_id')
+
+        if news_id:
+            mp4_feed = self._download_json(
+                'http://news.cts.com.tw/action/test_mp4feed.php',
+                news_id, note='Fetching feed', query={'news_id': news_id})
+            video_url = mp4_feed['source_url']
+        else:
+            self.to_screen('Not CTSPlayer video, trying Youtube...')
+            youtube_url = YoutubeIE._extract_url(page)
+
+            return self.url_result(youtube_url, ie='Youtube')
+
+        description = self._html_search_meta('description', page)
+        title = self._html_search_meta('title', page, fatal=True)
+        thumbnail = self._html_search_meta('image', page)
+
+        datetime_str = self._html_search_regex(
+            r'(\d{4}/\d{2}/\d{2} \d{2}:\d{2})', page, 'date and time', fatal=False)
+        timestamp = None
+        if datetime_str:
+            timestamp = unified_timestamp(datetime_str) - 8 * 3600
+
+        return {
+            'id': news_id,
+            'url': video_url,
+            'title': title,
+            'description': description,
+            'thumbnail': thumbnail,
+            'timestamp': timestamp,
+        }

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/ctv.py

@@ -0,0 +1,49 @@
+from .common import InfoExtractor
+
+
+class CTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?ctv\.ca/(?P<id>(?:show|movie)s/[^/]+/[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://www.ctv.ca/shows/your-morning/wednesday-december-23-2020-s5e88',
+        'info_dict': {
+            'id': '2102249',
+            'ext': 'flv',
+            'title': 'Wednesday, December 23, 2020',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'Your Morning delivers original perspectives and unique insights into the headlines of the day.',
+            'timestamp': 1608732000,
+            'upload_date': '20201223',
+            'series': 'Your Morning',
+            'season': '2020-2021',
+            'season_number': 5,
+            'episode_number': 88,
+            'tags': ['Your Morning'],
+            'categories': ['Talk Show'],
+            'duration': 7467.126,
+        },
+    }, {
+        'url': 'https://www.ctv.ca/movies/adam-sandlers-eight-crazy-nights/adam-sandlers-eight-crazy-nights',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        content = self._download_json(
+            'https://www.ctv.ca/space-graphql/graphql', display_id, query={
+                'query': '''{
+  resolvedPath(path: "/%s") {
+    lastSegment {
+      content {
+        ... on AxisContent {
+          axisId
+          videoPlayerDestCode
+        }
+      }
+    }
+  }
+}''' % display_id,
+            })['data']['resolvedPath']['lastSegment']['content']
+        video_id = content['axisId']
+        return self.url_result(
+            '9c9media:%s:%s' % (content['videoPlayerDestCode'], video_id),
+            'NineCNineMedia', video_id)

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/ctvnews.py

@@ -0,0 +1,70 @@
+import re
+
+from .common import InfoExtractor
+from ..utils import orderedSet
+
+
+class CTVNewsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:.+?\.)?ctvnews\.ca/(?:video\?(?:clip|playlist|bin)Id=|.*?)(?P<id>[0-9.]+)'
+    _TESTS = [{
+        'url': 'http://www.ctvnews.ca/video?clipId=901995',
+        'md5': '9b8624ba66351a23e0b6e1391971f9af',
+        'info_dict': {
+            'id': '901995',
+            'ext': 'flv',
+            'title': 'Extended: \'That person cannot be me\' Johnson says',
+            'description': 'md5:958dd3b4f5bbbf0ed4d045c790d89285',
+            'timestamp': 1467286284,
+            'upload_date': '20160630',
+        }
+    }, {
+        'url': 'http://www.ctvnews.ca/video?playlistId=1.2966224',
+        'info_dict':
+        {
+            'id': '1.2966224',
+        },
+        'playlist_mincount': 19,
+    }, {
+        'url': 'http://www.ctvnews.ca/video?binId=1.2876780',
+        'info_dict':
+        {
+            'id': '1.2876780',
+        },
+        'playlist_mincount': 100,
+    }, {
+        'url': 'http://www.ctvnews.ca/1.810401',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.ctvnews.ca/canadiens-send-p-k-subban-to-nashville-in-blockbuster-trade-1.2967231',
+        'only_matching': True,
+    }, {
+        'url': 'http://vancouverisland.ctvnews.ca/video?clipId=761241',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        page_id = self._match_id(url)
+
+        def ninecninemedia_url_result(clip_id):
+            return {
+                '_type': 'url_transparent',
+                'id': clip_id,
+                'url': '9c9media:ctvnews_web:%s' % clip_id,
+                'ie_key': 'NineCNineMedia',
+            }
+
+        if page_id.isdigit():
+            return ninecninemedia_url_result(page_id)
+        else:
+            webpage = self._download_webpage('http://www.ctvnews.ca/%s' % page_id, page_id, query={
+                'ot': 'example.AjaxPageLayout.ot',
+                'maxItemsPerPage': 1000000,
+            })
+            entries = [ninecninemedia_url_result(clip_id) for clip_id in orderedSet(
+                re.findall(r'clip\.id\s*=\s*(\d+);', webpage))]
+            if not entries:
+                webpage = self._download_webpage(url, page_id)
+                if 'getAuthStates("' in webpage:
+                    entries = [ninecninemedia_url_result(clip_id) for clip_id in
+                               self._search_regex(r'getAuthStates\("([\d+,]+)"', webpage, 'clip ids').split(',')]
+            return self.playlist_result(entries, page_id)

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/cultureunplugged.py

@@ -0,0 +1,65 @@
+import time
+
+from .common import InfoExtractor
+from ..networking import HEADRequest
+from ..utils import int_or_none
+
+
+class CultureUnpluggedIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?cultureunplugged\.com/documentary/watch-online/play/(?P<id>\d+)(?:/(?P<display_id>[^/]+))?'
+    _TESTS = [{
+        'url': 'http://www.cultureunplugged.com/documentary/watch-online/play/53662/The-Next--Best-West',
+        'md5': 'ac6c093b089f7d05e79934dcb3d228fc',
+        'info_dict': {
+            'id': '53662',
+            'display_id': 'The-Next--Best-West',
+            'ext': 'mp4',
+            'title': 'The Next, Best West',
+            'description': 'md5:0423cd00833dea1519cf014e9d0903b1',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'creator': 'Coldstream Creative',
+            'duration': 2203,
+            'view_count': int,
+        }
+    }, {
+        'url': 'http://www.cultureunplugged.com/documentary/watch-online/play/53662',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        mobj = self._match_valid_url(url)
+        video_id = mobj.group('id')
+        display_id = mobj.group('display_id') or video_id
+
+        # request setClientTimezone.php to get PHPSESSID cookie which is need to get valid json data in the next request
+        self._request_webpage(HEADRequest(
+            'http://www.cultureunplugged.com/setClientTimezone.php?timeOffset=%d' % -(time.timezone / 3600)), display_id)
+        movie_data = self._download_json(
+            'http://www.cultureunplugged.com/movie-data/cu-%s.json' % video_id, display_id)
+
+        video_url = movie_data['url']
+        title = movie_data['title']
+
+        description = movie_data.get('synopsis')
+        creator = movie_data.get('producer')
+        duration = int_or_none(movie_data.get('duration'))
+        view_count = int_or_none(movie_data.get('views'))
+
+        thumbnails = [{
+            'url': movie_data['%s_thumb' % size],
+            'id': size,
+            'preference': preference,
+        } for preference, size in enumerate((
+            'small', 'large')) if movie_data.get('%s_thumb' % size)]
+
+        return {
+            'id': video_id,
+            'display_id': display_id,
+            'url': video_url,
+            'title': title,
+            'description': description,
+            'creator': creator,
+            'duration': duration,
+            'view_count': view_count,
+            'thumbnails': thumbnails,
+        }