@tushar-br/desktop 1.0.282 → 1.0.283

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/openload.py

@@ -0,0 +1,243 @@
+import collections
+import contextlib
+import json
+import os
+import subprocess
+import tempfile
+
+from ..compat import compat_urlparse
+from ..utils import (
+    ExtractorError,
+    Popen,
+    check_executable,
+    format_field,
+    get_exe_version,
+    is_outdated_version,
+    shell_quote,
+)
+
+
+def cookie_to_dict(cookie):
+    cookie_dict = {
+        'name': cookie.name,
+        'value': cookie.value,
+    }
+    if cookie.port_specified:
+        cookie_dict['port'] = cookie.port
+    if cookie.domain_specified:
+        cookie_dict['domain'] = cookie.domain
+    if cookie.path_specified:
+        cookie_dict['path'] = cookie.path
+    if cookie.expires is not None:
+        cookie_dict['expires'] = cookie.expires
+    if cookie.secure is not None:
+        cookie_dict['secure'] = cookie.secure
+    if cookie.discard is not None:
+        cookie_dict['discard'] = cookie.discard
+    with contextlib.suppress(TypeError):
+        if (cookie.has_nonstandard_attr('httpOnly')
+                or cookie.has_nonstandard_attr('httponly')
+                or cookie.has_nonstandard_attr('HttpOnly')):
+            cookie_dict['httponly'] = True
+    return cookie_dict
+
+
+def cookie_jar_to_list(cookie_jar):
+    return [cookie_to_dict(cookie) for cookie in cookie_jar]
+
+
+class PhantomJSwrapper:
+    """PhantomJS wrapper class
+
+    This class is experimental.
+    """
+
+    INSTALL_HINT = 'Please download it from https://phantomjs.org/download.html'
+
+    _BASE_JS = R'''
+        phantom.onError = function(msg, trace) {{
+          var msgStack = ['PHANTOM ERROR: ' + msg];
+          if(trace && trace.length) {{
+            msgStack.push('TRACE:');
+            trace.forEach(function(t) {{
+              msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line
+                + (t.function ? ' (in function ' + t.function +')' : ''));
+            }});
+          }}
+          console.error(msgStack.join('\n'));
+          phantom.exit(1);
+        }};
+    '''
+
+    _TEMPLATE = R'''
+        var page = require('webpage').create();
+        var fs = require('fs');
+        var read = {{ mode: 'r', charset: 'utf-8' }};
+        var write = {{ mode: 'w', charset: 'utf-8' }};
+        JSON.parse(fs.read("{cookies}", read)).forEach(function(x) {{
+          phantom.addCookie(x);
+        }});
+        page.settings.resourceTimeout = {timeout};
+        page.settings.userAgent = "{ua}";
+        page.onLoadStarted = function() {{
+          page.evaluate(function() {{
+            delete window._phantom;
+            delete window.callPhantom;
+          }});
+        }};
+        var saveAndExit = function() {{
+          fs.write("{html}", page.content, write);
+          fs.write("{cookies}", JSON.stringify(phantom.cookies), write);
+          phantom.exit();
+        }};
+        page.onLoadFinished = function(status) {{
+          if(page.url === "") {{
+            page.setContent(fs.read("{html}", read), "{url}");
+          }}
+          else {{
+            {jscode}
+          }}
+        }};
+        page.open("");
+    '''
+
+    _TMP_FILE_NAMES = ['script', 'html', 'cookies']
+
+    @staticmethod
+    def _version():
+        return get_exe_version('phantomjs', version_re=r'([0-9.]+)')
+
+    def __init__(self, extractor, required_version=None, timeout=10000):
+        self._TMP_FILES = {}
+
+        self.exe = check_executable('phantomjs', ['-v'])
+        if not self.exe:
+            raise ExtractorError(f'PhantomJS not found, {self.INSTALL_HINT}', expected=True)
+
+        self.extractor = extractor
+
+        if required_version:
+            version = self._version()
+            if is_outdated_version(version, required_version):
+                self.extractor._downloader.report_warning(
+                    'Your copy of PhantomJS is outdated, update it to version '
+                    '%s or newer if you encounter any errors.' % required_version)
+
+        for name in self._TMP_FILE_NAMES:
+            tmp = tempfile.NamedTemporaryFile(delete=False)
+            tmp.close()
+            self._TMP_FILES[name] = tmp
+
+        self.options = collections.ChainMap({
+            'timeout': timeout,
+        }, {
+            x: self._TMP_FILES[x].name.replace('\\', '\\\\').replace('"', '\\"')
+            for x in self._TMP_FILE_NAMES
+        })
+
+    def __del__(self):
+        for name in self._TMP_FILE_NAMES:
+            with contextlib.suppress(OSError, KeyError):
+                os.remove(self._TMP_FILES[name].name)
+
+    def _save_cookies(self, url):
+        cookies = cookie_jar_to_list(self.extractor.cookiejar)
+        for cookie in cookies:
+            if 'path' not in cookie:
+                cookie['path'] = '/'
+            if 'domain' not in cookie:
+                cookie['domain'] = compat_urlparse.urlparse(url).netloc
+        with open(self._TMP_FILES['cookies'].name, 'wb') as f:
+            f.write(json.dumps(cookies).encode('utf-8'))
+
+    def _load_cookies(self):
+        with open(self._TMP_FILES['cookies'].name, 'rb') as f:
+            cookies = json.loads(f.read().decode('utf-8'))
+        for cookie in cookies:
+            if cookie['httponly'] is True:
+                cookie['rest'] = {'httpOnly': None}
+            if 'expiry' in cookie:
+                cookie['expire_time'] = cookie['expiry']
+            self.extractor._set_cookie(**cookie)
+
+    def get(self, url, html=None, video_id=None, note=None, note2='Executing JS on webpage', headers={}, jscode='saveAndExit();'):
+        """
+        Downloads webpage (if needed) and executes JS
+
+        Params:
+            url: website url
+            html: optional, html code of website
+            video_id: video id
+            note: optional, displayed when downloading webpage
+            note2: optional, displayed when executing JS
+            headers: custom http headers
+            jscode: code to be executed when page is loaded
+
+        Returns tuple with:
+            * downloaded website (after JS execution)
+            * anything you print with `console.log` (but not inside `page.execute`!)
+
+        In most cases you don't need to add any `jscode`.
+        It is executed in `page.onLoadFinished`.
+        `saveAndExit();` is mandatory, use it instead of `phantom.exit()`
+        It is possible to wait for some element on the webpage, e.g.
+            var check = function() {
+              var elementFound = page.evaluate(function() {
+                return document.querySelector('#b.done') !== null;
+              });
+              if(elementFound)
+                saveAndExit();
+              else
+                window.setTimeout(check, 500);
+            }
+
+            page.evaluate(function(){
+              document.querySelector('#a').click();
+            });
+            check();
+        """
+        if 'saveAndExit();' not in jscode:
+            raise ExtractorError('`saveAndExit();` not found in `jscode`')
+        if not html:
+            html = self.extractor._download_webpage(url, video_id, note=note, headers=headers)
+        with open(self._TMP_FILES['html'].name, 'wb') as f:
+            f.write(html.encode('utf-8'))
+
+        self._save_cookies(url)
+
+        user_agent = headers.get('User-Agent') or self.extractor.get_param('http_headers')['User-Agent']
+        jscode = self._TEMPLATE.format_map(self.options.new_child({
+            'url': url,
+            'ua': user_agent.replace('"', '\\"'),
+            'jscode': jscode,
+        }))
+
+        stdout = self.execute(jscode, video_id, note=note2)
+
+        with open(self._TMP_FILES['html'].name, 'rb') as f:
+            html = f.read().decode('utf-8')
+        self._load_cookies()
+
+        return html, stdout
+
+    def execute(self, jscode, video_id=None, *, note='Executing JS'):
+        """Execute JS and return stdout"""
+        if 'phantom.exit();' not in jscode:
+            jscode += ';\nphantom.exit();'
+        jscode = self._BASE_JS + jscode
+
+        with open(self._TMP_FILES['script'].name, 'w', encoding='utf-8') as f:
+            f.write(jscode)
+        self.extractor.to_screen(f'{format_field(video_id, None, "%s: ")}{note}')
+
+        cmd = [self.exe, '--ssl-protocol=any', self._TMP_FILES['script'].name]
+        self.extractor.write_debug(f'PhantomJS command line: {shell_quote(cmd)}')
+        try:
+            stdout, stderr, returncode = Popen.run(cmd, timeout=self.options['timeout'] / 1000,
+                                                   text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        except Exception as e:
+            raise ExtractorError(f'{note} failed: Unable to run PhantomJS binary', cause=e)
+        if returncode:
+            raise ExtractorError(f'{note} failed with returncode {returncode}:\n{stderr.strip()}')
+
+        return stdout

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/openrec.py

@@ -0,0 +1,151 @@
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    get_first,
+    int_or_none,
+    traverse_obj,
+    try_get,
+    unified_strdate,
+    unified_timestamp,
+)
+from ..compat import compat_str
+
+
+class OpenRecBaseIE(InfoExtractor):
+    _M3U8_HEADERS = {'Referer': 'https://www.openrec.tv/'}
+
+    def _extract_pagestore(self, webpage, video_id):
+        return self._parse_json(
+            self._search_regex(r'(?m)window\.pageStore\s*=\s*(\{.+?\});$', webpage, 'window.pageStore'), video_id)
+
+    def _expand_media(self, video_id, media):
+        for name, m3u8_url in (media or {}).items():
+            if not m3u8_url:
+                continue
+            yield from self._extract_m3u8_formats(
+                m3u8_url, video_id, ext='mp4', m3u8_id=name, headers=self._M3U8_HEADERS)
+
+    def _extract_movie(self, webpage, video_id, name, is_live):
+        window_stores = self._extract_pagestore(webpage, video_id)
+        movie_stores = [
+            # extract all three important data (most of data are duplicated each other, but slightly different!)
+            traverse_obj(window_stores, ('v8', 'state', 'movie'), expected_type=dict),
+            traverse_obj(window_stores, ('v8', 'movie'), expected_type=dict),
+            traverse_obj(window_stores, 'movieStore', expected_type=dict),
+        ]
+        if not any(movie_stores):
+            raise ExtractorError(f'Failed to extract {name} info')
+
+        formats = list(self._expand_media(video_id, get_first(movie_stores, 'media')))
+        if not formats:
+            # archived livestreams or subscriber-only videos
+            cookies = self._get_cookies('https://www.openrec.tv/')
+            detail = self._download_json(
+                f'https://apiv5.openrec.tv/api/v5/movies/{video_id}/detail', video_id,
+                headers={
+                    'Origin': 'https://www.openrec.tv',
+                    'Referer': 'https://www.openrec.tv/',
+                    'access-token': try_get(cookies, lambda x: x.get('access_token').value),
+                    'uuid': try_get(cookies, lambda x: x.get('uuid').value),
+                })
+            new_media = traverse_obj(detail, ('data', 'items', ..., 'media'), get_all=False)
+            formats = list(self._expand_media(video_id, new_media))
+            is_live = False
+
+        return {
+            'id': video_id,
+            'title': get_first(movie_stores, 'title'),
+            'description': get_first(movie_stores, 'introduction'),
+            'thumbnail': get_first(movie_stores, 'thumbnailUrl'),
+            'formats': formats,
+            'uploader': get_first(movie_stores, ('channel', 'user', 'name')),
+            'uploader_id': get_first(movie_stores, ('channel', 'user', 'id')),
+            'timestamp': int_or_none(get_first(movie_stores, ['publishedAt', 'time']), scale=1000) or unified_timestamp(get_first(movie_stores, 'publishedAt')),
+            'is_live': is_live,
+            'http_headers': self._M3U8_HEADERS,
+        }
+
+
+class OpenRecIE(OpenRecBaseIE):
+    IE_NAME = 'openrec'
+    _VALID_URL = r'https?://(?:www\.)?openrec\.tv/live/(?P<id>[^/]+)'
+    _TESTS = [{
+        'url': 'https://www.openrec.tv/live/2p8v31qe4zy',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.openrec.tv/live/wez93eqvjzl',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(f'https://www.openrec.tv/live/{video_id}', video_id)
+
+        return self._extract_movie(webpage, video_id, 'live', True)
+
+
+class OpenRecCaptureIE(OpenRecBaseIE):
+    IE_NAME = 'openrec:capture'
+    _VALID_URL = r'https?://(?:www\.)?openrec\.tv/capture/(?P<id>[^/]+)'
+    _TESTS = [{
+        'url': 'https://www.openrec.tv/capture/l9nk2x4gn14',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.openrec.tv/capture/mldjr82p7qk',
+        'info_dict': {
+            'id': 'mldjr82p7qk',
+            'title': 'たいじの恥ずかしい英語力',
+            'uploader': 'たいちゃんねる',
+            'uploader_id': 'Yaritaiji',
+            'upload_date': '20210803',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(f'https://www.openrec.tv/capture/{video_id}', video_id)
+
+        window_stores = self._extract_pagestore(webpage, video_id)
+        movie_store = window_stores.get('movie')
+
+        capture_data = window_stores.get('capture')
+        if not capture_data:
+            raise ExtractorError('Cannot extract title')
+
+        formats = self._extract_m3u8_formats(
+            capture_data.get('source'), video_id, ext='mp4', headers=self._M3U8_HEADERS)
+
+        return {
+            'id': video_id,
+            'title': capture_data.get('title'),
+            'thumbnail': capture_data.get('thumbnailUrl'),
+            'formats': formats,
+            'timestamp': unified_timestamp(traverse_obj(movie_store, 'createdAt', expected_type=compat_str)),
+            'uploader': traverse_obj(movie_store, ('channel', 'name'), expected_type=compat_str),
+            'uploader_id': traverse_obj(movie_store, ('channel', 'id'), expected_type=compat_str),
+            'upload_date': unified_strdate(capture_data.get('createdAt')),
+            'http_headers': self._M3U8_HEADERS,
+        }
+
+
+class OpenRecMovieIE(OpenRecBaseIE):
+    IE_NAME = 'openrec:movie'
+    _VALID_URL = r'https?://(?:www\.)?openrec\.tv/movie/(?P<id>[^/]+)'
+    _TESTS = [{
+        'url': 'https://www.openrec.tv/movie/nqz5xl5km8v',
+        'info_dict': {
+            'id': 'nqz5xl5km8v',
+            'title': '限定コミュニティ(Discord)参加方法ご説明動画',
+            'description': 'md5:ebd563e5f5b060cda2f02bf26b14d87f',
+            'thumbnail': r're:https://.+',
+            'uploader': 'タイキとカズヒロ',
+            'uploader_id': 'taiki_to_kazuhiro',
+            'timestamp': 1638856800,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(f'https://www.openrec.tv/movie/{video_id}', video_id)
+
+        return self._extract_movie(webpage, video_id, 'movie', False)

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/yt_dlp/extractor/ora.py

@@ -0,0 +1,71 @@
+import re
+from .common import InfoExtractor
+from ..compat import compat_urlparse
+from ..utils import (
+    get_element_by_attribute,
+    qualities,
+    unescapeHTML,
+)
+
+
+class OraTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?(?:ora\.tv|unsafespeech\.com)/([^/]+/)*(?P<id>[^/\?#]+)'
+    _TESTS = [{
+        'url': 'https://www.ora.tv/larrykingnow/2015/12/16/vine-youtube-stars-zach-king-king-bach-on-their-viral-videos-0_36jupg6090pq',
+        'md5': 'fa33717591c631ec93b04b0e330df786',
+        'info_dict': {
+            'id': '50178',
+            'ext': 'mp4',
+            'title': 'Vine & YouTube Stars Zach King & King Bach On Their Viral Videos!',
+            'description': 'md5:ebbc5b1424dd5dba7be7538148287ac1',
+        }
+    }, {
+        'url': 'http://www.unsafespeech.com/video/2016/5/10/student-self-censorship-and-the-thought-police-on-university-campuses-0_6622bnkppw4d',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        video_data = self._search_regex(
+            r'"(?:video|current)"\s*:\s*({[^}]+?})', webpage, 'current video')
+        m3u8_url = self._search_regex(
+            r'hls_stream"?\s*:\s*"([^"]+)', video_data, 'm3u8 url', None)
+        if m3u8_url:
+            formats = self._extract_m3u8_formats(
+                m3u8_url, display_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False)
+            # similar to GameSpotIE
+            m3u8_path = compat_urlparse.urlparse(m3u8_url).path
+            QUALITIES_RE = r'((,[a-z]+\d+)+,?)'
+            available_qualities = self._search_regex(
+                QUALITIES_RE, m3u8_path, 'qualities').strip(',').split(',')
+            http_path = m3u8_path[1:].split('/', 1)[1]
+            http_template = re.sub(QUALITIES_RE, r'%s', http_path)
+            http_template = http_template.replace('.csmil/master.m3u8', '')
+            http_template = compat_urlparse.urljoin(
+                'http://videocdn-pmd.ora.tv/', http_template)
+            preference = qualities(
+                ['mobile400', 'basic400', 'basic600', 'sd900', 'sd1200', 'sd1500', 'hd720', 'hd1080'])
+            for q in available_qualities:
+                formats.append({
+                    'url': http_template % q,
+                    'format_id': q,
+                    'quality': preference(q),
+                })
+        else:
+            return self.url_result(self._search_regex(
+                r'"youtube_id"\s*:\s*"([^"]+)', webpage, 'youtube id'), 'Youtube')
+
+        return {
+            'id': self._search_regex(
+                r'"id"\s*:\s*(\d+)', video_data, 'video id', default=display_id),
+            'display_id': display_id,
+            'title': unescapeHTML(self._og_search_title(webpage)),
+            'description': get_element_by_attribute(
+                'class', 'video_txt_decription', webpage),
+            'thumbnail': self._proto_relative_url(self._search_regex(
+                r'"thumb"\s*:\s*"([^"]+)', video_data, 'thumbnail', None)),
+            'formats': formats,
+        }