@tushar-br/desktop 1.0.282 → 1.0.283

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/pytube/request.py

@@ -0,0 +1,269 @@
+"""Implements a simple wrapper around urlopen."""
+import http.client
+import json
+import logging
+import re
+import socket
+from functools import lru_cache
+from urllib import parse
+from urllib.error import URLError
+from urllib.request import Request, urlopen
+
+from pytube.exceptions import RegexMatchError, MaxRetriesExceeded
+from pytube.helpers import regex_search
+
+logger = logging.getLogger(__name__)
+default_range_size = 9437184  # 9MB
+
+
+def _execute_request(
+    url,
+    method=None,
+    headers=None,
+    data=None,
+    timeout=socket._GLOBAL_DEFAULT_TIMEOUT
+):
+    base_headers = {"User-Agent": "Mozilla/5.0", "accept-language": "en-US,en"}
+    if headers:
+        base_headers.update(headers)
+    if data:
+        # encode data for request
+        if not isinstance(data, bytes):
+            data = bytes(json.dumps(data), encoding="utf-8")
+    if url.lower().startswith("http"):
+        request = Request(url, headers=base_headers, method=method, data=data)
+    else:
+        raise ValueError("Invalid URL")
+    return urlopen(request, timeout=timeout)  # nosec
+
+
+def get(url, extra_headers=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+    """Send an http GET request.
+
+    :param str url:
+        The URL to perform the GET request for.
+    :param dict extra_headers:
+        Extra headers to add to the request
+    :rtype: str
+    :returns:
+        UTF-8 encoded string of response
+    """
+    if extra_headers is None:
+        extra_headers = {}
+    response = _execute_request(url, headers=extra_headers, timeout=timeout)
+    return response.read().decode("utf-8")
+
+
+def post(url, extra_headers=None, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
+    """Send an http POST request.
+
+    :param str url:
+        The URL to perform the POST request for.
+    :param dict extra_headers:
+        Extra headers to add to the request
+    :param dict data:
+        The data to send on the POST request
+    :rtype: str
+    :returns:
+        UTF-8 encoded string of response
+    """
+    # could technically be implemented in get,
+    # but to avoid confusion implemented like this
+    if extra_headers is None:
+        extra_headers = {}
+    if data is None:
+        data = {}
+    # required because the youtube servers are strict on content type
+    # raises HTTPError [400]: Bad Request otherwise
+    extra_headers.update({"Content-Type": "application/json"})
+    response = _execute_request(
+        url,
+        headers=extra_headers,
+        data=data,
+        timeout=timeout
+    )
+    return response.read().decode("utf-8")
+
+
+def seq_stream(
+    url,
+    timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+    max_retries=0
+):
+    """Read the response in sequence.
+    :param str url: The URL to perform the GET request for.
+    :rtype: Iterable[bytes]
+    """
+    # YouTube expects a request sequence number as part of the parameters.
+    split_url = parse.urlsplit(url)
+    base_url = '%s://%s/%s?' % (split_url.scheme, split_url.netloc, split_url.path)
+
+    querys = dict(parse.parse_qsl(split_url.query))
+
+    # The 0th sequential request provides the file headers, which tell us
+    #  information about how the file is segmented.
+    querys['sq'] = 0
+    url = base_url + parse.urlencode(querys)
+
+    segment_data = b''
+    for chunk in stream(url, timeout=timeout, max_retries=max_retries):
+        yield chunk
+        segment_data += chunk
+
+    # We can then parse the header to find the number of segments
+    stream_info = segment_data.split(b'\r\n')
+    segment_count_pattern = re.compile(b'Segment-Count: (\\d+)')
+    for line in stream_info:
+        match = segment_count_pattern.search(line)
+        if match:
+            segment_count = int(match.group(1).decode('utf-8'))
+
+    # We request these segments sequentially to build the file.
+    seq_num = 1
+    while seq_num <= segment_count:
+        # Create sequential request URL
+        querys['sq'] = seq_num
+        url = base_url + parse.urlencode(querys)
+
+        yield from stream(url, timeout=timeout, max_retries=max_retries)
+        seq_num += 1
+    return  # pylint: disable=R1711
+
+
+def stream(
+    url,
+    timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
+    max_retries=0
+):
+    """Read the response in chunks.
+    :param str url: The URL to perform the GET request for.
+    :rtype: Iterable[bytes]
+    """
+    file_size: int = default_range_size  # fake filesize to start
+    downloaded = 0
+    while downloaded < file_size:
+        stop_pos = min(downloaded + default_range_size, file_size) - 1
+        range_header = f"bytes={downloaded}-{stop_pos}"
+        tries = 0
+
+        # Attempt to make the request multiple times as necessary.
+        while True:
+            # If the max retries is exceeded, raise an exception
+            if tries >= 1 + max_retries:
+                raise MaxRetriesExceeded()
+
+            # Try to execute the request, ignoring socket timeouts
+            try:
+                response = _execute_request(
+                    url + f"&range={downloaded}-{stop_pos}",
+                    method="GET",
+                    timeout=timeout
+                )
+            except URLError as e:
+                # We only want to skip over timeout errors, and
+                # raise any other URLError exceptions
+                if isinstance(e.reason, socket.timeout):
+                    pass
+                else:
+                    raise
+            except http.client.IncompleteRead:
+                # Allow retries on IncompleteRead errors for unreliable connections
+                pass
+            else:
+                # On a successful request, break from loop
+                break
+            tries += 1
+
+        if file_size == default_range_size:
+            try:
+                resp = _execute_request(
+                    url + f"&range={0}-{99999999999}",
+                    method="GET",
+                    timeout=timeout
+                )
+                content_range = resp.info()["Content-Length"]
+                file_size = int(content_range)
+            except (KeyError, IndexError, ValueError) as e:
+                logger.error(e)
+        while True:
+            chunk = response.read()
+            if not chunk:
+                break
+            downloaded += len(chunk)
+            yield chunk
+    return  # pylint: disable=R1711
+
+
+@lru_cache()
+def filesize(url):
+    """Fetch size in bytes of file at given URL
+
+    :param str url: The URL to get the size of
+    :returns: int: size in bytes of remote file
+    """
+    return int(head(url)["content-length"])
+
+
+@lru_cache()
+def seq_filesize(url):
+    """Fetch size in bytes of file at given URL from sequential requests
+
+    :param str url: The URL to get the size of
+    :returns: int: size in bytes of remote file
+    """
+    total_filesize = 0
+    # YouTube expects a request sequence number as part of the parameters.
+    split_url = parse.urlsplit(url)
+    base_url = '%s://%s/%s?' % (split_url.scheme, split_url.netloc, split_url.path)
+    querys = dict(parse.parse_qsl(split_url.query))
+
+    # The 0th sequential request provides the file headers, which tell us
+    #  information about how the file is segmented.
+    querys['sq'] = 0
+    url = base_url + parse.urlencode(querys)
+    response = _execute_request(
+        url, method="GET"
+    )
+
+    response_value = response.read()
+    # The file header must be added to the total filesize
+    total_filesize += len(response_value)
+
+    # We can then parse the header to find the number of segments
+    segment_count = 0
+    stream_info = response_value.split(b'\r\n')
+    segment_regex = b'Segment-Count: (\\d+)'
+    for line in stream_info:
+        # One of the lines should contain the segment count, but we don't know
+        #  which, so we need to iterate through the lines to find it
+        try:
+            segment_count = int(regex_search(segment_regex, line, 1))
+        except RegexMatchError:
+            pass
+
+    if segment_count == 0:
+        raise RegexMatchError('seq_filesize', segment_regex)
+
+    # We make HEAD requests to the segments sequentially to find the total filesize.
+    seq_num = 1
+    while seq_num <= segment_count:
+        # Create sequential request URL
+        querys['sq'] = seq_num
+        url = base_url + parse.urlencode(querys)
+
+        total_filesize += int(head(url)['content-length'])
+        seq_num += 1
+    return total_filesize
+
+
+def head(url):
+    """Fetch headers returned http GET request.
+
+    :param str url:
+        The URL to perform the GET request for.
+    :rtype: dict
+    :returns:
+        dictionary of lowercase headers
+    """
+    response_headers = _execute_request(url, method="HEAD").info()
+    return {k.lower(): v for k, v in response_headers.items()}

package/staging_area/micro projects/SD -Advanced Media Processing Toolkit Automated Downloader, Segmenter & Neural Dubber/.venv/Lib/site-packages/pytube/streams.py

@@ -0,0 +1,436 @@
+"""
+This module contains a container for stream manifest data.
+
+A container object for the media stream (video only / audio only / video+audio
+combined). This was referred to as ``Video`` in the legacy pytube version, but
+has been renamed to accommodate DASH (which serves the audio and video
+separately).
+"""
+import logging
+import os
+from math import ceil
+
+from datetime import datetime
+from typing import BinaryIO, Dict, Optional, Tuple
+from urllib.error import HTTPError
+from urllib.parse import parse_qs
+
+from pytube import extract, request
+from pytube.helpers import safe_filename, target_directory
+from pytube.itags import get_format_profile
+from pytube.monostate import Monostate
+
+logger = logging.getLogger(__name__)
+
+
+class Stream:
+    """Container for stream manifest data."""
+
+    def __init__(
+        self, stream: Dict, monostate: Monostate
+    ):
+        """Construct a :class:`Stream <Stream>`.
+
+        :param dict stream:
+            The unscrambled data extracted from YouTube.
+        :param dict monostate:
+            Dictionary of data shared across all instances of
+            :class:`Stream <Stream>`.
+        """
+        # A dictionary shared between all instances of :class:`Stream <Stream>`
+        # (Borg pattern).
+        self._monostate = monostate
+
+        self.url = stream["url"]  # signed download url
+        self.itag = int(
+            stream["itag"]
+        )  # stream format id (youtube nomenclature)
+
+        # set type and codec info
+
+        # 'video/webm; codecs="vp8, vorbis"' -> 'video/webm', ['vp8', 'vorbis']
+        self.mime_type, self.codecs = extract.mime_type_codec(stream["mimeType"])
+
+        # 'video/webm' -> 'video', 'webm'
+        self.type, self.subtype = self.mime_type.split("/")
+
+        # ['vp8', 'vorbis'] -> video_codec: vp8, audio_codec: vorbis. DASH
+        # streams return NoneType for audio/video depending.
+        self.video_codec, self.audio_codec = self.parse_codecs()
+
+        self.is_otf: bool = stream["is_otf"]
+        self.bitrate: Optional[int] = stream["bitrate"]
+
+        # filesize in bytes
+        self._filesize: Optional[int] = int(stream.get('contentLength', 0))
+        
+        # filesize in kilobytes
+        self._filesize_kb: Optional[float] = float(ceil(float(stream.get('contentLength', 0)) / 1024 * 1000) / 1000)
+        
+        # filesize in megabytes
+        self._filesize_mb: Optional[float] = float(ceil(float(stream.get('contentLength', 0)) / 1024 / 1024 * 1000) / 1000)
+        
+        # filesize in gigabytes(fingers crossed we don't need terabytes going forward though)
+        self._filesize_gb: Optional[float] = float(ceil(float(stream.get('contentLength', 0)) / 1024 / 1024 / 1024 * 1000) / 1000)
+
+        # Additional information about the stream format, such as resolution,
+        # frame rate, and whether the stream is live (HLS) or 3D.
+        itag_profile = get_format_profile(self.itag)
+        self.is_dash = itag_profile["is_dash"]
+        self.abr = itag_profile["abr"]  # average bitrate (audio streams only)
+        if 'fps' in stream:
+            self.fps = stream['fps']  # Video streams only
+        self.resolution = itag_profile[
+            "resolution"
+        ]  # resolution (e.g.: "480p")
+        self.is_3d = itag_profile["is_3d"]
+        self.is_hdr = itag_profile["is_hdr"]
+        self.is_live = itag_profile["is_live"]
+
+    @property
+    def is_adaptive(self) -> bool:
+        """Whether the stream is DASH.
+
+        :rtype: bool
+        """
+        # if codecs has two elements (e.g.: ['vp8', 'vorbis']): 2 % 2 = 0
+        # if codecs has one element (e.g.: ['vp8']) 1 % 2 = 1
+        return bool(len(self.codecs) % 2)
+
+    @property
+    def is_progressive(self) -> bool:
+        """Whether the stream is progressive.
+
+        :rtype: bool
+        """
+        return not self.is_adaptive
+
+    @property
+    def includes_audio_track(self) -> bool:
+        """Whether the stream only contains audio.
+
+        :rtype: bool
+        """
+        return self.is_progressive or self.type == "audio"
+
+    @property
+    def includes_video_track(self) -> bool:
+        """Whether the stream only contains video.
+
+        :rtype: bool
+        """
+        return self.is_progressive or self.type == "video"
+
+    def parse_codecs(self) -> Tuple[Optional[str], Optional[str]]:
+        """Get the video/audio codecs from list of codecs.
+
+        Parse a variable length sized list of codecs and returns a
+        constant two element tuple, with the video codec as the first element
+        and audio as the second. Returns None if one is not available
+        (adaptive only).
+
+        :rtype: tuple
+        :returns:
+            A two element tuple with audio and video codecs.
+
+        """
+        video = None
+        audio = None
+        if not self.is_adaptive:
+            video, audio = self.codecs
+        elif self.includes_video_track:
+            video = self.codecs[0]
+        elif self.includes_audio_track:
+            audio = self.codecs[0]
+        return video, audio
+
+    @property
+    def filesize(self) -> int:
+        """File size of the media stream in bytes.
+
+        :rtype: int
+        :returns:
+            Filesize (in bytes) of the stream.
+        """
+        if self._filesize == 0:
+            try:
+                self._filesize = request.filesize(self.url)
+            except HTTPError as e:
+                if e.code != 404:
+                    raise
+                self._filesize = request.seq_filesize(self.url)
+        return self._filesize
+    
+    @property
+    def filesize_kb(self) -> float:
+        """File size of the media stream in kilobytes.
+
+        :rtype: float
+        :returns:
+            Rounded filesize (in kilobytes) of the stream.
+        """
+        if self._filesize_kb == 0:
+            try:
+                self._filesize_kb = float(ceil(request.filesize(self.url)/1024 * 1000) / 1000)
+            except HTTPError as e:
+                if e.code != 404:
+                    raise
+                self._filesize_kb = float(ceil(request.seq_filesize(self.url)/1024 * 1000) / 1000)
+        return self._filesize_kb
+    
+    @property
+    def filesize_mb(self) -> float:
+        """File size of the media stream in megabytes.
+
+        :rtype: float
+        :returns:
+            Rounded filesize (in megabytes) of the stream.
+        """
+        if self._filesize_mb == 0:
+            try:
+                self._filesize_mb = float(ceil(request.filesize(self.url)/1024/1024 * 1000) / 1000)
+            except HTTPError as e:
+                if e.code != 404:
+                    raise
+                self._filesize_mb = float(ceil(request.seq_filesize(self.url)/1024/1024 * 1000) / 1000)
+        return self._filesize_mb
+
+    @property
+    def filesize_gb(self) -> float:
+        """File size of the media stream in gigabytes.
+
+        :rtype: float
+        :returns:
+            Rounded filesize (in gigabytes) of the stream.
+        """
+        if self._filesize_gb == 0:
+            try:
+                self._filesize_gb = float(ceil(request.filesize(self.url)/1024/1024/1024 * 1000) / 1000)
+            except HTTPError as e:
+                if e.code != 404:
+                    raise
+                self._filesize_gb = float(ceil(request.seq_filesize(self.url)/1024/1024/1024 * 1000) / 1000)
+        return self._filesize_gb
+    
+    @property
+    def title(self) -> str:
+        """Get title of video
+
+        :rtype: str
+        :returns:
+            Youtube video title
+        """
+        return self._monostate.title or "Unknown YouTube Video Title"
+
+    @property
+    def filesize_approx(self) -> int:
+        """Get approximate filesize of the video
+
+        Falls back to HTTP call if there is not sufficient information to approximate
+
+        :rtype: int
+        :returns: size of video in bytes
+        """
+        if self._monostate.duration and self.bitrate:
+            bits_in_byte = 8
+            return int(
+                (self._monostate.duration * self.bitrate) / bits_in_byte
+            )
+
+        return self.filesize
+
+    @property
+    def expiration(self) -> datetime:
+        expire = parse_qs(self.url.split("?")[1])["expire"][0]
+        return datetime.utcfromtimestamp(int(expire))
+
+    @property
+    def default_filename(self) -> str:
+        """Generate filename based on the video title.
+
+        :rtype: str
+        :returns:
+            An os file system compatible filename.
+        """
+        filename = safe_filename(self.title)
+        return f"{filename}.{self.subtype}"
+
+    def download(
+        self,
+        output_path: Optional[str] = None,
+        filename: Optional[str] = None,
+        filename_prefix: Optional[str] = None,
+        skip_existing: bool = True,
+        timeout: Optional[int] = None,
+        max_retries: Optional[int] = 0
+    ) -> str:
+        """Write the media stream to disk.
+
+        :param output_path:
+            (optional) Output path for writing media file. If one is not
+            specified, defaults to the current working directory.
+        :type output_path: str or None
+        :param filename:
+            (optional) Output filename (stem only) for writing media file.
+            If one is not specified, the default filename is used.
+        :type filename: str or None
+        :param filename_prefix:
+            (optional) A string that will be prepended to the filename.
+            For example a number in a playlist or the name of a series.
+            If one is not specified, nothing will be prepended
+            This is separate from filename so you can use the default
+            filename but still add a prefix.
+        :type filename_prefix: str or None
+        :param skip_existing:
+            (optional) Skip existing files, defaults to True
+        :type skip_existing: bool
+        :param timeout:
+            (optional) Request timeout length in seconds. Uses system default.
+        :type timeout: int
+        :param max_retries:
+            (optional) Number of retries to attempt after socket timeout. Defaults to 0.
+        :type max_retries: int
+        :returns:
+            Path to the saved video
+        :rtype: str
+
+        """
+        file_path = self.get_file_path(
+            filename=filename,
+            output_path=output_path,
+            filename_prefix=filename_prefix,
+        )
+
+        if skip_existing and self.exists_at_path(file_path):
+            logger.debug(f'file {file_path} already exists, skipping')
+            self.on_complete(file_path)
+            return file_path
+
+        bytes_remaining = self.filesize
+        logger.debug(f'downloading ({self.filesize} total bytes) file to {file_path}')
+
+        with open(file_path, "wb") as fh:
+            try:
+                for chunk in request.stream(
+                    self.url,
+                    timeout=timeout,
+                    max_retries=max_retries
+                ):
+                    # reduce the (bytes) remainder by the length of the chunk.
+                    bytes_remaining -= len(chunk)
+                    # send to the on_progress callback.
+                    self.on_progress(chunk, fh, bytes_remaining)
+            except HTTPError as e:
+                if e.code != 404:
+                    raise
+                # Some adaptive streams need to be requested with sequence numbers
+                for chunk in request.seq_stream(
+                    self.url,
+                    timeout=timeout,
+                    max_retries=max_retries
+                ):
+                    # reduce the (bytes) remainder by the length of the chunk.
+                    bytes_remaining -= len(chunk)
+                    # send to the on_progress callback.
+                    self.on_progress(chunk, fh, bytes_remaining)
+        self.on_complete(file_path)
+        return file_path
+
+    def get_file_path(
+        self,
+        filename: Optional[str] = None,
+        output_path: Optional[str] = None,
+        filename_prefix: Optional[str] = None,
+    ) -> str:
+        if not filename:
+            filename = self.default_filename
+        if filename_prefix:
+            filename = f"{filename_prefix}{filename}"
+        return os.path.join(target_directory(output_path), filename)
+
+    def exists_at_path(self, file_path: str) -> bool:
+        return (
+            os.path.isfile(file_path)
+            and os.path.getsize(file_path) == self.filesize
+        )
+
+    def stream_to_buffer(self, buffer: BinaryIO) -> None:
+        """Write the media stream to buffer
+
+        :rtype: io.BytesIO buffer
+        """
+        bytes_remaining = self.filesize
+        logger.info(
+            "downloading (%s total bytes) file to buffer", self.filesize,
+        )
+
+        for chunk in request.stream(self.url):
+            # reduce the (bytes) remainder by the length of the chunk.
+            bytes_remaining -= len(chunk)
+            # send to the on_progress callback.
+            self.on_progress(chunk, buffer, bytes_remaining)
+        self.on_complete(None)
+
+    def on_progress(
+        self, chunk: bytes, file_handler: BinaryIO, bytes_remaining: int
+    ):
+        """On progress callback function.
+
+        This function writes the binary data to the file, then checks if an
+        additional callback is defined in the monostate. This is exposed to
+        allow things like displaying a progress bar.
+
+        :param bytes chunk:
+            Segment of media file binary data, not yet written to disk.
+        :param file_handler:
+            The file handle where the media is being written to.
+        :type file_handler:
+            :py:class:`io.BufferedWriter`
+        :param int bytes_remaining:
+            The delta between the total file size in bytes and amount already
+            downloaded.
+
+        :rtype: None
+
+        """
+        file_handler.write(chunk)
+        logger.debug("download remaining: %s", bytes_remaining)
+        if self._monostate.on_progress:
+            self._monostate.on_progress(self, chunk, bytes_remaining)
+
+    def on_complete(self, file_path: Optional[str]):
+        """On download complete handler function.
+
+        :param file_path:
+            The file handle where the media is being written to.
+        :type file_path: str
+
+        :rtype: None
+
+        """
+        logger.debug("download finished")
+        on_complete = self._monostate.on_complete
+        if on_complete:
+            logger.debug("calling on_complete callback %s", on_complete)
+            on_complete(self, file_path)
+
+    def __repr__(self) -> str:
+        """Printable object representation.
+
+        :rtype: str
+        :returns:
+            A string representation of a :class:`Stream <Stream>` object.
+        """
+        parts = ['itag="{s.itag}"', 'mime_type="{s.mime_type}"']
+        if self.includes_video_track:
+            parts.extend(['res="{s.resolution}"', 'fps="{s.fps}fps"'])
+            if not self.is_adaptive:
+                parts.extend(
+                    ['vcodec="{s.video_codec}"', 'acodec="{s.audio_codec}"',]
+                )
+            else:
+                parts.extend(['vcodec="{s.video_codec}"'])
+        else:
+            parts.extend(['abr="{s.abr}"', 'acodec="{s.audio_codec}"'])
+        parts.extend(['progressive="{s.is_progressive}"', 'type="{s.type}"'])
+        return f"<Stream: {' '.join(parts).format(s=self)}>"