@jakende/media-info-cli 0.1.0

package/media_information_download/pipeline.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+from media_information_download.audio import convert_to_mp3
+from media_information_download.config import (
+    get_model_name,
+    get_output_dir,
+    get_whisper_language,
+    require_dependencies,
+)
+from media_information_download.downloaders.http import HTTPDownloader
+from media_information_download.downloaders.youtube import YouTubeDownloader
+from media_information_download.models import MediaItem, ProcessedMedia, ProgressCallback
+from media_information_download.output import write_transcript
+from media_information_download.sources.rss import RSSSource
+from media_information_download.sources.youtube import YouTubeSource
+from media_information_download.transcription import WhisperTranscriber
+
+
+@dataclass(frozen=True)
+class ProcessOptions:
+    source_type: str
+    raw_input: str
+    output_dir: Path | None = None
+    transcribe: bool = True
+    model_name: str | None = None
+    language: str | None = None
+    fps: int = 25
+    delay_seconds: float = 2.0
+
+
+class MediaPipeline:
+    def __init__(self, progress: ProgressCallback | None = None) -> None:
+        self.progress = progress or (lambda message: None)
+
+    def _source(self, source_type: str):
+        if source_type == "youtube":
+            return YouTubeSource()
+        if source_type == "rss":
+            return RSSSource()
+        raise ValueError(f"Unsupported source type: {source_type}")
+
+    def _downloader(self, item: MediaItem):
+        if item.source_type == "youtube":
+            return YouTubeDownloader()
+        if item.source_type == "rss":
+            return HTTPDownloader()
+        raise ValueError(f"Unsupported source type: {item.source_type}")
+
+    def collect_items(self, options: ProcessOptions) -> list[MediaItem]:
+        return self._source(options.source_type).collect(options.raw_input)
+
+    def process(self, options: ProcessOptions) -> list[ProcessedMedia]:
+        require_dependencies(include_transcription=options.transcribe)
+        output_dir = (options.output_dir or get_output_dir()).resolve()
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+        self.progress(f"Collecting media from {options.source_type} input...")
+        items = self.collect_items(options)
+        self.progress(f"Found {len(items)} media item(s).")
+
+        transcriber: WhisperTranscriber | None = None
+        if options.transcribe:
+            model_name = options.model_name or get_model_name()
+            transcriber = WhisperTranscriber(
+                model_name=model_name,
+                language=options.language if options.language is not None else get_whisper_language(),
+            )
+
+        results: list[ProcessedMedia] = []
+        for index, item in enumerate(items, start=1):
+            result = ProcessedMedia(item=item)
+            results.append(result)
+            self.progress(f"[{index}/{len(items)}] Downloading: {item.title}")
+            try:
+                downloaded_path = self._downloader(item).download(item, output_dir)
+                result.downloaded_path = downloaded_path
+                self.progress(f"Downloaded: {downloaded_path.name}")
+
+                self.progress(f"Converting to MP3: {downloaded_path.name}")
+                mp3_path = convert_to_mp3(downloaded_path, output_dir)
+                result.mp3_path = mp3_path
+                self.progress(f"MP3 ready: {mp3_path.name}")
+
+                if item.source_type == "rss" and downloaded_path.resolve() != mp3_path.resolve():
+                    downloaded_path.unlink(missing_ok=True)
+                    result.notes.append(f"Removed intermediate media file: {downloaded_path.name}")
+                    result.downloaded_path = None
+
+                if transcriber is not None:
+                    self.progress(f"Transcribing: {mp3_path.name}")
+                    transcript, language, device = transcriber.transcribe(mp3_path)
+                    result.transcript_path = write_transcript(
+                        output_dir=output_dir,
+                        item=item,
+                        audio_path=mp3_path,
+                        model_name=transcriber.model_name,
+                        device=device,
+                        language=language,
+                        fps=options.fps,
+                        segments=transcript.get("segments", []),
+                        full_text=(transcript.get("text") or "").strip(),
+                    )
+                    self.progress(f"Transcript written: {result.transcript_path.name}")
+            except Exception as exc:
+                result.error = str(exc)
+                self.progress(f"ERROR: {exc}")
+                if "HTTP Error 403" in str(exc):
+                    result.notes.append(
+                        "YouTube rejected the media request. Try setting "
+                        'YTDL_COOKIES_FROM_BROWSER="safari" or "chrome".'
+                    )
+
+            if index < len(items) and options.delay_seconds > 0:
+                time.sleep(max(0.0, options.delay_seconds))
+
+        return results
+
+    def transcribe_existing(
+        self,
+        audio_paths: list[Path],
+        output_dir: Path | None = None,
+        model_name: str | None = None,
+        language: str | None = None,
+        fps: int = 25,
+    ) -> list[ProcessedMedia]:
+        require_dependencies(include_transcription=True)
+        target_output_dir = (output_dir or get_output_dir()).resolve()
+        transcriber = WhisperTranscriber(
+            model_name=model_name or get_model_name(),
+            language=language if language is not None else get_whisper_language(),
+        )
+        results: list[ProcessedMedia] = []
+        for index, audio_path in enumerate(audio_paths, start=1):
+            item = MediaItem(
+                source_type="local",
+                source_url=str(audio_path),
+                media_url=str(audio_path),
+                title=audio_path.stem,
+            )
+            result = ProcessedMedia(item=item, mp3_path=audio_path)
+            results.append(result)
+            try:
+                self.progress(f"[{index}/{len(audio_paths)}] Transcribing: {audio_path.name}")
+                transcript, detected_language, device = transcriber.transcribe(audio_path)
+                result.transcript_path = write_transcript(
+                    output_dir=target_output_dir,
+                    item=item,
+                    audio_path=audio_path,
+                    model_name=transcriber.model_name,
+                    device=device,
+                    language=detected_language,
+                    fps=fps,
+                    segments=transcript.get("segments", []),
+                    full_text=(transcript.get("text") or "").strip(),
+                )
+                self.progress(f"Transcript written: {result.transcript_path.name}")
+            except Exception as exc:
+                result.error = str(exc)
+                self.progress(f"ERROR: {exc}")
+        return results

package/media_information_download/sources/rss.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+import mimetypes
+import urllib.parse
+import urllib.request
+import xml.etree.ElementTree as ET
+
+from media_information_download.config import (
+    SUPPORTED_MEDIA_EXTENSIONS,
+    SUPPORTED_MEDIA_MIME_PREFIXES,
+)
+from media_information_download.models import MediaItem
+
+
+def _tag_name(element: ET.Element) -> str:
+    return element.tag.rsplit("}", maxsplit=1)[-1].lower()
+
+
+def _child_text(element: ET.Element, names: set[str]) -> str:
+    for child in element:
+        if _tag_name(child) in names and child.text:
+            return child.text.strip()
+    return ""
+
+
+def _is_supported_media(url: str, mime_type: str = "") -> bool:
+    clean_path = urllib.parse.urlparse(url).path
+    extension = mimetypes.guess_extension(mime_type.split(";", maxsplit=1)[0].strip())
+    suffix = (extension or "").lower() or clean_path.lower().rsplit(".", maxsplit=1)[-1]
+    if suffix and not suffix.startswith("."):
+        suffix = f".{suffix}"
+
+    if suffix in SUPPORTED_MEDIA_EXTENSIONS:
+        return True
+
+    mime = mime_type.lower().strip()
+    return any(mime.startswith(prefix) for prefix in SUPPORTED_MEDIA_MIME_PREFIXES)
+
+
+def _first_attr(element: ET.Element, names: tuple[str, ...]) -> str:
+    for name in names:
+        value = element.attrib.get(name)
+        if value:
+            return value.strip()
+    return ""
+
+
+def parse_feed_xml(feed_xml: bytes | str, feed_url: str) -> list[MediaItem]:
+    root = ET.fromstring(feed_xml)
+    candidates = [
+        element
+        for element in root.iter()
+        if _tag_name(element) in {"item", "entry"}
+    ]
+
+    items: list[MediaItem] = []
+    for entry in candidates:
+        title = _child_text(entry, {"title"}) or "RSS media item"
+        published = _child_text(entry, {"pubdate", "published", "updated"})
+        description = _child_text(entry, {"description", "summary", "subtitle"})
+
+        media_candidates: list[tuple[str, str]] = []
+        for child in entry.iter():
+            name = _tag_name(child)
+            if name == "enclosure":
+                media_candidates.append(
+                    (
+                        _first_attr(child, ("url", "href")),
+                        _first_attr(child, ("type", "medium")),
+                    )
+                )
+            elif name in {"content", "player"} and (
+                "media" in child.tag.lower() or child.attrib.get("url")
+            ):
+                media_candidates.append(
+                    (
+                        _first_attr(child, ("url", "href")),
+                        _first_attr(child, ("type", "medium")),
+                    )
+                )
+            elif name == "link":
+                href = _first_attr(child, ("href", "url"))
+                link_type = _first_attr(child, ("type",))
+                if href and (link_type or _is_supported_media(href)):
+                    media_candidates.append((href, link_type))
+                elif child.text and _is_supported_media(child.text.strip()):
+                    media_candidates.append((child.text.strip(), ""))
+
+        for media_url, mime_type in media_candidates:
+            if not media_url:
+                continue
+            media_url = urllib.parse.urljoin(feed_url, media_url)
+            if not _is_supported_media(media_url, mime_type):
+                continue
+            items.append(
+                MediaItem(
+                    source_type="rss",
+                    source_url=feed_url,
+                    media_url=media_url,
+                    title=title,
+                    mime_type=mime_type,
+                    published=published,
+                    description=description,
+                )
+            )
+            break
+
+    return items
+
+
+class RSSSource:
+    name = "rss"
+
+    def collect(self, feed_url: str) -> list[MediaItem]:
+        parsed = urllib.parse.urlparse(feed_url)
+        if parsed.scheme not in {"http", "https"}:
+            raise ValueError("RSS feed URL must start with http:// or https://")
+
+        request = urllib.request.Request(
+            feed_url,
+            headers={
+                "User-Agent": "media-information-download/1.0",
+                "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml",
+            },
+        )
+        with urllib.request.urlopen(request, timeout=30) as response:
+            feed_xml = response.read()
+
+        items = parse_feed_xml(feed_xml, feed_url)
+        if not items:
+            raise ValueError("No supported audio or video enclosures were found in the RSS feed.")
+        return items

package/media_information_download/sources/youtube.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import re
+
+from media_information_download.models import MediaItem
+
+
+YOUTUBE_URL_RE = re.compile(
+    r"^(https?://)?(www\.)?(youtube\.com/watch\?v=|youtu\.be/)[\w\-]{6,}.*$",
+    re.IGNORECASE,
+)
+
+
+def validate_url(url: str) -> bool:
+    return bool(YOUTUBE_URL_RE.match(url.strip()))
+
+
+def parse_urls(raw_value: str) -> list[str]:
+    return [part.strip() for part in re.split(r"[\s,]+", raw_value) if part.strip()]
+
+
+class YouTubeSource:
+    name = "youtube"
+
+    def collect(self, raw_value: str) -> list[MediaItem]:
+        urls = parse_urls(raw_value)
+        invalid_urls = [url for url in urls if not validate_url(url)]
+        if invalid_urls:
+            raise ValueError(
+                "Invalid YouTube URL(s): " + ", ".join(invalid_urls)
+            )
+
+        return [
+            MediaItem(
+                source_type=self.name,
+                source_url=url,
+                media_url=url,
+                title="YouTube media",
+            )
+            for url in urls
+        ]

package/media_information_download/transcription.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import contextlib
+import os
+from pathlib import Path
+from typing import Any
+
+os.environ.setdefault("PYTORCH_ENABLE_MPS_FALLBACK", "1")
+
+
+def seconds_to_timecode(seconds: float, fps: int = 25) -> str:
+    total_frames = int(round(max(0.0, seconds) * fps))
+    frames_per_hour = 3600 * fps
+    frames_per_minute = 60 * fps
+
+    hours = total_frames // frames_per_hour
+    total_frames %= frames_per_hour
+
+    minutes = total_frames // frames_per_minute
+    total_frames %= frames_per_minute
+
+    secs = total_frames // fps
+    frames = total_frames % fps
+
+    return f"{hours:02d}:{minutes:02d}:{secs:02d}:{frames:02d}"
+
+
+def densify_sparse_buffers(module: Any) -> None:
+    import torch
+
+    for key, buf in list(module._buffers.items()):
+        if torch.is_tensor(buf) and getattr(buf, "is_sparse", False):
+            module._buffers[key] = buf.to_dense()
+    for child in module.children():
+        densify_sparse_buffers(child)
+
+
+def pick_preferred_device() -> str:
+    import torch
+
+    if torch.backends.mps.is_available():
+        return "mps"
+    if torch.cuda.is_available():
+        return "cuda"
+    return "cpu"
+
+
+def load_whisper_model_robust(model_name: str, preferred_device: str) -> tuple[Any, str]:
+    import whisper
+
+    model = whisper.load_model(model_name, device="cpu")
+    densify_sparse_buffers(model)
+
+    if preferred_device == "mps":
+        try:
+            model = model.to("mps")
+            return model, "mps"
+        except NotImplementedError:
+            return model, "cpu"
+
+    if preferred_device == "cuda":
+        try:
+            model = model.to("cuda")
+            return model, "cuda"
+        except Exception:
+            return model, "cpu"
+
+    return model, "cpu"
+
+
+class WhisperTranscriber:
+    def __init__(self, model_name: str, language: str | None = None) -> None:
+        self.model_name = model_name
+        self.language = language
+        self._model: Any | None = None
+        self.device = "unknown"
+
+    def _load(self) -> None:
+        if self._model is not None:
+            return
+
+        import torch
+
+        preferred = pick_preferred_device()
+        self._model, self.device = load_whisper_model_robust(self.model_name, preferred)
+        if self.device == "cpu":
+            try:
+                torch.set_num_threads(max(1, os.cpu_count() or 1))
+            except Exception:
+                pass
+
+    def transcribe(self, audio_path: Path) -> tuple[dict[str, Any], str, str]:
+        self._load()
+        options: dict[str, Any] = {
+            "verbose": False,
+            "fp16": False,
+            "beam_size": 1,
+            "best_of": 1,
+            "temperature": 0,
+            "condition_on_previous_text": False,
+        }
+        if self.language:
+            options["language"] = self.language
+
+        with open(os.devnull, "w", encoding="utf-8") as devnull:
+            with contextlib.redirect_stdout(devnull), contextlib.redirect_stderr(devnull):
+                result = self._model.transcribe(str(audio_path), **options)
+        detected_language = (result.get("language") or self.language or "unknown").strip()
+        return result, detected_language, self.device