@jakende/media-info-cli 0.1.0

package/README.md

@@ -0,0 +1,178 @@
+# Media Information Download
+
+Terminal application for downloading media from YouTube URLs or RSS feeds, converting audio to MP3, and generating Whisper Markdown transcripts.
+
+## Requirements
+
+- Python 3.10+
+- ffmpeg on PATH
+- A terminal with ANSI escape support for the framed TUI. macOS Terminal, iTerm2, Windows Terminal, and current PowerShell terminals are supported.
+
+Python dependencies are installed into the project-local `.venv` folder. Run commands from this folder so the local environment is used.
+
+## macOS Setup
+
+Option 1, installer script:
+
+```bash
+zsh scripts/install_macos.sh
+./run.sh
+```
+
+Option 2, manual setup:
+
+```bash
+brew install ffmpeg
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -r requirements-transcribe.txt
+```
+
+Optional editable install:
+
+```bash
+source .venv/bin/activate
+pip install -e ".[transcribe]"
+```
+
+## Windows Setup
+
+Use Windows Terminal or PowerShell. Install Python 3.10+ and ffmpeg first, then use the local project environment.
+
+Option 1, installer script:
+
+```powershell
+winget install Python.Python.3.12
+winget install Gyan.FFmpeg
+powershell -ExecutionPolicy Bypass -File .\scripts\install_windows.ps1
+.\run.ps1
+```
+
+Option 2, manual setup:
+
+```powershell
+py -3 -m venv .venv
+.\.venv\Scripts\python.exe -m pip install -r requirements-transcribe.txt
+.\.venv\Scripts\python.exe -m pip install -e ".[transcribe]"
+.\.venv\Scripts\python.exe media_tui.py
+```
+
+After editable install, these console commands are available inside the active environment on both platforms:
+
+```bash
+media-information-download
+media-info-download
+```
+
+## npm Install
+
+The package is also published as an npm CLI wrapper. It still requires Python 3.10+ and ffmpeg on `PATH`.
+
+```bash
+npm install -g @jakende/media-info-cli
+media-information-download
+```
+
+On first run, the npm wrapper creates a Python virtual environment in `~/.media-information-download/venv` and installs the Python dependencies there. To use a different venv location:
+
+```bash
+MEDIA_INFORMATION_DOWNLOAD_VENV=/path/to/venv media-information-download
+```
+
+## TUI
+
+```bash
+./run.sh
+```
+
+or:
+
+```bash
+source .venv/bin/activate
+python3 media_tui.py
+```
+
+On Windows:
+
+```powershell
+.\run.ps1
+```
+
+If the project is installed into the active environment, you can also run:
+
+```bash
+media-information-download
+```
+
+The TUI lets you choose YouTube or RSS input, start downloads, watch progress messages, trigger transcription, and list or open generated files.
+Long-running download, MP3 conversion, and transcription steps show a visible `WORKING` activity bar while active.
+Interactive screens render inside a left-aligned framed viewport that resizes with the current terminal window.
+
+Keyboard controls:
+
+- Navigation controls are shown at the bottom of the active menu or submenu
+- Up/Down: move through selectable menu items
+- Enter: select
+- Backspace: go back from a submenu or choice screen
+- Escape: cancel/back from submenus; quit from the main menu
+- URL entry screens show their own controls: type or paste text, Enter continues, Backspace/Escape goes back
+- Paste is supported in URL entry screens, including terminal bracketed paste, macOS clipboard paste, and Windows clipboard paste with Ctrl+V in PowerShell/Windows Terminal
+
+For multiple YouTube URLs, separate entries with commas, spaces, or line breaks:
+
+```text
+https://youtu.be/VIDEO_ONE
+https://www.youtube.com/watch?v=VIDEO_TWO, https://youtu.be/VIDEO_THREE
+```
+
+## Non-Interactive Usage
+
+Download and transcribe a YouTube URL:
+
+```bash
+python3 media_tui.py --source youtube --url "https://www.youtube.com/watch?v=VIDEO_ID"
+```
+
+Download and convert a YouTube URL to MP3 without transcription:
+
+```bash
+python3 media_tui.py --source youtube --url "https://www.youtube.com/watch?v=VIDEO_ID" --no-transcribe
+```
+
+Download supported media from an RSS feed and transcribe it:
+
+```bash
+python3 media_tui.py --source rss --url "https://example.com/feed.xml"
+```
+
+Compatibility commands still work:
+
+```bash
+python3 youtube_download.py --url "https://www.youtube.com/watch?v=VIDEO_ID"
+python3 youtube_download_transcribe.py --url "https://www.youtube.com/watch?v=VIDEO_ID"
+```
+
+## Configuration
+
+- `MEDIA_OUTPUT_DIR`: output folder. Defaults to `./output`
+- `YTDL_OUTPUT_DIR`: legacy output folder fallback
+- `WHISPER_MODEL`: Whisper model. Defaults to `large`
+- `WHISPER_LANGUAGE`: optional language code. If unset, Whisper auto-detects language
+- `YTDL_COOKIES_FROM_BROWSER`: optional browser cookies for YouTube, for example `safari` or `chrome`
+
+All generated audio is saved as `.mp3`. Non-MP3 RSS downloads are converted and removed as intermediates, so new RSS audio output does not remain as `.wav`. Transcripts are saved as `.md` next to the MP3 files.
+
+## Architecture
+
+- `media_tui.py`: direct script entry point
+- `youtube_download.py`: compatibility entry point for YouTube download and MP3 conversion
+- `youtube_download_transcribe.py`: compatibility entry point for YouTube download, MP3 conversion, and transcription
+- `media_information_download/sources/`: input source handling for YouTube and RSS
+- `media_information_download/downloaders/`: YouTube and HTTP media downloaders
+- `media_information_download/audio.py`: audio extraction and MP3 conversion
+- `media_information_download/transcription.py`: Whisper model loading and transcription, adapted from the macOS transcription workflow
+- `media_information_download/output.py`: Markdown transcript and output file handling
+- `media_information_download/pipeline.py`: orchestration across source, download, conversion, transcription, and output
+- `media_information_download/tui.py`: terminal UI and non-interactive CLI entry point
+
+This structure keeps new media sources or formats isolated to source handlers, downloaders, and supported format lists.

package/bin/media-information-download.js

@@ -0,0 +1,102 @@
+#!/usr/bin/env node
+"use strict";
+
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+const { spawnSync } = require("node:child_process");
+
+const packageRoot = path.resolve(__dirname, "..");
+const venvRoot = process.env.MEDIA_INFORMATION_DOWNLOAD_VENV
+  ? path.resolve(process.env.MEDIA_INFORMATION_DOWNLOAD_VENV)
+  : path.join(os.homedir(), ".media-information-download", "venv");
+const isWindows = process.platform === "win32";
+const venvPython = isWindows
+  ? path.join(venvRoot, "Scripts", "python.exe")
+  : path.join(venvRoot, "bin", "python");
+
+function run(command, args, options = {}) {
+  const result = spawnSync(command, args, {
+    cwd: packageRoot,
+    stdio: options.stdio || "inherit",
+    env: {
+      ...process.env,
+      PYTHONPATH: packageRoot,
+    },
+  });
+  return result;
+}
+
+function commandExists(command, args = ["--version"]) {
+  const result = spawnSync(command, args, { stdio: "ignore" });
+  return result.status === 0;
+}
+
+function pythonCommand() {
+  if (process.env.PYTHON) {
+    return { command: process.env.PYTHON, args: [] };
+  }
+  if (isWindows && commandExists("py", ["-3", "--version"])) {
+    return { command: "py", args: ["-3"] };
+  }
+  if (commandExists("python3")) {
+    return { command: "python3", args: [] };
+  }
+  if (commandExists("python")) {
+    return { command: "python", args: [] };
+  }
+  return null;
+}
+
+function ensureVenv() {
+  if (fs.existsSync(venvPython)) {
+    return;
+  }
+
+  fs.mkdirSync(path.dirname(venvRoot), { recursive: true });
+  const python = pythonCommand();
+  if (!python) {
+    console.error("Python 3.10+ is required. Install Python and rerun this command.");
+    process.exit(1);
+  }
+
+  const result = run(python.command, [...python.args, "-m", "venv", venvRoot]);
+  if (result.status !== 0) {
+    process.exit(result.status || 1);
+  }
+}
+
+function ensureDependencies() {
+  const check = run(venvPython, ["-c", "import yt_dlp, whisper, torch"], { stdio: "ignore" });
+  if (check.status === 0) {
+    return;
+  }
+
+  const pip = run(venvPython, [
+    "-m",
+    "pip",
+    "install",
+    "-r",
+    path.join(packageRoot, "requirements-transcribe.txt"),
+  ]);
+  if (pip.status !== 0) {
+    process.exit(pip.status || 1);
+  }
+}
+
+function warnIfFfmpegMissing() {
+  const command = isWindows ? "where" : "which";
+  if (!commandExists(command, ["ffmpeg"])) {
+    console.warn("Warning: ffmpeg is not available on PATH. Downloads may fail during MP3 conversion.");
+  }
+}
+
+ensureVenv();
+ensureDependencies();
+warnIfFfmpegMissing();
+
+const child = run(venvPython, [path.join(packageRoot, "media_tui.py"), ...process.argv.slice(2)]);
+if (child.signal) {
+  process.kill(process.pid, child.signal);
+}
+process.exit(child.status || 0);

package/media_information_download/audio.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+
+def convert_to_mp3(media_path: Path, output_dir: Path) -> Path:
+    mp3_path = output_dir / f"{media_path.stem}.mp3"
+    if media_path.resolve() == mp3_path.resolve() and mp3_path.exists():
+        return mp3_path
+
+    result = subprocess.run(
+        [
+            "ffmpeg",
+            "-y",
+            "-i",
+            str(media_path),
+            "-vn",
+            "-codec:a",
+            "libmp3lame",
+            "-q:a",
+            "2",
+            str(mp3_path),
+        ],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.PIPE,
+        text=True,
+        check=False,
+    )
+    if result.returncode != 0:
+        error_tail = (result.stderr or "").strip().splitlines()[-5:]
+        raise RuntimeError("ffmpeg failed to create MP3. " + " ".join(error_tail))
+    return mp3_path

package/media_information_download/config.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import os
+import shutil
+import sys
+from pathlib import Path
+
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+DEFAULT_OUTPUT_DIR = PROJECT_ROOT / "output"
+SUPPORTED_MEDIA_EXTENSIONS = {
+    ".aac",
+    ".avi",
+    ".flac",
+    ".m4a",
+    ".m4v",
+    ".mkv",
+    ".mov",
+    ".mp3",
+    ".mp4",
+    ".ogg",
+    ".opus",
+    ".wav",
+    ".webm",
+    ".wma",
+}
+SUPPORTED_MEDIA_MIME_PREFIXES = ("audio/", "video/")
+
+
+def get_output_dir() -> Path:
+    output_dir = os.environ.get("MEDIA_OUTPUT_DIR") or os.environ.get("YTDL_OUTPUT_DIR")
+    path = Path(output_dir).expanduser() if output_dir else DEFAULT_OUTPUT_DIR
+    path = path.resolve()
+    path.mkdir(parents=True, exist_ok=True)
+    return path
+
+
+def get_model_name() -> str:
+    return (os.environ.get("WHISPER_MODEL") or "large").strip() or "large"
+
+
+def get_whisper_language() -> str | None:
+    language = (os.environ.get("WHISPER_LANGUAGE") or "").strip()
+    return language or None
+
+
+def get_cookies_from_browser() -> tuple[str, ...] | None:
+    browser = os.environ.get("YTDL_COOKIES_FROM_BROWSER", "").strip()
+    if not browser:
+        return None
+
+    parts = [part.strip() for part in browser.split(":", maxsplit=1) if part.strip()]
+    return tuple(parts) if parts else None
+
+
+def is_project_venv_active() -> bool:
+    active_prefix = Path(sys.prefix).resolve()
+    project_venv = (PROJECT_ROOT / ".venv").resolve()
+    return active_prefix == project_venv or project_venv in active_prefix.parents
+
+
+def dependency_status(include_transcription: bool = True) -> list[str]:
+    messages: list[str] = []
+    if not is_project_venv_active():
+        messages.append(
+            f"Python is not running from the project venv: expected {PROJECT_ROOT / '.venv'}"
+        )
+
+    if shutil.which("ffmpeg") is None:
+        messages.append("ffmpeg is not available on PATH.")
+
+    try:
+        import yt_dlp  # noqa: F401
+    except Exception:
+        messages.append("Python package 'yt-dlp' is not installed in the active environment.")
+
+    if include_transcription:
+        try:
+            import torch  # noqa: F401
+            import whisper  # noqa: F401
+        except Exception:
+            messages.append(
+                "Whisper dependencies are missing. Install local deps with: "
+                ". .venv/bin/activate && pip install -r requirements-transcribe.txt"
+            )
+
+    return messages
+
+
+def require_dependencies(include_transcription: bool = True) -> None:
+    messages = dependency_status(include_transcription=include_transcription)
+    if messages:
+        raise RuntimeError("\n".join(messages))

package/media_information_download/downloaders/http.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import mimetypes
+import re
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+from media_information_download.models import MediaItem
+
+
+SAFE_FILENAME_RE = re.compile(r"[^A-Za-z0-9._() -]+")
+
+
+def safe_filename(value: str, fallback: str = "media") -> str:
+    cleaned = SAFE_FILENAME_RE.sub("_", value).strip(" ._")
+    return cleaned or fallback
+
+
+def _extension_from_item(item: MediaItem, response_url: str) -> str:
+    path_suffix = Path(urllib.parse.urlparse(response_url).path).suffix
+    if path_suffix:
+        return path_suffix
+
+    guessed = mimetypes.guess_extension(item.mime_type.split(";", maxsplit=1)[0].strip())
+    return guessed or ".media"
+
+
+def _dedupe_path(path: Path) -> Path:
+    if not path.exists():
+        return path
+
+    for index in range(1, 1000):
+        candidate = path.with_name(f"{path.stem}-{index}{path.suffix}")
+        if not candidate.exists():
+            return candidate
+    raise RuntimeError(f"Could not create unique output path for {path}")
+
+
+class HTTPDownloader:
+    def download(self, item: MediaItem, output_dir: Path) -> Path:
+        request = urllib.request.Request(
+            item.media_url,
+            headers={"User-Agent": "media-information-download/1.0"},
+        )
+        with urllib.request.urlopen(request, timeout=60) as response:
+            extension = _extension_from_item(item, response.geturl())
+            target = _dedupe_path(output_dir / f"{safe_filename(item.title)}{extension}")
+            with target.open("wb") as file:
+                while True:
+                    chunk = response.read(1024 * 1024)
+                    if not chunk:
+                        break
+                    file.write(chunk)
+
+        return target

package/media_information_download/downloaders/youtube.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from media_information_download.config import get_cookies_from_browser
+from media_information_download.models import MediaItem
+
+try:
+    import yt_dlp  # type: ignore
+except Exception:  # pragma: no cover - checked at runtime
+    yt_dlp = None
+
+
+def _build_ydl_opts(output_dir: Path, format_selector: str) -> dict[str, Any]:
+    ydl_opts: dict[str, Any] = {
+        "format": format_selector,
+        "merge_output_format": "mp4",
+        "outtmpl": str(output_dir / "%(title)s.%(ext)s"),
+        "noplaylist": True,
+        "quiet": True,
+        "no_warnings": True,
+        "retries": 10,
+        "fragment_retries": 10,
+        "extractor_args": {
+            "youtube": {
+                "player_client": ["ios", "android", "web"],
+            }
+        },
+        "http_headers": {
+            "User-Agent": (
+                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) "
+                "Chrome/136.0.0.0 Safari/537.36"
+            )
+        },
+    }
+
+    cookies_from_browser = get_cookies_from_browser()
+    if cookies_from_browser:
+        ydl_opts["cookiesfrombrowser"] = cookies_from_browser
+
+    return ydl_opts
+
+
+def _resolve_downloaded_path(output_dir: Path, prepared_filename: Path) -> Path:
+    if prepared_filename.exists():
+        return prepared_filename
+
+    stem = prepared_filename.stem
+    matches = sorted(
+        path
+        for path in output_dir.glob(f"{stem}.*")
+        if path.suffix.lower() not in {".part", ".ytdl"}
+    )
+    if matches:
+        return matches[0]
+
+    return prepared_filename
+
+
+class YouTubeDownloader:
+    def download(self, item: MediaItem, output_dir: Path) -> Path:
+        if yt_dlp is None:
+            raise RuntimeError("Python package 'yt-dlp' is not installed.")
+
+        format_attempts = [
+            ("bv*+ba/b", "best split video/audio streams"),
+            ("best[ext=mp4]/best", "progressive MP4 fallback"),
+        ]
+        last_error: Exception | None = None
+
+        for format_selector, description in format_attempts:
+            try:
+                ydl_opts = _build_ydl_opts(output_dir, format_selector)
+                with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+                    info = ydl.extract_info(item.media_url, download=True)
+                    filename = Path(ydl.prepare_filename(info))
+                return _resolve_downloaded_path(output_dir, filename)
+            except Exception as exc:
+                last_error = exc
+                if "HTTP Error 403" not in str(exc):
+                    raise
+                if description == format_attempts[-1][1]:
+                    break
+
+        if last_error is not None:
+            raise last_error
+        raise RuntimeError("YouTube download failed without a captured exception.")

package/media_information_download/models.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Callable
+
+
+@dataclass(frozen=True)
+class MediaItem:
+    source_type: str
+    source_url: str
+    media_url: str
+    title: str
+    mime_type: str = ""
+    published: str = ""
+    description: str = ""
+
+
+@dataclass
+class ProcessedMedia:
+    item: MediaItem
+    downloaded_path: Path | None = None
+    mp3_path: Path | None = None
+    transcript_path: Path | None = None
+    error: str | None = None
+    notes: list[str] = field(default_factory=list)
+
+
+ProgressCallback = Callable[[str], None]

package/media_information_download/output.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+from datetime import datetime
+from pathlib import Path
+
+from media_information_download.models import MediaItem
+from media_information_download.transcription import seconds_to_timecode
+
+
+def yaml_quote(value: str) -> str:
+    return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
+
+
+def build_markdown(
+    item: MediaItem,
+    audio_path: Path,
+    model_name: str,
+    device: str,
+    language: str,
+    fps: int,
+    segments: list[dict],
+    full_text: str,
+    include_timecodes: bool = True,
+) -> str:
+    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")
+    segment_blocks = []
+    for seg in segments:
+        start_tc = seconds_to_timecode(float(seg.get("start", 0.0)), fps=fps)
+        end_tc = seconds_to_timecode(float(seg.get("end", 0.0)), fps=fps)
+        seg_text = (seg.get("text") or "").strip()
+        if seg_text:
+            segment_blocks.append(f"{start_tc} - {end_tc}\n{seg_text}")
+
+    transcript_body = "\n\n".join(segment_blocks).strip() if include_timecodes and segment_blocks else full_text
+    return f"""\
+---
+created: {yaml_quote(timestamp)}
+model: {yaml_quote(model_name)}
+device: {yaml_quote(device)}
+language: {yaml_quote(language)}
+source_type: {yaml_quote(item.source_type)}
+source_url: {yaml_quote(item.source_url)}
+media_url: {yaml_quote(item.media_url)}
+source_file: {yaml_quote(audio_path.name)}
+fps_timecode: {fps}
+timecodes: {str(include_timecodes).lower()}
+---
+
+# Transkript: {item.title}
+
+---
+
+{transcript_body}
+"""
+
+
+def write_transcript(
+    output_dir: Path,
+    item: MediaItem,
+    audio_path: Path,
+    model_name: str,
+    device: str,
+    language: str,
+    fps: int,
+    segments: list[dict],
+    full_text: str,
+) -> Path:
+    md_path = output_dir / f"{audio_path.stem}.md"
+    markdown = build_markdown(
+        item=item,
+        audio_path=audio_path,
+        model_name=model_name,
+        device=device,
+        language=language,
+        fps=fps,
+        segments=segments,
+        full_text=full_text,
+    )
+    md_path.write_text(markdown, encoding="utf-8")
+    return md_path
+
+
+def list_output_files(output_dir: Path) -> list[Path]:
+    if not output_dir.exists():
+        return []
+    return sorted(path for path in output_dir.iterdir() if path.is_file())