@mkterswingman/5mghost-yonder 0.0.27 → 0.0.29

package/dist/utils/ytdlp.js

@@ -0,0 +1,205 @@
+import { spawn } from "node:child_process";
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { PATHS } from "./config.js";
+import { getYtDlpPath } from "./ytdlpPath.js";
+import { markGlobalYtDlpRateLimited, scheduleYtDlp } from "./ytdlpScheduler.js";
+import { classifyYtDlpFailure } from "./ytdlpFailures.js";
+function writeYtDlpFailureLog(payload) {
+    try {
+        mkdirSync(PATHS.logsDir, { recursive: true });
+        const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+        const filePath = join(PATHS.logsDir, `yt-dlp-failure-${stamp}.json`);
+        writeFileSync(filePath, JSON.stringify({
+            timestamp: new Date().toISOString(),
+            binary: getYtDlpPath(),
+            ...payload,
+        }, null, 2), "utf8");
+        return filePath;
+    }
+    catch {
+        return undefined;
+    }
+}
+export function createYtDlpStderrLineSplitter(onLine) {
+    let buffer = "";
+    let pendingCarriageReturn = false;
+    const emit = () => {
+        if (buffer.length === 0) {
+            return;
+        }
+        try {
+            onLine(buffer);
+        }
+        catch {
+            // Why: progress observers are best-effort and must not break downloads.
+        }
+        buffer = "";
+    };
+    return {
+        push(chunk) {
+            const text = typeof chunk === "string" ? chunk : chunk.toString("utf8");
+            for (const ch of text) {
+                if (pendingCarriageReturn) {
+                    if (ch === "\n") {
+                        pendingCarriageReturn = false;
+                        emit();
+                        continue;
+                    }
+                    pendingCarriageReturn = false;
+                    emit();
+                }
+                if (ch === "\r") {
+                    pendingCarriageReturn = true;
+                    continue;
+                }
+                if (ch === "\n") {
+                    emit();
+                    continue;
+                }
+                buffer += ch;
+            }
+        },
+        flush() {
+            if (pendingCarriageReturn) {
+                pendingCarriageReturn = false;
+            }
+            emit();
+        },
+    };
+}
+export function hasYtDlp(runSpawnSync = spawnSync) {
+    const result = runSpawnSync(getYtDlpPath(), ["--version"], {
+        stdio: "ignore",
+        timeout: 30_000,
+    });
+    return result.status === 0 && result.error == null;
+}
+export function buildYtDlpArgs(args, options = {}) {
+    const cookiesPath = options.cookiesPath ?? PATHS.cookiesTxt;
+    const cookiesExist = options.cookiesExist ?? existsSync(cookiesPath);
+    const finalArgs = ["--force-ipv4", "--no-warnings", "--js-runtimes", "node", ...args];
+    if (cookiesExist) {
+        finalArgs.push("--cookies", cookiesPath);
+    }
+    return finalArgs;
+}
+export function runYtDlp(args, timeoutMs = 45_000, onStderrLine) {
+    return scheduleYtDlp(() => new Promise((resolve, reject) => {
+        const start = Date.now();
+        const finalArgs = buildYtDlpArgs(args);
+        const proc = spawn(getYtDlpPath(), finalArgs, {
+            stdio: ["ignore", "pipe", "pipe"],
+        });
+        const stdoutChunks = [];
+        const stderrChunks = [];
+        const stderrSplitter = onStderrLine
+            ? createYtDlpStderrLineSplitter(onStderrLine)
+            : null;
+        proc.stdout.on("data", (chunk) => stdoutChunks.push(chunk));
+        proc.stderr.on("data", (chunk) => {
+            stderrChunks.push(chunk);
+            stderrSplitter?.push(chunk);
+        });
+        function flushStderrLineBuffer() {
+            stderrSplitter?.flush();
+        }
+        let settled = false;
+        const timer = setTimeout(() => {
+            if (!settled) {
+                settled = true;
+                proc.kill("SIGKILL");
+                flushStderrLineBuffer();
+                const stdout = Buffer.concat(stdoutChunks).toString("utf8");
+                const stderr = `yt-dlp timed out after ${timeoutMs}ms`;
+                resolve({
+                    exitCode: -1,
+                    stdout,
+                    stderr,
+                    durationMs: Date.now() - start,
+                    failureLogPath: writeYtDlpFailureLog({
+                        args: finalArgs,
+                        exitCode: -1,
+                        stdout,
+                        stderr,
+                        durationMs: Date.now() - start,
+                        timeoutMs,
+                        trigger: "timeout",
+                    }),
+                });
+            }
+        }, timeoutMs);
+        proc.on("close", (code) => {
+            clearTimeout(timer);
+            if (!settled) {
+                settled = true;
+                flushStderrLineBuffer();
+                const stdout = Buffer.concat(stdoutChunks).toString("utf8");
+                const stderr = Buffer.concat(stderrChunks).toString("utf8");
+                const exitCode = code ?? 1;
+                resolve({
+                    exitCode,
+                    stdout,
+                    stderr,
+                    durationMs: Date.now() - start,
+                    failureLogPath: exitCode === 0
+                        ? undefined
+                        : writeYtDlpFailureLog({
+                            args: finalArgs,
+                            exitCode,
+                            stdout,
+                            stderr,
+                            durationMs: Date.now() - start,
+                            timeoutMs,
+                            trigger: "exit",
+                        }),
+                });
+                if (exitCode !== 0 && classifyYtDlpFailure(stderr) === "RATE_LIMITED") {
+                    markGlobalYtDlpRateLimited();
+                }
+            }
+        });
+        proc.on("error", (err) => {
+            clearTimeout(timer);
+            if (!settled) {
+                settled = true;
+                const stdout = Buffer.concat(stdoutChunks).toString("utf8");
+                const stderr = Buffer.concat(stderrChunks).toString("utf8");
+                reject(Object.assign(err, {
+                    failureLogPath: writeYtDlpFailureLog({
+                        args: finalArgs,
+                        exitCode: 1,
+                        stdout,
+                        stderr,
+                        durationMs: Date.now() - start,
+                        timeoutMs,
+                        trigger: "spawn_error",
+                        spawnError: err instanceof Error ? err.message : String(err),
+                    }),
+                }));
+            }
+        });
+    }));
+}
+export async function runYtDlpJson(args, timeoutMs = 45_000) {
+    const result = await runYtDlp(args, timeoutMs);
+    if (result.exitCode !== 0) {
+        return {
+            ok: false,
+            error: result.stderr.slice(0, 500) || `yt-dlp exited with ${result.exitCode}`,
+        };
+    }
+    try {
+        return {
+            ok: true,
+            value: JSON.parse(result.stdout),
+        };
+    }
+    catch {
+        return {
+            ok: false,
+            error: "yt-dlp returned invalid JSON metadata.",
+        };
+    }
+}

package/dist/utils/ytdlpFailures.d.ts

@@ -0,0 +1,3 @@
+export type YtDlpFailureKind = "COOKIES_EXPIRED" | "COOKIES_INVALID" | "SIGN_IN_REQUIRED" | "RATE_LIMITED";
+export declare function classifyYtDlpFailure(stderr: string): YtDlpFailureKind | null;
+export declare function appendDiagnosticLog(message: string, logPath?: string): string;

package/dist/utils/ytdlpFailures.js

@@ -0,0 +1,27 @@
+export function classifyYtDlpFailure(stderr) {
+    const normalized = stderr.toLowerCase();
+    if (normalized.includes("cookies are no longer valid") ||
+        normalized.includes("cookie has expired") ||
+        normalized.includes("cookies have expired") ||
+        normalized.includes("session expired")) {
+        return "COOKIES_EXPIRED";
+    }
+    if (normalized.includes("rate-limited by youtube") ||
+        normalized.includes("current session has been rate-limited")) {
+        return "RATE_LIMITED";
+    }
+    if (normalized.includes("sign in to confirm you're not a bot") ||
+        normalized.includes("sign in to confirm you’re not a bot")) {
+        return "SIGN_IN_REQUIRED";
+    }
+    if (normalized.includes("sign in") ||
+        normalized.includes("login") ||
+        normalized.includes("cookies") ||
+        normalized.includes("not a bot")) {
+        return "COOKIES_INVALID";
+    }
+    return null;
+}
+export function appendDiagnosticLog(message, logPath) {
+    return logPath ? `${message}\nDiagnostic log: ${logPath}` : message;
+}

package/dist/utils/ytdlpPath.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Resolve the yt-dlp binary path.
+ *
+ * Priority:
+ *   1. YT_DLP_PATH env var (explicit override)
+ *   2. Runtime-managed binary at ~/.yt-mcp/runtime/bin/yt-dlp
+ *   3. "yt-dlp" — fall back to system PATH
+ */
+export interface ResolveYtDlpPathOptions {
+    envPath?: string | null;
+    runtimePath?: string;
+    runtimeExists?: boolean;
+}
+export declare function getRuntimeYtDlpPath(runtimeBinDir?: string): string;
+export declare function resolveYtDlpPath(options?: ResolveYtDlpPathOptions): string;
+/**
+ * Returns the absolute path to the yt-dlp binary, or the bare command name
+ * "yt-dlp" if only available on system PATH.
+ */
+export declare function getYtDlpPath(): string;
+export interface YtDlpVersionInfo {
+    version: string;
+    source: "runtime" | "system" | "env";
+}
+/**
+ * Get the version string and source of the resolved yt-dlp binary.
+ * Returns null if yt-dlp is not available at all.
+ *
+ * Uses a 30s timeout because macOS Gatekeeper performs a network verification
+ * on first run of ad-hoc signed binaries (~15-20s). The result is cached by
+ * the OS, so subsequent calls return instantly.
+ */
+export declare function getYtDlpVersion(binPath?: string): YtDlpVersionInfo | null;

package/dist/utils/ytdlpPath.js

@@ -0,0 +1,77 @@
+/**
+ * Resolve the yt-dlp binary path.
+ *
+ * Priority:
+ *   1. YT_DLP_PATH env var (explicit override)
+ *   2. Runtime-managed binary at ~/.yt-mcp/runtime/bin/yt-dlp
+ *   3. "yt-dlp" — fall back to system PATH
+ */
+import { existsSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { PATHS, buildRuntimeBinaryPath } from "./config.js";
+export function getRuntimeYtDlpPath(runtimeBinDir = PATHS.runtimeBinDir) {
+    const name = process.platform === "win32" ? "yt-dlp.exe" : "yt-dlp";
+    return buildRuntimeBinaryPath(runtimeBinDir, name);
+}
+export function resolveYtDlpPath(options = {}) {
+    const envPath = options.envPath ?? process.env.YT_DLP_PATH ?? null;
+    if (envPath) {
+        return envPath;
+    }
+    const runtimePath = options.runtimePath ?? getRuntimeYtDlpPath();
+    const runtimeExists = options.runtimeExists ?? existsSync(runtimePath);
+    if (runtimeExists) {
+        return runtimePath;
+    }
+    return "yt-dlp";
+}
+/**
+ * Returns the absolute path to the yt-dlp binary, or the bare command name
+ * "yt-dlp" if only available on system PATH.
+ */
+export function getYtDlpPath() {
+    return resolveYtDlpPath();
+}
+/**
+ * Get the version string and source of the resolved yt-dlp binary.
+ * Returns null if yt-dlp is not available at all.
+ *
+ * Uses a 30s timeout because macOS Gatekeeper performs a network verification
+ * on first run of ad-hoc signed binaries (~15-20s). The result is cached by
+ * the OS, so subsequent calls return instantly.
+ */
+export function getYtDlpVersion(binPath) {
+    const resolved = binPath ?? getYtDlpPath();
+    let source;
+    if (process.env.YT_DLP_PATH && resolved === process.env.YT_DLP_PATH) {
+        source = "env";
+    }
+    else if (resolved === getRuntimeYtDlpPath()) {
+        source = "runtime";
+    }
+    else {
+        source = "system";
+    }
+    try {
+        // macOS Gatekeeper: try clearing quarantine xattr before first execution.
+        // This helps in some environments; in others the OS re-applies it.
+        if (process.platform === "darwin" && resolved !== "yt-dlp") {
+            try {
+                execFileSync("xattr", ["-cr", resolved], { stdio: "ignore" });
+            }
+            catch {
+                // non-fatal
+            }
+        }
+        const ver = execFileSync(resolved, ["--version"], {
+            stdio: ["ignore", "pipe", "ignore"],
+            timeout: 30_000,
+        })
+            .toString()
+            .trim();
+        return { version: ver, source };
+    }
+    catch {
+        return null;
+    }
+}

package/dist/utils/ytdlpProgress.d.ts

@@ -0,0 +1,13 @@
+export interface ParsedYtDlpProgress {
+    progress: string;
+    downloaded_size?: string;
+    total_size?: string;
+    speed?: string;
+    eta?: string;
+}
+/**
+ * Parse the canonical yt-dlp download progress line used by this integration.
+ * Narrow contract: intentionally supports only representative `[download] ...`
+ * lines with percent, total size, speed, and ETA fields.
+ */
+export declare function parseYtDlpProgressLine(line: string): ParsedYtDlpProgress | null;

package/dist/utils/ytdlpProgress.js

@@ -0,0 +1,77 @@
+import { formatBytes, formatEta, formatPercent, formatSpeed } from "./formatters.js";
+const SIZE_UNITS = {
+    b: 1,
+    kb: 1000,
+    kib: 1024,
+    mb: 1000 ** 2,
+    mib: 1024 ** 2,
+    gb: 1000 ** 3,
+    gib: 1024 ** 3,
+    tb: 1000 ** 4,
+    tib: 1024 ** 4,
+};
+function parseSizeToBytes(text) {
+    const match = text.trim().match(/^(\d+(?:\.\d+)?)([kmgt]?i?b)$/i);
+    if (!match) {
+        return null;
+    }
+    const value = Number(match[1]);
+    const unit = match[2].toLowerCase();
+    const multiplier = SIZE_UNITS[unit];
+    if (!Number.isFinite(value) || multiplier == null) {
+        return null;
+    }
+    return value * multiplier;
+}
+function parseEtaToSeconds(text) {
+    const trimmed = text.trim();
+    if (!trimmed || trimmed === "N/A") {
+        return null;
+    }
+    const hms = trimmed.match(/^(\d+):(\d{2}):(\d{2})$/);
+    if (hms) {
+        return Number(hms[1]) * 3600 + Number(hms[2]) * 60 + Number(hms[3]);
+    }
+    const ms = trimmed.match(/^(\d{1,2}):(\d{2})$/);
+    if (ms) {
+        return Number(ms[1]) * 60 + Number(ms[2]);
+    }
+    const unitMatch = trimmed.match(/^(\d+(?:\.\d+)?)([smhd])$/i);
+    if (unitMatch) {
+        const value = Number(unitMatch[1]);
+        const unit = unitMatch[2].toLowerCase();
+        const multiplier = unit === "s" ? 1 : unit === "m" ? 60 : unit === "h" ? 3600 : 86400;
+        return value * multiplier;
+    }
+    return null;
+}
+/**
+ * Parse the canonical yt-dlp download progress line used by this integration.
+ * Narrow contract: intentionally supports only representative `[download] ...`
+ * lines with percent, total size, speed, and ETA fields.
+ */
+export function parseYtDlpProgressLine(line) {
+    const match = line.match(/^\[download\]\s+(?<progress>\d+(?:\.\d+)?)% of (?<total>\S+)(?: at (?<speed>\S+?\/s))?(?: ETA (?<eta>\S+))?(?: \((?<details>.+)\))?$/);
+    if (!match?.groups?.progress) {
+        return null;
+    }
+    const progress = formatPercent(Number(match.groups.progress));
+    const totalBytes = match.groups.total ? parseSizeToBytes(match.groups.total) : null;
+    const speedBytes = match.groups.speed ? parseSizeToBytes(match.groups.speed.replace(/\/s$/i, "")) : null;
+    const etaSeconds = match.groups.eta ? parseEtaToSeconds(match.groups.eta) : null;
+    const parsed = { progress };
+    if (totalBytes != null) {
+        parsed.total_size = formatBytes(totalBytes);
+    }
+    if (speedBytes != null) {
+        parsed.speed = formatSpeed(speedBytes);
+    }
+    if (etaSeconds != null) {
+        parsed.eta = formatEta(etaSeconds);
+    }
+    if (totalBytes != null && Number.isFinite(Number(match.groups.progress))) {
+        const downloadedBytes = (totalBytes * Number(match.groups.progress)) / 100;
+        parsed.downloaded_size = formatBytes(downloadedBytes);
+    }
+    return parsed;
+}

package/dist/utils/ytdlpScheduler.d.ts

@@ -0,0 +1,25 @@
+export interface YtDlpSchedulerState {
+    running: boolean;
+    pendingCount: number;
+    nextAvailableAt: number | null;
+    wouldThrottle: boolean;
+}
+interface YtDlpSchedulerOptions {
+    minIntervalMs?: number;
+    maxIntervalMs?: number;
+    rateLimitBackoffMs?: number;
+    nextIntervalMs?: () => number;
+    now?: () => number;
+    sleep?: (ms: number) => Promise<void>;
+}
+export interface YtDlpScheduler {
+    schedule<T>(task: () => Promise<T>): Promise<T>;
+    markRateLimited(): void;
+    getState(): YtDlpSchedulerState;
+    sampleNextIntervalMs(): number;
+}
+export declare function createYtDlpScheduler(options?: YtDlpSchedulerOptions): YtDlpScheduler;
+export declare function scheduleYtDlp<T>(task: () => Promise<T>): Promise<T>;
+export declare function markGlobalYtDlpRateLimited(): void;
+export declare function getGlobalYtDlpSchedulerState(): YtDlpSchedulerState;
+export {};

package/dist/utils/ytdlpScheduler.js

@@ -0,0 +1,78 @@
+const DEFAULT_MIN_INTERVAL_MS = 3_000;
+const DEFAULT_MAX_INTERVAL_MS = 7_000;
+const DEFAULT_RATE_LIMIT_BACKOFF_MS = 10 * 60_000;
+export function createYtDlpScheduler(options = {}) {
+    const now = options.now ?? Date.now;
+    const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    const minIntervalMs = options.minIntervalMs ?? DEFAULT_MIN_INTERVAL_MS;
+    const maxIntervalMs = options.maxIntervalMs ?? DEFAULT_MAX_INTERVAL_MS;
+    const rateLimitBackoffMs = options.rateLimitBackoffMs ?? DEFAULT_RATE_LIMIT_BACKOFF_MS;
+    const sampleNextIntervalMs = options.nextIntervalMs ??
+        (() => {
+            if (maxIntervalMs <= minIntervalMs) {
+                return minIntervalMs;
+            }
+            return minIntervalMs + Math.floor(Math.random() * (maxIntervalMs - minIntervalMs + 1));
+        });
+    let running = false;
+    let pendingCount = 0;
+    let nextAvailableAt = null;
+    let tail = Promise.resolve();
+    async function waitForAvailability() {
+        const currentNext = nextAvailableAt;
+        if (currentNext == null) {
+            return;
+        }
+        const delayMs = currentNext - now();
+        if (delayMs > 0) {
+            await sleep(delayMs);
+        }
+    }
+    return {
+        async schedule(task) {
+            pendingCount += 1;
+            const previousTail = tail;
+            let release;
+            tail = new Promise((resolve) => {
+                release = resolve;
+            });
+            await previousTail;
+            pendingCount -= 1;
+            await waitForAvailability();
+            running = true;
+            try {
+                return await task();
+            }
+            finally {
+                running = false;
+                nextAvailableAt = Math.max(nextAvailableAt ?? 0, now() + sampleNextIntervalMs());
+                release();
+            }
+        },
+        markRateLimited() {
+            nextAvailableAt = Math.max(nextAvailableAt ?? 0, now() + rateLimitBackoffMs);
+        },
+        getState() {
+            const next = nextAvailableAt;
+            return {
+                running,
+                pendingCount,
+                nextAvailableAt: next,
+                wouldThrottle: running || pendingCount > 0 || (next != null && next > now()),
+            };
+        },
+        sampleNextIntervalMs() {
+            return sampleNextIntervalMs();
+        },
+    };
+}
+const globalYtDlpScheduler = createYtDlpScheduler();
+export function scheduleYtDlp(task) {
+    return globalYtDlpScheduler.schedule(task);
+}
+export function markGlobalYtDlpRateLimited() {
+    globalYtDlpScheduler.markRateLimited();
+}
+export function getGlobalYtDlpSchedulerState() {
+    return globalYtDlpScheduler.getState();
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mkterswingman/5mghost-yonder",
-  "version": "0.0.27",
+  "version": "0.0.29",
   "description": "Internal MCP client with local data tools and remote API proxy",
   "type": "module",
   "bin": {
@@ -12,6 +12,7 @@
   "scripts": {
     "build": "tsc -p tsconfig.json",
     "test": "npm run build && node --test tests/*.test.mjs",
+    "prepublishOnly": "npm run build",
     "dev": "tsx src/cli/index.ts",
     "start": "node dist/cli/index.js"
   },