@mkterswingman/5mghost-yonder 0.0.12 → 0.0.14

package/dist/download/downloader.d.ts

@@ -1,6 +1,7 @@
 import type { DownloadJobManager } from "./jobManager.js";
 import type { DownloadJobItemSnapshot, DownloadMode } from "./types.js";
 import type { PreflightVideoMetadata } from "../utils/videoMetadata.js";
+import { type YtDlpSchedulerState } from "../utils/ytdlpScheduler.js";
 type DownloadingStep = "download_video" | "download_subtitles" | "merge";
 export interface DownloadOneItemProgressUpdate {
     step: DownloadingStep;
@@ -20,7 +21,7 @@ export interface DownloadOneItemInput {
     subtitleFormats: string[];
     subtitleLanguages?: string[];
     defaultSubtitleLanguages?: string[];
-    jobManager?: Pick<DownloadJobManager, "startItem" | "updateItemProgress" | "completeItem" | "failItem">;
+    jobManager?: Pick<DownloadJobManager, "startItem" | "updateItemProgress" | "completeItem" | "failItem" | "setItemQueuedByGlobalRateLimit">;
     jobId?: string;
     deps?: Partial<DownloadOneItemDeps>;
 }
@@ -38,6 +39,8 @@ interface DownloadOneItemDeps {
         videoQuality: string;
         onProgress: (update: DownloadOneItemProgressUpdate) => void;
     }) => Promise<string>;
+    refreshCookies: () => Promise<boolean>;
+    getYtDlpSchedulerState: () => YtDlpSchedulerState;
     downloadSubtitles: (input: {
         videoId: string;
         sourceUrl: string;

package/dist/download/downloader.js

@@ -2,10 +2,13 @@ import { mkdir, rm, stat, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { downloadSubtitlesForLanguages, toReadableSubtitleJobError } from "../tools/subtitles.js";
 import { loadConfig } from "../utils/config.js";
+import { tryHeadlessRefresh } from "../utils/cookieRefresh.js";
 import { hasFfmpeg, isRuntimeMissingMessage } from "../utils/ffmpeg.js";
 import { formatBytes } from "../utils/formatters.js";
 import { hasYtDlp, runYtDlp, runYtDlpJson } from "../utils/ytdlp.js";
+import { appendDiagnosticLog, classifyYtDlpFailure } from "../utils/ytdlpFailures.js";
 import { parseYtDlpProgressLine } from "../utils/ytdlpProgress.js";
+import { getGlobalYtDlpSchedulerState } from "../utils/ytdlpScheduler.js";
 export async function downloadOneItem(input) {
     const deps = resolveDeps(input.deps);
     const manager = input.jobManager;
@@ -50,26 +53,52 @@ export async function downloadOneItem(input) {
         if (needsVideo(input.mode)) {
             currentStep = "download_video";
             manager?.updateItemProgress(jobId, input.item.video_id, currentStep, 0);
-            result.video_file = await deps.downloadVideo({
-                sourceUrl: input.item.source_url,
-                outputFile: videoFile,
-                videoQuality,
-                onProgress,
-            });
+            manager?.setItemQueuedByGlobalRateLimit(jobId, input.item.video_id, deps.getYtDlpSchedulerState().wouldThrottle);
+            try {
+                result.video_file = await deps.downloadVideo({
+                    sourceUrl: input.item.source_url,
+                    outputFile: videoFile,
+                    videoQuality,
+                    onProgress,
+                });
+            }
+            catch (error) {
+                const failureKind = classifyVideoDownloadError(error);
+                if (failureKind != null && failureKind !== "RATE_LIMITED" && await deps.refreshCookies()) {
+                    result.video_file = await deps.downloadVideo({
+                        sourceUrl: input.item.source_url,
+                        outputFile: videoFile,
+                        videoQuality,
+                        onProgress,
+                    });
+                }
+                else {
+                    throw error;
+                }
+            }
+            finally {
+                manager?.setItemQueuedByGlobalRateLimit(jobId, input.item.video_id, false);
+            }
             result.final_file_size = formatBytes((await deps.stat(result.video_file)).size);
         }
         if (needsSubtitles(input.mode)) {
             currentStep = "download_subtitles";
             manager?.updateItemProgress(jobId, input.item.video_id, currentStep, 0);
-            result.subtitle_files = await deps.downloadSubtitles({
-                videoId: input.item.video_id,
-                sourceUrl: input.item.source_url,
-                subtitlesDir,
-                formats: subtitleFormats,
-                languages: subtitleLanguages,
-                skipMissingLanguages: usingDefaultSubtitleLanguages,
-                onProgress,
-            });
+            manager?.setItemQueuedByGlobalRateLimit(jobId, input.item.video_id, deps.getYtDlpSchedulerState().wouldThrottle);
+            try {
+                result.subtitle_files = await deps.downloadSubtitles({
+                    videoId: input.item.video_id,
+                    sourceUrl: input.item.source_url,
+                    subtitlesDir,
+                    formats: subtitleFormats,
+                    languages: subtitleLanguages,
+                    skipMissingLanguages: usingDefaultSubtitleLanguages,
+                    onProgress,
+                });
+            }
+            finally {
+                manager?.setItemQueuedByGlobalRateLimit(jobId, input.item.video_id, false);
+            }
         }
         currentStep = "write_metadata";
         manager?.updateItemProgress(jobId, input.item.video_id, currentStep, 100);
@@ -103,6 +132,8 @@ function resolveDeps(overrides) {
         hasFfmpeg,
         fetchMetadata: fetchMetadataWithYtDlp,
         downloadVideo: downloadVideoWithYtDlp,
+        refreshCookies: tryHeadlessRefresh,
+        getYtDlpSchedulerState: getGlobalYtDlpSchedulerState,
         downloadSubtitles: downloadSubtitlesWithYtDlp,
         mkdir: async (path, options) => {
             await mkdir(path, options);
@@ -166,7 +197,17 @@ async function downloadVideoWithYtDlp(input) {
         }
     });
     if (result.exitCode !== 0) {
-        throw new Error(result.stderr.slice(0, 500) || `yt-dlp exited with ${result.exitCode}`);
+        const failureKind = classifyYtDlpFailure(result.stderr);
+        if (failureKind === "RATE_LIMITED") {
+            throw new Error(appendDiagnosticLog("The current YouTube session has been rate-limited. Wait and retry later.", result.failureLogPath));
+        }
+        if (failureKind === "SIGN_IN_REQUIRED") {
+            throw new Error(appendDiagnosticLog("YouTube requested an additional sign-in confirmation for this video.", result.failureLogPath));
+        }
+        if (failureKind === "COOKIES_INVALID" || failureKind === "COOKIES_EXPIRED") {
+            throw new Error(appendDiagnosticLog("YouTube rejected the current cookies for video download.", result.failureLogPath));
+        }
+        throw new Error(appendDiagnosticLog(result.stderr.slice(0, 500) || `yt-dlp exited with ${result.exitCode}`, result.failureLogPath));
     }
     return resolveYtDlpOutputPath(input.outputFile, result.stdout);
 }
@@ -259,6 +300,10 @@ function sortSubtitleFiles(files) {
         .sort(([left], [right]) => left.localeCompare(right))
         .map(([format, paths]) => [format, [...paths].sort()]));
 }
+function classifyVideoDownloadError(error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return classifyYtDlpFailure(message);
+}
 function toErrorMessage(error) {
     return toReadableSubtitleJobError(error);
 }

package/dist/download/jobManager.d.ts

@@ -10,6 +10,7 @@ export declare class DownloadJobManager {
     pollJob(jobId: string): DownloadJobSnapshot;
     startItem(jobId: string, videoId: string, step?: DownloadStep): DownloadJobSnapshot;
     updateItemProgress(jobId: string, videoId: string, step: DownloadStep, progressPercent: number | null): DownloadJobSnapshot;
+    setItemQueuedByGlobalRateLimit(jobId: string, videoId: string, queued: boolean): DownloadJobSnapshot;
     completeItem(jobId: string, videoId: string, outputs?: Pick<DownloadJobItemSnapshot, "video_file" | "metadata_file" | "final_file_size" | "subtitle_files">): DownloadJobSnapshot;
     failItem(jobId: string, videoId: string, error: string, step?: DownloadStep): DownloadJobSnapshot;
     private applyItemMutation;

package/dist/download/jobManager.js

@@ -26,6 +26,7 @@ export class DownloadJobManager {
                 title: video.title,
                 output_dir: video.output_dir,
                 status: "queued",
+                queued_by_global_rate_limit: false,
                 step: null,
                 progress_percent: null,
                 error: null,
@@ -65,6 +66,12 @@ export class DownloadJobManager {
         });
         return this.toSnapshot(job);
     }
+    setItemQueuedByGlobalRateLimit(jobId, videoId, queued) {
+        const job = this.getJob(jobId);
+        const item = this.getItem(job, videoId);
+        item.queued_by_global_rate_limit = queued;
+        return this.toSnapshot(job);
+    }
     completeItem(jobId, videoId, outputs) {
         const job = this.getJob(jobId);
         const item = this.getItem(job, videoId);
@@ -96,6 +103,10 @@ export class DownloadJobManager {
         const nextStep = mutation.step === undefined ? item.step : mutation.step;
         const previousStatus = item.status;
         item.status = nextStatus;
+        item.queued_by_global_rate_limit =
+            mutation.queued_by_global_rate_limit === undefined
+                ? item.queued_by_global_rate_limit
+                : mutation.queued_by_global_rate_limit;
         item.step = nextStep;
         item.video_file = mutation.video_file === undefined ? item.video_file : mutation.video_file;
         item.metadata_file = mutation.metadata_file === undefined ? item.metadata_file : mutation.metadata_file;
@@ -107,6 +118,7 @@ export class DownloadJobManager {
             job.started_at = job.started_at ?? now;
             item.completed_at = null;
             item.error = null;
+            item.queued_by_global_rate_limit = false;
             item.progress_percent = resolveRunningProgress(item.progress_percent, mutation.progress_percent, previousStatus);
         }
         if (nextStatus === "completed" || nextStatus === "failed") {
@@ -114,6 +126,7 @@ export class DownloadJobManager {
             job.started_at = job.started_at ?? now;
             item.completed_at = now;
             item.error = mutation.error === undefined ? item.error : mutation.error;
+            item.queued_by_global_rate_limit = false;
             item.progress_percent =
                 mutation.progress_percent === undefined ? item.progress_percent : clampProgressOrNull(mutation.progress_percent);
         }

package/dist/download/types.d.ts

@@ -19,6 +19,7 @@ export interface DownloadJobItemSnapshot {
     title: string;
     output_dir: string;
     status: DownloadItemStatus;
+    queued_by_global_rate_limit: boolean;
     step: DownloadStep | null;
     progress_percent: number | null;
     error: string | null;

package/dist/tools/subtitles.js

@@ -5,12 +5,15 @@ import { PATHS } from "../utils/config.js";
 import { runYtDlp } from "../utils/ytdlp.js";
 import { hasSIDCookies, areCookiesExpired } from "../utils/cookies.js";
 import { tryHeadlessRefresh } from "../utils/cookieRefresh.js";
+import { appendDiagnosticLog, classifyYtDlpFailure } from "../utils/ytdlpFailures.js";
 import { resolveVideoInput, normalizeVideoInputs } from "../utils/videoInput.js";
 const AUTH_REQUIRED_MSG = "❌ 未认证。请先登录：\n• OAuth: npx @mkterswingman/5mghost-yonder setup\n• PAT: 设置环境变量 YT_MCP_TOKEN 或在 https://mkterswingman.com/pat/login 生成 token";
 const COOKIE_SETUP_COMMAND = "yt-mcp setup-cookies";
 const COOKIE_MISSING_MESSAGE = `No valid YouTube cookies found.\nPlease run in your terminal: ${COOKIE_SETUP_COMMAND}`;
 const COOKIE_EXPIRED_MESSAGE = `YouTube cookies have expired and auto-refresh failed.\nPlease run in your terminal: ${COOKIE_SETUP_COMMAND}`;
 const COOKIE_INVALID_MESSAGE = `YouTube rejected the current cookies and auto-refresh failed.\nPlease run in your terminal: ${COOKIE_SETUP_COMMAND}`;
+const SIGN_IN_REQUIRED_MESSAGE = `YouTube requested an additional sign-in confirmation and auto-refresh failed.\nPlease run in your terminal: ${COOKIE_SETUP_COMMAND}`;
+const RATE_LIMITED_MESSAGE = "The current YouTube session has been rate-limited. Wait and retry later.";
 const COOKIE_JOB_MESSAGE = `YouTube cookies are missing, expired, or invalid.\nPlease run in your terminal: ${COOKIE_SETUP_COMMAND}`;
 function toolOk(payload) {
     return {
@@ -56,30 +59,20 @@ function isCookieFailureText(error) {
         normalized.includes("login") ||
         normalized.includes("not a bot"));
 }
-function appendDiagnosticLog(message, logPath) {
-    return logPath ? `${message}\nDiagnostic log: ${logPath}` : message;
-}
 function classifyYtDlpCookieFailure(stderr) {
-    const normalized = stderr.toLowerCase();
-    if (normalized.includes("cookies are no longer valid") ||
-        normalized.includes("cookie has expired") ||
-        normalized.includes("cookies have expired") ||
-        normalized.includes("session expired")) {
-        return {
-            code: "COOKIES_EXPIRED",
-            message: COOKIE_EXPIRED_MESSAGE,
-        };
-    }
-    if (normalized.includes("sign in") ||
-        normalized.includes("login") ||
-        normalized.includes("cookies") ||
-        normalized.includes("not a bot")) {
-        return {
-            code: "COOKIES_INVALID",
-            message: COOKIE_INVALID_MESSAGE,
-        };
+    const kind = classifyYtDlpFailure(stderr);
+    switch (kind) {
+        case "COOKIES_EXPIRED":
+            return { code: kind, message: COOKIE_EXPIRED_MESSAGE };
+        case "COOKIES_INVALID":
+            return { code: kind, message: COOKIE_INVALID_MESSAGE };
+        case "SIGN_IN_REQUIRED":
+            return { code: kind, message: SIGN_IN_REQUIRED_MESSAGE };
+        case "RATE_LIMITED":
+            return { code: kind, message: RATE_LIMITED_MESSAGE };
+        default:
+            return null;
     }
-    return null;
 }
 export function toReadableSubtitleJobError(error) {
     const message = error instanceof Error ? error.message : String(error);
@@ -367,7 +360,7 @@ export async function downloadSubtitlesForLanguages(input) {
                 outputStem: `${lang}`,
                 targetFile,
             });
-            if (result.cookieFailureCode) {
+            if (result.cookieFailureCode && result.cookieFailureCode !== "RATE_LIMITED") {
                 const refreshed = await tryRefreshSubtitleCookies();
                 if (refreshed) {
                     result = await downloadSubtitle(input.videoId, lang, format, {
@@ -467,7 +460,14 @@ export function registerSubtitleTools(server, config, tokenManager) {
                     }
                     // Retry also failed with expired cookies — give up
                 }
-                return toolErr(dl.cookieFailureCode, dl.error ?? (dl.cookieFailureCode === "COOKIES_EXPIRED" ? COOKIE_EXPIRED_MESSAGE : COOKIE_INVALID_MESSAGE));
+                return toolErr(dl.cookieFailureCode, dl.error ??
+                    (dl.cookieFailureCode === "COOKIES_EXPIRED"
+                        ? COOKIE_EXPIRED_MESSAGE
+                        : dl.cookieFailureCode === "SIGN_IN_REQUIRED"
+                            ? SIGN_IN_REQUIRED_MESSAGE
+                            : dl.cookieFailureCode === "RATE_LIMITED"
+                                ? RATE_LIMITED_MESSAGE
+                                : COOKIE_INVALID_MESSAGE));
             }
             if (!dl.ok) {
                 // When using default languages, silently skip unavailable ones
@@ -554,6 +554,9 @@ export function registerSubtitleTools(server, config, tokenManager) {
                     lastError = dl.error;
                 }
             }
+            if (cookieFailureCode === "RATE_LIMITED") {
+                return toolErr("RATE_LIMITED", RATE_LIMITED_MESSAGE);
+            }
             if (cookieFailureCode) {
                 const refreshed = await tryRefreshSubtitleCookies();
                 if (refreshed) {
@@ -571,7 +574,11 @@ export function registerSubtitleTools(server, config, tokenManager) {
                     }
                 }
                 if (cookieFailureCode) {
-                    return toolErr(cookieFailureCode, cookieFailureCode === "COOKIES_EXPIRED" ? COOKIE_EXPIRED_MESSAGE : COOKIE_INVALID_MESSAGE);
+                    return toolErr(cookieFailureCode, cookieFailureCode === "COOKIES_EXPIRED"
+                        ? COOKIE_EXPIRED_MESSAGE
+                        : cookieFailureCode === "SIGN_IN_REQUIRED"
+                            ? SIGN_IN_REQUIRED_MESSAGE
+                            : COOKIE_INVALID_MESSAGE);
                 }
             }
             if (langFound.length > 0) {

package/dist/utils/ytdlp.js

@@ -4,6 +4,8 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { PATHS } from "./config.js";
 import { getYtDlpPath } from "./ytdlpPath.js";
+import { markGlobalYtDlpRateLimited, scheduleYtDlp } from "./ytdlpScheduler.js";
+import { classifyYtDlpFailure } from "./ytdlpFailures.js";
 function writeYtDlpFailureLog(payload) {
     try {
         mkdirSync(PATHS.logsDir, { recursive: true });
@@ -84,7 +86,7 @@ export function buildYtDlpArgs(args, options = {}) {
     return finalArgs;
 }
 export function runYtDlp(args, timeoutMs = 45_000, onStderrLine) {
-    return new Promise((resolve, reject) => {
+    return scheduleYtDlp(() => new Promise((resolve, reject) => {
         const start = Date.now();
         const finalArgs = buildYtDlpArgs(args);
         const proc = spawn(getYtDlpPath(), finalArgs, {
@@ -153,6 +155,9 @@ export function runYtDlp(args, timeoutMs = 45_000, onStderrLine) {
                             trigger: "exit",
                         }),
                 });
+                if (exitCode !== 0 && classifyYtDlpFailure(stderr) === "RATE_LIMITED") {
+                    markGlobalYtDlpRateLimited();
+                }
             }
         });
         proc.on("error", (err) => {
@@ -175,7 +180,7 @@ export function runYtDlp(args, timeoutMs = 45_000, onStderrLine) {
                 }));
             }
         });
-    });
+    }));
 }
 export async function runYtDlpJson(args, timeoutMs = 45_000) {
     const result = await runYtDlp(args, timeoutMs);

package/dist/utils/ytdlpFailures.d.ts

@@ -0,0 +1,3 @@
+export type YtDlpFailureKind = "COOKIES_EXPIRED" | "COOKIES_INVALID" | "SIGN_IN_REQUIRED" | "RATE_LIMITED";
+export declare function classifyYtDlpFailure(stderr: string): YtDlpFailureKind | null;
+export declare function appendDiagnosticLog(message: string, logPath?: string): string;

package/dist/utils/ytdlpFailures.js

@@ -0,0 +1,27 @@
+export function classifyYtDlpFailure(stderr) {
+    const normalized = stderr.toLowerCase();
+    if (normalized.includes("cookies are no longer valid") ||
+        normalized.includes("cookie has expired") ||
+        normalized.includes("cookies have expired") ||
+        normalized.includes("session expired")) {
+        return "COOKIES_EXPIRED";
+    }
+    if (normalized.includes("rate-limited by youtube") ||
+        normalized.includes("current session has been rate-limited")) {
+        return "RATE_LIMITED";
+    }
+    if (normalized.includes("sign in to confirm you're not a bot") ||
+        normalized.includes("sign in to confirm you’re not a bot")) {
+        return "SIGN_IN_REQUIRED";
+    }
+    if (normalized.includes("sign in") ||
+        normalized.includes("login") ||
+        normalized.includes("cookies") ||
+        normalized.includes("not a bot")) {
+        return "COOKIES_INVALID";
+    }
+    return null;
+}
+export function appendDiagnosticLog(message, logPath) {
+    return logPath ? `${message}\nDiagnostic log: ${logPath}` : message;
+}

package/dist/utils/ytdlpScheduler.d.ts

@@ -0,0 +1,22 @@
+export interface YtDlpSchedulerState {
+    running: boolean;
+    pendingCount: number;
+    nextAvailableAt: number | null;
+    wouldThrottle: boolean;
+}
+interface YtDlpSchedulerOptions {
+    minIntervalMs?: number;
+    rateLimitBackoffMs?: number;
+    now?: () => number;
+    sleep?: (ms: number) => Promise<void>;
+}
+export interface YtDlpScheduler {
+    schedule<T>(task: () => Promise<T>): Promise<T>;
+    markRateLimited(): void;
+    getState(): YtDlpSchedulerState;
+}
+export declare function createYtDlpScheduler(options?: YtDlpSchedulerOptions): YtDlpScheduler;
+export declare function scheduleYtDlp<T>(task: () => Promise<T>): Promise<T>;
+export declare function markGlobalYtDlpRateLimited(): void;
+export declare function getGlobalYtDlpSchedulerState(): YtDlpSchedulerState;
+export {};

package/dist/utils/ytdlpScheduler.js

@@ -0,0 +1,66 @@
+const DEFAULT_MIN_INTERVAL_MS = 5_000;
+const DEFAULT_RATE_LIMIT_BACKOFF_MS = 10 * 60_000;
+export function createYtDlpScheduler(options = {}) {
+    const now = options.now ?? Date.now;
+    const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    const minIntervalMs = options.minIntervalMs ?? DEFAULT_MIN_INTERVAL_MS;
+    const rateLimitBackoffMs = options.rateLimitBackoffMs ?? DEFAULT_RATE_LIMIT_BACKOFF_MS;
+    let running = false;
+    let pendingCount = 0;
+    let nextAvailableAt = null;
+    let tail = Promise.resolve();
+    async function waitForAvailability() {
+        const currentNext = nextAvailableAt;
+        if (currentNext == null) {
+            return;
+        }
+        const delayMs = currentNext - now();
+        if (delayMs > 0) {
+            await sleep(delayMs);
+        }
+    }
+    return {
+        async schedule(task) {
+            pendingCount += 1;
+            const previousTail = tail;
+            let release;
+            tail = new Promise((resolve) => {
+                release = resolve;
+            });
+            await previousTail;
+            pendingCount -= 1;
+            await waitForAvailability();
+            running = true;
+            try {
+                return await task();
+            }
+            finally {
+                running = false;
+                nextAvailableAt = Math.max(nextAvailableAt ?? 0, now() + minIntervalMs);
+                release();
+            }
+        },
+        markRateLimited() {
+            nextAvailableAt = Math.max(nextAvailableAt ?? 0, now() + rateLimitBackoffMs);
+        },
+        getState() {
+            const next = nextAvailableAt;
+            return {
+                running,
+                pendingCount,
+                nextAvailableAt: next,
+                wouldThrottle: running || pendingCount > 0 || (next != null && next > now()),
+            };
+        },
+    };
+}
+const globalYtDlpScheduler = createYtDlpScheduler();
+export function scheduleYtDlp(task) {
+    return globalYtDlpScheduler.schedule(task);
+}
+export function markGlobalYtDlpRateLimited() {
+    globalYtDlpScheduler.markRateLimited();
+}
+export function getGlobalYtDlpSchedulerState() {
+    return globalYtDlpScheduler.getState();
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mkterswingman/5mghost-yonder",
-  "version": "0.0.12",
+  "version": "0.0.14",
   "description": "Internal MCP client with local data tools and remote API proxy",
   "type": "module",
   "bin": {