@mkterswingman/5mghost-yonder 0.0.13 → 0.0.14

package/dist/download/downloader.d.ts

@@ -1,6 +1,7 @@
 import type { DownloadJobManager } from "./jobManager.js";
 import type { DownloadJobItemSnapshot, DownloadMode } from "./types.js";
 import type { PreflightVideoMetadata } from "../utils/videoMetadata.js";
+import { type YtDlpSchedulerState } from "../utils/ytdlpScheduler.js";
 type DownloadingStep = "download_video" | "download_subtitles" | "merge";
 export interface DownloadOneItemProgressUpdate {
     step: DownloadingStep;
@@ -20,7 +21,7 @@ export interface DownloadOneItemInput {
     subtitleFormats: string[];
     subtitleLanguages?: string[];
     defaultSubtitleLanguages?: string[];
-    jobManager?: Pick<DownloadJobManager, "startItem" | "updateItemProgress" | "completeItem" | "failItem">;
+    jobManager?: Pick<DownloadJobManager, "startItem" | "updateItemProgress" | "completeItem" | "failItem" | "setItemQueuedByGlobalRateLimit">;
     jobId?: string;
     deps?: Partial<DownloadOneItemDeps>;
 }
@@ -39,6 +40,7 @@ interface DownloadOneItemDeps {
         onProgress: (update: DownloadOneItemProgressUpdate) => void;
     }) => Promise<string>;
     refreshCookies: () => Promise<boolean>;
+    getYtDlpSchedulerState: () => YtDlpSchedulerState;
     downloadSubtitles: (input: {
         videoId: string;
         sourceUrl: string;

package/dist/download/downloader.js

@@ -8,6 +8,7 @@ import { formatBytes } from "../utils/formatters.js";
 import { hasYtDlp, runYtDlp, runYtDlpJson } from "../utils/ytdlp.js";
 import { appendDiagnosticLog, classifyYtDlpFailure } from "../utils/ytdlpFailures.js";
 import { parseYtDlpProgressLine } from "../utils/ytdlpProgress.js";
+import { getGlobalYtDlpSchedulerState } from "../utils/ytdlpScheduler.js";
 export async function downloadOneItem(input) {
     const deps = resolveDeps(input.deps);
     const manager = input.jobManager;
@@ -52,6 +53,7 @@ export async function downloadOneItem(input) {
         if (needsVideo(input.mode)) {
             currentStep = "download_video";
             manager?.updateItemProgress(jobId, input.item.video_id, currentStep, 0);
+            manager?.setItemQueuedByGlobalRateLimit(jobId, input.item.video_id, deps.getYtDlpSchedulerState().wouldThrottle);
             try {
                 result.video_file = await deps.downloadVideo({
                     sourceUrl: input.item.source_url,
@@ -74,20 +76,29 @@ export async function downloadOneItem(input) {
                     throw error;
                 }
             }
+            finally {
+                manager?.setItemQueuedByGlobalRateLimit(jobId, input.item.video_id, false);
+            }
             result.final_file_size = formatBytes((await deps.stat(result.video_file)).size);
         }
         if (needsSubtitles(input.mode)) {
             currentStep = "download_subtitles";
             manager?.updateItemProgress(jobId, input.item.video_id, currentStep, 0);
-            result.subtitle_files = await deps.downloadSubtitles({
-                videoId: input.item.video_id,
-                sourceUrl: input.item.source_url,
-                subtitlesDir,
-                formats: subtitleFormats,
-                languages: subtitleLanguages,
-                skipMissingLanguages: usingDefaultSubtitleLanguages,
-                onProgress,
-            });
+            manager?.setItemQueuedByGlobalRateLimit(jobId, input.item.video_id, deps.getYtDlpSchedulerState().wouldThrottle);
+            try {
+                result.subtitle_files = await deps.downloadSubtitles({
+                    videoId: input.item.video_id,
+                    sourceUrl: input.item.source_url,
+                    subtitlesDir,
+                    formats: subtitleFormats,
+                    languages: subtitleLanguages,
+                    skipMissingLanguages: usingDefaultSubtitleLanguages,
+                    onProgress,
+                });
+            }
+            finally {
+                manager?.setItemQueuedByGlobalRateLimit(jobId, input.item.video_id, false);
+            }
         }
         currentStep = "write_metadata";
         manager?.updateItemProgress(jobId, input.item.video_id, currentStep, 100);
@@ -122,6 +133,7 @@ function resolveDeps(overrides) {
         fetchMetadata: fetchMetadataWithYtDlp,
         downloadVideo: downloadVideoWithYtDlp,
         refreshCookies: tryHeadlessRefresh,
+        getYtDlpSchedulerState: getGlobalYtDlpSchedulerState,
         downloadSubtitles: downloadSubtitlesWithYtDlp,
         mkdir: async (path, options) => {
             await mkdir(path, options);

package/dist/download/jobManager.d.ts

@@ -10,6 +10,7 @@ export declare class DownloadJobManager {
     pollJob(jobId: string): DownloadJobSnapshot;
     startItem(jobId: string, videoId: string, step?: DownloadStep): DownloadJobSnapshot;
     updateItemProgress(jobId: string, videoId: string, step: DownloadStep, progressPercent: number | null): DownloadJobSnapshot;
+    setItemQueuedByGlobalRateLimit(jobId: string, videoId: string, queued: boolean): DownloadJobSnapshot;
     completeItem(jobId: string, videoId: string, outputs?: Pick<DownloadJobItemSnapshot, "video_file" | "metadata_file" | "final_file_size" | "subtitle_files">): DownloadJobSnapshot;
     failItem(jobId: string, videoId: string, error: string, step?: DownloadStep): DownloadJobSnapshot;
     private applyItemMutation;

package/dist/download/jobManager.js

@@ -26,6 +26,7 @@ export class DownloadJobManager {
                 title: video.title,
                 output_dir: video.output_dir,
                 status: "queued",
+                queued_by_global_rate_limit: false,
                 step: null,
                 progress_percent: null,
                 error: null,
@@ -65,6 +66,12 @@ export class DownloadJobManager {
         });
         return this.toSnapshot(job);
     }
+    setItemQueuedByGlobalRateLimit(jobId, videoId, queued) {
+        const job = this.getJob(jobId);
+        const item = this.getItem(job, videoId);
+        item.queued_by_global_rate_limit = queued;
+        return this.toSnapshot(job);
+    }
     completeItem(jobId, videoId, outputs) {
         const job = this.getJob(jobId);
         const item = this.getItem(job, videoId);
@@ -96,6 +103,10 @@ export class DownloadJobManager {
         const nextStep = mutation.step === undefined ? item.step : mutation.step;
         const previousStatus = item.status;
         item.status = nextStatus;
+        item.queued_by_global_rate_limit =
+            mutation.queued_by_global_rate_limit === undefined
+                ? item.queued_by_global_rate_limit
+                : mutation.queued_by_global_rate_limit;
         item.step = nextStep;
         item.video_file = mutation.video_file === undefined ? item.video_file : mutation.video_file;
         item.metadata_file = mutation.metadata_file === undefined ? item.metadata_file : mutation.metadata_file;
@@ -107,6 +118,7 @@ export class DownloadJobManager {
             job.started_at = job.started_at ?? now;
             item.completed_at = null;
             item.error = null;
+            item.queued_by_global_rate_limit = false;
             item.progress_percent = resolveRunningProgress(item.progress_percent, mutation.progress_percent, previousStatus);
         }
         if (nextStatus === "completed" || nextStatus === "failed") {
@@ -114,6 +126,7 @@ export class DownloadJobManager {
             job.started_at = job.started_at ?? now;
             item.completed_at = now;
             item.error = mutation.error === undefined ? item.error : mutation.error;
+            item.queued_by_global_rate_limit = false;
             item.progress_percent =
                 mutation.progress_percent === undefined ? item.progress_percent : clampProgressOrNull(mutation.progress_percent);
         }

package/dist/download/types.d.ts

@@ -19,6 +19,7 @@ export interface DownloadJobItemSnapshot {
     title: string;
     output_dir: string;
     status: DownloadItemStatus;
+    queued_by_global_rate_limit: boolean;
     step: DownloadStep | null;
     progress_percent: number | null;
     error: string | null;

package/dist/utils/ytdlp.js

@@ -4,6 +4,8 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { PATHS } from "./config.js";
 import { getYtDlpPath } from "./ytdlpPath.js";
+import { markGlobalYtDlpRateLimited, scheduleYtDlp } from "./ytdlpScheduler.js";
+import { classifyYtDlpFailure } from "./ytdlpFailures.js";
 function writeYtDlpFailureLog(payload) {
     try {
         mkdirSync(PATHS.logsDir, { recursive: true });
@@ -84,7 +86,7 @@ export function buildYtDlpArgs(args, options = {}) {
     return finalArgs;
 }
 export function runYtDlp(args, timeoutMs = 45_000, onStderrLine) {
-    return new Promise((resolve, reject) => {
+    return scheduleYtDlp(() => new Promise((resolve, reject) => {
         const start = Date.now();
         const finalArgs = buildYtDlpArgs(args);
         const proc = spawn(getYtDlpPath(), finalArgs, {
@@ -153,6 +155,9 @@ export function runYtDlp(args, timeoutMs = 45_000, onStderrLine) {
                             trigger: "exit",
                         }),
                 });
+                if (exitCode !== 0 && classifyYtDlpFailure(stderr) === "RATE_LIMITED") {
+                    markGlobalYtDlpRateLimited();
+                }
             }
         });
         proc.on("error", (err) => {
@@ -175,7 +180,7 @@ export function runYtDlp(args, timeoutMs = 45_000, onStderrLine) {
                 }));
             }
         });
-    });
+    }));
 }
 export async function runYtDlpJson(args, timeoutMs = 45_000) {
     const result = await runYtDlp(args, timeoutMs);

package/dist/utils/ytdlpScheduler.d.ts

@@ -0,0 +1,22 @@
+export interface YtDlpSchedulerState {
+    running: boolean;
+    pendingCount: number;
+    nextAvailableAt: number | null;
+    wouldThrottle: boolean;
+}
+interface YtDlpSchedulerOptions {
+    minIntervalMs?: number;
+    rateLimitBackoffMs?: number;
+    now?: () => number;
+    sleep?: (ms: number) => Promise<void>;
+}
+export interface YtDlpScheduler {
+    schedule<T>(task: () => Promise<T>): Promise<T>;
+    markRateLimited(): void;
+    getState(): YtDlpSchedulerState;
+}
+export declare function createYtDlpScheduler(options?: YtDlpSchedulerOptions): YtDlpScheduler;
+export declare function scheduleYtDlp<T>(task: () => Promise<T>): Promise<T>;
+export declare function markGlobalYtDlpRateLimited(): void;
+export declare function getGlobalYtDlpSchedulerState(): YtDlpSchedulerState;
+export {};

package/dist/utils/ytdlpScheduler.js

@@ -0,0 +1,66 @@
+const DEFAULT_MIN_INTERVAL_MS = 5_000;
+const DEFAULT_RATE_LIMIT_BACKOFF_MS = 10 * 60_000;
+export function createYtDlpScheduler(options = {}) {
+    const now = options.now ?? Date.now;
+    const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    const minIntervalMs = options.minIntervalMs ?? DEFAULT_MIN_INTERVAL_MS;
+    const rateLimitBackoffMs = options.rateLimitBackoffMs ?? DEFAULT_RATE_LIMIT_BACKOFF_MS;
+    let running = false;
+    let pendingCount = 0;
+    let nextAvailableAt = null;
+    let tail = Promise.resolve();
+    async function waitForAvailability() {
+        const currentNext = nextAvailableAt;
+        if (currentNext == null) {
+            return;
+        }
+        const delayMs = currentNext - now();
+        if (delayMs > 0) {
+            await sleep(delayMs);
+        }
+    }
+    return {
+        async schedule(task) {
+            pendingCount += 1;
+            const previousTail = tail;
+            let release;
+            tail = new Promise((resolve) => {
+                release = resolve;
+            });
+            await previousTail;
+            pendingCount -= 1;
+            await waitForAvailability();
+            running = true;
+            try {
+                return await task();
+            }
+            finally {
+                running = false;
+                nextAvailableAt = Math.max(nextAvailableAt ?? 0, now() + minIntervalMs);
+                release();
+            }
+        },
+        markRateLimited() {
+            nextAvailableAt = Math.max(nextAvailableAt ?? 0, now() + rateLimitBackoffMs);
+        },
+        getState() {
+            const next = nextAvailableAt;
+            return {
+                running,
+                pendingCount,
+                nextAvailableAt: next,
+                wouldThrottle: running || pendingCount > 0 || (next != null && next > now()),
+            };
+        },
+    };
+}
+const globalYtDlpScheduler = createYtDlpScheduler();
+export function scheduleYtDlp(task) {
+    return globalYtDlpScheduler.schedule(task);
+}
+export function markGlobalYtDlpRateLimited() {
+    globalYtDlpScheduler.markRateLimited();
+}
+export function getGlobalYtDlpSchedulerState() {
+    return globalYtDlpScheduler.getState();
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mkterswingman/5mghost-yonder",
-  "version": "0.0.13",
+  "version": "0.0.14",
   "description": "Internal MCP client with local data tools and remote API proxy",
   "type": "module",
   "bin": {