mcp-scraper 0.92.1 → 0.93.0

package/dist/index.js

@@ -1,4 +1,4 @@
-import{w as z}from"./chunk-ZMNUI5LB.js";import"./chunk-M5QHXNFZ.js";import"./chunk-4FROKQJN.js";import{h as M}from"./chunk-2SP57VCG.js";import"./chunk-6DTXIZY2.js";import"./chunk-2TZBO52D.js";import"./chunk-RECKUOEF.js";import"./chunk-WJ4XFLS4.js";import{execSync as ce}from"child_process";import{readFileSync as G,writeFileSync as _,unlinkSync as D,mkdirSync as Y}from"fs";import{tmpdir as W}from"os";import{join as l}from"path";import{fal as p}from"@fal-ai/client";var X="https://api.deepinfra.com/v1/openai/chat/completions",L="https://openrouter.ai/api/v1/chat/completions",A="Qwen/Qwen3.6-35B-A3B",Z=`You are a video prompt engineer for Seedance 2.0 text-to-video AI.
+import{w as z}from"./chunk-M5VZVMPC.js";import"./chunk-M5QHXNFZ.js";import"./chunk-SH5KB4P7.js";import{h as M}from"./chunk-2SP57VCG.js";import"./chunk-6DTXIZY2.js";import"./chunk-Y6MKMSOC.js";import"./chunk-WJ4XFLS4.js";import{execSync as ce}from"child_process";import{readFileSync as G,writeFileSync as _,unlinkSync as D,mkdirSync as Y}from"fs";import{tmpdir as W}from"os";import{join as l}from"path";import{fal as p}from"@fal-ai/client";var X="https://api.deepinfra.com/v1/openai/chat/completions",L="https://openrouter.ai/api/v1/chat/completions",A="Qwen/Qwen3.6-35B-A3B",Z=`You are a video prompt engineer for Seedance 2.0 text-to-video AI.
 
 IMPORTANT TECHNICAL CONTEXT: Clip 2 will be generated using image-to-video, meaning it will start from the exact last frame of Clip 1. The two clips will be visually seamless \u2014 same location, same characters, continuous motion. You must write prompts that make this feel like one uninterrupted 16-second video.
 
@@ -20,17 +20,17 @@ Rules for visuals:
 - Describe exactly what the camera sees
 
 Respond with JSON only:
-{"clip1": "...", "clip2": "...", "voiceover": "...", "audioMood": "..."}`;async function B(t,i,e,o){let n=await fetch(i,{method:"POST",headers:{Authorization:`Bearer ${t}`,"Content-Type":"application/json"},body:JSON.stringify({model:A,temperature:.7,messages:[{role:"system",content:Z},{role:"user",content:`Question: ${e}
+{"clip1": "...", "clip2": "...", "voiceover": "...", "audioMood": "..."}`;async function B(i,t,e,o){let n=await fetch(t,{method:"POST",headers:{Authorization:`Bearer ${i}`,"Content-Type":"application/json"},body:JSON.stringify({model:A,temperature:.7,messages:[{role:"system",content:Z},{role:"user",content:`Question: ${e}
 
-Answer: ${o.slice(0,500)}`}]})});if(!n.ok)throw new Error(`LLM call failed (${n.status}): ${await n.text()}`);let r=await n.json();i===L?M({vendor:"openrouter",model:A,units:r.usage?.cost??0,unitType:"usd"}):M({vendor:"deepinfra_qwen",model:A,units:r.usage?.completion_tokens??0,unitType:"tokens"});let a=r.choices[0]?.message?.content?.trim()??"",c=a.match(/\{[\s\S]*\}/);if(!c)throw new Error(`No JSON in QWEN response: ${a.slice(0,200)}`);let s=JSON.parse(c[0]);if(!s.clip1||!s.clip2||!s.voiceover||!s.audioMood)throw new Error(`QWEN response missing fields: ${a.slice(0,200)}`);return s}async function T(t,i){let e=process.env.DEEPINFRA_API_KEY,o=process.env.OPENROUTER_API_KEY;if(e)try{return await B(e,X,t,i)}catch(n){console.warn("[promptBuilder] DeepInfra failed, trying OpenRouter:",n.message)}if(o)return await B(o,L,t,i);throw new Error("No LLM key \u2014 set DEEPINFRA_API_KEY or OPENROUTER_API_KEY")}function V(t){if(!t.clip1||!t.clip2||!t.voiceover||!t.audioMood)throw new Error("Episode brief is missing prompt fields \u2014 run blog-to-video skill to regenerate");return{clip1:t.clip1,clip2:t.clip2,voiceover:t.voiceover,audioMood:t.audioMood}}import{writeFileSync as U,mkdirSync as ee}from"fs";import{join as F}from"path";import{tmpdir as oe}from"os";import{fal as x}from"@fal-ai/client";var ie="fal-ai/mmaudio-v2",te="fal-ai/elevenlabs/tts",re="fal-ai/google/gemini-2.5-flash-preview-tts";async function K(t){let i=await fetch(t);if(!i.ok)throw new Error(`Failed to download TTS audio (${i.status})`);return Buffer.from(await i.arrayBuffer())}async function O(t){console.log("[AudioGenerator] Generating voiceover...");let i=F(oe(),`tts-${Date.now()}`);ee(i,{recursive:!0});let e=F(i,"voiceover.mp3");try{let r=process.env.ELEVENLABS_VOICE_ID??"pNInz6obpgDQGcFmaJgB",a=await x.run(te,{input:{text:t,voice_id:r,model_id:"eleven_v3"}});return U(e,await K(a.audio.url)),console.log("[AudioGenerator] TTS: ElevenLabs via fal"),e}catch(r){console.warn("[AudioGenerator] ElevenLabs via fal failed, trying Gemini:",r.message)}let o=process.env.GEMINI_TTS_VOICE??"Kore",n=await x.run(re,{input:{text:t,voice:o}});return U(e,await K(n.audio.url)),console.log("[AudioGenerator] TTS: Gemini via fal"),e}async function $(t,i,e){return console.log("[AudioGenerator] Adding background audio via MMAudio V2..."),(await x.run(ie,{input:{video_url:t,prompt:i,negative_prompt:"speech, voice, talking, dialogue, narration, vocals, singing, human voice, conversation, words, lyrics, announcer, commentary",duration:e,cfg_strength:4.5}})).video.url}import{execSync as j}from"child_process";import{writeFileSync as ne,mkdirSync as ae}from"fs";import{join as u}from"path";import{fal as se}from"@fal-ai/client";async function I(t,i){let e=await fetch(t);if(!e.ok)throw new Error(`Download failed (${e.status}): ${t}`);ne(i,Buffer.from(await e.arrayBuffer()))}async function N(t,i,e){ae(e,{recursive:!0});let o=Date.now(),n=u(e,`clip1-${o}.mp4`),r=u(e,`clip2-${o}.mp4`),a=u(e,`combined-${o}.mp4`);return console.log("[VideoMixer] Downloading clips..."),await Promise.all([I(t,n),I(i,r)]),console.log("[VideoMixer] Concatenating..."),j(`ffmpeg -i "${n}" -i "${r}" -filter_complex "[0:v][1:v]concat=n=2:v=1:a=0[v]" -map "[v]" -y "${a}" -loglevel error`),a}async function b(t){let{readFileSync:i}=await import("fs"),e=new Blob([i(t)],{type:"video/mp4"}),o=await se.storage.upload(e);return console.log("[VideoMixer] Uploaded to fal:",o),o}async function R(t,i,e){let o=Date.now(),n=u(e,`voiceover-${o}.wav`),r=u(e,`final-${o}.mp4`);return console.log("[VideoMixer] Downloading voiceover..."),await I(i,n),console.log("[VideoMixer] Mixing voiceover over background audio..."),j(`ffmpeg -i "${t}" -i "${n}" -filter_complex "[0:a]volume=0.2[bg];[1:a]volume=1.0[vo];[bg][vo]amix=inputs=2:duration=first[a]" -map 0:v -map "[a]" -c:v copy -y "${r}" -loglevel error`),r}var q="bytedance/seedance-2.0/text-to-video",J="bytedance/seedance-2.0/image-to-video";function v(t,i,e,o){return{prompt:t,resolution:i.resolution??"720p",duration:i.clipDurationSeconds??8,aspect_ratio:i.aspectRatio??"16:9",generate_audio:!1,...e!==void 0?{seed:e}:{},...o!==void 0?{image_url:o}:{}}}async function w(t,i){let{request_id:e}=await p.queue.submit(t,{input:i});for(console.log(`[fal] submitted ${t} \u2192 ${e}`);;){await new Promise(r=>setTimeout(r,5e3));let o=await p.queue.status(t,{requestId:e,logs:!1});if(console.log(`[fal] ${e} \u2192 ${o.status}`),o.status==="FAILED")throw new Error(`Request ${e} failed`);if(o.status!=="COMPLETED")continue;return(await p.queue.result(t,{requestId:e})).data}}async function H(t,i){let e=Date.now(),o=l(i,`clip1-raw-${e}.mp4`),n=l(i,`last-frame-${e}.jpg`),r=await fetch(t);if(!r.ok)throw new Error(`Failed to download clip 1 (${r.status})`);_(o,Buffer.from(await r.arrayBuffer()));try{ce(`ffmpeg -sseof -0.1 -i "${o}" -vframes 1 -y "${n}" -loglevel error`)}finally{try{D(o)}catch{}}return n}var k=class{constructor(i){let e=i??process.env.FAL_KEY;if(!e)throw new Error("FAL_KEY is required");p.config({credentials:e})}async generateClipPair(i,e,o={}){let n=o.outputDir??l(W(),`paa-video-${Date.now()}`);Y(n,{recursive:!0}),console.log(`
-[1/7] Generating prompts via QWEN 3.6...`);let r=await T(i,e);console.log("  Voiceover:",r.voiceover),console.log("  Audio mood:",r.audioMood),console.log(`
+Answer: ${o.slice(0,500)}`}]})});if(!n.ok)throw new Error(`LLM call failed (${n.status}): ${await n.text()}`);let r=await n.json();t===L?M({vendor:"openrouter",model:A,units:r.usage?.cost??0,unitType:"usd"}):M({vendor:"deepinfra_qwen",model:A,units:r.usage?.completion_tokens??0,unitType:"tokens"});let a=r.choices[0]?.message?.content?.trim()??"",c=a.match(/\{[\s\S]*\}/);if(!c)throw new Error(`No JSON in QWEN response: ${a.slice(0,200)}`);let s=JSON.parse(c[0]);if(!s.clip1||!s.clip2||!s.voiceover||!s.audioMood)throw new Error(`QWEN response missing fields: ${a.slice(0,200)}`);return s}async function T(i,t){let e=process.env.DEEPINFRA_API_KEY,o=process.env.OPENROUTER_API_KEY;if(e)try{return await B(e,X,i,t)}catch(n){console.warn("[promptBuilder] DeepInfra failed, trying OpenRouter:",n.message)}if(o)return await B(o,L,i,t);throw new Error("No LLM key \u2014 set DEEPINFRA_API_KEY or OPENROUTER_API_KEY")}function V(i){if(!i.clip1||!i.clip2||!i.voiceover||!i.audioMood)throw new Error("Episode brief is missing prompt fields \u2014 run blog-to-video skill to regenerate");return{clip1:i.clip1,clip2:i.clip2,voiceover:i.voiceover,audioMood:i.audioMood}}import{writeFileSync as U,mkdirSync as ee}from"fs";import{join as F}from"path";import{tmpdir as oe}from"os";import{fal as x}from"@fal-ai/client";var te="fal-ai/mmaudio-v2",ie="fal-ai/elevenlabs/tts",re="fal-ai/google/gemini-2.5-flash-preview-tts";async function j(i){let t=await fetch(i);if(!t.ok)throw new Error(`Failed to download TTS audio (${t.status})`);return Buffer.from(await t.arrayBuffer())}async function O(i){console.log("[AudioGenerator] Generating voiceover...");let t=F(oe(),`tts-${Date.now()}`);ee(t,{recursive:!0});let e=F(t,"voiceover.mp3");try{let r=process.env.ELEVENLABS_VOICE_ID??"pNInz6obpgDQGcFmaJgB",a=await x.run(ie,{input:{text:i,voice_id:r,model_id:"eleven_v3"}});return U(e,await j(a.audio.url)),console.log("[AudioGenerator] TTS: ElevenLabs via fal"),e}catch(r){console.warn("[AudioGenerator] ElevenLabs via fal failed, trying Gemini:",r.message)}let o=process.env.GEMINI_TTS_VOICE??"Kore",n=await x.run(re,{input:{text:i,voice:o}});return U(e,await j(n.audio.url)),console.log("[AudioGenerator] TTS: Gemini via fal"),e}async function $(i,t,e){return console.log("[AudioGenerator] Adding background audio via MMAudio V2..."),(await x.run(te,{input:{video_url:i,prompt:t,negative_prompt:"speech, voice, talking, dialogue, narration, vocals, singing, human voice, conversation, words, lyrics, announcer, commentary",duration:e,cfg_strength:4.5}})).video.url}import{execSync as K}from"child_process";import{writeFileSync as ne,mkdirSync as ae}from"fs";import{join as u}from"path";import{fal as se}from"@fal-ai/client";async function I(i,t){let e=await fetch(i);if(!e.ok)throw new Error(`Download failed (${e.status}): ${i}`);ne(t,Buffer.from(await e.arrayBuffer()))}async function N(i,t,e){ae(e,{recursive:!0});let o=Date.now(),n=u(e,`clip1-${o}.mp4`),r=u(e,`clip2-${o}.mp4`),a=u(e,`combined-${o}.mp4`);return console.log("[VideoMixer] Downloading clips..."),await Promise.all([I(i,n),I(t,r)]),console.log("[VideoMixer] Concatenating..."),K(`ffmpeg -i "${n}" -i "${r}" -filter_complex "[0:v][1:v]concat=n=2:v=1:a=0[v]" -map "[v]" -y "${a}" -loglevel error`),a}async function b(i){let{readFileSync:t}=await import("fs"),e=new Blob([t(i)],{type:"video/mp4"}),o=await se.storage.upload(e);return console.log("[VideoMixer] Uploaded to fal:",o),o}async function R(i,t,e){let o=Date.now(),n=u(e,`voiceover-${o}.wav`),r=u(e,`final-${o}.mp4`);return console.log("[VideoMixer] Downloading voiceover..."),await I(t,n),console.log("[VideoMixer] Mixing voiceover over background audio..."),K(`ffmpeg -i "${i}" -i "${n}" -filter_complex "[0:a]volume=0.2[bg];[1:a]volume=1.0[vo];[bg][vo]amix=inputs=2:duration=first[a]" -map 0:v -map "[a]" -c:v copy -y "${r}" -loglevel error`),r}var q="bytedance/seedance-2.0/text-to-video",J="bytedance/seedance-2.0/image-to-video";function v(i,t,e,o){return{prompt:i,resolution:t.resolution??"720p",duration:t.clipDurationSeconds??8,aspect_ratio:t.aspectRatio??"16:9",generate_audio:!1,...e!==void 0?{seed:e}:{},...o!==void 0?{image_url:o}:{}}}async function w(i,t){let{request_id:e}=await p.queue.submit(i,{input:t});for(console.log(`[fal] submitted ${i} \u2192 ${e}`);;){await new Promise(r=>setTimeout(r,5e3));let o=await p.queue.status(i,{requestId:e,logs:!1});if(console.log(`[fal] ${e} \u2192 ${o.status}`),o.status==="FAILED")throw new Error(`Request ${e} failed`);if(o.status!=="COMPLETED")continue;return(await p.queue.result(i,{requestId:e})).data}}async function H(i,t){let e=Date.now(),o=l(t,`clip1-raw-${e}.mp4`),n=l(t,`last-frame-${e}.jpg`),r=await fetch(i);if(!r.ok)throw new Error(`Failed to download clip 1 (${r.status})`);_(o,Buffer.from(await r.arrayBuffer()));try{ce(`ffmpeg -sseof -0.1 -i "${o}" -vframes 1 -y "${n}" -loglevel error`)}finally{try{D(o)}catch{}}return n}var k=class{constructor(t){let e=t??process.env.FAL_KEY;if(!e)throw new Error("FAL_KEY is required");p.config({credentials:e})}async generateClipPair(t,e,o={}){let n=o.outputDir??l(W(),`paa-video-${Date.now()}`);Y(n,{recursive:!0}),console.log(`
+[1/7] Generating prompts via QWEN 3.6...`);let r=await T(t,e);console.log("  Voiceover:",r.voiceover),console.log("  Audio mood:",r.audioMood),console.log(`
 [2/7] Generating clip 1 (text-to-video)...`);let a=await w(q,v(r.clip1,o,o.seed));console.log(`
 [3/7] Extracting last frame \u2192 clip 2 start...`);let c=await H(a.video.url,n),s=new Blob([G(c)],{type:"image/jpeg"}),h=await p.storage.upload(s);try{D(c)}catch{}console.log(`
 [4/7] Generating clip 2 (image-to-video from last frame)...`);let d=o.seed!==void 0?o.seed+1:void 0,m=await w(J,v(r.clip2,o,d,h));console.log(`
 [5/7] Concatenating clips + generating voiceover (parallel)...`);let[P,y]=await Promise.all([N(a.video.url,m.video.url,n),O(r.voiceover)]);console.log(`
 [6/7] Adding background audio via MMAudio V2...`);let E=await b(P),S=(o.clipDurationSeconds??8)*2,g=await $(E,r.audioMood,S);console.log(`
-[7/7] Overlaying voiceover on final video...`);let f=l(n,`with-bg-audio-${Date.now()}.mp4`),C=await fetch(g);_(f,Buffer.from(await C.arrayBuffer()));let Q=await R(f,y,n);return{clip1Url:a.video.url,clip2Url:m.video.url,finalVideoPath:Q,seed:a.seed,promptClip1:r.clip1,promptClip2:r.clip2,voiceover:r.voiceover,audioMood:r.audioMood}}async generateEpisode(i,e={}){let o=e.outputDir??l(W(),`episode-${i.episodeNumber}-${Date.now()}`);Y(o,{recursive:!0});let n=V(i);console.log(`
-[Episode ${i.episodeNumber}/${i.episodeCount}] ${i.sectionTitle}`),console.log("  Voiceover:",n.voiceover),console.log(`
+[7/7] Overlaying voiceover on final video...`);let f=l(n,`with-bg-audio-${Date.now()}.mp4`),C=await fetch(g);_(f,Buffer.from(await C.arrayBuffer()));let Q=await R(f,y,n);return{clip1Url:a.video.url,clip2Url:m.video.url,finalVideoPath:Q,seed:a.seed,promptClip1:r.clip1,promptClip2:r.clip2,voiceover:r.voiceover,audioMood:r.audioMood}}async generateEpisode(t,e={}){let o=e.outputDir??l(W(),`episode-${t.episodeNumber}-${Date.now()}`);Y(o,{recursive:!0});let n=V(t);console.log(`
+[Episode ${t.episodeNumber}/${t.episodeCount}] ${t.sectionTitle}`),console.log("  Voiceover:",n.voiceover),console.log(`
 [2/7] Generating clip 1 (text-to-video)...`);let r=await w(q,v(n.clip1,e,e.seed));console.log(`
 [3/7] Extracting last frame \u2192 clip 2 start...`);let a=await H(r.video.url,o),c=new Blob([G(a)],{type:"image/jpeg"}),s=await p.storage.upload(c);try{D(a)}catch{}console.log(`
 [4/7] Generating clip 2 (image-to-video from last frame)...`);let h=e.seed!==void 0?e.seed+1:void 0,d=await w(J,v(n.clip2,e,h,s));console.log(`