mcp-scraper 0.88.2 → 0.89.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/CHANGELOG.md +36 -15
  2. package/README.md +6 -5
  3. package/THIRD_PARTY_NOTICES.html +203 -0
  4. package/dist/analytics-repository-2BMT5JNE.js +1 -0
  5. package/dist/bin/api-server.js +2 -41
  6. package/dist/bin/mcp-scraper-cli.js +39 -756
  7. package/dist/bin/mcp-scraper-core.js +1 -60
  8. package/dist/bin/mcp-scraper-install.js +2 -25
  9. package/dist/bin/mcp-stdio-server.js +1 -19
  10. package/dist/bin/paa-harvest.js +1 -41
  11. package/dist/chunk-3GP5CYZX.js +1 -0
  12. package/dist/chunk-4AI7DOS7.js +59 -0
  13. package/dist/chunk-4FROKQJN.js +1 -0
  14. package/dist/chunk-7YGVI5J4.js +21710 -0
  15. package/dist/chunk-CCYWSJNG.js +16 -0
  16. package/dist/chunk-CFI6CXIV.js +182 -0
  17. package/dist/chunk-E2WRWV3A.js +1 -0
  18. package/dist/chunk-GMWKPIYX.js +1172 -0
  19. package/dist/chunk-HDPYG3XV.js +102 -0
  20. package/dist/chunk-HE45FFBU.js +1 -0
  21. package/dist/chunk-HUV2WTRW.js +1 -0
  22. package/dist/chunk-KJQXUZ4Y.js +4 -0
  23. package/dist/chunk-L4CGLFPU.js +4 -0
  24. package/dist/chunk-M22MM4N4.js +84 -0
  25. package/dist/chunk-MASR22K4.js +73 -0
  26. package/dist/chunk-MZN4U5BL.js +1 -0
  27. package/dist/chunk-PUHFVA7P.js +1280 -0
  28. package/dist/chunk-QPWPR5XG.js +10 -0
  29. package/dist/chunk-TMB56NCA.js +1 -0
  30. package/dist/chunk-TXENITMS.js +20 -0
  31. package/dist/chunk-W2BVJ7S2.js +13 -0
  32. package/dist/chunk-WO3N5FH2.js +5 -0
  33. package/dist/chunk-WSCGYRWA.js +2595 -0
  34. package/dist/chunk-X54CQLK2.js +1 -0
  35. package/dist/chunk-XLWNEVUZ.js +27 -0
  36. package/dist/chunk-XPZVJIZ2.js +100 -0
  37. package/dist/chunk-YQZGZBB4.js +1 -0
  38. package/dist/chunk-Z2QGQJS2.js +1 -0
  39. package/dist/db-F2MX63GI.js +1 -0
  40. package/dist/extract-bundle-SNUIHM3J.js +26 -0
  41. package/dist/gmail-service-BZ3H75XC.js +1 -0
  42. package/dist/index.cjs +21750 -6045
  43. package/dist/index.d.cts +14 -14
  44. package/dist/index.d.ts +14 -14
  45. package/dist/index.js +18 -315
  46. package/dist/lead-list-enrichment-repository-S2H3U7T7.js +1 -0
  47. package/dist/location-data-repository-OTWHWMV6.js +1 -0
  48. package/dist/server-RFR2A5UJ.js +7303 -0
  49. package/dist/site-extract-repository-SE776XDC.js +1 -0
  50. package/dist/worker-XUDSM3AL.js +1 -0
  51. package/package.json +17 -124
  52. package/dist/analytics-repository-GGJJCVVP.js +0 -194
  53. package/dist/chunk-4QMUF6XM.js +0 -1013
  54. package/dist/chunk-6HAV7LCE.js +0 -265
  55. package/dist/chunk-ABF2CGOZ.js +0 -113
  56. package/dist/chunk-C5Z4OFKW.js +0 -404
  57. package/dist/chunk-DNM65UCK.js +0 -299
  58. package/dist/chunk-EQGTEHLZ.js +0 -592
  59. package/dist/chunk-F5GQJWZU.js +0 -732
  60. package/dist/chunk-GGZEC22A.js +0 -215
  61. package/dist/chunk-GXBZXWXB.js +0 -184
  62. package/dist/chunk-IHXAXYIS.js +0 -843
  63. package/dist/chunk-K3Z5AQYE.js +0 -683
  64. package/dist/chunk-K45K75OF.js +0 -6
  65. package/dist/chunk-LFW2FRPJ.js +0 -224
  66. package/dist/chunk-MZDNZQWT.js +0 -2078
  67. package/dist/chunk-NVUKO5NN.js +0 -256
  68. package/dist/chunk-OM7HVEJ3.js +0 -26
  69. package/dist/chunk-OPQIGAFB.js +0 -286
  70. package/dist/chunk-OZJMVCDK.js +0 -16
  71. package/dist/chunk-P7FWOMU7.js +0 -505
  72. package/dist/chunk-PGJQDMC2.js +0 -383
  73. package/dist/chunk-PKZS6SHW.js +0 -33139
  74. package/dist/chunk-RJ7JVYKU.js +0 -68
  75. package/dist/chunk-S24LFPL7.js +0 -5262
  76. package/dist/chunk-T3MZISOF.js +0 -240
  77. package/dist/chunk-UZPTGUDV.js +0 -1915
  78. package/dist/chunk-X623GTBV.js +0 -8290
  79. package/dist/chunk-YXNDOQXN.js +0 -4018
  80. package/dist/db-Z34LPZNR.js +0 -284
  81. package/dist/extract-bundle-565SBZCR.js +0 -1003
  82. package/dist/gmail-service-E6ALS7JG.js +0 -25
  83. package/dist/lead-list-enrichment-repository-36RPVV6N.js +0 -67
  84. package/dist/location-data-repository-WPRG62GE.js +0 -34
  85. package/dist/server-SQZ3A7SY.js +0 -86606
  86. package/dist/site-extract-repository-VYFZASPU.js +0 -69
  87. package/dist/worker-LDCAULWL.js +0 -146
package/dist/index.d.cts CHANGED
@@ -24,13 +24,13 @@ declare const HarvestOptionsSchema: z.ZodObject<{
24
24
  headless: z.ZodDefault<z.ZodBoolean>;
25
25
  profileDir: z.ZodOptional<z.ZodString>;
26
26
  proxy: z.ZodOptional<z.ZodString>;
27
- kernelApiKey: z.ZodOptional<z.ZodString>;
28
- kernelProxyId: z.ZodOptional<z.ZodString>;
29
- kernelProfileName: z.ZodOptional<z.ZodString>;
30
- kernelProfileSaveChanges: z.ZodOptional<z.ZodBoolean>;
31
- kernelStealth: z.ZodOptional<z.ZodBoolean>;
27
+ \u006b\u0065\u0072\u006e\u0065\u006cApiKey: z.ZodOptional<z.ZodString>;
28
+ \u006b\u0065\u0072\u006e\u0065\u006cProxyId: z.ZodOptional<z.ZodString>;
29
+ \u006b\u0065\u0072\u006e\u0065\u006cProfileName: z.ZodOptional<z.ZodString>;
30
+ \u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges: z.ZodOptional<z.ZodBoolean>;
31
+ \u006b\u0065\u0072\u006e\u0065\u006cStealth: z.ZodOptional<z.ZodBoolean>;
32
32
  serpIdentity: z.ZodOptional<z.ZodString>;
33
- kernelProxyResolution: z.ZodOptional<z.ZodUnknown>;
33
+ \u006b\u0065\u0072\u006e\u0065\u006cProxyResolution: z.ZodOptional<z.ZodUnknown>;
34
34
  outputDir: z.ZodDefault<z.ZodString>;
35
35
  format: z.ZodDefault<z.ZodEnum<{
36
36
  json: "json";
@@ -60,7 +60,7 @@ declare const HarvestOptionsSchema: z.ZodObject<{
60
60
  type CanonicalLocationName = string;
61
61
  type SerpDevice = 'desktop' | 'mobile';
62
62
  type ProxyMode = 'location' | 'configured' | 'none';
63
- interface KernelLocationProxyTarget {
63
+ interface \u004b\u0065\u0072\u006e\u0065\u006cLocationProxyTarget {
64
64
  canonicalLocation: CanonicalLocationName;
65
65
  level: 'zip' | 'city' | 'state';
66
66
  country: 'US';
@@ -76,14 +76,14 @@ interface KernelLocationProxyTarget {
76
76
  zip?: string;
77
77
  };
78
78
  }
79
- type KernelProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'isp_created' | 'backup_custom' | 'configured_fallback' | 'unavailable';
80
- interface KernelProxyResolutionDebug {
81
- source: KernelProxyResolutionSource;
79
+ type \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'isp_created' | 'backup_custom' | 'configured_fallback' | 'unavailable';
80
+ interface \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug {
81
+ source: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionSource;
82
82
  proxyType: string;
83
83
  proxyMode: ProxyMode;
84
84
  proxyIdPresent: boolean;
85
85
  proxyIdSuffix: string | null;
86
- target: KernelLocationProxyTarget | null;
86
+ target: \u004b\u0065\u0072\u006e\u0065\u006cLocationProxyTarget | null;
87
87
  error: string | null;
88
88
  disposable?: boolean;
89
89
  }
@@ -249,7 +249,7 @@ interface BrowserNetworkLocationDebug {
249
249
  timezone: string | null;
250
250
  error: string | null;
251
251
  }
252
- interface KernelSessionDebug {
252
+ interface \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug {
253
253
  sessionId: string | null;
254
254
  proxyMode: ProxyMode;
255
255
  requestedProxyIdPresent: boolean;
@@ -261,7 +261,7 @@ interface KernelSessionDebug {
261
261
  retrievedProxyIdMatchesRequested: boolean | null;
262
262
  defaultProxyDisabled: boolean | null;
263
263
  defaultProxyDisableError: string | null;
264
- proxyResolution: KernelProxyResolutionDebug | null;
264
+ proxyResolution: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug | null;
265
265
  timeoutSeconds: number | null;
266
266
  stealth: boolean | null;
267
267
  profilePresent: boolean | null;
@@ -291,7 +291,7 @@ interface SerpNavigationDebug {
291
291
  redirected: boolean | null;
292
292
  }
293
293
  interface BrowserDebugSnapshot {
294
- kernel: KernelSessionDebug | null;
294
+ \u006b\u0065\u0072\u006e\u0065\u006c: \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug | null;
295
295
  context: BrowserContextDebug | null;
296
296
  networkLocation: BrowserNetworkLocationDebug | null;
297
297
  serpNavigation: SerpNavigationDebug | null;
package/dist/index.d.ts CHANGED
@@ -24,13 +24,13 @@ declare const HarvestOptionsSchema: z.ZodObject<{
24
24
  headless: z.ZodDefault<z.ZodBoolean>;
25
25
  profileDir: z.ZodOptional<z.ZodString>;
26
26
  proxy: z.ZodOptional<z.ZodString>;
27
- kernelApiKey: z.ZodOptional<z.ZodString>;
28
- kernelProxyId: z.ZodOptional<z.ZodString>;
29
- kernelProfileName: z.ZodOptional<z.ZodString>;
30
- kernelProfileSaveChanges: z.ZodOptional<z.ZodBoolean>;
31
- kernelStealth: z.ZodOptional<z.ZodBoolean>;
27
+ \u006b\u0065\u0072\u006e\u0065\u006cApiKey: z.ZodOptional<z.ZodString>;
28
+ \u006b\u0065\u0072\u006e\u0065\u006cProxyId: z.ZodOptional<z.ZodString>;
29
+ \u006b\u0065\u0072\u006e\u0065\u006cProfileName: z.ZodOptional<z.ZodString>;
30
+ \u006b\u0065\u0072\u006e\u0065\u006cProfileSaveChanges: z.ZodOptional<z.ZodBoolean>;
31
+ \u006b\u0065\u0072\u006e\u0065\u006cStealth: z.ZodOptional<z.ZodBoolean>;
32
32
  serpIdentity: z.ZodOptional<z.ZodString>;
33
- kernelProxyResolution: z.ZodOptional<z.ZodUnknown>;
33
+ \u006b\u0065\u0072\u006e\u0065\u006cProxyResolution: z.ZodOptional<z.ZodUnknown>;
34
34
  outputDir: z.ZodDefault<z.ZodString>;
35
35
  format: z.ZodDefault<z.ZodEnum<{
36
36
  json: "json";
@@ -60,7 +60,7 @@ declare const HarvestOptionsSchema: z.ZodObject<{
60
60
  type CanonicalLocationName = string;
61
61
  type SerpDevice = 'desktop' | 'mobile';
62
62
  type ProxyMode = 'location' | 'configured' | 'none';
63
- interface KernelLocationProxyTarget {
63
+ interface \u004b\u0065\u0072\u006e\u0065\u006cLocationProxyTarget {
64
64
  canonicalLocation: CanonicalLocationName;
65
65
  level: 'zip' | 'city' | 'state';
66
66
  country: 'US';
@@ -76,14 +76,14 @@ interface KernelLocationProxyTarget {
76
76
  zip?: string;
77
77
  };
78
78
  }
79
- type KernelProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'isp_created' | 'backup_custom' | 'configured_fallback' | 'unavailable';
80
- interface KernelProxyResolutionDebug {
81
- source: KernelProxyResolutionSource;
79
+ type \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionSource = 'disabled' | 'location_reused' | 'location_created' | 'isp_created' | 'backup_custom' | 'configured_fallback' | 'unavailable';
80
+ interface \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug {
81
+ source: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionSource;
82
82
  proxyType: string;
83
83
  proxyMode: ProxyMode;
84
84
  proxyIdPresent: boolean;
85
85
  proxyIdSuffix: string | null;
86
- target: KernelLocationProxyTarget | null;
86
+ target: \u004b\u0065\u0072\u006e\u0065\u006cLocationProxyTarget | null;
87
87
  error: string | null;
88
88
  disposable?: boolean;
89
89
  }
@@ -249,7 +249,7 @@ interface BrowserNetworkLocationDebug {
249
249
  timezone: string | null;
250
250
  error: string | null;
251
251
  }
252
- interface KernelSessionDebug {
252
+ interface \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug {
253
253
  sessionId: string | null;
254
254
  proxyMode: ProxyMode;
255
255
  requestedProxyIdPresent: boolean;
@@ -261,7 +261,7 @@ interface KernelSessionDebug {
261
261
  retrievedProxyIdMatchesRequested: boolean | null;
262
262
  defaultProxyDisabled: boolean | null;
263
263
  defaultProxyDisableError: string | null;
264
- proxyResolution: KernelProxyResolutionDebug | null;
264
+ proxyResolution: \u004b\u0065\u0072\u006e\u0065\u006cProxyResolutionDebug | null;
265
265
  timeoutSeconds: number | null;
266
266
  stealth: boolean | null;
267
267
  profilePresent: boolean | null;
@@ -291,7 +291,7 @@ interface SerpNavigationDebug {
291
291
  redirected: boolean | null;
292
292
  }
293
293
  interface BrowserDebugSnapshot {
294
- kernel: KernelSessionDebug | null;
294
+ \u006b\u0065\u0072\u006e\u0065\u006c: \u004b\u0065\u0072\u006e\u0065\u006cSessionDebug | null;
295
295
  context: BrowserContextDebug | null;
296
296
  networkLocation: BrowserNetworkLocationDebug | null;
297
297
  serpNavigation: SerpNavigationDebug | null;
package/dist/index.js CHANGED
@@ -1,25 +1,4 @@
1
- import {
2
- harvest
3
- } from "./chunk-S24LFPL7.js";
4
- import "./chunk-OM7HVEJ3.js";
5
- import {
6
- recordVendorUsage
7
- } from "./chunk-OPQIGAFB.js";
8
- import "./chunk-GXBZXWXB.js";
9
- import "./chunk-YXNDOQXN.js";
10
-
11
- // src/video/VideoGenerator.ts
12
- import { execSync as execSync2 } from "child_process";
13
- import { readFileSync, writeFileSync as writeFileSync3, unlinkSync, mkdirSync as mkdirSync3 } from "fs";
14
- import { tmpdir as tmpdir2 } from "os";
15
- import { join as join3 } from "path";
16
- import { fal as fal3 } from "@fal-ai/client";
17
-
18
- // src/video/promptBuilder.ts
19
- var DEEPINFRA_URL = "https://api.deepinfra.com/v1/openai/chat/completions";
20
- var OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions";
21
- var QWEN_MODEL = "Qwen/Qwen3.6-35B-A3B";
22
- var SYSTEM_PROMPT = `You are a video prompt engineer for Seedance 2.0 text-to-video AI.
1
+ import{u as z}from"./chunk-7YGVI5J4.js";import"./chunk-QPWPR5XG.js";import"./chunk-4FROKQJN.js";import{h as M}from"./chunk-MASR22K4.js";import"./chunk-3GP5CYZX.js";import"./chunk-GMWKPIYX.js";import{execSync as ce}from"child_process";import{readFileSync as G,writeFileSync as _,unlinkSync as D,mkdirSync as Y}from"fs";import{tmpdir as W}from"os";import{join as l}from"path";import{fal as p}from"@fal-ai/client";var X="https://api.deepinfra.com/v1/openai/chat/completions",L="https://openrouter.ai/api/v1/chat/completions",A="Qwen/Qwen3.6-35B-A3B",Z=`You are a video prompt engineer for Seedance 2.0 text-to-video AI.
23
2
 
24
3
  IMPORTANT TECHNICAL CONTEXT: Clip 2 will be generated using image-to-video, meaning it will start from the exact last frame of Clip 1. The two clips will be visually seamless \u2014 same location, same characters, continuous motion. You must write prompts that make this feel like one uninterrupted 16-second video.
25
4
 
@@ -41,296 +20,20 @@ Rules for visuals:
41
20
  - Describe exactly what the camera sees
42
21
 
43
22
  Respond with JSON only:
44
- {"clip1": "...", "clip2": "...", "voiceover": "...", "audioMood": "..."}`;
45
- async function callLLM(apiKey, baseUrl, question, answer) {
46
- const res = await fetch(baseUrl, {
47
- method: "POST",
48
- headers: { "Authorization": `Bearer ${apiKey}`, "Content-Type": "application/json" },
49
- body: JSON.stringify({
50
- model: QWEN_MODEL,
51
- temperature: 0.7,
52
- messages: [
53
- { role: "system", content: SYSTEM_PROMPT },
54
- { role: "user", content: `Question: ${question}
55
-
56
- Answer: ${answer.slice(0, 500)}` }
57
- ]
58
- })
59
- });
60
- if (!res.ok) throw new Error(`LLM call failed (${res.status}): ${await res.text()}`);
61
- const data = await res.json();
62
- if (baseUrl === OPENROUTER_URL) {
63
- void recordVendorUsage({ vendor: "openrouter", model: QWEN_MODEL, units: data.usage?.cost ?? 0, unitType: "usd" });
64
- } else {
65
- void recordVendorUsage({ vendor: "deepinfra_qwen", model: QWEN_MODEL, units: data.usage?.completion_tokens ?? 0, unitType: "tokens" });
66
- }
67
- const raw = data.choices[0]?.message?.content?.trim() ?? "";
68
- const match = raw.match(/\{[\s\S]*\}/);
69
- if (!match) throw new Error(`No JSON in QWEN response: ${raw.slice(0, 200)}`);
70
- const parsed = JSON.parse(match[0]);
71
- if (!parsed.clip1 || !parsed.clip2 || !parsed.voiceover || !parsed.audioMood) {
72
- throw new Error(`QWEN response missing fields: ${raw.slice(0, 200)}`);
73
- }
74
- return parsed;
75
- }
76
- async function buildClipPrompts(question, answer) {
77
- const deepinfraKey = process.env["DEEPINFRA_API_KEY"];
78
- const openrouterKey = process.env["OPENROUTER_API_KEY"];
79
- if (deepinfraKey) {
80
- try {
81
- return await callLLM(deepinfraKey, DEEPINFRA_URL, question, answer);
82
- } catch (err) {
83
- console.warn("[promptBuilder] DeepInfra failed, trying OpenRouter:", err.message);
84
- }
85
- }
86
- if (openrouterKey) {
87
- return await callLLM(openrouterKey, OPENROUTER_URL, question, answer);
88
- }
89
- throw new Error("No LLM key \u2014 set DEEPINFRA_API_KEY or OPENROUTER_API_KEY");
90
- }
91
- function extractEpisodePrompts(brief) {
92
- if (!brief.clip1 || !brief.clip2 || !brief.voiceover || !brief.audioMood) {
93
- throw new Error("Episode brief is missing prompt fields \u2014 run blog-to-video skill to regenerate");
94
- }
95
- return { clip1: brief.clip1, clip2: brief.clip2, voiceover: brief.voiceover, audioMood: brief.audioMood };
96
- }
97
-
98
- // src/video/AudioGenerator.ts
99
- import { writeFileSync, mkdirSync } from "fs";
100
- import { join } from "path";
101
- import { tmpdir } from "os";
102
- import { fal } from "@fal-ai/client";
103
- var MMAUDIO_MODEL = "fal-ai/mmaudio-v2";
104
- var ELEVENLABS_MODEL = "fal-ai/elevenlabs/tts";
105
- var GEMINI_TTS_MODEL = "fal-ai/google/gemini-2.5-flash-preview-tts";
106
- async function downloadAudio(url) {
107
- const res = await fetch(url);
108
- if (!res.ok) throw new Error(`Failed to download TTS audio (${res.status})`);
109
- return Buffer.from(await res.arrayBuffer());
110
- }
111
- async function generateVoiceover(text) {
112
- console.log("[AudioGenerator] Generating voiceover...");
113
- const outDir = join(tmpdir(), `tts-${Date.now()}`);
114
- mkdirSync(outDir, { recursive: true });
115
- const outPath = join(outDir, "voiceover.mp3");
116
- try {
117
- const voiceId = process.env["ELEVENLABS_VOICE_ID"] ?? "pNInz6obpgDQGcFmaJgB";
118
- const result2 = await fal.run(ELEVENLABS_MODEL, {
119
- input: { text, voice_id: voiceId, model_id: "eleven_v3" }
120
- });
121
- writeFileSync(outPath, await downloadAudio(result2.audio.url));
122
- console.log("[AudioGenerator] TTS: ElevenLabs via fal");
123
- return outPath;
124
- } catch (err) {
125
- console.warn("[AudioGenerator] ElevenLabs via fal failed, trying Gemini:", err.message);
126
- }
127
- const voice = process.env["GEMINI_TTS_VOICE"] ?? "Kore";
128
- const result = await fal.run(GEMINI_TTS_MODEL, { input: { text, voice } });
129
- writeFileSync(outPath, await downloadAudio(result.audio.url));
130
- console.log("[AudioGenerator] TTS: Gemini via fal");
131
- return outPath;
132
- }
133
- async function addBackgroundAudio(videoUrl, mood, durationSeconds) {
134
- console.log("[AudioGenerator] Adding background audio via MMAudio V2...");
135
- const result = await fal.run(MMAUDIO_MODEL, {
136
- input: {
137
- video_url: videoUrl,
138
- prompt: mood,
139
- negative_prompt: "speech, voice, talking, dialogue, narration, vocals, singing, human voice, conversation, words, lyrics, announcer, commentary",
140
- duration: durationSeconds,
141
- cfg_strength: 4.5
142
- }
143
- });
144
- return result.video.url;
145
- }
146
-
147
- // src/video/VideoMixer.ts
148
- import { execSync } from "child_process";
149
- import { writeFileSync as writeFileSync2, mkdirSync as mkdirSync2 } from "fs";
150
- import { join as join2 } from "path";
151
- import { fal as fal2 } from "@fal-ai/client";
152
- async function download(url, destPath) {
153
- const res = await fetch(url);
154
- if (!res.ok) throw new Error(`Download failed (${res.status}): ${url}`);
155
- writeFileSync2(destPath, Buffer.from(await res.arrayBuffer()));
156
- }
157
- async function concatenateClips(clip1Url, clip2Url, outDir) {
158
- mkdirSync2(outDir, { recursive: true });
159
- const ts = Date.now();
160
- const p1 = join2(outDir, `clip1-${ts}.mp4`);
161
- const p2 = join2(outDir, `clip2-${ts}.mp4`);
162
- const out = join2(outDir, `combined-${ts}.mp4`);
163
- console.log("[VideoMixer] Downloading clips...");
164
- await Promise.all([download(clip1Url, p1), download(clip2Url, p2)]);
165
- console.log("[VideoMixer] Concatenating...");
166
- execSync(
167
- `ffmpeg -i "${p1}" -i "${p2}" -filter_complex "[0:v][1:v]concat=n=2:v=1:a=0[v]" -map "[v]" -y "${out}" -loglevel error`
168
- );
169
- return out;
170
- }
171
- async function uploadToFal(localPath) {
172
- const { readFileSync: readFileSync2 } = await import("fs");
173
- const blob = new Blob([readFileSync2(localPath)], { type: "video/mp4" });
174
- const url = await fal2.storage.upload(blob);
175
- console.log("[VideoMixer] Uploaded to fal:", url);
176
- return url;
177
- }
178
- async function overlayVoiceover(videoPath, voiceoverUrl, outDir) {
179
- const ts = Date.now();
180
- const wav = join2(outDir, `voiceover-${ts}.wav`);
181
- const out = join2(outDir, `final-${ts}.mp4`);
182
- console.log("[VideoMixer] Downloading voiceover...");
183
- await download(voiceoverUrl, wav);
184
- console.log("[VideoMixer] Mixing voiceover over background audio...");
185
- execSync(
186
- `ffmpeg -i "${videoPath}" -i "${wav}" -filter_complex "[0:a]volume=0.2[bg];[1:a]volume=1.0[vo];[bg][vo]amix=inputs=2:duration=first[a]" -map 0:v -map "[a]" -c:v copy -y "${out}" -loglevel error`
187
- );
188
- return out;
189
- }
190
-
191
- // src/video/VideoGenerator.ts
192
- var T2V = "bytedance/seedance-2.0/text-to-video";
193
- var I2V = "bytedance/seedance-2.0/image-to-video";
194
- function buildInput(prompt, opts, seed, imageUrl) {
195
- return {
196
- prompt,
197
- resolution: opts.resolution ?? "720p",
198
- duration: opts.clipDurationSeconds ?? 8,
199
- aspect_ratio: opts.aspectRatio ?? "16:9",
200
- generate_audio: false,
201
- ...seed !== void 0 ? { seed } : {},
202
- ...imageUrl !== void 0 ? { image_url: imageUrl } : {}
203
- };
204
- }
205
- async function generate(model, input) {
206
- const { request_id } = await fal3.queue.submit(model, { input });
207
- console.log(`[fal] submitted ${model} \u2192 ${request_id}`);
208
- while (true) {
209
- await new Promise((r) => setTimeout(r, 5e3));
210
- const s = await fal3.queue.status(model, { requestId: request_id, logs: false });
211
- console.log(`[fal] ${request_id} \u2192 ${s.status}`);
212
- if (s.status === "FAILED") throw new Error(`Request ${request_id} failed`);
213
- if (s.status !== "COMPLETED") continue;
214
- const result = await fal3.queue.result(model, { requestId: request_id });
215
- return result.data;
216
- }
217
- }
218
- async function extractLastFrame(videoUrl, outDir) {
219
- const ts = Date.now();
220
- const mp4Path = join3(outDir, `clip1-raw-${ts}.mp4`);
221
- const jpgPath = join3(outDir, `last-frame-${ts}.jpg`);
222
- const res = await fetch(videoUrl);
223
- if (!res.ok) throw new Error(`Failed to download clip 1 (${res.status})`);
224
- writeFileSync3(mp4Path, Buffer.from(await res.arrayBuffer()));
225
- try {
226
- execSync2(`ffmpeg -sseof -0.1 -i "${mp4Path}" -vframes 1 -y "${jpgPath}" -loglevel error`);
227
- } finally {
228
- try {
229
- unlinkSync(mp4Path);
230
- } catch {
231
- }
232
- }
233
- return jpgPath;
234
- }
235
- var VideoGenerator = class {
236
- constructor(apiKey) {
237
- const key = apiKey ?? process.env["FAL_KEY"];
238
- if (!key) throw new Error("FAL_KEY is required");
239
- fal3.config({ credentials: key });
240
- }
241
- async generateClipPair(question, answer, opts = {}) {
242
- const outDir = opts.outputDir ?? join3(tmpdir2(), `paa-video-${Date.now()}`);
243
- mkdirSync3(outDir, { recursive: true });
244
- console.log("\n[1/7] Generating prompts via QWEN 3.6...");
245
- const prompts = await buildClipPrompts(question, answer);
246
- console.log(" Voiceover:", prompts.voiceover);
247
- console.log(" Audio mood:", prompts.audioMood);
248
- console.log("\n[2/7] Generating clip 1 (text-to-video)...");
249
- const result1 = await generate(T2V, buildInput(prompts.clip1, opts, opts.seed));
250
- console.log("\n[3/7] Extracting last frame \u2192 clip 2 start...");
251
- const jpgPath = await extractLastFrame(result1.video.url, outDir);
252
- const imageBlob = new Blob([readFileSync(jpgPath)], { type: "image/jpeg" });
253
- const frameUrl = await fal3.storage.upload(imageBlob);
254
- try {
255
- unlinkSync(jpgPath);
256
- } catch {
257
- }
258
- console.log("\n[4/7] Generating clip 2 (image-to-video from last frame)...");
259
- const seed2 = opts.seed !== void 0 ? opts.seed + 1 : void 0;
260
- const result2 = await generate(I2V, buildInput(prompts.clip2, opts, seed2, frameUrl));
261
- console.log("\n[5/7] Concatenating clips + generating voiceover (parallel)...");
262
- const [combinedPath, voiceoverUrl] = await Promise.all([
263
- concatenateClips(result1.video.url, result2.video.url, outDir),
264
- generateVoiceover(prompts.voiceover)
265
- ]);
266
- console.log("\n[6/7] Adding background audio via MMAudio V2...");
267
- const falVideoUrl = await uploadToFal(combinedPath);
268
- const totalDuration = (opts.clipDurationSeconds ?? 8) * 2;
269
- const videoWithAudioUrl = await addBackgroundAudio(falVideoUrl, prompts.audioMood, totalDuration);
270
- console.log("\n[7/7] Overlaying voiceover on final video...");
271
- const videoWithAudioPath = join3(outDir, `with-bg-audio-${Date.now()}.mp4`);
272
- const bgRes = await fetch(videoWithAudioUrl);
273
- writeFileSync3(videoWithAudioPath, Buffer.from(await bgRes.arrayBuffer()));
274
- const finalVideoPath = await overlayVoiceover(videoWithAudioPath, voiceoverUrl, outDir);
275
- return {
276
- clip1Url: result1.video.url,
277
- clip2Url: result2.video.url,
278
- finalVideoPath,
279
- seed: result1.seed,
280
- promptClip1: prompts.clip1,
281
- promptClip2: prompts.clip2,
282
- voiceover: prompts.voiceover,
283
- audioMood: prompts.audioMood
284
- };
285
- }
286
- async generateEpisode(brief, opts = {}) {
287
- const outDir = opts.outputDir ?? join3(tmpdir2(), `episode-${brief.episodeNumber}-${Date.now()}`);
288
- mkdirSync3(outDir, { recursive: true });
289
- const prompts = extractEpisodePrompts(brief);
290
- console.log(`
291
- [Episode ${brief.episodeNumber}/${brief.episodeCount}] ${brief.sectionTitle}`);
292
- console.log(" Voiceover:", prompts.voiceover);
293
- console.log("\n[2/7] Generating clip 1 (text-to-video)...");
294
- const result1 = await generate(T2V, buildInput(prompts.clip1, opts, opts.seed));
295
- console.log("\n[3/7] Extracting last frame \u2192 clip 2 start...");
296
- const jpgPath = await extractLastFrame(result1.video.url, outDir);
297
- const imageBlob = new Blob([readFileSync(jpgPath)], { type: "image/jpeg" });
298
- const frameUrl = await fal3.storage.upload(imageBlob);
299
- try {
300
- unlinkSync(jpgPath);
301
- } catch {
302
- }
303
- console.log("\n[4/7] Generating clip 2 (image-to-video from last frame)...");
304
- const seed2 = opts.seed !== void 0 ? opts.seed + 1 : void 0;
305
- const result2 = await generate(I2V, buildInput(prompts.clip2, opts, seed2, frameUrl));
306
- console.log("\n[5/7] Concatenating clips + generating voiceover (parallel)...");
307
- const [combinedPath, voiceoverUrl] = await Promise.all([
308
- concatenateClips(result1.video.url, result2.video.url, outDir),
309
- generateVoiceover(prompts.voiceover)
310
- ]);
311
- console.log("\n[6/7] Adding background audio via MMAudio V2...");
312
- const falVideoUrl = await uploadToFal(combinedPath);
313
- const totalDuration = (opts.clipDurationSeconds ?? 8) * 2;
314
- const videoWithAudioUrl = await addBackgroundAudio(falVideoUrl, prompts.audioMood, totalDuration);
315
- console.log("\n[7/7] Overlaying voiceover on final video...");
316
- const videoWithAudioPath = join3(outDir, `with-bg-audio-${Date.now()}.mp4`);
317
- const bgRes = await fetch(videoWithAudioUrl);
318
- writeFileSync3(videoWithAudioPath, Buffer.from(await bgRes.arrayBuffer()));
319
- const finalVideoPath = await overlayVoiceover(videoWithAudioPath, voiceoverUrl, outDir);
320
- return {
321
- clip1Url: result1.video.url,
322
- clip2Url: result2.video.url,
323
- finalVideoPath,
324
- seed: result1.seed,
325
- promptClip1: prompts.clip1,
326
- promptClip2: prompts.clip2,
327
- voiceover: prompts.voiceover,
328
- audioMood: prompts.audioMood
329
- };
330
- }
331
- };
332
- export {
333
- VideoGenerator,
334
- buildClipPrompts,
335
- harvest
336
- };
23
+ {"clip1": "...", "clip2": "...", "voiceover": "...", "audioMood": "..."}`;async function B(t,i,e,o){let n=await fetch(i,{method:"POST",headers:{Authorization:`Bearer ${t}`,"Content-Type":"application/json"},body:JSON.stringify({model:A,temperature:.7,messages:[{role:"system",content:Z},{role:"user",content:`Question: ${e}
24
+
25
+ Answer: ${o.slice(0,500)}`}]})});if(!n.ok)throw new Error(`LLM call failed (${n.status}): ${await n.text()}`);let r=await n.json();i===L?M({vendor:"openrouter",model:A,units:r.usage?.cost??0,unitType:"usd"}):M({vendor:"deepinfra_qwen",model:A,units:r.usage?.completion_tokens??0,unitType:"tokens"});let a=r.choices[0]?.message?.content?.trim()??"",c=a.match(/\{[\s\S]*\}/);if(!c)throw new Error(`No JSON in QWEN response: ${a.slice(0,200)}`);let s=JSON.parse(c[0]);if(!s.clip1||!s.clip2||!s.voiceover||!s.audioMood)throw new Error(`QWEN response missing fields: ${a.slice(0,200)}`);return s}async function T(t,i){let e=process.env.DEEPINFRA_API_KEY,o=process.env.OPENROUTER_API_KEY;if(e)try{return await B(e,X,t,i)}catch(n){console.warn("[promptBuilder] DeepInfra failed, trying OpenRouter:",n.message)}if(o)return await B(o,L,t,i);throw new Error("No LLM key \u2014 set DEEPINFRA_API_KEY or OPENROUTER_API_KEY")}function V(t){if(!t.clip1||!t.clip2||!t.voiceover||!t.audioMood)throw new Error("Episode brief is missing prompt fields \u2014 run blog-to-video skill to regenerate");return{clip1:t.clip1,clip2:t.clip2,voiceover:t.voiceover,audioMood:t.audioMood}}import{writeFileSync as U,mkdirSync as ee}from"fs";import{join as F}from"path";import{tmpdir as oe}from"os";import{fal as x}from"@fal-ai/client";var ie="fal-ai/mmaudio-v2",te="fal-ai/elevenlabs/tts",re="fal-ai/google/gemini-2.5-flash-preview-tts";async function K(t){let i=await fetch(t);if(!i.ok)throw new Error(`Failed to download TTS audio (${i.status})`);return Buffer.from(await i.arrayBuffer())}async function O(t){console.log("[AudioGenerator] Generating voiceover...");let i=F(oe(),`tts-${Date.now()}`);ee(i,{recursive:!0});let e=F(i,"voiceover.mp3");try{let r=process.env.ELEVENLABS_VOICE_ID??"pNInz6obpgDQGcFmaJgB",a=await x.run(te,{input:{text:t,voice_id:r,model_id:"eleven_v3"}});return U(e,await K(a.audio.url)),console.log("[AudioGenerator] TTS: ElevenLabs via fal"),e}catch(r){console.warn("[AudioGenerator] ElevenLabs via fal failed, trying Gemini:",r.message)}let o=process.env.GEMINI_TTS_VOICE??"Kore",n=await x.run(re,{input:{text:t,voice:o}});return U(e,await K(n.audio.url)),console.log("[AudioGenerator] TTS: Gemini via fal"),e}async function $(t,i,e){return console.log("[AudioGenerator] Adding background audio via MMAudio V2..."),(await x.run(ie,{input:{video_url:t,prompt:i,negative_prompt:"speech, voice, talking, dialogue, narration, vocals, singing, human voice, conversation, words, lyrics, announcer, commentary",duration:e,cfg_strength:4.5}})).video.url}import{execSync as j}from"child_process";import{writeFileSync as ne,mkdirSync as ae}from"fs";import{join as u}from"path";import{fal as se}from"@fal-ai/client";async function I(t,i){let e=await fetch(t);if(!e.ok)throw new Error(`Download failed (${e.status}): ${t}`);ne(i,Buffer.from(await e.arrayBuffer()))}async function N(t,i,e){ae(e,{recursive:!0});let o=Date.now(),n=u(e,`clip1-${o}.mp4`),r=u(e,`clip2-${o}.mp4`),a=u(e,`combined-${o}.mp4`);return console.log("[VideoMixer] Downloading clips..."),await Promise.all([I(t,n),I(i,r)]),console.log("[VideoMixer] Concatenating..."),j(`ffmpeg -i "${n}" -i "${r}" -filter_complex "[0:v][1:v]concat=n=2:v=1:a=0[v]" -map "[v]" -y "${a}" -loglevel error`),a}async function b(t){let{readFileSync:i}=await import("fs"),e=new Blob([i(t)],{type:"video/mp4"}),o=await se.storage.upload(e);return console.log("[VideoMixer] Uploaded to fal:",o),o}async function R(t,i,e){let o=Date.now(),n=u(e,`voiceover-${o}.wav`),r=u(e,`final-${o}.mp4`);return console.log("[VideoMixer] Downloading voiceover..."),await I(i,n),console.log("[VideoMixer] Mixing voiceover over background audio..."),j(`ffmpeg -i "${t}" -i "${n}" -filter_complex "[0:a]volume=0.2[bg];[1:a]volume=1.0[vo];[bg][vo]amix=inputs=2:duration=first[a]" -map 0:v -map "[a]" -c:v copy -y "${r}" -loglevel error`),r}var q="bytedance/seedance-2.0/text-to-video",J="bytedance/seedance-2.0/image-to-video";function v(t,i,e,o){return{prompt:t,resolution:i.resolution??"720p",duration:i.clipDurationSeconds??8,aspect_ratio:i.aspectRatio??"16:9",generate_audio:!1,...e!==void 0?{seed:e}:{},...o!==void 0?{image_url:o}:{}}}async function w(t,i){let{request_id:e}=await p.queue.submit(t,{input:i});for(console.log(`[fal] submitted ${t} \u2192 ${e}`);;){await new Promise(r=>setTimeout(r,5e3));let o=await p.queue.status(t,{requestId:e,logs:!1});if(console.log(`[fal] ${e} \u2192 ${o.status}`),o.status==="FAILED")throw new Error(`Request ${e} failed`);if(o.status!=="COMPLETED")continue;return(await p.queue.result(t,{requestId:e})).data}}async function H(t,i){let e=Date.now(),o=l(i,`clip1-raw-${e}.mp4`),n=l(i,`last-frame-${e}.jpg`),r=await fetch(t);if(!r.ok)throw new Error(`Failed to download clip 1 (${r.status})`);_(o,Buffer.from(await r.arrayBuffer()));try{ce(`ffmpeg -sseof -0.1 -i "${o}" -vframes 1 -y "${n}" -loglevel error`)}finally{try{D(o)}catch{}}return n}var k=class{constructor(i){let e=i??process.env.FAL_KEY;if(!e)throw new Error("FAL_KEY is required");p.config({credentials:e})}async generateClipPair(i,e,o={}){let n=o.outputDir??l(W(),`paa-video-${Date.now()}`);Y(n,{recursive:!0}),console.log(`
26
+ [1/7] Generating prompts via QWEN 3.6...`);let r=await T(i,e);console.log(" Voiceover:",r.voiceover),console.log(" Audio mood:",r.audioMood),console.log(`
27
+ [2/7] Generating clip 1 (text-to-video)...`);let a=await w(q,v(r.clip1,o,o.seed));console.log(`
28
+ [3/7] Extracting last frame \u2192 clip 2 start...`);let c=await H(a.video.url,n),s=new Blob([G(c)],{type:"image/jpeg"}),h=await p.storage.upload(s);try{D(c)}catch{}console.log(`
29
+ [4/7] Generating clip 2 (image-to-video from last frame)...`);let d=o.seed!==void 0?o.seed+1:void 0,m=await w(J,v(r.clip2,o,d,h));console.log(`
30
+ [5/7] Concatenating clips + generating voiceover (parallel)...`);let[P,y]=await Promise.all([N(a.video.url,m.video.url,n),O(r.voiceover)]);console.log(`
31
+ [6/7] Adding background audio via MMAudio V2...`);let E=await b(P),S=(o.clipDurationSeconds??8)*2,g=await $(E,r.audioMood,S);console.log(`
32
+ [7/7] Overlaying voiceover on final video...`);let f=l(n,`with-bg-audio-${Date.now()}.mp4`),C=await fetch(g);_(f,Buffer.from(await C.arrayBuffer()));let Q=await R(f,y,n);return{clip1Url:a.video.url,clip2Url:m.video.url,finalVideoPath:Q,seed:a.seed,promptClip1:r.clip1,promptClip2:r.clip2,voiceover:r.voiceover,audioMood:r.audioMood}}async generateEpisode(i,e={}){let o=e.outputDir??l(W(),`episode-${i.episodeNumber}-${Date.now()}`);Y(o,{recursive:!0});let n=V(i);console.log(`
33
+ [Episode ${i.episodeNumber}/${i.episodeCount}] ${i.sectionTitle}`),console.log(" Voiceover:",n.voiceover),console.log(`
34
+ [2/7] Generating clip 1 (text-to-video)...`);let r=await w(q,v(n.clip1,e,e.seed));console.log(`
35
+ [3/7] Extracting last frame \u2192 clip 2 start...`);let a=await H(r.video.url,o),c=new Blob([G(a)],{type:"image/jpeg"}),s=await p.storage.upload(c);try{D(a)}catch{}console.log(`
36
+ [4/7] Generating clip 2 (image-to-video from last frame)...`);let h=e.seed!==void 0?e.seed+1:void 0,d=await w(J,v(n.clip2,e,h,s));console.log(`
37
+ [5/7] Concatenating clips + generating voiceover (parallel)...`);let[m,P]=await Promise.all([N(r.video.url,d.video.url,o),O(n.voiceover)]);console.log(`
38
+ [6/7] Adding background audio via MMAudio V2...`);let y=await b(m),E=(e.clipDurationSeconds??8)*2,S=await $(y,n.audioMood,E);console.log(`
39
+ [7/7] Overlaying voiceover on final video...`);let g=l(o,`with-bg-audio-${Date.now()}.mp4`),f=await fetch(S);_(g,Buffer.from(await f.arrayBuffer()));let C=await R(g,P,o);return{clip1Url:r.video.url,clip2Url:d.video.url,finalVideoPath:C,seed:r.seed,promptClip1:n.clip1,promptClip2:n.clip2,voiceover:n.voiceover,audioMood:n.audioMood}}};export{k as VideoGenerator,T as buildClipPrompts,z as harvest};
@@ -0,0 +1 @@
1
+ import{A,B,C,D,E,a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z}from"./chunk-XPZVJIZ2.js";import"./chunk-GMWKPIYX.js";export{a as LEAD_LIST_MAX_DISPATCH_ATTEMPTS,c as LEAD_LIST_RUNNING_STALE_MINUTES,b as LEAD_LIST_START_DEADLINE_MINUTES,s as cancelPendingLeadListEnrichmentRows,C as claimLeadListEnrichmentOutbox,n as claimLeadListEnrichmentRow,u as completeLeadListEnrichmentJob,j as createOrGetLeadListEnrichmentJob,g as ensureLeadListEnrichmentRepositorySchema,z as failExpiredLeadListEnrichmentStart,v as failLeadListEnrichmentJob,B as failStaleRunningLeadListEnrichmentJob,h as getLeadListEnrichmentJob,i as getLeadListEnrichmentJobByIdempotencyKey,o as getLeadListEnrichmentRow,e as leadListEnrichmentFingerprint,f as leadListRowInputDigest,y as listExpiredLeadListEnrichmentStarts,x as listLeadListEnrichmentJobsNeedingSettlement,p as listLeadListEnrichmentRows,A as listStaleRunningLeadListEnrichmentJobs,l as markLeadListEnrichmentJobQueued,m as markLeadListEnrichmentJobRunning,D as markLeadListEnrichmentOutboxDispatched,E as markLeadListEnrichmentOutboxFailed,w as recordLeadListEnrichmentSettlement,r as requestLeadListEnrichmentCancellation,t as saveLeadListEnrichmentDeliverable,q as saveLeadListEnrichmentRowResult,k as setInitializingLeadListEnrichmentConcurrencyLock,d as stableCanonicalJson};
@@ -0,0 +1 @@
1
+ import{a,b,c,d,e,f,g,h,i,j,k,l,m,n}from"./chunk-HDPYG3XV.js";import"./chunk-KJQXUZ4Y.js";import"./chunk-GMWKPIYX.js";export{b as HOSTED_CENSUS_DATASET_PREFIX,a as HOSTED_ZIP_DATASET_KIND,c as MIN_NATIONWIDE_ZIP_COUNT,d as REQUIRED_LOCATION_STATE_CODES,g as ensureHostedLocationDataSchema,h as getActiveHostedDataset,i as getActiveHostedLocationDataset,k as getHostedZipGroups,m as importHostedCensusPlacesCsv,l as importHostedZipGroupsCsv,j as listActiveHostedLocationDatasets,e as parseHostedCensusPlacesCsv,f as parseHostedZipGroupsCsv,n as queryHostedLocationMarkets};