@mixio-pro/kalaasetu-mcp 1.1.1 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/index.ts +23 -19
- package/src/tools/fal/config.ts +34 -0
- package/src/tools/fal/generate.ts +141 -0
- package/src/tools/fal/index.ts +14 -0
- package/src/tools/fal/models.ts +95 -0
- package/src/tools/fal/storage.ts +116 -0
- package/src/tools/image-to-video.ts +7 -5
- package/src/utils/index.ts +0 -1
- package/src/tools/hunyuan-avatar.ts +0 -160
- package/src/tools/infinitalk.ts +0 -156
- package/src/utils/fal.utils.ts +0 -53
package/package.json
CHANGED
package/src/index.ts
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
import { FastMCP } from "fastmcp";
|
|
3
|
-
import {
|
|
4
|
-
geminiTextToImage,
|
|
5
|
-
geminiEditImage,
|
|
6
|
-
geminiAnalyzeImages,
|
|
7
|
-
geminiSingleSpeakerTts,
|
|
8
|
-
geminiAnalyzeVideos,
|
|
9
|
-
} from "./tools/gemini";
|
|
10
|
-
import { analyzeYoutubeVideo } from "./tools/youtube";
|
|
3
|
+
import { geminiEditImage, geminiTextToImage } from "./tools/gemini";
|
|
11
4
|
import { imageToVideo } from "./tools/image-to-video";
|
|
12
|
-
import {
|
|
13
|
-
|
|
14
|
-
|
|
5
|
+
import {
|
|
6
|
+
falListModels,
|
|
7
|
+
falSearchModels,
|
|
8
|
+
falGetSchema,
|
|
9
|
+
falGenerateContent,
|
|
10
|
+
falGetResult,
|
|
11
|
+
falGetStatus,
|
|
12
|
+
falCancelRequest,
|
|
13
|
+
falUploadFile,
|
|
14
|
+
} from "./tools/fal";
|
|
15
15
|
|
|
16
16
|
const server = new FastMCP({
|
|
17
17
|
name: "Kalaasetu MCP Server",
|
|
18
|
-
version: "1.
|
|
18
|
+
version: "1.1.0",
|
|
19
19
|
});
|
|
20
20
|
|
|
21
21
|
// Gemini Image Tools
|
|
@@ -27,7 +27,7 @@ server.addTool(geminiEditImage);
|
|
|
27
27
|
// server.addTool(geminiSingleSpeakerTts);
|
|
28
28
|
|
|
29
29
|
// Gemini Video Analysis Tool
|
|
30
|
-
server.addTool(geminiAnalyzeVideos);
|
|
30
|
+
// server.addTool(geminiAnalyzeVideos);
|
|
31
31
|
|
|
32
32
|
// YouTube Analyzer Tool
|
|
33
33
|
// server.addTool(analyzeYoutubeVideo);
|
|
@@ -35,16 +35,20 @@ server.addTool(geminiAnalyzeVideos);
|
|
|
35
35
|
// Vertex AI Image-to-Video Tool
|
|
36
36
|
server.addTool(imageToVideo);
|
|
37
37
|
|
|
38
|
-
// FAL AI Infinitalk Tool
|
|
39
|
-
// server.addTool(infinitalk);
|
|
40
|
-
|
|
41
|
-
// FAL AI Hunyuan Avatar Tool
|
|
42
|
-
// server.addTool(hunyuanAvatar);
|
|
43
|
-
|
|
44
38
|
// Perplexity Search Tools
|
|
45
39
|
// server.addTool(perplexityImages);
|
|
46
40
|
// server.addTool(perplexityVideos);
|
|
47
41
|
|
|
42
|
+
// Fal AI Tools
|
|
43
|
+
server.addTool(falListModels);
|
|
44
|
+
server.addTool(falSearchModels);
|
|
45
|
+
server.addTool(falGetSchema);
|
|
46
|
+
server.addTool(falGenerateContent);
|
|
47
|
+
server.addTool(falGetResult);
|
|
48
|
+
server.addTool(falGetStatus);
|
|
49
|
+
server.addTool(falCancelRequest);
|
|
50
|
+
server.addTool(falUploadFile);
|
|
51
|
+
|
|
48
52
|
server.start({
|
|
49
53
|
transportType: "stdio",
|
|
50
54
|
});
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configuration module for the fal.ai MCP server.
|
|
3
|
+
* Provides centralized configuration settings for API endpoints, timeouts, and server metadata.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
// API URLs
|
|
7
|
+
export const FAL_BASE_URL = "https://fal.ai/api";
|
|
8
|
+
export const FAL_QUEUE_URL = "https://queue.fal.run";
|
|
9
|
+
export const FAL_DIRECT_URL = "https://fal.run";
|
|
10
|
+
export const FAL_REST_URL = "https://rest.alpha.fal.ai";
|
|
11
|
+
|
|
12
|
+
// Timeouts (in milliseconds)
|
|
13
|
+
export const DEFAULT_TIMEOUT = 30000;
|
|
14
|
+
export const AUTHENTICATED_TIMEOUT = 600000; // 10 minutes for long-running operations
|
|
15
|
+
|
|
16
|
+
// Environment variable names
|
|
17
|
+
export const API_KEY_ENV_VAR = "FAL_KEY";
|
|
18
|
+
|
|
19
|
+
// Server metadata
|
|
20
|
+
export const SERVER_NAME = "fal.ai MCP Server";
|
|
21
|
+
export const SERVER_DESCRIPTION =
|
|
22
|
+
"Access fal.ai models and generate content through MCP";
|
|
23
|
+
export const SERVER_VERSION = "1.0.0";
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Get the fal.ai API key from environment variables.
|
|
27
|
+
*/
|
|
28
|
+
export function getApiKey(): string {
|
|
29
|
+
const apiKey = process.env[API_KEY_ENV_VAR];
|
|
30
|
+
if (!apiKey) {
|
|
31
|
+
throw new Error(`${API_KEY_ENV_VAR} environment variable not set`);
|
|
32
|
+
}
|
|
33
|
+
return apiKey;
|
|
34
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Generate module for fal.ai MCP server.
|
|
3
|
+
* Provides tools for generating content and managing queue operations with fal.ai models.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { z } from "zod";
|
|
7
|
+
import {
|
|
8
|
+
FAL_QUEUE_URL,
|
|
9
|
+
FAL_DIRECT_URL,
|
|
10
|
+
AUTHENTICATED_TIMEOUT,
|
|
11
|
+
getApiKey,
|
|
12
|
+
} from "./config";
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Make an authenticated request to fal.ai API.
|
|
16
|
+
*/
|
|
17
|
+
async function authenticatedRequest(
|
|
18
|
+
url: string,
|
|
19
|
+
method: "GET" | "POST" | "PUT" = "GET",
|
|
20
|
+
jsonData?: Record<string, any>
|
|
21
|
+
): Promise<any> {
|
|
22
|
+
const headers: Record<string, string> = {
|
|
23
|
+
Authorization: `Key ${getApiKey()}`,
|
|
24
|
+
"Content-Type": "application/json",
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const options: RequestInit = {
|
|
28
|
+
method,
|
|
29
|
+
headers,
|
|
30
|
+
signal: AbortSignal.timeout(AUTHENTICATED_TIMEOUT),
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
if (jsonData && (method === "POST" || method === "PUT")) {
|
|
34
|
+
options.body = JSON.stringify(jsonData);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const response = await fetch(url, options);
|
|
38
|
+
|
|
39
|
+
if (!response.ok) {
|
|
40
|
+
const errorText = await response.text();
|
|
41
|
+
throw new Error(`[${response.status}] API error: ${errorText}`);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
return response.json();
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Sanitize parameters by removing null/undefined values.
|
|
49
|
+
*/
|
|
50
|
+
function sanitizeParameters(
|
|
51
|
+
parameters: Record<string, any>
|
|
52
|
+
): Record<string, any> {
|
|
53
|
+
return Object.fromEntries(
|
|
54
|
+
Object.entries(parameters).filter(([_, v]) => v != null)
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Generate content using a fal.ai model.
|
|
60
|
+
*/
|
|
61
|
+
export const falGenerateContent = {
|
|
62
|
+
name: "fal_generate_content",
|
|
63
|
+
description:
|
|
64
|
+
"Generate content using any fal.ai model. Supports both direct execution and queued execution for long-running tasks.",
|
|
65
|
+
parameters: z.object({
|
|
66
|
+
model: z.string().describe('The model ID to use (e.g., "fal-ai/flux/dev")'),
|
|
67
|
+
parameters: z
|
|
68
|
+
.record(z.string(), z.any())
|
|
69
|
+
.describe("Model-specific parameters as a dictionary"),
|
|
70
|
+
queue: z
|
|
71
|
+
.boolean()
|
|
72
|
+
.optional()
|
|
73
|
+
.default(false)
|
|
74
|
+
.describe(
|
|
75
|
+
"Whether to use the queuing system for long-running tasks. Default: false"
|
|
76
|
+
),
|
|
77
|
+
}),
|
|
78
|
+
execute: async (args: {
|
|
79
|
+
model: string;
|
|
80
|
+
parameters: Record<string, any>;
|
|
81
|
+
queue?: boolean;
|
|
82
|
+
}) => {
|
|
83
|
+
const sanitizedParams = sanitizeParameters(args.parameters);
|
|
84
|
+
|
|
85
|
+
const baseUrl = args.queue ? FAL_QUEUE_URL : FAL_DIRECT_URL;
|
|
86
|
+
const url = `${baseUrl}/${args.model}`;
|
|
87
|
+
|
|
88
|
+
console.error(`[fal_generate] Submitting request to ${url}...`);
|
|
89
|
+
|
|
90
|
+
const result = await authenticatedRequest(url, "POST", sanitizedParams);
|
|
91
|
+
|
|
92
|
+
console.error(`[fal_generate] Request completed successfully`);
|
|
93
|
+
|
|
94
|
+
return JSON.stringify(result);
|
|
95
|
+
},
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Get the result of a queued request.
|
|
100
|
+
*/
|
|
101
|
+
export const falGetResult = {
|
|
102
|
+
name: "fal_get_result",
|
|
103
|
+
description: "Get the result of a queued fal.ai request.",
|
|
104
|
+
parameters: z.object({
|
|
105
|
+
url: z.string().describe("The response_url from a queued request"),
|
|
106
|
+
}),
|
|
107
|
+
execute: async (args: { url: string }) => {
|
|
108
|
+
const result = await authenticatedRequest(args.url, "GET");
|
|
109
|
+
return JSON.stringify(result);
|
|
110
|
+
},
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Check the status of a queued request.
|
|
115
|
+
*/
|
|
116
|
+
export const falGetStatus = {
|
|
117
|
+
name: "fal_get_status",
|
|
118
|
+
description: "Check the status of a queued fal.ai request.",
|
|
119
|
+
parameters: z.object({
|
|
120
|
+
url: z.string().describe("The status_url from a queued request"),
|
|
121
|
+
}),
|
|
122
|
+
execute: async (args: { url: string }) => {
|
|
123
|
+
const result = await authenticatedRequest(args.url, "GET");
|
|
124
|
+
return JSON.stringify(result);
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Cancel a queued request.
|
|
130
|
+
*/
|
|
131
|
+
export const falCancelRequest = {
|
|
132
|
+
name: "fal_cancel_request",
|
|
133
|
+
description: "Cancel a queued fal.ai request.",
|
|
134
|
+
parameters: z.object({
|
|
135
|
+
url: z.string().describe("The cancel_url from a queued request"),
|
|
136
|
+
}),
|
|
137
|
+
execute: async (args: { url: string }) => {
|
|
138
|
+
const result = await authenticatedRequest(args.url, "PUT");
|
|
139
|
+
return JSON.stringify(result);
|
|
140
|
+
},
|
|
141
|
+
};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* fal.ai MCP Tools
|
|
3
|
+
* Export all fal.ai tools for registration with the MCP server.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
export { falListModels, falSearchModels, falGetSchema } from "./models";
|
|
7
|
+
export {
|
|
8
|
+
falGenerateContent,
|
|
9
|
+
falGetResult,
|
|
10
|
+
falGetStatus,
|
|
11
|
+
falCancelRequest,
|
|
12
|
+
} from "./generate";
|
|
13
|
+
export { falUploadFile } from "./storage";
|
|
14
|
+
export { SERVER_NAME, SERVER_DESCRIPTION, SERVER_VERSION } from "./config";
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Models module for fal.ai MCP server.
|
|
3
|
+
* Provides tools for listing, searching, and retrieving schemas for fal.ai models.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { z } from "zod";
|
|
7
|
+
import { FAL_BASE_URL, DEFAULT_TIMEOUT } from "./config";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Make a non-authenticated request to fal.ai API.
|
|
11
|
+
*/
|
|
12
|
+
async function publicRequest(url: string): Promise<any> {
|
|
13
|
+
const response = await fetch(url, {
|
|
14
|
+
method: "GET",
|
|
15
|
+
signal: AbortSignal.timeout(DEFAULT_TIMEOUT),
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
if (!response.ok) {
|
|
19
|
+
const errorText = await response.text();
|
|
20
|
+
throw new Error(`[${response.status}] API error: ${errorText}`);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
return response.json();
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* List available models on fal.ai with optional pagination.
|
|
28
|
+
*/
|
|
29
|
+
export const falListModels = {
|
|
30
|
+
name: "fal_list_models",
|
|
31
|
+
description:
|
|
32
|
+
"List available models on fal.ai. Use the page and total arguments for pagination. Avoid listing all models at once.",
|
|
33
|
+
parameters: z.object({
|
|
34
|
+
page: z
|
|
35
|
+
.number()
|
|
36
|
+
.optional()
|
|
37
|
+
.describe("The page number of models to retrieve (pagination)"),
|
|
38
|
+
total: z
|
|
39
|
+
.number()
|
|
40
|
+
.optional()
|
|
41
|
+
.describe("The total number of models to retrieve per page"),
|
|
42
|
+
}),
|
|
43
|
+
execute: async (args: { page?: number; total?: number }) => {
|
|
44
|
+
let url = `${FAL_BASE_URL}/models`;
|
|
45
|
+
|
|
46
|
+
const params: string[] = [];
|
|
47
|
+
if (args.page !== undefined) params.push(`page=${args.page}`);
|
|
48
|
+
if (args.total !== undefined) params.push(`total=${args.total}`);
|
|
49
|
+
|
|
50
|
+
if (params.length > 0) {
|
|
51
|
+
url += "?" + params.join("&");
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const result = await publicRequest(url);
|
|
55
|
+
return JSON.stringify(result);
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Search for models on fal.ai based on keywords.
|
|
61
|
+
*/
|
|
62
|
+
export const falSearchModels = {
|
|
63
|
+
name: "fal_search_models",
|
|
64
|
+
description: "Search for models on fal.ai based on keywords.",
|
|
65
|
+
parameters: z.object({
|
|
66
|
+
keywords: z.string().describe("The search terms to find models"),
|
|
67
|
+
}),
|
|
68
|
+
execute: async (args: { keywords: string }) => {
|
|
69
|
+
const url = `${FAL_BASE_URL}/models?keywords=${encodeURIComponent(
|
|
70
|
+
args.keywords
|
|
71
|
+
)}`;
|
|
72
|
+
const result = await publicRequest(url);
|
|
73
|
+
return JSON.stringify(result);
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Get the OpenAPI schema for a specific model.
|
|
79
|
+
*/
|
|
80
|
+
export const falGetSchema = {
|
|
81
|
+
name: "fal_get_schema",
|
|
82
|
+
description: "Get the OpenAPI schema for a specific fal.ai model.",
|
|
83
|
+
parameters: z.object({
|
|
84
|
+
model_id: z
|
|
85
|
+
.string()
|
|
86
|
+
.describe('The ID of the model (e.g., "fal-ai/flux/dev")'),
|
|
87
|
+
}),
|
|
88
|
+
execute: async (args: { model_id: string }) => {
|
|
89
|
+
const url = `${FAL_BASE_URL}/openapi/queue/openapi.json?endpoint_id=${encodeURIComponent(
|
|
90
|
+
args.model_id
|
|
91
|
+
)}`;
|
|
92
|
+
const result = await publicRequest(url);
|
|
93
|
+
return JSON.stringify(result);
|
|
94
|
+
},
|
|
95
|
+
};
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Storage module for fal.ai MCP server.
|
|
3
|
+
* Provides tools for uploading files to fal.ai storage.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { z } from "zod";
|
|
7
|
+
import * as fs from "fs";
|
|
8
|
+
import * as path from "path";
|
|
9
|
+
import { FAL_REST_URL, AUTHENTICATED_TIMEOUT, getApiKey } from "./config";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Get MIME type from file extension.
|
|
13
|
+
*/
|
|
14
|
+
function getMimeType(filePath: string): string {
|
|
15
|
+
const ext = path.extname(filePath).toLowerCase();
|
|
16
|
+
const mimeTypes: Record<string, string> = {
|
|
17
|
+
".jpg": "image/jpeg",
|
|
18
|
+
".jpeg": "image/jpeg",
|
|
19
|
+
".png": "image/png",
|
|
20
|
+
".gif": "image/gif",
|
|
21
|
+
".webp": "image/webp",
|
|
22
|
+
".mp4": "video/mp4",
|
|
23
|
+
".webm": "video/webm",
|
|
24
|
+
".mov": "video/quicktime",
|
|
25
|
+
".mp3": "audio/mpeg",
|
|
26
|
+
".wav": "audio/wav",
|
|
27
|
+
".ogg": "audio/ogg",
|
|
28
|
+
".pdf": "application/pdf",
|
|
29
|
+
".json": "application/json",
|
|
30
|
+
".txt": "text/plain",
|
|
31
|
+
};
|
|
32
|
+
return mimeTypes[ext] || "application/octet-stream";
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Upload a file to fal.ai storage.
|
|
37
|
+
*/
|
|
38
|
+
export const falUploadFile = {
|
|
39
|
+
name: "fal_upload_file",
|
|
40
|
+
description: "Upload a file to fal.ai CDN storage.",
|
|
41
|
+
parameters: z.object({
|
|
42
|
+
path: z.string().describe("The absolute path to the file to upload"),
|
|
43
|
+
}),
|
|
44
|
+
execute: async (args: { path: string }) => {
|
|
45
|
+
// Validate file exists
|
|
46
|
+
if (!fs.existsSync(args.path)) {
|
|
47
|
+
throw new Error(`File not found: ${args.path}`);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const filename = path.basename(args.path);
|
|
51
|
+
const fileSize = fs.statSync(args.path).size;
|
|
52
|
+
const contentType = getMimeType(args.path);
|
|
53
|
+
|
|
54
|
+
// Step 1: Initiate upload
|
|
55
|
+
const initiateUrl = `${FAL_REST_URL}/storage/upload/initiate?storage_type=fal-cdn-v3`;
|
|
56
|
+
const initiatePayload = {
|
|
57
|
+
content_type: contentType,
|
|
58
|
+
file_name: filename,
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
console.error(`[fal_upload] Initiating upload for ${filename}...`);
|
|
62
|
+
|
|
63
|
+
const initiateResponse = await fetch(initiateUrl, {
|
|
64
|
+
method: "POST",
|
|
65
|
+
headers: {
|
|
66
|
+
Authorization: `Key ${getApiKey()}`,
|
|
67
|
+
"Content-Type": "application/json",
|
|
68
|
+
},
|
|
69
|
+
body: JSON.stringify(initiatePayload),
|
|
70
|
+
signal: AbortSignal.timeout(AUTHENTICATED_TIMEOUT),
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
if (!initiateResponse.ok) {
|
|
74
|
+
const errorText = await initiateResponse.text();
|
|
75
|
+
throw new Error(
|
|
76
|
+
`[${initiateResponse.status}] Failed to initiate upload: ${errorText}`
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const initiateData = (await initiateResponse.json()) as {
|
|
81
|
+
file_url: string;
|
|
82
|
+
upload_url: string;
|
|
83
|
+
};
|
|
84
|
+
const { file_url, upload_url } = initiateData;
|
|
85
|
+
|
|
86
|
+
// Step 2: Upload file content
|
|
87
|
+
console.error(`[fal_upload] Uploading file content...`);
|
|
88
|
+
|
|
89
|
+
const fileContent = fs.readFileSync(args.path);
|
|
90
|
+
|
|
91
|
+
const uploadResponse = await fetch(upload_url, {
|
|
92
|
+
method: "PUT",
|
|
93
|
+
headers: {
|
|
94
|
+
"Content-Type": contentType,
|
|
95
|
+
},
|
|
96
|
+
body: fileContent,
|
|
97
|
+
signal: AbortSignal.timeout(AUTHENTICATED_TIMEOUT),
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
if (!uploadResponse.ok) {
|
|
101
|
+
const errorText = await uploadResponse.text();
|
|
102
|
+
throw new Error(
|
|
103
|
+
`[${uploadResponse.status}] Failed to upload file: ${errorText}`
|
|
104
|
+
);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
console.error(`[fal_upload] Upload completed successfully`);
|
|
108
|
+
|
|
109
|
+
return JSON.stringify({
|
|
110
|
+
file_url,
|
|
111
|
+
file_name: filename,
|
|
112
|
+
file_size: fileSize,
|
|
113
|
+
content_type: contentType,
|
|
114
|
+
});
|
|
115
|
+
},
|
|
116
|
+
};
|
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
import { GoogleAuth } from "google-auth-library";
|
|
2
|
-
import { exec } from "child_process";
|
|
3
|
-
import * as path from "path";
|
|
4
1
|
import { z } from "zod";
|
|
5
2
|
import { getStorage } from "../storage";
|
|
6
3
|
import { generateTimestampedFilename } from "../utils/filename";
|
|
@@ -57,10 +54,12 @@ export const imageToVideo = {
|
|
|
57
54
|
aspect_ratio: z
|
|
58
55
|
.string()
|
|
59
56
|
.optional()
|
|
60
|
-
.
|
|
57
|
+
.default("16:9")
|
|
58
|
+
.describe("Video aspect ratio: '16:9' or '9:16'"),
|
|
61
59
|
duration_seconds: z
|
|
62
60
|
.string()
|
|
63
61
|
.optional()
|
|
62
|
+
.default("6")
|
|
64
63
|
.describe(
|
|
65
64
|
"Video duration in seconds. MUST be one of: '4', '6', or '8' (default: '6'). Other values will be rejected by Vertex AI."
|
|
66
65
|
),
|
|
@@ -91,15 +90,18 @@ export const imageToVideo = {
|
|
|
91
90
|
project_id: z
|
|
92
91
|
.string()
|
|
93
92
|
.optional()
|
|
93
|
+
.default("mixio-pro")
|
|
94
94
|
.describe("GCP Project ID (default: mixio-pro)"),
|
|
95
95
|
location_id: z
|
|
96
96
|
.string()
|
|
97
97
|
.optional()
|
|
98
|
+
.default("us-central1")
|
|
98
99
|
.describe("Vertex region (default: us-central1)"),
|
|
99
100
|
model_id: z
|
|
100
101
|
.string()
|
|
101
102
|
.optional()
|
|
102
|
-
.
|
|
103
|
+
.default("veo-3.1-fast-generate-001")
|
|
104
|
+
.describe("Model ID (default: veo-3.1-fast-generate-001)"),
|
|
103
105
|
generate_audio: z
|
|
104
106
|
.boolean()
|
|
105
107
|
.optional()
|
package/src/utils/index.ts
CHANGED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export * from './fal.utils'
|
|
@@ -1,160 +0,0 @@
|
|
|
1
|
-
import { z } from "zod";
|
|
2
|
-
import { callFalModel } from "../utils/fal.utils";
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* Calculate number of frames based on audio duration at 25 FPS
|
|
6
|
-
* Adds 1 second buffer to ensure complete audio coverage
|
|
7
|
-
*/
|
|
8
|
-
function calculateFramesFromAudioDuration(
|
|
9
|
-
audioDurationSeconds: number
|
|
10
|
-
): number {
|
|
11
|
-
const totalDuration = audioDurationSeconds + 1; // Add 1 second buffer
|
|
12
|
-
const frames = Math.round(totalDuration * 25); // 25 FPS
|
|
13
|
-
|
|
14
|
-
// Clamp to valid range (129-401 frames)
|
|
15
|
-
return Math.max(129, Math.min(401, frames));
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
/**
|
|
19
|
-
* FAL AI Hunyuan Avatar - High-Fidelity Audio-Driven Human Animation
|
|
20
|
-
*/
|
|
21
|
-
export const hunyuanAvatar = {
|
|
22
|
-
name: "hunyuan_avatar",
|
|
23
|
-
description:
|
|
24
|
-
"Generate high-fidelity audio-driven human animation videos using FAL AI Hunyuan Avatar. Creates realistic talking avatar animations from an image and audio file.",
|
|
25
|
-
parameters: z.object({
|
|
26
|
-
image_url: z
|
|
27
|
-
.string()
|
|
28
|
-
.describe("Public URL of the reference image for the avatar."),
|
|
29
|
-
audio_url: z
|
|
30
|
-
.string()
|
|
31
|
-
.describe("Public URL of the audio file to drive the animation."),
|
|
32
|
-
audio_duration_seconds: z
|
|
33
|
-
.number()
|
|
34
|
-
.optional()
|
|
35
|
-
.describe(
|
|
36
|
-
"Duration of the audio in seconds. If provided, will automatically calculate optimal frames (audio duration + 1 second buffer at 25 FPS)."
|
|
37
|
-
),
|
|
38
|
-
text: z
|
|
39
|
-
.string()
|
|
40
|
-
.optional()
|
|
41
|
-
.describe(
|
|
42
|
-
"Text prompt describing the scene. Default: 'A cat is singing.'"
|
|
43
|
-
),
|
|
44
|
-
num_frames: z
|
|
45
|
-
.number()
|
|
46
|
-
.optional()
|
|
47
|
-
.describe(
|
|
48
|
-
"Number of video frames to generate at 25 FPS. Range: 129 to 401. If not provided and audio_duration_seconds is given, will be calculated automatically. Default: 129"
|
|
49
|
-
),
|
|
50
|
-
num_inference_steps: z
|
|
51
|
-
.number()
|
|
52
|
-
.optional()
|
|
53
|
-
.describe(
|
|
54
|
-
"Number of inference steps for sampling. Higher values give better quality but take longer. Range: 30 to 50. Default: 30"
|
|
55
|
-
),
|
|
56
|
-
turbo_mode: z
|
|
57
|
-
.boolean()
|
|
58
|
-
.optional()
|
|
59
|
-
.describe(
|
|
60
|
-
"If true, the video will be generated faster with no noticeable degradation in visual quality. Default: true"
|
|
61
|
-
),
|
|
62
|
-
seed: z.number().optional().describe("Random seed for generation."),
|
|
63
|
-
fal_key: z
|
|
64
|
-
.string()
|
|
65
|
-
.optional()
|
|
66
|
-
.describe(
|
|
67
|
-
"FAL API key. If not provided, will use FAL_KEY environment variable."
|
|
68
|
-
),
|
|
69
|
-
}),
|
|
70
|
-
execute: async (args: {
|
|
71
|
-
image_url: string;
|
|
72
|
-
audio_url: string;
|
|
73
|
-
audio_duration_seconds?: number;
|
|
74
|
-
text?: string;
|
|
75
|
-
num_frames?: number;
|
|
76
|
-
num_inference_steps?: number;
|
|
77
|
-
turbo_mode?: boolean;
|
|
78
|
-
seed?: number;
|
|
79
|
-
fal_key?: string;
|
|
80
|
-
}) => {
|
|
81
|
-
// Calculate frames from audio duration if provided and num_frames not specified
|
|
82
|
-
let calculatedFrames = args.num_frames;
|
|
83
|
-
if (
|
|
84
|
-
args.audio_duration_seconds !== undefined &&
|
|
85
|
-
args.num_frames === undefined
|
|
86
|
-
) {
|
|
87
|
-
calculatedFrames = calculateFramesFromAudioDuration(
|
|
88
|
-
args.audio_duration_seconds
|
|
89
|
-
);
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
// Validate num_frames range if provided
|
|
93
|
-
if (
|
|
94
|
-
calculatedFrames !== undefined &&
|
|
95
|
-
(calculatedFrames < 129 || calculatedFrames > 401)
|
|
96
|
-
) {
|
|
97
|
-
throw new Error("num_frames must be between 129 and 401");
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
// Validate num_inference_steps range if provided
|
|
101
|
-
if (
|
|
102
|
-
args.num_inference_steps !== undefined &&
|
|
103
|
-
(args.num_inference_steps < 30 || args.num_inference_steps > 50)
|
|
104
|
-
) {
|
|
105
|
-
throw new Error("num_inference_steps must be between 30 and 50");
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
// Build input payload
|
|
109
|
-
const input: any = {
|
|
110
|
-
image_url: args.image_url,
|
|
111
|
-
audio_url: args.audio_url,
|
|
112
|
-
};
|
|
113
|
-
|
|
114
|
-
// Add optional parameters if provided
|
|
115
|
-
if (args.text !== undefined) {
|
|
116
|
-
input.text = args.text;
|
|
117
|
-
}
|
|
118
|
-
if (calculatedFrames !== undefined) {
|
|
119
|
-
input.num_frames = calculatedFrames;
|
|
120
|
-
}
|
|
121
|
-
if (args.num_inference_steps !== undefined) {
|
|
122
|
-
input.num_inference_steps = args.num_inference_steps;
|
|
123
|
-
}
|
|
124
|
-
if (args.turbo_mode !== undefined) {
|
|
125
|
-
input.turbo_mode = args.turbo_mode;
|
|
126
|
-
}
|
|
127
|
-
if (args.seed !== undefined) {
|
|
128
|
-
input.seed = args.seed;
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
const result = await callFalModel("fal-ai/hunyuan-avatar", input, {
|
|
132
|
-
falKey: args.fal_key,
|
|
133
|
-
});
|
|
134
|
-
|
|
135
|
-
// Extract video data from the response
|
|
136
|
-
const videoData = result.data?.video;
|
|
137
|
-
|
|
138
|
-
if (!videoData || !videoData.url) {
|
|
139
|
-
throw new Error(
|
|
140
|
-
`No video data in completed response: ${JSON.stringify(result.data)}`
|
|
141
|
-
);
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
const videoUrl = videoData.url;
|
|
145
|
-
const fileName = videoData.file_name || "hunyuan_avatar.mp4";
|
|
146
|
-
|
|
147
|
-
return JSON.stringify({
|
|
148
|
-
videos: [
|
|
149
|
-
{
|
|
150
|
-
url: videoUrl,
|
|
151
|
-
filename: fileName,
|
|
152
|
-
mimeType: "video/mp4",
|
|
153
|
-
filesize: videoData.file_size,
|
|
154
|
-
},
|
|
155
|
-
],
|
|
156
|
-
message: "Hunyuan Avatar video generated successfully",
|
|
157
|
-
requestId: result.requestId,
|
|
158
|
-
});
|
|
159
|
-
},
|
|
160
|
-
};
|
package/src/tools/infinitalk.ts
DELETED
|
@@ -1,156 +0,0 @@
|
|
|
1
|
-
import { z } from "zod";
|
|
2
|
-
import { callFalModel } from "../utils/fal.utils";
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* Calculate number of frames based on audio duration at 25 FPS
|
|
6
|
-
* Adds 1 second buffer to ensure complete audio coverage
|
|
7
|
-
*/
|
|
8
|
-
function calculateFramesFromAudioDuration(
|
|
9
|
-
audioDurationSeconds: number
|
|
10
|
-
): number {
|
|
11
|
-
const totalDuration = audioDurationSeconds + 1; // Add 1 second buffer
|
|
12
|
-
const frames = Math.round(totalDuration * 25); // 25 FPS
|
|
13
|
-
|
|
14
|
-
// Clamp to valid range (41-721 frames)
|
|
15
|
-
return Math.max(41, Math.min(721, frames));
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
/**
|
|
19
|
-
* FAL AI Infinitalk - Generate talking avatar video from image and audio
|
|
20
|
-
*/
|
|
21
|
-
export const infinitalk = {
|
|
22
|
-
name: "infinitalk",
|
|
23
|
-
description:
|
|
24
|
-
"Generate a talking avatar video from an image and audio file using FAL AI Infinitalk. The avatar lip-syncs to the provided audio with natural facial expressions.",
|
|
25
|
-
parameters: z.object({
|
|
26
|
-
image_url: z
|
|
27
|
-
.string()
|
|
28
|
-
.describe(
|
|
29
|
-
"Public URL of the input image. If the input image does not match the chosen aspect ratio, it is resized and center cropped."
|
|
30
|
-
),
|
|
31
|
-
audio_url: z
|
|
32
|
-
.string()
|
|
33
|
-
.describe("The Public URL of the audio file for lip-sync generation."),
|
|
34
|
-
audio_duration_seconds: z
|
|
35
|
-
.number()
|
|
36
|
-
.optional()
|
|
37
|
-
.describe(
|
|
38
|
-
"Duration of the audio in seconds. If provided, will automatically calculate optimal frames (audio duration + 1 second buffer at 25 FPS)."
|
|
39
|
-
),
|
|
40
|
-
prompt: z
|
|
41
|
-
.string()
|
|
42
|
-
.describe(
|
|
43
|
-
"The text prompt to guide video generation (e.g., 'A woman with colorful hair talking on a podcast')"
|
|
44
|
-
),
|
|
45
|
-
num_frames: z
|
|
46
|
-
.number()
|
|
47
|
-
.optional()
|
|
48
|
-
.describe(
|
|
49
|
-
"Number of frames to generate. Must be between 41 to 721. If not provided and audio_duration_seconds is given, will be calculated automatically. Default: 145"
|
|
50
|
-
),
|
|
51
|
-
resolution: z
|
|
52
|
-
.enum(["480p", "720p"])
|
|
53
|
-
.optional()
|
|
54
|
-
.describe("Resolution of the video to generate. Default: '480p'"),
|
|
55
|
-
seed: z
|
|
56
|
-
.number()
|
|
57
|
-
.optional()
|
|
58
|
-
.describe(
|
|
59
|
-
"Random seed for reproducibility. If not provided, a random seed is chosen. Default: 42"
|
|
60
|
-
),
|
|
61
|
-
acceleration: z
|
|
62
|
-
.enum(["none", "regular", "high"])
|
|
63
|
-
.optional()
|
|
64
|
-
.describe(
|
|
65
|
-
"The acceleration level to use for generation. Default: 'regular'"
|
|
66
|
-
),
|
|
67
|
-
fal_key: z
|
|
68
|
-
.string()
|
|
69
|
-
.optional()
|
|
70
|
-
.describe(
|
|
71
|
-
"FAL API key. If not provided, will use FAL_KEY environment variable."
|
|
72
|
-
),
|
|
73
|
-
}),
|
|
74
|
-
execute: async (args: {
|
|
75
|
-
image_url: string;
|
|
76
|
-
audio_url: string;
|
|
77
|
-
audio_duration_seconds?: number;
|
|
78
|
-
prompt: string;
|
|
79
|
-
num_frames?: number;
|
|
80
|
-
resolution?: "480p" | "720p";
|
|
81
|
-
seed?: number;
|
|
82
|
-
acceleration?: "none" | "regular" | "high";
|
|
83
|
-
fal_key?: string;
|
|
84
|
-
}) => {
|
|
85
|
-
// Calculate frames from audio duration if provided and num_frames not specified
|
|
86
|
-
let calculatedFrames = args.num_frames;
|
|
87
|
-
if (
|
|
88
|
-
args.audio_duration_seconds !== undefined &&
|
|
89
|
-
args.num_frames === undefined
|
|
90
|
-
) {
|
|
91
|
-
calculatedFrames = calculateFramesFromAudioDuration(
|
|
92
|
-
args.audio_duration_seconds
|
|
93
|
-
);
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
// Validate num_frames range if provided
|
|
97
|
-
if (
|
|
98
|
-
calculatedFrames !== undefined &&
|
|
99
|
-
(calculatedFrames < 41 || calculatedFrames > 721)
|
|
100
|
-
) {
|
|
101
|
-
throw new Error("num_frames must be between 41 and 721");
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
// Build input payload
|
|
105
|
-
const input: any = {
|
|
106
|
-
image_url: args.image_url,
|
|
107
|
-
audio_url: args.audio_url,
|
|
108
|
-
prompt: args.prompt,
|
|
109
|
-
};
|
|
110
|
-
|
|
111
|
-
// Add optional parameters if provided
|
|
112
|
-
if (calculatedFrames !== undefined) {
|
|
113
|
-
input.num_frames = calculatedFrames;
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
input.resolution = args.resolution || "480p";
|
|
117
|
-
|
|
118
|
-
if (args.seed !== undefined) {
|
|
119
|
-
input.seed = args.seed;
|
|
120
|
-
}
|
|
121
|
-
if (args.acceleration !== undefined) {
|
|
122
|
-
input.acceleration = args.acceleration;
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
const result = await callFalModel("fal-ai/infinitalk", input, {
|
|
126
|
-
falKey: args.fal_key,
|
|
127
|
-
});
|
|
128
|
-
|
|
129
|
-
// Extract video data from the response
|
|
130
|
-
const videoData = result.data?.video;
|
|
131
|
-
const seed = result.data?.seed;
|
|
132
|
-
|
|
133
|
-
if (!videoData || !videoData.url) {
|
|
134
|
-
throw new Error(
|
|
135
|
-
`No video data in completed response: ${JSON.stringify(result.data)}`
|
|
136
|
-
);
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
const videoUrl = videoData.url;
|
|
140
|
-
const fileName = videoData.file_name || "infinitalk.mp4";
|
|
141
|
-
|
|
142
|
-
return JSON.stringify({
|
|
143
|
-
videos: [
|
|
144
|
-
{
|
|
145
|
-
url: videoUrl,
|
|
146
|
-
filename: fileName,
|
|
147
|
-
mimeType: "video/mp4",
|
|
148
|
-
filesize: videoData.file_size,
|
|
149
|
-
},
|
|
150
|
-
],
|
|
151
|
-
message: "Infinitalk video generated successfully",
|
|
152
|
-
seed: seed,
|
|
153
|
-
requestId: result.requestId,
|
|
154
|
-
});
|
|
155
|
-
},
|
|
156
|
-
};
|
package/src/utils/fal.utils.ts
DELETED
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
import { fal } from "@fal-ai/client";
|
|
2
|
-
|
|
3
|
-
export async function callFalModel(
|
|
4
|
-
modelName: string,
|
|
5
|
-
input: any,
|
|
6
|
-
options: { falKey?: string; logs?: boolean } = {}
|
|
7
|
-
) {
|
|
8
|
-
const { falKey, logs = true } = options;
|
|
9
|
-
const key = falKey || process.env.FAL_KEY;
|
|
10
|
-
if (!key) {
|
|
11
|
-
throw new Error(
|
|
12
|
-
"FAL_KEY is required. Provide it via fal_key parameter or FAL_KEY environment variable."
|
|
13
|
-
);
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
fal.config({
|
|
17
|
-
credentials: key,
|
|
18
|
-
});
|
|
19
|
-
|
|
20
|
-
console.error(`[${modelName}] Submitting request to FAL AI...`);
|
|
21
|
-
|
|
22
|
-
try {
|
|
23
|
-
const result = await fal.subscribe(modelName, {
|
|
24
|
-
input,
|
|
25
|
-
logs,
|
|
26
|
-
onQueueUpdate: (update) => {
|
|
27
|
-
if (update.status === "IN_PROGRESS") {
|
|
28
|
-
console.error(`[${modelName}] Status: ${update.status}`);
|
|
29
|
-
if (logs && "logs" in update && update.logs) {
|
|
30
|
-
update.logs.forEach((log) => {
|
|
31
|
-
console.error(`[${modelName}] ${log.message}`);
|
|
32
|
-
});
|
|
33
|
-
}
|
|
34
|
-
} else if (update.status === "IN_QUEUE") {
|
|
35
|
-
console.error(
|
|
36
|
-
`[${modelName}] Status: ${update.status} - Waiting in queue...`
|
|
37
|
-
);
|
|
38
|
-
}
|
|
39
|
-
},
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
console.error(`[${modelName}] Generation completed successfully`);
|
|
43
|
-
|
|
44
|
-
return result;
|
|
45
|
-
} catch (error: any) {
|
|
46
|
-
console.error(`[${modelName}] Error:`, error);
|
|
47
|
-
throw new Error(
|
|
48
|
-
`FAL AI ${modelName} generation failed: ${
|
|
49
|
-
error.message || JSON.stringify(error)
|
|
50
|
-
}`
|
|
51
|
-
);
|
|
52
|
-
}
|
|
53
|
-
}
|