@sogni-ai/sogni-client 4.2.0-alpha.2 → 4.2.0-alpha.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +148 -0
- package/CLAUDE.md +25 -3
- package/README.md +411 -136
- package/dist/Account/index.d.ts +4 -2
- package/dist/Account/index.js +27 -23
- package/dist/Account/index.js.map +1 -1
- package/dist/Account/types.d.ts +7 -0
- package/dist/ApiClient/WebSocketClient/BrowserWebSocketClient/index.d.ts +3 -1
- package/dist/ApiClient/WebSocketClient/BrowserWebSocketClient/index.js +26 -2
- package/dist/ApiClient/WebSocketClient/BrowserWebSocketClient/index.js.map +1 -1
- package/dist/ApiClient/WebSocketClient/eventSubscriptions.d.ts +33 -0
- package/dist/ApiClient/WebSocketClient/eventSubscriptions.js +39 -0
- package/dist/ApiClient/WebSocketClient/eventSubscriptions.js.map +1 -0
- package/dist/ApiClient/WebSocketClient/events.d.ts +24 -7
- package/dist/ApiClient/WebSocketClient/index.d.ts +5 -1
- package/dist/ApiClient/WebSocketClient/index.js +24 -1
- package/dist/ApiClient/WebSocketClient/index.js.map +1 -1
- package/dist/ApiClient/WebSocketClient/messages.d.ts +2 -0
- package/dist/ApiClient/WebSocketClient/types.d.ts +2 -0
- package/dist/ApiClient/index.d.ts +6 -1
- package/dist/ApiClient/index.js +7 -3
- package/dist/ApiClient/index.js.map +1 -1
- package/dist/Chat/ChatTools.d.ts +5 -49
- package/dist/Chat/ChatTools.js +311 -88
- package/dist/Chat/ChatTools.js.map +1 -1
- package/dist/Chat/index.d.ts +11 -2
- package/dist/Chat/index.js +78 -4
- package/dist/Chat/index.js.map +1 -1
- package/dist/Chat/modelRouting.d.ts +100 -0
- package/dist/Chat/modelRouting.js +441 -0
- package/dist/Chat/modelRouting.js.map +1 -0
- package/dist/Chat/sogniHostedTools.generated.json +529 -0
- package/dist/Chat/tools.d.ts +9 -55
- package/dist/Chat/tools.js +72 -228
- package/dist/Chat/tools.js.map +1 -1
- package/dist/Chat/types.d.ts +91 -2
- package/dist/CreativeWorkflows/index.d.ts +23 -0
- package/dist/CreativeWorkflows/index.js +274 -0
- package/dist/CreativeWorkflows/index.js.map +1 -0
- package/dist/CreativeWorkflows/types.d.ts +106 -0
- package/dist/CreativeWorkflows/types.js +3 -0
- package/dist/CreativeWorkflows/types.js.map +1 -0
- package/dist/Projects/Job.d.ts +6 -0
- package/dist/Projects/Job.js +60 -5
- package/dist/Projects/Job.js.map +1 -1
- package/dist/Projects/Project.js +15 -3
- package/dist/Projects/Project.js.map +1 -1
- package/dist/Projects/createJobRequestMessage.js +140 -6
- package/dist/Projects/createJobRequestMessage.js.map +1 -1
- package/dist/Projects/index.d.ts +10 -1
- package/dist/Projects/index.js +197 -58
- package/dist/Projects/index.js.map +1 -1
- package/dist/Projects/types/ModelOptions.d.ts +3 -3
- package/dist/Projects/types/ModelOptions.js +12 -5
- package/dist/Projects/types/ModelOptions.js.map +1 -1
- package/dist/Projects/types/ModelTiersRaw.d.ts +7 -7
- package/dist/Projects/types/RawProject.d.ts +2 -0
- package/dist/Projects/types/events.d.ts +5 -4
- package/dist/Projects/types/index.d.ts +77 -7
- package/dist/Projects/types/index.js.map +1 -1
- package/dist/Projects/utils/index.d.ts +8 -1
- package/dist/Projects/utils/index.js +22 -8
- package/dist/Projects/utils/index.js.map +1 -1
- package/dist/index.d.ts +28 -3
- package/dist/index.js +19 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/RestClient.d.ts +4 -1
- package/dist/lib/RestClient.js +17 -9
- package/dist/lib/RestClient.js.map +1 -1
- package/dist/lib/mediaValidation.d.ts +16 -0
- package/dist/lib/mediaValidation.js +280 -0
- package/dist/lib/mediaValidation.js.map +1 -0
- package/dist/lib/validation.d.ts +6 -1
- package/dist/lib/validation.js +28 -2
- package/dist/lib/validation.js.map +1 -1
- package/llms-full.txt +372 -133
- package/llms.txt +197 -86
- package/package.json +13 -4
- package/src/Account/index.ts +22 -2
- package/src/Account/types.ts +7 -0
- package/src/ApiClient/WebSocketClient/BrowserWebSocketClient/index.ts +47 -3
- package/src/ApiClient/WebSocketClient/eventSubscriptions.ts +92 -0
- package/src/ApiClient/WebSocketClient/events.ts +25 -7
- package/src/ApiClient/WebSocketClient/index.ts +33 -1
- package/src/ApiClient/WebSocketClient/messages.ts +2 -0
- package/src/ApiClient/WebSocketClient/types.ts +2 -0
- package/src/ApiClient/index.ts +32 -2
- package/src/Chat/ChatTools.ts +395 -95
- package/src/Chat/index.ts +149 -5
- package/src/Chat/modelRouting.ts +602 -0
- package/src/Chat/sogniHostedTools.generated.json +529 -0
- package/src/Chat/tools.ts +98 -245
- package/src/Chat/types.ts +100 -2
- package/src/CreativeWorkflows/index.ts +290 -0
- package/src/CreativeWorkflows/types.ts +134 -0
- package/src/Projects/Job.ts +76 -5
- package/src/Projects/Project.ts +13 -3
- package/src/Projects/createJobRequestMessage.ts +152 -13
- package/src/Projects/index.ts +230 -52
- package/src/Projects/types/ModelOptions.ts +15 -8
- package/src/Projects/types/ModelTiersRaw.ts +7 -7
- package/src/Projects/types/RawProject.ts +2 -0
- package/src/Projects/types/events.ts +5 -4
- package/src/Projects/types/index.ts +86 -6
- package/src/Projects/utils/index.ts +24 -8
- package/src/index.ts +93 -0
- package/src/lib/RestClient.ts +15 -5
- package/src/lib/mediaValidation.ts +367 -0
- package/src/lib/validation.ts +38 -2
package/src/Chat/ChatTools.ts
CHANGED
|
@@ -1,6 +1,26 @@
|
|
|
1
1
|
import type ProjectsApi from '../Projects';
|
|
2
2
|
import type { AvailableModel } from '../Projects/types';
|
|
3
|
-
import {
|
|
3
|
+
import { getMaxContextImages } from '../lib/validation';
|
|
4
|
+
import { parseInlineMediaDataUri } from '../lib/mediaValidation';
|
|
5
|
+
import type { MediaType } from '../lib/mediaValidation';
|
|
6
|
+
import {
|
|
7
|
+
assertHostedToolArguments,
|
|
8
|
+
asBooleanValue,
|
|
9
|
+
asFiniteNumber,
|
|
10
|
+
asStringArray,
|
|
11
|
+
getHostedVariationCount,
|
|
12
|
+
getVideoDefaults,
|
|
13
|
+
isEditImageModel,
|
|
14
|
+
isNonEmptyString,
|
|
15
|
+
normalizeTimeSignature,
|
|
16
|
+
normalizeVideoControlMode,
|
|
17
|
+
PREFERRED_MODEL_IDS,
|
|
18
|
+
resolveHostedToolModelSelector,
|
|
19
|
+
selectBackboneModel,
|
|
20
|
+
serializeUnknownError,
|
|
21
|
+
VideoWorkflow
|
|
22
|
+
} from './modelRouting';
|
|
23
|
+
import { SogniTools, isSogniToolCall, parseToolCallArguments } from './tools';
|
|
4
24
|
import {
|
|
5
25
|
ToolCall,
|
|
6
26
|
ToolExecutionOptions,
|
|
@@ -8,29 +28,43 @@ import {
|
|
|
8
28
|
ToolExecutionResult
|
|
9
29
|
} from './types';
|
|
10
30
|
|
|
11
|
-
|
|
12
|
-
const
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
31
|
+
const DEFAULT_TIMEOUT = 30 * 60 * 1000;
|
|
32
|
+
const MAX_SOGNI_TOOL_CALLS_PER_ROUND = 8;
|
|
33
|
+
|
|
34
|
+
const MAX_INPUT_MEDIA_BYTES: Record<MediaType, number> = {
|
|
35
|
+
image: 20 * 1024 * 1024,
|
|
36
|
+
audio: 50 * 1024 * 1024,
|
|
37
|
+
video: 100 * 1024 * 1024
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
function getVariationCount(args: Record<string, unknown>, options?: ToolExecutionOptions): number {
|
|
41
|
+
return getHostedVariationCount(args, options?.numberOfMedia);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function getStringArg(value: unknown): string | undefined {
|
|
45
|
+
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function normalizeImageOutputFormat(value: unknown): string | undefined {
|
|
49
|
+
const outputFormat = getStringArg(value)?.toLowerCase();
|
|
50
|
+
if (!outputFormat) return undefined;
|
|
51
|
+
if (outputFormat === 'jpeg') return 'jpg';
|
|
52
|
+
return outputFormat === 'png' || outputFormat === 'jpg' || outputFormat === 'webp'
|
|
53
|
+
? outputFormat
|
|
54
|
+
: undefined;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function applyHostedImageOptions(
|
|
58
|
+
projectParams: Record<string, unknown>,
|
|
59
|
+
args: Record<string, unknown>
|
|
60
|
+
) {
|
|
61
|
+
const gptImageQuality = getStringArg(args.gpt_image_quality ?? args.gptImageQuality);
|
|
62
|
+
if (gptImageQuality) projectParams.gptImageQuality = gptImageQuality.toLowerCase();
|
|
63
|
+
|
|
64
|
+
const outputFormat = normalizeImageOutputFormat(args.output_format ?? args.outputFormat);
|
|
65
|
+
if (outputFormat) projectParams.outputFormat = outputFormat;
|
|
66
|
+
}
|
|
67
|
+
|
|
34
68
|
class ChatToolsApi {
|
|
35
69
|
private projects: ProjectsApi;
|
|
36
70
|
|
|
@@ -38,14 +72,6 @@ class ChatToolsApi {
|
|
|
38
72
|
this.projects = projects;
|
|
39
73
|
}
|
|
40
74
|
|
|
41
|
-
/**
|
|
42
|
-
* Execute a single Sogni platform tool call.
|
|
43
|
-
*
|
|
44
|
-
* Maps tool call arguments to `sogni.projects.create()`, waits for the media
|
|
45
|
-
* generation to complete, and returns the result URLs.
|
|
46
|
-
*
|
|
47
|
-
* @throws Error if the tool call is not a Sogni tool (use `isSogniToolCall()` to check first)
|
|
48
|
-
*/
|
|
49
75
|
async execute(toolCall: ToolCall, options?: ToolExecutionOptions): Promise<ToolExecutionResult> {
|
|
50
76
|
if (!this.projects) {
|
|
51
77
|
throw new Error(
|
|
@@ -62,41 +88,44 @@ class ChatToolsApi {
|
|
|
62
88
|
const name = toolCall.function.name;
|
|
63
89
|
|
|
64
90
|
try {
|
|
91
|
+
assertHostedToolArguments(SogniTools.all, name, args);
|
|
92
|
+
|
|
65
93
|
switch (name) {
|
|
66
94
|
case 'sogni_generate_image':
|
|
67
95
|
return await this.executeImageGeneration(toolCall, args, options);
|
|
96
|
+
case 'sogni_edit_image':
|
|
97
|
+
return await this.executeImageEdit(toolCall, args, options);
|
|
68
98
|
case 'sogni_generate_video':
|
|
69
99
|
return await this.executeVideoGeneration(toolCall, args, options);
|
|
100
|
+
case 'sogni_sound_to_video':
|
|
101
|
+
return await this.executeSoundToVideo(toolCall, args, options);
|
|
102
|
+
case 'sogni_video_to_video':
|
|
103
|
+
return await this.executeVideoToVideo(toolCall, args, options);
|
|
70
104
|
case 'sogni_generate_music':
|
|
71
105
|
return await this.executeMusicGeneration(toolCall, args, options);
|
|
72
106
|
default:
|
|
73
107
|
return this.makeErrorResult(toolCall, `Unknown Sogni tool: ${name}`);
|
|
74
108
|
}
|
|
75
109
|
} catch (err) {
|
|
76
|
-
const error =
|
|
110
|
+
const error = serializeUnknownError(err);
|
|
77
111
|
return this.makeErrorResult(toolCall, error);
|
|
78
112
|
}
|
|
79
113
|
}
|
|
80
114
|
|
|
81
|
-
/**
|
|
82
|
-
* Execute multiple tool calls from a single LLM response.
|
|
83
|
-
*
|
|
84
|
-
* Sogni tool calls (prefixed with `sogni_`) are executed automatically via
|
|
85
|
-
* `projects.create()`. Non-Sogni tool calls are delegated to the `onToolCall`
|
|
86
|
-
* callback if provided, or returned as errors.
|
|
87
|
-
*
|
|
88
|
-
* @param toolCalls - Array of tool calls from `result.tool_calls`
|
|
89
|
-
* @param options - Execution options plus optional handler for non-Sogni tools
|
|
90
|
-
*/
|
|
91
115
|
async executeAll(
|
|
92
116
|
toolCalls: ToolCall[],
|
|
93
117
|
options?: ToolExecutionOptions & {
|
|
94
|
-
/** Handler for non-Sogni tool calls. Must return the tool result content string. */
|
|
95
118
|
onToolCall?: (toolCall: ToolCall) => Promise<string>;
|
|
96
|
-
/** Per-tool progress callback (wraps the per-tool onProgress with tool identity). */
|
|
97
119
|
onToolProgress?: (toolCall: ToolCall, progress: ToolExecutionProgress) => void;
|
|
98
120
|
}
|
|
99
121
|
): Promise<ToolExecutionResult[]> {
|
|
122
|
+
const sogniToolCallCount = toolCalls.filter((toolCall) => isSogniToolCall(toolCall)).length;
|
|
123
|
+
if (sogniToolCallCount > MAX_SOGNI_TOOL_CALLS_PER_ROUND) {
|
|
124
|
+
throw new Error(
|
|
125
|
+
`Too many Sogni tool calls in a single round (${sogniToolCallCount}); maximum is ${MAX_SOGNI_TOOL_CALLS_PER_ROUND}`
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
|
|
100
129
|
const results: ToolExecutionResult[] = [];
|
|
101
130
|
|
|
102
131
|
for (const toolCall of toolCalls) {
|
|
@@ -122,7 +151,7 @@ class ChatToolsApi {
|
|
|
122
151
|
content
|
|
123
152
|
});
|
|
124
153
|
} catch (err) {
|
|
125
|
-
const error =
|
|
154
|
+
const error = serializeUnknownError(err);
|
|
126
155
|
results.push(this.makeErrorResult(toolCall, error));
|
|
127
156
|
}
|
|
128
157
|
} else {
|
|
@@ -138,38 +167,24 @@ class ChatToolsApi {
|
|
|
138
167
|
return results;
|
|
139
168
|
}
|
|
140
169
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
*/
|
|
145
|
-
private async getDefaultModel(mediaType: 'image' | 'video' | 'audio'): Promise<string> {
|
|
146
|
-
const models: AvailableModel[] = await this.projects.waitForModels(10000);
|
|
147
|
-
const candidates = models.filter((m) => m.media === mediaType);
|
|
148
|
-
if (candidates.length === 0) {
|
|
149
|
-
throw new Error(`No ${mediaType} models currently available on the network`);
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
// For video, prefer LTX-2.3 text-to-video models
|
|
153
|
-
if (mediaType === 'video') {
|
|
154
|
-
const ltx2 = candidates.filter((m) => m.id.includes('ltx2') && m.id.includes('t2v'));
|
|
155
|
-
if (ltx2.length > 0) {
|
|
156
|
-
ltx2.sort((a, b) => b.workerCount - a.workerCount);
|
|
157
|
-
return ltx2[0].id;
|
|
158
|
-
}
|
|
159
|
-
}
|
|
170
|
+
private async getAvailableModels(): Promise<AvailableModel[]> {
|
|
171
|
+
return this.projects.waitForModels(10000);
|
|
172
|
+
}
|
|
160
173
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
174
|
+
private async selectModel(options: {
|
|
175
|
+
mediaType: MediaType;
|
|
176
|
+
requestedModel?: string;
|
|
177
|
+
workflows?: VideoWorkflow[];
|
|
178
|
+
filter?: (modelId: string) => boolean;
|
|
179
|
+
preferredModelIds?: string[];
|
|
180
|
+
}): Promise<string> {
|
|
181
|
+
const models = await this.getAvailableModels();
|
|
182
|
+
return selectBackboneModel(models, options).modelId;
|
|
164
183
|
}
|
|
165
184
|
|
|
166
|
-
/**
|
|
167
|
-
* Create a project, wait for completion with timeout, track per-job progress,
|
|
168
|
-
* and clean up on failure or timeout.
|
|
169
|
-
*/
|
|
170
185
|
private async executeProject(
|
|
171
186
|
toolCall: ToolCall,
|
|
172
|
-
mediaType:
|
|
187
|
+
mediaType: MediaType,
|
|
173
188
|
modelId: string,
|
|
174
189
|
projectParams: Record<string, unknown>,
|
|
175
190
|
prompt: string,
|
|
@@ -184,7 +199,6 @@ class ChatToolsApi {
|
|
|
184
199
|
let jobsFailed = 0;
|
|
185
200
|
const totalJobs = (projectParams.numberOfMedia as number) || 1;
|
|
186
201
|
|
|
187
|
-
// Track per-job progress via project events
|
|
188
202
|
const onJobCompleted = () => {
|
|
189
203
|
jobsCompleted++;
|
|
190
204
|
const percent = Math.round((jobsCompleted / totalJobs) * 100);
|
|
@@ -194,7 +208,6 @@ class ChatToolsApi {
|
|
|
194
208
|
jobsFailed++;
|
|
195
209
|
};
|
|
196
210
|
|
|
197
|
-
// Track step-level progress via project progress event
|
|
198
211
|
const onProgress = (percent: number) => {
|
|
199
212
|
options?.onProgress?.({
|
|
200
213
|
status: 'processing',
|
|
@@ -240,15 +253,13 @@ class ChatToolsApi {
|
|
|
240
253
|
})
|
|
241
254
|
};
|
|
242
255
|
} catch (err) {
|
|
243
|
-
// Attempt to cancel the project on timeout to free worker resources
|
|
244
256
|
try {
|
|
245
257
|
await project.cancel();
|
|
246
258
|
} catch {
|
|
247
|
-
|
|
259
|
+
// best-effort cleanup
|
|
248
260
|
}
|
|
249
261
|
|
|
250
262
|
options?.onProgress?.({ status: 'failed', percent: 0 });
|
|
251
|
-
|
|
252
263
|
throw err;
|
|
253
264
|
} finally {
|
|
254
265
|
if (timeoutId !== null) clearTimeout(timeoutId);
|
|
@@ -263,14 +274,18 @@ class ChatToolsApi {
|
|
|
263
274
|
args: Record<string, unknown>,
|
|
264
275
|
options?: ToolExecutionOptions
|
|
265
276
|
): Promise<ToolExecutionResult> {
|
|
266
|
-
const modelId =
|
|
277
|
+
const modelId = await this.selectModel({
|
|
278
|
+
mediaType: 'image',
|
|
279
|
+
requestedModel: resolveHostedToolModelSelector('sogni_generate_image', args)
|
|
280
|
+
});
|
|
267
281
|
|
|
268
282
|
const projectParams: Record<string, unknown> = {
|
|
269
283
|
type: 'image' as const,
|
|
270
284
|
modelId,
|
|
271
285
|
positivePrompt: args.prompt as string,
|
|
272
|
-
numberOfMedia: options
|
|
286
|
+
numberOfMedia: getVariationCount(args, options)
|
|
273
287
|
};
|
|
288
|
+
|
|
274
289
|
if (args.negative_prompt) projectParams.negativePrompt = args.negative_prompt;
|
|
275
290
|
if (args.width && args.height) {
|
|
276
291
|
projectParams.width = args.width;
|
|
@@ -279,6 +294,7 @@ class ChatToolsApi {
|
|
|
279
294
|
}
|
|
280
295
|
if (args.steps !== undefined) projectParams.steps = args.steps;
|
|
281
296
|
if (args.seed !== undefined) projectParams.seed = args.seed;
|
|
297
|
+
applyHostedImageOptions(projectParams, args);
|
|
282
298
|
if (options?.tokenType) projectParams.tokenType = options.tokenType;
|
|
283
299
|
if (options?.network) projectParams.network = options.network;
|
|
284
300
|
|
|
@@ -292,31 +308,293 @@ class ChatToolsApi {
|
|
|
292
308
|
);
|
|
293
309
|
}
|
|
294
310
|
|
|
295
|
-
private async
|
|
311
|
+
private async executeImageEdit(
|
|
296
312
|
toolCall: ToolCall,
|
|
297
313
|
args: Record<string, unknown>,
|
|
298
314
|
options?: ToolExecutionOptions
|
|
299
315
|
): Promise<ToolExecutionResult> {
|
|
300
|
-
const
|
|
316
|
+
const sourceImageUrl = isNonEmptyString(args.source_image_url) ? args.source_image_url : null;
|
|
317
|
+
const referenceImageUrls = asStringArray(args.reference_image_urls);
|
|
318
|
+
const inputUrls = [...(sourceImageUrl ? [sourceImageUrl] : []), ...referenceImageUrls];
|
|
319
|
+
|
|
320
|
+
if (inputUrls.length === 0) {
|
|
321
|
+
throw new Error('sogni_edit_image requires source_image_url or reference_image_urls');
|
|
322
|
+
}
|
|
301
323
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
324
|
+
const modelId = await this.selectModel({
|
|
325
|
+
mediaType: 'image',
|
|
326
|
+
requestedModel: resolveHostedToolModelSelector('sogni_edit_image', args),
|
|
327
|
+
filter: isEditImageModel
|
|
328
|
+
});
|
|
329
|
+
const maxContextImages = getMaxContextImages(modelId);
|
|
330
|
+
const contextImages = await Promise.all(
|
|
331
|
+
inputUrls.slice(0, maxContextImages).map(
|
|
332
|
+
(url) =>
|
|
333
|
+
parseInlineMediaDataUri(url, 'image', {
|
|
334
|
+
maxBytes: MAX_INPUT_MEDIA_BYTES.image
|
|
335
|
+
}).blob
|
|
336
|
+
)
|
|
337
|
+
);
|
|
307
338
|
|
|
308
339
|
const projectParams: Record<string, unknown> = {
|
|
309
|
-
type: '
|
|
340
|
+
type: 'image' as const,
|
|
310
341
|
modelId,
|
|
311
342
|
positivePrompt: args.prompt as string,
|
|
312
|
-
numberOfMedia: options
|
|
313
|
-
|
|
314
|
-
height: (args.height as number) || defaultHeight,
|
|
315
|
-
fps: (args.fps as number) || defaultFps
|
|
343
|
+
numberOfMedia: getVariationCount(args, options),
|
|
344
|
+
contextImages
|
|
316
345
|
};
|
|
346
|
+
|
|
317
347
|
if (args.negative_prompt) projectParams.negativePrompt = args.negative_prompt;
|
|
348
|
+
if (args.width && args.height) {
|
|
349
|
+
projectParams.width = args.width;
|
|
350
|
+
projectParams.height = args.height;
|
|
351
|
+
projectParams.sizePreset = 'custom';
|
|
352
|
+
}
|
|
353
|
+
if (args.seed !== undefined) projectParams.seed = args.seed;
|
|
354
|
+
applyHostedImageOptions(projectParams, args);
|
|
355
|
+
if (options?.tokenType) projectParams.tokenType = options.tokenType;
|
|
356
|
+
if (options?.network) projectParams.network = options.network;
|
|
357
|
+
|
|
358
|
+
return this.executeProject(
|
|
359
|
+
toolCall,
|
|
360
|
+
'image',
|
|
361
|
+
modelId,
|
|
362
|
+
projectParams,
|
|
363
|
+
args.prompt as string,
|
|
364
|
+
options
|
|
365
|
+
);
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
private async executeVideoGeneration(
|
|
369
|
+
toolCall: ToolCall,
|
|
370
|
+
args: Record<string, unknown>,
|
|
371
|
+
options?: ToolExecutionOptions
|
|
372
|
+
): Promise<ToolExecutionResult> {
|
|
373
|
+
const hasReferenceImages =
|
|
374
|
+
isNonEmptyString(args.reference_image_url) || isNonEmptyString(args.reference_image_end_url);
|
|
375
|
+
const workflowPreference: VideoWorkflow[] = hasReferenceImages ? ['i2v'] : ['t2v'];
|
|
376
|
+
const preferredModelIds = hasReferenceImages
|
|
377
|
+
? [PREFERRED_MODEL_IDS.video.i2v]
|
|
378
|
+
: [PREFERRED_MODEL_IDS.video.t2v];
|
|
379
|
+
|
|
380
|
+
const modelId = await this.selectModel({
|
|
381
|
+
mediaType: 'video',
|
|
382
|
+
requestedModel: resolveHostedToolModelSelector('sogni_generate_video', args),
|
|
383
|
+
workflows: workflowPreference,
|
|
384
|
+
preferredModelIds
|
|
385
|
+
});
|
|
386
|
+
const defaults = getVideoDefaults(modelId);
|
|
387
|
+
const isSeedanceModel = modelId.startsWith('seedance-2-0');
|
|
388
|
+
|
|
389
|
+
const projectParams: Record<string, unknown> = {
|
|
390
|
+
type: 'video' as const,
|
|
391
|
+
modelId,
|
|
392
|
+
positivePrompt: args.prompt as string,
|
|
393
|
+
numberOfMedia: getVariationCount(args, options),
|
|
394
|
+
width: (args.width as number) || defaults.width,
|
|
395
|
+
height: (args.height as number) || defaults.height,
|
|
396
|
+
fps: (args.fps as number) || defaults.fps
|
|
397
|
+
};
|
|
398
|
+
|
|
399
|
+
if (args.negative_prompt && !isSeedanceModel) {
|
|
400
|
+
projectParams.negativePrompt = args.negative_prompt;
|
|
401
|
+
}
|
|
318
402
|
if (args.duration !== undefined) projectParams.duration = args.duration;
|
|
319
403
|
if (args.seed !== undefined) projectParams.seed = args.seed;
|
|
404
|
+
if (isNonEmptyString(args.reference_image_url)) {
|
|
405
|
+
projectParams.referenceImage = parseInlineMediaDataUri(args.reference_image_url, 'image', {
|
|
406
|
+
maxBytes: MAX_INPUT_MEDIA_BYTES.image
|
|
407
|
+
}).blob;
|
|
408
|
+
}
|
|
409
|
+
if (isNonEmptyString(args.reference_image_end_url)) {
|
|
410
|
+
projectParams.referenceImageEnd = parseInlineMediaDataUri(
|
|
411
|
+
args.reference_image_end_url,
|
|
412
|
+
'image',
|
|
413
|
+
{
|
|
414
|
+
maxBytes: MAX_INPUT_MEDIA_BYTES.image
|
|
415
|
+
}
|
|
416
|
+
).blob;
|
|
417
|
+
}
|
|
418
|
+
if (isNonEmptyString(args.reference_audio_identity_url)) {
|
|
419
|
+
projectParams.referenceAudioIdentity = parseInlineMediaDataUri(
|
|
420
|
+
args.reference_audio_identity_url,
|
|
421
|
+
'audio',
|
|
422
|
+
{
|
|
423
|
+
maxBytes: MAX_INPUT_MEDIA_BYTES.audio
|
|
424
|
+
}
|
|
425
|
+
).blob;
|
|
426
|
+
}
|
|
427
|
+
if (args.audio_identity_strength !== undefined) {
|
|
428
|
+
projectParams.audioIdentityStrength = args.audio_identity_strength;
|
|
429
|
+
}
|
|
430
|
+
if (args.first_frame_strength !== undefined) {
|
|
431
|
+
projectParams.firstFrameStrength = args.first_frame_strength;
|
|
432
|
+
}
|
|
433
|
+
if (args.last_frame_strength !== undefined) {
|
|
434
|
+
projectParams.lastFrameStrength = args.last_frame_strength;
|
|
435
|
+
}
|
|
436
|
+
if (args.generate_audio !== undefined) {
|
|
437
|
+
projectParams.generateAudio = Boolean(args.generate_audio);
|
|
438
|
+
}
|
|
439
|
+
if (options?.tokenType) projectParams.tokenType = options.tokenType;
|
|
440
|
+
if (options?.network) projectParams.network = options.network;
|
|
441
|
+
|
|
442
|
+
return this.executeProject(
|
|
443
|
+
toolCall,
|
|
444
|
+
'video',
|
|
445
|
+
modelId,
|
|
446
|
+
projectParams,
|
|
447
|
+
args.prompt as string,
|
|
448
|
+
options
|
|
449
|
+
);
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
private async executeSoundToVideo(
|
|
453
|
+
toolCall: ToolCall,
|
|
454
|
+
args: Record<string, unknown>,
|
|
455
|
+
options?: ToolExecutionOptions
|
|
456
|
+
): Promise<ToolExecutionResult> {
|
|
457
|
+
if (!isNonEmptyString(args.reference_audio_url)) {
|
|
458
|
+
throw new Error('sogni_sound_to_video requires reference_audio_url');
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
const hasReferenceImage = isNonEmptyString(args.reference_image_url);
|
|
462
|
+
const workflows: VideoWorkflow[] = hasReferenceImage ? ['ia2v', 's2v'] : ['a2v'];
|
|
463
|
+
const preferredModelIds = hasReferenceImage
|
|
464
|
+
? [PREFERRED_MODEL_IDS.video.ia2v, PREFERRED_MODEL_IDS.video.s2v]
|
|
465
|
+
: [PREFERRED_MODEL_IDS.video.a2v];
|
|
466
|
+
const modelId = await this.selectModel({
|
|
467
|
+
mediaType: 'video',
|
|
468
|
+
requestedModel: resolveHostedToolModelSelector('sogni_sound_to_video', args),
|
|
469
|
+
workflows,
|
|
470
|
+
preferredModelIds
|
|
471
|
+
});
|
|
472
|
+
const defaults = getVideoDefaults(modelId);
|
|
473
|
+
const duration = asFiniteNumber(args.duration) ?? 5;
|
|
474
|
+
|
|
475
|
+
const projectParams: Record<string, unknown> = {
|
|
476
|
+
type: 'video' as const,
|
|
477
|
+
modelId,
|
|
478
|
+
positivePrompt: args.prompt as string,
|
|
479
|
+
numberOfMedia: getVariationCount(args, options),
|
|
480
|
+
referenceAudio: parseInlineMediaDataUri(args.reference_audio_url, 'audio', {
|
|
481
|
+
maxBytes: MAX_INPUT_MEDIA_BYTES.audio
|
|
482
|
+
}).blob,
|
|
483
|
+
width: (args.width as number) || defaults.width,
|
|
484
|
+
height: (args.height as number) || defaults.height,
|
|
485
|
+
fps: defaults.fps,
|
|
486
|
+
duration,
|
|
487
|
+
audioDuration: duration
|
|
488
|
+
};
|
|
489
|
+
|
|
490
|
+
if (isNonEmptyString(args.reference_image_url)) {
|
|
491
|
+
projectParams.referenceImage = parseInlineMediaDataUri(args.reference_image_url, 'image', {
|
|
492
|
+
maxBytes: MAX_INPUT_MEDIA_BYTES.image
|
|
493
|
+
}).blob;
|
|
494
|
+
}
|
|
495
|
+
if (args.audio_start !== undefined) projectParams.audioStart = args.audio_start;
|
|
496
|
+
if (args.generate_audio !== undefined) {
|
|
497
|
+
projectParams.generateAudio = Boolean(args.generate_audio);
|
|
498
|
+
}
|
|
499
|
+
if (args.seed !== undefined) projectParams.seed = args.seed;
|
|
500
|
+
if (options?.tokenType) projectParams.tokenType = options.tokenType;
|
|
501
|
+
if (options?.network) projectParams.network = options.network;
|
|
502
|
+
|
|
503
|
+
return this.executeProject(
|
|
504
|
+
toolCall,
|
|
505
|
+
'video',
|
|
506
|
+
modelId,
|
|
507
|
+
projectParams,
|
|
508
|
+
args.prompt as string,
|
|
509
|
+
options
|
|
510
|
+
);
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
private async executeVideoToVideo(
|
|
514
|
+
toolCall: ToolCall,
|
|
515
|
+
args: Record<string, unknown>,
|
|
516
|
+
options?: ToolExecutionOptions
|
|
517
|
+
): Promise<ToolExecutionResult> {
|
|
518
|
+
if (!isNonEmptyString(args.reference_video_url)) {
|
|
519
|
+
throw new Error('sogni_video_to_video requires reference_video_url');
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
const controlMode = normalizeVideoControlMode(args.control_mode);
|
|
523
|
+
const isAnimateMode = controlMode === 'animate-move' || controlMode === 'animate-replace';
|
|
524
|
+
const isSeedanceMode = controlMode === 'seedance-v2v';
|
|
525
|
+
const workflows: VideoWorkflow[] = isAnimateMode ? [controlMode] : ['v2v'];
|
|
526
|
+
const preferredModelIds = isAnimateMode
|
|
527
|
+
? [
|
|
528
|
+
controlMode === 'animate-move'
|
|
529
|
+
? PREFERRED_MODEL_IDS.video.animateMove
|
|
530
|
+
: PREFERRED_MODEL_IDS.video.animateReplace
|
|
531
|
+
]
|
|
532
|
+
: isSeedanceMode
|
|
533
|
+
? [PREFERRED_MODEL_IDS.video.seedanceV2v, PREFERRED_MODEL_IDS.video.v2v]
|
|
534
|
+
: [PREFERRED_MODEL_IDS.video.v2v];
|
|
535
|
+
const modelId = await this.selectModel({
|
|
536
|
+
mediaType: 'video',
|
|
537
|
+
requestedModel: resolveHostedToolModelSelector('sogni_video_to_video', args),
|
|
538
|
+
workflows,
|
|
539
|
+
preferredModelIds
|
|
540
|
+
});
|
|
541
|
+
const defaults = getVideoDefaults(modelId);
|
|
542
|
+
const isSeedanceModel = modelId.startsWith('seedance-2-0');
|
|
543
|
+
|
|
544
|
+
if (isAnimateMode && !isNonEmptyString(args.reference_image_url)) {
|
|
545
|
+
throw new Error(`${controlMode} requires reference_image_url`);
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
const projectParams: Record<string, unknown> = {
|
|
549
|
+
type: 'video' as const,
|
|
550
|
+
modelId,
|
|
551
|
+
positivePrompt: args.prompt as string,
|
|
552
|
+
numberOfMedia: getVariationCount(args, options),
|
|
553
|
+
referenceVideo: parseInlineMediaDataUri(args.reference_video_url, 'video', {
|
|
554
|
+
maxBytes: MAX_INPUT_MEDIA_BYTES.video
|
|
555
|
+
}).blob,
|
|
556
|
+
width: (args.width as number) || defaults.width,
|
|
557
|
+
height: (args.height as number) || defaults.height,
|
|
558
|
+
fps: defaults.fps,
|
|
559
|
+
duration: asFiniteNumber(args.duration) ?? 5
|
|
560
|
+
};
|
|
561
|
+
|
|
562
|
+
if (args.negative_prompt && !isSeedanceModel) {
|
|
563
|
+
projectParams.negativePrompt = args.negative_prompt;
|
|
564
|
+
}
|
|
565
|
+
if (args.seed !== undefined) projectParams.seed = args.seed;
|
|
566
|
+
if (isNonEmptyString(args.reference_image_url)) {
|
|
567
|
+
projectParams.referenceImage = parseInlineMediaDataUri(args.reference_image_url, 'image', {
|
|
568
|
+
maxBytes: MAX_INPUT_MEDIA_BYTES.image
|
|
569
|
+
}).blob;
|
|
570
|
+
}
|
|
571
|
+
if (isNonEmptyString(args.reference_audio_identity_url)) {
|
|
572
|
+
projectParams.referenceAudioIdentity = parseInlineMediaDataUri(
|
|
573
|
+
args.reference_audio_identity_url,
|
|
574
|
+
'audio',
|
|
575
|
+
{
|
|
576
|
+
maxBytes: MAX_INPUT_MEDIA_BYTES.audio
|
|
577
|
+
}
|
|
578
|
+
).blob;
|
|
579
|
+
}
|
|
580
|
+
if (args.audio_identity_strength !== undefined) {
|
|
581
|
+
projectParams.audioIdentityStrength = args.audio_identity_strength;
|
|
582
|
+
}
|
|
583
|
+
if (args.video_start !== undefined) {
|
|
584
|
+
projectParams.videoStart = args.video_start;
|
|
585
|
+
}
|
|
586
|
+
if (args.generate_audio !== undefined) {
|
|
587
|
+
projectParams.generateAudio = Boolean(args.generate_audio);
|
|
588
|
+
}
|
|
589
|
+
if (!isAnimateMode && !isSeedanceModel) {
|
|
590
|
+
projectParams.controlNet = {
|
|
591
|
+
name: controlMode,
|
|
592
|
+
strength: controlMode === 'detailer' ? 1 : 0.85
|
|
593
|
+
};
|
|
594
|
+
}
|
|
595
|
+
if (!isSeedanceModel && args.detailer_strength !== undefined) {
|
|
596
|
+
projectParams.detailerStrength = args.detailer_strength;
|
|
597
|
+
}
|
|
320
598
|
if (options?.tokenType) projectParams.tokenType = options.tokenType;
|
|
321
599
|
if (options?.network) projectParams.network = options.network;
|
|
322
600
|
|
|
@@ -335,19 +613,41 @@ class ChatToolsApi {
|
|
|
335
613
|
args: Record<string, unknown>,
|
|
336
614
|
options?: ToolExecutionOptions
|
|
337
615
|
): Promise<ToolExecutionResult> {
|
|
338
|
-
const modelId =
|
|
616
|
+
const modelId = await this.selectModel({
|
|
617
|
+
mediaType: 'audio',
|
|
618
|
+
requestedModel: resolveHostedToolModelSelector('sogni_generate_music', args),
|
|
619
|
+
preferredModelIds: [
|
|
620
|
+
PREFERRED_MODEL_IDS.audio.aceStepTurbo,
|
|
621
|
+
PREFERRED_MODEL_IDS.audio.aceStepSft
|
|
622
|
+
]
|
|
623
|
+
});
|
|
339
624
|
|
|
340
625
|
const projectParams: Record<string, unknown> = {
|
|
341
626
|
type: 'audio' as const,
|
|
342
627
|
modelId,
|
|
343
628
|
positivePrompt: args.prompt as string,
|
|
344
|
-
numberOfMedia: options
|
|
629
|
+
numberOfMedia: getVariationCount(args, options)
|
|
345
630
|
};
|
|
631
|
+
|
|
346
632
|
if (args.duration !== undefined) projectParams.duration = args.duration;
|
|
347
633
|
if (args.bpm !== undefined) projectParams.bpm = args.bpm;
|
|
348
634
|
if (args.keyscale) projectParams.keyscale = args.keyscale;
|
|
349
|
-
if (args.
|
|
635
|
+
if (args.lyrics) projectParams.lyrics = args.lyrics;
|
|
636
|
+
if (args.language) projectParams.language = args.language;
|
|
350
637
|
if (args.output_format) projectParams.outputFormat = args.output_format;
|
|
638
|
+
|
|
639
|
+
const timeSignature = normalizeTimeSignature(args.timesignature);
|
|
640
|
+
if (timeSignature) projectParams.timesignature = timeSignature;
|
|
641
|
+
|
|
642
|
+
const composerMode = asBooleanValue(args.composer_mode);
|
|
643
|
+
if (composerMode !== undefined) projectParams.composerMode = composerMode;
|
|
644
|
+
|
|
645
|
+
const promptStrength = asFiniteNumber(args.prompt_strength);
|
|
646
|
+
if (promptStrength !== undefined) projectParams.promptStrength = promptStrength;
|
|
647
|
+
|
|
648
|
+
const creativity = asFiniteNumber(args.creativity);
|
|
649
|
+
if (creativity !== undefined) projectParams.creativity = creativity;
|
|
650
|
+
|
|
351
651
|
if (args.seed !== undefined) projectParams.seed = args.seed;
|
|
352
652
|
if (options?.tokenType) projectParams.tokenType = options.tokenType;
|
|
353
653
|
if (options?.network) projectParams.network = options.network;
|