@sogni-ai/sogni-client 4.2.0-alpha.2 → 4.2.0-alpha.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/CHANGELOG.md +148 -0
  2. package/CLAUDE.md +25 -3
  3. package/README.md +411 -136
  4. package/dist/Account/index.d.ts +4 -2
  5. package/dist/Account/index.js +27 -23
  6. package/dist/Account/index.js.map +1 -1
  7. package/dist/Account/types.d.ts +7 -0
  8. package/dist/ApiClient/WebSocketClient/BrowserWebSocketClient/index.d.ts +3 -1
  9. package/dist/ApiClient/WebSocketClient/BrowserWebSocketClient/index.js +26 -2
  10. package/dist/ApiClient/WebSocketClient/BrowserWebSocketClient/index.js.map +1 -1
  11. package/dist/ApiClient/WebSocketClient/eventSubscriptions.d.ts +33 -0
  12. package/dist/ApiClient/WebSocketClient/eventSubscriptions.js +39 -0
  13. package/dist/ApiClient/WebSocketClient/eventSubscriptions.js.map +1 -0
  14. package/dist/ApiClient/WebSocketClient/events.d.ts +24 -7
  15. package/dist/ApiClient/WebSocketClient/index.d.ts +5 -1
  16. package/dist/ApiClient/WebSocketClient/index.js +24 -1
  17. package/dist/ApiClient/WebSocketClient/index.js.map +1 -1
  18. package/dist/ApiClient/WebSocketClient/messages.d.ts +2 -0
  19. package/dist/ApiClient/WebSocketClient/types.d.ts +2 -0
  20. package/dist/ApiClient/index.d.ts +6 -1
  21. package/dist/ApiClient/index.js +7 -3
  22. package/dist/ApiClient/index.js.map +1 -1
  23. package/dist/Chat/ChatTools.d.ts +5 -49
  24. package/dist/Chat/ChatTools.js +311 -88
  25. package/dist/Chat/ChatTools.js.map +1 -1
  26. package/dist/Chat/index.d.ts +11 -2
  27. package/dist/Chat/index.js +78 -4
  28. package/dist/Chat/index.js.map +1 -1
  29. package/dist/Chat/modelRouting.d.ts +100 -0
  30. package/dist/Chat/modelRouting.js +441 -0
  31. package/dist/Chat/modelRouting.js.map +1 -0
  32. package/dist/Chat/sogniHostedTools.generated.json +529 -0
  33. package/dist/Chat/tools.d.ts +9 -55
  34. package/dist/Chat/tools.js +72 -228
  35. package/dist/Chat/tools.js.map +1 -1
  36. package/dist/Chat/types.d.ts +91 -2
  37. package/dist/CreativeWorkflows/index.d.ts +23 -0
  38. package/dist/CreativeWorkflows/index.js +274 -0
  39. package/dist/CreativeWorkflows/index.js.map +1 -0
  40. package/dist/CreativeWorkflows/types.d.ts +106 -0
  41. package/dist/CreativeWorkflows/types.js +3 -0
  42. package/dist/CreativeWorkflows/types.js.map +1 -0
  43. package/dist/Projects/Job.d.ts +6 -0
  44. package/dist/Projects/Job.js +60 -5
  45. package/dist/Projects/Job.js.map +1 -1
  46. package/dist/Projects/Project.js +15 -3
  47. package/dist/Projects/Project.js.map +1 -1
  48. package/dist/Projects/createJobRequestMessage.js +140 -6
  49. package/dist/Projects/createJobRequestMessage.js.map +1 -1
  50. package/dist/Projects/index.d.ts +10 -1
  51. package/dist/Projects/index.js +197 -58
  52. package/dist/Projects/index.js.map +1 -1
  53. package/dist/Projects/types/ModelOptions.d.ts +3 -3
  54. package/dist/Projects/types/ModelOptions.js +12 -5
  55. package/dist/Projects/types/ModelOptions.js.map +1 -1
  56. package/dist/Projects/types/ModelTiersRaw.d.ts +7 -7
  57. package/dist/Projects/types/RawProject.d.ts +2 -0
  58. package/dist/Projects/types/events.d.ts +5 -4
  59. package/dist/Projects/types/index.d.ts +77 -7
  60. package/dist/Projects/types/index.js.map +1 -1
  61. package/dist/Projects/utils/index.d.ts +8 -1
  62. package/dist/Projects/utils/index.js +22 -8
  63. package/dist/Projects/utils/index.js.map +1 -1
  64. package/dist/index.d.ts +28 -3
  65. package/dist/index.js +19 -1
  66. package/dist/index.js.map +1 -1
  67. package/dist/lib/RestClient.d.ts +4 -1
  68. package/dist/lib/RestClient.js +17 -9
  69. package/dist/lib/RestClient.js.map +1 -1
  70. package/dist/lib/mediaValidation.d.ts +16 -0
  71. package/dist/lib/mediaValidation.js +280 -0
  72. package/dist/lib/mediaValidation.js.map +1 -0
  73. package/dist/lib/validation.d.ts +6 -1
  74. package/dist/lib/validation.js +28 -2
  75. package/dist/lib/validation.js.map +1 -1
  76. package/llms-full.txt +372 -133
  77. package/llms.txt +197 -86
  78. package/package.json +13 -4
  79. package/src/Account/index.ts +22 -2
  80. package/src/Account/types.ts +7 -0
  81. package/src/ApiClient/WebSocketClient/BrowserWebSocketClient/index.ts +47 -3
  82. package/src/ApiClient/WebSocketClient/eventSubscriptions.ts +92 -0
  83. package/src/ApiClient/WebSocketClient/events.ts +25 -7
  84. package/src/ApiClient/WebSocketClient/index.ts +33 -1
  85. package/src/ApiClient/WebSocketClient/messages.ts +2 -0
  86. package/src/ApiClient/WebSocketClient/types.ts +2 -0
  87. package/src/ApiClient/index.ts +32 -2
  88. package/src/Chat/ChatTools.ts +395 -95
  89. package/src/Chat/index.ts +149 -5
  90. package/src/Chat/modelRouting.ts +602 -0
  91. package/src/Chat/sogniHostedTools.generated.json +529 -0
  92. package/src/Chat/tools.ts +98 -245
  93. package/src/Chat/types.ts +100 -2
  94. package/src/CreativeWorkflows/index.ts +290 -0
  95. package/src/CreativeWorkflows/types.ts +134 -0
  96. package/src/Projects/Job.ts +76 -5
  97. package/src/Projects/Project.ts +13 -3
  98. package/src/Projects/createJobRequestMessage.ts +152 -13
  99. package/src/Projects/index.ts +230 -52
  100. package/src/Projects/types/ModelOptions.ts +15 -8
  101. package/src/Projects/types/ModelTiersRaw.ts +7 -7
  102. package/src/Projects/types/RawProject.ts +2 -0
  103. package/src/Projects/types/events.ts +5 -4
  104. package/src/Projects/types/index.ts +86 -6
  105. package/src/Projects/utils/index.ts +24 -8
  106. package/src/index.ts +93 -0
  107. package/src/lib/RestClient.ts +15 -5
  108. package/src/lib/mediaValidation.ts +367 -0
  109. package/src/lib/validation.ts +38 -2
@@ -1,6 +1,26 @@
1
1
  import type ProjectsApi from '../Projects';
2
2
  import type { AvailableModel } from '../Projects/types';
3
- import { isSogniToolCall, parseToolCallArguments } from './tools';
3
+ import { getMaxContextImages } from '../lib/validation';
4
+ import { parseInlineMediaDataUri } from '../lib/mediaValidation';
5
+ import type { MediaType } from '../lib/mediaValidation';
6
+ import {
7
+ assertHostedToolArguments,
8
+ asBooleanValue,
9
+ asFiniteNumber,
10
+ asStringArray,
11
+ getHostedVariationCount,
12
+ getVideoDefaults,
13
+ isEditImageModel,
14
+ isNonEmptyString,
15
+ normalizeTimeSignature,
16
+ normalizeVideoControlMode,
17
+ PREFERRED_MODEL_IDS,
18
+ resolveHostedToolModelSelector,
19
+ selectBackboneModel,
20
+ serializeUnknownError,
21
+ VideoWorkflow
22
+ } from './modelRouting';
23
+ import { SogniTools, isSogniToolCall, parseToolCallArguments } from './tools';
4
24
  import {
5
25
  ToolCall,
6
26
  ToolExecutionOptions,
@@ -8,29 +28,43 @@ import {
8
28
  ToolExecutionResult
9
29
  } from './types';
10
30
 
11
- /** Default timeout for media generation: 10 minutes. */
12
- const DEFAULT_TIMEOUT = 10 * 60 * 1000;
13
-
14
- /**
15
- * API for executing Sogni platform tool calls (image, video, music generation).
16
- *
17
- * Accessed via `sogni.chat.tools`. Provides methods to execute tool calls returned
18
- * by the LLM, mapping them to `sogni.projects.create()` calls automatically.
19
- *
20
- * @example
21
- * ```typescript
22
- * // Execute a single tool call
23
- * const result = await sogni.chat.tools.execute(toolCall, {
24
- * tokenType: 'sogni',
25
- * onProgress: (p) => console.log(`${p.status}: ${p.percent}%`),
26
- * });
27
- *
28
- * // Execute all tool calls from a completion
29
- * const results = await sogni.chat.tools.executeAll(result.tool_calls, {
30
- * onToolCall: async (tc) => myCustomHandler(tc), // for non-Sogni tools
31
- * });
32
- * ```
33
- */
31
+ const DEFAULT_TIMEOUT = 30 * 60 * 1000;
32
+ const MAX_SOGNI_TOOL_CALLS_PER_ROUND = 8;
33
+
34
+ const MAX_INPUT_MEDIA_BYTES: Record<MediaType, number> = {
35
+ image: 20 * 1024 * 1024,
36
+ audio: 50 * 1024 * 1024,
37
+ video: 100 * 1024 * 1024
38
+ };
39
+
40
+ function getVariationCount(args: Record<string, unknown>, options?: ToolExecutionOptions): number {
41
+ return getHostedVariationCount(args, options?.numberOfMedia);
42
+ }
43
+
44
+ function getStringArg(value: unknown): string | undefined {
45
+ return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
46
+ }
47
+
48
+ function normalizeImageOutputFormat(value: unknown): string | undefined {
49
+ const outputFormat = getStringArg(value)?.toLowerCase();
50
+ if (!outputFormat) return undefined;
51
+ if (outputFormat === 'jpeg') return 'jpg';
52
+ return outputFormat === 'png' || outputFormat === 'jpg' || outputFormat === 'webp'
53
+ ? outputFormat
54
+ : undefined;
55
+ }
56
+
57
+ function applyHostedImageOptions(
58
+ projectParams: Record<string, unknown>,
59
+ args: Record<string, unknown>
60
+ ) {
61
+ const gptImageQuality = getStringArg(args.gpt_image_quality ?? args.gptImageQuality);
62
+ if (gptImageQuality) projectParams.gptImageQuality = gptImageQuality.toLowerCase();
63
+
64
+ const outputFormat = normalizeImageOutputFormat(args.output_format ?? args.outputFormat);
65
+ if (outputFormat) projectParams.outputFormat = outputFormat;
66
+ }
67
+
34
68
  class ChatToolsApi {
35
69
  private projects: ProjectsApi;
36
70
 
@@ -38,14 +72,6 @@ class ChatToolsApi {
38
72
  this.projects = projects;
39
73
  }
40
74
 
41
- /**
42
- * Execute a single Sogni platform tool call.
43
- *
44
- * Maps tool call arguments to `sogni.projects.create()`, waits for the media
45
- * generation to complete, and returns the result URLs.
46
- *
47
- * @throws Error if the tool call is not a Sogni tool (use `isSogniToolCall()` to check first)
48
- */
49
75
  async execute(toolCall: ToolCall, options?: ToolExecutionOptions): Promise<ToolExecutionResult> {
50
76
  if (!this.projects) {
51
77
  throw new Error(
@@ -62,41 +88,44 @@ class ChatToolsApi {
62
88
  const name = toolCall.function.name;
63
89
 
64
90
  try {
91
+ assertHostedToolArguments(SogniTools.all, name, args);
92
+
65
93
  switch (name) {
66
94
  case 'sogni_generate_image':
67
95
  return await this.executeImageGeneration(toolCall, args, options);
96
+ case 'sogni_edit_image':
97
+ return await this.executeImageEdit(toolCall, args, options);
68
98
  case 'sogni_generate_video':
69
99
  return await this.executeVideoGeneration(toolCall, args, options);
100
+ case 'sogni_sound_to_video':
101
+ return await this.executeSoundToVideo(toolCall, args, options);
102
+ case 'sogni_video_to_video':
103
+ return await this.executeVideoToVideo(toolCall, args, options);
70
104
  case 'sogni_generate_music':
71
105
  return await this.executeMusicGeneration(toolCall, args, options);
72
106
  default:
73
107
  return this.makeErrorResult(toolCall, `Unknown Sogni tool: ${name}`);
74
108
  }
75
109
  } catch (err) {
76
- const error = err instanceof Error ? err.message : String(err);
110
+ const error = serializeUnknownError(err);
77
111
  return this.makeErrorResult(toolCall, error);
78
112
  }
79
113
  }
80
114
 
81
- /**
82
- * Execute multiple tool calls from a single LLM response.
83
- *
84
- * Sogni tool calls (prefixed with `sogni_`) are executed automatically via
85
- * `projects.create()`. Non-Sogni tool calls are delegated to the `onToolCall`
86
- * callback if provided, or returned as errors.
87
- *
88
- * @param toolCalls - Array of tool calls from `result.tool_calls`
89
- * @param options - Execution options plus optional handler for non-Sogni tools
90
- */
91
115
  async executeAll(
92
116
  toolCalls: ToolCall[],
93
117
  options?: ToolExecutionOptions & {
94
- /** Handler for non-Sogni tool calls. Must return the tool result content string. */
95
118
  onToolCall?: (toolCall: ToolCall) => Promise<string>;
96
- /** Per-tool progress callback (wraps the per-tool onProgress with tool identity). */
97
119
  onToolProgress?: (toolCall: ToolCall, progress: ToolExecutionProgress) => void;
98
120
  }
99
121
  ): Promise<ToolExecutionResult[]> {
122
+ const sogniToolCallCount = toolCalls.filter((toolCall) => isSogniToolCall(toolCall)).length;
123
+ if (sogniToolCallCount > MAX_SOGNI_TOOL_CALLS_PER_ROUND) {
124
+ throw new Error(
125
+ `Too many Sogni tool calls in a single round (${sogniToolCallCount}); maximum is ${MAX_SOGNI_TOOL_CALLS_PER_ROUND}`
126
+ );
127
+ }
128
+
100
129
  const results: ToolExecutionResult[] = [];
101
130
 
102
131
  for (const toolCall of toolCalls) {
@@ -122,7 +151,7 @@ class ChatToolsApi {
122
151
  content
123
152
  });
124
153
  } catch (err) {
125
- const error = err instanceof Error ? err.message : String(err);
154
+ const error = serializeUnknownError(err);
126
155
  results.push(this.makeErrorResult(toolCall, error));
127
156
  }
128
157
  } else {
@@ -138,38 +167,24 @@ class ChatToolsApi {
138
167
  return results;
139
168
  }
140
169
 
141
- /**
142
- * Get the default model for a media type. For video, prefers LTX-2.3 t2v models.
143
- * Falls back to the model with the most available workers.
144
- */
145
- private async getDefaultModel(mediaType: 'image' | 'video' | 'audio'): Promise<string> {
146
- const models: AvailableModel[] = await this.projects.waitForModels(10000);
147
- const candidates = models.filter((m) => m.media === mediaType);
148
- if (candidates.length === 0) {
149
- throw new Error(`No ${mediaType} models currently available on the network`);
150
- }
151
-
152
- // For video, prefer LTX-2.3 text-to-video models
153
- if (mediaType === 'video') {
154
- const ltx2 = candidates.filter((m) => m.id.includes('ltx2') && m.id.includes('t2v'));
155
- if (ltx2.length > 0) {
156
- ltx2.sort((a, b) => b.workerCount - a.workerCount);
157
- return ltx2[0].id;
158
- }
159
- }
170
+ private async getAvailableModels(): Promise<AvailableModel[]> {
171
+ return this.projects.waitForModels(10000);
172
+ }
160
173
 
161
- // Default: pick the model with most workers (highest availability)
162
- candidates.sort((a, b) => b.workerCount - a.workerCount);
163
- return candidates[0].id;
174
+ private async selectModel(options: {
175
+ mediaType: MediaType;
176
+ requestedModel?: string;
177
+ workflows?: VideoWorkflow[];
178
+ filter?: (modelId: string) => boolean;
179
+ preferredModelIds?: string[];
180
+ }): Promise<string> {
181
+ const models = await this.getAvailableModels();
182
+ return selectBackboneModel(models, options).modelId;
164
183
  }
165
184
 
166
- /**
167
- * Create a project, wait for completion with timeout, track per-job progress,
168
- * and clean up on failure or timeout.
169
- */
170
185
  private async executeProject(
171
186
  toolCall: ToolCall,
172
- mediaType: 'image' | 'video' | 'audio',
187
+ mediaType: MediaType,
173
188
  modelId: string,
174
189
  projectParams: Record<string, unknown>,
175
190
  prompt: string,
@@ -184,7 +199,6 @@ class ChatToolsApi {
184
199
  let jobsFailed = 0;
185
200
  const totalJobs = (projectParams.numberOfMedia as number) || 1;
186
201
 
187
- // Track per-job progress via project events
188
202
  const onJobCompleted = () => {
189
203
  jobsCompleted++;
190
204
  const percent = Math.round((jobsCompleted / totalJobs) * 100);
@@ -194,7 +208,6 @@ class ChatToolsApi {
194
208
  jobsFailed++;
195
209
  };
196
210
 
197
- // Track step-level progress via project progress event
198
211
  const onProgress = (percent: number) => {
199
212
  options?.onProgress?.({
200
213
  status: 'processing',
@@ -240,15 +253,13 @@ class ChatToolsApi {
240
253
  })
241
254
  };
242
255
  } catch (err) {
243
- // Attempt to cancel the project on timeout to free worker resources
244
256
  try {
245
257
  await project.cancel();
246
258
  } catch {
247
- /* best-effort */
259
+ // best-effort cleanup
248
260
  }
249
261
 
250
262
  options?.onProgress?.({ status: 'failed', percent: 0 });
251
-
252
263
  throw err;
253
264
  } finally {
254
265
  if (timeoutId !== null) clearTimeout(timeoutId);
@@ -263,14 +274,18 @@ class ChatToolsApi {
263
274
  args: Record<string, unknown>,
264
275
  options?: ToolExecutionOptions
265
276
  ): Promise<ToolExecutionResult> {
266
- const modelId = (args.model as string) || (await this.getDefaultModel('image'));
277
+ const modelId = await this.selectModel({
278
+ mediaType: 'image',
279
+ requestedModel: resolveHostedToolModelSelector('sogni_generate_image', args)
280
+ });
267
281
 
268
282
  const projectParams: Record<string, unknown> = {
269
283
  type: 'image' as const,
270
284
  modelId,
271
285
  positivePrompt: args.prompt as string,
272
- numberOfMedia: options?.numberOfMedia || 1
286
+ numberOfMedia: getVariationCount(args, options)
273
287
  };
288
+
274
289
  if (args.negative_prompt) projectParams.negativePrompt = args.negative_prompt;
275
290
  if (args.width && args.height) {
276
291
  projectParams.width = args.width;
@@ -279,6 +294,7 @@ class ChatToolsApi {
279
294
  }
280
295
  if (args.steps !== undefined) projectParams.steps = args.steps;
281
296
  if (args.seed !== undefined) projectParams.seed = args.seed;
297
+ applyHostedImageOptions(projectParams, args);
282
298
  if (options?.tokenType) projectParams.tokenType = options.tokenType;
283
299
  if (options?.network) projectParams.network = options.network;
284
300
 
@@ -292,31 +308,293 @@ class ChatToolsApi {
292
308
  );
293
309
  }
294
310
 
295
- private async executeVideoGeneration(
311
+ private async executeImageEdit(
296
312
  toolCall: ToolCall,
297
313
  args: Record<string, unknown>,
298
314
  options?: ToolExecutionOptions
299
315
  ): Promise<ToolExecutionResult> {
300
- const modelId = (args.model as string) || (await this.getDefaultModel('video'));
316
+ const sourceImageUrl = isNonEmptyString(args.source_image_url) ? args.source_image_url : null;
317
+ const referenceImageUrls = asStringArray(args.reference_image_urls);
318
+ const inputUrls = [...(sourceImageUrl ? [sourceImageUrl] : []), ...referenceImageUrls];
319
+
320
+ if (inputUrls.length === 0) {
321
+ throw new Error('sogni_edit_image requires source_image_url or reference_image_urls');
322
+ }
301
323
 
302
- // LTX-2.3 defaults: 1920x1088 landscape, 24fps
303
- const isLtx2 = modelId.includes('ltx2');
304
- const defaultWidth = isLtx2 ? 1920 : 848;
305
- const defaultHeight = isLtx2 ? 1088 : 480;
306
- const defaultFps = isLtx2 ? 24 : 16;
324
+ const modelId = await this.selectModel({
325
+ mediaType: 'image',
326
+ requestedModel: resolveHostedToolModelSelector('sogni_edit_image', args),
327
+ filter: isEditImageModel
328
+ });
329
+ const maxContextImages = getMaxContextImages(modelId);
330
+ const contextImages = await Promise.all(
331
+ inputUrls.slice(0, maxContextImages).map(
332
+ (url) =>
333
+ parseInlineMediaDataUri(url, 'image', {
334
+ maxBytes: MAX_INPUT_MEDIA_BYTES.image
335
+ }).blob
336
+ )
337
+ );
307
338
 
308
339
  const projectParams: Record<string, unknown> = {
309
- type: 'video' as const,
340
+ type: 'image' as const,
310
341
  modelId,
311
342
  positivePrompt: args.prompt as string,
312
- numberOfMedia: options?.numberOfMedia || 1,
313
- width: (args.width as number) || defaultWidth,
314
- height: (args.height as number) || defaultHeight,
315
- fps: (args.fps as number) || defaultFps
343
+ numberOfMedia: getVariationCount(args, options),
344
+ contextImages
316
345
  };
346
+
317
347
  if (args.negative_prompt) projectParams.negativePrompt = args.negative_prompt;
348
+ if (args.width && args.height) {
349
+ projectParams.width = args.width;
350
+ projectParams.height = args.height;
351
+ projectParams.sizePreset = 'custom';
352
+ }
353
+ if (args.seed !== undefined) projectParams.seed = args.seed;
354
+ applyHostedImageOptions(projectParams, args);
355
+ if (options?.tokenType) projectParams.tokenType = options.tokenType;
356
+ if (options?.network) projectParams.network = options.network;
357
+
358
+ return this.executeProject(
359
+ toolCall,
360
+ 'image',
361
+ modelId,
362
+ projectParams,
363
+ args.prompt as string,
364
+ options
365
+ );
366
+ }
367
+
368
+ private async executeVideoGeneration(
369
+ toolCall: ToolCall,
370
+ args: Record<string, unknown>,
371
+ options?: ToolExecutionOptions
372
+ ): Promise<ToolExecutionResult> {
373
+ const hasReferenceImages =
374
+ isNonEmptyString(args.reference_image_url) || isNonEmptyString(args.reference_image_end_url);
375
+ const workflowPreference: VideoWorkflow[] = hasReferenceImages ? ['i2v'] : ['t2v'];
376
+ const preferredModelIds = hasReferenceImages
377
+ ? [PREFERRED_MODEL_IDS.video.i2v]
378
+ : [PREFERRED_MODEL_IDS.video.t2v];
379
+
380
+ const modelId = await this.selectModel({
381
+ mediaType: 'video',
382
+ requestedModel: resolveHostedToolModelSelector('sogni_generate_video', args),
383
+ workflows: workflowPreference,
384
+ preferredModelIds
385
+ });
386
+ const defaults = getVideoDefaults(modelId);
387
+ const isSeedanceModel = modelId.startsWith('seedance-2-0');
388
+
389
+ const projectParams: Record<string, unknown> = {
390
+ type: 'video' as const,
391
+ modelId,
392
+ positivePrompt: args.prompt as string,
393
+ numberOfMedia: getVariationCount(args, options),
394
+ width: (args.width as number) || defaults.width,
395
+ height: (args.height as number) || defaults.height,
396
+ fps: (args.fps as number) || defaults.fps
397
+ };
398
+
399
+ if (args.negative_prompt && !isSeedanceModel) {
400
+ projectParams.negativePrompt = args.negative_prompt;
401
+ }
318
402
  if (args.duration !== undefined) projectParams.duration = args.duration;
319
403
  if (args.seed !== undefined) projectParams.seed = args.seed;
404
+ if (isNonEmptyString(args.reference_image_url)) {
405
+ projectParams.referenceImage = parseInlineMediaDataUri(args.reference_image_url, 'image', {
406
+ maxBytes: MAX_INPUT_MEDIA_BYTES.image
407
+ }).blob;
408
+ }
409
+ if (isNonEmptyString(args.reference_image_end_url)) {
410
+ projectParams.referenceImageEnd = parseInlineMediaDataUri(
411
+ args.reference_image_end_url,
412
+ 'image',
413
+ {
414
+ maxBytes: MAX_INPUT_MEDIA_BYTES.image
415
+ }
416
+ ).blob;
417
+ }
418
+ if (isNonEmptyString(args.reference_audio_identity_url)) {
419
+ projectParams.referenceAudioIdentity = parseInlineMediaDataUri(
420
+ args.reference_audio_identity_url,
421
+ 'audio',
422
+ {
423
+ maxBytes: MAX_INPUT_MEDIA_BYTES.audio
424
+ }
425
+ ).blob;
426
+ }
427
+ if (args.audio_identity_strength !== undefined) {
428
+ projectParams.audioIdentityStrength = args.audio_identity_strength;
429
+ }
430
+ if (args.first_frame_strength !== undefined) {
431
+ projectParams.firstFrameStrength = args.first_frame_strength;
432
+ }
433
+ if (args.last_frame_strength !== undefined) {
434
+ projectParams.lastFrameStrength = args.last_frame_strength;
435
+ }
436
+ if (args.generate_audio !== undefined) {
437
+ projectParams.generateAudio = Boolean(args.generate_audio);
438
+ }
439
+ if (options?.tokenType) projectParams.tokenType = options.tokenType;
440
+ if (options?.network) projectParams.network = options.network;
441
+
442
+ return this.executeProject(
443
+ toolCall,
444
+ 'video',
445
+ modelId,
446
+ projectParams,
447
+ args.prompt as string,
448
+ options
449
+ );
450
+ }
451
+
452
+ private async executeSoundToVideo(
453
+ toolCall: ToolCall,
454
+ args: Record<string, unknown>,
455
+ options?: ToolExecutionOptions
456
+ ): Promise<ToolExecutionResult> {
457
+ if (!isNonEmptyString(args.reference_audio_url)) {
458
+ throw new Error('sogni_sound_to_video requires reference_audio_url');
459
+ }
460
+
461
+ const hasReferenceImage = isNonEmptyString(args.reference_image_url);
462
+ const workflows: VideoWorkflow[] = hasReferenceImage ? ['ia2v', 's2v'] : ['a2v'];
463
+ const preferredModelIds = hasReferenceImage
464
+ ? [PREFERRED_MODEL_IDS.video.ia2v, PREFERRED_MODEL_IDS.video.s2v]
465
+ : [PREFERRED_MODEL_IDS.video.a2v];
466
+ const modelId = await this.selectModel({
467
+ mediaType: 'video',
468
+ requestedModel: resolveHostedToolModelSelector('sogni_sound_to_video', args),
469
+ workflows,
470
+ preferredModelIds
471
+ });
472
+ const defaults = getVideoDefaults(modelId);
473
+ const duration = asFiniteNumber(args.duration) ?? 5;
474
+
475
+ const projectParams: Record<string, unknown> = {
476
+ type: 'video' as const,
477
+ modelId,
478
+ positivePrompt: args.prompt as string,
479
+ numberOfMedia: getVariationCount(args, options),
480
+ referenceAudio: parseInlineMediaDataUri(args.reference_audio_url, 'audio', {
481
+ maxBytes: MAX_INPUT_MEDIA_BYTES.audio
482
+ }).blob,
483
+ width: (args.width as number) || defaults.width,
484
+ height: (args.height as number) || defaults.height,
485
+ fps: defaults.fps,
486
+ duration,
487
+ audioDuration: duration
488
+ };
489
+
490
+ if (isNonEmptyString(args.reference_image_url)) {
491
+ projectParams.referenceImage = parseInlineMediaDataUri(args.reference_image_url, 'image', {
492
+ maxBytes: MAX_INPUT_MEDIA_BYTES.image
493
+ }).blob;
494
+ }
495
+ if (args.audio_start !== undefined) projectParams.audioStart = args.audio_start;
496
+ if (args.generate_audio !== undefined) {
497
+ projectParams.generateAudio = Boolean(args.generate_audio);
498
+ }
499
+ if (args.seed !== undefined) projectParams.seed = args.seed;
500
+ if (options?.tokenType) projectParams.tokenType = options.tokenType;
501
+ if (options?.network) projectParams.network = options.network;
502
+
503
+ return this.executeProject(
504
+ toolCall,
505
+ 'video',
506
+ modelId,
507
+ projectParams,
508
+ args.prompt as string,
509
+ options
510
+ );
511
+ }
512
+
513
+ private async executeVideoToVideo(
514
+ toolCall: ToolCall,
515
+ args: Record<string, unknown>,
516
+ options?: ToolExecutionOptions
517
+ ): Promise<ToolExecutionResult> {
518
+ if (!isNonEmptyString(args.reference_video_url)) {
519
+ throw new Error('sogni_video_to_video requires reference_video_url');
520
+ }
521
+
522
+ const controlMode = normalizeVideoControlMode(args.control_mode);
523
+ const isAnimateMode = controlMode === 'animate-move' || controlMode === 'animate-replace';
524
+ const isSeedanceMode = controlMode === 'seedance-v2v';
525
+ const workflows: VideoWorkflow[] = isAnimateMode ? [controlMode] : ['v2v'];
526
+ const preferredModelIds = isAnimateMode
527
+ ? [
528
+ controlMode === 'animate-move'
529
+ ? PREFERRED_MODEL_IDS.video.animateMove
530
+ : PREFERRED_MODEL_IDS.video.animateReplace
531
+ ]
532
+ : isSeedanceMode
533
+ ? [PREFERRED_MODEL_IDS.video.seedanceV2v, PREFERRED_MODEL_IDS.video.v2v]
534
+ : [PREFERRED_MODEL_IDS.video.v2v];
535
+ const modelId = await this.selectModel({
536
+ mediaType: 'video',
537
+ requestedModel: resolveHostedToolModelSelector('sogni_video_to_video', args),
538
+ workflows,
539
+ preferredModelIds
540
+ });
541
+ const defaults = getVideoDefaults(modelId);
542
+ const isSeedanceModel = modelId.startsWith('seedance-2-0');
543
+
544
+ if (isAnimateMode && !isNonEmptyString(args.reference_image_url)) {
545
+ throw new Error(`${controlMode} requires reference_image_url`);
546
+ }
547
+
548
+ const projectParams: Record<string, unknown> = {
549
+ type: 'video' as const,
550
+ modelId,
551
+ positivePrompt: args.prompt as string,
552
+ numberOfMedia: getVariationCount(args, options),
553
+ referenceVideo: parseInlineMediaDataUri(args.reference_video_url, 'video', {
554
+ maxBytes: MAX_INPUT_MEDIA_BYTES.video
555
+ }).blob,
556
+ width: (args.width as number) || defaults.width,
557
+ height: (args.height as number) || defaults.height,
558
+ fps: defaults.fps,
559
+ duration: asFiniteNumber(args.duration) ?? 5
560
+ };
561
+
562
+ if (args.negative_prompt && !isSeedanceModel) {
563
+ projectParams.negativePrompt = args.negative_prompt;
564
+ }
565
+ if (args.seed !== undefined) projectParams.seed = args.seed;
566
+ if (isNonEmptyString(args.reference_image_url)) {
567
+ projectParams.referenceImage = parseInlineMediaDataUri(args.reference_image_url, 'image', {
568
+ maxBytes: MAX_INPUT_MEDIA_BYTES.image
569
+ }).blob;
570
+ }
571
+ if (isNonEmptyString(args.reference_audio_identity_url)) {
572
+ projectParams.referenceAudioIdentity = parseInlineMediaDataUri(
573
+ args.reference_audio_identity_url,
574
+ 'audio',
575
+ {
576
+ maxBytes: MAX_INPUT_MEDIA_BYTES.audio
577
+ }
578
+ ).blob;
579
+ }
580
+ if (args.audio_identity_strength !== undefined) {
581
+ projectParams.audioIdentityStrength = args.audio_identity_strength;
582
+ }
583
+ if (args.video_start !== undefined) {
584
+ projectParams.videoStart = args.video_start;
585
+ }
586
+ if (args.generate_audio !== undefined) {
587
+ projectParams.generateAudio = Boolean(args.generate_audio);
588
+ }
589
+ if (!isAnimateMode && !isSeedanceModel) {
590
+ projectParams.controlNet = {
591
+ name: controlMode,
592
+ strength: controlMode === 'detailer' ? 1 : 0.85
593
+ };
594
+ }
595
+ if (!isSeedanceModel && args.detailer_strength !== undefined) {
596
+ projectParams.detailerStrength = args.detailer_strength;
597
+ }
320
598
  if (options?.tokenType) projectParams.tokenType = options.tokenType;
321
599
  if (options?.network) projectParams.network = options.network;
322
600
 
@@ -335,19 +613,41 @@ class ChatToolsApi {
335
613
  args: Record<string, unknown>,
336
614
  options?: ToolExecutionOptions
337
615
  ): Promise<ToolExecutionResult> {
338
- const modelId = (args.model as string) || (await this.getDefaultModel('audio'));
616
+ const modelId = await this.selectModel({
617
+ mediaType: 'audio',
618
+ requestedModel: resolveHostedToolModelSelector('sogni_generate_music', args),
619
+ preferredModelIds: [
620
+ PREFERRED_MODEL_IDS.audio.aceStepTurbo,
621
+ PREFERRED_MODEL_IDS.audio.aceStepSft
622
+ ]
623
+ });
339
624
 
340
625
  const projectParams: Record<string, unknown> = {
341
626
  type: 'audio' as const,
342
627
  modelId,
343
628
  positivePrompt: args.prompt as string,
344
- numberOfMedia: options?.numberOfMedia || 1
629
+ numberOfMedia: getVariationCount(args, options)
345
630
  };
631
+
346
632
  if (args.duration !== undefined) projectParams.duration = args.duration;
347
633
  if (args.bpm !== undefined) projectParams.bpm = args.bpm;
348
634
  if (args.keyscale) projectParams.keyscale = args.keyscale;
349
- if (args.timesignature) projectParams.timesignature = args.timesignature;
635
+ if (args.lyrics) projectParams.lyrics = args.lyrics;
636
+ if (args.language) projectParams.language = args.language;
350
637
  if (args.output_format) projectParams.outputFormat = args.output_format;
638
+
639
+ const timeSignature = normalizeTimeSignature(args.timesignature);
640
+ if (timeSignature) projectParams.timesignature = timeSignature;
641
+
642
+ const composerMode = asBooleanValue(args.composer_mode);
643
+ if (composerMode !== undefined) projectParams.composerMode = composerMode;
644
+
645
+ const promptStrength = asFiniteNumber(args.prompt_strength);
646
+ if (promptStrength !== undefined) projectParams.promptStrength = promptStrength;
647
+
648
+ const creativity = asFiniteNumber(args.creativity);
649
+ if (creativity !== undefined) projectParams.creativity = creativity;
650
+
351
651
  if (args.seed !== undefined) projectParams.seed = args.seed;
352
652
  if (options?.tokenType) projectParams.tokenType = options.tokenType;
353
653
  if (options?.network) projectParams.network = options.network;