@sogni-ai/sogni-client 4.2.0-alpha.2 → 4.2.0-alpha.21

package/src/Chat/tools.ts

@@ -1,223 +1,83 @@
+import hostedToolManifest from './sogniHostedTools.generated.json';
 import { ToolDefinition, ToolCall } from './types';
+import { filterVideoModelsByWorkflow, isEditImageModel, VideoWorkflow } from './modelRouting';
 
-/**
- * Built-in Sogni platform tool definitions for use with LLM tool calling.
- *
- * These tools allow the LLM to generate images, videos, and music
- * through the Sogni Supernet. Include them in your `tools` array when
- * calling `sogni.chat.completions.create()`.
- *
- * @example
- * ```typescript
- * import { SogniTools } from '@sogni-ai/sogni-client';
- *
- * const stream = await sogni.chat.completions.create({
- *   model: 'qwen3.6-35b-a3b-gguf-iq4xs',
- *   messages: [{ role: 'user', content: 'Generate an image of a sunset' }],
- *   tools: SogniTools.all,
- *   tool_choice: 'auto',
- *   stream: true,
- * });
- * ```
- */
+type SogniHostedToolName =
+  | 'sogni_generate_image'
+  | 'sogni_edit_image'
+  | 'sogni_generate_video'
+  | 'sogni_sound_to_video'
+  | 'sogni_video_to_video'
+  | 'sogni_generate_music';
 
-/** Tool definition: Generate an image using the Sogni Supernet */
-export const generateImageTool: ToolDefinition = {
-  type: 'function',
-  function: {
-    name: 'sogni_generate_image',
-    description:
-      'Generate an image using AI image generation on the Sogni Supernet. Returns a URL to the generated image. Use this tool EVERY TIME the user asks to create, generate, draw, or make an image or picture. Do NOT generate URLs yourself — you MUST call this tool.',
-    parameters: {
-      type: 'object',
-      properties: {
-        prompt: {
-          type: 'string',
-          description:
-            'Detailed text description of the image to generate. Be specific about style, composition, lighting, colors, and subject matter.'
-        },
-        negative_prompt: {
-          type: 'string',
-          description:
-            'Things to avoid in the generated image (e.g., "blurry, low quality, distorted").'
-        },
-        width: {
-          type: 'number',
-          description: 'Image width in pixels. Must be a multiple of 16. Default: 1024. Max: 2048.'
-        },
-        height: {
-          type: 'number',
-          description: 'Image height in pixels. Must be a multiple of 16. Default: 1024. Max: 2048.'
-        },
-        model: {
-          type: 'string',
-          description: 'Image generation model to use.',
-          enum: [
-            'flux1-schnell-fp8',
-            'flux2-dev_fp8',
-            'chroma-v.46-flash_fp8',
-            'z_image_turbo_bf16'
-          ]
-        },
-        steps: {
-          type: 'number',
-          description:
-            'Number of inference steps. Higher = better quality but slower. Default depends on model (4-50).'
-        },
-        seed: {
-          type: 'number',
-          description: 'Random seed for reproducible generation. Use -1 for random.'
-        }
-      },
-      required: ['prompt']
-    }
-  }
-};
+interface SogniHostedToolManifest {
+  tools: ToolDefinition[];
+}
+
+const hostedTools = (hostedToolManifest as SogniHostedToolManifest).tools;
 
-/** Tool definition: Generate a video using the Sogni Supernet */
-export const generateVideoTool: ToolDefinition = {
-  type: 'function',
-  function: {
-    name: 'sogni_generate_video',
-    description:
-      'Generate a short video using AI video generation on the Sogni Supernet. Returns a URL to the generated video. Use this tool EVERY TIME the user asks to create, generate, or make a video, clip, or animation. Do NOT generate URLs yourself — you MUST call this tool. Write the prompt as a cohesive mini-scene in present tense, describing motion, camera movement, lighting, and atmosphere in flowing prose.',
-    parameters: {
-      type: 'object',
-      properties: {
-        prompt: {
-          type: 'string',
-          description:
-            'Detailed text description of the video to generate. Write it as a flowing present-tense scene: describe the subject, action, camera movement, lighting, and atmosphere. Clear camera-to-subject relationship improves motion consistency. Be specific and vivid.'
-        },
-        negative_prompt: {
-          type: 'string',
-          description:
-            'Things to avoid in the generated video (e.g., "blurry, low quality, distorted, watermark").'
-        },
-        width: {
-          type: 'number',
-          description:
-            'Video width in pixels. Default: 1920. Standard resolutions: 1920x1088 (landscape), 1088x1920 (portrait), 1280x720.'
-        },
-        height: {
-          type: 'number',
-          description: 'Video height in pixels. Default: 1088. Must be a multiple of 16.'
-        },
-        duration: {
-          type: 'number',
-          description: 'Video duration in seconds. Range: 1-20. Default: 5.'
-        },
-        fps: {
-          type: 'number',
-          description: 'Frames per second. Default: 24. Range: 1-60.'
-        },
-        model: {
-          type: 'string',
-          description:
-            'Video generation model to use. Prefer LTX-2.3 text-to-video (t2v) models for best quality.'
-        },
-        seed: {
-          type: 'number',
-          description: 'Random seed for reproducible generation. Use -1 for random.'
-        }
-      },
-      required: ['prompt']
-    }
+function getHostedTool(name: SogniHostedToolName): ToolDefinition {
+  const tool = hostedTools.find((candidate) => candidate.function.name === name);
+  if (!tool) {
+    throw new Error(`Missing hosted Sogni tool definition: ${name}`);
   }
-};
+  return tool;
+}
 
-/** Tool definition: Generate music using the Sogni Supernet */
-export const generateMusicTool: ToolDefinition = {
-  type: 'function',
-  function: {
-    name: 'sogni_generate_music',
-    description:
-      'Generate a music track using AI music generation on the Sogni Supernet. Returns a URL to the generated audio file. Use this tool EVERY TIME the user asks to create, generate, compose, or make music, a song, a beat, or audio. Do NOT generate URLs yourself — you MUST call this tool.',
-    parameters: {
-      type: 'object',
-      properties: {
-        prompt: {
-          type: 'string',
-          description:
-            'Description of the music to generate. Include genre, mood, tempo, instruments, and style. Can also include lyrics wrapped in [verse], [chorus], etc. tags.'
-        },
-        duration: {
-          type: 'number',
-          description: 'Duration of the generated music in seconds. Range: 10-600. Default: 30.'
-        },
-        bpm: {
-          type: 'number',
-          description: 'Beats per minute. Range: 30-300. Default: 120.'
-        },
-        keyscale: {
-          type: 'string',
-          description:
-            'Musical key and scale (e.g., "C major", "A minor", "F# minor", "Bb major"). Default: "C major".'
-        },
-        timesignature: {
-          type: 'string',
-          description: 'Time signature. "4" for 4/4, "3" for 3/4, "2" for 2/4. Default: "4".',
-          enum: ['4', '3', '2']
-        },
-        model: {
-          type: 'string',
-          description:
-            'Music generation model. "ace_step_1.5_turbo" for fast/catchy, "ace_step_1.5_sft" for more control over lyrics.',
-          enum: ['ace_step_1.5_turbo', 'ace_step_1.5_sft']
-        },
-        output_format: {
-          type: 'string',
-          description: 'Audio output format. Default: "mp3".',
-          enum: ['mp3', 'flac', 'wav']
-        },
-        seed: {
-          type: 'number',
-          description: 'Random seed for reproducible generation. Use -1 for random.'
-        }
-      },
-      required: ['prompt']
-    }
+function cloneTool(tool: ToolDefinition): ToolDefinition {
+  return structuredClone(tool);
+}
+
+function setModelEnum(
+  tool: ToolDefinition,
+  modelIds: string[],
+  description: string
+): ToolDefinition {
+  if (modelIds.length === 0) {
+    return tool;
   }
-};
+  (tool.function.parameters as any).properties.model = {
+    type: 'string',
+    description,
+    enum: modelIds
+  };
+  return tool;
+}
 
 /**
- * Collection of all Sogni platform tool definitions.
+ * Built-in Sogni platform tool definitions for use with LLM tool calling.
+ *
+ * These definitions are generated from the shared
+ * `@sogni/creative-agent` hosted tool backbone. The public SDK keeps a local
+ * generated copy so consumers do not need access to the private package.
  */
+
+export const generateImageTool: ToolDefinition = getHostedTool('sogni_generate_image');
+export const editImageTool: ToolDefinition = getHostedTool('sogni_edit_image');
+export const generateVideoTool: ToolDefinition = getHostedTool('sogni_generate_video');
+export const soundToVideoTool: ToolDefinition = getHostedTool('sogni_sound_to_video');
+export const videoToVideoTool: ToolDefinition = getHostedTool('sogni_video_to_video');
+export const generateMusicTool: ToolDefinition = getHostedTool('sogni_generate_music');
+
 export const SogniTools = {
-  /** Generate an image */
   generateImage: generateImageTool,
-  /** Generate a video */
+  editImage: editImageTool,
   generateVideo: generateVideoTool,
-  /** Generate music */
+  soundToVideo: soundToVideoTool,
+  videoToVideo: videoToVideoTool,
   generateMusic: generateMusicTool,
-  /** All Sogni tools combined — convenience array for `tools` param */
   get all(): ToolDefinition[] {
-    return [generateImageTool, generateVideoTool, generateMusicTool];
+    return [
+      generateImageTool,
+      editImageTool,
+      generateVideoTool,
+      soundToVideoTool,
+      videoToVideoTool,
+      generateMusicTool
+    ];
   }
 };
 
-/**
- * Build Sogni tool definitions with dynamically populated model enums
- * based on currently available models on the network.
- *
- * @param availableModels - Result of `sogni.projects.waitForModels()`. If omitted, returns
- *   the default tool definitions with static model lists.
- *
- * @example
- * ```typescript
- * import { buildSogniTools } from '@sogni-ai/sogni-client';
- *
- * const models = await sogni.projects.waitForModels();
- * const tools = buildSogniTools(models);
- *
- * const stream = await sogni.chat.completions.create({
- *   model: 'qwen3.6-35b-a3b-gguf-iq4xs',
- *   messages,
- *   tools,
- *   stream: true,
- * });
- * ```
- */
 export function buildSogniTools(
   availableModels?: Array<{ id: string; media?: string }>
 ): ToolDefinition[] {
@@ -225,56 +85,49 @@ export function buildSogniTools(
     return SogniTools.all;
   }
 
-  const imageModels = availableModels.filter((m) => m.media === 'image').map((m) => m.id);
-  const videoModels = availableModels.filter((m) => m.media === 'video').map((m) => m.id);
-  const audioModels = availableModels.filter((m) => m.media === 'audio').map((m) => m.id);
-
-  const tools: ToolDefinition[] = [];
-
-  // Image tool — override model enum if image models are available
-  const imageTool = structuredClone(generateImageTool);
-  if (imageModels.length > 0) {
-    (imageTool.function.parameters as any).properties.model.enum = imageModels;
-  }
-  tools.push(imageTool);
-
-  // Video tool — override model enum if video models are available
-  const videoTool = structuredClone(generateVideoTool);
-  if (videoModels.length > 0) {
-    (videoTool.function.parameters as any).properties.model = {
-      type: 'string',
-      description: 'Video generation model to use. Choose a text-to-video (t2v) model.',
-      enum: videoModels
-    };
-  }
-  tools.push(videoTool);
-
-  // Music tool — override model enum if audio models are available
-  const musicTool = structuredClone(generateMusicTool);
-  if (audioModels.length > 0) {
-    (musicTool.function.parameters as any).properties.model = {
-      type: 'string',
-      description:
-        'Music generation model. "ace_step_1.5_turbo" for fast/catchy, "ace_step_1.5_sft" for more control over lyrics.',
-      enum: audioModels
-    };
-  }
-  tools.push(musicTool);
+  const imageModels = availableModels
+    .filter((model) => model.media === 'image')
+    .map((model) => model.id);
+  const editImageModels = availableModels
+    .filter((model) => model.media === 'image' && isEditImageModel(model.id))
+    .map((model) => model.id);
+  const videoModels = filterVideoModelsByWorkflow(availableModels, ['t2v', 'i2v']);
+  const soundToVideoModels = filterVideoModelsByWorkflow(availableModels, ['s2v', 'ia2v', 'a2v']);
+  const videoToVideoModels = filterVideoModelsByWorkflow(availableModels, [
+    'animate-move',
+    'animate-replace',
+    'v2v'
+  ] as VideoWorkflow[]);
+  const audioModels = availableModels
+    .filter((model) => model.media === 'audio')
+    .map((model) => model.id);
 
-  return tools;
+  return [
+    setModelEnum(cloneTool(generateImageTool), imageModels, 'Image generation model to use.'),
+    setModelEnum(
+      cloneTool(editImageTool),
+      editImageModels,
+      'Image editing model to use. These models support reference-guided editing.'
+    ),
+    setModelEnum(
+      cloneTool(generateVideoTool),
+      videoModels,
+      'Video generation model to use. Prefer t2v models for text-only generation and i2v models when reference images are supplied.'
+    ),
+    setModelEnum(
+      cloneTool(soundToVideoTool),
+      soundToVideoModels,
+      'Audio-driven video model to use.'
+    ),
+    setModelEnum(cloneTool(videoToVideoTool), videoToVideoModels, 'Video-to-video model to use.'),
+    setModelEnum(cloneTool(generateMusicTool), audioModels, 'Music generation model to use.')
+  ];
 }
 
-/**
- * Check if a tool call is a Sogni platform tool.
- */
 export function isSogniToolCall(toolCall: ToolCall): boolean {
   return toolCall.function.name.startsWith('sogni_');
 }
 
-/**
- * Parse arguments from a tool call's JSON string.
- * Returns the parsed object or an empty object if parsing fails.
- */
 export function parseToolCallArguments(toolCall: ToolCall): Record<string, unknown> {
   try {
     return JSON.parse(toolCall.function.arguments);

package/src/Chat/types.ts

@@ -36,6 +36,31 @@ export type ToolChoice =
   | 'required'
   | { type: 'function'; function: { name: string } };
 
+export type SogniToolsMode = boolean | 'creative-agent' | 'creative-tools' | 'rich';
+
+/**
+ * OpenAI-compatible structured-output controls. Forwarded to the worker
+ * unchanged; honored natively by llama-server (compiles JSON Schema → GBNF
+ * internally) and vLLM (`xgrammar` / `outlines`). Per-request opt-in — when
+ * omitted, the model generates without constraint.
+ */
+export type ChatResponseFormat =
+  | { type: 'text' }
+  | { type: 'json_object' }
+  | {
+      type: 'json_schema';
+      json_schema: {
+        /** Identifier for cached grammar reuse. */
+        name: string;
+        /** JSON Schema describing the required output shape. */
+        schema: Record<string, unknown>;
+        /** When true, only fields named in `schema` may appear. Default: false. */
+        strict?: boolean;
+        /** Optional description shown to the model. */
+        description?: string;
+      };
+    };
+
 /** Text content part for multimodal messages. */
 export interface TextContentPart {
   type: 'text';
@@ -46,7 +71,7 @@ export interface TextContentPart {
 export interface ImageUrlContentPart {
   type: 'image_url';
   image_url: {
-    /** Supports http(s) URLs and data URIs (e.g., `data:image/jpeg;base64,...`). */
+    /** Supports inline base64-encoded JPEG or PNG data URIs only (e.g., `data:image/jpeg;base64,...`) for vision-model inputs. Max 20 images per request, 10MB each, longest side 1024px. */
     url: string;
     /** Controls how the model processes the image: 'auto' (default), 'low' (faster), 'high' (more detail). */
     detail?: 'auto' | 'low' | 'high';
@@ -67,6 +92,8 @@ export interface ChatMessage {
 export interface ChatCompletionParams {
   model: string;
   messages: ChatMessage[];
+  /** Optional source label for this request. Defaults to the client appSource when configured. */
+  appSource?: string;
   max_tokens?: number;
   temperature?: number;
   top_p?: number;
@@ -83,6 +110,19 @@ export interface ChatCompletionParams {
   tools?: ToolDefinition[];
   /** Controls which (if any) tool is called by the model. */
   tool_choice?: ToolChoice;
+  /**
+   * Ask the Sogni API to inject server-side media-generation tool families.
+   * `true` or `'creative-tools'` injects the hosted creative media/planning tool family.
+   * `'rich'` is accepted as a legacy alias for `'creative-tools'`.
+   * `'creative-agent'` adds hosted workflow-control and asset-manifest tools.
+   */
+  sogni_tools?: SogniToolsMode;
+  /**
+   * When `sogni_tools` is enabled, ask the Sogni API to execute requested Sogni
+   * tools server-side before returning the chat response. This is separate from
+   * the SDK-local `autoExecuteTools` loop.
+   */
+  sogni_tool_execution?: boolean;
   /**
    * Control thinking/reasoning mode for supported models (e.g. Qwen3/3.5).
    * When `false`, sends `chat_template_kwargs: { enable_thinking: false }` so
@@ -92,6 +132,12 @@ export interface ChatCompletionParams {
   think?: boolean;
   /** Hint for server-side preset selection. */
   taskProfile?: 'general' | 'coding' | 'reasoning';
+  /**
+   * Constrain output structure (OpenAI-compatible). Most useful on tool-call
+   * rounds where the model must emit a specific argument shape — eliminates
+   * JSON drift on quantized models. Forwarded to the worker unchanged.
+   */
+  response_format?: ChatResponseFormat;
   /**
    * Automatically execute Sogni tool calls (image/video/music generation) when the
    * model requests them. The SDK handles the full multi-round tool calling loop:
@@ -124,6 +170,7 @@ export interface ChatRequestMessage {
   type: 'llm';
   model: string;
   messages: ChatMessage[];
+  appSource?: string;
   max_tokens?: number;
   temperature?: number;
   top_p?: number;
@@ -137,9 +184,13 @@ export interface ChatRequestMessage {
   tokenType?: 'sogni' | 'spark';
   tools?: ToolDefinition[];
   tool_choice?: ToolChoice;
+  sogni_tools?: SogniToolsMode;
+  sogni_tool_execution?: boolean;
   taskProfile?: 'general' | 'coding' | 'reasoning';
   /** Per-request chat template arguments (e.g. `{ enable_thinking: false }` for llama.cpp). */
   chat_template_kwargs?: Record<string, unknown>;
+  /** Per-request structured-output constraint (OpenAI-compatible). */
+  response_format?: ChatResponseFormat;
 }
 
 export interface ChatCompletionChunk {
@@ -175,6 +226,52 @@ export interface ChatCompletionResult {
   toolHistory?: ToolHistoryEntry[];
 }
 
+export interface HostedChatCompletionMessage {
+  role: 'system' | 'user' | 'assistant' | 'tool';
+  content: string | null;
+  tool_calls?: ToolCall[];
+  tool_call_id?: string;
+}
+
+export interface HostedChatCompletionChoice {
+  index: number;
+  message: HostedChatCompletionMessage;
+  finish_reason: string | null;
+}
+
+export interface HostedCreativeWorkflowReference {
+  workflowId: string;
+  status: string;
+  url: string;
+  eventsUrl: string;
+  streamUrl: string;
+}
+
+export interface HostedChatCompletionResult {
+  id: string;
+  object: 'chat.completion';
+  created: number;
+  model: string;
+  choices: HostedChatCompletionChoice[];
+  usage?: TokenUsage;
+  creative_workflows?: HostedCreativeWorkflowReference[];
+  sogni_tool_results?: Record<string, unknown>[];
+}
+
+export type HostedChatCompletionParams = Omit<
+  ChatCompletionParams,
+  'stream' | 'autoExecuteTools' | 'onToolCall' | 'onToolProgress' | 'maxToolRounds'
+> & {
+  stream?: false;
+  token_type?: 'sogni' | 'spark' | 'auto';
+  app_source?: string;
+  chat_template_kwargs?: Record<string, unknown>;
+  media_references?: unknown[];
+  mediaReferences?: unknown[];
+  api_media_references?: unknown[];
+  apiMediaReferences?: unknown[];
+};
+
 export interface ChatJobStateEvent {
   jobID: string;
   type: string;
@@ -191,6 +288,7 @@ export interface LLMParamConstraint {
   max: number;
   decimals?: number;
   default: number;
+  thinkingComplexDefault?: number;
 }
 
 /** Recommended sampling defaults for a specific thinking mode. */
@@ -329,7 +427,7 @@ export interface ToolExecutionOptions {
    * complete within this time, the tool call will fail with a timeout error
    * and the project will be canceled.
    *
-   * Default: 600000 (10 minutes).
+   * Default: 1800000 (30 minutes).
    */
   timeout?: number;
 }