@sogni-ai/expo-client 1.0.0-alpha.10

package/src/Chat/ChatStream.ts

@@ -0,0 +1,164 @@
+import ErrorData from '../types/ErrorData';
+import {
+  ChatCompletionChunk,
+  ChatCompletionResult,
+  ToolCall,
+  ToolCallDelta,
+  TokenUsage
+} from './types';
+
+type Resolver = (value: IteratorResult<ChatCompletionChunk>) => void;
+type Rejecter = (err: Error) => void;
+
+/**
+ * Async iterable representing a streaming chat completion.
+ *
+ * Iterate with `for await (const chunk of stream)`. Each chunk has `content`
+ * (token text) and may carry a `finishReason` on the final chunk. After the
+ * stream completes, `stream.result` resolves to the full ChatCompletionResult.
+ *
+ * Cancel mid-stream with `stream.cancel()` (sends a server-side abort).
+ */
+class ChatStream implements AsyncIterable<ChatCompletionChunk> {
+  readonly jobId: string;
+  private _queue: ChatCompletionChunk[] = [];
+  private _pendingResolver: Resolver | null = null;
+  private _pendingRejecter: Rejecter | null = null;
+  private _content = '';
+  private _role: string = 'assistant';
+  private _finishReason: string | null = null;
+  private _usage: TokenUsage = { prompt_tokens: 0, completion_tokens: 0 };
+  private _timeTaken = 0;
+  private _workerName: string | undefined;
+  private _toolCallAccumulator = new Map<number, ToolCall>();
+  private _done = false;
+  private _error: Error | null = null;
+  private _resultPromise: Promise<ChatCompletionResult>;
+  private _resultResolve!: (r: ChatCompletionResult) => void;
+  private _resultReject!: (e: Error) => void;
+  private _onCancel: () => Promise<void>;
+
+  constructor(jobId: string, onCancel: () => Promise<void>) {
+    this.jobId = jobId;
+    this._onCancel = onCancel;
+    this._resultPromise = new Promise<ChatCompletionResult>((resolve, reject) => {
+      this._resultResolve = resolve;
+      this._resultReject = reject;
+    });
+  }
+
+  /** Resolves to the full result after the stream completes. */
+  get result(): Promise<ChatCompletionResult> {
+    return this._resultPromise;
+  }
+
+  async cancel(): Promise<void> {
+    await this._onCancel();
+  }
+
+  [Symbol.asyncIterator](): AsyncIterator<ChatCompletionChunk> {
+    return {
+      next: () => this._next()
+    };
+  }
+
+  private _next(): Promise<IteratorResult<ChatCompletionChunk>> {
+    return new Promise((resolve, reject) => {
+      if (this._error) {
+        reject(this._error);
+        return;
+      }
+      if (this._queue.length > 0) {
+        resolve({ value: this._queue.shift()!, done: false });
+        return;
+      }
+      if (this._done) {
+        resolve({ value: undefined as unknown as ChatCompletionChunk, done: true });
+        return;
+      }
+      this._pendingResolver = resolve;
+      this._pendingRejecter = reject;
+    });
+  }
+
+  _pushChunk(chunk: ChatCompletionChunk): void {
+    if (chunk.content) this._content += chunk.content;
+    if (chunk.role) this._role = chunk.role;
+    if (chunk.finishReason !== undefined && chunk.finishReason !== null) {
+      this._finishReason = chunk.finishReason;
+    }
+    if (chunk.usage) this._usage = chunk.usage;
+    if (chunk.tool_calls) this._accumulateToolCalls(chunk.tool_calls);
+
+    if (this._pendingResolver) {
+      const resolve = this._pendingResolver;
+      this._pendingResolver = null;
+      this._pendingRejecter = null;
+      resolve({ value: chunk, done: false });
+    } else {
+      this._queue.push(chunk);
+    }
+  }
+
+  _complete(timeTaken: number, usage?: TokenUsage, workerName?: string): void {
+    if (this._done) return;
+    if (usage) this._usage = usage;
+    this._timeTaken = timeTaken;
+    if (workerName) this._workerName = workerName;
+    this._done = true;
+
+    const toolCalls = Array.from(this._toolCallAccumulator.values()).filter(
+      (tc) => tc.id && tc.function.name
+    );
+    const result: ChatCompletionResult = {
+      jobId: this.jobId,
+      content: this._content,
+      role: this._role,
+      finishReason: this._finishReason,
+      usage: this._usage,
+      timeTaken: this._timeTaken,
+      ...(this._workerName ? { workerName: this._workerName } : {}),
+      ...(toolCalls.length ? { tool_calls: toolCalls } : {})
+    };
+    this._resultResolve(result);
+
+    if (this._pendingResolver) {
+      const resolve = this._pendingResolver;
+      this._pendingResolver = null;
+      this._pendingRejecter = null;
+      resolve({ value: undefined as unknown as ChatCompletionChunk, done: true });
+    }
+  }
+
+  _fail(error: ErrorData): void {
+    if (this._done) return;
+    this._done = true;
+    const err = new Error(error.message || 'Chat completion failed');
+    (err as Error & { code?: number; rawCode?: string }).code = error.code;
+    (err as Error & { code?: number; rawCode?: string }).rawCode = error.originalCode;
+    this._error = err;
+    this._resultReject(err);
+    if (this._pendingRejecter) {
+      const reject = this._pendingRejecter;
+      this._pendingResolver = null;
+      this._pendingRejecter = null;
+      reject(err);
+    }
+  }
+
+  private _accumulateToolCalls(deltas: ToolCallDelta[]): void {
+    for (const delta of deltas) {
+      const existing = this._toolCallAccumulator.get(delta.index) ?? {
+        id: delta.id ?? '',
+        type: 'function' as const,
+        function: { name: '', arguments: '' }
+      };
+      if (delta.id) existing.id = delta.id;
+      if (delta.function?.name) existing.function.name += delta.function.name;
+      if (delta.function?.arguments) existing.function.arguments += delta.function.arguments;
+      this._toolCallAccumulator.set(delta.index, existing);
+    }
+  }
+}
+
+export default ChatStream;

package/src/Chat/index.ts

@@ -0,0 +1,224 @@
+import ApiClient from '../ApiClient';
+import {
+  ChatErrorEvent,
+  ChatResultEvent,
+  ChatTokensEvent,
+  SwarmLLMModelsEvent
+} from '../ApiClient/WebSocketClient/events';
+import TypedEventEmitter from '../lib/TypedEventEmitter';
+import getUUID from '../lib/getUUID';
+import ChatStream from './ChatStream';
+import {
+  ChatCompletionChunk,
+  ChatCompletionParams,
+  ChatCompletionResult,
+  ChatMessage,
+  LLMModelInfo,
+  ToolRound
+} from './types';
+
+interface ChatApiEvents {
+  /** Fired for each streaming chunk on any active completion. */
+  token: ChatCompletionChunk;
+  /** Fired when any completion finishes successfully. */
+  completed: ChatCompletionResult;
+  /** Fired when any completion fails. */
+  failed: { jobId: string; error: Error };
+  /** Fired when the server pushes an updated LLM model list. */
+  modelsUpdated: Record<string, LLMModelInfo>;
+}
+
+/**
+ * Chat / LLM API — OpenAI-compatible completions over WebSocket.
+ *
+ * Phase 4b MVP: streaming + non-streaming text and multimodal (vision)
+ * completions. Tool-calling deltas are exposed on chunks/results for apps
+ * that drive their own tool loops, but the auto-execution loop and Sogni
+ * hosted tools are not yet wired up.
+ */
+class ChatApi extends TypedEventEmitter<ChatApiEvents> {
+  private _client: ApiClient;
+  private _streams = new Map<string, ChatStream>();
+  private _availableModels: Record<string, LLMModelInfo> = {};
+
+  constructor(client: ApiClient) {
+    super();
+    this._client = client;
+    client.socket.on('chatTokens', (e) => this._onTokens(e));
+    client.socket.on('chatResult', (e) => this._onResult(e));
+    client.socket.on('chatError', (e) => this._onError(e));
+    client.socket.on('swarmLLMModels', (e) => this._onModels(e));
+  }
+
+  /** Snapshot of the latest LLM model list pushed by the server. */
+  getAvailableModels(): Record<string, LLMModelInfo> {
+    return { ...this._availableModels };
+  }
+
+  /** Subscribe to model-list updates. Returns an unsubscribe function. */
+  onModelsUpdated(listener: (models: Record<string, LLMModelInfo>) => void): () => void {
+    return this.on('modelsUpdated', listener);
+  }
+
+  completions = {
+    create: ((params: ChatCompletionParams) => this._create(params)) as {
+      (params: ChatCompletionParams & { stream: true }): Promise<ChatStream>;
+      (params: ChatCompletionParams & { stream?: false }): Promise<ChatCompletionResult>;
+    }
+  };
+
+  private async _create(params: ChatCompletionParams): Promise<ChatStream | ChatCompletionResult> {
+    if (!this._client.socket.isConnected) {
+      throw new Error(
+        'Cannot start chat completion: WebSocket is not connected. Authenticate first.'
+      );
+    }
+    if (params.autoExecuteTools) {
+      if (params.stream) {
+        throw new Error(
+          'autoExecuteTools is not supported with stream: true. Drive the tool loop yourself instead.'
+        );
+      }
+      if (!params.onToolCall) {
+        throw new Error('autoExecuteTools requires an onToolCall handler');
+      }
+      return this._runAutoToolLoop(params);
+    }
+
+    const stream = await this._dispatchCompletion(params);
+    if (params.stream) return stream;
+    return stream.result;
+  }
+
+  private async _dispatchCompletion(params: ChatCompletionParams): Promise<ChatStream> {
+    const jobId = getUUID();
+    const request = {
+      jobID: jobId,
+      type: 'llm' as const,
+      model: params.model,
+      messages: params.messages,
+      max_tokens: params.max_tokens,
+      temperature: params.temperature,
+      top_p: params.top_p,
+      top_k: params.top_k,
+      min_p: params.min_p,
+      stream: !!params.stream,
+      repetition_penalty: params.repetition_penalty,
+      frequency_penalty: params.frequency_penalty,
+      presence_penalty: params.presence_penalty,
+      stop: params.stop,
+      tokenType: params.tokenType,
+      tools: params.tools,
+      tool_choice: params.tool_choice,
+      chat_template_kwargs: params.chat_template_kwargs,
+      taskProfile: params.taskProfile
+    };
+    const stream = new ChatStream(jobId, () => this._cancelStream(jobId));
+    this._streams.set(jobId, stream);
+    await this._client.socket.send('llmJobRequest', request);
+    return stream;
+  }
+
+  /**
+   * Multi-round tool-calling loop. Each round sends a completion, then
+   * executes any returned tool_calls via the user's onToolCall handler and
+   * appends the results to the conversation. Stops when the model returns a
+   * normal completion (no tool calls) or when maxToolRounds is reached.
+   */
+  private async _runAutoToolLoop(params: ChatCompletionParams): Promise<ChatCompletionResult> {
+    const maxRounds = params.maxToolRounds ?? 5;
+    const onToolCall = params.onToolCall!;
+    const history: ToolRound[] = [];
+    let messages: ChatMessage[] = params.messages.slice();
+
+    for (let round = 1; round <= maxRounds; round++) {
+      const stream = await this._dispatchCompletion({
+        ...params,
+        messages,
+        stream: false,
+        autoExecuteTools: false,
+        onToolCall: undefined
+      });
+      const result = await stream.result;
+
+      if (!result.tool_calls?.length || result.finishReason !== 'tool_calls') {
+        return { ...result, ...(history.length ? { toolHistory: history } : {}) };
+      }
+
+      const toolResults: Array<{ tool_call_id: string; content: string }> = [];
+      for (const call of result.tool_calls) {
+        const content = await onToolCall(call);
+        toolResults.push({ tool_call_id: call.id, content });
+      }
+      history.push({ round, toolCalls: result.tool_calls, toolResults });
+
+      messages = [
+        ...messages,
+        {
+          role: 'assistant',
+          content: result.content || null,
+          tool_calls: result.tool_calls
+        },
+        ...result.tool_calls.map((call, i) => ({
+          role: 'tool' as const,
+          content: toolResults[i]!.content,
+          tool_call_id: call.id,
+          name: call.function.name
+        }))
+      ];
+    }
+
+    throw new Error(
+      `autoExecuteTools exceeded maxToolRounds (${maxRounds}) — the model kept requesting tool calls`
+    );
+  }
+
+  private _onTokens(e: ChatTokensEvent) {
+    const stream = this._streams.get(e.jobId);
+    if (!stream) return;
+    const chunk: ChatCompletionChunk = {
+      jobId: e.jobId,
+      content: e.content,
+      ...(e.role ? { role: e.role } : {}),
+      ...(e.finishReason !== undefined ? { finishReason: e.finishReason } : {}),
+      ...(e.usage ? { usage: e.usage } : {}),
+      ...(e.tool_calls ? { tool_calls: e.tool_calls } : {})
+    };
+    stream._pushChunk(chunk);
+    this.emit('token', chunk);
+  }
+
+  private _onResult(e: ChatResultEvent) {
+    const stream = this._streams.get(e.jobId);
+    if (!stream) return;
+    stream._complete(e.timeTaken, e.usage, e.workerName);
+    stream.result.then((r) => this.emit('completed', r)).catch(() => undefined);
+    this._streams.delete(e.jobId);
+  }
+
+  private _onError(e: ChatErrorEvent) {
+    const stream = this._streams.get(e.jobId);
+    if (!stream) return;
+    stream._fail({ code: 5000, originalCode: e.rawCode, message: e.message });
+    this.emit('failed', {
+      jobId: e.jobId,
+      error: new Error(e.message)
+    });
+    this._streams.delete(e.jobId);
+  }
+
+  private _onModels(e: SwarmLLMModelsEvent) {
+    this._availableModels = e.models;
+    this.emit('modelsUpdated', this._availableModels);
+  }
+
+  private async _cancelStream(jobId: string): Promise<void> {
+    try {
+      await this._client.socket.send('llmJobCancel', { jobID: jobId });
+    } catch (e) {
+      this._client.logger.warn(`Failed to cancel chat job ${jobId}:`, e);
+    }
+  }
+}
+
+export default ChatApi;

package/src/Chat/tools.ts

@@ -0,0 +1,23 @@
+import { ToolCall } from './types';
+
+/**
+ * Parse a tool call's arguments JSON into a typed object.
+ *
+ * Tool call arguments arrive from the model as a string (so they can be
+ * streamed character-by-character). This helper does the JSON.parse with
+ * a clearer error message than a raw SyntaxError.
+ *
+ * @example
+ * const args = parseToolCallArguments<{ description: string; tags: string[] }>(call);
+ */
+export function parseToolCallArguments<T = Record<string, unknown>>(toolCall: ToolCall): T {
+  try {
+    return JSON.parse(toolCall.function.arguments) as T;
+  } catch (err) {
+    throw new Error(
+      `Failed to parse arguments for tool ${toolCall.function.name} (id=${toolCall.id}): ${
+        err instanceof Error ? err.message : String(err)
+      }. Raw arguments: ${toolCall.function.arguments}`
+    );
+  }
+}

package/src/Chat/types.ts

@@ -0,0 +1,165 @@
+import { TokenType } from '../types/token';
+
+export interface TextContentPart {
+  type: 'text';
+  text: string;
+}
+
+/**
+ * Image input for multimodal (vision) chat. Pass a base64-encoded data URI like
+ * `data:image/jpeg;base64,...`. Server limits: max 20 images per request, 10 MB
+ * each, longest side 1024px.
+ */
+export interface ImageUrlContentPart {
+  type: 'image_url';
+  image_url: {
+    url: string;
+    detail?: 'auto' | 'low' | 'high';
+  };
+}
+
+export type ContentPart = TextContentPart | ImageUrlContentPart;
+
+export interface ChatMessage {
+  role: 'system' | 'user' | 'assistant' | 'tool';
+  content: string | ContentPart[] | null;
+  /** Tool calls returned by an assistant message. */
+  tool_calls?: ToolCall[];
+  /** Identifies which tool call this message responds to. */
+  tool_call_id?: string;
+  name?: string;
+}
+
+/**
+ * Tool-calling types are kept structural only for Phase 4b. The SDK exposes
+ * the wire shape (so apps that want to drive their own tool loops can do so),
+ * but the multi-round auto-execution loop and Sogni hosted tools are deferred
+ * to a follow-up release.
+ */
+export interface ToolFunction {
+  name: string;
+  description?: string;
+  parameters?: Record<string, unknown>;
+}
+
+export interface ToolDefinition {
+  type: 'function';
+  function: ToolFunction;
+}
+
+export interface ToolCallFunction {
+  name: string;
+  arguments: string;
+}
+
+export interface ToolCall {
+  id: string;
+  type: 'function';
+  function: ToolCallFunction;
+}
+
+export interface ToolCallDelta {
+  index: number;
+  id?: string;
+  type?: 'function';
+  function?: {
+    name?: string;
+    arguments?: string;
+  };
+}
+
+export type ToolChoice =
+  | 'none'
+  | 'auto'
+  | 'required'
+  | { type: 'function'; function: { name: string } };
+
+export interface ChatCompletionParams {
+  model: string;
+  messages: ChatMessage[];
+  max_tokens?: number;
+  temperature?: number;
+  top_p?: number;
+  top_k?: number;
+  min_p?: number;
+  stream?: boolean;
+  repetition_penalty?: number;
+  frequency_penalty?: number;
+  presence_penalty?: number;
+  stop?: string | string[];
+  /** Token type to use for billing. Defaults to 'sogni'. */
+  tokenType?: TokenType;
+  tools?: ToolDefinition[];
+  tool_choice?: ToolChoice;
+  /** Per-request chat template arguments (e.g. `{ enable_thinking: false }`). */
+  chat_template_kwargs?: Record<string, unknown>;
+  /** Hint for server-side preset selection. */
+  taskProfile?: 'general' | 'coding' | 'reasoning';
+  /**
+   * Automatically run a multi-round tool calling loop. The SDK sends the
+   * completion, executes any returned tool calls via `onToolCall`, feeds the
+   * results back as `role: 'tool'` messages, and repeats until the model
+   * answers without calling more tools (or `maxToolRounds` is reached).
+   *
+   * Only supported when `stream: false`. For streaming flows, drive the loop
+   * yourself by inspecting `result.tool_calls` and issuing follow-up
+   * completions.
+   */
+  autoExecuteTools?: boolean;
+  /**
+   * Required when `autoExecuteTools` is true. Called for each tool call the
+   * model emits. Must return the tool result content (a string the model
+   * will see as the tool's output).
+   */
+  onToolCall?: (toolCall: ToolCall) => Promise<string> | string;
+  /**
+   * Max number of tool-calling rounds when `autoExecuteTools` is true.
+   * Defaults to 5. Prevents infinite loops if the model keeps calling tools.
+   */
+  maxToolRounds?: number;
+}
+
+/** Re-export for app-side use. */
+
+/** One round of tool calls + results captured during an autoExecuteTools loop. */
+export interface ToolRound {
+  round: number;
+  toolCalls: ToolCall[];
+  toolResults: Array<{ tool_call_id: string; content: string }>;
+}
+
+export interface TokenUsage {
+  prompt_tokens: number;
+  completion_tokens: number;
+  total_tokens?: number;
+}
+
+export interface ChatCompletionChunk {
+  jobId: string;
+  content: string;
+  role?: string;
+  finishReason?: string | null;
+  usage?: TokenUsage;
+  tool_calls?: ToolCallDelta[];
+}
+
+export interface ChatCompletionResult {
+  jobId: string;
+  content: string;
+  role: string;
+  finishReason: string | null;
+  usage: TokenUsage;
+  timeTaken: number;
+  workerName?: string;
+  tool_calls?: ToolCall[];
+  /** History of tool-calling rounds when `autoExecuteTools` was used. */
+  toolHistory?: ToolRound[];
+}
+
+export interface LLMModelInfo {
+  /** Worker count currently serving this model on the selected network. */
+  workers: number;
+  /** Optional richer metadata when the server sends the full LLMModelInfo shape. */
+  displayName?: string;
+  contextLength?: number;
+}

package/src/Projects/ControlNet.ts

@@ -0,0 +1,84 @@
+import { MediaInput } from '../lib/mediaInput';
+
+export type ControlNetName =
+  | 'canny'
+  | 'depth'
+  | 'inpaint'
+  | 'instrp2p'
+  | 'lineart'
+  | 'lineartanime'
+  | 'mlsd'
+  | 'normalbae'
+  | 'openpose'
+  | 'scribble'
+  | 'segmentation'
+  | 'shuffle'
+  | 'softedge'
+  | 'tile'
+  | 'instantid';
+
+export type ControlNetMode = 'balanced' | 'prompt_priority' | 'cn_priority';
+
+export interface ControlNetParams {
+  /** Short name of the ControlNet model. */
+  name: ControlNetName;
+  /** Control image input. Omit `image` and pass `imageAlreadyUploaded: true` if the image is already on the server. */
+  image?: MediaInput;
+  /** True when the control image was uploaded out of band and only the flag should be sent. */
+  imageAlreadyUploaded?: boolean;
+  /** 0..1. 0 = full control to prompt, 1 = full control to ControlNet. */
+  strength?: number;
+  mode?: ControlNetMode;
+  /** Step at which the ControlNet first applies (0..1). Must be < guidanceEnd. */
+  guidanceStart?: number;
+  /** Step at which the ControlNet last applies (0..1). Must be > guidanceStart. */
+  guidanceEnd?: number;
+}
+
+export interface ControlNetParamsRaw {
+  name: ControlNetName;
+  cnImageState: 'original';
+  hasImage: boolean;
+  controlStrength?: number;
+  controlMode?: 0 | 1 | 2;
+  controlGuidanceStart?: number;
+  controlGuidanceEnd?: number;
+}
+
+const MODE_MAP: Record<ControlNetMode, 0 | 1 | 2> = {
+  balanced: 0,
+  prompt_priority: 1,
+  cn_priority: 2
+};
+
+export function toControlNetRaw(params: ControlNetParams): ControlNetParamsRaw {
+  const raw: ControlNetParamsRaw = {
+    name: params.name,
+    cnImageState: 'original',
+    hasImage: !!params.image || !!params.imageAlreadyUploaded
+  };
+  if (params.strength !== undefined) raw.controlStrength = clamp01(params.strength, 'strength');
+  if (params.mode) raw.controlMode = MODE_MAP[params.mode];
+  if (params.guidanceStart !== undefined) {
+    raw.controlGuidanceStart = clamp01(params.guidanceStart, 'guidanceStart');
+  }
+  if (params.guidanceEnd !== undefined) {
+    raw.controlGuidanceEnd = clamp01(params.guidanceEnd, 'guidanceEnd');
+  }
+  return raw;
+}
+
+function clamp01(value: number, name: string): number {
+  if (Number.isNaN(value) || value < 0 || value > 1) {
+    throw new Error(`${name} must be between 0 and 1 (got ${value})`);
+  }
+  return value;
+}
+
+export function getMaxContextImages(modelId: string): number {
+  if (modelId === 'gpt-image-2') return 16;
+  if (modelId.startsWith('flux2_')) return 6;
+  if (modelId.startsWith('qwen_image_')) return 3;
+  if (modelId.includes('kontext')) return 2;
+  return 3;
+}