@bytexbyte/nxtlinq-ai-agent-core-development 0.2.0

package/dist/voice/types.d.ts

@@ -0,0 +1,138 @@
+import type { IceServerConfig } from '../ports/WebRTCPort';
+import type { Attachment } from '../types/ait-api';
+import type { VoiceUserInputOptions } from './voice-user-input';
+export type { VoiceUserInputOptions };
+export type VoicePhase = 'vad_start' | 'vad_end' | 'stt_done' | 'llm_start' | 'llm_first_token' | 'llm_end' | 'tts_audio_start' | 'tts_audio_end';
+export type VoiceStatus = 'listening' | 'transcribing' | 'thinking' | 'generating' | 'speaking' | 'idle';
+export type VoicePhaseExtra = {
+    turnId?: string;
+};
+export type VoiceTranscriptEvent = {
+    role: 'user' | 'assistant';
+    text: string;
+    interim: boolean;
+};
+export type VoiceToolCallEvent = {
+    name: string;
+    args: Record<string, unknown>;
+    toolCallId?: string;
+    bridgePhrase?: string;
+};
+export type VoiceToolResultEvent = {
+    name: string;
+    handled: boolean;
+    toolCallType: 'backend' | 'frontend' | null;
+    response?: unknown;
+};
+export type VoicePermissionRequiredEvent = {
+    toolName: string;
+    permission: string;
+};
+export type VoiceBillingBlockedEvent = {
+    reason: 'no_credits' | 'inactive' | 'suspended' | 'canceled' | 'grace_period_expired' | string;
+    message: string;
+    balanceCents?: number;
+    costCents?: number;
+};
+export type VoiceTtsWordEvent = {
+    text: string;
+    ptsMs: number;
+    ts: number;
+};
+export type VoiceDoneEvent = {
+    replyText?: string;
+    userMessageId?: string | null;
+    assistantMessageId?: string | null;
+    toolCall?: {
+        name: string;
+    };
+    timings?: {
+        totalDurationMs: number;
+        stepTimings: Array<{
+            name: string;
+            durationMs: number;
+        }>;
+    };
+    guardrailsBlocked?: boolean;
+    billingBlocked?: boolean;
+    billingReason?: VoiceBillingBlockedEvent['reason'];
+    error?: string;
+};
+export type StartVoiceSessionOptions = {
+    apiKey: string;
+    apiSecret: string;
+    pseudoId: string;
+    externalId?: string;
+    aitId?: string;
+    walletAddress?: string;
+    aitToken?: string;
+    metadata?: Record<string, unknown>;
+    voiceMode?: 'cascade' | 'realtime';
+    startWithMicMuted?: boolean;
+    /**
+     * Remote assistant playback gain (react-native-webrtc `_setVolume`, 0–10).
+     * @default 10 in {@link DEFAULT_REMOTE_AUDIO_GAIN}
+     */
+    remoteAudioGain?: number;
+    baseUrl?: string;
+    onOpen?: () => void;
+    onPhase?: (phase: VoicePhase, ts: number, extra?: VoicePhaseExtra) => void;
+    onStatus?: (status: VoiceStatus, ts: number) => void;
+    onTranscript?: (event: VoiceTranscriptEvent) => void;
+    onToolCall?: (event: VoiceToolCallEvent) => void;
+    onToolResult?: (event: VoiceToolResultEvent) => void;
+    onPermissionRequired?: (event: VoicePermissionRequiredEvent) => void;
+    onBillingBlocked?: (event: VoiceBillingBlockedEvent) => void;
+    onTtsWord?: (event: VoiceTtsWordEvent) => void;
+    onDone?: (event: VoiceDoneEvent) => void;
+    onError?: (error: Error) => void;
+    onClose?: (reason: string) => void;
+};
+/** Active voice call handle (platform layer attaches real MediaStream). */
+export type VoiceSession = {
+    id: string;
+    getRemoteAudioStream(): MediaStream | null;
+    /** Assistant playback RMS in 0–1 (poll from UI animation loop). */
+    getOutputAudioLevel(): number;
+    /** Whether the `app` data channel is open for client→server messages. */
+    isAppChannelOpen(): boolean;
+    /**
+     * Inject a text and/or image user turn during an active voice session
+     * (OpenAI Realtime–style conversation.item.create).
+     */
+    sendVoiceUserInput(options: VoiceUserInputOptions): void;
+    muteMic(muted: boolean): void;
+    interrupt(): void;
+    stop(reason?: string): Promise<void>;
+};
+export type VoiceGreetingOptions = {
+    text: string;
+    attachments?: Attachment[];
+    /**
+     * When true, do not append a user bubble in local chat state
+     * (Berify `_localOnly` auto-greet).
+     */
+    skipUserMessage?: boolean;
+    clientMessageId?: string;
+    /** Default false for opening greet (avoid barge-in on idle session). */
+    interruptFirst?: boolean;
+};
+export type VoiceSessionInitResponse = {
+    voiceSessionId: string;
+    signalingUrl: string;
+    iceServers: IceServerConfig[];
+    expiresAt: string;
+    mode?: 'cascade' | 'realtime';
+};
+export type CreateVoiceSessionParams = {
+    apiKey: string;
+    apiSecret: string;
+    pseudoId: string;
+    externalId?: string;
+    aitId?: string;
+    walletAddress?: string;
+    aitToken?: string;
+    voiceMode?: 'cascade' | 'realtime';
+    metadata?: Record<string, unknown>;
+};
+//# sourceMappingURL=types.d.ts.map

package/dist/voice/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/voice/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAEhE,YAAY,EAAE,qBAAqB,EAAE,CAAC;AAEtC,MAAM,MAAM,UAAU,GAClB,WAAW,GAAG,SAAS,GAAG,UAAU,GACpC,WAAW,GAAG,iBAAiB,GAAG,SAAS,GAC3C,iBAAiB,GAAG,eAAe,CAAC;AAExC,MAAM,MAAM,WAAW,GACnB,WAAW,GACX,cAAc,GACd,UAAU,GACV,YAAY,GACZ,UAAU,GACV,MAAM,CAAC;AAEX,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,SAAS,GAAG,UAAU,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,4BAA4B,GAAG;IACzC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,MAAM,EAAE,YAAY,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,GAAG,sBAAsB,GAAG,MAAM,CAAC;IAC/F,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;CACZ,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,kBAAkB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5B,OAAO,CAAC,EAAE;QAAE,eAAe,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,UAAU,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAChG,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,aAAa,CAAC,EAAE,wBAAwB,CAAC,QAAQ,CAAC,CAAC;IACnD,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,SAAS,CAAC,EAAE,SAAS,GAAG,UAAU,CAAC;IACnC,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,IAAI,CAAC;IACpB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IAC3E,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,KAAK,IAAI,CAAC;IACrD,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,IAAI,CAAC;IACrD,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,kBAAkB,KAAK,IAAI,CAAC;IACjD,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,IAAI,CAAC;IACrD,oBAAoB,CAAC,EAAE,CAAC,KAAK,EAAE,4BAA4B,KAAK,IAAI,CAAC;IACrE,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,wBAAwB,KAAK,IAAI,CAAC;IAC7D,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,CAAC;IAC/C,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;IACzC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IACjC,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;CACpC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,MAAM,YAAY,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,oBAAoB,IAAI,WAAW,GAAG,IAAI,CAAC;IAC3C,mEAAmE;IACnE,mBAAmB,IAAI,MAAM,CAAC;IAC9B,yEAAyE;IACzE,gBAAgB,IAAI,OAAO,CAAC;IAC5B;;;OAGG;IACH,kBAAkB,CAAC,OAAO,EAAE,qBAAqB,GAAG,IAAI,CAAC;IACzD,OAAO,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI,CAAC;IAC9B,SAAS,IAAI,IAAI,CAAC;IAClB,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC;IAC3B;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wEAAwE;IACxE,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,SAAS,GAAG,UAAU,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,SAAS,GAAG,UAAU,CAAC;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,CAAC"}

package/dist/voice/types.js

@@ -0,0 +1 @@
+export {};

package/dist/voice/voice-user-input.d.ts

@@ -0,0 +1,19 @@
+import type { Attachment } from '../types/ait-api';
+export type VoiceUserInputOptions = {
+    /** User text; omit or empty for image-only turns. */
+    text?: string;
+    attachments?: Attachment[];
+    /** Correlates with client UI message id (e.g. GiftedChat _id). */
+    clientMessageId?: string;
+    /** Send `interrupt` on the app channel before user input (default true). */
+    interruptFirst?: boolean;
+};
+export type AppChannelUserInputPayload = {
+    type: 'user_input';
+    text?: string;
+    attachments?: Attachment[];
+    clientMessageId?: string;
+    ts: number;
+};
+export declare function buildUserInputPayload(options: VoiceUserInputOptions): AppChannelUserInputPayload;
+//# sourceMappingURL=voice-user-input.d.ts.map

package/dist/voice/voice-user-input.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"voice-user-input.d.ts","sourceRoot":"","sources":["../../src/voice/voice-user-input.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAEnD,MAAM,MAAM,qBAAqB,GAAG;IAClC,qDAAqD;IACrD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC;IAC3B,kEAAkE;IAClE,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,4EAA4E;IAC5E,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG;IACvC,IAAI,EAAE,YAAY,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,UAAU,EAAE,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,EAAE,EAAE,MAAM,CAAC;CACZ,CAAC;AAEF,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,qBAAqB,GAC7B,0BAA0B,CAS5B"}

package/dist/voice/voice-user-input.js

@@ -0,0 +1,10 @@
+export function buildUserInputPayload(options) {
+    const text = typeof options.text === 'string' ? options.text.trim() : '';
+    return {
+        type: 'user_input',
+        ...(text ? { text } : {}),
+        ...(options.attachments?.length ? { attachments: options.attachments } : {}),
+        ...(options.clientMessageId ? { clientMessageId: options.clientMessageId } : {}),
+        ts: Date.now(),
+    };
+}

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@bytexbyte/nxtlinq-ai-agent-core-development",
+  "version": "0.2.0",
+  "description": "Platform-agnostic nxtlinq AI Agent core — API client, types, and orchestration ports",
+  "main": "dist/index.js",
+  "react-native": "src/index.ts",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist",
+    "src"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "clean": "rm -rf dist && find src -type f \\( -name '*.js' -o -name '*.d.ts' -o -name '*.js.map' -o -name '*.d.ts.map' \\) -delete 2>/dev/null || true",
+    "prepublishOnly": "yarn build"
+  },
+  "keywords": [
+    "nxtlinq",
+    "ai-agent",
+    "sdk"
+  ],
+  "author": "ByteXByte",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://dev.azure.com/nxtlinqLLC/nxtlinq/_git/nxtlinq-AI-Agent-SDK",
+    "directory": "packages/ai-agent-core"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "sideEffects": false,
+  "dependencies": {
+    "fast-json-stable-stringify": "^2.1.0",
+    "uuid": "^11.1.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.4.2"
+  }
+}

package/src/agent/ChatOrchestrator.ts

@@ -0,0 +1,380 @@
+import type { CoreAITApi } from '../api/nxtlinq-api';
+import {
+  appendServerHistoryIntoMessages,
+  mapServerHistoryToMessages,
+  VOICE_TURN_SYNC_LAST,
+  type ServerHistoryMessage,
+} from '../history/messageHistory';
+import type { AgentResponse, Attachment, Message } from '../types/ait-api';
+import type { AgentConfig, ToolUse } from '../types/agent-config';
+import { extractReplyText } from './extractReplyText';
+
+export type SendMessageOptions = {
+  text: string;
+  attachments?: Attachment[];
+  /** Defaults to `config.defaultModel` or `open-ai`. */
+  model?: string;
+  /** Skip appending a user bubble (retries / preset messages). */
+  skipUserMessage?: boolean;
+  isRetry?: boolean;
+  clientPipelineTimings?: Array<{ name: string; durationMs: number }>;
+};
+
+type AgentErrorResponse = AgentResponse & {
+  requiresWalletConnection?: boolean;
+  requiresPermissionUpdate?: boolean;
+  message?: string;
+  toolName?: string;
+  requiredPermission?: string;
+};
+
+/**
+ * Platform-agnostic text chat orchestration (no wallet UI, no TTS, no PII UI).
+ */
+export class ChatOrchestrator {
+  private messages: Message[] = [];
+  private isLoading = false;
+
+  constructor(
+    private readonly config: AgentConfig,
+    private readonly api: CoreAITApi,
+    private readonly onUpdate: () => void,
+  ) {}
+
+  getMessages(): Message[] {
+    return this.messages;
+  }
+
+  getIsLoading(): boolean {
+    return this.isLoading;
+  }
+
+  setMessages(messages: Message[]): void {
+    this.messages = messages;
+    this.onUpdate();
+  }
+
+  private emit(): void {
+    this.onUpdate();
+  }
+
+  private requirePseudoId(): string {
+    const id = this.config.pseudoId;
+    if (!id) {
+      throw new Error('pseudoId is required — set AgentConfig.pseudoId before chatting');
+    }
+    return id;
+  }
+
+  private resolveModel(model?: string): string {
+    return model ?? this.config.defaultModel ?? 'open-ai';
+  }
+
+  private patchMessage(id: string, patch: Partial<Message>): void {
+    this.messages = this.messages.map((m) =>
+      m.id === id ? { ...m, ...patch } : m,
+    );
+    this.emit();
+  }
+
+  private appendMessage(message: Message): void {
+    this.messages = [...this.messages, message];
+    this.config.onMessage?.(message);
+    this.emit();
+  }
+
+  async loadHistory(options?: { last?: number }): Promise<void> {
+    const pseudoId = this.requirePseudoId();
+    const result = await this.api.agent.getMessageHistory({
+      apiKey: this.config.apiKey,
+      apiSecret: this.config.apiSecret,
+      pseudoId,
+      externalId: this.config.externalId,
+      last: options?.last,
+    });
+
+    if (result && typeof result === 'object' && 'error' in result && result.error) {
+      throw new Error(String(result.error));
+    }
+
+    const rows = (result as { messages?: ServerHistoryMessage[] }).messages ?? [];
+    this.messages = mapServerHistoryToMessages(rows);
+    this.emit();
+  }
+
+  /**
+   * Fetch the latest voice-turn rows from the server and merge by message id
+   * (same behavior as web SDK `refreshMessageHistory`).
+   */
+  async syncVoiceTurnHistory(options?: { last?: number }): Promise<void> {
+    const pseudoId = this.requirePseudoId();
+    const result = await this.api.agent.getMessageHistory({
+      apiKey: this.config.apiKey,
+      apiSecret: this.config.apiSecret,
+      pseudoId,
+      externalId: this.config.externalId,
+      last: options?.last ?? VOICE_TURN_SYNC_LAST,
+    });
+
+    if (result && typeof result === 'object' && 'error' in result && result.error) {
+      throw new Error(String(result.error));
+    }
+
+    const rows = (result as { messages?: ServerHistoryMessage[] }).messages ?? [];
+    this.messages = appendServerHistoryIntoMessages(this.messages, rows);
+    this.emit();
+  }
+
+  async sendMessage(options: SendMessageOptions): Promise<void> {
+    const text = options.text?.trim() ?? '';
+    const hasContent = Boolean(text || (options.attachments && options.attachments.length > 0));
+    if (!hasContent || this.isLoading) return;
+
+    const model = this.resolveModel(options.model);
+    const pseudoId = this.requirePseudoId();
+
+    if (!options.skipUserMessage && !options.isRetry) {
+      this.appendMessage({
+        id: `${Date.now()}`,
+        content:
+          text
+          || (options.attachments?.length
+            ? `Uploaded ${options.attachments.length} file(s)`
+            : ''),
+        role: 'user',
+        timestamp: new Date().toISOString(),
+        attachments: options.attachments,
+        metadata: { model },
+      });
+    }
+
+    this.isLoading = true;
+    this.emit();
+
+    const isStreamModel = model.endsWith('-stream');
+    let streamAssistantId: string | null = null;
+    let streamedReplyBuffer = '';
+
+    if (isStreamModel && !options.isRetry) {
+      streamAssistantId = `stream-assistant-${Date.now()}`;
+      this.appendMessage({
+        id: streamAssistantId,
+        role: 'assistant',
+        content: '',
+        partialContent: '',
+        isStreaming: true,
+        timestamp: new Date().toISOString(),
+        metadata: { model, agentLlmStream: true },
+      });
+    }
+
+    try {
+      const apiAttachments = options.attachments?.map((att) => ({
+        type: att.type,
+        url: att.url,
+        name: att.name,
+        mimeType: att.mimeType,
+      }));
+
+      const response = (await this.api.agent.sendMessage({
+        model,
+        apiKey: this.config.apiKey,
+        apiSecret: this.config.apiSecret,
+        pseudoId,
+        externalId: this.config.externalId,
+        customUserInfo: this.config.customUserInfo,
+        customUsername: this.config.customUsername,
+        message:
+          text
+          || (options.attachments?.length
+            ? `Uploaded ${options.attachments.length} file(s)`
+            : ''),
+        attachments: apiAttachments,
+        isRetry: options.isRetry,
+        ...(options.clientPipelineTimings?.length
+          ? { clientPipelineTimings: options.clientPipelineTimings }
+          : {}),
+        onStreamDelta: streamAssistantId
+          ? (delta: string) => {
+            streamedReplyBuffer += delta;
+            const current = this.messages.find((m) => m.id === streamAssistantId);
+            this.patchMessage(streamAssistantId!, {
+              partialContent: `${current?.partialContent ?? ''}${delta}`,
+            });
+            if (delta) {
+              this.isLoading = false;
+              this.emit();
+            }
+          }
+          : undefined,
+      })) as AgentErrorResponse;
+
+      const actualModel = response.model ?? model;
+
+      if (response.error || response.result === 'error') {
+        if (response.requiresWalletConnection || response.requiresPermissionUpdate) {
+          const msg =
+            response.message
+            ?? response.error
+            ?? 'Additional setup is required before continuing.';
+          this.appendMessage({
+            id: `${Date.now()}-err`,
+            content: msg,
+            role: 'assistant',
+            timestamp: new Date().toISOString(),
+            metadata: {
+              model: actualModel,
+              toolName: response.toolName,
+              requiredPermission: response.requiredPermission,
+            },
+          });
+          return;
+        }
+        throw new Error(response.error ?? response.message ?? 'Failed to send message');
+      }
+
+      if (streamAssistantId && response.toolCall?.toolUse) {
+        this.messages = this.messages.filter((m) => m.id !== streamAssistantId);
+        streamAssistantId = null;
+        this.emit();
+      }
+
+      if (streamAssistantId && !response.toolCall?.toolUse) {
+        const replyText = extractReplyText(response, streamedReplyBuffer);
+        this.patchMessage(streamAssistantId, {
+          content: replyText,
+          isStreaming: false,
+          partialContent: undefined,
+          metadata: { model: actualModel, agentLlmStream: true },
+          piiProtection: response.piiProtection?.anonymizedReply
+            ? {
+              anonymizedContent: response.piiProtection.anonymizedReply ?? undefined,
+              mapping: response.piiProtection.mapping ?? undefined,
+            }
+            : undefined,
+        });
+        return;
+      }
+
+      if (response.toolCall?.toolUse) {
+        await this.handleToolCall(response, actualModel);
+        return;
+      }
+
+      const replyText = extractReplyText(response, streamedReplyBuffer);
+      this.appendMessage({
+        id: `${Date.now() + 1}`,
+        content: replyText,
+        role: 'assistant',
+        timestamp: new Date().toISOString(),
+        metadata: { model: actualModel },
+        piiProtection: response.piiProtection?.anonymizedReply
+          ? {
+            anonymizedContent: response.piiProtection.anonymizedReply ?? undefined,
+            mapping: response.piiProtection.mapping ?? undefined,
+          }
+          : undefined,
+      });
+
+      if (response.piiProtection?.anonymizedUserMessage) {
+        for (let i = this.messages.length - 1; i >= 0; i--) {
+          if (this.messages[i].role === 'user') {
+            this.patchMessage(this.messages[i].id, {
+              piiProtection: {
+                anonymizedContent: response.piiProtection!.anonymizedUserMessage!,
+                mapping: response.piiProtection!.mapping ?? undefined,
+              },
+              piiStatus: 'complete',
+            });
+            break;
+          }
+        }
+      }
+    } catch (error) {
+      const err = error instanceof Error ? error : new Error(String(error));
+      this.config.onError?.(err);
+      if (streamAssistantId) {
+        this.messages = this.messages.filter((m) => m.id !== streamAssistantId);
+      }
+      this.appendMessage({
+        id: `${Date.now()}-err`,
+        content: err.message,
+        role: 'assistant',
+        timestamp: new Date().toISOString(),
+        error: err.message,
+      });
+    } finally {
+      this.isLoading = false;
+      this.emit();
+    }
+  }
+
+  private async handleToolCall(
+    response: AgentResponse,
+    actualModel: string,
+  ): Promise<void> {
+    const toolUse = response.toolCall!.toolUse as ToolUse;
+    const streamingId = `streaming-${Date.now()}`;
+
+    this.appendMessage({
+      id: streamingId,
+      content: '',
+      role: 'assistant',
+      timestamp: new Date().toISOString(),
+      isStreaming: true,
+      streamingToolName: toolUse.name,
+      streamingStatus: `Starting ${toolUse.name}...`,
+      streamingProgress: 0,
+      metadata: { model: actualModel, toolUse },
+    });
+
+    let finalContent = `Tool ${toolUse.name} executed successfully`;
+
+    if (this.config.onToolUse) {
+      const toolResult = await this.config.onToolUse(toolUse, (update) => {
+        const current = this.messages.find((m) => m.id === streamingId);
+        this.patchMessage(streamingId, {
+          content: update.partialResult ?? current?.content ?? '',
+          partialContent: update.partialContent,
+          streamingProgress: update.progress,
+          streamingStatus: update.status,
+          streamingSteps: update.steps ?? current?.streamingSteps,
+        });
+      });
+      if (toolResult?.content) {
+        finalContent = toolResult.content;
+      }
+    }
+
+    const replyText = extractReplyText(response);
+    if (replyText && replyText !== 'Sorry, I cannot understand your question') {
+      finalContent = `${finalContent}\n\n${replyText}`;
+    }
+
+    this.patchMessage(streamingId, {
+      content: finalContent,
+      isStreaming: false,
+      streamingProgress: 100,
+      metadata: { model: actualModel, toolUse },
+      piiProtection: response.piiProtection?.anonymizedReply
+        ? {
+          anonymizedContent: response.piiProtection.anonymizedReply ?? undefined,
+          mapping: response.piiProtection.mapping ?? undefined,
+        }
+        : undefined,
+    });
+
+    if (response.assistantMessageId) {
+      try {
+        await this.api.agent.updateMessageContent({
+          apiKey: this.config.apiKey,
+          apiSecret: this.config.apiSecret,
+          messageId: response.assistantMessageId,
+          content: finalContent,
+        });
+      } catch {
+        /* best-effort */
+      }
+    }
+  }
+}