@bytexbyte/nxtlinq-ai-agent-web-development 0.1.1

package/src/hooks/useNxtlinqAgent.ts

@@ -0,0 +1,73 @@
+import type {
+  Message,
+  NxtlinqAgentSnapshot,
+  PostTextTtsResult,
+  SendMessageOptions,
+} from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import { useCallback, useSyncExternalStore } from 'react';
+import { useNxtlinqAgentContext } from '../context/NxtlinqAgentContext';
+
+export type UseNxtlinqAgentResult = NxtlinqAgentSnapshot & {
+  agent: ReturnType<typeof useNxtlinqAgentContext>;
+  sendMessage: (
+    text: string,
+    options?: Omit<SendMessageOptions, 'text'>,
+  ) => Promise<void>;
+  loadHistory: (options?: { last?: number }) => Promise<void>;
+  syncVoiceTurnHistory: (options?: { last?: number }) => Promise<void>;
+  setMessages: (messages: Message[]) => void;
+  postTextTts: (text: string) => Promise<PostTextTtsResult>;
+  buildTextTtsPlaybackUri: (result: PostTextTtsResult) => string;
+};
+
+/** Subscribe to {@link NxtlinqAgent} state for custom Web chat UIs. */
+export function useNxtlinqAgent(): UseNxtlinqAgentResult {
+  const agent = useNxtlinqAgentContext();
+  const snapshot = useSyncExternalStore(
+    (onStoreChange) => agent.subscribe(onStoreChange),
+    () => agent.getSnapshot(),
+    () => agent.getSnapshot(),
+  );
+
+  const sendMessage = useCallback(
+    (text: string, options?: Omit<SendMessageOptions, 'text'>) =>
+      agent.sendMessage(text, options),
+    [agent],
+  );
+
+  const loadHistory = useCallback(
+    (options?: { last?: number }) => agent.loadHistory(options),
+    [agent],
+  );
+
+  const syncVoiceTurnHistory = useCallback(
+    (options?: { last?: number }) => agent.syncVoiceTurnHistory(options),
+    [agent],
+  );
+
+  const setMessages = useCallback(
+    (messages: Message[]) => agent.setMessages(messages),
+    [agent],
+  );
+
+  const postTextTts = useCallback(
+    (text: string) => agent.postTextTts(text),
+    [agent],
+  );
+
+  const buildTextTtsPlaybackUri = useCallback(
+    (result: PostTextTtsResult) => agent.buildTextTtsPlaybackUri(result),
+    [agent],
+  );
+
+  return {
+    agent,
+    ...snapshot,
+    sendMessage,
+    loadHistory,
+    syncVoiceTurnHistory,
+    setMessages,
+    postTextTts,
+    buildTextTtsPlaybackUri,
+  };
+}

package/src/hooks/useNxtlinqVoice.ts

@@ -0,0 +1,143 @@
+import type {
+  StartVoiceSessionOptions,
+  VoiceGreetingOptions,
+  VoiceSession,
+  VoiceStatus,
+  VoiceUserInputOptions,
+} from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import { VoiceNotSupportedError } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import { useCallback, useMemo, useRef, useSyncExternalStore } from 'react';
+import { useNxtlinqAgentContext } from '../context/NxtlinqAgentContext';
+
+export type UseNxtlinqVoiceOptions = Partial<
+  Omit<StartVoiceSessionOptions, 'apiKey' | 'apiSecret' | 'pseudoId'>
+>;
+
+export type UseNxtlinqVoiceResult = {
+  voiceStatus: VoiceStatus;
+  voiceSessionId: string | null;
+  isVoiceActive: boolean;
+  startVoice: (options?: UseNxtlinqVoiceOptions) => Promise<VoiceSession>;
+  stopVoice: (reason?: string) => Promise<void>;
+  muteMic: (muted: boolean) => void;
+  interrupt: () => void;
+  getRemoteAudioStream: () => MediaStream | null;
+  getOutputAudioLevel: () => number;
+  sendVoiceUserInput: (options: VoiceUserInputOptions) => void;
+  triggerVoiceGreeting: (
+    options: VoiceGreetingOptions,
+    greetOptions?: { waitForChannel?: boolean; timeoutMs?: number },
+  ) => Promise<void>;
+};
+
+/** Voice controls for custom Web UIs (browser WebRTC). */
+export function useNxtlinqVoice(): UseNxtlinqVoiceResult {
+  const agent = useNxtlinqAgentContext();
+  const sessionRef = useRef<VoiceSession | null>(null);
+  const snapshot = useSyncExternalStore(
+    (onStoreChange) => agent.subscribe(onStoreChange),
+    () => agent.getSnapshot(),
+    () => agent.getSnapshot(),
+  );
+
+  const refreshSessionRef = useCallback(() => {
+    sessionRef.current = agent.getVoiceSession();
+  }, [agent]);
+
+  const startVoice = useCallback(
+    async (options?: UseNxtlinqVoiceOptions) => {
+      try {
+        const session = await agent.startVoice(options);
+        sessionRef.current = session;
+        return session;
+      } catch (err) {
+        if (err instanceof VoiceNotSupportedError) {
+          throw new Error(
+            'Voice is not available — check NxtlinqAgentProvider config and pseudoId.',
+          );
+        }
+        throw err;
+      }
+    },
+    [agent],
+  );
+
+  const stopVoice = useCallback(
+    async (reason?: string) => {
+      await agent.stopVoice(reason);
+      sessionRef.current = null;
+    },
+    [agent],
+  );
+
+  const muteMic = useCallback(
+    (muted: boolean) => {
+      refreshSessionRef();
+      sessionRef.current?.muteMic(muted);
+    },
+    [refreshSessionRef],
+  );
+
+  const interrupt = useCallback(() => {
+    refreshSessionRef();
+    sessionRef.current?.interrupt();
+  }, [refreshSessionRef]);
+
+  const getRemoteAudioStream = useCallback(
+    () => agent.getVoiceSession()?.getRemoteAudioStream() ?? null,
+    [agent],
+  );
+
+  const getOutputAudioLevel = useCallback(() => {
+    refreshSessionRef();
+    return sessionRef.current?.getOutputAudioLevel() ?? agent.getOutputAudioLevel();
+  }, [refreshSessionRef, agent]);
+
+  const sendVoiceUserInput = useCallback(
+    (options: VoiceUserInputOptions) => {
+      refreshSessionRef();
+      if (sessionRef.current) {
+        sessionRef.current.sendVoiceUserInput(options);
+        return;
+      }
+      agent.sendVoiceUserInput(options);
+    },
+    [refreshSessionRef, agent],
+  );
+
+  const triggerVoiceGreeting = useCallback(
+    (
+      options: VoiceGreetingOptions,
+      greetOptions?: { waitForChannel?: boolean; timeoutMs?: number },
+    ) => agent.triggerVoiceGreeting(options, greetOptions),
+    [agent],
+  );
+
+  return useMemo(
+    () => ({
+      voiceStatus: snapshot.voiceStatus,
+      voiceSessionId: snapshot.voiceSessionId,
+      isVoiceActive: snapshot.voiceSessionId != null,
+      startVoice,
+      stopVoice,
+      muteMic,
+      interrupt,
+      getRemoteAudioStream,
+      getOutputAudioLevel,
+      sendVoiceUserInput,
+      triggerVoiceGreeting,
+    }),
+    [
+      snapshot.voiceStatus,
+      snapshot.voiceSessionId,
+      startVoice,
+      stopVoice,
+      muteMic,
+      interrupt,
+      getRemoteAudioStream,
+      getOutputAudioLevel,
+      sendVoiceUserInput,
+      triggerVoiceGreeting,
+    ],
+  );
+}

package/src/index.ts

@@ -0,0 +1,84 @@
+export {
+  createNxtlinqAgentWeb,
+  type CreateNxtlinqAgentWebOptions,
+} from './createNxtlinqAgent';
+
+export {
+  NxtlinqAgentProvider,
+  useNxtlinqAgentContext,
+  type NxtlinqAgentProviderProps,
+  type AgentConfig,
+} from './context/NxtlinqAgentContext';
+
+export { useNxtlinqAgent, type UseNxtlinqAgentResult } from './hooks/useNxtlinqAgent';
+export {
+  useNxtlinqVoice,
+  type UseNxtlinqVoiceResult,
+  type UseNxtlinqVoiceOptions,
+} from './hooks/useNxtlinqVoice';
+
+export {
+  createWebPlatformPorts,
+  type CreateWebPlatformPortsOptions,
+} from './ports/createWebPlatformPorts';
+export { WEB_AGENT_DEFAULTS, applyWebAgentDefaults } from './webAgentDefaults';
+export { fileToAttachment } from './utils/fileToAttachment';
+
+export type {
+  Message,
+  Attachment,
+  AgentResponse,
+  SendMessageOptions,
+  NxtlinqAgentSnapshot,
+  ToolUse,
+  StartVoiceSessionOptions,
+  VoiceSession,
+  VoiceStatus,
+  VoiceTranscriptEvent,
+  VoiceDoneEvent,
+  VoiceUserInputOptions,
+  VoiceGreetingOptions,
+} from '@bytexbyte/nxtlinq-ai-agent-core-development';
+
+export {
+  NxtlinqAgent,
+  setApiHosts,
+  createBrowserWebRTCPort,
+  createBrowserStoragePort,
+  createDefaultHttpPort,
+  VoiceNotSupportedError,
+  mapServerHistoryToMessages,
+  appendServerHistoryIntoMessages,
+  STORAGE_KEYS,
+} from '@bytexbyte/nxtlinq-ai-agent-core-development';
+
+export {
+  createNxtlinqApi,
+  getAiAgentApiHost,
+  startVoiceSession,
+  createVoiceApi,
+  useDraggable,
+  useLocalStorage,
+  useSessionStorage,
+  useResizable,
+  synthesizeSpeechToBuffer,
+  useSpeechToTextFromMic,
+  useVoiceMode,
+  metakeepClient,
+  getEthers,
+  sleep,
+  walletTextUtils,
+  connectWallet,
+  disconnectWallet,
+  validateToken,
+  createAITMetadata,
+  generateAITId,
+  prepareAITCreation,
+  createNotification,
+  getNotificationIcon,
+  containsUrls,
+  convertUrlsToLinks,
+  convertUrlsToHtml,
+} from './legacy';
+
+export type { ResizeCorner } from './legacy';

package/src/legacy/api/nxtlinq-api.ts

@@ -0,0 +1,32 @@
+/**
+ * Web SDK wrapper: default browser storage + voice API from browser module.
+ */
+import type { AITApi } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import {
+  createBrowserStoragePort,
+  createDefaultHttpPort,
+  createNxtlinqApiWithDeps,
+  getAiAgentApiHost,
+  setApiHosts,
+} from '@bytexbyte/nxtlinq-ai-agent-core-development';
+import { createVoiceApi } from './voice';
+
+export {
+  setApiHosts,
+  getAiAgentApiHost,
+  createNxtlinqApiWithDeps,
+  createBrowserStoragePort,
+} from '@bytexbyte/nxtlinq-ai-agent-core-development';
+
+export type { CoreAITApi, ApiClientDeps } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+
+export const createNxtlinqApi = (apiKey: string, apiSecret: string): AITApi => {
+  const core = createNxtlinqApiWithDeps(apiKey, apiSecret, {
+    storage: createBrowserStoragePort(),
+    http: createDefaultHttpPort(),
+  });
+  return {
+    ...core,
+    voice: createVoiceApi(() => getAiAgentApiHost()),
+  };
+};

package/src/legacy/api/voice.ts

@@ -0,0 +1,72 @@
+/**
+ * Web voice client — thin wrapper over core {@link startVoiceSessionWithPort}.
+ */
+import {
+  createBrowserWebRTCPort,
+  startVoiceSessionWithPort,
+  startWsRealtimeSession,
+} from '@bytexbyte/nxtlinq-ai-agent-core-development';
+
+export {
+  normalizeTranscriptKey,
+  mergeStreamingTranscript,
+  appendTranscriptSegment,
+} from '@bytexbyte/nxtlinq-ai-agent-core-development';
+
+export type {
+  VoicePhase,
+  VoiceStatus,
+  VoicePhaseExtra,
+  VoiceTranscriptEvent,
+  VoiceToolCallEvent,
+  VoiceToolResultEvent,
+  VoicePermissionRequiredEvent,
+  VoiceBillingBlockedEvent,
+  VoiceTtsWordEvent,
+  VoiceDoneEvent,
+  StartVoiceSessionOptions,
+  VoiceSession,
+} from '@bytexbyte/nxtlinq-ai-agent-core-development';
+
+import type { StartVoiceSessionOptions, VoiceSession } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+
+/**
+ * Start a cascade voice agent session. Browser-only (uses {@link createBrowserWebRTCPort}).
+ */
+export async function startVoiceSession(
+  apiAgentHost: string,
+  options: StartVoiceSessionOptions,
+): Promise<VoiceSession> {
+  const baseUrl = (options.baseUrl ?? apiAgentHost ?? '').replace(/\/+$/, '');
+  if (!baseUrl) {
+    throw new Error(
+      'startVoiceSession: API host is empty. Did you forget to call setApiHosts() before createNxtlinqApi()?',
+    );
+  }
+
+  const transport = options.transport ?? 'ws-realtime';
+
+  if (transport === 'ws-realtime') {
+    return startWsRealtimeSession(
+      { baseUrl, fetchFn: fetch },
+      { ...options, baseUrl, transport },
+    );
+  }
+
+  return startVoiceSessionWithPort(
+    {
+      baseUrl,
+      webrtc: createBrowserWebRTCPort(),
+      fetchFn: fetch,
+    },
+    { ...options, baseUrl },
+  );
+}
+
+export const createVoiceApi = (apiAgentHost: string | (() => string)) => ({
+  startSession: (options: StartVoiceSessionOptions): Promise<VoiceSession> =>
+    startVoiceSession(
+      typeof apiAgentHost === 'function' ? apiAgentHost() : apiAgentHost,
+      options,
+    ),
+});

package/src/legacy/core/lib/messageHistory.ts

@@ -0,0 +1,6 @@
+export {
+  appendServerHistoryIntoMessages,
+  isVoicePlaceholderMessage,
+  mapServerHistoryToMessages,
+  type ServerHistoryMessage,
+} from '@bytexbyte/nxtlinq-ai-agent-core-development';

package/src/legacy/core/lib/textToSpeech.ts

@@ -0,0 +1,127 @@
+import { SpeechConfig, SpeechSynthesisResult, SpeechSynthesizer } from 'microsoft-cognitiveservices-speech-sdk';
+import { getAiAgentApiHost } from '../../api/nxtlinq-api';
+import type { ClientTtsVoiceSettings } from '@bytexbyte/nxtlinq-ai-agent-core-development';
+
+const DEFAULT_AZURE_VOICE = 'en-US-JennyNeural';
+
+async function fetchAzureTokenFromAgent(
+  agentHost: string,
+  apiKey: string,
+  apiSecret: string,
+): Promise<{ token: string; region: string } | { error: string }> {
+  try {
+    const res = await fetch(`${agentHost}/api/cognitive/token`, {
+      method: 'GET',
+      headers: {
+        'X-API-Key': apiKey,
+        'X-API-Secret': apiSecret,
+      },
+    });
+    const data = (await res.json()) as { token?: string; region?: string; error?: string };
+    if (!res.ok || !data.token || !data.region) {
+      return { error: data.error || 'Failed to get speech token' };
+    }
+    return { token: data.token, region: data.region };
+  } catch (e) {
+    return { error: e instanceof Error ? e.message : 'Network error' };
+  }
+}
+
+async function synthesizeAzureToBuffer(
+  text: string,
+  authToken: string,
+  region: string,
+  voiceName?: string,
+): Promise<ArrayBuffer | undefined> {
+  const speechConfig = SpeechConfig.fromAuthorizationToken(authToken, region) as SpeechConfig;
+  speechConfig.speechSynthesisVoiceName =
+    voiceName?.trim() ? voiceName.trim() : DEFAULT_AZURE_VOICE;
+
+  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+  // @ts-ignore — SDK typing allows null audio config
+  const synthesizer = new SpeechSynthesizer(speechConfig, null);
+
+  const res = await new Promise<SpeechSynthesisResult | string>((resolve) => {
+    synthesizer.speakTextAsync(
+      text,
+      (result) => {
+        resolve(result);
+      },
+      (err) => resolve(err),
+    );
+  });
+
+  synthesizer.close();
+
+  if (typeof res === 'string') {
+    console.error('Text-to-speech error:', res);
+    return undefined;
+  }
+
+  return res.audioData;
+}
+
+async function synthesizeOpenAiToBuffer(
+  agentHost: string,
+  apiKey: string,
+  apiSecret: string,
+  text: string,
+): Promise<ArrayBuffer | undefined> {
+  const res = await fetch(`${agentHost}/api/tts/openai`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ apiKey, apiSecret, text }),
+  });
+  if (!res.ok) {
+    return undefined;
+  }
+  return res.arrayBuffer();
+}
+
+/**
+ * TTS for chat playback: Azure (browser Speech SDK + token from Agent) or OpenAI (audio via Agent proxy).
+ */
+export async function synthesizeSpeechToBuffer(params: {
+  text: string;
+  apiKey: string;
+  apiSecret: string;
+  settings?: ClientTtsVoiceSettings;
+}): Promise<{ buffer: ArrayBuffer; mimeType: string } | undefined> {
+  const agentHost = getAiAgentApiHost();
+  if (!agentHost) {
+    console.error('AI Agent API host is not set (call setApiHosts first)');
+    return undefined;
+  }
+
+  const provider = params.settings?.provider ?? 'azure';
+  const resolvedVoice = params.settings?.resolvedVoice?.trim();
+
+  if (provider === 'openai' && resolvedVoice) {
+    const buffer = await synthesizeOpenAiToBuffer(
+      agentHost,
+      params.apiKey,
+      params.apiSecret,
+      params.text,
+    );
+    if (!buffer) {
+      console.warn('OpenAI TTS failed, falling back to Azure');
+    } else {
+      return { buffer, mimeType: 'audio/mpeg' };
+    }
+  }
+
+  const tok = await fetchAzureTokenFromAgent(agentHost, params.apiKey, params.apiSecret);
+  if ('error' in tok) {
+    console.error('Azure speech token:', tok.error);
+    return undefined;
+  }
+
+  const buffer = await synthesizeAzureToBuffer(
+    params.text,
+    tok.token,
+    tok.region,
+    provider === 'azure' ? resolvedVoice : undefined,
+  );
+  if (!buffer) return undefined;
+  return { buffer, mimeType: 'audio/wav' };
+}