@bytexbyte/nxtlinq-ai-agent-core-development 0.2.0

package/src/api/parse-sse.ts

@@ -0,0 +1,104 @@
+export type SSEHandlers = {
+  onPiiProgress?: (step: 'scan_start' | 'scan_complete' | 'send_start' | 'done', data?: unknown) => void;
+  onStreamDelta?: (text: string) => void;
+};
+
+/** Normalize CRLF and split into SSE message blocks. */
+function splitSSEMessages(raw: string): string[] {
+  const normalized = raw.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
+  return normalized.split('\n\n').filter((block) => block.trim().length > 0);
+}
+
+function dispatchSSEMessage(
+  msg: string,
+  handlers: SSEHandlers,
+  finalDataRef: { value: unknown | null },
+): void {
+  if (!msg.trim()) return;
+  let eventType = '';
+  const dataLines: string[] = [];
+  for (const line of msg.split('\n')) {
+    const trimmed = line.trim();
+    if (trimmed.startsWith('event:')) {
+      eventType = trimmed.slice(6).trim();
+    } else if (trimmed.startsWith('data:')) {
+      dataLines.push(trimmed.slice(5).trimStart());
+    }
+  }
+  if (!eventType) return;
+
+  const data = dataLines.join('\n');
+  const onProgress = handlers.onPiiProgress;
+
+  if (eventType === 'pii_scan_start') {
+    onProgress?.('scan_start');
+  } else if (eventType === 'pii_scan_complete') {
+    let parsed: unknown = undefined;
+    try {
+      parsed = data ? JSON.parse(data) : undefined;
+    } catch {
+      /* ignore */
+    }
+    onProgress?.('scan_complete', parsed);
+  } else if (eventType === 'pii_send_start') {
+    onProgress?.('send_start');
+  } else if (eventType === 'text_delta') {
+    let parsed: { text?: string } = {};
+    try {
+      parsed = data ? JSON.parse(data) : {};
+    } catch {
+      /* ignore */
+    }
+    const chunk = typeof parsed.text === 'string' ? parsed.text : '';
+    handlers.onStreamDelta?.(chunk);
+  } else if (eventType === 'error') {
+    let parsed: { error?: string } = {};
+    try {
+      parsed = data ? JSON.parse(data) : {};
+    } catch {
+      /* ignore */
+    }
+    throw new Error(parsed.error || 'SSE stream error');
+  } else if (eventType === 'done') {
+    try {
+      finalDataRef.value = data ? JSON.parse(data) : {};
+    } catch {
+      /* ignore */
+    }
+  }
+}
+
+function parseSSEPayload(raw: string, handlers: SSEHandlers): unknown {
+  const finalDataRef = { value: null as unknown | null };
+  for (const msg of splitSSEMessages(raw)) {
+    dispatchSSEMessage(msg, handlers, finalDataRef);
+  }
+  if (finalDataRef.value !== null && finalDataRef.value !== undefined) {
+    return finalDataRef.value;
+  }
+  throw new Error('SSE stream ended without done event');
+}
+
+/** Parse a complete SSE payload (React Native fetch often lacks a usable `response.body`). */
+export function parseSSEText(text: string, handlers: SSEHandlers): unknown {
+  return parseSSEPayload(text, handlers);
+}
+
+/** Parse Agent SSE: PII steps, optional LLM `text_delta`, final `done`. */
+export async function parseSSEResponse(
+  body: ReadableStream<Uint8Array>,
+  handlers: SSEHandlers,
+): Promise<unknown> {
+  const reader = body.getReader();
+  const decoder = new TextDecoder();
+  let buffer = '';
+
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    buffer += decoder.decode(value, { stream: true });
+  }
+
+  buffer += decoder.decode();
+  return parseSSEPayload(buffer, handlers);
+}

package/src/api/tts.ts

@@ -0,0 +1,69 @@
+import type { HttpFetch } from '../ports/HttpPort';
+import { getAiAgentApiHost } from './hosts';
+
+export type PostTextTtsParams = {
+  apiKey: string;
+  apiSecret: string;
+  text: string;
+};
+
+export type PostTextTtsResult = {
+  /** Raw MPEG audio bytes from `/api/tts/openai`. */
+  audio: ArrayBuffer;
+  mimeType: string;
+};
+
+function bytesToBase64(bytes: Uint8Array): string {
+  if (typeof globalThis.Buffer !== 'undefined') {
+    return globalThis.Buffer.from(bytes).toString('base64');
+  }
+  let binary = '';
+  const chunk = 0x8000;
+  for (let i = 0; i < bytes.length; i += chunk) {
+    binary += String.fromCharCode(...bytes.subarray(i, i + chunk));
+  }
+  if (typeof globalThis.btoa !== 'function') {
+    throw new Error('No base64 encoder available in this runtime');
+  }
+  return globalThis.btoa(binary);
+}
+
+/**
+ * Synthesize assistant text via nxtlinq AI Agent OpenAI TTS (Dashboard voice config).
+ * Berify-compatible name; returns audio bytes instead of Berify `audio_id` cache.
+ */
+export async function postTextTts(
+  params: PostTextTtsParams,
+  fetchFn: HttpFetch = globalThis.fetch.bind(globalThis) as HttpFetch,
+): Promise<PostTextTtsResult> {
+  const text = params.text.trim();
+  if (!text) {
+    throw new Error('postTextTts: text is empty');
+  }
+
+  const res = await fetchFn(`${getAiAgentApiHost()}/api/tts/openai`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      apiKey: params.apiKey,
+      apiSecret: params.apiSecret,
+      text,
+    }),
+  });
+
+  if (!res.ok) {
+    const errBody = await res.text().catch(() => '');
+    throw new Error(`postTextTts failed: ${res.status} ${errBody.slice(0, 200)}`);
+  }
+
+  const audio = await res.arrayBuffer();
+  const mimeType =
+    res.headers.get('content-type')?.split(';')[0]?.trim() || 'audio/mpeg';
+  return { audio, mimeType };
+}
+
+/** Build a data URI suitable for RN `Audio` / `Video` `source={{ uri }}`. */
+export function buildTextTtsDataUri(audio: ArrayBuffer, mimeType = 'audio/mpeg'): string {
+  const bytes = new Uint8Array(audio);
+  return `data:${mimeType};base64,${bytesToBase64(bytes)}`;
+}

package/src/constants/storageKeys.ts

@@ -0,0 +1,5 @@
+/** Keys used by the API client when reading wallet / AIT session from storage. */
+export const STORAGE_KEYS = {
+  AIT_SERVICE_ACCESS_TOKEN: 'nxtlinqAITServiceAccessToken',
+  WALLET_ADDRESS: 'walletAddress',
+} as const;

package/src/history/messageHistory.ts

@@ -0,0 +1,65 @@
+import type { Message } from '../types/ait-api';
+
+/** One voice turn is typically user + assistant (2 persisted messages). */
+export const VOICE_TURN_SYNC_LAST = 2;
+
+export type ServerHistoryMessage = {
+  id: string;
+  role: 'user' | 'assistant';
+  content: string;
+  createdAt?: string;
+  attachments?: Message['attachments'];
+};
+
+export function isVoicePlaceholderMessage(message: Message): boolean {
+  if (message.metadata?.voiceRealtime) return true;
+  return String(message.id).startsWith('voice-');
+}
+
+export function mapServerHistoryToMessages(rows: ServerHistoryMessage[]): Message[] {
+  return rows.map((row) => ({
+    id: row.id,
+    role: row.role,
+    content: row.content ?? '',
+    timestamp: row.createdAt ?? new Date().toISOString(),
+    attachments: row.attachments,
+    isStreaming: false,
+    partialContent: undefined,
+  }));
+}
+
+/**
+ * Append the latest server messages onto local history (one voice turn ≈ 2 rows).
+ * Strips voice placeholders, upserts by message id, preserves unrelated local rows.
+ */
+export function appendServerHistoryIntoMessages(
+  prev: Message[],
+  serverRows: ServerHistoryMessage[],
+): Message[] {
+  const withoutPlaceholders = prev.filter((m) => !isVoicePlaceholderMessage(m));
+  const incoming = mapServerHistoryToMessages(serverRows);
+  if (incoming.length === 0) return withoutPlaceholders;
+
+  const byId = new Map(withoutPlaceholders.map((m) => [m.id, m]));
+  for (const row of incoming) {
+    const existing = byId.get(row.id);
+    if (existing) {
+      byId.set(row.id, {
+        ...existing,
+        content: row.content,
+        role: row.role,
+        attachments: row.attachments ?? existing.attachments,
+        isStreaming: false,
+        partialContent: undefined,
+      });
+    } else {
+      byId.set(row.id, row);
+    }
+  }
+
+  const merged = Array.from(byId.values());
+  merged.sort(
+    (a, b) => new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime(),
+  );
+  return merged;
+}

package/src/index.ts

@@ -0,0 +1,70 @@
+// API
+export { setApiHosts, getAiAgentApiHost, getAitServiceApiHost } from './api/hosts';
+export {
+  createNxtlinqApiWithDeps,
+  type CoreAITApi,
+  type ApiClientDeps,
+} from './api/nxtlinq-api';
+export { parseSSEResponse, parseSSEText } from './api/parse-sse';
+export {
+  postTextTts,
+  buildTextTtsDataUri,
+  type PostTextTtsParams,
+  type PostTextTtsResult,
+} from './api/tts';
+
+// Ports
+export * from './ports';
+export { STORAGE_KEYS } from './constants/storageKeys';
+
+// Types
+export type * from './types/ait-api';
+export type { AgentConfig, ToolUse, AgentEnvironment } from './types/agent-config';
+
+// History
+export {
+  appendServerHistoryIntoMessages,
+  isVoicePlaceholderMessage,
+  mapServerHistoryToMessages,
+  VOICE_TURN_SYNC_LAST,
+  type ServerHistoryMessage,
+} from './history/messageHistory';
+
+// Voice (protocol + session mint; WebRTC connect in platform packages)
+export * from './voice/types';
+export {
+  normalizeTranscriptKey,
+  mergeStreamingTranscript,
+  appendTranscriptSegment,
+} from './voice/transcript';
+export {
+  createVoiceSession,
+  appendVoiceModeToSignalingUrl,
+} from './voice/create-voice-session';
+export {
+  startVoiceSessionWithPort,
+  type StartVoiceSessionDeps,
+} from './voice/start-voice-session';
+export { createAppChannelDispatcher } from './voice/app-channel-dispatcher';
+export { createBrowserWebRTCPort } from './ports/createBrowserWebRTCPort';
+export type { CreateVoiceSessionParams, VoiceGreetingOptions, VoiceUserInputOptions } from './voice/types';
+export { buildUserInputPayload } from './voice/voice-user-input';
+export {
+  applyRemoteAudioPlaybackGain,
+  applyRemoteAudioPlaybackGainToTrack,
+  DEFAULT_REMOTE_AUDIO_GAIN,
+  type RemoteAudioTrackLike,
+} from './voice/remote-audio-gain';
+export {
+  createOutputAudioLevelMeter,
+  createAnalyserOutputAudioLevelMeter,
+  createStatsOutputAudioLevelMeter,
+  type OutputAudioLevelMeter,
+} from './voice/output-audio-level';
+export { triggerVoiceGreeting } from './voice/trigger-voice-greeting';
+
+// Agent
+export { NxtlinqAgent, type NxtlinqAgentSnapshot, type SendMessageOptions } from './agent/NxtlinqAgent';
+export { ChatOrchestrator } from './agent/ChatOrchestrator';
+export { extractReplyText } from './agent/extractReplyText';
+export { AgentNotInitializedError } from './agent/errors';

package/src/ports/HttpPort.ts

@@ -0,0 +1,12 @@
+export type HttpFetch = (
+  url: string,
+  init?: RequestInit,
+) => Promise<Response>;
+
+export interface HttpPort {
+  fetch: HttpFetch;
+}
+
+export function createDefaultHttpPort(): HttpPort {
+  return { fetch: globalThis.fetch.bind(globalThis) };
+}

package/src/ports/PlatformPorts.ts

@@ -0,0 +1,12 @@
+import type { HttpPort } from './HttpPort';
+import type { StoragePort } from './StoragePort';
+import type { WebRTCPort } from './WebRTCPort';
+
+export interface PlatformPorts {
+  storage: StoragePort;
+  http: HttpPort;
+  /** Optional until M2; when missing, voice connect throws VoiceNotSupportedError. */
+  webrtc?: WebRTCPort;
+  /** IANA timezone for agent requests (default UTC). */
+  getTimezone?: () => string;
+}

package/src/ports/StoragePort.ts

@@ -0,0 +1,37 @@
+export interface StoragePort {
+  getItem(key: string): Promise<string | null>;
+  setItem(key: string, value: string): Promise<void>;
+  removeItem(key: string): Promise<void>;
+}
+
+/** Browser `localStorage` adapter (for Web SDK wrapper). */
+export function createBrowserStoragePort(): StoragePort {
+  return {
+    getItem: async (key) => {
+      if (typeof localStorage === 'undefined') return null;
+      return localStorage.getItem(key);
+    },
+    setItem: async (key, value) => {
+      if (typeof localStorage === 'undefined') return;
+      localStorage.setItem(key, value);
+    },
+    removeItem: async (key) => {
+      if (typeof localStorage === 'undefined') return;
+      localStorage.removeItem(key);
+    },
+  };
+}
+
+/** In-memory storage for tests or SSR. */
+export function createMemoryStoragePort(initial: Record<string, string> = {}): StoragePort {
+  const map = new Map(Object.entries(initial));
+  return {
+    getItem: async (key) => map.get(key) ?? null,
+    setItem: async (key, value) => {
+      map.set(key, value);
+    },
+    removeItem: async (key) => {
+      map.delete(key);
+    },
+  };
+}

package/src/ports/WebRTCPort.ts

@@ -0,0 +1,54 @@
+/**
+ * Platform WebRTC surface. Browser and React Native implementations live in
+ * `@nxtlinq/ai-agent-web` and `@bytexbyte/nxtlinq-ai-agent-react-native-development` (M2).
+ */
+
+export interface IceServerConfig {
+  urls: string | string[];
+  username?: string;
+  credential?: string;
+}
+
+export interface MediaStreamLike {
+  getAudioTracks(): Array<{ enabled: boolean; stop(): void }>;
+}
+
+export interface PeerConnectionLike {
+  connectionState: string;
+  iceConnectionState: string;
+  onconnectionstatechange: (() => void) | null;
+  oniceconnectionstatechange: (() => void) | null;
+  ontrack: ((event: { streams: MediaStreamLike[]; track: { kind: string } }) => void) | null;
+  /** Fired when the remote peer creates a data channel (e.g. Pipecat answer side). */
+  ondatachannel: ((event: { channel: DataChannelLike }) => void) | null;
+  createDataChannel(label: string): DataChannelLike;
+  addTrack(track: unknown, stream: MediaStreamLike): void;
+  createOffer(): Promise<{ sdp?: string; type?: string }>;
+  setLocalDescription(desc: unknown): Promise<void>;
+  setRemoteDescription(desc: unknown): Promise<void>;
+  getLocalDescription(): { sdp?: string; type?: string } | null;
+  close(): void;
+}
+
+export interface DataChannelLike {
+  readyState: string;
+  onopen: (() => void) | null;
+  onmessage: ((event: { data: string }) => void) | null;
+  onclose: (() => void) | null;
+  send(data: string): void;
+  close(): void;
+}
+
+export interface WebRTCPort {
+  isSupported(): boolean;
+  getUserMedia(constraints: { audio: boolean }): Promise<MediaStreamLike>;
+  createPeerConnection(config: { iceServers: IceServerConfig[] }): PeerConnectionLike;
+  waitForIceGathering(pc: PeerConnectionLike, timeoutMs: number): Promise<void>;
+}
+
+export class VoiceNotSupportedError extends Error {
+  constructor(message = 'WebRTC voice is not supported on this platform') {
+    super(message);
+    this.name = 'VoiceNotSupportedError';
+  }
+}

package/src/ports/createBrowserWebRTCPort.ts

@@ -0,0 +1,163 @@
+import type {
+  DataChannelLike,
+  IceServerConfig,
+  MediaStreamLike,
+  PeerConnectionLike,
+  WebRTCPort,
+} from './WebRTCPort';
+import { VoiceNotSupportedError } from './WebRTCPort';
+
+const ICE_GATHERING_TIMEOUT_MS = 5000;
+
+function wrapDataChannel(channel: RTCDataChannel): DataChannelLike {
+  const wrapper: DataChannelLike = {
+    get readyState() {
+      return channel.readyState;
+    },
+    onopen: null,
+    onmessage: null,
+    onclose: null,
+    send: (data) => channel.send(data),
+    close: () => channel.close(),
+  };
+  Object.defineProperty(wrapper, 'onopen', {
+    get: () => null,
+    set(handler: (() => void) | null) {
+      channel.onopen = handler ? () => handler() : null;
+    },
+  });
+  Object.defineProperty(wrapper, 'onmessage', {
+    get: () => null,
+    set(handler: ((event: { data: string }) => void) | null) {
+      channel.onmessage = handler
+        ? (event: MessageEvent) => handler({ data: String(event.data) })
+        : null;
+    },
+  });
+  Object.defineProperty(wrapper, 'onclose', {
+    get: () => null,
+    set(handler: (() => void) | null) {
+      channel.onclose = handler ? () => handler() : null;
+    },
+  });
+  return wrapper;
+}
+
+export type BrowserPeerConnectionLike = PeerConnectionLike & {
+  __nativePc: RTCPeerConnection;
+};
+
+function wrapPeerConnection(pc: RTCPeerConnection): BrowserPeerConnectionLike {
+  const wrapped: BrowserPeerConnectionLike = {
+    __nativePc: pc,
+    get connectionState() {
+      return pc.connectionState;
+    },
+    get iceConnectionState() {
+      return pc.iceConnectionState;
+    },
+    onconnectionstatechange: null,
+    oniceconnectionstatechange: null,
+    ontrack: null,
+    ondatachannel: null,
+    createDataChannel: (label) => wrapDataChannel(pc.createDataChannel(label)),
+    addTrack: (track, stream) => pc.addTrack(track as MediaStreamTrack, stream as MediaStream),
+    createOffer: () => pc.createOffer(),
+    setLocalDescription: (desc) => pc.setLocalDescription(desc as RTCSessionDescriptionInit),
+    setRemoteDescription: (desc) => pc.setRemoteDescription(desc as RTCSessionDescriptionInit),
+    getLocalDescription: () => {
+      const local = pc.localDescription;
+      return local ? { sdp: local.sdp ?? undefined, type: local.type } : null;
+    },
+    close: () => pc.close(),
+  };
+  Object.defineProperty(wrapped, 'onconnectionstatechange', {
+    get: () => null,
+    set(handler: (() => void) | null) {
+      pc.onconnectionstatechange = handler ? () => handler() : null;
+    },
+  });
+  Object.defineProperty(wrapped, 'oniceconnectionstatechange', {
+    get: () => null,
+    set(handler: (() => void) | null) {
+      pc.oniceconnectionstatechange = handler ? () => handler() : null;
+    },
+  });
+  Object.defineProperty(wrapped, 'ontrack', {
+    get: () => null,
+    set(handler: PeerConnectionLike['ontrack']) {
+      pc.ontrack = handler
+        ? (event: RTCTrackEvent) => {
+          handler({
+            streams: Array.from(event.streams) as unknown as MediaStreamLike[],
+            track: { kind: event.track.kind },
+          });
+        }
+        : null;
+    },
+  });
+  Object.defineProperty(wrapped, 'ondatachannel', {
+    get: () => null,
+    set(handler: PeerConnectionLike['ondatachannel']) {
+      pc.ondatachannel = handler
+        ? (event: RTCDataChannelEvent) => {
+          handler({ channel: wrapDataChannel(event.channel) });
+        }
+        : null;
+    },
+  });
+  return wrapped;
+}
+
+/** Browser `RTCPeerConnection` adapter for Web SDK. */
+export function createBrowserWebRTCPort(): WebRTCPort {
+  if (
+    typeof RTCPeerConnection === 'undefined'
+    || typeof navigator === 'undefined'
+    || !navigator.mediaDevices?.getUserMedia
+  ) {
+    throw new VoiceNotSupportedError(
+      'WebRTC is not available — use a browser with RTCPeerConnection support.',
+    );
+  }
+
+  return {
+    isSupported: () =>
+      typeof RTCPeerConnection !== 'undefined'
+      && typeof navigator !== 'undefined'
+      && !!navigator.mediaDevices?.getUserMedia,
+
+    getUserMedia: async (constraints) => {
+      const stream = await navigator.mediaDevices.getUserMedia(constraints);
+      return stream as unknown as MediaStreamLike;
+    },
+
+    createPeerConnection: (config: { iceServers: IceServerConfig[] }) =>
+      wrapPeerConnection(
+        new RTCPeerConnection({
+          iceServers: config.iceServers as RTCIceServer[],
+        }),
+      ),
+
+    waitForIceGathering: async (pc, timeoutMs = ICE_GATHERING_TIMEOUT_MS) => {
+      const native = (pc as BrowserPeerConnectionLike).__nativePc;
+      if (native.iceGatheringState === 'complete') return;
+      await new Promise<void>((resolve) => {
+        let done = false;
+        const finish = () => {
+          if (done) return;
+          done = true;
+          native.removeEventListener('icegatheringstatechange', check);
+          clearTimeout(timer);
+          resolve();
+        };
+        const check = () => {
+          if (native.iceGatheringState === 'complete') finish();
+        };
+        native.addEventListener('icegatheringstatechange', check);
+        if (native.iceGatheringState === 'complete') finish();
+        const timer = setTimeout(finish, timeoutMs);
+      });
+    },
+  };
+}

package/src/ports/index.ts

@@ -0,0 +1,4 @@
+export * from './StoragePort';
+export * from './HttpPort';
+export * from './WebRTCPort';
+export * from './PlatformPorts';

package/src/types/agent-config.ts

@@ -0,0 +1,51 @@
+import type { Message } from './ait-api';
+
+export type ToolUse = {
+  name: string;
+  input: Record<string, unknown>;
+};
+
+export type AgentEnvironment = 'production' | 'staging';
+
+/**
+ * Core configuration shared by Web ChatBot and RN custom UI.
+ * UI-only fields belong in `@nxtlinq/chatbot-ui` / app layers.
+ */
+export interface AgentConfig {
+  serviceId: string;
+  apiKey: string;
+  apiSecret: string;
+  environment?: AgentEnvironment;
+
+  onMessage?: (message: Message) => void;
+  onError?: (error: Error) => void;
+  onToolUse?: (
+    toolUse: ToolUse,
+    onProgress?: (update: {
+      status?: string;
+      progress?: number;
+      partialResult?: string;
+      steps?: string[];
+      partialContent?: string;
+    }) => void,
+  ) => Promise<Message | void>;
+
+  customUserInfo?: Record<string, unknown>;
+  customUsername?: string;
+  permissionGroup?: string;
+
+  pseudoId?: string;
+  externalId?: string;
+
+  /** Agent model route, e.g. `open-ai`, `open-ai-stream`. */
+  defaultModel?: string;
+
+  /**
+   * Voice pipeline: `cascade` (STT → LLM → TTS) or `realtime` (OpenAI Realtime).
+   * Passed to `/api/voice/session` when {@link NxtlinqAgent.startVoice} runs.
+   */
+  voiceMode?: 'cascade' | 'realtime';
+
+  maxRetries?: number;
+  retryDelay?: number;
+}