@simfinity/constellation-react 0.0.1

package/dist/index.d.cts

@@ -0,0 +1,261 @@
+import * as react_jsx_runtime from 'react/jsx-runtime';
+import React from 'react';
+import { WebClientConfig, SessionStartParameters, SessionStats, EventHandlers, SessionConfig } from '@simfinity/constellation-client';
+
+/**
+ * State machine
+ */
+type AudioPlaybackState = {
+    playing: boolean;
+};
+
+/**
+ * Audio capturing settings.
+ * Includes a basic Voice-Activation-Detection implementation with minimum energy threshold
+ * and silence tracking to detect end-of-speech.
+ * Default values are provided and should be adequate in most typical setups.
+ */
+declare class AudioCaptureConfig {
+    readonly sampleRate = 24000;
+    /**
+     * Minimum amount of energy for a noise to be considered as actual input and be recorded.
+     *  - Below it, audio data is silently discarded
+     *  - Above it, onAudioChunk events containing the audio stream will be fired
+     *
+     *  @remarks
+     *  This is the main parameter for noise-cancelling
+     */
+    silenceThreshold: number;
+    /**
+     * Once an input was detected, this is the duration of silence required before
+     * detecting the end-of-speech and triggering the onSilenceCommit event.
+     *
+     * IMPORTANT:
+     * Internally, duration is computed on audio chunks count, not system clock.
+     * A chunk is usually 8ms, therefore to see the effect of a silence duration change, it must be
+     * such that the number of chunks count changes e.g. going from 800ms to 810ms increases
+     * the chunk count by one.
+     *
+     * @remarks
+     * This is the main voice-activation-detection parameter.
+     */
+    silenceDurationMs: number;
+    /**
+     * Minimum input duration: below it, an input will not trigger an onSilenceCommit event.
+     *
+     * @remarks
+     * This is an important interruption parameter: it prevents very short inputs from
+     * interrupting an ongoing audio response.
+     */
+    minSpeechDurationMs: number;
+}
+/**
+ * Client callback functions for the different audio events, provided at start time.
+ */
+type CaptureStartParams = {
+    /**
+     * Fired when a valid audio chunk containing sound was captured:
+     * This event only occurs when confirmed input voice was detected.
+     *
+     * @param base64Chunk
+     */
+    onAudioChunk: (base64Chunk: string) => void;
+    /**
+     * Fired after a silence period, following a confirmed input and marks the end of the input.
+     * This event necessarily occurs after at least one onAudioChunk (most likely many).
+     */
+    onSilenceCommit?: () => void;
+    /**
+     * Fired very early in the voice detection process. This event marks the beginning
+     * of an internal process to assess whether the sound captured should convert to an actual input.
+     */
+    onSpeechConsidered?: () => void;
+    /**
+     * Fired when speech is confirmed: marks the end of the process started when onSpeechConsidered was fired.
+     * The captured input has finally been determined to be real voice input.
+     * Upon receiving this event, it is not yet known what the input is or how long it will be, but there is
+     * a guarantee that an input audio message will be produced:
+     *  - onAudioChunk events can be expected next
+     *  - When working with conversational-interruptions, this is typically the right moment to trigger them
+     */
+    onSpeechConfirmed?: () => void;
+};
+/**
+ * State machine
+ */
+type AudioCaptureState = {
+    recording: boolean;
+    speaking: boolean;
+    muted: boolean;
+};
+
+/**
+ * React context provider wrapping a Constellation WebClient instance.
+ *
+ * This component exposes session lifecycle state and client control
+ * functions to its descendant components.
+ *
+ * A ConstellationProvider MUST wrap any component that calls:
+ *  - useConstellationClient()
+ *  - useConstellationSession()
+ *
+ * @param client A pre-configured WebClient instance from "@simfinity/constellation-client".
+ * @param children The DOM descendant components
+ *
+ * @example
+ * ```tsx
+ * import WebClient from "@simfinity/constellation-client";
+ * import { ConstellationProvider } from "@simfinity/constellation-ui";
+ *
+ * const client = new WebClient({
+ *   sessionEndpoint: "https://your-api-endpoint",
+ *   streamingEndpoint: "wss://your-stream-endpoint",
+ *   key: "YOUR_SECRET_KEY",
+ * });
+ *
+ * function App() {
+ *   return (
+ *     <ConstellationProvider client={client}>
+ *       <Chat />
+ *     </ConstellationProvider>
+ *   );
+ * }
+ * ```
+ *
+ * @remarks
+ * This provider does NOT automatically start or join sessions.
+ * The lifecycle must be controlled explicitly via useConstellationClient().
+ */
+declare function ConstellationProvider({ config, children, }: {
+    config: WebClientConfig;
+    children: React.ReactNode;
+}): react_jsx_runtime.JSX.Element;
+
+/**
+ * An instance of constellation client.
+ * If a constellation session is already started & in context, this client
+ * can resume interacting with it:
+ * Either stop it & start a new session, or keep it alive and re-join it.
+ */
+interface ConstellationSession {
+    startSession: (params: SessionStartParameters) => Promise<void>;
+    endSession: () => Promise<SessionStats>;
+    joinSession: (audio: boolean, handlers: ConstellationEventHandlers) => Promise<void>;
+    configureSession: (settings: SessionConfig) => void;
+    sendText: (text: string) => void;
+    sendWhisper: (text: string) => void;
+    audioDevice: () => AudioDevice;
+    sessionId: string | null;
+}
+type ConstellationEventHandlers = Omit<EventHandlers, 'onAudioResponseChunk' | 'onAudioResponseEnd'>;
+interface AudioDevice {
+    in: {
+        start: () => Promise<void>;
+        stop: () => Promise<void>;
+        setMuted: (muted: boolean) => void;
+    };
+    out: {};
+}
+/**
+ * Hook exposing all lifecycle control functions required to
+ * interact with a Constellation session.
+ *
+ * Internally uses the WebClient instance maintained in context by the ConstellationProvider.
+ *
+ * IMPORTANT SESSION ORDER:
+ *
+ * 1) startSession(params)
+ *    → Creates a persistent server-side session (REST)
+ *    → A session remains alive and can be re-joined within 5 minutes of inactivity
+ *    → params.voiceEnabled true means this session will support both text and audio message exchange
+ *
+ * 2) joinSession(audioEnabled, handlers)
+ *    → Opens WebSocket stream and begins event flow
+ *    → audioEnabled (when joining an audioEnabled session) to join in voice mode
+ *    → handlers are the callback functions to receive server events
+ *
+ * 3) configureSession(settings?) (optional)
+ *    → Update of the settings affecting the LLM behaviour/context
+ *    → This can be done at any time mid-stream
+ *    → May not be supported by some LLMs
+ *    → Can be provided initially in the startSession parameters
+ *
+ * 4) sendText(...) OR sendAudioChunk(...) + commitAudio()
+ *    → Input-sending methods text & audio
+ *
+ * 5) endSession()
+ *    → Closes the persistent server-side session
+ *    → For a clean implementation the client should close the stream first
+ *
+ * This hook does NOT expose the underlying WebClient directly.
+ * It provides a controlled abstraction safe for React usage.
+ *
+ * @throws Error if used outside ConstellationProvider.
+ *
+ * @returns a SessionClient instance providing lifecycle control methods:
+ *
+ * startSession
+ *   - Creates a new persistent session on the server.
+ *   - Must be called before joinSession().
+ *
+ * joinSession
+ *   - Opens the WebSocket stream.
+ *   - Handlers are required to receive model responses.
+ *
+ * configureSession
+ *   - Updates temperature, instructions, max tokens.
+ *   - Does NOT trigger a response.
+ *   - The instructions are the "system prompt" giving context to the model
+ *
+ * sendText
+ *   - Sends a user text message.
+ *   - Triggers model response.
+ *
+ * sendAudioChunk
+ *   - Sends base64 PCM16 16k Hertz, audio chunk.
+ *   - Accumulates until commitAudio() or silence detection.
+ *
+ * commitAudio
+ *   - Forces model to process accumulated audio and generate a response.
+ *
+ * endSession
+ *   - Closes session and frees server resources.
+ *
+ * @example
+ * ```tsx
+ * const client = useConstellationClient();
+ * const params: SessionStartParameters = {
+ *   llmProvider: "openai",
+ *   voiceEnabled: false,
+ *   voiceName: "ballad",
+ *   behaviour: {
+ *     temperature: 0.9,
+ *     instructions: "Just have a nice and casual conversation.",
+ *   }
+ * }
+ * await client.startSession(params);
+ * await client.joinSession(false, {
+ *   onStreamClosed: () => console.log,
+ *   onTranscriptResponse: (text) => console.log(text),
+ * });
+ * client.sendText("Hello world");
+ * client.endSession();
+ * ```
+ */
+declare function useConstellationSession(): ConstellationSession;
+
+/**
+ * React hook to get UI reactive AudioPlayback state updates.
+ *
+ * @param selector
+ */
+declare function useAudioPlaybackState<T>(selector: (state: AudioPlaybackState) => T): T;
+
+/**
+ * React hook to subscribe to AudioCapture state updates.
+ *
+ * @param selector
+ */
+declare function useAudioCaptureState<T>(selector: (state: AudioCaptureState) => T): T;
+
+export { AudioCaptureConfig, type AudioCaptureState, type AudioPlaybackState, type CaptureStartParams, ConstellationProvider, type ConstellationSession, useAudioCaptureState, useAudioPlaybackState, useConstellationSession };

package/dist/index.d.ts

@@ -0,0 +1,261 @@
+import * as react_jsx_runtime from 'react/jsx-runtime';
+import React from 'react';
+import { WebClientConfig, SessionStartParameters, SessionStats, EventHandlers, SessionConfig } from '@simfinity/constellation-client';
+
+/**
+ * State machine
+ */
+type AudioPlaybackState = {
+    playing: boolean;
+};
+
+/**
+ * Audio capturing settings.
+ * Includes a basic Voice-Activation-Detection implementation with minimum energy threshold
+ * and silence tracking to detect end-of-speech.
+ * Default values are provided and should be adequate in most typical setups.
+ */
+declare class AudioCaptureConfig {
+    readonly sampleRate = 24000;
+    /**
+     * Minimum amount of energy for a noise to be considered as actual input and be recorded.
+     *  - Below it, audio data is silently discarded
+     *  - Above it, onAudioChunk events containing the audio stream will be fired
+     *
+     *  @remarks
+     *  This is the main parameter for noise-cancelling
+     */
+    silenceThreshold: number;
+    /**
+     * Once an input was detected, this is the duration of silence required before
+     * detecting the end-of-speech and triggering the onSilenceCommit event.
+     *
+     * IMPORTANT:
+     * Internally, duration is computed on audio chunks count, not system clock.
+     * A chunk is usually 8ms, therefore to see the effect of a silence duration change, it must be
+     * such that the number of chunks count changes e.g. going from 800ms to 810ms increases
+     * the chunk count by one.
+     *
+     * @remarks
+     * This is the main voice-activation-detection parameter.
+     */
+    silenceDurationMs: number;
+    /**
+     * Minimum input duration: below it, an input will not trigger an onSilenceCommit event.
+     *
+     * @remarks
+     * This is an important interruption parameter: it prevents very short inputs from
+     * interrupting an ongoing audio response.
+     */
+    minSpeechDurationMs: number;
+}
+/**
+ * Client callback functions for the different audio events, provided at start time.
+ */
+type CaptureStartParams = {
+    /**
+     * Fired when a valid audio chunk containing sound was captured:
+     * This event only occurs when confirmed input voice was detected.
+     *
+     * @param base64Chunk
+     */
+    onAudioChunk: (base64Chunk: string) => void;
+    /**
+     * Fired after a silence period, following a confirmed input and marks the end of the input.
+     * This event necessarily occurs after at least one onAudioChunk (most likely many).
+     */
+    onSilenceCommit?: () => void;
+    /**
+     * Fired very early in the voice detection process. This event marks the beginning
+     * of an internal process to assess whether the sound captured should convert to an actual input.
+     */
+    onSpeechConsidered?: () => void;
+    /**
+     * Fired when speech is confirmed: marks the end of the process started when onSpeechConsidered was fired.
+     * The captured input has finally been determined to be real voice input.
+     * Upon receiving this event, it is not yet known what the input is or how long it will be, but there is
+     * a guarantee that an input audio message will be produced:
+     *  - onAudioChunk events can be expected next
+     *  - When working with conversational-interruptions, this is typically the right moment to trigger them
+     */
+    onSpeechConfirmed?: () => void;
+};
+/**
+ * State machine
+ */
+type AudioCaptureState = {
+    recording: boolean;
+    speaking: boolean;
+    muted: boolean;
+};
+
+/**
+ * React context provider wrapping a Constellation WebClient instance.
+ *
+ * This component exposes session lifecycle state and client control
+ * functions to its descendant components.
+ *
+ * A ConstellationProvider MUST wrap any component that calls:
+ *  - useConstellationClient()
+ *  - useConstellationSession()
+ *
+ * @param client A pre-configured WebClient instance from "@simfinity/constellation-client".
+ * @param children The DOM descendant components
+ *
+ * @example
+ * ```tsx
+ * import WebClient from "@simfinity/constellation-client";
+ * import { ConstellationProvider } from "@simfinity/constellation-ui";
+ *
+ * const client = new WebClient({
+ *   sessionEndpoint: "https://your-api-endpoint",
+ *   streamingEndpoint: "wss://your-stream-endpoint",
+ *   key: "YOUR_SECRET_KEY",
+ * });
+ *
+ * function App() {
+ *   return (
+ *     <ConstellationProvider client={client}>
+ *       <Chat />
+ *     </ConstellationProvider>
+ *   );
+ * }
+ * ```
+ *
+ * @remarks
+ * This provider does NOT automatically start or join sessions.
+ * The lifecycle must be controlled explicitly via useConstellationClient().
+ */
+declare function ConstellationProvider({ config, children, }: {
+    config: WebClientConfig;
+    children: React.ReactNode;
+}): react_jsx_runtime.JSX.Element;
+
+/**
+ * An instance of constellation client.
+ * If a constellation session is already started & in context, this client
+ * can resume interacting with it:
+ * Either stop it & start a new session, or keep it alive and re-join it.
+ */
+interface ConstellationSession {
+    startSession: (params: SessionStartParameters) => Promise<void>;
+    endSession: () => Promise<SessionStats>;
+    joinSession: (audio: boolean, handlers: ConstellationEventHandlers) => Promise<void>;
+    configureSession: (settings: SessionConfig) => void;
+    sendText: (text: string) => void;
+    sendWhisper: (text: string) => void;
+    audioDevice: () => AudioDevice;
+    sessionId: string | null;
+}
+type ConstellationEventHandlers = Omit<EventHandlers, 'onAudioResponseChunk' | 'onAudioResponseEnd'>;
+interface AudioDevice {
+    in: {
+        start: () => Promise<void>;
+        stop: () => Promise<void>;
+        setMuted: (muted: boolean) => void;
+    };
+    out: {};
+}
+/**
+ * Hook exposing all lifecycle control functions required to
+ * interact with a Constellation session.
+ *
+ * Internally uses the WebClient instance maintained in context by the ConstellationProvider.
+ *
+ * IMPORTANT SESSION ORDER:
+ *
+ * 1) startSession(params)
+ *    → Creates a persistent server-side session (REST)
+ *    → A session remains alive and can be re-joined within 5 minutes of inactivity
+ *    → params.voiceEnabled true means this session will support both text and audio message exchange
+ *
+ * 2) joinSession(audioEnabled, handlers)
+ *    → Opens WebSocket stream and begins event flow
+ *    → audioEnabled (when joining an audioEnabled session) to join in voice mode
+ *    → handlers are the callback functions to receive server events
+ *
+ * 3) configureSession(settings?) (optional)
+ *    → Update of the settings affecting the LLM behaviour/context
+ *    → This can be done at any time mid-stream
+ *    → May not be supported by some LLMs
+ *    → Can be provided initially in the startSession parameters
+ *
+ * 4) sendText(...) OR sendAudioChunk(...) + commitAudio()
+ *    → Input-sending methods text & audio
+ *
+ * 5) endSession()
+ *    → Closes the persistent server-side session
+ *    → For a clean implementation the client should close the stream first
+ *
+ * This hook does NOT expose the underlying WebClient directly.
+ * It provides a controlled abstraction safe for React usage.
+ *
+ * @throws Error if used outside ConstellationProvider.
+ *
+ * @returns a SessionClient instance providing lifecycle control methods:
+ *
+ * startSession
+ *   - Creates a new persistent session on the server.
+ *   - Must be called before joinSession().
+ *
+ * joinSession
+ *   - Opens the WebSocket stream.
+ *   - Handlers are required to receive model responses.
+ *
+ * configureSession
+ *   - Updates temperature, instructions, max tokens.
+ *   - Does NOT trigger a response.
+ *   - The instructions are the "system prompt" giving context to the model
+ *
+ * sendText
+ *   - Sends a user text message.
+ *   - Triggers model response.
+ *
+ * sendAudioChunk
+ *   - Sends base64 PCM16 16k Hertz, audio chunk.
+ *   - Accumulates until commitAudio() or silence detection.
+ *
+ * commitAudio
+ *   - Forces model to process accumulated audio and generate a response.
+ *
+ * endSession
+ *   - Closes session and frees server resources.
+ *
+ * @example
+ * ```tsx
+ * const client = useConstellationClient();
+ * const params: SessionStartParameters = {
+ *   llmProvider: "openai",
+ *   voiceEnabled: false,
+ *   voiceName: "ballad",
+ *   behaviour: {
+ *     temperature: 0.9,
+ *     instructions: "Just have a nice and casual conversation.",
+ *   }
+ * }
+ * await client.startSession(params);
+ * await client.joinSession(false, {
+ *   onStreamClosed: () => console.log,
+ *   onTranscriptResponse: (text) => console.log(text),
+ * });
+ * client.sendText("Hello world");
+ * client.endSession();
+ * ```
+ */
+declare function useConstellationSession(): ConstellationSession;
+
+/**
+ * React hook to get UI reactive AudioPlayback state updates.
+ *
+ * @param selector
+ */
+declare function useAudioPlaybackState<T>(selector: (state: AudioPlaybackState) => T): T;
+
+/**
+ * React hook to subscribe to AudioCapture state updates.
+ *
+ * @param selector
+ */
+declare function useAudioCaptureState<T>(selector: (state: AudioCaptureState) => T): T;
+
+export { AudioCaptureConfig, type AudioCaptureState, type AudioPlaybackState, type CaptureStartParams, ConstellationProvider, type ConstellationSession, useAudioCaptureState, useAudioPlaybackState, useConstellationSession };