@simfinity/constellation-react 0.0.1 → 0.0.2

package/README.md

@@ -1,4 +1,4 @@
-# @simfinity/constellation-ui
+# @simfinity/constellation-react
 
 React bindings for Simfinity Constellation Client — persistent real-time LLM chat rooms (text + audio).
 
@@ -20,14 +20,9 @@ It provides context + hooks for lifecycle management.
 
 ## Installation
 ```bash
-npm install @simfinity/constellation-ui
+npm install @simfinity/constellation-react
 # or
-yarn add @simfinity/constellation-ui
-
-# Dependency:
-npm install @simfinity/constellation-client
-# or
-yarn add @simfinity/constellation-client
+yarn add @simfinity/constellation-react
 ```
 
 ## Architecture
@@ -51,20 +46,13 @@ endSession()    → REST call
 
 ```Typescript
 import React, { useEffect } from "react";
-import WebClient from "@simfinity/constellation-client";
 import {
     ConstellationProvider,
-    useConstellationClient
-} from "@simfinity/constellation-ui";
-
-const client = new WebClient({
-    sessionEndpoint: "https://your-api",
-    streamingEndpoint: "wss://your-stream",
-    key: "YOUR_SECRET_KEY",
-});
+    useConstellationSession
+} from "@simfinity/constellation-react";
 
 function Chat() {
-    const client = useConstellationClient();
+    const constellationSession = useConstellationSession();
     
     useEffect(() => {
         async function init() {
@@ -77,22 +65,21 @@ function Chat() {
                 }
             }
             
-            await client.startSession(params);
-
-            await client.joinSession(false, {
+            await constellationSession.startSession(params);
+            await constellationSession.joinSession(false, {
                 onStreamClosed: console.log,
                 onTranscriptResponse: (msg) => {
                   console.log("Model:", msg);
                 }
-              });
+            });
 
-            client.sendText("Hello!");
+            constellationSession.sendText("Hello!");
         }
     
         init();
     
         return () => {
-            client.endSession();
+            constellationSession.endSession();
         };
     }, []);
     
@@ -101,7 +88,11 @@ function Chat() {
 
 export default function App() {
     return (
-        <ConstellationProvider client={client}>
+        <ConstellationProvider config={{
+            sessionEndpoint: "https://your-api",
+            streamingEndpoint: "wss://your-stream",
+            key: "YOUR_SECRET_KEY"
+        }}>
           <Chat />
         </ConstellationProvider>
     );
@@ -117,53 +108,45 @@ To enable audio:
 const params: SessionStartParameters = {
     llmProvider: "openai",
     voiceEnabled: true,
+    interruptions: true,
     voiceName: "alloy",
     behaviour: {
         temperature: 0.9,
         instructions: "Just have a nice and casual conversation.",
     }
 }
-await startSession(params);
 
+await constellationSession.startSession(params);
 // Join a stream subscribing to audio events
-await joinSession(true, {
+await constellationSession.joinSession(true, {
     onStreamClosed: console.log,
-    onAudioResponseStart: () => console.log("Speaking..."),
-    onAudioResponseChunk: (chunk) => audioPlayer.enqueue(chunk),
-    onAudioResponseEnd: () => console.log("Done")
 });
+
+// Use built-in audio capture and playback device
+await constellationSession.audioDevice().in.start();
+constellationSession.audioDevice().in.setMuted(true);
 ```
 
 Audio requirements:
 
 ### Input:
-* Format: PCM, 16k Hertz
+* Format: PCM, 24k Hertz
 * Encoding: Base64
 * Transcription: handled server-side
 
-Send audio:
-
-```Typescript
-sendAudioChunk(base64PcmChunk);
-commitAudio();
-```
-
-commitAudio() is MANDATORY: server-side VAD (voice activation detection) is not provided.
-
-⚠️ Client should always implement audio-input noise detection and explicit commits:
-* Current constellation version does not provide it server-side
-* Avoids continuously streaming audio data which reduces network and token consumption
-* Allows to provide a more responsive experience as VAD introduces a constant delay and is potentially less stable
-
 ### Output:
 Audio responses are:
 * Format: PCM, 24k Hertz
 * Encoding: Base64
 
+Sending audio:
+
+Handled internally by the constellationSession package. 
+
 
 ## Text and Transcript
 
-Text is always enabled in a session. However, the client must provide the appropriate handlers to receive events:
+Text is always enabled in a session. However, the client application must provide the appropriate handlers to receive events:
 ```Typescript
 interface EventHandlers {
   // ...
@@ -177,15 +160,14 @@ interface EventHandlers {
 // Pseudo-code:
 
 // Text:
-constellationClient.sendText("Hello");
+constellationSession.sendText("Hello");
 
 // Triggers:
 // 1) onTranscriptInput(transcript) -> transcript is "Hello"
 // 2) onTranscriptResponse(transcript) -> transcript is the response from the LLM
 
 // Audio:
-constellationClient.sendAudioChunk("... PCM16 audio data for 'Hello'...");
-constellationClient.commitAudio();
+// ... Internally, constellationSession does 'send audio data + commit audio message' ...
 
 // Triggers:
 // 1) onTranscriptInput(transcript) -> transcript is "Hello"
@@ -226,8 +208,6 @@ joinSession() requires at least:
 Optional handlers:
 
 * onSessionConfigured: acknowledgment of configureSession call
-* onAudioResponseChunk: audio data chunk of a model audio response
-* onAudioResponseEnd: model has finished streaming an audio response
 * onResponseEnd: model has finished generating and streaming a response
 * onTranscriptInput: the echo of a user text input or transcript of a user audio input
 * onTranscriptInputPart: the echo of a user text input or a piece of transcript of a user audio input

package/dist/index.cjs

@@ -30,7 +30,6 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
-  AudioCaptureConfig: () => AudioCaptureConfig,
   ConstellationProvider: () => ConstellationProvider,
   useAudioCaptureState: () => useAudioCaptureState,
   useAudioPlaybackState: () => useAudioPlaybackState,
@@ -911,7 +910,6 @@ function useAudioCaptureState(selector) {
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  AudioCaptureConfig,
   ConstellationProvider,
   useAudioCaptureState,
   useAudioPlaybackState,

package/dist/index.d.cts

@@ -1,6 +1,6 @@
 import * as react_jsx_runtime from 'react/jsx-runtime';
 import React from 'react';
-import { WebClientConfig, SessionStartParameters, SessionStats, EventHandlers, SessionConfig } from '@simfinity/constellation-client';
+import { WebClientConfig, EventHandlers, SessionStartParameters, SessionStats, SessionConfig } from '@simfinity/constellation-client';
 
 /**
  * State machine
@@ -9,77 +9,6 @@ type AudioPlaybackState = {
     playing: boolean;
 };
 
-/**
- * Audio capturing settings.
- * Includes a basic Voice-Activation-Detection implementation with minimum energy threshold
- * and silence tracking to detect end-of-speech.
- * Default values are provided and should be adequate in most typical setups.
- */
-declare class AudioCaptureConfig {
-    readonly sampleRate = 24000;
-    /**
-     * Minimum amount of energy for a noise to be considered as actual input and be recorded.
-     *  - Below it, audio data is silently discarded
-     *  - Above it, onAudioChunk events containing the audio stream will be fired
-     *
-     *  @remarks
-     *  This is the main parameter for noise-cancelling
-     */
-    silenceThreshold: number;
-    /**
-     * Once an input was detected, this is the duration of silence required before
-     * detecting the end-of-speech and triggering the onSilenceCommit event.
-     *
-     * IMPORTANT:
-     * Internally, duration is computed on audio chunks count, not system clock.
-     * A chunk is usually 8ms, therefore to see the effect of a silence duration change, it must be
-     * such that the number of chunks count changes e.g. going from 800ms to 810ms increases
-     * the chunk count by one.
-     *
-     * @remarks
-     * This is the main voice-activation-detection parameter.
-     */
-    silenceDurationMs: number;
-    /**
-     * Minimum input duration: below it, an input will not trigger an onSilenceCommit event.
-     *
-     * @remarks
-     * This is an important interruption parameter: it prevents very short inputs from
-     * interrupting an ongoing audio response.
-     */
-    minSpeechDurationMs: number;
-}
-/**
- * Client callback functions for the different audio events, provided at start time.
- */
-type CaptureStartParams = {
-    /**
-     * Fired when a valid audio chunk containing sound was captured:
-     * This event only occurs when confirmed input voice was detected.
-     *
-     * @param base64Chunk
-     */
-    onAudioChunk: (base64Chunk: string) => void;
-    /**
-     * Fired after a silence period, following a confirmed input and marks the end of the input.
-     * This event necessarily occurs after at least one onAudioChunk (most likely many).
-     */
-    onSilenceCommit?: () => void;
-    /**
-     * Fired very early in the voice detection process. This event marks the beginning
-     * of an internal process to assess whether the sound captured should convert to an actual input.
-     */
-    onSpeechConsidered?: () => void;
-    /**
-     * Fired when speech is confirmed: marks the end of the process started when onSpeechConsidered was fired.
-     * The captured input has finally been determined to be real voice input.
-     * Upon receiving this event, it is not yet known what the input is or how long it will be, but there is
-     * a guarantee that an input audio message will be produced:
-     *  - onAudioChunk events can be expected next
-     *  - When working with conversational-interruptions, this is typically the right moment to trigger them
-     */
-    onSpeechConfirmed?: () => void;
-};
 /**
  * State machine
  */
@@ -96,26 +25,22 @@ type AudioCaptureState = {
  * functions to its descendant components.
  *
  * A ConstellationProvider MUST wrap any component that calls:
- *  - useConstellationClient()
  *  - useConstellationSession()
  *
- * @param client A pre-configured WebClient instance from "@simfinity/constellation-client".
+ * @param config the configuration for underlying WebClient instance from "@simfinity/constellation-client".
  * @param children The DOM descendant components
  *
  * @example
  * ```tsx
- * import WebClient from "@simfinity/constellation-client";
- * import { ConstellationProvider } from "@simfinity/constellation-ui";
- *
- * const client = new WebClient({
- *   sessionEndpoint: "https://your-api-endpoint",
- *   streamingEndpoint: "wss://your-stream-endpoint",
- *   key: "YOUR_SECRET_KEY",
- * });
+ * import { ConstellationProvider } from "@simfinity/constellation-react";
  *
  * function App() {
  *   return (
- *     <ConstellationProvider client={client}>
+ *     <ConstellationProvider config={{
+ *          sessionEndpoint: "https://your-api-endpoint",
+ *          streamingEndpoint: "wss://your-stream-endpoint",
+ *          key: "YOUR_SECRET_KEY",
+ *     }}>
  *       <Chat />
  *     </ConstellationProvider>
  *   );
@@ -124,7 +49,7 @@ type AudioCaptureState = {
  *
  * @remarks
  * This provider does NOT automatically start or join sessions.
- * The lifecycle must be controlled explicitly via useConstellationClient().
+ * The lifecycle must be controlled explicitly via useConstellationSession().
  */
 declare function ConstellationProvider({ config, children, }: {
     config: WebClientConfig;
@@ -132,8 +57,8 @@ declare function ConstellationProvider({ config, children, }: {
 }): react_jsx_runtime.JSX.Element;
 
 /**
- * An instance of constellation client.
- * If a constellation session is already started & in context, this client
+ * An instance of constellation session.
+ * If a constellation session is already started & in context, this instance
  * can resume interacting with it:
  * Either stop it & start a new session, or keep it alive and re-join it.
  */
@@ -160,7 +85,7 @@ interface AudioDevice {
  * Hook exposing all lifecycle control functions required to
  * interact with a Constellation session.
  *
- * Internally uses the WebClient instance maintained in context by the ConstellationProvider.
+ * Internally uses a unique constellation client instance maintained in context by the ConstellationProvider.
  *
  * IMPORTANT SESSION ORDER:
  *
@@ -180,10 +105,13 @@ interface AudioDevice {
  *    → May not be supported by some LLMs
  *    → Can be provided initially in the startSession parameters
  *
- * 4) sendText(...) OR sendAudioChunk(...) + commitAudio()
- *    → Input-sending methods text & audio
+ * 4) sendText(...)
+ *    → Input-sending method for text
  *
- * 5) endSession()
+ * 5) await audioDevice().in.start()
+ *    → Start capture sound in an audio session
+ *
+ * 6) endSession()
  *    → Closes the persistent server-side session
  *    → For a clean implementation the client should close the stream first
  *
@@ -192,38 +120,11 @@ interface AudioDevice {
  *
  * @throws Error if used outside ConstellationProvider.
  *
- * @returns a SessionClient instance providing lifecycle control methods:
- *
- * startSession
- *   - Creates a new persistent session on the server.
- *   - Must be called before joinSession().
- *
- * joinSession
- *   - Opens the WebSocket stream.
- *   - Handlers are required to receive model responses.
- *
- * configureSession
- *   - Updates temperature, instructions, max tokens.
- *   - Does NOT trigger a response.
- *   - The instructions are the "system prompt" giving context to the model
- *
- * sendText
- *   - Sends a user text message.
- *   - Triggers model response.
- *
- * sendAudioChunk
- *   - Sends base64 PCM16 16k Hertz, audio chunk.
- *   - Accumulates until commitAudio() or silence detection.
- *
- * commitAudio
- *   - Forces model to process accumulated audio and generate a response.
- *
- * endSession
- *   - Closes session and frees server resources.
+ * @returns a ConstellationSession instance providing lifecycle control methods:
  *
  * @example
  * ```tsx
- * const client = useConstellationClient();
+ * const constellationSession = useConstellationSession();
  * const params: SessionStartParameters = {
  *   llmProvider: "openai",
  *   voiceEnabled: false,
@@ -233,13 +134,13 @@ interface AudioDevice {
  *     instructions: "Just have a nice and casual conversation.",
  *   }
  * }
- * await client.startSession(params);
- * await client.joinSession(false, {
+ * await constellationSession.startSession(params);
+ * await constellationSession.joinSession(false, {
  *   onStreamClosed: () => console.log,
  *   onTranscriptResponse: (text) => console.log(text),
  * });
- * client.sendText("Hello world");
- * client.endSession();
+ * constellationSession.sendText("Hello world");
+ * constellationSession.endSession();
  * ```
  */
 declare function useConstellationSession(): ConstellationSession;
@@ -258,4 +159,4 @@ declare function useAudioPlaybackState<T>(selector: (state: AudioPlaybackState)
  */
 declare function useAudioCaptureState<T>(selector: (state: AudioCaptureState) => T): T;
 
-export { AudioCaptureConfig, type AudioCaptureState, type AudioPlaybackState, type CaptureStartParams, ConstellationProvider, type ConstellationSession, useAudioCaptureState, useAudioPlaybackState, useConstellationSession };
+export { type AudioCaptureState, type AudioDevice, type AudioPlaybackState, type ConstellationEventHandlers, ConstellationProvider, type ConstellationSession, useAudioCaptureState, useAudioPlaybackState, useConstellationSession };

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 import * as react_jsx_runtime from 'react/jsx-runtime';
 import React from 'react';
-import { WebClientConfig, SessionStartParameters, SessionStats, EventHandlers, SessionConfig } from '@simfinity/constellation-client';
+import { WebClientConfig, EventHandlers, SessionStartParameters, SessionStats, SessionConfig } from '@simfinity/constellation-client';
 
 /**
  * State machine
@@ -9,77 +9,6 @@ type AudioPlaybackState = {
     playing: boolean;
 };
 
-/**
- * Audio capturing settings.
- * Includes a basic Voice-Activation-Detection implementation with minimum energy threshold
- * and silence tracking to detect end-of-speech.
- * Default values are provided and should be adequate in most typical setups.
- */
-declare class AudioCaptureConfig {
-    readonly sampleRate = 24000;
-    /**
-     * Minimum amount of energy for a noise to be considered as actual input and be recorded.
-     *  - Below it, audio data is silently discarded
-     *  - Above it, onAudioChunk events containing the audio stream will be fired
-     *
-     *  @remarks
-     *  This is the main parameter for noise-cancelling
-     */
-    silenceThreshold: number;
-    /**
-     * Once an input was detected, this is the duration of silence required before
-     * detecting the end-of-speech and triggering the onSilenceCommit event.
-     *
-     * IMPORTANT:
-     * Internally, duration is computed on audio chunks count, not system clock.
-     * A chunk is usually 8ms, therefore to see the effect of a silence duration change, it must be
-     * such that the number of chunks count changes e.g. going from 800ms to 810ms increases
-     * the chunk count by one.
-     *
-     * @remarks
-     * This is the main voice-activation-detection parameter.
-     */
-    silenceDurationMs: number;
-    /**
-     * Minimum input duration: below it, an input will not trigger an onSilenceCommit event.
-     *
-     * @remarks
-     * This is an important interruption parameter: it prevents very short inputs from
-     * interrupting an ongoing audio response.
-     */
-    minSpeechDurationMs: number;
-}
-/**
- * Client callback functions for the different audio events, provided at start time.
- */
-type CaptureStartParams = {
-    /**
-     * Fired when a valid audio chunk containing sound was captured:
-     * This event only occurs when confirmed input voice was detected.
-     *
-     * @param base64Chunk
-     */
-    onAudioChunk: (base64Chunk: string) => void;
-    /**
-     * Fired after a silence period, following a confirmed input and marks the end of the input.
-     * This event necessarily occurs after at least one onAudioChunk (most likely many).
-     */
-    onSilenceCommit?: () => void;
-    /**
-     * Fired very early in the voice detection process. This event marks the beginning
-     * of an internal process to assess whether the sound captured should convert to an actual input.
-     */
-    onSpeechConsidered?: () => void;
-    /**
-     * Fired when speech is confirmed: marks the end of the process started when onSpeechConsidered was fired.
-     * The captured input has finally been determined to be real voice input.
-     * Upon receiving this event, it is not yet known what the input is or how long it will be, but there is
-     * a guarantee that an input audio message will be produced:
-     *  - onAudioChunk events can be expected next
-     *  - When working with conversational-interruptions, this is typically the right moment to trigger them
-     */
-    onSpeechConfirmed?: () => void;
-};
 /**
  * State machine
  */
@@ -96,26 +25,22 @@ type AudioCaptureState = {
  * functions to its descendant components.
  *
  * A ConstellationProvider MUST wrap any component that calls:
- *  - useConstellationClient()
  *  - useConstellationSession()
  *
- * @param client A pre-configured WebClient instance from "@simfinity/constellation-client".
+ * @param config the configuration for underlying WebClient instance from "@simfinity/constellation-client".
  * @param children The DOM descendant components
  *
  * @example
  * ```tsx
- * import WebClient from "@simfinity/constellation-client";
- * import { ConstellationProvider } from "@simfinity/constellation-ui";
- *
- * const client = new WebClient({
- *   sessionEndpoint: "https://your-api-endpoint",
- *   streamingEndpoint: "wss://your-stream-endpoint",
- *   key: "YOUR_SECRET_KEY",
- * });
+ * import { ConstellationProvider } from "@simfinity/constellation-react";
  *
  * function App() {
  *   return (
- *     <ConstellationProvider client={client}>
+ *     <ConstellationProvider config={{
+ *          sessionEndpoint: "https://your-api-endpoint",
+ *          streamingEndpoint: "wss://your-stream-endpoint",
+ *          key: "YOUR_SECRET_KEY",
+ *     }}>
  *       <Chat />
  *     </ConstellationProvider>
  *   );
@@ -124,7 +49,7 @@ type AudioCaptureState = {
  *
  * @remarks
  * This provider does NOT automatically start or join sessions.
- * The lifecycle must be controlled explicitly via useConstellationClient().
+ * The lifecycle must be controlled explicitly via useConstellationSession().
  */
 declare function ConstellationProvider({ config, children, }: {
     config: WebClientConfig;
@@ -132,8 +57,8 @@ declare function ConstellationProvider({ config, children, }: {
 }): react_jsx_runtime.JSX.Element;
 
 /**
- * An instance of constellation client.
- * If a constellation session is already started & in context, this client
+ * An instance of constellation session.
+ * If a constellation session is already started & in context, this instance
  * can resume interacting with it:
  * Either stop it & start a new session, or keep it alive and re-join it.
  */
@@ -160,7 +85,7 @@ interface AudioDevice {
  * Hook exposing all lifecycle control functions required to
  * interact with a Constellation session.
  *
- * Internally uses the WebClient instance maintained in context by the ConstellationProvider.
+ * Internally uses a unique constellation client instance maintained in context by the ConstellationProvider.
  *
  * IMPORTANT SESSION ORDER:
  *
@@ -180,10 +105,13 @@ interface AudioDevice {
  *    → May not be supported by some LLMs
  *    → Can be provided initially in the startSession parameters
  *
- * 4) sendText(...) OR sendAudioChunk(...) + commitAudio()
- *    → Input-sending methods text & audio
+ * 4) sendText(...)
+ *    → Input-sending method for text
  *
- * 5) endSession()
+ * 5) await audioDevice().in.start()
+ *    → Start capture sound in an audio session
+ *
+ * 6) endSession()
  *    → Closes the persistent server-side session
  *    → For a clean implementation the client should close the stream first
  *
@@ -192,38 +120,11 @@ interface AudioDevice {
  *
  * @throws Error if used outside ConstellationProvider.
  *
- * @returns a SessionClient instance providing lifecycle control methods:
- *
- * startSession
- *   - Creates a new persistent session on the server.
- *   - Must be called before joinSession().
- *
- * joinSession
- *   - Opens the WebSocket stream.
- *   - Handlers are required to receive model responses.
- *
- * configureSession
- *   - Updates temperature, instructions, max tokens.
- *   - Does NOT trigger a response.
- *   - The instructions are the "system prompt" giving context to the model
- *
- * sendText
- *   - Sends a user text message.
- *   - Triggers model response.
- *
- * sendAudioChunk
- *   - Sends base64 PCM16 16k Hertz, audio chunk.
- *   - Accumulates until commitAudio() or silence detection.
- *
- * commitAudio
- *   - Forces model to process accumulated audio and generate a response.
- *
- * endSession
- *   - Closes session and frees server resources.
+ * @returns a ConstellationSession instance providing lifecycle control methods:
  *
  * @example
  * ```tsx
- * const client = useConstellationClient();
+ * const constellationSession = useConstellationSession();
  * const params: SessionStartParameters = {
  *   llmProvider: "openai",
  *   voiceEnabled: false,
@@ -233,13 +134,13 @@ interface AudioDevice {
  *     instructions: "Just have a nice and casual conversation.",
  *   }
  * }
- * await client.startSession(params);
- * await client.joinSession(false, {
+ * await constellationSession.startSession(params);
+ * await constellationSession.joinSession(false, {
  *   onStreamClosed: () => console.log,
  *   onTranscriptResponse: (text) => console.log(text),
  * });
- * client.sendText("Hello world");
- * client.endSession();
+ * constellationSession.sendText("Hello world");
+ * constellationSession.endSession();
  * ```
  */
 declare function useConstellationSession(): ConstellationSession;
@@ -258,4 +159,4 @@ declare function useAudioPlaybackState<T>(selector: (state: AudioPlaybackState)
  */
 declare function useAudioCaptureState<T>(selector: (state: AudioCaptureState) => T): T;
 
-export { AudioCaptureConfig, type AudioCaptureState, type AudioPlaybackState, type CaptureStartParams, ConstellationProvider, type ConstellationSession, useAudioCaptureState, useAudioPlaybackState, useConstellationSession };
+export { type AudioCaptureState, type AudioDevice, type AudioPlaybackState, type ConstellationEventHandlers, ConstellationProvider, type ConstellationSession, useAudioCaptureState, useAudioPlaybackState, useConstellationSession };

package/dist/index.js

@@ -872,7 +872,6 @@ function useAudioCaptureState(selector) {
   );
 }
 export {
-  AudioCaptureConfig,
   ConstellationProvider,
   useAudioCaptureState,
   useAudioPlaybackState,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@simfinity/constellation-react",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "type": "module",
   "exports": {
     ".": {