@lumavox/react 0.7.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -55,8 +55,10 @@ redial.
55
55
  | `session` | the session it opened, with its last messages |
56
56
  | `error` | `{ kind, message }`: the session, an `error` frame, the close, the microphone |
57
57
  | `speaking` `recording` | the adapter is playing; the recorder is open |
58
+ | `interrupted` | the device has the audio — a call, Siri, another app |
58
59
  | `send` `startAudio` `sendAudio` `endAudio` `cancel` `mute` `reconnect` `close` | the conversation's |
59
60
  | `startRecording` `stopRecording({ transcribeOnly, discard })` | through the `audio` adapter |
61
+ | `listen(options)` `stopListening()` `listening` `hearing` | hands-free: each utterance a turn, talking over the agent stopping it (`@lumavox/client`'s `listen`) |
60
62
  | `newSession()` | a new conversation with the same agent |
61
63
  | `conversation` | the `Conversation` itself |
62
64
 
@@ -67,9 +69,13 @@ running the latest render's handler. **`audio`**, an `AgentAudio` — a player a
67
69
  recorder — makes replies audible and `startRecording` work: replies play
68
70
  as they arrive and stop on `cancel`, `mute` or a new message; recording
69
71
  opens the microphone first and the recording at the rate it was granted.
70
- Without an adapter or an `onAudio`, no speech is asked for. The SDK ships
71
- no player or recorder in this release: write one on Web Audio, or on your
72
- React Native audio library.
72
+ Without an adapter or an `onAudio`, no speech is asked for. In a browser,
73
+ `createWebAudio()` from `@lumavox/client/web-audio` is one — call its
74
+ `prime()` in the click that starts the conversation; on React Native,
75
+ `@lumavox/react-native` is one install with these hooks and `createAudio()`.
76
+ An adapter that tells an interruption — a call, Siri, the app leaving the
77
+ screen — has the recording in progress thrown away, `interrupted` set, and
78
+ `onAudioInterrupted` and `onAudioResumed` called.
73
79
 
74
80
  ## Reading
75
81
 
@@ -12,11 +12,21 @@
12
12
  * *then* the recording, at the rate the recorder was granted, so a
13
13
  * refused microphone never opens a window on the socket. The adapter is
14
14
  * the app's: the hook never disposes it.
15
+ * - **An interruption throws the recording away**: when the adapter says
16
+ * the device took the audio (`onInterruption`) — a call, Siri, the app
17
+ * leaving the screen — a recording open is discarded as
18
+ * `stopRecording({ discard: true })` would, and `interrupted` is true
19
+ * until the adapter gives the audio back.
15
20
  * - **A late answer is dropped quietly**: a session opened after unmount
16
21
  * is never connected, and a press released while the microphone was
17
22
  * still opening gives it back.
23
+ * - **Hands-free is `listen()`** (`@lumavox/client`'s): the recorder stays
24
+ * open and each utterance is a turn; talking over the agent stops it on
25
+ * the edge's `barge_in`, and its voice stays stopped for the rest of
26
+ * that reply. Push-to-talk waits while it listens, and it ends with the
27
+ * conversation.
18
28
  */
19
- import { type AgentAudio, type Client, type CloseInfo, type ConnectOptions, type Conversation, type ConversationHandlers, type ErrorFrame, type PlatformError, type SessionDetail, type Snapshot, type StartAudioOptions, type TranscriptOutcome, type TurnOptions, type TurnOutcome } from "@lumavox/client";
29
+ import { type AgentAudio, type Client, type CloseInfo, type ConnectOptions, type Conversation, type ConversationHandlers, type ErrorFrame, type ListenOptions, type PlatformError, type SessionDetail, type Snapshot, type StartAudioOptions, type TranscriptOutcome, type TurnOptions, type TurnOutcome } from "@lumavox/client";
20
30
  /** Whom to talk to; `null` for nobody yet — nothing is opened. */
21
31
  export type AgentTarget =
22
32
  /** An agent: the person's last conversation with it, by default — `resume: false` for a new one. */
@@ -34,6 +44,10 @@ export interface UseAgentOptions extends ConversationHandlers, ConnectOptions {
34
44
  client?: Client;
35
45
  /** A player and a recorder: replies are played, and `startRecording` records. Yours to dispose. */
36
46
  audio?: AgentAudio;
47
+ /** The device took the audio away — a call, Siri, another app: playback paused, a recording thrown away. */
48
+ onAudioInterrupted?: () => void;
49
+ /** The device gave the audio back: playback goes on where it stopped. */
50
+ onAudioResumed?: () => void;
37
51
  }
38
52
  /** What went wrong last; cleared by the next turn. */
39
53
  export type AgentError =
@@ -81,6 +95,12 @@ export interface AgentControls extends Snapshot {
81
95
  speaking: boolean;
82
96
  /** The recorder is open. */
83
97
  recording: boolean;
98
+ /** The device has the audio — a call, Siri, another app: playback is paused until it gives it back. */
99
+ interrupted: boolean;
100
+ /** Hands-free is on: the recorder is open, waiting for the person. */
101
+ listening: boolean;
102
+ /** Hands-free hears the person: an utterance is being recorded. */
103
+ hearing: boolean;
84
104
  /** The conversation itself, once it is made. */
85
105
  conversation: Conversation | null;
86
106
  send(text: string, options?: TurnOptions): Promise<TurnOutcome>;
@@ -91,6 +111,15 @@ export interface AgentControls extends Snapshot {
91
111
  startRecording(options?: RecordingOptions): Promise<boolean>;
92
112
  /** Close the recorder and end the recording: as a turn, as words, or discarded. */
93
113
  stopRecording(options?: StopRecordingOptions): Promise<TurnOutcome | TranscriptOutcome | null>;
114
+ /**
115
+ * Hands-free: the recorder left open, each utterance a turn, talking over
116
+ * the agent stopping it. Answers whether it listens; where the agent
117
+ * would hear itself it does not, and `error` says why — push-to-talk
118
+ * still works there.
119
+ */
120
+ listen(options?: ListenOptions): Promise<boolean>;
121
+ /** Hands-free off: the recorder closed, a half-heard utterance thrown away. */
122
+ stopListening(): Promise<void>;
94
123
  cancel(): void;
95
124
  mute(): void;
96
125
  reconnect(): void;
package/dist/use-agent.js CHANGED
@@ -12,11 +12,21 @@
12
12
  * *then* the recording, at the rate the recorder was granted, so a
13
13
  * refused microphone never opens a window on the socket. The adapter is
14
14
  * the app's: the hook never disposes it.
15
+ * - **An interruption throws the recording away**: when the adapter says
16
+ * the device took the audio (`onInterruption`) — a call, Siri, the app
17
+ * leaving the screen — a recording open is discarded as
18
+ * `stopRecording({ discard: true })` would, and `interrupted` is true
19
+ * until the adapter gives the audio back.
15
20
  * - **A late answer is dropped quietly**: a session opened after unmount
16
21
  * is never connected, and a press released while the microphone was
17
22
  * still opening gives it back.
23
+ * - **Hands-free is `listen()`** (`@lumavox/client`'s): the recorder stays
24
+ * open and each utterance is a turn; talking over the agent stops it on
25
+ * the edge's `barge_in`, and its voice stays stopped for the rest of
26
+ * that reply. Push-to-talk waits while it listens, and it ends with the
27
+ * conversation.
18
28
  */
19
- import { RECORDING_RATES, } from "@lumavox/client";
29
+ import { listen as listenOn, RECORDING_RATES, } from "@lumavox/client";
20
30
  import { useCallback, useEffect, useMemo, useRef, useState, useSyncExternalStore, } from "react";
21
31
  import { useClient, useLive } from "./provider.js";
22
32
  import { asPlatformError } from "./read.js";
@@ -74,6 +84,9 @@ export function useAgent(target, options = {}) {
74
84
  const [error, setError] = useState(null);
75
85
  const [speaking, setSpeaking] = useState(false);
76
86
  const [recording, setRecording] = useState(false);
87
+ const [interrupted, setInterrupted] = useState(false);
88
+ const [listening, setListening] = useState(false);
89
+ const [hearing, setHearing] = useState(false);
77
90
  const [failed, setFailed] = useState(false);
78
91
  const current = useRef(null);
79
92
  const waiting = useRef([]);
@@ -81,6 +94,8 @@ export function useAgent(target, options = {}) {
81
94
  /** The reply's frames still arriving after a stop are not played. */
82
95
  const silenced = useRef(false);
83
96
  const capture = useRef(null);
97
+ /** Hands-free, once `listen` opened it; `starting` while it opens. */
98
+ const handsFree = useRef(null);
84
99
  const sessionId = target && "sessionId" in target ? target.sessionId : null;
85
100
  const agentId = target && "agentId" in target ? target.agentId : null;
86
101
  const resume = target && "agentId" in target ? target.resume : undefined;
@@ -195,6 +210,12 @@ export function useAgent(target, options = {}) {
195
210
  return () => {
196
211
  alive = false;
197
212
  opening.current = false;
213
+ const free = handsFree.current;
214
+ handsFree.current = null;
215
+ if (free && free !== "starting")
216
+ void free.stop();
217
+ setListening(false);
218
+ setHearing(false);
198
219
  const open = capture.current;
199
220
  if (open) {
200
221
  open.cancelled = open.over = true;
@@ -248,8 +269,9 @@ export function useAgent(target, options = {}) {
248
269
  async startRecording(record = {}) {
249
270
  const recorder = latest.current.audio;
250
271
  const c = current.current;
251
- if (!recorder || !c || capture.current)
272
+ if (!recorder || !c || capture.current || handsFree.current) {
252
273
  return false;
274
+ }
253
275
  if (!record.transcribeOnly)
254
276
  hush();
255
277
  const mine = {
@@ -334,6 +356,48 @@ export function useAgent(target, options = {}) {
334
356
  hush();
335
357
  return c.endAudio();
336
358
  },
359
+ async listen(listenOptions = {}) {
360
+ const audio = latest.current.audio;
361
+ const c = current.current ?? (await connected());
362
+ if (!audio || !c || capture.current || handsFree.current)
363
+ return false;
364
+ handsFree.current = "starting";
365
+ const outcome = await listenOn(c, audio, {
366
+ ...listenOptions,
367
+ onState(state) {
368
+ setHearing(state === "hearing");
369
+ if (state === "stopped")
370
+ setListening(false);
371
+ listenOptions.onState?.(state);
372
+ },
373
+ onBargeIn(frame) {
374
+ hush();
375
+ listenOptions.onBargeIn?.(frame);
376
+ },
377
+ });
378
+ // Stopped, unmounted or redialled while it opened: given back.
379
+ if (handsFree.current !== "starting" || current.current !== c) {
380
+ if (outcome.ok)
381
+ await outcome.listening.stop();
382
+ return false;
383
+ }
384
+ if (!outcome.ok) {
385
+ handsFree.current = null;
386
+ setError({ kind: "audio", message: outcome.reason, error: null });
387
+ return false;
388
+ }
389
+ handsFree.current = outcome.listening;
390
+ setListening(true);
391
+ return true;
392
+ },
393
+ async stopListening() {
394
+ const free = handsFree.current;
395
+ handsFree.current = null;
396
+ setListening(false);
397
+ setHearing(false);
398
+ if (free && free !== "starting")
399
+ await free.stop();
400
+ },
337
401
  cancel() {
338
402
  hush();
339
403
  current.current?.cancel();
@@ -348,6 +412,21 @@ export function useAgent(target, options = {}) {
348
412
  current.current?.close();
349
413
  },
350
414
  }), [connected, hush]);
415
+ // The device took the audio, or gave it back: an open recording is
416
+ // thrown away, its speaker cut off mid-sentence.
417
+ const { stopRecording } = controls;
418
+ useEffect(() => {
419
+ setInterrupted(false);
420
+ return audio?.onInterruption?.((now) => {
421
+ setInterrupted(now);
422
+ if (now) {
423
+ void stopRecording({ discard: true });
424
+ latest.current.onAudioInterrupted?.();
425
+ }
426
+ else
427
+ latest.current.onAudioResumed?.();
428
+ });
429
+ }, [audio, stopRecording]);
351
430
  const readyAgent = snapshot.ready?.agent_id ?? null;
352
431
  const newSession = useCallback(() => {
353
432
  const agent = agentId ?? readyAgent;
@@ -365,6 +444,9 @@ export function useAgent(target, options = {}) {
365
444
  error,
366
445
  speaking,
367
446
  recording,
447
+ interrupted,
448
+ listening,
449
+ hearing,
368
450
  conversation,
369
451
  ...controls,
370
452
  newSession,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@lumavox/react",
3
- "version": "0.7.0",
3
+ "version": "0.10.0",
4
4
  "description": "React and React Native hooks for Lumavox agents: a conversation that keeps itself connected, sessions, memory and relations, over @lumavox/client.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -29,7 +29,7 @@
29
29
  "typecheck": "tsc --noEmit && tsc --noEmit -p test"
30
30
  },
31
31
  "dependencies": {
32
- "@lumavox/client": "0.7.0"
32
+ "@lumavox/client": "0.10.0"
33
33
  },
34
34
  "peerDependencies": {
35
35
  "react": ">=18"