@lumavox/react 0.7.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -3
- package/dist/use-agent.d.ts +30 -1
- package/dist/use-agent.js +84 -2
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -55,8 +55,10 @@ redial.
|
|
|
55
55
|
| `session` | the session it opened, with its last messages |
|
|
56
56
|
| `error` | `{ kind, message }`: the session, an `error` frame, the close, the microphone |
|
|
57
57
|
| `speaking` `recording` | the adapter is playing; the recorder is open |
|
|
58
|
+
| `interrupted` | the device has the audio — a call, Siri, another app |
|
|
58
59
|
| `send` `startAudio` `sendAudio` `endAudio` `cancel` `mute` `reconnect` `close` | the conversation's |
|
|
59
60
|
| `startRecording` `stopRecording({ transcribeOnly, discard })` | through the `audio` adapter |
|
|
61
|
+
| `listen(options)` `stopListening()` `listening` `hearing` | hands-free: each utterance a turn, talking over the agent stopping it (`@lumavox/client`'s `listen`) |
|
|
60
62
|
| `newSession()` | a new conversation with the same agent |
|
|
61
63
|
| `conversation` | the `Conversation` itself |
|
|
62
64
|
|
|
@@ -67,9 +69,13 @@ running the latest render's handler. **`audio`**, an `AgentAudio` — a player a
|
|
|
67
69
|
recorder — makes replies audible and `startRecording` work: replies play
|
|
68
70
|
as they arrive and stop on `cancel`, `mute` or a new message; recording
|
|
69
71
|
opens the microphone first and the recording at the rate it was granted.
|
|
70
|
-
Without an adapter or an `onAudio`, no speech is asked for.
|
|
71
|
-
|
|
72
|
-
React Native
|
|
72
|
+
Without an adapter or an `onAudio`, no speech is asked for. In a browser,
|
|
73
|
+
`createWebAudio()` from `@lumavox/client/web-audio` is one — call its
|
|
74
|
+
`prime()` in the click that starts the conversation; on React Native,
|
|
75
|
+
`@lumavox/react-native` is one install with these hooks and `createAudio()`.
|
|
76
|
+
An adapter that tells an interruption — a call, Siri, the app leaving the
|
|
77
|
+
screen — has the recording in progress thrown away, `interrupted` set, and
|
|
78
|
+
`onAudioInterrupted` and `onAudioResumed` called.
|
|
73
79
|
|
|
74
80
|
## Reading
|
|
75
81
|
|
package/dist/use-agent.d.ts
CHANGED
|
@@ -12,11 +12,21 @@
|
|
|
12
12
|
* *then* the recording, at the rate the recorder was granted, so a
|
|
13
13
|
* refused microphone never opens a window on the socket. The adapter is
|
|
14
14
|
* the app's: the hook never disposes it.
|
|
15
|
+
* - **An interruption throws the recording away**: when the adapter says
|
|
16
|
+
* the device took the audio (`onInterruption`) — a call, Siri, the app
|
|
17
|
+
* leaving the screen — a recording open is discarded as
|
|
18
|
+
* `stopRecording({ discard: true })` would, and `interrupted` is true
|
|
19
|
+
* until the adapter gives the audio back.
|
|
15
20
|
* - **A late answer is dropped quietly**: a session opened after unmount
|
|
16
21
|
* is never connected, and a press released while the microphone was
|
|
17
22
|
* still opening gives it back.
|
|
23
|
+
* - **Hands-free is `listen()`** (`@lumavox/client`'s): the recorder stays
|
|
24
|
+
* open and each utterance is a turn; talking over the agent stops it on
|
|
25
|
+
* the edge's `barge_in`, and its voice stays stopped for the rest of
|
|
26
|
+
* that reply. Push-to-talk waits while it listens, and it ends with the
|
|
27
|
+
* conversation.
|
|
18
28
|
*/
|
|
19
|
-
import { type AgentAudio, type Client, type CloseInfo, type ConnectOptions, type Conversation, type ConversationHandlers, type ErrorFrame, type PlatformError, type SessionDetail, type Snapshot, type StartAudioOptions, type TranscriptOutcome, type TurnOptions, type TurnOutcome } from "@lumavox/client";
|
|
29
|
+
import { type AgentAudio, type Client, type CloseInfo, type ConnectOptions, type Conversation, type ConversationHandlers, type ErrorFrame, type ListenOptions, type PlatformError, type SessionDetail, type Snapshot, type StartAudioOptions, type TranscriptOutcome, type TurnOptions, type TurnOutcome } from "@lumavox/client";
|
|
20
30
|
/** Whom to talk to; `null` for nobody yet — nothing is opened. */
|
|
21
31
|
export type AgentTarget =
|
|
22
32
|
/** An agent: the person's last conversation with it, by default — `resume: false` for a new one. */
|
|
@@ -34,6 +44,10 @@ export interface UseAgentOptions extends ConversationHandlers, ConnectOptions {
|
|
|
34
44
|
client?: Client;
|
|
35
45
|
/** A player and a recorder: replies are played, and `startRecording` records. Yours to dispose. */
|
|
36
46
|
audio?: AgentAudio;
|
|
47
|
+
/** The device took the audio away — a call, Siri, another app: playback paused, a recording thrown away. */
|
|
48
|
+
onAudioInterrupted?: () => void;
|
|
49
|
+
/** The device gave the audio back: playback goes on where it stopped. */
|
|
50
|
+
onAudioResumed?: () => void;
|
|
37
51
|
}
|
|
38
52
|
/** What went wrong last; cleared by the next turn. */
|
|
39
53
|
export type AgentError =
|
|
@@ -81,6 +95,12 @@ export interface AgentControls extends Snapshot {
|
|
|
81
95
|
speaking: boolean;
|
|
82
96
|
/** The recorder is open. */
|
|
83
97
|
recording: boolean;
|
|
98
|
+
/** The device has the audio — a call, Siri, another app: playback is paused until it gives it back. */
|
|
99
|
+
interrupted: boolean;
|
|
100
|
+
/** Hands-free is on: the recorder is open, waiting for the person. */
|
|
101
|
+
listening: boolean;
|
|
102
|
+
/** Hands-free hears the person: an utterance is being recorded. */
|
|
103
|
+
hearing: boolean;
|
|
84
104
|
/** The conversation itself, once it is made. */
|
|
85
105
|
conversation: Conversation | null;
|
|
86
106
|
send(text: string, options?: TurnOptions): Promise<TurnOutcome>;
|
|
@@ -91,6 +111,15 @@ export interface AgentControls extends Snapshot {
|
|
|
91
111
|
startRecording(options?: RecordingOptions): Promise<boolean>;
|
|
92
112
|
/** Close the recorder and end the recording: as a turn, as words, or discarded. */
|
|
93
113
|
stopRecording(options?: StopRecordingOptions): Promise<TurnOutcome | TranscriptOutcome | null>;
|
|
114
|
+
/**
|
|
115
|
+
* Hands-free: the recorder left open, each utterance a turn, talking over
|
|
116
|
+
* the agent stopping it. Answers whether it listens; where the agent
|
|
117
|
+
* would hear itself it does not, and `error` says why — push-to-talk
|
|
118
|
+
* still works there.
|
|
119
|
+
*/
|
|
120
|
+
listen(options?: ListenOptions): Promise<boolean>;
|
|
121
|
+
/** Hands-free off: the recorder closed, a half-heard utterance thrown away. */
|
|
122
|
+
stopListening(): Promise<void>;
|
|
94
123
|
cancel(): void;
|
|
95
124
|
mute(): void;
|
|
96
125
|
reconnect(): void;
|
package/dist/use-agent.js
CHANGED
|
@@ -12,11 +12,21 @@
|
|
|
12
12
|
* *then* the recording, at the rate the recorder was granted, so a
|
|
13
13
|
* refused microphone never opens a window on the socket. The adapter is
|
|
14
14
|
* the app's: the hook never disposes it.
|
|
15
|
+
* - **An interruption throws the recording away**: when the adapter says
|
|
16
|
+
* the device took the audio (`onInterruption`) — a call, Siri, the app
|
|
17
|
+
* leaving the screen — a recording open is discarded as
|
|
18
|
+
* `stopRecording({ discard: true })` would, and `interrupted` is true
|
|
19
|
+
* until the adapter gives the audio back.
|
|
15
20
|
* - **A late answer is dropped quietly**: a session opened after unmount
|
|
16
21
|
* is never connected, and a press released while the microphone was
|
|
17
22
|
* still opening gives it back.
|
|
23
|
+
* - **Hands-free is `listen()`** (`@lumavox/client`'s): the recorder stays
|
|
24
|
+
* open and each utterance is a turn; talking over the agent stops it on
|
|
25
|
+
* the edge's `barge_in`, and its voice stays stopped for the rest of
|
|
26
|
+
* that reply. Push-to-talk waits while it listens, and it ends with the
|
|
27
|
+
* conversation.
|
|
18
28
|
*/
|
|
19
|
-
import { RECORDING_RATES, } from "@lumavox/client";
|
|
29
|
+
import { listen as listenOn, RECORDING_RATES, } from "@lumavox/client";
|
|
20
30
|
import { useCallback, useEffect, useMemo, useRef, useState, useSyncExternalStore, } from "react";
|
|
21
31
|
import { useClient, useLive } from "./provider.js";
|
|
22
32
|
import { asPlatformError } from "./read.js";
|
|
@@ -74,6 +84,9 @@ export function useAgent(target, options = {}) {
|
|
|
74
84
|
const [error, setError] = useState(null);
|
|
75
85
|
const [speaking, setSpeaking] = useState(false);
|
|
76
86
|
const [recording, setRecording] = useState(false);
|
|
87
|
+
const [interrupted, setInterrupted] = useState(false);
|
|
88
|
+
const [listening, setListening] = useState(false);
|
|
89
|
+
const [hearing, setHearing] = useState(false);
|
|
77
90
|
const [failed, setFailed] = useState(false);
|
|
78
91
|
const current = useRef(null);
|
|
79
92
|
const waiting = useRef([]);
|
|
@@ -81,6 +94,8 @@ export function useAgent(target, options = {}) {
|
|
|
81
94
|
/** The reply's frames still arriving after a stop are not played. */
|
|
82
95
|
const silenced = useRef(false);
|
|
83
96
|
const capture = useRef(null);
|
|
97
|
+
/** Hands-free, once `listen` opened it; `starting` while it opens. */
|
|
98
|
+
const handsFree = useRef(null);
|
|
84
99
|
const sessionId = target && "sessionId" in target ? target.sessionId : null;
|
|
85
100
|
const agentId = target && "agentId" in target ? target.agentId : null;
|
|
86
101
|
const resume = target && "agentId" in target ? target.resume : undefined;
|
|
@@ -195,6 +210,12 @@ export function useAgent(target, options = {}) {
|
|
|
195
210
|
return () => {
|
|
196
211
|
alive = false;
|
|
197
212
|
opening.current = false;
|
|
213
|
+
const free = handsFree.current;
|
|
214
|
+
handsFree.current = null;
|
|
215
|
+
if (free && free !== "starting")
|
|
216
|
+
void free.stop();
|
|
217
|
+
setListening(false);
|
|
218
|
+
setHearing(false);
|
|
198
219
|
const open = capture.current;
|
|
199
220
|
if (open) {
|
|
200
221
|
open.cancelled = open.over = true;
|
|
@@ -248,8 +269,9 @@ export function useAgent(target, options = {}) {
|
|
|
248
269
|
async startRecording(record = {}) {
|
|
249
270
|
const recorder = latest.current.audio;
|
|
250
271
|
const c = current.current;
|
|
251
|
-
if (!recorder || !c || capture.current)
|
|
272
|
+
if (!recorder || !c || capture.current || handsFree.current) {
|
|
252
273
|
return false;
|
|
274
|
+
}
|
|
253
275
|
if (!record.transcribeOnly)
|
|
254
276
|
hush();
|
|
255
277
|
const mine = {
|
|
@@ -334,6 +356,48 @@ export function useAgent(target, options = {}) {
|
|
|
334
356
|
hush();
|
|
335
357
|
return c.endAudio();
|
|
336
358
|
},
|
|
359
|
+
async listen(listenOptions = {}) {
|
|
360
|
+
const audio = latest.current.audio;
|
|
361
|
+
const c = current.current ?? (await connected());
|
|
362
|
+
if (!audio || !c || capture.current || handsFree.current)
|
|
363
|
+
return false;
|
|
364
|
+
handsFree.current = "starting";
|
|
365
|
+
const outcome = await listenOn(c, audio, {
|
|
366
|
+
...listenOptions,
|
|
367
|
+
onState(state) {
|
|
368
|
+
setHearing(state === "hearing");
|
|
369
|
+
if (state === "stopped")
|
|
370
|
+
setListening(false);
|
|
371
|
+
listenOptions.onState?.(state);
|
|
372
|
+
},
|
|
373
|
+
onBargeIn(frame) {
|
|
374
|
+
hush();
|
|
375
|
+
listenOptions.onBargeIn?.(frame);
|
|
376
|
+
},
|
|
377
|
+
});
|
|
378
|
+
// Stopped, unmounted or redialled while it opened: given back.
|
|
379
|
+
if (handsFree.current !== "starting" || current.current !== c) {
|
|
380
|
+
if (outcome.ok)
|
|
381
|
+
await outcome.listening.stop();
|
|
382
|
+
return false;
|
|
383
|
+
}
|
|
384
|
+
if (!outcome.ok) {
|
|
385
|
+
handsFree.current = null;
|
|
386
|
+
setError({ kind: "audio", message: outcome.reason, error: null });
|
|
387
|
+
return false;
|
|
388
|
+
}
|
|
389
|
+
handsFree.current = outcome.listening;
|
|
390
|
+
setListening(true);
|
|
391
|
+
return true;
|
|
392
|
+
},
|
|
393
|
+
async stopListening() {
|
|
394
|
+
const free = handsFree.current;
|
|
395
|
+
handsFree.current = null;
|
|
396
|
+
setListening(false);
|
|
397
|
+
setHearing(false);
|
|
398
|
+
if (free && free !== "starting")
|
|
399
|
+
await free.stop();
|
|
400
|
+
},
|
|
337
401
|
cancel() {
|
|
338
402
|
hush();
|
|
339
403
|
current.current?.cancel();
|
|
@@ -348,6 +412,21 @@ export function useAgent(target, options = {}) {
|
|
|
348
412
|
current.current?.close();
|
|
349
413
|
},
|
|
350
414
|
}), [connected, hush]);
|
|
415
|
+
// The device took the audio, or gave it back: an open recording is
|
|
416
|
+
// thrown away, its speaker cut off mid-sentence.
|
|
417
|
+
const { stopRecording } = controls;
|
|
418
|
+
useEffect(() => {
|
|
419
|
+
setInterrupted(false);
|
|
420
|
+
return audio?.onInterruption?.((now) => {
|
|
421
|
+
setInterrupted(now);
|
|
422
|
+
if (now) {
|
|
423
|
+
void stopRecording({ discard: true });
|
|
424
|
+
latest.current.onAudioInterrupted?.();
|
|
425
|
+
}
|
|
426
|
+
else
|
|
427
|
+
latest.current.onAudioResumed?.();
|
|
428
|
+
});
|
|
429
|
+
}, [audio, stopRecording]);
|
|
351
430
|
const readyAgent = snapshot.ready?.agent_id ?? null;
|
|
352
431
|
const newSession = useCallback(() => {
|
|
353
432
|
const agent = agentId ?? readyAgent;
|
|
@@ -365,6 +444,9 @@ export function useAgent(target, options = {}) {
|
|
|
365
444
|
error,
|
|
366
445
|
speaking,
|
|
367
446
|
recording,
|
|
447
|
+
interrupted,
|
|
448
|
+
listening,
|
|
449
|
+
hearing,
|
|
368
450
|
conversation,
|
|
369
451
|
...controls,
|
|
370
452
|
newSession,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lumavox/react",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.0",
|
|
4
4
|
"description": "React and React Native hooks for Lumavox agents: a conversation that keeps itself connected, sessions, memory and relations, over @lumavox/client.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
"typecheck": "tsc --noEmit && tsc --noEmit -p test"
|
|
30
30
|
},
|
|
31
31
|
"dependencies": {
|
|
32
|
-
"@lumavox/client": "0.
|
|
32
|
+
"@lumavox/client": "0.10.0"
|
|
33
33
|
},
|
|
34
34
|
"peerDependencies": {
|
|
35
35
|
"react": ">=18"
|