@alexkroman1/aai-ui 16.1.0 → 18.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/_page-shell.d.ts +11 -0
- package/dist/{chat-view-DsylXJsw.js → chat-view-P12pHKQU.js} +1 -1
- package/dist/{client-config-DJQHnYjm.js → client-config-CFhkMkmv.js} +15 -13
- package/dist/client-config.d.ts +14 -13
- package/dist/components/chat-view.js +1 -1
- package/dist/components/controls.js +1 -1
- package/dist/components/message-list.js +1 -1
- package/dist/components/tool-call-block.js +1 -1
- package/dist/context.d.ts +27 -5
- package/dist/context.js +3 -3
- package/dist/default-client/assets/index-5XG8PqGw.css +2 -0
- package/dist/default-client/assets/index-tc-nAWq3.js +121 -0
- package/dist/default-client/index.html +2 -2
- package/dist/default-client.d.ts +21 -0
- package/dist/define-client.js +4 -4
- package/dist/hooks.d.ts +19 -9
- package/dist/hooks.js +1 -1
- package/dist/index.d.ts +17 -13
- package/dist/index.js +734 -433
- package/dist/internal.js +3 -3
- package/dist/{message-list-BLyPRS1Q.js → message-list-CJX560b5.js} +1 -1
- package/dist/page.d.ts +47 -21
- package/dist/{session-core-gwePM95B.js → session-core-TOdm_Con.js} +424 -174
- package/dist/session-core-audio-effects.d.ts +26 -0
- package/dist/session-core-audio-setup.d.ts +38 -31
- package/dist/session-core-audio-state.d.ts +152 -0
- package/dist/session-core-messages.d.ts +9 -10
- package/dist/session-core-types.d.ts +60 -21
- package/dist/session-core-user-turn.d.ts +30 -0
- package/dist/session-core.js +1 -1
- package/dist/use-push-to-talk.d.ts +105 -0
- package/dist/{use-workflow-run-CNahvlgL.js → use-workflow-run-ZqJWJn2-.js} +24 -11
- package/dist/workflow-client.d.ts +26 -12
- package/package.json +2 -2
- package/dist/default-client/assets/index-CMh4E-Ci.css +0 -2
- package/dist/default-client/assets/index-eOmPq2sS.js +0 -284
- package/dist/{tool-call-block-C2t_5fpp.js → tool-call-block-4gXoL_eP.js} +1 -1
- package/dist/{url-chips-D62RTz7v.js → url-chips-0n35fPJY.js} +1 -1
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The session core's side of the audio path: every socket frame and snapshot
|
|
3
|
+
* write the statechart decides on but cannot make itself.
|
|
4
|
+
*
|
|
5
|
+
* The split is the one `s2s-lifecycle.ts` draws in the runtime. WHEN a
|
|
6
|
+
* microphone opens, whether an opening is still wanted, and where audio goes
|
|
7
|
+
* before there is anywhere to play it are `session-core-audio-state.ts`'s;
|
|
8
|
+
* this module is HOW — and it is the only half that touches `conn`, the
|
|
9
|
+
* snapshot, or the wire, which is what lets the machine be specced without a
|
|
10
|
+
* browser.
|
|
11
|
+
*/
|
|
12
|
+
import type { SessionCommand } from "@alexkroman1/aai/protocol";
|
|
13
|
+
import type { AudioPathEffects } from "./session-core-audio-state.ts";
|
|
14
|
+
import type { SessionStateMachine } from "./session-core-state.ts";
|
|
15
|
+
import { type ConnState, type SessionSnapshot } from "./session-core-types.ts";
|
|
16
|
+
/** What the audio effects need from the owning session core. */
|
|
17
|
+
export type AudioEffectsDeps = {
|
|
18
|
+
conn: ConnState;
|
|
19
|
+
updateState: (partial: Partial<SessionSnapshot>) => void;
|
|
20
|
+
/** The session's state and error, as one fact — see `session-core-state.ts`. */
|
|
21
|
+
agentState: SessionStateMachine;
|
|
22
|
+
sendJson: (msg: SessionCommand) => void;
|
|
23
|
+
sendAudio: (bytes: ArrayBuffer) => void;
|
|
24
|
+
};
|
|
25
|
+
/** Build the effects for one session's audio path. */
|
|
26
|
+
export declare function createAudioEffects(deps: AudioEffectsDeps): AudioPathEffects;
|
|
@@ -1,20 +1,15 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* How the audio path is OPENED: the module load, the mic grant, the worklet
|
|
3
|
+
* registration, and the {@link VoiceIO} those three produce.
|
|
4
|
+
*
|
|
5
|
+
* WHEN one is opened, whether it is still wanted by the time it settles, and
|
|
6
|
+
* what happens to audio that arrives meanwhile all live in
|
|
7
|
+
* `session-core-audio-state.ts` — the same split `s2s-lifecycle.ts` and
|
|
8
|
+
* `pipeline-speech-edges.ts` draw in the runtime, and for the same reason:
|
|
9
|
+
* nothing here reads or writes session state, so it is callable from an
|
|
10
|
+
* invoked actor and testable without one.
|
|
11
|
+
*/
|
|
2
12
|
import type { VoiceIO } from "./audio.ts";
|
|
3
|
-
import type { SessionStateMachine } from "./session-core-state.ts";
|
|
4
|
-
import { type ConnState, type SessionSnapshot } from "./session-core-types.ts";
|
|
5
|
-
/** Dependencies `initAudioCapture` needs from the owning session core. */
|
|
6
|
-
export type AudioSetupDeps = {
|
|
7
|
-
sendJson: (msg: SessionCommand) => void;
|
|
8
|
-
sendAudio: (bytes: ArrayBuffer) => void;
|
|
9
|
-
updateState: (partial: Partial<SessionSnapshot>) => void;
|
|
10
|
-
/** The session's state and error, as one fact — see `session-core-state.ts`. */
|
|
11
|
-
agentState: SessionStateMachine;
|
|
12
|
-
/** Turn-boundary-guarded drain from the message handlers — replays a
|
|
13
|
-
* buffered `audio_done` without stomping a barge-in's state. */
|
|
14
|
-
settleWhenAudioDrained: (io: VoiceIO) => void;
|
|
15
|
-
/** Release the mic/VoiceIO (used when the audio path dies non-fatally). */
|
|
16
|
-
cleanupAudio: () => void;
|
|
17
|
-
};
|
|
18
13
|
type AudioModules = [typeof import("./audio.ts"), string, string];
|
|
19
14
|
/**
|
|
20
15
|
* Load the audio module and worklet sources, memoized so a prefetch at
|
|
@@ -23,23 +18,35 @@ type AudioModules = [typeof import("./audio.ts"), string, string];
|
|
|
23
18
|
* load clears the memo so the next attempt retries the import.
|
|
24
19
|
*/
|
|
25
20
|
export declare function loadAudioModules(): Promise<AudioModules>;
|
|
21
|
+
/** The audio parameters the server's `config` frame carries. */
|
|
22
|
+
export type AudioPathConfig = {
|
|
23
|
+
sampleRate: number;
|
|
24
|
+
ttsSampleRate: number;
|
|
25
|
+
};
|
|
26
26
|
/**
|
|
27
|
-
*
|
|
27
|
+
* What a live audio path reports back.
|
|
28
28
|
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
29
|
+
* Both of the last two used to be guarded at the callback by
|
|
30
|
+
* `conn.generation.isCurrent(gen)`; they are plain reports now, and whether
|
|
31
|
+
* one still matters is the state machine's answer — see
|
|
32
|
+
* `session-core-audio-state.ts`.
|
|
33
|
+
*/
|
|
34
|
+
export type AudioPathCallbacks = {
|
|
35
|
+
/** Buffered PCM16 from the microphone, for the socket. */
|
|
36
|
+
onMicData(pcm16: ArrayBuffer): void;
|
|
37
|
+
/** Unplayed agent audio in the buffer, in ms — the host's closed playback loop. */
|
|
38
|
+
onProgress(bufferedMs: number): void;
|
|
39
|
+
/** A worklet processor died after setup: the audio path is gone. */
|
|
40
|
+
onFailure(message: string): void;
|
|
41
|
+
};
|
|
42
|
+
/**
|
|
43
|
+
* Open one audio path: load the worklets, ask for the microphone, and build
|
|
44
|
+
* the {@link VoiceIO} wired to `callbacks`.
|
|
37
45
|
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
46
|
+
* Rejects when any of the three refuses — a denied mic prompt being the
|
|
47
|
+
* ordinary case. The caller owns closing what this resolves: a path that comes
|
|
48
|
+
* up for a connection nobody wants any more is a live microphone, and no
|
|
49
|
+
* amount of cancellation upstream releases the device.
|
|
40
50
|
*/
|
|
41
|
-
export declare function
|
|
42
|
-
sampleRate: number;
|
|
43
|
-
ttsSampleRate: number;
|
|
44
|
-
}, deps: AudioSetupDeps): Promise<void>;
|
|
51
|
+
export declare function openAudioPath(config: AudioPathConfig, callbacks: AudioPathCallbacks): Promise<VoiceIO>;
|
|
45
52
|
export {};
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The browser session's AUDIO PATH — the mic grant, the worklets, and the
|
|
3
|
+
* {@link VoiceIO} they produce — as a statechart.
|
|
4
|
+
*
|
|
5
|
+
* It was a latch, an epoch counter and three buffers spread across
|
|
6
|
+
* `ConnState`, and every one of them was a question the position already
|
|
7
|
+
* answers:
|
|
8
|
+
*
|
|
9
|
+
* - **`audioSetupInFlight` was a dedup, and dedup is a state.**
|
|
10
|
+
* `initAudioCapture` opened with `if (conn.audioSetupInFlight) return` and
|
|
11
|
+
* closed with "only the init that still owns the flag may clear it", which is
|
|
12
|
+
* an ownership problem a boolean cannot express and a position does not have.
|
|
13
|
+
* Here `starting` simply does not handle `START`, and there is no flag to
|
|
14
|
+
* hand back.
|
|
15
|
+
* - **`conn.generation` existed for this module and nothing else.** Four
|
|
16
|
+
* `bump()` sites, four `isCurrent(gen)` reads, and its own field on
|
|
17
|
+
* `ConnState`. All four bumps sat immediately after a `cleanupAudio()` that
|
|
18
|
+
* had *just* cleared the in-flight flag — the reconnect close handler said so
|
|
19
|
+
* outright ("cleanupAudio just cleared the in-flight flag, so a survivor
|
|
20
|
+
* would otherwise pass the same-generation guard and double-run"). Two
|
|
21
|
+
* mechanisms for one question, the second one layered on because the first
|
|
22
|
+
* could not stop the work already running. A stopped `invoke` can, so the
|
|
23
|
+
* counter is gone.
|
|
24
|
+
* - **A counter cannot tell two bring-ups in ONE generation apart, and they
|
|
25
|
+
* happen.** A repeated `config` frame on a live connection re-opened the path
|
|
26
|
+
* without bumping anything, so the outgoing `VoiceIO`'s `onError` still
|
|
27
|
+
* passed `isCurrent(gen)` and could tear down the replacement that had just
|
|
28
|
+
* taken its slot. `PROGRESS` and `IO_FAILED` carry the instance that fired
|
|
29
|
+
* them now, and identity is exact where a counter was approximate.
|
|
30
|
+
* - **`preInitAudio` / `preInitDone` are only meaningful while the path is
|
|
31
|
+
* down.** As connection fields they were readable — and writable — from
|
|
32
|
+
* anywhere; as context of `down`/`starting` they are cleared by entering
|
|
33
|
+
* `down` rather than by three call sites remembering to.
|
|
34
|
+
*
|
|
35
|
+
* ## What cancellation does NOT do, and the one guard that had to survive
|
|
36
|
+
*
|
|
37
|
+
* Stopping the `bringUp` actor hides its resolution; it does not close a
|
|
38
|
+
* microphone the browser has already granted. So `bringUp` checks its own
|
|
39
|
+
* `signal` after `openAudioPath` settles and releases what it built. That is
|
|
40
|
+
* the single remaining "is this still wanted?" test, and it lives in the
|
|
41
|
+
* producer — which owns the value nobody adopted — rather than at the four
|
|
42
|
+
* consumer sites that used to re-ask.
|
|
43
|
+
*
|
|
44
|
+
* ## Effects stay in the session core
|
|
45
|
+
*
|
|
46
|
+
* The machine owns WHEN. Every HOW — the socket frame, the snapshot write, the
|
|
47
|
+
* turn-boundary drain — arrives as an injected {@link AudioPathEffects} call,
|
|
48
|
+
* the same shape `s2s-lifecycle.ts` uses. So this module touches no socket and
|
|
49
|
+
* no snapshot.
|
|
50
|
+
*/
|
|
51
|
+
import type { VoiceIO } from "./audio.ts";
|
|
52
|
+
import type { AudioPathCallbacks, AudioPathConfig } from "./session-core-audio-setup.ts";
|
|
53
|
+
/** Where the audio path is. */
|
|
54
|
+
export type AudioPathPhase = "down" | "starting" | "up";
|
|
55
|
+
/**
|
|
56
|
+
* The session core's side of the machine: everything the audio path decides to
|
|
57
|
+
* do but does not know how to.
|
|
58
|
+
*/
|
|
59
|
+
export type AudioPathEffects = {
|
|
60
|
+
/**
|
|
61
|
+
* Open one audio path — `openAudioPath` in `session-core-audio-setup.ts`.
|
|
62
|
+
* Injected rather than imported so this module needs no browser: the specs
|
|
63
|
+
* hand it a promise they resolve by hand, which is the only way to hold a
|
|
64
|
+
* bring-up open across the events that abandon it.
|
|
65
|
+
*/
|
|
66
|
+
open(config: AudioPathConfig, callbacks: AudioPathCallbacks): Promise<VoiceIO>;
|
|
67
|
+
/** Relay captured microphone audio to the server. */
|
|
68
|
+
sendMicAudio(pcm16: ArrayBuffer): void;
|
|
69
|
+
/** Tell the server how much agent audio is still unplayed. */
|
|
70
|
+
reportProgress(bufferedMs: number): void;
|
|
71
|
+
/** Tell the server the microphone is live. */
|
|
72
|
+
announceReady(): void;
|
|
73
|
+
/** The microphone is live: the snapshot's `recording`. */
|
|
74
|
+
micLive(): void;
|
|
75
|
+
/** Back to the mic, with no audio left to wait for. */
|
|
76
|
+
listen(): void;
|
|
77
|
+
/**
|
|
78
|
+
* Wait for `io` to drain, then listen — guarded by the turn boundary, so a
|
|
79
|
+
* completion landing after a barge-in or a teardown is discarded.
|
|
80
|
+
*/
|
|
81
|
+
settleWhenDrained(io: VoiceIO): void;
|
|
82
|
+
/**
|
|
83
|
+
* Report a non-fatal audio failure and stop the mic. Non-fatal because the
|
|
84
|
+
* socket may well still be fine, so a later server frame is allowed to
|
|
85
|
+
* recover the banner — see `session-core-state.ts`.
|
|
86
|
+
*/
|
|
87
|
+
reportFailure(message: string): void;
|
|
88
|
+
/**
|
|
89
|
+
* The audio path is gone: end whatever turn it was playing.
|
|
90
|
+
*
|
|
91
|
+
* Paired with the release below rather than folded into it, because the
|
|
92
|
+
* bump has to happen on every path into `down` — including a bring-up that
|
|
93
|
+
* failed before there was anything to close.
|
|
94
|
+
*/
|
|
95
|
+
endTurn(): void;
|
|
96
|
+
/** Close `io` and let go of the microphone. Best-effort. */
|
|
97
|
+
release(io: VoiceIO): void;
|
|
98
|
+
};
|
|
99
|
+
/** Everything that happens to the audio path. */
|
|
100
|
+
export type AudioPathEvent =
|
|
101
|
+
/** The server's `config` frame: bring the path up for these rates. */
|
|
102
|
+
{
|
|
103
|
+
type: "START";
|
|
104
|
+
config: AudioPathConfig;
|
|
105
|
+
}
|
|
106
|
+
/** A PCM16 chunk arrived for playback. */
|
|
107
|
+
| {
|
|
108
|
+
type: "CHUNK";
|
|
109
|
+
chunk: ArrayBuffer;
|
|
110
|
+
}
|
|
111
|
+
/** The server finished sending this turn's audio. */
|
|
112
|
+
| {
|
|
113
|
+
type: "DONE";
|
|
114
|
+
}
|
|
115
|
+
/** A live path reported its unplayed depth. */
|
|
116
|
+
| {
|
|
117
|
+
type: "PROGRESS";
|
|
118
|
+
io: VoiceIO;
|
|
119
|
+
bufferedMs: number;
|
|
120
|
+
}
|
|
121
|
+
/** A live path's worklet died. */
|
|
122
|
+
| {
|
|
123
|
+
type: "IO_FAILED";
|
|
124
|
+
io: VoiceIO;
|
|
125
|
+
message: string;
|
|
126
|
+
}
|
|
127
|
+
/** Release the audio path: hang-up, reconnect, fatal frame, session end. */
|
|
128
|
+
| {
|
|
129
|
+
type: "TEARDOWN";
|
|
130
|
+
};
|
|
131
|
+
/** The session core's handle on its audio path. */
|
|
132
|
+
export type AudioPath = {
|
|
133
|
+
/** Where the path is. */
|
|
134
|
+
phase(): AudioPathPhase;
|
|
135
|
+
/** The server named the audio rates: bring the path up. */
|
|
136
|
+
start(config: AudioPathConfig): void;
|
|
137
|
+
/** Play a chunk, or hold it until there is somewhere to play it. */
|
|
138
|
+
enqueue(chunk: ArrayBuffer): void;
|
|
139
|
+
/** The server finished this turn's audio. */
|
|
140
|
+
done(): void;
|
|
141
|
+
/**
|
|
142
|
+
* Barge-in: discard what is queued for playback.
|
|
143
|
+
*
|
|
144
|
+
* A read rather than an event — it moves the path nowhere, and a turn
|
|
145
|
+
* boundary is the caller's to declare (see `bargeIn`).
|
|
146
|
+
*/
|
|
147
|
+
flush(): void;
|
|
148
|
+
/** Release the microphone and forget anything buffered. */
|
|
149
|
+
teardown(): void;
|
|
150
|
+
};
|
|
151
|
+
/** Create the audio path for one browser session. */
|
|
152
|
+
export declare function createAudioPath(effects: AudioPathEffects): AudioPath;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { AudioPath } from "./session-core-audio-state.ts";
|
|
1
2
|
import type { SessionStateMachine } from "./session-core-state.ts";
|
|
2
3
|
import { type ConnState, type SessionSnapshot } from "./session-core-types.ts";
|
|
3
4
|
/**
|
|
@@ -28,8 +29,14 @@ type MessageHandlerDeps = {
|
|
|
28
29
|
conn: ConnState;
|
|
29
30
|
/** The session's state and error, as one fact — see `session-core-state.ts`. */
|
|
30
31
|
agentState: SessionStateMachine;
|
|
31
|
-
/**
|
|
32
|
-
|
|
32
|
+
/**
|
|
33
|
+
* The microphone and the playback queue — see
|
|
34
|
+
* `session-core-audio-state.ts`. Frames say what happened (`enqueue`,
|
|
35
|
+
* `done`, `teardown`); whether the path is up, and what to do with audio
|
|
36
|
+
* that arrives before it is, are the machine's answers rather than this
|
|
37
|
+
* module's.
|
|
38
|
+
*/
|
|
39
|
+
audio: AudioPath;
|
|
33
40
|
};
|
|
34
41
|
type MessageHandlers = {
|
|
35
42
|
/**
|
|
@@ -42,14 +49,6 @@ type MessageHandlers = {
|
|
|
42
49
|
* otherwise `undefined`.
|
|
43
50
|
*/
|
|
44
51
|
handleMessage(data: unknown): SessionConfigMessage | undefined;
|
|
45
|
-
/**
|
|
46
|
-
* Wait for `io`'s playback queue to drain, then transition to `"listening"`
|
|
47
|
-
* — guarded by the same turn-boundary generation the live `audio.completed`
|
|
48
|
-
* path uses. `initAudioCapture` routes the pre-init greeting replay
|
|
49
|
-
* through this so a barge-in mid-greeting can't be stomped by the
|
|
50
|
-
* replayed completion resolving late.
|
|
51
|
-
*/
|
|
52
|
-
settleWhenAudioDrained(io: NonNullable<ConnState["voiceIO"]>): void;
|
|
53
52
|
};
|
|
54
53
|
/**
|
|
55
54
|
* Create the server→client message handlers for one session core.
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Split out of `session-core.ts` to keep that module focused on behaviour.
|
|
5
5
|
*/
|
|
6
6
|
import type { Epoch } from "@alexkroman1/aai/internal";
|
|
7
|
-
import type {
|
|
7
|
+
import type { AudioPath } from "./session-core-audio-state.ts";
|
|
8
8
|
import type { AgentState, ChatMessage, SessionError, ToolCallInfo, WebSocketConstructor } from "./types.ts";
|
|
9
9
|
/**
|
|
10
10
|
* A custom event emitted by the agent via `ctx.send(event, data)` — the
|
|
@@ -140,6 +140,45 @@ export type SessionSnapshot = {
|
|
|
140
140
|
*/
|
|
141
141
|
readonly running: boolean;
|
|
142
142
|
};
|
|
143
|
+
/**
|
|
144
|
+
* The seal on a {@link BrowserSession}.
|
|
145
|
+
*
|
|
146
|
+
* TYPE-ONLY — there is no value at run time, so an object literal cannot carry
|
|
147
|
+
* the key and only `createBrowserSession` mints a session. That is what lets
|
|
148
|
+
* the handle grow a member without breaking anybody: push-to-talk added three
|
|
149
|
+
* REQUIRED methods and broke every hand-written double, which is the failure a
|
|
150
|
+
* handle a caller RECEIVES should not be able to cause.
|
|
151
|
+
*
|
|
152
|
+
* @public
|
|
153
|
+
*/
|
|
154
|
+
export declare const browserSessionBrand: unique symbol;
|
|
155
|
+
/**
|
|
156
|
+
* Push-to-talk's three edges on a {@link BrowserSession} — `session.userTurn`.
|
|
157
|
+
*
|
|
158
|
+
* Only an agent declaring `turnDetection: "manual"` honours them; any other
|
|
159
|
+
* agent logs once and ignores them, because its transcriber already ends each
|
|
160
|
+
* turn on a pause. `usePushToTalk` is the hook a button is built on, and the
|
|
161
|
+
* way a `client.tsx` reaches these: a sub-handle rather than three methods on
|
|
162
|
+
* the session, so a feature one agent in many declares is not three members
|
|
163
|
+
* every session carries in its autocomplete.
|
|
164
|
+
*
|
|
165
|
+
* @public
|
|
166
|
+
*/
|
|
167
|
+
export type UserTurnControls = {
|
|
168
|
+
/**
|
|
169
|
+
* OPEN a turn — the button went down. Stops the agent if it is speaking
|
|
170
|
+
* (discarding its queued audio here at once, rather than a round trip later)
|
|
171
|
+
* and lets the microphone through to the transcriber.
|
|
172
|
+
*/
|
|
173
|
+
start(): void;
|
|
174
|
+
/** CLOSE the turn and have the agent answer everything said since `start()` — the button came up. */
|
|
175
|
+
commit(): void;
|
|
176
|
+
/**
|
|
177
|
+
* Close the turn and THROW AWAY what was said in it — a cancelled press (the
|
|
178
|
+
* pointer left the button, Escape). The agent answers nothing.
|
|
179
|
+
*/
|
|
180
|
+
clear(): void;
|
|
181
|
+
};
|
|
143
182
|
/**
|
|
144
183
|
* A framework-agnostic voice session that manages WebSocket communication,
|
|
145
184
|
* audio capture/playback, and agent state transitions.
|
|
@@ -147,9 +186,15 @@ export type SessionSnapshot = {
|
|
|
147
186
|
* Uses a subscribe/getSnapshot pattern (compatible with React's
|
|
148
187
|
* `useSyncExternalStore`). Implements `Disposable` for resource cleanup.
|
|
149
188
|
*
|
|
189
|
+
* @sealed Only `createBrowserSession` produces one — see
|
|
190
|
+
* {@link browserSessionBrand}. A test double for a component is a real
|
|
191
|
+
* session with no socket, not an object literal.
|
|
192
|
+
*
|
|
150
193
|
* @public
|
|
151
194
|
*/
|
|
152
195
|
export type BrowserSession = {
|
|
196
|
+
/** The seal — see {@link browserSessionBrand}. */
|
|
197
|
+
readonly [browserSessionBrand]: true;
|
|
153
198
|
/** Return the current immutable state snapshot. */
|
|
154
199
|
getSnapshot(): SessionSnapshot;
|
|
155
200
|
/** Subscribe to state changes. Returns an unsubscribe function. */
|
|
@@ -166,6 +211,8 @@ export type BrowserSession = {
|
|
|
166
211
|
}): void;
|
|
167
212
|
/** Cancel the current agent turn and discard in-flight TTS audio. */
|
|
168
213
|
cancel(): void;
|
|
214
|
+
/** Push-to-talk's three edges — see {@link UserTurnControls}. */
|
|
215
|
+
readonly userTurn: UserTurnControls;
|
|
169
216
|
/**
|
|
170
217
|
* Clear messages, transcripts, and error state while keeping the current
|
|
171
218
|
* connection (unlike `reset()`, which also reconnects).
|
|
@@ -223,11 +270,18 @@ export type BrowserSession = {
|
|
|
223
270
|
[Symbol.dispose](): void;
|
|
224
271
|
};
|
|
225
272
|
/**
|
|
226
|
-
* Shared mutable connection state
|
|
273
|
+
* Shared mutable connection state.
|
|
274
|
+
*
|
|
275
|
+
* **It used to carry the audio path too**, and the four fields that went are
|
|
276
|
+
* worth naming because the shape recurs: a `voiceIO` slot, an
|
|
277
|
+
* `audioSetupInFlight` latch, a `generation` epoch whose only readers were the
|
|
278
|
+
* bring-up's four staleness checks, and the two pre-init buffers. Each was a
|
|
279
|
+
* question about a position the connection did not have, readable and writable
|
|
280
|
+
* from anywhere. They are the audio path's own statechart now — see
|
|
281
|
+
* `session-core-audio-state.ts`.
|
|
227
282
|
*
|
|
228
|
-
*
|
|
229
|
-
*
|
|
230
|
-
* assigning their results to a newer connection after a reconnect.
|
|
283
|
+
* What is left is what the CONNECTION really owns: the socket, the server's
|
|
284
|
+
* idle retirement, and the turn boundary.
|
|
231
285
|
*/
|
|
232
286
|
export type ConnState = {
|
|
233
287
|
ws: InstanceType<WebSocketConstructor> | null;
|
|
@@ -242,12 +296,6 @@ export type ConnState = {
|
|
|
242
296
|
* reconnect on demand).
|
|
243
297
|
*/
|
|
244
298
|
retiredByServer: boolean;
|
|
245
|
-
voiceIO: VoiceIO | null;
|
|
246
|
-
audioSetupInFlight: boolean;
|
|
247
|
-
/** Connection epoch, bumped on each connect()/retry (see `createEpoch`).
|
|
248
|
-
* Prevents a stale initAudioCapture from assigning its voiceIO to a newer
|
|
249
|
-
* connection. */
|
|
250
|
-
generation: Epoch;
|
|
251
299
|
/**
|
|
252
300
|
* Turn epoch, bumped at every turn boundary — a committed user turn, a
|
|
253
301
|
* barge-in, a reset, AND every audio-path teardown (`cleanupAudio`).
|
|
@@ -261,15 +309,6 @@ export type ConnState = {
|
|
|
261
309
|
* later — a dead session reporting a live mic.
|
|
262
310
|
*/
|
|
263
311
|
turn: Epoch;
|
|
264
|
-
/** Audio chunks that arrived before `voiceIO` was initialized — drained into
|
|
265
|
-
* the playback worklet once init completes. Closes the race between the
|
|
266
|
-
* server starting greeting audio (immediately on S2S connect) and the
|
|
267
|
-
* client awaiting mic permission + worklet registration. */
|
|
268
|
-
preInitAudio: ArrayBuffer[];
|
|
269
|
-
/** True if `audio_done` arrived before `voiceIO` was initialized. The done
|
|
270
|
-
* signal must be replayed after draining preInitAudio, or a short greeting
|
|
271
|
-
* buffered during mic-permission never finishes playing. */
|
|
272
|
-
preInitDone: boolean;
|
|
273
312
|
};
|
|
274
313
|
/**
|
|
275
314
|
* The two liveness fields at rest.
|
|
@@ -303,4 +342,4 @@ export declare const STOPPED: {
|
|
|
303
342
|
* bump on its own now reads as a deliberate choice rather than a forgotten
|
|
304
343
|
* flush.
|
|
305
344
|
*/
|
|
306
|
-
export declare function bargeIn(conn: ConnState): void;
|
|
345
|
+
export declare function bargeIn(conn: ConnState, audio: AudioPath): void;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Push-to-talk's three edges on the browser session — `session.userTurn`'s
|
|
3
|
+
* `start`, `commit` and `clear` — for an agent that declares
|
|
4
|
+
* `turnDetection: "manual"`.
|
|
5
|
+
*
|
|
6
|
+
* Split out of `session-core.ts` at the source-length cap. Each is a frame to
|
|
7
|
+
* the server; what the server does with it is `aai-runtime`'s
|
|
8
|
+
* `pipeline-manual-turn.ts`. The two things done HERE rather than waiting a
|
|
9
|
+
* round trip for are the reason these are more than `sendJson` calls: a press
|
|
10
|
+
* stops the agent's queued audio at once, and a discarded turn's caption is
|
|
11
|
+
* cleared because no transcript for it will ever arrive.
|
|
12
|
+
*
|
|
13
|
+
* @module
|
|
14
|
+
*/
|
|
15
|
+
import type { SessionCommand } from "@alexkroman1/aai/protocol";
|
|
16
|
+
import type { SessionStateMachine } from "./session-core-state.ts";
|
|
17
|
+
import type { SessionSnapshot, UserTurnControls } from "./session-core-types.ts";
|
|
18
|
+
/** What the three edges need from the session around them. @internal */
|
|
19
|
+
export type UserTurnDeps = {
|
|
20
|
+
snapshot: () => SessionSnapshot;
|
|
21
|
+
/** A socket is open and can carry a frame. */
|
|
22
|
+
connected: () => boolean;
|
|
23
|
+
/** The local turn boundary — see `bargeIn` in `session-core-types.ts`. */
|
|
24
|
+
bargeIn: () => void;
|
|
25
|
+
agentState: SessionStateMachine;
|
|
26
|
+
updateState: (partial: Partial<SessionSnapshot>) => void;
|
|
27
|
+
sendJson: (msg: SessionCommand) => void;
|
|
28
|
+
};
|
|
29
|
+
/** Build the three push-to-talk edges. @internal */
|
|
30
|
+
export declare function createUserTurnActions(deps: UserTurnDeps): UserTurnControls;
|
package/dist/session-core.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { t as createBrowserSession } from "./session-core-
|
|
1
|
+
import { t as createBrowserSession } from "./session-core-TOdm_Con.js";
|
|
2
2
|
export { createBrowserSession };
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `usePushToTalk` — a hold-to-speak button, for an agent that declares
|
|
3
|
+
* `turnDetection: "manual"`.
|
|
4
|
+
*
|
|
5
|
+
* The three session edges underneath (`session.userTurn`'s `start`, `commit`
|
|
6
|
+
* and `clear`) are one line each; what a button needs on top of them is
|
|
7
|
+
* where hand-written versions go wrong, and every one of the four is a turn
|
|
8
|
+
* that is left OPEN — the microphone live and nothing ever answered:
|
|
9
|
+
*
|
|
10
|
+
* - **The pointer leaves.** A `mouseup` outside the button never reaches it.
|
|
11
|
+
* The handlers here capture the pointer on the way down, so the release
|
|
12
|
+
* arrives wherever it happens; an `onPointerCancel` (a scroll gesture, a
|
|
13
|
+
* system dialog) DISCARDS the turn rather than answering half a sentence.
|
|
14
|
+
* - **The key repeats.** Holding a key fires `keydown` over and over, and each
|
|
15
|
+
* one would re-open the turn. Only the first counts.
|
|
16
|
+
* - **The window loses focus mid-hold.** The `keyup` goes to another window
|
|
17
|
+
* and is never delivered, so the turn is committed on `blur` — what the
|
|
18
|
+
* caller would have done had the release been seen.
|
|
19
|
+
* - **The component unmounts mid-hold.** The turn is discarded on the way out.
|
|
20
|
+
*
|
|
21
|
+
* And one that is a nuisance rather than a stuck turn: the hold key is ignored
|
|
22
|
+
* while focus is in a text field, where Space is a character.
|
|
23
|
+
*/
|
|
24
|
+
/** Options for {@link usePushToTalk}. */
|
|
25
|
+
export type UsePushToTalkOptions = {
|
|
26
|
+
/**
|
|
27
|
+
* The keyboard key that holds the turn open, as a `KeyboardEvent.code` —
|
|
28
|
+
* `"Space"` by default, so the whole page is a walkie-talkie. `false` turns
|
|
29
|
+
* the global key off; the button itself still answers Space and Enter while
|
|
30
|
+
* it has focus.
|
|
31
|
+
*/
|
|
32
|
+
holdKey?: string | false;
|
|
33
|
+
};
|
|
34
|
+
/**
|
|
35
|
+
* What {@link usePushToTalk} returns.
|
|
36
|
+
*
|
|
37
|
+
* @public
|
|
38
|
+
*/
|
|
39
|
+
export type UsePushToTalkResult = {
|
|
40
|
+
/** Whether a turn is being held open right now — the button is DOWN. */
|
|
41
|
+
talking: boolean;
|
|
42
|
+
/**
|
|
43
|
+
* Whether pressing would do anything: the call is live. False before Start
|
|
44
|
+
* and while paused, which is when a button should render disabled.
|
|
45
|
+
*/
|
|
46
|
+
ready: boolean;
|
|
47
|
+
/** Open a turn. Interrupts the agent if it is speaking. Ignored while held. */
|
|
48
|
+
press: () => void;
|
|
49
|
+
/** Close the turn and have the agent answer it. Ignored unless held. */
|
|
50
|
+
release: () => void;
|
|
51
|
+
/** Close the turn and discard it — nothing is answered. Ignored unless held. */
|
|
52
|
+
cancel: () => void;
|
|
53
|
+
/**
|
|
54
|
+
* Spread onto a `<button>`: pointer capture, the keyboard pair, and
|
|
55
|
+
* `aria-pressed`. The handlers are the whole contract — style it however you
|
|
56
|
+
* like.
|
|
57
|
+
*/
|
|
58
|
+
buttonProps: {
|
|
59
|
+
onPointerDown: (event: {
|
|
60
|
+
currentTarget: Element;
|
|
61
|
+
pointerId: number;
|
|
62
|
+
}) => void;
|
|
63
|
+
onPointerUp: () => void;
|
|
64
|
+
onPointerCancel: () => void;
|
|
65
|
+
onKeyDown: (event: {
|
|
66
|
+
key: string;
|
|
67
|
+
repeat: boolean;
|
|
68
|
+
preventDefault(): void;
|
|
69
|
+
}) => void;
|
|
70
|
+
onKeyUp: (event: {
|
|
71
|
+
key: string;
|
|
72
|
+
preventDefault(): void;
|
|
73
|
+
}) => void;
|
|
74
|
+
onContextMenu: (event: {
|
|
75
|
+
preventDefault(): void;
|
|
76
|
+
}) => void;
|
|
77
|
+
disabled: boolean;
|
|
78
|
+
"aria-pressed": boolean;
|
|
79
|
+
};
|
|
80
|
+
};
|
|
81
|
+
/**
|
|
82
|
+
* Hold-to-speak over the session's push-to-talk methods, with the four ways a
|
|
83
|
+
* turn gets stuck open handled — see this module's doc.
|
|
84
|
+
*
|
|
85
|
+
* Must be used inside the provider `mountClient()` installs, against an agent
|
|
86
|
+
* declaring `turnDetection: "manual"`; any other agent ignores the commands and
|
|
87
|
+
* its server says so once.
|
|
88
|
+
*
|
|
89
|
+
* @example A hold-to-talk button
|
|
90
|
+
* ```tsx
|
|
91
|
+
* import { usePushToTalk } from "@alexkroman1/aai-ui";
|
|
92
|
+
*
|
|
93
|
+
* function TalkButton() {
|
|
94
|
+
* const { talking, buttonProps } = usePushToTalk();
|
|
95
|
+
* return (
|
|
96
|
+
* <button type="button" {...buttonProps}>
|
|
97
|
+
* {talking ? "Listening… release to send" : "Hold to talk (or hold Space)"}
|
|
98
|
+
* </button>
|
|
99
|
+
* );
|
|
100
|
+
* }
|
|
101
|
+
* ```
|
|
102
|
+
*
|
|
103
|
+
* @public
|
|
104
|
+
*/
|
|
105
|
+
export declare function usePushToTalk(options?: UsePushToTalkOptions): UsePushToTalkResult;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { t as pageBaseUrl } from "./_utils-DnQDM9Uy.js";
|
|
2
|
-
import { useCallback, useEffect, useRef, useState } from "react";
|
|
3
2
|
import { errorMessage } from "@alexkroman1/aai";
|
|
3
|
+
import { useCallback, useEffect, useRef, useState } from "react";
|
|
4
4
|
import { omitUndefined } from "@alexkroman1/aai/utils";
|
|
5
|
-
import {
|
|
5
|
+
import { createAgentClient, isTerminal, isTerminal as isTerminal$1, readEventStream } from "@alexkroman1/aai/workflow-api";
|
|
6
6
|
//#region src/workflow-client.ts
|
|
7
7
|
/**
|
|
8
8
|
* Browser client for the workflow HTTP API (`aai/host/workflow-api.ts`).
|
|
@@ -31,20 +31,28 @@ import { createWorkflowApiClient, isTerminal, isTerminal as isTerminal$1, readEv
|
|
|
31
31
|
* fast path under it in `workflow-events.ts`.
|
|
32
32
|
*/
|
|
33
33
|
/**
|
|
34
|
-
* Create a
|
|
34
|
+
* Create a client for the agent serving this page.
|
|
35
|
+
*
|
|
36
|
+
* **You usually do not need one.** Every hook and component here builds this
|
|
37
|
+
* exact client lazily and once when no `api` is passed
|
|
38
|
+
* (`_workflow-api-ref.ts`), so `useWorkflowSubmit("digest")` already talks to
|
|
39
|
+
* the right agent. Reach for this when the client has to be DIFFERENT from that
|
|
40
|
+
* default — another agent's `baseUrl`, or a `token` — or when a page wants
|
|
41
|
+
* `config()` and the run calls on one object.
|
|
35
42
|
*
|
|
36
|
-
*
|
|
37
|
-
* in a ref precisely so a fresh object per
|
|
38
|
-
* but a client built in render is still a new
|
|
39
|
-
* reads as though it were free.
|
|
43
|
+
* If you do build one, hoist it out of the component that uses it.
|
|
44
|
+
* `useWorkflowRun` holds the client in a ref precisely so a fresh object per
|
|
45
|
+
* render does not restart its watch, but a client built in render is still a new
|
|
46
|
+
* `fetch` closure every time and reads as though it were free.
|
|
40
47
|
*
|
|
41
48
|
* @example
|
|
42
49
|
* ```tsx
|
|
43
50
|
* import { createWorkflowApi, useWorkflowRun } from "@alexkroman1/aai-ui";
|
|
44
51
|
* import { useState } from "react";
|
|
45
52
|
*
|
|
46
|
-
* //
|
|
47
|
-
*
|
|
53
|
+
* // A DIFFERENT agent than the one serving this page, so the client is
|
|
54
|
+
* // explicit — and module scope, not render scope, per above.
|
|
55
|
+
* const api = createWorkflowApi({ baseUrl: "https://agents.example/digest" });
|
|
48
56
|
*
|
|
49
57
|
* function StartDigest() {
|
|
50
58
|
* const [runId, setRunId] = useState<string>();
|
|
@@ -62,12 +70,17 @@ import { createWorkflowApiClient, isTerminal, isTerminal as isTerminal$1, readEv
|
|
|
62
70
|
*
|
|
63
71
|
* @param options - See {@link WorkflowApiOptions}. Both fields are optional; the
|
|
64
72
|
* default base URL is the page's own origin and path.
|
|
65
|
-
* @returns
|
|
73
|
+
* @returns Every workflow call plus `config()` and `baseUrl` — see
|
|
74
|
+
* {@link AgentClient}. It was the narrow {@link WorkflowApi}, which left a page
|
|
75
|
+
* that also wanted the agent's own name building a SECOND client (or a bare
|
|
76
|
+
* `fetch` and a hand-written URL join) for one read; the SDK documents
|
|
77
|
+
* `createAgentClient` as a superset of the same routes, so delegating to it
|
|
78
|
+
* widens the return without a second implementation of anything.
|
|
66
79
|
*
|
|
67
80
|
* @public
|
|
68
81
|
*/
|
|
69
82
|
function createWorkflowApi(options = {}) {
|
|
70
|
-
return
|
|
83
|
+
return createAgentClient({
|
|
71
84
|
baseUrl: options.baseUrl ?? pageBaseUrl(),
|
|
72
85
|
...omitUndefined({ token: options.token })
|
|
73
86
|
});
|