@qafka/react-native 2.3.8 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/android/src/main/AndroidManifest.xml +5 -0
- package/android/src/main/java/com/qafka/audio/QafkaAudioModule.kt +161 -16
- package/dist/components/ChatPage.js +5 -5
- package/dist/decide-interrupt.d.ts +9 -0
- package/dist/decide-interrupt.js +12 -0
- package/dist/decide-mic-forwarding.d.ts +23 -0
- package/dist/decide-mic-forwarding.js +28 -0
- package/dist/hooks/useChatMessages.js +73 -22
- package/dist/hooks/useVoiceChat.js +16 -0
- package/dist/native/QafkaAudio.d.ts +3 -0
- package/dist/native/QafkaAudio.js +13 -0
- package/dist/services/RealtimeService.d.ts +18 -0
- package/dist/services/RealtimeService.js +119 -3
- package/dist/utils/extractionContinuation.d.ts +32 -0
- package/dist/utils/extractionContinuation.js +48 -0
- package/dist/utils/joinNarration.d.ts +15 -0
- package/dist/utils/joinNarration.js +25 -0
- package/dist/utils/toolNarration.d.ts +21 -0
- package/dist/utils/toolNarration.js +28 -0
- package/ios/QafkaAudio.m +9 -0
- package/ios/QafkaAudio.swift +171 -1
- package/package.json +2 -2
|
@@ -11,6 +11,8 @@ export type RealtimeEvent = {
|
|
|
11
11
|
delta: string;
|
|
12
12
|
} | {
|
|
13
13
|
type: 'response.done';
|
|
14
|
+
} | {
|
|
15
|
+
type: 'response.interrupted';
|
|
14
16
|
} | {
|
|
15
17
|
type: 'conversation.item.input_audio_transcription.completed';
|
|
16
18
|
transcript: string;
|
|
@@ -47,6 +49,12 @@ export declare class RealtimeService {
|
|
|
47
49
|
private apiKey;
|
|
48
50
|
private ws;
|
|
49
51
|
private audioUnsubscribe;
|
|
52
|
+
private bargeInUnsubscribe;
|
|
53
|
+
private bargeInTimes;
|
|
54
|
+
private echoDowngraded;
|
|
55
|
+
private micEnabled;
|
|
56
|
+
private aiSpeaking;
|
|
57
|
+
private fullDuplex;
|
|
50
58
|
private eventHandler;
|
|
51
59
|
private audioChunksSent;
|
|
52
60
|
private sessionTokenGetter;
|
|
@@ -62,8 +70,18 @@ export declare class RealtimeService {
|
|
|
62
70
|
}): Promise<void>;
|
|
63
71
|
private audioDeltasReceived;
|
|
64
72
|
private handleEvent;
|
|
73
|
+
/**
|
|
74
|
+
* Native detected the user talking over the AI (client-side barge-in) — it has
|
|
75
|
+
* already cut playback locally and is suppressing the cancelled turn. Mirror a
|
|
76
|
+
* server interrupt so the consumer UI returns to listening immediately; the
|
|
77
|
+
* gateway's own (slower) interrupt follows and lifts the suppression.
|
|
78
|
+
*/
|
|
79
|
+
private handleLocalBargeIn;
|
|
80
|
+
private guardAgainstEcho;
|
|
65
81
|
private attachAudioListener;
|
|
66
82
|
private detachAudioListener;
|
|
83
|
+
private syncMicForwarding;
|
|
84
|
+
private setAiSpeaking;
|
|
67
85
|
private startAudioPipeline;
|
|
68
86
|
/**
|
|
69
87
|
* Fully stop the native audio engine (capture + playback). Use when leaving
|
|
@@ -3,11 +3,41 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.RealtimeService = void 0;
|
|
4
4
|
const QafkaAudio_1 = require("../native/QafkaAudio");
|
|
5
5
|
const QafkaAttestation_1 = require("../native/QafkaAttestation");
|
|
6
|
+
const decide_interrupt_1 = require("../decide-interrupt");
|
|
7
|
+
const decide_mic_forwarding_1 = require("../decide-mic-forwarding");
|
|
8
|
+
// Full-duplex voice: keep the mic open while the AI speaks and rely on the
|
|
9
|
+
// native OS echo canceler to strip the AI's playback out of the mic, so the
|
|
10
|
+
// user can interrupt by voice (barge-in). When false, the SDK falls back to
|
|
11
|
+
// half-duplex (mic gated while the AI speaks). A failing on-device echo
|
|
12
|
+
// canceler auto-downgrades the session to half-duplex at runtime (see the
|
|
13
|
+
// echo guard below).
|
|
14
|
+
const DEFAULT_FULL_DUPLEX = true;
|
|
15
|
+
// Echo guard: if the device's AEC is failing, the AI's own voice leaks into the
|
|
16
|
+
// mic and the barge-in detector reads it as the user — firing repeatedly. A real
|
|
17
|
+
// user never barges in this often, so this many barge-ins within the window means
|
|
18
|
+
// the AEC isn't working: downgrade the session to half-duplex.
|
|
19
|
+
const ECHO_GUARD_BARGE_INS = 3;
|
|
20
|
+
const ECHO_GUARD_WINDOW_MS = 12000;
|
|
6
21
|
class RealtimeService {
|
|
7
22
|
apiUrl;
|
|
8
23
|
apiKey;
|
|
9
24
|
ws = null;
|
|
10
25
|
audioUnsubscribe = null;
|
|
26
|
+
// Native client-side barge-in subscription (mic-energy spike while AI speaks).
|
|
27
|
+
bargeInUnsubscribe = null;
|
|
28
|
+
// Echo guard — recent barge-in timestamps + whether this session downgraded.
|
|
29
|
+
bargeInTimes = [];
|
|
30
|
+
echoDowngraded = false;
|
|
31
|
+
// Half-duplex mic gating. `micEnabled` reflects the user/system mute state;
|
|
32
|
+
// `aiSpeaking` is true while the AI's audio is playing. Mic audio is
|
|
33
|
+
// forwarded to the server only when both allow it — this stops the AI's own
|
|
34
|
+
// voice (echoed into the mic) from being transcribed as a user turn, since
|
|
35
|
+
// there is no server-side echo cancellation upstream.
|
|
36
|
+
micEnabled = true;
|
|
37
|
+
aiSpeaking = false;
|
|
38
|
+
// Full-duplex (keep mic open while AI speaks, rely on native hardware AEC).
|
|
39
|
+
// When false, half-duplex mic gating is used. See DEFAULT_FULL_DUPLEX.
|
|
40
|
+
fullDuplex = DEFAULT_FULL_DUPLEX;
|
|
11
41
|
eventHandler = null;
|
|
12
42
|
audioChunksSent = 0;
|
|
13
43
|
sessionTokenGetter = null;
|
|
@@ -50,6 +80,12 @@ class RealtimeService {
|
|
|
50
80
|
Object.keys(options.endUserData).length > 0) {
|
|
51
81
|
params.set('endUserData', JSON.stringify(options.endUserData));
|
|
52
82
|
}
|
|
83
|
+
// Tell the gateway this session is full-duplex so it enables the server-side
|
|
84
|
+
// barge-in VAD config ONLY for us. Half-duplex builds omit it → the gateway
|
|
85
|
+
// keeps their exact prior behavior.
|
|
86
|
+
if (this.fullDuplex) {
|
|
87
|
+
params.set('fullDuplex', '1');
|
|
88
|
+
}
|
|
53
89
|
const qs = params.toString();
|
|
54
90
|
const fullUrl = `${wsUrl}/v1/realtime${qs ? `?${qs}` : ''}`;
|
|
55
91
|
// Start the audio engine FIRST so the player is ready to schedule buffers
|
|
@@ -117,12 +153,63 @@ class RealtimeService {
|
|
|
117
153
|
handleEvent(event) {
|
|
118
154
|
if (event.type === 'response.audio.delta' && 'delta' in event) {
|
|
119
155
|
this.audioDeltasReceived++;
|
|
156
|
+
// Half-duplex: the AI is speaking — stop forwarding mic audio so its own
|
|
157
|
+
// voice can't echo back into the STT as a user turn.
|
|
158
|
+
this.setAiSpeaking(true);
|
|
120
159
|
QafkaAudio_1.QafkaAudio.playAudioChunk(event.delta).catch(() => {
|
|
121
160
|
// playback errors are surfaced to the consumer via eventHandler
|
|
122
161
|
});
|
|
123
162
|
}
|
|
163
|
+
else if ((0, decide_interrupt_1.shouldStopPlayback)(event)) {
|
|
164
|
+
// Barge-in: the server signalled the user interrupted. Purge native
|
|
165
|
+
// playback so queued audio stops; the WS is ordered, so no further
|
|
166
|
+
// audio delta arrives after this for the cancelled turn. Also lift any
|
|
167
|
+
// local-barge-in suppression so the AI's NEXT turn can play.
|
|
168
|
+
this.setAiSpeaking(false);
|
|
169
|
+
QafkaAudio_1.QafkaAudio.stopPlayback().catch(() => { });
|
|
170
|
+
QafkaAudio_1.QafkaAudio.resumePlayback().catch(() => { });
|
|
171
|
+
}
|
|
172
|
+
else if (event.type === 'response.done') {
|
|
173
|
+
// AI finished its turn — reopen mic forwarding for the user and clear any
|
|
174
|
+
// lingering local-barge-in suppression.
|
|
175
|
+
this.setAiSpeaking(false);
|
|
176
|
+
QafkaAudio_1.QafkaAudio.resumePlayback().catch(() => { });
|
|
177
|
+
}
|
|
124
178
|
this.eventHandler?.(event);
|
|
125
179
|
}
|
|
180
|
+
/**
|
|
181
|
+
* Native detected the user talking over the AI (client-side barge-in) — it has
|
|
182
|
+
* already cut playback locally and is suppressing the cancelled turn. Mirror a
|
|
183
|
+
* server interrupt so the consumer UI returns to listening immediately; the
|
|
184
|
+
* gateway's own (slower) interrupt follows and lifts the suppression.
|
|
185
|
+
*/
|
|
186
|
+
handleLocalBargeIn() {
|
|
187
|
+
if (!this.aiSpeaking)
|
|
188
|
+
return; // only meaningful while the AI is talking
|
|
189
|
+
this.setAiSpeaking(false);
|
|
190
|
+
this.eventHandler?.({ type: 'response.interrupted' });
|
|
191
|
+
this.guardAgainstEcho();
|
|
192
|
+
}
|
|
193
|
+
// A device whose AEC fails leaks the AI's own voice into the mic, which the
|
|
194
|
+
// barge-in detector reads as the user talking — firing repeatedly. A real user
|
|
195
|
+
// never barges in this often, so a burst means the AEC isn't working: downgrade
|
|
196
|
+
// THIS session to half-duplex (mic gated while the AI speaks + native barge-in
|
|
197
|
+
// disarmed) so the echo can't keep cutting the AI. Per-session — a fresh
|
|
198
|
+
// session re-tries full-duplex.
|
|
199
|
+
guardAgainstEcho() {
|
|
200
|
+
if (this.echoDowngraded)
|
|
201
|
+
return;
|
|
202
|
+
const now = Date.now();
|
|
203
|
+
this.bargeInTimes.push(now);
|
|
204
|
+
this.bargeInTimes = this.bargeInTimes.filter((t) => now - t <= ECHO_GUARD_WINDOW_MS);
|
|
205
|
+
if (this.bargeInTimes.length >= ECHO_GUARD_BARGE_INS) {
|
|
206
|
+
this.echoDowngraded = true;
|
|
207
|
+
this.fullDuplex = false;
|
|
208
|
+
QafkaAudio_1.QafkaAudio.setBargeInEnabled(false).catch(() => { });
|
|
209
|
+
QafkaAudio_1.QafkaAudio.resumePlayback().catch(() => { });
|
|
210
|
+
this.syncMicForwarding();
|
|
211
|
+
}
|
|
212
|
+
}
|
|
126
213
|
attachAudioListener() {
|
|
127
214
|
if (this.audioUnsubscribe)
|
|
128
215
|
return; // already attached
|
|
@@ -137,6 +224,26 @@ class RealtimeService {
|
|
|
137
224
|
this.audioUnsubscribe?.();
|
|
138
225
|
this.audioUnsubscribe = null;
|
|
139
226
|
}
|
|
227
|
+
// Attach/detach mic→server forwarding from the combined mute + AI-speaking
|
|
228
|
+
// state. Both underlying calls are idempotent, so this is safe to call often.
|
|
229
|
+
syncMicForwarding() {
|
|
230
|
+
if ((0, decide_mic_forwarding_1.shouldForwardMicAudio)({
|
|
231
|
+
micEnabled: this.micEnabled,
|
|
232
|
+
aiSpeaking: this.aiSpeaking,
|
|
233
|
+
fullDuplex: this.fullDuplex,
|
|
234
|
+
})) {
|
|
235
|
+
this.attachAudioListener();
|
|
236
|
+
}
|
|
237
|
+
else {
|
|
238
|
+
this.detachAudioListener();
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
setAiSpeaking(speaking) {
|
|
242
|
+
if (this.aiSpeaking === speaking)
|
|
243
|
+
return;
|
|
244
|
+
this.aiSpeaking = speaking;
|
|
245
|
+
this.syncMicForwarding();
|
|
246
|
+
}
|
|
140
247
|
async startAudioPipeline() {
|
|
141
248
|
try {
|
|
142
249
|
// Android needs the RECORD_AUDIO runtime permission requested explicitly
|
|
@@ -152,7 +259,12 @@ class RealtimeService {
|
|
|
152
259
|
}
|
|
153
260
|
// Attach listener BEFORE startCapture so the native tap
|
|
154
261
|
// doesn't drop early buffers (hasListeners/listenerCount gate).
|
|
155
|
-
|
|
262
|
+
// Fresh capture: the AI isn't speaking yet; sync respects any mute state.
|
|
263
|
+
this.aiSpeaking = false;
|
|
264
|
+
this.syncMicForwarding();
|
|
265
|
+
if (!this.bargeInUnsubscribe) {
|
|
266
|
+
this.bargeInUnsubscribe = QafkaAudio_1.QafkaAudio.onBargeIn(() => this.handleLocalBargeIn());
|
|
267
|
+
}
|
|
156
268
|
await QafkaAudio_1.QafkaAudio.startCapture();
|
|
157
269
|
}
|
|
158
270
|
catch (error) {
|
|
@@ -181,10 +293,12 @@ class RealtimeService {
|
|
|
181
293
|
* tool flows where the user shouldn't interrupt but the AI may still speak.
|
|
182
294
|
*/
|
|
183
295
|
muteMic() {
|
|
184
|
-
this.
|
|
296
|
+
this.micEnabled = false;
|
|
297
|
+
this.syncMicForwarding();
|
|
185
298
|
}
|
|
186
299
|
unmuteMic() {
|
|
187
|
-
this.
|
|
300
|
+
this.micEnabled = true;
|
|
301
|
+
this.syncMicForwarding();
|
|
188
302
|
}
|
|
189
303
|
sendToolResult(payload) {
|
|
190
304
|
if (this.ws?.readyState !== WebSocket.OPEN) {
|
|
@@ -213,6 +327,8 @@ class RealtimeService {
|
|
|
213
327
|
async cleanup() {
|
|
214
328
|
this.audioUnsubscribe?.();
|
|
215
329
|
this.audioUnsubscribe = null;
|
|
330
|
+
this.bargeInUnsubscribe?.();
|
|
331
|
+
this.bargeInUnsubscribe = null;
|
|
216
332
|
try {
|
|
217
333
|
await QafkaAudio_1.QafkaAudio.stopCapture();
|
|
218
334
|
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Helpers for auto-continuing a multi-step file-input tool flow after a
|
|
3
|
+
* document is uploaded and its fields are extracted.
|
|
4
|
+
*
|
|
5
|
+
* Background: uploading a file + extracting its fields does NOT, on its own,
|
|
6
|
+
* start a new model turn. Only a fresh `/chat` turn re-enables tools, which is
|
|
7
|
+
* what lets the model advance to the next step (e.g. ask the first question).
|
|
8
|
+
* For multi-step file tools we therefore fire a hidden continuation turn that
|
|
9
|
+
* carries the extracted fields so the model can proceed.
|
|
10
|
+
*
|
|
11
|
+
* These helpers are intentionally generic — they know nothing about any
|
|
12
|
+
* specific tool, field name, or domain.
|
|
13
|
+
*/
|
|
14
|
+
/** A suggested tool spans multiple steps AND collects a file when it exposes
|
|
15
|
+
* both a `step` parameter and a file-input module. Single-shot file tools
|
|
16
|
+
* (extract → render → done) are excluded. */
|
|
17
|
+
export declare function isMultiStepFileTool(tool: any): boolean;
|
|
18
|
+
/** Auto-advance only once a multi-step file tool's extraction has completed.
|
|
19
|
+
* Incomplete extractions and single-shot tools stay on the normal path. */
|
|
20
|
+
export declare function shouldAutoContinueAfterExtraction(args: {
|
|
21
|
+
isMultiStepFileTool: boolean;
|
|
22
|
+
extractionStatus?: string;
|
|
23
|
+
}): boolean;
|
|
24
|
+
/** Build the hidden continuation message. The extracted fields are embedded
|
|
25
|
+
* inline as JSON: in practice the model only reliably uses the values (and
|
|
26
|
+
* stays on-script / in-language) when they are present in the message itself —
|
|
27
|
+
* a generic "read it from context" trigger derails it. The data IS also in the
|
|
28
|
+
* `context` prop, but this inline copy is what keeps the continuation reliable.
|
|
29
|
+
* Keeping it out of the persisted transcript is handled separately, server side,
|
|
30
|
+
* not by stripping it here.
|
|
31
|
+
*/
|
|
32
|
+
export declare function buildExtractionContinuationMessage(data: Record<string, any> | undefined | null): string;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Helpers for auto-continuing a multi-step file-input tool flow after a
|
|
4
|
+
* document is uploaded and its fields are extracted.
|
|
5
|
+
*
|
|
6
|
+
* Background: uploading a file + extracting its fields does NOT, on its own,
|
|
7
|
+
* start a new model turn. Only a fresh `/chat` turn re-enables tools, which is
|
|
8
|
+
* what lets the model advance to the next step (e.g. ask the first question).
|
|
9
|
+
* For multi-step file tools we therefore fire a hidden continuation turn that
|
|
10
|
+
* carries the extracted fields so the model can proceed.
|
|
11
|
+
*
|
|
12
|
+
* These helpers are intentionally generic — they know nothing about any
|
|
13
|
+
* specific tool, field name, or domain.
|
|
14
|
+
*/
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
exports.isMultiStepFileTool = isMultiStepFileTool;
|
|
17
|
+
exports.shouldAutoContinueAfterExtraction = shouldAutoContinueAfterExtraction;
|
|
18
|
+
exports.buildExtractionContinuationMessage = buildExtractionContinuationMessage;
|
|
19
|
+
/** A suggested tool spans multiple steps AND collects a file when it exposes
|
|
20
|
+
* both a `step` parameter and a file-input module. Single-shot file tools
|
|
21
|
+
* (extract → render → done) are excluded. */
|
|
22
|
+
function isMultiStepFileTool(tool) {
|
|
23
|
+
const params = tool?.parameters;
|
|
24
|
+
const hasStep = !!params && typeof params === 'object' && 'step' in params;
|
|
25
|
+
return Boolean(tool?.modules?.hasFileInput) && hasStep;
|
|
26
|
+
}
|
|
27
|
+
/** Auto-advance only once a multi-step file tool's extraction has completed.
|
|
28
|
+
* Incomplete extractions and single-shot tools stay on the normal path. */
|
|
29
|
+
function shouldAutoContinueAfterExtraction(args) {
|
|
30
|
+
return args.isMultiStepFileTool && args.extractionStatus === 'complete';
|
|
31
|
+
}
|
|
32
|
+
/** Build the hidden continuation message. The extracted fields are embedded
|
|
33
|
+
* inline as JSON: in practice the model only reliably uses the values (and
|
|
34
|
+
* stays on-script / in-language) when they are present in the message itself —
|
|
35
|
+
* a generic "read it from context" trigger derails it. The data IS also in the
|
|
36
|
+
* `context` prop, but this inline copy is what keeps the continuation reliable.
|
|
37
|
+
* Keeping it out of the persisted transcript is handled separately, server side,
|
|
38
|
+
* not by stripping it here.
|
|
39
|
+
*/
|
|
40
|
+
function buildExtractionContinuationMessage(data) {
|
|
41
|
+
const fields = data && typeof data === 'object' ? JSON.stringify(data) : '{}';
|
|
42
|
+
return [
|
|
43
|
+
'[Uploaded document — extracted fields]',
|
|
44
|
+
fields,
|
|
45
|
+
'',
|
|
46
|
+
'(The user just uploaded a document and it was processed. Use these fields to continue the tool flow — proceed to the next step. Do not restart or repeat the greeting.)',
|
|
47
|
+
].join('\n');
|
|
48
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Joins two pieces of assistant narration into a single text body with a
|
|
3
|
+
* consistent paragraph break.
|
|
4
|
+
*
|
|
5
|
+
* A custom-with-ai tool produces a reply in two streamed parts (e.g. a first
|
|
6
|
+
* reply, then a follow-up after a file upload). Those parts land in the same
|
|
7
|
+
* assistant bubble, so they must be separated the same way everywhere — both in
|
|
8
|
+
* the live render and in what the backend persists — otherwise the live text
|
|
9
|
+
* comes out glued together with no break (".. yeterli!Harika! ..").
|
|
10
|
+
*
|
|
11
|
+
* Rules: empty parts pass through untouched; if `prev` already ends with a
|
|
12
|
+
* newline, no extra break is added; otherwise the two parts are joined with a
|
|
13
|
+
* blank line.
|
|
14
|
+
*/
|
|
15
|
+
export declare function joinNarration(prev: string, next: string): string;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.joinNarration = joinNarration;
|
|
4
|
+
/**
|
|
5
|
+
* Joins two pieces of assistant narration into a single text body with a
|
|
6
|
+
* consistent paragraph break.
|
|
7
|
+
*
|
|
8
|
+
* A custom-with-ai tool produces a reply in two streamed parts (e.g. a first
|
|
9
|
+
* reply, then a follow-up after a file upload). Those parts land in the same
|
|
10
|
+
* assistant bubble, so they must be separated the same way everywhere — both in
|
|
11
|
+
* the live render and in what the backend persists — otherwise the live text
|
|
12
|
+
* comes out glued together with no break (".. yeterli!Harika! ..").
|
|
13
|
+
*
|
|
14
|
+
* Rules: empty parts pass through untouched; if `prev` already ends with a
|
|
15
|
+
* newline, no extra break is added; otherwise the two parts are joined with a
|
|
16
|
+
* blank line.
|
|
17
|
+
*/
|
|
18
|
+
function joinNarration(prev, next) {
|
|
19
|
+
if (!prev)
|
|
20
|
+
return next || '';
|
|
21
|
+
if (!next)
|
|
22
|
+
return prev;
|
|
23
|
+
const sep = prev.endsWith('\n') ? '' : '\n\n';
|
|
24
|
+
return prev + sep + next;
|
|
25
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whether to skip the turn-2 narration (the post-tool-result round-trip) for a
|
|
3
|
+
* suggested tool.
|
|
4
|
+
*
|
|
5
|
+
* custom-with-ai tools answer in two streamed parts: the model's first reply
|
|
6
|
+
* (the "bridge", produced alongside the tool call) and a turn-2 narration of
|
|
7
|
+
* the tool result. Which one carries the real content is per-tool:
|
|
8
|
+
* - **File-input tools** (e.g. an interview driven by an uploaded CV) are
|
|
9
|
+
* bridge-primary: the model writes the full reply in the bridge (it responds
|
|
10
|
+
* to the post-upload continuation), so the turn-2 narration just repeats it.
|
|
11
|
+
* Skip turn-2 for these.
|
|
12
|
+
* - **Non-file tools** (e.g. a date-availability check) are turn-2-primary:
|
|
13
|
+
* the bridge is a short "checking…" filler and the SUBSTANTIVE result lives
|
|
14
|
+
* only in the turn-2 narration. Skipping it would leave the flow hanging, so
|
|
15
|
+
* turn-2 must still run.
|
|
16
|
+
*
|
|
17
|
+
* We therefore gate on `hasFileInput` in addition to custom-with-ai + card.
|
|
18
|
+
* (A per-tool flag would be cleaner long-term; this heuristic distinguishes the
|
|
19
|
+
* current bridge-primary vs turn-2-primary tools.)
|
|
20
|
+
*/
|
|
21
|
+
export declare function shouldSkipTurn2Narration(tool: any): boolean;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.shouldSkipTurn2Narration = shouldSkipTurn2Narration;
|
|
4
|
+
/**
|
|
5
|
+
* Whether to skip the turn-2 narration (the post-tool-result round-trip) for a
|
|
6
|
+
* suggested tool.
|
|
7
|
+
*
|
|
8
|
+
* custom-with-ai tools answer in two streamed parts: the model's first reply
|
|
9
|
+
* (the "bridge", produced alongside the tool call) and a turn-2 narration of
|
|
10
|
+
* the tool result. Which one carries the real content is per-tool:
|
|
11
|
+
* - **File-input tools** (e.g. an interview driven by an uploaded CV) are
|
|
12
|
+
* bridge-primary: the model writes the full reply in the bridge (it responds
|
|
13
|
+
* to the post-upload continuation), so the turn-2 narration just repeats it.
|
|
14
|
+
* Skip turn-2 for these.
|
|
15
|
+
* - **Non-file tools** (e.g. a date-availability check) are turn-2-primary:
|
|
16
|
+
* the bridge is a short "checking…" filler and the SUBSTANTIVE result lives
|
|
17
|
+
* only in the turn-2 narration. Skipping it would leave the flow hanging, so
|
|
18
|
+
* turn-2 must still run.
|
|
19
|
+
*
|
|
20
|
+
* We therefore gate on `hasFileInput` in addition to custom-with-ai + card.
|
|
21
|
+
* (A per-tool flag would be cleaner long-term; this heuristic distinguishes the
|
|
22
|
+
* current bridge-primary vs turn-2-primary tools.)
|
|
23
|
+
*/
|
|
24
|
+
function shouldSkipTurn2Narration(tool) {
|
|
25
|
+
return (tool?.executionMode === 'custom-with-ai' &&
|
|
26
|
+
tool?.uiConfig?.responseType === 'card' &&
|
|
27
|
+
tool?.modules?.hasFileInput === true);
|
|
28
|
+
}
|
package/ios/QafkaAudio.m
CHANGED
|
@@ -20,4 +20,13 @@ RCT_EXTERN_METHOD(stopPlayback:
|
|
|
20
20
|
(RCTPromiseResolveBlock)resolve
|
|
21
21
|
rejecter:(RCTPromiseRejectBlock)reject)
|
|
22
22
|
|
|
23
|
+
RCT_EXTERN_METHOD(resumePlayback:
|
|
24
|
+
(RCTPromiseResolveBlock)resolve
|
|
25
|
+
rejecter:(RCTPromiseRejectBlock)reject)
|
|
26
|
+
|
|
27
|
+
RCT_EXTERN_METHOD(setBargeInEnabled:
|
|
28
|
+
(BOOL)enabled
|
|
29
|
+
resolver:(RCTPromiseResolveBlock)resolve
|
|
30
|
+
rejecter:(RCTPromiseRejectBlock)reject)
|
|
31
|
+
|
|
23
32
|
@end
|
package/ios/QafkaAudio.swift
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import Foundation
|
|
2
2
|
import AVFoundation
|
|
3
3
|
import React
|
|
4
|
+
import os
|
|
4
5
|
|
|
5
6
|
#if DEBUG
|
|
6
7
|
private func qafkaLog(_ message: String) { NSLog("%@", message) }
|
|
@@ -18,6 +19,58 @@ class QafkaAudio: RCTEventEmitter {
|
|
|
18
19
|
private var configChangeObserver: NSObjectProtocol?
|
|
19
20
|
private var playChunkCount = 0
|
|
20
21
|
|
|
22
|
+
// MARK: - Client-side barge-in state
|
|
23
|
+
//
|
|
24
|
+
// While the AI is playing, a sustained mic-energy spike means the user is
|
|
25
|
+
// talking over it — cut playback locally and emit `onBargeIn` instead of
|
|
26
|
+
// waiting for the server's slower interrupt signal. VPIO (Apple AEC) keeps
|
|
27
|
+
// the mic near-silent during playback, so a spike is a clear user signal.
|
|
28
|
+
// `suppressPlayback` then drops the rest of the cancelled turn until JS lifts
|
|
29
|
+
// it at the next turn boundary (resumePlayback) or the safety cap elapses.
|
|
30
|
+
//
|
|
31
|
+
// These scalars are read/written from both the capture-tap thread and the
|
|
32
|
+
// bridge thread (playAudioChunk / resumePlayback / setBargeInEnabled), so all
|
|
33
|
+
// access goes through `stateLock`. (Android uses @Volatile for the same set.)
|
|
34
|
+
private let stateLock: os_unfair_lock_t = {
|
|
35
|
+
let lock = os_unfair_lock_t.allocate(capacity: 1)
|
|
36
|
+
lock.initialize(to: os_unfair_lock())
|
|
37
|
+
return lock
|
|
38
|
+
}()
|
|
39
|
+
private var lastPlaybackWriteMs: Double = 0
|
|
40
|
+
private var suppressPlayback = false
|
|
41
|
+
private var bargeInAtMs: Double = 0
|
|
42
|
+
// Gates client-side barge-in detection. JS disarms it when it detects the
|
|
43
|
+
// device's AEC is failing (a burst of barge-ins = the AI's own echo, not a
|
|
44
|
+
// user) and downgrades to half-duplex, so the echo can't keep cutting the AI.
|
|
45
|
+
private var bargeInEnabled = true
|
|
46
|
+
// Consecutive over-threshold reads; only touched on the capture-tap thread.
|
|
47
|
+
private var bargeInHighReads = 0
|
|
48
|
+
|
|
49
|
+
// Barge-in tuning. iOS mic RMS is normalized to [0,1] (sample / Int16.max),
|
|
50
|
+
// unlike Android's raw Int16 (its threshold 1200/32767 ≈ 0.035). Require a
|
|
51
|
+
// couple of consecutive reads to reject transient clicks; only armed while the
|
|
52
|
+
// AI is actively playing. Tune on device.
|
|
53
|
+
private static let bargeInRmsThreshold: Float = 0.035
|
|
54
|
+
private static let bargeInSustainReads = 2
|
|
55
|
+
private static let playbackActiveWindowMs: Double = 350
|
|
56
|
+
// Safety cap: if no turn-end signal lifts suppression, resume anyway so a
|
|
57
|
+
// missed event can't permanently mute the next turn.
|
|
58
|
+
private static let suppressMaxMs: Double = 6000
|
|
59
|
+
|
|
60
|
+
@inline(__always)
|
|
61
|
+
private func withStateLock<T>(_ body: () -> T) -> T {
|
|
62
|
+
os_unfair_lock_lock(stateLock)
|
|
63
|
+
defer { os_unfair_lock_unlock(stateLock) }
|
|
64
|
+
return body()
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// Monotonic milliseconds for the playback-active window and suppression cap
|
|
68
|
+
// (wall-clock jumps must not affect timing). Deltas only.
|
|
69
|
+
@inline(__always)
|
|
70
|
+
private func nowMs() -> Double {
|
|
71
|
+
return Double(DispatchTime.now().uptimeNanoseconds) / 1_000_000.0
|
|
72
|
+
}
|
|
73
|
+
|
|
21
74
|
// MARK: - RCTEventEmitter overrides
|
|
22
75
|
|
|
23
76
|
override static func requiresMainQueueSetup() -> Bool {
|
|
@@ -25,7 +78,7 @@ class QafkaAudio: RCTEventEmitter {
|
|
|
25
78
|
}
|
|
26
79
|
|
|
27
80
|
override func supportedEvents() -> [String]! {
|
|
28
|
-
return ["onAudioData", "onAmplitude"]
|
|
81
|
+
return ["onAudioData", "onAmplitude", "onBargeIn"]
|
|
29
82
|
}
|
|
30
83
|
|
|
31
84
|
override func startObserving() {
|
|
@@ -106,6 +159,16 @@ class QafkaAudio: RCTEventEmitter {
|
|
|
106
159
|
) {
|
|
107
160
|
qafkaLog("[QafkaAudio] === doStartCapture begin ===")
|
|
108
161
|
|
|
162
|
+
// Fresh session starts full-duplex (barge-in armed) with no suppression;
|
|
163
|
+
// JS downgrades to half-duplex later if it detects the AEC is failing.
|
|
164
|
+
withStateLock {
|
|
165
|
+
bargeInEnabled = true
|
|
166
|
+
suppressPlayback = false
|
|
167
|
+
lastPlaybackWriteMs = 0
|
|
168
|
+
bargeInAtMs = 0
|
|
169
|
+
}
|
|
170
|
+
bargeInHighReads = 0
|
|
171
|
+
|
|
109
172
|
// Defensive startup cleanup — recover from any leaked VPIO/audio-unit state
|
|
110
173
|
// left by a prior crashed session. Apple Forums 721535 confirms the combo
|
|
111
174
|
// of (deactivate) + (force category) + (output-port toggle) is needed to
|
|
@@ -215,6 +278,29 @@ class QafkaAudio: RCTEventEmitter {
|
|
|
215
278
|
let rms = sqrt(sumSquares / Float(sampleCount))
|
|
216
279
|
let level = min(rms * 3.0, 1.0)
|
|
217
280
|
self.sendEvent(withName: "onAmplitude", body: ["level": level, "source": "mic"])
|
|
281
|
+
|
|
282
|
+
// Client-side barge-in: while the AI is actively playing, a sustained
|
|
283
|
+
// mic-energy spike means the user is talking over it — cut locally
|
|
284
|
+
// instead of waiting for the server's slower interrupt signal.
|
|
285
|
+
// Reuse the raw normalized `rms` (not the *3 display level).
|
|
286
|
+
let now = self.nowMs()
|
|
287
|
+
let (suppress, enabled, lastWrite) = self.withStateLock {
|
|
288
|
+
(self.suppressPlayback, self.bargeInEnabled, self.lastPlaybackWriteMs)
|
|
289
|
+
}
|
|
290
|
+
let playbackActive = (now - lastWrite) < QafkaAudio.playbackActiveWindowMs
|
|
291
|
+
if playbackActive && !suppress && enabled {
|
|
292
|
+
if rms > QafkaAudio.bargeInRmsThreshold {
|
|
293
|
+
self.bargeInHighReads += 1
|
|
294
|
+
if self.bargeInHighReads >= QafkaAudio.bargeInSustainReads {
|
|
295
|
+
self.bargeInHighReads = 0
|
|
296
|
+
self.triggerBargeIn()
|
|
297
|
+
}
|
|
298
|
+
} else {
|
|
299
|
+
self.bargeInHighReads = 0
|
|
300
|
+
}
|
|
301
|
+
} else {
|
|
302
|
+
self.bargeInHighReads = 0
|
|
303
|
+
}
|
|
218
304
|
}
|
|
219
305
|
|
|
220
306
|
let base64String = audioData.base64EncodedString()
|
|
@@ -394,6 +480,8 @@ class QafkaAudio: RCTEventEmitter {
|
|
|
394
480
|
engine.reset()
|
|
395
481
|
try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation)
|
|
396
482
|
}
|
|
483
|
+
stateLock.deinitialize(count: 1)
|
|
484
|
+
stateLock.deallocate()
|
|
397
485
|
}
|
|
398
486
|
|
|
399
487
|
// MARK: - Playback
|
|
@@ -406,6 +494,30 @@ class QafkaAudio: RCTEventEmitter {
|
|
|
406
494
|
playChunkCount += 1
|
|
407
495
|
let isLoggedChunk = playChunkCount <= 3 || playChunkCount % 50 == 0
|
|
408
496
|
|
|
497
|
+
// Drop the AI's remaining audio after a local barge-in until the turn
|
|
498
|
+
// boundary lifts suppression (or the safety cap elapses). Without this the
|
|
499
|
+
// in-flight deltas of the just-cancelled turn would refill the player and
|
|
500
|
+
// the AI would audibly resume right after we cut it.
|
|
501
|
+
let now = nowMs()
|
|
502
|
+
var autoLifted = false
|
|
503
|
+
let drop: Bool = withStateLock {
|
|
504
|
+
guard suppressPlayback else { return false }
|
|
505
|
+
if now - bargeInAtMs > QafkaAudio.suppressMaxMs {
|
|
506
|
+
suppressPlayback = false
|
|
507
|
+
autoLifted = true
|
|
508
|
+
return false
|
|
509
|
+
}
|
|
510
|
+
return true
|
|
511
|
+
}
|
|
512
|
+
if autoLifted {
|
|
513
|
+
qafkaLog("[QafkaAudio] ⏱️ playAudioChunk #\(playChunkCount): suppression auto-lifted (6s safety cap)")
|
|
514
|
+
}
|
|
515
|
+
if drop {
|
|
516
|
+
qafkaLog("[QafkaAudio] 🚫 playAudioChunk #\(playChunkCount): dropped — barge-in suppression active")
|
|
517
|
+
resolve(true)
|
|
518
|
+
return
|
|
519
|
+
}
|
|
520
|
+
|
|
409
521
|
guard let player = playerNode, let engine = audioEngine else {
|
|
410
522
|
qafkaLog("[QafkaAudio] ❌ playAudioChunk #\(playChunkCount): engine not running")
|
|
411
523
|
reject("NOT_RUNNING", "Audio engine is not running. Call startCapture first.", nil)
|
|
@@ -493,6 +605,9 @@ class QafkaAudio: RCTEventEmitter {
|
|
|
493
605
|
}
|
|
494
606
|
|
|
495
607
|
player.scheduleBuffer(outputBuffer, completionHandler: nil)
|
|
608
|
+
// Mark the playback as active so the capture tap's barge-in detector only
|
|
609
|
+
// arms while the AI is genuinely streaming audio (see playbackActiveWindowMs).
|
|
610
|
+
withStateLock { lastPlaybackWriteMs = nowMs() }
|
|
496
611
|
if isLoggedChunk {
|
|
497
612
|
qafkaLog("[QafkaAudio] 🔊 playAudioChunk #\(playChunkCount): scheduled \(outputBuffer.frameLength)f @ \(mixerFormat.sampleRate)Hz playerPlaying=\(player.isPlaying) engineRunning=\(engine.isRunning)")
|
|
498
613
|
}
|
|
@@ -516,4 +631,59 @@ class QafkaAudio: RCTEventEmitter {
|
|
|
516
631
|
|
|
517
632
|
resolve(true)
|
|
518
633
|
}
|
|
634
|
+
|
|
635
|
+
// MARK: - Barge-in
|
|
636
|
+
|
|
637
|
+
/// Called from the capture tap when a sustained mic-energy spike is detected
|
|
638
|
+
/// while the AI is playing. Cuts playback instantly (purging scheduled
|
|
639
|
+
/// buffers) and starts dropping the cancelled turn's remaining audio; JS lifts
|
|
640
|
+
/// suppression at the next turn boundary via resumePlayback().
|
|
641
|
+
private func triggerBargeIn() {
|
|
642
|
+
withStateLock {
|
|
643
|
+
suppressPlayback = true
|
|
644
|
+
bargeInAtMs = nowMs()
|
|
645
|
+
}
|
|
646
|
+
// Tell JS immediately (UI → listening). sendEvent from the tap thread is
|
|
647
|
+
// already used above for onAudioData/onAmplitude.
|
|
648
|
+
sendEvent(withName: "onBargeIn", body: [String: Any]())
|
|
649
|
+
// Purge the cancelled turn's queued buffers OFF the capture-tap thread.
|
|
650
|
+
// AVAudioPlayerNode.stop() called from the tap (audio IO) callback can stall
|
|
651
|
+
// the IO thread (sluggish cut + choppy mic) and leave the node not rendering
|
|
652
|
+
// afterwards (the next turn schedules buffers that never sound — "stuck in
|
|
653
|
+
// listening"). The suppress flag above already drops NEW audio synchronously,
|
|
654
|
+
// so the cut still feels instant; deferring only the buffer purge to the main
|
|
655
|
+
// queue (where the config-change handler also does engine ops) is safe. Same
|
|
656
|
+
// stop()+play() pattern as stopPlayback, just off the IO thread.
|
|
657
|
+
DispatchQueue.main.async { [weak self] in
|
|
658
|
+
guard let self = self else { return }
|
|
659
|
+
self.playerNode?.stop()
|
|
660
|
+
self.playerNode?.play()
|
|
661
|
+
qafkaLog("[QafkaAudio] ⚡ Local barge-in cut applied (playerPlaying=\(self.playerNode?.isPlaying ?? false) engineRunning=\(self.audioEngine?.isRunning ?? false))")
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
@objc func resumePlayback(
|
|
666
|
+
_ resolve: @escaping RCTPromiseResolveBlock,
|
|
667
|
+
rejecter reject: @escaping RCTPromiseRejectBlock
|
|
668
|
+
) {
|
|
669
|
+
withStateLock { suppressPlayback = false }
|
|
670
|
+
qafkaLog("[QafkaAudio] ▶️ resumePlayback — suppression lifted")
|
|
671
|
+
resolve(true)
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/// Arm/disarm client-side barge-in. JS disarms it after detecting the device's
|
|
675
|
+
/// echo cancellation is failing (downgrade to half-duplex), so the AI's own
|
|
676
|
+
/// leaked echo can't keep triggering a cut. Disarming also clears suppression.
|
|
677
|
+
@objc func setBargeInEnabled(
|
|
678
|
+
_ enabled: Bool,
|
|
679
|
+
resolver resolve: @escaping RCTPromiseResolveBlock,
|
|
680
|
+
rejecter reject: @escaping RCTPromiseRejectBlock
|
|
681
|
+
) {
|
|
682
|
+
withStateLock {
|
|
683
|
+
bargeInEnabled = enabled
|
|
684
|
+
if !enabled { suppressPlayback = false }
|
|
685
|
+
}
|
|
686
|
+
qafkaLog("[QafkaAudio] setBargeInEnabled=\(enabled)")
|
|
687
|
+
resolve(true)
|
|
688
|
+
}
|
|
519
689
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qafka/react-native",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.4.0",
|
|
4
4
|
"description": "Drop-in AI assistant for React Native: chat, voice, and tool execution with screen-aware navigation.",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
"react-native.config.js",
|
|
16
16
|
"qafka.config.js",
|
|
17
17
|
"qafka.config.d.ts",
|
|
18
|
-
"docs",
|
|
18
|
+
"docs/qafka-tool-authoring-guide.md",
|
|
19
19
|
"README.md",
|
|
20
20
|
"CHANGELOG.md",
|
|
21
21
|
"CONTRIBUTING.md",
|