@volley/recognition-client-sdk 0.1.423 → 0.1.621
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.bundled.d.ts +237 -7
- package/dist/index.bundled.d.ts +346 -10
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +248 -12
- package/dist/index.js.map +4 -4
- package/dist/recog-client-sdk.browser.js +236 -11
- package/dist/recog-client-sdk.browser.js.map +4 -4
- package/dist/recognition-client.d.ts +32 -1
- package/dist/recognition-client.d.ts.map +1 -1
- package/dist/recognition-client.types.d.ts +20 -0
- package/dist/recognition-client.types.d.ts.map +1 -1
- package/dist/simplified-vgf-recognition-client.d.ts +17 -0
- package/dist/simplified-vgf-recognition-client.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/index.ts +2 -0
- package/src/recognition-client.ts +160 -5
- package/src/recognition-client.types.ts +23 -0
- package/src/simplified-vgf-recognition-client.integration.spec.ts +15 -3
- package/src/simplified-vgf-recognition-client.ts +30 -3
- package/src/utils/audio-ring-buffer.spec.ts +335 -0
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
* ```
|
|
34
34
|
*/
|
|
35
35
|
import { WebSocketAudioClient } from '@recog/websocket';
|
|
36
|
-
import { type TranscriptionResultV1 } from '@recog/shared-types';
|
|
36
|
+
import { type TranscriptionResultV1, type GameContextV1 } from '@recog/shared-types';
|
|
37
37
|
import { ClientState } from './recognition-client.types.js';
|
|
38
38
|
import type { IRecognitionClient, IRecognitionClientStats, RealTimeTwoWayWebSocketRecognitionClientConfig } from './recognition-client.types.js';
|
|
39
39
|
/**
|
|
@@ -55,8 +55,11 @@ export type { RealTimeTwoWayWebSocketRecognitionClientConfig } from './recogniti
|
|
|
55
55
|
*/
|
|
56
56
|
export declare class RealTimeTwoWayWebSocketRecognitionClient extends WebSocketAudioClient<number, any, any> implements IRecognitionClient {
|
|
57
57
|
private static readonly PROTOCOL_VERSION;
|
|
58
|
+
private static readonly MAX_PREFIX_BUFFER_BYTES;
|
|
58
59
|
private config;
|
|
59
60
|
private audioBuffer;
|
|
61
|
+
private prefixBuffer;
|
|
62
|
+
private prefixBufferBytes;
|
|
60
63
|
private messageHandler;
|
|
61
64
|
private state;
|
|
62
65
|
private connectionPromise;
|
|
@@ -87,6 +90,10 @@ export declare class RealTimeTwoWayWebSocketRecognitionClient extends WebSocketA
|
|
|
87
90
|
private connectWithRetry;
|
|
88
91
|
sendAudio(audioData: ArrayBuffer | ArrayBufferView | Blob): void;
|
|
89
92
|
private sendAudioInternal;
|
|
93
|
+
/**
|
|
94
|
+
* Only active ehwne client is in READY state. otherwise it will return immediately.
|
|
95
|
+
* @returns Promise that resolves when the recording is stopped
|
|
96
|
+
*/
|
|
90
97
|
stopRecording(): Promise<void>;
|
|
91
98
|
stopAbnormally(): void;
|
|
92
99
|
getAudioUtteranceId(): string;
|
|
@@ -97,6 +104,8 @@ export declare class RealTimeTwoWayWebSocketRecognitionClient extends WebSocketA
|
|
|
97
104
|
isStopping(): boolean;
|
|
98
105
|
isTranscriptionFinished(): boolean;
|
|
99
106
|
isBufferOverflowing(): boolean;
|
|
107
|
+
isServerReady(): boolean;
|
|
108
|
+
sendGameContext(context: GameContextV1): void;
|
|
100
109
|
getStats(): IRecognitionClientStats;
|
|
101
110
|
protected onConnected(): void;
|
|
102
111
|
protected onDisconnected(code: number, reason: string): void;
|
|
@@ -120,5 +129,27 @@ export declare class RealTimeTwoWayWebSocketRecognitionClient extends WebSocketA
|
|
|
120
129
|
* @param audioData - Audio data to send
|
|
121
130
|
*/
|
|
122
131
|
private sendAudioNow;
|
|
132
|
+
/**
|
|
133
|
+
* Send prefix audio to the server.
|
|
134
|
+
* Prefix audio is sent before user audio and is used for context/priming.
|
|
135
|
+
* The server will process it but adjust timing so transcripts reflect user audio timing.
|
|
136
|
+
*
|
|
137
|
+
* Note: Prefix audio is buffered until READY state, then flushed before user audio.
|
|
138
|
+
* This ensures proper ordering even if called before server is ready.
|
|
139
|
+
*
|
|
140
|
+
* @param audioData - Prefix audio data (ArrayBuffer, ArrayBufferView, or Blob)
|
|
141
|
+
*/
|
|
142
|
+
sendPrefixAudio(audioData: ArrayBuffer | ArrayBufferView | Blob): void;
|
|
143
|
+
/**
|
|
144
|
+
* Internal method to handle prefix audio with buffering
|
|
145
|
+
* Buffers if not READY, sends immediately if READY
|
|
146
|
+
*/
|
|
147
|
+
private sendPrefixAudioInternal;
|
|
148
|
+
/**
|
|
149
|
+
* Send prefix audio immediately to the server (without buffering)
|
|
150
|
+
* Uses encoding offset to mark as prefix audio
|
|
151
|
+
* @param audioData - Prefix audio data to send
|
|
152
|
+
*/
|
|
153
|
+
private sendPrefixAudioNow;
|
|
123
154
|
}
|
|
124
155
|
//# sourceMappingURL=recognition-client.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recognition-client.d.ts","sourceRoot":"","sources":["../src/recognition-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAML,KAAK,qBAAqB,
|
|
1
|
+
{"version":3,"file":"recognition-client.d.ts","sourceRoot":"","sources":["../src/recognition-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAML,KAAK,qBAAqB,EAO1B,KAAK,aAAa,EAGnB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,KAAK,EACV,kBAAkB,EAClB,uBAAuB,EACvB,8CAA8C,EAE/C,MAAM,+BAA+B,CAAC;AAUvC;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAE3D;AAgCD;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,qBAAqB,CAAC;AAGxD,YAAY,EAAE,8CAA8C,EAAE,MAAM,+BAA+B,CAAC;AAgCpG;;;;;GAKG;AACH,qBAAa,wCACX,SAAQ,oBAAoB,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAC7C,YAAW,kBAAkB;IAE7B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAK;IAC7C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAAoB;IAEnE,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,YAAY,CAAyC;IAC7D,OAAO,CAAC,iBAAiB,CAAK;IAC9B,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,KAAK,CAAoC;IACjD,OAAO,CAAC,iBAAiB,CAA4B;IAGrD,OAAO,CAAC,iBAAiB,CAAS;IAGlC,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,qBAAqB,CAAO;IACpC,OAAO,CAAC,iBAAiB,CAAK;gBAElB,MAAM,EAAE,8CAA8C;IA+ElE;;;;;;OAMG;IACH,OAAO,CAAC,GAAG;IAWX;;;OAGG;IACH,OAAO,CAAC,OAAO;IAqBA,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IA6BvC;;;OAGG;YACW,gBAAgB;IAkIrB,SAAS,CAAC,SAAS,EAAE,WAAW,GAAG,eAAe,GAAG,IAAI,GAAG,IAAI;IAiBzE,OAAO,CAAC,iBAAiB;IAsCzB;;;OAGG;IAEG,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IAoCpC,cAAc,IAAI,IAAI;IAwBtB,mBAAmB,IAAI,MAAM;IAI7B,MAAM,IAAI,MAAM;IAIhB,QAAQ,IAAI,WAAW;IAIvB,WAAW,IAAI,OAAO;IAItB,YAAY,IAAI,OAAO;IAIvB,UAAU,IAAI,OAAO;IAIrB,uBAAuB,IAAI,OAAO;IAIlC,mBAAmB,IAAI,OAAO;IAI9B,aAAa,IAAI,OAAO;IAIxB,eAAe,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI;IAmB7C,QAAQ,IAAI,uBAAuB;IAgBnC,SAAS,CAAC,WAAW,IAAI,IAAI;IAiF7B,SAAS,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI;IA8C5D;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAwB/B,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;cAYlB,SAAS,CAAC,GAAG,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,GAAG,CAAA;KAAE,GAAG,IAAI;IAQ/E;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAiC5B;;;OAGG;IACH,OAAO,CAAC,YAAY;IAwBpB;;;;;;;;;OASG;IACH,eAAe,CAAC,SAAS,EAAE,WAAW,GAAG,eAAe,GAAG,IAAI,GAAG,IAAI;IAiBtE;;;OAGG;IACH,OAAO,CAAC,uBAAuB;IAiC/B;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;CA2B3B"}
|
|
@@ -243,6 +243,26 @@ export interface IRecognitionClient {
|
|
|
243
243
|
* @returns WebSocket URL string
|
|
244
244
|
*/
|
|
245
245
|
getUrl(): string;
|
|
246
|
+
/**
|
|
247
|
+
* Send game context after connection is established (for preconnect flow).
|
|
248
|
+
*
|
|
249
|
+
* Preconnect flow: Create client with asrRequestConfig (useContext: true) but
|
|
250
|
+
* WITHOUT gameContext → call connect() → WS opens, ASRRequest sent, server
|
|
251
|
+
* waits in PENDING_CONTEXT → later call sendGameContext() with slotMap →
|
|
252
|
+
* server attaches provider and sends READY.
|
|
253
|
+
*
|
|
254
|
+
* This enables connecting early (before slotMap is known) and sending
|
|
255
|
+
* game context later when question data is available.
|
|
256
|
+
*
|
|
257
|
+
* @param context - Game context including slotMap for keyword boosting
|
|
258
|
+
*/
|
|
259
|
+
sendGameContext(context: GameContextV1): void;
|
|
260
|
+
/**
|
|
261
|
+
* Check if server has sent READY signal (provider is connected and ready for audio).
|
|
262
|
+
* In preconnect flow, this becomes true after sendGameContext() triggers provider attachment.
|
|
263
|
+
* @returns true if server is ready to receive audio
|
|
264
|
+
*/
|
|
265
|
+
isServerReady(): boolean;
|
|
246
266
|
}
|
|
247
267
|
/**
|
|
248
268
|
* Client statistics interface
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recognition-client.types.d.ts","sourceRoot":"","sources":["../src/recognition-client.types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,qBAAqB,EACrB,oBAAoB,EACpB,gBAAgB,EAChB,aAAa,EACb,gBAAgB,EAChB,aAAa,EACb,KAAK,EACN,MAAM,qBAAqB,CAAC;AAE7B;;;GAGG;AACH,oBAAY,WAAW;IACrB,+CAA+C;IAC/C,OAAO,YAAY;IAEnB,iDAAiD;IACjD,UAAU,eAAe;IAEzB,8DAA8D;IAC9D,SAAS,cAAc;IAEvB,mCAAmC;IACnC,KAAK,UAAU;IAEf,qDAAqD;IACrD,QAAQ,aAAa;IAErB,4CAA4C;IAC5C,OAAO,YAAY;IAEnB,6CAA6C;IAC7C,MAAM,WAAW;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,gCAAgC;IAChC,GAAG,EAAE,MAAM,CAAC;IAEZ,yFAAyF;IACzF,YAAY,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;CACvC;AAGD,MAAM,MAAM,uBAAuB,GAAG,sBAAsB,CAAC;AAE7D,MAAM,WAAW,wBAAwB;IACvC;;;;;;;;;OASG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IAEvB,qEAAqE;IACrE,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IAEpC,qDAAqD;IACrD,WAAW,CAAC,EAAE,aAAa,CAAC;IAE5B;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,mFAAmF;IACnF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;OAEG;IACH,YAAY,CAAC,EAAE,sBAAsB,EAAE,CAAC;IAExC,qCAAqC;IACrC,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,6FAA6F;IAC7F,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,uCAAuC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,gGAAgG;IAChG,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B,6FAA6F;IAC7F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,2CAA2C;IAC3C,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAEvD;;;OAGG;IACH,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,oBAAoB,KAAK,IAAI,CAAC;IAExD,oFAAoF;IACpF,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAElD,iCAAiC;IACjC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;IAEzC,2CAA2C;IAC3C,WAAW,CAAC,EAAE,MAAM,IAAI,CAAC;IAEzB;;;;OAIG;IACH,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IAExD,uDAAuD;IACvD,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,sDAAsD;IACtD,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,wDAAwD;IACxD,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B,uEAAuE;IACvE,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;;;;;;;;;;;;;OAiBG;IACH,eAAe,CAAC,EAAE;QAChB,yEAAyE;QACzE,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,oEAAoE;QACpE,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;IAEF;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,GAAG,KAAK,IAAI,CAAC;CAC5F;AAED;;;;;GAKG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEzB;;;;OAIG;IACH,SAAS,CAAC,SAAS,EAAE,WAAW,GAAG,eAAe,GAAG,IAAI,GAAG,IAAI,CAAC;IAEjE;;;;OAIG;IACH,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/B;;;;;;;;;;;;;;;;OAgBG;IACH,cAAc,IAAI,IAAI,CAAC;IAEvB;;;;OAIG;IACH,mBAAmB,IAAI,MAAM,CAAC;IAE9B;;;OAGG;IACH,QAAQ,IAAI,WAAW,CAAC;IAExB;;;OAGG;IACH,WAAW,IAAI,OAAO,CAAC;IAEvB;;;OAGG;IACH,YAAY,IAAI,OAAO,CAAC;IAExB;;;OAGG;IACH,UAAU,IAAI,OAAO,CAAC;IAEtB;;;OAGG;IACH,uBAAuB,IAAI,OAAO,CAAC;IAEnC;;;OAGG;IACH,mBAAmB,IAAI,OAAO,CAAC;IAE/B;;;OAGG;IACH,QAAQ,IAAI,uBAAuB,CAAC;IAEpC;;;;OAIG;IACH,MAAM,IAAI,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"recognition-client.types.d.ts","sourceRoot":"","sources":["../src/recognition-client.types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,qBAAqB,EACrB,oBAAoB,EACpB,gBAAgB,EAChB,aAAa,EACb,gBAAgB,EAChB,aAAa,EACb,KAAK,EACN,MAAM,qBAAqB,CAAC;AAE7B;;;GAGG;AACH,oBAAY,WAAW;IACrB,+CAA+C;IAC/C,OAAO,YAAY;IAEnB,iDAAiD;IACjD,UAAU,eAAe;IAEzB,8DAA8D;IAC9D,SAAS,cAAc;IAEvB,mCAAmC;IACnC,KAAK,UAAU;IAEf,qDAAqD;IACrD,QAAQ,aAAa;IAErB,4CAA4C;IAC5C,OAAO,YAAY;IAEnB,6CAA6C;IAC7C,MAAM,WAAW;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,gCAAgC;IAChC,GAAG,EAAE,MAAM,CAAC;IAEZ,yFAAyF;IACzF,YAAY,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;CACvC;AAGD,MAAM,MAAM,uBAAuB,GAAG,sBAAsB,CAAC;AAE7D,MAAM,WAAW,wBAAwB;IACvC;;;;;;;;;OASG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IAEvB,qEAAqE;IACrE,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IAEpC,qDAAqD;IACrD,WAAW,CAAC,EAAE,aAAa,CAAC;IAE5B;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,mFAAmF;IACnF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;OAEG;IACH,YAAY,CAAC,EAAE,sBAAsB,EAAE,CAAC;IAExC,qCAAqC;IACrC,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB,6FAA6F;IAC7F,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,uCAAuC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,gGAAgG;IAChG,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B,6FAA6F;IAC7F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,2CAA2C;IAC3C,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAEvD;;;OAGG;IACH,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,oBAAoB,KAAK,IAAI,CAAC;IAExD,oFAAoF;IACpF,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAElD,iCAAiC;IACjC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;IAEzC,2CAA2C;IAC3C,WAAW,CAAC,EAAE,MAAM,IAAI,CAAC;IAEzB;;;;OAIG;IACH,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IAExD,uDAAuD;IACvD,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB,sDAAsD;IACtD,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,wDAAwD;IACxD,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAE9B,uEAAuE;IACvE,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;;;;;;;;;;;;;OAiBG;IACH,eAAe,CAAC,EAAE;QAChB,yEAAyE;QACzE,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,oEAAoE;QACpE,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;IAEF;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,GAAG,KAAK,IAAI,CAAC;CAC5F;AAED;;;;;GAKG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEzB;;;;OAIG;IACH,SAAS,CAAC,SAAS,EAAE,WAAW,GAAG,eAAe,GAAG,IAAI,GAAG,IAAI,CAAC;IAEjE;;;;OAIG;IACH,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/B;;;;;;;;;;;;;;;;OAgBG;IACH,cAAc,IAAI,IAAI,CAAC;IAEvB;;;;OAIG;IACH,mBAAmB,IAAI,MAAM,CAAC;IAE9B;;;OAGG;IACH,QAAQ,IAAI,WAAW,CAAC;IAExB;;;OAGG;IACH,WAAW,IAAI,OAAO,CAAC;IAEvB;;;OAGG;IACH,YAAY,IAAI,OAAO,CAAC;IAExB;;;OAGG;IACH,UAAU,IAAI,OAAO,CAAC;IAEtB;;;OAGG;IACH,uBAAuB,IAAI,OAAO,CAAC;IAEnC;;;OAGG;IACH,mBAAmB,IAAI,OAAO,CAAC;IAE/B;;;OAGG;IACH,QAAQ,IAAI,uBAAuB,CAAC;IAEpC;;;;OAIG;IACH,MAAM,IAAI,MAAM,CAAC;IAEjB;;;;;;;;;;;;OAYG;IACH,eAAe,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI,CAAC;IAE9C;;;;OAIG;IACH,aAAa,IAAI,OAAO,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,uCAAuC;IACvC,cAAc,EAAE,MAAM,CAAC;IAEvB,wCAAwC;IACxC,eAAe,EAAE,MAAM,CAAC;IAExB,4CAA4C;IAC5C,mBAAmB,EAAE,MAAM,CAAC;IAE5B,iDAAiD;IACjD,mBAAmB,EAAE,MAAM,CAAC;IAE5B,yCAAyC;IACzC,qBAAqB,EAAE,MAAM,CAAC;IAE9B,iEAAiE;IACjE,UAAU,EAAE,OAAO,CAAC;CACrB;AAED;;;;GAIG;AAEH,MAAM,WAAW,8CAA+C,SAAQ,wBAAwB;CAG/F"}
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { RecognitionState } from './vgf-recognition-state.js';
|
|
11
11
|
import { IRecognitionClientConfig, ClientState } from './recognition-client.types.js';
|
|
12
|
+
import { type GameContextV1 } from '@recog/shared-types';
|
|
12
13
|
/**
|
|
13
14
|
* Configuration for SimplifiedVGFRecognitionClient
|
|
14
15
|
*/
|
|
@@ -89,6 +90,20 @@ export interface ISimplifiedVGFRecognitionClient {
|
|
|
89
90
|
* Check if the audio buffer has overflowed
|
|
90
91
|
*/
|
|
91
92
|
isBufferOverflowing(): boolean;
|
|
93
|
+
/**
|
|
94
|
+
* Send game context after connection is established (for preconnect flow).
|
|
95
|
+
*
|
|
96
|
+
* Preconnect flow: Create client with asrRequestConfig (useContext: true) but
|
|
97
|
+
* WITHOUT gameContext → call connect() → later call sendGameContext() with slotMap.
|
|
98
|
+
*
|
|
99
|
+
* @param context - Game context including slotMap for keyword boosting
|
|
100
|
+
*/
|
|
101
|
+
sendGameContext(context: GameContextV1): void;
|
|
102
|
+
/**
|
|
103
|
+
* Check if server has sent READY signal (provider connected, ready for audio).
|
|
104
|
+
* In preconnect flow, this becomes true after sendGameContext() triggers provider attachment.
|
|
105
|
+
*/
|
|
106
|
+
isServerReady(): boolean;
|
|
92
107
|
/**
|
|
93
108
|
* Get the audio utterance ID for this session
|
|
94
109
|
*/
|
|
@@ -127,6 +142,8 @@ export declare class SimplifiedVGFRecognitionClient implements ISimplifiedVGFRec
|
|
|
127
142
|
isStopping(): boolean;
|
|
128
143
|
isTranscriptionFinished(): boolean;
|
|
129
144
|
isBufferOverflowing(): boolean;
|
|
145
|
+
sendGameContext(context: GameContextV1): void;
|
|
146
|
+
isServerReady(): boolean;
|
|
130
147
|
getVGFState(): RecognitionState;
|
|
131
148
|
private isTerminalStatus;
|
|
132
149
|
private notifyStateChange;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"simplified-vgf-recognition-client.d.ts","sourceRoot":"","sources":["../src/simplified-vgf-recognition-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACH,gBAAgB,EAInB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAEH,wBAAwB,EACxB,WAAW,EACd,MAAM,+BAA+B,CAAC;
|
|
1
|
+
{"version":3,"file":"simplified-vgf-recognition-client.d.ts","sourceRoot":"","sources":["../src/simplified-vgf-recognition-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACH,gBAAgB,EAInB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAEH,wBAAwB,EACxB,WAAW,EACd,MAAM,+BAA+B,CAAC;AASvC,OAAO,EAA4B,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEnF;;GAEG;AACH,MAAM,WAAW,yBAA0B,SAAQ,wBAAwB;IACvE;;;OAGG;IACH,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAElD;;;OAGG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACnC;AAED;;;;;GAKG;AACH,MAAM,WAAW,+BAA+B;IAE5C;;;OAGG;IACH,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEzB;;;OAGG;IACH,SAAS,CAAC,SAAS,EAAE,WAAW,GAAG,eAAe,GAAG,IAAI,GAAG,IAAI,CAAC;IAEjE;;;OAGG;IACH,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/B;;;;;;;;;;;;;;;;OAgBG;IACH,cAAc,IAAI,IAAI,CAAC;IAGvB;;;OAGG;IACH,WAAW,IAAI,gBAAgB,CAAC;IAGhC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC;IAEvB;;OAEG;IACH,YAAY,IAAI,OAAO,CAAC;IAExB;;OAEG;IACH,UAAU,IAAI,OAAO,CAAC;IAEtB;;OAEG;IACH,uBAAuB,IAAI,OAAO,CAAC;IAEnC;;OAEG;IACH,mBAAmB,IAAI,OAAO,CAAC;IAG/B;;;;;;;OAOG;IACH,eAAe,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI,CAAC;IAE9C;;;OAGG;IACH,aAAa,IAAI,OAAO,CAAC;IAGzB;;OAEG;IACH,mBAAmB,IAAI,MAAM,CAAC;IAE9B;;OAEG;IACH,MAAM,IAAI,MAAM,CAAC;IAEjB;;OAEG;IACH,QAAQ,IAAI,WAAW,CAAC;CAE3B;AAED;;;GAGG;AACH,qBAAa,8BAA+B,YAAW,+BAA+B;IAClF,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,KAAK,CAAmB;IAChC,OAAO,CAAC,gBAAgB,CAAkB;IAC1C,OAAO,CAAC,mBAAmB,CAAkD;IAC7E,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,MAAM,CAAqC;IACnD,OAAO,CAAC,oBAAoB,CAAuB;gBAEvC,MAAM,EAAE,yBAAyB;IAqKvC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAK9B,SAAS,CAAC,SAAS,EAAE,WAAW,GAAG,eAAe,GAAG,IAAI,GAAG,IAAI;IAc1D,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IA4BpC,cAAc,IAAI,IAAI;IAiCtB,mBAAmB,IAAI,MAAM;IAI7B,MAAM,IAAI,MAAM;IAIhB,QAAQ,IAAI,WAAW;IAIvB,WAAW,IAAI,OAAO;IAItB,YAAY,IAAI,OAAO;IAIvB,UAAU,IAAI,OAAO;IAIrB,uBAAuB,IAAI,OAAO;IAIlC,mBAAmB,IAAI,OAAO;IAI9B,eAAe,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI;IAI7C,aAAa,IAAI,OAAO;IAMxB,WAAW,IAAI,gBAAgB;IAI/B,OAAO,CAAC,gBAAgB;IAMxB,OAAO,CAAC,iBAAiB;CA8B5B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,yBAAyB,GAAG,+BAA+B,CAE5G"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@volley/recognition-client-sdk",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.621",
|
|
4
4
|
"description": "Recognition Service TypeScript/Node.js Client SDK",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -43,21 +43,21 @@
|
|
|
43
43
|
"@semantic-release/github": "12.0.0",
|
|
44
44
|
"@semantic-release/npm": "13.1.1",
|
|
45
45
|
"@semantic-release/release-notes-generator": "14.1.0",
|
|
46
|
-
"@types/jest": "
|
|
47
|
-
"@types/node": "20.
|
|
46
|
+
"@types/jest": "29.5.14",
|
|
47
|
+
"@types/node": "20.11.5",
|
|
48
48
|
"@types/uuid": "10.0.0",
|
|
49
49
|
"@types/ws": "8.5.5",
|
|
50
50
|
"esbuild": "0.25.0",
|
|
51
|
-
"jest": "29.
|
|
51
|
+
"jest": "29.7.0",
|
|
52
52
|
"rollup": "4.52.5",
|
|
53
53
|
"rollup-plugin-dts": "6.2.3",
|
|
54
54
|
"semantic-release": "25.0.1",
|
|
55
|
-
"ts-jest": "29.
|
|
55
|
+
"ts-jest": "29.2.5",
|
|
56
56
|
"typescript": "5.1.6",
|
|
57
57
|
"@recog/shared-config": "1.0.0",
|
|
58
|
-
"@recog/shared-types": "1.0.0",
|
|
59
58
|
"@recog/shared-utils": "1.0.0",
|
|
60
|
-
"@recog/websocket": "1.0.0"
|
|
59
|
+
"@recog/websocket": "1.0.0",
|
|
60
|
+
"@recog/shared-types": "1.0.0"
|
|
61
61
|
},
|
|
62
62
|
"keywords": [
|
|
63
63
|
"recognition",
|
package/src/index.ts
CHANGED
|
@@ -48,7 +48,8 @@ import {
|
|
|
48
48
|
type ASRRequestConfig,
|
|
49
49
|
type ASRRequestV1,
|
|
50
50
|
type GameContextV1,
|
|
51
|
-
SampleRate
|
|
51
|
+
SampleRate,
|
|
52
|
+
PREFIX_AUDIO_ENCODING_OFFSET,
|
|
52
53
|
} from '@recog/shared-types';
|
|
53
54
|
import { v4 as uuidv4 } from 'uuid';
|
|
54
55
|
import { ClientState } from './recognition-client.types.js';
|
|
@@ -155,9 +156,12 @@ export class RealTimeTwoWayWebSocketRecognitionClient
|
|
|
155
156
|
implements IRecognitionClient
|
|
156
157
|
{
|
|
157
158
|
private static readonly PROTOCOL_VERSION = 1;
|
|
159
|
+
private static readonly MAX_PREFIX_BUFFER_BYTES = 10 * 1024 * 1024; // 10MB max (~5+ minutes of audio)
|
|
158
160
|
|
|
159
161
|
private config: InternalConfig;
|
|
160
162
|
private audioBuffer: AudioRingBuffer;
|
|
163
|
+
private prefixBuffer: (ArrayBuffer | ArrayBufferView)[] = []; // Buffer prefix audio until READY
|
|
164
|
+
private prefixBufferBytes = 0; // Track total bytes in prefix buffer
|
|
161
165
|
private messageHandler: MessageHandler;
|
|
162
166
|
private state: ClientState = ClientState.INITIAL;
|
|
163
167
|
private connectionPromise: Promise<void> | undefined;
|
|
@@ -275,8 +279,10 @@ export class RealTimeTwoWayWebSocketRecognitionClient
|
|
|
275
279
|
private cleanup(): void {
|
|
276
280
|
this.log('debug', 'Cleaning up resources');
|
|
277
281
|
|
|
278
|
-
// Clear audio
|
|
282
|
+
// Clear audio buffers to free memory
|
|
279
283
|
this.audioBuffer.clear();
|
|
284
|
+
this.prefixBuffer = [];
|
|
285
|
+
this.prefixBufferBytes = 0;
|
|
280
286
|
|
|
281
287
|
// Reset stats
|
|
282
288
|
this.audioBytesSent = 0;
|
|
@@ -509,9 +515,14 @@ export class RealTimeTwoWayWebSocketRecognitionClient
|
|
|
509
515
|
}
|
|
510
516
|
}
|
|
511
517
|
|
|
518
|
+
/**
|
|
519
|
+
* Only active ehwne client is in READY state. otherwise it will return immediately.
|
|
520
|
+
* @returns Promise that resolves when the recording is stopped
|
|
521
|
+
*/
|
|
522
|
+
|
|
512
523
|
async stopRecording(): Promise<void> {
|
|
513
524
|
if (this.state !== ClientState.READY) {
|
|
514
|
-
this.log('
|
|
525
|
+
this.log('warn', 'stopRecording called but not in READY state', { state: this.state });
|
|
515
526
|
return;
|
|
516
527
|
}
|
|
517
528
|
|
|
@@ -601,6 +612,29 @@ export class RealTimeTwoWayWebSocketRecognitionClient
|
|
|
601
612
|
return this.audioBuffer.isOverflowing();
|
|
602
613
|
}
|
|
603
614
|
|
|
615
|
+
isServerReady(): boolean {
|
|
616
|
+
return this.state === ClientState.READY;
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
sendGameContext(context: GameContextV1): void {
|
|
620
|
+
if (this.state !== ClientState.CONNECTED && this.state !== ClientState.READY) {
|
|
621
|
+
this.log('warn', 'sendGameContext called in wrong state', { state: this.state });
|
|
622
|
+
return;
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
this.log('debug', 'Sending game context (deferred)', {
|
|
626
|
+
gameId: context.gameId,
|
|
627
|
+
gamePhase: context.gamePhase,
|
|
628
|
+
hasSlotMap: !!context.slotMap
|
|
629
|
+
});
|
|
630
|
+
|
|
631
|
+
super.sendMessage(
|
|
632
|
+
RealTimeTwoWayWebSocketRecognitionClient.PROTOCOL_VERSION,
|
|
633
|
+
'message',
|
|
634
|
+
context
|
|
635
|
+
);
|
|
636
|
+
}
|
|
637
|
+
|
|
604
638
|
getStats(): IRecognitionClientStats {
|
|
605
639
|
const bufferStats = this.audioBuffer.getStats();
|
|
606
640
|
return {
|
|
@@ -634,6 +668,9 @@ export class RealTimeTwoWayWebSocketRecognitionClient
|
|
|
634
668
|
this.log('debug', 'Sending ASR request', this.config.asrRequestConfig);
|
|
635
669
|
}
|
|
636
670
|
|
|
671
|
+
// Extract fallbackModels if present
|
|
672
|
+
const fallbackModels = (this.config.asrRequestConfig as any).fallbackModels;
|
|
673
|
+
|
|
637
674
|
const asrRequest: ASRRequestV1 = {
|
|
638
675
|
type: RecognitionContextTypeV1.ASR_REQUEST,
|
|
639
676
|
audioUtteranceId: this.config.audioUtteranceId,
|
|
@@ -655,7 +692,20 @@ export class RealTimeTwoWayWebSocketRecognitionClient
|
|
|
655
692
|
...(this.config.asrRequestConfig.finalTranscriptStability && {
|
|
656
693
|
finalTranscriptStability: this.config.asrRequestConfig.finalTranscriptStability
|
|
657
694
|
}),
|
|
658
|
-
|
|
695
|
+
// Include fallbackModels if provided (for circuit breaker fallback)
|
|
696
|
+
...(fallbackModels && { fallbackModels }),
|
|
697
|
+
...(debugCommand && { debugCommand }),
|
|
698
|
+
// Include prefix mode if provided (for server-side stored prefix injection)
|
|
699
|
+
...(this.config.asrRequestConfig.prefixMode && {
|
|
700
|
+
prefixMode: this.config.asrRequestConfig.prefixMode
|
|
701
|
+
}),
|
|
702
|
+
...(this.config.asrRequestConfig.prefixId && {
|
|
703
|
+
prefixId: this.config.asrRequestConfig.prefixId
|
|
704
|
+
}),
|
|
705
|
+
// Include prefix text to remove if provided (for server-side prefix text removal)
|
|
706
|
+
...(this.config.asrRequestConfig.prefixTextToRemove && {
|
|
707
|
+
prefixTextToRemove: this.config.asrRequestConfig.prefixTextToRemove
|
|
708
|
+
})
|
|
659
709
|
};
|
|
660
710
|
|
|
661
711
|
super.sendMessage(
|
|
@@ -786,7 +836,15 @@ export class RealTimeTwoWayWebSocketRecognitionClient
|
|
|
786
836
|
this.state = ClientState.READY;
|
|
787
837
|
this.messageHandler.setSessionStartTime(Date.now());
|
|
788
838
|
|
|
789
|
-
// Flush buffered audio
|
|
839
|
+
// Flush buffered PREFIX audio FIRST (must be sent before user audio)
|
|
840
|
+
if (this.prefixBuffer.length > 0) {
|
|
841
|
+
this.log('debug', 'Flushing buffered prefix audio', { chunks: this.prefixBuffer.length });
|
|
842
|
+
this.prefixBuffer.forEach((chunk) => this.sendPrefixAudioNow(chunk));
|
|
843
|
+
this.prefixBuffer = [];
|
|
844
|
+
this.prefixBufferBytes = 0;
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
// Then flush buffered user audio
|
|
790
848
|
const bufferedChunks = this.audioBuffer.flush();
|
|
791
849
|
if (bufferedChunks.length > 0) {
|
|
792
850
|
this.log('debug', 'Flushing buffered audio', { chunks: bufferedChunks.length });
|
|
@@ -831,4 +889,101 @@ export class RealTimeTwoWayWebSocketRecognitionClient
|
|
|
831
889
|
this.audioBytesSent += byteLength;
|
|
832
890
|
this.audioChunksSent++;
|
|
833
891
|
}
|
|
892
|
+
|
|
893
|
+
/**
|
|
894
|
+
* Send prefix audio to the server.
|
|
895
|
+
* Prefix audio is sent before user audio and is used for context/priming.
|
|
896
|
+
* The server will process it but adjust timing so transcripts reflect user audio timing.
|
|
897
|
+
*
|
|
898
|
+
* Note: Prefix audio is buffered until READY state, then flushed before user audio.
|
|
899
|
+
* This ensures proper ordering even if called before server is ready.
|
|
900
|
+
*
|
|
901
|
+
* @param audioData - Prefix audio data (ArrayBuffer, ArrayBufferView, or Blob)
|
|
902
|
+
*/
|
|
903
|
+
sendPrefixAudio(audioData: ArrayBuffer | ArrayBufferView | Blob): void {
|
|
904
|
+
// Handle Blob by converting to ArrayBuffer asynchronously
|
|
905
|
+
if (audioData instanceof Blob) {
|
|
906
|
+
blobToArrayBuffer(audioData)
|
|
907
|
+
.then((arrayBuffer) => {
|
|
908
|
+
this.sendPrefixAudioInternal(arrayBuffer);
|
|
909
|
+
})
|
|
910
|
+
.catch((error) => {
|
|
911
|
+
this.log('error', 'Failed to convert Blob to ArrayBuffer for prefix audio', error);
|
|
912
|
+
});
|
|
913
|
+
return;
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
// Handle ArrayBuffer and ArrayBufferView synchronously
|
|
917
|
+
this.sendPrefixAudioInternal(audioData);
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
/**
|
|
921
|
+
* Internal method to handle prefix audio with buffering
|
|
922
|
+
* Buffers if not READY, sends immediately if READY
|
|
923
|
+
*/
|
|
924
|
+
private sendPrefixAudioInternal(audioData: ArrayBuffer | ArrayBufferView): void {
|
|
925
|
+
const bytes = ArrayBuffer.isView(audioData) ? audioData.byteLength : audioData.byteLength;
|
|
926
|
+
if (bytes === 0) return;
|
|
927
|
+
|
|
928
|
+
// Guard against stale async callbacks after cleanup (e.g., Blob conversion completing after disconnect)
|
|
929
|
+
if (this.state === ClientState.STOPPED || this.state === ClientState.FAILED) {
|
|
930
|
+
this.log('debug', 'Ignoring prefix audio in terminal state', { bytes, state: this.state });
|
|
931
|
+
return;
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
// Buffer if not ready, send immediately if ready
|
|
935
|
+
if (this.state === ClientState.READY) {
|
|
936
|
+
this.log('debug', 'Sending prefix audio immediately', { bytes });
|
|
937
|
+
this.sendPrefixAudioNow(audioData);
|
|
938
|
+
} else {
|
|
939
|
+
// Check buffer size limit to prevent OOM
|
|
940
|
+
if (
|
|
941
|
+
this.prefixBufferBytes + bytes >
|
|
942
|
+
RealTimeTwoWayWebSocketRecognitionClient.MAX_PREFIX_BUFFER_BYTES
|
|
943
|
+
) {
|
|
944
|
+
this.log('warn', 'Prefix buffer limit exceeded, dropping chunk', {
|
|
945
|
+
bytes,
|
|
946
|
+
current: this.prefixBufferBytes,
|
|
947
|
+
max: RealTimeTwoWayWebSocketRecognitionClient.MAX_PREFIX_BUFFER_BYTES,
|
|
948
|
+
});
|
|
949
|
+
return;
|
|
950
|
+
}
|
|
951
|
+
this.log('debug', 'Buffering prefix audio until READY', { bytes, state: this.state });
|
|
952
|
+
this.prefixBuffer.push(audioData);
|
|
953
|
+
this.prefixBufferBytes += bytes;
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
/**
|
|
958
|
+
* Send prefix audio immediately to the server (without buffering)
|
|
959
|
+
* Uses encoding offset to mark as prefix audio
|
|
960
|
+
* @param audioData - Prefix audio data to send
|
|
961
|
+
*/
|
|
962
|
+
private sendPrefixAudioNow(audioData: ArrayBuffer | ArrayBufferView): void {
|
|
963
|
+
const byteLength = ArrayBuffer.isView(audioData)
|
|
964
|
+
? audioData.byteLength
|
|
965
|
+
: audioData.byteLength;
|
|
966
|
+
|
|
967
|
+
if (byteLength === 0) return;
|
|
968
|
+
|
|
969
|
+
const baseEncodingId = (this.config.asrRequestConfig?.encoding ||
|
|
970
|
+
AudioEncoding.LINEAR16) as AudioEncoding;
|
|
971
|
+
|
|
972
|
+
// Add offset to mark as prefix audio
|
|
973
|
+
const prefixEncodingId = baseEncodingId + PREFIX_AUDIO_ENCODING_OFFSET;
|
|
974
|
+
|
|
975
|
+
const sampleRate =
|
|
976
|
+
typeof this.config.asrRequestConfig?.sampleRate === 'number'
|
|
977
|
+
? this.config.asrRequestConfig.sampleRate
|
|
978
|
+
: SampleRate.RATE_16000;
|
|
979
|
+
|
|
980
|
+
this.log('debug', 'Sending prefix audio', { bytes: byteLength, encoding: prefixEncodingId });
|
|
981
|
+
|
|
982
|
+
super.sendAudio(
|
|
983
|
+
audioData,
|
|
984
|
+
RealTimeTwoWayWebSocketRecognitionClient.PROTOCOL_VERSION,
|
|
985
|
+
prefixEncodingId,
|
|
986
|
+
sampleRate
|
|
987
|
+
);
|
|
988
|
+
}
|
|
834
989
|
}
|
|
@@ -301,6 +301,28 @@ export interface IRecognitionClient {
|
|
|
301
301
|
* @returns WebSocket URL string
|
|
302
302
|
*/
|
|
303
303
|
getUrl(): string;
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* Send game context after connection is established (for preconnect flow).
|
|
307
|
+
*
|
|
308
|
+
* Preconnect flow: Create client with asrRequestConfig (useContext: true) but
|
|
309
|
+
* WITHOUT gameContext → call connect() → WS opens, ASRRequest sent, server
|
|
310
|
+
* waits in PENDING_CONTEXT → later call sendGameContext() with slotMap →
|
|
311
|
+
* server attaches provider and sends READY.
|
|
312
|
+
*
|
|
313
|
+
* This enables connecting early (before slotMap is known) and sending
|
|
314
|
+
* game context later when question data is available.
|
|
315
|
+
*
|
|
316
|
+
* @param context - Game context including slotMap for keyword boosting
|
|
317
|
+
*/
|
|
318
|
+
sendGameContext(context: GameContextV1): void;
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* Check if server has sent READY signal (provider is connected and ready for audio).
|
|
322
|
+
* In preconnect flow, this becomes true after sendGameContext() triggers provider attachment.
|
|
323
|
+
* @returns true if server is ready to receive audio
|
|
324
|
+
*/
|
|
325
|
+
isServerReady(): boolean;
|
|
304
326
|
}
|
|
305
327
|
|
|
306
328
|
/**
|
|
@@ -331,6 +353,7 @@ export interface IRecognitionClientStats {
|
|
|
331
353
|
* This extends IRecognitionClientConfig and is the main configuration interface
|
|
332
354
|
* for creating a new RealTimeTwoWayWebSocketRecognitionClient instance.
|
|
333
355
|
*/
|
|
356
|
+
// eslint-disable-next-line @typescript-eslint/no-empty-object-type -- Backward compatibility alias
|
|
334
357
|
export interface RealTimeTwoWayWebSocketRecognitionClientConfig extends IRecognitionClientConfig {
|
|
335
358
|
// All fields are inherited from IRecognitionClientConfig
|
|
336
359
|
// This interface exists for backward compatibility and clarity
|
|
@@ -19,8 +19,12 @@ import {
|
|
|
19
19
|
TranscriptionStatus,
|
|
20
20
|
RecordingStatus
|
|
21
21
|
} from './vgf-recognition-state.js';
|
|
22
|
+
import { ClientState } from './recognition-client.types.js';
|
|
22
23
|
import { AudioEncoding } from '@recog/shared-types';
|
|
23
24
|
|
|
25
|
+
// Track the current mock state - can be changed per test
|
|
26
|
+
let mockClientState: ClientState = ClientState.READY;
|
|
27
|
+
|
|
24
28
|
// Mock the underlying client
|
|
25
29
|
jest.mock('./recognition-client.js', () => {
|
|
26
30
|
const mockClient = {
|
|
@@ -30,7 +34,7 @@ jest.mock('./recognition-client.js', () => {
|
|
|
30
34
|
stopAbnormally: jest.fn(),
|
|
31
35
|
getAudioUtteranceId: jest.fn().mockReturnValue('mock-uuid'),
|
|
32
36
|
getUrl: jest.fn().mockReturnValue('wss://mock-url'),
|
|
33
|
-
getState: jest.fn().
|
|
37
|
+
getState: jest.fn().mockImplementation(() => mockClientState),
|
|
34
38
|
isConnected: jest.fn().mockReturnValue(true),
|
|
35
39
|
isConnecting: jest.fn().mockReturnValue(false),
|
|
36
40
|
isStopping: jest.fn().mockReturnValue(false),
|
|
@@ -46,6 +50,8 @@ jest.mock('./recognition-client.js', () => {
|
|
|
46
50
|
describe('SimplifiedVGFRecognitionClient Integration - State Transitions', () => {
|
|
47
51
|
beforeEach(() => {
|
|
48
52
|
jest.clearAllMocks();
|
|
53
|
+
// Reset to default READY state
|
|
54
|
+
mockClientState = ClientState.READY;
|
|
49
55
|
});
|
|
50
56
|
|
|
51
57
|
describe('Normal Recognition Flow', () => {
|
|
@@ -238,6 +244,9 @@ describe('SimplifiedVGFRecognitionClient Integration - State Transitions', () =>
|
|
|
238
244
|
|
|
239
245
|
describe('Early Termination with Synthetic Finalization', () => {
|
|
240
246
|
it('should handle stopRecording without ever sending audio', async () => {
|
|
247
|
+
// Mock client in CONNECTED state (not yet READY) - synthetic finalization should trigger
|
|
248
|
+
mockClientState = ClientState.CONNECTED;
|
|
249
|
+
|
|
241
250
|
let stateChangeCounter = 0;
|
|
242
251
|
|
|
243
252
|
const trackingCallback = jest.fn((state: RecognitionState) => {
|
|
@@ -264,7 +273,7 @@ describe('SimplifiedVGFRecognitionClient Integration - State Transitions', () =>
|
|
|
264
273
|
expect(currentState.transcriptionStatus).toBe(TranscriptionStatus.NOT_STARTED);
|
|
265
274
|
|
|
266
275
|
// Step 2: User calls stopRecording WITHOUT ever sending audio
|
|
267
|
-
// SDK should emit synthetic finalization
|
|
276
|
+
// Since client is in CONNECTED state (not READY), SDK should emit synthetic finalization
|
|
268
277
|
await client.stopRecording();
|
|
269
278
|
|
|
270
279
|
// Should have 2 callbacks: stopRecording (FINISHED) + synthetic finalization (FINALIZED)
|
|
@@ -282,6 +291,9 @@ describe('SimplifiedVGFRecognitionClient Integration - State Transitions', () =>
|
|
|
282
291
|
});
|
|
283
292
|
|
|
284
293
|
it('should emit synthetic finalization and suppress late server transcripts', async () => {
|
|
294
|
+
// Mock client in CONNECTED state (not yet READY) - synthetic finalization should trigger
|
|
295
|
+
mockClientState = ClientState.CONNECTED;
|
|
296
|
+
|
|
285
297
|
let stateChangeCounter = 0;
|
|
286
298
|
const stateHistory: RecognitionState[] = [];
|
|
287
299
|
const loggerCalls: Array<{ level: string; message: string }> = [];
|
|
@@ -319,7 +331,7 @@ describe('SimplifiedVGFRecognitionClient Integration - State Transitions', () =>
|
|
|
319
331
|
expect(stateChangeCounter).toBe(1);
|
|
320
332
|
|
|
321
333
|
// Step 2: User calls stopRecording BEFORE any transcript received
|
|
322
|
-
//
|
|
334
|
+
// Since client is in CONNECTED state (not READY), SDK should emit synthetic finalization
|
|
323
335
|
await client.stopRecording();
|
|
324
336
|
|
|
325
337
|
// Should have 3 callbacks: sendAudio, stopRecording (FINISHED), synthetic finalization (FINALIZED)
|
|
@@ -27,7 +27,7 @@ import {
|
|
|
27
27
|
updateStateOnStop,
|
|
28
28
|
resetRecognitionVGFState
|
|
29
29
|
} from './vgf-recognition-mapper.js';
|
|
30
|
-
import { RecognitionContextTypeV1 } from '@recog/shared-types';
|
|
30
|
+
import { RecognitionContextTypeV1, type GameContextV1 } from '@recog/shared-types';
|
|
31
31
|
|
|
32
32
|
/**
|
|
33
33
|
* Configuration for SimplifiedVGFRecognitionClient
|
|
@@ -124,6 +124,23 @@ export interface ISimplifiedVGFRecognitionClient {
|
|
|
124
124
|
*/
|
|
125
125
|
isBufferOverflowing(): boolean;
|
|
126
126
|
|
|
127
|
+
// ============= Preconnect Methods =============
|
|
128
|
+
/**
|
|
129
|
+
* Send game context after connection is established (for preconnect flow).
|
|
130
|
+
*
|
|
131
|
+
* Preconnect flow: Create client with asrRequestConfig (useContext: true) but
|
|
132
|
+
* WITHOUT gameContext → call connect() → later call sendGameContext() with slotMap.
|
|
133
|
+
*
|
|
134
|
+
* @param context - Game context including slotMap for keyword boosting
|
|
135
|
+
*/
|
|
136
|
+
sendGameContext(context: GameContextV1): void;
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Check if server has sent READY signal (provider connected, ready for audio).
|
|
140
|
+
* In preconnect flow, this becomes true after sendGameContext() triggers provider attachment.
|
|
141
|
+
*/
|
|
142
|
+
isServerReady(): boolean;
|
|
143
|
+
|
|
127
144
|
// ============= Utility Methods =============
|
|
128
145
|
/**
|
|
129
146
|
* Get the audio utterance ID for this session
|
|
@@ -202,6 +219,8 @@ export class SimplifiedVGFRecognitionClient implements ISimplifiedVGFRecognition
|
|
|
202
219
|
} else {
|
|
203
220
|
// Initialize VGF state from config
|
|
204
221
|
this.state = createVGFStateFromConfig(clientConfig);
|
|
222
|
+
// Ensure clientConfig uses the same UUID as VGF state
|
|
223
|
+
clientConfig.audioUtteranceId = this.state.audioUtteranceId;
|
|
205
224
|
}
|
|
206
225
|
|
|
207
226
|
// Client is immediately ready to accept audio (will buffer if not connected)
|
|
@@ -342,9 +361,9 @@ export class SimplifiedVGFRecognitionClient implements ISimplifiedVGFRecognition
|
|
|
342
361
|
this.state = updateStateOnStop(this.state);
|
|
343
362
|
this.notifyStateChange();
|
|
344
363
|
|
|
345
|
-
// Early termination: If
|
|
364
|
+
// Early termination: If connection is not yet established with server, emit synthetic finalization immediately
|
|
346
365
|
// This prevents games from getting stuck waiting for a server response that may never come
|
|
347
|
-
if (this.
|
|
366
|
+
if (this.client.getState() === ClientState.CONNECTED || this.client.getState() === ClientState.CONNECTING) {
|
|
348
367
|
if (this.logger) {
|
|
349
368
|
this.logger('info',
|
|
350
369
|
`[RecogSDK:VGF] Early termination detected (transcriptionStatus: NOT_STARTED) - emitting synthetic finalization`
|
|
@@ -430,6 +449,14 @@ export class SimplifiedVGFRecognitionClient implements ISimplifiedVGFRecognition
|
|
|
430
449
|
return this.client.isBufferOverflowing();
|
|
431
450
|
}
|
|
432
451
|
|
|
452
|
+
sendGameContext(context: GameContextV1): void {
|
|
453
|
+
this.client.sendGameContext(context);
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
isServerReady(): boolean {
|
|
457
|
+
return this.client.isServerReady();
|
|
458
|
+
}
|
|
459
|
+
|
|
433
460
|
|
|
434
461
|
// VGF State access (read-only for consumers)
|
|
435
462
|
getVGFState(): RecognitionState {
|