@hoanghong/speaking-core 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -12
- package/README.md +54 -96
- package/dist/audio.d.ts +9 -0
- package/dist/audio.js +36 -0
- package/dist/batch.d.ts +1 -0
- package/dist/batch.js +1 -0
- package/dist/contracts.d.ts +10 -0
- package/dist/gateway.d.ts +2 -0
- package/dist/gateway.js +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/microphone.d.ts +3 -2
- package/dist/microphone.js +17 -1
- package/dist/pronunciation.d.ts +58 -0
- package/dist/pronunciation.js +52 -0
- package/dist/public-service.d.ts +9 -0
- package/dist/public-service.js +43 -0
- package/dist/realtime.d.ts +2 -1
- package/dist/realtime.js +1 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,12 +1,28 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
1
|
+
## 0.4.0
|
|
2
|
+
|
|
3
|
+
- Add US English pronunciation assessment client: 120-second audio, reference passage, optional IPA tokens and uncalibrated phoneme diagnostics.
|
|
4
|
+
- Share existing gateway/public application sessions; no service credentials in client apps.
|
|
5
|
+
- Local test release; requires server pronunciation endpoint and model provisioning.
|
|
6
|
+
|
|
7
|
+
# Changelog
|
|
8
|
+
|
|
9
|
+
## 0.3.0
|
|
10
|
+
|
|
11
|
+
- Per-application keys with server-managed quota; appKey setup without a service URL.
|
|
12
|
+
|
|
13
|
+
- Hosted public service configuration with automatic short-lived sessions and quota errors.
|
|
14
|
+
- Select realtime/upload using serviceType; no URL or token setup for hosted use.
|
|
15
|
+
- Completed callback includes text and audio: recorded PCM as WAV, or original uploaded file.
|
|
16
|
+
- Bounded five-minute recording buffer and preserved final residual audio.
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
## 0.2.0
|
|
20
|
+
|
|
21
|
+
- Add createSpeakingGateway for shared HTTP/WebSocket/ticket configuration.
|
|
22
|
+
- Validate gateway URLs and malformed ticket responses.
|
|
23
|
+
- Add complete microphone/upload examples and gateway/asset setup instructions.
|
|
24
|
+
- Preserve existing low-level 0.1 APIs.
|
|
25
|
+
|
|
26
|
+
## 0.1.0
|
|
27
|
+
|
|
28
|
+
- Initial batch, realtime and browser microphone SDK.
|
package/README.md
CHANGED
|
@@ -1,120 +1,78 @@
|
|
|
1
1
|
# @hoanghong/speaking-core
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
SDK online cho người tự làm UI. Bản test 0.4.0 cài bằng `npm install ./vendor/hoanghong-speaking-core-0.4.0.tgz` từ bộ ZIP, chưa publish registry. Angular dùng @hoanghong/speaking-angular với component có sẵn; Flutter dùng fsel_speaking. MIT.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
## Đánh giá phát âm tiếng Anh Mỹ
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
## 2. Chuẩn bị gateway và asset
|
|
16
|
-
|
|
17
|
-
Backend ứng dụng cung cấp prefix như `/api/speaking`, xác thực người dùng và relay giao thức FSEL ASR tới server. Chỉ backend giữ service token. Không nhúng khóa S3/service token vào bundle. Địa chỉ gateway quan sát được qua Network; không thể giấu URL mạng trong SDK. Package không kèm server ASR hoặc tài khoản miễn phí; server phải có weights cho ngôn ngữ được chọn.
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
Gateway forward `POST /v1/realtime-tickets`, `WS /v1/realtime`, `POST /v1/transcription-jobs`, `GET/DELETE /v1/transcription-jobs/{id}` sau khi bỏ prefix. Bảo vệ route bằng đăng nhập, quyền sở hữu job và quota; không mở relay công khai dùng service token.
|
|
21
|
-
|
|
22
|
-
Copy `node_modules/@hoanghong/speaking-core/assets/pcm-worklet.js` sang static asset `assets/speaking/pcm-worklet.js`. Với Angular thêm vào `angular.json` assets:
|
|
23
|
-
|
|
24
|
-
```json
|
|
25
|
-
{"glob":"pcm-worklet.js","input":"node_modules/@hoanghong/speaking-core/assets","output":"assets/speaking"}
|
|
26
|
-
```
|
|
27
|
-
|
|
28
|
-
Với bundler khác, dùng chức năng copy static assets tương ứng; URL phải trả JavaScript. Không dùng một HTML fallback làm worklet.
|
|
29
|
-
|
|
30
|
-
## 3. Ví dụ microphone đầy đủ
|
|
31
|
-
|
|
32
|
-
HTML của trang:
|
|
33
|
-
|
|
34
|
-
```html
|
|
35
|
-
<button id="start">Bắt đầu</button>
|
|
36
|
-
<button id="stop" disabled>Dừng</button>
|
|
37
|
-
<button id="cancel" disabled>Hủy</button>
|
|
38
|
-
<pre id="transcript" aria-live="polite"></pre>
|
|
7
|
+
```ts
|
|
8
|
+
import { PronunciationClient, createSpeakingService } from '@hoanghong/speaking-core';
|
|
9
|
+
const pronunciation = new PronunciationClient(createSpeakingService().pronunciation);
|
|
10
|
+
export async function assess(file: File) {
|
|
11
|
+
return pronunciation.assessAudio(file, {
|
|
12
|
+
referenceText: 'cat', expectedPhonemes: ['k', 'æ', 't'], filename: file.name,
|
|
13
|
+
});
|
|
14
|
+
}
|
|
39
15
|
```
|
|
40
16
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
17
|
+
Audio tối đa 120 giây/25 MiB, reference tối đa 4096 ký tự; timeout client 11 phút.
|
|
18
|
+
Đọc cả đoạn: thay referenceText bằng đoạn mẫu; bỏ expectedPhonemes khi server có eSpeak.
|
|
19
|
+
Các âm tùy chọn là token vocabulary, không tách IPA theo ký tự. Mỗi lần gọi cấp phiên riêng,
|
|
20
|
+
không tự retry. AbortSignal được hỗ trợ. Server phải triển khai endpoint/model phát âm;
|
|
21
|
+
bộ cài này chưa xác nhận hosted endpoint đã sẵn sàng. Gateway riêng dùng cấu hình
|
|
22
|
+
`createSpeakingGateway(...).pronunciation`, giữ credential của service ở backend.
|
|
23
|
+
Response typed PronunciationAssessment: âm mẫu/nhận dạng, comparisons, timestamp ước lượng,
|
|
24
|
+
formant/pitch/intensity/PSD và warnings. `calibrated=false`, không dùng similarity làm điểm
|
|
25
|
+
phát âm chính thức. Các API realtime/upload dưới đây giữ nguyên.
|
|
26
|
+
|
|
27
|
+
## Kết nối tự động, không cấu hình URL/token
|
|
28
|
+
|
|
29
|
+
```ts
|
|
30
|
+
import { createSpeakingService, RealtimeSpeakingClient, SpeakingMicrophone } from '@hoanghong/speaking-core';
|
|
31
|
+
const config = createSpeakingService();
|
|
49
32
|
const client = new RealtimeSpeakingClient(config.realtime);
|
|
50
|
-
const
|
|
51
|
-
const start = document.querySelector('#start');
|
|
52
|
-
const stop = document.querySelector('#stop');
|
|
53
|
-
const cancel = document.querySelector('#cancel');
|
|
54
|
-
const transcript = document.querySelector('#transcript');
|
|
55
|
-
function idle() { start.disabled = false; stop.disabled = true; cancel.disabled = true; }
|
|
33
|
+
const mic = new SpeakingMicrophone(client, config.microphone);
|
|
56
34
|
const unsubscribe = client.onEvent(event => {
|
|
57
|
-
if (event.type === 'partial')
|
|
58
|
-
if (event.type === 'clientError')
|
|
35
|
+
if (event.type === 'partial') console.log(event.text);
|
|
36
|
+
if (event.type === 'clientError') console.error(event.error);
|
|
59
37
|
});
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
};
|
|
69
|
-
stop.onclick = async () => {
|
|
70
|
-
stop.disabled = true;
|
|
71
|
-
try {
|
|
72
|
-
const final = await microphone.stop();
|
|
73
|
-
transcript.textContent = final.result?.text ?? 'Không có lời nói.';
|
|
74
|
-
} catch (error) { transcript.textContent = String(error); }
|
|
75
|
-
finally { idle(); }
|
|
76
|
-
};
|
|
77
|
-
cancel.onclick = async () => {
|
|
78
|
-
try { await microphone.cancel(); transcript.textContent = 'Đã hủy.'; }
|
|
79
|
-
catch (error) { transcript.textContent = String(error); }
|
|
80
|
-
finally { idle(); }
|
|
81
|
-
};
|
|
82
|
-
window.addEventListener('pagehide', () => {
|
|
83
|
-
unsubscribe();
|
|
84
|
-
void microphone.dispose();
|
|
85
|
-
}, { once: true });
|
|
38
|
+
// Gọi từ handler nút bắt đầu; không tự ghi âm khi load trang:
|
|
39
|
+
export async function start() { await mic.start({ language: 'vi' }); }
|
|
40
|
+
export async function stop() {
|
|
41
|
+
const result = await mic.stop();
|
|
42
|
+
return { text: result.result?.text ?? '', audio: result.audio };
|
|
43
|
+
}
|
|
44
|
+
export async function cancel() { await mic.cancel(); }
|
|
45
|
+
export async function dispose() { unsubscribe(); await mic.dispose(); }
|
|
86
46
|
```
|
|
87
47
|
|
|
88
|
-
|
|
48
|
+
Audio là File WAV PCM16 mono 16 kHz, gồm phần audio cuối khi dừng. Dùng URL.createObjectURL để nghe/tải rồi revokeObjectURL; hoặc FormData để lưu. Cần browser AudioWorklet/getUserMedia và HTTPS/localhost. Public worklet được tải tự động; không copy asset. Gọi dispose ở lifecycle unmount.
|
|
89
49
|
|
|
90
|
-
##
|
|
50
|
+
## Upload
|
|
91
51
|
|
|
92
|
-
```
|
|
93
|
-
import { BatchSpeakingClient,
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
export async function transcribeFile(file, signal) {
|
|
52
|
+
```ts
|
|
53
|
+
import { BatchSpeakingClient, createSpeakingService } from '@hoanghong/speaking-core';
|
|
54
|
+
const batch = new BatchSpeakingClient(createSpeakingService().batch);
|
|
55
|
+
export async function upload(file: File) {
|
|
97
56
|
if (file.size > 25 * 1024 * 1024) throw new Error('File tối đa 25 MiB');
|
|
98
|
-
let job = await batch.submitAudio(file, 'vi', crypto.randomUUID(), 'short_answer', file.name
|
|
99
|
-
const deadline = Date.now() +
|
|
57
|
+
let job = await batch.submitAudio(file, 'vi', crypto.randomUUID(), 'short_answer', file.name);
|
|
58
|
+
const deadline = Date.now() + 600000;
|
|
100
59
|
while (!['succeeded', 'no_speech', 'failed', 'cancelled'].includes(job.status)) {
|
|
101
|
-
if (Date.now() > deadline) throw new Error('Hết thời gian
|
|
102
|
-
if (signal?.aborted) throw new DOMException('Aborted', 'AbortError');
|
|
60
|
+
if (Date.now() > deadline) throw new Error('Hết thời gian chờ');
|
|
103
61
|
await new Promise(resolve => setTimeout(resolve, 1500));
|
|
104
|
-
job = await batch.getResult(job.job_id
|
|
62
|
+
job = await batch.getResult(job.job_id);
|
|
105
63
|
}
|
|
106
64
|
if (job.error) throw new Error(job.error.message);
|
|
107
|
-
|
|
65
|
+
if (job.status === 'cancelled') throw new Error('Đã hủy');
|
|
66
|
+
return { text: job.result?.text ?? '', audio: file, language: 'vi', serviceType: 'upload' };
|
|
108
67
|
}
|
|
109
|
-
// Truyền File lấy từ input[type=file] vào transcribeFile(file).
|
|
110
|
-
// Chủ động hủy server job: await batch.cancelJob(jobId).
|
|
111
|
-
// AbortSignal chỉ dừng request phía client, không tự hủy job trên server.
|
|
112
68
|
```
|
|
113
69
|
|
|
114
|
-
|
|
70
|
+
Public client quản lý capability cho một job tại một thời điểm; tạo client/config riêng nếu xử lý nhiều job song song. Giữ client khi cần getResult/cancelJob cho job đó. Không chia sẻ capability hay serialize audio vào JSON.
|
|
71
|
+
|
|
72
|
+
Ngôn ngữ en/ja/ko/zh/vi. Public service mặc định 5 phiên/IP/ngày, 100 phiên/ngày/instance (UTC); có thể được FSEL điều chỉnh. Mỗi phiên mới tính lượt kể cả khi lỗi; 429/QUOTA_EXCEEDED là hết lượt. SDK không tự retry/replay. Realtime tối đa 300 giây với buffer PCM 9,6 MB; dọn buffer khi lỗi/hủy. Upload không tự lưu vĩnh viễn file trên S3. Mạng/CSP cần cho phép truy cập dịch vụ FSEL. 404/503 có thể do server chưa triển khai public SDK/model. Cần server hỗ trợ bản 0.3 trước khi sử dụng.
|
|
115
73
|
|
|
116
|
-
|
|
74
|
+
createSpeakingGateway và các constructor cấu hình thủ công cũ vẫn được giữ cho nhu cầu nâng cao; không cần dùng trong luồng public trên.
|
|
117
75
|
|
|
118
|
-
|
|
76
|
+
## Key ứng dụng
|
|
119
77
|
|
|
120
|
-
|
|
78
|
+
Dùng `createSpeakingService({ appKey: "KEY_DUOC_FSEL_CAP" })` thay cho lời gọi không tham số. SDK gửi key khi cấp phiên; quota quản lý phía server, không có URL cần cấu hình. Key này là định danh ứng dụng có thể quan sát ở frontend, không có quyền quản trị. INVALID_APP_KEY/401 nghĩa là sai hoặc đã bị khóa; không tự fallback ẩn danh.
|
package/dist/audio.d.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Five minutes of mono PCM16 at 16 kHz; bounded independently of server limits. */
|
|
2
|
+
export declare const MAX_RECORDING_BYTES: number;
|
|
3
|
+
export declare class SpeakingAudioBuffer {
|
|
4
|
+
private chunks;
|
|
5
|
+
private size;
|
|
6
|
+
append(pcm: Uint8Array): void;
|
|
7
|
+
file(): File;
|
|
8
|
+
clear(): void;
|
|
9
|
+
}
|
package/dist/audio.js
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { SpeakingError } from './contracts.js';
|
|
2
|
+
/** Five minutes of mono PCM16 at 16 kHz; bounded independently of server limits. */
|
|
3
|
+
export const MAX_RECORDING_BYTES = 16000 * 2 * 300;
|
|
4
|
+
export class SpeakingAudioBuffer {
|
|
5
|
+
chunks = [];
|
|
6
|
+
size = 0;
|
|
7
|
+
append(pcm) {
|
|
8
|
+
if (pcm.byteLength % 2)
|
|
9
|
+
throw new SpeakingError('INCOMPLETE_AUDIO', 'PCM16 requires complete samples');
|
|
10
|
+
if (this.size + pcm.byteLength > MAX_RECORDING_BYTES)
|
|
11
|
+
throw new SpeakingError('AUDIO_TOO_LONG', 'Recording exceeds five minutes');
|
|
12
|
+
this.chunks.push(pcm.slice());
|
|
13
|
+
this.size += pcm.byteLength;
|
|
14
|
+
}
|
|
15
|
+
file() {
|
|
16
|
+
const header = new ArrayBuffer(44);
|
|
17
|
+
const view = new DataView(header);
|
|
18
|
+
const text = (at, value) => { for (let i = 0; i < value.length; ++i)
|
|
19
|
+
view.setUint8(at + i, value.charCodeAt(i)); };
|
|
20
|
+
text(0, 'RIFF');
|
|
21
|
+
view.setUint32(4, 36 + this.size, true);
|
|
22
|
+
text(8, 'WAVE');
|
|
23
|
+
text(12, 'fmt ');
|
|
24
|
+
view.setUint32(16, 16, true);
|
|
25
|
+
view.setUint16(20, 1, true);
|
|
26
|
+
view.setUint16(22, 1, true);
|
|
27
|
+
view.setUint32(24, 16000, true);
|
|
28
|
+
view.setUint32(28, 32000, true);
|
|
29
|
+
view.setUint16(32, 2, true);
|
|
30
|
+
view.setUint16(34, 16, true);
|
|
31
|
+
text(36, 'data');
|
|
32
|
+
view.setUint32(40, this.size, true);
|
|
33
|
+
return new File([header, ...this.chunks], 'speaking.wav', { type: 'audio/wav' });
|
|
34
|
+
}
|
|
35
|
+
clear() { this.chunks = []; this.size = 0; }
|
|
36
|
+
}
|
package/dist/batch.d.ts
CHANGED
|
@@ -4,6 +4,7 @@ export interface BatchOptions {
|
|
|
4
4
|
/** User/gateway auth only. Never distribute an ASR service credential to an app. */
|
|
5
5
|
headers: () => Promise<HeadersInit>;
|
|
6
6
|
fetch?: typeof globalThis.fetch;
|
|
7
|
+
beforeSubmit?: (idempotencyKey: string) => Promise<void>;
|
|
7
8
|
}
|
|
8
9
|
export declare class BatchSpeakingClient {
|
|
9
10
|
private readonly options;
|
package/dist/batch.js
CHANGED
|
@@ -17,6 +17,7 @@ export class BatchSpeakingClient {
|
|
|
17
17
|
async submitAudio(audio, language, idempotencyKey, audioProfile = 'short_answer', filename = 'recording.wav', signal) {
|
|
18
18
|
if (!idempotencyKey.trim())
|
|
19
19
|
throw new SpeakingError('INVALID_REQUEST', 'An idempotency key is required');
|
|
20
|
+
await this.options.beforeSubmit?.(idempotencyKey);
|
|
20
21
|
const form = new FormData();
|
|
21
22
|
form.append('file', audio, filename);
|
|
22
23
|
form.append('language', language);
|
package/dist/contracts.d.ts
CHANGED
|
@@ -1,4 +1,14 @@
|
|
|
1
1
|
export type Language = 'en' | 'ja' | 'ko' | 'zh' | 'vi';
|
|
2
|
+
export type SpeakingServiceType = 'realtime' | 'upload';
|
|
3
|
+
export type SpeakingResult = (RealtimeFinal | Job) & {
|
|
4
|
+
text: string;
|
|
5
|
+
audio: File;
|
|
6
|
+
language: Language;
|
|
7
|
+
serviceType: SpeakingServiceType;
|
|
8
|
+
};
|
|
9
|
+
export type RecordedSpeakingFinal = RealtimeFinal & {
|
|
10
|
+
audio: File;
|
|
11
|
+
};
|
|
2
12
|
export type AudioProfile = 'short_answer' | 'passage';
|
|
3
13
|
export type SpeakingState = 'idle' | 'connecting' | 'recording' | 'stopping' | 'stopped' | 'cancelled' | 'failed';
|
|
4
14
|
export interface Transcript {
|
package/dist/gateway.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { BatchOptions } from './batch.js';
|
|
2
2
|
import { MicrophoneOptions } from './microphone.js';
|
|
3
3
|
import { RealtimeOptions } from './realtime.js';
|
|
4
|
+
import { PronunciationOptions } from './pronunciation.js';
|
|
4
5
|
export interface SpeakingGatewayOptions {
|
|
5
6
|
/** Application gateway, e.g. /api/speaking. It must relay /v1 HTTP + WebSocket. */
|
|
6
7
|
baseUrl?: string;
|
|
@@ -13,4 +14,5 @@ export declare function createSpeakingGateway(options?: SpeakingGatewayOptions):
|
|
|
13
14
|
batch: BatchOptions;
|
|
14
15
|
realtime: RealtimeOptions;
|
|
15
16
|
microphone: MicrophoneOptions;
|
|
17
|
+
pronunciation: PronunciationOptions;
|
|
16
18
|
};
|
package/dist/gateway.js
CHANGED
|
@@ -12,6 +12,7 @@ export function createSpeakingGateway(options = {}) {
|
|
|
12
12
|
socket.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
|
|
13
13
|
return {
|
|
14
14
|
batch: { baseUrl, headers, fetch: request },
|
|
15
|
+
pronunciation: { baseUrl, headers, fetch: request },
|
|
15
16
|
realtime: {
|
|
16
17
|
websocketUrl: socket.href,
|
|
17
18
|
ticketProvider: async (start) => {
|
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
package/dist/microphone.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { RealtimeSpeakingClient } from './realtime.js';
|
|
2
|
-
import {
|
|
2
|
+
import { RecordedSpeakingFinal, StartOptions } from './contracts.js';
|
|
3
3
|
export interface MicrophoneOptions {
|
|
4
4
|
/** Host the packaged pcm-worklet.js with your app's normal CSP; no inline/blob worker. */
|
|
5
5
|
workletUrl: string;
|
|
@@ -18,9 +18,10 @@ export declare class SpeakingMicrophone {
|
|
|
18
18
|
private starting;
|
|
19
19
|
private generation;
|
|
20
20
|
private stopPromise?;
|
|
21
|
+
private readonly recording;
|
|
21
22
|
constructor(client: RealtimeSpeakingClient, options: MicrophoneOptions);
|
|
22
23
|
start(options: StartOptions): Promise<void>;
|
|
23
|
-
stop(): Promise<
|
|
24
|
+
stop(): Promise<RecordedSpeakingFinal>;
|
|
24
25
|
private release;
|
|
25
26
|
cancel(): Promise<void>;
|
|
26
27
|
dispose(): Promise<void>;
|
package/dist/microphone.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { SpeakingError } from './contracts.js';
|
|
2
|
+
import { SpeakingAudioBuffer } from './audio.js';
|
|
2
3
|
export class SpeakingMicrophone {
|
|
3
4
|
client;
|
|
4
5
|
options;
|
|
@@ -13,6 +14,7 @@ export class SpeakingMicrophone {
|
|
|
13
14
|
starting = false;
|
|
14
15
|
generation = 0;
|
|
15
16
|
stopPromise;
|
|
17
|
+
recording = new SpeakingAudioBuffer();
|
|
16
18
|
constructor(client, options) {
|
|
17
19
|
this.client = client;
|
|
18
20
|
this.options = options;
|
|
@@ -25,6 +27,7 @@ export class SpeakingMicrophone {
|
|
|
25
27
|
this.stopPromise = undefined;
|
|
26
28
|
this.audioError = undefined;
|
|
27
29
|
this.sends = Promise.resolve();
|
|
30
|
+
this.recording.clear();
|
|
28
31
|
try {
|
|
29
32
|
const stream = await navigator.mediaDevices.getUserMedia({
|
|
30
33
|
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true },
|
|
@@ -55,6 +58,15 @@ export class SpeakingMicrophone {
|
|
|
55
58
|
if (generation !== this.generation)
|
|
56
59
|
return;
|
|
57
60
|
if (data.type === 'pcm') {
|
|
61
|
+
try {
|
|
62
|
+
this.recording.append(new Uint8Array(data.buffer));
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
this.audioError = error;
|
|
66
|
+
this.client.audioFailed(this.audioError);
|
|
67
|
+
void this.release();
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
58
70
|
this.client.audioLevel(data.level);
|
|
59
71
|
// Enqueue immediately to let the transport enforce a bounded audio backlog.
|
|
60
72
|
const pending = this.client.pushAudio(new Uint8Array(data.buffer));
|
|
@@ -106,7 +118,10 @@ export class SpeakingMicrophone {
|
|
|
106
118
|
throw new SpeakingError('CANCELLED', 'Microphone session changed');
|
|
107
119
|
if (this.audioError)
|
|
108
120
|
throw this.audioError;
|
|
109
|
-
|
|
121
|
+
const result = await this.client.stop();
|
|
122
|
+
if (generation !== this.generation)
|
|
123
|
+
throw new SpeakingError('CANCELLED', 'Microphone session changed');
|
|
124
|
+
return { ...result, audio: this.recording.file() };
|
|
110
125
|
}
|
|
111
126
|
catch (error) {
|
|
112
127
|
if (generation === this.generation)
|
|
@@ -121,6 +136,7 @@ export class SpeakingMicrophone {
|
|
|
121
136
|
return this.stopPromise;
|
|
122
137
|
}
|
|
123
138
|
async release() {
|
|
139
|
+
this.recording.clear();
|
|
124
140
|
this.unsubscribe?.();
|
|
125
141
|
this.unsubscribe = undefined;
|
|
126
142
|
this.source?.disconnect();
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
export interface PronunciationOptions {
|
|
2
|
+
baseUrl: string;
|
|
3
|
+
/** Short-lived app/gateway authentication; never embed a service credential. */
|
|
4
|
+
headers: () => Promise<HeadersInit>;
|
|
5
|
+
fetch?: typeof globalThis.fetch;
|
|
6
|
+
timeoutMs?: number;
|
|
7
|
+
}
|
|
8
|
+
export interface PronunciationPhone {
|
|
9
|
+
phoneme: string;
|
|
10
|
+
start: number;
|
|
11
|
+
end: number;
|
|
12
|
+
}
|
|
13
|
+
export interface PronunciationComparison {
|
|
14
|
+
operation: 'match' | 'substitution' | 'deletion' | 'insertion';
|
|
15
|
+
expected: string | null;
|
|
16
|
+
observed: string | null;
|
|
17
|
+
start: number | null;
|
|
18
|
+
end: number | null;
|
|
19
|
+
}
|
|
20
|
+
export interface PronunciationAcousticFrame {
|
|
21
|
+
time: number;
|
|
22
|
+
formants_hz: Array<number | null>;
|
|
23
|
+
delta_formants_hz: Array<number | null>;
|
|
24
|
+
f2_minus_f1_hz: number | null;
|
|
25
|
+
pitch_hz: number | null;
|
|
26
|
+
intensity_db: number | null;
|
|
27
|
+
psd_power_per_hz: number[];
|
|
28
|
+
}
|
|
29
|
+
export interface PronunciationAssessment {
|
|
30
|
+
reference_text: string;
|
|
31
|
+
language: 'en-US';
|
|
32
|
+
expected_phonemes: string[];
|
|
33
|
+
observed_phonemes: PronunciationPhone[];
|
|
34
|
+
comparisons: PronunciationComparison[];
|
|
35
|
+
phoneme_error_rate: number;
|
|
36
|
+
phoneme_similarity: number;
|
|
37
|
+
calibrated: false;
|
|
38
|
+
timing_method: 'ctc_greedy_approximate';
|
|
39
|
+
duration: number;
|
|
40
|
+
acoustic_frames: PronunciationAcousticFrame[];
|
|
41
|
+
psd_frequencies_hz: number[];
|
|
42
|
+
model: string;
|
|
43
|
+
warnings: string[];
|
|
44
|
+
}
|
|
45
|
+
export interface AssessPronunciationOptions {
|
|
46
|
+
referenceText: string;
|
|
47
|
+
/** Vocabulary tokens, not individual IPA characters. Optional when server has eSpeak. */
|
|
48
|
+
expectedPhonemes?: string[];
|
|
49
|
+
filename?: string;
|
|
50
|
+
signal?: AbortSignal;
|
|
51
|
+
}
|
|
52
|
+
export declare class PronunciationClient {
|
|
53
|
+
private readonly options;
|
|
54
|
+
private readonly request;
|
|
55
|
+
constructor(options: PronunciationOptions);
|
|
56
|
+
/** US English, <=120 seconds and <=25 MiB. No automatic retries or duplicate quota charges. */
|
|
57
|
+
assessAudio(audio: Blob, options: AssessPronunciationOptions): Promise<PronunciationAssessment>;
|
|
58
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { SpeakingError } from './contracts.js';
|
|
2
|
+
export class PronunciationClient {
|
|
3
|
+
options;
|
|
4
|
+
request;
|
|
5
|
+
constructor(options) {
|
|
6
|
+
this.options = options;
|
|
7
|
+
this.request = options.fetch ?? globalThis.fetch.bind(globalThis);
|
|
8
|
+
}
|
|
9
|
+
/** US English, <=120 seconds and <=25 MiB. No automatic retries or duplicate quota charges. */
|
|
10
|
+
async assessAudio(audio, options) {
|
|
11
|
+
if (!options.referenceText.trim() || options.referenceText.length > 4096) {
|
|
12
|
+
throw new SpeakingError('INVALID_REFERENCE', 'Provide reference text of 1 to 4096 characters');
|
|
13
|
+
}
|
|
14
|
+
if (!audio.size || audio.size > 25 * 1024 * 1024) {
|
|
15
|
+
throw new SpeakingError('INVALID_AUDIO', 'Audio must be nonempty and at most 25 MiB');
|
|
16
|
+
}
|
|
17
|
+
const controller = new AbortController();
|
|
18
|
+
const abort = () => controller.abort(options.signal?.reason);
|
|
19
|
+
if (options.signal?.aborted)
|
|
20
|
+
abort();
|
|
21
|
+
else
|
|
22
|
+
options.signal?.addEventListener('abort', abort, { once: true });
|
|
23
|
+
const timer = setTimeout(() => controller.abort(), this.options.timeoutMs ?? 660_000);
|
|
24
|
+
try {
|
|
25
|
+
if (controller.signal.aborted)
|
|
26
|
+
throw new SpeakingError('CANCELLED', 'Assessment was cancelled');
|
|
27
|
+
const headers = new Headers(await this.options.headers());
|
|
28
|
+
headers.delete('Content-Type');
|
|
29
|
+
const form = new FormData();
|
|
30
|
+
form.append('file', audio, options.filename ?? 'recording.wav');
|
|
31
|
+
form.append('reference_text', options.referenceText);
|
|
32
|
+
form.append('language', 'en-US');
|
|
33
|
+
if (options.expectedPhonemes !== undefined) {
|
|
34
|
+
form.append('expected_phonemes', JSON.stringify(options.expectedPhonemes));
|
|
35
|
+
}
|
|
36
|
+
const response = await this.request(this.options.baseUrl.replace(/\/$/, '') + '/v1/pronunciation-assessments', { method: 'POST', credentials: 'same-origin', headers, body: form, signal: controller.signal });
|
|
37
|
+
const body = await response.json().catch(() => null);
|
|
38
|
+
if (!response.ok) {
|
|
39
|
+
throw new SpeakingError(body?.code ?? 'HTTP_ERROR', body?.message ?? 'Assessment failed', response.status);
|
|
40
|
+
}
|
|
41
|
+
if (!body || body.language !== 'en-US' || body.calibrated !== false ||
|
|
42
|
+
typeof body.phoneme_similarity !== 'number' || !Array.isArray(body.comparisons)) {
|
|
43
|
+
throw new SpeakingError('INVALID_RESPONSE', 'Invalid pronunciation assessment response');
|
|
44
|
+
}
|
|
45
|
+
return body;
|
|
46
|
+
}
|
|
47
|
+
finally {
|
|
48
|
+
clearTimeout(timer);
|
|
49
|
+
options.signal?.removeEventListener('abort', abort);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Hosted public service. The SDK handles short-lived capabilities internally. */
|
|
2
|
+
export declare function createSpeakingService(options?: {
|
|
3
|
+
appKey?: string;
|
|
4
|
+
}): {
|
|
5
|
+
batch: import("./batch.js").BatchOptions;
|
|
6
|
+
realtime: import("./realtime.js").RealtimeOptions;
|
|
7
|
+
microphone: import("./microphone.js").MicrophoneOptions;
|
|
8
|
+
pronunciation: import("./pronunciation.js").PronunciationOptions;
|
|
9
|
+
};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { createSpeakingGateway } from './gateway.js';
|
|
2
|
+
import { SpeakingError } from './contracts.js';
|
|
3
|
+
const SERVICE = 'https://ai-service.fsel.edu.vn/sdk';
|
|
4
|
+
/** Hosted public service. The SDK handles short-lived capabilities internally. */
|
|
5
|
+
export function createSpeakingService(options = {}) {
|
|
6
|
+
if (options.appKey !== undefined && !options.appKey.trim())
|
|
7
|
+
throw new SpeakingError('INVALID_CONFIG', 'Application key cannot be empty');
|
|
8
|
+
const request = globalThis.fetch.bind(globalThis);
|
|
9
|
+
const session = async () => {
|
|
10
|
+
const response = await request(SERVICE + '/session', {
|
|
11
|
+
method: 'POST', credentials: 'omit',
|
|
12
|
+
headers: options.appKey ? { 'X-Speaking-Key': options.appKey } : undefined,
|
|
13
|
+
});
|
|
14
|
+
const body = await response.json().catch(() => ({}));
|
|
15
|
+
if (!response.ok)
|
|
16
|
+
throw new SpeakingError(body?.code ?? 'HTTP_ERROR', body?.message ?? 'Speaking service unavailable', response.status);
|
|
17
|
+
if (typeof body?.access_token !== 'string' || !body.access_token)
|
|
18
|
+
throw new SpeakingError('INVALID_RESPONSE', 'Speaking session unavailable');
|
|
19
|
+
return body.access_token;
|
|
20
|
+
};
|
|
21
|
+
let token;
|
|
22
|
+
let previousKey;
|
|
23
|
+
const config = createSpeakingGateway({
|
|
24
|
+
baseUrl: SERVICE,
|
|
25
|
+
headers: async () => {
|
|
26
|
+
if (!token)
|
|
27
|
+
throw new SpeakingError('INVALID_STATE', 'Submit audio before reading a public job');
|
|
28
|
+
return { Authorization: 'Bearer ' + token };
|
|
29
|
+
},
|
|
30
|
+
workletUrl: SERVICE + '/pcm-worklet.js',
|
|
31
|
+
});
|
|
32
|
+
config.batch.beforeSubmit = async (key) => {
|
|
33
|
+
if (!token || key !== previousKey) {
|
|
34
|
+
token = await session();
|
|
35
|
+
previousKey = key;
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
const realtime = createSpeakingGateway({ baseUrl: SERVICE, headers: async () => ({ Authorization: 'Bearer ' + await session() }) });
|
|
39
|
+
config.realtime = realtime.realtime;
|
|
40
|
+
// Every synchronous assessment consumes its own short-lived, single-use session.
|
|
41
|
+
config.pronunciation.headers = async () => ({ Authorization: 'Bearer ' + await session() });
|
|
42
|
+
return config;
|
|
43
|
+
}
|
package/dist/realtime.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { RealtimeFinal, SpeakingEvent, SpeakingState, Started, StartOptions } from './contracts.js';
|
|
1
|
+
import { RealtimeFinal, SpeakingError, SpeakingEvent, SpeakingState, Started, StartOptions } from './contracts.js';
|
|
2
2
|
export interface RealtimeOptions {
|
|
3
3
|
websocketUrl: string;
|
|
4
4
|
/** Ask an authenticated application backend for a fresh single-use ticket per start. */
|
|
@@ -30,6 +30,7 @@ export declare class RealtimeSpeakingClient {
|
|
|
30
30
|
private emit;
|
|
31
31
|
private setState;
|
|
32
32
|
audioLevel(level: number): void;
|
|
33
|
+
audioFailed(error: SpeakingError): void;
|
|
33
34
|
private wait;
|
|
34
35
|
private rejectWaiters;
|
|
35
36
|
private fail;
|
package/dist/realtime.js
CHANGED
|
@@ -35,6 +35,7 @@ export class RealtimeSpeakingClient {
|
|
|
35
35
|
this.emit({ type: 'state', state });
|
|
36
36
|
}
|
|
37
37
|
audioLevel(level) { this.emit({ type: 'audioLevel', level }); }
|
|
38
|
+
audioFailed(error) { this.fail(error); }
|
|
38
39
|
wait(type, timeout) {
|
|
39
40
|
return new Promise((resolve, reject) => {
|
|
40
41
|
const waiter = { type, resolve, reject, timer: setTimeout(() => {
|