@automattic/jetpack-ai-client 0.8.1 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/build/audio-transcription/index.js +8 -5
- package/build/hooks/use-audio-transcription/index.js +34 -1
- package/build/hooks/use-audio-validation/index.d.ts +21 -0
- package/build/hooks/use-audio-validation/index.js +61 -0
- package/build/hooks/use-media-recording/index.d.ts +1 -5
- package/build/hooks/use-media-recording/index.js +13 -7
- package/build/index.d.ts +1 -0
- package/build/index.js +1 -0
- package/build/types.d.ts +2 -0
- package/package.json +13 -13
- package/src/audio-transcription/index.ts +11 -19
- package/src/hooks/use-audio-transcription/index.ts +59 -1
- package/src/hooks/use-audio-validation/index.ts +110 -0
- package/src/hooks/use-media-recording/index.ts +16 -14
- package/src/index.ts +1 -0
- package/src/types.ts +10 -0
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,20 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.8.2] - 2024-03-04
|
|
9
|
+
### Added
|
|
10
|
+
- AI Client: add audio validation hook. [#36043]
|
|
11
|
+
- Voice to Content: Close audio stream on hook destruction [#36086]
|
|
12
|
+
|
|
13
|
+
### Changed
|
|
14
|
+
- AI Client: change loading and error state handling on media recording hook. [#36001]
|
|
15
|
+
- AI Client: publish audio information on the validation success callback of the audio validation hook. [#36094]
|
|
16
|
+
- Updated package dependencies. [#36095]
|
|
17
|
+
- Updated package dependencies. [#36143]
|
|
18
|
+
|
|
19
|
+
### Fixed
|
|
20
|
+
- AI Client: fixed transcription request from P2 editor [#36081]
|
|
21
|
+
|
|
8
22
|
## [0.8.1] - 2024-02-27
|
|
9
23
|
### Changed
|
|
10
24
|
- AI Client: support audio transcription and transcription post-processing canceling. [#35923]
|
|
@@ -233,6 +247,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
233
247
|
- Updated package dependencies. [#31659]
|
|
234
248
|
- Updated package dependencies. [#31785]
|
|
235
249
|
|
|
250
|
+
[0.8.2]: https://github.com/Automattic/jetpack-ai-client/compare/v0.8.1...v0.8.2
|
|
236
251
|
[0.8.1]: https://github.com/Automattic/jetpack-ai-client/compare/v0.8.0...v0.8.1
|
|
237
252
|
[0.8.0]: https://github.com/Automattic/jetpack-ai-client/compare/v0.7.0...v0.8.0
|
|
238
253
|
[0.7.0]: https://github.com/Automattic/jetpack-ai-client/compare/v0.6.1...v0.7.0
|
|
@@ -5,7 +5,6 @@ import debugFactory from 'debug';
|
|
|
5
5
|
/**
|
|
6
6
|
* Internal dependencies
|
|
7
7
|
*/
|
|
8
|
-
import apiFetch from '../api-fetch/index.js';
|
|
9
8
|
import requestJwt from '../jwt/index.js';
|
|
10
9
|
const debug = debugFactory('jetpack-ai-client:audio-transcription');
|
|
11
10
|
/**
|
|
@@ -34,15 +33,19 @@ export default async function transcribeAudio(audio, feature, requestAbortSignal
|
|
|
34
33
|
const headers = {
|
|
35
34
|
Authorization: `Bearer ${token}`,
|
|
36
35
|
};
|
|
37
|
-
const
|
|
38
|
-
|
|
36
|
+
const URL = `https://public-api.wordpress.com/wpcom/v2/jetpack-ai-transcription${feature ? `?feature=${feature}` : ''}`;
|
|
37
|
+
return fetch(URL, {
|
|
39
38
|
method: 'POST',
|
|
40
39
|
body: formData,
|
|
41
40
|
headers,
|
|
42
41
|
signal: requestAbortSignal ?? undefined,
|
|
42
|
+
}).then(response => {
|
|
43
|
+
debug('Transcription response: %o', response);
|
|
44
|
+
if (response.ok) {
|
|
45
|
+
return response.json().then(data => data?.text);
|
|
46
|
+
}
|
|
47
|
+
return response.json().then(data => Promise.reject(data));
|
|
43
48
|
});
|
|
44
|
-
debug('Transcription response: %o', response);
|
|
45
|
-
return response.text;
|
|
46
49
|
}
|
|
47
50
|
catch (error) {
|
|
48
51
|
debug('Transcription error response: %o', error);
|
|
@@ -2,12 +2,45 @@
|
|
|
2
2
|
* External dependencies
|
|
3
3
|
*/
|
|
4
4
|
import { useCallback, useState, useRef } from '@wordpress/element';
|
|
5
|
+
import { __ } from '@wordpress/i18n';
|
|
5
6
|
import debugFactory from 'debug';
|
|
6
7
|
/**
|
|
7
8
|
* Internal dependencies
|
|
8
9
|
*/
|
|
9
10
|
import transcribeAudio from '../../audio-transcription/index.js';
|
|
10
11
|
const debug = debugFactory('jetpack-ai-client:use-audio-transcription');
|
|
12
|
+
/**
|
|
13
|
+
* Map error response to a string.
|
|
14
|
+
* @param {Error | string | AudioTranscriptionErrorResponse} error - The error response from the audio transcription service.
|
|
15
|
+
* @returns {string} the translated error message
|
|
16
|
+
*/
|
|
17
|
+
const mapErrorResponse = (error) => {
|
|
18
|
+
if (typeof error === 'string') {
|
|
19
|
+
return error;
|
|
20
|
+
}
|
|
21
|
+
if ('code' in error) {
|
|
22
|
+
switch (error.code) {
|
|
23
|
+
case 'error_quota_exceeded':
|
|
24
|
+
return __('You exceeded your current quota, please check your plan details.', 'jetpack-ai-client');
|
|
25
|
+
case 'jetpack_ai_missing_audio_param':
|
|
26
|
+
return __('The audio_file is required to perform a transcription.', 'jetpack-ai-client');
|
|
27
|
+
case 'jetpack_ai_service_unavailable':
|
|
28
|
+
return __('The Jetpack AI service is temporarily unavailable.', 'jetpack-ai-client');
|
|
29
|
+
case 'file_size_not_supported':
|
|
30
|
+
return __('The provided audio file is too big.', 'jetpack-ai-client');
|
|
31
|
+
case 'file_type_not_supported':
|
|
32
|
+
return __('The provided audio file type is not supported.', 'jetpack-ai-client');
|
|
33
|
+
case 'jetpack_ai_error':
|
|
34
|
+
return __('There was an error processing the transcription request.', 'jetpack-ai-client');
|
|
35
|
+
default:
|
|
36
|
+
return error.message;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
if ('message' in error) {
|
|
40
|
+
return error.message;
|
|
41
|
+
}
|
|
42
|
+
return __('There was an error processing the transcription request.', 'jetpack-ai-client');
|
|
43
|
+
};
|
|
11
44
|
/**
|
|
12
45
|
* A hook to handle audio transcription.
|
|
13
46
|
*
|
|
@@ -43,7 +76,7 @@ export default function useAudioTranscription({ feature, onReady, onError, }) {
|
|
|
43
76
|
.catch(error => {
|
|
44
77
|
if (!controller.signal.aborted) {
|
|
45
78
|
setTranscriptionError(error.message);
|
|
46
|
-
onError?.(error
|
|
79
|
+
onError?.(mapErrorResponse(error));
|
|
47
80
|
}
|
|
48
81
|
})
|
|
49
82
|
.finally(() => setIsTranscribingAudio(false));
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The return value for the audio validation hook.
|
|
3
|
+
*/
|
|
4
|
+
export type UseAudioValidationReturn = {
|
|
5
|
+
isValidatingAudio: boolean;
|
|
6
|
+
validateAudio: (audio: Blob, successCallback: (info?: ValidatedAudioInformation) => void, errorCallback: (error: string) => void) => void;
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* The validated audio information.
|
|
10
|
+
*/
|
|
11
|
+
export type ValidatedAudioInformation = {
|
|
12
|
+
duration: number;
|
|
13
|
+
isFile: boolean;
|
|
14
|
+
size: number;
|
|
15
|
+
};
|
|
16
|
+
/**
|
|
17
|
+
* Hook to handle the validation of an audio file.
|
|
18
|
+
*
|
|
19
|
+
* @returns {UseAudioValidationReturn} - Object with the audio validation state and the function to validate the audio.
|
|
20
|
+
*/
|
|
21
|
+
export default function useAudioValidation(): UseAudioValidationReturn;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* External dependencies
|
|
3
|
+
*/
|
|
4
|
+
import { useCallback, useState } from '@wordpress/element';
|
|
5
|
+
import { __ } from '@wordpress/i18n';
|
|
6
|
+
const MAX_AUDIO_SIZE = 25000000; // 25MB
|
|
7
|
+
const MAX_AUDIO_DURATION = 25 * 60; // 25 minutes
|
|
8
|
+
const ALLOWED_MEDIA_TYPES = [
|
|
9
|
+
'audio/mpeg',
|
|
10
|
+
'audio/mp3',
|
|
11
|
+
'audio/ogg',
|
|
12
|
+
'audio/flac',
|
|
13
|
+
'audio/x-flac',
|
|
14
|
+
'audio/m4a',
|
|
15
|
+
'audio/x-m4a',
|
|
16
|
+
'audio/mp4',
|
|
17
|
+
'audio/wav',
|
|
18
|
+
'audio/wave',
|
|
19
|
+
'audio/x-wav',
|
|
20
|
+
'audio/webm',
|
|
21
|
+
];
|
|
22
|
+
/**
|
|
23
|
+
* Hook to handle the validation of an audio file.
|
|
24
|
+
*
|
|
25
|
+
* @returns {UseAudioValidationReturn} - Object with the audio validation state and the function to validate the audio.
|
|
26
|
+
*/
|
|
27
|
+
export default function useAudioValidation() {
|
|
28
|
+
const [isValidatingAudio, setIsValidatingAudio] = useState(false);
|
|
29
|
+
const validateAudio = useCallback((audio, successCallback, errorCallback) => {
|
|
30
|
+
setIsValidatingAudio(true);
|
|
31
|
+
// Check if the audio file is too large
|
|
32
|
+
if (audio?.size > MAX_AUDIO_SIZE) {
|
|
33
|
+
setIsValidatingAudio(false);
|
|
34
|
+
return errorCallback(__('The audio file is too large. The maximum file size is 25MB.', 'jetpack-ai-client'));
|
|
35
|
+
}
|
|
36
|
+
// When it's a file, check the media type
|
|
37
|
+
const isFile = audio instanceof File;
|
|
38
|
+
if (isFile) {
|
|
39
|
+
if (!ALLOWED_MEDIA_TYPES.includes(audio.type)) {
|
|
40
|
+
setIsValidatingAudio(false);
|
|
41
|
+
return errorCallback(__('The audio file type is not supported. Please use a supported audio file type.', 'jetpack-ai-client'));
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
// Check the duration of the audio
|
|
45
|
+
const audioContext = new AudioContext();
|
|
46
|
+
// Map blob to an array buffer
|
|
47
|
+
audio.arrayBuffer().then(arrayBuffer => {
|
|
48
|
+
// Decode audio file data contained in an ArrayBuffer
|
|
49
|
+
audioContext.decodeAudioData(arrayBuffer, function (audioBuffer) {
|
|
50
|
+
const duration = Math.ceil(audioBuffer.duration);
|
|
51
|
+
if (duration > MAX_AUDIO_DURATION) {
|
|
52
|
+
setIsValidatingAudio(false);
|
|
53
|
+
return errorCallback(__('The audio file is too long. The maximum recording time is 25 minutes.', 'jetpack-ai-client'));
|
|
54
|
+
}
|
|
55
|
+
setIsValidatingAudio(false);
|
|
56
|
+
return successCallback({ duration, isFile, size: audio?.size });
|
|
57
|
+
});
|
|
58
|
+
});
|
|
59
|
+
}, [setIsValidatingAudio]);
|
|
60
|
+
return { isValidatingAudio, validateAudio };
|
|
61
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type RecordingState = 'inactive' | 'recording' | 'paused' | '
|
|
1
|
+
export type RecordingState = 'inactive' | 'recording' | 'paused' | 'error';
|
|
2
2
|
type UseMediaRecordingProps = {
|
|
3
3
|
onDone?: (blob: Blob) => void;
|
|
4
4
|
};
|
|
@@ -27,10 +27,6 @@ type UseMediaRecordingReturn = {
|
|
|
27
27
|
* The error handler
|
|
28
28
|
*/
|
|
29
29
|
onError: (err: string | Error) => void;
|
|
30
|
-
/**
|
|
31
|
-
* The processing handler
|
|
32
|
-
*/
|
|
33
|
-
onProcessing: () => void;
|
|
34
30
|
controls: {
|
|
35
31
|
/**
|
|
36
32
|
* `start` recording handler
|
|
@@ -11,7 +11,7 @@ import { useRef, useState, useEffect, useCallback } from '@wordpress/element';
|
|
|
11
11
|
export default function useMediaRecording({ onDone, } = {}) {
|
|
12
12
|
// Reference to the media recorder instance
|
|
13
13
|
const mediaRecordRef = useRef(null);
|
|
14
|
-
// Recording state: `inactive`, `recording`, `paused`, `
|
|
14
|
+
// Recording state: `inactive`, `recording`, `paused`, `error`
|
|
15
15
|
const [state, setState] = useState('inactive');
|
|
16
16
|
// reference to the paused state to be used in the `onDataAvailable` event listener,
|
|
17
17
|
// as the `mediaRecordRef.current.state` is already `inactive` when the recorder is stopped,
|
|
@@ -19,6 +19,7 @@ export default function useMediaRecording({ onDone, } = {}) {
|
|
|
19
19
|
const isPaused = useRef(false);
|
|
20
20
|
const recordStartTimestamp = useRef(0);
|
|
21
21
|
const [duration, setDuration] = useState(0);
|
|
22
|
+
const audioStream = useRef(null);
|
|
22
23
|
// The recorded blob
|
|
23
24
|
const [blob, setBlob] = useState(null);
|
|
24
25
|
// Store the recorded chunks
|
|
@@ -111,6 +112,7 @@ export default function useMediaRecording({ onDone, } = {}) {
|
|
|
111
112
|
navigator.mediaDevices
|
|
112
113
|
.getUserMedia(constraints)
|
|
113
114
|
.then(stream => {
|
|
115
|
+
audioStream.current = stream;
|
|
114
116
|
const source = audioCtx.createMediaStreamSource(stream);
|
|
115
117
|
source.connect(analyser.current);
|
|
116
118
|
mediaRecordRef.current = new MediaRecorder(stream);
|
|
@@ -131,10 +133,6 @@ export default function useMediaRecording({ onDone, } = {}) {
|
|
|
131
133
|
setError(typeof err === 'string' ? err : err.message);
|
|
132
134
|
setState('error');
|
|
133
135
|
}, []);
|
|
134
|
-
// manually set the state to `processing` for the file upload case
|
|
135
|
-
const onProcessing = useCallback(() => {
|
|
136
|
-
setState('processing');
|
|
137
|
-
}, []);
|
|
138
136
|
/**
|
|
139
137
|
* `start` event listener for the media recorder instance.
|
|
140
138
|
*/
|
|
@@ -148,7 +146,6 @@ export default function useMediaRecording({ onDone, } = {}) {
|
|
|
148
146
|
* @returns {void}
|
|
149
147
|
*/
|
|
150
148
|
function onStopListener() {
|
|
151
|
-
setState('processing');
|
|
152
149
|
const lastBlob = getBlob();
|
|
153
150
|
onDone?.(lastBlob);
|
|
154
151
|
// Clear the recorded chunks
|
|
@@ -192,10 +189,20 @@ export default function useMediaRecording({ onDone, } = {}) {
|
|
|
192
189
|
});
|
|
193
190
|
}
|
|
194
191
|
}
|
|
192
|
+
/**
|
|
193
|
+
* Close the audio stream
|
|
194
|
+
*/
|
|
195
|
+
function closeStream() {
|
|
196
|
+
if (audioStream.current) {
|
|
197
|
+
const tracks = audioStream.current.getTracks();
|
|
198
|
+
tracks.forEach(track => track.stop());
|
|
199
|
+
}
|
|
200
|
+
}
|
|
195
201
|
// Remove listeners and clear the recorded chunks
|
|
196
202
|
useEffect(() => {
|
|
197
203
|
reset();
|
|
198
204
|
return () => {
|
|
205
|
+
closeStream();
|
|
199
206
|
clearListeners();
|
|
200
207
|
};
|
|
201
208
|
}, []);
|
|
@@ -206,7 +213,6 @@ export default function useMediaRecording({ onDone, } = {}) {
|
|
|
206
213
|
duration,
|
|
207
214
|
analyser: analyser.current,
|
|
208
215
|
onError,
|
|
209
|
-
onProcessing,
|
|
210
216
|
controls: {
|
|
211
217
|
start,
|
|
212
218
|
pause,
|
package/build/index.d.ts
CHANGED
|
@@ -6,6 +6,7 @@ export { default as useAiSuggestions } from './hooks/use-ai-suggestions/index.js
|
|
|
6
6
|
export { default as useMediaRecording } from './hooks/use-media-recording/index.js';
|
|
7
7
|
export { default as useAudioTranscription } from './hooks/use-audio-transcription/index.js';
|
|
8
8
|
export { default as useTranscriptionPostProcessing } from './hooks/use-transcription-post-processing/index.js';
|
|
9
|
+
export { default as useAudioValidation } from './hooks/use-audio-validation/index.js';
|
|
9
10
|
export * from './icons/index.js';
|
|
10
11
|
export * from './components/index.js';
|
|
11
12
|
export * from './data-flow/index.js';
|
package/build/index.js
CHANGED
|
@@ -12,6 +12,7 @@ export { default as useAiSuggestions } from './hooks/use-ai-suggestions/index.js
|
|
|
12
12
|
export { default as useMediaRecording } from './hooks/use-media-recording/index.js';
|
|
13
13
|
export { default as useAudioTranscription } from './hooks/use-audio-transcription/index.js';
|
|
14
14
|
export { default as useTranscriptionPostProcessing } from './hooks/use-transcription-post-processing/index.js';
|
|
15
|
+
export { default as useAudioValidation } from './hooks/use-audio-validation/index.js';
|
|
15
16
|
/*
|
|
16
17
|
* Components: Icons
|
|
17
18
|
*/
|
package/build/types.d.ts
CHANGED
|
@@ -17,6 +17,7 @@ export type { UseAiContextOptions } from './data-flow/use-ai-context.js';
|
|
|
17
17
|
export type { RequestingErrorProps } from './hooks/use-ai-suggestions/index.js';
|
|
18
18
|
export type { UseAudioTranscriptionProps, UseAudioTranscriptionReturn, } from './hooks/use-audio-transcription/index.js';
|
|
19
19
|
export type { UseTranscriptionPostProcessingProps, UseTranscriptionPostProcessingReturn, PostProcessingAction, } from './hooks/use-transcription-post-processing/index.js';
|
|
20
|
+
export type { UseAudioValidationReturn, ValidatedAudioInformation, } from './hooks/use-audio-validation/index.js';
|
|
20
21
|
export { TRANSCRIPTION_POST_PROCESSING_ACTION_SIMPLE_DRAFT } from './hooks/use-transcription-post-processing/index.js';
|
|
21
22
|
export declare const REQUESTING_STATES: readonly ["init", "requesting", "suggesting", "done", "error"];
|
|
22
23
|
export type RequestingStateProp = (typeof REQUESTING_STATES)[number];
|
|
@@ -27,6 +28,7 @@ export type { RecordingState } from './hooks/use-media-recording/index.js';
|
|
|
27
28
|
export type CancelablePromise<T = void> = Promise<T> & {
|
|
28
29
|
canceled?: boolean;
|
|
29
30
|
};
|
|
31
|
+
export type TranscriptionState = RecordingState | 'validating' | 'processing' | 'error';
|
|
30
32
|
interface JPConnectionInitialState {
|
|
31
33
|
apiNonce: string;
|
|
32
34
|
siteSuffix: string;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"private": false,
|
|
3
3
|
"name": "@automattic/jetpack-ai-client",
|
|
4
|
-
"version": "0.8.
|
|
4
|
+
"version": "0.8.2",
|
|
5
5
|
"description": "A JS client for consuming Jetpack AI services",
|
|
6
6
|
"homepage": "https://github.com/Automattic/jetpack/tree/HEAD/projects/js-packages/ai-client/#readme",
|
|
7
7
|
"bugs": {
|
|
@@ -38,19 +38,19 @@
|
|
|
38
38
|
"main": "./build/index.js",
|
|
39
39
|
"types": "./build/index.d.ts",
|
|
40
40
|
"dependencies": {
|
|
41
|
-
"@automattic/jetpack-base-styles": "^0.6.
|
|
42
|
-
"@automattic/jetpack-connection": "^0.
|
|
43
|
-
"@automattic/jetpack-shared-extension-utils": "^0.14.
|
|
41
|
+
"@automattic/jetpack-base-styles": "^0.6.18",
|
|
42
|
+
"@automattic/jetpack-connection": "^0.33.1",
|
|
43
|
+
"@automattic/jetpack-shared-extension-utils": "^0.14.5",
|
|
44
44
|
"@microsoft/fetch-event-source": "2.0.1",
|
|
45
|
-
"@types/react": "18.2.
|
|
46
|
-
"@wordpress/api-fetch": "6.
|
|
47
|
-
"@wordpress/block-editor": "12.
|
|
48
|
-
"@wordpress/components": "
|
|
49
|
-
"@wordpress/compose": "6.
|
|
50
|
-
"@wordpress/data": "9.
|
|
51
|
-
"@wordpress/element": "5.
|
|
52
|
-
"@wordpress/i18n": "4.
|
|
53
|
-
"@wordpress/icons": "9.
|
|
45
|
+
"@types/react": "18.2.61",
|
|
46
|
+
"@wordpress/api-fetch": "6.49.0",
|
|
47
|
+
"@wordpress/block-editor": "12.20.0",
|
|
48
|
+
"@wordpress/components": "27.0.0",
|
|
49
|
+
"@wordpress/compose": "6.29.0",
|
|
50
|
+
"@wordpress/data": "9.22.0",
|
|
51
|
+
"@wordpress/element": "5.29.0",
|
|
52
|
+
"@wordpress/i18n": "4.52.0",
|
|
53
|
+
"@wordpress/icons": "9.43.0",
|
|
54
54
|
"classnames": "2.3.2",
|
|
55
55
|
"debug": "4.3.4",
|
|
56
56
|
"react": "18.2.0",
|
|
@@ -5,21 +5,10 @@ import debugFactory from 'debug';
|
|
|
5
5
|
/**
|
|
6
6
|
* Internal dependencies
|
|
7
7
|
*/
|
|
8
|
-
import apiFetch from '../api-fetch/index.js';
|
|
9
8
|
import requestJwt from '../jwt/index.js';
|
|
10
9
|
|
|
11
10
|
const debug = debugFactory( 'jetpack-ai-client:audio-transcription' );
|
|
12
11
|
|
|
13
|
-
/**
|
|
14
|
-
* The response from the audio transcription service.
|
|
15
|
-
*/
|
|
16
|
-
type AudioTranscriptionResponse = {
|
|
17
|
-
/**
|
|
18
|
-
* The transcribed text.
|
|
19
|
-
*/
|
|
20
|
-
text: string;
|
|
21
|
-
};
|
|
22
|
-
|
|
23
12
|
/**
|
|
24
13
|
* A function that takes an audio blob and transcribes it.
|
|
25
14
|
*
|
|
@@ -53,19 +42,22 @@ export default async function transcribeAudio(
|
|
|
53
42
|
Authorization: `Bearer ${ token }`,
|
|
54
43
|
};
|
|
55
44
|
|
|
56
|
-
const
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
45
|
+
const URL = `https://public-api.wordpress.com/wpcom/v2/jetpack-ai-transcription${
|
|
46
|
+
feature ? `?feature=${ feature }` : ''
|
|
47
|
+
}`;
|
|
48
|
+
|
|
49
|
+
return fetch( URL, {
|
|
60
50
|
method: 'POST',
|
|
61
51
|
body: formData,
|
|
62
52
|
headers,
|
|
63
53
|
signal: requestAbortSignal ?? undefined,
|
|
54
|
+
} ).then( response => {
|
|
55
|
+
debug( 'Transcription response: %o', response );
|
|
56
|
+
if ( response.ok ) {
|
|
57
|
+
return response.json().then( data => data?.text );
|
|
58
|
+
}
|
|
59
|
+
return response.json().then( data => Promise.reject( data ) );
|
|
64
60
|
} );
|
|
65
|
-
|
|
66
|
-
debug( 'Transcription response: %o', response );
|
|
67
|
-
|
|
68
|
-
return response.text;
|
|
69
61
|
} catch ( error ) {
|
|
70
62
|
debug( 'Transcription error response: %o', error );
|
|
71
63
|
return Promise.reject( error );
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* External dependencies
|
|
3
3
|
*/
|
|
4
4
|
import { useCallback, useState, useRef } from '@wordpress/element';
|
|
5
|
+
import { __ } from '@wordpress/i18n';
|
|
5
6
|
import debugFactory from 'debug';
|
|
6
7
|
/**
|
|
7
8
|
* Internal dependencies
|
|
@@ -30,6 +31,63 @@ export type UseAudioTranscriptionProps = {
|
|
|
30
31
|
onError?: ( error: string ) => void;
|
|
31
32
|
};
|
|
32
33
|
|
|
34
|
+
/**
|
|
35
|
+
* The error response from the audio transcription service.
|
|
36
|
+
*/
|
|
37
|
+
type AudioTranscriptionErrorResponse = {
|
|
38
|
+
/**
|
|
39
|
+
* The error message.
|
|
40
|
+
*/
|
|
41
|
+
message: string;
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The error code.
|
|
45
|
+
*/
|
|
46
|
+
code: string;
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Map error response to a string.
|
|
51
|
+
* @param {Error | string | AudioTranscriptionErrorResponse} error - The error response from the audio transcription service.
|
|
52
|
+
* @returns {string} the translated error message
|
|
53
|
+
*/
|
|
54
|
+
const mapErrorResponse = ( error: Error | string | AudioTranscriptionErrorResponse ): string => {
|
|
55
|
+
if ( typeof error === 'string' ) {
|
|
56
|
+
return error;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
if ( 'code' in error ) {
|
|
60
|
+
switch ( error.code ) {
|
|
61
|
+
case 'error_quota_exceeded':
|
|
62
|
+
return __(
|
|
63
|
+
'You exceeded your current quota, please check your plan details.',
|
|
64
|
+
'jetpack-ai-client'
|
|
65
|
+
);
|
|
66
|
+
case 'jetpack_ai_missing_audio_param':
|
|
67
|
+
return __( 'The audio_file is required to perform a transcription.', 'jetpack-ai-client' );
|
|
68
|
+
case 'jetpack_ai_service_unavailable':
|
|
69
|
+
return __( 'The Jetpack AI service is temporarily unavailable.', 'jetpack-ai-client' );
|
|
70
|
+
case 'file_size_not_supported':
|
|
71
|
+
return __( 'The provided audio file is too big.', 'jetpack-ai-client' );
|
|
72
|
+
case 'file_type_not_supported':
|
|
73
|
+
return __( 'The provided audio file type is not supported.', 'jetpack-ai-client' );
|
|
74
|
+
case 'jetpack_ai_error':
|
|
75
|
+
return __(
|
|
76
|
+
'There was an error processing the transcription request.',
|
|
77
|
+
'jetpack-ai-client'
|
|
78
|
+
);
|
|
79
|
+
default:
|
|
80
|
+
return error.message;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
if ( 'message' in error ) {
|
|
85
|
+
return error.message;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return __( 'There was an error processing the transcription request.', 'jetpack-ai-client' );
|
|
89
|
+
};
|
|
90
|
+
|
|
33
91
|
/**
|
|
34
92
|
* A hook to handle audio transcription.
|
|
35
93
|
*
|
|
@@ -74,7 +132,7 @@ export default function useAudioTranscription( {
|
|
|
74
132
|
.catch( error => {
|
|
75
133
|
if ( ! controller.signal.aborted ) {
|
|
76
134
|
setTranscriptionError( error.message );
|
|
77
|
-
onError?.( error
|
|
135
|
+
onError?.( mapErrorResponse( error ) );
|
|
78
136
|
}
|
|
79
137
|
} )
|
|
80
138
|
.finally( () => setIsTranscribingAudio( false ) );
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* External dependencies
|
|
3
|
+
*/
|
|
4
|
+
import { useCallback, useState } from '@wordpress/element';
|
|
5
|
+
import { __ } from '@wordpress/i18n';
|
|
6
|
+
|
|
7
|
+
const MAX_AUDIO_SIZE = 25000000; // 25MB
|
|
8
|
+
const MAX_AUDIO_DURATION = 25 * 60; // 25 minutes
|
|
9
|
+
const ALLOWED_MEDIA_TYPES = [
|
|
10
|
+
'audio/mpeg',
|
|
11
|
+
'audio/mp3',
|
|
12
|
+
'audio/ogg',
|
|
13
|
+
'audio/flac',
|
|
14
|
+
'audio/x-flac',
|
|
15
|
+
'audio/m4a',
|
|
16
|
+
'audio/x-m4a',
|
|
17
|
+
'audio/mp4',
|
|
18
|
+
'audio/wav',
|
|
19
|
+
'audio/wave',
|
|
20
|
+
'audio/x-wav',
|
|
21
|
+
'audio/webm',
|
|
22
|
+
];
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* The return value for the audio validation hook.
|
|
26
|
+
*/
|
|
27
|
+
export type UseAudioValidationReturn = {
|
|
28
|
+
isValidatingAudio: boolean;
|
|
29
|
+
validateAudio: (
|
|
30
|
+
audio: Blob,
|
|
31
|
+
successCallback: ( info?: ValidatedAudioInformation ) => void,
|
|
32
|
+
errorCallback: ( error: string ) => void
|
|
33
|
+
) => void;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* The validated audio information.
|
|
38
|
+
*/
|
|
39
|
+
export type ValidatedAudioInformation = {
|
|
40
|
+
duration: number;
|
|
41
|
+
isFile: boolean;
|
|
42
|
+
size: number;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Hook to handle the validation of an audio file.
|
|
47
|
+
*
|
|
48
|
+
* @returns {UseAudioValidationReturn} - Object with the audio validation state and the function to validate the audio.
|
|
49
|
+
*/
|
|
50
|
+
export default function useAudioValidation(): UseAudioValidationReturn {
|
|
51
|
+
const [ isValidatingAudio, setIsValidatingAudio ] = useState< boolean >( false );
|
|
52
|
+
|
|
53
|
+
const validateAudio = useCallback(
|
|
54
|
+
(
|
|
55
|
+
audio: Blob,
|
|
56
|
+
successCallback: ( info?: ValidatedAudioInformation ) => void,
|
|
57
|
+
errorCallback: ( error: string ) => void
|
|
58
|
+
) => {
|
|
59
|
+
setIsValidatingAudio( true );
|
|
60
|
+
|
|
61
|
+
// Check if the audio file is too large
|
|
62
|
+
if ( audio?.size > MAX_AUDIO_SIZE ) {
|
|
63
|
+
setIsValidatingAudio( false );
|
|
64
|
+
return errorCallback(
|
|
65
|
+
__( 'The audio file is too large. The maximum file size is 25MB.', 'jetpack-ai-client' )
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// When it's a file, check the media type
|
|
70
|
+
const isFile = audio instanceof File;
|
|
71
|
+
if ( isFile ) {
|
|
72
|
+
if ( ! ALLOWED_MEDIA_TYPES.includes( audio.type ) ) {
|
|
73
|
+
setIsValidatingAudio( false );
|
|
74
|
+
return errorCallback(
|
|
75
|
+
__(
|
|
76
|
+
'The audio file type is not supported. Please use a supported audio file type.',
|
|
77
|
+
'jetpack-ai-client'
|
|
78
|
+
)
|
|
79
|
+
);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Check the duration of the audio
|
|
84
|
+
const audioContext = new AudioContext();
|
|
85
|
+
|
|
86
|
+
// Map blob to an array buffer
|
|
87
|
+
audio.arrayBuffer().then( arrayBuffer => {
|
|
88
|
+
// Decode audio file data contained in an ArrayBuffer
|
|
89
|
+
audioContext.decodeAudioData( arrayBuffer, function ( audioBuffer ) {
|
|
90
|
+
const duration = Math.ceil( audioBuffer.duration );
|
|
91
|
+
|
|
92
|
+
if ( duration > MAX_AUDIO_DURATION ) {
|
|
93
|
+
setIsValidatingAudio( false );
|
|
94
|
+
return errorCallback(
|
|
95
|
+
__(
|
|
96
|
+
'The audio file is too long. The maximum recording time is 25 minutes.',
|
|
97
|
+
'jetpack-ai-client'
|
|
98
|
+
)
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
setIsValidatingAudio( false );
|
|
102
|
+
return successCallback( { duration, isFile, size: audio?.size } );
|
|
103
|
+
} );
|
|
104
|
+
} );
|
|
105
|
+
},
|
|
106
|
+
[ setIsValidatingAudio ]
|
|
107
|
+
);
|
|
108
|
+
|
|
109
|
+
return { isValidatingAudio, validateAudio };
|
|
110
|
+
}
|
|
@@ -5,7 +5,7 @@ import { useRef, useState, useEffect, useCallback } from '@wordpress/element';
|
|
|
5
5
|
/*
|
|
6
6
|
* Types
|
|
7
7
|
*/
|
|
8
|
-
export type RecordingState = 'inactive' | 'recording' | 'paused' | '
|
|
8
|
+
export type RecordingState = 'inactive' | 'recording' | 'paused' | 'error';
|
|
9
9
|
type UseMediaRecordingProps = {
|
|
10
10
|
onDone?: ( blob: Blob ) => void;
|
|
11
11
|
};
|
|
@@ -41,11 +41,6 @@ type UseMediaRecordingReturn = {
|
|
|
41
41
|
*/
|
|
42
42
|
onError: ( err: string | Error ) => void;
|
|
43
43
|
|
|
44
|
-
/**
|
|
45
|
-
* The processing handler
|
|
46
|
-
*/
|
|
47
|
-
onProcessing: () => void;
|
|
48
|
-
|
|
49
44
|
controls: {
|
|
50
45
|
/**
|
|
51
46
|
* `start` recording handler
|
|
@@ -90,7 +85,7 @@ export default function useMediaRecording( {
|
|
|
90
85
|
// Reference to the media recorder instance
|
|
91
86
|
const mediaRecordRef = useRef( null );
|
|
92
87
|
|
|
93
|
-
// Recording state: `inactive`, `recording`, `paused`, `
|
|
88
|
+
// Recording state: `inactive`, `recording`, `paused`, `error`
|
|
94
89
|
const [ state, setState ] = useState< RecordingState >( 'inactive' );
|
|
95
90
|
|
|
96
91
|
// reference to the paused state to be used in the `onDataAvailable` event listener,
|
|
@@ -101,6 +96,8 @@ export default function useMediaRecording( {
|
|
|
101
96
|
const recordStartTimestamp = useRef< number >( 0 );
|
|
102
97
|
const [ duration, setDuration ] = useState< number >( 0 );
|
|
103
98
|
|
|
99
|
+
const audioStream = useRef< MediaStream | null >( null );
|
|
100
|
+
|
|
104
101
|
// The recorded blob
|
|
105
102
|
const [ blob, setBlob ] = useState< Blob | null >( null );
|
|
106
103
|
|
|
@@ -216,6 +213,7 @@ export default function useMediaRecording( {
|
|
|
216
213
|
navigator.mediaDevices
|
|
217
214
|
.getUserMedia( constraints )
|
|
218
215
|
.then( stream => {
|
|
216
|
+
audioStream.current = stream;
|
|
219
217
|
const source = audioCtx.createMediaStreamSource( stream );
|
|
220
218
|
source.connect( analyser.current );
|
|
221
219
|
|
|
@@ -239,11 +237,6 @@ export default function useMediaRecording( {
|
|
|
239
237
|
setState( 'error' );
|
|
240
238
|
}, [] );
|
|
241
239
|
|
|
242
|
-
// manually set the state to `processing` for the file upload case
|
|
243
|
-
const onProcessing = useCallback( () => {
|
|
244
|
-
setState( 'processing' );
|
|
245
|
-
}, [] );
|
|
246
|
-
|
|
247
240
|
/**
|
|
248
241
|
* `start` event listener for the media recorder instance.
|
|
249
242
|
*/
|
|
@@ -258,7 +251,6 @@ export default function useMediaRecording( {
|
|
|
258
251
|
* @returns {void}
|
|
259
252
|
*/
|
|
260
253
|
function onStopListener(): void {
|
|
261
|
-
setState( 'processing' );
|
|
262
254
|
const lastBlob = getBlob();
|
|
263
255
|
onDone?.( lastBlob );
|
|
264
256
|
|
|
@@ -310,11 +302,22 @@ export default function useMediaRecording( {
|
|
|
310
302
|
}
|
|
311
303
|
}
|
|
312
304
|
|
|
305
|
+
/**
|
|
306
|
+
* Close the audio stream
|
|
307
|
+
*/
|
|
308
|
+
function closeStream() {
|
|
309
|
+
if ( audioStream.current ) {
|
|
310
|
+
const tracks = audioStream.current.getTracks();
|
|
311
|
+
tracks.forEach( track => track.stop() );
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
313
315
|
// Remove listeners and clear the recorded chunks
|
|
314
316
|
useEffect( () => {
|
|
315
317
|
reset();
|
|
316
318
|
|
|
317
319
|
return () => {
|
|
320
|
+
closeStream();
|
|
318
321
|
clearListeners();
|
|
319
322
|
};
|
|
320
323
|
}, [] );
|
|
@@ -326,7 +329,6 @@ export default function useMediaRecording( {
|
|
|
326
329
|
duration,
|
|
327
330
|
analyser: analyser.current,
|
|
328
331
|
onError,
|
|
329
|
-
onProcessing,
|
|
330
332
|
|
|
331
333
|
controls: {
|
|
332
334
|
start,
|
package/src/index.ts
CHANGED
|
@@ -13,6 +13,7 @@ export { default as useAiSuggestions } from './hooks/use-ai-suggestions/index.js
|
|
|
13
13
|
export { default as useMediaRecording } from './hooks/use-media-recording/index.js';
|
|
14
14
|
export { default as useAudioTranscription } from './hooks/use-audio-transcription/index.js';
|
|
15
15
|
export { default as useTranscriptionPostProcessing } from './hooks/use-transcription-post-processing/index.js';
|
|
16
|
+
export { default as useAudioValidation } from './hooks/use-audio-validation/index.js';
|
|
16
17
|
|
|
17
18
|
/*
|
|
18
19
|
* Components: Icons
|
package/src/types.ts
CHANGED
|
@@ -46,6 +46,11 @@ export type {
|
|
|
46
46
|
UseTranscriptionPostProcessingReturn,
|
|
47
47
|
PostProcessingAction,
|
|
48
48
|
} from './hooks/use-transcription-post-processing/index.js';
|
|
49
|
+
export type {
|
|
50
|
+
UseAudioValidationReturn,
|
|
51
|
+
ValidatedAudioInformation,
|
|
52
|
+
} from './hooks/use-audio-validation/index.js';
|
|
53
|
+
|
|
49
54
|
/*
|
|
50
55
|
* Hook constants
|
|
51
56
|
*/
|
|
@@ -88,6 +93,11 @@ export type { RecordingState } from './hooks/use-media-recording/index.js';
|
|
|
88
93
|
*/
|
|
89
94
|
export type CancelablePromise< T = void > = Promise< T > & { canceled?: boolean };
|
|
90
95
|
|
|
96
|
+
/*
|
|
97
|
+
* Transcription types
|
|
98
|
+
*/
|
|
99
|
+
export type TranscriptionState = RecordingState | 'validating' | 'processing' | 'error';
|
|
100
|
+
|
|
91
101
|
// Connection initial state
|
|
92
102
|
// @todo: it should be provided by the connection package
|
|
93
103
|
interface JPConnectionInitialState {
|