@automattic/jetpack-ai-client 0.6.1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/build/api-fetch/index.d.ts +7 -0
- package/build/api-fetch/index.js +6 -0
- package/build/ask-question/index.d.ts +2 -2
- package/build/ask-question/index.js +1 -1
- package/build/audio-transcription/index.d.ts +12 -0
- package/build/audio-transcription/index.js +51 -0
- package/build/components/ai-control/index.d.ts +7 -6
- package/build/components/ai-control/index.js +6 -6
- package/build/components/ai-control/message.d.ts +10 -0
- package/build/components/ai-control/message.js +15 -2
- package/build/components/ai-status-indicator/index.d.ts +1 -1
- package/build/components/audio-duration-display/index.d.ts +3 -2
- package/build/components/audio-duration-display/index.js +3 -14
- package/build/components/audio-duration-display/lib/media.d.ts +13 -14
- package/build/components/audio-duration-display/lib/media.js +7 -32
- package/build/components/index.d.ts +4 -4
- package/build/components/index.js +4 -4
- package/build/data-flow/context.d.ts +9 -6
- package/build/data-flow/context.js +1 -1
- package/build/data-flow/index.d.ts +3 -3
- package/build/data-flow/index.js +3 -3
- package/build/data-flow/use-ai-context.d.ts +3 -3
- package/build/data-flow/use-ai-context.js +3 -3
- package/build/data-flow/with-ai-assistant-data.js +2 -2
- package/build/hooks/use-ai-suggestions/index.d.ts +4 -4
- package/build/hooks/use-ai-suggestions/index.js +2 -2
- package/build/hooks/use-audio-transcription/index.d.ts +28 -0
- package/build/hooks/use-audio-transcription/index.js +56 -0
- package/build/hooks/use-media-recording/index.d.ts +34 -12
- package/build/hooks/use-media-recording/index.js +116 -41
- package/build/hooks/use-transcription-post-processing/index.d.ts +30 -0
- package/build/hooks/use-transcription-post-processing/index.js +84 -0
- package/build/icons/index.d.ts +7 -7
- package/build/icons/index.js +7 -7
- package/build/icons/mic.js +2 -2
- package/build/icons/player-pause.js +1 -1
- package/build/index.d.ts +12 -9
- package/build/index.js +12 -9
- package/build/jwt/index.js +4 -1
- package/build/suggestions-event-source/index.d.ts +1 -1
- package/build/suggestions-event-source/index.js +3 -3
- package/build/types.d.ts +9 -2
- package/build/types.js +4 -0
- package/package.json +5 -5
- package/src/api-fetch/index.ts +13 -0
- package/src/ask-question/index.ts +2 -2
- package/src/audio-transcription/index.ts +75 -0
- package/src/components/ai-control/index.tsx +12 -16
- package/src/components/ai-control/message.tsx +34 -2
- package/src/components/ai-control/style.scss +6 -0
- package/src/components/ai-status-indicator/index.tsx +1 -1
- package/src/components/audio-duration-display/index.tsx +6 -17
- package/src/components/audio-duration-display/lib/media.ts +17 -40
- package/src/components/index.ts +8 -4
- package/src/data-flow/context.tsx +7 -8
- package/src/data-flow/index.ts +3 -3
- package/src/data-flow/use-ai-context.ts +6 -6
- package/src/data-flow/with-ai-assistant-data.tsx +2 -2
- package/src/hooks/use-ai-suggestions/index.ts +6 -6
- package/src/hooks/use-audio-transcription/index.ts +95 -0
- package/src/hooks/use-media-recording/Readme.md +2 -2
- package/src/hooks/use-media-recording/index.ts +179 -59
- package/src/hooks/use-transcription-post-processing/index.ts +135 -0
- package/src/icons/index.ts +7 -7
- package/src/icons/mic.tsx +14 -16
- package/src/icons/player-pause.tsx +5 -5
- package/src/index.ts +12 -9
- package/src/jwt/index.ts +4 -1
- package/src/suggestions-event-source/index.ts +4 -4
- package/src/types.ts +29 -2
|
@@ -23,7 +23,7 @@ The hook returns an object with the following properties and methods:
|
|
|
23
23
|
Here's an example React component that utilizes the `useMediaRecording` hook.
|
|
24
24
|
|
|
25
25
|
```jsx
|
|
26
|
-
import useMediaRecording from './useMediaRecording';
|
|
26
|
+
import useMediaRecording from './useMediaRecording/index.js';
|
|
27
27
|
|
|
28
28
|
const MediaRecorderComponent = () => {
|
|
29
29
|
const { start, pause, resume, stop, state } = useMediaRecording();
|
|
@@ -47,4 +47,4 @@ const MediaRecorderComponent = () => {
|
|
|
47
47
|
</div>
|
|
48
48
|
);
|
|
49
49
|
};
|
|
50
|
-
```
|
|
50
|
+
```
|
|
@@ -2,11 +2,10 @@
|
|
|
2
2
|
* External dependencies
|
|
3
3
|
*/
|
|
4
4
|
import { useRef, useState, useEffect, useCallback } from '@wordpress/element';
|
|
5
|
-
|
|
6
5
|
/*
|
|
7
6
|
* Types
|
|
8
7
|
*/
|
|
9
|
-
type
|
|
8
|
+
export type RecordingState = 'inactive' | 'recording' | 'paused' | 'processing' | 'error';
|
|
10
9
|
type UseMediaRecordingProps = {
|
|
11
10
|
onDone?: ( blob: Blob ) => void;
|
|
12
11
|
};
|
|
@@ -15,7 +14,7 @@ type UseMediaRecordingReturn = {
|
|
|
15
14
|
/**
|
|
16
15
|
* The current recording state
|
|
17
16
|
*/
|
|
18
|
-
state:
|
|
17
|
+
state: RecordingState;
|
|
19
18
|
|
|
20
19
|
/**
|
|
21
20
|
* The recorded blob
|
|
@@ -23,29 +22,56 @@ type UseMediaRecordingReturn = {
|
|
|
23
22
|
blob: Blob | null;
|
|
24
23
|
|
|
25
24
|
/**
|
|
26
|
-
* The
|
|
25
|
+
* The error message
|
|
27
26
|
*/
|
|
28
|
-
|
|
27
|
+
error: string | null;
|
|
29
28
|
|
|
30
29
|
/**
|
|
31
|
-
*
|
|
30
|
+
* The duration of the recorded audio
|
|
32
31
|
*/
|
|
33
|
-
|
|
32
|
+
duration: number;
|
|
34
33
|
|
|
35
34
|
/**
|
|
36
|
-
*
|
|
35
|
+
* The audio analyser node
|
|
37
36
|
*/
|
|
38
|
-
|
|
37
|
+
analyser?: AnalyserNode;
|
|
39
38
|
|
|
40
39
|
/**
|
|
41
|
-
*
|
|
40
|
+
* The error handler
|
|
42
41
|
*/
|
|
43
|
-
|
|
42
|
+
onError: ( err: string | Error ) => void;
|
|
44
43
|
|
|
45
44
|
/**
|
|
46
|
-
*
|
|
45
|
+
* The processing handler
|
|
47
46
|
*/
|
|
48
|
-
|
|
47
|
+
onProcessing: () => void;
|
|
48
|
+
|
|
49
|
+
controls: {
|
|
50
|
+
/**
|
|
51
|
+
* `start` recording handler
|
|
52
|
+
*/
|
|
53
|
+
start: ( timeslice?: number ) => void;
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* `pause` recording handler
|
|
57
|
+
*/
|
|
58
|
+
pause: () => void;
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* `resume` recording handler
|
|
62
|
+
*/
|
|
63
|
+
resume: () => void;
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* `stop` recording handler
|
|
67
|
+
*/
|
|
68
|
+
stop: () => void;
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* `reset` recording handler
|
|
72
|
+
*/
|
|
73
|
+
reset: () => void;
|
|
74
|
+
};
|
|
49
75
|
};
|
|
50
76
|
|
|
51
77
|
type MediaRecorderEvent = {
|
|
@@ -64,8 +90,16 @@ export default function useMediaRecording( {
|
|
|
64
90
|
// Reference to the media recorder instance
|
|
65
91
|
const mediaRecordRef = useRef( null );
|
|
66
92
|
|
|
67
|
-
// Recording state: `inactive`, `recording`, `paused`
|
|
68
|
-
const [ state, setState ] = useState<
|
|
93
|
+
// Recording state: `inactive`, `recording`, `paused`, `processing`, `error`
|
|
94
|
+
const [ state, setState ] = useState< RecordingState >( 'inactive' );
|
|
95
|
+
|
|
96
|
+
// reference to the paused state to be used in the `onDataAvailable` event listener,
|
|
97
|
+
// as the `mediaRecordRef.current.state` is already `inactive` when the recorder is stopped,
|
|
98
|
+
// and the event listener does not react to state changes
|
|
99
|
+
const isPaused = useRef< boolean >( false );
|
|
100
|
+
|
|
101
|
+
const recordStartTimestamp = useRef< number >( 0 );
|
|
102
|
+
const [ duration, setDuration ] = useState< number >( 0 );
|
|
69
103
|
|
|
70
104
|
// The recorded blob
|
|
71
105
|
const [ blob, setBlob ] = useState< Blob | null >( null );
|
|
@@ -73,6 +107,10 @@ export default function useMediaRecording( {
|
|
|
73
107
|
// Store the recorded chunks
|
|
74
108
|
const recordedChunks = useRef< Array< Blob > >( [] ).current;
|
|
75
109
|
|
|
110
|
+
const [ error, setError ] = useState< string | null >( null );
|
|
111
|
+
|
|
112
|
+
const analyser = useRef< AnalyserNode >( null );
|
|
113
|
+
|
|
76
114
|
/**
|
|
77
115
|
* Get the recorded blob.
|
|
78
116
|
*
|
|
@@ -86,6 +124,8 @@ export default function useMediaRecording( {
|
|
|
86
124
|
|
|
87
125
|
// `start` recording handler
|
|
88
126
|
const start = useCallback( ( timeslice: number ) => {
|
|
127
|
+
clearData();
|
|
128
|
+
|
|
89
129
|
if ( ! timeslice ) {
|
|
90
130
|
return mediaRecordRef?.current?.start();
|
|
91
131
|
}
|
|
@@ -93,22 +133,115 @@ export default function useMediaRecording( {
|
|
|
93
133
|
if ( timeslice < 100 ) {
|
|
94
134
|
timeslice = 100; // set minimum timeslice to 100ms
|
|
95
135
|
}
|
|
136
|
+
|
|
137
|
+
// Record the start time
|
|
138
|
+
recordStartTimestamp.current = Date.now();
|
|
139
|
+
|
|
96
140
|
mediaRecordRef?.current?.start( timeslice );
|
|
97
141
|
}, [] );
|
|
98
142
|
|
|
99
143
|
// `pause` recording handler
|
|
100
144
|
const pause = useCallback( () => {
|
|
145
|
+
isPaused.current = true;
|
|
101
146
|
mediaRecordRef?.current?.pause();
|
|
147
|
+
|
|
148
|
+
// Calculate the duration of the recorded audio from the start time
|
|
149
|
+
setDuration( currentDuration => currentDuration + Date.now() - recordStartTimestamp.current );
|
|
102
150
|
}, [] );
|
|
103
151
|
|
|
104
152
|
// `resume` recording handler
|
|
105
153
|
const resume = useCallback( () => {
|
|
154
|
+
isPaused.current = false;
|
|
106
155
|
mediaRecordRef?.current?.resume();
|
|
156
|
+
|
|
157
|
+
// Record the start time
|
|
158
|
+
recordStartTimestamp.current = Date.now();
|
|
107
159
|
}, [] );
|
|
108
160
|
|
|
109
161
|
// `stop` recording handler
|
|
110
162
|
const stop = useCallback( () => {
|
|
111
163
|
mediaRecordRef?.current?.stop();
|
|
164
|
+
|
|
165
|
+
if ( state === 'recording' ) {
|
|
166
|
+
// Calculate the duration of the recorded audio from the start time
|
|
167
|
+
setDuration( currentDuration => currentDuration + Date.now() - recordStartTimestamp.current );
|
|
168
|
+
}
|
|
169
|
+
}, [] );
|
|
170
|
+
|
|
171
|
+
// clears the recording state
|
|
172
|
+
const clearData = useCallback( () => {
|
|
173
|
+
recordedChunks.length = 0;
|
|
174
|
+
setBlob( null );
|
|
175
|
+
setError( null );
|
|
176
|
+
setDuration( 0 );
|
|
177
|
+
isPaused.current = false;
|
|
178
|
+
recordStartTimestamp.current = 0;
|
|
179
|
+
}, [] );
|
|
180
|
+
|
|
181
|
+
// removes the event listeners
|
|
182
|
+
const clearListeners = useCallback( () => {
|
|
183
|
+
/*
|
|
184
|
+
* mediaRecordRef is not defined when
|
|
185
|
+
* the getUserMedia API is not supported,
|
|
186
|
+
* or when the user has not granted access
|
|
187
|
+
*/
|
|
188
|
+
if ( ! mediaRecordRef?.current ) {
|
|
189
|
+
return;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
mediaRecordRef.current.removeEventListener( 'start', onStartListener );
|
|
193
|
+
mediaRecordRef.current.removeEventListener( 'stop', onStopListener );
|
|
194
|
+
mediaRecordRef.current.removeEventListener( 'pause', onPauseListener );
|
|
195
|
+
mediaRecordRef.current.removeEventListener( 'resume', onResumeListener );
|
|
196
|
+
mediaRecordRef.current.removeEventListener( 'dataavailable', onDataAvailableListener );
|
|
197
|
+
mediaRecordRef.current = null;
|
|
198
|
+
}, [] );
|
|
199
|
+
|
|
200
|
+
// resets the recording state, initializing the media recorder instance
|
|
201
|
+
const reset = useCallback( () => {
|
|
202
|
+
setState( 'inactive' );
|
|
203
|
+
clearData();
|
|
204
|
+
clearListeners();
|
|
205
|
+
|
|
206
|
+
// Check if the getUserMedia API is supported
|
|
207
|
+
if ( ! navigator.mediaDevices?.getUserMedia ) {
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const audioCtx = new AudioContext();
|
|
212
|
+
analyser.current = audioCtx.createAnalyser();
|
|
213
|
+
|
|
214
|
+
const constraints = { audio: true };
|
|
215
|
+
|
|
216
|
+
navigator.mediaDevices
|
|
217
|
+
.getUserMedia( constraints )
|
|
218
|
+
.then( stream => {
|
|
219
|
+
const source = audioCtx.createMediaStreamSource( stream );
|
|
220
|
+
source.connect( analyser.current );
|
|
221
|
+
|
|
222
|
+
mediaRecordRef.current = new MediaRecorder( stream );
|
|
223
|
+
mediaRecordRef.current.addEventListener( 'start', onStartListener );
|
|
224
|
+
mediaRecordRef.current.addEventListener( 'stop', onStopListener );
|
|
225
|
+
mediaRecordRef.current.addEventListener( 'pause', onPauseListener );
|
|
226
|
+
mediaRecordRef.current.addEventListener( 'resume', onResumeListener );
|
|
227
|
+
mediaRecordRef.current.addEventListener( 'dataavailable', onDataAvailableListener );
|
|
228
|
+
} )
|
|
229
|
+
.catch( err => {
|
|
230
|
+
// @todo: handle error
|
|
231
|
+
throw err;
|
|
232
|
+
} );
|
|
233
|
+
}, [] );
|
|
234
|
+
|
|
235
|
+
// stops the recording and sets the error state
|
|
236
|
+
const onError = useCallback( ( err: string | Error ) => {
|
|
237
|
+
stop();
|
|
238
|
+
setError( typeof err === 'string' ? err : err.message );
|
|
239
|
+
setState( 'error' );
|
|
240
|
+
}, [] );
|
|
241
|
+
|
|
242
|
+
// manually set the state to `processing` for the file upload case
|
|
243
|
+
const onProcessing = useCallback( () => {
|
|
244
|
+
setState( 'processing' );
|
|
112
245
|
}, [] );
|
|
113
246
|
|
|
114
247
|
/**
|
|
@@ -120,12 +253,14 @@ export default function useMediaRecording( {
|
|
|
120
253
|
|
|
121
254
|
/**
|
|
122
255
|
* `stop` event listener for the media recorder instance.
|
|
256
|
+
* Happens after the last `dataavailable` event.
|
|
123
257
|
*
|
|
124
258
|
* @returns {void}
|
|
125
259
|
*/
|
|
126
260
|
function onStopListener(): void {
|
|
127
|
-
setState( '
|
|
128
|
-
|
|
261
|
+
setState( 'processing' );
|
|
262
|
+
const lastBlob = getBlob();
|
|
263
|
+
onDone?.( lastBlob );
|
|
129
264
|
|
|
130
265
|
// Clear the recorded chunks
|
|
131
266
|
recordedChunks.length = 0;
|
|
@@ -162,58 +297,43 @@ export default function useMediaRecording( {
|
|
|
162
297
|
|
|
163
298
|
// Create and store the Blob for the recorded chunks
|
|
164
299
|
setBlob( getBlob() );
|
|
165
|
-
}
|
|
166
300
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
301
|
+
// If the recorder was paused, it is the last data available event, so we do not update the duration
|
|
302
|
+
if ( ! isPaused.current ) {
|
|
303
|
+
setDuration( currentDuration => {
|
|
304
|
+
const now = Date.now();
|
|
305
|
+
const difference = now - recordStartTimestamp.current;
|
|
306
|
+
// Update the start time
|
|
307
|
+
recordStartTimestamp.current = now;
|
|
308
|
+
return currentDuration + difference;
|
|
309
|
+
} );
|
|
172
310
|
}
|
|
311
|
+
}
|
|
173
312
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
.getUserMedia( constraints )
|
|
178
|
-
.then( stream => {
|
|
179
|
-
mediaRecordRef.current = new MediaRecorder( stream );
|
|
313
|
+
// Remove listeners and clear the recorded chunks
|
|
314
|
+
useEffect( () => {
|
|
315
|
+
reset();
|
|
180
316
|
|
|
181
|
-
mediaRecordRef.current.addEventListener( 'start', onStartListener );
|
|
182
|
-
mediaRecordRef.current.addEventListener( 'stop', onStopListener );
|
|
183
|
-
mediaRecordRef.current.addEventListener( 'pause', onPauseListener );
|
|
184
|
-
mediaRecordRef.current.addEventListener( 'resume', onResumeListener );
|
|
185
|
-
mediaRecordRef.current.addEventListener( 'dataavailable', onDataAvailableListener );
|
|
186
|
-
} )
|
|
187
|
-
.catch( err => {
|
|
188
|
-
// @todo: handle error
|
|
189
|
-
throw err;
|
|
190
|
-
} );
|
|
191
317
|
return () => {
|
|
192
|
-
|
|
193
|
-
* mediaRecordRef is not defined when
|
|
194
|
-
* the getUserMedia API is not supported,
|
|
195
|
-
* or when the user has not granted access
|
|
196
|
-
*/
|
|
197
|
-
if ( ! mediaRecordRef?.current ) {
|
|
198
|
-
return;
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
mediaRecordRef.current.removeEventListener( 'start', onStartListener );
|
|
202
|
-
mediaRecordRef.current.removeEventListener( 'stop', onStopListener );
|
|
203
|
-
mediaRecordRef.current.removeEventListener( 'pause', onPauseListener );
|
|
204
|
-
mediaRecordRef.current.removeEventListener( 'resume', onResumeListener );
|
|
205
|
-
mediaRecordRef.current.removeEventListener( 'dataavailable', onDataAvailableListener );
|
|
318
|
+
clearListeners();
|
|
206
319
|
};
|
|
207
320
|
}, [] );
|
|
208
321
|
|
|
209
322
|
return {
|
|
210
323
|
state,
|
|
211
324
|
blob,
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
325
|
+
error,
|
|
326
|
+
duration,
|
|
327
|
+
analyser: analyser.current,
|
|
328
|
+
onError,
|
|
329
|
+
onProcessing,
|
|
330
|
+
|
|
331
|
+
controls: {
|
|
332
|
+
start,
|
|
333
|
+
pause,
|
|
334
|
+
resume,
|
|
335
|
+
stop,
|
|
336
|
+
reset,
|
|
337
|
+
},
|
|
218
338
|
};
|
|
219
339
|
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* External dependencies
|
|
3
|
+
*/
|
|
4
|
+
import { useCallback, useState } from '@wordpress/element';
|
|
5
|
+
import debugFactory from 'debug';
|
|
6
|
+
/**
|
|
7
|
+
* Internal dependencies
|
|
8
|
+
*/
|
|
9
|
+
import useAiSuggestions, { RequestingErrorProps } from '../use-ai-suggestions/index.js';
|
|
10
|
+
import type { PromptProp } from '../../types.js';
|
|
11
|
+
|
|
12
|
+
const debug = debugFactory( 'jetpack-ai-client:use-transcription-post-processing' );
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Post-processing types.
|
|
16
|
+
*/
|
|
17
|
+
export const TRANSCRIPTION_POST_PROCESSING_ACTION_SIMPLE_DRAFT = 'voice-to-content-simple-draft';
|
|
18
|
+
export type PostProcessingAction = typeof TRANSCRIPTION_POST_PROCESSING_ACTION_SIMPLE_DRAFT;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* The return value for the transcription post-processing hook.
|
|
22
|
+
*/
|
|
23
|
+
export type UseTranscriptionPostProcessingReturn = {
|
|
24
|
+
postProcessingResult: string;
|
|
25
|
+
isProcessingTranscription: boolean;
|
|
26
|
+
postProcessingError: string;
|
|
27
|
+
processTranscription: ( action: PostProcessingAction, transcription: string ) => void;
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The props for the transcription post-processing hook.
|
|
32
|
+
*/
|
|
33
|
+
export type UseTranscriptionPostProcessingProps = {
|
|
34
|
+
feature: string;
|
|
35
|
+
onReady?: ( postProcessingResult: string ) => void;
|
|
36
|
+
onError?: ( error: string ) => void;
|
|
37
|
+
onUpdate?: ( currentPostProcessingResult: string ) => void;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* A hook to handle transcription post-processing.
|
|
42
|
+
*
|
|
43
|
+
* @param {string} feature - The feature name that is calling the post-processing actions.
|
|
44
|
+
* @returns {UseTranscriptionPostProcessingReturn} - Object with properties to get the post-processing results.
|
|
45
|
+
*/
|
|
46
|
+
export default function useTranscriptionPostProcessing( {
|
|
47
|
+
feature,
|
|
48
|
+
onReady,
|
|
49
|
+
onError,
|
|
50
|
+
onUpdate,
|
|
51
|
+
}: UseTranscriptionPostProcessingProps ): UseTranscriptionPostProcessingReturn {
|
|
52
|
+
const [ postProcessingResult, setPostProcessingResult ] = useState< string >( '' );
|
|
53
|
+
const [ postProcessingError, setPostProcessingError ] = useState< string >( '' );
|
|
54
|
+
const [ isProcessingTranscription, setIsProcessingTranscription ] = useState( false );
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Set-up the useAiSuggestions hook.
|
|
58
|
+
*/
|
|
59
|
+
const handleOnSuggestion = useCallback(
|
|
60
|
+
( suggestion: string ) => {
|
|
61
|
+
setPostProcessingResult( suggestion );
|
|
62
|
+
onUpdate?.( suggestion );
|
|
63
|
+
},
|
|
64
|
+
[ setPostProcessingResult, onUpdate ]
|
|
65
|
+
);
|
|
66
|
+
|
|
67
|
+
const handleOnDone = useCallback(
|
|
68
|
+
( result: string ) => {
|
|
69
|
+
setPostProcessingResult( result );
|
|
70
|
+
onUpdate?.( result );
|
|
71
|
+
onReady?.( result );
|
|
72
|
+
},
|
|
73
|
+
[ setPostProcessingResult, onUpdate, onReady ]
|
|
74
|
+
);
|
|
75
|
+
|
|
76
|
+
const handleOnError = useCallback(
|
|
77
|
+
( errorData: RequestingErrorProps ) => {
|
|
78
|
+
setPostProcessingError( errorData.message );
|
|
79
|
+
onError?.( errorData.message );
|
|
80
|
+
},
|
|
81
|
+
[ setPostProcessingError, onError ]
|
|
82
|
+
);
|
|
83
|
+
|
|
84
|
+
const { request } = useAiSuggestions( {
|
|
85
|
+
autoRequest: false,
|
|
86
|
+
onSuggestion: handleOnSuggestion,
|
|
87
|
+
onDone: handleOnDone,
|
|
88
|
+
onError: handleOnError,
|
|
89
|
+
} );
|
|
90
|
+
|
|
91
|
+
const handleTranscriptionPostProcessing = useCallback(
|
|
92
|
+
( action: PostProcessingAction, transcription: string ) => {
|
|
93
|
+
debug( 'Post-processing transcription' );
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Reset the transcription result and error.
|
|
97
|
+
*/
|
|
98
|
+
setPostProcessingResult( '' );
|
|
99
|
+
setPostProcessingError( '' );
|
|
100
|
+
setIsProcessingTranscription( true );
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Build the prompt to call the suggestion hook.
|
|
104
|
+
*/
|
|
105
|
+
const messages: PromptProp = [
|
|
106
|
+
{
|
|
107
|
+
role: 'jetpack-ai',
|
|
108
|
+
context: {
|
|
109
|
+
type: action,
|
|
110
|
+
content: transcription,
|
|
111
|
+
},
|
|
112
|
+
},
|
|
113
|
+
];
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Call the suggestion hook using the message.
|
|
117
|
+
*/
|
|
118
|
+
request( messages, { feature } );
|
|
119
|
+
},
|
|
120
|
+
[
|
|
121
|
+
setPostProcessingResult,
|
|
122
|
+
setPostProcessingError,
|
|
123
|
+
setIsProcessingTranscription,
|
|
124
|
+
request,
|
|
125
|
+
feature,
|
|
126
|
+
]
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
return {
|
|
130
|
+
postProcessingResult,
|
|
131
|
+
isProcessingTranscription,
|
|
132
|
+
postProcessingError,
|
|
133
|
+
processTranscription: handleTranscriptionPostProcessing,
|
|
134
|
+
};
|
|
135
|
+
}
|
package/src/icons/index.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
export { default as aiAssistantIcon } from './ai-assistant';
|
|
2
|
-
export { default as micIcon } from './mic';
|
|
3
|
-
export { default as origamiPlaneIcon } from './origami-plane';
|
|
4
|
-
export { default as playerPlayIcon } from './player-play';
|
|
5
|
-
export { default as playerStopIcon } from './player-stop';
|
|
6
|
-
export { default as playerPauseIcon } from './player-pause';
|
|
7
|
-
export { default as speakToneIcon } from './speak-tone';
|
|
1
|
+
export { default as aiAssistantIcon } from './ai-assistant.js';
|
|
2
|
+
export { default as micIcon } from './mic.js';
|
|
3
|
+
export { default as origamiPlaneIcon } from './origami-plane.js';
|
|
4
|
+
export { default as playerPlayIcon } from './player-play.js';
|
|
5
|
+
export { default as playerStopIcon } from './player-stop.js';
|
|
6
|
+
export { default as playerPauseIcon } from './player-pause.js';
|
|
7
|
+
export { default as speakToneIcon } from './speak-tone.js';
|
package/src/icons/mic.tsx
CHANGED
|
@@ -1,27 +1,25 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* External dependencies
|
|
3
3
|
*/
|
|
4
|
-
import { SVG, Rect,
|
|
4
|
+
import { SVG, Rect, Path } from '@wordpress/components';
|
|
5
5
|
import React from 'react';
|
|
6
6
|
|
|
7
7
|
const mic = (
|
|
8
|
-
<SVG width="24" height="24" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/SVG">
|
|
9
|
-
<
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
8
|
+
<SVG width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/SVG">
|
|
9
|
+
<Path
|
|
10
|
+
d="M12 4.75C11.0479 4.75 10.1052 4.93753 9.22554 5.30187C8.34593 5.66622 7.5467 6.20025 6.87348 6.87348C6.20025 7.5467 5.66622 8.34593 5.30187 9.22554C4.93753 10.1052 4.75 11.0479 4.75 12C4.75 12.9521 4.93753 13.8948 5.30187 14.7745C5.66622 15.6541 6.20025 16.4533 6.87348 17.1265C7.5467 17.7997 8.34593 18.3338 9.22554 18.6981C10.1052 19.0625 11.0479 19.25 12 19.25C13.9228 19.25 15.7669 18.4862 17.1265 17.1265C18.4862 15.7669 19.25 13.9228 19.25 12C19.25 10.0772 18.4862 8.23312 17.1265 6.87348C15.7669 5.51384 13.9228 4.75 12 4.75ZM3.25 12C3.25 9.67936 4.17187 7.45376 5.81282 5.81282C7.45376 4.17187 9.67936 3.25 12 3.25C14.3206 3.25 16.5462 4.17187 18.1872 5.81282C19.8281 7.45376 20.75 9.67936 20.75 12C20.75 14.3206 19.8281 16.5462 18.1872 18.1872C16.5462 19.8281 14.3206 20.75 12 20.75C9.67936 20.75 7.45376 19.8281 5.81282 18.1872C4.17187 16.5462 3.25 14.3206 3.25 12Z"
|
|
11
|
+
fill="#008710"
|
|
12
|
+
/>
|
|
13
|
+
<Rect x="10.5" y="8.25" width="3" height="4.875" rx="1.5" fill="#008710" />
|
|
14
|
+
<Path
|
|
15
|
+
d="M15 11.25C15 11.644 14.9224 12.0341 14.7716 12.3981C14.6209 12.762 14.3999 13.0927 14.1213 13.3713C13.8427 13.6499 13.512 13.8709 13.1481 14.0216C12.7841 14.1724 12.394 14.25 12 14.25C11.606 14.25 11.2159 14.1724 10.8519 14.0216C10.488 13.8709 10.1573 13.6499 9.87868 13.3713C9.6001 13.0927 9.37913 12.762 9.22836 12.3981C9.0776 12.0341 9 11.644 9 11.25"
|
|
16
|
+
stroke="#008710"
|
|
17
|
+
strokeWidth="1.125"
|
|
18
|
+
strokeLinecap="round"
|
|
18
19
|
/>
|
|
19
|
-
<Line x1="12" y1="17" x2="12" y2="21" stroke="currentColor" strokeWidth="1.5" fill="none" />
|
|
20
20
|
<Path
|
|
21
|
-
d="
|
|
22
|
-
|
|
23
|
-
strokeWidth="1.5"
|
|
24
|
-
fill="none"
|
|
21
|
+
d="M11.4375 15.75C11.4375 16.0607 11.6893 16.3125 12 16.3125C12.3107 16.3125 12.5625 16.0607 12.5625 15.75L11.4375 15.75ZM11.4375 14.25L11.4375 15.75L12.5625 15.75L12.5625 14.25L11.4375 14.25Z"
|
|
22
|
+
fill="#008710"
|
|
25
23
|
/>
|
|
26
24
|
</SVG>
|
|
27
25
|
);
|
|
@@ -5,13 +5,13 @@ import { SVG, Path, Rect } from '@wordpress/components';
|
|
|
5
5
|
import React from 'react';
|
|
6
6
|
|
|
7
7
|
const playerPause = (
|
|
8
|
-
<SVG width="
|
|
8
|
+
<SVG width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/SVG">
|
|
9
9
|
<Path
|
|
10
|
-
d="
|
|
11
|
-
fill="
|
|
10
|
+
d="M12 4.75C11.0479 4.75 10.1052 4.93753 9.22554 5.30187C8.34593 5.66622 7.5467 6.20025 6.87348 6.87348C6.20025 7.5467 5.66622 8.34593 5.30187 9.22554C4.93753 10.1052 4.75 11.0479 4.75 12C4.75 12.9521 4.93753 13.8948 5.30187 14.7745C5.66622 15.6541 6.20025 16.4533 6.87348 17.1265C7.5467 17.7997 8.34593 18.3338 9.22554 18.6981C10.1052 19.0625 11.0479 19.25 12 19.25C13.9228 19.25 15.7669 18.4862 17.1265 17.1265C18.4862 15.7669 19.25 13.9228 19.25 12C19.25 10.0772 18.4862 8.23312 17.1265 6.87348C15.7669 5.51384 13.9228 4.75 12 4.75ZM3.25 12C3.25 9.67936 4.17187 7.45376 5.81282 5.81282C7.45376 4.17187 9.67936 3.25 12 3.25C14.3206 3.25 16.5462 4.17187 18.1872 5.81282C19.8281 7.45376 20.75 9.67936 20.75 12C20.75 14.3206 19.8281 16.5462 18.1872 18.1872C16.5462 19.8281 14.3206 20.75 12 20.75C9.67936 20.75 7.45376 19.8281 5.81282 18.1872C4.17187 16.5462 3.25 14.3206 3.25 12Z"
|
|
11
|
+
fill="#008710"
|
|
12
12
|
/>
|
|
13
|
-
<Rect x="
|
|
14
|
-
<Rect x="
|
|
13
|
+
<Rect x="12.75" y="9" width="2.25" height="6" fill="#008710" />
|
|
14
|
+
<Rect x="9" y="9" width="2.25" height="6" fill="#008710" />
|
|
15
15
|
</SVG>
|
|
16
16
|
);
|
|
17
17
|
|
package/src/index.ts
CHANGED
|
@@ -1,32 +1,35 @@
|
|
|
1
1
|
/*
|
|
2
2
|
* Core library exports
|
|
3
3
|
*/
|
|
4
|
-
export { default as requestJwt } from './jwt';
|
|
5
|
-
export { default as SuggestionsEventSource } from './suggestions-event-source';
|
|
6
|
-
export { default as askQuestion } from './ask-question';
|
|
4
|
+
export { default as requestJwt } from './jwt/index.js';
|
|
5
|
+
export { default as SuggestionsEventSource } from './suggestions-event-source/index.js';
|
|
6
|
+
export { default as askQuestion } from './ask-question/index.js';
|
|
7
|
+
export { default as transcribeAudio } from './audio-transcription/index.js';
|
|
7
8
|
|
|
8
9
|
/*
|
|
9
10
|
* Hooks
|
|
10
11
|
*/
|
|
11
|
-
export { default as useAiSuggestions } from './hooks/use-ai-suggestions';
|
|
12
|
-
export { default as useMediaRecording } from './hooks/use-media-recording';
|
|
12
|
+
export { default as useAiSuggestions } from './hooks/use-ai-suggestions/index.js';
|
|
13
|
+
export { default as useMediaRecording } from './hooks/use-media-recording/index.js';
|
|
14
|
+
export { default as useAudioTranscription } from './hooks/use-audio-transcription/index.js';
|
|
15
|
+
export { default as useTranscriptionPostProcessing } from './hooks/use-transcription-post-processing/index.js';
|
|
13
16
|
|
|
14
17
|
/*
|
|
15
18
|
* Components: Icons
|
|
16
19
|
*/
|
|
17
|
-
export * from './icons';
|
|
20
|
+
export * from './icons/index.js';
|
|
18
21
|
|
|
19
22
|
/*
|
|
20
23
|
* Components
|
|
21
24
|
*/
|
|
22
|
-
export * from './components';
|
|
25
|
+
export * from './components/index.js';
|
|
23
26
|
|
|
24
27
|
/*
|
|
25
28
|
* Contexts
|
|
26
29
|
*/
|
|
27
|
-
export * from './data-flow';
|
|
30
|
+
export * from './data-flow/index.js';
|
|
28
31
|
|
|
29
32
|
/*
|
|
30
33
|
* Types
|
|
31
34
|
*/
|
|
32
|
-
export * from './types';
|
|
35
|
+
export * from './types.js';
|
package/src/jwt/index.ts
CHANGED
|
@@ -2,8 +2,11 @@
|
|
|
2
2
|
* External dependencies
|
|
3
3
|
*/
|
|
4
4
|
import { isSimpleSite } from '@automattic/jetpack-shared-extension-utils';
|
|
5
|
-
import apiFetch from '@wordpress/api-fetch';
|
|
6
5
|
import debugFactory from 'debug';
|
|
6
|
+
/**
|
|
7
|
+
* Internal dependencies
|
|
8
|
+
*/
|
|
9
|
+
import apiFetch from '../api-fetch/index.js';
|
|
7
10
|
/*
|
|
8
11
|
* Types & constants
|
|
9
12
|
*/
|
|
@@ -6,8 +6,8 @@ import debugFactory from 'debug';
|
|
|
6
6
|
/**
|
|
7
7
|
* Internal dependencies
|
|
8
8
|
*/
|
|
9
|
-
import { getErrorData } from '../hooks/use-ai-suggestions';
|
|
10
|
-
import requestJwt from '../jwt';
|
|
9
|
+
import { getErrorData } from '../hooks/use-ai-suggestions/index.js';
|
|
10
|
+
import requestJwt from '../jwt/index.js';
|
|
11
11
|
/*
|
|
12
12
|
* Types & constants
|
|
13
13
|
*/
|
|
@@ -19,13 +19,13 @@ import {
|
|
|
19
19
|
ERROR_RESPONSE,
|
|
20
20
|
ERROR_SERVICE_UNAVAILABLE,
|
|
21
21
|
ERROR_UNCLEAR_PROMPT,
|
|
22
|
-
} from '../types';
|
|
22
|
+
} from '../types.js';
|
|
23
23
|
import type {
|
|
24
24
|
AiModelTypeProp,
|
|
25
25
|
PromptMessagesProp,
|
|
26
26
|
PromptProp,
|
|
27
27
|
SuggestionErrorCode,
|
|
28
|
-
} from '../types';
|
|
28
|
+
} from '../types.js';
|
|
29
29
|
|
|
30
30
|
type SuggestionsEventSourceConstructorArgs = {
|
|
31
31
|
url?: string;
|