@automattic/jetpack-ai-client 0.8.2 → 0.10.0

package/CHANGELOG.md

@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.10.0] - 2024-03-18
+### Added
+- Add image generator hook [#36415]
+
+## [0.9.0] - 2024-03-12
+### Changed
+- Fix typescript errors [#35904]
+- Updated package dependencies. [#36325]
+
+### Fixed
+- AI Client: Fix audio recording where WebM is not supported (iOS for example). [#36160]
+
 ## [0.8.2] - 2024-03-04
 ### Added
 - AI Client: add audio validation hook. [#36043]
@@ -247,6 +259,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Updated package dependencies. [#31659]
 - Updated package dependencies. [#31785]
 
+[0.10.0]: https://github.com/Automattic/jetpack-ai-client/compare/v0.9.0...v0.10.0
+[0.9.0]: https://github.com/Automattic/jetpack-ai-client/compare/v0.8.2...v0.9.0
 [0.8.2]: https://github.com/Automattic/jetpack-ai-client/compare/v0.8.1...v0.8.2
 [0.8.1]: https://github.com/Automattic/jetpack-ai-client/compare/v0.8.0...v0.8.1
 [0.8.0]: https://github.com/Automattic/jetpack-ai-client/compare/v0.7.0...v0.8.0

package/build/api-fetch/index.d.ts

@@ -2,6 +2,5 @@
  * External dependencies
  */
 import apiFetchMod from '@wordpress/api-fetch';
-type ApiFetchType = typeof apiFetchMod.default;
-declare const apiFetch: ApiFetchType;
-export default apiFetch;
+declare const _default: typeof apiFetchMod.default;
+export default _default;

package/build/api-fetch/index.js

@@ -2,5 +2,9 @@
  * External dependencies
  */
 import apiFetchMod from '@wordpress/api-fetch';
-const apiFetch = (apiFetchMod.default ?? apiFetchMod);
+// @wordpress/api-fetch (as of 6.47.0) declares itself in such a way that tsc and node see the function at apiFetchMod.default
+// while some other environments (including code running inside WordPress itself) see it at apiFetch.
+// See https://arethetypeswrong.github.io/?p=@wordpress/api-fetch@6.47.0
+// This is a helper to simplify the usage of the api-fetch module on the ai-client package.
+const apiFetch = 'default' in apiFetchMod ? apiFetchMod.default : apiFetchMod;
 export default apiFetch;

package/build/components/ai-control/index.d.ts

@@ -35,6 +35,6 @@ type AiControlProps = {
  * @param {React.MutableRefObject} ref - Ref to the component.
  * @returns {ReactElement}         Rendered component.
  */
-export declare function AIControl({ disabled, value, placeholder, showAccept, acceptLabel, showButtonLabels, isTransparent, state, showGuideLine, customFooter, onChange, onSend, onStop, onAccept, onDiscard, showRemove, bannerComponent, errorComponent, }: AiControlProps, ref: React.MutableRefObject<null>): ReactElement;
-declare const _default: React.ForwardRefExoticComponent<AiControlProps & React.RefAttributes<null>>;
+export declare function AIControl({ disabled, value, placeholder, showAccept, acceptLabel, showButtonLabels, isTransparent, state, showGuideLine, customFooter, onChange, onSend, onStop, onAccept, onDiscard, showRemove, bannerComponent, errorComponent, }: AiControlProps, ref: React.MutableRefObject<HTMLInputElement>): ReactElement;
+declare const _default: React.ForwardRefExoticComponent<AiControlProps & React.RefAttributes<HTMLInputElement>>;
 export default _default;

package/build/hooks/use-image-generator/index.d.ts

@@ -0,0 +1,10 @@
+declare const useImageGenerator: () => {
+    generateImage: ({ feature, }: {
+        feature: string;
+    }) => Promise<{
+        data: Array<{
+            url: string;
+        }>;
+    }>;
+};
+export default useImageGenerator;

package/build/hooks/use-image-generator/index.js

@@ -0,0 +1,48 @@
+/**
+ * External dependencies
+ */
+import debugFactory from 'debug';
+/**
+ * Internal dependencies
+ */
+import requestJwt from '../../jwt/index.js';
+const debug = debugFactory('ai-client:use-image-generator');
+const useImageGenerator = () => {
+    const generateImage = async function ({ feature, }) {
+        let token = '';
+        try {
+            token = (await requestJwt()).token;
+        }
+        catch (error) {
+            debug('Error getting token: %o', error);
+            return Promise.reject(error);
+        }
+        try {
+            debug('Generating image');
+            // TODO: Find a proper prompt for the image generation
+            const imageGenerationPrompt = ``;
+            const URL = 'https://public-api.wordpress.com/wpcom/v2/jetpack-ai-image';
+            const body = {
+                prompt: imageGenerationPrompt,
+                response_format: 'url',
+                feature,
+            };
+            const headers = {
+                Authorization: `Bearer ${token}`,
+            };
+            const data = await fetch(URL, {
+                method: 'POST',
+                headers,
+                body: JSON.stringify(body),
+            }).then(response => response.json());
+            return data;
+        }
+        catch (error) {
+            return;
+        }
+    };
+    return {
+        generateImage,
+    };
+};
+export default useImageGenerator;

package/build/hooks/use-media-recording/index.js

@@ -2,6 +2,12 @@
  * External dependencies
  */
 import { useRef, useState, useEffect, useCallback } from '@wordpress/element';
+/**
+ * Media types
+ */
+const MEDIA_TYPE_MP4_MP4A = 'audio/mp4;codecs=mp4a';
+const MEDIA_TYPE_MP4 = 'audio/mp4';
+const MEDIA_TYPE_WEBM = 'audio/webm';
 /**
  * react custom hook to handle media recording.
  *
@@ -32,9 +38,10 @@ export default function useMediaRecording({ onDone, } = {}) {
      * @returns {Blob} The recorded blob
      */
     function getBlob() {
-        return new Blob(recordedChunks, {
-            type: 'audio/webm',
-        });
+        if (MediaRecorder.isTypeSupported(MEDIA_TYPE_MP4_MP4A)) {
+            return new Blob(recordedChunks, { type: MEDIA_TYPE_MP4 }); // omit the codecs parameter
+        }
+        return new Blob(recordedChunks, { type: MEDIA_TYPE_WEBM });
     }
     // `start` recording handler
     const start = useCallback((timeslice) => {
@@ -115,7 +122,15 @@ export default function useMediaRecording({ onDone, } = {}) {
             audioStream.current = stream;
             const source = audioCtx.createMediaStreamSource(stream);
             source.connect(analyser.current);
-            mediaRecordRef.current = new MediaRecorder(stream);
+            /**
+             * Special handling for iOS devices.
+             */
+            if (MediaRecorder.isTypeSupported(MEDIA_TYPE_MP4_MP4A)) {
+                mediaRecordRef.current = new MediaRecorder(stream, { mimeType: MEDIA_TYPE_MP4_MP4A });
+            }
+            else {
+                mediaRecordRef.current = new MediaRecorder(stream, { mimeType: MEDIA_TYPE_WEBM });
+            }
             mediaRecordRef.current.addEventListener('start', onStartListener);
             mediaRecordRef.current.addEventListener('stop', onStopListener);
             mediaRecordRef.current.addEventListener('pause', onPauseListener);

package/build/index.d.ts

@@ -7,6 +7,7 @@ export { default as useMediaRecording } from './hooks/use-media-recording/index.
 export { default as useAudioTranscription } from './hooks/use-audio-transcription/index.js';
 export { default as useTranscriptionPostProcessing } from './hooks/use-transcription-post-processing/index.js';
 export { default as useAudioValidation } from './hooks/use-audio-validation/index.js';
+export { default as useImageGenerator } from './hooks/use-image-generator/index.js';
 export * from './icons/index.js';
 export * from './components/index.js';
 export * from './data-flow/index.js';

package/build/index.js

@@ -13,6 +13,7 @@ export { default as useMediaRecording } from './hooks/use-media-recording/index.
 export { default as useAudioTranscription } from './hooks/use-audio-transcription/index.js';
 export { default as useTranscriptionPostProcessing } from './hooks/use-transcription-post-processing/index.js';
 export { default as useAudioValidation } from './hooks/use-audio-validation/index.js';
+export { default as useImageGenerator } from './hooks/use-image-generator/index.js';
 /*
  * Components: Icons
  */

package/build/types.d.ts

@@ -29,15 +29,3 @@ export type CancelablePromise<T = void> = Promise<T> & {
     canceled?: boolean;
 };
 export type TranscriptionState = RecordingState | 'validating' | 'processing' | 'error';
-interface JPConnectionInitialState {
-    apiNonce: string;
-    siteSuffix: string;
-    connectionStatus: {
-        isActive: boolean;
-    };
-}
-declare global {
-    interface Window {
-        JP_CONNECTION_INITIAL_STATE: JPConnectionInitialState;
-    }
-}

package/package.json

@@ -1,7 +1,7 @@
 {
 	"private": false,
 	"name": "@automattic/jetpack-ai-client",
-	"version": "0.8.2",
+	"version": "0.10.0",
 	"description": "A JS client for consuming Jetpack AI services",
 	"homepage": "https://github.com/Automattic/jetpack/tree/HEAD/projects/js-packages/ai-client/#readme",
 	"bugs": {
@@ -38,19 +38,19 @@
 	"main": "./build/index.js",
 	"types": "./build/index.d.ts",
 	"dependencies": {
-		"@automattic/jetpack-base-styles": "^0.6.18",
-		"@automattic/jetpack-connection": "^0.33.1",
-		"@automattic/jetpack-shared-extension-utils": "^0.14.5",
+		"@automattic/jetpack-base-styles": "^0.6.19",
+		"@automattic/jetpack-connection": "^0.33.3",
+		"@automattic/jetpack-shared-extension-utils": "^0.14.7",
 		"@microsoft/fetch-event-source": "2.0.1",
 		"@types/react": "18.2.61",
-		"@wordpress/api-fetch": "6.49.0",
-		"@wordpress/block-editor": "12.20.0",
-		"@wordpress/components": "27.0.0",
-		"@wordpress/compose": "6.29.0",
-		"@wordpress/data": "9.22.0",
-		"@wordpress/element": "5.29.0",
-		"@wordpress/i18n": "4.52.0",
-		"@wordpress/icons": "9.43.0",
+		"@wordpress/api-fetch": "6.50.0",
+		"@wordpress/block-editor": "12.21.0",
+		"@wordpress/components": "27.1.0",
+		"@wordpress/compose": "6.30.0",
+		"@wordpress/data": "9.23.0",
+		"@wordpress/element": "5.30.0",
+		"@wordpress/i18n": "4.53.0",
+		"@wordpress/icons": "9.44.0",
 		"classnames": "2.3.2",
 		"debug": "4.3.4",
 		"react": "18.2.0",

package/src/api-fetch/index.ts

@@ -7,7 +7,8 @@ import apiFetchMod from '@wordpress/api-fetch';
 // while some other environments (including code running inside WordPress itself) see it at apiFetch.
 // See https://arethetypeswrong.github.io/?p=@wordpress/api-fetch@6.47.0
 // This is a helper to simplify the usage of the api-fetch module on the ai-client package.
-type ApiFetchType = typeof apiFetchMod.default;
-const apiFetch: ApiFetchType = ( apiFetchMod.default ?? apiFetchMod ) as ApiFetchType;
+const apiFetch = 'default' in apiFetchMod ? apiFetchMod.default : apiFetchMod;
+// eslint-disable-next-line @typescript-eslint/ban-types
+type ApiFetchType = typeof apiFetch extends Function ? typeof apiFetch : typeof apiFetchMod;
 
-export default apiFetch;
+export default apiFetch as ApiFetchType;

package/src/components/ai-control/index.tsx

@@ -73,7 +73,7 @@ export function AIControl(
 		bannerComponent = null,
 		errorComponent = null,
 	}: AiControlProps,
-	ref: React.MutableRefObject< null >
+	ref: React.MutableRefObject< HTMLInputElement >
 ): ReactElement {
 	const promptUserInputRef = useRef( null );
 	const loading = state === 'requesting' || state === 'suggesting';

package/src/hooks/use-image-generator/index.ts

@@ -0,0 +1,62 @@
+/**
+ * External dependencies
+ */
+import debugFactory from 'debug';
+/**
+ * Internal dependencies
+ */
+import requestJwt from '../../jwt/index.js';
+
+const debug = debugFactory( 'ai-client:use-image-generator' );
+
+const useImageGenerator = () => {
+	const generateImage = async function ( {
+		feature,
+	}: {
+		feature: string;
+	} ): Promise< { data: Array< { url: string } > } > {
+		let token = '';
+
+		try {
+			token = ( await requestJwt() ).token;
+		} catch ( error ) {
+			debug( 'Error getting token: %o', error );
+			return Promise.reject( error );
+		}
+
+		try {
+			debug( 'Generating image' );
+
+			// TODO: Find a proper prompt for the image generation
+			const imageGenerationPrompt = ``;
+
+			const URL = 'https://public-api.wordpress.com/wpcom/v2/jetpack-ai-image';
+
+			const body = {
+				prompt: imageGenerationPrompt,
+				response_format: 'url',
+				feature,
+			};
+
+			const headers = {
+				Authorization: `Bearer ${ token }`,
+			};
+
+			const data = await fetch( URL, {
+				method: 'POST',
+				headers,
+				body: JSON.stringify( body ),
+			} ).then( response => response.json() );
+
+			return data as { data: { url: string }[] };
+		} catch ( error ) {
+			return;
+		}
+	};
+
+	return {
+		generateImage,
+	};
+};
+
+export default useImageGenerator;

package/src/hooks/use-media-recording/index.ts

@@ -10,6 +10,13 @@ type UseMediaRecordingProps = {
 	onDone?: ( blob: Blob ) => void;
 };
 
+/**
+ * Media types
+ */
+const MEDIA_TYPE_MP4_MP4A = 'audio/mp4;codecs=mp4a';
+const MEDIA_TYPE_MP4 = 'audio/mp4';
+const MEDIA_TYPE_WEBM = 'audio/webm';
+
 type UseMediaRecordingReturn = {
 	/**
 	 * The current recording state
@@ -114,9 +121,11 @@ export default function useMediaRecording( {
 	 * @returns {Blob} The recorded blob
 	 */
 	function getBlob() {
-		return new Blob( recordedChunks, {
-			type: 'audio/webm',
-		} );
+		if ( MediaRecorder.isTypeSupported( MEDIA_TYPE_MP4_MP4A ) ) {
+			return new Blob( recordedChunks, { type: MEDIA_TYPE_MP4 } ); // omit the codecs parameter
+		}
+
+		return new Blob( recordedChunks, { type: MEDIA_TYPE_WEBM } );
 	}
 
 	// `start` recording handler
@@ -217,7 +226,15 @@ export default function useMediaRecording( {
 				const source = audioCtx.createMediaStreamSource( stream );
 				source.connect( analyser.current );
 
-				mediaRecordRef.current = new MediaRecorder( stream );
+				/**
+				 * Special handling for iOS devices.
+				 */
+				if ( MediaRecorder.isTypeSupported( MEDIA_TYPE_MP4_MP4A ) ) {
+					mediaRecordRef.current = new MediaRecorder( stream, { mimeType: MEDIA_TYPE_MP4_MP4A } );
+				} else {
+					mediaRecordRef.current = new MediaRecorder( stream, { mimeType: MEDIA_TYPE_WEBM } );
+				}
+
 				mediaRecordRef.current.addEventListener( 'start', onStartListener );
 				mediaRecordRef.current.addEventListener( 'stop', onStopListener );
 				mediaRecordRef.current.addEventListener( 'pause', onPauseListener );

package/src/index.ts

@@ -14,6 +14,7 @@ export { default as useMediaRecording } from './hooks/use-media-recording/index.
 export { default as useAudioTranscription } from './hooks/use-audio-transcription/index.js';
 export { default as useTranscriptionPostProcessing } from './hooks/use-transcription-post-processing/index.js';
 export { default as useAudioValidation } from './hooks/use-audio-validation/index.js';
+export { default as useImageGenerator } from './hooks/use-image-generator/index.js';
 
 /*
  * Components: Icons

package/src/types.ts

@@ -97,20 +97,3 @@ export type CancelablePromise< T = void > = Promise< T > & { canceled?: boolean
  * Transcription types
  */
 export type TranscriptionState = RecordingState | 'validating' | 'processing' | 'error';
-
-// Connection initial state
-// @todo: it should be provided by the connection package
-interface JPConnectionInitialState {
-	apiNonce: string;
-	siteSuffix: string;
-	connectionStatus: {
-		isActive: boolean;
-	};
-}
-
-// Global
-declare global {
-	interface Window {
-		JP_CONNECTION_INITIAL_STATE: JPConnectionInitialState;
-	}
-}