@aws-amplify/predictions 5.5.6-api-v6-models.b3abc9b.0 → 6.0.0

package/src/{Providers → providers}/AmazonAIConvertPredictionsProvider.ts

@@ -1,77 +1,110 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
-import { AbstractConvertPredictionsProvider } from '../types/Providers/AbstractConvertPredictionsProvider';
+import {
+	Amplify,
+	ConsoleLogger as Logger,
+	fetchAuthSession,
+} from '@aws-amplify/core';
+import {
+	AWSCredentials,
+	Category,
+	PredictionsAction,
+	Signer,
+	getAmplifyUserAgentObject,
+} from '@aws-amplify/core/internals/utils';
+import { PollyClient, SynthesizeSpeechCommand } from '@aws-sdk/client-polly';
 import {
 	TranslateClient,
 	TranslateTextCommand,
 } from '@aws-sdk/client-translate';
-import { PollyClient, SynthesizeSpeechCommand } from '@aws-sdk/client-polly';
 import {
-	TranslateTextInput,
-	TextToSpeechInput,
+	EventStreamCodec,
+	MessageHeaderValue,
+} from '@smithy/eventstream-codec';
+import { fromUtf8, toUtf8 } from '@smithy/util-utf8';
+import { Buffer } from 'buffer';
+import { PredictionsValidationErrorCode } from '../errors/types/validation';
+import { assertValidationError } from '../errors/utils/assertValidationError';
+import {
+	ConvertBytes,
 	SpeechToTextInput,
-	TranslateTextOutput,
-	TextToSpeechOutput,
 	SpeechToTextOutput,
-	isBytesSource,
+	TextToSpeechInput,
+	TextToSpeechOutput,
+	TranscribeData,
+	TranslateTextInput,
+	TranslateTextOutput,
+	isConvertBytesSource,
+	isTextToSpeechInput,
+	isTranslateTextInput,
+	isValidConvertInput,
 } from '../types';
-import {
-	Credentials,
-	ConsoleLogger as Logger,
-	Signer,
-	getAmplifyUserAgentObject,
-	Category,
-	PredictionsAction,
-} from '@aws-amplify/core';
-import {
-	EventStreamMarshaller,
-	MessageHeaderValue,
-} from '@aws-sdk/eventstream-marshaller';
-import { fromUtf8, toUtf8 } from '@aws-sdk/util-utf8-node';
-import { Buffer } from 'buffer';
 
 const logger = new Logger('AmazonAIConvertPredictionsProvider');
-const eventBuilder = new EventStreamMarshaller(toUtf8, fromUtf8);
+const eventBuilder = new EventStreamCodec(toUtf8, fromUtf8);
 
 const LANGUAGES_CODE_IN_8KHZ = ['fr-FR', 'en-AU', 'en-GB', 'fr-CA'];
 
-export class AmazonAIConvertPredictionsProvider extends AbstractConvertPredictionsProvider {
-	private translateClient: TranslateClient;
-	private pollyClient: PollyClient;
-	constructor() {
-		super();
-	}
+export class AmazonAIConvertPredictionsProvider {
+	private translateClient?: TranslateClient;
+	private pollyClient?: PollyClient;
 
 	getProviderName() {
 		return 'AmazonAIConvertPredictionsProvider';
 	}
 
+	convert(
+		input: TranslateTextInput | TextToSpeechInput | SpeechToTextInput
+	): Promise<TextToSpeechOutput | TranslateTextOutput | SpeechToTextOutput> {
+		assertValidationError(
+			isValidConvertInput(input),
+			PredictionsValidationErrorCode.InvalidInput
+		);
+
+		if (isTranslateTextInput(input)) {
+			logger.debug('translateText');
+			return this.translateText(input);
+		} else if (isTextToSpeechInput(input)) {
+			logger.debug('textToSpeech');
+			return this.convertTextToSpeech(input);
+		} else {
+			logger.debug('textToSpeech');
+			return this.convertSpeechToText(input);
+		}
+	}
+
 	protected async translateText(
 		input: TranslateTextInput
 	): Promise<TranslateTextOutput> {
 		logger.debug('Starting translation');
-		const {
-			translateText: {
-				defaults: { sourceLanguage = '', targetLanguage = '' } = {},
-				region = '',
-			} = {},
-		} = this._config;
-
-		if (!region) {
-			return Promise.reject('region not configured for transcription');
-		}
 
-		const credentials = await Credentials.get();
-		if (!credentials) {
-			return Promise.reject('No credentials');
-		}
+		const { translateText = {} } =
+			Amplify.getConfig().Predictions?.convert ?? {};
+		assertValidationError(
+			!!translateText.region,
+			PredictionsValidationErrorCode.NoRegion
+		);
+		const { defaults = {}, region } = translateText;
+
+		const { credentials } = await fetchAuthSession();
+		assertValidationError(
+			!!credentials,
+			PredictionsValidationErrorCode.NoCredentials
+		);
+
+		const { sourceLanguage, targetLanguage } = defaults;
 		const sourceLanguageCode =
-			input.translateText.source.language || sourceLanguage;
+			input.translateText?.source?.language ?? sourceLanguage;
 		const targetLanguageCode =
-			input.translateText.targetLanguage || targetLanguage;
-		if (!sourceLanguageCode || !targetLanguageCode) {
-			return Promise.reject('Please provide both source and target language');
-		}
+			input.translateText?.targetLanguage ?? targetLanguage;
+		assertValidationError(
+			!!sourceLanguageCode,
+			PredictionsValidationErrorCode.NoSourceLanguage
+		);
+		assertValidationError(
+			!!targetLanguageCode,
+			PredictionsValidationErrorCode.NoTargetLanguage
+		);
 
 		this.translateClient = new TranslateClient({
 			region,
@@ -84,43 +117,39 @@ export class AmazonAIConvertPredictionsProvider extends AbstractConvertPredictio
 		const translateTextCommand = new TranslateTextCommand({
 			SourceLanguageCode: sourceLanguageCode,
 			TargetLanguageCode: targetLanguageCode,
-			Text: input.translateText.source.text,
+			Text: input.translateText?.source?.text,
 		});
-		try {
-			const data = await this.translateClient.send(translateTextCommand);
-			return {
-				text: data.TranslatedText,
-				language: data.TargetLanguageCode,
-			} as TranslateTextOutput;
-		} catch (err) {
-			return Promise.reject(err);
-		}
+		const data = await this.translateClient.send(translateTextCommand);
+		return {
+			text: data.TranslatedText,
+			language: data.TargetLanguageCode,
+		} as TranslateTextOutput;
 	}
 
 	protected async convertTextToSpeech(
 		input: TextToSpeechInput
 	): Promise<TextToSpeechOutput> {
-		const credentials = await Credentials.get();
-		if (!credentials) {
-			return Promise.reject('No credentials');
-		}
-		const {
-			speechGenerator: { defaults: { VoiceId = '' } = {}, region = '' } = {},
-		} = this._config;
+		const { credentials } = await fetchAuthSession();
+		assertValidationError(
+			!!credentials,
+			PredictionsValidationErrorCode.NoCredentials
+		);
+		assertValidationError(
+			!!input.textToSpeech?.source,
+			PredictionsValidationErrorCode.NoSource
+		);
 
-		if (!input.textToSpeech.source) {
-			return Promise.reject('Source needs to be provided in the input');
-		}
-		const voiceId = input.textToSpeech.voiceId || VoiceId;
-		if (!region) {
-			return Promise.reject(
-				'Region was undefined. Did you enable speech generator using amplify CLI?'
-			);
-		}
+		const { speechGenerator } = Amplify.getConfig().Predictions?.convert ?? {};
+		assertValidationError(
+			!!speechGenerator?.region,
+			PredictionsValidationErrorCode.NoRegion
+		);
 
-		if (!voiceId) {
-			return Promise.reject('VoiceId was undefined.');
-		}
+		const { defaults = {}, region } = speechGenerator;
+
+		const { voiceId: defaultVoiceId } = defaults;
+		const voiceId = input.textToSpeech?.voiceId ?? defaultVoiceId;
+		assertValidationError(!!voiceId, PredictionsValidationErrorCode.NoVoiceId);
 
 		this.pollyClient = new PollyClient({
 			region,
@@ -132,93 +161,77 @@ export class AmazonAIConvertPredictionsProvider extends AbstractConvertPredictio
 		});
 		const synthesizeSpeechCommand = new SynthesizeSpeechCommand({
 			OutputFormat: 'mp3',
-			Text: input.textToSpeech.source.text,
+			Text: input.textToSpeech?.source?.text,
 			VoiceId: voiceId,
 			TextType: 'text',
 			SampleRate: '24000',
 			// tslint:disable-next-line: align
 		});
-		try {
-			const data = await this.pollyClient.send(synthesizeSpeechCommand);
-			const response = new Response(data.AudioStream as ReadableStream);
-			const arrayBuffer = await response.arrayBuffer();
-			const blob = new Blob([arrayBuffer], {
-				type: data.ContentType,
-			});
-			const url = URL.createObjectURL(blob);
-			return {
-				speech: { url },
-				audioStream: arrayBuffer,
-				text: input.textToSpeech.source.text,
-			} as TextToSpeechOutput;
-		} catch (err) {
-			return Promise.reject(err);
-		}
+		const data = await this.pollyClient.send(synthesizeSpeechCommand);
+		const response = new Response(data.AudioStream as ReadableStream);
+		const arrayBuffer = await response.arrayBuffer();
+		const blob = new Blob([arrayBuffer], {
+			type: data.ContentType,
+		});
+		const url = URL.createObjectURL(blob);
+		return {
+			speech: { url },
+			audioStream: arrayBuffer,
+			text: input.textToSpeech?.source?.text,
+		} as TextToSpeechOutput;
 	}
 
 	protected async convertSpeechToText(
 		input: SpeechToTextInput
 	): Promise<SpeechToTextOutput> {
-		try {
-			logger.debug('starting transcription..');
-			const credentials = await Credentials.get();
-			if (!credentials) {
-				return Promise.reject('No credentials');
-			}
-			const {
-				transcription: {
-					defaults: { language: languageCode = '' } = {},
-					region = '',
-				} = {},
-			} = this._config;
-			if (!region) {
-				return Promise.reject('region not configured for transcription');
-			}
-			if (!languageCode) {
-				return Promise.reject(
-					'languageCode not configured or provided for transcription'
-				);
-			}
-			const {
-				transcription: { source, language = languageCode },
-			} = input;
+		logger.debug('starting transcription..');
+		const { credentials } = await fetchAuthSession();
+		assertValidationError(
+			!!credentials,
+			PredictionsValidationErrorCode.NoCredentials
+		);
 
-			if (isBytesSource(source)) {
-				const connection = await this.openConnectionWithTranscribe({
-					credentials,
-					region,
-					languageCode: language,
-				});
+		const { transcription } = Amplify.getConfig().Predictions?.convert ?? {};
+		assertValidationError(
+			!!transcription?.region,
+			PredictionsValidationErrorCode.NoRegion
+		);
 
-				try {
-					const fullText = await this.sendDataToTranscribe({
-						connection,
-						raw: source.bytes,
-						languageCode: language,
-					});
-					return {
-						transcription: {
-							fullText,
-						},
-					};
-				} catch (err) {
-					return Promise.reject(err);
-				}
-			}
+		const { defaults, region } = transcription;
+		const language = input.transcription?.language ?? defaults?.language;
 
-			return Promise.reject(
-				'Source types other than byte source are not supported.'
-			);
-		} catch (err) {
-			return Promise.reject(err.name + ': ' + err.message);
-		}
+		assertValidationError(
+			!!language,
+			PredictionsValidationErrorCode.NoLanguage
+		);
+
+		const source = input.transcription?.source;
+		assertValidationError(
+			isConvertBytesSource(source),
+			PredictionsValidationErrorCode.InvalidSource
+		);
+
+		const connection = await this.openConnectionWithTranscribe({
+			credentials,
+			region,
+			languageCode: language,
+		});
+
+		const fullText = await this.sendDataToTranscribe({
+			connection,
+			raw: source.bytes,
+			languageCode: language,
+		});
+		return {
+			transcription: {
+				fullText,
+			},
+		};
 	}
 
-	public static serializeDataFromTranscribe(message) {
+	public static serializeDataFromTranscribe(message: MessageEvent) {
 		let decodedMessage = '';
-		const transcribeMessage = eventBuilder.unmarshall(
-			Buffer.from(message.data)
-		);
+		const transcribeMessage = eventBuilder.decode(Buffer.from(message.data));
 		const transcribeMessageJson = JSON.parse(toUtf8(transcribeMessage.body));
 		if (transcribeMessage.headers[':message-type'].value === 'exception') {
 			logger.debug(
@@ -259,7 +272,7 @@ export class AmazonAIConvertPredictionsProvider extends AbstractConvertPredictio
 		connection,
 		raw,
 		languageCode,
-	}): Promise<string> {
+	}: TranscribeData): Promise<string> {
 		return new Promise((res, rej) => {
 			let fullText = '';
 			connection.onmessage = message => {
@@ -271,9 +284,9 @@ export class AmazonAIConvertPredictionsProvider extends AbstractConvertPredictio
 					if (decodedMessage) {
 						fullText += decodedMessage + ' ';
 					}
-				} catch (err) {
+				} catch (err: unknown) {
 					logger.debug(err);
-					rej(err.message);
+					rej(err);
 				}
 			};
 
@@ -301,27 +314,32 @@ export class AmazonAIConvertPredictionsProvider extends AbstractConvertPredictio
 
 			// sending end frame
 			const endFrameEventMessage = this.getAudioEventMessage(Buffer.from([]));
-			const endFrameBinary = eventBuilder.marshall(endFrameEventMessage);
+			const endFrameBinary = eventBuilder.encode(endFrameEventMessage);
 			connection.send(endFrameBinary);
 		});
 	}
 
-	private sendEncodedDataToTranscribe(connection, data, languageCode) {
+	private sendEncodedDataToTranscribe(
+		connection: WebSocket,
+		data: ConvertBytes | any[],
+		languageCode: string
+	) {
 		const downsampledBuffer = this.downsampleBuffer({
 			buffer: data,
 			outputSampleRate: LANGUAGES_CODE_IN_8KHZ.includes(languageCode)
 				? 8000
 				: 16000,
 		});
+
 		const pcmEncodedBuffer = this.pcmEncode(downsampledBuffer);
 		const audioEventMessage = this.getAudioEventMessage(
 			Buffer.from(pcmEncodedBuffer)
 		);
-		const binary = eventBuilder.marshall(audioEventMessage);
+		const binary = eventBuilder.encode(audioEventMessage);
 		connection.send(binary);
 	}
 
-	private getAudioEventMessage(buffer) {
+	private getAudioEventMessage(buffer: Buffer) {
 		const audioEventMessage = {
 			body: buffer as Uint8Array,
 			headers: {
@@ -339,8 +357,13 @@ export class AmazonAIConvertPredictionsProvider extends AbstractConvertPredictio
 		return audioEventMessage;
 	}
 
-	private pcmEncode(input) {
+	private pcmEncode(input: Float32Array | ConvertBytes | any[]) {
 		let offset = 0;
+		// ArrayBuffer cannot be processed using length property
+		if (input instanceof ArrayBuffer) {
+			return input;
+		}
+
 		const buffer = new ArrayBuffer(input.length * 2);
 		const view = new DataView(buffer);
 		for (let i = 0; i < input.length; i++, offset += 2) {
@@ -352,8 +375,18 @@ export class AmazonAIConvertPredictionsProvider extends AbstractConvertPredictio
 
 	private inputSampleRate = 44100;
 
-	private downsampleBuffer({ buffer, outputSampleRate = 16000 }) {
-		if (outputSampleRate === this.inputSampleRate) {
+	private downsampleBuffer({
+		buffer,
+		outputSampleRate = 16000,
+	}: {
+		buffer: ConvertBytes | any[];
+		outputSampleRate: number;
+	}) {
+		// Cannot process ArrayBuffer using length property
+		if (
+			outputSampleRate === this.inputSampleRate ||
+			buffer instanceof ArrayBuffer
+		) {
 			return buffer;
 		}
 
@@ -383,23 +416,15 @@ export class AmazonAIConvertPredictionsProvider extends AbstractConvertPredictio
 	}
 
 	private openConnectionWithTranscribe({
-		credentials: userCredentials,
+		credentials,
 		region,
 		languageCode,
+	}: {
+		credentials: AWSCredentials;
+		region: string;
+		languageCode: string;
 	}): Promise<WebSocket> {
 		return new Promise(async (res, rej) => {
-			const {
-				accessKeyId: access_key,
-				secretAccessKey: secret_key,
-				sessionToken: session_token,
-			} = userCredentials;
-
-			const credentials = {
-				access_key,
-				secret_key,
-				session_token,
-			};
-
 			const signedUrl = this.generateTranscribeUrl({
 				credentials,
 				region,
@@ -417,7 +442,21 @@ export class AmazonAIConvertPredictionsProvider extends AbstractConvertPredictio
 		});
 	}
 
-	private generateTranscribeUrl({ credentials, region, languageCode }): string {
+	private generateTranscribeUrl({
+		credentials: { accessKeyId, secretAccessKey, sessionToken },
+		region,
+		languageCode,
+	}: {
+		credentials: AWSCredentials;
+		region: string;
+		languageCode: string;
+	}): string {
+		const credentials = {
+			access_key: accessKeyId,
+			secret_key: secretAccessKey,
+			session_token: sessionToken,
+		};
+
 		const url = [
 			`wss://transcribestreaming.${region}.amazonaws.com:8443`,
 			'/stream-transcription-websocket?',