ai.muna.muna 0.0.44 → 0.0.45

package/Runtime/Beta/OpenAI/SpeechService.cs

@@ -0,0 +1,250 @@
+/* 
+*   Muna
+*   Copyright © 2025 NatML Inc. All rights reserved.
+*/
+
+#nullable enable
+
+namespace Muna.Beta.OpenAI {
+
+    using System;
+    using System.Collections.Generic;
+    using System.Linq;
+    using System.Runtime.Serialization;
+    using System.Threading.Tasks;
+    using Newtonsoft.Json;
+    using Newtonsoft.Json.Converters;
+    using Services;
+    using PredictorService = global::Muna.Services.PredictorService;
+    using EdgePredictionService = global::Muna.Services.PredictionService;
+
+    /// <summary>
+    /// Create speech.
+    /// </summary>
+    public sealed class SpeechService {
+
+        #region --Client API--
+        /// <summary>
+        /// Audio output format.
+        /// </summary>
+        [JsonConverter(typeof(StringEnumConverter))]
+        public enum ResponseFormat {
+            /// <summary>
+            /// MP3 audio.
+            /// </summary>
+            [EnumMember(Value = @"mp3")]
+            MP3 = 1,
+            /// <summary>
+            /// Opus.
+            /// </summary>
+            [EnumMember(Value = @"opus")]
+            Opus = 2,
+            /// <summary>
+            /// AAC.
+            /// </summary>
+            [EnumMember(Value = @"aac")]
+            AAC = 3,
+            /// <summary>
+            /// FLAC losseless audio.
+            /// </summary>
+            [EnumMember(Value = @"flac")]
+            FLAC = 4,
+            /// <summary>
+            /// Waveform audio.
+            /// </summary>
+            [EnumMember(Value = @"wav")]
+            WAV = 5,
+            /// <summary>
+            /// Linear PCM audio.
+            /// </summary>
+            [EnumMember(Value = @"pcm")]
+            PCM = 6,
+        }
+
+        /// <summary>
+        /// The format to stream the audio in.
+        /// </summary>
+        [JsonConverter(typeof(StringEnumConverter))]
+        public enum StreamFormat {
+            /// <summary>
+            /// Raw audio.
+            /// </summary>
+            Audio = 1,
+            /// <summary>
+            /// Server-sent events.
+            /// </summary>
+            SSE = 2,
+        }
+
+        /// <summary>
+        /// Generate audio from the input text.
+        /// </summary>
+        /// <param name="model">Speech generation predictor tag.</param>
+        /// <param name="input">The text to generate audio for.</param>
+        /// <param name="voice">The voice to use when generating the audio.</param>
+        /// <param name="speed">The speed of the generated audio. Defaults to 1.0.</param>
+        /// <param name="responseFormat">The format to return audio in. Currently only `ResponseFormat.PCM` is supported.</param>
+        /// <param name="streamFormat">The format to stream the audio in. Currently only `StreamFormat.Audio` is supported.</param>
+        /// <param name="acceleration">Prediction acceleration. Must be `Acceleration` or `RemoteAcceleration` instance.</param>
+        /// <returns>Generated audio.</returns>
+        public async Task<BinaryData> Create(
+            string model,
+            string input,
+            string voice,
+            float speed = 1f,
+            ResponseFormat responseFormat = ResponseFormat.MP3,
+            StreamFormat streamFormat = StreamFormat.Audio,
+            object? acceleration = null
+        ) {
+            // Ensure we have a delegate
+            if (!cache.ContainsKey(model)) {
+                var @delegate = await CreateSpeechDelegate(model);
+                cache.Add(model, @delegate);
+            }
+            // Make prediction
+            var handler = cache[model];
+            var result = await handler(
+                model,
+                input,
+                voice,
+                speed,
+                responseFormat,
+                streamFormat,
+                acceleration: acceleration ?? Acceleration.Auto
+            );
+            // Return
+            return result;
+        }
+        #endregion
+
+
+        #region --Operations--
+        private readonly PredictorService predictors;
+        private readonly EdgePredictionService predictions;
+        private readonly RemotePredictionService remotePredictions;
+        private readonly Dictionary<string, SpeechDelegate> cache;
+        private delegate Task<BinaryData> SpeechDelegate(
+            string model,
+            string input,
+            string voice,
+            float speed,
+            ResponseFormat responseFormat,
+            StreamFormat streamFormat,
+            object acceleration
+        );
+
+        internal SpeechService(
+            PredictorService predictors,
+            EdgePredictionService predictions,
+            RemotePredictionService remotePredictions
+        ) {
+            this.predictors = predictors;
+            this.predictions = predictions;
+            this.remotePredictions = remotePredictions;
+            this.cache = new();
+        }
+
+        private async Task<SpeechDelegate> CreateSpeechDelegate(string tag) {
+            // Retrieve predictor
+            var predictor = await predictors.Retrieve(tag);
+            if (predictor == null)
+                throw new ArgumentException($"{tag} cannot be used for OpenAI speech API because the predictor could not be found.");
+            // Get required inputs
+            var signature = predictor.signature!;
+            var requiredInputParams = signature.inputs.Where(parameter => parameter.optional == false).ToArray();
+            if (requiredInputParams.Length != 2)
+                throw new InvalidOperationException($"${tag} cannot be used with OpenAI speech API because it does not have exactly two required input parameters.");
+            // Get the text input parameter
+            var inputParam = requiredInputParams.FirstOrDefault(parameter => parameter.type == Dtype.String);
+            if (inputParam == null)
+                throw new InvalidOperationException($"${tag} cannot be used with OpenAI speech API because it does not have the required speech input parameter.");
+            // Get the voice input parameter
+            var voiceParam = requiredInputParams.FirstOrDefault(parameter =>
+                parameter.type == Dtype.String &&
+                parameter.denotation == "audio.voice"
+            );
+            if (voiceParam == null)
+                throw new InvalidOperationException($"${tag} cannot be used with OpenAI speech API because it does not have the required speech voice parameter.");
+            // Get the speed input parameter (optional)
+            var speedParam = signature.inputs.FirstOrDefault(parameter =>
+                new[] { Dtype.Float32, Dtype.Float64 }.Contains((Dtype)parameter.type!) &&
+                parameter.denotation == "audio.speed"
+            );
+            // Get the audio output parameter
+            var (audioParamIdx, audioParam) = signature.outputs
+                .Select((parameter, idx) => (idx, parameter))
+                .Where(pair =>
+                    pair.parameter.type == Dtype.Float32 &&
+                    pair.parameter.denotation == "audio"
+                )
+                .FirstOrDefault();
+            if (audioParam == null)
+                throw new InvalidOperationException($"{tag} cannot be used with OpenAI speech API because it has no outputs with an `audio` denotation.");
+            // Create delegate
+            SpeechDelegate result = async(
+                string model,
+                string input,
+                string voice,
+                float speed,
+                ResponseFormat responseFormat,
+                StreamFormat streamFormat,
+                object acceleration
+            ) => {
+                // Build prediction input map
+                var inputMap = new Dictionary<string, object?> {
+                    [inputParam.name] = input,
+                    [voiceParam.name] = voice
+                };
+                if (speedParam != null)
+                    inputMap[speedParam.name] = speed;
+                // Create prediction
+                var prediction = await CreatePrediction(
+                    model,
+                    inputs: inputMap,
+                    acceleration: acceleration
+                );
+                // Check for error
+                if (prediction.error != null)
+                    throw new InvalidOperationException(prediction.error);
+                // Check returned audio
+                var result = prediction.results![audioParamIdx]!;
+                if (!(result is Tensor<float> tensor))
+                    throw new InvalidOperationException($"{tag} cannot be used with OpenAI speech API because it returned an object of type {result.GetType()} instead of an audio tensor.");
+                if (tensor.shape.Length != 1 && tensor.shape.Length != 2) {
+                    var shapeStr = "(" + string.Join(",", tensor.shape) + ")";
+                    throw new InvalidOperationException($"{tag} cannot be used with OpenAI speech API because it returned an audio tensor with an invalid shape: {shapeStr}");
+                }
+                // Create response
+                var channels = tensor.shape.Length == 2 ? tensor.shape[0] : 1; // Assume planar
+                var mediaType = $"audio/pcm;rate={audioParam.sampleRate};channels={channels}";
+                var response = ToBinaryData(tensor, mediaType);
+                // Return
+                return response;
+            };
+            // Return
+            return result;
+        }
+
+        private Task<Prediction> CreatePrediction(
+            string tag,
+            Dictionary<string, object?> inputs,
+            object acceleration
+        ) => acceleration switch {
+            Acceleration acc => predictions.Create(tag, inputs, acc),
+            RemoteAcceleration acc => remotePredictions.Create(tag, inputs, acc),
+            _ => throw new InvalidOperationException($"Cannot create {tag} prediction because acceleration is invalid: {acceleration}")
+        };
+        
+        private static unsafe BinaryData ToBinaryData<T>(
+            Tensor<T> tensor,
+            string? mediaType = null
+        ) where T : unmanaged {
+            var elementCount = tensor.shape.Aggregate(1, (a, b) => a * b);
+            var data = new byte[elementCount * sizeof(T)];
+            fixed (void* src = tensor, dst = data)
+                Buffer.MemoryCopy(src, dst, data.Length, data.Length);
+            return new BinaryData(data, mediaType);
+        }
+        #endregion
+    }
+}

package/Runtime/Beta/OpenAI/SpeechService.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: aa80e003d14a045c59c487cf206d4f95
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

package/Runtime/Beta/OpenAI/Types.cs

@@ -0,0 +1,365 @@
+/* 
+*   Muna
+*   Copyright © 2025 NatML Inc. All rights reserved.
+*/
+
+#nullable enable
+
+namespace Muna.Beta.OpenAI {
+
+    using System;
+    using System.Runtime.Serialization;
+    using Newtonsoft.Json;
+    using Newtonsoft.Json.Converters;
+
+    /// <summary>
+    /// Binary data with an optional content type.
+    /// </summary>
+    public sealed class BinaryData {
+
+        /// <summary>
+        /// Whether this data is empty.
+        /// </summary>
+        public bool IsEmpty => data.Length == 0;
+
+        /// <summary>
+        /// Number of bytes of this data.
+        /// </summary>
+        public int Length => data.Length;
+
+        /// <summary>
+        /// MIME type of this data
+        /// </summary>
+        public string? MediaType { get; private set; }
+
+        /// <summary>
+        /// Convert the `BinaryData` to a byte array.
+        /// </summary>
+        /// <returns></returns>
+        public byte[] ToArray() => data;
+
+        /// <summary>
+        /// Gets the value of this instance as bytes without any further interpretation.
+        /// </summary>
+        public ReadOnlyMemory<byte> ToMemory() => new ReadOnlyMemory<byte>(data);
+
+        private readonly byte[] data;
+        
+        public BinaryData(byte[] data, string? mediaType = null) {
+            this.data = data;
+            this.MediaType = mediaType;
+        }
+    }
+
+    /// <summary>
+    /// Embedding.
+    /// </summary>
+    public sealed class Embedding {
+
+        /// <summary>
+        /// The object type, which is always "embedding".
+        /// </summary>
+        [JsonProperty(@"object")]
+#pragma warning disable 8618
+        public string Object;
+#pragma warning restore 8618
+
+        /// <summary>
+        /// Embedding vector as number array.
+        /// </summary>
+        [JsonProperty(@"embedding")]
+        public float[]? Floats;
+
+        /// <summary>
+        /// Index of the embedding in the list of embeddings.
+        /// </summary>
+        [JsonProperty(@"index")]
+        public int Index;
+
+        /// <summary>
+        /// Embedding vector as a base64 string.
+        /// </summary>
+        [JsonIgnore]
+        public string? Base64;
+    }
+
+    public sealed class CreateEmbeddingResponse {
+
+#pragma warning disable 8618
+        /// <summary>
+        /// The object type, which is always "list".
+        /// </summary>
+        [JsonProperty(@"object")]
+        public string Object;
+
+        /// <summary>
+        /// The name of the model used to generate the embedding.
+        /// </summary>
+        [JsonProperty(@"model")]
+        public string Model;
+
+        /// <summary>
+        /// The list of embeddings generated by the model.
+        /// </summary>
+        [JsonProperty(@"data")]
+        public Embedding[] Data;
+#pragma warning restore 8618
+
+        /// <summary>
+        /// The usage information for the request.
+        /// </summary>
+        [JsonProperty(@"usage")]
+        public UsageInfo Usage;
+
+        /// <summary>
+        /// Usage information for the request.
+        /// </summary>
+        public struct UsageInfo {
+
+            /// <summary>
+            /// Number of tokens used by the prompt.
+            /// </summary>
+            [JsonProperty(@"prompt_tokens")]
+            public int PromptTokens;
+
+            /// <summary>
+            /// Total number of tokens used by the request.
+            /// </summary>
+            [JsonProperty(@"total_tokens")]
+            public int TotalTokens;
+        }
+    }
+
+    /// <summary>
+    /// Chat message.
+    /// </summary>
+    public sealed class ChatMessage {
+
+        /// <summary>
+        /// Message role.
+        /// </summary>
+        [JsonConverter(typeof(StringEnumConverter))]
+        public enum ChatRole {
+            /// <summary>
+            /// Assistant.
+            /// </summary>
+            [EnumMember(Value = @"assistant")]
+            Assistant = 1,
+            /// <summary>
+            /// User
+            /// </summary>
+            [EnumMember(Value = @"user")]
+            User = 2,
+            /// <summary>
+            /// Developer.
+            /// </summary>
+            [EnumMember(Value = @"developer")]
+            Developer = 3,
+            /// <summary>
+            /// System.
+            /// </summary>
+            [EnumMember(Value = @"system")]
+            System = 4,
+            /// <summary>
+            /// Tool.
+            /// </summary>
+            [EnumMember(Value = @"tool")]
+            Tool = 5,
+        }
+
+        /// <summary>
+        /// Message role.
+        /// </summary>
+        [JsonProperty(@"role")]
+        public ChatRole Role;
+
+        /// <summary>
+        /// Message content.
+        /// </summary>
+        [JsonProperty(@"content")]
+        public string? Content;
+    }
+
+    /// <summary>
+    /// Chat completion.
+    /// </summary>
+    public sealed class ChatCompletion {
+
+#pragma warning disable 8618
+        /// <summary>
+        /// The object type, which is always `chat.completion`.
+        /// </summary>
+        [JsonProperty(@"object")]
+        public string Object;
+
+        /// <summary>
+        /// Completion unique identifier.
+        /// </summary>
+        [JsonProperty(@"id")]
+        public string Id;
+
+        /// <summary>
+        /// Model that generated the chat completion.
+        /// </summary>
+        [JsonProperty(@"model")]
+        public string Model;
+
+        /// <summary>
+        /// Chat completion choices.
+        /// Can be more than one if `n` is greater than 1.
+        /// </summary>
+        [JsonProperty(@"choices")]
+        public Choice[] Choices;
+
+        /// <summary>
+        /// Date created as a unix timestamp (in seconds).
+        /// </summary>
+        [JsonProperty(@"created")]
+        public long Created;
+
+        /// <summary>
+        /// Usage statistics for the completion request.
+        /// </summary>
+        [JsonProperty(@"usage")]
+        public UsageInfo Usage;
+
+        public sealed class Choice {
+
+            /// <summary>
+            /// Index of this choice in the list of choices.
+            /// </summary>
+            [JsonProperty(@"index")]
+            public int Index;
+
+            /// <summary>
+            /// Completion message generated by the model.
+            /// </summary>
+            [JsonProperty(@"message")]
+            public ChatMessage Message;
+
+            /// <summary>
+            /// The reason the model stopped generating tokens. This will be:
+            /// 
+            /// `stop` if the model hit a natural stop point or a provided stop sequence.
+            /// `length` if the maximum number of tokens specified in the request was reached.
+            /// `content_filter` if content was omitted due to a flag from our content filters.
+            /// `tool_calls` if the model called a tool.
+            /// `function_call` (deprecated) if the model called a function.
+            /// </summary>
+            [JsonProperty(@"finish_reason")]
+            public string? FinishReason;
+        }
+#pragma warning restore 8618
+
+        public struct UsageInfo {
+
+            /// <summary>
+            /// Number of tokens in the generated completion.
+            /// </summary>
+            [JsonProperty(@"completion_tokens")]
+            public int CompletionTokens;
+
+            /// <summary>
+            /// Number of tokens in the prompt.
+            /// </summary>
+            [JsonProperty(@"prompt_tokens")]
+            public int PromptTokens;
+
+            /// <summary>
+            /// Total number of tokens used in the request (prompt + completion).
+            /// </summary>
+            [JsonProperty(@"total_tokens")]
+            public int TotalTokens;
+        }
+    }
+    
+    /// <summary>
+    /// Chat completion chunk.
+    /// </summary>
+    public sealed class ChatCompletionChunk {
+
+#pragma warning disable 8618
+        /// <summary>
+        /// The object type, which is always `chat.completion.chunk`.
+        /// </summary>
+        [JsonProperty(@"object")]
+        public string Object;
+
+        /// <summary>
+        /// Completion unique identifier.
+        /// All chunks from a given completion share the same identifier.
+        /// </summary>
+        [JsonProperty(@"id")]
+        public string Id;
+
+        /// <summary>
+        /// Model that generated the chat completion.
+        /// </summary>
+        [JsonProperty(@"model")]
+        public string Model;
+
+        /// <summary>
+        /// Chat completion choices.
+        /// Can be more than one if `n` is greater than 1.
+        /// Can also be empty for the last chunk if you set `stream_options: {"include_usage": true}`.
+        /// </summary>
+        [JsonProperty(@"choices")]
+        public Choice[] Choices;
+#pragma warning restore 8618
+
+        /// <summary>
+        /// Date created as a unix timestamp (in seconds).
+        /// All chunks from a given completion share the same creation timestamp.
+        /// </summary>
+        [JsonProperty(@"created")]
+        public long Created;
+
+        /// <summary>
+        /// Usage statistics for the completion request.
+        /// </summary>
+        [JsonProperty(@"usage")]
+        public ChatCompletion.UsageInfo? Usage;
+
+        public sealed class Choice {
+
+            /// <summary>
+            /// Index of this choice in the list of choices.
+            /// </summary>
+            [JsonProperty(@"index")]
+            public int Index;
+
+            /// <summary>
+            /// Completion message delta generated by the model when streaming.
+            /// </summary>
+            [JsonProperty(@"delta")]
+            public MessageDelta? Delta;
+
+            /// <summary>
+            /// The reason the model stopped generating tokens. This will be:
+            /// 
+            /// `stop` if the model hit a natural stop point or a provided stop sequence.
+            /// `length` if the maximum number of tokens specified in the request was reached.
+            /// `content_filter` if content was omitted due to a flag from our content filters.
+            /// `tool_calls` if the model called a tool.
+            /// `function_call` (deprecated) if the model called a function.
+            /// </summary>
+            [JsonProperty(@"finish_reason")]
+            public string? FinishReason;
+
+            public sealed class MessageDelta {
+
+                /// <summary>
+                /// Role of the author of this message.
+                /// </summary>
+                [JsonProperty(@"role")]
+                public ChatMessage.ChatRole? Role;
+
+                /// <summary>
+                /// Content of the message chunk.
+                /// </summary>
+                [JsonProperty(@"content")]
+                public string? Content;
+            }
+        }
+    }
+}

package/Runtime/Beta/OpenAI/Types.cs.meta

@@ -0,0 +1,11 @@
+fileFormatVersion: 2
+guid: cc44ab5ef48204d5bade71a3dc9640ed
+MonoImporter:
+  externalObjects: {}
+  serializedVersion: 2
+  defaultReferences: []
+  executionOrder: 0
+  icon: {instanceID: 0}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: 

package/Runtime/Beta/OpenAI.meta

@@ -0,0 +1,8 @@
+fileFormatVersion: 2
+guid: b6c7d82c12a544ba98437c05417089e8
+folderAsset: yes
+DefaultImporter:
+  externalObjects: {}
+  userData: 
+  assetBundleName: 
+  assetBundleVariant: