ai.muna.muna 0.0.44 → 0.0.46

package/Unity/MunaUnity.cs

@@ -20,9 +20,11 @@ namespace Muna {
 
     using System;
     using System.Collections.Generic;
+    using System.Text.RegularExpressions;
     using UnityEngine;
     using Unity.Collections.LowLevel.Unsafe;
     using API;
+    using Beta.OpenAI;
     using Internal;
 
     /// <summary>
@@ -36,7 +38,6 @@ namespace Muna {
         /// </summary>
         /// <param name="accessKey">Muna access key. This defaults to your access key in Project Settings.</param>
         /// <param name="url">Muna API URL.</param>
-        /// <param name="cachePath">Predictor cache path.</param>
         /// <returns>Muna client.</returns>
         public static Muna Create(
             string? accessKey = null,
@@ -62,34 +63,34 @@ namespace Muna {
             this Texture2D texture,
             byte[]? pixelBuffer = null
         ) {
+            // Check texture
             if (texture == null)
                 throw new ArgumentNullException(nameof(texture));
+            // Check that texture is readable
             if (!texture.isReadable)
                 throw new InvalidOperationException(@"Texture cannot be converted to a Muna image because it is not readable");
-            var FormatChannelMap = new Dictionary<TextureFormat, int> {
-                [TextureFormat.R8] = 1,
-                [TextureFormat.Alpha8] = 1,
-                [TextureFormat.RGB24] = 3,
-                [TextureFormat.RGBA32] = 4,
-            };
-            if (!FormatChannelMap.TryGetValue(texture.format, out var channels))
-                throw new InvalidOperationException($"Texture cannot be converted to a Muna image because it has unsupported format: {texture.format}");
+            // Allocate buffer
+            var channels = TextureFormatToImageChannels.GetValueOrDefault(texture.format, 4);
             var rowStride = texture.width * channels;
             var bufferSize = rowStride * texture.height;
             pixelBuffer ??= new byte[bufferSize];
             if (pixelBuffer.Length < bufferSize)
                 throw new InvalidOperationException($"Texture cannot be converted to a Muna image because pixel buffer length was expected to be greater than or equal to {bufferSize} but got {pixelBuffer.Length}");
-            fixed (void* dst = pixelBuffer)
+            // Copy
+            var colorData = !TextureFormatToImageChannels.ContainsKey(texture.format) ? texture.GetPixels32() : null;
+            fixed (void* dst = pixelBuffer, colors = colorData) {
+                var src = colors == null ? texture.GetRawTextureData<byte>().GetUnsafePtr() : colors;
                 UnsafeUtility.MemCpyStride(
                     dst,
                     rowStride,
-                    (byte*)texture.GetRawTextureData<byte>().GetUnsafePtr() + (rowStride * (texture.height - 1)),
+                    (byte*)src + (rowStride * (texture.height - 1)),
                     -rowStride,
                     rowStride,
                     texture.height
                 );
-            var image = new Image(pixelBuffer, texture.width, texture.height, channels);
-            return image;
+            }
+            // Return
+            return new Image(pixelBuffer, texture.width, texture.height, channels);
         }
 
         /// <summary>
@@ -102,12 +103,7 @@ namespace Muna {
             this Image image,
             Texture2D? texture = null
         ) {
-            var ChannelFormatMap = new Dictionary<int, TextureFormat> {
-                [1] = TextureFormat.Alpha8,
-                [3] = TextureFormat.RGB24,
-                [4] = TextureFormat.RGBA32
-            };
-            if (!ChannelFormatMap.TryGetValue(image.channels, out var format))
+            if (!ImageChannelsToTextureFormat.TryGetValue(image.channels, out var format))
                 throw new InvalidOperationException($"Image cannot be converted to a Texture2D because it has unsupported channel count: {image.channels}");
             texture = texture != null ? texture : new Texture2D(image.width, image.height, format, false);
             if (texture.width != image.width || texture.height != image.height || texture.format != format)
@@ -125,6 +121,58 @@ namespace Muna {
             texture.Apply();
             return texture;
         }
+
+        /// <summary>
+        /// Convert a `BinaryData` containing linear PCM audio into an audio clip.
+        /// </summary>
+        /// <param name="data">Binary data containing linear PCM audio.</param>
+        /// <returns>Audio clip.</returns>
+        public static unsafe AudioClip ToAudioClip(this BinaryData data) {
+            // Check that this contains LPCM data
+            if (string.IsNullOrEmpty(data.MediaType) || !data.MediaType.StartsWith(@"audio/pcm"))
+                throw new ArgumentException($"Failed to create audio clip from binary data because media type was expected to be 'audio/pcm' but got: '{data.MediaType}'");
+            // Match sample rate and channel count
+            var rateMatch = Regex.Match(data.MediaType, @"rate=(\d+)");
+            var channelsMatch = Regex.Match(data.MediaType, @"channels=(\d+)");
+            if (!rateMatch.Success || !channelsMatch.Success)
+                throw new ArgumentException($"Failed to create audio clip from binary data because media type is invalid: '{data.MediaType}'");
+            // Parse
+            if (!int.TryParse(rateMatch.Groups[1].Value, out var sampleRate))
+                throw new ArgumentException($"Failed to create audio clip from binary data because sample rate is invalid: '{rateMatch.Value}'");
+            if (!int.TryParse(channelsMatch.Groups[1].Value, out var channelCount))
+                throw new ArgumentException($"Failed to create audio clip from binary data because channel count is invalid: '{channelsMatch.Value}'");
+            // Create clip
+            var sampleCount = data.Length / sizeof(float);
+            var frameCount = sampleCount / channelCount;
+            var clip = AudioClip.Create(
+                "audio",
+                lengthSamples: frameCount,
+                channels: channelCount,
+                frequency: sampleRate,
+                stream: false
+            );
+            // Copy data
+            var samples = new float[sampleCount];
+            Buffer.BlockCopy(data.ToArray(), 0, samples, 0, data.Length);
+            clip.SetData(samples, 0);
+            // Return
+            return clip;
+        }
+        #endregion
+
+
+        #region --Operations--
+        private static readonly Dictionary<TextureFormat, int> TextureFormatToImageChannels = new() {
+            [TextureFormat.R8] = 1,
+            [TextureFormat.Alpha8] = 1,
+            [TextureFormat.RGB24] = 3,
+            [TextureFormat.RGBA32] = 4,
+        };
+        private static readonly Dictionary<int, TextureFormat> ImageChannelsToTextureFormat = new() {
+            [1] = TextureFormat.Alpha8,
+            [3] = TextureFormat.RGB24,
+            [4] = TextureFormat.RGBA32,
+        };
         #endregion
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "ai.muna.muna",
-    "version": "0.0.44",
+    "version": "0.0.46",
     "displayName": "Muna",
     "description": "Run AI inference in Unity Engine.",
     "unity": "2022.3",