smart_prompt 0.4.3 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 683f259828b34a687bb598abff6fe3ef547e918954b7422eecf2d357ba237495
-  data.tar.gz: 5410ee6c08d46643ac1c6c9ccbccc38f5729973a65063454c8b75c7787a587a6
+  metadata.gz: a1b5288acfef3c366b16a0e08dc0b4f43a7f8613f73879f92420a3cf60ce9332
+  data.tar.gz: c2fd82bf35e96c6784492dfe2dd9ee30dbf02e56b5cce41ee8d9895f4b050b13
 SHA512:
-  metadata.gz: 25755db9db82c9af27d8007753da625767831f835bd5767726db738c2863dfea9a3429d4a600e7dbd5005729cfbd3b679b75e4657eab13dd7b6e446379984ea0
-  data.tar.gz: 4fb6b5f037c504b7a0ae440066aa4608fafa5a97c516ac220ffcd0817ffd03e880d0d7473e0e6537b50c6e771ca59f3b08aa9cbb3b24fabb41665a06ebe69ca2
+  metadata.gz: ccd75b4af683bb4585ca46a24d60833fac4344e214f5fef6730e9b55e86e80a35951d39e6cbff8ad1f364623e62aade8bce065ade1cd8c6ddea7e1b1faea2126
+  data.tar.gz: 68cba98948160fc872d2b5661661fe16ae76f5b43025fa96ea0c546563e01be5a0930d76fd13ec327017e07b23c7b739953794a3b385185c448888862374190c

data/lib/smart_prompt/conversation.rb

@@ -1,6 +1,7 @@
 require "yaml"
 require "retriable"
 require "numo/narray"
+require "base64"
 
 module SmartPrompt
   class Conversation
@@ -234,8 +235,8 @@ module SmartPrompt
 
     def multimodal_content(text)
       parts = @pending_content_parts
-      images_and_videos = parts.select { |part| ["image", "video"].include?(part[:type] || part["type"]) }
-      audio_parts = parts.select { |part| (part[:type] || part["type"]) == "audio" }
+      images_and_videos = parts.select { |part| ["image_url", "image", "video_url", "video"].include?(part[:type] || part["type"]) }
+      audio_parts = parts.select { |part| ["input_audio", "audio"].include?(part[:type] || part["type"]) }
       other_parts = parts - images_and_videos - audio_parts
       normalize_content_parts(images_and_videos + other_parts + [{ type: "text", text: text.to_s }] + audio_parts)
     end
@@ -249,14 +250,26 @@ module SmartPrompt
     end
 
     def media_part(type, source, **metadata)
-      part = { type: type }
       case type
       when "image"
-        part[:url] = source
+        mime_type = detect_image_mime(source)
+        data = File.binread(source)
+        base64_data = Base64.strict_encode64(data)
+        url = "data:#{mime_type};base64,#{base64_data}"
+        part = { type: "image_url", image_url: { url: url } }
       when "audio"
-        part[:audio] = source
+        format = detect_audio_format(source)
+        data = File.binread(source)
+        base64_data = Base64.strict_encode64(data)
+        part = { type: "input_audio", input_audio: { data: base64_data, format: format } }
       when "video"
-        part[:video] = source
+        mime_type = detect_video_mime(source)
+        data = File.binread(source)
+        base64_data = Base64.strict_encode64(data)
+        url = "data:#{mime_type};base64,#{base64_data}"
+        part = { type: "video_url", video_url: { url: url } }
+      else
+        part = { type: type }
       end
       metadata.each do |key, value|
         part[key] = value unless value.nil?
@@ -264,6 +277,35 @@ module SmartPrompt
       part
     end
 
+    def detect_image_mime(path)
+      ext = File.extname(path).downcase
+      case ext
+      when ".png"  then "image/png"
+      when ".jpg", ".jpeg" then "image/jpeg"
+      when ".gif"  then "image/gif"
+      when ".webp" then "image/webp"
+      when ".bmp"  then "image/bmp"
+      when ".svg"  then "image/svg+xml"
+      else "application/octet-stream"
+      end
+    end
+
+    def detect_audio_format(path)
+      ext = File.extname(path).downcase.delete_prefix(".")
+      %w[wav mp3 ogg flac aac m4a].include?(ext) ? ext : "wav"
+    end
+
+    def detect_video_mime(path)
+      ext = File.extname(path).downcase
+      case ext
+      when ".mp4"  then "video/mp4"
+      when ".webm" then "video/webm"
+      when ".mov"  then "video/quicktime"
+      when ".avi"  then "video/x-msvideo"
+      else "application/octet-stream"
+      end
+    end
+
     def thinking_system_message(message)
       message = message.to_s.sub(/\A<\|think\|>\n?/, "")
       return message if @thinking_enabled == false

data/lib/smart_prompt/version.rb

@@ -1,3 +1,3 @@
 module SmartPrompt
-  VERSION = "0.4.3"
+  VERSION = "0.4.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smart_prompt
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.4.4
 platform: ruby
 authors:
 - zhuang biaowei
@@ -93,6 +93,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.2.1
+- !ruby/object:Gem::Dependency
+  name: base64
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
 description: SmartPrompt provides a flexible DSL for managing prompts, interacting
   with multiple LLMs, and creating composable task workers.
 email: