smart_prompt 0.4.2 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 396c6097973289a34143e86b65f428d55919d0e755916992a5c714e289ebf5a2
-  data.tar.gz: 5d2c2d81b486e1fb05b53f116047ab1f90be77c9536fd6440a96b573bdad00c3
+  metadata.gz: a1b5288acfef3c366b16a0e08dc0b4f43a7f8613f73879f92420a3cf60ce9332
+  data.tar.gz: c2fd82bf35e96c6784492dfe2dd9ee30dbf02e56b5cce41ee8d9895f4b050b13
 SHA512:
-  metadata.gz: c6880395149678a195ea6efc46a81623d61c14c56534936041a625616a9e6b716597c078ccee0ac0d47c3b2a8e67d272742ee36940fa64321426799db0b26e4d
-  data.tar.gz: 63f0b8ae0f6f62363443731cae6fed3eafe746c463d259c9b88eee8563d6ef0cdcd84e684d6daeedd7329c3dcab84748ab62358fa2935b9d0278ec347df45ffb
+  metadata.gz: ccd75b4af683bb4585ca46a24d60833fac4344e214f5fef6730e9b55e86e80a35951d39e6cbff8ad1f364623e62aade8bce065ade1cd8c6ddea7e1b1faea2126
+  data.tar.gz: 68cba98948160fc872d2b5661661fe16ae76f5b43025fa96ea0c546563e01be5a0930d76fd13ec327017e07b23c7b739953794a3b385185c448888862374190c

data/README.cn.md

@@ -75,6 +75,14 @@ llms:
     adapter: openai
     url: http://localhost:11434/
     default_model: deepseek-r1
+  gemma4_local:
+    adapter: openai
+    url: http://localhost:8000/v1
+    api_key: dummy
+    default_model: gemma-4-12B-it
+    temperature: 1.0
+    top_p: 0.95
+    top_k: 64
   deepseek:
     adapter: openai
     url: https://api.deepseek.com
@@ -89,6 +97,10 @@ models:
   deepseekv3.2:
     use: SiliconFlow
     model: Pro/deepseek-ai/DeepSeek-V3.2
+  gemma4/12b:
+    use: gemma4_local
+    model: gemma-4-12B-it
+    max_tokens: 1024
 
 # 默认设置
 default_llm: SiliconFlow
@@ -170,6 +182,26 @@ engine.call_worker_by_stream(:streaming_chat, {
 end
 ```
 
+### Gemma 4 12B 多模态
+
+Gemma 4 12B 可以通过 LiteRT-LM、LM Studio、Ollama、llama.cpp 等 OpenAI 兼容本地服务接入。SmartPrompt 会把图片放在文本前、音频放在文本后，以匹配 Gemma 4 的多模态最佳实践。
+
+```ruby
+SmartPrompt.define_worker :gemma_multimodal_assistant do
+  use_model "gemma4/12b"
+  thinking params.fetch(:thinking, true)
+  sys_msg("你是一个严谨的本地多模态助手。", params)
+
+  image(params[:image], token_budget: params[:token_budget] || 280) if params[:image]
+  video(params[:video], fps: 1, max_seconds: 60) if params[:video]
+  audio(params[:audio]) if params[:audio]
+  prompt(params[:message])
+
+  request_options(response_format: { type: "json_object" }) if params[:json]
+  send_msg
+end
+```
+
 ### 工具集成
 
 ```ruby

data/README.md

@@ -75,6 +75,14 @@ llms:
     adapter: openai
     url: http://localhost:11434/
     default_model: deepseek-r1
+  gemma4_local:
+    adapter: openai
+    url: http://localhost:8000/v1
+    api_key: dummy
+    default_model: gemma-4-12B-it
+    temperature: 1.0
+    top_p: 0.95
+    top_k: 64
   deepseek:
     adapter: openai
     url: https://api.deepseek.com
@@ -89,6 +97,10 @@ models:
   deepseekv3.2:
     use: SiliconFlow
     model: Pro/deepseek-ai/DeepSeek-V3.2
+  gemma4/12b:
+    use: gemma4_local
+    model: gemma-4-12B-it
+    max_tokens: 1024
 
 # Default settings
 default_llm: SiliconFlow
@@ -170,6 +182,26 @@ engine.call_worker_by_stream(:streaming_chat, {
 end
 ```
 
+### Gemma 4 12B Multimodal
+
+Gemma 4 12B can be connected through OpenAI-compatible local servers such as LiteRT-LM, LM Studio, Ollama, or llama.cpp. SmartPrompt places images before text and audio after text to match Gemma 4 multimodal best practices.
+
+```ruby
+SmartPrompt.define_worker :gemma_multimodal_assistant do
+  use_model "gemma4/12b"
+  thinking params.fetch(:thinking, true)
+  sys_msg("You are a precise local multimodal assistant.", params)
+
+  image(params[:image], token_budget: params[:token_budget] || 280) if params[:image]
+  video(params[:video], fps: 1, max_seconds: 60) if params[:video]
+  audio(params[:audio]) if params[:audio]
+  prompt(params[:message])
+
+  request_options(response_format: { type: "json_object" }) if params[:json]
+  send_msg
+end
+```
+
 ### Tool Integration
 
 ```ruby

data/lib/smart_prompt/conversation.rb

@@ -1,10 +1,23 @@
 require "yaml"
 require "retriable"
 require "numo/narray"
+require "base64"
 
 module SmartPrompt
   class Conversation
     include APIHandler
+    MODEL_REQUEST_OPTION_KEYS = %w[
+      max_tokens
+      max_completion_tokens
+      top_p
+      top_k
+      response_format
+      tool_choice
+      parallel_tool_calls
+      seed
+      stop
+    ].freeze
+
     attr_reader :messages, :last_response, :config_file
     attr_reader :last_call_id
 
@@ -21,6 +34,9 @@ module SmartPrompt
       @current_adapter = engine.current_adapter
       @last_response = nil
       @tools = tools
+      @request_options = {}
+      @pending_content_parts = []
+      @thinking_enabled = nil
     end
 
     def use(llm_name)
@@ -43,6 +59,7 @@ module SmartPrompt
 
       use(llm_name)
       model(configured_model_name)
+      merge_model_request_options(model_config)
       self
     end
 
@@ -54,6 +71,20 @@ module SmartPrompt
       @temperature = temperature
     end
 
+    def request_options(options = {})
+      @request_options.merge!(options || {})
+      self
+    end
+
+    def thinking(enabled = true)
+      @thinking_enabled = enabled
+      if @sys_msg
+        @sys_msg = thinking_system_message(@sys_msg)
+        refresh_system_message(@sys_msg)
+      end
+      self
+    end
+
     def history_messages
       @engine.history_messages
     end
@@ -71,23 +102,43 @@ module SmartPrompt
         SmartPrompt.logger.info "Use template #{template_name}"
         raise "Template #{template_name} not found" unless @templates.key?(template_name)
         content = @templates[template_name].render(params)
-        add_message({ role: "user", content: content }, with_history)
+        add_user_content(content, with_history)
         self
       else
-        add_message({ role: "user", content: template_name }, with_history)
+        add_user_content(template_name, with_history)
         self
       end
     end
 
     def sys_msg(message, params)
-      @sys_msg = message
-      add_message({ role: "system", content: message }, params[:with_history])
+      @sys_msg = thinking_system_message(message)
+      add_message({ role: "system", content: @sys_msg }, params[:with_history])
+      self
+    end
+
+    def multimodal_prompt(parts, with_history: false)
+      add_message({ role: "user", content: normalize_content_parts(parts) }, with_history)
+      self
+    end
+
+    def image(source, token_budget: nil, **metadata)
+      @pending_content_parts << media_part("image", source, token_budget: token_budget, **metadata)
+      self
+    end
+
+    def audio(source, **metadata)
+      @pending_content_parts << media_part("audio", source, **metadata)
+      self
+    end
+
+    def video(source, fps: nil, max_seconds: nil, **metadata)
+      @pending_content_parts << media_part("video", source, fps: fps, max_seconds: max_seconds, **metadata)
       self
     end
 
     def send_msg_once
       raise "No LLM selected" if @current_llm.nil?
-      @last_response = @current_llm.send_request(@messages, @model_name, @temperature)
+      @last_response = send_llm_request(@messages, nil)
       @messages = []
       @messages << { role: "system", content: @sys_msg }
       @last_response
@@ -97,9 +148,9 @@ module SmartPrompt
       Retriable.retriable(RETRY_OPTIONS) do
         raise ConfigurationError, "No LLM selected" if @current_llm.nil?
         if params[:with_history]
-          @last_response = @current_llm.send_request(history_messages, @model_name, @temperature, @tools, nil)
+          @last_response = send_llm_request(history_messages, nil)
         else
-          @last_response = @current_llm.send_request(@messages, @model_name, @temperature, @tools, nil)
+          @last_response = send_llm_request(@messages, nil)
         end
         if @last_response == ""
           @last_response = @current_llm.last_response
@@ -116,9 +167,9 @@ module SmartPrompt
       Retriable.retriable(RETRY_OPTIONS) do
         raise ConfigurationError, "No LLM selected" if @current_llm.nil?
         if params[:with_history]
-          @current_llm.send_request(history_messages, @model_name, @temperature, @tools, proc)
+          send_llm_request(history_messages, proc)
         else
-          @current_llm.send_request(@messages, @model_name, @temperature, @tools, proc)
+          send_llm_request(@messages, proc)
         end
         @messages = []
         @messages << { role: "system", content: @sys_msg }
@@ -152,5 +203,120 @@ module SmartPrompt
         normalize(@last_response, length)
       end
     end
+
+    private
+
+    def send_llm_request(messages, proc)
+      parameters = @current_llm.method(:send_request).parameters
+      if parameters.length >= 6
+        @current_llm.send_request(messages, @model_name, @temperature, @tools, proc, @request_options)
+      else
+        @current_llm.send_request(messages, @model_name, @temperature, @tools, proc)
+      end
+    end
+
+    def merge_model_request_options(model_config)
+      explicit_options = model_config["request_options"] || model_config[:request_options] || {}
+      @request_options.merge!(explicit_options)
+      MODEL_REQUEST_OPTION_KEYS.each do |key|
+        value = model_config[key] || model_config[key.to_sym]
+        @request_options[key.to_sym] = value unless value.nil?
+      end
+    end
+
+    def add_user_content(content, with_history)
+      if @pending_content_parts.empty?
+        add_message({ role: "user", content: content }, with_history)
+      else
+        add_message({ role: "user", content: multimodal_content(content) }, with_history)
+        @pending_content_parts = []
+      end
+    end
+
+    def multimodal_content(text)
+      parts = @pending_content_parts
+      images_and_videos = parts.select { |part| ["image_url", "image", "video_url", "video"].include?(part[:type] || part["type"]) }
+      audio_parts = parts.select { |part| ["input_audio", "audio"].include?(part[:type] || part["type"]) }
+      other_parts = parts - images_and_videos - audio_parts
+      normalize_content_parts(images_and_videos + other_parts + [{ type: "text", text: text.to_s }] + audio_parts)
+    end
+
+    def normalize_content_parts(parts)
+      parts.map do |part|
+        normalized = part.transform_keys(&:to_s)
+        normalized["text"] = normalized.delete("content") if normalized["type"] == "text" && normalized.key?("content")
+        normalized
+      end
+    end
+
+    def media_part(type, source, **metadata)
+      case type
+      when "image"
+        mime_type = detect_image_mime(source)
+        data = File.binread(source)
+        base64_data = Base64.strict_encode64(data)
+        url = "data:#{mime_type};base64,#{base64_data}"
+        part = { type: "image_url", image_url: { url: url } }
+      when "audio"
+        format = detect_audio_format(source)
+        data = File.binread(source)
+        base64_data = Base64.strict_encode64(data)
+        part = { type: "input_audio", input_audio: { data: base64_data, format: format } }
+      when "video"
+        mime_type = detect_video_mime(source)
+        data = File.binread(source)
+        base64_data = Base64.strict_encode64(data)
+        url = "data:#{mime_type};base64,#{base64_data}"
+        part = { type: "video_url", video_url: { url: url } }
+      else
+        part = { type: type }
+      end
+      metadata.each do |key, value|
+        part[key] = value unless value.nil?
+      end
+      part
+    end
+
+    def detect_image_mime(path)
+      ext = File.extname(path).downcase
+      case ext
+      when ".png"  then "image/png"
+      when ".jpg", ".jpeg" then "image/jpeg"
+      when ".gif"  then "image/gif"
+      when ".webp" then "image/webp"
+      when ".bmp"  then "image/bmp"
+      when ".svg"  then "image/svg+xml"
+      else "application/octet-stream"
+      end
+    end
+
+    def detect_audio_format(path)
+      ext = File.extname(path).downcase.delete_prefix(".")
+      %w[wav mp3 ogg flac aac m4a].include?(ext) ? ext : "wav"
+    end
+
+    def detect_video_mime(path)
+      ext = File.extname(path).downcase
+      case ext
+      when ".mp4"  then "video/mp4"
+      when ".webm" then "video/webm"
+      when ".mov"  then "video/quicktime"
+      when ".avi"  then "video/x-msvideo"
+      else "application/octet-stream"
+      end
+    end
+
+    def thinking_system_message(message)
+      message = message.to_s.sub(/\A<\|think\|>\n?/, "")
+      return message if @thinking_enabled == false
+      return message unless @thinking_enabled == true
+
+      "<|think|>\n#{message}"
+    end
+
+    def refresh_system_message(message)
+      system_message = @messages.find { |item| (item[:role] || item["role"]) == "system" }
+      system_message[:content] = message if system_message
+    end
   end
 end

data/lib/smart_prompt/engine.rb

@@ -123,15 +123,12 @@ module SmartPrompt
         if result.class == String
           recive_message = {
             "role": "assistant",
-            "content": result,
+            "content": sanitize_history_content(result),
           }
         elsif result.class == Array
           recive_message = nil
         else
-          recive_message = {
-            "role": result.dig("choices", 0, "message", "role"),
-            "content": result.dig("choices", 0, "message", "content").to_s + result.dig("choices", 0, "message", "tool_calls").to_s,
-          }
+          recive_message = assistant_history_message(result)
         end
         worker.conversation.add_message(recive_message) if recive_message
         SmartPrompt.logger.info "Worker result is: #{result}"
@@ -175,5 +172,22 @@ module SmartPrompt
     def clear_history_messages
       @history_messages = []
     end
+
+    private
+
+    def assistant_history_message(result)
+      message = result.dig("choices", 0, "message") || {}
+      history_message = {
+        "role": message["role"] || "assistant",
+        "content": sanitize_history_content(message["content"].to_s),
+      }
+      tool_calls = message["tool_calls"]
+      history_message["tool_calls"] = tool_calls if tool_calls && !tool_calls.empty?
+      history_message
+    end
+
+    def sanitize_history_content(content)
+      content.to_s.gsub(/<\|channel\>thought\n.*?<channel\|>/m, "")
+    end
   end
 end

data/lib/smart_prompt/openai_adapter.rb

@@ -31,7 +31,19 @@ module SmartPrompt
       end
     end
 
-    def send_request(messages, model = nil, temperature = 0.7, tools = nil, proc = nil)
+    REQUEST_PARAMETER_KEYS = %w[
+      max_tokens
+      max_completion_tokens
+      top_p
+      top_k
+      response_format
+      tool_choice
+      parallel_tool_calls
+      seed
+      stop
+    ].freeze
+
+    def send_request(messages, model = nil, temperature = 0.7, tools = nil, proc = nil, request_options = {})
       SmartPrompt.logger.info "OpenAIAdapter: Sending request to OpenAI"
       temperature = 0.7 if temperature == nil
       if model
@@ -46,6 +58,8 @@ module SmartPrompt
           messages: messages,
           temperature: @config["temperature"] || temperature,
         }
+        parameters.merge!(configured_request_parameters)
+        parameters.merge!(request_options || {})
         if proc
           parameters[:stream] = proc
         end
@@ -99,5 +113,15 @@ module SmartPrompt
       end
       return response.dig("data", 0, "embedding")
     end
+
+    private
+
+    def configured_request_parameters
+      REQUEST_PARAMETER_KEYS.each_with_object({}) do |key, parameters|
+        next unless @config.key?(key)
+
+        parameters[key.to_sym] = @config[key]
+      end
+    end
   end
 end

data/lib/smart_prompt/version.rb

@@ -1,3 +1,3 @@
 module SmartPrompt
-  VERSION = "0.4.2"
+  VERSION = "0.4.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smart_prompt
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.4
 platform: ruby
 authors:
 - zhuang biaowei
@@ -93,6 +93,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.2.1
+- !ruby/object:Gem::Dependency
+  name: base64
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
 description: SmartPrompt provides a flexible DSL for managing prompts, interacting
   with multiple LLMs, and creating composable task workers.
 email:
@@ -138,7 +152,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.13
 specification_version: 4
 summary: A smart prompt management and LLM interaction gem
 test_files: []