smart_prompt 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 396c6097973289a34143e86b65f428d55919d0e755916992a5c714e289ebf5a2
-  data.tar.gz: 5d2c2d81b486e1fb05b53f116047ab1f90be77c9536fd6440a96b573bdad00c3
+  metadata.gz: 683f259828b34a687bb598abff6fe3ef547e918954b7422eecf2d357ba237495
+  data.tar.gz: 5410ee6c08d46643ac1c6c9ccbccc38f5729973a65063454c8b75c7787a587a6
 SHA512:
-  metadata.gz: c6880395149678a195ea6efc46a81623d61c14c56534936041a625616a9e6b716597c078ccee0ac0d47c3b2a8e67d272742ee36940fa64321426799db0b26e4d
-  data.tar.gz: 63f0b8ae0f6f62363443731cae6fed3eafe746c463d259c9b88eee8563d6ef0cdcd84e684d6daeedd7329c3dcab84748ab62358fa2935b9d0278ec347df45ffb
+  metadata.gz: 25755db9db82c9af27d8007753da625767831f835bd5767726db738c2863dfea9a3429d4a600e7dbd5005729cfbd3b679b75e4657eab13dd7b6e446379984ea0
+  data.tar.gz: 4fb6b5f037c504b7a0ae440066aa4608fafa5a97c516ac220ffcd0817ffd03e880d0d7473e0e6537b50c6e771ca59f3b08aa9cbb3b24fabb41665a06ebe69ca2

data/README.cn.md

@@ -75,6 +75,14 @@ llms:
     adapter: openai
     url: http://localhost:11434/
     default_model: deepseek-r1
+  gemma4_local:
+    adapter: openai
+    url: http://localhost:8000/v1
+    api_key: dummy
+    default_model: gemma-4-12B-it
+    temperature: 1.0
+    top_p: 0.95
+    top_k: 64
   deepseek:
     adapter: openai
     url: https://api.deepseek.com
@@ -89,6 +97,10 @@ models:
   deepseekv3.2:
     use: SiliconFlow
     model: Pro/deepseek-ai/DeepSeek-V3.2
+  gemma4/12b:
+    use: gemma4_local
+    model: gemma-4-12B-it
+    max_tokens: 1024
 
 # 默认设置
 default_llm: SiliconFlow
@@ -170,6 +182,26 @@ engine.call_worker_by_stream(:streaming_chat, {
 end
 ```
 
+### Gemma 4 12B 多模态
+
+Gemma 4 12B 可以通过 LiteRT-LM、LM Studio、Ollama、llama.cpp 等 OpenAI 兼容本地服务接入。SmartPrompt 会把图片放在文本前、音频放在文本后，以匹配 Gemma 4 的多模态最佳实践。
+
+```ruby
+SmartPrompt.define_worker :gemma_multimodal_assistant do
+  use_model "gemma4/12b"
+  thinking params.fetch(:thinking, true)
+  sys_msg("你是一个严谨的本地多模态助手。", params)
+
+  image(params[:image], token_budget: params[:token_budget] || 280) if params[:image]
+  video(params[:video], fps: 1, max_seconds: 60) if params[:video]
+  audio(params[:audio]) if params[:audio]
+  prompt(params[:message])
+
+  request_options(response_format: { type: "json_object" }) if params[:json]
+  send_msg
+end
+```
+
 ### 工具集成
 
 ```ruby

data/README.md

@@ -75,6 +75,14 @@ llms:
     adapter: openai
     url: http://localhost:11434/
     default_model: deepseek-r1
+  gemma4_local:
+    adapter: openai
+    url: http://localhost:8000/v1
+    api_key: dummy
+    default_model: gemma-4-12B-it
+    temperature: 1.0
+    top_p: 0.95
+    top_k: 64
   deepseek:
     adapter: openai
     url: https://api.deepseek.com
@@ -89,6 +97,10 @@ models:
   deepseekv3.2:
     use: SiliconFlow
     model: Pro/deepseek-ai/DeepSeek-V3.2
+  gemma4/12b:
+    use: gemma4_local
+    model: gemma-4-12B-it
+    max_tokens: 1024
 
 # Default settings
 default_llm: SiliconFlow
@@ -170,6 +182,26 @@ engine.call_worker_by_stream(:streaming_chat, {
 end
 ```
 
+### Gemma 4 12B Multimodal
+
+Gemma 4 12B can be connected through OpenAI-compatible local servers such as LiteRT-LM, LM Studio, Ollama, or llama.cpp. SmartPrompt places images before text and audio after text to match Gemma 4 multimodal best practices.
+
+```ruby
+SmartPrompt.define_worker :gemma_multimodal_assistant do
+  use_model "gemma4/12b"
+  thinking params.fetch(:thinking, true)
+  sys_msg("You are a precise local multimodal assistant.", params)
+
+  image(params[:image], token_budget: params[:token_budget] || 280) if params[:image]
+  video(params[:video], fps: 1, max_seconds: 60) if params[:video]
+  audio(params[:audio]) if params[:audio]
+  prompt(params[:message])
+
+  request_options(response_format: { type: "json_object" }) if params[:json]
+  send_msg
+end
+```
+
 ### Tool Integration
 
 ```ruby

data/lib/smart_prompt/conversation.rb

@@ -5,6 +5,18 @@ require "numo/narray"
 module SmartPrompt
   class Conversation
     include APIHandler
+    MODEL_REQUEST_OPTION_KEYS = %w[
+      max_tokens
+      max_completion_tokens
+      top_p
+      top_k
+      response_format
+      tool_choice
+      parallel_tool_calls
+      seed
+      stop
+    ].freeze
+
     attr_reader :messages, :last_response, :config_file
     attr_reader :last_call_id
 
@@ -21,6 +33,9 @@ module SmartPrompt
       @current_adapter = engine.current_adapter
       @last_response = nil
       @tools = tools
+      @request_options = {}
+      @pending_content_parts = []
+      @thinking_enabled = nil
     end
 
     def use(llm_name)
@@ -43,6 +58,7 @@ module SmartPrompt
 
       use(llm_name)
       model(configured_model_name)
+      merge_model_request_options(model_config)
       self
     end
 
@@ -54,6 +70,20 @@ module SmartPrompt
       @temperature = temperature
     end
 
+    def request_options(options = {})
+      @request_options.merge!(options || {})
+      self
+    end
+
+    def thinking(enabled = true)
+      @thinking_enabled = enabled
+      if @sys_msg
+        @sys_msg = thinking_system_message(@sys_msg)
+        refresh_system_message(@sys_msg)
+      end
+      self
+    end
+
     def history_messages
       @engine.history_messages
     end
@@ -71,23 +101,43 @@ module SmartPrompt
         SmartPrompt.logger.info "Use template #{template_name}"
         raise "Template #{template_name} not found" unless @templates.key?(template_name)
         content = @templates[template_name].render(params)
-        add_message({ role: "user", content: content }, with_history)
+        add_user_content(content, with_history)
         self
       else
-        add_message({ role: "user", content: template_name }, with_history)
+        add_user_content(template_name, with_history)
         self
       end
     end
 
     def sys_msg(message, params)
-      @sys_msg = message
-      add_message({ role: "system", content: message }, params[:with_history])
+      @sys_msg = thinking_system_message(message)
+      add_message({ role: "system", content: @sys_msg }, params[:with_history])
+      self
+    end
+
+    def multimodal_prompt(parts, with_history: false)
+      add_message({ role: "user", content: normalize_content_parts(parts) }, with_history)
+      self
+    end
+
+    def image(source, token_budget: nil, **metadata)
+      @pending_content_parts << media_part("image", source, token_budget: token_budget, **metadata)
+      self
+    end
+
+    def audio(source, **metadata)
+      @pending_content_parts << media_part("audio", source, **metadata)
+      self
+    end
+
+    def video(source, fps: nil, max_seconds: nil, **metadata)
+      @pending_content_parts << media_part("video", source, fps: fps, max_seconds: max_seconds, **metadata)
       self
     end
 
     def send_msg_once
       raise "No LLM selected" if @current_llm.nil?
-      @last_response = @current_llm.send_request(@messages, @model_name, @temperature)
+      @last_response = send_llm_request(@messages, nil)
       @messages = []
       @messages << { role: "system", content: @sys_msg }
       @last_response
@@ -97,9 +147,9 @@ module SmartPrompt
       Retriable.retriable(RETRY_OPTIONS) do
         raise ConfigurationError, "No LLM selected" if @current_llm.nil?
         if params[:with_history]
-          @last_response = @current_llm.send_request(history_messages, @model_name, @temperature, @tools, nil)
+          @last_response = send_llm_request(history_messages, nil)
         else
-          @last_response = @current_llm.send_request(@messages, @model_name, @temperature, @tools, nil)
+          @last_response = send_llm_request(@messages, nil)
         end
         if @last_response == ""
           @last_response = @current_llm.last_response
@@ -116,9 +166,9 @@ module SmartPrompt
       Retriable.retriable(RETRY_OPTIONS) do
         raise ConfigurationError, "No LLM selected" if @current_llm.nil?
         if params[:with_history]
-          @current_llm.send_request(history_messages, @model_name, @temperature, @tools, proc)
+          send_llm_request(history_messages, proc)
         else
-          @current_llm.send_request(@messages, @model_name, @temperature, @tools, proc)
+          send_llm_request(@messages, proc)
         end
         @messages = []
         @messages << { role: "system", content: @sys_msg }
@@ -152,5 +202,79 @@ module SmartPrompt
         normalize(@last_response, length)
       end
     end
+
+    private
+
+    def send_llm_request(messages, proc)
+      parameters = @current_llm.method(:send_request).parameters
+      if parameters.length >= 6
+        @current_llm.send_request(messages, @model_name, @temperature, @tools, proc, @request_options)
+      else
+        @current_llm.send_request(messages, @model_name, @temperature, @tools, proc)
+      end
+    end
+
+    def merge_model_request_options(model_config)
+      explicit_options = model_config["request_options"] || model_config[:request_options] || {}
+      @request_options.merge!(explicit_options)
+      MODEL_REQUEST_OPTION_KEYS.each do |key|
+        value = model_config[key] || model_config[key.to_sym]
+        @request_options[key.to_sym] = value unless value.nil?
+      end
+    end
+
+    def add_user_content(content, with_history)
+      if @pending_content_parts.empty?
+        add_message({ role: "user", content: content }, with_history)
+      else
+        add_message({ role: "user", content: multimodal_content(content) }, with_history)
+        @pending_content_parts = []
+      end
+    end
+
+    def multimodal_content(text)
+      parts = @pending_content_parts
+      images_and_videos = parts.select { |part| ["image", "video"].include?(part[:type] || part["type"]) }
+      audio_parts = parts.select { |part| (part[:type] || part["type"]) == "audio" }
+      other_parts = parts - images_and_videos - audio_parts
+      normalize_content_parts(images_and_videos + other_parts + [{ type: "text", text: text.to_s }] + audio_parts)
+    end
+
+    def normalize_content_parts(parts)
+      parts.map do |part|
+        normalized = part.transform_keys(&:to_s)
+        normalized["text"] = normalized.delete("content") if normalized["type"] == "text" && normalized.key?("content")
+        normalized
+      end
+    end
+
+    def media_part(type, source, **metadata)
+      part = { type: type }
+      case type
+      when "image"
+        part[:url] = source
+      when "audio"
+        part[:audio] = source
+      when "video"
+        part[:video] = source
+      end
+      metadata.each do |key, value|
+        part[key] = value unless value.nil?
+      end
+      part
+    end
+
+    def thinking_system_message(message)
+      message = message.to_s.sub(/\A<\|think\|>\n?/, "")
+      return message if @thinking_enabled == false
+      return message unless @thinking_enabled == true
+
+      "<|think|>\n#{message}"
+    end
+
+    def refresh_system_message(message)
+      system_message = @messages.find { |item| (item[:role] || item["role"]) == "system" }
+      system_message[:content] = message if system_message
+    end
   end
 end

data/lib/smart_prompt/engine.rb

@@ -123,15 +123,12 @@ module SmartPrompt
         if result.class == String
           recive_message = {
             "role": "assistant",
-            "content": result,
+            "content": sanitize_history_content(result),
           }
         elsif result.class == Array
           recive_message = nil
         else
-          recive_message = {
-            "role": result.dig("choices", 0, "message", "role"),
-            "content": result.dig("choices", 0, "message", "content").to_s + result.dig("choices", 0, "message", "tool_calls").to_s,
-          }
+          recive_message = assistant_history_message(result)
         end
         worker.conversation.add_message(recive_message) if recive_message
         SmartPrompt.logger.info "Worker result is: #{result}"
@@ -175,5 +172,22 @@ module SmartPrompt
     def clear_history_messages
       @history_messages = []
     end
+
+    private
+
+    def assistant_history_message(result)
+      message = result.dig("choices", 0, "message") || {}
+      history_message = {
+        "role": message["role"] || "assistant",
+        "content": sanitize_history_content(message["content"].to_s),
+      }
+      tool_calls = message["tool_calls"]
+      history_message["tool_calls"] = tool_calls if tool_calls && !tool_calls.empty?
+      history_message
+    end
+
+    def sanitize_history_content(content)
+      content.to_s.gsub(/<\|channel\>thought\n.*?<channel\|>/m, "")
+    end
   end
 end

data/lib/smart_prompt/openai_adapter.rb

@@ -31,7 +31,19 @@ module SmartPrompt
       end
     end
 
-    def send_request(messages, model = nil, temperature = 0.7, tools = nil, proc = nil)
+    REQUEST_PARAMETER_KEYS = %w[
+      max_tokens
+      max_completion_tokens
+      top_p
+      top_k
+      response_format
+      tool_choice
+      parallel_tool_calls
+      seed
+      stop
+    ].freeze
+
+    def send_request(messages, model = nil, temperature = 0.7, tools = nil, proc = nil, request_options = {})
       SmartPrompt.logger.info "OpenAIAdapter: Sending request to OpenAI"
       temperature = 0.7 if temperature == nil
       if model
@@ -46,6 +58,8 @@ module SmartPrompt
           messages: messages,
           temperature: @config["temperature"] || temperature,
         }
+        parameters.merge!(configured_request_parameters)
+        parameters.merge!(request_options || {})
         if proc
           parameters[:stream] = proc
         end
@@ -99,5 +113,15 @@ module SmartPrompt
       end
       return response.dig("data", 0, "embedding")
     end
+
+    private
+
+    def configured_request_parameters
+      REQUEST_PARAMETER_KEYS.each_with_object({}) do |key, parameters|
+        next unless @config.key?(key)
+
+        parameters[key.to_sym] = @config[key]
+      end
+    end
   end
 end

data/lib/smart_prompt/version.rb

@@ -1,3 +1,3 @@
 module SmartPrompt
-  VERSION = "0.4.2"
+  VERSION = "0.4.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: smart_prompt
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors:
 - zhuang biaowei
@@ -138,7 +138,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.13
 specification_version: 4
 summary: A smart prompt management and LLM interaction gem
 test_files: []