turnkit 0.4.0 → 0.4.2

data/lib/turnkit/media_analysis_result.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module TurnKit
+  class MediaAnalysisResult
+    attr_reader :text, :data, :model, :provider, :usage, :params, :media, :metadata, :error
+
+    def self.from_h(value)
+      new(**value.transform_keys(&:to_sym))
+    end
+
+    def initialize(text: "", data: nil, model: nil, provider: nil, usage: Usage.new, params: {}, media: {}, metadata: {}, error: nil, **)
+      @text = text.to_s
+      @data = data
+      @model = model
+      @provider = provider
+      @usage = usage.is_a?(Usage) ? usage : Usage.from_h(usage || {})
+      @params = params || {}
+      @media = media || {}
+      @metadata = metadata || {}
+      @error = error
+    end
+
+    def data?
+      !data.nil?
+    end
+
+    def cost
+      Cost.from_usage(usage, model: model)
+    end
+
+    def to_h
+      {
+        "text" => text,
+        "data" => data,
+        "model" => model,
+        "provider" => provider,
+        "usage" => usage.to_h,
+        "cost" => cost.to_h,
+        "params" => params,
+        "media" => media,
+        "metadata" => metadata,
+        "error" => error
+      }.compact
+    end
+  end
+end

data/lib/turnkit/media_input.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+require "pathname"
+require "stringio"
+require "uri"
+
+module TurnKit
+  class MediaInput
+    SUPPORTED_MIME_TYPES = %w[image/png image/jpeg image/webp image/gif application/pdf].freeze
+    EXTENSION_MIME_TYPES = {
+      ".png" => "image/png",
+      ".jpg" => "image/jpeg",
+      ".jpeg" => "image/jpeg",
+      ".webp" => "image/webp",
+      ".gif" => "image/gif",
+      ".pdf" => "application/pdf",
+      ".mp3" => "audio/mpeg",
+      ".wav" => "audio/wav",
+      ".m4a" => "audio/mp4",
+      ".mp4" => "video/mp4",
+      ".mov" => "video/quicktime",
+      ".webm" => "video/webm"
+    }.freeze
+
+    attr_reader :source, :mime_type, :filename, :metadata, :source_type
+
+    def self.wrap(value, **options)
+      value.is_a?(self) && options.empty? ? value : new(value, **options)
+    end
+
+    def self.bytes(data, mime_type:, filename: nil, metadata: {})
+      new(data, source_type: :bytes, mime_type: mime_type, filename: filename, metadata: metadata)
+    end
+
+    def initialize(source, mime_type: nil, filename: nil, metadata: {}, source_type: nil)
+      @source = source
+      @source_type = (source_type || infer_source_type).to_s
+      @filename = filename || infer_filename
+      @mime_type = mime_type || infer_mime_type
+      @metadata = metadata || {}
+
+      validate!
+    end
+
+    def kind
+      return "image" if mime_type&.start_with?("image/")
+      return "audio" if mime_type&.start_with?("audio/")
+      return "video" if mime_type&.start_with?("video/")
+      return "pdf" if mime_type == "application/pdf"
+
+      nil
+    end
+
+    def byte_size
+      case source_type
+      when "path"
+        File.size(source.to_s) if File.file?(source.to_s)
+      when "bytes"
+        source.bytesize
+      when "io"
+        source.size if source.respond_to?(:size)
+      when "active_storage"
+        active_storage_byte_size
+      end
+    end
+
+    def url
+      source.to_s if source_type == "url"
+    end
+
+    def path
+      source.to_s if source_type == "path"
+    end
+
+    def attachment_source
+      case source_type
+      when "bytes"
+        StringIO.new(source)
+      else
+        source
+      end
+    end
+
+    def to_h
+      {
+        "kind" => kind,
+        "mime_type" => mime_type,
+        "filename" => filename,
+        "byte_size" => byte_size,
+        "url" => url,
+        "path" => path,
+        "metadata" => metadata
+      }.compact
+    end
+
+    private
+      def infer_source_type
+        return :url if source.to_s.match?(%r{\Ahttps?://})
+        return :active_storage if active_storage?
+        return :path if source.is_a?(Pathname) || (source.is_a?(String) && File.exist?(source))
+        return :io if source.respond_to?(:read)
+        return :bytes if source.is_a?(String)
+
+        raise ArgumentError, "unsupported media input: #{source.class}"
+      end
+
+      def infer_filename
+        case source_type
+        when "url"
+          basename = File.basename(URI(source.to_s).path).to_s
+          basename.empty? ? nil : basename
+        when "path"
+          File.basename(source.to_s)
+        when "io"
+          source.respond_to?(:path) ? File.basename(source.path.to_s) : nil
+        when "active_storage"
+          active_storage_filename
+        end
+      end
+
+      def infer_mime_type
+        active_storage_content_type || mime_from_filename || mime_from_marcel
+      end
+
+      def mime_from_filename
+        EXTENSION_MIME_TYPES[File.extname(filename.to_s).downcase]
+      end
+
+      def mime_from_marcel
+        require "marcel"
+
+        Marcel::MimeType.for(marcel_io, name: filename)
+      rescue LoadError
+        nil
+      ensure
+        rewind_source
+      end
+
+      def marcel_io
+        case source_type
+        when "path"
+          Pathname.new(source.to_s)
+        when "bytes"
+          StringIO.new(source)
+        when "io"
+          source
+        else
+          nil
+        end
+      end
+
+      def validate!
+        return if mime_type.nil?
+        return if SUPPORTED_MIME_TYPES.include?(mime_type)
+        return if mime_type.start_with?("audio/", "video/")
+
+        raise ArgumentError, "unsupported media type: #{mime_type}"
+      end
+
+      def active_storage?
+        return false unless defined?(ActiveStorage)
+
+        (defined?(ActiveStorage::Blob) && source.is_a?(ActiveStorage::Blob)) ||
+          (defined?(ActiveStorage::Attached::One) && source.is_a?(ActiveStorage::Attached::One)) ||
+          (defined?(ActiveStorage::Attached::Many) && source.is_a?(ActiveStorage::Attached::Many))
+      end
+
+      def active_storage_filename
+        if defined?(ActiveStorage::Blob) && source.is_a?(ActiveStorage::Blob)
+          source.filename.to_s
+        elsif source.respond_to?(:filename)
+          source.filename.to_s
+        elsif source.respond_to?(:blob)
+          source.blob&.filename&.to_s
+        elsif source.respond_to?(:blobs)
+          source.blobs.first&.filename&.to_s
+        end
+      end
+
+      def active_storage_content_type
+        if defined?(ActiveStorage::Blob) && source.is_a?(ActiveStorage::Blob)
+          source.content_type
+        elsif source.respond_to?(:content_type)
+          source.content_type
+        elsif source.respond_to?(:blob)
+          source.blob&.content_type
+        elsif source.respond_to?(:blobs)
+          source.blobs.first&.content_type
+        end
+      end
+
+      def active_storage_byte_size
+        if defined?(ActiveStorage::Blob) && source.is_a?(ActiveStorage::Blob)
+          source.byte_size
+        elsif source.respond_to?(:byte_size)
+          source.byte_size
+        elsif source.respond_to?(:blob)
+          source.blob&.byte_size
+        elsif source.respond_to?(:blobs)
+          source.blobs.first&.byte_size
+        end
+      end
+
+      def rewind_source
+        source.rewind if source_type == "io" && source.respond_to?(:rewind)
+      end
+  end
+end

data/lib/turnkit/message.rb

@@ -3,7 +3,7 @@
 module TurnKit
   class Message
     ROLES = %w[user assistant tool].freeze
-    KINDS = %w[text tool_call tool_result context_summary image].freeze
+    KINDS = %w[text tool_call tool_result context_summary image media_analysis].freeze
 
     attr_reader :id, :conversation_id, :turn_id, :role, :kind, :sequence
     attr_reader :content, :tool_execution_id, :provider_message_id, :metadata, :created_at
@@ -61,6 +61,10 @@ module TurnKit
       kind == "image"
     end
 
+    def media_analysis?
+      kind == "media_analysis"
+    end
+
     def text
       content.filter_map do |part|
         attrs = stringify(part)

data/lib/turnkit/message_projection.rb

@@ -46,6 +46,8 @@ module TurnKit
         { role: :tool, content: part&.fetch("text", message.text) || message.text, tool_call_id: part&.fetch("tool_call_id", nil) }
       when "image"
         { role: :assistant, content: projected_images }
+      when "media_analysis"
+        { role: :assistant, content: projected_media_analyses }
       else
         { role: message.role.to_sym, content: message.text }
       end
@@ -76,5 +78,14 @@ module TurnKit
           "Generated image: #{attrs.to_json}"
         end.join("\n")
       end
+
+      def projected_media_analyses
+        message.content.filter_map do |part|
+          next unless part.fetch("type") == "media_analysis"
+
+          media = part.fetch("media", {}).slice("kind", "mime_type", "filename", "url").compact
+          [ "Media analysis: #{media.to_json}", part["text"].to_s ].reject(&:empty?).join("\n")
+        end.join("\n")
+      end
   end
 end

data/lib/turnkit/model_request.rb

@@ -2,13 +2,14 @@
 
 module TurnKit
   class ModelRequest
-    attr_reader :model, :messages, :tools, :instructions, :thinking, :output_schema, :metadata, :report
+    attr_reader :model, :messages, :tools, :instructions, :dynamic_instructions, :thinking, :output_schema, :metadata, :report
 
-    def initialize(model:, messages:, tools:, instructions:, thinking: nil, output_schema: nil, metadata: {}, report: nil)
+    def initialize(model:, messages:, tools:, instructions:, dynamic_instructions: nil, thinking: nil, output_schema: nil, metadata: {}, report: nil)
       @model = model
       @messages = Array(messages)
       @tools = Array(tools)
       @instructions = instructions.to_s
+      @dynamic_instructions = dynamic_instructions.to_s
       @thinking = thinking
       @output_schema = output_schema
       @metadata = metadata || {}
@@ -25,6 +26,7 @@ module TurnKit
         "messages" => messages,
         "tools" => tool_names,
         "instructions" => instructions,
+        "dynamic_instructions" => dynamic_instructions,
         "thinking" => thinking,
         "output_schema" => output_schema,
         "metadata" => metadata,

data/lib/turnkit/output_policy.rb

@@ -40,6 +40,15 @@ module TurnKit
       end
     end
 
+    def self.require_media_analysis
+      lambda do |output, output_data: nil, turn: nil, **|
+        data = output_data.is_a?(Hash) ? output_data : output
+        analyses = data.is_a?(Hash) ? data["media_analyses"] || data[:media_analyses] : nil
+        has_analysis = Array(analyses).any? || turn&.conversation&.messages_for_turn(turn)&.any?(&:media_analysis?)
+        { rule: "media_analysis_required", message: "output must include a media analysis result" } unless has_analysis
+      end
+    end
+
     def initialize(content:, name: "output_policy", model: nil, thinking: nil, client: nil)
       @name = name.to_s
       @content = content.to_s
@@ -66,7 +75,7 @@ module TurnKit
       else
         audit_client = client || TurnKit.client
         audit_client.validate!(model: model_name)
-        chat(audit_client, model: model_name, messages: audit_messages(output), tools: [], instructions: audit_instructions, thinking: thinking, output_schema: DEFAULT_SCHEMA, metadata: { output_policy: name })
+        audit_client.chat(model: model_name, messages: audit_messages(output), tools: [], instructions: audit_instructions, thinking: thinking, output_schema: DEFAULT_SCHEMA, metadata: { output_policy: name })
       end
       data = result.output_data || parse_json(result.text)
       return if data.fetch("approved", false)
@@ -102,20 +111,6 @@ module TurnKit
         [ { role: :user, content: JSON.generate(output: output) } ]
       end
 
-      def chat(client, **kwargs)
-        accepted = chat_keyword_names(client)
-        kwargs = kwargs.slice(*accepted) unless accepted.include?(:keyrest)
-        client.chat(**kwargs)
-      end
-
-      def chat_keyword_names(client)
-        client.method(:chat).parameters.filter_map do |kind, name|
-          return [ :keyrest ] if kind == :keyrest
-
-          name if %i[key keyreq].include?(kind)
-        end
-      end
-
       def parse_json(value)
         JSON.parse(extract_json(value.to_s))
       rescue JSON::ParserError

data/lib/turnkit/result.rb

@@ -40,6 +40,18 @@ module TurnKit
       images.any?
     end
 
+    def media_analyses
+      parts.filter_map do |part|
+        next unless part["type"] == "media_analysis"
+
+        MediaAnalysisResult.from_h(part)
+      end
+    end
+
+    def media_analysis?
+      media_analyses.any?
+    end
+
     private
       def synthesize_parts(text:, tool_calls:)
         parts = []

data/lib/turnkit/run.rb

@@ -11,16 +11,12 @@ module TurnKit
     def id = turn.id
     def root_turn_id = turn.root_turn_id
     def status = turn.status
-    def output = output_text
     def output_text = turn.output_text
     def output_data = turn.output_data
     def policy_audit = turn.policy_audit
     def policy_clean? = policy_audit.nil? || policy_audit.fetch("clean", false)
     def usage = Usage.from_records(turn_records)
     def cost = Cost.from_records(turn_records)
-    def steps = turn_records.length
-    def tool_calls = tool_executions
-    def persisted? = true
 
     def error
       turn.store.load_turn(id)["error"]

data/lib/turnkit/store.rb

@@ -12,12 +12,10 @@ module TurnKit
     def create_turn(_attributes) = raise(NotImplementedError)
     def load_turn(_id) = raise(NotImplementedError)
     def update_turn(_id, _attributes) = raise(NotImplementedError)
-    def claim_turn(id, from: "pending", to: "running", **attributes)
-      turn = load_turn(id)
-      return nil unless turn["status"] == from
-
-      update_turn(id, attributes.merge(status: to))
-    end
+    # claim_turn is the concurrency-safety point: it must atomically
+    # compare-and-set status from `from` to `to` (returning nil when the turn
+    # is not in `from`), so concurrent workers cannot both claim a turn.
+    def claim_turn(_id, from: "pending", to: "running", **_attributes) = raise(NotImplementedError)
     def list_turns(root_turn_id: nil, conversation_id: nil, agent_name: nil) = raise(NotImplementedError)
 
     def create_tool_execution(_attributes) = raise(NotImplementedError)

data/lib/turnkit/sub_agent_tool.rb

@@ -2,8 +2,7 @@
 
 module TurnKit
   class SubAgentTool < Tool
-    parameter :task, :string, required: true, description: "The task for the sub-agent to complete."
-    parameter :context, :string, required: false, description: "Relevant context for the sub-agent."
+    parameter :task, :string, required: true, description: "The complete task for the sub-agent, including all relevant context."
 
     def self.for(agent)
       Class.new(self) do
@@ -18,25 +17,21 @@ module TurnKit
       end
     end
 
-    def call(task:, context: nil, turnkit_context:)
+    def call(task:, context:)
       sub_agent = self.class.agent
-      parent_turn = turnkit_context.turn
-      prompt = [ task, context ].compact.join("\n\n")
-      conversation = sub_agent.conversation(metadata: {
+      parent_turn = context.turn
+      lineage = {
         "parent_conversation_id" => parent_turn.conversation.id,
         "parent_turn_id" => parent_turn.id,
-        "parent_tool_execution_id" => turnkit_context.execution.id
-      })
-      trigger = conversation.say(prompt, metadata: {
-        "parent_conversation_id" => parent_turn.conversation.id,
-        "parent_turn_id" => parent_turn.id,
-        "parent_tool_execution_id" => turnkit_context.execution.id
-      })
+        "parent_tool_execution_id" => context.execution.id
+      }
+      conversation = sub_agent.conversation(metadata: lineage)
+      trigger = conversation.say(task, metadata: lineage)
       child = conversation.run!(
         trigger_message_id: trigger.id,
         budget: parent_turn.budget,
         parent_turn: parent_turn,
-        parent_tool_execution: turnkit_context.execution,
+        parent_tool_execution: context.execution,
         depth: parent_turn.depth + 1,
         model: sub_agent.effective_model,
         agent: sub_agent,

data/lib/turnkit/system_prompt.rb

@@ -3,7 +3,6 @@
 module TurnKit
   class SystemPrompt
     DEFAULT_SECTIONS = %i[agent instructions behavior loaded_skills available_skills tools subject live_context environment].freeze
-    CACHE_BOUNDARY = "<!-- TURNKIT_DYNAMIC_PROMPT_BOUNDARY -->"
     NONE_PROMPT = "You are an assistant running inside TurnKit."
     PROMPT_MODES = %i[full minimal task none].freeze
     MODE_SECTIONS = {
@@ -13,7 +12,6 @@ module TurnKit
       none: []
     }.freeze
     DYNAMIC_SECTIONS = %i[subject live_context environment].freeze
-    OVERRIDABLE_SECTIONS = %i[behavior tools].freeze
 
     SECTION_METHODS = {
       agent: :agent_section,
@@ -92,44 +90,28 @@ module TurnKit
       raise ArgumentError, "unknown prompt mode: #{@mode}" unless PROMPT_MODES.include?(@mode)
 
       @sections = Array(sections || prompt_sections_for_mode)
-      @prompt_contribution = nil
     end
 
-    def to_s
-      return NONE_PROMPT if mode == :none
-
-      values = []
-      contribution = prompt_contribution
-      values << contribution.stable_prefix unless contribution.stable_prefix.empty?
-
-      boundary_inserted = false
-      sections.each do |section|
-        rendered = render(section)
-        next if rendered.nil? || rendered.strip.empty?
-
-        if dynamic_section?(section) && !boundary_inserted
-          values << CACHE_BOUNDARY
-          boundary_inserted = true
-        end
-
-        values << rendered
-      end
+    # The prompt splits into a stable part (identical across turns of the same
+    # agent, safe to cache) and a dynamic part (subject, live context,
+    # environment) recomputed each turn. Adapters that support prompt caching
+    # receive both via ModelRequest#instructions and #dynamic_instructions.
+    def stable
+      parts.fetch(0).join("\n\n")
+    end
 
-      unless contribution.dynamic_suffix.empty?
-        values << CACHE_BOUNDARY unless boundary_inserted
-        values << contribution.dynamic_suffix
-      end
+    def dynamic
+      parts.fetch(1).join("\n\n")
+    end
 
-      values.compact.reject { |value| value.strip.empty? }.join("\n\n")
+    def to_s
+      [ stable, dynamic ].reject(&:empty?).join("\n\n")
     end
 
     def render(section)
       method = SECTION_METHODS[section.to_sym]
       raise ArgumentError, "unknown prompt section: #{section}" unless method
 
-      override = section_override(section)
-      return tagged(section, override) if override
-
       public_send(method)
     end
 
@@ -292,11 +274,9 @@ module TurnKit
 
     def report
       text = to_s
-      stable, dynamic = self.class.split_cache_boundary(text)
       {
         "chars" => text.length,
         "hash" => Digest::SHA256.hexdigest(text),
-        "has_cache_boundary" => text.include?(CACHE_BOUNDARY),
         "stable_chars" => stable.length,
         "dynamic_chars" => dynamic.length,
         "sections" => sections.map(&:to_s),
@@ -304,12 +284,27 @@ module TurnKit
       }
     end
 
-    def self.split_cache_boundary(text)
-      stable, dynamic = text.to_s.split(CACHE_BOUNDARY, 2)
-      [ stable.to_s, dynamic.to_s ]
-    end
-
     private
+      def parts
+        @parts ||= build_parts
+      end
+
+      def build_parts
+        return [ [ NONE_PROMPT ], [] ] if mode == :none
+
+        stable_parts = []
+        dynamic_parts = []
+        target = stable_parts
+        sections.each do |section|
+          rendered = render(section)
+          next if rendered.nil? || rendered.strip.empty?
+
+          target = dynamic_parts if dynamic_section?(section)
+          target << rendered
+        end
+        [ stable_parts, dynamic_parts ]
+      end
+
       def tagged(name, content)
         "<#{name}>\n#{content}\n</#{name}>"
       end
@@ -383,64 +378,5 @@ module TurnKit
         end
       end
 
-      def prompt_contribution
-        @prompt_contribution ||= merge_prompt_contributions(resolve_prompt_contributions)
-      end
-
-      def resolve_prompt_contributions
-        contributors = Array(TurnKit.system_prompt_contributors)
-        contributors += matching_model_prompt_contributors
-        contributors.filter_map do |contributor|
-          value = contributor.respond_to?(:call) ? contributor.call(prompt_build_context) : contributor
-          normalize_prompt_contribution(value)
-        end
-      end
-
-      def matching_model_prompt_contributors
-        model_name = (turn.model || agent.effective_model).to_s
-        TurnKit.model_prompt_contributors.flat_map do |matcher, contributor|
-          matches = case matcher
-          when Regexp
-            matcher.match?(model_name)
-          else
-            matcher.to_s == model_name
-          end
-          matches ? Array(contributor) : []
-        end
-      end
-
-      def normalize_prompt_contribution(value)
-        case value
-        when nil, false
-          nil
-        when PromptContribution
-          value
-        when Hash
-          PromptContribution.new(
-            stable_prefix: value[:stable_prefix] || value["stable_prefix"],
-            dynamic_suffix: value[:dynamic_suffix] || value["dynamic_suffix"],
-            section_overrides: value[:section_overrides] || value["section_overrides"]
-          )
-        else
-          PromptContribution.new(stable_prefix: value.to_s)
-        end
-      end
-
-      def merge_prompt_contributions(contributions)
-        stable_prefix = contributions.map(&:stable_prefix).reject(&:empty?).join("\n\n")
-        dynamic_suffix = contributions.map(&:dynamic_suffix).reject(&:empty?).join("\n\n")
-        section_overrides = contributions.each_with_object({}) do |contribution, overrides|
-          overrides.merge!(contribution.section_overrides)
-        end
-        PromptContribution.new(stable_prefix: stable_prefix, dynamic_suffix: dynamic_suffix, section_overrides: section_overrides)
-      end
-
-      def section_override(section)
-        key = section.to_sym
-        return nil unless OVERRIDABLE_SECTIONS.include?(key)
-
-        value = prompt_contribution.section_overrides[key]
-        value.to_s unless value.nil?
-      end
   end
 end