llm.rb 4.7.0 → 4.9.0

data/lib/llm/tracer/langsmith.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+module LLM
+  ##
+  # LangSmith-specific tracer built on top of Telemetry. Supports extra
+  # inputs/outputs and metadata on traces and spans via {#merge_extra} and
+  # {#start_trace}(metadata:).
+  #
+  # @example Constructor metadata and tags
+  #   llm.tracer = LLM::Tracer::Langsmith.new(
+  #     llm,
+  #     session_id: "123e4567-e89b-12d3-a456-426614174000",
+  #     metadata: {env: "dev"},
+  #     tags: ["changelog"]
+  #   )
+  #
+  # @example Per-request extra metadata and inputs (e.g. from chatbot)
+  #   tracer.merge_extra(
+  #     metadata: { turn_id: turn.id, component: "chatbot_message_stream" },
+  #     inputs: { "gen_ai.input.messages" => messages_json }
+  #   )
+  #   bot.chat(prompt)
+  #
+  # @example Trace-level metadata via start_trace
+  #   tracer.start_trace(trace_group_id: turn.id, name: "chatbot.turn", metadata: { turn_id: turn.id })
+  class Tracer::Langsmith < Tracer::Telemetry
+    THREAD_EXTRA_KEY = :llm_langsmith_extra
+
+    UUID = /\A
+      [0-9a-f]{8}-
+      [0-9a-f]{4}-
+      [1-5][0-9a-f]{3}-
+      [89ab][0-9a-f]{3}-
+      [0-9a-f]{12}
+    \z/ix
+
+    def initialize(provider, options = {})
+      super
+      setup_langsmith!(options)
+    end
+
+    def start_trace(trace_group_id: nil, name: "llm", attributes: {}, metadata: nil)
+      merge_extra(metadata: metadata) if metadata && !metadata.empty?
+      super
+    end
+
+    def stop_trace
+      clear_thread_extra!
+      super
+    end
+
+    def merge_extra(metadata: nil, inputs: nil, outputs: nil)
+      store = thread_extra
+      store[:metadata].merge!(metadata) if metadata && !metadata.empty?
+      store[:inputs].merge!(inputs) if inputs && !inputs.empty?
+      store[:outputs].merge!(outputs) if outputs && !outputs.empty?
+      self
+    end
+
+    def current_extra
+      store = thread_extra
+      {
+        metadata: store[:metadata].dup,
+        inputs: store[:inputs].dup,
+        outputs: store[:outputs].dup
+      }
+    end
+
+    def consume_extra_inputs
+      thread_extra[:inputs].tap { thread_extra[:inputs] = {} }
+    end
+
+    def consume_extra_outputs
+      thread_extra[:outputs].tap { thread_extra[:outputs] = {} }
+    end
+
+    private
+
+    def trace_attributes(span_kind:)
+      attributes = {}
+      unless @langsmith_session_id.to_s.empty?
+        attributes["langsmith.trace.session_id"] = @langsmith_session_id
+      end
+      merged_metadata = @langsmith_metadata.merge(thread_extra[:metadata])
+      merged_metadata.each do |key, value|
+        next if value.nil?
+
+        attr_key = key.to_s.start_with?("langsmith.metadata.") ? key.to_s : "langsmith.metadata.#{key}"
+        attributes[attr_key] = serialize_langsmith_value(value)
+      end
+      unless @langsmith_tags.empty?
+        attributes["langsmith.span.tags"] = @langsmith_tags.map(&:to_s).join(",")
+      end
+      attributes["langsmith.span.kind"] = span_kind
+      attributes
+    end
+
+    def thread_extra
+      Thread.current[THREAD_EXTRA_KEY] ||= {
+        metadata: {},
+        inputs: {},
+        outputs: {}
+      }
+    end
+
+    def clear_thread_extra!
+      Thread.current[THREAD_EXTRA_KEY] = nil
+    end
+
+    def setup_langsmith!(options)
+      options ||= {}
+      @langsmith_metadata = options[:metadata] || {}
+      @langsmith_session_id = normalize_langsmith_session_id(
+        options[:session_id],
+        metadata: @langsmith_metadata
+      )
+      @langsmith_tags = options[:tags] || []
+    end
+
+    def serialize_langsmith_value(value)
+      case value
+      when String, Numeric, TrueClass, FalseClass
+        value
+      else
+        LLM.json.dump(value)
+      end
+    end
+
+    def normalize_langsmith_session_id(session_id, metadata:)
+      raw = session_id&.to_s
+      return nil if raw.to_s.empty?
+      return raw if uuid?(raw)
+
+      # Keep arbitrary identifiers in metadata instead of forcing
+      # them into langsmith.trace.session_id, which expects a UUID.
+      metadata[:session_id] ||= raw
+      nil
+    end
+
+    def uuid?(value)
+      value.match?(UUID)
+    end
+  end
+end

data/lib/llm/tracer/logger.rb

@@ -23,7 +23,7 @@ module LLM
     ##
     # @param (see LLM::Tracer#on_request_start)
     # @return [void]
-    def on_request_start(operation:, model: nil)
+    def on_request_start(operation:, model: nil, **)
       case operation
       when "chat" then start_chat(operation:, model:)
       when "retrieval" then start_retrieval(operation:)
@@ -188,5 +188,13 @@ module LLM
         **finish_attributes(operation, res)
       )
     end
+
+    ##
+    # @param (see LLM::Tracer#set_finish_metadata_proc)
+    # @return [self]
+    def set_finish_metadata_proc(_proc = nil)
+      Thread.current[LLM::Tracer::FINISH_METADATA_PROC_KEY] = nil
+      self
+    end
   end
 end

data/lib/llm/tracer/null.rb

@@ -45,5 +45,13 @@ module LLM
     def on_tool_error(**)
       nil
     end
+
+    ##
+    # @param (see LLM::Tracer#set_finish_metadata_proc)
+    # @return [self]
+    def set_finish_metadata_proc(_proc = nil)
+      Thread.current[LLM::Tracer::FINISH_METADATA_PROC_KEY] = nil
+      self
+    end
   end
 end

data/lib/llm/tracer/telemetry.rb

@@ -7,8 +7,7 @@ module LLM
   # The {LLM::Tracer::Telemetry LLM::Tracer::Telemetry} tracer provides
   # telemetry support through the [opentelemetry-ruby](https://github.com/open-telemetry/opentelemetry-ruby)
   # RubyGem. The gem should be installed separately since this feature is opt-in
-  # and disabled by default. This feature exists to support integration with tools
-  # like [LangSmith](https://www.langsmith.com).
+  # and disabled by default.
   #
   # @see https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai Telemetry specs (index)
   # @see https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/openai.md Telemetry specs (OpenAI)
@@ -21,10 +20,10 @@ module LLM
   #   llm = LLM.openai(key: ENV["KEY"])
   #   llm.tracer = LLM::Tracer::Telemetry.new(llm)
   #
-  #   ses = LLM::Session.new(llm)
-  #   ses.talk "hello"
-  #   ses.talk "how are you?"
-  #   ses.tracer.spans.each { |span| pp span }
+  #   ctx = LLM::Context.new(llm)
+  #   ctx.talk "hello"
+  #   ctx.talk "how are you?"
+  #   ctx.tracer.spans.each { |span| pp span }
   #
   # @example OTLP export
   #   #!/usr/bin/env ruby
@@ -37,9 +36,9 @@ module LLM
   #   llm = LLM.openai(key: ENV["KEY"])
   #   llm.tracer = LLM::Tracer::Telemetry.new(llm, exporter:)
   #
-  #   ses = LLM::Session.new(llm)
-  #   ses.talk "hello"
-  #   ses.talk "how are you?"
+  #   ctx = LLM::Context.new(llm)
+  #   ctx.talk "hello"
+  #   ctx.talk "how are you?"
   class Tracer::Telemetry < Tracer
     ##
     # param [LLM::Provider] provider
@@ -48,7 +47,6 @@ module LLM
     def initialize(provider, options = {})
       super
       @exporter = options.delete(:exporter)
-      setup_langsmith!(options.delete(:langsmith))
       setup!
     end
 
@@ -59,7 +57,7 @@ module LLM
     #
     # @param (see LLM::Tracer#start_trace)
     # @return [self]
-    def start_trace(trace_group_id: nil, name: "llm", attributes: {})
+    def start_trace(trace_group_id: nil, name: "llm", attributes: {}, metadata: nil)
       return self if trace_group_id.to_s.empty?
 
       span_context = span_context_from_trace_group_id(trace_group_id.to_s)
@@ -74,25 +72,25 @@ module LLM
         attributes: attrs,
         with_parent: parent_ctx
       )
-      thread[thread_root_span_key] = root_span
-      thread[thread_root_context_key] = ::OpenTelemetry::Trace.context_with_span(root_span)
+      @root_span = root_span
+      @root_context = ::OpenTelemetry::Trace.context_with_span(root_span)
       self
     end
 
     ##
     # @return [self]
     def stop_trace
-      thread[thread_root_span_key]&.finish
-      thread[thread_root_span_key] = nil
-      thread[thread_root_context_key] = nil
+      @root_span&.finish
+      @root_span = nil
+      @root_context = nil
       self
     end
 
     ##
     # @param (see LLM::Tracer#on_request_start)
-    def on_request_start(operation:, model: nil)
+    def on_request_start(operation:, model: nil, inputs: nil)
       case operation
-      when "chat" then start_chat(operation:, model:)
+      when "chat" then start_chat(operation:, model:, inputs:)
       when "retrieval" then start_retrieval(operation:)
       else nil
       end
@@ -100,10 +98,10 @@ module LLM
 
     ##
     # @param (see LLM::Tracer#on_request_finish)
-    def on_request_finish(operation:, res:, model: nil, span: nil)
+    def on_request_finish(operation:, res:, model: nil, span: nil, outputs: nil, metadata: nil)
       return nil unless span
       case operation
-      when "chat" then finish_chat(operation:, model:, res:, span:)
+      when "chat" then finish_chat(operation:, model:, res:, span:, outputs:, metadata:)
       when "retrieval" then finish_retrieval(operation:, res:, span:)
       else nil
       end
@@ -133,7 +131,7 @@ module LLM
         "gen_ai.provider.name" => provider_name,
         "server.address" => provider_host,
         "server.port" => provider_port
-      }.merge!(langsmith_attributes(span_kind: "tool")).compact
+      }.merge!(trace_attributes(span_kind: "tool")).compact
       span_name = ["execute_tool", name].compact.join(" ")
       span = create_span(span_name.empty? ? "gen_ai.tool" : span_name, attributes:)
       span.add_event("gen_ai.tool.start")
@@ -197,30 +195,12 @@ module LLM
     ##
     # @api private
     def create_span(name, kind: :client, attributes: {})
-      root_context = thread[thread_root_context_key]
+      root_context = @root_context
       opts = {kind:, attributes:}
       opts[:with_parent] = root_context if root_context
       @tracer.start_span(name, **opts)
     end
 
-    ##
-    # @api private
-    def thread_root_span_key
-      @thread_root_span_key ||= :"llm.telemetry.root_span.#{object_id}"
-    end
-
-    ##
-    # @api private
-    def thread_root_context_key
-      @thread_root_context_key ||= :"llm.telemetry.root_context.#{object_id}"
-    end
-
-    ##
-    # @api private
-    def thread
-      Thread.current
-    end
-
     ##
     # Converts a string trace_group_id to an OpenTelemetry SpanContext so all
     # spans created with this context share the same trace_id.
@@ -282,16 +262,20 @@ module LLM
     ##
     # start_*
 
-    def start_chat(operation:, model:)
+    def start_chat(operation:, model:, inputs: nil)
+      request_metadata = consume_request_metadata
+      input_value = request_metadata[:user_input]
       attributes = {
         "gen_ai.operation.name" => operation,
         "gen_ai.request.model" => model,
         "gen_ai.provider.name" => provider_name,
         "server.address" => provider_host,
-        "server.port" => provider_port
-      }.merge!(langsmith_attributes(span_kind: "llm")).compact
+        "server.port" => provider_port,
+        "input.value" => serialize_request_value(input_value)
+      }.merge!(trace_attributes(span_kind: "llm")).compact
       span_name = [operation, model].compact.join(" ")
       span = create_span(span_name.empty? ? "gen_ai.request" : span_name, attributes:)
+      set_span_attributes(span, consume_extra_inputs.merge(inputs || {}))
       span.add_event("gen_ai.request.start")
       span
     end
@@ -302,7 +286,7 @@ module LLM
         "gen_ai.provider.name" => provider_name,
         "server.address" => provider_host,
         "server.port" => provider_port
-      }.merge!(langsmith_attributes(span_kind: "retriever")).compact
+      }.merge!(trace_attributes(span_kind: "retriever")).compact
       span = create_span(operation, attributes:)
       span.add_event("gen_ai.request.start")
       span
@@ -311,16 +295,26 @@ module LLM
     ##
     # finish_*
 
-    def finish_chat(operation:, model:, res:, span:)
+    def finish_chat(operation:, model:, res:, span:, outputs: nil, metadata: nil)
+      output_value = if res.respond_to?(:output_text)
+        res.output_text
+      else
+        (res.respond_to?(:content) ? res.content : nil)
+      end
       attributes = {
         "gen_ai.operation.name" => operation,
         "gen_ai.request.model" => model,
         "gen_ai.response.id" => res.id,
         "gen_ai.response.model" => model,
         "gen_ai.usage.input_tokens" => res.usage.input_tokens,
-        "gen_ai.usage.output_tokens" => res.usage.output_tokens
+        "gen_ai.usage.output_tokens" => res.usage.output_tokens,
+        "output.value" => serialize_request_value(output_value)
       }.merge!(finish_attributes(operation, res)).compact
       attributes.each { span.set_attribute(_1, _2) }
+      set_span_attributes(span, consume_extra_outputs.merge(outputs || {}))
+      finish_metadata = consume_finish_metadata_proc(res)
+      metadata = (metadata || {}).merge(finish_metadata || {})
+      set_span_attributes(span, metadata.transform_keys { "langsmith.metadata.#{_1}" })
       span.add_event("gen_ai.request.finish")
       span.tap(&:finish)
     end
@@ -329,57 +323,83 @@ module LLM
       attributes = {
         "gen_ai.operation.name" => operation
       }.merge!(finish_attributes(operation, res)).compact
+      chunks_json = retrieval_chunks_json(res)
+      attributes["langsmith.metadata.chunks"] = chunks_json if chunks_json
       attributes.each { span.set_attribute(_1, _2) }
       span.add_event("gen_ai.request.finish")
       span.tap(&:finish)
     end
 
-    def setup_langsmith!(options)
-      options ||= {}
-      @langsmith_metadata = options[:metadata] || {}
-      @langsmith_session_id = normalize_langsmith_session_id(options[:session_id], metadata: @langsmith_metadata)
-      @langsmith_tags = options[:tags] || []
+    ##
+    # @api private
+    # Serialize retrieval response chunks for span attributes (e.g. langsmith.metadata.chunks).
+    # Returns a JSON string or nil when res has no data.
+    def consume_finish_metadata_proc(res)
+      key = LLM::Tracer::FINISH_METADATA_PROC_KEY
+      proc = Thread.current[key]
+      Thread.current[key] = nil
+      return {} unless proc.respond_to?(:call)
+
+      proc.call(res) || {}
+    rescue
+      {}
     end
 
-    def langsmith_attributes(span_kind:)
-      attributes = {}
-      unless @langsmith_session_id.to_s.empty?
-        attributes["langsmith.trace.session_id"] = @langsmith_session_id
-      end
-      @langsmith_metadata.each do |key, value|
-        next if value.nil?
+    def retrieval_chunks_json(res)
+      return nil unless res.respond_to?(:data)
 
-        attributes["langsmith.metadata.#{key}"] = serialize_langsmith_value(value)
-      end
-      unless @langsmith_tags.empty?
-        attributes["langsmith.span.tags"] = @langsmith_tags.map(&:to_s).join(",")
+      data = res.data
+      return nil unless data.is_a?(Array)
+
+      payload = data.map { |c| c.respond_to?(:to_h) ? c.to_h : c }
+      LLM.json.dump(payload)
+    rescue
+      nil
+    end
+
+    ##
+    # @api private
+    # Hook for tracer-specific span attributes.
+    # Subclasses can override this to inject provider-agnostic tags.
+    def trace_attributes(span_kind:)
+      {}
+    end
+
+    ##
+    # @api private
+    # Sets attribute key-value pairs on the span, serializing non-primitive values to JSON.
+    def set_span_attributes(span, attrs)
+      return if attrs.nil? || attrs.empty?
+
+      attrs.each do |key, value|
+        span.set_attribute(key.to_s, serialize_span_value(value))
       end
-      attributes["langsmith.span.kind"] = span_kind
-      attributes
     end
 
-    def serialize_langsmith_value(value)
+    ##
+    # @api private
+    # OpenTelemetry attributes accept String, Numeric, Boolean, or Array of those.
+    # Complex values (hashes, arrays of objects) are serialized to JSON strings.
+    def serialize_span_value(value)
       case value
       when String, Numeric, TrueClass, FalseClass
         value
+      when Array
+        value.all? { |v| v.is_a?(String) || v.is_a?(Numeric) || v == true || v == false } ? value : LLM.json.dump(value)
       else
         LLM.json.dump(value)
       end
     end
 
-    def normalize_langsmith_session_id(session_id, metadata:)
-      raw = session_id&.to_s
-      return nil if raw.to_s.empty?
-      return raw if uuid?(raw)
-
-      # Keep arbitrary thread identifiers in metadata instead of forcing
-      # them into langsmith.trace.session_id, which expects a known UUID.
-      metadata[:session_id] ||= raw
-      nil
-    end
-
-    def uuid?(value)
-      value.match?(/\A[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\z/i)
+    def serialize_request_value(value)
+      case value
+      when nil
+        nil
+      when String
+        value
+      else
+        LLM.json.dump(value)
+      end
     end
   end
 end

data/lib/llm/tracer.rb

@@ -11,6 +11,7 @@ module LLM
   class Tracer
     require_relative "tracer/logger"
     require_relative "tracer/telemetry"
+    require_relative "tracer/langsmith"
     require_relative "tracer/null"
 
     ##
@@ -27,19 +28,22 @@ module LLM
     # Called before an LLM provider request is executed.
     # @param [String] operation
     # @param [String] model
+    # @param [Hash, nil] inputs Optional span attributes (e.g. gen_ai.input.messages) from llm.rb or caller.
     # @return [void]
-    def on_request_start(operation:, model: nil)
+    def on_request_start(operation:, model: nil, inputs: nil)
       raise NotImplementedError, "#{self.class} does not implement '#{__method__}'"
     end
 
     ##
     # Called after an LLM provider request succeeds.
     # @param [String] operation
-    # @param [String] model
     # @param [LLM::Response] res
     # @param [Object, nil] span
+    # @param [String] model
+    # @param [Hash, nil] outputs Optional span attributes (e.g. gen_ai.output.messages) from llm.rb or caller.
+    # @param [Hash, nil] metadata Optional metadata (emitted as langsmith.metadata.*) from llm.rb or caller.
     # @return [void]
-    def on_request_finish(operation:, res:, model: nil, span: nil)
+    def on_request_finish(operation:, res:, model: nil, span: nil, outputs: nil, metadata: nil)
       raise NotImplementedError, "#{self.class} does not implement '#{__method__}'"
     end
 
@@ -101,8 +105,11 @@ module LLM
     #  Name for the root span (e.g. "chatbot.turn").
     # @param [Hash] attributes
     #  OpenTelemetry attributes to set on the root span.
+    # @param [Hash, nil] metadata
+    #  Optional. Trace-level metadata merged into the trace (e.g. langsmith.metadata.*).
+    #  Only used by tracers that support it (e.g. {LLM::Tracer::Langsmith}).
     # @return [self]
-    def start_trace(trace_group_id: nil, name: "llm", attributes: {})
+    def start_trace(trace_group_id: nil, name: "llm", attributes: {}, metadata: nil)
       self
     end
 
@@ -136,8 +143,105 @@ module LLM
       nil
     end
 
+    ##
+    # Merges extra attributes for the current trace/span. Used by applications
+    # (e.g. chatbot) to add metadata, span inputs, or span outputs to the next
+    # span or to the trace. No-op by default; {LLM::Tracer::Langsmith} merges
+    # into fiber-local storage and emits them as langsmith/GenAI attributes.
+    #
+    # @param [Hash, nil] metadata
+    #  Key-value pairs merged into trace/span metadata (e.g. langsmith.metadata.*).
+    # @param [Hash, nil] inputs
+    #  Key-value pairs set on the next span at start (e.g. gen_ai.input.messages).
+    #  Consumed when the span is created.
+    # @param [Hash, nil] outputs
+    #  Key-value pairs set on the current span at finish (e.g. gen_ai.output.messages).
+    #  Must be set before the request finishes (e.g. in a block passed to the provider).
+    # @return [self]
+    def merge_extra(metadata: nil, inputs: nil, outputs: nil)
+      self
+    end
+
+    ##
+    # Optional: set a proc to supply metadata when the next chat span finishes.
+    # The proc is called with the response (res) and should return a Hash of
+    # metadata (e.g. { intent: "...", confidence: 1.0 }) to merge onto the span
+    # as langsmith.metadata.*. Cleared after use. Used by apps to attach
+    # routing/intent that is only known after the response.
+    #
+    # @param [Proc, nil] proc (res) -> Hash or nil
+    # @return [self]
+    def set_finish_metadata_proc(proc)
+      thread[FINISH_METADATA_PROC_KEY] = proc
+      self
+    end
+
+    FINISH_METADATA_PROC_KEY = :"llm.tracer.finish_metadata_proc"
+
+    ##
+    # Returns the current extra bag (metadata, inputs, outputs) for the current
+    # thread/trace. Used by subclasses; default returns empty hashes.
+    #
+    # @return [Hash] { metadata: {}, inputs: {}, outputs: {} }
+    def current_extra
+      {}
+    end
+
+    ##
+    # Returns and clears extra inputs for the next span. Called by the telemetry
+    # tracer when starting a span. Subclasses (e.g. Langsmith) override to
+    # return fiber-local inputs; default returns {}.
+    #
+    # @return [Hash] Attribute key => value to set on the span at start
+    def consume_extra_inputs
+      {}
+    end
+
+    ##
+    # Returns and clears extra outputs for the current span. Called by the
+    # telemetry tracer when finishing a span. Subclasses override to return
+    # fiber-local outputs; default returns {}.
+    #
+    # @return [Hash] Attribute key => value to set on the span at finish
+    def consume_extra_outputs
+      {}
+    end
+
+    ##
+    # Store per-request metadata (e.g. user_input) to be consumed by tracers
+    # when starting the next span. Used for plain-text input.value / output.value.
+    #
+    # @param [Hash] metadata e.g. { user_input: "the user question" }
+    # @return [nil]
+    def set_request_metadata(metadata)
+      return nil unless metadata && !metadata.empty?
+      key = thread_request_metadata_key
+      current = thread[key] || {}
+      thread[key] = current.merge(metadata.compact)
+      nil
+    end
+
+    ##
+    # Consume and clear per-request metadata. Called by the telemetry tracer at span start.
+    #
+    # @return [Hash]
+    def consume_request_metadata
+      key = thread_request_metadata_key
+      data = thread[key] || {}
+      thread[key] = nil
+      data
+    end
+
     private
 
+    def thread_request_metadata_key
+      @thread_request_metadata_key ||= :"llm.tracer.request_metadata.#{object_id}"
+    end
+
+    def thread
+      Thread.current
+    end
+
     ##
     # @return [String]
     def provider_name

data/lib/llm/usage.rb

@@ -8,4 +8,9 @@
 # It can also help track usage of the context window (which may
 # vary by model).
 class LLM::Usage < Struct.new(:input_tokens, :output_tokens, :reasoning_tokens, :total_tokens, keyword_init: true)
+  ##
+  # @return [String]
+  def to_json(...)
+    LLM.json.dump({input_tokens:, output_tokens:, reasoning_tokens:, total_tokens:})
+  end
 end

data/lib/llm/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LLM
-  VERSION = "4.7.0"
+  VERSION = "4.9.0"
 end