langfuse-ruby 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0e15fd7f7f4e2e8b1a8018f27de35f42a3124973c8a455b50781a0d08924aa3d
-  data.tar.gz: 91253f7addf89cef33fcf047376f8667fae8e2e9d75f096b4096d4c876d06b20
+  metadata.gz: 98dc87bcfe8834aa485a810f1d955216a0c17e74a6b058f42f76539350b5c542
+  data.tar.gz: b826b87fc097c83637969f81a0aa0ff93e520c63a73519c78bd84cee3258af16
 SHA512:
-  metadata.gz: 62d8773683844ecc44a1d258765f0522f77e83b02cbb7b7441d892e3e6a9252348f8ccfd331e185186b46aa9fb30203146e7beffd29a1330b5ab14ff75862a47
-  data.tar.gz: 23e33a0b7d7b8b25c07e9663cf320771a6db8160fb18302179acc8aeaaf99c1740add31c84448106df46efe22e89b2c36ce7bae6b66e600e525226854a2003d7
+  metadata.gz: 80b9e696b8157772731a519712087f19e751caf72009a87995c5684793a9fba1d507ab38eaedccd31ce5fac9280634d07ce7646454a7854383312891a076c525
+  data.tar.gz: a05b595c3cc2e437511dbafa44df1331b5d64802f2008a51797419cb1eba9691ec7edef68435f55a972dff4b699b449f57603ef1161a7e73127d22a7c346a799

data/.github/workflows/release.yml

@@ -9,6 +9,8 @@ jobs:
   release:
     runs-on: ubuntu-latest
     if: startsWith(github.ref, 'refs/tags/v')
+    permissions:
+      contents: write
     
     steps:
     - uses: actions/checkout@v4

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    langfuse-ruby (0.1.6)
+    langfuse-ruby (0.1.7)
       concurrent-ruby (~> 1.0)
       faraday (>= 1.8, < 3.0)
       faraday-multipart (~> 1.0)

data/Makefile

@@ -26,7 +26,7 @@ lint-fix: ## Run RuboCop with auto-correct
 build: ## Build the gem
 	bundle exec rake build
 
-release: tag build ## Release: tag + build + push to RubyGems
+release: tag ## Release: tag + build + push to RubyGems
 	bundle exec rake release_gem
 
 clean: ## Remove built gem files
@@ -39,7 +39,7 @@ console: ## Start an IRB console with the gem loaded
 tag: ## Create and push a version tag. Usage: make tag [VERSION=x.y.z]
 	@git fetch --tags; \
 	if [ -z "$(VERSION)" ]; then \
-		LATEST=$$(git tag -l 'v*' --sort=-v:refname | head -n1); \
+		LATEST=$$(git tag -l 'v[0-9]*' --sort=-v:refname | head -n1); \
 		if [ -z "$$LATEST" ]; then \
 			NEW_TAG="v0.0.1"; \
 		else \
@@ -51,7 +51,7 @@ tag: ## Create and push a version tag. Usage: make tag [VERSION=x.y.z]
 		fi; \
 	else \
 		NEW_TAG="v$(VERSION)"; \
-		LATEST=$$(git tag -l 'v*' --sort=-v:refname | head -n1); \
+		LATEST=$$(git tag -l 'v[0-9]*' --sort=-v:refname | head -n1); \
 		if [ "$$LATEST" = "$$NEW_TAG" ]; then \
 			echo "Tag $$NEW_TAG already exists on remote, deleting and re-pushing..."; \
 			git tag -d "$$NEW_TAG" 2>/dev/null || true; \

data/README.md

@@ -58,6 +58,36 @@ end
 client = Langfuse.new
 ```
 
+### OpenTelemetry (OTEL) Ingestion Mode
+
+Langfuse v4 introduces a faster data model powered by OpenTelemetry. To use
+it, enable the OTEL ingestion mode. This sends data via the OTLP/HTTP JSON
+endpoint (`/api/public/otel/v1/traces`) with the `x-langfuse-ingestion-version: 4`
+header for real-time ingestion and observation-level online evaluators.
+
+```ruby
+# Via constructor
+client = Langfuse.new(
+  public_key: "pk-lf-...",
+  secret_key: "sk-lf-...",
+  ingestion_mode: :otel
+)
+
+# Via global configuration
+Langfuse.configure do |config|
+  config.public_key = "pk-lf-..."
+  config.secret_key = "sk-lf-..."
+  config.ingestion_mode = :otel
+end
+
+# Via environment variable
+# LANGFUSE_INGESTION_MODE=otel
+```
+
+All existing tracing APIs work unchanged. The SDK maps Langfuse events to
+OpenTelemetry spans with the appropriate `langfuse.*` and `gen_ai.*` attributes.
+No additional dependencies are required.
+
 ### 2. Basic Tracing
 
 ```ruby

data/lib/langfuse/client.rb

@@ -9,10 +9,11 @@ require 'concurrent'
 
 module Langfuse
   class Client
-    attr_reader :public_key, :secret_key, :host, :debug, :timeout, :retries, :flush_interval, :auto_flush
+    attr_reader :public_key, :secret_key, :host, :debug, :timeout, :retries, :flush_interval, :auto_flush,
+                :ingestion_mode
 
     def initialize(public_key: nil, secret_key: nil, host: nil, debug: false, timeout: 30, retries: 3,
-                   flush_interval: nil, auto_flush: nil)
+                   flush_interval: nil, auto_flush: nil, ingestion_mode: nil)
       @public_key = public_key || ENV['LANGFUSE_PUBLIC_KEY'] || Langfuse.configuration.public_key
       @secret_key = secret_key || ENV['LANGFUSE_SECRET_KEY'] || Langfuse.configuration.secret_key
       @host = host || ENV['LANGFUSE_HOST'] || Langfuse.configuration.host
@@ -25,11 +26,14 @@ module Langfuse
                     else
                       auto_flush
                     end
+      @ingestion_mode = resolve_ingestion_mode(ingestion_mode)
 
       raise AuthenticationError, 'Public key is required' unless @public_key
       raise AuthenticationError, 'Secret key is required' unless @secret_key
 
       @connection = build_connection
+      @otel_connection = build_otel_connection if @ingestion_mode == :otel
+      @otel_exporter = OtelExporter.new(connection: @otel_connection, debug: @debug) if @ingestion_mode == :otel
       @event_queue = Concurrent::Array.new
       @flush_thread = start_flush_thread if @auto_flush
     end
@@ -452,16 +456,38 @@ module Langfuse
         return
       end
 
+      if @ingestion_mode == :otel
+        send_batch_otel(valid_events)
+      else
+        send_batch_legacy(valid_events)
+      end
+    end
+
+    def send_batch_legacy(valid_events)
       batch_data = build_batch_data(valid_events)
       puts "Sending batch data: #{batch_data}" if @debug
 
       begin
         response = post('/api/public/ingestion', batch_data)
-        puts "Flushed #{valid_events.length} events" if @debug
+        puts "Flushed #{valid_events.length} events (legacy)" if @debug
         response
       rescue StandardError => e
         puts "Failed to flush events: #{e.message}" if @debug
-        # Re-queue events on failure
+        valid_events.each { |event| @event_queue << event }
+        raise
+      end
+    end
+
+    def send_batch_otel(valid_events)
+      puts "Sending #{valid_events.length} events via OTEL" if @debug
+
+      begin
+        response = @otel_exporter.export(valid_events)
+        handle_response(response)
+        puts "Flushed #{valid_events.length} events (otel)" if @debug
+        response
+      rescue StandardError => e
+        puts "Failed to flush OTEL events: #{e.message}" if @debug
         valid_events.each { |event| @event_queue << event }
         raise
       end
@@ -493,6 +519,15 @@ module Langfuse
       end
     end
 
+    def resolve_ingestion_mode(explicit_mode)
+      return explicit_mode.to_sym if explicit_mode
+
+      env_mode = ENV['LANGFUSE_INGESTION_MODE']
+      return env_mode.to_sym if env_mode && !env_mode.empty?
+
+      Langfuse.configuration.ingestion_mode || :legacy
+    end
+
     def build_connection
       Faraday.new(url: @host) do |conn|
         # 配置请求和响应处理
@@ -516,6 +551,22 @@ module Langfuse
       end
     end
 
+    # Build a separate Faraday connection for OTEL with the v4 ingestion header.
+    def build_otel_connection
+      Faraday.new(url: @host) do |conn|
+        conn.response :json, content_type: /\bjson$/
+
+        conn.headers['User-Agent'] = "langfuse-ruby/#{Langfuse::VERSION}"
+        conn.headers['Authorization'] = "Basic #{Base64.strict_encode64("#{@public_key}:#{@secret_key}")}"
+        conn.headers['x-langfuse-ingestion-version'] = '4'
+        conn.headers['Content-Type'] = 'application/json'
+
+        conn.options.timeout = @timeout
+        conn.response :logger if @debug
+        conn.adapter Faraday.default_adapter
+      end
+    end
+
     # HTTP methods
     def get(path, params = {})
       request(:get, path, params: params)

data/lib/langfuse/otel_exporter.rb

@@ -0,0 +1,333 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'securerandom'
+
+module Langfuse
+  # Converts batched Langfuse events into OTLP/HTTP JSON (ExportTraceServiceRequest)
+  # and sends them to the Langfuse OTEL endpoint for v4-compatible ingestion.
+  class OtelExporter
+    OTEL_ENDPOINT = '/api/public/otel/v1/traces'
+
+    # @param connection [Faraday::Connection] HTTP connection to Langfuse host
+    # @param debug [Boolean] whether to print debug output
+    def initialize(connection:, debug: false)
+      @connection = connection
+      @debug = debug
+    end
+
+    # Export a batch of Langfuse events as OTLP spans.
+    # @param events [Array<Hash>] array of event hashes from the event queue
+    # @return [Faraday::Response]
+    def export(events)
+      resource_spans = build_resource_spans(events)
+      payload = { resourceSpans: resource_spans }
+
+      puts "OTEL export payload: #{JSON.pretty_generate(payload)}" if @debug
+
+      @connection.post(OTEL_ENDPOINT) do |req|
+        req.headers['Content-Type'] = 'application/json'
+        req.body = JSON.generate(payload)
+      end
+    end
+
+    private
+
+    # Build the top-level resourceSpans array from events.
+    # Groups events by trace_id, producing one scopeSpan per trace.
+    def build_resource_spans(events)
+      grouped = group_events_by_trace(events)
+
+      scope_spans = grouped.map do |_trace_id, trace_events|
+        spans = trace_events.filter_map { |event| convert_event_to_span(event) }
+        next if spans.empty?
+
+        { scope: { name: 'langfuse-ruby', version: Langfuse::VERSION }, spans: spans }
+      end.compact
+
+      return [] if scope_spans.empty?
+
+      [{
+        resource: {
+          attributes: [
+            { key: 'service.name', value: { stringValue: 'langfuse-ruby' } },
+            { key: 'telemetry.sdk.name', value: { stringValue: 'langfuse-ruby' } },
+            { key: 'telemetry.sdk.version', value: { stringValue: Langfuse::VERSION } }
+          ]
+        },
+        scopeSpans: scope_spans
+      }]
+    end
+
+    # Group events by their trace ID for proper OTEL span hierarchy.
+    def group_events_by_trace(events)
+      groups = Hash.new { |h, k| h[k] = [] }
+
+      events.each do |event|
+        body = event[:body] || {}
+        trace_id = body['traceId'] || body['trace_id'] || body['id'] || 'unknown'
+        groups[trace_id] << event
+      end
+
+      groups
+    end
+
+    # Convert a single Langfuse event to an OTLP span hash, or nil if not convertible.
+    def convert_event_to_span(event)
+      type = event[:type]
+      body = event[:body] || {}
+
+      case type
+      when 'trace-create'
+        build_trace_span(body)
+      when 'span-create', 'span-update'
+        build_observation_span(body, 'span')
+      when 'generation-create', 'generation-update'
+        build_observation_span(body, 'generation')
+      when 'event-create'
+        build_event_span(body)
+      when 'score-create'
+        build_score_span(body)
+      end
+    end
+
+    # Build a root OTEL span for a Langfuse trace.
+    def build_trace_span(body)
+      trace_id = to_otel_trace_id(body['id'])
+      span_id = to_otel_span_id(body['id'])
+
+      attributes = []
+      add_attr(attributes, 'langfuse.trace.name', body['name'])
+      add_attr(attributes, 'langfuse.user.id', body['userId'])
+      add_attr(attributes, 'langfuse.session.id', body['sessionId'])
+      add_attr(attributes, 'langfuse.release', body['release'])
+      add_attr(attributes, 'langfuse.version', body['version'])
+      add_json_attr(attributes, 'langfuse.trace.input', body['input'])
+      add_json_attr(attributes, 'langfuse.trace.output', body['output'])
+      add_json_attr(attributes, 'langfuse.trace.metadata', body['metadata'])
+
+      tags = body['tags']
+      if tags.is_a?(Array) && !tags.empty?
+        add_array_attr(attributes, 'langfuse.trace.tags', tags)
+      end
+
+      {
+        traceId: trace_id,
+        spanId: span_id,
+        name: body['name'] || 'trace',
+        kind: 1, # SPAN_KIND_INTERNAL
+        startTimeUnixNano: to_unix_nano(body['timestamp']),
+        endTimeUnixNano: to_unix_nano(body['timestamp']),
+        attributes: attributes,
+        status: { code: 1 } # STATUS_CODE_OK
+      }
+    end
+
+    # Build an OTEL span for a Langfuse span or generation observation.
+    def build_observation_span(body, obs_type)
+      trace_id = to_otel_trace_id(body['traceId'])
+      span_id = to_otel_span_id(body['id'])
+
+      span = {
+        traceId: trace_id,
+        spanId: span_id,
+        name: body['name'] || obs_type,
+        kind: 1, # SPAN_KIND_INTERNAL
+        startTimeUnixNano: to_unix_nano(body['startTime']),
+        endTimeUnixNano: to_unix_nano(body['endTime'] || body['startTime']),
+        attributes: build_observation_attributes(body, obs_type),
+        status: { code: 1 }
+      }
+
+      # Set parent span ID
+      parent_id = body['parentObservationId']
+      if parent_id
+        span[:parentSpanId] = to_otel_span_id(parent_id)
+      else
+        # Parent is the trace root span
+        span[:parentSpanId] = to_otel_span_id(body['traceId'])
+      end
+
+      span
+    end
+
+    # Build an OTEL span for a Langfuse event (zero-duration span).
+    def build_event_span(body)
+      trace_id = to_otel_trace_id(body['traceId'])
+      span_id = to_otel_span_id(body['id'])
+      timestamp = to_unix_nano(body['startTime'])
+
+      attributes = []
+      add_attr(attributes, 'langfuse.observation.type', 'event')
+      add_json_attr(attributes, 'langfuse.observation.input', body['input'])
+      add_json_attr(attributes, 'langfuse.observation.output', body['output'])
+      add_json_attr(attributes, 'langfuse.observation.metadata', body['metadata'])
+      add_attr(attributes, 'langfuse.observation.level', body['level'])
+      add_attr(attributes, 'langfuse.observation.status_message', body['statusMessage'])
+
+      span = {
+        traceId: trace_id,
+        spanId: span_id,
+        name: body['name'] || 'event',
+        kind: 1,
+        startTimeUnixNano: timestamp,
+        endTimeUnixNano: timestamp,
+        attributes: attributes,
+        status: { code: 1 }
+      }
+
+      parent_id = body['parentObservationId']
+      if parent_id
+        span[:parentSpanId] = to_otel_span_id(parent_id)
+      elsif body['traceId']
+        span[:parentSpanId] = to_otel_span_id(body['traceId'])
+      end
+
+      span
+    end
+
+    # Build a minimal OTEL span for a score event.
+    def build_score_span(body)
+      trace_id_raw = body['traceId']
+      return nil unless trace_id_raw
+
+      trace_id = to_otel_trace_id(trace_id_raw)
+      span_id = to_otel_span_id(body['id'] || SecureRandom.uuid)
+      timestamp = to_unix_nano(body['timestamp'] || Time.now.utc.iso8601(3))
+
+      attributes = []
+      add_attr(attributes, 'langfuse.score.name', body['name'])
+      add_attr(attributes, 'langfuse.score.value', body['value'])
+      add_attr(attributes, 'langfuse.score.data_type', body['dataType'])
+      add_attr(attributes, 'langfuse.score.comment', body['comment'])
+      add_attr(attributes, 'langfuse.observation.type', 'score')
+
+      if body['observationId']
+        add_attr(attributes, 'langfuse.score.observation_id', body['observationId'])
+      end
+
+      span = {
+        traceId: trace_id,
+        spanId: span_id,
+        name: "score-#{body['name']}",
+        kind: 1,
+        startTimeUnixNano: timestamp,
+        endTimeUnixNano: timestamp,
+        attributes: attributes,
+        status: { code: 1 }
+      }
+
+      # Parent is either the observation or the trace
+      parent_raw = body['observationId'] || trace_id_raw
+      span[:parentSpanId] = to_otel_span_id(parent_raw) if parent_raw
+
+      span
+    end
+
+    # Build OTEL attributes for a span/generation observation.
+    def build_observation_attributes(body, obs_type)
+      attributes = []
+      effective_type = body['type'] || obs_type
+      add_attr(attributes, 'langfuse.observation.type', effective_type)
+      add_json_attr(attributes, 'langfuse.observation.input', body['input'])
+      add_json_attr(attributes, 'langfuse.observation.output', body['output'])
+      add_json_attr(attributes, 'langfuse.observation.metadata', body['metadata'])
+      add_attr(attributes, 'langfuse.observation.level', body['level'])
+      add_attr(attributes, 'langfuse.observation.status_message', body['statusMessage'])
+
+      if obs_type == 'generation'
+        add_generation_attributes(attributes, body)
+      end
+
+      attributes
+    end
+
+    # Add generation-specific gen_ai.* attributes.
+    def add_generation_attributes(attributes, body)
+      add_attr(attributes, 'gen_ai.request.model', body['model'])
+
+      model_params = body['modelParameters']
+      if model_params.is_a?(Hash)
+        model_params.each do |key, value|
+          add_attr(attributes, "gen_ai.request.#{key}", value) unless value.nil?
+        end
+      end
+
+      usage = body['usage']
+      if usage.is_a?(Hash)
+        add_attr(attributes, 'gen_ai.usage.prompt_tokens', usage['promptTokens'] || usage['prompt_tokens'])
+        add_attr(attributes, 'gen_ai.usage.completion_tokens', usage['completionTokens'] || usage['completion_tokens'])
+        total = usage['totalTokens'] || usage['total_tokens']
+        add_attr(attributes, 'gen_ai.usage.total_tokens', total) if total
+      end
+
+      add_attr(attributes, 'langfuse.observation.completion_start_time', body['completionStartTime'])
+    end
+
+    # Convert a UUID string to OTEL 32-char hex trace ID.
+    # OTEL trace IDs are 16 bytes (32 hex chars).
+    def to_otel_trace_id(uuid_str)
+      return '0' * 32 unless uuid_str
+
+      hex = uuid_str.to_s.delete('-')
+      hex.ljust(32, '0')[0, 32]
+    end
+
+    # Convert a UUID string to OTEL 16-char hex span ID.
+    # OTEL span IDs are 8 bytes (16 hex chars).
+    def to_otel_span_id(uuid_str)
+      return '0' * 16 unless uuid_str
+
+      hex = uuid_str.to_s.delete('-')
+      hex[0, 16]
+    end
+
+    # Convert an ISO8601 timestamp string to nanoseconds since epoch.
+    def to_unix_nano(timestamp_str)
+      return '0' unless timestamp_str
+
+      time = Time.parse(timestamp_str.to_s)
+      ((time.to_f * 1_000_000_000).to_i).to_s
+    rescue ArgumentError
+      '0'
+    end
+
+    # Add a string/numeric attribute to the attributes array.
+    def add_attr(attributes, key, value)
+      return if value.nil?
+
+      otel_value = case value
+                   when String
+                     { stringValue: value }
+                   when Integer
+                     { intValue: value.to_s }
+                   when Float
+                     { doubleValue: value }
+                   when TrueClass, FalseClass
+                     { boolValue: value }
+                   else
+                     { stringValue: value.to_s }
+                   end
+
+      attributes << { key: key, value: otel_value }
+    end
+
+    # Add a JSON-serialized attribute (for complex objects like input/output).
+    def add_json_attr(attributes, key, value)
+      return if value.nil?
+      return if value.is_a?(Hash) && value.empty?
+      return if value.is_a?(Array) && value.empty?
+
+      json_str = value.is_a?(String) ? value : JSON.generate(value)
+      attributes << { key: key, value: { stringValue: json_str } }
+    end
+
+    # Add an array attribute for tags.
+    def add_array_attr(attributes, key, values)
+      return if values.nil? || values.empty?
+
+      array_values = values.map { |v| { stringValue: v.to_s } }
+      attributes << { key: key, value: { arrayValue: { values: array_values } } }
+    end
+  end
+end

data/lib/langfuse/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langfuse
-  VERSION = '0.1.6'
+  VERSION = '0.1.7'
 end

data/lib/langfuse.rb

@@ -12,6 +12,7 @@ require_relative 'langfuse/evaluation'
 require_relative 'langfuse/errors'
 require_relative 'langfuse/utils'
 require_relative 'langfuse/null_objects'
+require_relative 'langfuse/otel_exporter'
 
 # Ruby SDK for Langfuse - Open source LLM engineering platform
 module Langfuse
@@ -157,7 +158,8 @@ module Langfuse
 
   # Configuration class for Langfuse client settings
   class Configuration
-    attr_accessor :public_key, :secret_key, :host, :debug, :timeout, :retries, :flush_interval, :auto_flush
+    attr_accessor :public_key, :secret_key, :host, :debug, :timeout, :retries, :flush_interval, :auto_flush,
+                  :ingestion_mode
 
     def initialize
       @public_key = nil
@@ -168,6 +170,7 @@ module Langfuse
       @retries = 3
       @flush_interval = 5
       @auto_flush = true
+      @ingestion_mode = :legacy # :legacy or :otel
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: langfuse-ruby
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Richard Sun
@@ -229,6 +229,7 @@ files:
 - lib/langfuse/generation.rb
 - lib/langfuse/null_objects.rb
 - lib/langfuse/observation_types.rb
+- lib/langfuse/otel_exporter.rb
 - lib/langfuse/prompt.rb
 - lib/langfuse/span.rb
 - lib/langfuse/trace.rb