langfuse-rb 0.10.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +38 -1
- data/lib/langfuse/api_client.rb +75 -16
- data/lib/langfuse/app_root_tracking.rb +164 -0
- data/lib/langfuse/chat_prompt_client.rb +18 -1
- data/lib/langfuse/client.rb +114 -32
- data/lib/langfuse/config.rb +306 -63
- data/lib/langfuse/evaluation.rb +8 -4
- data/lib/langfuse/exit_hook.rb +77 -0
- data/lib/langfuse/fork_safety.rb +71 -0
- data/lib/langfuse/masking_exporter.rb +98 -0
- data/lib/langfuse/observations.rb +2 -1
- data/lib/langfuse/otel_attributes.rb +1 -0
- data/lib/langfuse/otel_setup.rb +32 -44
- data/lib/langfuse/otel_span_batch.rb +113 -0
- data/lib/langfuse/otel_span_masking.rb +89 -0
- data/lib/langfuse/otel_span_patch_applier.rb +97 -0
- data/lib/langfuse/pending_score_queue.rb +62 -0
- data/lib/langfuse/prompt_cache.rb +8 -0
- data/lib/langfuse/prompt_variables.rb +54 -0
- data/lib/langfuse/propagation.rb +101 -34
- data/lib/langfuse/rails_cache_adapter.rb +17 -1
- data/lib/langfuse/read_api.rb +242 -0
- data/lib/langfuse/resilient_metrics_reporter.rb +60 -0
- data/lib/langfuse/score_client.rb +201 -80
- data/lib/langfuse/score_value.rb +58 -0
- data/lib/langfuse/span_processor.rb +53 -4
- data/lib/langfuse/stale_while_revalidate.rb +3 -4
- data/lib/langfuse/text_prompt_client.rb +13 -1
- data/lib/langfuse/trace_export_guard.rb +47 -0
- data/lib/langfuse/traced_execution.rb +18 -12
- data/lib/langfuse/types.rb +15 -1
- data/lib/langfuse/version.rb +1 -1
- data/lib/langfuse.rb +156 -44
- metadata +19 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: cdad4b4c1edea98dac2e1929430a30fd8cf7b8831a6603b41192055f20f76516
|
|
4
|
+
data.tar.gz: 526062c59b48af70225ee68a44ef70ef127cb2cf83ba33f5f5e314597e680d9a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4bf11c145c2c13a2ed6af21a8c0936781034c66516084896d22dc4510537da26234a4bd8f3ff070472e42f13aba88d94efabbfa753bb9dd837e0cf978a1d8fb5
|
|
7
|
+
data.tar.gz: 2ef511ee676296bdfb78523a04b345e3ea6705e3c82bed4102bfd49722775929a651c5059572b76f9b49ee3308962d8b562abbab2b99b4dfc839cd6af414e075
|
data/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,42 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.11.0] - 2026-08-19
|
|
11
|
+
|
|
12
|
+
> [!NOTE]
|
|
13
|
+
> **Contributor acknowledgment:** Thank you to [Alan Marx](https://github.com/Alan-Marx) for contributions in #98, #99, #100, and #102.
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
- Read current observations, metrics, and typed scores through the client API (#95).
|
|
17
|
+
- Create and read text and correction scores (#97, #102).
|
|
18
|
+
- Create scores synchronously and receive the created score ID (#102).
|
|
19
|
+
- Check global client readiness locally with `Langfuse.configured?` (#98).
|
|
20
|
+
- Send OpenTelemetry batch processor metrics to an application reporter (#99).
|
|
21
|
+
- Inject an OpenTelemetry span exporter into the Langfuse tracing pipeline (#100).
|
|
22
|
+
- Inspect referenced Mustache variables in source order (#111).
|
|
23
|
+
- Disable Langfuse tracing and scoring with one configuration switch (#110).
|
|
24
|
+
|
|
25
|
+
### Changed
|
|
26
|
+
- Export traces through direct Langfuse v4 OTLP ingestion (#103).
|
|
27
|
+
- Read timeout, batch, flush interval, and debug defaults from environment variables (#106).
|
|
28
|
+
- Bound the asynchronous score queue and split large batch payloads (#109).
|
|
29
|
+
- Reject invalid batch settings during client construction (#109).
|
|
30
|
+
- Require non-empty API keys and an absolute HTTP or HTTPS base URL (#98).
|
|
31
|
+
- Use a null logger when `Config#logger` is `nil` (#98).
|
|
32
|
+
- Use a no-op tracer for implicit observations when tracing configuration is invalid (#98).
|
|
33
|
+
|
|
34
|
+
### Fixed
|
|
35
|
+
- Mask third-party OpenTelemetry spans without changing spans for other exporters (#96).
|
|
36
|
+
- Reset SDK queues and workers after Ruby `fork` (#107).
|
|
37
|
+
- Flush pending spans and scores during normal process exit (#108).
|
|
38
|
+
- Apply the configured environment to scores without an explicit override (#105).
|
|
39
|
+
- Report invalid numeric, cache, and tracing settings as `ConfigurationError` (#98).
|
|
40
|
+
- Validate tracing batch and sampling settings before processor creation (#98).
|
|
41
|
+
- Reject explicit Rails cache configuration when `Rails.cache` is unavailable (#98).
|
|
42
|
+
|
|
43
|
+
### Documentation
|
|
44
|
+
- Organize the SDK guides by task and add data-access verification guidance (#113).
|
|
45
|
+
|
|
10
46
|
## [0.10.1] - 2026-05-05
|
|
11
47
|
|
|
12
48
|
### Changed
|
|
@@ -129,7 +165,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
129
165
|
- Migrated from legacy ingestion API to OTLP endpoint
|
|
130
166
|
- Removed `tracing_enabled` configuration flag (#2)
|
|
131
167
|
|
|
132
|
-
[Unreleased]: https://github.com/simplepractice/langfuse-rb/compare/v0.
|
|
168
|
+
[Unreleased]: https://github.com/simplepractice/langfuse-rb/compare/v0.11.0...HEAD
|
|
169
|
+
[0.11.0]: https://github.com/simplepractice/langfuse-rb/compare/v0.10.1...v0.11.0
|
|
133
170
|
[0.10.1]: https://github.com/simplepractice/langfuse-rb/compare/v0.10.0...v0.10.1
|
|
134
171
|
[0.10.0]: https://github.com/simplepractice/langfuse-rb/compare/v0.9.0...v0.10.0
|
|
135
172
|
[0.9.0]: https://github.com/simplepractice/langfuse-rb/compare/v0.8.0...v0.9.0
|
data/lib/langfuse/api_client.rb
CHANGED
|
@@ -7,6 +7,7 @@ require "json"
|
|
|
7
7
|
require "uri"
|
|
8
8
|
require_relative "prompt_fetch_result"
|
|
9
9
|
require_relative "prompt_cache_coordinator"
|
|
10
|
+
require_relative "read_api"
|
|
10
11
|
|
|
11
12
|
module Langfuse
|
|
12
13
|
# HTTP client for Langfuse API
|
|
@@ -25,6 +26,7 @@ module Langfuse
|
|
|
25
26
|
#
|
|
26
27
|
class ApiClient # rubocop:disable Metrics/ClassLength
|
|
27
28
|
include PromptCacheEvents
|
|
29
|
+
include ReadApi
|
|
28
30
|
|
|
29
31
|
# @return [String] Langfuse public API key
|
|
30
32
|
attr_reader :public_key
|
|
@@ -301,7 +303,21 @@ module Langfuse
|
|
|
301
303
|
handle_batch_response(e.response)
|
|
302
304
|
rescue Faraday::Error => e
|
|
303
305
|
logger.error("Langfuse batch send failed: #{e.message}")
|
|
304
|
-
raise
|
|
306
|
+
raise BatchDeliveryError.new("Batch send failed: #{e.message}", retryable: true)
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
# Create a score through the synchronous Scores API.
|
|
310
|
+
#
|
|
311
|
+
# @param payload [Hash] Validated score attributes in API format
|
|
312
|
+
# @return [String] ID of the created score
|
|
313
|
+
# @raise [UnauthorizedError] if authentication fails
|
|
314
|
+
# @raise [ApiError] if the API request fails or omits the created score ID
|
|
315
|
+
def create_score(payload:)
|
|
316
|
+
response = request(:post, "/api/public/scores", body: payload)
|
|
317
|
+
score_id = response["id"]
|
|
318
|
+
return score_id if score_id.is_a?(String) && !score_id.empty?
|
|
319
|
+
|
|
320
|
+
raise ApiError, "Score creation response did not include an id"
|
|
305
321
|
end
|
|
306
322
|
|
|
307
323
|
# Create a dataset run item (link a trace to a dataset item within a run)
|
|
@@ -625,10 +641,14 @@ module Langfuse
|
|
|
625
641
|
# @param path [String] Request path
|
|
626
642
|
# @param params [Hash, nil] Query string params (GET/DELETE)
|
|
627
643
|
# @param body [Hash, nil] JSON body (POST/PATCH)
|
|
644
|
+
# @param params_encoder [Object, nil] Faraday query encoder for this request
|
|
628
645
|
# @return [Hash] Parsed response body
|
|
629
|
-
def request(verb, path, params: nil, body: nil)
|
|
646
|
+
def request(verb, path, params: nil, body: nil, params_encoder: nil)
|
|
630
647
|
with_faraday_error_handling do
|
|
631
|
-
|
|
648
|
+
response = connection.public_send(verb, path, body || params) do |faraday_request|
|
|
649
|
+
faraday_request.options.params_encoder = params_encoder if params_encoder
|
|
650
|
+
end
|
|
651
|
+
handle_response(response)
|
|
632
652
|
end
|
|
633
653
|
end
|
|
634
654
|
|
|
@@ -689,7 +709,7 @@ module Langfuse
|
|
|
689
709
|
# - Max 2 retries (3 total attempts)
|
|
690
710
|
# - Exponential backoff (0.05s * 2^retry_count)
|
|
691
711
|
# - Retries GET, PATCH, and DELETE requests (idempotent operations)
|
|
692
|
-
# - Retries
|
|
712
|
+
# - Retries ingestion batches and score creation (both include stable IDs)
|
|
693
713
|
# - Note: POST to create_prompt is NOT idempotent; retries may create duplicate versions
|
|
694
714
|
# - Retries on: 429 (rate limit), 503 (service unavailable), 504 (gateway timeout)
|
|
695
715
|
# - Does NOT retry on: 4xx errors (except 429), 5xx errors (except 503, 504)
|
|
@@ -780,38 +800,77 @@ module Langfuse
|
|
|
780
800
|
|
|
781
801
|
# Handle HTTP response for batch requests
|
|
782
802
|
#
|
|
803
|
+
# Per-event input errors can arrive with HTTP 207 instead of a 4xx response.
|
|
804
|
+
# The `errors` array reports rejected events, so HTTP status alone cannot
|
|
805
|
+
# identify a batch with rejected events. An empty array confirms only that
|
|
806
|
+
# the response reported no rejection; it does not prove downstream
|
|
807
|
+
# processing or immediate read visibility.
|
|
808
|
+
#
|
|
783
809
|
# @param response [Faraday::Response] The HTTP response
|
|
784
810
|
# @return [void]
|
|
785
811
|
# @raise [UnauthorizedError] if status is 401
|
|
786
|
-
# @raise [ApiError] for other error statuses
|
|
812
|
+
# @raise [ApiError] for other error statuses, or if the body lists rejected events
|
|
787
813
|
def handle_batch_response(response)
|
|
788
814
|
case response.status
|
|
789
815
|
when 200, 201, 204, 207
|
|
790
|
-
|
|
816
|
+
raise_on_batch_errors(response)
|
|
791
817
|
when 401
|
|
792
818
|
raise UnauthorizedError, "Authentication failed. Check your API keys."
|
|
793
819
|
else
|
|
794
820
|
error_message = extract_error_message(response)
|
|
795
|
-
raise
|
|
821
|
+
raise BatchDeliveryError.new(
|
|
822
|
+
"Batch send failed (#{response.status}): #{error_message}",
|
|
823
|
+
retryable: retryable_batch_status?(response.status)
|
|
824
|
+
)
|
|
796
825
|
end
|
|
797
826
|
end
|
|
798
827
|
|
|
828
|
+
# @param response [Faraday::Response] The HTTP response
|
|
829
|
+
# @return [void]
|
|
830
|
+
# @raise [ApiError] if the response body's `errors` array is non-empty
|
|
831
|
+
def raise_on_batch_errors(response)
|
|
832
|
+
errors = Array(parse_response_body(response)["errors"])
|
|
833
|
+
return if errors.empty?
|
|
834
|
+
|
|
835
|
+
messages = errors.filter_map { |e| e["message"] || e["error"] }
|
|
836
|
+
summary = messages.empty? ? "#{errors.size} event(s) rejected" : messages.join("; ")
|
|
837
|
+
raise BatchDeliveryError.new(
|
|
838
|
+
"Batch send failed: #{summary}",
|
|
839
|
+
retryable: retryable_batch_errors?(errors)
|
|
840
|
+
)
|
|
841
|
+
end
|
|
842
|
+
|
|
843
|
+
def retryable_batch_errors?(errors)
|
|
844
|
+
statuses = errors.filter_map { |error| Integer(error["status"], exception: false) }
|
|
845
|
+
statuses.length == errors.length && statuses.all? { |status| retryable_batch_status?(status) }
|
|
846
|
+
end
|
|
847
|
+
|
|
848
|
+
def retryable_batch_status?(status)
|
|
849
|
+
status == 429 || status >= 500
|
|
850
|
+
end
|
|
851
|
+
|
|
799
852
|
# Extract error message from response body
|
|
800
853
|
#
|
|
801
854
|
# @param response [Faraday::Response] The HTTP response
|
|
802
855
|
# @return [String] The error message
|
|
803
856
|
def extract_error_message(response)
|
|
804
|
-
body_hash =
|
|
805
|
-
in Hash => h then h
|
|
806
|
-
in String => s then begin
|
|
807
|
-
JSON.parse(s)
|
|
808
|
-
rescue StandardError
|
|
809
|
-
{}
|
|
810
|
-
end
|
|
811
|
-
else {}
|
|
812
|
-
end
|
|
857
|
+
body_hash = parse_response_body(response)
|
|
813
858
|
|
|
814
859
|
%w[message error].filter_map { |key| body_hash[key] }.first || "Unknown error"
|
|
815
860
|
end
|
|
861
|
+
|
|
862
|
+
# @param response [Faraday::Response] The HTTP response
|
|
863
|
+
# @return [Hash] The parsed JSON body, or {} if absent/unparsable
|
|
864
|
+
def parse_response_body(response)
|
|
865
|
+
case response.body
|
|
866
|
+
in Hash => h then h
|
|
867
|
+
in String => s then begin
|
|
868
|
+
JSON.parse(s)
|
|
869
|
+
rescue StandardError
|
|
870
|
+
{}
|
|
871
|
+
end
|
|
872
|
+
else {}
|
|
873
|
+
end
|
|
874
|
+
end
|
|
816
875
|
end
|
|
817
876
|
end
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Langfuse
|
|
4
|
+
# Owns application-root state for active span trees.
|
|
5
|
+
#
|
|
6
|
+
# @api private
|
|
7
|
+
module AppRootTracking
|
|
8
|
+
APP_ROOT_INELIGIBLE_SPANS = [
|
|
9
|
+
%w[litellm raw_gen_ai_request]
|
|
10
|
+
].freeze
|
|
11
|
+
private_constant :APP_ROOT_INELIGIBLE_SPANS
|
|
12
|
+
|
|
13
|
+
# Return whether an exported span can be an application root when its parent is untracked.
|
|
14
|
+
#
|
|
15
|
+
# LiteLLM can emit `raw_gen_ai_request` after its exported parent finishes.
|
|
16
|
+
# The late child must not become a second application root.
|
|
17
|
+
#
|
|
18
|
+
# @param span [OpenTelemetry::SDK::Trace::Span] Span to inspect
|
|
19
|
+
# @return [Boolean] Whether the span can be an application root
|
|
20
|
+
# @api private
|
|
21
|
+
def self.eligible_without_tracked_parent?(span)
|
|
22
|
+
return true if span.parent_span_id == OpenTelemetry::Trace::INVALID_SPAN_ID
|
|
23
|
+
|
|
24
|
+
identity = [span.instrumentation_scope&.name, span.name]
|
|
25
|
+
!APP_ROOT_INELIGIBLE_SPANS.include?(identity)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Defers a finished span until its active ancestors have final export decisions.
|
|
29
|
+
#
|
|
30
|
+
# @api private
|
|
31
|
+
class Tracker
|
|
32
|
+
ReadySpan = Struct.new(:span, :app_root, keyword_init: true)
|
|
33
|
+
private_constant :ReadySpan
|
|
34
|
+
|
|
35
|
+
State = Struct.new(
|
|
36
|
+
:span,
|
|
37
|
+
:trace_claimed,
|
|
38
|
+
:untracked_parent_root_eligible,
|
|
39
|
+
:parent_span_id,
|
|
40
|
+
:active_child_count,
|
|
41
|
+
:finished,
|
|
42
|
+
:exportable,
|
|
43
|
+
:enqueued,
|
|
44
|
+
keyword_init: true
|
|
45
|
+
)
|
|
46
|
+
private_constant :State
|
|
47
|
+
|
|
48
|
+
def initialize
|
|
49
|
+
@mutex = Mutex.new
|
|
50
|
+
@state_by_span_id = {}
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# @param span [OpenTelemetry::SDK::Trace::Span] The active span
|
|
54
|
+
# @param trace_claimed [Boolean] Whether propagated context already owns the root
|
|
55
|
+
# @param untracked_parent_root_eligible [Boolean] Whether an untracked parent permits a root
|
|
56
|
+
# @return [void]
|
|
57
|
+
def remember(span, trace_claimed:, untracked_parent_root_eligible:)
|
|
58
|
+
@mutex.synchronize do
|
|
59
|
+
parent_state = @state_by_span_id[span.parent_span_id]
|
|
60
|
+
parent_state.active_child_count += 1 if parent_state
|
|
61
|
+
@state_by_span_id[span.context.span_id] = build_state(
|
|
62
|
+
span,
|
|
63
|
+
trace_claimed: trace_claimed,
|
|
64
|
+
untracked_parent_root_eligible: untracked_parent_root_eligible
|
|
65
|
+
)
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Resolve finished spans whose ancestor export decisions are final.
|
|
70
|
+
#
|
|
71
|
+
# @param span [OpenTelemetry::SDK::Trace::Span] The finished span
|
|
72
|
+
# @param exportable [Boolean] Whether the final export filter accepted the span
|
|
73
|
+
# @return [Array<ReadySpan>] Spans that the batch processor can enqueue
|
|
74
|
+
def finish(span, exportable:)
|
|
75
|
+
@mutex.synchronize do
|
|
76
|
+
state = @state_by_span_id[span.context.span_id]
|
|
77
|
+
return [] unless state
|
|
78
|
+
|
|
79
|
+
state.finished = true
|
|
80
|
+
state.exportable = exportable
|
|
81
|
+
ready_spans = resolve_ready_spans
|
|
82
|
+
release_finished_states
|
|
83
|
+
ready_spans
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# @return [Boolean] Whether the tracker has no active span trees
|
|
88
|
+
def empty?
|
|
89
|
+
@mutex.synchronize { @state_by_span_id.empty? }
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
private
|
|
93
|
+
|
|
94
|
+
def build_state(span, trace_claimed:, untracked_parent_root_eligible:)
|
|
95
|
+
State.new(
|
|
96
|
+
span: span,
|
|
97
|
+
trace_claimed: trace_claimed,
|
|
98
|
+
untracked_parent_root_eligible: untracked_parent_root_eligible,
|
|
99
|
+
parent_span_id: span.parent_span_id,
|
|
100
|
+
active_child_count: 0,
|
|
101
|
+
finished: false,
|
|
102
|
+
exportable: nil,
|
|
103
|
+
enqueued: false
|
|
104
|
+
)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def resolve_ready_spans
|
|
108
|
+
@state_by_span_id.values.filter_map do |state|
|
|
109
|
+
next unless state.finished && state.exportable && !state.enqueued
|
|
110
|
+
|
|
111
|
+
app_root = app_root_status(state)
|
|
112
|
+
next if app_root.nil?
|
|
113
|
+
|
|
114
|
+
state.enqueued = true
|
|
115
|
+
ReadySpan.new(span: state.span, app_root: app_root)
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def app_root_status(state)
|
|
120
|
+
trace_claimed = state.trace_claimed
|
|
121
|
+
parent_span_id = state.parent_span_id
|
|
122
|
+
parent_state = @state_by_span_id[parent_span_id]
|
|
123
|
+
return false unless parent_state || state.untracked_parent_root_eligible
|
|
124
|
+
|
|
125
|
+
while parent_state
|
|
126
|
+
return nil unless parent_state.finished
|
|
127
|
+
return false if parent_state.exportable
|
|
128
|
+
|
|
129
|
+
trace_claimed = parent_state.trace_claimed
|
|
130
|
+
parent_span_id = parent_state.parent_span_id
|
|
131
|
+
parent_state = @state_by_span_id[parent_span_id]
|
|
132
|
+
end
|
|
133
|
+
!trace_claimed
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def release_finished_states
|
|
137
|
+
releasable_span_ids = @state_by_span_id.filter_map do |span_id, state|
|
|
138
|
+
span_id if releasable?(state)
|
|
139
|
+
end
|
|
140
|
+
until releasable_span_ids.empty?
|
|
141
|
+
span_id = releasable_span_ids.pop
|
|
142
|
+
state = @state_by_span_id[span_id]
|
|
143
|
+
next unless releasable?(state)
|
|
144
|
+
|
|
145
|
+
parent_span_id = release_state(span_id, state)
|
|
146
|
+
releasable_span_ids << parent_span_id if parent_span_id
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def releasable?(state)
|
|
151
|
+
state&.finished && state.active_child_count.zero? && (!state.exportable || state.enqueued)
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def release_state(span_id, state)
|
|
155
|
+
@state_by_span_id.delete(span_id)
|
|
156
|
+
parent_state = @state_by_span_id[state.parent_span_id]
|
|
157
|
+
return unless parent_state
|
|
158
|
+
|
|
159
|
+
parent_state.active_child_count -= 1
|
|
160
|
+
state.parent_span_id
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
end
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "prompt_renderer"
|
|
4
|
+
require_relative "prompt_variables"
|
|
4
5
|
|
|
5
6
|
module Langfuse
|
|
6
7
|
# Chat prompt client for compiling chat prompts with variable substitution
|
|
@@ -11,7 +12,7 @@ module Langfuse
|
|
|
11
12
|
# @example Basic usage
|
|
12
13
|
# prompt_data = api_client.get_prompt("support_chat")
|
|
13
14
|
# chat_prompt = Langfuse::ChatPromptClient.new(prompt_data)
|
|
14
|
-
# chat_prompt.compile(
|
|
15
|
+
# chat_prompt.compile(user_name: "Alice", issue: "login")
|
|
15
16
|
# # => [{ role: "system", content: "You are a support agent..." }, ...]
|
|
16
17
|
#
|
|
17
18
|
# @example Accessing metadata
|
|
@@ -73,6 +74,22 @@ module Langfuse
|
|
|
73
74
|
"chat"
|
|
74
75
|
end
|
|
75
76
|
|
|
77
|
+
# Return the unique variables referenced by all message templates
|
|
78
|
+
#
|
|
79
|
+
# Section names are included because callers must provide their values.
|
|
80
|
+
# Message placeholder entries are not Mustache templates and are excluded.
|
|
81
|
+
#
|
|
82
|
+
# @return [Array<String>] Referenced variable names in message and source order
|
|
83
|
+
# @raise [Mustache::Parser::SyntaxError] if a message contains invalid Mustache syntax
|
|
84
|
+
def variables
|
|
85
|
+
prompt.each_with_object([]) do |message, names|
|
|
86
|
+
normalized = symbolize_keys(message)
|
|
87
|
+
next if normalized[:type].to_s == PLACEHOLDER_TYPE
|
|
88
|
+
|
|
89
|
+
names.concat(PromptVariables.extract(normalized[:content] || ""))
|
|
90
|
+
end.uniq
|
|
91
|
+
end
|
|
92
|
+
|
|
76
93
|
# Compile the chat prompt with variable substitution and message placeholders
|
|
77
94
|
#
|
|
78
95
|
# Returns an array of message hashes with roles and compiled content.
|
data/lib/langfuse/client.rb
CHANGED
|
@@ -29,8 +29,13 @@ module Langfuse
|
|
|
29
29
|
# @return [Config] The client configuration
|
|
30
30
|
attr_reader :config
|
|
31
31
|
|
|
32
|
+
# Return the underlying API client, building it on first non-telemetry use.
|
|
33
|
+
#
|
|
32
34
|
# @return [ApiClient] The underlying API client
|
|
33
|
-
|
|
35
|
+
# @raise [ConfigurationError] if the full client configuration is invalid
|
|
36
|
+
def api_client
|
|
37
|
+
@api_client || @api_client_mutex.synchronize { @api_client ||= build_validated_api_client }
|
|
38
|
+
end
|
|
34
39
|
|
|
35
40
|
# Pure pass-throughs to {ApiClient}. See {ApiClient} for parameter and
|
|
36
41
|
# return-value documentation; the public surface here is identical.
|
|
@@ -44,6 +49,9 @@ module Langfuse
|
|
|
44
49
|
# @!method validate_prompt_cache_backend!
|
|
45
50
|
# @!method list_traces(**options)
|
|
46
51
|
# @!method get_trace(id)
|
|
52
|
+
# @!method list_observations(**options)
|
|
53
|
+
# @!method query_metrics(query:)
|
|
54
|
+
# @!method list_scores(**options)
|
|
47
55
|
# @!method list_datasets(page: nil, limit: nil)
|
|
48
56
|
# @!method get_dataset_run(dataset_name:, run_name:)
|
|
49
57
|
# @!method create_dataset_run_item(**)
|
|
@@ -57,6 +65,9 @@ module Langfuse
|
|
|
57
65
|
:validate_prompt_cache_backend!,
|
|
58
66
|
:list_traces,
|
|
59
67
|
:get_trace,
|
|
68
|
+
:list_observations,
|
|
69
|
+
:query_metrics,
|
|
70
|
+
:list_scores,
|
|
60
71
|
:list_datasets,
|
|
61
72
|
:get_dataset_run,
|
|
62
73
|
:create_dataset_run_item
|
|
@@ -67,29 +78,19 @@ module Langfuse
|
|
|
67
78
|
# @return [Client]
|
|
68
79
|
def initialize(config)
|
|
69
80
|
@config = config
|
|
70
|
-
@
|
|
71
|
-
|
|
72
|
-
# Create cache if enabled
|
|
73
|
-
cache = create_cache if cache_enabled?
|
|
74
|
-
|
|
75
|
-
# Create API client with cache
|
|
76
|
-
@api_client = ApiClient.new(
|
|
77
|
-
public_key: config.public_key,
|
|
78
|
-
secret_key: config.secret_key,
|
|
79
|
-
base_url: config.base_url,
|
|
80
|
-
timeout: config.timeout,
|
|
81
|
-
logger: config.logger,
|
|
82
|
-
cache: cache,
|
|
83
|
-
cache_observer: config.prompt_cache_observer
|
|
84
|
-
)
|
|
85
|
-
|
|
81
|
+
@api_client_mutex = Mutex.new
|
|
82
|
+
@score_client_mutex = Mutex.new
|
|
86
83
|
@project_id = nil
|
|
87
84
|
# One-shot lookup: avoids repeated blocking API calls in URL helpers
|
|
88
85
|
# (trace_url, dataset_url, dataset_run_url) when the project endpoint is down.
|
|
89
86
|
@project_id_fetched = false
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
87
|
+
if telemetry_enabled?
|
|
88
|
+
config.validate!
|
|
89
|
+
@api_client = build_api_client
|
|
90
|
+
@score_client = build_score_client
|
|
91
|
+
else
|
|
92
|
+
config.validate_telemetry_disabled!
|
|
93
|
+
end
|
|
93
94
|
end
|
|
94
95
|
|
|
95
96
|
# Fetch a prompt and return the appropriate client
|
|
@@ -347,14 +348,14 @@ module Langfuse
|
|
|
347
348
|
#
|
|
348
349
|
# @param name [String] Score name (required)
|
|
349
350
|
# @param value [Numeric, Integer, String] Score value (type depends on data_type)
|
|
350
|
-
# @param id [String, nil] Score ID
|
|
351
|
+
# @param id [String, nil] Score ID; use a stable value as an idempotency key
|
|
351
352
|
# @param trace_id [String, nil] Trace ID to associate with the score
|
|
352
353
|
# @param session_id [String, nil] Session ID to associate with the score
|
|
353
354
|
# @param observation_id [String, nil] Observation ID to associate with the score
|
|
354
355
|
# @param comment [String, nil] Optional comment
|
|
355
356
|
# @param metadata [Hash, nil] Optional metadata hash
|
|
356
|
-
# @param environment [String, nil] Optional environment
|
|
357
|
-
# @param data_type [Symbol] Data type (:numeric, :boolean, :categorical)
|
|
357
|
+
# @param environment [String, nil] Optional per-score environment override
|
|
358
|
+
# @param data_type [Symbol] Data type (:numeric, :boolean, :categorical, :text, :correction)
|
|
358
359
|
# @param dataset_run_id [String, nil] Optional dataset run ID to associate with the score
|
|
359
360
|
# @param config_id [String, nil] Optional score config ID
|
|
360
361
|
# @return [void]
|
|
@@ -368,10 +369,59 @@ module Langfuse
|
|
|
368
369
|
#
|
|
369
370
|
# @example Categorical score
|
|
370
371
|
# client.create_score(name: "category", value: "high", trace_id: "abc123", data_type: :categorical)
|
|
372
|
+
#
|
|
373
|
+
# @example Text score (1 to 500 characters)
|
|
374
|
+
# client.create_score(name: "reviewer_notes", value: "Helpful but verbose",
|
|
375
|
+
# trace_id: "abc123", data_type: :text)
|
|
376
|
+
#
|
|
377
|
+
# @example Corrected output (conventionally named "output")
|
|
378
|
+
# client.create_score(name: "output", value: "The corrected output", trace_id: "abc123",
|
|
379
|
+
# observation_id: "def456", data_type: :correction)
|
|
371
380
|
# rubocop:disable Metrics/ParameterLists
|
|
372
381
|
def create_score(name:, value:, id: nil, trace_id: nil, session_id: nil, observation_id: nil, comment: nil,
|
|
373
382
|
metadata: nil, environment: nil, data_type: :numeric, dataset_run_id: nil, config_id: nil)
|
|
374
|
-
|
|
383
|
+
active_score_client&.create(
|
|
384
|
+
name: name,
|
|
385
|
+
value: value,
|
|
386
|
+
id: id,
|
|
387
|
+
trace_id: trace_id,
|
|
388
|
+
session_id: session_id,
|
|
389
|
+
observation_id: observation_id,
|
|
390
|
+
comment: comment,
|
|
391
|
+
metadata: metadata,
|
|
392
|
+
environment: environment,
|
|
393
|
+
data_type: data_type,
|
|
394
|
+
dataset_run_id: dataset_run_id,
|
|
395
|
+
config_id: config_id
|
|
396
|
+
)
|
|
397
|
+
end
|
|
398
|
+
# rubocop:enable Metrics/ParameterLists
|
|
399
|
+
|
|
400
|
+
# Create a score immediately through the Scores API. See {ScoreClient#create!}.
|
|
401
|
+
#
|
|
402
|
+
# @param name [String] Score name (required)
|
|
403
|
+
# @param value [Numeric, Integer, String] Score value (type depends on data_type)
|
|
404
|
+
# @param id [String, nil] Score ID; use a stable value as an idempotency key
|
|
405
|
+
# @param trace_id [String, nil] Trace ID to associate with the score
|
|
406
|
+
# @param session_id [String, nil] Session ID to associate with the score
|
|
407
|
+
# @param observation_id [String, nil] Observation ID to associate with the score
|
|
408
|
+
# @param comment [String, nil] Optional comment
|
|
409
|
+
# @param metadata [Hash, nil] Optional metadata hash
|
|
410
|
+
# @param environment [String, nil] Optional per-score environment override
|
|
411
|
+
# @param data_type [Symbol] Data type (:numeric, :boolean, :categorical, :text, :correction)
|
|
412
|
+
# @param dataset_run_id [String, nil] Optional dataset run ID to associate with the score
|
|
413
|
+
# @param config_id [String, nil] Optional score config ID
|
|
414
|
+
# @return [String, nil] ID of the created score, or nil when telemetry is disabled
|
|
415
|
+
# @raise [ArgumentError] if validation fails
|
|
416
|
+
# @raise [UnauthorizedError] if authentication fails
|
|
417
|
+
# @raise [ApiError] if the API request fails
|
|
418
|
+
#
|
|
419
|
+
# @example Create a score with an idempotency key
|
|
420
|
+
# client.create_score!(id: "feedback-abc123", name: "quality", value: 0.85, trace_id: "abc123")
|
|
421
|
+
# rubocop:disable Metrics/ParameterLists
|
|
422
|
+
def create_score!(name:, value:, id: nil, trace_id: nil, session_id: nil, observation_id: nil, comment: nil,
|
|
423
|
+
metadata: nil, environment: nil, data_type: :numeric, dataset_run_id: nil, config_id: nil)
|
|
424
|
+
active_score_client&.create!(
|
|
375
425
|
name: name,
|
|
376
426
|
value: value,
|
|
377
427
|
id: id,
|
|
@@ -396,7 +446,7 @@ module Langfuse
|
|
|
396
446
|
# @param value [Numeric, Integer, String] Score value
|
|
397
447
|
# @param comment [String, nil] Optional comment
|
|
398
448
|
# @param metadata [Hash, nil] Optional metadata hash
|
|
399
|
-
# @param data_type [Symbol] Data type (:numeric, :boolean, :categorical)
|
|
449
|
+
# @param data_type [Symbol] Data type (:numeric, :boolean, :categorical, :text, :correction)
|
|
400
450
|
# @return [void]
|
|
401
451
|
# @raise [ArgumentError] if no active span or validation fails
|
|
402
452
|
#
|
|
@@ -405,7 +455,7 @@ module Langfuse
|
|
|
405
455
|
# client.score_active_observation(name: "accuracy", value: 0.92)
|
|
406
456
|
# end
|
|
407
457
|
def score_active_observation(name:, value:, comment: nil, metadata: nil, data_type: :numeric)
|
|
408
|
-
|
|
458
|
+
active_score_client&.score_active_observation(
|
|
409
459
|
name: name,
|
|
410
460
|
value: value,
|
|
411
461
|
comment: comment,
|
|
@@ -422,7 +472,7 @@ module Langfuse
|
|
|
422
472
|
# @param value [Numeric, Integer, String] Score value
|
|
423
473
|
# @param comment [String, nil] Optional comment
|
|
424
474
|
# @param metadata [Hash, nil] Optional metadata hash
|
|
425
|
-
# @param data_type [Symbol] Data type (:numeric, :boolean, :categorical)
|
|
475
|
+
# @param data_type [Symbol] Data type (:numeric, :boolean, :categorical, :text, :correction)
|
|
426
476
|
# @return [void]
|
|
427
477
|
# @raise [ArgumentError] if no active span or validation fails
|
|
428
478
|
#
|
|
@@ -431,7 +481,7 @@ module Langfuse
|
|
|
431
481
|
# client.score_active_trace(name: "overall_quality", value: 5)
|
|
432
482
|
# end
|
|
433
483
|
def score_active_trace(name:, value:, comment: nil, metadata: nil, data_type: :numeric)
|
|
434
|
-
|
|
484
|
+
active_score_client&.score_active_trace(
|
|
435
485
|
name: name,
|
|
436
486
|
value: value,
|
|
437
487
|
comment: comment,
|
|
@@ -449,7 +499,7 @@ module Langfuse
|
|
|
449
499
|
# @example
|
|
450
500
|
# client.flush_scores
|
|
451
501
|
def flush_scores
|
|
452
|
-
|
|
502
|
+
active_score_client&.flush
|
|
453
503
|
end
|
|
454
504
|
|
|
455
505
|
# Shutdown the client and flush any pending scores
|
|
@@ -458,8 +508,8 @@ module Langfuse
|
|
|
458
508
|
#
|
|
459
509
|
# @return [void]
|
|
460
510
|
def shutdown
|
|
461
|
-
@score_client
|
|
462
|
-
@api_client
|
|
511
|
+
@score_client&.shutdown
|
|
512
|
+
@api_client_mutex.synchronize { @api_client }&.shutdown
|
|
463
513
|
end
|
|
464
514
|
|
|
465
515
|
# Create a new dataset
|
|
@@ -660,6 +710,38 @@ module Langfuse
|
|
|
660
710
|
|
|
661
711
|
private
|
|
662
712
|
|
|
713
|
+
def build_validated_api_client
|
|
714
|
+
config.validate!
|
|
715
|
+
build_api_client
|
|
716
|
+
end
|
|
717
|
+
|
|
718
|
+
def build_api_client
|
|
719
|
+
cache = create_cache if cache_enabled?
|
|
720
|
+
ApiClient.new(
|
|
721
|
+
public_key: config.public_key,
|
|
722
|
+
secret_key: config.secret_key,
|
|
723
|
+
base_url: config.base_url,
|
|
724
|
+
timeout: config.timeout,
|
|
725
|
+
logger: config.logger,
|
|
726
|
+
cache: cache,
|
|
727
|
+
cache_observer: config.prompt_cache_observer
|
|
728
|
+
)
|
|
729
|
+
end
|
|
730
|
+
|
|
731
|
+
def telemetry_enabled?
|
|
732
|
+
config.telemetry_enabled?
|
|
733
|
+
end
|
|
734
|
+
|
|
735
|
+
def active_score_client
|
|
736
|
+
return unless telemetry_enabled?
|
|
737
|
+
|
|
738
|
+
@score_client_mutex.synchronize { @score_client ||= build_score_client }
|
|
739
|
+
end
|
|
740
|
+
|
|
741
|
+
def build_score_client
|
|
742
|
+
ScoreClient.new(api_client: api_client, config: config)
|
|
743
|
+
end
|
|
744
|
+
|
|
663
745
|
attr_reader :score_client
|
|
664
746
|
|
|
665
747
|
# Build a project-scoped URL, returning nil if project ID is unavailable
|
|
@@ -790,7 +872,7 @@ module Langfuse
|
|
|
790
872
|
end
|
|
791
873
|
|
|
792
874
|
def rails_cache_available?
|
|
793
|
-
|
|
875
|
+
RailsCacheAdapter.available?
|
|
794
876
|
end
|
|
795
877
|
|
|
796
878
|
# Create in-memory cache with SWR support if enabled
|