langchainrb_datadog 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.rspec +3 -0
- data/.rubocop.yml +14 -0
- data/LICENSE.txt +21 -0
- data/README.md +58 -0
- data/Rakefile +12 -0
- data/langchainrb_datadog.gemspec +39 -0
- data/lib/langchain/datadog/llm.rb +115 -0
- data/lib/langchain/datadog/tracing.rb +117 -0
- data/lib/langchain/datadog/vectorsearch.rb +125 -0
- data/lib/langchain/datadog/version.rb +7 -0
- data/lib/langchain/datadog.rb +89 -0
- data/lib/langchainrb_datadog.rb +3 -0
- metadata +89 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: abb023d18e8b8502d417d70b5cf6e004aa4d490f47bf5eb7f71af13ea6d48fc7
|
4
|
+
data.tar.gz: 8d8be0154c9707258f9a306834c2818a302225294001ab3155c2b9584e05ddd6
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 3cb22c5a223473a51557211ebdfcfb0d41b309e39469fcd01b193e5cccb702f483d5d3a7033adbdd90c86eefa7a1b769f8550a53088783a18a471d1f0ef140c3
|
7
|
+
data.tar.gz: 998cbea79f0ca64a80622816e036d8ad4af6ce6abc2552491d6b9c604ff5828b052b59b2e3ee2516a281c8c513502f7cb2f2bc4c5dbbaf2034d2086538955f6c
|
data/.rspec
ADDED
data/.rubocop.yml
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2024 Nikolaos Anastopoulos
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
# Langchain.rb Datadog
|
2
|
+
|
3
|
+
Enables LLM observability with Datadog for Langchain.rb.
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Install the gem and add to the application's Gemfile by executing:
|
8
|
+
|
9
|
+
$ bundle add langchainrb_datadog
|
10
|
+
|
11
|
+
If bundler is not being used to manage dependencies, install the gem by executing:
|
12
|
+
|
13
|
+
$ gem install langchainrb_datadog
|
14
|
+
|
15
|
+
## Usage
|
16
|
+
|
17
|
+
Its function is automatic: it hooks into Langchain.rb methods to capture LLM calls and report them to the Datadog LLM Observability API.
|
18
|
+
|
19
|
+
Configure with the following environment variables or Ruby methods:
|
20
|
+
|
21
|
+
- `DD_SITE` (optional, default: `datadoghq.com`)
|
22
|
+
```ruby
|
23
|
+
Langchain::Datadog.site = 'datadoghq.com'
|
24
|
+
```
|
25
|
+
The Datadog site to submit your LLM data.
|
26
|
+
|
27
|
+
- `DD_API_KEY` (required)
|
28
|
+
```ruby
|
29
|
+
Langchain::Datadog.api_key = '1a2b3c4d5e6f'
|
30
|
+
```
|
31
|
+
Your Datadog API key.
|
32
|
+
|
33
|
+
- `DD_LLMOBS_ENABLED` (optional, default: `1`)
|
34
|
+
```ruby
|
35
|
+
Langchain::Datadog.enabled = true
|
36
|
+
```
|
37
|
+
Toggle to disable submitting data to LLM Observability.
|
38
|
+
|
39
|
+
- `DD_LLMOBS_ML_APP` (required)
|
40
|
+
```ruby
|
41
|
+
Langchain::Datadog.ml_app = 'langchainrb_datadog'
|
42
|
+
```
|
43
|
+
The name of your LLM application, service, or project, under which all traces and spans are grouped. This helps distinguish between different applications or experiments.
|
44
|
+
See [Application naming guidelines](https://docs.datadoghq.com/llm_observability/setup/api/#application-naming-guidelines) for allowed characters and other constraints.
|
45
|
+
|
46
|
+
## Development
|
47
|
+
|
48
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
49
|
+
|
50
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
51
|
+
|
52
|
+
## Contributing
|
53
|
+
|
54
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/GaggleAMP/langchainrb_datadog.
|
55
|
+
|
56
|
+
## License
|
57
|
+
|
58
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative 'lib/langchain/datadog/version'
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = 'langchainrb_datadog'
|
7
|
+
spec.version = Langchain::Datadog::VERSION
|
8
|
+
spec.authors = ['GaggleAMP', 'Nikolaos Anastopoulos']
|
9
|
+
spec.email = ['info@gaggleamp.com', 'ebababi@ebababi.net']
|
10
|
+
|
11
|
+
spec.summary = 'Enables LLM observability with Datadog for Langchain.rb.'
|
12
|
+
spec.description = 'Hooks into Langchain.rb methods to capture LLM calls and ' \
|
13
|
+
'report them to the Datadog LLM Observability API.'
|
14
|
+
spec.homepage = 'https://github.com/GaggleAMP/langchainrb_datadog'
|
15
|
+
spec.license = 'MIT'
|
16
|
+
spec.required_ruby_version = '>= 3.1.0'
|
17
|
+
|
18
|
+
spec.metadata['homepage_uri'] = spec.homepage
|
19
|
+
spec.metadata['source_code_uri'] = 'https://github.com/GaggleAMP/langchainrb_datadog'
|
20
|
+
spec.metadata['github_repo'] = 'git@github.com:GaggleAMP/langchainrb_datadog.git'
|
21
|
+
|
22
|
+
# Specify which files should be added to the gem when it is released.
|
23
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
24
|
+
spec.files = Dir.chdir(__dir__) do
|
25
|
+
`git ls-files -z`.split("\x0").reject do |f|
|
26
|
+
(File.expand_path(f) == __FILE__) ||
|
27
|
+
f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile])
|
28
|
+
end
|
29
|
+
end
|
30
|
+
spec.bindir = 'exe'
|
31
|
+
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
32
|
+
spec.require_paths = ['lib']
|
33
|
+
|
34
|
+
spec.add_dependency 'faraday'
|
35
|
+
spec.add_dependency 'langchainrb', '~> 0.17.1'
|
36
|
+
|
37
|
+
# For more information and examples about making a new gem, check out our
|
38
|
+
# guide at: https://bundler.io/guides/creating_gem.html
|
39
|
+
end
|
@@ -0,0 +1,115 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Langchain
|
4
|
+
module Datadog
|
5
|
+
# Implements hooks for Langchain::LLM module classes to capture LLM calls
|
6
|
+
# and report them to the Datadog LLM Observability API.
|
7
|
+
module LLM
|
8
|
+
include Langchain::Datadog::Tracing
|
9
|
+
|
10
|
+
def chat(params = {}, ...)
|
11
|
+
return super unless Datadog.enabled?
|
12
|
+
|
13
|
+
parameters = chat_parameters.to_params(params)
|
14
|
+
|
15
|
+
span(parameters) { super }
|
16
|
+
end
|
17
|
+
|
18
|
+
def complete(prompt:, **)
|
19
|
+
return super unless Datadog.enabled?
|
20
|
+
|
21
|
+
parameters = { prompt: }
|
22
|
+
|
23
|
+
span(parameters) { super }
|
24
|
+
end
|
25
|
+
|
26
|
+
def embed(text:, **)
|
27
|
+
return super unless Datadog.enabled?
|
28
|
+
|
29
|
+
parameters = { text: }
|
30
|
+
|
31
|
+
span(parameters, kind: 'embedding') { super }
|
32
|
+
end
|
33
|
+
|
34
|
+
def summarize(text:, **)
|
35
|
+
return super unless Datadog.enabled?
|
36
|
+
|
37
|
+
parameters = { text: }
|
38
|
+
|
39
|
+
span(parameters, kind: 'task') { super }
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def span(parameters, kind: 'llm')
|
45
|
+
previous_parent_id = Tracing.active_parent_id
|
46
|
+
Tracing.start_span
|
47
|
+
|
48
|
+
start_ns = (Time.now.to_r * 1_000_000_000).to_i
|
49
|
+
response = yield
|
50
|
+
duration = (Time.now.to_r * 1_000_000_000).to_i - start_ns
|
51
|
+
|
52
|
+
trace([{
|
53
|
+
name: caller_locations(1, 1)[0].label,
|
54
|
+
span_id: Tracing.active_span_id.to_s,
|
55
|
+
trace_id: Tracing.active_trace_id.to_s,
|
56
|
+
parent_id: Tracing.active_parent_id&.to_s || 'undefined',
|
57
|
+
start_ns:,
|
58
|
+
duration:,
|
59
|
+
meta: {
|
60
|
+
kind:,
|
61
|
+
input: input(parameters),
|
62
|
+
output: output(response),
|
63
|
+
metadata: metadata(parameters, response)
|
64
|
+
}.compact,
|
65
|
+
metrics: metrics(response)
|
66
|
+
}.compact])
|
67
|
+
|
68
|
+
response
|
69
|
+
ensure
|
70
|
+
Tracing.end_span(parent_id: previous_parent_id)
|
71
|
+
end
|
72
|
+
|
73
|
+
def input(parameters)
|
74
|
+
case parameters
|
75
|
+
in { messages: messages } then { messages: }
|
76
|
+
in { prompt: value } then { value: }
|
77
|
+
in { text: value } then { value: }
|
78
|
+
else nil
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def output(response)
|
83
|
+
completions = begin; response.completions; rescue NotImplementedError; nil; end
|
84
|
+
embeddings = begin; response.embeddings; rescue NotImplementedError; nil; end
|
85
|
+
|
86
|
+
if completions
|
87
|
+
{ messages: [completions.dig(0, 'message')&.slice('content', 'role')] }
|
88
|
+
elsif embeddings
|
89
|
+
{ value: embeddings.first&.to_s }
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def metadata(parameters, response)
|
94
|
+
temperature = parameters[:temperature]
|
95
|
+
max_tokens = parameters[:max_tokens] || parameters[:maxTokens]
|
96
|
+
model_name = begin; response.model; rescue NotImplementedError; nil; end
|
97
|
+
model_provider = response.class.name.split('::').last.sub(/Response$/, '')
|
98
|
+
|
99
|
+
metadata = { temperature:, max_tokens:, model_name:, model_provider: }.compact
|
100
|
+
|
101
|
+
metadata.empty? ? nil : metadata
|
102
|
+
end
|
103
|
+
|
104
|
+
def metrics(response)
|
105
|
+
input_tokens = begin; response.prompt_tokens; rescue NotImplementedError; nil; end
|
106
|
+
output_tokens = begin; response.completion_tokens; rescue NotImplementedError; nil; end
|
107
|
+
total_tokens = begin; response.total_tokens; rescue NotImplementedError; nil; end
|
108
|
+
|
109
|
+
metrics = { input_tokens:, output_tokens:, total_tokens: }.compact
|
110
|
+
|
111
|
+
metrics.empty? ? nil : metrics
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
@@ -0,0 +1,117 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Langchain
|
4
|
+
module Datadog
|
5
|
+
# Implements the Datadog LLM Observability API tracing.
|
6
|
+
module Tracing
|
7
|
+
# Returns the active trace ID.
|
8
|
+
def self.active_trace_id
|
9
|
+
@active_trace_id ||=
|
10
|
+
(::Datadog::Tracing.active_trace&.id if defined? ::Datadog) ||
|
11
|
+
next_id
|
12
|
+
end
|
13
|
+
|
14
|
+
# Returns the active span ID.
|
15
|
+
def self.active_span_id
|
16
|
+
@active_span_id ||
|
17
|
+
(::Datadog::Tracing.active_span&.id if defined? ::Datadog)
|
18
|
+
end
|
19
|
+
|
20
|
+
# Returns the active parent span ID.
|
21
|
+
def self.active_parent_id
|
22
|
+
@active_parent_id
|
23
|
+
end
|
24
|
+
|
25
|
+
# Starts a new span, setting the parent ID to the active span ID.
|
26
|
+
def self.start_span(parent_id: active_span_id)
|
27
|
+
@active_parent_id = parent_id
|
28
|
+
@active_span_id = next_id
|
29
|
+
end
|
30
|
+
|
31
|
+
# Ends the active span, setting the parent ID to the given value.
|
32
|
+
def self.end_span(parent_id: nil)
|
33
|
+
@active_span_id = active_parent_id
|
34
|
+
@active_parent_id = parent_id
|
35
|
+
end
|
36
|
+
|
37
|
+
# Starts a new workflow span.
|
38
|
+
def self.workflow(input = nil, name: nil, &block)
|
39
|
+
span(input, name:, kind: 'workflow', &block)
|
40
|
+
end
|
41
|
+
|
42
|
+
# Starts a new agent span.
|
43
|
+
def self.agent(input = nil, name: nil, &block)
|
44
|
+
span(input, name:, kind: 'agent', &block)
|
45
|
+
end
|
46
|
+
|
47
|
+
private_class_method def self.span(input = nil, name: nil, kind: 'workflow')
|
48
|
+
previous_parent_id = active_parent_id
|
49
|
+
start_span
|
50
|
+
|
51
|
+
start_ns = (Time.now.to_r * 1_000_000_000).to_i
|
52
|
+
output = yield
|
53
|
+
duration = (Time.now.to_r * 1_000_000_000).to_i - start_ns
|
54
|
+
|
55
|
+
input_value = { value: input } if input
|
56
|
+
output_value = { value: output } if output
|
57
|
+
|
58
|
+
trace([{
|
59
|
+
name: name || caller_locations(1, 1)[0].label.gsub(' ', '_'),
|
60
|
+
span_id: active_span_id.to_s,
|
61
|
+
trace_id: active_trace_id.to_s,
|
62
|
+
parent_id: active_parent_id&.to_s || 'undefined',
|
63
|
+
start_ns:,
|
64
|
+
duration:,
|
65
|
+
meta: {
|
66
|
+
kind:,
|
67
|
+
input: input_value,
|
68
|
+
output: output_value
|
69
|
+
}.compact
|
70
|
+
}.compact])
|
71
|
+
|
72
|
+
output
|
73
|
+
ensure
|
74
|
+
end_span(parent_id: previous_parent_id)
|
75
|
+
end
|
76
|
+
|
77
|
+
module_function
|
78
|
+
|
79
|
+
def next_id
|
80
|
+
return ::Datadog::Tracing::Utils.next_id if defined? ::Datadog
|
81
|
+
|
82
|
+
rand 1..((1 << 62) - 1)
|
83
|
+
end
|
84
|
+
private_class_method :next_id
|
85
|
+
|
86
|
+
def trace(spans)
|
87
|
+
datadog_client.post do |request|
|
88
|
+
request.url 'trace/spans'
|
89
|
+
request.body = {
|
90
|
+
data: {
|
91
|
+
type: 'span',
|
92
|
+
attributes: {
|
93
|
+
ml_app: Datadog.ml_app,
|
94
|
+
spans:
|
95
|
+
}
|
96
|
+
}
|
97
|
+
}
|
98
|
+
end
|
99
|
+
end
|
100
|
+
private_class_method :trace
|
101
|
+
|
102
|
+
def datadog_client
|
103
|
+
@datadog_client ||= Faraday.new(
|
104
|
+
url: "https://api.#{Datadog.site}/api/intake/llm-obs/v1/",
|
105
|
+
headers: { 'DD-API-KEY' => Datadog.api_key, 'Content-Type' => 'application/json' }
|
106
|
+
) do |faraday|
|
107
|
+
faraday.request :json
|
108
|
+
faraday.response :json
|
109
|
+
faraday.response :logger, Langchain.logger, headers: true, bodies: true do |formatter|
|
110
|
+
formatter.filter(/(DD-API-KEY: )("\w+")/, '\1[REDACTED]')
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
private_class_method :datadog_client
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
@@ -0,0 +1,125 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Langchain
|
4
|
+
module Datadog
|
5
|
+
# Implements hooks for Langchain::Vectorsearch module classes to capture LLM
|
6
|
+
# calls and report them to the Datadog LLM Observability API.
|
7
|
+
module Vectorsearch
|
8
|
+
include Langchain::Datadog::Tracing
|
9
|
+
|
10
|
+
# List of various attributes that store vectors.
|
11
|
+
VECTOR_ATTRIBUTES = %w[embedding vector input_vector values].freeze
|
12
|
+
|
13
|
+
def add_texts(...)
|
14
|
+
return super unless Datadog.enabled?
|
15
|
+
|
16
|
+
span(kind: 'workflow') { super }
|
17
|
+
end
|
18
|
+
|
19
|
+
def update_texts(...)
|
20
|
+
return super unless Datadog.enabled?
|
21
|
+
|
22
|
+
span(kind: 'workflow') { super }
|
23
|
+
end
|
24
|
+
|
25
|
+
def similarity_search(query:, **)
|
26
|
+
return super unless Datadog.enabled?
|
27
|
+
|
28
|
+
parameters = { query: }
|
29
|
+
|
30
|
+
# Retrieval is not a valid root span kind, therefore when called
|
31
|
+
# directly it's wrapped in a workflow span.
|
32
|
+
if Tracing.active_parent_id == 'undefined'
|
33
|
+
return span(parameters, kind: 'workflow') do
|
34
|
+
span(parameters) { super }
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
span(parameters) { super }
|
39
|
+
end
|
40
|
+
|
41
|
+
def similarity_search_with_hyde(query:, **)
|
42
|
+
return super unless Datadog.enabled?
|
43
|
+
|
44
|
+
parameters = { query: }
|
45
|
+
|
46
|
+
span(parameters, kind: 'workflow') { super }
|
47
|
+
end
|
48
|
+
|
49
|
+
def ask(question:, **)
|
50
|
+
return super unless Datadog.enabled?
|
51
|
+
|
52
|
+
parameters = { question: }
|
53
|
+
|
54
|
+
span(parameters, kind: 'workflow') { super }
|
55
|
+
end
|
56
|
+
|
57
|
+
private
|
58
|
+
|
59
|
+
def span(parameters = {}, kind: 'retrieval')
|
60
|
+
previous_parent_id = Tracing.active_parent_id
|
61
|
+
Tracing.start_span
|
62
|
+
|
63
|
+
start_ns = (Time.now.to_r * 1_000_000_000).to_i
|
64
|
+
results = yield
|
65
|
+
duration = (Time.now.to_r * 1_000_000_000).to_i - start_ns
|
66
|
+
|
67
|
+
trace([{
|
68
|
+
name: caller_locations(1, 1)[0].label.gsub(' ', '_'),
|
69
|
+
span_id: Tracing.active_span_id.to_s,
|
70
|
+
trace_id: Tracing.active_trace_id.to_s,
|
71
|
+
parent_id: Tracing.active_parent_id&.to_s || 'undefined',
|
72
|
+
start_ns:,
|
73
|
+
duration:,
|
74
|
+
meta: {
|
75
|
+
kind:,
|
76
|
+
input: input(parameters),
|
77
|
+
output: (output(results) if kind == 'retrieval')
|
78
|
+
}.compact
|
79
|
+
}.compact])
|
80
|
+
|
81
|
+
results
|
82
|
+
ensure
|
83
|
+
Tracing.end_span(parent_id: previous_parent_id)
|
84
|
+
end
|
85
|
+
|
86
|
+
def input(parameters)
|
87
|
+
case parameters
|
88
|
+
in { query: value } then { value: }
|
89
|
+
in { question: value } then { value: }
|
90
|
+
else nil
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def output(results)
|
95
|
+
documents = results.map do |result|
|
96
|
+
next { text: result.to_s } unless result.is_a? Hash
|
97
|
+
|
98
|
+
id = result['id']
|
99
|
+
score = result['score']
|
100
|
+
|
101
|
+
name =
|
102
|
+
result['name'] ||
|
103
|
+
result.dig('metadata', 'name') ||
|
104
|
+
result.dig('metadata', 'title') ||
|
105
|
+
result.dig('metadata', 'filename') ||
|
106
|
+
result.dig('metadata', 'url') ||
|
107
|
+
result.dig('metadata', 'id')
|
108
|
+
|
109
|
+
text =
|
110
|
+
result['content'] ||
|
111
|
+
result['document'] ||
|
112
|
+
result['input'] ||
|
113
|
+
result['payload'] ||
|
114
|
+
result.dig('data', 'content') ||
|
115
|
+
result.dig('metadata', 'content') ||
|
116
|
+
result.except(*VECTOR_ATTRIBUTES, 'name', 'score', 'id').to_json
|
117
|
+
|
118
|
+
{ text: text&.to_s, name:, score:, id: }.compact
|
119
|
+
end
|
120
|
+
|
121
|
+
{ documents: }
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'langchainrb'
|
4
|
+
require 'faraday'
|
5
|
+
|
6
|
+
require_relative 'datadog/version'
|
7
|
+
require_relative 'datadog/tracing'
|
8
|
+
require_relative 'datadog/llm'
|
9
|
+
require_relative 'datadog/vectorsearch'
|
10
|
+
|
11
|
+
module Langchain
|
12
|
+
# Datadog LLM Observability integration with Langchain.rb.
|
13
|
+
module Datadog
|
14
|
+
extend self # rubocop:disable Style/ModuleFunction
|
15
|
+
|
16
|
+
# Values that are considered false when parsing environment variables..
|
17
|
+
FALSE_VALUES = [false, 0, '0', 'f', 'F', 'false', 'FALSE', 'off', 'OFF'].freeze
|
18
|
+
|
19
|
+
# @!attribute [rw] enabled
|
20
|
+
# @return [Boolean] whether to submit data to LLM Observability
|
21
|
+
#
|
22
|
+
# @!attribute [rw] site
|
23
|
+
# @return [String] the Datadog site to submit the LLM data.
|
24
|
+
#
|
25
|
+
# @!attribute [rw] api_key
|
26
|
+
# @return [String] the Datadog API key
|
27
|
+
#
|
28
|
+
# @!attribute [rw] ml_app
|
29
|
+
# @return [String] the name of the LLM application
|
30
|
+
attr_writer :enabled, :site, :api_key, :ml_app
|
31
|
+
|
32
|
+
def enabled
|
33
|
+
return @enabled unless @enabled.nil?
|
34
|
+
|
35
|
+
@enabled = !FALSE_VALUES.include?(ENV.fetch('DD_LLMOBS_ENABLED', '1'))
|
36
|
+
end
|
37
|
+
|
38
|
+
def enabled? = !!enabled
|
39
|
+
|
40
|
+
def site
|
41
|
+
@site ||= if defined? ::Datadog
|
42
|
+
::Datadog.configuration.site
|
43
|
+
else
|
44
|
+
ENV.fetch('DD_SITE', 'datadoghq.com')
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def api_key
|
49
|
+
@api_key ||= if defined? ::Datadog
|
50
|
+
::Datadog.configuration.api_key
|
51
|
+
else
|
52
|
+
ENV.fetch('DD_API_KEY')
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def ml_app
|
57
|
+
@ml_app ||= ENV['DD_LLMOBS_ML_APP'] || \
|
58
|
+
if defined? ::Datadog
|
59
|
+
::Datadog.configuration.service
|
60
|
+
else
|
61
|
+
ENV.fetch('DD_LLMOBS_ML_APP')
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
# Eager load all Langchain::LLM classes to ensure that the Datadog::LLM
|
68
|
+
# module is prepended to all of them.
|
69
|
+
Zeitwerk::Loader.eager_load_namespace(Langchain::LLM) rescue Zeitwerk::NameError ? nil : raise \
|
70
|
+
if defined?(Zeitwerk)
|
71
|
+
|
72
|
+
# Find all classes inheriting from Langchain::LLM::Base and prepend the
|
73
|
+
# Datadog::LLM module to capture LLM calls and report them to the Datadog
|
74
|
+
# LLM Observability API.
|
75
|
+
Langchain::LLM::Base.subclasses.each do |klass|
|
76
|
+
klass.prepend Langchain::Datadog::LLM
|
77
|
+
end
|
78
|
+
|
79
|
+
# Eager load all Langchain::Vectorsearch classes to ensure that the
|
80
|
+
# Datadog::Vectorsearch module is prepended to all of them.
|
81
|
+
Zeitwerk::Loader.eager_load_namespace(Langchain::Vectorsearch) rescue Zeitwerk::NameError ? nil : raise \
|
82
|
+
if defined?(Zeitwerk)
|
83
|
+
|
84
|
+
# Find all classes inheriting from Langchain::Vectorsearch::Base and prepend the
|
85
|
+
# Datadog::Vectorsearch module to capture LLM calls and report them to the
|
86
|
+
# Datadog LLM Observability API.
|
87
|
+
Langchain::Vectorsearch::Base.subclasses.each do |klass|
|
88
|
+
klass.prepend Langchain::Datadog::Vectorsearch
|
89
|
+
end
|
metadata
ADDED
@@ -0,0 +1,89 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: langchainrb_datadog
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- GaggleAMP
|
8
|
+
- Nikolaos Anastopoulos
|
9
|
+
autorequire:
|
10
|
+
bindir: exe
|
11
|
+
cert_chain: []
|
12
|
+
date: 2024-10-12 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: faraday
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - ">="
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '0'
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '0'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: langchainrb
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - "~>"
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: 0.17.1
|
35
|
+
type: :runtime
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - "~>"
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: 0.17.1
|
42
|
+
description: Hooks into Langchain.rb methods to capture LLM calls and report them
|
43
|
+
to the Datadog LLM Observability API.
|
44
|
+
email:
|
45
|
+
- info@gaggleamp.com
|
46
|
+
- ebababi@ebababi.net
|
47
|
+
executables: []
|
48
|
+
extensions: []
|
49
|
+
extra_rdoc_files: []
|
50
|
+
files:
|
51
|
+
- ".rspec"
|
52
|
+
- ".rubocop.yml"
|
53
|
+
- LICENSE.txt
|
54
|
+
- README.md
|
55
|
+
- Rakefile
|
56
|
+
- langchainrb_datadog.gemspec
|
57
|
+
- lib/langchain/datadog.rb
|
58
|
+
- lib/langchain/datadog/llm.rb
|
59
|
+
- lib/langchain/datadog/tracing.rb
|
60
|
+
- lib/langchain/datadog/vectorsearch.rb
|
61
|
+
- lib/langchain/datadog/version.rb
|
62
|
+
- lib/langchainrb_datadog.rb
|
63
|
+
homepage: https://github.com/GaggleAMP/langchainrb_datadog
|
64
|
+
licenses:
|
65
|
+
- MIT
|
66
|
+
metadata:
|
67
|
+
homepage_uri: https://github.com/GaggleAMP/langchainrb_datadog
|
68
|
+
source_code_uri: https://github.com/GaggleAMP/langchainrb_datadog
|
69
|
+
github_repo: git@github.com:GaggleAMP/langchainrb_datadog.git
|
70
|
+
post_install_message:
|
71
|
+
rdoc_options: []
|
72
|
+
require_paths:
|
73
|
+
- lib
|
74
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
75
|
+
requirements:
|
76
|
+
- - ">="
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
version: 3.1.0
|
79
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - ">="
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '0'
|
84
|
+
requirements: []
|
85
|
+
rubygems_version: 3.5.16
|
86
|
+
signing_key:
|
87
|
+
specification_version: 4
|
88
|
+
summary: Enables LLM observability with Datadog for Langchain.rb.
|
89
|
+
test_files: []
|