coolhand 0.1.5 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +14 -1
- data/CHANGELOG.md +99 -1
- data/CLAUDE.md +34 -0
- data/README.md +165 -46
- data/coolhand-ruby.gemspec +10 -6
- data/docs/anthropic.md +518 -0
- data/docs/elevenlabs.md +6 -4
- data/lib/coolhand/api_service.rb +264 -0
- data/lib/coolhand/base_interceptor.rb +213 -0
- data/lib/coolhand/collector.rb +19 -0
- data/lib/coolhand/configuration.rb +84 -0
- data/lib/coolhand/default_exclude_api_patterns.yml +9 -0
- data/lib/coolhand/default_intercept_addresses.yml +15 -0
- data/lib/coolhand/feedback_service.rb +15 -0
- data/lib/coolhand/logger_service.rb +112 -0
- data/lib/coolhand/net_http_interceptor.rb +163 -0
- data/lib/coolhand/open_ai/batch_result_processor.rb +139 -0
- data/lib/coolhand/open_ai/webhook_validator.rb +127 -0
- data/lib/coolhand/{ruby/version.rb → version.rb} +1 -3
- data/lib/coolhand/vertex/batch_result_processor.rb +84 -0
- data/lib/coolhand/webhook_interceptor.rb +39 -0
- data/lib/coolhand.rb +109 -2
- metadata +41 -19
- data/lib/coolhand/ruby/api_service.rb +0 -211
- data/lib/coolhand/ruby/collector.rb +0 -21
- data/lib/coolhand/ruby/configuration.rb +0 -38
- data/lib/coolhand/ruby/feedback_service.rb +0 -17
- data/lib/coolhand/ruby/interceptor.rb +0 -119
- data/lib/coolhand/ruby/logger_service.rb +0 -114
- data/lib/coolhand/ruby.rb +0 -90
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6a7e878a6cb441cee3e29de825ba5b0d5b01285475f42d5a4f66386f37379098
|
|
4
|
+
data.tar.gz: fcc866bd1ca6e7348fbd3ff2daf75ba9203981c09acfaa6062873a08e11b5370
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 7b7f36ff49b826fb34691488c3839de760c7f01c58f811e1bb9108b896da71d958f2e1a714486513da0c1a264278078784ee4065be165b72b349939128e4a9e8
|
|
7
|
+
data.tar.gz: c3bcb5e2a7d3bb07e8e66483bb5c64d3a27b4ad7b7cd677a7bc7aa74b8d612ad53b3b373f21244d3fdf14fb32a34ccee1f80e227b740d3634b7fb6074b7ee64a
|
data/.rubocop.yml
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
# This is the configuration used to check the rubocop source code.
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
plugins:
|
|
4
4
|
- rubocop-performance
|
|
5
|
+
require:
|
|
5
6
|
- rubocop-rspec
|
|
6
7
|
inherit_gem:
|
|
7
8
|
test-prof: config/rubocop-rspec.yml
|
|
@@ -124,6 +125,12 @@ Layout/EndAlignment:
|
|
|
124
125
|
Naming/VariableNumber:
|
|
125
126
|
Enabled: false
|
|
126
127
|
|
|
128
|
+
Naming/PredicateMethod:
|
|
129
|
+
Enabled: false
|
|
130
|
+
|
|
131
|
+
Naming/MethodParameterName:
|
|
132
|
+
Enabled: false
|
|
133
|
+
|
|
127
134
|
# Lint rules
|
|
128
135
|
Lint/EmptyClass:
|
|
129
136
|
Enabled: false
|
|
@@ -138,6 +145,12 @@ RSpec/HookArgument:
|
|
|
138
145
|
RSpec/MultipleExpectations:
|
|
139
146
|
Enabled: false
|
|
140
147
|
|
|
148
|
+
RSpec/VerifiedDoubleReference:
|
|
149
|
+
Enabled: false
|
|
150
|
+
|
|
151
|
+
RSpec/MessageChain:
|
|
152
|
+
Enabled: false
|
|
153
|
+
|
|
141
154
|
RSpec/ExampleLength:
|
|
142
155
|
Enabled: false
|
|
143
156
|
|
data/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,104 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.3.0] - 2026-05-14
|
|
9
|
+
|
|
10
|
+
### 🚀 Major Changes
|
|
11
|
+
- **Unified Net::HTTP Interceptor** - Replaced dual interceptor architecture (Faraday + Anthropic) with a single `NetHttpInterceptor` that captures all HTTP traffic via `Module#prepend`
|
|
12
|
+
- **Simplified Namespace** - Removed `Coolhand::Ruby` namespace; all classes now under `Coolhand` directly (e.g., `Coolhand::FeedbackService` instead of `Coolhand::Ruby::FeedbackService`)
|
|
13
|
+
- **Ruby 4.0 Compatibility** - Full support for Ruby 4.0 with conditional debugger dependencies
|
|
14
|
+
- **Google Gemini API Support** - `generativelanguage.googleapis.com` and `:streamGenerateContent` added to default `intercept_addresses`; both `generateContent` and `streamGenerateContent` endpoints are intercepted out of the box
|
|
15
|
+
- **Anthropic API Support Restored** - `api.anthropic.com` added to default `intercept_addresses`; accidentally dropped during the v0.3.0 refactor that replaced `AnthropicInterceptor` with the unified `NetHttpInterceptor`
|
|
16
|
+
- **URL Query Parameter Sanitization** - New `sanitize_url` helper redacts sensitive query parameters (`key`, `api_key`, `apikey`, `token`, `access_token`, `secret`) before logging; protects API keys passed as URL params (common with Gemini's `?key=` pattern)
|
|
17
|
+
|
|
18
|
+
### ✨ New Features
|
|
19
|
+
- **GitHub Models API** - `models.github.ai` (current endpoint) and `models.inference.ai.azure.com` (deprecated endpoint) added to default `intercept_addresses`; calls routed through GitHub Copilot credentials are now captured automatically without manual configuration. Default intercept addresses are loaded from `default_intercept_addresses.yml` to make future additions a single-line YAML change.
|
|
20
|
+
- **`config.base_url`** - Configurable API destination for self-hosted deployments. Defaults to `https://coolhandlabs.com/api`; set to any `https://` URL to redirect logs and feedback POSTs to your own backend. `http://localhost` and `http://127.0.0.1` are also accepted for local development. Trailing slashes are normalized automatically.
|
|
21
|
+
- **Feedback `sentiment` field** - New string field for feedback: `'like'`, `'dislike'`, or `'neutral'`. Preferred over the boolean `like` field for richer signal.
|
|
22
|
+
- **Feedback `workload_hashid` field** - New string field to associate feedback with a specific workload.
|
|
23
|
+
- **Batch Processing Support** - New `Coolhand::OpenAi::BatchResultProcessor` and `Coolhand::Vertex::BatchResultProcessor` for logging completed async batch jobs as individual `llm_request_log` entries
|
|
24
|
+
- **OpenAI Webhook Validation** - New `Coolhand::OpenAi::WebhookValidator` verifies webhook signatures using HMAC-SHA256 with timing-safe comparison; lenient in development, strict in production/staging
|
|
25
|
+
- **WebhookInterceptor Rails Module** - `Coolhand::WebhookInterceptor` mixin for Rails controllers to validate and dispatch OpenAI batch completion webhooks automatically
|
|
26
|
+
- **Capture Control** - New `config.capture` global toggle (default: `true`) and `config.debug_mode` (captures locally, skips API forwarding) for fine-grained interception control
|
|
27
|
+
- **Thread-Safe Block Control** - `Coolhand.with_capture { }` and `Coolhand.without_capture { }` for scoped override of capture behavior within a block; uses thread-local storage
|
|
28
|
+
- **Exclude API Patterns** - New `config.exclude_api_patterns` deny-list checked after the `intercept_addresses` allow-list; default excludes `["/batchPredictionJobs/"]` to suppress Vertex AI batch job management noise
|
|
29
|
+
|
|
30
|
+
### 🚫 Deprecated
|
|
31
|
+
- **Feedback `like` field** - The boolean `like` field is deprecated. Use `sentiment: 'like'` or `sentiment: 'dislike'` instead.
|
|
32
|
+
|
|
33
|
+
### 🏗️ Architecture Improvements
|
|
34
|
+
- **Single Interceptor** - `NetHttpInterceptor` patches `Net::HTTP#request` and `Net::HTTPResponse#read_body`; removed ~1,400 lines of interceptor-specific code
|
|
35
|
+
- **Thread-Safe Streaming** - Uses `Thread.current[:coolhand_stream_buffer]` for streaming response capture
|
|
36
|
+
- **Capture Priority Hierarchy** - `debug_mode` (always capture) > thread-local override > global `capture` config
|
|
37
|
+
|
|
38
|
+
### 🐛 Bug Fixes
|
|
39
|
+
- **Streaming Response Encoding** - Streamed response content is now force-encoded to UTF-8 before JSON parsing, eliminating noisy `BINARY` encoding warnings for multi-byte responses.
|
|
40
|
+
- **Double-Capture with `Net::HTTP.new` Pattern** - Fixed double-logging when callers use `Net::HTTP.new(host, port).request(req)` without an explicit `start` block. The re-entry guard is now per-connection-object (using a `compare_by_identity` Hash) rather than a boolean thread-local, so independent requests on a different `Net::HTTP` instance inside a callback are still captured.
|
|
41
|
+
- **Provider-Neutral Readiness Log** - Startup console message no longer names a single provider; it now reflects all monitored inference URIs.
|
|
42
|
+
- **Interceptor No Longer Silently Drops Logs on HTTP Errors** - Wrapped `Net::HTTP#request` in `begin/rescue/ensure` so `send_complete_request_log` is always called even when the SDK raises an exception (e.g., `Anthropic::Errors::NotFoundError` on a 404). Status is extracted from the exception via `.status`, `.response.status`, or message parsing.
|
|
43
|
+
|
|
44
|
+
### 📦 Dependencies
|
|
45
|
+
- Bumped `faraday` from 2.14.0 to 2.14.1
|
|
46
|
+
|
|
47
|
+
### 💔 Breaking Changes
|
|
48
|
+
- **`config.base_url` validation** - `Coolhand.configure` now raises `Coolhand::Error` if `base_url` is set to a plain `http://` URL (non-localhost). Previously any string was accepted silently. If you were pointing at an internal `http://` host (e.g. `http://logs.internal/api`), you will need to either enable TLS on that host or use an `https://` proxy in front of it.
|
|
49
|
+
- **Namespace Change** - `Coolhand::Ruby::*` references must be updated to `Coolhand::*`
|
|
50
|
+
- **Removed Files** - `faraday_interceptor.rb` and `anthropic_interceptor.rb` replaced by `net_http_interceptor.rb`
|
|
51
|
+
- **`environment` Config Behavior** - The `environment` attribute no longer controls whether requests are forwarded to the API. Use `config.debug_mode = true` instead if you previously relied on `environment: "development"` to suppress API calls.
|
|
52
|
+
|
|
53
|
+
### 🔄 Migration Guide
|
|
54
|
+
1. Update gem dependency to `~> 0.3.0`
|
|
55
|
+
2. Replace `Coolhand::Ruby::` with `Coolhand::` in all class references
|
|
56
|
+
3. If using `environment: "development"` to prevent API calls, switch to `config.debug_mode = true`
|
|
57
|
+
4. If `config.base_url` was set to a plain `http://` (non-localhost) URL, switch to `https://` or use an `https://` proxy
|
|
58
|
+
5. No other changes needed to `Coolhand.configure` blocks for basic usage
|
|
59
|
+
|
|
60
|
+
## [0.2.0] - 2025-12-16
|
|
61
|
+
|
|
62
|
+
### ✨ Major New Features
|
|
63
|
+
- **Official Anthropic Gem Support** - Added comprehensive monitoring support for the official `anthropic` gem (v1.8+) through direct Net::HTTP interception
|
|
64
|
+
- **Dual Gem Compatibility** - Support for both `anthropic` (official) and `ruby-anthropic` (community) gems with automatic detection and appropriate interceptor selection
|
|
65
|
+
- **Streaming Response Support** - Enhanced SSE (Server-Sent Events) parsing for Anthropic streaming responses with proper message accumulation and reconstruction
|
|
66
|
+
- **Graceful Gem Conflict Handling** - Automatic detection when both anthropic gems are installed, with graceful degradation to ruby-anthropic monitoring
|
|
67
|
+
|
|
68
|
+
### 🏗️ Architecture Improvements
|
|
69
|
+
- **AnthropicInterceptor Module** - New dedicated interceptor for official anthropic gem requests with streaming response support
|
|
70
|
+
- **BaseInterceptor Module** - Shared functionality across interceptors with unified API logging format and DRY principles
|
|
71
|
+
- **Modular Design** - Moved from single `interceptor.rb` to specialized interceptors (`faraday_interceptor.rb`, `anthropic_interceptor.rb`)
|
|
72
|
+
- **Enhanced Configuration** - Automatic gem detection in `configure` block with appropriate interceptor selection
|
|
73
|
+
|
|
74
|
+
### 🔧 API & Format Changes
|
|
75
|
+
- **Unified Logging Format** - Standardized API request/response logging with `raw_request` wrapper and collector data integration
|
|
76
|
+
- **Headers Field Update** - API logs now use `headers` instead of `request_headers` for consistency
|
|
77
|
+
- **Silent Mode Override** - Critical warnings (like gem conflicts) now always display regardless of silent mode settings
|
|
78
|
+
|
|
79
|
+
### 🧪 Testing & Quality
|
|
80
|
+
- **Comprehensive Test Coverage** - Added 16 new specs covering all interceptor scenarios including gem conflict handling
|
|
81
|
+
- **RuboCop Compliance** - Applied linting with proper line length, verified doubles, and RSpec best practices
|
|
82
|
+
- **Thread Safety** - Enhanced request correlation with thread-local storage for streaming requests
|
|
83
|
+
|
|
84
|
+
### 🗂️ Supported Environments
|
|
85
|
+
- **Development Environment** - Uses official `anthropic` gem for Net::HTTP-based requests
|
|
86
|
+
- **AR_Dev Environment** - Uses `ruby-anthropic` gem for Faraday-based requests
|
|
87
|
+
- **Automatic Detection** - Coolhand detects which gem is loaded and applies appropriate interception
|
|
88
|
+
|
|
89
|
+
### 💔 Breaking Changes
|
|
90
|
+
- **Removed** - `lib/coolhand/ruby/interceptor.rb` replaced by specialized interceptor modules
|
|
91
|
+
- **API Change** - Logging format now uses `headers` field instead of `request_headers`
|
|
92
|
+
|
|
93
|
+
### 🔄 Migration Guide
|
|
94
|
+
For users upgrading from v0.1.x:
|
|
95
|
+
- No code changes required for basic usage
|
|
96
|
+
- If depending on old `interceptor.rb` directly, update imports to use `faraday_interceptor.rb` or `anthropic_interceptor.rb`
|
|
97
|
+
- API log consumers should expect `headers` field instead of `request_headers`
|
|
98
|
+
|
|
99
|
+
### 📊 Compatibility Matrix
|
|
100
|
+
| Gem | Version | Interceptor | Status |
|
|
101
|
+
|-----|---------|-------------|--------|
|
|
102
|
+
| `anthropic` | 1.8+ | AnthropicInterceptor | ✅ Full Support |
|
|
103
|
+
| `ruby-anthropic` | 0.4+ | FaradayInterceptor | ✅ Full Support |
|
|
104
|
+
| Both gems | Any | FaradayInterceptor | ⚠️ Graceful Degradation |
|
|
105
|
+
|
|
8
106
|
## [0.1.5] - 2024-12-09
|
|
9
107
|
|
|
10
108
|
### 🐛 Critical Bug Fixes
|
|
@@ -19,7 +117,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
19
117
|
- **Collection Method Tracking** - Support for optional collection method suffix (`manual`, `auto-monitor`)
|
|
20
118
|
|
|
21
119
|
### 🏗️ Internal Improvements
|
|
22
|
-
- **Added Collector Module** - New `Coolhand::
|
|
120
|
+
- **Added Collector Module** - New `Coolhand::Collector` module for generating SDK identification strings
|
|
23
121
|
- **Updated ApiService** - Base service now automatically adds collector field to all API payloads
|
|
24
122
|
- **Enhanced Logging** - Both LoggerService and FeedbackService now send collector information
|
|
25
123
|
|
data/CLAUDE.md
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Development Guidelines
|
|
2
|
+
|
|
3
|
+
## Optional Provider Dependencies
|
|
4
|
+
|
|
5
|
+
Coolhand supports multiple LLM providers (OpenAI, Anthropic, Google Gemini, etc.). These provider gems should **never** be required at gem load time, as clients may not use all providers and shouldn't be forced to install unnecessary dependencies.
|
|
6
|
+
|
|
7
|
+
**Rule**: Any require for provider SDKs (openai, anthropic, google-generativeai, etc.) must be:
|
|
8
|
+
1. Placed in the file where it's actually used (not in the main coolhand.rb)
|
|
9
|
+
2. Only executed when that provider's functionality is accessed
|
|
10
|
+
3. Not declared as a hard dependency in coolhand-ruby.gemspec
|
|
11
|
+
|
|
12
|
+
Example pattern:
|
|
13
|
+
```ruby
|
|
14
|
+
# ❌ DON'T: In lib/coolhand.rb (loads unconditionally)
|
|
15
|
+
require "openai"
|
|
16
|
+
|
|
17
|
+
# ✅ DO: In lib/coolhand/open_ai/batch_result_processor.rb (only when needed)
|
|
18
|
+
require "openai"
|
|
19
|
+
|
|
20
|
+
module Coolhand
|
|
21
|
+
module OpenAi
|
|
22
|
+
class BatchResultProcessor
|
|
23
|
+
def client
|
|
24
|
+
@client ||= OpenAI::Client.new
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
This ensures:
|
|
32
|
+
- Gem loads cleanly regardless of what providers are installed
|
|
33
|
+
- Apps using path gems (local development) don't break from missing optional dependencies
|
|
34
|
+
- Users only need gems for providers they actually use
|
data/README.md
CHANGED
|
@@ -24,7 +24,7 @@ gem 'coolhand'
|
|
|
24
24
|
|
|
25
25
|
```ruby
|
|
26
26
|
# Add this configuration at the start of your application
|
|
27
|
-
require 'coolhand
|
|
27
|
+
require 'coolhand'
|
|
28
28
|
|
|
29
29
|
Coolhand.configure do |config|
|
|
30
30
|
config.api_key = 'your_api_key_here'
|
|
@@ -47,15 +47,38 @@ end
|
|
|
47
47
|
- ⚡ **Performance optimized** - Negligible overhead via async logging
|
|
48
48
|
- 🛡️ **Future-proof** - Automatically captures new AI calls added by your team
|
|
49
49
|
|
|
50
|
+
## Self-Hosted Deployments
|
|
51
|
+
|
|
52
|
+
For compliance, data-residency, or cost reasons you can run your own Coolhand-compatible endpoint and point the SDK at it via `config.base_url`:
|
|
53
|
+
|
|
54
|
+
```ruby
|
|
55
|
+
Coolhand.configure do |config|
|
|
56
|
+
config.api_key = ENV['COOLHAND_API_KEY']
|
|
57
|
+
config.base_url = ENV['COOLHAND_BASE_URL'] # e.g. "https://coolhand.internal.example.com/api"
|
|
58
|
+
end
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
When `base_url` is unset the SDK defaults to `https://coolhandlabs.com/api` and behaviour is unchanged.
|
|
62
|
+
|
|
63
|
+
**Accepted values:**
|
|
64
|
+
- Any `https://` URL — required for production use
|
|
65
|
+
- `http://localhost` or `http://127.0.0.1` — accepted for local development only
|
|
66
|
+
|
|
67
|
+
**Trailing slashes** are stripped automatically, so `"https://example.com/api/"` and `"https://example.com/api"` are equivalent.
|
|
68
|
+
|
|
69
|
+
The SDK raises `Coolhand::Error` at configure time if `base_url` is set to a plain `http://` URL pointing at a non-localhost host.
|
|
70
|
+
|
|
50
71
|
## Feedback API
|
|
51
72
|
|
|
52
|
-
Collect feedback on LLM responses to improve model performance
|
|
73
|
+
Collect feedback on LLM responses to improve model performance.
|
|
74
|
+
|
|
75
|
+
> **Frontend Feedback Widget**: For browser-based feedback collection, see [coolhand-js](https://github.com/Coolhand-Labs/coolhand-js) - an accessible, lightweight JavaScript widget that leverages best UX practices to capture actionable user feedback on any AI output.
|
|
53
76
|
|
|
54
77
|
```ruby
|
|
55
|
-
require 'coolhand
|
|
78
|
+
require 'coolhand'
|
|
56
79
|
|
|
57
80
|
# Create feedback for an LLM response
|
|
58
|
-
feedback_service = Coolhand::
|
|
81
|
+
feedback_service = Coolhand::FeedbackService.new(Coolhand.configuration)
|
|
59
82
|
|
|
60
83
|
feedback = feedback_service.create_feedback(
|
|
61
84
|
llm_request_log_id: 123,
|
|
@@ -65,7 +88,7 @@ feedback = feedback_service.create_feedback(
|
|
|
65
88
|
original_output: 'Here is the original LLM response!',
|
|
66
89
|
revised_output: 'Here is the human edit of the original LLM response.',
|
|
67
90
|
explanation: 'Tone of the original response read like AI-generated open source README docs',
|
|
68
|
-
|
|
91
|
+
sentiment: 'dislike'
|
|
69
92
|
)
|
|
70
93
|
```
|
|
71
94
|
|
|
@@ -80,7 +103,9 @@ feedback = feedback_service.create_feedback(
|
|
|
80
103
|
### Quality Data
|
|
81
104
|
- **`revised_output`** ⭐ *Best Signal* - End user revision of the LLM response. The highest value data for improving quality scores.
|
|
82
105
|
- **`explanation`** 💬 *Medium Signal* - End user explanation of why the response was good or bad. Valuable qualitative data.
|
|
83
|
-
- **`
|
|
106
|
+
- **`sentiment`** 🎭 *Preferred* - String sentiment: `'like'`, `'dislike'`, or `'neutral'`. Takes precedence over `like` if both are provided. The gem automatically converts `like` to `sentiment` before sending.
|
|
107
|
+
- **`like`** 👍 *Low Signal (Deprecated)* - Boolean: `true` = like, `false` = dislike. Use `sentiment` instead. Conversion: `true` → `"like"`, `false` → `"dislike"`.
|
|
108
|
+
- **`workload_hashid`** 🔗 *Workload Association* - Hashid of a workload to associate this feedback with.
|
|
84
109
|
- **`creator_unique_id`** 👤 *User Tracking* - Unique ID to match feedback to the end user who created it
|
|
85
110
|
|
|
86
111
|
## Rails Integration
|
|
@@ -100,7 +125,7 @@ Coolhand.configure do |config|
|
|
|
100
125
|
config.silent = Rails.env.production?
|
|
101
126
|
|
|
102
127
|
# Specify which LLM endpoints to intercept (array of strings)
|
|
103
|
-
# Optional - defaults to
|
|
128
|
+
# Optional - defaults to OpenAI, Anthropic, ElevenLabs, Google Gemini, and GitHub Models
|
|
104
129
|
# config.intercept_addresses = ["api.openai.com", "api.anthropic.com", "api.cohere.ai"]
|
|
105
130
|
end
|
|
106
131
|
```
|
|
@@ -110,7 +135,7 @@ end
|
|
|
110
135
|
```ruby
|
|
111
136
|
class ChatController < ApplicationController
|
|
112
137
|
def create_feedback
|
|
113
|
-
feedback_service = Coolhand::
|
|
138
|
+
feedback_service = Coolhand::FeedbackService.new(Coolhand.configuration)
|
|
114
139
|
|
|
115
140
|
feedback = feedback_service.create_feedback(
|
|
116
141
|
llm_request_log_id: params[:log_id],
|
|
@@ -118,7 +143,7 @@ class ChatController < ApplicationController
|
|
|
118
143
|
original_output: params[:original_response],
|
|
119
144
|
revised_output: params[:edited_response],
|
|
120
145
|
explanation: params[:feedback_text],
|
|
121
|
-
|
|
146
|
+
sentiment: params[:sentiment]
|
|
122
147
|
)
|
|
123
148
|
|
|
124
149
|
if feedback
|
|
@@ -135,14 +160,14 @@ end
|
|
|
135
160
|
```ruby
|
|
136
161
|
class FeedbackCollectionJob < ApplicationJob
|
|
137
162
|
def perform(feedback_data)
|
|
138
|
-
feedback_service = Coolhand::
|
|
163
|
+
feedback_service = Coolhand::FeedbackService.new(Coolhand.configuration)
|
|
139
164
|
|
|
140
165
|
feedback_service.create_feedback(
|
|
141
166
|
llm_provider_unique_id: feedback_data[:request_id],
|
|
142
167
|
creator_unique_id: feedback_data[:user_id],
|
|
143
168
|
original_output: feedback_data[:original],
|
|
144
169
|
explanation: feedback_data[:reason],
|
|
145
|
-
|
|
170
|
+
sentiment: feedback_data[:sentiment]
|
|
146
171
|
)
|
|
147
172
|
end
|
|
148
173
|
end
|
|
@@ -164,7 +189,7 @@ end
|
|
|
164
189
|
|
|
165
190
|
```ruby
|
|
166
191
|
require 'openai'
|
|
167
|
-
require 'coolhand
|
|
192
|
+
require 'coolhand'
|
|
168
193
|
|
|
169
194
|
# Configure Coolhand
|
|
170
195
|
Coolhand.configure do |config|
|
|
@@ -186,29 +211,7 @@ puts response.dig("choices", 0, "message", "content")
|
|
|
186
211
|
# The request and response have been automatically logged to Coolhand!
|
|
187
212
|
```
|
|
188
213
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
```ruby
|
|
192
|
-
require 'anthropic'
|
|
193
|
-
require 'coolhand/ruby'
|
|
194
|
-
|
|
195
|
-
# Configure Coolhand
|
|
196
|
-
Coolhand.configure do |config|
|
|
197
|
-
config.api_key = 'your_api_key_here'
|
|
198
|
-
end
|
|
199
|
-
|
|
200
|
-
# Use Anthropic normally - requests are automatically logged
|
|
201
|
-
anthropic = Anthropic::Client.new(access_token: ENV['ANTHROPIC_API_KEY'])
|
|
202
|
-
|
|
203
|
-
response = anthropic.messages(
|
|
204
|
-
model: "claude-3-opus",
|
|
205
|
-
max_tokens: 1024,
|
|
206
|
-
messages: [{ role: "user", content: "Hello, Claude!" }]
|
|
207
|
-
)
|
|
208
|
-
|
|
209
|
-
puts response["content"]
|
|
210
|
-
# Automatically logged to Coolhand!
|
|
211
|
-
```
|
|
214
|
+
📖 **[Complete Anthropic Integration Guide →](docs/anthropic.md)** - Supports both official and community gems with automatic detection
|
|
212
215
|
|
|
213
216
|
## Logging Inbound Webhooks
|
|
214
217
|
|
|
@@ -285,23 +288,41 @@ The filtering is automatic and applies to all monitored API calls and webhook lo
|
|
|
285
288
|
|
|
286
289
|
## Supported Libraries
|
|
287
290
|
|
|
288
|
-
The monitor works with
|
|
291
|
+
The monitor works with multiple transport layers and Ruby libraries:
|
|
289
292
|
|
|
293
|
+
**Faraday-based libraries:**
|
|
290
294
|
- OpenAI Ruby SDK
|
|
291
|
-
- Anthropic
|
|
295
|
+
- ruby-anthropic gem (community Anthropic gem)
|
|
292
296
|
- ruby-openai gem
|
|
293
297
|
- LangChain.rb
|
|
294
298
|
- Direct Faraday requests
|
|
295
299
|
- Any other Faraday-based HTTP client
|
|
296
300
|
|
|
301
|
+
**Native HTTP libraries:**
|
|
302
|
+
- Official Anthropic Ruby SDK (using Net::HTTP)
|
|
303
|
+
- GitHub Models SDK / any client using `models.github.ai`
|
|
304
|
+
- Any library using Net::HTTP directly
|
|
305
|
+
|
|
306
|
+
**Universal Coverage**: Since most Ruby HTTP libraries use Net::HTTP under the hood, Coolhand's single interceptor provides comprehensive monitoring without needing library-specific integrations.
|
|
307
|
+
|
|
297
308
|
## How It Works
|
|
298
309
|
|
|
299
|
-
|
|
310
|
+
Coolhand uses a unified Net::HTTP interceptor to monitor all HTTP traffic to configured LLM endpoints:
|
|
311
|
+
|
|
312
|
+
### Net::HTTP Interceptor
|
|
313
|
+
- Patches Ruby's core `Net::HTTP` library using `Module#prepend`
|
|
314
|
+
- Monitors **all** HTTP libraries that use Net::HTTP under the hood (which is most of them)
|
|
315
|
+
- Handles both standard requests and streaming responses via `read_body` interception
|
|
316
|
+
- Thread-safe design using thread-local storage for streaming buffers
|
|
300
317
|
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
318
|
+
### Request Flow
|
|
319
|
+
When a request matches configured LLM endpoints:
|
|
320
|
+
|
|
321
|
+
1. The original request executes normally with zero performance impact
|
|
322
|
+
2. Request and response data (body, headers, status) are captured by the interceptor
|
|
323
|
+
3. For streaming requests, the complete accumulated response is captured (not individual chunks)
|
|
324
|
+
4. Data is sent to the Coolhand API asynchronously in a background thread
|
|
325
|
+
5. Your application continues without interruption
|
|
305
326
|
|
|
306
327
|
For non-matching endpoints, requests pass through unchanged.
|
|
307
328
|
|
|
@@ -338,7 +359,7 @@ For standard Ruby scripts or non-Rails applications:
|
|
|
338
359
|
|
|
339
360
|
```ruby
|
|
340
361
|
#!/usr/bin/env ruby
|
|
341
|
-
require 'coolhand
|
|
362
|
+
require 'coolhand'
|
|
342
363
|
|
|
343
364
|
Coolhand.configure do |config|
|
|
344
365
|
config.api_key = 'your_api_key_here' # Store securely, don't commit to git
|
|
@@ -367,8 +388,105 @@ The monitor handles errors gracefully:
|
|
|
367
388
|
- Invalid API keys will be reported but won't crash your app
|
|
368
389
|
- Network issues are handled with appropriate error messages
|
|
369
390
|
|
|
391
|
+
|
|
392
|
+
## Batch webhook handler (OpenAI)
|
|
393
|
+
|
|
394
|
+
Automatically handle OpenAI batch event logs (batch.completed, batch.failed, batch.expired, batch.cancelled)
|
|
395
|
+
by intercepting webhook requests and enqueuing your batch result processor.
|
|
396
|
+
|
|
397
|
+
Usage:
|
|
398
|
+
- Include the interceptor in your controller:
|
|
399
|
+
include Coolhand::WebhookInterceptor
|
|
400
|
+
- Add the before_action to validate and populate @validator payload:
|
|
401
|
+
before_action :intercept_batch_request, only: :openai
|
|
402
|
+
- Ensure you skip CSRF for the webhook endpoint:
|
|
403
|
+
skip_before_action :verify_authenticity_token
|
|
404
|
+
- Override the webhook_secret method to return your OpenAI webhook secret
|
|
405
|
+
|
|
406
|
+
Minimal example (only key lines shown):
|
|
407
|
+
|
|
408
|
+
```ruby
|
|
409
|
+
# app/controllers/webhooks/batch_api_requests_controller.rb
|
|
410
|
+
# ...existing code...
|
|
411
|
+
include Coolhand::WebhookInterceptor
|
|
412
|
+
|
|
413
|
+
skip_before_action :verify_authenticity_token
|
|
414
|
+
before_action :intercept_batch_request, only: :openai
|
|
415
|
+
|
|
416
|
+
def openai
|
|
417
|
+
event = JSON.parse(@validator.payload)
|
|
418
|
+
case event["type"]
|
|
419
|
+
when "batch.completed", "batch.failed", "batch.expired", "batch.cancelled"
|
|
420
|
+
batch_id = event.dig("data", "id")
|
|
421
|
+
batch_request = BatchApiRequest.find_by(provider: "openai", provider_batch_id: batch_id)
|
|
422
|
+
|
|
423
|
+
if batch_request
|
|
424
|
+
OpenAi::BatchResultProcessor.perform_async(batch_request.id)
|
|
425
|
+
Rails.logger.info("Queued batch result processing for BatchApiRequest #{batch_request.id}")
|
|
426
|
+
else
|
|
427
|
+
Rails.logger.warn("Could not find BatchApiRequest for OpenAI batch ID: #{batch_id}")
|
|
428
|
+
end
|
|
429
|
+
else
|
|
430
|
+
Rails.logger.info("Unhandled OpenAI webhook event type: #{event["type"]}")
|
|
431
|
+
end
|
|
432
|
+
|
|
433
|
+
head :ok
|
|
434
|
+
rescue JSON::ParserError
|
|
435
|
+
head :bad_request
|
|
436
|
+
rescue StandardError => e
|
|
437
|
+
Rails.logger.error("OpenAI webhook error: #{e.message}")
|
|
438
|
+
head :internal_server_error
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
def webhook_secret
|
|
442
|
+
Rails.application.credentials.openai_webhook_secret
|
|
443
|
+
end
|
|
444
|
+
# ...existing code...
|
|
445
|
+
```
|
|
446
|
+
|
|
447
|
+
## Batch webhook handler (Vertex)
|
|
448
|
+
|
|
449
|
+
Automatically handle Vertex batch event logs.
|
|
450
|
+
|
|
451
|
+
Usage:
|
|
452
|
+
- call Coolhand::Vertex::BatchResultProcessor service with batch_info and download batch results
|
|
453
|
+
|
|
454
|
+
Minimal example (only key lines shown):
|
|
455
|
+
|
|
456
|
+
```ruby
|
|
457
|
+
class Vertex::BatchCallbackProcessor < BaseService
|
|
458
|
+
option :batch_request, model: BatchApiRequest
|
|
459
|
+
option :batch_info
|
|
460
|
+
|
|
461
|
+
def call
|
|
462
|
+
case batch_info["state"]
|
|
463
|
+
when "JOB_STATE_PENDING"
|
|
464
|
+
nil
|
|
465
|
+
when "JOB_STATE_RUNNING", "JOB_STATE_QUEUED"
|
|
466
|
+
batch_request.update!(status: "processing")
|
|
467
|
+
|
|
468
|
+
Coolhand::Vertex::BatchResultProcessor.new(batch_info:).call
|
|
469
|
+
when "JOB_STATE_SUCCEEDED"
|
|
470
|
+
output_file_id = batch_info["outputInfo"]["gcsOutputDirectory"]
|
|
471
|
+
results = download_batch_results(output_file_id)
|
|
472
|
+
results.each { |batch_item| process_batch_result(batch_item) }
|
|
473
|
+
|
|
474
|
+
batch_request.update!(status: "completed", completed_at: Time.current, output_file_id:)
|
|
475
|
+
|
|
476
|
+
Coolhand::Vertex::BatchResultProcessor.new(batch_info:).call(results)
|
|
477
|
+
|
|
478
|
+
# Clean up GCS files after successful processing
|
|
479
|
+
cleanup_gcs_files(output_file_id)
|
|
480
|
+
when "JOB_STATE_FAILED"
|
|
481
|
+
handle_failed_batch(batch_info["error"]["message"])
|
|
482
|
+
end
|
|
483
|
+
end
|
|
484
|
+
end
|
|
485
|
+
```
|
|
486
|
+
|
|
370
487
|
## Integration Guides
|
|
371
488
|
|
|
489
|
+
- **[Anthropic Integration](docs/anthropic.md)** - Complete guide for both official and community Anthropic gems, including streaming, dual gem handling, and troubleshooting
|
|
372
490
|
- **[ElevenLabs Integration](docs/elevenlabs.md)** - Complete guide for integrating ElevenLabs Conversational AI with webhook capture and feedback submission
|
|
373
491
|
|
|
374
492
|
## Security
|
|
@@ -377,10 +495,11 @@ The monitor handles errors gracefully:
|
|
|
377
495
|
- No sensitive data is exposed in logs
|
|
378
496
|
- All data is sent via HTTPS to Coolhand servers
|
|
379
497
|
|
|
380
|
-
##
|
|
498
|
+
## Related Packages
|
|
381
499
|
|
|
382
|
-
- **
|
|
383
|
-
- **
|
|
500
|
+
- **Frontend (Feedback Collection Widget)**: [coolhand-js](https://github.com/Coolhand-Labs/coolhand-js) - Frontend feedback widget for collecting user feedback on AI outputs
|
|
501
|
+
- **Node.js**: [coolhand-node package](https://github.com/Coolhand-Labs/coolhand-node) - Coolhand monitoring for Node.js applications
|
|
502
|
+
- **Python**: [coolhand package](https://github.com/Coolhand-Labs/coolhand-python) - Coolhand monitoring for Python applications
|
|
384
503
|
|
|
385
504
|
## Community
|
|
386
505
|
|
data/coolhand-ruby.gemspec
CHANGED
|
@@ -1,16 +1,18 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require_relative "lib/coolhand/
|
|
3
|
+
require_relative "lib/coolhand/version"
|
|
4
4
|
|
|
5
5
|
Gem::Specification.new do |spec|
|
|
6
6
|
spec.name = "coolhand"
|
|
7
|
-
spec.version = Coolhand::
|
|
7
|
+
spec.version = Coolhand::VERSION
|
|
8
8
|
spec.authors = ["Michael Carroll", "Yaroslav Malyk"]
|
|
9
9
|
spec.email = ["mc@coolhandlabs.com"]
|
|
10
10
|
|
|
11
|
-
spec.summary = "
|
|
12
|
-
spec.description = "
|
|
13
|
-
"
|
|
11
|
+
spec.summary = "Monitor and log LLM API calls from OpenAI, Anthropic, and other providers to Coolhand analytics."
|
|
12
|
+
spec.description = "Automatically intercept and log LLM requests from Ruby applications. Supports OpenAI, " \
|
|
13
|
+
"official Anthropic gem, ruby-anthropic gem, and other Faraday-based libraries. Features " \
|
|
14
|
+
"dual interceptor architecture, streaming support, thread-safe operation, and automatic " \
|
|
15
|
+
"duplicate request prevention."
|
|
14
16
|
spec.homepage = "https://coolhandlabs.com/"
|
|
15
17
|
spec.license = "Apache-2.0"
|
|
16
18
|
spec.required_ruby_version = ">= 3.0.0"
|
|
@@ -19,7 +21,7 @@ Gem::Specification.new do |spec|
|
|
|
19
21
|
|
|
20
22
|
spec.metadata["homepage_uri"] = spec.homepage
|
|
21
23
|
spec.metadata["source_code_uri"] = "https://github.com/Coolhand-Labs/coolhand-ruby"
|
|
22
|
-
spec.metadata["changelog_uri"] = "https://github.com/Coolhand-Labs/coolhand-ruby"
|
|
24
|
+
spec.metadata["changelog_uri"] = "https://github.com/Coolhand-Labs/coolhand-ruby/blob/main/CHANGELOG.md"
|
|
23
25
|
|
|
24
26
|
# Specify which files should be added to the gem when it is released.
|
|
25
27
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
|
@@ -36,6 +38,8 @@ Gem::Specification.new do |spec|
|
|
|
36
38
|
# Uncomment to register a new dependency of your gem
|
|
37
39
|
# spec.add_dependency "example-gem", "~> 1.0"
|
|
38
40
|
|
|
41
|
+
spec.add_dependency "base64", "~> 0.2"
|
|
42
|
+
|
|
39
43
|
# For more information and examples about making a new gem, check out our
|
|
40
44
|
# guide at: https://bundler.io/guides/creating_gem.html
|
|
41
45
|
spec.metadata["rubygems_mfa_required"] = "true"
|