active_harness 0.3.5 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/active_harness/configuration.rb +45 -0
- data/lib/active_harness/providers/audio/deepgram.rb +81 -0
- data/lib/active_harness/providers/audio/elevenlabs.rb +103 -0
- data/lib/active_harness/providers/hetzner.rb +69 -0
- data/lib/active_harness/providers/ollama_cloud.rb +63 -0
- data/lib/active_harness/providers/typesafe.rb +35 -0
- data/lib/active_harness/providers/vercel.rb +28 -5
- data/lib/active_harness/request/providers.rb +7 -2
- data/lib/active_harness.rb +6 -1
- data/lib/generators/active_harness/install/templates/initializers/active_harness.rb +24 -0
- metadata +6 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 347d7944af01b44bda7568ac47fc335fd58a48320f80ae71c4a52b83458a710f
|
|
4
|
+
data.tar.gz: 13343b28d031fd4dd86b4114a7d4c3f7b7223db6402b3c1f2bf9e135073e3bc3
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3eb2200ec4c40e7ecf626d9cff3c5f56917b07dbf5756fa54d986d327ce740b18b95b5806f4e9924674a010196c76bbb11f9c66fbe9f54256ac0cfce4c901a7c
|
|
7
|
+
data.tar.gz: 8c1c186d855e6ef3ee0e8961665904da7175cf7018c0e4e352eb2b6a15bcd42f2d019963cb996d0bb8407f8eeb9eda98bac13b8def11fd2c43b382a8dce8d8f5
|
|
@@ -105,6 +105,36 @@ module ActiveHarness
|
|
|
105
105
|
attr_accessor :vercel_api_key
|
|
106
106
|
attr_accessor :vercel_api_url
|
|
107
107
|
|
|
108
|
+
# -------------------------------------------------------------------------
|
|
109
|
+
# TypeSafe — direct Jev (System One) API; also any Jev-compatible server
|
|
110
|
+
# -------------------------------------------------------------------------
|
|
111
|
+
attr_accessor :typesafe_api_key
|
|
112
|
+
attr_accessor :typesafe_api_url
|
|
113
|
+
|
|
114
|
+
# -------------------------------------------------------------------------
|
|
115
|
+
# Ollama Cloud (hosted — key required)
|
|
116
|
+
# -------------------------------------------------------------------------
|
|
117
|
+
attr_accessor :ollama_cloud_api_key
|
|
118
|
+
attr_accessor :ollama_cloud_api_url
|
|
119
|
+
|
|
120
|
+
# -------------------------------------------------------------------------
|
|
121
|
+
# Hetzner Inference
|
|
122
|
+
# -------------------------------------------------------------------------
|
|
123
|
+
attr_accessor :hetzner_api_key
|
|
124
|
+
attr_accessor :hetzner_api_url
|
|
125
|
+
|
|
126
|
+
# -------------------------------------------------------------------------
|
|
127
|
+
# Deepgram (audio transcription only)
|
|
128
|
+
# -------------------------------------------------------------------------
|
|
129
|
+
attr_accessor :deepgram_api_key
|
|
130
|
+
attr_accessor :deepgram_api_url
|
|
131
|
+
|
|
132
|
+
# -------------------------------------------------------------------------
|
|
133
|
+
# ElevenLabs (audio transcription only)
|
|
134
|
+
# -------------------------------------------------------------------------
|
|
135
|
+
attr_accessor :elevenlabs_api_key
|
|
136
|
+
attr_accessor :elevenlabs_api_url
|
|
137
|
+
|
|
108
138
|
# -------------------------------------------------------------------------
|
|
109
139
|
# Custom providers
|
|
110
140
|
#
|
|
@@ -179,6 +209,21 @@ module ActiveHarness
|
|
|
179
209
|
|
|
180
210
|
@vercel_api_key = ENV["VERCEL_API_KEY"]
|
|
181
211
|
@vercel_api_url = "https://ai-gateway.vercel.sh/typesafe/v1/systemone"
|
|
212
|
+
|
|
213
|
+
@typesafe_api_key = ENV["TYPESAFE_API_KEY"]
|
|
214
|
+
@typesafe_api_url = "https://api.typesafe.ai/v1/systemone"
|
|
215
|
+
|
|
216
|
+
@ollama_cloud_api_key = ENV["OLLAMA_CLOUD_API_KEY"]
|
|
217
|
+
@ollama_cloud_api_url = "https://ollama.com/v1/chat/completions"
|
|
218
|
+
|
|
219
|
+
@hetzner_api_key = ENV["HETZNER_API_KEY"]
|
|
220
|
+
@hetzner_api_url = "https://inference.hetzner.com/api/v1/chat/completions"
|
|
221
|
+
|
|
222
|
+
@deepgram_api_key = ENV["DEEPGRAM_API_KEY"]
|
|
223
|
+
@deepgram_api_url = "https://api.deepgram.com/v1/listen"
|
|
224
|
+
|
|
225
|
+
@elevenlabs_api_key = ENV["ELEVENLABS_API_KEY"]
|
|
226
|
+
@elevenlabs_api_url = "https://api.elevenlabs.io/v1/speech-to-text"
|
|
182
227
|
end
|
|
183
228
|
end
|
|
184
229
|
end
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
require "uri"
|
|
2
|
+
|
|
3
|
+
module ActiveHarness
|
|
4
|
+
module Providers
|
|
5
|
+
module Audio
|
|
6
|
+
class Deepgram < Base
|
|
7
|
+
CONTENT_TYPES = {
|
|
8
|
+
"mp3" => "audio/mpeg",
|
|
9
|
+
"mp4" => "audio/mp4",
|
|
10
|
+
"m4a" => "audio/mp4",
|
|
11
|
+
"wav" => "audio/wav",
|
|
12
|
+
"flac" => "audio/flac",
|
|
13
|
+
"ogg" => "audio/ogg",
|
|
14
|
+
"opus" => "audio/ogg",
|
|
15
|
+
"webm" => "audio/webm",
|
|
16
|
+
"aac" => "audio/aac"
|
|
17
|
+
}.freeze
|
|
18
|
+
|
|
19
|
+
# @param model [String] e.g. "nova-3", "nova-2"
|
|
20
|
+
# @param audio_data [String] raw binary audio bytes
|
|
21
|
+
# @param audio_format [String] one of CONTENT_TYPES.keys
|
|
22
|
+
# @param language [String] BCP-47 code, e.g. "en" (optional)
|
|
23
|
+
#
|
|
24
|
+
# Synchronous pre-recorded transcription: the audio bytes are the raw
|
|
25
|
+
# request body (no multipart, no base64), options go in the query string.
|
|
26
|
+
# Deepgram bills by audio duration and reports no token usage, so
|
|
27
|
+
# `usage` is always nil.
|
|
28
|
+
def call(model:, audio_data:, audio_format:, language: nil, **_)
|
|
29
|
+
content_type = CONTENT_TYPES[audio_format]
|
|
30
|
+
unless content_type
|
|
31
|
+
raise Errors::InvalidRequestError,
|
|
32
|
+
"deepgram transcription does not support .#{audio_format} — use one of: #{CONTENT_TYPES.keys.join(', ')}"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
query = { "model" => model, "smart_format" => "true" }
|
|
36
|
+
query["language"] = language if language
|
|
37
|
+
|
|
38
|
+
url = URI(config.deepgram_api_url)
|
|
39
|
+
url.query = URI.encode_www_form(query)
|
|
40
|
+
|
|
41
|
+
headers = { "Content-Type" => content_type, "Authorization" => "Token #{api_key}" }
|
|
42
|
+
|
|
43
|
+
raw = HTTP.post(url, headers: headers, body: audio_data, timeout: 90)
|
|
44
|
+
data = parse!(raw)
|
|
45
|
+
handle_error!(data)
|
|
46
|
+
|
|
47
|
+
text = data.dig("results", "channels", 0, "alternatives", 0, "transcript")
|
|
48
|
+
raise Errors::ProviderError, "No transcription text in response: #{data.keys}" if text.nil?
|
|
49
|
+
|
|
50
|
+
{ content: text, provider: :deepgram, model: model, usage: nil }
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
def api_key
|
|
56
|
+
key = config.deepgram_api_key.to_s
|
|
57
|
+
raise Errors::InvalidApiKeyError, "deepgram_api_key is not configured" if key.empty?
|
|
58
|
+
key
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Deepgram errors look like { "err_code": "INVALID_AUTH", "err_msg": "..." }.
|
|
62
|
+
def handle_error!(data)
|
|
63
|
+
return unless data["err_code"] || data["err_msg"]
|
|
64
|
+
|
|
65
|
+
msg = data["err_msg"].to_s
|
|
66
|
+
code = data["err_code"].to_s
|
|
67
|
+
meta = data["request_id"] ? { request_id: data["request_id"] } : nil
|
|
68
|
+
|
|
69
|
+
case code
|
|
70
|
+
when "INVALID_AUTH", "INSUFFICIENT_PERMISSIONS"
|
|
71
|
+
raise Errors::InvalidApiKeyError.new(msg, error_code: code, metadata: meta)
|
|
72
|
+
when "TOO_MANY_REQUESTS", "RATE_LIMIT"
|
|
73
|
+
raise Errors::RateLimitError.new(msg, error_code: code, metadata: meta)
|
|
74
|
+
else
|
|
75
|
+
raise Errors::InvalidRequestError.new(msg, error_code: code, metadata: meta)
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
require "uri"
|
|
2
|
+
require "securerandom"
|
|
3
|
+
|
|
4
|
+
module ActiveHarness
|
|
5
|
+
module Providers
|
|
6
|
+
module Audio
|
|
7
|
+
class ElevenLabs < Base
|
|
8
|
+
CONTENT_TYPES = {
|
|
9
|
+
"mp3" => "audio/mpeg",
|
|
10
|
+
"mp4" => "audio/mp4",
|
|
11
|
+
"m4a" => "audio/mp4",
|
|
12
|
+
"wav" => "audio/wav",
|
|
13
|
+
"flac" => "audio/flac",
|
|
14
|
+
"ogg" => "audio/ogg",
|
|
15
|
+
"webm" => "audio/webm",
|
|
16
|
+
"aac" => "audio/aac"
|
|
17
|
+
}.freeze
|
|
18
|
+
|
|
19
|
+
# @param model [String] e.g. "scribe_v1", "scribe_v2"
|
|
20
|
+
# @param audio_data [String] raw binary audio bytes
|
|
21
|
+
# @param audio_format [String] one of CONTENT_TYPES.keys
|
|
22
|
+
# @param language [String] ISO-639-1/3 code (optional — auto-detected if omitted)
|
|
23
|
+
#
|
|
24
|
+
# Synchronous multipart/form-data request. ElevenLabs reports no token
|
|
25
|
+
# usage for speech-to-text, so `usage` is always nil.
|
|
26
|
+
def call(model:, audio_data:, audio_format:, language: nil, **_)
|
|
27
|
+
content_type = CONTENT_TYPES[audio_format]
|
|
28
|
+
unless content_type
|
|
29
|
+
raise Errors::InvalidRequestError,
|
|
30
|
+
"elevenlabs transcription does not support .#{audio_format} — use one of: #{CONTENT_TYPES.keys.join(', ')}"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
boundary = SecureRandom.hex(16)
|
|
34
|
+
fields = { "model_id" => model }
|
|
35
|
+
fields["language_code"] = language if language
|
|
36
|
+
|
|
37
|
+
headers = {
|
|
38
|
+
"Content-Type" => "multipart/form-data; boundary=#{boundary}",
|
|
39
|
+
"xi-api-key" => api_key
|
|
40
|
+
}
|
|
41
|
+
body = build_multipart_body(boundary, fields, audio_data, "audio.#{audio_format}", content_type)
|
|
42
|
+
|
|
43
|
+
raw = HTTP.post(URI(config.elevenlabs_api_url), headers: headers, body: body, timeout: 90)
|
|
44
|
+
data = parse!(raw)
|
|
45
|
+
handle_error!(data)
|
|
46
|
+
|
|
47
|
+
text = data["text"]
|
|
48
|
+
raise Errors::ProviderError, "No transcription text in response: #{data.keys}" if text.nil?
|
|
49
|
+
|
|
50
|
+
{ content: text, provider: :elevenlabs, model: model, usage: nil }
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
def build_multipart_body(boundary, fields, file_data, filename, content_type)
|
|
56
|
+
body = +""
|
|
57
|
+
fields.each do |name, value|
|
|
58
|
+
body << "--#{boundary}\r\n"
|
|
59
|
+
body << "Content-Disposition: form-data; name=\"#{name}\"\r\n\r\n"
|
|
60
|
+
body << "#{value}\r\n"
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
body << "--#{boundary}\r\n"
|
|
64
|
+
body << "Content-Disposition: form-data; name=\"file\"; filename=\"#{filename}\"\r\n"
|
|
65
|
+
body << "Content-Type: #{content_type}\r\n\r\n"
|
|
66
|
+
body << file_data
|
|
67
|
+
body << "\r\n--#{boundary}--\r\n"
|
|
68
|
+
body
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def api_key
|
|
72
|
+
key = config.elevenlabs_api_key.to_s
|
|
73
|
+
raise Errors::InvalidApiKeyError, "elevenlabs_api_key is not configured" if key.empty?
|
|
74
|
+
key
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Errors are { "detail": { "status": "...", "message": "..." } }, or for
|
|
78
|
+
# request-validation failures { "detail": [ { "msg": "...", ... } ] }.
|
|
79
|
+
def handle_error!(data)
|
|
80
|
+
detail = data["detail"]
|
|
81
|
+
return unless detail
|
|
82
|
+
|
|
83
|
+
if detail.is_a?(Hash)
|
|
84
|
+
msg = detail["message"].to_s
|
|
85
|
+
code = detail["status"].to_s
|
|
86
|
+
else
|
|
87
|
+
msg = Array(detail).map { |d| d.is_a?(Hash) ? d["msg"] : d }.join("; ")
|
|
88
|
+
code = "validation_error"
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
case code
|
|
92
|
+
when "invalid_api_key", "unauthorized", "needs_authorization"
|
|
93
|
+
raise Errors::InvalidApiKeyError.new(msg, error_code: code)
|
|
94
|
+
when "too_many_concurrent_requests", "rate_limit_exceeded", "system_busy"
|
|
95
|
+
raise Errors::RateLimitError.new(msg, error_code: code)
|
|
96
|
+
else
|
|
97
|
+
raise Errors::InvalidRequestError.new(msg, error_code: code)
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
require "uri"
|
|
2
|
+
|
|
3
|
+
module ActiveHarness
|
|
4
|
+
module Providers
|
|
5
|
+
# Hetzner Inference — OpenAI-compatible Chat Completions API.
|
|
6
|
+
# https://inference.hetzner.com
|
|
7
|
+
#
|
|
8
|
+
# Hetzner's servers cannot fetch image URLs, so image inputs must be sent
|
|
9
|
+
# inline (data URIs); plain-text chat is unaffected.
|
|
10
|
+
#
|
|
11
|
+
# Example:
|
|
12
|
+
# model do
|
|
13
|
+
# use provider: :hetzner, model: "meta-llama/Llama-3.3-70B-Instruct"
|
|
14
|
+
# end
|
|
15
|
+
class Hetzner < Base
|
|
16
|
+
def call(model:, messages:, temperature: 0.7, stream: nil)
|
|
17
|
+
headers = { "Content-Type" => "application/json", "Authorization" => "Bearer #{api_key}" }
|
|
18
|
+
body = { model: model, messages: messages, temperature: temperature }
|
|
19
|
+
|
|
20
|
+
return call_streaming(url: config.hetzner_api_url, headers: headers, body: body, stream: stream, provider: :hetzner, model: model) if stream
|
|
21
|
+
|
|
22
|
+
raw = post_json(URI(config.hetzner_api_url), headers: headers, body: body)
|
|
23
|
+
data = parse!(raw)
|
|
24
|
+
handle_error!(data)
|
|
25
|
+
|
|
26
|
+
{
|
|
27
|
+
content: data.dig("choices", 0, "message", "content").to_s.strip,
|
|
28
|
+
provider: :hetzner,
|
|
29
|
+
model: data["model"] || model,
|
|
30
|
+
usage: extract_usage_openai(data)
|
|
31
|
+
}
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
private
|
|
35
|
+
|
|
36
|
+
def api_key
|
|
37
|
+
key = config.hetzner_api_key.to_s
|
|
38
|
+
raise Errors::InvalidApiKeyError, "hetzner_api_key is not configured" if key.empty?
|
|
39
|
+
key
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def handle_error!(data)
|
|
43
|
+
return unless data["error"]
|
|
44
|
+
|
|
45
|
+
err = data["error"]
|
|
46
|
+
msg = err.is_a?(Hash) ? err["message"].to_s : err.to_s
|
|
47
|
+
code = err.is_a?(Hash) ? err["code"].to_s : ""
|
|
48
|
+
type = err.is_a?(Hash) ? err["type"].to_s : ""
|
|
49
|
+
metadata = err.is_a?(Hash) ? err.reject { |k, _| %w[message code type].include?(k) } : {}
|
|
50
|
+
metadata = nil if metadata.empty?
|
|
51
|
+
|
|
52
|
+
case code
|
|
53
|
+
when "401", "invalid_api_key", "unauthorized"
|
|
54
|
+
raise Errors::InvalidApiKeyError.new(msg, error_code: code, metadata: metadata)
|
|
55
|
+
when "429", "rate_limit_exceeded"
|
|
56
|
+
raise Errors::RateLimitError.new(msg, error_code: code, metadata: metadata)
|
|
57
|
+
when "500", "502", "503", "504"
|
|
58
|
+
raise Errors::ProviderUnavailableError.new(msg, error_code: code, metadata: metadata)
|
|
59
|
+
else
|
|
60
|
+
if type == "server_error"
|
|
61
|
+
raise Errors::ServerError.new(msg, error_code: code, metadata: metadata)
|
|
62
|
+
else
|
|
63
|
+
raise Errors::InvalidRequestError.new(msg, error_code: code, metadata: metadata)
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
require "uri"
|
|
2
|
+
|
|
3
|
+
module ActiveHarness
|
|
4
|
+
module Providers
|
|
5
|
+
# Ollama Cloud — hosted Ollama models, OpenAI-compatible API (key required).
|
|
6
|
+
# https://docs.ollama.com/cloud
|
|
7
|
+
#
|
|
8
|
+
# For a local Ollama server use provider: :ollama instead.
|
|
9
|
+
#
|
|
10
|
+
# Example:
|
|
11
|
+
# model do
|
|
12
|
+
# use provider: :ollama_cloud, model: "gpt-oss:120b"
|
|
13
|
+
# end
|
|
14
|
+
class OllamaCloud < Base
|
|
15
|
+
def call(model:, messages:, temperature: 0.7, stream: nil)
|
|
16
|
+
headers = { "Content-Type" => "application/json", "Authorization" => "Bearer #{api_key}" }
|
|
17
|
+
body = { model: model, messages: messages, temperature: temperature }
|
|
18
|
+
|
|
19
|
+
return call_streaming(url: config.ollama_cloud_api_url, headers: headers, body: body, stream: stream, provider: :ollama_cloud, model: model) if stream
|
|
20
|
+
|
|
21
|
+
raw = post_json(URI(config.ollama_cloud_api_url), headers: headers, body: body)
|
|
22
|
+
data = parse!(raw)
|
|
23
|
+
handle_error!(data)
|
|
24
|
+
|
|
25
|
+
{
|
|
26
|
+
content: data.dig("choices", 0, "message", "content").to_s.strip,
|
|
27
|
+
provider: :ollama_cloud,
|
|
28
|
+
model: data["model"] || model,
|
|
29
|
+
usage: extract_usage_openai(data)
|
|
30
|
+
}
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def api_key
|
|
36
|
+
key = config.ollama_cloud_api_key.to_s
|
|
37
|
+
raise Errors::InvalidApiKeyError, "ollama_cloud_api_key is not configured" if key.empty?
|
|
38
|
+
key
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def handle_error!(data)
|
|
42
|
+
return unless data["error"]
|
|
43
|
+
|
|
44
|
+
err = data["error"]
|
|
45
|
+
msg = err.is_a?(Hash) ? err["message"].to_s : err.to_s
|
|
46
|
+
code = err.is_a?(Hash) ? err["code"].to_s : ""
|
|
47
|
+
metadata = err.is_a?(Hash) ? err.reject { |k, _| %w[message code].include?(k) } : {}
|
|
48
|
+
metadata = nil if metadata.empty?
|
|
49
|
+
|
|
50
|
+
case code
|
|
51
|
+
when "401", "invalid_api_key", "unauthorized"
|
|
52
|
+
raise Errors::InvalidApiKeyError.new(msg, error_code: code, metadata: metadata)
|
|
53
|
+
when "429", "rate_limit_exceeded"
|
|
54
|
+
raise Errors::RateLimitError.new(msg, error_code: code, metadata: metadata)
|
|
55
|
+
when "500", "502", "503", "504"
|
|
56
|
+
raise Errors::ProviderUnavailableError.new(msg, error_code: code, metadata: metadata)
|
|
57
|
+
else
|
|
58
|
+
raise Errors::InvalidRequestError.new(msg, error_code: code, metadata: metadata)
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
require "uri"
|
|
2
|
+
|
|
3
|
+
module ActiveHarness
|
|
4
|
+
module Providers
|
|
5
|
+
# TypeSafe — direct access to Jev, TypeSafe's "System One" evaluation model
|
|
6
|
+
# (https://api.typesafe.ai/v1/systemone). Same request/response contract as
|
|
7
|
+
# provider: :vercel (which reaches the same model through Vercel AI Gateway),
|
|
8
|
+
# so `questions:` and everything else in Providers::Vercel apply unchanged.
|
|
9
|
+
#
|
|
10
|
+
# Also works with any Jev-compatible server: point `typesafe_api_url` at it.
|
|
11
|
+
#
|
|
12
|
+
# model do
|
|
13
|
+
# use provider: :typesafe, model: "jev-latest", questions: {
|
|
14
|
+
# urgent: { type: "noul", instructions: "Is this urgent?" }
|
|
15
|
+
# }
|
|
16
|
+
# end
|
|
17
|
+
class TypeSafe < Vercel
|
|
18
|
+
private
|
|
19
|
+
|
|
20
|
+
def provider_name
|
|
21
|
+
:typesafe
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def endpoint_url
|
|
25
|
+
config.typesafe_api_url
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def api_key
|
|
29
|
+
key = config.typesafe_api_key.to_s
|
|
30
|
+
raise Errors::InvalidApiKeyError, "typesafe_api_key is not configured" if key.empty?
|
|
31
|
+
key
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -44,7 +44,7 @@ module ActiveHarness
|
|
|
44
44
|
DEFAULT_INSTRUCTIONS = "Evaluate the given state.".freeze
|
|
45
45
|
|
|
46
46
|
def call(model:, messages:, temperature: nil, stream: nil, questions: nil)
|
|
47
|
-
raise Errors::InvalidRequestError, "provider:
|
|
47
|
+
raise Errors::InvalidRequestError, "provider: :#{provider_name} (Jev) does not support token streaming" if stream
|
|
48
48
|
|
|
49
49
|
state = messages.reverse.find { |m| m[:role] == "user" }&.fetch(:content, nil).to_s
|
|
50
50
|
|
|
@@ -58,13 +58,13 @@ module ActiveHarness
|
|
|
58
58
|
questions: questions || default_questions(messages)
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
-
raw = post_json(URI(
|
|
61
|
+
raw = post_json(URI(endpoint_url), headers: headers, body: body)
|
|
62
62
|
data = parse!(raw)
|
|
63
63
|
handle_error!(data)
|
|
64
64
|
|
|
65
65
|
{
|
|
66
66
|
content: JSON.pretty_generate(data["answers"]),
|
|
67
|
-
provider:
|
|
67
|
+
provider: provider_name,
|
|
68
68
|
model: data["model"] || model,
|
|
69
69
|
usage: extract_usage(data)
|
|
70
70
|
}
|
|
@@ -78,6 +78,15 @@ module ActiveHarness
|
|
|
78
78
|
{ answer: { type: "noul", instructions: instructions } }
|
|
79
79
|
end
|
|
80
80
|
|
|
81
|
+
# Subclasses (e.g. TypeSafe) reuse everything above and only swap these.
|
|
82
|
+
def provider_name
|
|
83
|
+
:vercel
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def endpoint_url
|
|
87
|
+
config.vercel_api_url
|
|
88
|
+
end
|
|
89
|
+
|
|
81
90
|
def api_key
|
|
82
91
|
key = config.vercel_api_key.to_s
|
|
83
92
|
raise Errors::InvalidApiKeyError, "vercel_api_key is not configured" if key.empty?
|
|
@@ -95,12 +104,26 @@ module ActiveHarness
|
|
|
95
104
|
{ input_tokens: input, output_tokens: output, total_tokens: input + output }
|
|
96
105
|
end
|
|
97
106
|
|
|
98
|
-
#
|
|
107
|
+
# TypeSafe's own API reports { "detail": String | { "message": ... } |
|
|
108
|
+
# [ { "loc": [...], "msg": "..." } ] } (FastAPI-style).
|
|
109
|
+
def detail_message(detail)
|
|
110
|
+
case detail
|
|
111
|
+
when Hash then detail["message"].to_s
|
|
112
|
+
when Array then detail.map { |e| e.is_a?(Hash) ? [Array(e["loc"]).join("."), e["msg"]].reject { |x| x.to_s.empty? }.join(": ") : e.to_s }.join("; ")
|
|
113
|
+
else detail.to_s
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Error shapes seen in practice:
|
|
118
|
+
# - TypeSafe direct API: { "detail": ... } (see detail_message)
|
|
99
119
|
# - TypeSafe request-validation errors: { "message": "...", "error_type": "..." }
|
|
100
120
|
# - AI Gateway account/billing errors (OpenAI-style, nested): { "error": { "message": "...", "type": "..." } }
|
|
101
121
|
def handle_error!(data)
|
|
102
122
|
msg, type =
|
|
103
|
-
if data
|
|
123
|
+
if data.key?("detail")
|
|
124
|
+
msg = detail_message(data["detail"])
|
|
125
|
+
[msg, msg =~ /api key|unauthori|authenticat|invalid token|credentials/i ? "unauthorized" : "invalid_request"]
|
|
126
|
+
elsif data["message"] && data["error_type"]
|
|
104
127
|
[data["message"].to_s, data["error_type"].to_s]
|
|
105
128
|
elsif data["error"].is_a?(Hash)
|
|
106
129
|
[data["error"]["message"].to_s, data["error"]["type"].to_s]
|
|
@@ -36,7 +36,10 @@ module ActiveHarness
|
|
|
36
36
|
bedrock: -> { Providers::Bedrock.new },
|
|
37
37
|
vertexai: -> { Providers::VertexAI.new },
|
|
38
38
|
custom: -> { Providers::Custom.new },
|
|
39
|
-
vercel: -> { Providers::Vercel.new }
|
|
39
|
+
vercel: -> { Providers::Vercel.new },
|
|
40
|
+
typesafe: -> { Providers::TypeSafe.new },
|
|
41
|
+
ollama_cloud: -> { Providers::OllamaCloud.new },
|
|
42
|
+
hetzner: -> { Providers::Hetzner.new }
|
|
40
43
|
}.freeze
|
|
41
44
|
|
|
42
45
|
IMAGE_PROVIDERS = {
|
|
@@ -46,7 +49,9 @@ module ActiveHarness
|
|
|
46
49
|
|
|
47
50
|
TRANSCRIPTION_PROVIDERS = {
|
|
48
51
|
openai: -> { Providers::Audio::OpenAI.new },
|
|
49
|
-
openrouter: -> { Providers::Audio::OpenRouter.new }
|
|
52
|
+
openrouter: -> { Providers::Audio::OpenRouter.new },
|
|
53
|
+
deepgram: -> { Providers::Audio::Deepgram.new },
|
|
54
|
+
elevenlabs: -> { Providers::Audio::ElevenLabs.new }
|
|
50
55
|
}.freeze
|
|
51
56
|
|
|
52
57
|
private
|
data/lib/active_harness.rb
CHANGED
|
@@ -22,10 +22,15 @@ require_relative "active_harness/providers/bedrock"
|
|
|
22
22
|
require_relative "active_harness/providers/vertexai"
|
|
23
23
|
require_relative "active_harness/providers/custom"
|
|
24
24
|
require_relative "active_harness/providers/vercel"
|
|
25
|
+
require_relative "active_harness/providers/typesafe"
|
|
26
|
+
require_relative "active_harness/providers/ollama_cloud"
|
|
27
|
+
require_relative "active_harness/providers/hetzner"
|
|
25
28
|
require_relative "active_harness/providers/images/openai"
|
|
26
29
|
require_relative "active_harness/providers/images/openrouter"
|
|
27
30
|
require_relative "active_harness/providers/audio/openai"
|
|
28
31
|
require_relative "active_harness/providers/audio/openrouter"
|
|
32
|
+
require_relative "active_harness/providers/audio/deepgram"
|
|
33
|
+
require_relative "active_harness/providers/audio/elevenlabs"
|
|
29
34
|
require "active_harness_pricing"
|
|
30
35
|
require_relative "active_harness/memory"
|
|
31
36
|
require_relative "active_harness/request"
|
|
@@ -35,7 +40,7 @@ require_relative "active_harness/pipeline"
|
|
|
35
40
|
require_relative "active_harness/railtie" if defined?(Rails::Railtie)
|
|
36
41
|
|
|
37
42
|
module ActiveHarness
|
|
38
|
-
VERSION = "0.3.
|
|
43
|
+
VERSION = "0.3.6"
|
|
39
44
|
|
|
40
45
|
class << self
|
|
41
46
|
# Configure ActiveHarness.
|
|
@@ -81,6 +81,30 @@ ActiveHarness.configure do |config|
|
|
|
81
81
|
# config.azure_ai_auth_token = ENV["AZURE_AI_AUTH_TOKEN"] # OAuth bearer (alternative)
|
|
82
82
|
# config.azure_api_version = "2024-05-01-preview"
|
|
83
83
|
|
|
84
|
+
# ---------------------------------------------------------------------------
|
|
85
|
+
# TypeSafe — Jev (System One) evaluation model, direct API
|
|
86
|
+
# ---------------------------------------------------------------------------
|
|
87
|
+
# config.typesafe_api_key = ENV["TYPESAFE_API_KEY"]
|
|
88
|
+
# config.typesafe_api_url = "https://api.typesafe.ai/v1/systemone"
|
|
89
|
+
|
|
90
|
+
# ---------------------------------------------------------------------------
|
|
91
|
+
# Ollama Cloud (hosted — API key required)
|
|
92
|
+
# ---------------------------------------------------------------------------
|
|
93
|
+
# config.ollama_cloud_api_key = ENV["OLLAMA_CLOUD_API_KEY"]
|
|
94
|
+
# config.ollama_cloud_api_url = "https://ollama.com/v1/chat/completions"
|
|
95
|
+
|
|
96
|
+
# ---------------------------------------------------------------------------
|
|
97
|
+
# Hetzner Inference
|
|
98
|
+
# ---------------------------------------------------------------------------
|
|
99
|
+
# config.hetzner_api_key = ENV["HETZNER_API_KEY"]
|
|
100
|
+
# config.hetzner_api_url = "https://inference.hetzner.com/api/v1/chat/completions"
|
|
101
|
+
|
|
102
|
+
# ---------------------------------------------------------------------------
|
|
103
|
+
# Deepgram / ElevenLabs (audio transcription only)
|
|
104
|
+
# ---------------------------------------------------------------------------
|
|
105
|
+
# config.deepgram_api_key = ENV["DEEPGRAM_API_KEY"]
|
|
106
|
+
# config.elevenlabs_api_key = ENV["ELEVENLABS_API_KEY"]
|
|
107
|
+
|
|
84
108
|
# ---------------------------------------------------------------------------
|
|
85
109
|
# Custom providers — any OpenAI-compatible endpoint
|
|
86
110
|
# Register as many as you need under arbitrary names.
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: active_harness
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.6
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- the-teacher
|
|
@@ -63,6 +63,8 @@ files:
|
|
|
63
63
|
- lib/active_harness/pipeline/step.rb
|
|
64
64
|
- lib/active_harness/providers/PROVIDER_CONTRACT.md
|
|
65
65
|
- lib/active_harness/providers/anthropic.rb
|
|
66
|
+
- lib/active_harness/providers/audio/deepgram.rb
|
|
67
|
+
- lib/active_harness/providers/audio/elevenlabs.rb
|
|
66
68
|
- lib/active_harness/providers/audio/openai.rb
|
|
67
69
|
- lib/active_harness/providers/audio/openrouter.rb
|
|
68
70
|
- lib/active_harness/providers/azure.rb
|
|
@@ -73,13 +75,16 @@ files:
|
|
|
73
75
|
- lib/active_harness/providers/gemini.rb
|
|
74
76
|
- lib/active_harness/providers/gpustack.rb
|
|
75
77
|
- lib/active_harness/providers/groq.rb
|
|
78
|
+
- lib/active_harness/providers/hetzner.rb
|
|
76
79
|
- lib/active_harness/providers/images/openai.rb
|
|
77
80
|
- lib/active_harness/providers/images/openrouter.rb
|
|
78
81
|
- lib/active_harness/providers/mistral.rb
|
|
79
82
|
- lib/active_harness/providers/ollama.rb
|
|
83
|
+
- lib/active_harness/providers/ollama_cloud.rb
|
|
80
84
|
- lib/active_harness/providers/openai.rb
|
|
81
85
|
- lib/active_harness/providers/openrouter.rb
|
|
82
86
|
- lib/active_harness/providers/perplexity.rb
|
|
87
|
+
- lib/active_harness/providers/typesafe.rb
|
|
83
88
|
- lib/active_harness/providers/vercel.rb
|
|
84
89
|
- lib/active_harness/providers/vertexai.rb
|
|
85
90
|
- lib/active_harness/providers/xai.rb
|