typesafe-jev 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/lib/typesafe/answer.rb +134 -0
- data/lib/typesafe/choice_answer.rb +53 -0
- data/lib/typesafe/client.rb +8 -5
- data/lib/typesafe/jev.rb +5 -4
- data/lib/typesafe/noul_answer.rb +34 -0
- data/lib/typesafe/response.rb +127 -0
- data/lib/typesafe/score_answer.rb +84 -0
- data/lib/typesafe/usage.rb +64 -0
- data/lib/typesafe/version.rb +1 -1
- data/lib/typesafe.rb +9 -0
- metadata +11 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 331918548db2a5fdbd2f3dc3c2ac3bf650ff80d530e1bf5df205ddabede2299b
|
|
4
|
+
data.tar.gz: ce1c7bef8487e643d53fcfa8cb338fdb4abd26a085c19478e50cb53eac01d336
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 18dee0e01b1df600771dcc3eff223841f7074df76f309494dbdfc3b200ff6db4cc6627c6ce8cd0a3934665302385b07986e6a66f3bd96b51af3126f61d95e823
|
|
7
|
+
data.tar.gz: cb4261224cfec035eed1b1bac36a02f84cc388a9ff05997aff7d9bc59a35a31108465eb6bd277cd94c0296408321ae17c8c8883b41718b1d95a9b363236ec120
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.6.0
|
|
4
|
+
|
|
5
|
+
- **Breaking**: `Typesafe::Client#evaluate` (and `Typesafe::Jev.evaluate`) now returns a typed `Typesafe::Response` instead of the raw parsed Hash; answers come back as `NoulAnswer`/`ChoiceAnswer`/`ScoreAnswer` objects accessible via `response[question_id]` (String or Symbol). Use `response.to_h` for the previous raw-Hash behavior.
|
|
6
|
+
|
|
7
|
+
## 0.5.0
|
|
8
|
+
|
|
9
|
+
- Add response classes mirroring the question classes: abstract `Typesafe::Answer` base with `Answer.from_h` dispatching on the `type` tag, plus `Typesafe::NoulAnswer` (`noul`), `Typesafe::ChoiceAnswer` (`choice`, `probabilities`, `confidence`) and `Typesafe::ScoreAnswer` (`score`, `legend`, `probabilities`, `confidence`). Frozen, immutable value objects that validate their inputs and serialize via `#to_h`/`#to_json`.
|
|
10
|
+
- Add `Typesafe::Usage` (token counts) and `Typesafe::Response` (`model`, `answers`, `usage`), with `Response.from_json`/`from_h` to parse an API response body into typed Answer objects; `Response#[]` accepts String or Symbol question ids.
|
|
11
|
+
|
|
12
|
+
## 0.4.1
|
|
13
|
+
|
|
14
|
+
- Fix gemspec metadata: point `source_code_uri`/`changelog_uri` and author/email at the `dtheofr` GitHub account.
|
|
15
|
+
|
|
3
16
|
## 0.4.0
|
|
4
17
|
|
|
5
18
|
- Add `Typesafe::Jev`, a `Typesafe::Client` subclass with the model pinned to `jev-latest`: `Jev.new(api_key: nil).evaluate(...)` plus a one-shot `Typesafe::Jev.evaluate(state:, questions:, api_key: nil)`.
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Typesafe
|
|
4
|
+
# Abstract base class for the three answer types returned by the TypeSafe
|
|
5
|
+
# System One API: {NoulAnswer}, {ChoiceAnswer}, and {ScoreAnswer}.
|
|
6
|
+
#
|
|
7
|
+
# An answer is an immutable value object mirroring its question: it carries
|
|
8
|
+
# what the model returned for one question, but not the question id — that
|
|
9
|
+
# is the Hash key under which the answer appears in a {Response}.
|
|
10
|
+
#
|
|
11
|
+
# Use {Answer.from_h} (or {Response.from_json}) to build typed answers from
|
|
12
|
+
# a parsed response body:
|
|
13
|
+
#
|
|
14
|
+
# Typesafe::Answer.from_h({ "type" => "noul", "noul" => 0.95 })
|
|
15
|
+
# # => #<Typesafe::NoulAnswer @noul=0.95>
|
|
16
|
+
class Answer
|
|
17
|
+
# Builds the typed answer matching +hash+'s +type+ tag.
|
|
18
|
+
#
|
|
19
|
+
# @param hash [Hash] one entry of the API response's +answers+ map, with
|
|
20
|
+
# String or Symbol keys.
|
|
21
|
+
# @return [NoulAnswer, ChoiceAnswer, ScoreAnswer]
|
|
22
|
+
# @raise [ArgumentError] if +hash+ is not a Hash or its +type+ is missing
|
|
23
|
+
# or unknown.
|
|
24
|
+
def self.from_h(hash)
|
|
25
|
+
unless hash.is_a?(Hash)
|
|
26
|
+
raise ArgumentError, "answer must be a Hash, got #{hash.inspect}"
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
type = hash["type"] || hash[:type]
|
|
30
|
+
case type
|
|
31
|
+
when "noul" then NoulAnswer.from_h(hash)
|
|
32
|
+
when "choice" then ChoiceAnswer.from_h(hash)
|
|
33
|
+
when "score" then ScoreAnswer.from_h(hash)
|
|
34
|
+
else
|
|
35
|
+
raise ArgumentError,
|
|
36
|
+
"unknown answer type #{type.inspect} (expected \"noul\", \"choice\", or \"score\")"
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# @return [String] the API answer type tag ("noul", "choice", or "score").
|
|
41
|
+
# @raise [NotImplementedError] on the abstract base class.
|
|
42
|
+
def type
|
|
43
|
+
raise NotImplementedError, "#{self.class} must implement #type"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# @return [Hash] the answer shape returned by the TypeSafe API.
|
|
47
|
+
def to_h
|
|
48
|
+
{ type: type }
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# @return [String] the answer serialized as JSON.
|
|
52
|
+
def to_json(state = nil)
|
|
53
|
+
JSON.generate(to_h, state)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Answers compare equal when they are of the same class and serialize
|
|
57
|
+
# to the same API shape.
|
|
58
|
+
def ==(other)
|
|
59
|
+
other.class == self.class && other.to_h == to_h
|
|
60
|
+
end
|
|
61
|
+
alias eql? ==
|
|
62
|
+
|
|
63
|
+
def hash
|
|
64
|
+
[self.class, to_h].hash
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
private
|
|
68
|
+
|
|
69
|
+
# Validates that +value+ is a non-empty String and returns a frozen copy.
|
|
70
|
+
def validate_string(value, name)
|
|
71
|
+
unless value.is_a?(String) && !value.strip.empty?
|
|
72
|
+
raise ArgumentError, "#{name} must be a non-empty String"
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
value.dup.freeze
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Validates that +value+ is a finite Numeric, optionally within +range+.
|
|
79
|
+
def validate_number(value, name, range: nil)
|
|
80
|
+
unless value.is_a?(Numeric) && value.finite?
|
|
81
|
+
raise ArgumentError, "#{name} must be a finite Numeric, got #{value.inspect}"
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
if range && !range.cover?(value)
|
|
85
|
+
raise ArgumentError, "#{name} must be between #{range.min} and #{range.max}, got #{value.inspect}"
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
value
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Validates that +value+ is a probability between 0 and 1.
|
|
92
|
+
def validate_probability(value, name)
|
|
93
|
+
validate_number(value, name, range: 0.0..1.0)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Validates a probability distribution: a non-empty Hash with String or
|
|
97
|
+
# Symbol keys mapping to probabilities in 0..1 that sum to 1. When
|
|
98
|
+
# +allowed_keys+ is given, the keys must match it exactly.
|
|
99
|
+
def normalize_probabilities(probabilities, allowed_keys: nil)
|
|
100
|
+
unless probabilities.is_a?(Hash) && !probabilities.empty?
|
|
101
|
+
raise ArgumentError, "probabilities must be a non-empty Hash of option or level keys to probabilities"
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
normalized = probabilities.each_with_object({}) do |(key, probability), result|
|
|
105
|
+
unless key.is_a?(String) || key.is_a?(Symbol)
|
|
106
|
+
raise ArgumentError, "probabilities keys must be Strings or Symbols, got #{key.inspect}"
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
result[key.is_a?(String) ? key.dup.freeze : key] =
|
|
110
|
+
validate_probability(probability, "probabilities[#{key.inspect}]")
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
sum = normalized.values.sum
|
|
114
|
+
unless (sum - 1.0).abs <= 1e-6
|
|
115
|
+
raise ArgumentError, "probabilities must sum to 1, got #{sum}"
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
if allowed_keys
|
|
119
|
+
expected = allowed_keys.map(&:to_s)
|
|
120
|
+
actual = normalized.keys.map(&:to_s)
|
|
121
|
+
missing = expected - actual
|
|
122
|
+
unexpected = actual - expected
|
|
123
|
+
unless missing.empty? && unexpected.empty?
|
|
124
|
+
problems = []
|
|
125
|
+
problems << "missing #{missing.map(&:inspect).join(", ")}" unless missing.empty?
|
|
126
|
+
problems << "unexpected #{unexpected.map(&:inspect).join(", ")}" unless unexpected.empty?
|
|
127
|
+
raise ArgumentError, "probabilities keys must match the legend keys (#{problems.join("; ")})"
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
normalized.freeze
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Typesafe
|
|
4
|
+
# The answer to a {Choice} question: the selected option, the probability
|
|
5
|
+
# distribution across every option, and a confidence between 0 and 1.
|
|
6
|
+
#
|
|
7
|
+
# Typesafe::ChoiceAnswer.new(
|
|
8
|
+
# choice: "billing",
|
|
9
|
+
# probabilities: { "billing" => 0.88, "technical" => 0.12, "sales" => 0.0 },
|
|
10
|
+
# confidence: 0.81
|
|
11
|
+
# )
|
|
12
|
+
class ChoiceAnswer < Answer
|
|
13
|
+
# @return [String] the highest-probability option.
|
|
14
|
+
attr_reader :choice
|
|
15
|
+
|
|
16
|
+
# @return [Hash] every option mapped to its probability.
|
|
17
|
+
attr_reader :probabilities
|
|
18
|
+
|
|
19
|
+
# @return [Numeric] how certain the model is, between 0 and 1.
|
|
20
|
+
attr_reader :confidence
|
|
21
|
+
|
|
22
|
+
# @param choice [String] the selected option.
|
|
23
|
+
# @param probabilities [Hash] option keys (String or Symbol) mapped to
|
|
24
|
+
# probabilities in 0..1; the values must sum to 1.
|
|
25
|
+
# @param confidence [Numeric] the model's confidence, between 0 and 1.
|
|
26
|
+
# @raise [ArgumentError] if any attribute is invalid.
|
|
27
|
+
def initialize(choice:, probabilities:, confidence:)
|
|
28
|
+
@choice = validate_string(choice, "choice")
|
|
29
|
+
@probabilities = normalize_probabilities(probabilities)
|
|
30
|
+
@confidence = validate_probability(confidence, "confidence")
|
|
31
|
+
freeze
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# @param hash [Hash] a +choice+ answer entry from the API response, with
|
|
35
|
+
# String or Symbol keys.
|
|
36
|
+
# @return [ChoiceAnswer]
|
|
37
|
+
def self.from_h(hash)
|
|
38
|
+
new(
|
|
39
|
+
choice: hash["choice"] || hash[:choice],
|
|
40
|
+
probabilities: hash["probabilities"] || hash[:probabilities],
|
|
41
|
+
confidence: hash["confidence"] || hash[:confidence]
|
|
42
|
+
)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def type
|
|
46
|
+
"choice"
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def to_h
|
|
50
|
+
{ type: type, choice: choice, probabilities: probabilities, confidence: confidence }
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
data/lib/typesafe/client.rb
CHANGED
|
@@ -40,8 +40,9 @@ module Typesafe
|
|
|
40
40
|
freeze
|
|
41
41
|
end
|
|
42
42
|
|
|
43
|
-
# Evaluates a state against a map of questions and returns
|
|
44
|
-
#
|
|
43
|
+
# Evaluates a state against a map of questions and returns a {Response}
|
|
44
|
+
# with one typed {Answer} per question, e.g. `response[:refund_requested]`.
|
|
45
|
+
# `response.to_h` gives the raw parsed body as a Hash.
|
|
45
46
|
#
|
|
46
47
|
# @param state [String, Object, Array] the content to evaluate; passed
|
|
47
48
|
# through as-is.
|
|
@@ -49,9 +50,11 @@ module Typesafe
|
|
|
49
50
|
# to Question objects; answers come back under the same keys.
|
|
50
51
|
# @param model [String, nil] model for this call; falls back to the model
|
|
51
52
|
# given at initialization.
|
|
52
|
-
# @return [
|
|
53
|
+
# @return [Response] the parsed response.
|
|
53
54
|
# @raise [ArgumentError] if +questions+ is not a non-empty Hash of
|
|
54
|
-
# String/Symbol keys to Question values,
|
|
55
|
+
# String/Symbol keys to Question values, +model+ is invalid, or the
|
|
56
|
+
# response body is not a valid response shape.
|
|
57
|
+
# @raise [JSON::ParserError] if the response body is not valid JSON.
|
|
55
58
|
# @raise [Net::HTTPClientException, Net::HTTPFatalError] on any non-2xx
|
|
56
59
|
# HTTP response.
|
|
57
60
|
def evaluate(state:, questions:, model: nil)
|
|
@@ -66,7 +69,7 @@ module Typesafe
|
|
|
66
69
|
|
|
67
70
|
response = post(body)
|
|
68
71
|
response.value
|
|
69
|
-
|
|
72
|
+
Response.from_json(response.body)
|
|
70
73
|
end
|
|
71
74
|
|
|
72
75
|
private
|
data/lib/typesafe/jev.rb
CHANGED
|
@@ -31,9 +31,10 @@ module Typesafe
|
|
|
31
31
|
# @param questions [Hash{String, Symbol => Question}] question ids mapped
|
|
32
32
|
# to Question objects; answers come back under the same keys.
|
|
33
33
|
# @param model [String, nil] must be nil or "jev-latest".
|
|
34
|
-
# @return [
|
|
35
|
-
# @raise [ArgumentError] if +questions+ is invalid
|
|
36
|
-
# pinned one.
|
|
34
|
+
# @return [Response] the parsed response.
|
|
35
|
+
# @raise [ArgumentError] if +questions+ is invalid, +model+ is not the
|
|
36
|
+
# pinned one, or the response body is not a valid response shape.
|
|
37
|
+
# @raise [JSON::ParserError] if the response body is not valid JSON.
|
|
37
38
|
# @raise [Net::HTTPClientException, Net::HTTPFatalError] on any non-2xx
|
|
38
39
|
# HTTP response.
|
|
39
40
|
def evaluate(state:, questions:, model: nil)
|
|
@@ -53,7 +54,7 @@ module Typesafe
|
|
|
53
54
|
# to Question objects.
|
|
54
55
|
# @param api_key [String, nil] the TypeSafe API key; falls back to the
|
|
55
56
|
# +TYPESAFE_API_KEY+ environment variable.
|
|
56
|
-
# @return [
|
|
57
|
+
# @return [Response] the parsed response.
|
|
57
58
|
def self.evaluate(state:, questions:, api_key: nil)
|
|
58
59
|
new(api_key: api_key).evaluate(state: state, questions: questions)
|
|
59
60
|
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Typesafe
|
|
4
|
+
# The answer to a {Noul} question: the yes/no probability on a scale from
|
|
5
|
+
# 0 (no) to 1 (yes).
|
|
6
|
+
#
|
|
7
|
+
# Typesafe::NoulAnswer.new(noul: 0.95)
|
|
8
|
+
class NoulAnswer < Answer
|
|
9
|
+
# @return [Numeric] the yes/no answer, between 0 (no) and 1 (yes).
|
|
10
|
+
attr_reader :noul
|
|
11
|
+
|
|
12
|
+
# @param noul [Numeric] the yes/no probability, between 0 and 1.
|
|
13
|
+
# @raise [ArgumentError] if +noul+ is not a Numeric between 0 and 1.
|
|
14
|
+
def initialize(noul:)
|
|
15
|
+
@noul = validate_probability(noul, "noul")
|
|
16
|
+
freeze
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# @param hash [Hash] a +noul+ answer entry from the API response, with
|
|
20
|
+
# String or Symbol keys.
|
|
21
|
+
# @return [NoulAnswer]
|
|
22
|
+
def self.from_h(hash)
|
|
23
|
+
new(noul: hash["noul"] || hash[:noul])
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def type
|
|
27
|
+
"noul"
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def to_h
|
|
31
|
+
{ type: type, noul: noul }
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Typesafe
|
|
4
|
+
# A System One evaluation response: the model that performed the
|
|
5
|
+
# evaluation, one typed {Answer} per question (keyed by the question ids
|
|
6
|
+
# from the request), and the token {Usage}.
|
|
7
|
+
#
|
|
8
|
+
# response = Typesafe::Response.from_json(json)
|
|
9
|
+
# response[:refund_requested] # => #<Typesafe::NoulAnswer @noul=0.95>
|
|
10
|
+
#
|
|
11
|
+
# The answer objects do not carry their question id — it is the Hash key
|
|
12
|
+
# under which each answer appears in {#answers}, mirroring how questions
|
|
13
|
+
# are sent.
|
|
14
|
+
class Response
|
|
15
|
+
# @return [String] the model that performed the evaluation.
|
|
16
|
+
attr_reader :model
|
|
17
|
+
|
|
18
|
+
# @return [Hash] question ids (String or Symbol) mapped to Answer objects.
|
|
19
|
+
attr_reader :answers
|
|
20
|
+
|
|
21
|
+
# @return [Usage] the token usage for the request.
|
|
22
|
+
attr_reader :usage
|
|
23
|
+
|
|
24
|
+
# @param model [String] the model that performed the evaluation.
|
|
25
|
+
# @param answers [Hash] question ids (String or Symbol) mapped to Answer
|
|
26
|
+
# objects or answer Hashes; Hashes are parsed via {Answer.from_h}.
|
|
27
|
+
# @param usage [Usage, Hash] a Usage object, or a +usage+ Hash parsed
|
|
28
|
+
# via {Usage.from_h}.
|
|
29
|
+
# @raise [ArgumentError] if any attribute is invalid or an answer is not
|
|
30
|
+
# parseable.
|
|
31
|
+
def initialize(model:, answers:, usage:)
|
|
32
|
+
@model = freeze_string(model, "model must be a non-empty String")
|
|
33
|
+
@answers = normalize_answers(answers)
|
|
34
|
+
@usage = usage.is_a?(Usage) ? usage : Usage.from_h(usage)
|
|
35
|
+
freeze
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Parses a raw JSON response body from the TypeSafe API.
|
|
39
|
+
#
|
|
40
|
+
# @param json [String] the response body.
|
|
41
|
+
# @return [Response]
|
|
42
|
+
# @raise [ArgumentError] if the body is not a valid response shape.
|
|
43
|
+
# @raise [JSON::ParserError] if +json+ is not valid JSON.
|
|
44
|
+
def self.from_json(json)
|
|
45
|
+
from_h(JSON.parse(json))
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# @param hash [Hash] the parsed response body, with String or Symbol keys.
|
|
49
|
+
# @return [Response]
|
|
50
|
+
# @raise [ArgumentError] if +hash+ is not a valid response shape.
|
|
51
|
+
def self.from_h(hash)
|
|
52
|
+
unless hash.is_a?(Hash)
|
|
53
|
+
raise ArgumentError, "response must be a Hash, got #{hash.inspect}"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
new(
|
|
57
|
+
model: hash["model"] || hash[:model],
|
|
58
|
+
answers: hash["answers"] || hash[:answers],
|
|
59
|
+
usage: hash["usage"] || hash[:usage]
|
|
60
|
+
)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# The answer for a question id.
|
|
64
|
+
#
|
|
65
|
+
# @param id [String, Symbol] the question id used in the request.
|
|
66
|
+
# @return [Answer, nil] the matching answer, or nil if the id is unknown.
|
|
67
|
+
def [](id)
|
|
68
|
+
answers[id] || (id.is_a?(Symbol) ? answers[id.to_s] : nil)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# @return [Hash] the response shape returned by the TypeSafe API.
|
|
72
|
+
def to_h
|
|
73
|
+
{ model: model, answers: answers.transform_values(&:to_h), usage: usage.to_h }
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# @return [String] the response serialized as JSON.
|
|
77
|
+
def to_json(state = nil)
|
|
78
|
+
JSON.generate(to_h, state)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Responses compare equal when they are of the same class and serialize
|
|
82
|
+
# to the same API shape.
|
|
83
|
+
def ==(other)
|
|
84
|
+
other.class == self.class && other.to_h == to_h
|
|
85
|
+
end
|
|
86
|
+
alias eql? ==
|
|
87
|
+
|
|
88
|
+
def hash
|
|
89
|
+
[self.class, to_h].hash
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
private
|
|
93
|
+
|
|
94
|
+
# Validates that +model+ is a non-empty String and returns a frozen copy.
|
|
95
|
+
def freeze_string(value, error_message)
|
|
96
|
+
unless value.is_a?(String) && !value.strip.empty?
|
|
97
|
+
raise ArgumentError, error_message
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
value.dup.freeze
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Validates that +answers+ is a non-empty Hash of question ids to
|
|
104
|
+
# (or parseable) Answer objects, and freezes its keys.
|
|
105
|
+
def normalize_answers(answers)
|
|
106
|
+
unless answers.is_a?(Hash) && !answers.empty?
|
|
107
|
+
raise ArgumentError, "answers must be a non-empty Hash mapping question ids to answers"
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
answers.each_with_object({}) do |(id, answer), normalized|
|
|
111
|
+
unless (id.is_a?(String) && !id.strip.empty?) || (id.is_a?(Symbol) && !id.to_s.strip.empty?)
|
|
112
|
+
raise ArgumentError, "answers keys must be non-empty String or Symbol question ids, got #{id.inspect}"
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
normalized[id.is_a?(String) ? id.dup.freeze : id] =
|
|
116
|
+
if answer.is_a?(Answer)
|
|
117
|
+
answer
|
|
118
|
+
elsif answer.is_a?(Hash)
|
|
119
|
+
Answer.from_h(answer)
|
|
120
|
+
else
|
|
121
|
+
raise ArgumentError,
|
|
122
|
+
"answers[#{id.inspect}] must be an answer Hash or Answer object, got #{answer.inspect}"
|
|
123
|
+
end
|
|
124
|
+
end.freeze
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Typesafe
|
|
4
|
+
# The answer to a {Score} question: the probability-weighted value across
|
|
5
|
+
# the levels (which can land between two levels), the legend mapping each
|
|
6
|
+
# level back to its description, the probability distribution across
|
|
7
|
+
# levels, and a confidence between 0 and 1.
|
|
8
|
+
#
|
|
9
|
+
# Typesafe::ScoreAnswer.new(
|
|
10
|
+
# score: 1.6,
|
|
11
|
+
# legend: { "1" => "Calm", "2" => "Frustrated", "3" => "Very angry" },
|
|
12
|
+
# probabilities: { "1" => 0.6, "2" => 0.35, "3" => 0.05 },
|
|
13
|
+
# confidence: 0.7
|
|
14
|
+
# )
|
|
15
|
+
class ScoreAnswer < Answer
|
|
16
|
+
# @return [Numeric] the probability-weighted answer across the levels;
|
|
17
|
+
# can land between levels.
|
|
18
|
+
attr_reader :score
|
|
19
|
+
|
|
20
|
+
# @return [Hash] each level number mapped back to its description.
|
|
21
|
+
attr_reader :legend
|
|
22
|
+
|
|
23
|
+
# @return [Hash] each level mapped to its probability; keys match the
|
|
24
|
+
# legend.
|
|
25
|
+
attr_reader :probabilities
|
|
26
|
+
|
|
27
|
+
# @return [Numeric] how certain the model is, between 0 and 1.
|
|
28
|
+
attr_reader :confidence
|
|
29
|
+
|
|
30
|
+
# @param score [Numeric] the probability-weighted answer across levels.
|
|
31
|
+
# @param legend [Hash] level keys (String or Symbol) mapped to non-empty
|
|
32
|
+
# String descriptions.
|
|
33
|
+
# @param probabilities [Hash] the same level keys mapped to probabilities
|
|
34
|
+
# in 0..1; the values must sum to 1 and the keys must match +legend+.
|
|
35
|
+
# @param confidence [Numeric] the model's confidence, between 0 and 1.
|
|
36
|
+
# @raise [ArgumentError] if any attribute is invalid.
|
|
37
|
+
def initialize(score:, legend:, probabilities:, confidence:)
|
|
38
|
+
@score = validate_number(score, "score")
|
|
39
|
+
@legend = normalize_legend(legend)
|
|
40
|
+
@probabilities = normalize_probabilities(probabilities, allowed_keys: @legend.keys)
|
|
41
|
+
@confidence = validate_probability(confidence, "confidence")
|
|
42
|
+
freeze
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# @param hash [Hash] a +score+ answer entry from the API response, with
|
|
46
|
+
# String or Symbol keys.
|
|
47
|
+
# @return [ScoreAnswer]
|
|
48
|
+
def self.from_h(hash)
|
|
49
|
+
new(
|
|
50
|
+
score: hash["score"] || hash[:score],
|
|
51
|
+
legend: hash["legend"] || hash[:legend],
|
|
52
|
+
probabilities: hash["probabilities"] || hash[:probabilities],
|
|
53
|
+
confidence: hash["confidence"] || hash[:confidence]
|
|
54
|
+
)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def type
|
|
58
|
+
"score"
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def to_h
|
|
62
|
+
{ type: type, score: score, legend: legend, probabilities: probabilities, confidence: confidence }
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
private
|
|
66
|
+
|
|
67
|
+
# Validates that +legend+ is a non-empty Hash of level keys to
|
|
68
|
+
# non-empty String descriptions, with frozen copies of the descriptions.
|
|
69
|
+
def normalize_legend(legend)
|
|
70
|
+
unless legend.is_a?(Hash) && !legend.empty?
|
|
71
|
+
raise ArgumentError, "legend must be a non-empty Hash of level keys to descriptions"
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
legend.each_with_object({}) do |(key, description), normalized|
|
|
75
|
+
unless key.is_a?(String) || key.is_a?(Symbol)
|
|
76
|
+
raise ArgumentError, "legend keys must be Strings or Symbols, got #{key.inspect}"
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
normalized[key.is_a?(String) ? key.dup.freeze : key] =
|
|
80
|
+
validate_string(description, "legend[#{key.inspect}]")
|
|
81
|
+
end.freeze
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Typesafe
|
|
4
|
+
# Token usage for one evaluation request.
|
|
5
|
+
#
|
|
6
|
+
# Typesafe::Usage.new(input_tokens: 296, output_tokens: 20)
|
|
7
|
+
class Usage
|
|
8
|
+
# @return [Integer] the number of input tokens.
|
|
9
|
+
attr_reader :input_tokens
|
|
10
|
+
|
|
11
|
+
# @return [Integer] the number of output tokens.
|
|
12
|
+
attr_reader :output_tokens
|
|
13
|
+
|
|
14
|
+
# @param input_tokens [Integer] the number of input tokens.
|
|
15
|
+
# @param output_tokens [Integer] the number of output tokens.
|
|
16
|
+
# @raise [ArgumentError] if a token count is not a non-negative Integer.
|
|
17
|
+
def initialize(input_tokens:, output_tokens:)
|
|
18
|
+
@input_tokens = validate_token_count(input_tokens, "input_tokens")
|
|
19
|
+
@output_tokens = validate_token_count(output_tokens, "output_tokens")
|
|
20
|
+
freeze
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# @param hash [Hash] a +usage+ entry from the API response, with String
|
|
24
|
+
# or Symbol keys.
|
|
25
|
+
# @return [Usage]
|
|
26
|
+
# @raise [ArgumentError] if +hash+ is not a Hash or a token count is
|
|
27
|
+
# missing or invalid.
|
|
28
|
+
def self.from_h(hash)
|
|
29
|
+
unless hash.is_a?(Hash)
|
|
30
|
+
raise ArgumentError, "usage must be a Hash, got #{hash.inspect}"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
new(
|
|
34
|
+
input_tokens: hash["input_tokens"] || hash[:input_tokens],
|
|
35
|
+
output_tokens: hash["output_tokens"] || hash[:output_tokens]
|
|
36
|
+
)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# @return [Hash] the +usage+ shape returned by the TypeSafe API.
|
|
40
|
+
def to_h
|
|
41
|
+
{ input_tokens: input_tokens, output_tokens: output_tokens }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Usages compare equal when their token counts match.
|
|
45
|
+
def ==(other)
|
|
46
|
+
other.class == self.class && other.to_h == to_h
|
|
47
|
+
end
|
|
48
|
+
alias eql? ==
|
|
49
|
+
|
|
50
|
+
def hash
|
|
51
|
+
[self.class, to_h].hash
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
def validate_token_count(value, name)
|
|
57
|
+
unless value.is_a?(Integer) && value >= 0
|
|
58
|
+
raise ArgumentError, "#{name} must be a non-negative Integer, got #{value.inspect}"
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
value
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
data/lib/typesafe/version.rb
CHANGED
data/lib/typesafe.rb
CHANGED
|
@@ -3,10 +3,19 @@
|
|
|
3
3
|
require "json"
|
|
4
4
|
|
|
5
5
|
require_relative "typesafe/version"
|
|
6
|
+
|
|
6
7
|
require_relative "typesafe/question"
|
|
7
8
|
require_relative "typesafe/noul"
|
|
8
9
|
require_relative "typesafe/choice"
|
|
9
10
|
require_relative "typesafe/score"
|
|
11
|
+
|
|
12
|
+
require_relative "typesafe/answer"
|
|
13
|
+
require_relative "typesafe/noul_answer"
|
|
14
|
+
require_relative "typesafe/choice_answer"
|
|
15
|
+
require_relative "typesafe/score_answer"
|
|
16
|
+
|
|
17
|
+
require_relative "typesafe/usage"
|
|
18
|
+
require_relative "typesafe/response"
|
|
10
19
|
require_relative "typesafe/client"
|
|
11
20
|
require_relative "typesafe/jev"
|
|
12
21
|
|
metadata
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: typesafe-jev
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.6.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
|
-
-
|
|
7
|
+
- dtheofr
|
|
8
8
|
bindir: bin
|
|
9
9
|
cert_chain: []
|
|
10
10
|
date: 1980-01-02 00:00:00.000000000 Z
|
|
@@ -53,7 +53,7 @@ dependencies:
|
|
|
53
53
|
version: '3.0'
|
|
54
54
|
description: Use Jev, TypeSafe's System One model, from Ruby or Rails.
|
|
55
55
|
email:
|
|
56
|
-
-
|
|
56
|
+
- dtheofr@users.noreply.github.com
|
|
57
57
|
executables: []
|
|
58
58
|
extensions: []
|
|
59
59
|
extra_rdoc_files: []
|
|
@@ -62,20 +62,26 @@ files:
|
|
|
62
62
|
- LICENSE
|
|
63
63
|
- README.md
|
|
64
64
|
- lib/typesafe.rb
|
|
65
|
+
- lib/typesafe/answer.rb
|
|
65
66
|
- lib/typesafe/choice.rb
|
|
67
|
+
- lib/typesafe/choice_answer.rb
|
|
66
68
|
- lib/typesafe/client.rb
|
|
67
69
|
- lib/typesafe/jev.rb
|
|
68
70
|
- lib/typesafe/noul.rb
|
|
71
|
+
- lib/typesafe/noul_answer.rb
|
|
69
72
|
- lib/typesafe/question.rb
|
|
73
|
+
- lib/typesafe/response.rb
|
|
70
74
|
- lib/typesafe/score.rb
|
|
75
|
+
- lib/typesafe/score_answer.rb
|
|
76
|
+
- lib/typesafe/usage.rb
|
|
71
77
|
- lib/typesafe/version.rb
|
|
72
78
|
homepage: https://docs.typesafe.ai
|
|
73
79
|
licenses:
|
|
74
80
|
- MIT
|
|
75
81
|
metadata:
|
|
76
82
|
homepage_uri: https://docs.typesafe.ai
|
|
77
|
-
source_code_uri: https://github.com/
|
|
78
|
-
changelog_uri: https://github.com/
|
|
83
|
+
source_code_uri: https://github.com/dtheofr/typesafe-jev-ruby
|
|
84
|
+
changelog_uri: https://github.com/dtheofr/typesafe-jev-ruby/blob/main/CHANGELOG.md
|
|
79
85
|
rdoc_options: []
|
|
80
86
|
require_paths:
|
|
81
87
|
- lib
|