completion-kit 0.28.46 → 0.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +48 -3
- data/app/assets/javascripts/completion_kit/application.js +435 -23
- data/app/assets/stylesheets/completion_kit/application.css +44 -14
- data/app/controllers/completion_kit/api/v1/metric_versions_controller.rb +2 -0
- data/app/controllers/completion_kit/api/v1/metrics_controller.rb +1 -1
- data/app/controllers/completion_kit/api/v1/runs_controller.rb +4 -1
- data/app/controllers/completion_kit/mcp_controller.rb +4 -0
- data/app/controllers/completion_kit/metrics_controller.rb +48 -46
- data/app/controllers/completion_kit/runs_controller.rb +4 -1
- data/app/helpers/completion_kit/application_helper.rb +99 -45
- data/app/jobs/completion_kit/check_review_job.rb +27 -6
- data/app/models/completion_kit/dataset.rb +53 -4
- data/app/models/completion_kit/metric.rb +103 -28
- data/app/models/completion_kit/metric_version.rb +16 -1
- data/app/models/completion_kit/prompt.rb +8 -0
- data/app/models/completion_kit/review.rb +2 -0
- data/app/models/completion_kit/run.rb +20 -9
- data/app/services/completion_kit/api_config.rb +2 -1
- data/app/services/completion_kit/checks/contains.rb +2 -8
- data/app/services/completion_kit/checks/equals.rb +2 -7
- data/app/services/completion_kit/checks/expected_resolver.rb +17 -3
- data/app/services/completion_kit/checks/flag.rb +13 -0
- data/app/services/completion_kit/checks/json_path_equals.rb +2 -15
- data/app/services/completion_kit/checks/labels.rb +104 -0
- data/app/services/completion_kit/checks/length_bounds.rb +4 -4
- data/app/services/completion_kit/checks/list_overlap.rb +96 -0
- data/app/services/completion_kit/checks/not_contains.rb +2 -8
- data/app/services/completion_kit/checks/numeric_bounds.rb +22 -0
- data/app/services/completion_kit/checks/numeric_equals.rb +37 -0
- data/app/services/completion_kit/checks/numeric_value.rb +45 -0
- data/app/services/completion_kit/checks/regex.rb +14 -4
- data/app/services/completion_kit/checks/registry.rb +107 -4
- data/app/services/completion_kit/checks/result.rb +5 -1
- data/app/services/completion_kit/checks/target_resolver.rb +43 -8
- data/app/services/completion_kit/checks/textual.rb +19 -0
- data/app/services/completion_kit/csv_processor.rb +14 -2
- data/app/services/completion_kit/mcp_tools/metrics.rb +21 -7
- data/app/services/completion_kit/model_discovery_service.rb +2 -1
- data/app/services/completion_kit/ollama_client.rb +19 -2
- data/app/services/completion_kit/promptfoo_importer.rb +4 -4
- data/app/services/completion_kit/provider_endpoint.rb +10 -0
- data/app/services/completion_kit/run_comparison.rb +6 -0
- data/app/views/completion_kit/api_reference/_authentication.html.erb +2 -2
- data/app/views/completion_kit/api_reference/_body.html.erb +7 -30
- data/app/views/completion_kit/api_reference/_example.html.erb +1 -1
- data/app/views/completion_kit/api_reference/_resource_card.html.erb +1 -1
- data/app/views/completion_kit/datasets/_form.html.erb +0 -28
- data/app/views/completion_kit/datasets/index.html.erb +1 -1
- data/app/views/completion_kit/metric_groups/_form.html.erb +1 -20
- data/app/views/completion_kit/metric_groups/index.html.erb +1 -1
- data/app/views/completion_kit/metrics/_check_spec.html.erb +5 -11
- data/app/views/completion_kit/metrics/_form.html.erb +62 -47
- data/app/views/completion_kit/metrics/_starter_card.html.erb +1 -1
- data/app/views/completion_kit/metrics/index.html.erb +2 -2
- data/app/views/completion_kit/metrics/show.html.erb +4 -14
- data/app/views/completion_kit/prompts/_form.html.erb +1 -1
- data/app/views/completion_kit/prompts/index.html.erb +3 -3
- data/app/views/completion_kit/prompts/show.html.erb +8 -8
- data/app/views/completion_kit/provider_credentials/_models_card.html.erb +2 -2
- data/app/views/completion_kit/responses/show.html.erb +1 -1
- data/app/views/completion_kit/runs/_compare_score.html.erb +1 -1
- data/app/views/completion_kit/runs/_form.html.erb +4 -174
- data/app/views/completion_kit/runs/_response_row.html.erb +5 -5
- data/app/views/completion_kit/runs/_row.html.erb +13 -11
- data/app/views/completion_kit/runs/_run_config.html.erb +2 -2
- data/app/views/completion_kit/runs/_status_panel.html.erb +2 -2
- data/app/views/completion_kit/runs/compare.html.erb +10 -6
- data/app/views/completion_kit/runs/show.html.erb +5 -19
- data/app/views/completion_kit/shared/_branded_select.html.erb +1 -1
- data/app/views/completion_kit/tags/index.html.erb +1 -1
- data/app/views/layouts/completion_kit/application.html.erb +2 -2
- data/config/locales/en.yml +2 -0
- data/db/migrate/20260903000001_add_check_score_to_completion_kit_reviews.rb +5 -0
- data/lib/completion_kit/version.rb +1 -1
- data/lib/completion_kit.rb +14 -0
- data/lib/generators/completion_kit/templates/initializer.rb +18 -0
- metadata +9 -1
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
module CompletionKit
|
|
2
|
+
module Checks
|
|
3
|
+
class ListOverlap
|
|
4
|
+
MEASURES = %w[recall precision f1 jaccard].freeze
|
|
5
|
+
DEFAULT_MEASURE = "recall".freeze
|
|
6
|
+
DEFAULT_MIN = 1.0
|
|
7
|
+
SEPARATORS = /[,;\n]/
|
|
8
|
+
DECORATION = /\A(?:["'`]+|[-*•]\s+)|["'`]+\z/
|
|
9
|
+
PLAIN_NUMBER = /\A-?(?:0|[1-9]\d*)(?:\.\d+)?\z/
|
|
10
|
+
|
|
11
|
+
def call(target, config)
|
|
12
|
+
actual = to_set(target, config)
|
|
13
|
+
expected = to_set(config["value"], config)
|
|
14
|
+
measure = MEASURES.include?(config["score_by"]) ? config["score_by"] : DEFAULT_MEASURE
|
|
15
|
+
overlap = (actual & expected).length
|
|
16
|
+
score = score_for(measure, overlap, actual.length, expected.length).round(4)
|
|
17
|
+
threshold = NumericValue.parse(config["min"]) || DEFAULT_MIN
|
|
18
|
+
|
|
19
|
+
Result.new(
|
|
20
|
+
passed: score >= threshold,
|
|
21
|
+
detail: "found #{overlap} of #{expected.length} expected, #{actual.length} returned (score #{score.round(2)})",
|
|
22
|
+
score: score
|
|
23
|
+
)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def score_for(measure, overlap, actual_size, expected_size)
|
|
29
|
+
return 1.0 if actual_size.zero? && expected_size.zero?
|
|
30
|
+
|
|
31
|
+
precision = ratio(overlap, actual_size)
|
|
32
|
+
recall = expected_size.zero? ? 1.0 : ratio(overlap, expected_size)
|
|
33
|
+
case measure
|
|
34
|
+
when "precision" then precision
|
|
35
|
+
when "jaccard" then ratio(overlap, actual_size + expected_size - overlap)
|
|
36
|
+
when "f1" then f1(precision, recall)
|
|
37
|
+
else recall
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def ratio(numerator, denominator)
|
|
42
|
+
return 0.0 if denominator.zero?
|
|
43
|
+
|
|
44
|
+
numerator.to_f / denominator
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def f1(precision, recall)
|
|
48
|
+
return 0.0 if (precision + recall).zero?
|
|
49
|
+
|
|
50
|
+
(2 * precision * recall) / (precision + recall)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def to_set(value, config)
|
|
54
|
+
items = coerce_list(value).map { |item| normalize(item, config) }
|
|
55
|
+
items.reject(&:empty?).uniq
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def normalize(item, config)
|
|
59
|
+
return number_text(item.to_s) if item.is_a?(::Numeric)
|
|
60
|
+
|
|
61
|
+
text = item.to_s.strip.gsub(DECORATION, "").strip
|
|
62
|
+
return number_text(text) if PLAIN_NUMBER.match?(text)
|
|
63
|
+
|
|
64
|
+
Flag.on?(config, "case_sensitive") ? text : text.downcase
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def number_text(text)
|
|
68
|
+
number = BigDecimal(text)
|
|
69
|
+
number.frac.zero? ? number.to_i.to_s : number.to_s("F")
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def coerce_list(value)
|
|
73
|
+
case value
|
|
74
|
+
when Array then value
|
|
75
|
+
when Hash, nil then []
|
|
76
|
+
else
|
|
77
|
+
text = value.to_s.strip
|
|
78
|
+
return [] if text.empty?
|
|
79
|
+
|
|
80
|
+
from_json(text) || text.split(SEPARATORS)
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def from_json(text)
|
|
85
|
+
parsed = JSON.parse(text)
|
|
86
|
+
return parsed if parsed.is_a?(Array)
|
|
87
|
+
return parsed.split(SEPARATORS) if parsed.is_a?(String)
|
|
88
|
+
return [] if parsed.is_a?(Hash)
|
|
89
|
+
|
|
90
|
+
nil
|
|
91
|
+
rescue JSON::ParserError
|
|
92
|
+
nil
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
@@ -2,15 +2,9 @@ module CompletionKit
|
|
|
2
2
|
module Checks
|
|
3
3
|
class NotContains
|
|
4
4
|
def call(target, config)
|
|
5
|
-
value = config
|
|
6
|
-
haystack = target.to_s
|
|
7
|
-
present = if config["case_sensitive"] == true
|
|
8
|
-
haystack.include?(value)
|
|
9
|
-
else
|
|
10
|
-
haystack.downcase.include?(value.downcase)
|
|
11
|
-
end
|
|
5
|
+
value, haystack = Textual.operands(target, config)
|
|
12
6
|
|
|
13
|
-
if
|
|
7
|
+
if Textual.include?(haystack, value, config)
|
|
14
8
|
Result.new(passed: false, detail: "contains #{value.inspect}")
|
|
15
9
|
else
|
|
16
10
|
Result.new(passed: true, detail: "does not contain #{value.inspect}")
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
module CompletionKit
|
|
2
|
+
module Checks
|
|
3
|
+
class NumericBounds
|
|
4
|
+
def call(target, config)
|
|
5
|
+
number = NumericValue.parse(target)
|
|
6
|
+
return Result.new(passed: false, detail: "#{NumericValue.preview(target)} is not a number") if number.nil?
|
|
7
|
+
|
|
8
|
+
min = NumericValue.parse(config["min"])
|
|
9
|
+
max = NumericValue.parse(config["max"])
|
|
10
|
+
shown = NumericValue.format(number)
|
|
11
|
+
|
|
12
|
+
if min && number < min
|
|
13
|
+
Result.new(passed: false, detail: "#{shown} is below min #{NumericValue.format(min)}")
|
|
14
|
+
elsif max && number > max
|
|
15
|
+
Result.new(passed: false, detail: "#{shown} is above max #{NumericValue.format(max)}")
|
|
16
|
+
else
|
|
17
|
+
Result.new(passed: true, detail: "#{shown} is within bounds")
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
module CompletionKit
|
|
2
|
+
module Checks
|
|
3
|
+
class NumericEquals
|
|
4
|
+
PERCENT = /\A(.*)%\z/
|
|
5
|
+
|
|
6
|
+
def call(target, config)
|
|
7
|
+
actual = NumericValue.decimal(target)
|
|
8
|
+
return Result.new(passed: false, detail: "#{NumericValue.preview(target)} is not a number") if actual.nil?
|
|
9
|
+
|
|
10
|
+
expected = NumericValue.decimal(config["value"])
|
|
11
|
+
return Result.new(passed: false, detail: "expected #{NumericValue.preview(config["value"])} is not a number") if expected.nil?
|
|
12
|
+
|
|
13
|
+
difference = (actual - expected).abs
|
|
14
|
+
allowed = self.class.allowance(config["tolerance"], expected)
|
|
15
|
+
shown = "#{NumericValue.format(actual)} vs #{NumericValue.format(expected)}"
|
|
16
|
+
|
|
17
|
+
if difference <= allowed
|
|
18
|
+
Result.new(passed: true, detail: "#{shown}, within #{NumericValue.format(allowed)}")
|
|
19
|
+
else
|
|
20
|
+
Result.new(passed: false, detail: "#{shown}, off by #{NumericValue.format(difference)} (allowed #{NumericValue.format(allowed)})")
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def self.allowance(raw, expected)
|
|
25
|
+
share = PERCENT.match(raw.to_s)
|
|
26
|
+
return (NumericValue.decimal(share[1]) || BigDecimal("0")) / 100 * expected.abs if share
|
|
27
|
+
|
|
28
|
+
NumericValue.decimal(raw) || BigDecimal("0")
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def self.tolerance_value(raw)
|
|
32
|
+
share = PERCENT.match(raw.to_s)
|
|
33
|
+
NumericValue.parse(share ? share[1] : raw)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
module CompletionKit
|
|
2
|
+
module Checks
|
|
3
|
+
module NumericValue
|
|
4
|
+
STRIPPED = /[\s_ ]/
|
|
5
|
+
CURRENCY = /\A([+-]?)[$£€]/
|
|
6
|
+
GROUPED = /\A[+-]?[1-9]\d{0,2}(?:,\d{3})+(?:\.\d*)?(?:[eE][+-]?\d+)?\z/
|
|
7
|
+
NUMERIC = /\A[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?\z/
|
|
8
|
+
PREVIEW_LENGTH = 40
|
|
9
|
+
|
|
10
|
+
def self.parse(value)
|
|
11
|
+
return finite(value.to_f) if value.is_a?(::Numeric)
|
|
12
|
+
|
|
13
|
+
text = clean(value)
|
|
14
|
+
return nil unless NUMERIC.match?(text)
|
|
15
|
+
|
|
16
|
+
finite(Float(text))
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def self.decimal(value)
|
|
20
|
+
return nil if parse(value).nil?
|
|
21
|
+
|
|
22
|
+
BigDecimal(value.is_a?(::Numeric) ? value.to_s : clean(value))
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def self.clean(value)
|
|
26
|
+
text = value.to_s.gsub(STRIPPED, "").sub(CURRENCY) { ::Regexp.last_match(1) }
|
|
27
|
+
GROUPED.match?(text) ? text.delete(",") : text
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def self.finite(number)
|
|
31
|
+
number.finite? ? number : nil
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def self.format(number)
|
|
35
|
+
return number.to_i.to_s if number == number.to_i
|
|
36
|
+
|
|
37
|
+
number.to_s
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def self.preview(value)
|
|
41
|
+
value.inspect.truncate(PREVIEW_LENGTH)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -1,17 +1,27 @@
|
|
|
1
1
|
module CompletionKit
|
|
2
2
|
module Checks
|
|
3
3
|
class Regex
|
|
4
|
-
def
|
|
4
|
+
def self.build(config)
|
|
5
5
|
options = 0
|
|
6
|
-
options |= Regexp::IGNORECASE
|
|
7
|
-
options |= Regexp::MULTILINE if config
|
|
8
|
-
|
|
6
|
+
options |= Regexp::IGNORECASE unless Flag.on?(config, "case_sensitive")
|
|
7
|
+
options |= Regexp::MULTILINE if Flag.on?(config, "multiline")
|
|
8
|
+
Regexp.new(config["pattern"].to_s, options, timeout: timeout)
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def self.timeout
|
|
12
|
+
[CompletionKit.config.regex_timeout, Regexp.timeout].compact.min
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def call(target, config)
|
|
16
|
+
pattern = self.class.build(config)
|
|
9
17
|
|
|
10
18
|
if pattern.match?(target.to_s)
|
|
11
19
|
Result.new(passed: true, detail: "matched /#{config["pattern"]}/")
|
|
12
20
|
else
|
|
13
21
|
Result.new(passed: false, detail: "no match for /#{config["pattern"]}/")
|
|
14
22
|
end
|
|
23
|
+
rescue Regexp::TimeoutError
|
|
24
|
+
Result.new(passed: false, detail: "timed out after #{pattern.timeout || Regexp.timeout}s matching /#{config["pattern"]}/")
|
|
15
25
|
rescue RegexpError => e
|
|
16
26
|
Result.new(passed: false, detail: "invalid pattern: #{e.message}")
|
|
17
27
|
end
|
|
@@ -8,7 +8,10 @@ module CompletionKit
|
|
|
8
8
|
"regex" => Regex,
|
|
9
9
|
"valid_json" => ValidJson,
|
|
10
10
|
"json_path_equals" => JsonPathEquals,
|
|
11
|
-
"length_bounds" => LengthBounds
|
|
11
|
+
"length_bounds" => LengthBounds,
|
|
12
|
+
"list_overlap" => ListOverlap,
|
|
13
|
+
"numeric_bounds" => NumericBounds,
|
|
14
|
+
"numeric_equals" => NumericEquals
|
|
12
15
|
}.freeze
|
|
13
16
|
|
|
14
17
|
REQUIRED_KEYS = {
|
|
@@ -18,10 +21,43 @@ module CompletionKit
|
|
|
18
21
|
"regex" => %w[pattern],
|
|
19
22
|
"valid_json" => [],
|
|
20
23
|
"json_path_equals" => %w[json_path expected],
|
|
21
|
-
"length_bounds" => []
|
|
24
|
+
"length_bounds" => [],
|
|
25
|
+
"list_overlap" => %w[value],
|
|
26
|
+
"numeric_bounds" => [],
|
|
27
|
+
"numeric_equals" => %w[value]
|
|
22
28
|
}.freeze
|
|
23
29
|
|
|
24
|
-
|
|
30
|
+
CONFIG_KEYS = {
|
|
31
|
+
"contains" => %w[value case_sensitive trim],
|
|
32
|
+
"not_contains" => %w[value case_sensitive trim],
|
|
33
|
+
"equals" => %w[value case_sensitive trim],
|
|
34
|
+
"regex" => %w[pattern case_sensitive multiline],
|
|
35
|
+
"valid_json" => [],
|
|
36
|
+
"json_path_equals" => %w[json_path expected],
|
|
37
|
+
"length_bounds" => %w[min max],
|
|
38
|
+
"list_overlap" => %w[value score_by min case_sensitive],
|
|
39
|
+
"numeric_bounds" => %w[min max],
|
|
40
|
+
"numeric_equals" => %w[value tolerance]
|
|
41
|
+
}.freeze
|
|
42
|
+
|
|
43
|
+
SHARED_KEYS = %w[check_kind target target_path compare_to expected_path].freeze
|
|
44
|
+
FLAG_KEYS = %w[case_sensitive multiline trim].freeze
|
|
45
|
+
NUMBER_KEYS = %w[min max tolerance].freeze
|
|
46
|
+
DEFAULTS = { "target" => "response_text", "score_by" => ListOverlap::DEFAULT_MEASURE }.freeze
|
|
47
|
+
|
|
48
|
+
EXPECTED_KEYS = {
|
|
49
|
+
"contains" => "value",
|
|
50
|
+
"not_contains" => "value",
|
|
51
|
+
"equals" => "value",
|
|
52
|
+
"json_path_equals" => "expected",
|
|
53
|
+
"list_overlap" => "value",
|
|
54
|
+
"numeric_equals" => "value"
|
|
55
|
+
}.freeze
|
|
56
|
+
|
|
57
|
+
RAW_TARGET_KINDS = %w[list_overlap numeric_bounds numeric_equals].freeze
|
|
58
|
+
RAW_EXPECTED_KINDS = %w[json_path_equals list_overlap numeric_equals].freeze
|
|
59
|
+
BOUNDED_KINDS = %w[length_bounds numeric_bounds].freeze
|
|
60
|
+
SCORING_KINDS = %w[list_overlap].freeze
|
|
25
61
|
|
|
26
62
|
KINDS = CHECKS.keys.freeze
|
|
27
63
|
|
|
@@ -33,13 +69,80 @@ module CompletionKit
|
|
|
33
69
|
REQUIRED_KEYS
|
|
34
70
|
end
|
|
35
71
|
|
|
72
|
+
def self.config_keys(kind)
|
|
73
|
+
CONFIG_KEYS.fetch(kind, [])
|
|
74
|
+
end
|
|
75
|
+
|
|
36
76
|
def self.compares_value?(kind)
|
|
37
|
-
|
|
77
|
+
EXPECTED_KEYS.key?(kind)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def self.expected_key(kind)
|
|
81
|
+
EXPECTED_KEYS[kind]
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def self.comparable_kinds
|
|
85
|
+
EXPECTED_KEYS.keys
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def self.raw_target?(kind)
|
|
89
|
+
RAW_TARGET_KINDS.include?(kind)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def self.raw_expected?(kind)
|
|
93
|
+
RAW_EXPECTED_KINDS.include?(kind)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def self.bounded?(kind)
|
|
97
|
+
BOUNDED_KINDS.include?(kind)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def self.scores?(kind)
|
|
101
|
+
SCORING_KINDS.include?(kind)
|
|
38
102
|
end
|
|
39
103
|
|
|
40
104
|
def self.fetch(kind)
|
|
41
105
|
CHECKS.fetch(kind).new
|
|
42
106
|
end
|
|
107
|
+
|
|
108
|
+
def self.unused_keys(config)
|
|
109
|
+
kind = config["check_kind"]
|
|
110
|
+
keys = config["compare_to"] == "expected" && compares_value?(kind) ? [expected_key(kind)] : %w[compare_to expected_path]
|
|
111
|
+
config["target"] == "json_path" ? keys : keys + %w[target_path]
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def self.canonical(config)
|
|
115
|
+
hash = config.to_h.stringify_keys
|
|
116
|
+
relevant = hash.slice(*SHARED_KEYS, *config_keys(hash["check_kind"])).except(*unused_keys(hash))
|
|
117
|
+
relevant.to_h { |key, value| [key, canonical_value(key, value)] }.reject { |key, value| omitted?(key, value) }
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def self.canonical_value(key, value)
|
|
121
|
+
case key
|
|
122
|
+
when *FLAG_KEYS then Flag.cast(value)
|
|
123
|
+
when *NUMBER_KEYS then NumericValue.parse(value) || value
|
|
124
|
+
when "value" then canonical_operand(value)
|
|
125
|
+
else value
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def self.canonical_operand(value)
|
|
130
|
+
return value.to_s if value.is_a?(::Numeric)
|
|
131
|
+
return value unless value.is_a?(String)
|
|
132
|
+
|
|
133
|
+
parsed = parse_json(value)
|
|
134
|
+
parsed.is_a?(Array) ? parsed : value
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def self.parse_json(text)
|
|
138
|
+
JSON.parse(text)
|
|
139
|
+
rescue JSON::ParserError
|
|
140
|
+
text
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def self.omitted?(key, value)
|
|
144
|
+
value.nil? || value == "" || (FLAG_KEYS.include?(key) && !value) || DEFAULTS[key] == value
|
|
145
|
+
end
|
|
43
146
|
end
|
|
44
147
|
end
|
|
45
148
|
end
|
|
@@ -3,28 +3,63 @@ module CompletionKit
|
|
|
3
3
|
module TargetResolver
|
|
4
4
|
TARGETS = %w[response_text input_data json_path].freeze
|
|
5
5
|
UNRESOLVED = Object.new.freeze
|
|
6
|
+
ARRAY_INDEX = /\A\d+\z/
|
|
7
|
+
ROOT = /\A\$(?:\.|\z)/
|
|
6
8
|
|
|
7
9
|
def self.call(response, config)
|
|
10
|
+
stringify(call_value(response, config))
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def self.call_value(response, config)
|
|
8
14
|
case config["target"]
|
|
9
15
|
when "input_data"
|
|
10
16
|
response.input_data
|
|
11
17
|
when "json_path"
|
|
12
|
-
|
|
18
|
+
dig_json(response.response_text, config["target_path"])
|
|
13
19
|
else
|
|
14
20
|
response.response_text
|
|
15
21
|
end
|
|
16
22
|
end
|
|
17
23
|
|
|
18
|
-
def self.
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
24
|
+
def self.dig_json(text, path)
|
|
25
|
+
walk(JSON.parse(text.to_s), path)
|
|
26
|
+
rescue JSON::ParserError
|
|
27
|
+
UNRESOLVED
|
|
28
|
+
end
|
|
22
29
|
|
|
23
|
-
|
|
30
|
+
def self.walk(document, path)
|
|
31
|
+
normalize_path(path).split(".").reduce(document) do |node, key|
|
|
32
|
+
case node
|
|
33
|
+
when Hash
|
|
34
|
+
return UNRESOLVED unless node.key?(key)
|
|
35
|
+
|
|
36
|
+
node[key]
|
|
37
|
+
when Array
|
|
38
|
+
index = array_index(node, key)
|
|
39
|
+
return UNRESOLVED if index.nil?
|
|
40
|
+
|
|
41
|
+
node[index]
|
|
42
|
+
else
|
|
43
|
+
return UNRESOLVED
|
|
44
|
+
end
|
|
24
45
|
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def self.normalize_path(path)
|
|
49
|
+
path.to_s.strip.sub(ROOT, "")
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def self.array_index(array, key)
|
|
53
|
+
return nil unless ARRAY_INDEX.match?(key)
|
|
54
|
+
|
|
55
|
+
index = key.to_i
|
|
56
|
+
index < array.length ? index : nil
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def self.stringify(value)
|
|
60
|
+
return value if value.equal?(UNRESOLVED)
|
|
61
|
+
|
|
25
62
|
value.to_s
|
|
26
|
-
rescue JSON::ParserError
|
|
27
|
-
UNRESOLVED
|
|
28
63
|
end
|
|
29
64
|
end
|
|
30
65
|
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
module CompletionKit
|
|
2
|
+
module Checks
|
|
3
|
+
module Textual
|
|
4
|
+
def self.operands(target, config)
|
|
5
|
+
value = config["value"].to_s
|
|
6
|
+
haystack = target.to_s
|
|
7
|
+
return [value.strip, haystack.strip] if Flag.on?(config, "trim")
|
|
8
|
+
|
|
9
|
+
[value, haystack]
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def self.include?(haystack, value, config)
|
|
13
|
+
return haystack.include?(value) if Flag.on?(config, "case_sensitive")
|
|
14
|
+
|
|
15
|
+
haystack.downcase.include?(value.downcase)
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -37,7 +37,18 @@ module CompletionKit
|
|
|
37
37
|
def self.extract_variables(prompt)
|
|
38
38
|
return [] if prompt.nil? || prompt.template.blank?
|
|
39
39
|
|
|
40
|
-
prompt.template
|
|
40
|
+
template = prompt.template
|
|
41
|
+
names = []
|
|
42
|
+
cursor = 0
|
|
43
|
+
while (opener = template.byteindex("{{", cursor)) && (closer = template.byteindex("}", opener + 2))
|
|
44
|
+
if closer > opener + 2 && template.byteslice(closer + 1, 1) == "}"
|
|
45
|
+
names << template.byteslice(opener + 2, closer - opener - 2)
|
|
46
|
+
cursor = closer + 2
|
|
47
|
+
else
|
|
48
|
+
cursor = closer + 1
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
names.map(&:strip).uniq
|
|
41
52
|
end
|
|
42
53
|
|
|
43
54
|
def self.validate_variables(run, headers)
|
|
@@ -57,7 +68,8 @@ module CompletionKit
|
|
|
57
68
|
return prompt.template.dup if values.empty?
|
|
58
69
|
|
|
59
70
|
names = values.keys.map { |name| Regexp.escape(name) }.join("|")
|
|
60
|
-
|
|
71
|
+
pattern = Regexp.new("\\{\\{\\s*(#{names})\\s*\\}\\}", timeout: Checks::Regex.timeout)
|
|
72
|
+
prompt.template.gsub(pattern) { values[Regexp.last_match(1)].to_s }
|
|
61
73
|
end
|
|
62
74
|
end
|
|
63
75
|
end
|
|
@@ -3,30 +3,44 @@ module CompletionKit
|
|
|
3
3
|
module Metrics
|
|
4
4
|
extend Base
|
|
5
5
|
|
|
6
|
+
CASE_SENSITIVE_NOTE = "A check with no case_sensitive matches case-insensitively, regex and list_overlap included; " \
|
|
7
|
+
"send case_sensitive: true to match case exactly. Before 0.29.0 a regex check read a missing case_sensitive as case-sensitive."
|
|
8
|
+
|
|
6
9
|
CHECK_CONFIG_SCHEMA = {
|
|
7
10
|
type: "object",
|
|
8
11
|
properties: {
|
|
9
12
|
check_kind: {type: "string", enum: CompletionKit::Checks::Registry.kinds},
|
|
10
13
|
target: {type: "string", enum: CompletionKit::Checks::TargetResolver::TARGETS},
|
|
11
14
|
target_path: {type: "string"},
|
|
12
|
-
value: {type:
|
|
15
|
+
value: {type: %w[string number array], items: {type: %w[string number]}, description: "Only list_overlap takes a list, as a JSON array or as comma-separated text; every other rule refuses one"},
|
|
13
16
|
pattern: {type: "string"},
|
|
14
17
|
json_path: {type: "string"},
|
|
15
18
|
expected: {},
|
|
16
19
|
compare_to: {type: "string", enum: %w[constant expected]},
|
|
17
20
|
expected_path: {type: "string"},
|
|
18
|
-
min: {type: "
|
|
19
|
-
max: {type: "
|
|
20
|
-
|
|
21
|
+
min: {type: "number"},
|
|
22
|
+
max: {type: "number"},
|
|
23
|
+
score_by: {type: "string", enum: CompletionKit::Checks::ListOverlap::MEASURES},
|
|
24
|
+
tolerance: {type: %w[number string], description: "A plain number for a fixed amount, or a percentage string such as \"2%\" for a share of the expected value"},
|
|
25
|
+
case_sensitive: {type: "boolean", description: CASE_SENSITIVE_NOTE},
|
|
21
26
|
multiline: {type: "boolean"},
|
|
22
27
|
trim: {type: "boolean"}
|
|
23
28
|
}
|
|
24
29
|
}.freeze
|
|
25
30
|
|
|
26
31
|
CHECK_CONFIG_HINT = "For a deterministic check set metric_type:\"check\" and check_config. Per-kind required keys: " \
|
|
27
|
-
"value (contains/not_contains/equals), pattern (regex), json_path+expected (json_path_equals), " \
|
|
28
|
-
"min and/or max (length_bounds); valid_json takes no extra keys. target_path is required when target is json_path
|
|
29
|
-
"
|
|
32
|
+
"value (contains/not_contains/equals/list_overlap/numeric_equals), pattern (regex), json_path+expected (json_path_equals), " \
|
|
33
|
+
"min and/or max (length_bounds/numeric_bounds); valid_json takes no extra keys. target_path is required when target is json_path; " \
|
|
34
|
+
"a target_path sent with no target sets target to json_path, and one sent with any other target is refused. " \
|
|
35
|
+
"target_path walks array indices, as in items.0.name; any path may start at the root with $., as in $.items.0.name. " \
|
|
36
|
+
"list_overlap grades a list-valued field and stores a fraction rather than a verdict: score_by is recall, precision, f1 or jaccard (recall by default), " \
|
|
37
|
+
"and min is the fraction that counts as a pass (1, a perfect score, by default). " \
|
|
38
|
+
"numeric_bounds and numeric_equals read a number, ignoring the thousands commas and currency symbol a figure is written with; " \
|
|
39
|
+
"a comma counts only as a thousands separator, with three digits after it and no decimal point before it, so 1,5 is not a number; " \
|
|
40
|
+
"numeric_equals takes a tolerance, either a plain number for a fixed amount or a percentage string such as 2% for a share of the expected value. " \
|
|
41
|
+
"#{CASE_SENSITIVE_NOTE} " \
|
|
42
|
+
"Set compare_to:\"expected\" on #{CompletionKit::Checks::Registry.comparable_kinds.join(", ")} to grade against each row's own expected_output (ground truth) " \
|
|
43
|
+
"instead of a constant; drop value (or expected, for json_path_equals) and add expected_path to dig into the answer key when it is JSON."
|
|
30
44
|
|
|
31
45
|
TOOLS = {
|
|
32
46
|
"metrics_list" => {
|
|
@@ -12,8 +12,9 @@ module CompletionKit
|
|
|
12
12
|
|
|
13
13
|
def initialize(config:)
|
|
14
14
|
@provider = config[:provider]
|
|
15
|
-
@api_key = config[:api_key]
|
|
16
15
|
@api_endpoint = config[:api_endpoint]
|
|
16
|
+
@api_key = config[:api_key]
|
|
17
|
+
@api_key = @api_key.presence || OllamaClient.fallback_api_key(@api_endpoint) if @provider == "ollama"
|
|
17
18
|
@api_version = config[:api_version]
|
|
18
19
|
@catalog_model_count = nil
|
|
19
20
|
end
|
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
module CompletionKit
|
|
2
2
|
class OllamaClient < LlmClient
|
|
3
|
+
DEFAULT_ENDPOINT = "http://localhost:11434/v1"
|
|
4
|
+
|
|
5
|
+
def self.configured_endpoint
|
|
6
|
+
endpoint = CompletionKit.config.ollama_api_endpoint || ENV["OLLAMA_API_ENDPOINT"]
|
|
7
|
+
endpoint || (DEFAULT_ENDPOINT if CompletionKit.config.allow_loopback_endpoints)
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def self.fallback_api_key(endpoint)
|
|
11
|
+
return unless ProviderEndpoint.same_destination?(root_url(endpoint), root_url(configured_endpoint))
|
|
12
|
+
|
|
13
|
+
env_key = ENV["OLLAMA_API_KEY"]
|
|
14
|
+
CompletionKit.config.tenant_scope ? env_key : CompletionKit.config.ollama_api_key || env_key
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def self.root_url(endpoint)
|
|
18
|
+
endpoint.to_s.strip.delete_suffix("/").delete_suffix("/v1")
|
|
19
|
+
end
|
|
3
20
|
|
|
4
21
|
def generate_completion(prompt, options = {})
|
|
5
22
|
@temperature_dropped = false
|
|
@@ -75,12 +92,12 @@ module CompletionKit
|
|
|
75
92
|
end
|
|
76
93
|
|
|
77
94
|
def api_key
|
|
78
|
-
@config[:api_key] ||
|
|
95
|
+
@config[:api_key].presence || self.class.fallback_api_key(api_endpoint)
|
|
79
96
|
end
|
|
80
97
|
|
|
81
98
|
def api_endpoint
|
|
82
99
|
raw = @config[:api_endpoint] || ENV["OLLAMA_API_ENDPOINT"]
|
|
83
|
-
raw ||=
|
|
100
|
+
raw ||= DEFAULT_ENDPOINT if CompletionKit.config.allow_loopback_endpoints
|
|
84
101
|
raw.to_s.delete_suffix("/")
|
|
85
102
|
end
|
|
86
103
|
|
|
@@ -149,17 +149,17 @@ module CompletionKit
|
|
|
149
149
|
def check_mapping(type, text)
|
|
150
150
|
case type
|
|
151
151
|
when "contains"
|
|
152
|
-
[{ "check_kind" => "contains", "target" => "response_text", "value" => text }, "Contains #{text.truncate(30).inspect}"]
|
|
152
|
+
[{ "check_kind" => "contains", "target" => "response_text", "value" => text, "case_sensitive" => true }, "Contains #{text.truncate(30).inspect}"]
|
|
153
153
|
when "icontains"
|
|
154
154
|
[{ "check_kind" => "contains", "target" => "response_text", "value" => text, "case_sensitive" => false }, "Contains #{text.truncate(30).inspect}"]
|
|
155
155
|
when "not-contains"
|
|
156
|
-
[{ "check_kind" => "not_contains", "target" => "response_text", "value" => text }, "Does not contain #{text.truncate(30).inspect}"]
|
|
156
|
+
[{ "check_kind" => "not_contains", "target" => "response_text", "value" => text, "case_sensitive" => true }, "Does not contain #{text.truncate(30).inspect}"]
|
|
157
157
|
when "not-icontains"
|
|
158
158
|
[{ "check_kind" => "not_contains", "target" => "response_text", "value" => text, "case_sensitive" => false }, "Does not contain #{text.truncate(30).inspect}"]
|
|
159
159
|
when "equals"
|
|
160
|
-
[{ "check_kind" => "equals", "target" => "response_text", "value" => text }, "Equals #{text.truncate(30).inspect}"]
|
|
160
|
+
[{ "check_kind" => "equals", "target" => "response_text", "value" => text, "case_sensitive" => true }, "Equals #{text.truncate(30).inspect}"]
|
|
161
161
|
when "regex"
|
|
162
|
-
[{ "check_kind" => "regex", "target" => "response_text", "pattern" => text }, "Matches /#{text.truncate(30)}/"]
|
|
162
|
+
[{ "check_kind" => "regex", "target" => "response_text", "pattern" => text, "case_sensitive" => true }, "Matches /#{text.truncate(30)}/"]
|
|
163
163
|
when "is-json"
|
|
164
164
|
[{ "check_kind" => "valid_json", "target" => "response_text" }, "Valid JSON"]
|
|
165
165
|
end
|
|
@@ -51,6 +51,16 @@ module CompletionKit
|
|
|
51
51
|
nil
|
|
52
52
|
end
|
|
53
53
|
|
|
54
|
+
def same_destination?(url, other)
|
|
55
|
+
destination = destination(url)
|
|
56
|
+
!destination.nil? && destination == destination(other)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def destination(url)
|
|
60
|
+
uri = parse(url)
|
|
61
|
+
[uri.scheme, uri.host.downcase, uri.port, uri.path.delete_suffix("/")] if uri
|
|
62
|
+
end
|
|
63
|
+
|
|
54
64
|
def issues_for(addrs)
|
|
55
65
|
return [:unresolvable] if addrs.empty?
|
|
56
66
|
return [:unsafe_host] if addrs.any? { |ip| unsafe?(ip) }
|