completion-kit 0.28.45 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/javascripts/completion_kit/application.js +52 -23
- data/app/assets/stylesheets/completion_kit/application.css +56 -15
- data/app/controllers/completion_kit/api/v1/metric_versions_controller.rb +2 -0
- data/app/controllers/completion_kit/api/v1/metrics_controller.rb +1 -1
- data/app/controllers/completion_kit/api/v1/runs_controller.rb +4 -1
- data/app/controllers/completion_kit/metrics_controller.rb +48 -46
- data/app/controllers/completion_kit/onboarding_controller.rb +1 -1
- data/app/controllers/completion_kit/runs_controller.rb +18 -6
- data/app/helpers/completion_kit/application_helper.rb +110 -52
- data/app/jobs/completion_kit/check_review_job.rb +27 -6
- data/app/models/completion_kit/metric.rb +103 -28
- data/app/models/completion_kit/metric_version.rb +16 -1
- data/app/models/completion_kit/prompt.rb +1 -1
- data/app/models/completion_kit/review.rb +2 -0
- data/app/models/completion_kit/run.rb +36 -18
- data/app/services/completion_kit/checks/contains.rb +2 -8
- data/app/services/completion_kit/checks/equals.rb +2 -7
- data/app/services/completion_kit/checks/expected_resolver.rb +17 -3
- data/app/services/completion_kit/checks/flag.rb +13 -0
- data/app/services/completion_kit/checks/json_path_equals.rb +2 -15
- data/app/services/completion_kit/checks/labels.rb +104 -0
- data/app/services/completion_kit/checks/length_bounds.rb +4 -4
- data/app/services/completion_kit/checks/list_overlap.rb +96 -0
- data/app/services/completion_kit/checks/not_contains.rb +2 -8
- data/app/services/completion_kit/checks/numeric_bounds.rb +22 -0
- data/app/services/completion_kit/checks/numeric_equals.rb +37 -0
- data/app/services/completion_kit/checks/numeric_value.rb +45 -0
- data/app/services/completion_kit/checks/regex.rb +2 -2
- data/app/services/completion_kit/checks/registry.rb +107 -4
- data/app/services/completion_kit/checks/result.rb +5 -1
- data/app/services/completion_kit/checks/target_resolver.rb +43 -8
- data/app/services/completion_kit/checks/textual.rb +19 -0
- data/app/services/completion_kit/mcp_tools/metric_versions.rb +1 -1
- data/app/services/completion_kit/mcp_tools/metrics.rb +21 -7
- data/app/services/completion_kit/mcp_tools/prompts.rb +1 -1
- data/app/services/completion_kit/prompt_improvement_service.rb +1 -1
- data/app/services/completion_kit/promptfoo_importer.rb +4 -4
- data/app/services/completion_kit/run_comparison.rb +6 -0
- data/app/views/completion_kit/agreements/_buttons.html.erb +1 -1
- data/app/views/completion_kit/api_reference/_body.html.erb +9 -5
- data/app/views/completion_kit/metric_groups/_form.html.erb +13 -2
- data/app/views/completion_kit/metric_groups/edit.html.erb +1 -1
- data/app/views/completion_kit/metrics/_check_spec.html.erb +5 -11
- data/app/views/completion_kit/metrics/_form.html.erb +62 -47
- data/app/views/completion_kit/onboarding/show.html.erb +2 -2
- data/app/views/completion_kit/prompts/_form.html.erb +2 -1
- data/app/views/completion_kit/provider_credentials/_models_card.html.erb +2 -2
- data/app/views/completion_kit/responses/show.html.erb +1 -1
- data/app/views/completion_kit/runs/_compare_score.html.erb +1 -1
- data/app/views/completion_kit/runs/_form.html.erb +21 -11
- data/app/views/completion_kit/runs/_response_row.html.erb +5 -3
- data/app/views/completion_kit/runs/_row.html.erb +10 -8
- data/app/views/completion_kit/runs/_run_config.html.erb +1 -1
- data/app/views/completion_kit/runs/_status_panel.html.erb +2 -2
- data/app/views/completion_kit/runs/compare.html.erb +10 -6
- data/app/views/completion_kit/runs/edit.html.erb +1 -1
- data/app/views/completion_kit/runs/new.html.erb +1 -1
- data/app/views/completion_kit/shared/_branded_select.html.erb +1 -1
- data/app/views/layouts/completion_kit/application.html.erb +1 -1
- data/config/locales/en.yml +2 -0
- data/db/migrate/20260903000001_add_check_score_to_completion_kit_reviews.rb +5 -0
- data/db/migrate/20261013000001_drop_completion_kit_mcp_sessions.rb +16 -0
- data/lib/completion_kit/errors.rb +2 -2
- data/lib/completion_kit/version.rb +1 -1
- metadata +10 -1
|
@@ -2,14 +2,9 @@ module CompletionKit
|
|
|
2
2
|
module Checks
|
|
3
3
|
class Equals
|
|
4
4
|
def call(target, config)
|
|
5
|
-
actual = target
|
|
6
|
-
expected = config["value"].to_s
|
|
7
|
-
if config["trim"] == true
|
|
8
|
-
actual = actual.strip
|
|
9
|
-
expected = expected.strip
|
|
10
|
-
end
|
|
5
|
+
expected, actual = Textual.operands(target, config)
|
|
11
6
|
|
|
12
|
-
match = if config
|
|
7
|
+
match = if Flag.on?(config, "case_sensitive")
|
|
13
8
|
actual == expected
|
|
14
9
|
else
|
|
15
10
|
actual.casecmp?(expected)
|
|
@@ -4,13 +4,27 @@ module CompletionKit
|
|
|
4
4
|
UNRESOLVED = TargetResolver::UNRESOLVED
|
|
5
5
|
|
|
6
6
|
def self.call(response, config)
|
|
7
|
+
TargetResolver.stringify(resolve(response, config) { |raw| raw.to_s })
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def self.call_value(response, config)
|
|
11
|
+
resolve(response, config) { |raw| parse(raw) }
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def self.resolve(response, config)
|
|
7
15
|
raw = response.expected_output
|
|
8
16
|
return UNRESOLVED if raw.blank?
|
|
9
17
|
|
|
10
|
-
path = config["expected_path"]
|
|
11
|
-
return raw
|
|
18
|
+
path = TargetResolver.normalize_path(config["expected_path"])
|
|
19
|
+
return yield(raw) if path.empty?
|
|
20
|
+
|
|
21
|
+
TargetResolver.dig_json(raw.to_s, path)
|
|
22
|
+
end
|
|
12
23
|
|
|
13
|
-
|
|
24
|
+
def self.parse(raw)
|
|
25
|
+
JSON.parse(raw.to_s)
|
|
26
|
+
rescue JSON::ParserError
|
|
27
|
+
raw
|
|
14
28
|
end
|
|
15
29
|
end
|
|
16
30
|
end
|
|
@@ -1,13 +1,10 @@
|
|
|
1
1
|
module CompletionKit
|
|
2
2
|
module Checks
|
|
3
3
|
class JsonPathEquals
|
|
4
|
-
MISSING = Object.new
|
|
5
|
-
|
|
6
4
|
def call(target, config)
|
|
7
|
-
|
|
8
|
-
value = dig(parsed, config["json_path"].to_s)
|
|
5
|
+
value = TargetResolver.walk(JSON.parse(target.to_s), config["json_path"])
|
|
9
6
|
|
|
10
|
-
if value.equal?(
|
|
7
|
+
if value.equal?(TargetResolver::UNRESOLVED)
|
|
11
8
|
Result.new(passed: false, detail: "path #{config["json_path"]} not found")
|
|
12
9
|
elsif value == config["expected"]
|
|
13
10
|
Result.new(passed: true, detail: "#{config["json_path"]} == #{config["expected"].inspect}")
|
|
@@ -17,16 +14,6 @@ module CompletionKit
|
|
|
17
14
|
rescue JSON::ParserError
|
|
18
15
|
Result.new(passed: false, detail: "not valid JSON")
|
|
19
16
|
end
|
|
20
|
-
|
|
21
|
-
private
|
|
22
|
-
|
|
23
|
-
def dig(data, path)
|
|
24
|
-
path.split(".").reduce(data) do |node, key|
|
|
25
|
-
return MISSING unless node.is_a?(Hash) && node.key?(key)
|
|
26
|
-
|
|
27
|
-
node[key]
|
|
28
|
-
end
|
|
29
|
-
end
|
|
30
17
|
end
|
|
31
18
|
end
|
|
32
19
|
end
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
module CompletionKit
|
|
2
|
+
module Checks
|
|
3
|
+
module Labels
|
|
4
|
+
KINDS = {
|
|
5
|
+
"contains" => "Contains a phrase",
|
|
6
|
+
"not_contains" => "Does not contain a phrase",
|
|
7
|
+
"equals" => "Equals exactly",
|
|
8
|
+
"regex" => "Matches a pattern",
|
|
9
|
+
"valid_json" => "Is valid JSON",
|
|
10
|
+
"json_path_equals" => "A JSON field equals a value",
|
|
11
|
+
"length_bounds" => "Length is within a range",
|
|
12
|
+
"list_overlap" => "How much of a list was found",
|
|
13
|
+
"numeric_bounds" => "A number is within a range",
|
|
14
|
+
"numeric_equals" => "A number is close enough"
|
|
15
|
+
}.freeze
|
|
16
|
+
|
|
17
|
+
TARGETS = {
|
|
18
|
+
"response_text" => "The response text",
|
|
19
|
+
"input_data" => "The input row",
|
|
20
|
+
"json_path" => "A value from the response JSON"
|
|
21
|
+
}.freeze
|
|
22
|
+
|
|
23
|
+
COMPARE_TO = {
|
|
24
|
+
"constant" => "A value you type",
|
|
25
|
+
"expected" => "Each row's expected value"
|
|
26
|
+
}.freeze
|
|
27
|
+
|
|
28
|
+
SCORE_BY = {
|
|
29
|
+
"recall" => "Coverage: how much of the expected list it found",
|
|
30
|
+
"precision" => "Accuracy: how much of what it returned was expected",
|
|
31
|
+
"f1" => "Balance: coverage and accuracy weighed equally",
|
|
32
|
+
"jaccard" => "Overlap: shared items out of every item in either list"
|
|
33
|
+
}.freeze
|
|
34
|
+
|
|
35
|
+
FIELDS = {
|
|
36
|
+
"check_kind" => "Rule",
|
|
37
|
+
"target" => "What to check",
|
|
38
|
+
"target_path" => "Path into the JSON",
|
|
39
|
+
"compare_to" => "Compare against",
|
|
40
|
+
"value" => "Text to look for",
|
|
41
|
+
"pattern" => "Pattern",
|
|
42
|
+
"json_path" => "JSON path",
|
|
43
|
+
"expected" => "Expected value",
|
|
44
|
+
"expected_path" => "Field in the answer key",
|
|
45
|
+
"score_by" => "How to score it",
|
|
46
|
+
"min" => "Minimum",
|
|
47
|
+
"max" => "Maximum",
|
|
48
|
+
"tolerance" => "How far off is still right",
|
|
49
|
+
"case_sensitive" => "Case sensitive",
|
|
50
|
+
"multiline" => "Multiline",
|
|
51
|
+
"trim" => "Trim whitespace"
|
|
52
|
+
}.freeze
|
|
53
|
+
|
|
54
|
+
FIELDS_BY_KIND = {
|
|
55
|
+
"length_bounds" => { "min" => "Shortest allowed", "max" => "Longest allowed" },
|
|
56
|
+
"numeric_bounds" => { "min" => "Lowest allowed", "max" => "Highest allowed" },
|
|
57
|
+
"list_overlap" => { "min" => "Lowest score that passes", "value" => "Expected list" },
|
|
58
|
+
"numeric_equals" => { "value" => "Expected number" }
|
|
59
|
+
}.freeze
|
|
60
|
+
|
|
61
|
+
HINTS = {
|
|
62
|
+
"score_by" => "Coverage ignores extra items, and accuracy ignores missing ones. Balance weighs the two equally. " \
|
|
63
|
+
"Overlap divides the shared items by every item in either list, so it never scores higher than the other three.",
|
|
64
|
+
"min" => "Leave blank for no minimum.",
|
|
65
|
+
"max" => "Leave blank for no maximum."
|
|
66
|
+
}.freeze
|
|
67
|
+
|
|
68
|
+
HINTS_BY_KIND = {
|
|
69
|
+
"list_overlap" => { "min" => "Give a value from 0 to 1, like 0.6 for 60%. Leave blank to require a perfect score." }
|
|
70
|
+
}.freeze
|
|
71
|
+
|
|
72
|
+
VALUES = { "compare_to" => COMPARE_TO, "score_by" => SCORE_BY }.freeze
|
|
73
|
+
|
|
74
|
+
def self.kind(kind)
|
|
75
|
+
KINDS.fetch(kind.to_s) { kind.to_s.humanize }
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def self.target(target)
|
|
79
|
+
TARGETS.fetch(target.to_s) { target.to_s.humanize }
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def self.field(key, kind = nil)
|
|
83
|
+
FIELDS_BY_KIND.dig(kind.to_s, key.to_s) || FIELDS.fetch(key.to_s) { key.to_s.humanize }
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def self.hint(key, kind = nil)
|
|
87
|
+
HINTS_BY_KIND.dig(kind.to_s, key.to_s) || HINTS[key.to_s]
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def self.value(key, value)
|
|
91
|
+
VALUES.fetch(key.to_s, {}).fetch(value.to_s, value)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def self.worded?(key)
|
|
95
|
+
VALUES.key?(key.to_s)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def self.in_form_order(config)
|
|
99
|
+
order = FIELDS.keys
|
|
100
|
+
config.sort_by.with_index { |(key, _value), index| [order.index(key) || order.length, index] }.to_h
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
@@ -3,13 +3,13 @@ module CompletionKit
|
|
|
3
3
|
class LengthBounds
|
|
4
4
|
def call(target, config)
|
|
5
5
|
length = target.to_s.length
|
|
6
|
-
min = config["min"]
|
|
7
|
-
max = config["max"]
|
|
6
|
+
min = NumericValue.parse(config["min"])
|
|
7
|
+
max = NumericValue.parse(config["max"])
|
|
8
8
|
|
|
9
9
|
if min && length < min
|
|
10
|
-
Result.new(passed: false, detail: "length #{length} below min #{min}")
|
|
10
|
+
Result.new(passed: false, detail: "length #{length} below min #{NumericValue.format(min)}")
|
|
11
11
|
elsif max && length > max
|
|
12
|
-
Result.new(passed: false, detail: "length #{length} above max #{max}")
|
|
12
|
+
Result.new(passed: false, detail: "length #{length} above max #{NumericValue.format(max)}")
|
|
13
13
|
else
|
|
14
14
|
Result.new(passed: true, detail: "length #{length} within bounds")
|
|
15
15
|
end
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
module CompletionKit
|
|
2
|
+
module Checks
|
|
3
|
+
class ListOverlap
|
|
4
|
+
MEASURES = %w[recall precision f1 jaccard].freeze
|
|
5
|
+
DEFAULT_MEASURE = "recall".freeze
|
|
6
|
+
DEFAULT_MIN = 1.0
|
|
7
|
+
SEPARATORS = /[,;\n]/
|
|
8
|
+
DECORATION = /\A(?:["'`]+|[-*•]\s+)|["'`]+\z/
|
|
9
|
+
PLAIN_NUMBER = /\A-?(?:0|[1-9]\d*)(?:\.\d+)?\z/
|
|
10
|
+
|
|
11
|
+
def call(target, config)
|
|
12
|
+
actual = to_set(target, config)
|
|
13
|
+
expected = to_set(config["value"], config)
|
|
14
|
+
measure = MEASURES.include?(config["score_by"]) ? config["score_by"] : DEFAULT_MEASURE
|
|
15
|
+
overlap = (actual & expected).length
|
|
16
|
+
score = score_for(measure, overlap, actual.length, expected.length).round(4)
|
|
17
|
+
threshold = NumericValue.parse(config["min"]) || DEFAULT_MIN
|
|
18
|
+
|
|
19
|
+
Result.new(
|
|
20
|
+
passed: score >= threshold,
|
|
21
|
+
detail: "found #{overlap} of #{expected.length} expected, #{actual.length} returned (score #{score.round(2)})",
|
|
22
|
+
score: score
|
|
23
|
+
)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def score_for(measure, overlap, actual_size, expected_size)
|
|
29
|
+
return 1.0 if actual_size.zero? && expected_size.zero?
|
|
30
|
+
|
|
31
|
+
precision = ratio(overlap, actual_size)
|
|
32
|
+
recall = expected_size.zero? ? 1.0 : ratio(overlap, expected_size)
|
|
33
|
+
case measure
|
|
34
|
+
when "precision" then precision
|
|
35
|
+
when "jaccard" then ratio(overlap, actual_size + expected_size - overlap)
|
|
36
|
+
when "f1" then f1(precision, recall)
|
|
37
|
+
else recall
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def ratio(numerator, denominator)
|
|
42
|
+
return 0.0 if denominator.zero?
|
|
43
|
+
|
|
44
|
+
numerator.to_f / denominator
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def f1(precision, recall)
|
|
48
|
+
return 0.0 if (precision + recall).zero?
|
|
49
|
+
|
|
50
|
+
(2 * precision * recall) / (precision + recall)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def to_set(value, config)
|
|
54
|
+
items = coerce_list(value).map { |item| normalize(item, config) }
|
|
55
|
+
items.reject(&:empty?).uniq
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def normalize(item, config)
|
|
59
|
+
return number_text(item.to_s) if item.is_a?(::Numeric)
|
|
60
|
+
|
|
61
|
+
text = item.to_s.strip.gsub(DECORATION, "").strip
|
|
62
|
+
return number_text(text) if PLAIN_NUMBER.match?(text)
|
|
63
|
+
|
|
64
|
+
Flag.on?(config, "case_sensitive") ? text : text.downcase
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def number_text(text)
|
|
68
|
+
number = BigDecimal(text)
|
|
69
|
+
number.frac.zero? ? number.to_i.to_s : number.to_s("F")
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def coerce_list(value)
|
|
73
|
+
case value
|
|
74
|
+
when Array then value
|
|
75
|
+
when Hash, nil then []
|
|
76
|
+
else
|
|
77
|
+
text = value.to_s.strip
|
|
78
|
+
return [] if text.empty?
|
|
79
|
+
|
|
80
|
+
from_json(text) || text.split(SEPARATORS)
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def from_json(text)
|
|
85
|
+
parsed = JSON.parse(text)
|
|
86
|
+
return parsed if parsed.is_a?(Array)
|
|
87
|
+
return parsed.split(SEPARATORS) if parsed.is_a?(String)
|
|
88
|
+
return [] if parsed.is_a?(Hash)
|
|
89
|
+
|
|
90
|
+
nil
|
|
91
|
+
rescue JSON::ParserError
|
|
92
|
+
nil
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
@@ -2,15 +2,9 @@ module CompletionKit
|
|
|
2
2
|
module Checks
|
|
3
3
|
class NotContains
|
|
4
4
|
def call(target, config)
|
|
5
|
-
value = config
|
|
6
|
-
haystack = target.to_s
|
|
7
|
-
present = if config["case_sensitive"] == true
|
|
8
|
-
haystack.include?(value)
|
|
9
|
-
else
|
|
10
|
-
haystack.downcase.include?(value.downcase)
|
|
11
|
-
end
|
|
5
|
+
value, haystack = Textual.operands(target, config)
|
|
12
6
|
|
|
13
|
-
if
|
|
7
|
+
if Textual.include?(haystack, value, config)
|
|
14
8
|
Result.new(passed: false, detail: "contains #{value.inspect}")
|
|
15
9
|
else
|
|
16
10
|
Result.new(passed: true, detail: "does not contain #{value.inspect}")
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
module CompletionKit
|
|
2
|
+
module Checks
|
|
3
|
+
class NumericBounds
|
|
4
|
+
def call(target, config)
|
|
5
|
+
number = NumericValue.parse(target)
|
|
6
|
+
return Result.new(passed: false, detail: "#{NumericValue.preview(target)} is not a number") if number.nil?
|
|
7
|
+
|
|
8
|
+
min = NumericValue.parse(config["min"])
|
|
9
|
+
max = NumericValue.parse(config["max"])
|
|
10
|
+
shown = NumericValue.format(number)
|
|
11
|
+
|
|
12
|
+
if min && number < min
|
|
13
|
+
Result.new(passed: false, detail: "#{shown} is below min #{NumericValue.format(min)}")
|
|
14
|
+
elsif max && number > max
|
|
15
|
+
Result.new(passed: false, detail: "#{shown} is above max #{NumericValue.format(max)}")
|
|
16
|
+
else
|
|
17
|
+
Result.new(passed: true, detail: "#{shown} is within bounds")
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
module CompletionKit
|
|
2
|
+
module Checks
|
|
3
|
+
class NumericEquals
|
|
4
|
+
PERCENT = /\A(.*)%\z/
|
|
5
|
+
|
|
6
|
+
def call(target, config)
|
|
7
|
+
actual = NumericValue.decimal(target)
|
|
8
|
+
return Result.new(passed: false, detail: "#{NumericValue.preview(target)} is not a number") if actual.nil?
|
|
9
|
+
|
|
10
|
+
expected = NumericValue.decimal(config["value"])
|
|
11
|
+
return Result.new(passed: false, detail: "expected #{NumericValue.preview(config["value"])} is not a number") if expected.nil?
|
|
12
|
+
|
|
13
|
+
difference = (actual - expected).abs
|
|
14
|
+
allowed = self.class.allowance(config["tolerance"], expected)
|
|
15
|
+
shown = "#{NumericValue.format(actual)} vs #{NumericValue.format(expected)}"
|
|
16
|
+
|
|
17
|
+
if difference <= allowed
|
|
18
|
+
Result.new(passed: true, detail: "#{shown}, within #{NumericValue.format(allowed)}")
|
|
19
|
+
else
|
|
20
|
+
Result.new(passed: false, detail: "#{shown}, off by #{NumericValue.format(difference)} (allowed #{NumericValue.format(allowed)})")
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def self.allowance(raw, expected)
|
|
25
|
+
share = PERCENT.match(raw.to_s)
|
|
26
|
+
return (NumericValue.decimal(share[1]) || BigDecimal("0")) / 100 * expected.abs if share
|
|
27
|
+
|
|
28
|
+
NumericValue.decimal(raw) || BigDecimal("0")
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def self.tolerance_value(raw)
|
|
32
|
+
share = PERCENT.match(raw.to_s)
|
|
33
|
+
NumericValue.parse(share ? share[1] : raw)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
module CompletionKit
|
|
2
|
+
module Checks
|
|
3
|
+
module NumericValue
|
|
4
|
+
STRIPPED = /[\s_ ]/
|
|
5
|
+
CURRENCY = /\A([+-]?)[$£€]/
|
|
6
|
+
GROUPED = /\A[+-]?[1-9]\d{0,2}(?:,\d{3})+(?:\.\d*)?(?:[eE][+-]?\d+)?\z/
|
|
7
|
+
NUMERIC = /\A[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?\z/
|
|
8
|
+
PREVIEW_LENGTH = 40
|
|
9
|
+
|
|
10
|
+
def self.parse(value)
|
|
11
|
+
return finite(value.to_f) if value.is_a?(::Numeric)
|
|
12
|
+
|
|
13
|
+
text = clean(value)
|
|
14
|
+
return nil unless NUMERIC.match?(text)
|
|
15
|
+
|
|
16
|
+
finite(Float(text))
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def self.decimal(value)
|
|
20
|
+
return nil if parse(value).nil?
|
|
21
|
+
|
|
22
|
+
BigDecimal(value.is_a?(::Numeric) ? value.to_s : clean(value))
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def self.clean(value)
|
|
26
|
+
text = value.to_s.gsub(STRIPPED, "").sub(CURRENCY) { ::Regexp.last_match(1) }
|
|
27
|
+
GROUPED.match?(text) ? text.delete(",") : text
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def self.finite(number)
|
|
31
|
+
number.finite? ? number : nil
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def self.format(number)
|
|
35
|
+
return number.to_i.to_s if number == number.to_i
|
|
36
|
+
|
|
37
|
+
number.to_s
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def self.preview(value)
|
|
41
|
+
value.inspect.truncate(PREVIEW_LENGTH)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -3,8 +3,8 @@ module CompletionKit
|
|
|
3
3
|
class Regex
|
|
4
4
|
def call(target, config)
|
|
5
5
|
options = 0
|
|
6
|
-
options |= Regexp::IGNORECASE
|
|
7
|
-
options |= Regexp::MULTILINE if config
|
|
6
|
+
options |= Regexp::IGNORECASE unless Flag.on?(config, "case_sensitive")
|
|
7
|
+
options |= Regexp::MULTILINE if Flag.on?(config, "multiline")
|
|
8
8
|
pattern = Regexp.new(config["pattern"].to_s, options)
|
|
9
9
|
|
|
10
10
|
if pattern.match?(target.to_s)
|
|
@@ -8,7 +8,10 @@ module CompletionKit
|
|
|
8
8
|
"regex" => Regex,
|
|
9
9
|
"valid_json" => ValidJson,
|
|
10
10
|
"json_path_equals" => JsonPathEquals,
|
|
11
|
-
"length_bounds" => LengthBounds
|
|
11
|
+
"length_bounds" => LengthBounds,
|
|
12
|
+
"list_overlap" => ListOverlap,
|
|
13
|
+
"numeric_bounds" => NumericBounds,
|
|
14
|
+
"numeric_equals" => NumericEquals
|
|
12
15
|
}.freeze
|
|
13
16
|
|
|
14
17
|
REQUIRED_KEYS = {
|
|
@@ -18,10 +21,43 @@ module CompletionKit
|
|
|
18
21
|
"regex" => %w[pattern],
|
|
19
22
|
"valid_json" => [],
|
|
20
23
|
"json_path_equals" => %w[json_path expected],
|
|
21
|
-
"length_bounds" => []
|
|
24
|
+
"length_bounds" => [],
|
|
25
|
+
"list_overlap" => %w[value],
|
|
26
|
+
"numeric_bounds" => [],
|
|
27
|
+
"numeric_equals" => %w[value]
|
|
22
28
|
}.freeze
|
|
23
29
|
|
|
24
|
-
|
|
30
|
+
CONFIG_KEYS = {
|
|
31
|
+
"contains" => %w[value case_sensitive trim],
|
|
32
|
+
"not_contains" => %w[value case_sensitive trim],
|
|
33
|
+
"equals" => %w[value case_sensitive trim],
|
|
34
|
+
"regex" => %w[pattern case_sensitive multiline],
|
|
35
|
+
"valid_json" => [],
|
|
36
|
+
"json_path_equals" => %w[json_path expected],
|
|
37
|
+
"length_bounds" => %w[min max],
|
|
38
|
+
"list_overlap" => %w[value score_by min case_sensitive],
|
|
39
|
+
"numeric_bounds" => %w[min max],
|
|
40
|
+
"numeric_equals" => %w[value tolerance]
|
|
41
|
+
}.freeze
|
|
42
|
+
|
|
43
|
+
SHARED_KEYS = %w[check_kind target target_path compare_to expected_path].freeze
|
|
44
|
+
FLAG_KEYS = %w[case_sensitive multiline trim].freeze
|
|
45
|
+
NUMBER_KEYS = %w[min max tolerance].freeze
|
|
46
|
+
DEFAULTS = { "target" => "response_text", "score_by" => ListOverlap::DEFAULT_MEASURE }.freeze
|
|
47
|
+
|
|
48
|
+
EXPECTED_KEYS = {
|
|
49
|
+
"contains" => "value",
|
|
50
|
+
"not_contains" => "value",
|
|
51
|
+
"equals" => "value",
|
|
52
|
+
"json_path_equals" => "expected",
|
|
53
|
+
"list_overlap" => "value",
|
|
54
|
+
"numeric_equals" => "value"
|
|
55
|
+
}.freeze
|
|
56
|
+
|
|
57
|
+
RAW_TARGET_KINDS = %w[list_overlap numeric_bounds numeric_equals].freeze
|
|
58
|
+
RAW_EXPECTED_KINDS = %w[json_path_equals list_overlap numeric_equals].freeze
|
|
59
|
+
BOUNDED_KINDS = %w[length_bounds numeric_bounds].freeze
|
|
60
|
+
SCORING_KINDS = %w[list_overlap].freeze
|
|
25
61
|
|
|
26
62
|
KINDS = CHECKS.keys.freeze
|
|
27
63
|
|
|
@@ -33,13 +69,80 @@ module CompletionKit
|
|
|
33
69
|
REQUIRED_KEYS
|
|
34
70
|
end
|
|
35
71
|
|
|
72
|
+
def self.config_keys(kind)
|
|
73
|
+
CONFIG_KEYS.fetch(kind, [])
|
|
74
|
+
end
|
|
75
|
+
|
|
36
76
|
def self.compares_value?(kind)
|
|
37
|
-
|
|
77
|
+
EXPECTED_KEYS.key?(kind)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def self.expected_key(kind)
|
|
81
|
+
EXPECTED_KEYS[kind]
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def self.comparable_kinds
|
|
85
|
+
EXPECTED_KEYS.keys
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def self.raw_target?(kind)
|
|
89
|
+
RAW_TARGET_KINDS.include?(kind)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def self.raw_expected?(kind)
|
|
93
|
+
RAW_EXPECTED_KINDS.include?(kind)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def self.bounded?(kind)
|
|
97
|
+
BOUNDED_KINDS.include?(kind)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def self.scores?(kind)
|
|
101
|
+
SCORING_KINDS.include?(kind)
|
|
38
102
|
end
|
|
39
103
|
|
|
40
104
|
def self.fetch(kind)
|
|
41
105
|
CHECKS.fetch(kind).new
|
|
42
106
|
end
|
|
107
|
+
|
|
108
|
+
def self.unused_keys(config)
|
|
109
|
+
kind = config["check_kind"]
|
|
110
|
+
keys = config["compare_to"] == "expected" && compares_value?(kind) ? [expected_key(kind)] : %w[compare_to expected_path]
|
|
111
|
+
config["target"] == "json_path" ? keys : keys + %w[target_path]
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def self.canonical(config)
|
|
115
|
+
hash = config.to_h.stringify_keys
|
|
116
|
+
relevant = hash.slice(*SHARED_KEYS, *config_keys(hash["check_kind"])).except(*unused_keys(hash))
|
|
117
|
+
relevant.to_h { |key, value| [key, canonical_value(key, value)] }.reject { |key, value| omitted?(key, value) }
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def self.canonical_value(key, value)
|
|
121
|
+
case key
|
|
122
|
+
when *FLAG_KEYS then Flag.cast(value)
|
|
123
|
+
when *NUMBER_KEYS then NumericValue.parse(value) || value
|
|
124
|
+
when "value" then canonical_operand(value)
|
|
125
|
+
else value
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def self.canonical_operand(value)
|
|
130
|
+
return value.to_s if value.is_a?(::Numeric)
|
|
131
|
+
return value unless value.is_a?(String)
|
|
132
|
+
|
|
133
|
+
parsed = parse_json(value)
|
|
134
|
+
parsed.is_a?(Array) ? parsed : value
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def self.parse_json(text)
|
|
138
|
+
JSON.parse(text)
|
|
139
|
+
rescue JSON::ParserError
|
|
140
|
+
text
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def self.omitted?(key, value)
|
|
144
|
+
value.nil? || value == "" || (FLAG_KEYS.include?(key) && !value) || DEFAULTS[key] == value
|
|
145
|
+
end
|
|
43
146
|
end
|
|
44
147
|
end
|
|
45
148
|
end
|
|
@@ -3,28 +3,63 @@ module CompletionKit
|
|
|
3
3
|
module TargetResolver
|
|
4
4
|
TARGETS = %w[response_text input_data json_path].freeze
|
|
5
5
|
UNRESOLVED = Object.new.freeze
|
|
6
|
+
ARRAY_INDEX = /\A\d+\z/
|
|
7
|
+
ROOT = /\A\$(?:\.|\z)/
|
|
6
8
|
|
|
7
9
|
def self.call(response, config)
|
|
10
|
+
stringify(call_value(response, config))
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def self.call_value(response, config)
|
|
8
14
|
case config["target"]
|
|
9
15
|
when "input_data"
|
|
10
16
|
response.input_data
|
|
11
17
|
when "json_path"
|
|
12
|
-
|
|
18
|
+
dig_json(response.response_text, config["target_path"])
|
|
13
19
|
else
|
|
14
20
|
response.response_text
|
|
15
21
|
end
|
|
16
22
|
end
|
|
17
23
|
|
|
18
|
-
def self.
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
24
|
+
def self.dig_json(text, path)
|
|
25
|
+
walk(JSON.parse(text.to_s), path)
|
|
26
|
+
rescue JSON::ParserError
|
|
27
|
+
UNRESOLVED
|
|
28
|
+
end
|
|
22
29
|
|
|
23
|
-
|
|
30
|
+
def self.walk(document, path)
|
|
31
|
+
normalize_path(path).split(".").reduce(document) do |node, key|
|
|
32
|
+
case node
|
|
33
|
+
when Hash
|
|
34
|
+
return UNRESOLVED unless node.key?(key)
|
|
35
|
+
|
|
36
|
+
node[key]
|
|
37
|
+
when Array
|
|
38
|
+
index = array_index(node, key)
|
|
39
|
+
return UNRESOLVED if index.nil?
|
|
40
|
+
|
|
41
|
+
node[index]
|
|
42
|
+
else
|
|
43
|
+
return UNRESOLVED
|
|
44
|
+
end
|
|
24
45
|
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def self.normalize_path(path)
|
|
49
|
+
path.to_s.strip.sub(ROOT, "")
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def self.array_index(array, key)
|
|
53
|
+
return nil unless ARRAY_INDEX.match?(key)
|
|
54
|
+
|
|
55
|
+
index = key.to_i
|
|
56
|
+
index < array.length ? index : nil
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def self.stringify(value)
|
|
60
|
+
return value if value.equal?(UNRESOLVED)
|
|
61
|
+
|
|
25
62
|
value.to_s
|
|
26
|
-
rescue JSON::ParserError
|
|
27
|
-
UNRESOLVED
|
|
28
63
|
end
|
|
29
64
|
end
|
|
30
65
|
end
|