completion-kit 0.28.45 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/app/assets/javascripts/completion_kit/application.js +52 -23
  3. data/app/assets/stylesheets/completion_kit/application.css +56 -15
  4. data/app/controllers/completion_kit/api/v1/metric_versions_controller.rb +2 -0
  5. data/app/controllers/completion_kit/api/v1/metrics_controller.rb +1 -1
  6. data/app/controllers/completion_kit/api/v1/runs_controller.rb +4 -1
  7. data/app/controllers/completion_kit/metrics_controller.rb +48 -46
  8. data/app/controllers/completion_kit/onboarding_controller.rb +1 -1
  9. data/app/controllers/completion_kit/runs_controller.rb +18 -6
  10. data/app/helpers/completion_kit/application_helper.rb +110 -52
  11. data/app/jobs/completion_kit/check_review_job.rb +27 -6
  12. data/app/models/completion_kit/metric.rb +103 -28
  13. data/app/models/completion_kit/metric_version.rb +16 -1
  14. data/app/models/completion_kit/prompt.rb +1 -1
  15. data/app/models/completion_kit/review.rb +2 -0
  16. data/app/models/completion_kit/run.rb +36 -18
  17. data/app/services/completion_kit/checks/contains.rb +2 -8
  18. data/app/services/completion_kit/checks/equals.rb +2 -7
  19. data/app/services/completion_kit/checks/expected_resolver.rb +17 -3
  20. data/app/services/completion_kit/checks/flag.rb +13 -0
  21. data/app/services/completion_kit/checks/json_path_equals.rb +2 -15
  22. data/app/services/completion_kit/checks/labels.rb +104 -0
  23. data/app/services/completion_kit/checks/length_bounds.rb +4 -4
  24. data/app/services/completion_kit/checks/list_overlap.rb +96 -0
  25. data/app/services/completion_kit/checks/not_contains.rb +2 -8
  26. data/app/services/completion_kit/checks/numeric_bounds.rb +22 -0
  27. data/app/services/completion_kit/checks/numeric_equals.rb +37 -0
  28. data/app/services/completion_kit/checks/numeric_value.rb +45 -0
  29. data/app/services/completion_kit/checks/regex.rb +2 -2
  30. data/app/services/completion_kit/checks/registry.rb +107 -4
  31. data/app/services/completion_kit/checks/result.rb +5 -1
  32. data/app/services/completion_kit/checks/target_resolver.rb +43 -8
  33. data/app/services/completion_kit/checks/textual.rb +19 -0
  34. data/app/services/completion_kit/mcp_tools/metric_versions.rb +1 -1
  35. data/app/services/completion_kit/mcp_tools/metrics.rb +21 -7
  36. data/app/services/completion_kit/mcp_tools/prompts.rb +1 -1
  37. data/app/services/completion_kit/prompt_improvement_service.rb +1 -1
  38. data/app/services/completion_kit/promptfoo_importer.rb +4 -4
  39. data/app/services/completion_kit/run_comparison.rb +6 -0
  40. data/app/views/completion_kit/agreements/_buttons.html.erb +1 -1
  41. data/app/views/completion_kit/api_reference/_body.html.erb +9 -5
  42. data/app/views/completion_kit/metric_groups/_form.html.erb +13 -2
  43. data/app/views/completion_kit/metric_groups/edit.html.erb +1 -1
  44. data/app/views/completion_kit/metrics/_check_spec.html.erb +5 -11
  45. data/app/views/completion_kit/metrics/_form.html.erb +62 -47
  46. data/app/views/completion_kit/onboarding/show.html.erb +2 -2
  47. data/app/views/completion_kit/prompts/_form.html.erb +2 -1
  48. data/app/views/completion_kit/provider_credentials/_models_card.html.erb +2 -2
  49. data/app/views/completion_kit/responses/show.html.erb +1 -1
  50. data/app/views/completion_kit/runs/_compare_score.html.erb +1 -1
  51. data/app/views/completion_kit/runs/_form.html.erb +21 -11
  52. data/app/views/completion_kit/runs/_response_row.html.erb +5 -3
  53. data/app/views/completion_kit/runs/_row.html.erb +10 -8
  54. data/app/views/completion_kit/runs/_run_config.html.erb +1 -1
  55. data/app/views/completion_kit/runs/_status_panel.html.erb +2 -2
  56. data/app/views/completion_kit/runs/compare.html.erb +10 -6
  57. data/app/views/completion_kit/runs/edit.html.erb +1 -1
  58. data/app/views/completion_kit/runs/new.html.erb +1 -1
  59. data/app/views/completion_kit/shared/_branded_select.html.erb +1 -1
  60. data/app/views/layouts/completion_kit/application.html.erb +1 -1
  61. data/config/locales/en.yml +2 -0
  62. data/db/migrate/20260903000001_add_check_score_to_completion_kit_reviews.rb +5 -0
  63. data/db/migrate/20261013000001_drop_completion_kit_mcp_sessions.rb +16 -0
  64. data/lib/completion_kit/errors.rb +2 -2
  65. data/lib/completion_kit/version.rb +1 -1
  66. metadata +10 -1
@@ -2,14 +2,9 @@ module CompletionKit
2
2
  module Checks
3
3
  class Equals
4
4
  def call(target, config)
5
- actual = target.to_s
6
- expected = config["value"].to_s
7
- if config["trim"] == true
8
- actual = actual.strip
9
- expected = expected.strip
10
- end
5
+ expected, actual = Textual.operands(target, config)
11
6
 
12
- match = if config["case_sensitive"] == true
7
+ match = if Flag.on?(config, "case_sensitive")
13
8
  actual == expected
14
9
  else
15
10
  actual.casecmp?(expected)
@@ -4,13 +4,27 @@ module CompletionKit
4
4
  UNRESOLVED = TargetResolver::UNRESOLVED
5
5
 
6
6
  def self.call(response, config)
7
+ TargetResolver.stringify(resolve(response, config) { |raw| raw.to_s })
8
+ end
9
+
10
+ def self.call_value(response, config)
11
+ resolve(response, config) { |raw| parse(raw) }
12
+ end
13
+
14
+ def self.resolve(response, config)
7
15
  raw = response.expected_output
8
16
  return UNRESOLVED if raw.blank?
9
17
 
10
- path = config["expected_path"].to_s.strip
11
- return raw.to_s if path.empty?
18
+ path = TargetResolver.normalize_path(config["expected_path"])
19
+ return yield(raw) if path.empty?
20
+
21
+ TargetResolver.dig_json(raw.to_s, path)
22
+ end
12
23
 
13
- TargetResolver.resolve_json_path(raw.to_s, path)
24
+ def self.parse(raw)
25
+ JSON.parse(raw.to_s)
26
+ rescue JSON::ParserError
27
+ raw
14
28
  end
15
29
  end
16
30
  end
@@ -0,0 +1,13 @@
1
+ module CompletionKit
2
+ module Checks
3
+ module Flag
4
+ def self.on?(config, key)
5
+ cast(config[key])
6
+ end
7
+
8
+ def self.cast(value)
9
+ ActiveModel::Type::Boolean.new.cast(value) == true
10
+ end
11
+ end
12
+ end
13
+ end
@@ -1,13 +1,10 @@
1
1
  module CompletionKit
2
2
  module Checks
3
3
  class JsonPathEquals
4
- MISSING = Object.new
5
-
6
4
  def call(target, config)
7
- parsed = JSON.parse(target.to_s)
8
- value = dig(parsed, config["json_path"].to_s)
5
+ value = TargetResolver.walk(JSON.parse(target.to_s), config["json_path"])
9
6
 
10
- if value.equal?(MISSING)
7
+ if value.equal?(TargetResolver::UNRESOLVED)
11
8
  Result.new(passed: false, detail: "path #{config["json_path"]} not found")
12
9
  elsif value == config["expected"]
13
10
  Result.new(passed: true, detail: "#{config["json_path"]} == #{config["expected"].inspect}")
@@ -17,16 +14,6 @@ module CompletionKit
17
14
  rescue JSON::ParserError
18
15
  Result.new(passed: false, detail: "not valid JSON")
19
16
  end
20
-
21
- private
22
-
23
- def dig(data, path)
24
- path.split(".").reduce(data) do |node, key|
25
- return MISSING unless node.is_a?(Hash) && node.key?(key)
26
-
27
- node[key]
28
- end
29
- end
30
17
  end
31
18
  end
32
19
  end
@@ -0,0 +1,104 @@
1
+ module CompletionKit
2
+ module Checks
3
+ module Labels
4
+ KINDS = {
5
+ "contains" => "Contains a phrase",
6
+ "not_contains" => "Does not contain a phrase",
7
+ "equals" => "Equals exactly",
8
+ "regex" => "Matches a pattern",
9
+ "valid_json" => "Is valid JSON",
10
+ "json_path_equals" => "A JSON field equals a value",
11
+ "length_bounds" => "Length is within a range",
12
+ "list_overlap" => "How much of a list was found",
13
+ "numeric_bounds" => "A number is within a range",
14
+ "numeric_equals" => "A number is close enough"
15
+ }.freeze
16
+
17
+ TARGETS = {
18
+ "response_text" => "The response text",
19
+ "input_data" => "The input row",
20
+ "json_path" => "A value from the response JSON"
21
+ }.freeze
22
+
23
+ COMPARE_TO = {
24
+ "constant" => "A value you type",
25
+ "expected" => "Each row's expected value"
26
+ }.freeze
27
+
28
+ SCORE_BY = {
29
+ "recall" => "Coverage: how much of the expected list it found",
30
+ "precision" => "Accuracy: how much of what it returned was expected",
31
+ "f1" => "Balance: coverage and accuracy weighed equally",
32
+ "jaccard" => "Overlap: shared items out of every item in either list"
33
+ }.freeze
34
+
35
+ FIELDS = {
36
+ "check_kind" => "Rule",
37
+ "target" => "What to check",
38
+ "target_path" => "Path into the JSON",
39
+ "compare_to" => "Compare against",
40
+ "value" => "Text to look for",
41
+ "pattern" => "Pattern",
42
+ "json_path" => "JSON path",
43
+ "expected" => "Expected value",
44
+ "expected_path" => "Field in the answer key",
45
+ "score_by" => "How to score it",
46
+ "min" => "Minimum",
47
+ "max" => "Maximum",
48
+ "tolerance" => "How far off is still right",
49
+ "case_sensitive" => "Case sensitive",
50
+ "multiline" => "Multiline",
51
+ "trim" => "Trim whitespace"
52
+ }.freeze
53
+
54
+ FIELDS_BY_KIND = {
55
+ "length_bounds" => { "min" => "Shortest allowed", "max" => "Longest allowed" },
56
+ "numeric_bounds" => { "min" => "Lowest allowed", "max" => "Highest allowed" },
57
+ "list_overlap" => { "min" => "Lowest score that passes", "value" => "Expected list" },
58
+ "numeric_equals" => { "value" => "Expected number" }
59
+ }.freeze
60
+
61
+ HINTS = {
62
+ "score_by" => "Coverage ignores extra items, and accuracy ignores missing ones. Balance weighs the two equally. " \
63
+ "Overlap divides the shared items by every item in either list, so it never scores higher than the other three.",
64
+ "min" => "Leave blank for no minimum.",
65
+ "max" => "Leave blank for no maximum."
66
+ }.freeze
67
+
68
+ HINTS_BY_KIND = {
69
+ "list_overlap" => { "min" => "Give a value from 0 to 1, like 0.6 for 60%. Leave blank to require a perfect score." }
70
+ }.freeze
71
+
72
+ VALUES = { "compare_to" => COMPARE_TO, "score_by" => SCORE_BY }.freeze
73
+
74
+ def self.kind(kind)
75
+ KINDS.fetch(kind.to_s) { kind.to_s.humanize }
76
+ end
77
+
78
+ def self.target(target)
79
+ TARGETS.fetch(target.to_s) { target.to_s.humanize }
80
+ end
81
+
82
+ def self.field(key, kind = nil)
83
+ FIELDS_BY_KIND.dig(kind.to_s, key.to_s) || FIELDS.fetch(key.to_s) { key.to_s.humanize }
84
+ end
85
+
86
+ def self.hint(key, kind = nil)
87
+ HINTS_BY_KIND.dig(kind.to_s, key.to_s) || HINTS[key.to_s]
88
+ end
89
+
90
+ def self.value(key, value)
91
+ VALUES.fetch(key.to_s, {}).fetch(value.to_s, value)
92
+ end
93
+
94
+ def self.worded?(key)
95
+ VALUES.key?(key.to_s)
96
+ end
97
+
98
+ def self.in_form_order(config)
99
+ order = FIELDS.keys
100
+ config.sort_by.with_index { |(key, _value), index| [order.index(key) || order.length, index] }.to_h
101
+ end
102
+ end
103
+ end
104
+ end
@@ -3,13 +3,13 @@ module CompletionKit
3
3
  class LengthBounds
4
4
  def call(target, config)
5
5
  length = target.to_s.length
6
- min = config["min"] && config["min"].to_i
7
- max = config["max"] && config["max"].to_i
6
+ min = NumericValue.parse(config["min"])
7
+ max = NumericValue.parse(config["max"])
8
8
 
9
9
  if min && length < min
10
- Result.new(passed: false, detail: "length #{length} below min #{min}")
10
+ Result.new(passed: false, detail: "length #{length} below min #{NumericValue.format(min)}")
11
11
  elsif max && length > max
12
- Result.new(passed: false, detail: "length #{length} above max #{max}")
12
+ Result.new(passed: false, detail: "length #{length} above max #{NumericValue.format(max)}")
13
13
  else
14
14
  Result.new(passed: true, detail: "length #{length} within bounds")
15
15
  end
@@ -0,0 +1,96 @@
1
+ module CompletionKit
2
+ module Checks
3
+ class ListOverlap
4
+ MEASURES = %w[recall precision f1 jaccard].freeze
5
+ DEFAULT_MEASURE = "recall".freeze
6
+ DEFAULT_MIN = 1.0
7
+ SEPARATORS = /[,;\n]/
8
+ DECORATION = /\A(?:["'`]+|[-*•]\s+)|["'`]+\z/
9
+ PLAIN_NUMBER = /\A-?(?:0|[1-9]\d*)(?:\.\d+)?\z/
10
+
11
+ def call(target, config)
12
+ actual = to_set(target, config)
13
+ expected = to_set(config["value"], config)
14
+ measure = MEASURES.include?(config["score_by"]) ? config["score_by"] : DEFAULT_MEASURE
15
+ overlap = (actual & expected).length
16
+ score = score_for(measure, overlap, actual.length, expected.length).round(4)
17
+ threshold = NumericValue.parse(config["min"]) || DEFAULT_MIN
18
+
19
+ Result.new(
20
+ passed: score >= threshold,
21
+ detail: "found #{overlap} of #{expected.length} expected, #{actual.length} returned (score #{score.round(2)})",
22
+ score: score
23
+ )
24
+ end
25
+
26
+ private
27
+
28
+ def score_for(measure, overlap, actual_size, expected_size)
29
+ return 1.0 if actual_size.zero? && expected_size.zero?
30
+
31
+ precision = ratio(overlap, actual_size)
32
+ recall = expected_size.zero? ? 1.0 : ratio(overlap, expected_size)
33
+ case measure
34
+ when "precision" then precision
35
+ when "jaccard" then ratio(overlap, actual_size + expected_size - overlap)
36
+ when "f1" then f1(precision, recall)
37
+ else recall
38
+ end
39
+ end
40
+
41
+ def ratio(numerator, denominator)
42
+ return 0.0 if denominator.zero?
43
+
44
+ numerator.to_f / denominator
45
+ end
46
+
47
+ def f1(precision, recall)
48
+ return 0.0 if (precision + recall).zero?
49
+
50
+ (2 * precision * recall) / (precision + recall)
51
+ end
52
+
53
+ def to_set(value, config)
54
+ items = coerce_list(value).map { |item| normalize(item, config) }
55
+ items.reject(&:empty?).uniq
56
+ end
57
+
58
+ def normalize(item, config)
59
+ return number_text(item.to_s) if item.is_a?(::Numeric)
60
+
61
+ text = item.to_s.strip.gsub(DECORATION, "").strip
62
+ return number_text(text) if PLAIN_NUMBER.match?(text)
63
+
64
+ Flag.on?(config, "case_sensitive") ? text : text.downcase
65
+ end
66
+
67
+ def number_text(text)
68
+ number = BigDecimal(text)
69
+ number.frac.zero? ? number.to_i.to_s : number.to_s("F")
70
+ end
71
+
72
+ def coerce_list(value)
73
+ case value
74
+ when Array then value
75
+ when Hash, nil then []
76
+ else
77
+ text = value.to_s.strip
78
+ return [] if text.empty?
79
+
80
+ from_json(text) || text.split(SEPARATORS)
81
+ end
82
+ end
83
+
84
+ def from_json(text)
85
+ parsed = JSON.parse(text)
86
+ return parsed if parsed.is_a?(Array)
87
+ return parsed.split(SEPARATORS) if parsed.is_a?(String)
88
+ return [] if parsed.is_a?(Hash)
89
+
90
+ nil
91
+ rescue JSON::ParserError
92
+ nil
93
+ end
94
+ end
95
+ end
96
+ end
@@ -2,15 +2,9 @@ module CompletionKit
2
2
  module Checks
3
3
  class NotContains
4
4
  def call(target, config)
5
- value = config["value"].to_s
6
- haystack = target.to_s
7
- present = if config["case_sensitive"] == true
8
- haystack.include?(value)
9
- else
10
- haystack.downcase.include?(value.downcase)
11
- end
5
+ value, haystack = Textual.operands(target, config)
12
6
 
13
- if present
7
+ if Textual.include?(haystack, value, config)
14
8
  Result.new(passed: false, detail: "contains #{value.inspect}")
15
9
  else
16
10
  Result.new(passed: true, detail: "does not contain #{value.inspect}")
@@ -0,0 +1,22 @@
1
+ module CompletionKit
2
+ module Checks
3
+ class NumericBounds
4
+ def call(target, config)
5
+ number = NumericValue.parse(target)
6
+ return Result.new(passed: false, detail: "#{NumericValue.preview(target)} is not a number") if number.nil?
7
+
8
+ min = NumericValue.parse(config["min"])
9
+ max = NumericValue.parse(config["max"])
10
+ shown = NumericValue.format(number)
11
+
12
+ if min && number < min
13
+ Result.new(passed: false, detail: "#{shown} is below min #{NumericValue.format(min)}")
14
+ elsif max && number > max
15
+ Result.new(passed: false, detail: "#{shown} is above max #{NumericValue.format(max)}")
16
+ else
17
+ Result.new(passed: true, detail: "#{shown} is within bounds")
18
+ end
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,37 @@
1
+ module CompletionKit
2
+ module Checks
3
+ class NumericEquals
4
+ PERCENT = /\A(.*)%\z/
5
+
6
+ def call(target, config)
7
+ actual = NumericValue.decimal(target)
8
+ return Result.new(passed: false, detail: "#{NumericValue.preview(target)} is not a number") if actual.nil?
9
+
10
+ expected = NumericValue.decimal(config["value"])
11
+ return Result.new(passed: false, detail: "expected #{NumericValue.preview(config["value"])} is not a number") if expected.nil?
12
+
13
+ difference = (actual - expected).abs
14
+ allowed = self.class.allowance(config["tolerance"], expected)
15
+ shown = "#{NumericValue.format(actual)} vs #{NumericValue.format(expected)}"
16
+
17
+ if difference <= allowed
18
+ Result.new(passed: true, detail: "#{shown}, within #{NumericValue.format(allowed)}")
19
+ else
20
+ Result.new(passed: false, detail: "#{shown}, off by #{NumericValue.format(difference)} (allowed #{NumericValue.format(allowed)})")
21
+ end
22
+ end
23
+
24
+ def self.allowance(raw, expected)
25
+ share = PERCENT.match(raw.to_s)
26
+ return (NumericValue.decimal(share[1]) || BigDecimal("0")) / 100 * expected.abs if share
27
+
28
+ NumericValue.decimal(raw) || BigDecimal("0")
29
+ end
30
+
31
+ def self.tolerance_value(raw)
32
+ share = PERCENT.match(raw.to_s)
33
+ NumericValue.parse(share ? share[1] : raw)
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,45 @@
1
+ module CompletionKit
2
+ module Checks
3
+ module NumericValue
4
+ STRIPPED = /[\s_   ]/
5
+ CURRENCY = /\A([+-]?)[$£€]/
6
+ GROUPED = /\A[+-]?[1-9]\d{0,2}(?:,\d{3})+(?:\.\d*)?(?:[eE][+-]?\d+)?\z/
7
+ NUMERIC = /\A[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?\z/
8
+ PREVIEW_LENGTH = 40
9
+
10
+ def self.parse(value)
11
+ return finite(value.to_f) if value.is_a?(::Numeric)
12
+
13
+ text = clean(value)
14
+ return nil unless NUMERIC.match?(text)
15
+
16
+ finite(Float(text))
17
+ end
18
+
19
+ def self.decimal(value)
20
+ return nil if parse(value).nil?
21
+
22
+ BigDecimal(value.is_a?(::Numeric) ? value.to_s : clean(value))
23
+ end
24
+
25
+ def self.clean(value)
26
+ text = value.to_s.gsub(STRIPPED, "").sub(CURRENCY) { ::Regexp.last_match(1) }
27
+ GROUPED.match?(text) ? text.delete(",") : text
28
+ end
29
+
30
+ def self.finite(number)
31
+ number.finite? ? number : nil
32
+ end
33
+
34
+ def self.format(number)
35
+ return number.to_i.to_s if number == number.to_i
36
+
37
+ number.to_s
38
+ end
39
+
40
+ def self.preview(value)
41
+ value.inspect.truncate(PREVIEW_LENGTH)
42
+ end
43
+ end
44
+ end
45
+ end
@@ -3,8 +3,8 @@ module CompletionKit
3
3
  class Regex
4
4
  def call(target, config)
5
5
  options = 0
6
- options |= Regexp::IGNORECASE if config["case_sensitive"] == false
7
- options |= Regexp::MULTILINE if config["multiline"] == true
6
+ options |= Regexp::IGNORECASE unless Flag.on?(config, "case_sensitive")
7
+ options |= Regexp::MULTILINE if Flag.on?(config, "multiline")
8
8
  pattern = Regexp.new(config["pattern"].to_s, options)
9
9
 
10
10
  if pattern.match?(target.to_s)
@@ -8,7 +8,10 @@ module CompletionKit
8
8
  "regex" => Regex,
9
9
  "valid_json" => ValidJson,
10
10
  "json_path_equals" => JsonPathEquals,
11
- "length_bounds" => LengthBounds
11
+ "length_bounds" => LengthBounds,
12
+ "list_overlap" => ListOverlap,
13
+ "numeric_bounds" => NumericBounds,
14
+ "numeric_equals" => NumericEquals
12
15
  }.freeze
13
16
 
14
17
  REQUIRED_KEYS = {
@@ -18,10 +21,43 @@ module CompletionKit
18
21
  "regex" => %w[pattern],
19
22
  "valid_json" => [],
20
23
  "json_path_equals" => %w[json_path expected],
21
- "length_bounds" => []
24
+ "length_bounds" => [],
25
+ "list_overlap" => %w[value],
26
+ "numeric_bounds" => [],
27
+ "numeric_equals" => %w[value]
22
28
  }.freeze
23
29
 
24
- VALUE_COMPARISON_KINDS = %w[contains not_contains equals].freeze
30
+ CONFIG_KEYS = {
31
+ "contains" => %w[value case_sensitive trim],
32
+ "not_contains" => %w[value case_sensitive trim],
33
+ "equals" => %w[value case_sensitive trim],
34
+ "regex" => %w[pattern case_sensitive multiline],
35
+ "valid_json" => [],
36
+ "json_path_equals" => %w[json_path expected],
37
+ "length_bounds" => %w[min max],
38
+ "list_overlap" => %w[value score_by min case_sensitive],
39
+ "numeric_bounds" => %w[min max],
40
+ "numeric_equals" => %w[value tolerance]
41
+ }.freeze
42
+
43
+ SHARED_KEYS = %w[check_kind target target_path compare_to expected_path].freeze
44
+ FLAG_KEYS = %w[case_sensitive multiline trim].freeze
45
+ NUMBER_KEYS = %w[min max tolerance].freeze
46
+ DEFAULTS = { "target" => "response_text", "score_by" => ListOverlap::DEFAULT_MEASURE }.freeze
47
+
48
+ EXPECTED_KEYS = {
49
+ "contains" => "value",
50
+ "not_contains" => "value",
51
+ "equals" => "value",
52
+ "json_path_equals" => "expected",
53
+ "list_overlap" => "value",
54
+ "numeric_equals" => "value"
55
+ }.freeze
56
+
57
+ RAW_TARGET_KINDS = %w[list_overlap numeric_bounds numeric_equals].freeze
58
+ RAW_EXPECTED_KINDS = %w[json_path_equals list_overlap numeric_equals].freeze
59
+ BOUNDED_KINDS = %w[length_bounds numeric_bounds].freeze
60
+ SCORING_KINDS = %w[list_overlap].freeze
25
61
 
26
62
  KINDS = CHECKS.keys.freeze
27
63
 
@@ -33,13 +69,80 @@ module CompletionKit
33
69
  REQUIRED_KEYS
34
70
  end
35
71
 
72
+ def self.config_keys(kind)
73
+ CONFIG_KEYS.fetch(kind, [])
74
+ end
75
+
36
76
  def self.compares_value?(kind)
37
- VALUE_COMPARISON_KINDS.include?(kind)
77
+ EXPECTED_KEYS.key?(kind)
78
+ end
79
+
80
+ def self.expected_key(kind)
81
+ EXPECTED_KEYS[kind]
82
+ end
83
+
84
+ def self.comparable_kinds
85
+ EXPECTED_KEYS.keys
86
+ end
87
+
88
+ def self.raw_target?(kind)
89
+ RAW_TARGET_KINDS.include?(kind)
90
+ end
91
+
92
+ def self.raw_expected?(kind)
93
+ RAW_EXPECTED_KINDS.include?(kind)
94
+ end
95
+
96
+ def self.bounded?(kind)
97
+ BOUNDED_KINDS.include?(kind)
98
+ end
99
+
100
+ def self.scores?(kind)
101
+ SCORING_KINDS.include?(kind)
38
102
  end
39
103
 
40
104
  def self.fetch(kind)
41
105
  CHECKS.fetch(kind).new
42
106
  end
107
+
108
+ def self.unused_keys(config)
109
+ kind = config["check_kind"]
110
+ keys = config["compare_to"] == "expected" && compares_value?(kind) ? [expected_key(kind)] : %w[compare_to expected_path]
111
+ config["target"] == "json_path" ? keys : keys + %w[target_path]
112
+ end
113
+
114
+ def self.canonical(config)
115
+ hash = config.to_h.stringify_keys
116
+ relevant = hash.slice(*SHARED_KEYS, *config_keys(hash["check_kind"])).except(*unused_keys(hash))
117
+ relevant.to_h { |key, value| [key, canonical_value(key, value)] }.reject { |key, value| omitted?(key, value) }
118
+ end
119
+
120
+ def self.canonical_value(key, value)
121
+ case key
122
+ when *FLAG_KEYS then Flag.cast(value)
123
+ when *NUMBER_KEYS then NumericValue.parse(value) || value
124
+ when "value" then canonical_operand(value)
125
+ else value
126
+ end
127
+ end
128
+
129
+ def self.canonical_operand(value)
130
+ return value.to_s if value.is_a?(::Numeric)
131
+ return value unless value.is_a?(String)
132
+
133
+ parsed = parse_json(value)
134
+ parsed.is_a?(Array) ? parsed : value
135
+ end
136
+
137
+ def self.parse_json(text)
138
+ JSON.parse(text)
139
+ rescue JSON::ParserError
140
+ text
141
+ end
142
+
143
+ def self.omitted?(key, value)
144
+ value.nil? || value == "" || (FLAG_KEYS.include?(key) && !value) || DEFAULTS[key] == value
145
+ end
43
146
  end
44
147
  end
45
148
  end
@@ -1,5 +1,9 @@
1
1
  module CompletionKit
2
2
  module Checks
3
- Result = Data.define(:passed, :detail)
3
+ Result = Data.define(:passed, :detail, :score) do
4
+ def initialize(passed:, detail:, score: nil)
5
+ super
6
+ end
7
+ end
4
8
  end
5
9
  end
@@ -3,28 +3,63 @@ module CompletionKit
3
3
  module TargetResolver
4
4
  TARGETS = %w[response_text input_data json_path].freeze
5
5
  UNRESOLVED = Object.new.freeze
6
+ ARRAY_INDEX = /\A\d+\z/
7
+ ROOT = /\A\$(?:\.|\z)/
6
8
 
7
9
  def self.call(response, config)
10
+ stringify(call_value(response, config))
11
+ end
12
+
13
+ def self.call_value(response, config)
8
14
  case config["target"]
9
15
  when "input_data"
10
16
  response.input_data
11
17
  when "json_path"
12
- resolve_json_path(response.response_text, config["target_path"].to_s)
18
+ dig_json(response.response_text, config["target_path"])
13
19
  else
14
20
  response.response_text
15
21
  end
16
22
  end
17
23
 
18
- def self.resolve_json_path(text, path)
19
- parsed = JSON.parse(text.to_s)
20
- value = path.split(".").reduce(parsed) do |node, key|
21
- return UNRESOLVED unless node.is_a?(Hash) && node.key?(key)
24
+ def self.dig_json(text, path)
25
+ walk(JSON.parse(text.to_s), path)
26
+ rescue JSON::ParserError
27
+ UNRESOLVED
28
+ end
22
29
 
23
- node[key]
30
+ def self.walk(document, path)
31
+ normalize_path(path).split(".").reduce(document) do |node, key|
32
+ case node
33
+ when Hash
34
+ return UNRESOLVED unless node.key?(key)
35
+
36
+ node[key]
37
+ when Array
38
+ index = array_index(node, key)
39
+ return UNRESOLVED if index.nil?
40
+
41
+ node[index]
42
+ else
43
+ return UNRESOLVED
44
+ end
24
45
  end
46
+ end
47
+
48
+ def self.normalize_path(path)
49
+ path.to_s.strip.sub(ROOT, "")
50
+ end
51
+
52
+ def self.array_index(array, key)
53
+ return nil unless ARRAY_INDEX.match?(key)
54
+
55
+ index = key.to_i
56
+ index < array.length ? index : nil
57
+ end
58
+
59
+ def self.stringify(value)
60
+ return value if value.equal?(UNRESOLVED)
61
+
25
62
  value.to_s
26
- rescue JSON::ParserError
27
- UNRESOLVED
28
63
  end
29
64
  end
30
65
  end