completion-kit 0.28.46 → 0.29.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +48 -3
  3. data/app/assets/javascripts/completion_kit/application.js +435 -23
  4. data/app/assets/stylesheets/completion_kit/application.css +44 -14
  5. data/app/controllers/completion_kit/api/v1/metric_versions_controller.rb +2 -0
  6. data/app/controllers/completion_kit/api/v1/metrics_controller.rb +1 -1
  7. data/app/controllers/completion_kit/api/v1/runs_controller.rb +4 -1
  8. data/app/controllers/completion_kit/mcp_controller.rb +4 -0
  9. data/app/controllers/completion_kit/metrics_controller.rb +48 -46
  10. data/app/controllers/completion_kit/runs_controller.rb +4 -1
  11. data/app/helpers/completion_kit/application_helper.rb +99 -45
  12. data/app/jobs/completion_kit/check_review_job.rb +27 -6
  13. data/app/models/completion_kit/dataset.rb +53 -4
  14. data/app/models/completion_kit/metric.rb +103 -28
  15. data/app/models/completion_kit/metric_version.rb +16 -1
  16. data/app/models/completion_kit/prompt.rb +8 -0
  17. data/app/models/completion_kit/review.rb +2 -0
  18. data/app/models/completion_kit/run.rb +20 -9
  19. data/app/services/completion_kit/api_config.rb +2 -1
  20. data/app/services/completion_kit/checks/contains.rb +2 -8
  21. data/app/services/completion_kit/checks/equals.rb +2 -7
  22. data/app/services/completion_kit/checks/expected_resolver.rb +17 -3
  23. data/app/services/completion_kit/checks/flag.rb +13 -0
  24. data/app/services/completion_kit/checks/json_path_equals.rb +2 -15
  25. data/app/services/completion_kit/checks/labels.rb +104 -0
  26. data/app/services/completion_kit/checks/length_bounds.rb +4 -4
  27. data/app/services/completion_kit/checks/list_overlap.rb +96 -0
  28. data/app/services/completion_kit/checks/not_contains.rb +2 -8
  29. data/app/services/completion_kit/checks/numeric_bounds.rb +22 -0
  30. data/app/services/completion_kit/checks/numeric_equals.rb +37 -0
  31. data/app/services/completion_kit/checks/numeric_value.rb +45 -0
  32. data/app/services/completion_kit/checks/regex.rb +14 -4
  33. data/app/services/completion_kit/checks/registry.rb +107 -4
  34. data/app/services/completion_kit/checks/result.rb +5 -1
  35. data/app/services/completion_kit/checks/target_resolver.rb +43 -8
  36. data/app/services/completion_kit/checks/textual.rb +19 -0
  37. data/app/services/completion_kit/csv_processor.rb +14 -2
  38. data/app/services/completion_kit/mcp_tools/metrics.rb +21 -7
  39. data/app/services/completion_kit/model_discovery_service.rb +2 -1
  40. data/app/services/completion_kit/ollama_client.rb +19 -2
  41. data/app/services/completion_kit/promptfoo_importer.rb +4 -4
  42. data/app/services/completion_kit/provider_endpoint.rb +10 -0
  43. data/app/services/completion_kit/run_comparison.rb +6 -0
  44. data/app/views/completion_kit/api_reference/_authentication.html.erb +2 -2
  45. data/app/views/completion_kit/api_reference/_body.html.erb +7 -30
  46. data/app/views/completion_kit/api_reference/_example.html.erb +1 -1
  47. data/app/views/completion_kit/api_reference/_resource_card.html.erb +1 -1
  48. data/app/views/completion_kit/datasets/_form.html.erb +0 -28
  49. data/app/views/completion_kit/datasets/index.html.erb +1 -1
  50. data/app/views/completion_kit/metric_groups/_form.html.erb +1 -20
  51. data/app/views/completion_kit/metric_groups/index.html.erb +1 -1
  52. data/app/views/completion_kit/metrics/_check_spec.html.erb +5 -11
  53. data/app/views/completion_kit/metrics/_form.html.erb +62 -47
  54. data/app/views/completion_kit/metrics/_starter_card.html.erb +1 -1
  55. data/app/views/completion_kit/metrics/index.html.erb +2 -2
  56. data/app/views/completion_kit/metrics/show.html.erb +4 -14
  57. data/app/views/completion_kit/prompts/_form.html.erb +1 -1
  58. data/app/views/completion_kit/prompts/index.html.erb +3 -3
  59. data/app/views/completion_kit/prompts/show.html.erb +8 -8
  60. data/app/views/completion_kit/provider_credentials/_models_card.html.erb +2 -2
  61. data/app/views/completion_kit/responses/show.html.erb +1 -1
  62. data/app/views/completion_kit/runs/_compare_score.html.erb +1 -1
  63. data/app/views/completion_kit/runs/_form.html.erb +4 -174
  64. data/app/views/completion_kit/runs/_response_row.html.erb +5 -5
  65. data/app/views/completion_kit/runs/_row.html.erb +13 -11
  66. data/app/views/completion_kit/runs/_run_config.html.erb +2 -2
  67. data/app/views/completion_kit/runs/_status_panel.html.erb +2 -2
  68. data/app/views/completion_kit/runs/compare.html.erb +10 -6
  69. data/app/views/completion_kit/runs/show.html.erb +5 -19
  70. data/app/views/completion_kit/shared/_branded_select.html.erb +1 -1
  71. data/app/views/completion_kit/tags/index.html.erb +1 -1
  72. data/app/views/layouts/completion_kit/application.html.erb +2 -2
  73. data/config/locales/en.yml +2 -0
  74. data/db/migrate/20260903000001_add_check_score_to_completion_kit_reviews.rb +5 -0
  75. data/lib/completion_kit/version.rb +1 -1
  76. data/lib/completion_kit.rb +14 -0
  77. data/lib/generators/completion_kit/templates/initializer.rb +18 -0
  78. metadata +9 -1
@@ -2053,6 +2053,10 @@ label.ck-checkbox[hidden] {
2053
2053
  flex: 1;
2054
2054
  }
2055
2055
 
2056
+ .ck-field-row:not(:has(> .ck-field:not([hidden]))) {
2057
+ display: none;
2058
+ }
2059
+
2056
2060
  .ck-metric-type-chip {
2057
2061
  text-transform: uppercase;
2058
2062
  letter-spacing: 0.04em;
@@ -2060,32 +2064,42 @@ label.ck-checkbox[hidden] {
2060
2064
  }
2061
2065
 
2062
2066
  .ck-check-spec {
2063
- display: flex;
2064
- flex-direction: column;
2065
- gap: 0.4rem;
2067
+ display: grid;
2068
+ grid-template-columns: max-content minmax(0, 1fr);
2069
+ column-gap: 1rem;
2070
+ row-gap: 0.4rem;
2066
2071
  margin: 0.5rem 0 0;
2067
2072
  }
2068
2073
 
2069
2074
  .ck-check-spec__row {
2070
- display: flex;
2071
- gap: 0.75rem;
2075
+ display: grid;
2076
+ grid-column: 1 / -1;
2077
+ grid-template-columns: subgrid;
2072
2078
  align-items: baseline;
2073
2079
  }
2074
2080
 
2075
2081
  .ck-check-spec__term {
2076
- flex: 0 0 7rem;
2077
2082
  margin: 0;
2078
2083
  font-size: 0.8rem;
2079
2084
  text-transform: uppercase;
2080
2085
  letter-spacing: 0.04em;
2086
+ white-space: nowrap;
2081
2087
  color: var(--ck-muted);
2082
2088
  }
2083
2089
 
2084
2090
  .ck-check-spec__val {
2085
2091
  margin: 0;
2092
+ min-width: 0;
2093
+ overflow-wrap: anywhere;
2086
2094
  color: var(--ck-text);
2087
2095
  }
2088
2096
 
2097
+ @media (max-width: 640px) {
2098
+ .ck-check-spec { grid-template-columns: minmax(0, 1fr); row-gap: 0.6rem; }
2099
+ .ck-check-spec__row { grid-template-columns: minmax(0, 1fr); row-gap: 0.15rem; }
2100
+ .ck-check-spec__term { white-space: normal; }
2101
+ }
2102
+
2089
2103
  .ck-check-spec__code {
2090
2104
  font-family: var(--ck-mono);
2091
2105
  font-size: 0.85rem;
@@ -2098,6 +2112,7 @@ label.ck-checkbox[hidden] {
2098
2112
 
2099
2113
  .ck-select {
2100
2114
  position: relative;
2115
+ min-width: 0;
2101
2116
  }
2102
2117
 
2103
2118
  .ck-select__trigger {
@@ -3271,13 +3286,13 @@ select.ck-input {
3271
3286
 
3272
3287
  .ck-run-compare-table td { vertical-align: top; }
3273
3288
 
3274
- .ck-table-scroll { overflow-x: auto; }
3289
+ .ck-table-scroll { position: relative; overflow-x: auto; }
3275
3290
 
3276
3291
  .ck-run-compare-table th { white-space: nowrap; }
3277
3292
  .ck-run-compare-table th:nth-child(1) { width: 7rem; }
3278
- .ck-run-compare-table th:nth-child(2) { width: 15rem; }
3279
- .ck-run-compare-table th:nth-child(3) { width: 12rem; }
3280
- .ck-run-compare-table th:nth-child(4) { width: 8rem; }
3293
+ .ck-run-compare-table th:nth-child(2) { width: 12rem; }
3294
+ .ck-run-compare-table th:nth-child(3) { width: 16rem; text-align: center; }
3295
+ .ck-run-compare-table th:nth-child(4) { width: 9rem; }
3281
3296
  .ck-run-compare-table th:nth-child(5) { width: auto; }
3282
3297
 
3283
3298
  .ck-run-compare-table th.ck-num,
@@ -3292,13 +3307,26 @@ select.ck-input {
3292
3307
  font-weight: 500;
3293
3308
  }
3294
3309
 
3295
- .ck-cmp-score__pair { display: inline-flex; align-items: center; gap: 0.45rem; }
3310
+ .ck-cmp-score__pair {
3311
+ display: grid;
3312
+ grid-template-columns: minmax(max-content, 1fr) auto minmax(max-content, 1fr);
3313
+ align-items: center;
3314
+ gap: 0.45rem;
3315
+ width: 100%;
3316
+ }
3317
+ .ck-cmp-score__pair > :first-child { justify-self: end; }
3318
+ .ck-cmp-score__pair > :last-child { justify-self: start; }
3319
+ @media (max-width: 640px) {
3320
+ .ck-cmp-score__pair { display: inline-grid; width: auto; }
3321
+ }
3322
+ .ck-cmp-score__pair .ck-badge { white-space: nowrap; }
3296
3323
  .ck-cmp-score__arrow { color: var(--ck-dim); font-size: 0.78rem; }
3297
3324
  .ck-cmp-score__link { display: inline-flex; text-decoration: none; border-radius: 7px; transition: box-shadow 0.12s ease; }
3298
3325
  .ck-cmp-score__link:hover { box-shadow: 0 0 0 1px var(--ck-accent); }
3299
3326
  .ck-cmp-score__none { color: var(--ck-dim); }
3300
3327
 
3301
- .ck-cmp-delta .ck-delta { display: inline-flex; align-items: baseline; gap: 0.2rem; }
3328
+ .ck-cmp-delta .ck-delta { display: inline-flex; align-items: baseline; gap: 0.2rem; white-space: nowrap; }
3329
+ .ck-cmp-delta__pair { display: inline-flex; align-items: baseline; justify-content: flex-end; gap: 0.35rem; }
3302
3330
  .ck-delta__arrow { font-size: 0.72rem; }
3303
3331
 
3304
3332
  .ck-cmp-ver__one, .ck-cmp-ver__pair {
@@ -5185,12 +5213,14 @@ a.tag-mark {
5185
5213
 
5186
5214
 
5187
5215
  .ck-input--error,
5188
- .ck-input[aria-invalid="true"] {
5216
+ .ck-input[aria-invalid="true"],
5217
+ .ck-select__trigger[aria-invalid="true"] {
5189
5218
  border-color: var(--ck-danger);
5190
5219
  background-color: rgba(248, 113, 113, 0.05);
5191
5220
  }
5192
5221
  .ck-input--error:focus,
5193
- .ck-input[aria-invalid="true"]:focus {
5222
+ .ck-input[aria-invalid="true"]:focus,
5223
+ .ck-select__trigger[aria-invalid="true"]:focus-visible {
5194
5224
  box-shadow: 0 0 0 3px rgba(248, 113, 113, 0.15);
5195
5225
  border-color: var(--ck-danger);
5196
5226
  }
@@ -16,6 +16,8 @@ module CompletionKit
16
16
  def publish
17
17
  @version.publish!
18
18
  render json: @version.reload
19
+ rescue ActiveRecord::RecordInvalid => e
20
+ render_validation_errors(e.record)
19
21
  end
20
22
 
21
23
  def destroy
@@ -70,7 +70,7 @@ module CompletionKit
70
70
  def metric_params
71
71
  params.permit(:name, :instruction, :metric_type,
72
72
  rubric_bands: [:stars, :description],
73
- check_config: %i[check_kind target target_path compare_to value pattern json_path expected expected_path min max case_sensitive multiline trim],
73
+ check_config: [*%i[check_kind target target_path compare_to value pattern json_path expected expected_path min max score_by tolerance case_sensitive multiline trim], { value: [] }],
74
74
  tag_names: [])
75
75
  end
76
76
  end
@@ -112,9 +112,12 @@ module CompletionKit
112
112
  right_score: r_review ? r_review.ai_score : nil,
113
113
  left_passed: l_review&.passed,
114
114
  right_passed: r_review&.passed,
115
+ left_check_score: l_review&.check_score&.to_f,
116
+ right_check_score: r_review&.check_score&.to_f,
115
117
  left_metric_version_id: l_review&.metric_version_id,
116
118
  right_metric_version_id: r_review&.metric_version_id,
117
- delta: (l_review&.ai_score && r_review&.ai_score) ? (r_review.ai_score.to_f - l_review.ai_score.to_f).round(2) : nil,
119
+ delta: CompletionKit::RunComparison.score_delta(l_review&.ai_score, r_review&.ai_score),
120
+ check_score_delta: CompletionKit::RunComparison.score_delta(l_review&.check_score, r_review&.check_score),
118
121
  result_change: CompletionKit::RunComparison.result_change(l_review&.passed, r_review&.passed)
119
122
  }
120
123
  end.compact
@@ -3,6 +3,7 @@ module CompletionKit
3
3
  PUBLIC_METHODS = %w[initialize notifications/initialized tools/list].freeze
4
4
 
5
5
  FOREIGN_KEY_MESSAGE = "This change references a record that does not exist or is still in use.".freeze
6
+ TIMEOUT_MESSAGE = "The input was too large or too complex to process.".freeze
6
7
 
7
8
  skip_before_action :authenticate_api!, only: [:stream, :handle], raise: false
8
9
 
@@ -47,6 +48,9 @@ module CompletionKit
47
48
  rescue ActiveRecord::InvalidForeignKey => e
48
49
  Rails.error.report(e, handled: true, context: { controller: "CompletionKit::McpController" })
49
50
  render json: jsonrpc_error(request_body.dig("id"), -32602, FOREIGN_KEY_MESSAGE), status: :ok
51
+ rescue Regexp::TimeoutError => e
52
+ Rails.error.report(e, handled: true, context: { controller: "CompletionKit::McpController" })
53
+ render json: jsonrpc_error(request_body.dig("id"), -32602, TIMEOUT_MESSAGE), status: :ok
50
54
  rescue StandardError => e
51
55
  Rails.error.report(e, handled: true, context: { controller: "CompletionKit::McpController" })
52
56
  render json: jsonrpc_error(request_body.dig("id"), -32603, "Internal error"), status: :ok
@@ -61,7 +61,9 @@ module CompletionKit
61
61
  @published_metric_version = MetricVersion.published.where(metric_id: @metric.id, current: true).first
62
62
  @improve_disagreement_count = @metric.check? ? 0 : Agreement.where(metric_id: @metric.id, verdict: "disagree", run_id: CompletionKit::Run.visible_run_ids).count
63
63
 
64
- if @edit_draft
64
+ if @edit_draft&.check?
65
+ @metric.check_config = @edit_draft.check_config
66
+ elsif @edit_draft
65
67
  @metric.instruction = @edit_draft.instruction
66
68
  @metric.rubric_bands = @edit_draft.rubric_bands
67
69
  end
@@ -78,17 +80,12 @@ module CompletionKit
78
80
  end
79
81
 
80
82
  def update
81
- meta_attrs = metric_params.except(:instruction, :rubric_bands, :check_config)
82
-
83
- unless @metric.update(meta_attrs)
84
- return render(:edit, status: :unprocessable_entity)
85
- end
83
+ attrs = metric_params
84
+ @metric.assign_attributes(attrs.except(:instruction, :rubric_bands, :check_config))
85
+ return update_check_definition(attrs.to_unsafe_h[:check_config]) if @metric.check?
86
+ return render(:edit, status: :unprocessable_entity) unless @metric.save
86
87
 
87
- if @metric.check?
88
- update_check_definition
89
- else
90
- update_judge_definition
91
- end
88
+ update_judge_definition
92
89
  end
93
90
 
94
91
  def destroy
@@ -157,7 +154,11 @@ module CompletionKit
157
154
  reverting = was_published_already && !version.current?
158
155
  previously_current = MetricVersion.current.find_by(metric_id: @metric.id)
159
156
 
160
- version.publish!
157
+ begin
158
+ version.publish!
159
+ rescue ActiveRecord::RecordInvalid => e
160
+ return redirect_to(metric_path(@metric), alert: "#{version.version_label} was not published. #{e.record.errors.full_messages.to_sentence}.")
161
+ end
161
162
 
162
163
  if reverting
163
164
  redirect_to metric_path(@metric),
@@ -207,28 +208,28 @@ module CompletionKit
207
208
  end
208
209
  end
209
210
 
210
- def update_check_definition
211
- raw = metric_params[:check_config]
212
- proposed = raw.nil? ? nil : normalize_check_config(raw)
211
+ def update_check_definition(proposed)
212
+ live = @metric.check_config
213
+ changed = !proposed.nil? && CompletionKit::Checks::Registry.canonical(proposed) != CompletionKit::Checks::Registry.canonical(live)
214
+ @metric.check_config = proposed if changed
215
+ return render(:edit, status: :unprocessable_entity) unless @metric.valid?
216
+ return save_check_draft(live, proposed) if changed && @metric.reviews.exists?
213
217
 
214
- unless !proposed.nil? && proposed != @metric.check_config
215
- return redirect_to(metric_path(@metric), notice: "Metric was successfully updated.")
216
- end
218
+ @metric.save!
219
+ MetricVersion.published.find_by(metric_id: @metric.id, current: true)&.update!(metric_type: "check", check_config: proposed) if changed
220
+ redirect_to metric_path(@metric), notice: "Metric was successfully updated."
221
+ end
217
222
 
218
- if @metric.reviews.exists?
219
- MetricVersion.drafts.where(metric_id: @metric.id, source: "edit").destroy_all
220
- draft = MetricVersion.create!(
221
- metric: @metric, metric_type: "check", check_config: proposed,
222
- state: "draft", source: "edit", current: false
223
- )
224
- redirect_to edit_metric_path(@metric),
225
- notice: "Saved as draft #{draft.version_label}. Publish to make these changes the metric's live version."
226
- else
227
- @metric.update!(check_config: proposed)
228
- current_pub = MetricVersion.published.where(metric_id: @metric.id, current: true).first
229
- current_pub&.update!(metric_type: "check", check_config: proposed)
230
- redirect_to metric_path(@metric), notice: "Metric was successfully updated."
231
- end
223
+ def save_check_draft(live, proposed)
224
+ @metric.check_config = live
225
+ @metric.save!(validate: false)
226
+ MetricVersion.drafts.where(metric_id: @metric.id, source: "edit").destroy_all
227
+ draft = MetricVersion.create!(
228
+ metric: @metric, metric_type: "check", check_config: proposed,
229
+ state: "draft", source: "edit", current: false
230
+ )
231
+ redirect_to edit_metric_path(@metric),
232
+ notice: "Saved as draft #{draft.version_label}. Publish to make these changes the metric's live version."
232
233
  end
233
234
 
234
235
  def set_metric
@@ -247,7 +248,7 @@ module CompletionKit
247
248
  def metric_params
248
249
  permitted = params.require(:metric).permit(:name, :instruction, :metric_type,
249
250
  rubric_bands: [:stars, :description],
250
- check_config: %i[check_kind target target_path compare_to value pattern json_path expected expected_path min max case_sensitive multiline trim],
251
+ check_config: %i[check_kind target target_path compare_to value pattern json_path expected expected_path min max score_by tolerance case_sensitive multiline trim],
251
252
  tag_names: [])
252
253
  permitted[:check_config] = normalize_check_config(permitted[:check_config]) if permitted.key?(:check_config)
253
254
  permitted
@@ -255,22 +256,23 @@ module CompletionKit
255
256
 
256
257
  def normalize_check_config(config)
257
258
  hash = config.to_unsafe_h.stringify_keys
258
- %w[min max].each { |key| hash[key] = hash[key].to_i if hash[key].present? }
259
+ %w[min max tolerance].each { |key| hash[key] = coerce_number(hash[key]) if hash[key].present? }
259
260
  %w[case_sensitive multiline trim].each { |key| hash[key] = ActiveModel::Type::Boolean.new.cast(hash[key]) if hash.key?(key) }
260
- hash["expected"] = coerce_scalar(hash["expected"]) if hash["expected"].present?
261
- if hash["compare_to"] == "expected"
262
- hash.delete("value")
263
- else
264
- hash.delete("compare_to")
265
- hash.delete("expected_path")
266
- end
267
- hash.reject { |_, value| value.nil? || value == "" }
261
+ hash["expected"] = CompletionKit::Checks::Registry.parse_json(hash["expected"]) if hash["expected"].present?
262
+ prune_check_config(hash.except(*CompletionKit::Checks::Registry.unused_keys(hash)))
268
263
  end
269
264
 
270
- def coerce_scalar(value)
271
- JSON.parse(value)
272
- rescue JSON::ParserError
273
- value
265
+ def prune_check_config(hash)
266
+ allowed = CompletionKit::Checks::Registry::SHARED_KEYS +
267
+ CompletionKit::Checks::Registry.config_keys(hash["check_kind"])
268
+ hash.slice(*allowed).reject { |_, value| value.nil? || value == "" }
269
+ end
270
+
271
+ def coerce_number(value)
272
+ number = CompletionKit::Checks::NumericValue.parse(value)
273
+ return value if number.nil?
274
+
275
+ number == number.to_i ? number.to_i : number
274
276
  end
275
277
 
276
278
  def normalize_rubric_bands_for_update(bands)
@@ -202,9 +202,12 @@ module CompletionKit
202
202
  right_score: r_review ? r_review.ai_score : nil,
203
203
  left_passed: l_review&.passed,
204
204
  right_passed: r_review&.passed,
205
+ left_check_score: l_review&.check_score&.to_f,
206
+ right_check_score: r_review&.check_score&.to_f,
205
207
  left_version_label: version_label_for(l_review, metric_versions),
206
208
  right_version_label: version_label_for(r_review, metric_versions),
207
- delta: (l_review&.ai_score && r_review&.ai_score) ? (r_review.ai_score.to_f - l_review.ai_score.to_f).round(2) : nil,
209
+ delta: RunComparison.score_delta(l_review&.ai_score, r_review&.ai_score),
210
+ check_score_delta: RunComparison.score_delta(l_review&.check_score, r_review&.check_score),
208
211
  result_change: RunComparison.result_change(l_review&.passed, r_review&.passed)
209
212
  }
210
213
  end.compact
@@ -204,11 +204,52 @@ module CompletionKit
204
204
  :low
205
205
  end
206
206
 
207
- def ck_check_badge(passed)
207
+ def ck_percent_points(fraction)
208
+ (BigDecimal(fraction.to_s) * 100).round
209
+ end
210
+
211
+ def ck_percent(fraction)
212
+ "#{ck_percent_points(fraction)}%"
213
+ end
214
+
215
+ def ck_check_verdict(passed, score = nil)
216
+ verdict = passed ? "Pass" : "Fail"
217
+ score.nil? ? verdict : "#{verdict} #{ck_percent(score)}"
218
+ end
219
+
220
+ def ck_check_average(summary)
221
+ ck_percent(summary[:avg_check_score] || summary[:pass_rate])
222
+ end
223
+
224
+ def ck_check_average_label(name, summary)
225
+ pass_rate = "#{ck_percent(summary[:pass_rate])} pass rate"
226
+ return "#{name} #{pass_rate}" unless summary[:avg_check_score]
227
+
228
+ "#{name} average score #{ck_percent(summary[:avg_check_score])}, #{pass_rate}"
229
+ end
230
+
231
+ def ck_check_score_change(left, right)
232
+ return nil if left.nil? || right.nil?
233
+
234
+ ck_percent_points(right) - ck_percent_points(left)
235
+ end
236
+
237
+ def ck_delta_chip(delta, unit = "")
238
+ tone, arrow = if delta.positive? then %w[positive ▲]
239
+ elsif delta.negative? then %w[negative ▼]
240
+ else %w[zero =]
241
+ end
242
+ content_tag(:span, class: "ck-delta ck-delta--#{tone}") do
243
+ content_tag(:span, arrow, class: "ck-delta__arrow", "aria-hidden": "true") + "#{"+" if delta.positive?}#{delta}#{unit}"
244
+ end
245
+ end
246
+
247
+ def ck_check_badge(passed, score = nil)
248
+ suffix = score.nil? ? "" : " #{ck_percent(score)}"
208
249
  if passed == true
209
- content_tag(:span, "Pass", class: ck_badge_classes(:high))
250
+ content_tag(:span, "Pass#{suffix}", class: ck_badge_classes(:high))
210
251
  elsif passed == false
211
- content_tag(:span, "Fail", class: ck_badge_classes(:low))
252
+ content_tag(:span, "Fail#{suffix}", class: ck_badge_classes(:low))
212
253
  else
213
254
  content_tag(:span, "Pending", class: ck_badge_classes(:pending))
214
255
  end
@@ -232,58 +273,71 @@ module CompletionKit
232
273
  {heading: heading, judge_model: judge ? judge_model.presence : nil}
233
274
  end
234
275
 
235
- CHECK_KIND_LABELS = {
236
- "contains" => "Contains a phrase",
237
- "not_contains" => "Does not contain a phrase",
238
- "equals" => "Equals exactly",
239
- "regex" => "Matches a pattern",
240
- "valid_json" => "Is valid JSON",
241
- "json_path_equals" => "A JSON field equals a value",
242
- "length_bounds" => "Length is within a range"
243
- }.freeze
244
-
245
- CHECK_COMPARE_TO_LABELS = {
246
- "constant" => "A value you type",
247
- "expected" => "Each row's expected value"
248
- }.freeze
249
-
250
- CHECK_TARGET_LABELS = {
251
- "response_text" => "The response text",
252
- "input_data" => "The input row",
253
- "json_path" => "A value from the response JSON"
254
- }.freeze
255
-
256
- CHECK_FIELD_LABELS = {
257
- "value" => "Text to look for",
258
- "pattern" => "Pattern",
259
- "json_path" => "JSON path",
260
- "expected" => "Expected value",
261
- "expected_path" => "Field in the answer key",
262
- "compare_to" => "Compare against",
263
- "target_path" => "Path into the JSON",
264
- "min" => "Shortest allowed",
265
- "max" => "Longest allowed",
266
- "case_sensitive" => "Case sensitive",
267
- "multiline" => "Multiline",
268
- "trim" => "Trim whitespace"
269
- }.freeze
270
-
271
276
  def ck_check_kind_label(kind)
272
- CHECK_KIND_LABELS.fetch(kind.to_s) { kind.to_s.humanize }
277
+ CompletionKit::Checks::Labels.kind(kind)
273
278
  end
274
279
 
275
280
  def ck_check_target_label(target)
276
- CHECK_TARGET_LABELS.fetch(target.to_s) { target.to_s.humanize }
281
+ CompletionKit::Checks::Labels.target(target)
282
+ end
283
+
284
+ def ck_check_field_label(key, kind = nil)
285
+ CompletionKit::Checks::Labels.field(key, kind)
277
286
  end
278
287
 
279
- def ck_check_field_label(key)
280
- CHECK_FIELD_LABELS.fetch(key.to_s) { key.to_s.humanize }
288
+ def ck_check_field_hint(key, kind = nil)
289
+ CompletionKit::Checks::Labels.hint(key, kind)
281
290
  end
282
291
 
283
292
  def ck_check_field_value(key, value)
284
- return CHECK_COMPARE_TO_LABELS.fetch(value.to_s, value) if key.to_s == "compare_to"
293
+ CompletionKit::Checks::Labels.value(key, value)
294
+ end
295
+
296
+ def ck_check_input_value(value)
297
+ value.is_a?(Array) || value.is_a?(Hash) ? value.to_json : value
298
+ end
299
+
300
+ def ck_check_expected_input_value(value)
301
+ return value if value.nil? || (value.is_a?(String) && CompletionKit::Checks::Registry.parse_json(value) == value)
302
+
303
+ value.to_json
304
+ end
305
+
306
+ def ck_check_spec_fields(config)
307
+ shown = config.except("check_kind", "target").reject do |key, _value|
308
+ CompletionKit::Checks::Registry::FLAG_KEYS.include?(key) && !CompletionKit::Checks::Flag.on?(config, key)
309
+ end
310
+ CompletionKit::Checks::Labels.in_form_order(shown)
311
+ end
312
+
313
+ def ck_check_spec_value(key, value)
314
+ return "Yes" if CompletionKit::Checks::Registry::FLAG_KEYS.include?(key.to_s)
315
+ return ck_check_field_value(key, value) if CompletionKit::Checks::Labels.worded?(key)
316
+
317
+ shown = key.to_s == "expected" ? ck_check_expected_input_value(value) : ck_check_input_value(value)
318
+ content_tag(:code, shown.to_s, class: "ck-check-spec__code")
319
+ end
320
+
321
+ def ck_check_field_invalid?(metric, key)
322
+ metric.check_error_keys.include?(key)
323
+ end
324
+
325
+ def ck_check_field_aria(metric, key)
326
+ ck_check_field_invalid?(metric, key) ? { "aria-invalid" => "true" } : {}
327
+ end
328
+
329
+ def ck_check_score_by_options
330
+ CompletionKit::Checks::Labels::SCORE_BY.to_a
331
+ end
285
332
 
286
- value
333
+ def ck_check_form_data
334
+ labels = CompletionKit::Checks::Labels
335
+ {
336
+ fields: CompletionKit::Checks::Registry::CONFIG_KEYS,
337
+ expected: CompletionKit::Checks::Registry::EXPECTED_KEYS,
338
+ labels: labels::FIELDS_BY_KIND, defaultLabels: labels::FIELDS,
339
+ hints: labels::HINTS_BY_KIND, defaultHints: labels::HINTS
340
+ }
287
341
  end
288
342
 
289
343
  def ck_result_change_badge(change)
@@ -25,6 +25,7 @@ module CompletionKit
25
25
  status: "succeeded",
26
26
  passed: result.passed,
27
27
  ai_score: nil,
28
+ check_score: result.score,
28
29
  ai_feedback: result.detail,
29
30
  error_provider: nil, error_class: nil, error_status: nil, error_message: nil
30
31
  )
@@ -36,23 +37,43 @@ module CompletionKit
36
37
  private
37
38
 
38
39
  def evaluate(response, config)
39
- target_value = Checks::TargetResolver.call(response, config)
40
+ kind = config["check_kind"]
41
+ target_value = resolve_target(response, config, kind)
40
42
  if target_value.equal?(Checks::TargetResolver::UNRESOLVED)
41
- return Checks::Result.new(passed: false, detail: "could not resolve target")
43
+ return ungraded(kind, "could not resolve target")
42
44
  end
43
45
 
44
- kind = config["check_kind"]
45
46
  if config["compare_to"] == "expected" && Checks::Registry.compares_value?(kind)
46
- expected_value = Checks::ExpectedResolver.call(response, config)
47
+ expected_value = resolve_expected(response, config, kind)
47
48
  if expected_value.equal?(Checks::ExpectedResolver::UNRESOLVED)
48
- return Checks::Result.new(passed: false, detail: "no expected value for this row")
49
+ return ungraded(kind, "no expected value for this row")
49
50
  end
50
- config = config.merge("value" => expected_value)
51
+ config = config.merge(Checks::Registry.expected_key(kind) => expected_value)
51
52
  end
52
53
 
53
54
  Checks::Registry.fetch(kind).call(target_value, config)
54
55
  end
55
56
 
57
+ def ungraded(kind, detail)
58
+ Checks::Result.new(passed: false, detail: detail, score: Checks::Registry.scores?(kind) ? 0.0 : nil)
59
+ end
60
+
61
+ def resolve_target(response, config, kind)
62
+ if Checks::Registry.raw_target?(kind)
63
+ Checks::TargetResolver.call_value(response, config)
64
+ else
65
+ Checks::TargetResolver.call(response, config)
66
+ end
67
+ end
68
+
69
+ def resolve_expected(response, config, kind)
70
+ if Checks::Registry.raw_expected?(kind)
71
+ Checks::ExpectedResolver.call_value(response, config)
72
+ else
73
+ Checks::ExpectedResolver.call(response, config)
74
+ end
75
+ end
76
+
56
77
  def record_terminal_failure!(error)
57
78
  response = Response.find_by(id: @response_id)
58
79
  return unless response
@@ -2,14 +2,17 @@ module CompletionKit
2
2
  class Dataset < ApplicationRecord
3
3
  include CompletionKit::Taggable
4
4
 
5
+ HEADER_ROW_LIMIT = 64 * 1024
6
+
5
7
  has_many :runs, dependent: :destroy
6
8
 
7
9
  validates :name, presence: true
8
- validates :csv_data, presence: true
10
+ validate :csv_data_present
9
11
  validate :csv_data_within_size_limit
12
+ validate :header_row_usable, if: :will_save_change_to_csv_data?
10
13
 
11
14
  def csv_data_within_size_limit
12
- return if csv_data.blank?
15
+ return if csv_data_blank?
13
16
  return if csv_data.bytesize <= CompletionKit.config.max_upload_bytes
14
17
 
15
18
  errors.add(:csv_data, "is too large (limit #{CompletionKit.config.max_upload_bytes / (1024 * 1024)} MB)")
@@ -37,12 +40,58 @@ module CompletionKit
37
40
  end
38
41
 
39
42
  def headers
40
- return [] if csv_data.blank?
43
+ row = header_row
44
+ return [] if row.nil?
41
45
 
42
46
  require "csv"
43
- ::CSV.parse(csv_data.lines.first.to_s).first.to_a.map(&:to_s).map(&:strip)
47
+ ::CSV.parse(row).first.to_a.map(&:to_s).map(&:strip)
44
48
  rescue ::CSV::MalformedCSVError
45
49
  []
46
50
  end
51
+
52
+ def any_rows?
53
+ stop = header_row_end
54
+ return false unless stop
55
+
56
+ stop += 1 if csv_data.byteslice(stop, 2) == "\r\n"
57
+ csv_data.bytesize > stop + 1
58
+ end
59
+
60
+ private
61
+
62
+ def csv_data_present
63
+ errors.add(:csv_data, :blank) if csv_data_blank?
64
+ end
65
+
66
+ def csv_data_blank?
67
+ csv_data.to_s.empty? || (csv_data.valid_encoding? && header_row.blank? && csv_data.lstrip.empty?)
68
+ end
69
+
70
+ def header_row
71
+ return if csv_data.to_s.empty?
72
+
73
+ stop = header_row_end
74
+ return csv_data.byteslice(0, stop) if stop
75
+
76
+ csv_data if csv_data.bytesize <= HEADER_ROW_LIMIT
77
+ end
78
+
79
+ def header_row_end
80
+ head = csv_data.to_s.byteslice(0, HEADER_ROW_LIMIT + 1)
81
+ [head.byteindex("\n"), head.byteindex("\r")].compact.min
82
+ end
83
+
84
+ def header_row_usable
85
+ return if csv_data_blank?
86
+
87
+ row = header_row
88
+ if !csv_data.valid_encoding?
89
+ errors.add(:csv_data, "is not valid UTF-8")
90
+ elsif row.nil?
91
+ errors.add(:csv_data, "has a header row longer than #{HEADER_ROW_LIMIT / 1024} KB")
92
+ elsif row.blank?
93
+ errors.add(:csv_data, "has a blank header row")
94
+ end
95
+ end
47
96
  end
48
97
  end