lex-eval 0.3.13 → 0.3.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: bcf09bf5fdfb1a7c80e050e440c874f6e099ddd41ac0f85155733d1c4aa80bed
4
- data.tar.gz: f1291ca88a8a4f7cf30abce7616bf8ab4145d1db0fe6ed6c0112e09482bda77b
3
+ metadata.gz: c6475ecd12f917b2a19176ee554b5e66be2fe5d647dc2a93a71ffa1307a5bcaf
4
+ data.tar.gz: 6cbbef414bbc6d385023e58943136eca6f23f222e94b4c1fc1f7649f94dc3f2b
5
5
  SHA512:
6
- metadata.gz: 1df90ff664f437acbfa6eb6e1b7dd5efc7620a8b4ee73b8f11a4fb5b35f8b2ab7324d887186abf56c929a84799e68eba3a30a375b05760048fab6eb6f1078b2f
7
- data.tar.gz: 1887981dc5dc141deb187f1d8ed7e2c5075c911d87b7a367773b93fd96ea6b52c6564b44a90ee8c32a1e9e6670c5ba381b371a8440fdad39b8ea9fda2d42cb9b
6
+ metadata.gz: db8527cd54cf93e2d30ed4c689c11eae043580f164a38bd80cd53e338ee7eac4c89e2302360d01fea36450f699f1f0aa8c76c95494d709f603341e133b22a07d
7
+ data.tar.gz: 0c25a627d242be6d38b37f3b2aa25957312e2e74db138f3c209df790cf9e4f950e77532bfce7fdd17f9f68b2a51c2fff86573465913d8991a437152f5edb63ec
@@ -8,7 +8,7 @@ module Legion
8
8
  module Eval
9
9
  module Evaluators
10
10
  class CodeEvaluator < Base
11
- def evaluate(input:, output:, expected: nil, context: {}) # rubocop:disable Lint/UnusedMethodArgument
11
+ def evaluate(output:, **)
12
12
  checks = @config[:checks] || []
13
13
  failures = checks.reject { |check| run_check(check, output.to_s) }
14
14
  score = checks.empty? ? 1.0 : (checks.size - failures.size).to_f / checks.size
@@ -22,7 +22,7 @@ module Legion
22
22
  required: %i[score passed explanation]
23
23
  }.freeze
24
24
 
25
- def evaluate(input:, output:, expected: nil, context: {}) # rubocop:disable Lint/UnusedMethodArgument
25
+ def evaluate(input:, output:, expected: nil, **)
26
26
  if defined?(Legion::Telemetry::OpenInference)
27
27
  Legion::Telemetry::OpenInference.evaluator_span(template: @config[:name] || 'unknown') do |_span|
28
28
  evaluate_impl(input: input, output: output, expected: expected)
@@ -26,13 +26,16 @@ module Legion
26
26
  def review_output(input:, output:, review_prompt: nil, model: nil, provider: nil, **)
27
27
  prompt = build_review_message(review_prompt || default_review_prompt, input, output)
28
28
  llm_kwargs = {
29
- message: prompt, schema: REVIEW_SCHEMA,
29
+ messages: [{ role: 'user', content: prompt }], schema: REVIEW_SCHEMA,
30
30
  intent: { capability: :reasoning },
31
31
  caller: { extension: 'lex-eval', operation: 'agentic_review' }
32
32
  }
33
33
  llm_kwargs[:model] = model if model
34
34
  llm_kwargs[:provider] = provider if provider
35
- Legion::LLM.structured(**llm_kwargs) # rubocop:disable Legion/HelperMigration/DirectLlm
35
+ result = Legion::LLM.structured(**llm_kwargs) # rubocop:disable Legion/HelperMigration/DirectLlm
36
+ raise "structured output invalid: #{result[:error]}" unless result[:valid] && result[:data].is_a?(Hash)
37
+
38
+ result[:data].transform_keys(&:to_sym)
36
39
  rescue StandardError => e
37
40
  log.warn(e.message) if respond_to?(:log, true) # rubocop:disable Legion/HelperMigration/LoggingGuard
38
41
  { confidence: 0.0, recommendation: 'reject',
@@ -3,7 +3,7 @@
3
3
  module Legion
4
4
  module Extensions
5
5
  module Eval
6
- VERSION = '0.3.13'
6
+ VERSION = '0.3.14'
7
7
  end
8
8
  end
9
9
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: lex-eval
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.13
4
+ version: 0.3.14
5
5
  platform: ruby
6
6
  authors:
7
7
  - Matthew Iverson