lex-eval 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/legion/extensions/eval/runners/online.rb +3 -2
- data/lib/legion/extensions/eval/version.rb +1 -1
- metadata +100 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 42e93ebdc972bbb947ca388705947747e0dc4050e7cb5483efc57471adfbca29
|
|
4
|
+
data.tar.gz: f3a45e7d94bd7c4670f92f11a6e838e9a0913fd8823726436efa889c017afea3
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9597e0c4cf93d703c40d83a80107a1c2271022710dadfc666ea3ce510d327eefceb33005e456b0af1064d77c5d7d88502d3476cf7554382fc0b7d18e07421539
|
|
7
|
+
data.tar.gz: b8e858dbd043dfd71fe5d964c608d438ad34f56ba3da487fd3a58f8fd76ffd244f01266a0c72c892c1257ac87f0041a1a7e7aaf1fb09f6ff2ef48e53ab0c13f7
|
|
@@ -5,6 +5,7 @@ module Legion
|
|
|
5
5
|
module Eval
|
|
6
6
|
module Runners
|
|
7
7
|
module Online
|
|
8
|
+
include Legion::Extensions::Helpers::Lex if defined?(Legion::Extensions::Helpers::Lex)
|
|
8
9
|
def evaluate_response(response:, evaluators: nil, sample_rate: 1.0, **)
|
|
9
10
|
evaluator_names = evaluators || configured_evaluators
|
|
10
11
|
effective_rate = sample_rate || configured_sample_rate
|
|
@@ -18,7 +19,7 @@ module Legion
|
|
|
18
19
|
|
|
19
20
|
{ evaluated: true, scores: scores, sampled: true }
|
|
20
21
|
rescue StandardError => e
|
|
21
|
-
|
|
22
|
+
log.warn("lex-eval online: evaluate_response failed: #{e.message}")
|
|
22
23
|
{ evaluated: false, reason: :error, error: e.message, sampled: true }
|
|
23
24
|
end
|
|
24
25
|
|
|
@@ -34,7 +35,7 @@ module Legion
|
|
|
34
35
|
)
|
|
35
36
|
result.dig(:summary, :avg_score)
|
|
36
37
|
rescue StandardError => e
|
|
37
|
-
|
|
38
|
+
log.warn("lex-eval online: evaluator #{name} failed: #{e.message}")
|
|
38
39
|
nil
|
|
39
40
|
end
|
|
40
41
|
|
metadata
CHANGED
|
@@ -1,14 +1,112 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: lex-eval
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Matthew Iverson
|
|
8
8
|
bindir: bin
|
|
9
9
|
cert_chain: []
|
|
10
10
|
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
-
dependencies:
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: legion-cache
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - ">="
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: 1.3.11
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - ">="
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: 1.3.11
|
|
26
|
+
- !ruby/object:Gem::Dependency
|
|
27
|
+
name: legion-crypt
|
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
|
29
|
+
requirements:
|
|
30
|
+
- - ">="
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: 1.4.9
|
|
33
|
+
type: :runtime
|
|
34
|
+
prerelease: false
|
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - ">="
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: 1.4.9
|
|
40
|
+
- !ruby/object:Gem::Dependency
|
|
41
|
+
name: legion-data
|
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
|
43
|
+
requirements:
|
|
44
|
+
- - ">="
|
|
45
|
+
- !ruby/object:Gem::Version
|
|
46
|
+
version: 1.4.17
|
|
47
|
+
type: :runtime
|
|
48
|
+
prerelease: false
|
|
49
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - ">="
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: 1.4.17
|
|
54
|
+
- !ruby/object:Gem::Dependency
|
|
55
|
+
name: legion-json
|
|
56
|
+
requirement: !ruby/object:Gem::Requirement
|
|
57
|
+
requirements:
|
|
58
|
+
- - ">="
|
|
59
|
+
- !ruby/object:Gem::Version
|
|
60
|
+
version: 1.2.1
|
|
61
|
+
type: :runtime
|
|
62
|
+
prerelease: false
|
|
63
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
64
|
+
requirements:
|
|
65
|
+
- - ">="
|
|
66
|
+
- !ruby/object:Gem::Version
|
|
67
|
+
version: 1.2.1
|
|
68
|
+
- !ruby/object:Gem::Dependency
|
|
69
|
+
name: legion-logging
|
|
70
|
+
requirement: !ruby/object:Gem::Requirement
|
|
71
|
+
requirements:
|
|
72
|
+
- - ">="
|
|
73
|
+
- !ruby/object:Gem::Version
|
|
74
|
+
version: 1.3.2
|
|
75
|
+
type: :runtime
|
|
76
|
+
prerelease: false
|
|
77
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
78
|
+
requirements:
|
|
79
|
+
- - ">="
|
|
80
|
+
- !ruby/object:Gem::Version
|
|
81
|
+
version: 1.3.2
|
|
82
|
+
- !ruby/object:Gem::Dependency
|
|
83
|
+
name: legion-settings
|
|
84
|
+
requirement: !ruby/object:Gem::Requirement
|
|
85
|
+
requirements:
|
|
86
|
+
- - ">="
|
|
87
|
+
- !ruby/object:Gem::Version
|
|
88
|
+
version: 1.3.14
|
|
89
|
+
type: :runtime
|
|
90
|
+
prerelease: false
|
|
91
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
92
|
+
requirements:
|
|
93
|
+
- - ">="
|
|
94
|
+
- !ruby/object:Gem::Version
|
|
95
|
+
version: 1.3.14
|
|
96
|
+
- !ruby/object:Gem::Dependency
|
|
97
|
+
name: legion-transport
|
|
98
|
+
requirement: !ruby/object:Gem::Requirement
|
|
99
|
+
requirements:
|
|
100
|
+
- - ">="
|
|
101
|
+
- !ruby/object:Gem::Version
|
|
102
|
+
version: 1.3.9
|
|
103
|
+
type: :runtime
|
|
104
|
+
prerelease: false
|
|
105
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
106
|
+
requirements:
|
|
107
|
+
- - ">="
|
|
108
|
+
- !ruby/object:Gem::Version
|
|
109
|
+
version: 1.3.9
|
|
12
110
|
description: Provides LLM-as-judge and code-based evaluators for scoring LLM outputs,
|
|
13
111
|
with built-in templates for hallucination, relevance, and toxicity detection.
|
|
14
112
|
email:
|