@aigne/cli 1.55.2 → 1.55.3-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/dist/utils/evaluation/evaluator.js +13 -14
- package/package.json +11 -11
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,48 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [1.55.3-beta.1](https://github.com/AIGNE-io/aigne-framework/compare/cli-v1.55.3-beta...cli-v1.55.3-beta.1) (2025-11-18)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Bug Fixes
|
|
7
|
+
|
|
8
|
+
* **observability:** introduce TraceContext and improve delete UX ([#755](https://github.com/AIGNE-io/aigne-framework/issues/755)) ([dee54f1](https://github.com/AIGNE-io/aigne-framework/commit/dee54f1c548ed1046781e919f8c51a642b6b0dac))
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Dependencies
|
|
12
|
+
|
|
13
|
+
* The following workspace dependencies were updated
|
|
14
|
+
* dependencies
|
|
15
|
+
* @aigne/afs-history bumped to 1.0.3-beta.1
|
|
16
|
+
* @aigne/afs-local-fs bumped to 1.1.3-beta.1
|
|
17
|
+
* @aigne/agent-library bumped to 1.21.56-beta.1
|
|
18
|
+
* @aigne/agentic-memory bumped to 1.0.56-beta.1
|
|
19
|
+
* @aigne/aigne-hub bumped to 0.10.10-beta.1
|
|
20
|
+
* @aigne/core bumped to 1.68.3-beta.1
|
|
21
|
+
* @aigne/default-memory bumped to 1.2.19-beta.1
|
|
22
|
+
* @aigne/observability-api bumped to 0.11.9-beta.1
|
|
23
|
+
* @aigne/openai bumped to 0.16.10-beta.1
|
|
24
|
+
* devDependencies
|
|
25
|
+
* @aigne/test-utils bumped to 0.5.63-beta.1
|
|
26
|
+
|
|
27
|
+
## [1.55.3-beta](https://github.com/AIGNE-io/aigne-framework/compare/cli-v1.55.2...cli-v1.55.3-beta) (2025-11-17)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
### Dependencies
|
|
31
|
+
|
|
32
|
+
* The following workspace dependencies were updated
|
|
33
|
+
* dependencies
|
|
34
|
+
* @aigne/afs-history bumped to 1.0.3-beta
|
|
35
|
+
* @aigne/afs-local-fs bumped to 1.1.3-beta
|
|
36
|
+
* @aigne/agent-library bumped to 1.21.56-beta
|
|
37
|
+
* @aigne/agentic-memory bumped to 1.0.56-beta
|
|
38
|
+
* @aigne/aigne-hub bumped to 0.10.10-beta
|
|
39
|
+
* @aigne/core bumped to 1.68.3-beta
|
|
40
|
+
* @aigne/default-memory bumped to 1.2.19-beta
|
|
41
|
+
* @aigne/observability-api bumped to 0.11.9-beta
|
|
42
|
+
* @aigne/openai bumped to 0.16.10-beta
|
|
43
|
+
* devDependencies
|
|
44
|
+
* @aigne/test-utils bumped to 0.5.63-beta
|
|
45
|
+
|
|
3
46
|
## [1.55.2](https://github.com/AIGNE-io/aigne-framework/compare/cli-v1.55.2-beta...cli-v1.55.2) (2025-11-15)
|
|
4
47
|
|
|
5
48
|
|
|
@@ -24,21 +24,21 @@ Then assign a rating and a score based on the overall quality.
|
|
|
24
24
|
- **Fluency**: Is the AI response clear, structured, and easy to read?
|
|
25
25
|
|
|
26
26
|
## Rating Rubric (1–5)
|
|
27
|
-
- **5 - Very Good**: Highly relevant, closely aligned with the expected output, accurate, complete, and fluent.
|
|
28
|
-
- **4 - Good**: Relevant, mostly aligned with the expected output, generally accurate and complete, only minor issues.
|
|
29
|
-
- **3 - Ok**: Somewhat relevant, partially aligned, or missing important details.
|
|
30
|
-
- **2 - Bad**: Weak relevance, low similarity with expected output, contains significant errors or omissions.
|
|
31
|
-
- **1 - Very Bad**: Irrelevant, fails to align with expected output, or completely incorrect.
|
|
27
|
+
- **5 - Very Good**: Highly relevant, closely aligned with the expected output, accurate, complete, and fluent.
|
|
28
|
+
- **4 - Good**: Relevant, mostly aligned with the expected output, generally accurate and complete, only minor issues.
|
|
29
|
+
- **3 - Ok**: Somewhat relevant, partially aligned, or missing important details.
|
|
30
|
+
- **2 - Bad**: Weak relevance, low similarity with expected output, contains significant errors or omissions.
|
|
31
|
+
- **1 - Very Bad**: Irrelevant, fails to align with expected output, or completely incorrect.
|
|
32
32
|
|
|
33
33
|
## Evaluation Steps
|
|
34
|
-
1. Compare the **semantic content** of AI Output vs Expected Output.
|
|
35
|
-
- Ignore JSON keys, object structure, formatting, whitespace, capitalization, and minor punctuation differences.
|
|
36
|
-
- If meaning is the same but phrasing differs slightly, assign a higher score (4–5).
|
|
37
|
-
- If AI output deviates significantly, assign a lower score (1–2).
|
|
34
|
+
1. Compare the **semantic content** of AI Output vs Expected Output.
|
|
35
|
+
- Ignore JSON keys, object structure, formatting, whitespace, capitalization, and minor punctuation differences.
|
|
36
|
+
- If meaning is the same but phrasing differs slightly, assign a higher score (4–5).
|
|
37
|
+
- If AI output deviates significantly, assign a lower score (1–2).
|
|
38
38
|
- If AI output is empty, assign a lower score (1–2).
|
|
39
|
-
2. Assess against criteria: instruction following, groundedness, completeness, correctness, fluency.
|
|
40
|
-
3. Assign a 1–5 integer score.
|
|
41
|
-
4. Provide reasoning, and explicitly justify why this result is **not** a 1/2/3 case (why it avoids being a negative example).
|
|
39
|
+
2. Assess against criteria: instruction following, groundedness, completeness, correctness, fluency.
|
|
40
|
+
3. Assign a 1–5 integer score.
|
|
41
|
+
4. Provide reasoning, and explicitly justify why this result is **not** a 1/2/3 case (why it avoids being a negative example).
|
|
42
42
|
|
|
43
43
|
# Response Output Format
|
|
44
44
|
Your output must strictly follow this three-line format:
|
|
@@ -77,12 +77,11 @@ const defaultAgent = AIAgent.from({
|
|
|
77
77
|
score: z.number().int().min(1).max(5).describe("The score of the output, 1–5, 5 is the best"),
|
|
78
78
|
}),
|
|
79
79
|
});
|
|
80
|
-
const defaultAigne = new AIGNE();
|
|
81
80
|
export class LLMEvaluator {
|
|
82
81
|
aigne;
|
|
83
82
|
agent;
|
|
84
83
|
name = "llm-as-judge";
|
|
85
|
-
constructor(aigne =
|
|
84
|
+
constructor(aigne = new AIGNE(), agent = defaultAgent) {
|
|
86
85
|
this.aigne = aigne;
|
|
87
86
|
this.agent = agent;
|
|
88
87
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aigne/cli",
|
|
3
|
-
"version": "1.55.
|
|
3
|
+
"version": "1.55.3-beta.1",
|
|
4
4
|
"description": "Your command center for agent development",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public"
|
|
@@ -89,16 +89,16 @@
|
|
|
89
89
|
"yoctocolors-cjs": "^2.1.3",
|
|
90
90
|
"zod": "^3.25.67",
|
|
91
91
|
"zod-to-json-schema": "^3.24.6",
|
|
92
|
-
"@aigne/afs-history": "^1.0.2",
|
|
93
|
-
"@aigne/agent-library": "^1.21.55",
|
|
94
|
-
"@aigne/afs-local-fs": "^1.1.2",
|
|
95
|
-
"@aigne/agentic-memory": "^1.0.55",
|
|
96
92
|
"@aigne/afs": "^1.2.1",
|
|
97
|
-
"@aigne/
|
|
98
|
-
"@aigne/
|
|
99
|
-
"@aigne/
|
|
100
|
-
"@aigne/
|
|
101
|
-
"@aigne/
|
|
93
|
+
"@aigne/afs-history": "^1.0.3-beta.1",
|
|
94
|
+
"@aigne/afs-local-fs": "^1.1.3-beta.1",
|
|
95
|
+
"@aigne/agentic-memory": "^1.0.56-beta.1",
|
|
96
|
+
"@aigne/agent-library": "^1.21.56-beta.1",
|
|
97
|
+
"@aigne/aigne-hub": "^0.10.10-beta.1",
|
|
98
|
+
"@aigne/core": "^1.68.3-beta.1",
|
|
99
|
+
"@aigne/observability-api": "^0.11.9-beta.1",
|
|
100
|
+
"@aigne/default-memory": "^1.2.19-beta.1",
|
|
101
|
+
"@aigne/openai": "^0.16.10-beta.1"
|
|
102
102
|
},
|
|
103
103
|
"devDependencies": {
|
|
104
104
|
"@inquirer/testing": "^2.1.50",
|
|
@@ -115,7 +115,7 @@
|
|
|
115
115
|
"rimraf": "^6.0.1",
|
|
116
116
|
"typescript": "^5.9.2",
|
|
117
117
|
"ufo": "^1.6.1",
|
|
118
|
-
"@aigne/test-utils": "^0.5.
|
|
118
|
+
"@aigne/test-utils": "^0.5.63-beta.1"
|
|
119
119
|
},
|
|
120
120
|
"scripts": {
|
|
121
121
|
"lint": "tsc --noEmit",
|