@mutagent/evaluator 0.1.0-alpha.5 → 0.2.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/mutagent-evaluator/SKILL.md +28 -30
- package/.claude/skills/mutagent-evaluator/assets/agents/dataset-builder.md +6 -5
- package/.claude/skills/mutagent-evaluator/assets/agents/evaluator.md +93 -30
- package/.claude/skills/mutagent-evaluator/assets/code-quality-criteria.yaml +75 -0
- package/.claude/skills/mutagent-evaluator/assets/templates/discover-report.template.html +397 -0
- package/.claude/skills/mutagent-evaluator/assets/templates/eval-report.template.html +594 -0
- package/.claude/skills/mutagent-evaluator/assets/templates/review-report.template.html +560 -0
- package/.claude/skills/mutagent-evaluator/golden/judge-trajectory.prose.md +69 -0
- package/.claude/skills/mutagent-evaluator/lenses/methodology-critic-lens.md +1 -1
- package/.claude/skills/mutagent-evaluator/references/data-registry.md +43 -0
- package/.claude/skills/mutagent-evaluator/references/edd-loop.md +6 -6
- package/.claude/skills/mutagent-evaluator/references/eval-layers.md +52 -0
- package/.claude/skills/mutagent-evaluator/references/eval-stage.md +37 -4
- package/.claude/skills/mutagent-evaluator/references/memory-format.md +1 -1
- package/.claude/skills/mutagent-evaluator/schemas/methodology-review.schema.yaml +1 -1
- package/.claude/skills/mutagent-evaluator/scripts/agent-dispatch.ts +0 -0
- package/.claude/skills/mutagent-evaluator/scripts/aggregate-discover.ts +254 -9
- package/.claude/skills/mutagent-evaluator/scripts/audience.ts +84 -0
- package/.claude/skills/mutagent-evaluator/scripts/build-review-ui.ts +320 -6
- package/.claude/skills/mutagent-evaluator/scripts/cli/prep.ts +32 -2
- package/.claude/skills/mutagent-evaluator/scripts/cli/profile-subject.ts +1 -1
- package/.claude/skills/mutagent-evaluator/scripts/code-eval-library.ts +201 -0
- package/.claude/skills/mutagent-evaluator/scripts/code-quality-verdict.ts +60 -68
- package/.claude/skills/mutagent-evaluator/scripts/config/schema.ts +3 -3
- package/.claude/skills/mutagent-evaluator/scripts/contracts/agentspec-evals.ts +101 -39
- package/.claude/skills/mutagent-evaluator/scripts/contracts/eval-matrix.ts +326 -3
- package/.claude/skills/mutagent-evaluator/scripts/contracts/eval-types.ts +30 -4
- package/.claude/skills/mutagent-evaluator/scripts/determine-outcome.ts +8 -0
- package/.claude/skills/mutagent-evaluator/scripts/edd/change-request.ts +1 -1
- package/.claude/skills/mutagent-evaluator/scripts/edd/edd-types.ts +2 -2
- package/.claude/skills/mutagent-evaluator/scripts/judge-prompt-template.ts +144 -15
- package/.claude/skills/mutagent-evaluator/scripts/materialize-dataset.ts +130 -68
- package/.claude/skills/mutagent-evaluator/scripts/matrix-judge.ts +219 -1
- package/.claude/skills/mutagent-evaluator/scripts/memory/append.ts +1 -1
- package/.claude/skills/mutagent-evaluator/scripts/memory/ratify.ts +165 -0
- package/.claude/skills/mutagent-evaluator/scripts/merge-criteria.ts +849 -0
- package/.claude/skills/mutagent-evaluator/scripts/prep-tasks.ts +15 -4
- package/.claude/skills/mutagent-evaluator/scripts/read-manifest.ts +120 -0
- package/.claude/skills/mutagent-evaluator/scripts/read-unitf-traces.ts +34 -2
- package/.claude/skills/mutagent-evaluator/scripts/render-build-cards.ts +37 -0
- package/.claude/skills/mutagent-evaluator/scripts/render-discover-report-v3.ts +1155 -0
- package/.claude/skills/mutagent-evaluator/scripts/render-eval-report-v3.ts +610 -0
- package/.claude/skills/mutagent-evaluator/scripts/render-eval-report.ts +11 -92
- package/.claude/skills/mutagent-evaluator/scripts/render-review-report-v3.ts +691 -0
- package/.claude/skills/mutagent-evaluator/scripts/report-fragments.ts +173 -0
- package/.claude/skills/mutagent-evaluator/scripts/route-failures.ts +12 -9
- package/.claude/skills/mutagent-evaluator/scripts/run-evaluate.ts +46 -1
- package/.claude/skills/mutagent-evaluator/scripts/run-pipeline.ts +39 -4
- package/.claude/skills/mutagent-evaluator/scripts/run-review.ts +266 -0
- package/.claude/skills/mutagent-evaluator/scripts/subject-profile.ts +82 -0
- package/.claude/skills/mutagent-evaluator/scripts/sync-eval-criteria.ts +2 -2
- package/.claude/skills/mutagent-evaluator/scripts/unitf-to-evaltrace.ts +64 -21
- package/.claude/skills/mutagent-evaluator/scripts/verify-render.ts +728 -0
- package/bin/mutagent-cli.mjs +9 -5
- package/package.json +2 -2
package/bin/mutagent-cli.mjs
CHANGED
|
@@ -5045,8 +5045,12 @@ var SpanSchema = Type.Object({
|
|
|
5045
5045
|
role: Type.Optional(RoleSchema),
|
|
5046
5046
|
toolName: Type.Optional(Type.String()),
|
|
5047
5047
|
toolCallId: Type.Optional(Type.String()),
|
|
5048
|
-
input: Type.Optional(Type.Unknown(
|
|
5049
|
-
|
|
5048
|
+
input: Type.Optional(Type.Unknown({
|
|
5049
|
+
description: "Message text for the INPUT-side layers (role user\xB7system\xB7tool = user-input / system-input / tool-call arguments), or a chat-messages array on a multi-layer LLM span. See the field-per-layer rule; enforced by checkFieldConformance()."
|
|
5050
|
+
})),
|
|
5051
|
+
output: Type.Optional(Type.Unknown({
|
|
5052
|
+
description: "Message text for the OUTPUT-side layers (role assistant\xB7toolResult = assistant-output / tool-result). See the field-per-layer rule; enforced by checkFieldConformance()."
|
|
5053
|
+
})),
|
|
5050
5054
|
tokens: Type.Optional(TokensSchema),
|
|
5051
5055
|
costUsd: Type.Optional(Type.Number()),
|
|
5052
5056
|
attributes: Type.Optional(Type.Record(Type.String(), Type.Unknown()))
|
|
@@ -6527,7 +6531,7 @@ function normalizeRecords(records, malformedIn) {
|
|
|
6527
6531
|
if (ev.timestamp)
|
|
6528
6532
|
span.startTime = ev.timestamp;
|
|
6529
6533
|
if (text)
|
|
6530
|
-
span.
|
|
6534
|
+
span.input = text;
|
|
6531
6535
|
spans.push(span);
|
|
6532
6536
|
spanIdx++;
|
|
6533
6537
|
}
|
|
@@ -6542,7 +6546,7 @@ function normalizeRecords(records, malformedIn) {
|
|
|
6542
6546
|
if (ev.timestamp)
|
|
6543
6547
|
span.startTime = ev.timestamp;
|
|
6544
6548
|
if (typeof content === "string" && content)
|
|
6545
|
-
span.
|
|
6549
|
+
span.input = content;
|
|
6546
6550
|
spans.push(span);
|
|
6547
6551
|
spanIdx++;
|
|
6548
6552
|
}
|
|
@@ -6797,7 +6801,7 @@ function normalizeRecords2(records, malformedIn) {
|
|
|
6797
6801
|
span.startTime = ts;
|
|
6798
6802
|
const msg = str2(p, "message");
|
|
6799
6803
|
if (msg)
|
|
6800
|
-
span.
|
|
6804
|
+
span.input = msg;
|
|
6801
6805
|
spans.push(span);
|
|
6802
6806
|
spanIdx++;
|
|
6803
6807
|
} else if (pType === "agent_message") {
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mutagent/evaluator",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "mutagent-evaluator: a generic, subject-agnostic AI-agent auditor
|
|
3
|
+
"version": "0.2.0-alpha.1",
|
|
4
|
+
"description": "mutagent-evaluator: a generic, subject-agnostic AI-agent auditor \u2014 reviewer, never executor. Audits any skill/agent against a generated subject profile and emits a 4-tab master-audit report.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"publishConfig": {
|