@mutagent/evaluator 0.1.0-alpha.5 → 0.2.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/.claude/skills/mutagent-evaluator/SKILL.md +28 -30
  2. package/.claude/skills/mutagent-evaluator/assets/agents/dataset-builder.md +6 -5
  3. package/.claude/skills/mutagent-evaluator/assets/agents/evaluator.md +93 -30
  4. package/.claude/skills/mutagent-evaluator/assets/code-quality-criteria.yaml +75 -0
  5. package/.claude/skills/mutagent-evaluator/assets/templates/discover-report.template.html +397 -0
  6. package/.claude/skills/mutagent-evaluator/assets/templates/eval-report.template.html +594 -0
  7. package/.claude/skills/mutagent-evaluator/assets/templates/review-report.template.html +560 -0
  8. package/.claude/skills/mutagent-evaluator/golden/judge-trajectory.prose.md +69 -0
  9. package/.claude/skills/mutagent-evaluator/lenses/methodology-critic-lens.md +1 -1
  10. package/.claude/skills/mutagent-evaluator/references/data-registry.md +43 -0
  11. package/.claude/skills/mutagent-evaluator/references/edd-loop.md +6 -6
  12. package/.claude/skills/mutagent-evaluator/references/eval-layers.md +52 -0
  13. package/.claude/skills/mutagent-evaluator/references/eval-stage.md +37 -4
  14. package/.claude/skills/mutagent-evaluator/references/memory-format.md +1 -1
  15. package/.claude/skills/mutagent-evaluator/schemas/methodology-review.schema.yaml +1 -1
  16. package/.claude/skills/mutagent-evaluator/scripts/agent-dispatch.ts +0 -0
  17. package/.claude/skills/mutagent-evaluator/scripts/aggregate-discover.ts +254 -9
  18. package/.claude/skills/mutagent-evaluator/scripts/audience.ts +84 -0
  19. package/.claude/skills/mutagent-evaluator/scripts/build-review-ui.ts +320 -6
  20. package/.claude/skills/mutagent-evaluator/scripts/cli/prep.ts +32 -2
  21. package/.claude/skills/mutagent-evaluator/scripts/cli/profile-subject.ts +1 -1
  22. package/.claude/skills/mutagent-evaluator/scripts/code-eval-library.ts +201 -0
  23. package/.claude/skills/mutagent-evaluator/scripts/code-quality-verdict.ts +60 -68
  24. package/.claude/skills/mutagent-evaluator/scripts/config/schema.ts +3 -3
  25. package/.claude/skills/mutagent-evaluator/scripts/contracts/agentspec-evals.ts +101 -39
  26. package/.claude/skills/mutagent-evaluator/scripts/contracts/eval-matrix.ts +326 -3
  27. package/.claude/skills/mutagent-evaluator/scripts/contracts/eval-types.ts +30 -4
  28. package/.claude/skills/mutagent-evaluator/scripts/determine-outcome.ts +8 -0
  29. package/.claude/skills/mutagent-evaluator/scripts/edd/change-request.ts +1 -1
  30. package/.claude/skills/mutagent-evaluator/scripts/edd/edd-types.ts +2 -2
  31. package/.claude/skills/mutagent-evaluator/scripts/judge-prompt-template.ts +144 -15
  32. package/.claude/skills/mutagent-evaluator/scripts/materialize-dataset.ts +130 -68
  33. package/.claude/skills/mutagent-evaluator/scripts/matrix-judge.ts +219 -1
  34. package/.claude/skills/mutagent-evaluator/scripts/memory/append.ts +1 -1
  35. package/.claude/skills/mutagent-evaluator/scripts/memory/ratify.ts +165 -0
  36. package/.claude/skills/mutagent-evaluator/scripts/merge-criteria.ts +849 -0
  37. package/.claude/skills/mutagent-evaluator/scripts/prep-tasks.ts +15 -4
  38. package/.claude/skills/mutagent-evaluator/scripts/read-manifest.ts +120 -0
  39. package/.claude/skills/mutagent-evaluator/scripts/read-unitf-traces.ts +34 -2
  40. package/.claude/skills/mutagent-evaluator/scripts/render-build-cards.ts +37 -0
  41. package/.claude/skills/mutagent-evaluator/scripts/render-discover-report-v3.ts +1155 -0
  42. package/.claude/skills/mutagent-evaluator/scripts/render-eval-report-v3.ts +610 -0
  43. package/.claude/skills/mutagent-evaluator/scripts/render-eval-report.ts +11 -92
  44. package/.claude/skills/mutagent-evaluator/scripts/render-review-report-v3.ts +691 -0
  45. package/.claude/skills/mutagent-evaluator/scripts/report-fragments.ts +173 -0
  46. package/.claude/skills/mutagent-evaluator/scripts/route-failures.ts +12 -9
  47. package/.claude/skills/mutagent-evaluator/scripts/run-evaluate.ts +46 -1
  48. package/.claude/skills/mutagent-evaluator/scripts/run-pipeline.ts +39 -4
  49. package/.claude/skills/mutagent-evaluator/scripts/run-review.ts +266 -0
  50. package/.claude/skills/mutagent-evaluator/scripts/subject-profile.ts +82 -0
  51. package/.claude/skills/mutagent-evaluator/scripts/sync-eval-criteria.ts +2 -2
  52. package/.claude/skills/mutagent-evaluator/scripts/unitf-to-evaltrace.ts +64 -21
  53. package/.claude/skills/mutagent-evaluator/scripts/verify-render.ts +728 -0
  54. package/bin/mutagent-cli.mjs +9 -5
  55. package/package.json +2 -2
@@ -5045,8 +5045,12 @@ var SpanSchema = Type.Object({
5045
5045
  role: Type.Optional(RoleSchema),
5046
5046
  toolName: Type.Optional(Type.String()),
5047
5047
  toolCallId: Type.Optional(Type.String()),
5048
- input: Type.Optional(Type.Unknown()),
5049
- output: Type.Optional(Type.Unknown()),
5048
+ input: Type.Optional(Type.Unknown({
5049
+ description: "Message text for the INPUT-side layers (role user\xB7system\xB7tool = user-input / system-input / tool-call arguments), or a chat-messages array on a multi-layer LLM span. See the field-per-layer rule; enforced by checkFieldConformance()."
5050
+ })),
5051
+ output: Type.Optional(Type.Unknown({
5052
+ description: "Message text for the OUTPUT-side layers (role assistant\xB7toolResult = assistant-output / tool-result). See the field-per-layer rule; enforced by checkFieldConformance()."
5053
+ })),
5050
5054
  tokens: Type.Optional(TokensSchema),
5051
5055
  costUsd: Type.Optional(Type.Number()),
5052
5056
  attributes: Type.Optional(Type.Record(Type.String(), Type.Unknown()))
@@ -6527,7 +6531,7 @@ function normalizeRecords(records, malformedIn) {
6527
6531
  if (ev.timestamp)
6528
6532
  span.startTime = ev.timestamp;
6529
6533
  if (text)
6530
- span.output = text;
6534
+ span.input = text;
6531
6535
  spans.push(span);
6532
6536
  spanIdx++;
6533
6537
  }
@@ -6542,7 +6546,7 @@ function normalizeRecords(records, malformedIn) {
6542
6546
  if (ev.timestamp)
6543
6547
  span.startTime = ev.timestamp;
6544
6548
  if (typeof content === "string" && content)
6545
- span.output = content;
6549
+ span.input = content;
6546
6550
  spans.push(span);
6547
6551
  spanIdx++;
6548
6552
  }
@@ -6797,7 +6801,7 @@ function normalizeRecords2(records, malformedIn) {
6797
6801
  span.startTime = ts;
6798
6802
  const msg = str2(p, "message");
6799
6803
  if (msg)
6800
- span.output = msg;
6804
+ span.input = msg;
6801
6805
  spans.push(span);
6802
6806
  spanIdx++;
6803
6807
  } else if (pType === "agent_message") {
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@mutagent/evaluator",
3
- "version": "0.1.0-alpha.5",
4
- "description": "mutagent-evaluator: a generic, subject-agnostic AI-agent auditor reviewer, never executor. Audits any skill/agent against a generated subject profile and emits a 4-tab master-audit report.",
3
+ "version": "0.2.0-alpha.1",
4
+ "description": "mutagent-evaluator: a generic, subject-agnostic AI-agent auditor \u2014 reviewer, never executor. Audits any skill/agent against a generated subject profile and emits a 4-tab master-audit report.",
5
5
  "license": "Apache-2.0",
6
6
  "type": "module",
7
7
  "publishConfig": {