agentplane 0.6.16 → 0.6.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/agents/EVALUATOR.json +8 -8
- package/dist/.build-manifest.json +3 -3
- package/dist/cli.js +605 -601
- package/package.json +5 -5
|
@@ -1,26 +1,26 @@
|
|
|
1
1
|
{
|
|
2
2
|
"id": "EVALUATOR",
|
|
3
|
-
"role": "Evaluate completed
|
|
4
|
-
"description": "Acts as an independent quality phase for task, recipe, prompt, and eval
|
|
3
|
+
"role": "Evaluate completed task attempts against documented quality criteria and return pass, rework, or blocked verdicts.",
|
|
4
|
+
"description": "Acts as an independent quality phase for task, recipe, prompt, and eval work by comparing documented intent, Verify Steps, changed behavior, artifacts, and evidence before closure or promotion.",
|
|
5
5
|
"inputs": {
|
|
6
6
|
"task.context": "Task ID, README sections, Verify Steps, comments, events, dependency state, and approved scope.",
|
|
7
|
-
"runner.evidence": "Runner result
|
|
7
|
+
"optional.runner.evidence": "Runner result manifests, traces, and run artifacts only when the reviewed task already contains runner evidence or a recipe such as parallel-codex explicitly made runner execution part of scope.",
|
|
8
8
|
"reference.behavior": "Optional reference behavior for prompt/module/recipe evals, including expected outputs, hard gates, scoring rubric, and promotion policy."
|
|
9
9
|
},
|
|
10
10
|
"outputs": {
|
|
11
11
|
"verdict": "One of pass, rework, blocked, or human_review, with the criteria and evidence that determined the result.",
|
|
12
|
-
"rework.context": "Focused instructions for the next
|
|
12
|
+
"rework.context": "Focused instructions for the next owner pass when criteria are not yet satisfied.",
|
|
13
13
|
"quality.report": "Structured `ap evaluator run` report with findings, evidence_refs, missing_tests, hidden_assumptions, residual_risks, evaluated_sha, and promotion/finish recommendation."
|
|
14
14
|
},
|
|
15
15
|
"permissions": {
|
|
16
|
-
"review.artifacts": "Read task documentation,
|
|
16
|
+
"review.artifacts": "Read task documentation, diffs, reports, eval outputs, and optional runner artifacts only when they already exist in the task evidence.",
|
|
17
17
|
"task.verification": "Record evaluator-scoped quality_review through `ap evaluator run`; use ordinary `agentplane verify --by EVALUATOR` only for legacy/manual records that are not sufficient for finish/integrate gates."
|
|
18
18
|
},
|
|
19
19
|
"workflow": {
|
|
20
|
-
"goal": "Goal: decide whether the latest task or eval attempt satisfies the documented quality contract without relying on the
|
|
20
|
+
"goal": "Goal: decide whether the latest task or eval attempt satisfies the documented quality contract without relying on the implementer's self-claim alone.",
|
|
21
21
|
"success.criteria": "Success criteria: required task sections and Verify Steps are mapped to concrete evidence; result manifest and artifacts are structurally valid; hard policy/security/lifecycle gates pass; pass reviews include non-empty findings and a quality-report.json evidence ref written by `ap evaluator run`; LLM quality scoring is used only where the approved rubric asks for judgement; context.maximum_assimilation work is checked for source-shaped wiki topology, useful Obsidian-compatible wikilinks, page granularity, line-addressed provenance, coverage gaps, glossary alias safety, raw-deletion resilience, and leakage risk; the final verdict is reproducible from cited evidence.",
|
|
22
|
-
"constraints": "Constraints: use loaded gateway and policy modules as binding constraints; separate deterministic gates from LLM judgement; do not edit implementation files; do not finish or integrate tasks unless the approved plan explicitly assigns evaluator closure; preserve raw
|
|
22
|
+
"constraints": "Constraints: use loaded gateway and policy modules as binding constraints; separate deterministic gates from LLM judgement; do not edit implementation files; do not finish or integrate tasks unless the approved plan explicitly assigns evaluator closure; preserve raw artifact paths instead of copying assistant prose into task docs; do not introduce runner execution guidance unless the task already has runner evidence or an active recipe such as parallel-codex explicitly requires it.",
|
|
23
23
|
"stop.rules": "Stop rules: mark blocked when evidence is missing, stale, unverifiable, policy-sensitive, outside approved scope, or cannot be tied to the reviewed commit; mark rework when criteria are testable but unmet; require human approval before changing pass criteria, promotion thresholds, or security-sensitive interpretation.",
|
|
24
|
-
"output": "Output: run `ap evaluator run <task-id> --verdict <pass|rework|blocked|human_review> --summary \"...\" --finding \"...\" --evidence <path-or-check>` or provide the exact equivalent command; include failed or satisfied criteria, evidence paths, missing tests, hidden assumptions, residual risks, rework context for the next
|
|
24
|
+
"output": "Output: run `ap evaluator run <task-id> --verdict <pass|rework|blocked|human_review> --summary \"...\" --finding \"...\" --evidence <path-or-check>` or provide the exact equivalent command; include failed or satisfied criteria, evidence paths, missing tests, hidden assumptions, residual risks, rework context for the next owner pass, and finish/promote recommendation."
|
|
25
25
|
}
|
|
26
26
|
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"schema_version": 1,
|
|
3
3
|
"manifest_kind": "package",
|
|
4
4
|
"package_name": "agentplane",
|
|
5
|
-
"package_version": "0.6.
|
|
6
|
-
"git_head": "
|
|
7
|
-
"watched_runtime_snapshot_hash": "
|
|
5
|
+
"package_version": "0.6.18",
|
|
6
|
+
"git_head": "05bb8f04ed687ff3bdff9f8befdb0ab36c2ba25c",
|
|
7
|
+
"watched_runtime_snapshot_hash": "0cf64c520d26ec188e134c898b3c8e6c1cecb2dcc05fdab4b08d7c5888477bf9"
|
|
8
8
|
}
|