dsh-harbor-evolution 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -7
- package/index.js +109 -2
- package/lib/candidate.js +29 -2
- package/lib/client.js +767 -260
- package/lib/dashboard.js +644 -67
- package/lib/evolution.js +164 -26
- package/lib/process.js +2 -1
- package/lib/service.js +137 -2
- package/lib/web.js +76 -15
- package/package.json +7 -1
- package/schemas/evaluation-result.schema.json +27 -0
- package/schemas/evaluator-observations.schema.json +51 -0
- package/schemas/ground-truth.schema.json +61 -0
- package/schemas/meta-evaluation-report.schema.json +23 -0
- package/skills/evolve-agent-with-harbor/SKILL.md +149 -78
- package/skills/evolve-agent-with-harbor/references/evaluator-upgrade.md +61 -0
- package/skills/evolve-agent-with-harbor/references/initialization.md +100 -81
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"title": "Evaluator Ground Truth v1",
|
|
4
|
+
"type": "object",
|
|
5
|
+
"additionalProperties": false,
|
|
6
|
+
"required": ["schema_version", "protocol", "ground_truth_id", "version", "source", "criteria", "cases"],
|
|
7
|
+
"properties": {
|
|
8
|
+
"schema_version": { "const": 1 },
|
|
9
|
+
"protocol": { "const": "ground-truth/v1" },
|
|
10
|
+
"ground_truth_id": { "type": "string", "minLength": 1 },
|
|
11
|
+
"version": { "type": "string", "minLength": 1 },
|
|
12
|
+
"source": {
|
|
13
|
+
"type": "object",
|
|
14
|
+
"required": ["kind", "description", "provenance", "independent_of_candidate"],
|
|
15
|
+
"properties": {
|
|
16
|
+
"kind": { "enum": ["human", "programmatic", "consensus", "model", "external"] },
|
|
17
|
+
"description": { "type": "string", "minLength": 1 },
|
|
18
|
+
"provenance": { "type": "string", "minLength": 1 },
|
|
19
|
+
"independent_of_candidate": { "const": true }
|
|
20
|
+
}
|
|
21
|
+
},
|
|
22
|
+
"criteria": {
|
|
23
|
+
"type": "array",
|
|
24
|
+
"minItems": 1,
|
|
25
|
+
"items": {
|
|
26
|
+
"type": "object",
|
|
27
|
+
"required": ["id", "label"],
|
|
28
|
+
"properties": {
|
|
29
|
+
"id": { "type": "string", "minLength": 1 },
|
|
30
|
+
"label": { "type": "string", "minLength": 1 }
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
},
|
|
34
|
+
"cases": {
|
|
35
|
+
"type": "array",
|
|
36
|
+
"items": {
|
|
37
|
+
"type": "object",
|
|
38
|
+
"required": ["id", "artifact_ref", "criteria"],
|
|
39
|
+
"properties": {
|
|
40
|
+
"id": { "type": "string", "minLength": 1 },
|
|
41
|
+
"artifact_ref": { "type": "string", "minLength": 1 },
|
|
42
|
+
"badcase": { "type": "boolean" },
|
|
43
|
+
"criteria": {
|
|
44
|
+
"type": "array",
|
|
45
|
+
"minItems": 1,
|
|
46
|
+
"items": {
|
|
47
|
+
"type": "object",
|
|
48
|
+
"required": ["id", "score", "weight", "reason"],
|
|
49
|
+
"properties": {
|
|
50
|
+
"id": { "type": "string", "minLength": 1 },
|
|
51
|
+
"score": { "enum": [0, 0.5, 1] },
|
|
52
|
+
"weight": { "type": "number", "exclusiveMinimum": 0 },
|
|
53
|
+
"reason": { "type": "string", "minLength": 1 }
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"title": "Evaluator Meta-Evaluation Report v1",
|
|
4
|
+
"type": "object",
|
|
5
|
+
"required": ["schema_version", "protocol", "ground_truth", "evaluator", "coverage", "metrics", "disagreements"],
|
|
6
|
+
"properties": {
|
|
7
|
+
"schema_version": { "const": 1 },
|
|
8
|
+
"protocol": { "const": "meta-evaluation-report/v1" },
|
|
9
|
+
"ground_truth": { "type": "object" },
|
|
10
|
+
"evaluator": { "type": "object" },
|
|
11
|
+
"coverage": { "type": "object" },
|
|
12
|
+
"metrics": {
|
|
13
|
+
"type": "object",
|
|
14
|
+
"required": ["esf", "sce", "rcr"],
|
|
15
|
+
"properties": {
|
|
16
|
+
"esf": { "type": "number", "minimum": 0, "maximum": 1 },
|
|
17
|
+
"sce": { "type": "number", "minimum": 0, "maximum": 1 },
|
|
18
|
+
"rcr": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }
|
|
19
|
+
}
|
|
20
|
+
},
|
|
21
|
+
"disagreements": { "type": "array" }
|
|
22
|
+
}
|
|
23
|
+
}
|
|
@@ -1,127 +1,198 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: evolve-agent-with-harbor
|
|
3
|
-
description:
|
|
3
|
+
description: Architect, initialize, run, diagnose, compare, and safely improve a DeepSeek Harness business Agent with Harbor Trial Lifecycle, Score Validity, Evidence Provenance, Evaluation Stack, Context v2, Architecture Doctor, and explicit Promotion Gate. Use for Harbor setup, Agent self-evolution, vertical-search evaluation loops, running Job inspection, failed Trial diagnosis, Candidate optimization, evaluator governance or meta-evaluation, and promotion decisions.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Evolve Agent With Harbor
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
Build a reproducible improvement loop around three boundaries:
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
- **Generator/Candidate**: the immutable DSH Agent composition being improved.
|
|
11
|
+
- **Evaluator/Evaluation Stack**: Integration, Renderer, Evaluator, Rubric, Diagnoser, Optimizer, Runner, Reporter, and Judge identities.
|
|
12
|
+
- **Optimizer**: proposes one evidence-linked Candidate change; it never controls the final Gate.
|
|
11
13
|
|
|
12
|
-
|
|
14
|
+
Treat Harbor as the experiment boundary. Deployment, CI/CD, and Champion replacement remain external actions requiring separate authority.
|
|
13
15
|
|
|
14
|
-
|
|
15
|
-
- **Initialize**: create the missing Candidate, Dataset, and Promotion Policy structure.
|
|
16
|
-
- **Evaluate**: snapshot and run one Candidate.
|
|
17
|
-
- **Compare**: compare an existing baseline Job with a new Candidate Job.
|
|
18
|
-
- **Evolve**: run the complete baseline, diagnosis, controlled change, regression, and gate loop.
|
|
19
|
-
- **Meta-evaluate**: optimize a Verifier or Judge against human ground truth rather than optimizing the business Agent.
|
|
16
|
+
## Select the narrowest mode
|
|
20
17
|
|
|
21
|
-
|
|
18
|
+
- **Clarify**: define progress, identities, constraints, and promotion ownership.
|
|
19
|
+
- **Architecture**: inspect role boundaries and run `harbor_evolution_doctor`.
|
|
20
|
+
- **Initialize**: read `references/initialization.md`, obtain explicit values, then call `harbor_evolution_init`.
|
|
21
|
+
- **Diagnostic**: investigate failures without making a promotion claim.
|
|
22
|
+
- **Promotion**: run a `promotion-eligible` Job and apply the deterministic Gate.
|
|
23
|
+
- **Evolve**: baseline → diagnose → one controlled change → regression Job → Gate.
|
|
24
|
+
- **Meta-evaluate**: improve an Evaluator/Judge against independently maintained, provenance-bearing GT.
|
|
25
|
+
- **Govern**: inspect Evaluator/Rubric/Judge source and identities; preview whether a change requires a fresh baseline.
|
|
22
26
|
|
|
23
|
-
|
|
27
|
+
Do not turn an inspection or diagnostic request into Agent mutation or deployment.
|
|
24
28
|
|
|
25
|
-
|
|
29
|
+
## Clarify before initializing
|
|
26
30
|
|
|
27
|
-
|
|
31
|
+
Inspect the workspace first. Resolve only material gaps, preferably in no more than three grouped questions. Obtain:
|
|
28
32
|
|
|
29
|
-
1. Business behavior and
|
|
30
|
-
2.
|
|
31
|
-
3.
|
|
32
|
-
4.
|
|
33
|
-
5.
|
|
34
|
-
6.
|
|
35
|
-
7.
|
|
36
|
-
8. Promotion owner and external CI/CD
|
|
33
|
+
1. Business behavior, failure pattern, and Candidate path/product identity.
|
|
34
|
+
2. Dataset path/id/version, task population, holdout boundary, and side-effect sandbox.
|
|
35
|
+
3. Evaluation Stack id/version and one entry for every required role.
|
|
36
|
+
4. Judge provider/model/version/parameters without credentials.
|
|
37
|
+
5. Evaluation Contract id/version, primary metric and direction, diagnostic metrics, groups, and hard requirements.
|
|
38
|
+
6. Promotion Policy id/version, delta, minimums, maximums, non-regression metrics, and metric directions.
|
|
39
|
+
7. Baseline Job/Candidate, repeat policy, run budget, stopping rule, allowed mutation surface, and forbidden files.
|
|
40
|
+
8. Promotion owner and external CI/CD handoff.
|
|
37
41
|
|
|
38
|
-
|
|
42
|
+
Never invent GT, a Judge model, reward definitions, thresholds, credentials, or deployment authority. Offer draft values only when clearly labeled and accepted.
|
|
39
43
|
|
|
40
|
-
##
|
|
44
|
+
## Enforce the strict architecture
|
|
41
45
|
|
|
42
|
-
|
|
46
|
+
Require these before every Job:
|
|
43
47
|
|
|
44
|
-
-
|
|
45
|
-
-
|
|
46
|
-
-
|
|
47
|
-
-
|
|
48
|
-
- Version the Promotion Policy. A policy or Verifier change requires a fresh baseline.
|
|
49
|
-
- Use test accounts, mocks, or sandboxes for business side effects.
|
|
48
|
+
- `candidate-manifest.json` verified against the Candidate files.
|
|
49
|
+
- `dataset-manifest.json` with unique task ids, non-empty instructions, safe paths, and a matching source digest.
|
|
50
|
+
- `.harbor/evaluation-stack.yml` with all eight roles, Judge identity, and Evaluation Contract.
|
|
51
|
+
- Evaluation Context v2 preview.
|
|
50
52
|
|
|
51
|
-
|
|
53
|
+
Require `input_integrity`, `agent_completed`, `integration_valid`, `renderer_valid`, `judge_completed`, and `artifact_schema_valid` in the Trial validity contract. Specify which failures are hard requirements. Never infer that a numeric raw verifier reward is a valid Candidate quality score.
|
|
52
54
|
|
|
53
|
-
|
|
55
|
+
Before a formal Job, call in order:
|
|
54
56
|
|
|
55
|
-
|
|
57
|
+
1. `harbor_candidate_snapshot`
|
|
58
|
+
2. `harbor_dataset_validate`
|
|
59
|
+
3. `harbor_evolution_doctor`
|
|
60
|
+
4. `harbor_context_preview`
|
|
56
61
|
|
|
57
|
-
|
|
62
|
+
Do not launch a `promotion-eligible` Job when Doctor reports an error, no comparable baseline exists, or `fresh_baseline_required` is true. A diagnostic Job may investigate architecture warnings, but still requires a valid Candidate, Dataset Manifest, Evaluation Stack, and Context v2.
|
|
58
63
|
|
|
59
|
-
|
|
64
|
+
Keep Runner orchestration-only. Treat these as architecture errors:
|
|
60
65
|
|
|
61
|
-
|
|
66
|
+
- Runner combines HTTP integration, rubric, and Judge logic.
|
|
67
|
+
- Runner makes a promotion/Champion decision.
|
|
62
68
|
|
|
63
|
-
|
|
69
|
+
## Initialize without overwriting
|
|
64
70
|
|
|
65
|
-
|
|
71
|
+
Read `references/initialization.md` when required files are missing. After the user accepts all required identities and metric semantics, call `harbor_evolution_init`. It preserves existing files and creates explicit placeholders that still require business implementation.
|
|
66
72
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
-
|
|
73
|
+
After initialization:
|
|
74
|
+
|
|
75
|
+
- Replace placeholders with real role implementations.
|
|
76
|
+
- Pin Candidate dependencies and keep secrets runtime-injected.
|
|
77
|
+
- Re-snapshot the Dataset after intentional Dataset changes.
|
|
78
|
+
- Run Doctor again; initialization success is not evaluation readiness.
|
|
79
|
+
|
|
80
|
+
## Determine comparability correctly
|
|
81
|
+
|
|
82
|
+
Use the Context v2 `digest`, not timestamps or Job names.
|
|
83
|
+
|
|
84
|
+
A fresh baseline is required when any of these change:
|
|
85
|
+
|
|
86
|
+
- Dataset id, version, or source digest.
|
|
87
|
+
- Integration, Renderer, Evaluator, or Rubric identity.
|
|
88
|
+
- Judge provider, model, version, or parameters.
|
|
89
|
+
- Runner marked `semantic: true`.
|
|
90
|
+
- Harbor or integration runtime identity.
|
|
91
|
+
|
|
92
|
+
Diagnoser, Optimizer, Reporter, and non-semantic Runner changes remain comparable but change the full audit digest. A Candidate digest must differ from the baseline Candidate digest. Promotion Policy is reapplied as a separately versioned decision contract; changing it does not rewrite Evaluation Context.
|
|
93
|
+
|
|
94
|
+
## Run the evolution loop
|
|
95
|
+
|
|
96
|
+
### Establish a baseline
|
|
97
|
+
|
|
98
|
+
Call `harbor_eval_run` with Candidate, Dataset, Stack, explicit `mode`, and a Policy for `promotion-eligible`. Preserve Candidate, Dataset, Stack, Context, Doctor, Contract, Trial assessments, Population report, Summary, and later Promotion report.
|
|
99
|
+
|
|
100
|
+
Never cherry-pick stochastic runs. Apply the accepted repeat/seed policy symmetrically.
|
|
101
|
+
|
|
102
|
+
### Diagnose before changing
|
|
103
|
+
|
|
104
|
+
Use `harbor_eval_result` to reopen evidence without guessing local artifact paths: default `view=summary`, `view=job` for capabilities and stage artifacts, `view=dataset` for Agent-visible instructions, `view=progress` while running, `view=trial` with a returned `trialId` for the generated output and sanitized evidence, and `view=governance` for Evaluator/Rubric/Judge source and upgrade impact. Inspect in this order:
|
|
105
|
+
|
|
106
|
+
1. Confirm every Dataset item reached a terminal Trial state. Running, queued, cancelled, or missing Trials are not quality evidence.
|
|
107
|
+
2. Check `score.valid` and every validity requirement. Display an invalid score as `—`, never `0`.
|
|
108
|
+
3. Inspect evidence provenance. Keep `Real Renderer`, `ACP Agent Output Fallback`, raw transport evidence, Judge explanation, and deterministic diagnosis distinct.
|
|
109
|
+
4. Inspect findings, recommendations, user-visible output, criteria, and timing.
|
|
110
|
+
5. Classify the owning layer before proposing a mutation.
|
|
111
|
+
|
|
112
|
+
Treat `raw_rewards` as audit-only when `score.valid=false`. Aggregate and compare only valid quality scores. Inspect Trial assessments and classify each failure as:
|
|
113
|
+
|
|
114
|
+
- Candidate capability or policy.
|
|
115
|
+
- Tool-call, invalid search, citation, or output-contract failure.
|
|
116
|
+
- Dataset, Evaluator, Rubric, Judge, or GT defect.
|
|
70
117
|
- Infrastructure, dependency, permission, timeout, or deployment failure.
|
|
71
|
-
- Stochastic
|
|
118
|
+
- Stochastic uncertainty.
|
|
119
|
+
|
|
120
|
+
Do not optimize the Candidate around broken evaluation infrastructure. Never leak holdout answers or GT into Candidate prompts, skills, tools, or memory.
|
|
121
|
+
|
|
122
|
+
Use the formal terminal states precisely:
|
|
123
|
+
|
|
124
|
+
- `candidate-quality-failed`: valid execution reached evaluation, but a Candidate-owned hard requirement failed.
|
|
125
|
+
- `infrastructure-error`: dependency, sandbox, permission, transport, timeout, or runtime failure; no Candidate quality score.
|
|
126
|
+
- `evaluation-error`: Renderer/Judge/Verifier did not complete; no Candidate quality score.
|
|
127
|
+
- `cancelled`: preserve the attempt and do not score it.
|
|
72
128
|
|
|
73
|
-
|
|
129
|
+
For retry or resume, retain the old attempt and create a new attempt. Never replace an assessment or event history in place.
|
|
74
130
|
|
|
75
|
-
###
|
|
131
|
+
### Propose one controlled change
|
|
76
132
|
|
|
77
|
-
|
|
133
|
+
Require every optimization hypothesis to include:
|
|
78
134
|
|
|
79
|
-
|
|
135
|
+
- Evidence references to Job/Trial/findings.
|
|
136
|
+
- Root-cause classification.
|
|
137
|
+
- Expected metric effect.
|
|
138
|
+
- Exact mutation surface and forbidden surface.
|
|
139
|
+
- Rollback condition.
|
|
80
140
|
|
|
81
|
-
|
|
141
|
+
Create a new immutable Candidate version; never edit the baseline in place. Stop when the new digest is unchanged.
|
|
82
142
|
|
|
83
|
-
|
|
143
|
+
### Re-run and gate
|
|
84
144
|
|
|
85
|
-
|
|
145
|
+
Call `harbor_context_preview`; establish a fresh baseline if needed. Run the Candidate under the same comparable Context. Then call `harbor_candidate_compare`.
|
|
86
146
|
|
|
87
|
-
|
|
147
|
+
- `PROMOTE`: recommend external promotion with the complete evidence package.
|
|
148
|
+
- `REJECT`: keep the Champion and explain every structured reason code.
|
|
88
149
|
|
|
89
|
-
|
|
150
|
+
Never bypass `INFRASTRUCTURE_EXCEPTION_PRESENT`, `ARTIFACT_SCHEMA_INVALID`, Dataset/Stack/Rubric/Judge mismatch, or non-regression failures.
|
|
90
151
|
|
|
91
|
-
|
|
92
|
-
- `REJECT`: keep the current Champion and explain each failed criterion.
|
|
152
|
+
A `diagnostic` Job must never invoke Gate. Reading the Workbench, generating a Reporter summary, or producing a non-reward Optimization Report also must not promote, deploy, publish, or replace the Champion. Gate remains a separate, explicit comparison action.
|
|
93
153
|
|
|
94
|
-
|
|
154
|
+
## Govern evaluator changes
|
|
95
155
|
|
|
96
|
-
|
|
156
|
+
Read `references/evaluator-upgrade.md` whenever the user asks to improve, replace, align, calibrate, debug, or explain an Evaluator, Rubric, Judge, reward, or meta-evaluation loop.
|
|
97
157
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
158
|
+
Use the Workbench Governance view to read component identity, source, Rubric, Judge parameters, Contract, and Context impact. Before any Evaluator/Rubric/Judge edit:
|
|
159
|
+
|
|
160
|
+
1. Show the current source and proposed diff.
|
|
161
|
+
2. State which reward semantics change.
|
|
162
|
+
3. Create a new component and Stack version; never overwrite historical identity.
|
|
163
|
+
4. Establish a fresh baseline when a reward-affecting digest or Judge identity changes.
|
|
164
|
+
5. Run meta-evaluation against independently maintained GT when aligning the Evaluator itself.
|
|
165
|
+
|
|
166
|
+
Saving a new identity does not automatically launch an evaluation or Gate.
|
|
167
|
+
|
|
168
|
+
An Evaluator implementation must use `harbor-dsh-evaluator/v1`. It may declare `kind=script` or `kind=llm-as-judge`, but both kinds accept `evaluation-input/v1` and return `evaluation-result/v1`. Every Descriptor-declared Criterion must return its declared score plus a non-empty `reason` string and a non-empty `recommendation` string. Missing explanations or recommendations invalidate the evaluator result; Reporter must not invent them. Use `harbor_evaluator_inspect` before proposing a change. After the user approves, use `harbor_evaluator_update` only for an exact `editable_files` path and provide the current digest plus new Evaluator and Stack versions. The tool creates a new versioned bundle; it does not overwrite the old implementation, run meta-evaluation, establish a baseline, or invoke Gate.
|
|
105
169
|
|
|
106
170
|
## Handle evaluator meta-evaluation
|
|
107
171
|
|
|
108
|
-
|
|
172
|
+
Rotate roles when improving the Evaluator:
|
|
109
173
|
|
|
110
|
-
- Candidate is
|
|
111
|
-
- Dataset contains
|
|
112
|
-
- Metrics
|
|
113
|
-
-
|
|
174
|
+
- Candidate is the Evaluator/Rubric/Judge version.
|
|
175
|
+
- Dataset contains fixed artifacts plus independently maintained GT with explicit source kind and provenance.
|
|
176
|
+
- Metrics include RCR, bias, variance, calibration, latency, and cost as appropriate.
|
|
177
|
+
- The Candidate evaluator must not author its own GT or final promotion decision.
|
|
114
178
|
|
|
115
|
-
|
|
179
|
+
GT may be human, programmatic, consensus-based, produced by an independently pinned model, or imported from an external standard. Independence and provenance matter more than the author type. The Candidate evaluator must never see labels before producing its observation.
|
|
116
180
|
|
|
117
|
-
|
|
181
|
+
When GT is missing, clarify its id/version, source kind, owner, provenance, Criteria, case selection, and adjudication process. Then call `harbor_ground_truth_init`; it creates a non-overwriting draft and never invents cases or labels. After cases are populated, collect repeated `evaluator-observations/v1` and call `harbor_evaluator_meta_evaluate`. Report ESF, SCE, RCR, coverage, disagreement slices, latency, and cost as applicable.
|
|
182
|
+
|
|
183
|
+
Manage evaluator Candidates and meta-evaluation Jobs with the same Manifest, Context v2, Doctor, evidence, and Gate rules.
|
|
118
184
|
|
|
119
|
-
|
|
185
|
+
## Report each cycle
|
|
120
186
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
- Evaluation
|
|
124
|
-
-
|
|
125
|
-
-
|
|
126
|
-
-
|
|
127
|
-
-
|
|
187
|
+
Return:
|
|
188
|
+
|
|
189
|
+
- Accepted Evaluation Contract and unresolved assumptions.
|
|
190
|
+
- Candidate, Dataset, Stack, Context, Judge, and Policy identities.
|
|
191
|
+
- Comparable baseline or fresh-baseline decision.
|
|
192
|
+
- Metric deltas, exception counts, Population groups, and artifact validation.
|
|
193
|
+
- Dataset coverage, terminal-state counts, valid/invalid score counts, and selected attempt policy.
|
|
194
|
+
- Representative Trial evidence and root-cause classes.
|
|
195
|
+
- Evidence provenance and any capability unavailable on a legacy Job.
|
|
196
|
+
- Controlled change hypothesis and mutation surface.
|
|
197
|
+
- Gate decision with exact reason codes.
|
|
198
|
+
- External CI/CD action still required.
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# Evaluator Upgrade Workflow
|
|
2
|
+
|
|
3
|
+
Load this reference when the user wants to improve score reliability or change Evaluator, Rubric, Judge, reward semantics, or meta-evaluation.
|
|
4
|
+
|
|
5
|
+
## Clarify the evaluator objective
|
|
6
|
+
|
|
7
|
+
Inspect `harbor_eval_result view=governance` and representative Trial evidence first. Resolve:
|
|
8
|
+
|
|
9
|
+
1. Which false positives, false negatives, disagreements, or calibration failures matter.
|
|
10
|
+
2. Who owns the independently maintained GT, its source kind and provenance, and how adjudication works.
|
|
11
|
+
3. Which meta-metrics define improvement: RCR, precision/recall, bias, variance, calibration, latency, or cost.
|
|
12
|
+
4. The allowed Evaluator/Rubric/Judge mutation surface and budget.
|
|
13
|
+
5. The approval owner for adopting new score semantics.
|
|
14
|
+
|
|
15
|
+
Do not use the Candidate Agent to author its own GT. Do not infer GT from the current Evaluator output.
|
|
16
|
+
|
|
17
|
+
## Create a new immutable evaluator identity
|
|
18
|
+
|
|
19
|
+
Show the current identity, source, proposed diff, and expected semantic impact. Create new files and increment component plus Evaluation Stack versions. Never edit a historical Evaluator, Rubric, Judge identity, or old Job artifact in place.
|
|
20
|
+
|
|
21
|
+
Use the `harbor-dsh-evaluator/v1` Descriptor as the implementation boundary:
|
|
22
|
+
|
|
23
|
+
- `kind=script` for deterministic code, rules, or local models.
|
|
24
|
+
- `kind=llm-as-judge` for a model-backed judge; keep credentials out of source and identity artifacts.
|
|
25
|
+
- Both kinds consume `evaluation-input/v1` and return `evaluation-result/v1` with Descriptor-declared Criterion ids and score values.
|
|
26
|
+
- Every Criterion also requires non-empty `reason` and `recommendation` strings. Missing fields invalidate the evaluator result; Reporter never fabricates them.
|
|
27
|
+
- `editable_files` is the exact source allowlist used by the Workbench and `harbor_evaluator_update`.
|
|
28
|
+
|
|
29
|
+
Call `harbor_evaluator_inspect` to capture the active digest. After explicit approval, `harbor_evaluator_update` requires the expected file digest plus new Evaluator and Stack versions, copies the whole bundle to a new version directory, and switches the active Stack. It never launches evaluation or Gate.
|
|
30
|
+
|
|
31
|
+
Treat changes to any of these as reward-semantic changes requiring a new Context and fresh Agent baseline:
|
|
32
|
+
|
|
33
|
+
- Evaluator source or digest.
|
|
34
|
+
- Rubric source or digest.
|
|
35
|
+
- Judge provider, model, version, or parameters.
|
|
36
|
+
- Evaluation Contract metric meaning or hard requirements.
|
|
37
|
+
|
|
38
|
+
## Meta-evaluate before adopting
|
|
39
|
+
|
|
40
|
+
Rotate roles:
|
|
41
|
+
|
|
42
|
+
- Candidate: the new Evaluator/Rubric/Judge version.
|
|
43
|
+
- Dataset: independently maintained examples with provenance-bearing GT and disagreement metadata.
|
|
44
|
+
- Evaluator: deterministic comparison between evaluator decisions and GT.
|
|
45
|
+
- Reporter: RCR and accepted diagnostic slices.
|
|
46
|
+
- Gate: explicit human-approved adoption decision.
|
|
47
|
+
|
|
48
|
+
Use the same GT set, repeat policy, and measurement procedure for old and new evaluator Candidates. Report coverage, invalid measurements, aggregate metrics, disagreement slices, latency, cost, and representative errors. Do not select only favorable runs.
|
|
49
|
+
|
|
50
|
+
GT is not synonymous with human labeling. It may be human, programmatic, consensus-based, produced by a separately pinned model, or imported from an external standard. Require explicit provenance and independence from the Candidate evaluator in every case. Use `harbor_ground_truth_init` to create the versioned draft and `harbor_evaluator_meta_evaluate` to calculate ESF, SCE, and RCR from repeated observations.
|
|
51
|
+
|
|
52
|
+
## Re-baseline Agent progress
|
|
53
|
+
|
|
54
|
+
After the evaluator Candidate passes its explicit Gate:
|
|
55
|
+
|
|
56
|
+
1. Update Evaluation Stack identity.
|
|
57
|
+
2. Preview Context v2 and confirm old Agent Jobs are no longer comparable.
|
|
58
|
+
3. Run the current Champion Agent as a fresh baseline under the new evaluator.
|
|
59
|
+
4. Compare later Agent Candidates only against Jobs sharing the new Context digest.
|
|
60
|
+
|
|
61
|
+
Adopting an evaluator does not mutate, promote, deploy, or publish an Agent automatically.
|