dsh-harbor-evolution 0.5.0 → 0.6.1

package/lib/evolution.js

@@ -33,13 +33,20 @@ export function makeJobName(manifest, now = new Date()) {
   return `${identity}-${suffix}`
 }
 
-async function cliJson(config, args, { allowedExitCodes = [0] } = {}) {
-  const result = await runProcess(config.harborDshBin, args, {
-    cwd: config.projectRoot,
-    timeoutMs: config.timeoutMs,
-    allowedExitCodes,
-    env: { ...process.env, ...(config.pythonPath ? { PYTHONPATH: config.pythonPath } : {}) },
-  })
+async function cliJson(config, args, { allowedExitCodes = [0], input } = {}) {
+  let result
+  try {
+    result = await runProcess(config.harborDshBin, args, {
+      cwd: config.projectRoot,
+      timeoutMs: config.timeoutMs,
+      allowedExitCodes,
+      input,
+      env: { ...process.env, ...(config.pythonPath ? { PYTHONPATH: config.pythonPath } : {}) },
+    })
+  } catch (error) {
+    const detail = error?.result?.stderr?.trim().split('\n').at(-1)?.replace(/^[A-Za-z]+Error:\s*/, '')
+    throw new Error(detail || error.message)
+  }
   try {
     return JSON.parse(result.stdout)
   } catch {
@@ -47,6 +54,58 @@ async function cliJson(config, args, { allowedExitCodes = [0] } = {}) {
   }
 }
 
+export async function inspectEvaluator(config, args = {}) {
+  const stack = resolveWithin(config.projectRoot, args.stackPath ?? '.harbor/evaluation-stack.yml', 'stackPath')
+  return cliJson(config, [
+    'evaluator', 'inspect',
+    '--project-root', config.projectRoot,
+    '--stack', stack,
+  ])
+}
+
+export async function updateEvaluator(config, args) {
+  const stack = resolveWithin(config.projectRoot, args.stackPath ?? '.harbor/evaluation-stack.yml', 'stackPath')
+  if (typeof args.content !== 'string') throw new Error('content is required')
+  return cliJson(config, [
+    'evaluator', 'update',
+    '--project-root', config.projectRoot,
+    '--stack', stack,
+    '--file', String(args.filePath ?? ''),
+    '--expected-digest', String(args.expectedDigest ?? ''),
+    '--new-evaluator-version', String(args.newEvaluatorVersion ?? ''),
+    '--new-stack-version', String(args.newStackVersion ?? ''),
+    '--content-stdin',
+  ], { input: args.content })
+}
+
+export async function initializeGroundTruth(config, args) {
+  const output = resolveWithin(config.projectRoot, args.outputPath ?? '.harbor/ground-truth.json', 'outputPath')
+  return cliJson(config, [
+    'ground-truth', 'init',
+    '--project-root', config.projectRoot,
+    '--output', output,
+    '--id', String(args.groundTruthId ?? ''),
+    '--version', String(args.version ?? ''),
+    '--source-kind', String(args.sourceKind ?? ''),
+    '--source-description', String(args.sourceDescription ?? ''),
+    '--provenance', String(args.provenance ?? ''),
+    '--criteria', String(args.criteria ?? ''),
+  ])
+}
+
+export async function runMetaEvaluation(config, args) {
+  const groundTruth = resolveWithin(config.projectRoot, args.groundTruthPath ?? '.harbor/ground-truth.json', 'groundTruthPath')
+  const observations = resolveWithin(config.projectRoot, args.observationsPath, 'observationsPath')
+  const output = resolveWithin(config.projectRoot, args.outputPath ?? '.harbor/meta-evaluation-report.json', 'outputPath')
+  return cliJson(config, [
+    'meta-evaluate',
+    '--project-root', config.projectRoot,
+    '--ground-truth', groundTruth,
+    '--observations', observations,
+    '--output', output,
+  ])
+}
+
 function strictInputs(config, args) {
   const projectRoot = path.resolve(config.projectRoot)
   const candidate = resolveWithin(projectRoot, args.candidatePath, 'candidatePath')

package/lib/process.js

@@ -6,7 +6,7 @@ export function runProcess(command, args, options = {}) {
       cwd: options.cwd,
       env: options.env ?? process.env,
       shell: false,
-      stdio: ['ignore', 'pipe', 'pipe'],
+      stdio: [options.input === undefined ? 'ignore' : 'pipe', 'pipe', 'pipe'],
     })
     let stdout = ''
     let stderr = ''
@@ -16,6 +16,7 @@ export function runProcess(command, args, options = {}) {
     }, options.timeoutMs ?? 1_800_000)
     child.stdout.on('data', chunk => { stdout += chunk })
     child.stderr.on('data', chunk => { stderr += chunk })
+    if (options.input !== undefined) child.stdin.end(options.input)
     child.on('error', error => {
       clearTimeout(timeout)
       reject(error)

package/lib/service.js

@@ -1,15 +1,51 @@
-import { readDashboardSnapshot, readJobDetail, readTrialDetail, readTrialsPage } from './dashboard.js'
+import { access } from 'node:fs/promises'
+import path from 'node:path'
+
+import {
+  readComparison,
+  readDashboardSnapshot,
+  readDatasetPreview,
+  readEvaluatorGovernance,
+  readJobDetail,
+  readJobProgress,
+  readMetaEvaluation,
+  readTrialDetail,
+  readTrialsPage,
+} from './dashboard.js'
 import {
   compareCandidates,
+  initializeGroundTruth,
   initializeProject,
+  inspectEvaluator,
   previewContext,
   readEvaluation,
   runDoctor,
   runEvaluation,
+  runMetaEvaluation,
   snapshot,
+  updateEvaluator,
   validateDataset,
 } from './evolution.js'
 
+export async function resolveEvaluatorStackPath(config, governance, explicitPath) {
+  if (explicitPath) return explicitPath
+  const root = path.resolve(config.projectRoot)
+  const entry = governance.components?.evaluator?.entry
+  if (typeof entry !== 'string' || !entry) return undefined
+  let directory = path.dirname(path.resolve(root, entry))
+  if (directory !== root && !directory.startsWith(`${root}${path.sep}`)) return undefined
+  while (directory === root || directory.startsWith(`${root}${path.sep}`)) {
+    const candidate = path.join(directory, '.harbor', 'evaluation-stack.yml')
+    try {
+      await access(candidate)
+      return path.relative(root, candidate)
+    } catch {}
+    if (directory === root) break
+    directory = path.dirname(directory)
+  }
+  return undefined
+}
+
 /** One Host-side boundary shared by Agent tools and the Web dashboard. */
 export class EvolutionService {
   constructor(config, metadata = {}) {
@@ -30,6 +66,15 @@ export class EvolutionService {
   }
 
   result(args) {
+    const job = String(args.jobPath ?? '').split(/[\\/]/).filter(Boolean).at(-1)
+    if (args.view === 'job') return readJobDetail(this.config, { job })
+    if (args.view === 'progress') return readJobProgress(this.config, { job, since: args.since })
+    if (args.view === 'trial') {
+      if (!args.trialId) throw new Error('trialId is required when view=trial')
+      return readTrialDetail(this.config, { job, trial: args.trialId })
+    }
+    if (args.view === 'dataset') return readDatasetPreview(this.config, { job })
+    if (args.view === 'governance') return readEvaluatorGovernance(this.config, { job, compareJob: args.compareJob })
     return readEvaluation(this.config, args)
   }
 
@@ -64,4 +109,57 @@ export class EvolutionService {
   trial(args) {
     return readTrialDetail(this.config, args)
   }
+
+  dataset(args) {
+    return readDatasetPreview(this.config, args)
+  }
+
+  progress(args) {
+    return readJobProgress(this.config, args)
+  }
+
+  comparison(args) {
+    return readComparison(this.config, args)
+  }
+
+  async governance(args) {
+    const governance = await readEvaluatorGovernance(this.config, args)
+    try {
+      const stackPath = await resolveEvaluatorStackPath(this.config, governance, args.stackPath)
+      governance.evaluatorInterface = await inspectEvaluator(this.config, { ...args, stackPath })
+      governance.editingPolicy.browserWriteEnabled = true
+      governance.editingPolicy.stackPath = governance.evaluatorInterface.stack?.path
+      governance.editingPolicy.saveBehavior = 'Update one descriptor-authorized file with optimistic concurrency and create new Evaluator and Stack identities.'
+    } catch (error) {
+      governance.evaluatorInterface = { error: error instanceof Error ? error.message : String(error) }
+    }
+    return governance
+  }
+
+  evaluator(args) {
+    return updateEvaluator(this.config, args)
+  }
+
+  evaluatorInspect(args) {
+    return inspectEvaluator(this.config, args)
+  }
+
+  groundTruthInitialize(args) {
+    return initializeGroundTruth(this.config, args)
+  }
+
+  evaluatorMetaEvaluate(args) {
+    return runMetaEvaluation(this.config, args)
+  }
+
+  async meta(args) {
+    const governance = await readEvaluatorGovernance(this.config, args)
+    const stackPath = await resolveEvaluatorStackPath(this.config, governance, args.stackPath)
+    if (!stackPath) return readMetaEvaluation(this.config)
+    const stackDirectory = path.dirname(path.resolve(this.config.projectRoot, stackPath))
+    const evaluationRoot = path.dirname(stackDirectory)
+    return readMetaEvaluation(this.config, {
+      evaluationRoot: path.relative(this.config.projectRoot, evaluationRoot),
+    })
+  }
 }

package/lib/setup.js

@@ -267,7 +267,11 @@ export async function setupIntegration(raw = {}, dependencies = {}) {
   try {
     await run('pnpm', [
       '--silent', 'dlx', `@deepseek-ai/dsh@${DSH_VERSION}`,
-      'plugin', '--profile', config.profile, 'add', '-w', '--save-exact', config.pluginSpec,
+      // Registry packages ship their built client and need no lifecycle
+      // scripts. Skipping scripts also prevents pnpm 11 from reclassifying
+      // unrelated DSH native dependencies as newly unapproved builds while
+      // adding this plugin to an existing profile.
+      'plugin', '--profile', config.profile, 'add', '-w', '--save-exact', '--ignore-scripts', config.pluginSpec,
     ], { env: { ...env, DSH_HOME: config.dshHome } })
   } catch (error) {
     throw new Error(processFailure(error))

package/lib/web.js

@@ -2,6 +2,13 @@ export const DASHBOARD_ROUTE = '/_dsh/harbor-evolution/dashboard'
 export const JOB_ROUTE = '/_dsh/harbor-evolution/job'
 export const TRIALS_ROUTE = '/_dsh/harbor-evolution/trials'
 export const TRIAL_ROUTE = '/_dsh/harbor-evolution/trial'
+export const DATASET_ROUTE = '/_dsh/harbor-evolution/dataset'
+export const PROGRESS_ROUTE = '/_dsh/harbor-evolution/progress'
+export const COMPARE_ROUTE = '/_dsh/harbor-evolution/compare'
+export const GOVERNANCE_ROUTE = '/_dsh/harbor-evolution/governance'
+export const EVALUATOR_ROUTE = '/_dsh/harbor-evolution/evaluator'
+export const META_ROUTE = '/_dsh/harbor-evolution/meta'
+const MAX_MUTATION_BYTES = 256 * 1024
 
 function sendJson(response, status, body) {
   response.writeHead(status, {
@@ -60,6 +67,41 @@ export function createDashboardHandler(service) {
   return createApiHandler(() => service.dashboard(), 'dashboard-unavailable')
 }
 
+export function createMutationHandler(update, code = 'update-failed') {
+  return async (request, response) => {
+    if (request.method !== 'POST') {
+      response.writeHead(405, { allow: 'POST' })
+      response.end()
+      return
+    }
+    if (!isSameOriginRequest(request)) {
+      sendJson(response, 403, { ok: false, error: { code: 'forbidden', message: 'same-origin request required' } })
+      return
+    }
+    if (!String(request.headers['content-type'] ?? '').toLowerCase().startsWith('application/json')) {
+      sendJson(response, 415, { ok: false, error: { code: 'unsupported-media-type', message: 'application/json required' } })
+      return
+    }
+    try {
+      const chunks = []
+      let size = 0
+      for await (const chunk of request) {
+        size += chunk.length
+        if (size > MAX_MUTATION_BYTES) {
+          sendJson(response, 413, { ok: false, error: { code: 'payload-too-large', message: 'request body is too large' } })
+          return
+        }
+        chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
+      }
+      const body = JSON.parse(Buffer.concat(chunks).toString('utf8'))
+      const value = await update(body)
+      sendJson(response, 200, { ok: true, value })
+    } catch (error) {
+      sendJson(response, 400, { ok: false, error: { code, message: safeError(error) } })
+    }
+  }
+}
+
 export function installDashboardWeb(ctx, service) {
   if (typeof ctx.inject !== 'function') return
   ctx.inject(['webServer'], (webCtx) => {
@@ -68,6 +110,12 @@ export function installDashboardWeb(ctx, service) {
       [JOB_ROUTE, createApiHandler(args => service.job(args), 'job-unavailable')],
       [TRIALS_ROUTE, createApiHandler(args => service.trials(args), 'trials-unavailable')],
       [TRIAL_ROUTE, createApiHandler(args => service.trial(args), 'trial-unavailable')],
+      [DATASET_ROUTE, createApiHandler(args => service.dataset(args), 'dataset-unavailable')],
+      [PROGRESS_ROUTE, createApiHandler(args => service.progress(args), 'progress-unavailable')],
+      [COMPARE_ROUTE, createApiHandler(args => service.comparison(args), 'comparison-unavailable')],
+      [GOVERNANCE_ROUTE, createApiHandler(args => service.governance(args), 'governance-unavailable')],
+      [EVALUATOR_ROUTE, createMutationHandler(args => service.evaluator(args), 'evaluator-update-failed')],
+      [META_ROUTE, createApiHandler(args => service.meta(args), 'meta-evaluation-unavailable')],
     ]
     for (const [route, handler] of routes) {
       webCtx.effect(() => webCtx.webServer.register({ kind: 'exact', path: route, handler }), `harbor-evolution: ${route}`)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-harbor-evolution",
-  "version": "0.5.0",
+  "version": "0.6.1",
   "description": "DeepSeek Harness plugin and bundled Skill for safely evolving Cordis Candidates with Harbor.",
   "type": "module",
   "main": "index.js",
@@ -8,6 +8,10 @@
     ".": "./index.js",
     "./client": "./lib/client.js",
     "./cordis.patch.yml": "./cordis.patch.yml",
+    "./schemas/evaluation-result.schema.json": "./schemas/evaluation-result.schema.json",
+    "./schemas/ground-truth.schema.json": "./schemas/ground-truth.schema.json",
+    "./schemas/evaluator-observations.schema.json": "./schemas/evaluator-observations.schema.json",
+    "./schemas/meta-evaluation-report.schema.json": "./schemas/meta-evaluation-report.schema.json",
     "./package.json": "./package.json"
   },
   "bin": {
@@ -18,6 +22,7 @@
     "lib/",
     "bin/",
     "skills/",
+    "schemas/",
     "cordis.patch.yml",
     "README.md",
     "LICENSE"

package/schemas/evaluation-result.schema.json

@@ -0,0 +1,27 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Harbor DSH Evaluator Result v1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "protocol", "criteria"],
+  "properties": {
+    "schema_version": { "const": 1 },
+    "protocol": { "const": "evaluation-result/v1" },
+    "criteria": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["id", "score", "reason", "recommendation"],
+        "properties": {
+          "id": { "type": "string", "minLength": 1 },
+          "score": { "enum": [0, 0.5, 1] },
+          "reason": { "type": "string", "minLength": 1 },
+          "recommendation": { "type": "string", "minLength": 1 },
+          "evidence_refs": { "type": "array", "items": { "type": "string" } }
+        }
+      }
+    }
+  }
+}

package/schemas/evaluator-observations.schema.json

@@ -0,0 +1,51 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Evaluator Observations v1",
+  "type": "object",
+  "required": ["schema_version", "protocol", "evaluator", "repeat_policy", "observations"],
+  "properties": {
+    "schema_version": { "const": 1 },
+    "protocol": { "const": "evaluator-observations/v1" },
+    "evaluator": {
+      "type": "object",
+      "required": ["id", "version"],
+      "properties": {
+        "id": { "type": "string", "minLength": 1 },
+        "version": { "type": "string", "minLength": 1 },
+        "digest": { "type": "string" }
+      }
+    },
+    "repeat_policy": {
+      "type": "object",
+      "required": ["repeats"],
+      "properties": {
+        "repeats": { "type": "integer", "minimum": 1 },
+        "seed_policy": { "type": "string" }
+      }
+    },
+    "observations": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "required": ["case_id", "repeat", "criteria"],
+        "properties": {
+          "case_id": { "type": "string", "minLength": 1 },
+          "repeat": { "type": "integer", "minimum": 1 },
+          "criteria": {
+            "type": "array",
+            "minItems": 1,
+            "items": {
+              "type": "object",
+              "required": ["id", "score"],
+              "properties": {
+                "id": { "type": "string", "minLength": 1 },
+                "score": { "enum": [0, 0.5, 1] }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

package/schemas/ground-truth.schema.json

@@ -0,0 +1,61 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Evaluator Ground Truth v1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "protocol", "ground_truth_id", "version", "source", "criteria", "cases"],
+  "properties": {
+    "schema_version": { "const": 1 },
+    "protocol": { "const": "ground-truth/v1" },
+    "ground_truth_id": { "type": "string", "minLength": 1 },
+    "version": { "type": "string", "minLength": 1 },
+    "source": {
+      "type": "object",
+      "required": ["kind", "description", "provenance", "independent_of_candidate"],
+      "properties": {
+        "kind": { "enum": ["human", "programmatic", "consensus", "model", "external"] },
+        "description": { "type": "string", "minLength": 1 },
+        "provenance": { "type": "string", "minLength": 1 },
+        "independent_of_candidate": { "const": true }
+      }
+    },
+    "criteria": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "required": ["id", "label"],
+        "properties": {
+          "id": { "type": "string", "minLength": 1 },
+          "label": { "type": "string", "minLength": 1 }
+        }
+      }
+    },
+    "cases": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["id", "artifact_ref", "criteria"],
+        "properties": {
+          "id": { "type": "string", "minLength": 1 },
+          "artifact_ref": { "type": "string", "minLength": 1 },
+          "badcase": { "type": "boolean" },
+          "criteria": {
+            "type": "array",
+            "minItems": 1,
+            "items": {
+              "type": "object",
+              "required": ["id", "score", "weight", "reason"],
+              "properties": {
+                "id": { "type": "string", "minLength": 1 },
+                "score": { "enum": [0, 0.5, 1] },
+                "weight": { "type": "number", "exclusiveMinimum": 0 },
+                "reason": { "type": "string", "minLength": 1 }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

package/schemas/meta-evaluation-report.schema.json

@@ -0,0 +1,23 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Evaluator Meta-Evaluation Report v1",
+  "type": "object",
+  "required": ["schema_version", "protocol", "ground_truth", "evaluator", "coverage", "metrics", "disagreements"],
+  "properties": {
+    "schema_version": { "const": 1 },
+    "protocol": { "const": "meta-evaluation-report/v1" },
+    "ground_truth": { "type": "object" },
+    "evaluator": { "type": "object" },
+    "coverage": { "type": "object" },
+    "metrics": {
+      "type": "object",
+      "required": ["esf", "sce", "rcr"],
+      "properties": {
+        "esf": { "type": "number", "minimum": 0, "maximum": 1 },
+        "sce": { "type": "number", "minimum": 0, "maximum": 1 },
+        "rcr": { "type": ["number", "null"], "minimum": 0, "maximum": 1 }
+      }
+    },
+    "disagreements": { "type": "array" }
+  }
+}

package/skills/evolve-agent-with-harbor/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: evolve-agent-with-harbor
-description: Architect, initialize, diagnose, evaluate, compare, and safely improve a DeepSeek Harness business Agent with Harbor Evaluation Stack, Dataset Manifest, Context v2, Architecture Doctor, and Promotion Gate. Use for Harbor setup, Agent self-evolution, vertical-search evaluation loops, evaluation architecture review, failed Job diagnosis, Candidate optimization, evaluator meta-evaluation, or promotion decisions.
+description: Architect, initialize, run, diagnose, compare, and safely improve a DeepSeek Harness business Agent with Harbor Trial Lifecycle, Score Validity, Evidence Provenance, Evaluation Stack, Context v2, Architecture Doctor, and explicit Promotion Gate. Use for Harbor setup, Agent self-evolution, vertical-search evaluation loops, running Job inspection, failed Trial diagnosis, Candidate optimization, evaluator governance or meta-evaluation, and promotion decisions.
 ---
 
 # Evolve Agent With Harbor
@@ -21,7 +21,8 @@ Treat Harbor as the experiment boundary. Deployment, CI/CD, and Champion replace
 - **Diagnostic**: investigate failures without making a promotion claim.
 - **Promotion**: run a `promotion-eligible` Job and apply the deterministic Gate.
 - **Evolve**: baseline → diagnose → one controlled change → regression Job → Gate.
-- **Meta-evaluate**: improve an Evaluator/Judge against independently maintained human GT.
+- **Meta-evaluate**: improve an Evaluator/Judge against independently maintained, provenance-bearing GT.
+- **Govern**: inspect Evaluator/Rubric/Judge source and identities; preview whether a change requires a fresh baseline.
 
 Do not turn an inspection or diagnostic request into Agent mutation or deployment.
 
@@ -49,6 +50,8 @@ Require these before every Job:
 - `.harbor/evaluation-stack.yml` with all eight roles, Judge identity, and Evaluation Contract.
 - Evaluation Context v2 preview.
 
+Require `input_integrity`, `agent_completed`, `integration_valid`, `renderer_valid`, `judge_completed`, and `artifact_schema_valid` in the Trial validity contract. Specify which failures are hard requirements. Never infer that a numeric raw verifier reward is a valid Candidate quality score.
+
 Before a formal Job, call in order:
 
 1. `harbor_candidate_snapshot`
@@ -98,7 +101,15 @@ Never cherry-pick stochastic runs. Apply the accepted repeat/seed policy symmetr
 
 ### Diagnose before changing
 
-Use `harbor_eval_result` only to reopen a stable Job summary. Inspect Trial assessments and classify each failure as:
+Use `harbor_eval_result` to reopen evidence without guessing local artifact paths: default `view=summary`, `view=job` for capabilities and stage artifacts, `view=dataset` for Agent-visible instructions, `view=progress` while running, `view=trial` with a returned `trialId` for the generated output and sanitized evidence, and `view=governance` for Evaluator/Rubric/Judge source and upgrade impact. Inspect in this order:
+
+1. Confirm every Dataset item reached a terminal Trial state. Running, queued, cancelled, or missing Trials are not quality evidence.
+2. Check `score.valid` and every validity requirement. Display an invalid score as `—`, never `0`.
+3. Inspect evidence provenance. Keep `Real Renderer`, `ACP Agent Output Fallback`, raw transport evidence, Judge explanation, and deterministic diagnosis distinct.
+4. Inspect findings, recommendations, user-visible output, criteria, and timing.
+5. Classify the owning layer before proposing a mutation.
+
+Treat `raw_rewards` as audit-only when `score.valid=false`. Aggregate and compare only valid quality scores. Inspect Trial assessments and classify each failure as:
 
 - Candidate capability or policy.
 - Tool-call, invalid search, citation, or output-contract failure.
@@ -108,6 +119,15 @@ Use `harbor_eval_result` only to reopen a stable Job summary. Inspect Trial asse
 
 Do not optimize the Candidate around broken evaluation infrastructure. Never leak holdout answers or GT into Candidate prompts, skills, tools, or memory.
 
+Use the formal terminal states precisely:
+
+- `candidate-quality-failed`: valid execution reached evaluation, but a Candidate-owned hard requirement failed.
+- `infrastructure-error`: dependency, sandbox, permission, transport, timeout, or runtime failure; no Candidate quality score.
+- `evaluation-error`: Renderer/Judge/Verifier did not complete; no Candidate quality score.
+- `cancelled`: preserve the attempt and do not score it.
+
+For retry or resume, retain the old attempt and create a new attempt. Never replace an assessment or event history in place.
+
 ### Propose one controlled change
 
 Require every optimization hypothesis to include:
@@ -129,15 +149,37 @@ Call `harbor_context_preview`; establish a fresh baseline if needed. Run the Can
 
 Never bypass `INFRASTRUCTURE_EXCEPTION_PRESENT`, `ARTIFACT_SCHEMA_INVALID`, Dataset/Stack/Rubric/Judge mismatch, or non-regression failures.
 
+A `diagnostic` Job must never invoke Gate. Reading the Workbench, generating a Reporter summary, or producing a non-reward Optimization Report also must not promote, deploy, publish, or replace the Champion. Gate remains a separate, explicit comparison action.
+
+## Govern evaluator changes
+
+Read `references/evaluator-upgrade.md` whenever the user asks to improve, replace, align, calibrate, debug, or explain an Evaluator, Rubric, Judge, reward, or meta-evaluation loop.
+
+Use the Workbench Governance view to read component identity, source, Rubric, Judge parameters, Contract, and Context impact. Before any Evaluator/Rubric/Judge edit:
+
+1. Show the current source and proposed diff.
+2. State which reward semantics change.
+3. Create a new component and Stack version; never overwrite historical identity.
+4. Establish a fresh baseline when a reward-affecting digest or Judge identity changes.
+5. Run meta-evaluation against independently maintained GT when aligning the Evaluator itself.
+
+Saving a new identity does not automatically launch an evaluation or Gate.
+
+An Evaluator implementation must use `harbor-dsh-evaluator/v1`. It may declare `kind=script` or `kind=llm-as-judge`, but both kinds accept `evaluation-input/v1` and return `evaluation-result/v1`. Every Descriptor-declared Criterion must return its declared score plus a non-empty `reason` string and a non-empty `recommendation` string. Missing explanations or recommendations invalidate the evaluator result; Reporter must not invent them. Use `harbor_evaluator_inspect` before proposing a change. After the user approves, use `harbor_evaluator_update` only for an exact `editable_files` path and provide the current digest plus new Evaluator and Stack versions. The tool creates a new versioned bundle; it does not overwrite the old implementation, run meta-evaluation, establish a baseline, or invoke Gate.
+
 ## Handle evaluator meta-evaluation
 
 Rotate roles when improving the Evaluator:
 
 - Candidate is the Evaluator/Rubric/Judge version.
-- Dataset contains independently maintained human GT.
+- Dataset contains fixed artifacts plus independently maintained GT with explicit source kind and provenance.
 - Metrics include RCR, bias, variance, calibration, latency, and cost as appropriate.
 - The Candidate evaluator must not author its own GT or final promotion decision.
 
+GT may be human, programmatic, consensus-based, produced by an independently pinned model, or imported from an external standard. Independence and provenance matter more than the author type. The Candidate evaluator must never see labels before producing its observation.
+
+When GT is missing, clarify its id/version, source kind, owner, provenance, Criteria, case selection, and adjudication process. Then call `harbor_ground_truth_init`; it creates a non-overwriting draft and never invents cases or labels. After cases are populated, collect repeated `evaluator-observations/v1` and call `harbor_evaluator_meta_evaluate`. Report ESF, SCE, RCR, coverage, disagreement slices, latency, and cost as applicable.
+
 Manage evaluator Candidates and meta-evaluation Jobs with the same Manifest, Context v2, Doctor, evidence, and Gate rules.
 
 ## Report each cycle
@@ -148,7 +190,9 @@ Return:
 - Candidate, Dataset, Stack, Context, Judge, and Policy identities.
 - Comparable baseline or fresh-baseline decision.
 - Metric deltas, exception counts, Population groups, and artifact validation.
+- Dataset coverage, terminal-state counts, valid/invalid score counts, and selected attempt policy.
 - Representative Trial evidence and root-cause classes.
+- Evidence provenance and any capability unavailable on a legacy Job.
 - Controlled change hypothesis and mutation surface.
 - Gate decision with exact reason codes.
 - External CI/CD action still required.