dsh-harbor-evolution 0.7.2 → 0.8.0

package/schemas/historical-evaluation-context.schema.json

@@ -0,0 +1,66 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/istarwyh/harbor-self-evolving/schemas/historical-evaluation-context.schema.json",
+  "title": "Historical Generation Evaluation Context v1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "protocol", "job_kind", "mode", "promotion_eligible", "execution_mode", "evaluation_level", "evaluation_target", "generation_source", "dataset", "evaluation_stack", "execution_adapter", "runtime", "downstream_analysis", "digest", "full_digest"],
+  "properties": {
+    "schema_version": { "const": 1 },
+    "protocol": { "const": "historical-generation-evaluation-context/v1" },
+    "job_kind": { "const": "historical-generation-evaluation" },
+    "mode": { "const": "diagnostic" },
+    "promotion_eligible": { "const": false },
+    "execution_mode": { "const": "observe-existing" },
+    "evaluation_level": { "const": "trial" },
+    "evaluation_target": {
+      "type": "object",
+      "required": ["kind", "source_kind", "batch_id", "digest", "record_count", "generator_population"],
+      "properties": {
+        "kind": { "const": "generation-record-batch" },
+        "source_kind": { "const": "dsh-session" },
+        "batch_id": { "type": "string", "minLength": 1 },
+        "digest": { "$ref": "#/$defs/digest" },
+        "record_count": { "type": "integer", "minimum": 1, "maximum": 10 },
+        "generator_population": { "type": "object" }
+      }
+    },
+    "generation_source": {
+      "type": "object",
+      "required": ["mode", "kind", "adapter_id", "selection", "redaction_policy"],
+      "properties": {
+        "mode": { "const": "existing-records" },
+        "kind": { "const": "dsh-session" },
+        "adapter_id": { "const": "dsh-session-query" },
+        "adapter": { "const": "dsh-session-query" },
+        "selection": { "type": "object" },
+        "redaction_policy": { "type": "object" }
+      }
+    },
+    "dataset": { "type": "object" },
+    "evaluation_stack": { "type": "object", "required": ["stack_id", "version", "digest", "comparison_digest", "components", "judge"] },
+    "execution_adapter": { "type": "object", "required": ["id", "version", "import_path", "digest", "model_invocation", "tool_reexecution"] },
+    "runtime": { "type": "object", "required": ["harbor_version", "integration_version"] },
+    "downstream_analysis": {
+      "type": "object",
+      "required": ["population_analysis", "generator_diagnosis", "optimizer", "evaluator_meta_evaluation"],
+      "properties": {
+        "population_analysis": { "const": true },
+        "generator_diagnosis": { "type": ["boolean", "object"] },
+        "optimizer": { "type": "object" },
+        "evaluator_meta_evaluation": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["status", "validation_report_ref"],
+          "properties": {
+            "status": { "const": "not-run" },
+            "validation_report_ref": { "type": "null" }
+          }
+        }
+      }
+    },
+    "digest": { "$ref": "#/$defs/digest" },
+    "full_digest": { "$ref": "#/$defs/digest" }
+  },
+  "$defs": { "digest": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" } }
+}

package/schemas/historical-evaluation-summary.schema.json

@@ -0,0 +1,49 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/istarwyh/harbor-self-evolving/schemas/historical-evaluation-summary.schema.json",
+  "title": "Historical Generation Evaluation Summary v4",
+  "type": "object",
+  "required": ["schema_version", "job", "job_kind", "mode", "execution_mode", "evaluation_target", "generation_source", "evaluation_context", "n_trials", "n_discovered_trials", "n_completed_trials", "n_valid_scores", "n_invalid_scores", "n_unscored_trials", "status_counts", "coverage", "criterion_status_counts", "metrics", "trials", "artifact_validation", "evaluator_meta_evaluation"],
+  "properties": {
+    "schema_version": { "const": 4 },
+    "job": { "type": "string", "minLength": 1 },
+    "job_kind": { "const": "historical-generation-evaluation" },
+    "mode": { "const": "diagnostic" },
+    "execution_mode": { "const": "observe-existing" },
+    "evaluation_target": { "type": "object", "required": ["kind", "source_kind", "batch_id", "digest", "record_count"] },
+    "generation_source": { "type": "object", "required": ["kind"] },
+    "evaluation_context": { "type": "object" },
+    "n_trials": { "type": "integer", "minimum": 1, "maximum": 10 },
+    "n_discovered_trials": { "type": "integer", "minimum": 0, "maximum": 10 },
+    "n_completed_trials": { "type": "integer", "minimum": 0, "maximum": 10 },
+    "n_valid_scores": { "type": "integer", "minimum": 0, "maximum": 10 },
+    "n_invalid_scores": { "type": "integer", "minimum": 0, "maximum": 10 },
+    "n_unscored_trials": { "type": "integer", "minimum": 0, "maximum": 10 },
+    "status_counts": { "type": "object", "additionalProperties": { "type": "integer", "minimum": 0 } },
+    "coverage": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["scored_trials", "unscored_trials", "total_trials", "trial_rate", "criterion_scored", "criterion_total", "criterion_rate"],
+      "properties": {
+        "scored_trials": { "type": "integer", "minimum": 0 },
+        "unscored_trials": { "type": "integer", "minimum": 0 },
+        "total_trials": { "type": "integer", "minimum": 1, "maximum": 10 },
+        "trial_rate": { "type": "number", "minimum": 0, "maximum": 1 },
+        "criterion_scored": { "type": "integer", "minimum": 0 },
+        "criterion_total": { "type": "integer", "minimum": 0 },
+        "criterion_rate": { "type": "number", "minimum": 0, "maximum": 1 }
+      }
+    },
+    "criterion_status_counts": { "type": "object", "additionalProperties": { "type": "integer", "minimum": 0 } },
+    "metrics": { "type": "object", "additionalProperties": { "type": "number" } },
+    "trials": { "type": "array", "maxItems": 10, "items": { "type": "object" } },
+    "artifact_validation": { "type": "object" },
+    "evaluator_meta_evaluation": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["status", "validation_report_ref"],
+      "properties": { "status": { "const": "not-run" }, "validation_report_ref": { "type": "null" } }
+    }
+  },
+  "not": { "required": ["candidate"] }
+}

package/schemas/historical-generation-batch.schema.json

@@ -0,0 +1,76 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/istarwyh/harbor-self-evolving/schemas/historical-generation-batch.schema.json",
+  "title": "Historical Generation Batch v1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "protocol", "batch_id", "created_at", "project", "selection", "source", "redaction_policy", "records", "generator_population", "digest"],
+  "properties": {
+    "schema_version": { "const": 1 },
+    "protocol": { "const": "historical-generation-batch/v1" },
+    "batch_id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$" },
+    "created_at": { "type": "string", "format": "date-time" },
+    "project": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["cwd_digest"],
+      "properties": { "cwd_digest": { "$ref": "#/$defs/digest" } }
+    },
+    "selection": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["scope", "order", "requested_limit", "selected_count", "current_session_excluded"],
+      "properties": {
+        "scope": { "const": "exact-cwd" },
+        "order": { "const": "last-activity-desc" },
+        "requested_limit": { "type": "integer", "minimum": 1, "maximum": 10 },
+        "selected_count": { "type": "integer", "minimum": 1, "maximum": 10 },
+        "current_session_excluded": { "const": true },
+        "created_after": { "type": "string", "format": "date-time" }
+      }
+    },
+    "source": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "adapter", "session_format_versions"],
+      "properties": {
+        "kind": { "const": "dsh-session" },
+        "adapter": { "const": "dsh-session-query" },
+        "session_format_versions": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "type": "integer", "minimum": 0 } }
+      }
+    },
+    "redaction_policy": {
+      "type": "object",
+      "required": ["id", "version", "digest"],
+      "properties": {
+        "id": { "type": "string", "minLength": 1 },
+        "version": { "type": "string", "minLength": 1 },
+        "digest": { "$ref": "#/$defs/digest" }
+      }
+    },
+    "records": {
+      "type": "array",
+      "minItems": 1,
+      "maxItems": 10,
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["trial_id", "record_kind", "source_ref", "captured_through_seq", "source_digest", "observation_digest", "last_activity_at", "generator", "observation_path"],
+        "properties": {
+          "trial_id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$" },
+          "record_kind": { "const": "dsh-session" },
+          "source_ref": { "$ref": "#/$defs/digest" },
+          "captured_through_seq": { "type": "integer", "minimum": 0 },
+          "source_digest": { "$ref": "#/$defs/digest" },
+          "observation_digest": { "$ref": "#/$defs/digest" },
+          "last_activity_at": { "type": "string", "format": "date-time" },
+          "generator": { "type": "object" },
+          "observation_path": { "type": "string", "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$))[A-Za-z0-9._/-]+$" }
+        }
+      }
+    },
+    "generator_population": { "type": "object" },
+    "digest": { "$ref": "#/$defs/digest" }
+  },
+  "$defs": { "digest": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" } }
+}

package/skills/evolve-agent-with-harbor/SKILL.md

@@ -1,11 +1,11 @@
 ---
 name: evolve-agent-with-harbor
-description: Architect, initialize, run, diagnose, compare, and safely improve a DeepSeek Harness business Agent with Harbor Trial Lifecycle, Score Validity, Evidence Provenance, Evaluation Stack, Context v2, Architecture Doctor, and explicit Promotion Gate. Use for Harbor setup, Agent self-evolution, vertical-search evaluation loops, running Job inspection, failed Trial diagnosis, Candidate optimization, evaluator governance or meta-evaluation, and promotion decisions.
+description: Architect, initialize, run, diagnose, compare, and safely improve a DeepSeek Harness business Agent or Evaluator with Harbor. Use for low-friction Harbor setup, evaluating recent completed DSH Sessions when no Dataset is supplied, Agent self-evolution, vertical-search evaluation loops, running Job inspection, failed Trial diagnosis, Candidate optimization, evaluator governance, turning reviewed reports and natural-language scoring feedback into evaluator meta-evaluation data, or explicit promotion decisions.
 ---
 
 # Evolve Agent With Harbor
 
-Build a reproducible improvement loop around four concepts that users can describe in business language:
+Build a maintainable, evidence-bearing improvement loop around four concepts that users can describe in business language. The DSH and Candidate ACP runtime follows the latest published release by default; record that policy honestly and do not block a Job on an older pinned runtime:
 
 - **评测集 (Dataset)** — what should be tested: one Query, a file, a directory of instructions, or an existing Harbor Dataset.
 - **生成器 (Generator)** — who produces the answer or artifact: a curl request, a local Agent entry, or an Agent already found in the workspace.
@@ -16,12 +16,24 @@ Keep these four names visible during onboarding and confirmation; they establish
 
 Treat Harbor as the experiment boundary. Deployment, CI/CD, and Champion replacement remain external actions requiring separate authority.
 
+## Keep the Generator model explicit
+
+For a DSH/Cordis Generator, offer “使用当前 Harbor Agent 模型” as the default model choice. Explain that this creates a **model binding**, not a live pointer:
+
+> 创建 Candidate 时会固定本次 `provider / model / reasoning`；之后切换聊天模型不会改写已经建立的 Candidate。
+
+After the user accepts, call `harbor_model_binding` and write its `candidate_model_binding` output verbatim to `model-binding.json` before `harbor_candidate_snapshot`. Show the resolved provider/model in the confirmation card. The file contains identity only and becomes part of the Candidate digest.
+
+The runtime must remain `dsh-host-broker` / `dsh-host-model-gateway/v1`: the Candidate receives a random, short-lived Job capability, never GPT Auth, Codex OAuth, an API key, or another Host credential. Do not add a provider credential, auth file path, or secret value to the Candidate, Dataset, Stack, Job, prompt, report, or tool arguments. A pinned Candidate that needs a different model must become a new Candidate version; do not override its binding in place.
+
 ## Select the narrowest mode
 
 - **Clarify**: identify the Dataset, Generator, Evaluator/criteria, and Optimizer with the least user effort.
 - **Architecture**: inspect role boundaries and run `harbor_evolution_doctor`.
 - **Initialize**: read `references/initialization.md`, compile the accepted four-concept card, then call `harbor_evolution_init`.
 - **Diagnostic**: investigate failures without making a promotion claim.
+- **Historical generation diagnostic**: when no Dataset was supplied, preview recent completed DSH Sessions and, only after confirmation, evaluate the immutable records without re-executing a Candidate.
+- **Quick diagnostic**: after confirmation, call `harbor_quick_diagnostic_init` for one Query plus a Rubric draft. It generates a Harbor 1.4 wiring project that reuses the current DSH model and is permanently marked non-promotable.
 - **Promotion**: run a `promotion-eligible` Job and apply the deterministic Gate.
 - **Evolve**: baseline → diagnose → one controlled change → regression Job → Gate.
 - **Meta-evaluate**: improve an Evaluator/Judge against independently maintained, provenance-bearing GT.
@@ -29,16 +41,54 @@ Treat Harbor as the experiment boundary. Deployment, CI/CD, and Champion replace
 
 Do not turn an inspection or diagnostic request into Agent mutation or deployment.
 
+## Default to recent Sessions only when Dataset is absent
+
+Preserve explicit user input. If the user supplies any Dataset, Query, instruction file/directory, Dataset path, or Dataset-bearing curl workflow, use the normal four-concept flow below. Never replace or augment an explicit Dataset with Session history unless the user separately asks for that change.
+
+Only when no Dataset was supplied and `harbor_session_diagnostic_preview` is available:
+
+1. Call `harbor_session_diagnostic_preview` with `limit=10`. This is a read-only Preview, not a Job. If the user selected a different Judge, pass its provider/model/reasoning options here so that identity is part of the confirmation token.
+2. Present the returned safe Session metadata, exact-cwd scope, last-activity order, excluded counts, warnings, estimated Judge requests, token expiry, and confirmation text. Do not expose or reconstruct raw Session ids, transcripts, tool payloads, or credentials.
+3. Explain the role mapping plainly: the DSH Agent that produced each Session remains the **Generator**; the completed Session is immutable Generation Record evidence; one Historical Generation Evaluation Job contains up to 10 Trials; one selected Session becomes one Trial.
+4. Ask for explicit confirmation. Do not call the run tool merely because Preview succeeded. If the sample changed or the token expired, preview again instead of widening scope.
+5. After confirmation, call `harbor_session_diagnostic_run` with the returned `selectionToken` (and only an optional `jobName`). Evaluator/Judge overrides belong to Preview and are rejected at Run so the confirmed identity cannot change. The tool synchronously materializes the private Batch into its matching Dataset and immutable Historical Evaluation Stack before starting the Job. Do not pass `stackPath`: the MVP rejects custom Historical Stacks so the executed Evaluator cannot drift from the declared Stack. Do not call `harbor_candidate_snapshot`, `harbor_model_binding`, `harbor_context_preview`, or `harbor_eval_run` for this branch.
+
+Render this compact confirmation card before running:
+
+```text
+会话历史评测确认
+- 范围：当前工作目录，按最后活动时间选取最近 <N>/10 条已完成会话
+- 生成器：产生这些会话的 DSH Agent（本次不会重新执行）
+- 评测对象：<N> 条已有 Generation Records；1 条会话 = 1 Trial
+- 评测器：<evaluation.evaluator.id>@<version> · Judge <evaluation.judge.provider>/<model>
+- 评测耦合：<evaluation.coupling；同模型或 Generator 模型未知时明确仅用于诊断，不声称独立>
+- 成本上界：<estimatedJudgeRequests> 次 Judge 请求
+- 用途：诊断、群体分析与生成器问题定位
+- 本地保留：`.harbor/private` 和 `jobs` 会保存脱敏后的真实业务会话证据，默认不会自动删除
+- VCS 风险：private 根不存在规则时会创建 ignore-all `.gitignore`，但不会覆盖已有规则；`jobs` 的忽略、上传与保留策略仍由项目负责
+- 不会执行：Candidate 生成、本 Historical Job 内的评测器元评测、Promotion Gate 或部署
+```
+
+Treat the resulting `historical-generation-evaluation` Job as `diagnostic` and `observe-existing`. `completed-unscored` is a normal abstention when evidence is insufficient; report scored/unscored Trial and Criterion coverage separately, and never convert abstention into business score `0`.
+
+Population Analysis and Generator Diagnosis summarize the observed records and Generator population. They are not Evaluator Meta-Evaluation. The existing independent-GT flow (`harbor_ground_truth_init` plus repeated evaluator observations and `harbor_evaluator_meta_evaluate`) remains available as a separate governance action, but this Historical Job never invokes or inherits it automatically. For the Historical Job report `evaluator_meta_evaluation.status=not-run`, say that its Evaluator reliability remains unvalidated, and never claim ESF/SCE/RCR evidence from Session scores. A future dedicated `evaluator-meta-evaluation` Job may package that existing flow into a Job lifecycle; do not describe the underlying meta-evaluation capability as absent.
+
+Historical Generation Jobs are never comparable Candidate baselines or Promotion Gate inputs. Report Gate as `N/A`; if comparison is requested, explain `UNSUPPORTED_JOB_KIND_FOR_PROMOTION` and first convert reviewed badcases into a fixed regression Dataset.
+
+If the Session Query capability or Preview tool is unavailable, state that limitation and continue with the ordinary four-concept intake. Do not invent a filesystem transcript scan as a fallback.
+
+State current MVP limits instead of suggesting unsupported controls: selection is exact-cwd, reads at most the configured `sessionMaxReads` candidates (100 by default), accepts an optional ISO-8601 `createdAfter` lower bound, and exposes no cursor. On `SESSION_SELECTION_TOO_EXPENSIVE`, preview again with a narrower `createdAfter`, use an explicit Query/Dataset, or ask an administrator to review the read limit. A token binds Feedback availability, failure state, and content digest without retaining raw Feedback; if any of them changes, Run fails before writing the Batch and requires a new Preview. Do not claim that a real Docker/Harbor/Workbench journey passed from unit tests, generated files, or a zero process exit code alone.
+
 ## Start with four clear concepts
 
 Inspect the current workspace before asking questions. Look for Agent entry files, package metadata, curl examples, Dataset instructions, existing Harbor configuration/Jobs, tests, and available Codex or Claude Code commands. Reuse reliable findings and say what was inferred; do not ask the user to transcribe information already present in files.
 
-When no Harbor workspace exists, propose `./harbor-evolution/` under the current session working directory as the managed evaluation workspace. Agent-facing Harbor Tools derive `projectRoot` from the calling session for every invocation and keep imported snapshots and generated evaluation files inside that request-local root. Treat the Plugin's configured `projectRoot` only as the Web Workbench and non-Agent fallback; do not block initialization merely because it differs from the current session working directory.
+When no Harbor workspace exists, propose `./harbor-evolution/` under the current session working directory as the managed evaluation workspace. Agent-facing Harbor Tools derive `projectRoot` from the calling session for every invocation and keep imported snapshots and generated evaluation files inside that request-local root. Every Harbor Tool call also activates that Session root for the Web Workbench; the Plugin's configured `projectRoot` is only the startup/manual fallback. Do not block initialization merely because the fallback differs from the current session working directory.
 
 Ask only for missing parts of the four-concept intake, using the user's language and short examples:
 
 1. **评测集：测什么？** Accept one Query, a file path, a directory containing multiple instructions, or an existing Dataset path.
-2. **生成器：谁来回答？** Accept a curl request or a local Agent file/directory. Offer an Agent entry discovered in the workspace instead of asking again.
+2. **生成器：谁来回答？** Accept a curl request or a local Agent file/directory. For a DSH/Cordis Agent, offer “使用当前 Harbor Agent 模型” alongside a discovered entry; resolve it with `harbor_model_binding` only after the user agrees.
 3. **评测器（评测标准）：怎样算好？** Accept an evaluator curl request, a local evaluator path, or “请你生成”. If no evaluator exists, ask only for natural-language criteria and draft a versioned evaluator plus Rubric for confirmation.
 4. **优化器：谁根据结果改进？** Default to the current Agent. If Codex CLI or Claude Code is available, present it as an optional alternative; also accept a local command or Agent path.
 
@@ -65,8 +115,12 @@ Before creating files, show one confirmation card:
 - 暂不启用：<holdout / formal promotion Gate / deployment, when unresolved>
 ```
 
+When model binding is selected, render the Generator row as `<local Agent> · <provider>/<model>（已固定）`. Never show credential locations or values.
+
 Offer three next actions in natural language: **开始初始化**, **修改以上内容**, or **查看高级配置**. Call `harbor_evolution_init` only after the user accepts the card. Generate internal ids and initial versions from the workspace/project identity, use `reward`/`maximize` as a visible draft when the criteria imply quality scoring, and do not use the generated Policy for a `promotion-eligible` Job until real business thresholds are accepted.
 
+For a single-Query wiring check, call `harbor_quick_diagnostic_init` after confirmation. State before and after the call that its score proves only Candidate → Harbor → verifier connectivity: the supplied Rubric is saved as a draft but is not executed. Never use its Job as a Baseline or pass it to Gate.
+
 Ask advanced questions just in time:
 
 - Ask for holdout boundaries and side-effect constraints before they can affect a real run.
@@ -78,23 +132,23 @@ Never invent GT labels, business thresholds, credentials, production side-effect
 
 ## Enforce the strict architecture
 
-Require these before every Job:
+Require these before every Candidate execution Job:
 
 - `candidate-manifest.json` verified against the Candidate files.
-- `dataset-manifest.json` with unique task ids, non-empty instructions, safe paths, and a matching source digest.
+- `dataset-manifest.json` with unique task ids, non-empty instructions, safe paths, a matching source digest, and the same Task population that Harbor resolves at runtime. A local Dataset contains immediate Task child directories; each Task uses `schema_version = "1.4"`, `[task].name = "org/name"`, `instruction.md`, `environment/`, and `tests/test.sh`.
 - `.harbor/evaluation-stack.yml` with all eight roles, Judge identity, and Evaluation Contract.
 - Evaluation Context v2 preview.
 
 Require `input_integrity`, `agent_completed`, `integration_valid`, `renderer_valid`, `judge_completed`, and `artifact_schema_valid` in the Trial validity contract. Specify which failures are hard requirements. Never infer that a numeric raw verifier reward is a valid Candidate quality score.
 
-Before a formal Job, call in order:
+Before a formal Candidate execution Job, call in order:
 
 1. `harbor_candidate_snapshot`
 2. `harbor_dataset_validate`
 3. `harbor_evolution_doctor`
 4. `harbor_context_preview`
 
-Do not launch a `promotion-eligible` Job when Doctor reports an error, no comparable baseline exists, or `fresh_baseline_required` is true. A diagnostic Job may investigate architecture warnings, but still requires a valid Candidate, Dataset Manifest, Evaluation Stack, and Context v2.
+Do not launch a `promotion-eligible` Job when Doctor reports an error, no comparable baseline exists, or `fresh_baseline_required` is true. A Candidate-execution diagnostic Job may investigate architecture warnings, but still requires a valid Candidate, Dataset Manifest, Evaluation Stack, and Context v2. The observe-existing Session branch instead uses its frozen Historical Generation Batch, Historical Evaluation Context, and non-promotion Stack.
 
 Keep Runner orchestration-only. Treat these as architecture errors:
 
@@ -105,6 +159,8 @@ Keep Runner orchestration-only. Treat these as architecture errors:
 
 Read `references/initialization.md` when required files are missing. Translate the accepted four-concept card into strict internal identities and call `harbor_evolution_init`; do not send the user back a second architecture questionnaire. It preserves existing files and creates explicit placeholders that still require business implementation.
 
+If `.harbor/evaluation-stack.yml` exists with another `stack_id`, do not report initialization success. Explain `STACK_ALREADY_EXISTS_DIFFERENT_ID` and choose an accepted `workspaceSubdir` so independent Harbor projects can coexist. Never overwrite or silently preserve a different Stack identity.
+
 After initialization:
 
 - Replace placeholders with real role implementations.
@@ -122,7 +178,7 @@ A fresh baseline is required when any of these change:
 - Integration, Renderer, Evaluator, or Rubric identity.
 - Judge provider, model, version, or parameters.
 - Runner marked `semantic: true`.
-- Harbor or integration runtime identity.
+- Harbor or Adapter integration identity. DSH and Candidate ACP themselves follow `latest`; do not reject a Job for an older pinned rc. If latest-runtime drift plausibly changes behavior, recommend a fresh baseline on the current latest runtime instead of restoring and maintaining the old runtime.
 
 Diagnoser, Optimizer, Reporter, and non-semantic Runner changes remain comparable but change the full audit digest. A Candidate digest must differ from the baseline Candidate digest. Promotion Policy is reapplied as a separately versioned decision contract; changing it does not rewrite Evaluation Context.
 
@@ -159,10 +215,31 @@ Use the formal terminal states precisely:
 - `candidate-quality-failed`: valid execution reached evaluation, but a Candidate-owned hard requirement failed.
 - `infrastructure-error`: dependency, sandbox, permission, transport, timeout, or runtime failure; no Candidate quality score.
 - `evaluation-error`: Renderer/Judge/Verifier did not complete; no Candidate quality score.
+- `completed-unscored`: a Historical Generation Trial completed but the Evaluator abstained for insufficient evidence; preserve it in coverage and do not count it as a quality failure or score `0`.
 - `cancelled`: preserve the attempt and do not score it.
 
 For retry or resume, retain the old attempt and create a new attempt. Never replace an assessment or event history in place.
 
+### Synthesize one Dataset-level recommendation
+
+When the user opens an evaluation report or asks what to improve, do not stop at aggregate metrics and do not merely repeat per-Trial recommendations. The current Agent acting as Optimizer must synthesize one concrete **评测集整体优化建议** from the complete Dataset evidence. This synthesis is non-reward-affecting Optimizer output, not a new Evaluator score and not a recommendation invented on behalf of the Evaluator.
+
+1. Inspect all Trial assessments, including every server-side page. Never infer a Dataset conclusion from only the first page, selected badcases, or the lowest score.
+2. Confirm terminal-state and valid-score coverage first. Keep infrastructure and evaluation errors outside Candidate-quality patterns. If coverage is insufficient, say that a trustworthy business optimization recommendation cannot yet be made and recommend repairing the owning evaluation layer.
+3. Group valid results by Criterion, recurring reason/recommendation, Query or population slice. Report the affected count as `N / valid Trials`, distinguish repeated patterns from isolated cases, and identify representative Trial ids or instructions.
+4. Read the corresponding generated artifacts before assigning ownership. Choose the highest-leverage repeated weakness that is Candidate-owned; do not optimize the Candidate around a Dataset, Evaluator, Rubric, Judge, Renderer, or infrastructure defect.
+5. Produce one prioritized recommendation with this user-facing shape:
+
+```text
+评测集整体结论：<what is already reliable and the dominant weakness>
+关键证据：<Criterion and score distribution; N/M affected; representative Trials>
+优先优化建议：<one specific Candidate behavior or implementation change>
+预期效果：<which metric/pattern should improve and what must not regress>
+验证方式：<same Dataset/Context regression Job; protected metrics and rollback condition>
+```
+
+Base the recommendation on the Evaluator's recorded scores, reasons, recommendations, and the actual Candidate artifacts. Do not invent missing reasons, average incompatible Criteria, or present correlation as a proven root cause. If the evidence supports several changes, rank them but recommend only one controlled next experiment. If the user accepts it, translate it into an `optimization-report/v2`-compatible hypothesis with evidence refs, mutation and forbidden surfaces, guardrails, rollback condition, and a comparable next Job before changing the Candidate.
+
 ### Propose one controlled change
 
 Require every optimization hypothesis to include:
@@ -184,7 +261,7 @@ Call `harbor_context_preview`; establish a fresh baseline if needed. Run the Can
 
 Never bypass `INFRASTRUCTURE_EXCEPTION_PRESENT`, `ARTIFACT_SCHEMA_INVALID`, Dataset/Stack/Rubric/Judge mismatch, or non-regression failures.
 
-A `diagnostic` Job must never invoke Gate. Reading the Workbench, generating a Reporter summary, or producing a non-reward Optimization Report also must not promote, deploy, publish, or replace the Champion. Gate remains a separate, explicit comparison action.
+A `diagnostic` Job must never invoke Gate. A Historical Generation Job always displays Gate as `N/A` and must not be passed to `harbor_candidate_compare`. Reading the Workbench, generating a Reporter summary, or producing a non-reward Optimization Report also must not promote, deploy, publish, or replace the Champion. Gate remains a separate, explicit Candidate comparison action.
 
 ## Govern evaluator changes
 
@@ -202,8 +279,40 @@ Saving a new identity does not automatically launch an evaluation or Gate.
 
 An Evaluator implementation must use `harbor-dsh-evaluator/v1`. It may declare `kind=script` or `kind=llm-as-judge`, but both kinds accept `evaluation-input/v1` and return `evaluation-result/v1`. Every Descriptor-declared Criterion must return its declared score plus a non-empty `reason` string and a non-empty `recommendation` string. Missing explanations or recommendations invalidate the evaluator result; Reporter must not invent them. Use `harbor_evaluator_inspect` before proposing a change. After the user approves, use `harbor_evaluator_update` only for an exact `editable_files` path and provide the current digest plus new Evaluator and Stack versions. The tool creates a new versioned bundle; it does not overwrite the old implementation, run meta-evaluation, establish a baseline, or invoke Gate.
 
+The Task verifier must write `/logs/verifier/evaluation-result.json`; `reward.json` alone is not a valid `harbor-dsh-evaluator/v1` result. Summary and Trial views must use the same validity decision.
+
+## Explain failures with the next action
+
+Use the structured diagnostic tail returned by `harbor_eval_run`; never answer with only an exit code. Redact credentials and map common signatures:
+
+- `AgentSetupTimeoutError` → use an image with Python, curl, Node.js, npm, `stdbuf`, ACP, and DSH dependencies preinstalled.
+- `evaluation-result.json is missing` → fix the Task verifier to emit `evaluation-result/v1` with reasons and recommendations.
+- `Either datasets or tasks must be provided` / `HARBOR_RUNTIME_NO_TASKS` → repair the Dataset's immediate Harbor 1.4 Task structure and re-snapshot it.
+- `docker-credential-*` → repair the configured helper or use a verified local base image.
+
+Rerun `harbor_dataset_validate` and `harbor_evolution_doctor` before retrying. Preserve the failed Job as evidence; do not mutate it in place.
+
 ## Handle evaluator meta-evaluation
 
+Read `references/evaluator-upgrade.md` before handling reviewed reports, expert comments, scoring notes, evaluator calibration, or meta-evaluation. Keep protocol names out of the initial user interaction.
+
+Start from evidence the user already has. Ask only:
+
+> 请提供一些已经被评价过的报告，以及对应的评分、问题或修改建议。你可以直接粘贴文本，也可以提供文件或目录路径。我会整理成评测器元评测集，并只请你确认有歧义的评分。
+
+Accept one report, several pasted report/review pairs, or a directory. Do not initially ask the user for GT JSON, Criterion ids, provenance fields, repeat policy, Evaluator identity, or meta-metric thresholds. Inspect the active Rubric and infer stable internal ids and versions after understanding the material.
+
+For every report/review pair:
+
+1. Preserve the original report and review as source evidence. Never replace them with only the normalized JSON.
+2. Extract Criterion, score, reason, any reviewer-provided recommendation, and the exact review excerpt supporting the extraction.
+3. Mark each extracted decision internally as `explicit`, `inferred`, or `unresolved`.
+4. Map a natural-language judgment to the active score scale only when the Rubric makes the mapping defensible. Treat the mapped value as a draft, not confirmed GT.
+5. Show one compact table with report, Criterion, proposed score, reason, recommendation, and status. Ask only targeted questions for `inferred` or `unresolved` rows.
+6. Create formal GT only after the user confirms the draft. Never silently fill a missing score, reason, source, or independence claim. A missing reviewer recommendation may remain empty and must not be attributed to the reviewer.
+
+Use plain language in the confirmation. Say “标准评分” instead of `ground-truth/v1`, “评测器重复评分” instead of `evaluator-observations/v1`, and “评测器可靠性报告” instead of `meta-evaluation-report/v1`. Protocol names may appear later in an audit or advanced view.
+
 Rotate roles when improving the Evaluator:
 
 - Candidate is the Evaluator/Rubric/Judge version.
@@ -213,9 +322,13 @@ Rotate roles when improving the Evaluator:
 
 GT may be human, programmatic, consensus-based, produced by an independently pinned model, or imported from an external standard. Independence and provenance matter more than the author type. The Candidate evaluator must never see labels before producing its observation.
 
-When GT is missing, clarify its id/version, source kind, owner, provenance, Criteria, case selection, and adjudication process. Then call `harbor_ground_truth_init`; it creates a non-overwriting draft and never invents cases or labels. After cases are populated, collect repeated `evaluator-observations/v1` and call `harbor_evaluator_meta_evaluate`. Report ESF, SCE, RCR, coverage, disagreement slices, latency, and cost as applicable.
+After confirmation, infer a readable GT id/version, source kind, provenance, Criteria, case ids, and initial weights from the accepted material. Show any consequential inference. Call `harbor_ground_truth_init`; it creates a non-overwriting draft and never invents cases or labels. Populate the draft from confirmed rows using ordinary safe file operations, keeping artifact references inside the request-local project root.
+
+Treat one reviewed report as a diagnostic calibration example, not evidence that an Evaluator is generally reliable. With enough cases, propose a tuning/holdout split without burdening the user with the terminology: explain that one group helps improve the Evaluator and an untouched group checks whether the improvement generalizes. Never expose holdout labels to the Candidate evaluator or use its own prior output as GT.
+
+After cases are populated, run the same Evaluator repeatedly on the fixed reports, collect `evaluator-observations/v1`, and call `harbor_evaluator_meta_evaluate`. The user should not have to hand-author either JSON file. Report ESF, SCE, RCR, coverage, disagreement slices, latency, and cost as applicable, then translate them back into direct conclusions: missed problems, false alarms, unstable judgments, and the smallest justified Evaluator/Rubric change.
 
-Manage evaluator Candidates and meta-evaluation Jobs with the same Manifest, Context v2, Doctor, evidence, and Gate rules.
+Manage Evaluator Candidates and the existing independent-GT meta-evaluation artifacts with immutable identities, provenance, comparable observations, and an explicit human adoption decision. A dedicated `evaluator-meta-evaluation` Harbor Job lifecycle is future work; do not claim that `harbor_evaluator_meta_evaluate` created such a Job.
 
 ## Report each cycle
 
@@ -227,7 +340,8 @@ Return:
 - Metric deltas, exception counts, Population groups, and artifact validation.
 - Dataset coverage, terminal-state counts, valid/invalid score counts, and selected attempt policy.
 - Representative Trial evidence and root-cause classes.
+- One Dataset-level overall conclusion and one prioritized, evidence-linked optimization recommendation; explicitly state when score validity or coverage is insufficient for one.
 - Evidence provenance and any capability unavailable on a legacy Job.
 - Controlled change hypothesis and mutation surface.
-- Gate decision with exact reason codes.
+- Gate decision with exact reason codes for Candidate comparison Jobs, or explicit `N/A` for Historical Generation Jobs.
 - External CI/CD action still required.

package/skills/evolve-agent-with-harbor/evals/evals.json

@@ -4,12 +4,14 @@
     {
       "id": 1,
       "prompt": "帮我给当前目录里的业务 Agent 建一个 Harbor 自进化流程，我还没有准备任何评测配置。",
-      "expected_output": "先检查工作区，再用评测集、生成器、评测器（评测标准）和优化器四个业务概念提出少量短问题；不得要求用户枚举 Evaluation Stack 八个角色或版本号。",
+      "expected_output": "先检查工作区；用户未提供显式 Dataset 时，先调用 harbor_session_diagnostic_preview(limit=10) 只读预览当前 cwd 最近完成会话，说明原 DSH Agent 是生成器、1 会话是 1 Trial，展示成本、排除原因、Evaluator/Judge identity 与 coupling 后等待确认，不得直接 run。Judge 选择只允许在 Preview，确认后只传 selectionToken；确认卡还要提示 .harbor/private 与 jobs 会本地保留脱敏业务会话及其 VCS 风险。Batch 同步物化匹配的 Dataset/Stack，MVP 不接受自定义 stackPath。明确无 Candidate 重执行、Historical Job 内元评测 status=not-run、Gate N/A，同时说明已有独立 GT/meta-evaluate 流程仍可单独使用；仅当 Session Preview 不可用时，再用评测集、生成器、评测器（评测标准）和优化器四个业务概念继续普通冷启动。",
       "files": [],
       "assertions": [
-        "The response visibly uses the four concepts Dataset, Generator, Evaluator with criteria, and Optimizer.",
-        "The response does not ask for eight Evaluation Stack roles, Judge version parameters, Contract ids, or Policy ids.",
-        "The response proposes a managed workspace and states that inferred values will be confirmed before files are created."
+        "The response calls the read-only Session Preview with a limit of 10 before proposing any Historical Job run.",
+        "The response maps the original DSH Agent to Generator and each selected completed Session to one immutable Trial.",
+        "The response pauses for explicit confirmation and warns that redacted business Session evidence remains locally under .harbor/private and jobs with VCS risk.",
+        "The response says Run receives the selection token, synchronously materializes the matching Dataset and immutable Stack, and does not accept a custom stackPath.",
+        "The response keeps Evaluator Meta-Evaluation not-run inside the Historical Job while acknowledging the separate independent-GT meta-evaluate flow, and reports Gate as N/A."
       ]
     },
     {
@@ -26,23 +28,69 @@
     {
       "id": 3,
       "prompt": "评测集在 ./evals，生成器调用这个 curl：curl -X POST http://127.0.0.1:9000/run -H 'Authorization: Bearer secret' -d '{\"input\":\"hi\"}'。评测器在 ./judge.py，优化交给 codex。帮我初始化。",
-      "expected_output": "自动解析四个概念并展示确认卡；不回显或持久化 secret；底层身份与适配器由 Skill 推断，不再追问专业字段；未确认前不写文件或运行 Job。",
+      "expected_output": "保留显式 ./evals Dataset，不用会话历史替换或追加它，也不调用 Session Preview。自动解析四个概念并展示确认卡；不回显或持久化 secret；底层身份与适配器由 Skill 推断，不再追问专业字段；未确认前不写文件或运行 Job。",
       "files": [],
       "assertions": [
         "The Authorization credential is redacted and explicitly excluded from persisted configuration.",
         "The response maps the supplied paths and curl into the four-concept confirmation card.",
-        "The response does not run initialization or evaluation before confirmation."
+        "The response does not run initialization or evaluation before confirmation.",
+        "The response keeps the explicit Dataset authoritative and does not invoke the Session-history Preview."
       ]
     },
     {
       "id": 4,
-      "prompt": "当前 Session 工作目录是 /Users/mac/Documents/Harness，但 Harbor Plugin 配置里显示的 projectRoot 是 /Users/mac/Library/Application Support/XiaoHui Harness/workspace。请在当前项目下初始化 ./harbor-evolution。",
-      "expected_output": "把当前 Session 工作目录作为 Agent Tool 的项目根目录，继续准备 /Users/mac/Documents/Harness/harbor-evolution 的初始化确认卡；说明配置值只供 Web Workbench 或非 Agent 场景回退使用，不要求修改配置，也不因路径不同而拒绝初始化。",
+      "prompt": "当前 Session 工作目录是 /workspace/business-agent，但 Harbor Plugin 配置里显示的 projectRoot 是 /workspace/default。请在当前项目下初始化 ./harbor-evolution。",
+      "expected_output": "把当前 Session 工作目录作为 Agent Tool 的项目根目录，继续准备 /workspace/business-agent/harbor-evolution 的初始化确认卡；说明配置值只供 Web Workbench 或非 Agent 场景回退使用，不要求修改配置，也不因路径不同而拒绝初始化。",
       "files": [],
       "assertions": [
         "The response treats the calling session working directory as the Agent Tool project root.",
         "The response does not block initialization or require a projectRoot configuration change because the fallback differs.",
-        "The proposed managed workspace stays under /Users/mac/Documents/Harness."
+        "The proposed managed workspace stays under /workspace/business-agent."
+      ]
+    },
+    {
+      "id": 5,
+      "prompt": "生成器用本机 DSH Agent，我希望它固定使用当前 Harbor Agent 的 GPT Auth 模型，不要把登录信息放进 Candidate。",
+      "expected_output": "把同模型解释为固定的 Candidate model binding；确认后调用 harbor_model_binding 取得非敏感 provider/model/reasoning，写入 model-binding.json，并说明运行仍通过短期 dsh-host-broker Capability，Host OAuth 文件不会进入容器或 Harbor 产物。",
+      "files": [],
+      "assertions": [
+        "The response offers the current Harbor Agent model as an explicit Generator choice and snapshots it instead of live-following later model changes.",
+        "The response keeps Host authentication outside the Candidate and uses the short-lived Host Model Broker capability.",
+        "The response requires a new Candidate identity instead of overriding a pinned model binding in place."
+      ]
+    },
+    {
+      "id": 6,
+      "prompt": "我想提升 DeepResearch 的评测器。这里有一份 report.md，专家的评价是：基本回答了问题，但没有解释颜色和光的关系，回应问题 0.5；表达平淡、没有例子，有趣性 0.5；引用了不存在的资料，引用规范性 0。建议补充形成机制并核对所有引用。帮我做元评测。",
+      "expected_output": "读取报告和当前 Rubric，把专家自然语言评价解析为评分、原因与建议的简洁确认表，保留原始评价并标明哪些值是明确给出、推断或待确认；不要求用户编写 GT/Observation JSON，且说明单个样本只能用于诊断校准。",
+      "files": [],
+      "assertions": [
+        "The response accepts the report and natural-language review as the user-facing meta-evaluation input without requesting JSON schemas or protocol fields.",
+        "The response presents or proposes a compact confirmation of scores, reasons, recommendations, and extraction status while preserving the raw review as provenance.",
+        "The response classifies a single reviewed report as diagnostic evidence rather than sufficient evaluator promotion evidence."
+      ]
+    },
+    {
+      "id": 7,
+      "prompt": "./reviewed-reports 里有十份报告和对应的专家评语，但有些评语只写了‘引用有问题’或者‘基本可用’，没有结构化分数。请用这些材料校准评测器。",
+      "expected_output": "先检查目录配对和当前 Rubric，根据评分标准形成带 explicit/inferred/unresolved 状态的草稿；只对有歧义的映射提出合并后的短问题，用户确认后再生成严格元评测数据，并建议保留未参与调优的一组报告验证泛化。",
+      "files": [],
+      "assertions": [
+        "The response inspects and pairs existing files before asking the user to transcribe their contents.",
+        "The response does not silently convert vague comments into confirmed scores and asks only targeted questions for inferred or unresolved mappings.",
+        "The response proposes a tuning versus untouched validation split in plain language and keeps labels hidden from the Candidate evaluator."
+      ]
+    },
+    {
+      "id": 8,
+      "prompt": "这个 DeepResearch Job 已经跑完 10 个任务。回应问题大多是 1，但引用规范性有 6 个 0.5、2 个 0，低分主要出现在需要多来源交叉验证的问题。请基于整个评测集告诉我下一步最值得做什么，不要逐条复述。",
+      "expected_output": "读取全部 Trial 与产物后，归纳为评测集级整体结论：用受影响数量、评分维度和代表性样本支撑一条优先的 Candidate 优化建议，说明预期效果、保护指标、回归验证和回滚条件；不把建议写成新的 Evaluator 分数或直接执行改动。",
+      "files": [],
+      "assertions": [
+        "The response synthesizes one Dataset-level conclusion from all Trial pages instead of repeating Trial recommendations or relying on selected badcases.",
+        "The response quantifies the recurring citation weakness and cites representative Trial evidence before assigning it to the Candidate.",
+        "The response proposes one specific controlled optimization with expected metric effect, protected metrics, comparable regression validation, and a rollback condition.",
+        "The response treats the synthesis as non-reward Optimizer guidance and does not mutate the Candidate or claim promotion without user approval."
       ]
     }
   ]