dsh-harbor-evolution 0.4.0 → 0.6.0

package/README.md

@@ -2,7 +2,7 @@
 
 Installable DeepSeek Harness Plugin + Skill for running stable Harbor evaluation and controlled Agent evolution loops, with a native DSH Web dashboard.
 
-The package gives DSH four deterministic Harbor tools, dedicated Tool cards, a Harbor conversation tab, an installation Doctor, and the model- and user-invocable `evolve-agent-with-harbor` Skill. The Skill clarifies the evaluation contract, initializes missing structures, establishes a baseline, diagnoses evidence, limits each iteration to one controlled Candidate change, runs regression evaluation, and produces a Promotion Gate recommendation.
+The package gives DSH twelve strict Harbor tools, dedicated Tool cards, a nine-stage Evaluation Workbench, an installation Doctor, and the model- and user-invocable `evolve-agent-with-harbor` Skill. The Skill clarifies and initializes the Evaluation Stack, validates Dataset identity, checks Trial Lifecycle and Score Validity, governs independent Ground Truth meta-evaluation, diagnoses evidence provenance, limits each iteration to one controlled Candidate change, and invokes the Promotion Gate only as an explicit action.
 
 ## Install
 
@@ -14,8 +14,8 @@ npx --yes dsh-harbor-evolution@latest setup --project-root "$PWD"
 
 The setup command installs both required runtimes:
 
-- `harbor-dsh-evolution==0.4.0` in a managed Python environment.
-- `dsh-harbor-evolution@0.4.0` in the selected DSH profile.
+- `harbor-dsh-evolution==0.6.0` in a managed Python environment.
+- `dsh-harbor-evolution@0.6.0` in the selected DSH profile.
 
 It then stores the absolute Harbor executable paths and `projectRoot` in the profile's `harbor-evolution` block and verifies the integration. Existing unrelated profile entries are preserved, and rerunning setup updates the same block.
 
@@ -35,19 +35,27 @@ Inspect this workspace and help me clarify and initialize a stable Harbor self-e
 The Plugin registers:
 
 - `harbor_candidate_snapshot`
+- `harbor_evolution_init`
+- `harbor_evolution_doctor`
+- `harbor_dataset_validate`
+- `harbor_context_preview`
 - `harbor_eval_run`
 - `harbor_eval_result`
 - `harbor_candidate_compare`
 
 In the `web` profile, the same package also registers:
 
-- a `Harbor` conversation tab for recent Jobs, metrics, Candidate identity, and evaluation-context digests;
-- four compact result cards for the Harbor Tool calls;
-- a `Harbor Evolution` Settings section that checks the configured project, Jobs directory, and CLI paths.
+- a localized nine-stage Workbench that directly exposes fixed experiment identities, Agent-visible Dataset queries/instructions, safe business-artifact previews, Ground Truth meta-evaluation, paginated per-Trial evidence and recommendations, Population validity/coverage, controlled optimization hypotheses, and Baseline/Gate deltas; raw JSON remains in the audit drawer;
+- descriptor-authorized Evaluator/Rubric source editing for `script` and `llm-as-judge` implementations, with optimistic concurrency and mandatory new identities;
+- a `harbor-dsh-evaluator/v1` interface shared by deterministic scripts and LLM-as-Judge implementations;
+- compact result cards for all Harbor Tool calls;
+- a `Harbor Evolution` Settings section that checks the configured project, Evaluation Stack, Jobs directory, and CLI paths.
 
 The Web UI is intentionally read-only. Starting an evaluation or deciding promotion remains an explicit Agent + Skill workflow, so a page refresh can never launch an expensive Job.
 
-The shortest direct evaluation call needs `candidatePath` and `datasetPath`. Prefer the Skill for a new project because it will not run or compare Jobs until the material evaluation contract is resolved.
+A direct evaluation requires `candidatePath`, `datasetPath`, `stackPath`, and explicit `mode`; `promotion-eligible` additionally requires `policyPath`. Prefer the Skill because it will not run or compare Jobs until the material identities and evaluation contract are resolved.
+
+`harbor_eval_result` defaults to the stable Summary. Use `view=job`, `view=dataset`, `view=progress`, `view=trial` plus a returned `trialId`, or `view=governance` to inspect sanitized instructions, generated output, evidence, and evaluator source without coupling the Agent to artifact file paths.
 
 ## What setup writes
 

package/index.js

@@ -72,26 +72,133 @@ export function apply(ctx, config) {
     },
   }, args => service.snapshot(args)))
 
+  ctx.tools.register(jsonTool({
+    name: 'harbor_evolution_init',
+    description: 'Initialize a strict, non-overwriting Evaluation Stack project after the Skill has clarified identities, primary metric, judge, and promotion threshold.',
+    parameters: {
+      datasetPath: { type: 'string', required: true },
+      stackId: { type: 'string', required: true },
+      stackVersion: { type: 'string', required: true },
+      datasetId: { type: 'string', required: true },
+      datasetVersion: { type: 'string', required: true },
+      contractId: { type: 'string', required: true },
+      contractVersion: { type: 'string', required: true },
+      primaryMetric: { type: 'string', required: true },
+      primaryDirection: { type: 'string', required: true },
+      judgeProvider: { type: 'string', required: true },
+      judgeModel: { type: 'string', required: true },
+      judgeVersion: { type: 'string', required: true },
+      policyId: { type: 'string', required: true },
+      policyVersion: { type: 'string', required: true },
+      minImprovement: { type: 'number', required: true },
+    },
+  }, args => service.initialize(args)))
+
+  ctx.tools.register(jsonTool({
+    name: 'harbor_evolution_doctor',
+    description: 'Validate the Evaluation Stack architecture, Dataset manifest, Candidate, and optional Promotion Policy before an expensive Harbor Job.',
+    parameters: {
+      candidatePath: { type: 'string', required: true },
+      datasetPath: { type: 'string', required: true },
+      stackPath: { type: 'string', required: true },
+      policyPath: { type: 'string' },
+      mode: { type: 'string', required: true },
+    },
+  }, args => service.doctor(args)))
+
+  ctx.tools.register(jsonTool({
+    name: 'harbor_dataset_validate',
+    description: 'Validate dataset-manifest.json, task uniqueness, instructions, paths, sensitive metadata, and the immutable source digest.',
+    parameters: {
+      datasetPath: { type: 'string', required: true },
+    },
+  }, args => service.validateDataset(args)))
+
+  ctx.tools.register(jsonTool({
+    name: 'harbor_context_preview',
+    description: 'Preview Evaluation Context v2 and find comparable baselines before launching a Job.',
+    parameters: {
+      candidatePath: { type: 'string', required: true },
+      candidateId: { type: 'string' },
+      version: { type: 'string' },
+      datasetPath: { type: 'string', required: true },
+      stackPath: { type: 'string', required: true },
+      mode: { type: 'string', required: true },
+    },
+  }, args => service.previewContext(args)))
+
   ctx.tools.register(jsonTool({
     name: 'harbor_eval_run',
-    description: 'Snapshot and run one DeepSeek Harness Candidate against a Harbor dataset, then return the completed evaluation summary.',
+    description: 'Run a strict diagnostic or promotion-eligible Harbor Job bound to Candidate, Dataset Manifest, Evaluation Stack, and Context v2 identities.',
     parameters: {
       candidatePath: { type: 'string', required: true },
       candidateId: { type: 'string' },
       version: { type: 'string' },
       datasetPath: { type: 'string', required: true },
+      stackPath: { type: 'string', required: true },
+      mode: { type: 'string', required: true },
+      policyPath: { type: 'string' },
       jobName: { type: 'string' },
     },
   }, args => service.run(args)))
 
   ctx.tools.register(jsonTool({
     name: 'harbor_eval_result',
-    description: 'Read the stable evaluation summary produced for a Harbor Job.',
+    description: 'Read a stable Job summary or a sanitized Workbench, Dataset instruction, Trial output/evidence, progress, or Evaluator governance view. Invalid scores remain distinct from raw verifier rewards.',
     parameters: {
       jobPath: { type: 'string', required: true },
+      view: { type: 'string', description: 'summary (default), job, dataset, progress, trial, or governance' },
+      trialId: { type: 'string', description: 'Required only for view=trial; use an id returned by the Job/Progress view' },
+      compareJob: { type: 'string', description: 'Optional previous Job for view=governance impact analysis' },
+      since: { type: 'string', description: 'Optional ISO timestamp for incremental progress changes' },
     },
   }, args => service.result(args)))
 
+  ctx.tools.register(jsonTool({
+    name: 'harbor_evaluator_inspect',
+    description: 'Inspect the active harbor-dsh-evaluator/v1 descriptor, implementation kind, ternary Criteria, and safely editable source files.',
+    parameters: {
+      stackPath: { type: 'string', description: 'Defaults to .harbor/evaluation-stack.yml' },
+    },
+  }, args => service.evaluatorInspect(args)))
+
+  ctx.tools.register(jsonTool({
+    name: 'harbor_evaluator_update',
+    description: 'Update one descriptor-authorized Evaluator source file with optimistic concurrency. Requires new Evaluator and Stack identities and never runs evaluation or Gate automatically.',
+    parameters: {
+      stackPath: { type: 'string', description: 'Defaults to .harbor/evaluation-stack.yml' },
+      filePath: { type: 'string', required: true },
+      content: { type: 'string', required: true },
+      expectedDigest: { type: 'string', required: true },
+      newEvaluatorVersion: { type: 'string', required: true },
+      newStackVersion: { type: 'string', required: true },
+    },
+  }, args => service.evaluator(args)))
+
+  ctx.tools.register(jsonTool({
+    name: 'harbor_ground_truth_init',
+    description: 'Create a non-overwriting Ground Truth draft for evaluator meta-evaluation. GT may be human, programmatic, consensus, model, or external, but must have explicit provenance and remain independent of the Candidate evaluator.',
+    parameters: {
+      outputPath: { type: 'string', description: 'Defaults to .harbor/ground-truth.json' },
+      groundTruthId: { type: 'string', required: true },
+      version: { type: 'string', required: true },
+      sourceKind: { type: 'string', required: true, description: 'human, programmatic, consensus, model, or external' },
+      sourceDescription: { type: 'string', required: true },
+      provenance: { type: 'string', required: true },
+      criteria: { type: 'string', required: true, description: 'Comma-separated criterion ids' },
+    },
+  }, args => service.groundTruthInitialize(args)))
+
+  ctx.tools.register(jsonTool({
+    name: 'harbor_evaluator_meta_evaluate',
+    description: 'Compare repeated evaluator-observations/v1 with independent ground-truth/v1 and write an ESF, SCE, and RCR meta-evaluation report.',
+    parameters: {
+      groundTruthPath: { type: 'string', description: 'Defaults to .harbor/ground-truth.json' },
+      observationsPath: { type: 'string', required: true },
+      outputPath: { type: 'string', description: 'Defaults to .harbor/meta-evaluation-report.json' },
+    },
+  }, args => service.evaluatorMetaEvaluate(args)))
+
   ctx.tools.register(jsonTool({
     name: 'harbor_candidate_compare',
     description: 'Apply the deterministic Promotion Gate to a baseline Job and a Candidate Job.',

package/lib/candidate.js

@@ -6,6 +6,16 @@ export const MANIFEST_NAME = 'candidate-manifest.json'
 const DIGEST_PREFIX = Buffer.from('harbor-dsh-candidate-v1\0')
 const EXCLUDED_DIRS = new Set(['.git', 'node_modules', '__pycache__'])
 const EXCLUDED_FILES = new Set([MANIFEST_NAME, '.DS_Store'])
+const LOCKFILES = ['package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'bun.lock', 'bun.lockb']
+const CREDENTIAL_FILES = new Set([
+  'credentials.json',
+  'service-account.json',
+  'secrets.json',
+  'secrets.yaml',
+  'secrets.yml',
+  'id_rsa',
+  'id_ed25519',
+])
 
 async function walk(root, current = root) {
   const entries = await readdir(current, { withFileTypes: true })
@@ -46,8 +56,7 @@ export async function computeCandidate(candidateDir) {
   return { digest: `sha256:${digest.digest('hex')}`, files }
 }
 
-export async function snapshotCandidate(candidateDir, options = {}) {
-  const root = path.resolve(candidateDir)
+async function validateCandidateContract(root) {
   for (const required of ['cordis.yml', 'package.json']) {
     try {
       if (!(await stat(path.join(root, required))).isFile()) throw new Error()
@@ -55,6 +64,24 @@ export async function snapshotCandidate(candidateDir, options = {}) {
       throw new Error(`Candidate is missing required file: ${required}`)
     }
   }
+  const files = await walk(root)
+  if (!files.some(item => LOCKFILES.includes(item.relative))) {
+    throw new Error(`Candidate requires a JavaScript lockfile: ${LOCKFILES.join(', ')}`)
+  }
+  const credentialPaths = files
+    .map(item => item.relative)
+    .filter((relative) => {
+      const name = path.basename(relative).toLowerCase()
+      return name.startsWith('.env') || CREDENTIAL_FILES.has(name)
+    })
+  if (credentialPaths.length > 0) {
+    throw new Error(`Candidate contains credential-bearing files: ${credentialPaths.join(', ')}; inject credentials at runtime instead`)
+  }
+}
+
+export async function snapshotCandidate(candidateDir, options = {}) {
+  const root = path.resolve(candidateDir)
+  await validateCandidateContract(root)
   let packageJson
   try {
     packageJson = JSON.parse(await readFile(path.join(root, 'package.json'), 'utf8'))