dsh-harbor-evolution 0.4.0 → 0.6.0

package/lib/evolution.js

@@ -23,47 +23,194 @@ export async function snapshot(config, args) {
 }
 
 function slug(value) {
-  return String(value)
-    .toLowerCase()
-    .replace(/[^a-z0-9._-]+/g, '-')
-    .replace(/^[._-]+|[._-]+$/g, '') || 'candidate'
+  return String(value).toLowerCase().replace(/[^a-z0-9._-]+/g, '-').replace(/^[._-]+|[._-]+$/g, '') || 'candidate'
 }
 
 export function makeJobName(manifest, now = new Date()) {
   const timestamp = now.toISOString().replace(/\.\d{3}Z$/, 'Z').replace(/[-:]/g, '')
   const suffix = `${timestamp}-${manifest.digest.slice(7, 15)}`
-  const available = 100 - suffix.length - 1
-  const identity = `${slug(manifest.candidate_id)}-${slug(manifest.version)}`.slice(0, available)
+  const identity = `${slug(manifest.candidate_id)}-${slug(manifest.version)}`.slice(0, 99 - suffix.length)
   return `${identity}-${suffix}`
 }
 
+async function cliJson(config, args, { allowedExitCodes = [0], input } = {}) {
+  let result
+  try {
+    result = await runProcess(config.harborDshBin, args, {
+      cwd: config.projectRoot,
+      timeoutMs: config.timeoutMs,
+      allowedExitCodes,
+      input,
+      env: { ...process.env, ...(config.pythonPath ? { PYTHONPATH: config.pythonPath } : {}) },
+    })
+  } catch (error) {
+    const detail = error?.result?.stderr?.trim().split('\n').at(-1)?.replace(/^[A-Za-z]+Error:\s*/, '')
+    throw new Error(detail || error.message)
+  }
+  try {
+    return JSON.parse(result.stdout)
+  } catch {
+    throw new Error(`harbor-dsh returned invalid JSON for ${args.slice(0, 2).join(' ')}`)
+  }
+}
+
+export async function inspectEvaluator(config, args = {}) {
+  const stack = resolveWithin(config.projectRoot, args.stackPath ?? '.harbor/evaluation-stack.yml', 'stackPath')
+  return cliJson(config, [
+    'evaluator', 'inspect',
+    '--project-root', config.projectRoot,
+    '--stack', stack,
+  ])
+}
+
+export async function updateEvaluator(config, args) {
+  const stack = resolveWithin(config.projectRoot, args.stackPath ?? '.harbor/evaluation-stack.yml', 'stackPath')
+  if (typeof args.content !== 'string') throw new Error('content is required')
+  return cliJson(config, [
+    'evaluator', 'update',
+    '--project-root', config.projectRoot,
+    '--stack', stack,
+    '--file', String(args.filePath ?? ''),
+    '--expected-digest', String(args.expectedDigest ?? ''),
+    '--new-evaluator-version', String(args.newEvaluatorVersion ?? ''),
+    '--new-stack-version', String(args.newStackVersion ?? ''),
+    '--content-stdin',
+  ], { input: args.content })
+}
+
+export async function initializeGroundTruth(config, args) {
+  const output = resolveWithin(config.projectRoot, args.outputPath ?? '.harbor/ground-truth.json', 'outputPath')
+  return cliJson(config, [
+    'ground-truth', 'init',
+    '--project-root', config.projectRoot,
+    '--output', output,
+    '--id', String(args.groundTruthId ?? ''),
+    '--version', String(args.version ?? ''),
+    '--source-kind', String(args.sourceKind ?? ''),
+    '--source-description', String(args.sourceDescription ?? ''),
+    '--provenance', String(args.provenance ?? ''),
+    '--criteria', String(args.criteria ?? ''),
+  ])
+}
+
+export async function runMetaEvaluation(config, args) {
+  const groundTruth = resolveWithin(config.projectRoot, args.groundTruthPath ?? '.harbor/ground-truth.json', 'groundTruthPath')
+  const observations = resolveWithin(config.projectRoot, args.observationsPath, 'observationsPath')
+  const output = resolveWithin(config.projectRoot, args.outputPath ?? '.harbor/meta-evaluation-report.json', 'outputPath')
+  return cliJson(config, [
+    'meta-evaluate',
+    '--project-root', config.projectRoot,
+    '--ground-truth', groundTruth,
+    '--observations', observations,
+    '--output', output,
+  ])
+}
+
+function strictInputs(config, args) {
+  const projectRoot = path.resolve(config.projectRoot)
+  const candidate = resolveWithin(projectRoot, args.candidatePath, 'candidatePath')
+  const dataset = resolveWithin(projectRoot, args.datasetPath, 'datasetPath')
+  const stack = resolveWithin(projectRoot, args.stackPath, 'stackPath')
+  const jobs = resolveWithin(projectRoot, config.jobsDir, 'jobsDir')
+  const mode = args.mode
+  if (!['diagnostic', 'promotion-eligible'].includes(mode)) throw new Error('mode must be diagnostic or promotion-eligible')
+  const policy = args.policyPath ? resolveWithin(projectRoot, args.policyPath, 'policyPath') : undefined
+  if (mode === 'promotion-eligible' && !policy) throw new Error('promotion-eligible mode requires policyPath')
+  return { projectRoot, candidate, dataset, stack, jobs, mode, policy }
+}
+
+export async function validateDataset(config, args) {
+  const dataset = resolveWithin(config.projectRoot, args.datasetPath, 'datasetPath')
+  return cliJson(config, ['dataset', 'validate', dataset, '--project-root', config.projectRoot], { allowedExitCodes: [0, 2] })
+}
+
+export async function initializeProject(config, args) {
+  const dataset = resolveWithin(config.projectRoot, args.datasetPath, 'datasetPath')
+  return cliJson(config, [
+    'init', '--project-root', config.projectRoot, '--dataset', dataset,
+    '--stack-id', args.stackId, '--stack-version', args.stackVersion,
+    '--dataset-id', args.datasetId, '--dataset-version', args.datasetVersion,
+    '--contract-id', args.contractId, '--contract-version', args.contractVersion,
+    '--primary-metric', args.primaryMetric, '--primary-direction', args.primaryDirection,
+    '--judge-provider', args.judgeProvider, '--judge-model', args.judgeModel, '--judge-version', args.judgeVersion,
+    '--policy-id', args.policyId, '--policy-version', args.policyVersion,
+    '--min-improvement', String(args.minImprovement),
+  ])
+}
+
+export async function runDoctor(config, args) {
+  const inputs = strictInputs(config, { ...args, mode: args.mode ?? 'diagnostic' })
+  const command = ['doctor', '--architecture', '--project-root', inputs.projectRoot, '--stack', inputs.stack, '--dataset', inputs.dataset]
+  if (args.candidatePath) command.push('--candidate', inputs.candidate)
+  if (inputs.policy) command.push('--policy', inputs.policy)
+  return cliJson(config, command, { allowedExitCodes: [0, 2] })
+}
+
+export async function previewContext(config, args) {
+  const manifest = await snapshot(config, args)
+  const inputs = strictInputs(config, args)
+  const preview = await cliJson(config, [
+    'context', 'preview',
+    '--project-root', inputs.projectRoot,
+    '--candidate', inputs.candidate,
+    '--dataset', inputs.dataset,
+    '--stack', inputs.stack,
+    '--jobs-dir', inputs.jobs,
+    '--mode', inputs.mode,
+  ])
+  return { manifest, ...preview }
+}
+
 export async function runEvaluation(config, args) {
   const manifest = await snapshot(config, args)
-  const candidateDir = resolveWithin(config.projectRoot, args.candidatePath, 'candidatePath')
-  const dataset = resolveWithin(config.projectRoot, args.datasetPath, 'datasetPath')
-  const jobsDir = resolveWithin(config.projectRoot, config.jobsDir, 'jobsDir')
+  const inputs = strictInputs(config, args)
+  const datasetValidation = await validateDataset(config, args)
+  if (!datasetValidation.valid) {
+    throw new Error(`Dataset validation failed: ${datasetValidation.findings.map(item => item.code).join(', ')}`)
+  }
+  const doctor = await runDoctor(config, args)
+  if (inputs.mode === 'promotion-eligible' && !doctor.promotion_ready) {
+    throw new Error(`Architecture Doctor blocked promotion-eligible Job: ${doctor.findings.filter(item => item.level === 'error').map(item => item.code).join(', ')}`)
+  }
+  const preview = await cliJson(config, [
+    'context', 'preview', '--project-root', inputs.projectRoot,
+    '--candidate', inputs.candidate, '--dataset', inputs.dataset,
+    '--stack', inputs.stack, '--jobs-dir', inputs.jobs, '--mode', inputs.mode,
+  ])
   const jobName = args.jobName ?? makeJobName(manifest)
   if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$/.test(jobName)) throw new Error('jobName contains unsupported characters')
 
   const harborArgs = [
-    'run', '-p', dataset,
+    'run', '-p', inputs.dataset,
     '-a', config.agentImportPath,
-    '--ak', `candidate_path=${candidateDir}`,
+    '--ak', `candidate_path=${inputs.candidate}`,
     '--ak', `candidate_version=${manifest.version}`,
     '--ak', `candidate_digest=${manifest.digest}`,
     '--job-name', jobName,
-    '--jobs-dir', jobsDir,
+    '--jobs-dir', inputs.jobs,
     '--plugin', config.pluginImportPath,
-    '--plugin-kwarg', `candidate_manifest=${path.join(candidateDir, MANIFEST_NAME)}`,
+    '--plugin-kwarg', `candidate_manifest=${path.join(inputs.candidate, MANIFEST_NAME)}`,
+    '--plugin-kwarg', `dataset_path=${inputs.dataset}`,
+    '--plugin-kwarg', `stack_path=${inputs.stack}`,
+    '--plugin-kwarg', `project_root=${inputs.projectRoot}`,
+    '--plugin-kwarg', `mode=${inputs.mode}`,
   ]
-  const result = await runProcess(config.harborBin, harborArgs, {
+  if (inputs.policy) harborArgs.push('--plugin-kwarg', `policy_path=${inputs.policy}`)
+  const processResult = await runProcess(config.harborBin, harborArgs, {
     cwd: config.projectRoot,
     timeoutMs: config.timeoutMs,
     env: { ...process.env, ...(config.pythonPath ? { PYTHONPATH: config.pythonPath } : {}) },
   })
-  const jobDir = path.join(jobsDir, jobName)
+  const jobDir = path.join(inputs.jobs, jobName)
   const summary = JSON.parse(await readFile(path.join(jobDir, 'evaluation-summary.json'), 'utf8'))
-  return { manifest, jobDir, summary, process: { code: result.code } }
+  return {
+    manifest,
+    job: path.relative(inputs.projectRoot, jobDir),
+    summary,
+    doctor,
+    contextPreview: preview,
+    process: { code: processResult.code },
+  }
 }
 
 export async function readEvaluation(config, args) {
@@ -75,14 +222,5 @@ export async function compareCandidates(config, args) {
   const baseline = resolveWithin(config.projectRoot, args.baselineJob, 'baselineJob')
   const candidate = resolveWithin(config.projectRoot, args.candidateJob, 'candidateJob')
   const policy = resolveWithin(config.projectRoot, args.policyPath, 'policyPath')
-  const result = await runProcess(config.harborDshBin, [
-    'promote', baseline, candidate, '--policy', policy,
-  ], {
-    cwd: config.projectRoot,
-    timeoutMs: config.timeoutMs,
-    // harbor-dsh uses exit 1 for a valid REJECT decision. The report remains
-    // model-readable; infrastructure failures still reject the subprocess.
-    allowedExitCodes: [0, 1],
-  })
-  return JSON.parse(result.stdout)
+  return cliJson(config, ['promote', baseline, candidate, '--policy', policy], { allowedExitCodes: [0, 1] })
 }

package/lib/process.js

@@ -6,7 +6,7 @@ export function runProcess(command, args, options = {}) {
       cwd: options.cwd,
       env: options.env ?? process.env,
       shell: false,
-      stdio: ['ignore', 'pipe', 'pipe'],
+      stdio: [options.input === undefined ? 'ignore' : 'pipe', 'pipe', 'pipe'],
     })
     let stdout = ''
     let stderr = ''
@@ -16,6 +16,7 @@ export function runProcess(command, args, options = {}) {
     }, options.timeoutMs ?? 1_800_000)
     child.stdout.on('data', chunk => { stdout += chunk })
     child.stderr.on('data', chunk => { stderr += chunk })
+    if (options.input !== undefined) child.stdin.end(options.input)
     child.on('error', error => {
       clearTimeout(timeout)
       reject(error)

package/lib/service.js

@@ -1,5 +1,50 @@
-import { readDashboardSnapshot } from './dashboard.js'
-import { compareCandidates, readEvaluation, runEvaluation, snapshot } from './evolution.js'
+import { access } from 'node:fs/promises'
+import path from 'node:path'
+
+import {
+  readComparison,
+  readDashboardSnapshot,
+  readDatasetPreview,
+  readEvaluatorGovernance,
+  readJobDetail,
+  readJobProgress,
+  readMetaEvaluation,
+  readTrialDetail,
+  readTrialsPage,
+} from './dashboard.js'
+import {
+  compareCandidates,
+  initializeGroundTruth,
+  initializeProject,
+  inspectEvaluator,
+  previewContext,
+  readEvaluation,
+  runDoctor,
+  runEvaluation,
+  runMetaEvaluation,
+  snapshot,
+  updateEvaluator,
+  validateDataset,
+} from './evolution.js'
+
+export async function resolveEvaluatorStackPath(config, governance, explicitPath) {
+  if (explicitPath) return explicitPath
+  const root = path.resolve(config.projectRoot)
+  const entry = governance.components?.evaluator?.entry
+  if (typeof entry !== 'string' || !entry) return undefined
+  let directory = path.dirname(path.resolve(root, entry))
+  if (directory !== root && !directory.startsWith(`${root}${path.sep}`)) return undefined
+  while (directory === root || directory.startsWith(`${root}${path.sep}`)) {
+    const candidate = path.join(directory, '.harbor', 'evaluation-stack.yml')
+    try {
+      await access(candidate)
+      return path.relative(root, candidate)
+    } catch {}
+    if (directory === root) break
+    directory = path.dirname(directory)
+  }
+  return undefined
+}
 
 /** One Host-side boundary shared by Agent tools and the Web dashboard. */
 export class EvolutionService {
@@ -12,11 +57,24 @@ export class EvolutionService {
     return snapshot(this.config, args)
   }
 
+  initialize(args) {
+    return initializeProject(this.config, args)
+  }
+
   run(args) {
     return runEvaluation(this.config, args)
   }
 
   result(args) {
+    const job = String(args.jobPath ?? '').split(/[\\/]/).filter(Boolean).at(-1)
+    if (args.view === 'job') return readJobDetail(this.config, { job })
+    if (args.view === 'progress') return readJobProgress(this.config, { job, since: args.since })
+    if (args.view === 'trial') {
+      if (!args.trialId) throw new Error('trialId is required when view=trial')
+      return readTrialDetail(this.config, { job, trial: args.trialId })
+    }
+    if (args.view === 'dataset') return readDatasetPreview(this.config, { job })
+    if (args.view === 'governance') return readEvaluatorGovernance(this.config, { job, compareJob: args.compareJob })
     return readEvaluation(this.config, args)
   }
 
@@ -24,7 +82,84 @@ export class EvolutionService {
     return compareCandidates(this.config, args)
   }
 
+  doctor(args) {
+    return runDoctor(this.config, args)
+  }
+
+  validateDataset(args) {
+    return validateDataset(this.config, args)
+  }
+
+  previewContext(args) {
+    return previewContext(this.config, args)
+  }
+
   dashboard() {
     return readDashboardSnapshot(this.config, this.metadata)
   }
+
+  job(args) {
+    return readJobDetail(this.config, args)
+  }
+
+  trials(args) {
+    return readTrialsPage(this.config, args)
+  }
+
+  trial(args) {
+    return readTrialDetail(this.config, args)
+  }
+
+  dataset(args) {
+    return readDatasetPreview(this.config, args)
+  }
+
+  progress(args) {
+    return readJobProgress(this.config, args)
+  }
+
+  comparison(args) {
+    return readComparison(this.config, args)
+  }
+
+  async governance(args) {
+    const governance = await readEvaluatorGovernance(this.config, args)
+    try {
+      const stackPath = await resolveEvaluatorStackPath(this.config, governance, args.stackPath)
+      governance.evaluatorInterface = await inspectEvaluator(this.config, { ...args, stackPath })
+      governance.editingPolicy.browserWriteEnabled = true
+      governance.editingPolicy.stackPath = governance.evaluatorInterface.stack?.path
+      governance.editingPolicy.saveBehavior = 'Update one descriptor-authorized file with optimistic concurrency and create new Evaluator and Stack identities.'
+    } catch (error) {
+      governance.evaluatorInterface = { error: error instanceof Error ? error.message : String(error) }
+    }
+    return governance
+  }
+
+  evaluator(args) {
+    return updateEvaluator(this.config, args)
+  }
+
+  evaluatorInspect(args) {
+    return inspectEvaluator(this.config, args)
+  }
+
+  groundTruthInitialize(args) {
+    return initializeGroundTruth(this.config, args)
+  }
+
+  evaluatorMetaEvaluate(args) {
+    return runMetaEvaluation(this.config, args)
+  }
+
+  async meta(args) {
+    const governance = await readEvaluatorGovernance(this.config, args)
+    const stackPath = await resolveEvaluatorStackPath(this.config, governance, args.stackPath)
+    if (!stackPath) return readMetaEvaluation(this.config)
+    const stackDirectory = path.dirname(path.resolve(this.config.projectRoot, stackPath))
+    const evaluationRoot = path.dirname(stackDirectory)
+    return readMetaEvaluation(this.config, {
+      evaluationRoot: path.relative(this.config.projectRoot, evaluationRoot),
+    })
+  }
 }

package/lib/web.js

@@ -1,4 +1,14 @@
 export const DASHBOARD_ROUTE = '/_dsh/harbor-evolution/dashboard'
+export const JOB_ROUTE = '/_dsh/harbor-evolution/job'
+export const TRIALS_ROUTE = '/_dsh/harbor-evolution/trials'
+export const TRIAL_ROUTE = '/_dsh/harbor-evolution/trial'
+export const DATASET_ROUTE = '/_dsh/harbor-evolution/dataset'
+export const PROGRESS_ROUTE = '/_dsh/harbor-evolution/progress'
+export const COMPARE_ROUTE = '/_dsh/harbor-evolution/compare'
+export const GOVERNANCE_ROUTE = '/_dsh/harbor-evolution/governance'
+export const EVALUATOR_ROUTE = '/_dsh/harbor-evolution/evaluator'
+export const META_ROUTE = '/_dsh/harbor-evolution/meta'
+const MAX_MUTATION_BYTES = 256 * 1024
 
 function sendJson(response, status, body) {
   response.writeHead(status, {
@@ -16,20 +26,24 @@ export function isSameOriginRequest(request) {
   if (!origin) {
     if (fetchSite === 'same-origin' || fetchSite === 'none') return true
     const address = request.socket?.remoteAddress ?? ''
-    return address === '::1' || address === '127.0.0.1' || address.startsWith('127.')
-      || address.startsWith('::ffff:127.')
+    return address === '::1' || address === '127.0.0.1' || address.startsWith('127.') || address.startsWith('::ffff:127.')
   }
   const host = request.headers.host
   if (!host) return false
   try {
     const parsed = new URL(origin)
-    return (parsed.protocol === 'http:' || parsed.protocol === 'https:') && parsed.host === host
+    return ['http:', 'https:'].includes(parsed.protocol) && parsed.host === host
   } catch {
     return false
   }
 }
 
-export function createDashboardHandler(service) {
+function safeError(error) {
+  const message = error instanceof Error ? error.message : String(error)
+  return message.replace(/(?:\/[A-Za-z0-9._ -]+){2,}/g, '[local path]').replace(/[A-Za-z]:\\[^\s]+/g, '[local path]')
+}
+
+export function createApiHandler(load, code = 'request-failed') {
   return (request, response) => {
     if (request.method !== 'GET') {
       response.writeHead(405, { allow: 'GET' })
@@ -40,24 +54,71 @@ export function createDashboardHandler(service) {
       sendJson(response, 403, { ok: false, error: { code: 'forbidden', message: 'same-origin request required' } })
       return
     }
-    Promise.resolve(service.dashboard()).then(
+    const url = new URL(request.url ?? '/', 'http://localhost')
+    const args = Object.fromEntries(url.searchParams)
+    Promise.resolve(load(args)).then(
       value => sendJson(response, 200, { ok: true, value }),
-      error => sendJson(response, 500, {
-        ok: false,
-        error: { code: 'dashboard-unavailable', message: error instanceof Error ? error.message : String(error) },
-      }),
+      error => sendJson(response, 500, { ok: false, error: { code, message: safeError(error) } }),
     )
   }
 }
 
-/** Add the dashboard route only in profiles that provide the optional Web service. */
+export function createDashboardHandler(service) {
+  return createApiHandler(() => service.dashboard(), 'dashboard-unavailable')
+}
+
+export function createMutationHandler(update, code = 'update-failed') {
+  return async (request, response) => {
+    if (request.method !== 'POST') {
+      response.writeHead(405, { allow: 'POST' })
+      response.end()
+      return
+    }
+    if (!isSameOriginRequest(request)) {
+      sendJson(response, 403, { ok: false, error: { code: 'forbidden', message: 'same-origin request required' } })
+      return
+    }
+    if (!String(request.headers['content-type'] ?? '').toLowerCase().startsWith('application/json')) {
+      sendJson(response, 415, { ok: false, error: { code: 'unsupported-media-type', message: 'application/json required' } })
+      return
+    }
+    try {
+      const chunks = []
+      let size = 0
+      for await (const chunk of request) {
+        size += chunk.length
+        if (size > MAX_MUTATION_BYTES) {
+          sendJson(response, 413, { ok: false, error: { code: 'payload-too-large', message: 'request body is too large' } })
+          return
+        }
+        chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
+      }
+      const body = JSON.parse(Buffer.concat(chunks).toString('utf8'))
+      const value = await update(body)
+      sendJson(response, 200, { ok: true, value })
+    } catch (error) {
+      sendJson(response, 400, { ok: false, error: { code, message: safeError(error) } })
+    }
+  }
+}
+
 export function installDashboardWeb(ctx, service) {
   if (typeof ctx.inject !== 'function') return
   ctx.inject(['webServer'], (webCtx) => {
-    webCtx.effect(() => webCtx.webServer.register({
-      kind: 'exact',
-      path: DASHBOARD_ROUTE,
-      handler: createDashboardHandler(service),
-    }), 'harbor-evolution: dashboard route')
+    const routes = [
+      [DASHBOARD_ROUTE, createDashboardHandler(service)],
+      [JOB_ROUTE, createApiHandler(args => service.job(args), 'job-unavailable')],
+      [TRIALS_ROUTE, createApiHandler(args => service.trials(args), 'trials-unavailable')],
+      [TRIAL_ROUTE, createApiHandler(args => service.trial(args), 'trial-unavailable')],
+      [DATASET_ROUTE, createApiHandler(args => service.dataset(args), 'dataset-unavailable')],
+      [PROGRESS_ROUTE, createApiHandler(args => service.progress(args), 'progress-unavailable')],
+      [COMPARE_ROUTE, createApiHandler(args => service.comparison(args), 'comparison-unavailable')],
+      [GOVERNANCE_ROUTE, createApiHandler(args => service.governance(args), 'governance-unavailable')],
+      [EVALUATOR_ROUTE, createMutationHandler(args => service.evaluator(args), 'evaluator-update-failed')],
+      [META_ROUTE, createApiHandler(args => service.meta(args), 'meta-evaluation-unavailable')],
+    ]
+    for (const [route, handler] of routes) {
+      webCtx.effect(() => webCtx.webServer.register({ kind: 'exact', path: route, handler }), `harbor-evolution: ${route}`)
+    }
   })
 }

package/package.json

@@ -1,12 +1,17 @@
 {
   "name": "dsh-harbor-evolution",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "description": "DeepSeek Harness plugin and bundled Skill for safely evolving Cordis Candidates with Harbor.",
   "type": "module",
   "main": "index.js",
   "exports": {
     ".": "./index.js",
     "./client": "./lib/client.js",
+    "./cordis.patch.yml": "./cordis.patch.yml",
+    "./schemas/evaluation-result.schema.json": "./schemas/evaluation-result.schema.json",
+    "./schemas/ground-truth.schema.json": "./schemas/ground-truth.schema.json",
+    "./schemas/evaluator-observations.schema.json": "./schemas/evaluator-observations.schema.json",
+    "./schemas/meta-evaluation-report.schema.json": "./schemas/meta-evaluation-report.schema.json",
     "./package.json": "./package.json"
   },
   "bin": {
@@ -17,6 +22,7 @@
     "lib/",
     "bin/",
     "skills/",
+    "schemas/",
     "cordis.patch.yml",
     "README.md",
     "LICENSE"

package/schemas/evaluation-result.schema.json

@@ -0,0 +1,27 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Harbor DSH Evaluator Result v1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "protocol", "criteria"],
+  "properties": {
+    "schema_version": { "const": 1 },
+    "protocol": { "const": "evaluation-result/v1" },
+    "criteria": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["id", "score", "reason", "recommendation"],
+        "properties": {
+          "id": { "type": "string", "minLength": 1 },
+          "score": { "enum": [0, 0.5, 1] },
+          "reason": { "type": "string", "minLength": 1 },
+          "recommendation": { "type": "string", "minLength": 1 },
+          "evidence_refs": { "type": "array", "items": { "type": "string" } }
+        }
+      }
+    }
+  }
+}

package/schemas/evaluator-observations.schema.json

@@ -0,0 +1,51 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Evaluator Observations v1",
+  "type": "object",
+  "required": ["schema_version", "protocol", "evaluator", "repeat_policy", "observations"],
+  "properties": {
+    "schema_version": { "const": 1 },
+    "protocol": { "const": "evaluator-observations/v1" },
+    "evaluator": {
+      "type": "object",
+      "required": ["id", "version"],
+      "properties": {
+        "id": { "type": "string", "minLength": 1 },
+        "version": { "type": "string", "minLength": 1 },
+        "digest": { "type": "string" }
+      }
+    },
+    "repeat_policy": {
+      "type": "object",
+      "required": ["repeats"],
+      "properties": {
+        "repeats": { "type": "integer", "minimum": 1 },
+        "seed_policy": { "type": "string" }
+      }
+    },
+    "observations": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "required": ["case_id", "repeat", "criteria"],
+        "properties": {
+          "case_id": { "type": "string", "minLength": 1 },
+          "repeat": { "type": "integer", "minimum": 1 },
+          "criteria": {
+            "type": "array",
+            "minItems": 1,
+            "items": {
+              "type": "object",
+              "required": ["id", "score"],
+              "properties": {
+                "id": { "type": "string", "minLength": 1 },
+                "score": { "enum": [0, 0.5, 1] }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}