dsh-harbor-evolution 0.7.0 → 0.7.1

package/README.md

@@ -14,8 +14,8 @@ npx --yes dsh-harbor-evolution@latest setup --project-root "$PWD"
 
 The setup command installs both required runtimes:
 
-- `harbor-dsh-evolution==0.7.0` in a managed Python environment.
-- `dsh-harbor-evolution@0.7.0` in the selected DSH profile.
+- `harbor-dsh-evolution==0.7.1` in a managed Python environment.
+- `dsh-harbor-evolution@0.7.1` in the selected DSH profile.
 
 It then stores the absolute Harbor executable paths and `projectRoot` in the profile's `harbor-evolution` block and verifies the integration. Existing unrelated profile entries are preserved, and rerunning setup updates the same block.
 
@@ -57,6 +57,12 @@ The Web UI is intentionally read-only. Starting an evaluation or deciding promot
 
 A direct evaluation requires `candidatePath`, `datasetPath`, `stackPath`, and explicit `mode`; `promotion-eligible` additionally requires `policyPath`. Prefer the Skill because it will not run or compare Jobs until the material identities and evaluation contract are resolved.
 
+## Candidate model binding
+
+Before each Job, the Plugin snapshots the current DSH Agent selection—provider, model, and reasoning effort—then starts a per-Job local Model Broker. The Candidate uses the temporary `dsh-host` adapter through `dsh-host-broker` / `dsh-host-model-gateway/v1`; it receives only a short-lived Job capability file, never GPT Auth, Codex OAuth, or an upstream API key.
+
+`harbor_eval_run`, `harbor_context_preview`, and `harbor_evolution_doctor` inherit that selection by default. Advanced callers can override `candidateProvider` and `candidateModel` only as a pair, plus an optional `candidateReasoningEffort`. `openai-codex` performs a GPT Auth sign-in check before Harbor starts. The resulting model binding is part of Context v2 comparison identity, so any provider/model/reasoning change requires a new baseline.
+
 `harbor_eval_result` defaults to the stable Summary. Use `view=job`, `view=dataset`, `view=progress`, `view=trial` plus a returned `trialId`, or `view=governance` to inspect sanitized instructions, generated output, evidence, and evaluator source without coupling the Agent to artifact file paths.
 
 ## What setup writes

package/index.js

@@ -5,11 +5,12 @@ import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 
 import { loadBundledSkill } from './lib/official-skill.js'
+import { CandidateModelRuntime } from './lib/model-runtime.js'
 import { EvolutionService } from './lib/service.js'
 import { installDashboardWeb } from './lib/web.js'
 
 export const name = 'harbor-evolution'
-export const inject = ['tools', 'skills']
+export const inject = ['tools', 'skills', 'llm', 'agentDefaultModel']
 
 const packageDir = path.dirname(fileURLToPath(import.meta.url))
 const checkoutPythonPackage = path.resolve(packageDir, '../harbor-plugin')
@@ -30,6 +31,13 @@ export const Config = Schema.object({
   pluginImportPath: Schema.string().default('dsh-evolution'),
   pythonPath: Schema.string().default(''),
   timeoutMs: Schema.number().default(1800000),
+  candidateProvider: Schema.string().default(''),
+  candidateModel: Schema.string().default(''),
+  candidateReasoningEffort: Schema.string().default(''),
+  modelBrokerBindHost: Schema.string().default('127.0.0.1'),
+  modelBrokerAdvertisedHost: Schema.string().default('host.docker.internal'),
+  modelBrokerMaxRequests: Schema.number().min(1).default(1000),
+  modelBrokerMaxRequestBytes: Schema.number().min(1024).default(33554432),
 })
 
 function jsonTool(definition, execute) {
@@ -57,7 +65,8 @@ export function apply(ctx, config) {
         : ''
     ),
   }
-  const service = new EvolutionService(resolved, { pluginVersion: packageJson.version })
+  const modelRuntime = new CandidateModelRuntime(ctx, resolved)
+  const service = new EvolutionService(resolved, { pluginVersion: packageJson.version }, modelRuntime)
 
   ctx.skills.register(loadBundledSkill())
   installDashboardWeb(ctx, service)
@@ -103,6 +112,9 @@ export function apply(ctx, config) {
       stackPath: { type: 'string', required: true },
       policyPath: { type: 'string' },
       mode: { type: 'string', required: true },
+      candidateProvider: { type: 'string', description: 'Optional Candidate provider. Supply it together with candidateModel; defaults to the current DSH Agent model.' },
+      candidateModel: { type: 'string' },
+      candidateReasoningEffort: { type: 'string' },
     },
   }, args => service.doctor(args)))
 
@@ -124,6 +136,9 @@ export function apply(ctx, config) {
       datasetPath: { type: 'string', required: true },
       stackPath: { type: 'string', required: true },
       mode: { type: 'string', required: true },
+      candidateProvider: { type: 'string', description: 'Optional Candidate provider. Supply it together with candidateModel; defaults to the current DSH Agent model.' },
+      candidateModel: { type: 'string' },
+      candidateReasoningEffort: { type: 'string' },
     },
   }, args => service.previewContext(args)))
 
@@ -139,6 +154,9 @@ export function apply(ctx, config) {
       mode: { type: 'string', required: true },
       policyPath: { type: 'string' },
       jobName: { type: 'string' },
+      candidateProvider: { type: 'string', description: 'Optional Candidate provider. Supply it together with candidateModel; defaults to the current DSH Agent model and is frozen before the Job starts.' },
+      candidateModel: { type: 'string' },
+      candidateReasoningEffort: { type: 'string' },
     },
   }, args => service.run(args)))
 

package/lib/candidate.js

@@ -4,7 +4,7 @@ import path from 'node:path'
 
 export const MANIFEST_NAME = 'candidate-manifest.json'
 const DIGEST_PREFIX = Buffer.from('harbor-dsh-candidate-v1\0')
-const EXCLUDED_DIRS = new Set(['.git', 'node_modules', '__pycache__'])
+const EXCLUDED_DIRS = new Set(['.git', 'node_modules', '__pycache__', '.harbor-runtime'])
 const EXCLUDED_FILES = new Set([MANIFEST_NAME, '.DS_Store'])
 const LOCKFILES = ['package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'bun.lock', 'bun.lockb']
 const CREDENTIAL_FILES = new Set([
@@ -57,6 +57,12 @@ export async function computeCandidate(candidateDir) {
 }
 
 async function validateCandidateContract(root) {
+  try {
+    await stat(path.join(root, '.harbor-runtime'))
+    throw new Error('Candidate must not contain the reserved .harbor-runtime path')
+  } catch (error) {
+    if (error.code !== 'ENOENT') throw error
+  }
   for (const required of ['cordis.yml', 'package.json']) {
     try {
       if (!(await stat(path.join(root, required))).isFile()) throw new Error()

package/lib/evolution.js

@@ -119,6 +119,19 @@ function strictInputs(config, args) {
   return { projectRoot, candidate, dataset, stack, jobs, mode, policy }
 }
 
+function candidateModelCliArgs(binding) {
+  if (!binding?.provider || !binding?.model) throw new Error('Candidate model binding is required')
+  return [
+    '--candidate-model-provider', binding.provider,
+    '--candidate-model', binding.model,
+    ...(binding.reasoning_effort === undefined
+      ? []
+      : ['--candidate-reasoning-effort', binding.reasoning_effort]),
+    '--candidate-model-transport', binding.transport,
+    '--candidate-model-protocol', binding.protocol,
+  ]
+}
+
 export async function validateDataset(config, args) {
   const dataset = resolveWithin(config.projectRoot, args.datasetPath, 'datasetPath')
   return cliJson(config, ['dataset', 'validate', dataset, '--project-root', config.projectRoot], { allowedExitCodes: [0, 2] })
@@ -157,11 +170,12 @@ export async function previewContext(config, args) {
     '--stack', inputs.stack,
     '--jobs-dir', inputs.jobs,
     '--mode', inputs.mode,
+    ...candidateModelCliArgs(args.candidateModelBinding),
   ])
   return { manifest, ...preview }
 }
 
-export async function runEvaluation(config, args) {
+export async function runEvaluation(config, args, modelRuntime) {
   const manifest = await snapshot(config, args)
   const inputs = strictInputs(config, args)
   const datasetValidation = await validateDataset(config, args)
@@ -176,6 +190,7 @@ export async function runEvaluation(config, args) {
     'context', 'preview', '--project-root', inputs.projectRoot,
     '--candidate', inputs.candidate, '--dataset', inputs.dataset,
     '--stack', inputs.stack, '--jobs-dir', inputs.jobs, '--mode', inputs.mode,
+    ...candidateModelCliArgs(args.candidateModelBinding),
   ])
   const jobName = args.jobName ?? makeJobName(manifest)
   if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$/.test(jobName)) throw new Error('jobName contains unsupported characters')
@@ -186,6 +201,8 @@ export async function runEvaluation(config, args) {
     '--ak', `candidate_path=${inputs.candidate}`,
     '--ak', `candidate_version=${manifest.version}`,
     '--ak', `candidate_digest=${manifest.digest}`,
+    '--ak', `candidate_model_provider=${args.candidateModelBinding.provider}`,
+    '--ak', `candidate_model=${args.candidateModelBinding.model}`,
     '--job-name', jobName,
     '--jobs-dir', inputs.jobs,
     '--plugin', config.pluginImportPath,
@@ -194,22 +211,47 @@ export async function runEvaluation(config, args) {
     '--plugin-kwarg', `stack_path=${inputs.stack}`,
     '--plugin-kwarg', `project_root=${inputs.projectRoot}`,
     '--plugin-kwarg', `mode=${inputs.mode}`,
+    '--plugin-kwarg', `candidate_model_provider=${args.candidateModelBinding.provider}`,
+    '--plugin-kwarg', `candidate_model=${args.candidateModelBinding.model}`,
+    '--plugin-kwarg', `candidate_model_transport=${args.candidateModelBinding.transport}`,
+    '--plugin-kwarg', `candidate_model_protocol=${args.candidateModelBinding.protocol}`,
   ]
+  if (args.candidateModelBinding.reasoning_effort !== undefined) {
+    harborArgs.push('--ak', `candidate_reasoning_effort=${args.candidateModelBinding.reasoning_effort}`)
+    harborArgs.push('--plugin-kwarg', `candidate_reasoning_effort=${args.candidateModelBinding.reasoning_effort}`)
+  }
   if (inputs.policy) harborArgs.push('--plugin-kwarg', `policy_path=${inputs.policy}`)
-  const processResult = await runProcess(config.harborBin, harborArgs, {
-    cwd: config.projectRoot,
-    timeoutMs: config.timeoutMs,
-    env: { ...process.env, ...(config.pythonPath ? { PYTHONPATH: config.pythonPath } : {}) },
+  const lease = await modelRuntime.openLease(args.candidateModelBinding, {
+    candidateDigest: manifest.digest,
+    jobName,
   })
-  const jobDir = path.join(inputs.jobs, jobName)
-  const summary = JSON.parse(await readFile(path.join(jobDir, 'evaluation-summary.json'), 'utf8'))
-  return {
-    manifest,
-    job: path.relative(inputs.projectRoot, jobDir),
-    summary,
-    doctor,
-    contextPreview: preview,
-    process: { code: processResult.code },
+  try {
+    const processResult = await runProcess(config.harborBin, harborArgs, {
+      cwd: config.projectRoot,
+      timeoutMs: config.timeoutMs,
+      env: {
+        ...process.env,
+        ...(config.pythonPath ? { PYTHONPATH: config.pythonPath } : {}),
+        HSE_MODEL_GATEWAY_URL: lease.endpoint,
+        HSE_MODEL_GATEWAY_TOKEN: lease.token,
+        HSE_MODEL_GATEWAY_PROVIDER: lease.candidateProvider,
+        HSE_MODEL_GATEWAY_INFO: JSON.stringify(lease.modelInfo),
+        HSE_MODEL_GATEWAY_PROTOCOL: lease.protocol,
+      },
+    })
+    const jobDir = path.join(inputs.jobs, jobName)
+    const summary = JSON.parse(await readFile(path.join(jobDir, 'evaluation-summary.json'), 'utf8'))
+    return {
+      manifest,
+      candidateModelBinding: args.candidateModelBinding,
+      job: path.relative(inputs.projectRoot, jobDir),
+      summary,
+      doctor,
+      contextPreview: preview,
+      process: { code: processResult.code },
+    }
+  } finally {
+    await lease.close()
   }
 }
 

package/lib/model-runtime.js

@@ -0,0 +1,216 @@
+import { randomBytes, timingSafeEqual } from 'node:crypto'
+import { once } from 'node:events'
+import { createServer } from 'node:http'
+
+export const MODEL_GATEWAY_PROTOCOL = 'dsh-host-model-gateway/v1'
+export const CANDIDATE_GATEWAY_PROVIDER = 'dsh-host'
+
+function nonBlank(value) {
+  return typeof value === 'string' && value.trim() ? value.trim() : undefined
+}
+
+function sameSecret(expected, actual) {
+  const left = Buffer.from(expected)
+  const right = Buffer.from(actual)
+  return left.length === right.length && timingSafeEqual(left, right)
+}
+
+async function readJsonBody(request, maxBytes) {
+  const chunks = []
+  let size = 0
+  for await (const chunk of request) {
+    size += chunk.length
+    if (size > maxBytes) {
+      throw Object.assign(new Error('model gateway request is too large'), { statusCode: 413 })
+    }
+    chunks.push(chunk)
+  }
+  let value
+  try {
+    value = JSON.parse(Buffer.concat(chunks).toString('utf8'))
+  } catch {
+    throw Object.assign(new Error('model gateway request must be valid JSON'), { statusCode: 400 })
+  }
+  if (value === null || typeof value !== 'object' || Array.isArray(value) || !Array.isArray(value.messages)) {
+    throw Object.assign(new Error('model gateway request requires a messages array'), { statusCode: 400 })
+  }
+  return value
+}
+
+function sendJson(response, statusCode, value) {
+  const body = `${JSON.stringify(value)}\n`
+  response.writeHead(statusCode, {
+    'content-type': 'application/json; charset=utf-8',
+    'content-length': Buffer.byteLength(body),
+    'cache-control': 'no-store',
+  })
+  response.end(body)
+}
+
+function authorized(request, token) {
+  const value = request.headers.authorization
+  return typeof value === 'string' && value.startsWith('Bearer ')
+    && sameSecret(token, value.slice('Bearer '.length))
+}
+
+async function listen(server, host) {
+  server.listen(0, host)
+  await once(server, 'listening')
+  const address = server.address()
+  if (address === null || typeof address === 'string') {
+    throw new Error('model gateway did not bind a TCP port')
+  }
+  return address.port
+}
+
+/**
+ * Freeze the current DSH Agent model per Job, then proxy only that model to an
+ * immutable Candidate. The Candidate receives a short-lived capability, never
+ * the Host provider's credential or route selection.
+ */
+export class CandidateModelRuntime {
+  constructor(ctx, config) {
+    this.ctx = ctx
+    this.config = config
+  }
+
+  async resolve(args = {}) {
+    const explicitProvider = nonBlank(args.candidateProvider)
+    const explicitModel = nonBlank(args.candidateModel)
+    if (Boolean(explicitProvider) !== Boolean(explicitModel)) {
+      throw new Error('candidateProvider and candidateModel must be supplied together')
+    }
+    const configuredProvider = nonBlank(this.config.candidateProvider)
+    const configuredModel = nonBlank(this.config.candidateModel)
+    if (Boolean(configuredProvider) !== Boolean(configuredModel)) {
+      throw new Error('Harbor candidateProvider and candidateModel configuration must be supplied together')
+    }
+
+    const inherited = this.ctx.agentDefaultModel.currentSelection()
+    const provider = explicitProvider ?? configuredProvider ?? inherited.provider
+    const model = explicitModel ?? configuredModel ?? inherited.model
+    const explicitReasoning = nonBlank(args.candidateReasoningEffort)
+    const configuredReasoning = nonBlank(this.config.candidateReasoningEffort)
+    const canInheritReasoning = provider === inherited.provider && model === inherited.model
+    const reasoningEffort = explicitReasoning ?? configuredReasoning
+      ?? (canInheritReasoning ? inherited.reasoningEffort : undefined)
+
+    if (!this.ctx.llm.listProviders().some(item => item.id === provider)) {
+      throw new Error(`Candidate model provider "${provider}" is not registered in DeepSeek Harness`)
+    }
+    const modelInfo = await this.ctx.llm.resolveModelInfo(provider, model)
+    if (provider === 'openai-codex') {
+      const auth = this.ctx.get('codexAuth')
+      if (auth === undefined || typeof auth.status !== 'function') {
+        throw new Error('Candidate model openai-codex requires the dsh-codex-auth service')
+      }
+      const status = await auth.status()
+      if (!status.configured) {
+        throw new Error('Candidate model openai-codex is not signed in; complete GPT Auth in Settings before starting Harbor')
+      }
+    }
+
+    return {
+      provider,
+      model,
+      ...(reasoningEffort === undefined ? {} : { reasoning_effort: String(reasoningEffort) }),
+      transport: 'dsh-host-broker',
+      protocol: MODEL_GATEWAY_PROTOCOL,
+      model_info: modelInfo,
+    }
+  }
+
+  async openLease(binding, scope) {
+    const token = randomBytes(32).toString('base64url')
+    const route = `/harbor-model-gateway/v1/${randomBytes(18).toString('base64url')}`
+    const controllers = new Set()
+    let requestCount = 0
+    const server = createServer(async (request, response) => {
+      if (request.url !== route || !authorized(request, token)) {
+        sendJson(response, 404, { error: 'not found' })
+        return
+      }
+      if (request.method === 'GET') {
+        sendJson(response, 200, {
+          protocol: MODEL_GATEWAY_PROTOCOL,
+          candidate_digest: scope.candidateDigest,
+          job: scope.jobName,
+          binding: {
+            provider: binding.provider,
+            model: binding.model,
+            ...(binding.reasoning_effort === undefined ? {} : { reasoning_effort: binding.reasoning_effort }),
+          },
+        })
+        return
+      }
+      if (request.method !== 'POST') {
+        sendJson(response, 405, { error: 'method not allowed' })
+        return
+      }
+      if (requestCount >= this.config.modelBrokerMaxRequests) {
+        sendJson(response, 429, { error: 'model gateway request budget exhausted' })
+        return
+      }
+      requestCount += 1
+      const controller = new AbortController()
+      controllers.add(controller)
+      response.on('close', () => {
+        if (!response.writableEnded) controller.abort(new Error('Candidate disconnected'))
+      })
+      try {
+        const body = await readJsonBody(request, this.config.modelBrokerMaxRequestBytes)
+        const {
+          provider: _provider,
+          model: _model,
+          reasoningEffort: _reasoningEffort,
+          signal: _signal,
+          ...requestOptions
+        } = body
+        response.writeHead(200, {
+          'content-type': 'application/x-ndjson; charset=utf-8',
+          'cache-control': 'no-store',
+        })
+        const stream = this.ctx.llm.stream({
+          ...requestOptions,
+          provider: binding.provider,
+          model: binding.model,
+          ...(binding.reasoning_effort === undefined ? {} : { reasoningEffort: binding.reasoning_effort }),
+          signal: controller.signal,
+        })
+        for await (const chunk of stream) {
+          if (!response.write(`${JSON.stringify(chunk)}\n`)) await once(response, 'drain')
+        }
+        response.end()
+      } catch (error) {
+        if (!response.headersSent) {
+          const statusCode = Number.isInteger(error?.statusCode) ? error.statusCode : 502
+          const message = error instanceof Error ? error.message : String(error)
+          sendJson(response, statusCode, { error: message })
+        } else if (!response.writableEnded) {
+          response.destroy(error instanceof Error ? error : new Error(String(error)))
+        }
+      } finally {
+        controllers.delete(controller)
+      }
+    })
+
+    const port = await listen(server, this.config.modelBrokerBindHost)
+    const endpoint = `http://${this.config.modelBrokerAdvertisedHost}:${port}${route}`
+    let closed = false
+    return {
+      protocol: MODEL_GATEWAY_PROTOCOL,
+      endpoint,
+      token,
+      candidateProvider: CANDIDATE_GATEWAY_PROVIDER,
+      modelInfo: binding.model_info,
+      async close() {
+        if (closed) return
+        closed = true
+        for (const controller of controllers) controller.abort(new Error('Harbor Job ended'))
+        const close = new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve()))
+        server.closeAllConnections()
+        await close
+      },
+    }
+  }
+}

package/lib/service.js

@@ -48,9 +48,10 @@ export async function resolveEvaluatorStackPath(config, governance, explicitPath
 
 /** One Host-side boundary shared by Agent tools and the Web dashboard. */
 export class EvolutionService {
-  constructor(config, metadata = {}) {
+  constructor(config, metadata = {}, modelRuntime) {
     this.config = config
     this.metadata = metadata
+    this.modelRuntime = modelRuntime
   }
 
   snapshot(args) {
@@ -61,8 +62,9 @@ export class EvolutionService {
     return initializeProject(this.config, args)
   }
 
-  run(args) {
-    return runEvaluation(this.config, args)
+  async run(args) {
+    const candidateModelBinding = await this.modelRuntime.resolve(args)
+    return runEvaluation(this.config, { ...args, candidateModelBinding }, this.modelRuntime)
   }
 
   result(args) {
@@ -82,16 +84,19 @@ export class EvolutionService {
     return compareCandidates(this.config, args)
   }
 
-  doctor(args) {
-    return runDoctor(this.config, args)
+  async doctor(args) {
+    const candidateModelBinding = await this.modelRuntime.resolve(args)
+    const result = await runDoctor(this.config, args)
+    return { ...result, candidate_model_binding: candidateModelBinding }
   }
 
   validateDataset(args) {
     return validateDataset(this.config, args)
   }
 
-  previewContext(args) {
-    return previewContext(this.config, args)
+  async previewContext(args) {
+    const candidateModelBinding = await this.modelRuntime.resolve(args)
+    return previewContext(this.config, { ...args, candidateModelBinding })
   }
 
   dashboard() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-harbor-evolution",
-  "version": "0.7.0",
+  "version": "0.7.1",
   "description": "DeepSeek Harness plugin and bundled Skill for safely evolving Cordis Candidates with Harbor.",
   "type": "module",
   "main": "index.js",