dsh-harbor-evolution 0.7.3 → 0.8.0

package/lib/dashboard.js

@@ -5,6 +5,7 @@ import path from 'node:path'
 import { resolveWithin } from './evolution.js'
 
 const SUMMARY_NAME = 'evaluation-summary.json'
+const HISTORICAL_COMPLETION_NAME = 'historical-evaluation-complete.json'
 const DEFAULT_JOB_PAGE_SIZE = 20
 const MAX_JOB_PAGE_SIZE = 100
 const MAX_JSON_BYTES = 2 * 1024 * 1024
@@ -107,19 +108,65 @@ function isObject(value) {
   return value !== null && typeof value === 'object' && !Array.isArray(value)
 }
 
+const CANDIDATE_JOB_KIND = 'candidate-evaluation'
+const HISTORICAL_JOB_KIND = 'historical-generation-evaluation'
+
+function normalizedJobKind(summary, context) {
+  const declared = summary?.job_kind ?? context?.job_kind
+  if (typeof declared === 'string' && declared) return declared
+  if (context?.protocol === 'historical-generation-evaluation-context/v1') return HISTORICAL_JOB_KIND
+  return CANDIDATE_JOB_KIND
+}
+
+function coverageView(summary) {
+  if (isObject(summary?.coverage)) return summary.coverage
+  const total = Number(summary?.n_trials ?? 0)
+  const scored = Number(summary?.scored_trial_count ?? summary?.n_valid_scores ?? 0)
+  const unscored = Number(
+    summary?.unscored_trial_count
+    ?? summary?.status_counts?.['completed-unscored']
+    ?? 0,
+  )
+  return {
+    scored_trials: scored,
+    unscored_trials: unscored,
+    total_trials: total,
+    trial_rate: total ? scored / total : undefined,
+  }
+}
+
+function evaluatorMetaEvaluation(summary, context) {
+  return summary?.evaluator_meta_evaluation
+    ?? context?.downstream_analysis?.evaluator_meta_evaluation
+    ?? (normalizedJobKind(summary, context) === HISTORICAL_JOB_KIND
+      ? { status: 'not-run', validation_report_ref: null }
+      : undefined)
+}
+
 function capabilityMap(summary, context, lifecycle, registry, stack) {
+  const jobKind = normalizedJobKind(summary, context)
+  const historicalGeneration = jobKind === HISTORICAL_JOB_KIND
   const contextV2 = context?.schema_version === 2
+  const historicalContext = context?.schema_version === 1
+    && context?.protocol === 'historical-generation-evaluation-context/v1'
   const scoreValidity = summary?.schema_version === 3
+    || summary?.schema_version === 4
   return {
+    jobKind,
     contextV2,
+    contextSupported: contextV2 || historicalContext,
+    historicalGeneration,
+    candidateEvaluation: !historicalGeneration,
     trialLifecycle: lifecycle?.schema_version === 1,
     scoreValidity,
     evidenceProvenance: scoreValidity,
-    artifactRegistry: registry?.schema_version === 1,
-    compare: contextV2,
+    artifactRegistry: [1, 2].includes(registry?.schema_version),
+    source: historicalGeneration,
+    compare: contextV2 && !historicalGeneration,
     evaluatorGovernance: stack?.schema_version === 1,
-    gate: contextV2 && summary?.mode === 'promotion-eligible',
-    readOnlyLegacy: !contextV2,
+    evaluatorMetaEvaluation: evaluatorMetaEvaluation(summary, context),
+    gate: contextV2 && !historicalGeneration && summary?.mode === 'promotion-eligible',
+    readOnlyLegacy: !contextV2 && !historicalContext,
   }
 }
 
@@ -131,7 +178,7 @@ function primaryMetric(summary, contract) {
 }
 
 function progressView(summary, lifecycle, updatedAt) {
-  const total = Number(lifecycle?.dataset_total ?? summary?.n_trials ?? 0)
+  const total = Number(lifecycle?.dataset_total ?? summary?.n_trials ?? summary?.coverage?.total_trials ?? 0)
   const lifecycleTrials = selectedLifecycleTrials(lifecycle)
   const completed = lifecycle
     ? lifecycleTrials.filter(item => item.terminal).length
@@ -151,19 +198,53 @@ function progressView(summary, lifecycle, updatedAt) {
   }
 }
 
-function jobStatus(summary, lifecycle, progress) {
+const HISTORICAL_COVERAGE_KEYS = [
+  'scored_trials', 'unscored_trials', 'total_trials', 'trial_rate',
+  'criterion_scored', 'criterion_total', 'criterion_rate',
+]
+
+function historicalCompletionValid(summary, completion, jobName) {
+  return (
+    summary?.schema_version === 4
+    && summary?.job === jobName
+    && summary?.job_kind === HISTORICAL_JOB_KIND
+    && summary?.mode === 'diagnostic'
+    && summary?.execution_mode === 'observe-existing'
+    && summary?.artifact_validation?.valid === true
+    && summary?.candidate === undefined
+    && completion?.schema_version === 1
+    && completion?.job_kind === HISTORICAL_JOB_KIND
+    && completion?.status === 'completed'
+    && completion?.valid === true
+    && completion?.job === jobName
+    && completion?.summary_path === SUMMARY_NAME
+    && completion?.artifact_registry_path === 'artifact-registry.json'
+    && HISTORICAL_COVERAGE_KEYS.every(key => (
+      typeof summary?.coverage?.[key] === 'number'
+      && summary.coverage[key] === completion?.coverage?.[key]
+    ))
+  )
+}
+
+function jobStatus(summary, lifecycle, progress, jobKind, completion, jobName) {
   if (summary?.__readError) return 'failed'
   if (!summary && !lifecycle) return 'pending'
   if (progress.active) return 'running'
   if (!summary && lifecycle) return 'running'
+  if (summary?.artifact_validation?.valid === false) return 'failed'
+  if (
+    jobKind === HISTORICAL_JOB_KIND
+    && !historicalCompletionValid(summary, completion, jobName)
+  ) return 'failed'
   if (Number(summary.n_infrastructure_exceptions ?? summary.n_exceptions ?? 0) > 0 || Number(summary.n_evaluation_exceptions ?? 0) > 0) return 'partial'
-  if (Number(summary.n_invalid_scores ?? 0) > 0) return 'attention'
+  const invalidScores = Number(summary.n_invalid_scores ?? 0)
+  if (invalidScores > 0) return 'attention'
   return 'completed'
 }
 
 async function readJob(jobsDir, entry, details) {
   const directory = path.join(jobsDir, entry.name)
-  const [summary, contextFile, promotion, contract, lifecycle, registry, stack] = await Promise.all([
+  const [summary, contextFile, promotion, contract, lifecycle, registry, stack, completion] = await Promise.all([
     readJson(path.join(directory, SUMMARY_NAME)),
     readJson(path.join(directory, 'evaluation-context.json')),
     readJson(path.join(directory, 'promotion-report.json')),
@@ -171,25 +252,38 @@ async function readJob(jobsDir, entry, details) {
     readJson(path.join(directory, 'trial-lifecycle.json')),
     readJson(path.join(directory, 'artifact-registry.json')),
     readJson(path.join(directory, 'evaluation-stack-manifest.json')),
+    readJson(path.join(directory, HISTORICAL_COMPLETION_NAME)),
   ])
   const evaluationContext = summary?.evaluation_context ?? contextFile
   if (!evaluationContext && !summary && !lifecycle) return undefined
   const updatedAt = details.mtime.toISOString()
   const progress = progressView(summary, lifecycle, updatedAt)
+  const jobKind = normalizedJobKind(summary, evaluationContext)
   const capabilities = capabilityMap(summary, evaluationContext, lifecycle, registry, stack)
+  const evaluationTarget = summary?.evaluation_target ?? evaluationContext?.evaluation_target
+  const generationSource = summary?.generation_source ?? evaluationContext?.generation_source
+  const coverage = coverageView(summary)
   return {
     name: entry.name,
     updatedAt,
-    status: jobStatus(summary, lifecycle, progress),
+    status: jobStatus(summary, lifecycle, progress, jobKind, completion, entry.name),
+    jobKind,
     mode: summary?.mode ?? evaluationContext?.mode,
+    executionMode: summary?.execution_mode ?? evaluationContext?.execution_mode,
     nTrials: progress.total,
     nDiscoveredTrials: Number(summary?.n_discovered_trials ?? lifecycle?.attempt_count ?? 0),
     nValidScores: summary?.n_valid_scores,
     nInvalidScores: summary?.n_invalid_scores,
+    nUnscoredTrials: Number(coverage.unscored_trials ?? 0),
     nExceptions: Number(summary?.n_exceptions ?? 0),
     primaryMetric: primaryMetric(summary, contract),
     metrics: summary?.metrics ?? {},
     candidate: summary?.candidate ?? evaluationContext?.candidate,
+    evaluationTarget,
+    generationSource,
+    generatorPopulation: evaluationTarget?.generator_population,
+    coverage,
+    evaluatorMetaEvaluation: evaluatorMetaEvaluation(summary, evaluationContext),
     dataset: evaluationContext?.dataset,
     evaluationContext,
     progress,
@@ -353,6 +447,7 @@ const DETAIL_ARTIFACTS = {
   diagnosis: 'diagnosis-report.json',
   optimization: 'optimization-report.json',
   promotion: 'promotion-report.json',
+  completion: HISTORICAL_COMPLETION_NAME,
 }
 
 function schemaIssue(key, value) {
@@ -360,8 +455,8 @@ function schemaIssue(key, value) {
   if (value?.__readError) return value.__readError
   if (!isObject(value)) return 'artifact must be an object'
   const versions = {
-    summary: [2, 3], candidate: [1], dataset: [1], datasetPreview: [1], stack: [1], stackSources: [1], context: [1, 2], contract: [1],
-    doctor: [1], population: [1, 2], lifecycle: [1], registry: [1], diagnosis: [1], optimization: [1, 2], promotion: [2],
+    summary: [2, 3, 4], candidate: [1], dataset: [1], datasetPreview: [1], stack: [1], stackSources: [1], context: [1, 2], contract: [1],
+    doctor: [1], population: [1, 2, 3], lifecycle: [1], registry: [1, 2], diagnosis: [1, 2], optimization: [1, 2, 3], promotion: [2], completion: [1],
   }[key]
   if (versions && !versions.includes(value.schema_version)) return `schema_version must be one of ${versions.join(', ')}`
   const required = {
@@ -369,8 +464,12 @@ function schemaIssue(key, value) {
     stack: ['stack_id', 'version', 'digest', 'components', 'judge'], stackSources: ['stack_digest', 'components'], context: ['digest'], contract: ['contract_id', 'version', 'primary_metric', 'metrics'],
     doctor: ['promotion_ready', 'findings'], population: ['population_size', 'groups', 'metrics'], lifecycle: ['dataset_total', 'trials'],
     registry: ['artifacts'], diagnosis: ['diagnoses'], optimization: ['hypotheses'], promotion: ['decision', 'reasons', 'policy_digest'],
+    completion: ['job_kind', 'status', 'valid', 'job', 'summary_path', 'artifact_registry_path', 'coverage'],
   }[key] ?? []
-  const missing = required.filter(field => value[field] === undefined)
+  const requiredFields = key === 'population' && value.schema_version === 3
+    ? ['population_size', 'coverage', 'metrics']
+    : required
+  const missing = requiredFields.filter(field => value[field] === undefined)
   return missing.length ? `missing fields: ${missing.join(', ')}` : undefined
 }
 
@@ -390,10 +489,35 @@ export async function readJobDetail(config, args) {
     return [key, value === undefined ? { status: 'unavailable', reason: 'capability-not-produced' } : issue ? { status: 'invalid', error: issue } : { status: 'valid' }]
   }))
   const context = artifacts.context ?? values[Object.keys(DETAIL_ARTIFACTS).indexOf('context')]
+  const summary = values[Object.keys(DETAIL_ARTIFACTS).indexOf('summary')]
+  const jobKind = normalizedJobKind(summary, context)
+  if (
+    jobKind === HISTORICAL_JOB_KIND
+    && !historicalCompletionValid(summary, artifacts.completion, job)
+  ) {
+    validation.completion = {
+      status: 'invalid',
+      error: 'Historical completion sentinel is missing, stale, or inconsistent with the Summary',
+    }
+  }
   const capabilities = capabilityMap(
-    values[Object.keys(DETAIL_ARTIFACTS).indexOf('summary')], context, artifacts.lifecycle, artifacts.registry, artifacts.stack,
+    summary, context, artifacts.lifecycle, artifacts.registry, artifacts.stack,
   )
-  return { schemaVersion: 2, job, capabilities, artifacts, validation }
+  const evaluationTarget = summary?.evaluation_target ?? context?.evaluation_target
+  return {
+    schemaVersion: 3,
+    job,
+    jobKind,
+    evaluationTarget,
+    generationSource: summary?.generation_source ?? context?.generation_source,
+    generatorPopulation: evaluationTarget?.generator_population,
+    executionMode: summary?.execution_mode ?? context?.execution_mode,
+    coverage: coverageView(summary),
+    evaluatorMetaEvaluation: evaluatorMetaEvaluation(summary, context),
+    capabilities,
+    artifacts,
+    validation,
+  }
 }
 
 function selectedLifecycleTrials(lifecycle) {
@@ -408,13 +532,15 @@ function selectedLifecycleTrials(lifecycle) {
 function normalizeTrial(trial, order) {
   const score = trial.score ?? { value: undefined, valid: trial.exception ? false : true, invalid_reasons: trial.exception ? ['infrastructure-error'] : [] }
   const datasetOrder = Number(trial.datasetOrder ?? trial.dataset_order ?? order)
+  const status = trial.status ?? trial.phase ?? (trial.exception ? 'infrastructure-error' : 'completed')
   return {
     id: trial.id ?? trial.execution_id ?? `dataset-${datasetOrder}`,
     name: trial.name ?? trial.trial_name ?? trial.dataset_trial ?? trial.trial,
     datasetTrial: trial.datasetTrial ?? trial.dataset_trial ?? trial.trial,
     datasetOrder,
     attempt: Number(trial.attempt ?? 1),
-    status: trial.status ?? trial.phase ?? (trial.exception ? 'infrastructure-error' : 'completed'),
+    status,
+    scoringStatus: status === 'completed-unscored' ? 'unscored' : score.valid ? 'scored' : 'invalid',
     terminal: trial.terminal ?? true,
     updatedAt: trial.updatedAt ?? trial.updated_at,
     score,
@@ -737,6 +863,32 @@ export async function readComparison(config, args) {
   if (!baseline || baseline.__readError || !candidate || candidate.__readError) throw new Error('Both Job summaries are required')
   const baselineContext = baseline.evaluation_context ?? await readJson(path.join(jobDirectory(config, baselineJob), 'evaluation-context.json'))
   const candidateContext = candidate.evaluation_context ?? await readJson(path.join(jobDirectory(config, candidateJob), 'evaluation-context.json'))
+  const baselineKind = normalizedJobKind(baseline, baselineContext)
+  const candidateKind = normalizedJobKind(candidate, candidateContext)
+  if (baselineKind !== CANDIDATE_JOB_KIND || candidateKind !== CANDIDATE_JOB_KIND) {
+    const error = {
+      code: 'UNSUPPORTED_JOB_KIND_FOR_PROMOTION',
+      message: 'Historical Generation Evaluation Jobs are diagnostic evidence and cannot be used as a Candidate baseline, comparison, or Promotion Gate input.',
+    }
+    return {
+      schemaVersion: 1,
+      baselineJob,
+      candidateJob,
+      baselineJobKind: baselineKind,
+      candidateJobKind: candidateKind,
+      comparable: false,
+      comparabilityReasons: [error],
+      metrics: {},
+      population: {},
+      improvedTrials: [],
+      regressedTrials: [],
+      newExceptions: [],
+      artifactRegressions: [],
+      gateEligibility: 'not-applicable',
+      error,
+      note: 'Convert reviewed badcases into a fixed regression Dataset before running Candidate comparison or Gate.',
+    }
+  }
   const reasons = []
   if (baselineContext?.schema_version !== 2 || candidateContext?.schema_version !== 2) reasons.push('Context v2 is required')
   if (!baselineContext?.digest || baselineContext.digest !== candidateContext?.digest) reasons.push('Evaluation Context differs; establish a fresh baseline')

package/lib/evolution.js

@@ -66,6 +66,73 @@ export function makeJobName(manifest, now = new Date()) {
   return `${identity}-${suffix}`
 }
 
+export function makeHistoricalJobName(batch, now = new Date()) {
+  const timestamp = now.toISOString().replace(/\.\d{3}Z$/, 'Z').replace(/[-:]/g, '')
+  const digest = String(batch?.digest ?? '').replace(/^sha256:/, '').slice(0, 8) || 'historical'
+  return `dsh-session-history-${timestamp}-${digest}`
+}
+
+const HISTORICAL_COVERAGE_KEYS = [
+  'scored_trials',
+  'unscored_trials',
+  'total_trials',
+  'trial_rate',
+  'criterion_scored',
+  'criterion_total',
+  'criterion_rate',
+]
+
+function historicalCoverageMatches(summaryCoverage, completionCoverage) {
+  if (!summaryCoverage || !completionCoverage) return false
+  return HISTORICAL_COVERAGE_KEYS.every(key => (
+    typeof summaryCoverage[key] === 'number'
+    && summaryCoverage[key] === completionCoverage[key]
+  ))
+}
+
+/**
+ * Harbor 0.21 treats Job-plugin finalization failures as warnings, so a zero
+ * process exit is not proof that Historical artifacts completed. Validate the
+ * fresh plugin-owned sentinel and its core Summary cross-links fail-closed.
+ */
+export function assertHistoricalCompletion(summary, completion, {
+  jobName,
+  batchDigest,
+  batchId,
+  recordCount,
+}) {
+  const summaryValid = (
+    summary?.schema_version === 4
+    && summary?.job === jobName
+    && summary?.job_kind === 'historical-generation-evaluation'
+    && summary?.mode === 'diagnostic'
+    && summary?.execution_mode === 'observe-existing'
+    && summary?.evaluation_target?.digest === batchDigest
+    && summary?.evaluation_target?.batch_id === batchId
+    && summary?.evaluation_target?.record_count === recordCount
+    && summary?.n_trials === recordCount
+    && summary?.coverage?.total_trials === recordCount
+    && summary?.artifact_validation?.valid === true
+    && summary?.candidate === undefined
+  )
+  if (!summaryValid) {
+    throw new Error('HISTORICAL_JOB_IDENTITY_INVALID: the Summary is not a validated, Candidate-free Historical Generation Evaluation for this Batch')
+  }
+  const completionValid = (
+    completion?.schema_version === 1
+    && completion?.job_kind === 'historical-generation-evaluation'
+    && completion?.status === 'completed'
+    && completion?.valid === true
+    && completion?.job === jobName
+    && completion?.summary_path === 'evaluation-summary.json'
+    && completion?.artifact_registry_path === 'artifact-registry.json'
+    && historicalCoverageMatches(summary.coverage, completion.coverage)
+  )
+  if (!completionValid) {
+    throw new Error('HISTORICAL_JOB_ARTIFACT_VALIDATION_FAILED: the completion sentinel is missing, stale, or inconsistent with the validated Summary')
+  }
+}
+
 async function cliJson(config, args, { allowedExitCodes = [0], input } = {}) {
   let result
   try {
@@ -374,6 +441,128 @@ export async function runEvaluation(config, args, modelRuntime) {
   }
 }
 
+/**
+ * Materialize an immutable Historical Generation Batch before Harbor creates
+ * its Job, then run a deterministic Observation Adapter. This path never
+ * snapshots or executes a Candidate.
+ */
+export async function runHistoricalEvaluation(config, args, modelRuntime) {
+  const projectRoot = path.resolve(config.projectRoot)
+  const batchPath = resolveWithin(projectRoot, args.batchPath, 'batchPath')
+  const batchDir = resolveWithin(projectRoot, args.batchDir ?? path.dirname(batchPath), 'batchDir')
+  const output = resolveWithin(projectRoot, path.join(batchDir, 'dataset'), 'historicalDataset')
+  if (!args.judgeBinding?.provider || !args.judgeBinding?.model) {
+    throw new Error('Historical Judge model binding is required before materialization')
+  }
+  const materialized = await cliJson(config, [
+    'historical', 'materialize',
+    '--project-root', projectRoot,
+    '--batch', batchPath,
+    '--output', output,
+    '--judge-provider', args.judgeBinding.provider,
+    '--judge-model', args.judgeBinding.model,
+    ...(args.judgeBinding.reasoning_effort === undefined
+      ? []
+      : ['--judge-reasoning-effort', args.judgeBinding.reasoning_effort]),
+  ])
+  const dataset = resolveWithin(projectRoot, materialized.dataset_path ?? output, 'historicalDataset')
+  const stack = resolveWithin(projectRoot, materialized.stack_path, 'historicalStack')
+  const batch = JSON.parse(await readFile(batchPath, 'utf8'))
+  const jobs = resolveWithin(projectRoot, config.jobsDir, 'jobsDir')
+  const jobName = args.jobName ?? makeHistoricalJobName(batch)
+  if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$/.test(jobName)) {
+    throw new Error('jobName contains unsupported characters')
+  }
+  const agentImportPath = config.historicalAgentImportPath
+    ?? 'harbor_dsh_evolution.session_agent:SessionObservationAgent'
+  const pluginImportPath = config.historicalPluginImportPath
+    ?? 'dsh-historical-evaluation'
+  const harborArgs = [
+    'run', '-y', '-p', dataset,
+    '-a', agentImportPath,
+    '--job-name', jobName,
+    '--jobs-dir', jobs,
+    '--plugin', pluginImportPath,
+    '--plugin-kwarg', `batch_path=${batchPath}`,
+    '--plugin-kwarg', `dataset_path=${dataset}`,
+    '--plugin-kwarg', `stack_path=${stack}`,
+    '--plugin-kwarg', `project_root=${projectRoot}`,
+    '--plugin-kwarg', 'mode=diagnostic',
+  ]
+  const lease = await modelRuntime.openLease(args.judgeBinding, {
+    candidateDigest: batch.digest,
+    jobName,
+  })
+  const jobDir = path.join(jobs, jobName)
+  try {
+    let processResult
+    try {
+      processResult = await runProcess(config.harborBin, harborArgs, {
+        cwd: projectRoot,
+        timeoutMs: config.timeoutMs,
+        env: {
+          ...process.env,
+          ...(config.pythonPath ? { PYTHONPATH: config.pythonPath } : {}),
+          HSE_JUDGE_GATEWAY_URL: lease.endpoint,
+          HSE_JUDGE_GATEWAY_TOKEN: lease.token,
+          HSE_JUDGE_GATEWAY_PROVIDER: lease.candidateProvider,
+          HSE_JUDGE_GATEWAY_INFO: JSON.stringify({
+            protocol: lease.protocol,
+            candidate_digest: batch.digest,
+            job: jobName,
+            binding: {
+              provider: args.judgeBinding.provider,
+              model: args.judgeBinding.model,
+              ...(args.judgeBinding.reasoning_effort === undefined
+                ? {}
+                : { reasoning_effort: args.judgeBinding.reasoning_effort }),
+            },
+            model_info: lease.modelInfo,
+          }),
+          HSE_JUDGE_GATEWAY_PROTOCOL: lease.protocol,
+        },
+      })
+    } catch (error) {
+      throw await explainHarborFailure(error, jobDir)
+    }
+    let summary
+    let completion
+    try {
+      summary = JSON.parse(await readFile(path.join(jobDir, 'evaluation-summary.json'), 'utf8'))
+      completion = JSON.parse(await readFile(path.join(jobDir, 'historical-evaluation-complete.json'), 'utf8'))
+    } catch (error) {
+      throw new Error(
+        `HISTORICAL_JOB_INCOMPLETE: Harbor exited successfully but the Historical plugin did not write its summary/completion sentinel (${error.message})`,
+      )
+    }
+    assertHistoricalCompletion(summary, completion, {
+      jobName,
+      batchDigest: batch.digest,
+      batchId: batch.batch_id,
+      recordCount: Array.isArray(batch.records) ? batch.records.length : undefined,
+    })
+    return {
+      judgeModelBinding: {
+        provider: args.judgeBinding.provider,
+        model: args.judgeBinding.model,
+        ...(args.judgeBinding.reasoning_effort === undefined
+          ? {}
+          : { reasoning_effort: args.judgeBinding.reasoning_effort }),
+      },
+      job: path.relative(projectRoot, jobDir).split(path.sep).join('/'),
+      summary,
+      completion,
+      materialized: {
+        dataset: path.relative(projectRoot, dataset).split(path.sep).join('/'),
+        stack: path.relative(projectRoot, stack).split(path.sep).join('/'),
+      },
+      process: { code: processResult.code },
+    }
+  } finally {
+    await lease.close()
+  }
+}
+
 export async function readEvaluation(config, args) {
   const jobDir = resolveWithin(config.projectRoot, args.jobPath, 'jobPath')
   return JSON.parse(await readFile(path.join(jobDir, 'evaluation-summary.json'), 'utf8'))

package/lib/model-runtime.js

@@ -74,13 +74,17 @@ export class CandidateModelRuntime {
     this.config = config
   }
 
-  async currentBinding() {
+  async resolveCurrent() {
     const inherited = this.ctx.agentDefaultModel.currentSelection()
-    const binding = await this.resolve({
+    return this.resolve({
       candidateProvider: inherited.provider,
       candidateModel: inherited.model,
       candidateReasoningEffort: inherited.reasoningEffort,
-    })
+    }, undefined, { ignoreConfigured: true })
+  }
+
+  async currentBinding() {
+    const binding = await this.resolveCurrent()
     return {
       schema_version: 1,
       source: 'skill-agent-default',
@@ -92,14 +96,14 @@ export class CandidateModelRuntime {
     }
   }
 
-  async resolve(args = {}, pinnedBinding) {
+  async resolve(args = {}, pinnedBinding, { ignoreConfigured = false } = {}) {
     const explicitProvider = nonBlank(args.candidateProvider)
     const explicitModel = nonBlank(args.candidateModel)
     if (Boolean(explicitProvider) !== Boolean(explicitModel)) {
       throw new Error('candidateProvider and candidateModel must be supplied together')
     }
-    const configuredProvider = nonBlank(this.config.candidateProvider)
-    const configuredModel = nonBlank(this.config.candidateModel)
+    const configuredProvider = ignoreConfigured ? undefined : nonBlank(this.config.candidateProvider)
+    const configuredModel = ignoreConfigured ? undefined : nonBlank(this.config.candidateModel)
     if (Boolean(configuredProvider) !== Boolean(configuredModel)) {
       throw new Error('Harbor candidateProvider and candidateModel configuration must be supplied together')
     }
@@ -129,7 +133,7 @@ export class CandidateModelRuntime {
     const provider = pinnedProvider ?? explicitProvider ?? configuredProvider ?? inherited.provider
     const model = pinnedModel ?? explicitModel ?? configuredModel ?? inherited.model
     const explicitReasoning = nonBlank(args.candidateReasoningEffort)
-    const configuredReasoning = nonBlank(this.config.candidateReasoningEffort)
+    const configuredReasoning = ignoreConfigured ? undefined : nonBlank(this.config.candidateReasoningEffort)
     const canInheritReasoning = provider === inherited.provider && model === inherited.model
     const reasoningEffort = pinnedProvider
       ? pinnedReasoning