dsh-harbor-evolution 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -4
- package/index.js +72 -6
- package/lib/candidate.js +58 -5
- package/lib/client.js +283 -63
- package/lib/dashboard.js +307 -34
- package/lib/evolution.js +328 -20
- package/lib/model-runtime.js +53 -8
- package/lib/runtime-identity.js +7 -0
- package/lib/service.js +187 -33
- package/lib/session-diagnostic.js +320 -0
- package/lib/session-materializer.js +194 -0
- package/lib/session-projection.js +161 -0
- package/lib/session-redaction.js +311 -0
- package/lib/session-selection.js +294 -0
- package/lib/setup.js +11 -5
- package/lib/version.js +128 -0
- package/lib/web.js +5 -1
- package/package.json +13 -3
- package/schemas/dsh-session-observation.schema.json +69 -0
- package/schemas/evaluation-result-v2.schema.json +45 -0
- package/schemas/historical-evaluation-context.schema.json +66 -0
- package/schemas/historical-evaluation-summary.schema.json +49 -0
- package/schemas/historical-generation-batch.schema.json +76 -0
- package/skills/evolve-agent-with-harbor/SKILL.md +127 -13
- package/skills/evolve-agent-with-harbor/evals/evals.json +57 -9
- package/skills/evolve-agent-with-harbor/references/evaluator-upgrade.md +34 -1
- package/skills/evolve-agent-with-harbor/references/initialization.md +9 -2
package/lib/evolution.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { readFile } from 'node:fs/promises'
|
|
1
|
+
import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'
|
|
2
2
|
import path from 'node:path'
|
|
3
3
|
|
|
4
4
|
import { MANIFEST_NAME, snapshotCandidate } from './candidate.js'
|
|
@@ -8,17 +8,50 @@ export function resolveWithin(root, value, label) {
|
|
|
8
8
|
const base = path.resolve(root)
|
|
9
9
|
const resolved = path.resolve(base, value)
|
|
10
10
|
if (resolved !== base && !resolved.startsWith(`${base}${path.sep}`)) {
|
|
11
|
-
throw new Error(
|
|
11
|
+
throw new Error(
|
|
12
|
+
`PATH_OUTSIDE_PROJECT_ROOT: ${label} must stay under projectRoot.\n` +
|
|
13
|
+
`projectRoot: ${base}\n` +
|
|
14
|
+
`${label}: ${value}\n` +
|
|
15
|
+
'Recommended fix: use a path inside the current Agent session directory, or open the intended project as the session working directory. The Web Workbench projectRoot can be switched and reloaded in Harbor settings.',
|
|
16
|
+
)
|
|
12
17
|
}
|
|
13
18
|
return resolved
|
|
14
19
|
}
|
|
15
20
|
|
|
21
|
+
const META_ARTIFACT_INDEX = '.harbor/meta-artifacts.json'
|
|
22
|
+
|
|
23
|
+
function inferEvaluationRoot(projectRoot, artifactPath, explicitRoot) {
|
|
24
|
+
if (explicitRoot) return resolveWithin(projectRoot, explicitRoot, 'evaluationRoot')
|
|
25
|
+
const relative = path.relative(projectRoot, artifactPath)
|
|
26
|
+
const parts = relative.split(path.sep)
|
|
27
|
+
const marker = parts.lastIndexOf('.harbor')
|
|
28
|
+
return marker >= 0 ? path.resolve(projectRoot, ...parts.slice(0, marker)) : path.resolve(projectRoot)
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
async function recordMetaArtifact(config, artifactPath, key, explicitRoot) {
|
|
32
|
+
const evaluationRoot = inferEvaluationRoot(config.projectRoot, artifactPath, explicitRoot)
|
|
33
|
+
const registeredArtifact = resolveWithin(evaluationRoot, artifactPath, key)
|
|
34
|
+
const indexPath = resolveWithin(evaluationRoot, META_ARTIFACT_INDEX, 'metaArtifactIndex')
|
|
35
|
+
let current = { schema_version: 1, artifacts: {} }
|
|
36
|
+
try {
|
|
37
|
+
const parsed = JSON.parse(await readFile(indexPath, 'utf8'))
|
|
38
|
+
if (parsed?.schema_version === 1 && parsed.artifacts && typeof parsed.artifacts === 'object') current = parsed
|
|
39
|
+
} catch (error) {
|
|
40
|
+
if (error.code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error
|
|
41
|
+
}
|
|
42
|
+
current.artifacts[key] = path.relative(evaluationRoot, registeredArtifact).split(path.sep).join('/')
|
|
43
|
+
await mkdir(path.dirname(indexPath), { recursive: true })
|
|
44
|
+
const temporary = `${indexPath}.${process.pid}.tmp`
|
|
45
|
+
await writeFile(temporary, `${JSON.stringify(current, null, 2)}\n`, 'utf8')
|
|
46
|
+
await rename(temporary, indexPath)
|
|
47
|
+
return path.relative(config.projectRoot, indexPath).split(path.sep).join('/')
|
|
48
|
+
}
|
|
49
|
+
|
|
16
50
|
export async function snapshot(config, args) {
|
|
17
51
|
const candidateDir = resolveWithin(config.projectRoot, args.candidatePath, 'candidatePath')
|
|
18
52
|
return snapshotCandidate(candidateDir, {
|
|
19
53
|
candidateId: args.candidateId,
|
|
20
54
|
version: args.version,
|
|
21
|
-
runtimeVersion: config.dshVersion,
|
|
22
55
|
})
|
|
23
56
|
}
|
|
24
57
|
|
|
@@ -33,6 +66,73 @@ export function makeJobName(manifest, now = new Date()) {
|
|
|
33
66
|
return `${identity}-${suffix}`
|
|
34
67
|
}
|
|
35
68
|
|
|
69
|
+
export function makeHistoricalJobName(batch, now = new Date()) {
|
|
70
|
+
const timestamp = now.toISOString().replace(/\.\d{3}Z$/, 'Z').replace(/[-:]/g, '')
|
|
71
|
+
const digest = String(batch?.digest ?? '').replace(/^sha256:/, '').slice(0, 8) || 'historical'
|
|
72
|
+
return `dsh-session-history-${timestamp}-${digest}`
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const HISTORICAL_COVERAGE_KEYS = [
|
|
76
|
+
'scored_trials',
|
|
77
|
+
'unscored_trials',
|
|
78
|
+
'total_trials',
|
|
79
|
+
'trial_rate',
|
|
80
|
+
'criterion_scored',
|
|
81
|
+
'criterion_total',
|
|
82
|
+
'criterion_rate',
|
|
83
|
+
]
|
|
84
|
+
|
|
85
|
+
function historicalCoverageMatches(summaryCoverage, completionCoverage) {
|
|
86
|
+
if (!summaryCoverage || !completionCoverage) return false
|
|
87
|
+
return HISTORICAL_COVERAGE_KEYS.every(key => (
|
|
88
|
+
typeof summaryCoverage[key] === 'number'
|
|
89
|
+
&& summaryCoverage[key] === completionCoverage[key]
|
|
90
|
+
))
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Harbor 0.21 treats Job-plugin finalization failures as warnings, so a zero
|
|
95
|
+
* process exit is not proof that Historical artifacts completed. Validate the
|
|
96
|
+
* fresh plugin-owned sentinel and its core Summary cross-links fail-closed.
|
|
97
|
+
*/
|
|
98
|
+
export function assertHistoricalCompletion(summary, completion, {
|
|
99
|
+
jobName,
|
|
100
|
+
batchDigest,
|
|
101
|
+
batchId,
|
|
102
|
+
recordCount,
|
|
103
|
+
}) {
|
|
104
|
+
const summaryValid = (
|
|
105
|
+
summary?.schema_version === 4
|
|
106
|
+
&& summary?.job === jobName
|
|
107
|
+
&& summary?.job_kind === 'historical-generation-evaluation'
|
|
108
|
+
&& summary?.mode === 'diagnostic'
|
|
109
|
+
&& summary?.execution_mode === 'observe-existing'
|
|
110
|
+
&& summary?.evaluation_target?.digest === batchDigest
|
|
111
|
+
&& summary?.evaluation_target?.batch_id === batchId
|
|
112
|
+
&& summary?.evaluation_target?.record_count === recordCount
|
|
113
|
+
&& summary?.n_trials === recordCount
|
|
114
|
+
&& summary?.coverage?.total_trials === recordCount
|
|
115
|
+
&& summary?.artifact_validation?.valid === true
|
|
116
|
+
&& summary?.candidate === undefined
|
|
117
|
+
)
|
|
118
|
+
if (!summaryValid) {
|
|
119
|
+
throw new Error('HISTORICAL_JOB_IDENTITY_INVALID: the Summary is not a validated, Candidate-free Historical Generation Evaluation for this Batch')
|
|
120
|
+
}
|
|
121
|
+
const completionValid = (
|
|
122
|
+
completion?.schema_version === 1
|
|
123
|
+
&& completion?.job_kind === 'historical-generation-evaluation'
|
|
124
|
+
&& completion?.status === 'completed'
|
|
125
|
+
&& completion?.valid === true
|
|
126
|
+
&& completion?.job === jobName
|
|
127
|
+
&& completion?.summary_path === 'evaluation-summary.json'
|
|
128
|
+
&& completion?.artifact_registry_path === 'artifact-registry.json'
|
|
129
|
+
&& historicalCoverageMatches(summary.coverage, completion.coverage)
|
|
130
|
+
)
|
|
131
|
+
if (!completionValid) {
|
|
132
|
+
throw new Error('HISTORICAL_JOB_ARTIFACT_VALIDATION_FAILED: the completion sentinel is missing, stale, or inconsistent with the validated Summary')
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
36
136
|
async function cliJson(config, args, { allowedExitCodes = [0], input } = {}) {
|
|
37
137
|
let result
|
|
38
138
|
try {
|
|
@@ -80,7 +180,7 @@ export async function updateEvaluator(config, args) {
|
|
|
80
180
|
|
|
81
181
|
export async function initializeGroundTruth(config, args) {
|
|
82
182
|
const output = resolveWithin(config.projectRoot, args.outputPath ?? '.harbor/ground-truth.json', 'outputPath')
|
|
83
|
-
|
|
183
|
+
const result = await cliJson(config, [
|
|
84
184
|
'ground-truth', 'init',
|
|
85
185
|
'--project-root', config.projectRoot,
|
|
86
186
|
'--output', output,
|
|
@@ -91,19 +191,25 @@ export async function initializeGroundTruth(config, args) {
|
|
|
91
191
|
'--provenance', String(args.provenance ?? ''),
|
|
92
192
|
'--criteria', String(args.criteria ?? ''),
|
|
93
193
|
])
|
|
194
|
+
result.artifact_index = await recordMetaArtifact(config, output, 'ground_truth', args.evaluationRoot)
|
|
195
|
+
return result
|
|
94
196
|
}
|
|
95
197
|
|
|
96
198
|
export async function runMetaEvaluation(config, args) {
|
|
97
199
|
const groundTruth = resolveWithin(config.projectRoot, args.groundTruthPath ?? '.harbor/ground-truth.json', 'groundTruthPath')
|
|
98
200
|
const observations = resolveWithin(config.projectRoot, args.observationsPath, 'observationsPath')
|
|
99
201
|
const output = resolveWithin(config.projectRoot, args.outputPath ?? '.harbor/meta-evaluation-report.json', 'outputPath')
|
|
100
|
-
|
|
202
|
+
const result = await cliJson(config, [
|
|
101
203
|
'meta-evaluate',
|
|
102
204
|
'--project-root', config.projectRoot,
|
|
103
205
|
'--ground-truth', groundTruth,
|
|
104
206
|
'--observations', observations,
|
|
105
207
|
'--output', output,
|
|
106
208
|
])
|
|
209
|
+
const evaluationRoot = args.evaluationRoot
|
|
210
|
+
?? inferEvaluationRoot(config.projectRoot, groundTruth)
|
|
211
|
+
result.artifact_index = await recordMetaArtifact(config, output, 'meta_evaluation_report', evaluationRoot)
|
|
212
|
+
return result
|
|
107
213
|
}
|
|
108
214
|
|
|
109
215
|
function strictInputs(config, args) {
|
|
@@ -132,6 +238,63 @@ function candidateModelCliArgs(binding) {
|
|
|
132
238
|
]
|
|
133
239
|
}
|
|
134
240
|
|
|
241
|
+
export function redactDiagnostic(value) {
|
|
242
|
+
return String(value ?? '')
|
|
243
|
+
.replace(/(authorization\s*[:=]\s*(?:bearer\s+)?)[^\s,'"}]+/gi, '$1[redacted]')
|
|
244
|
+
.replace(/(bearer\s+)[A-Za-z0-9._~+\/-]+/gi, '$1[redacted]')
|
|
245
|
+
.replace(/((?:api[_-]?key|token|secret|password)\s*[:=]\s*)[^\s,'"}]+/gi, '$1[redacted]')
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
export function classifyHarborFailure(value) {
|
|
249
|
+
const text = String(value ?? '')
|
|
250
|
+
const suggestions = []
|
|
251
|
+
if (/AgentSetupTimeoutError|agent setup.{0,30}time(?:d out|out)/i.test(text)) {
|
|
252
|
+
suggestions.push({ code: 'AGENT_SETUP_TIMEOUT', action: 'Use a base image with Python, curl, Node.js, npm, and ACP/DSH dependencies already installed; then rerun Doctor.' })
|
|
253
|
+
}
|
|
254
|
+
if (/evaluation-result\.json is missing/i.test(text)) {
|
|
255
|
+
suggestions.push({ code: 'EVALUATOR_RESULT_MISSING', action: 'Update tests/test.sh or its evaluator script to write /logs/verifier/evaluation-result.json using evaluation-result/v1.' })
|
|
256
|
+
}
|
|
257
|
+
if (/Either datasets or tasks must be provided|HARBOR_RUNTIME_NO_TASKS/i.test(text)) {
|
|
258
|
+
suggestions.push({ code: 'DATASET_NOT_RESOLVED', action: 'Make the Dataset root contain immediate Task subdirectories with schema_version = "1.4", [task] name = "org/name", instruction.md, environment/, and tests/test.sh.' })
|
|
259
|
+
}
|
|
260
|
+
if (/docker-credential-|credential helper/i.test(text)) {
|
|
261
|
+
suggestions.push({ code: 'DOCKER_CREDENTIAL_HELPER', action: 'Repair the configured Docker credential helper or use a verified image already present in the local Docker daemon.' })
|
|
262
|
+
}
|
|
263
|
+
if (/Cannot connect to the Docker daemon|DOCKER_DAEMON_UNAVAILABLE/i.test(text)) {
|
|
264
|
+
suggestions.push({ code: 'DOCKER_DAEMON_UNAVAILABLE', action: 'Start Docker and confirm `docker version` can reach the server.' })
|
|
265
|
+
}
|
|
266
|
+
if (!suggestions.length) {
|
|
267
|
+
suggestions.push({ code: 'INSPECT_JOB_LOG', action: 'Review the stderr/job.log excerpt below, fix the first causal error, rerun Doctor, then retry the Job.' })
|
|
268
|
+
}
|
|
269
|
+
return suggestions
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
async function optionalTail(pathname, maxChars = 6000) {
|
|
273
|
+
try {
|
|
274
|
+
const value = await readFile(pathname, 'utf8')
|
|
275
|
+
return value.slice(-maxChars)
|
|
276
|
+
} catch {
|
|
277
|
+
return ''
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
export async function explainHarborFailure(error, jobDir) {
|
|
282
|
+
const stderr = error?.result?.stderr ?? ''
|
|
283
|
+
const stdout = error?.result?.stdout ?? ''
|
|
284
|
+
const logCandidates = ['job.log', 'harbor.log']
|
|
285
|
+
const logParts = (await Promise.all(logCandidates.map(name => optionalTail(path.join(jobDir, name)))))
|
|
286
|
+
.filter(Boolean)
|
|
287
|
+
const detail = redactDiagnostic([stderr.slice(-8000), ...logParts, stdout.slice(-2000)].filter(Boolean).join('\n'))
|
|
288
|
+
const suggestions = classifyHarborFailure(detail || error?.message)
|
|
289
|
+
const lines = [
|
|
290
|
+
`HARBOR_JOB_FAILED: Harbor exited with code ${error?.result?.code ?? 'unknown'}.`,
|
|
291
|
+
`jobPath: ${jobDir}`,
|
|
292
|
+
...suggestions.map(item => `nextStep[${item.code}]: ${item.action}`),
|
|
293
|
+
]
|
|
294
|
+
if (detail.trim()) lines.push('diagnosticTail:', detail.trim())
|
|
295
|
+
return new Error(lines.join('\n'))
|
|
296
|
+
}
|
|
297
|
+
|
|
135
298
|
export async function validateDataset(config, args) {
|
|
136
299
|
const dataset = resolveWithin(config.projectRoot, args.datasetPath, 'datasetPath')
|
|
137
300
|
return cliJson(config, ['dataset', 'validate', dataset, '--project-root', config.projectRoot], { allowedExitCodes: [0, 2] })
|
|
@@ -148,12 +311,23 @@ export async function initializeProject(config, args) {
|
|
|
148
311
|
'--judge-provider', args.judgeProvider, '--judge-model', args.judgeModel, '--judge-version', args.judgeVersion,
|
|
149
312
|
'--policy-id', args.policyId, '--policy-version', args.policyVersion,
|
|
150
313
|
'--min-improvement', String(args.minImprovement),
|
|
314
|
+
'--workspace-subdir', String(args.workspaceSubdir ?? '.'),
|
|
315
|
+
])
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
export async function initializeQuickDiagnostic(config, args) {
|
|
319
|
+
return cliJson(config, [
|
|
320
|
+
'quick', 'diagnostic',
|
|
321
|
+
'--project-root', config.projectRoot,
|
|
322
|
+
'--query', String(args.query ?? ''),
|
|
323
|
+
'--rubric', String(args.rubric ?? ''),
|
|
324
|
+
'--workspace-subdir', String(args.workspaceSubdir ?? 'harbor-diagnostic'),
|
|
151
325
|
])
|
|
152
326
|
}
|
|
153
327
|
|
|
154
328
|
export async function runDoctor(config, args) {
|
|
155
329
|
const inputs = strictInputs(config, { ...args, mode: args.mode ?? 'diagnostic' })
|
|
156
|
-
const command = ['doctor', '--architecture', '--project-root', inputs.projectRoot, '--stack', inputs.stack, '--dataset', inputs.dataset]
|
|
330
|
+
const command = ['doctor', '--architecture', '--runtime', '--project-root', inputs.projectRoot, '--stack', inputs.stack, '--dataset', inputs.dataset]
|
|
157
331
|
if (args.candidatePath) command.push('--candidate', inputs.candidate)
|
|
158
332
|
if (inputs.policy) command.push('--policy', inputs.policy)
|
|
159
333
|
return cliJson(config, command, { allowedExitCodes: [0, 2] })
|
|
@@ -180,9 +354,16 @@ export async function runEvaluation(config, args, modelRuntime) {
|
|
|
180
354
|
const inputs = strictInputs(config, args)
|
|
181
355
|
const datasetValidation = await validateDataset(config, args)
|
|
182
356
|
if (!datasetValidation.valid) {
|
|
183
|
-
throw new Error(
|
|
357
|
+
throw new Error(
|
|
358
|
+
`Dataset validation failed under projectRoot ${inputs.projectRoot}:\n` +
|
|
359
|
+
datasetValidation.findings.map(item => `${item.code}: ${item.message}`).join('\n'),
|
|
360
|
+
)
|
|
184
361
|
}
|
|
185
362
|
const doctor = await runDoctor(config, args)
|
|
363
|
+
const runtimeBlockers = doctor.findings.filter(item => item.level === 'error' && item.code.startsWith('DOCKER_'))
|
|
364
|
+
if (runtimeBlockers.length) {
|
|
365
|
+
throw new Error(`Runtime Doctor blocked Harbor Job:\n${runtimeBlockers.map(item => `${item.code}: ${item.message}`).join('\n')}`)
|
|
366
|
+
}
|
|
186
367
|
if (inputs.mode === 'promotion-eligible' && !doctor.promotion_ready) {
|
|
187
368
|
throw new Error(`Architecture Doctor blocked promotion-eligible Job: ${doctor.findings.filter(item => item.level === 'error').map(item => item.code).join(', ')}`)
|
|
188
369
|
}
|
|
@@ -226,20 +407,25 @@ export async function runEvaluation(config, args, modelRuntime) {
|
|
|
226
407
|
jobName,
|
|
227
408
|
})
|
|
228
409
|
try {
|
|
229
|
-
const processResult = await runProcess(config.harborBin, harborArgs, {
|
|
230
|
-
cwd: config.projectRoot,
|
|
231
|
-
timeoutMs: config.timeoutMs,
|
|
232
|
-
env: {
|
|
233
|
-
...process.env,
|
|
234
|
-
...(config.pythonPath ? { PYTHONPATH: config.pythonPath } : {}),
|
|
235
|
-
HSE_MODEL_GATEWAY_URL: lease.endpoint,
|
|
236
|
-
HSE_MODEL_GATEWAY_TOKEN: lease.token,
|
|
237
|
-
HSE_MODEL_GATEWAY_PROVIDER: lease.candidateProvider,
|
|
238
|
-
HSE_MODEL_GATEWAY_INFO: JSON.stringify(lease.modelInfo),
|
|
239
|
-
HSE_MODEL_GATEWAY_PROTOCOL: lease.protocol,
|
|
240
|
-
},
|
|
241
|
-
})
|
|
242
410
|
const jobDir = path.join(inputs.jobs, jobName)
|
|
411
|
+
let processResult
|
|
412
|
+
try {
|
|
413
|
+
processResult = await runProcess(config.harborBin, harborArgs, {
|
|
414
|
+
cwd: config.projectRoot,
|
|
415
|
+
timeoutMs: config.timeoutMs,
|
|
416
|
+
env: {
|
|
417
|
+
...process.env,
|
|
418
|
+
...(config.pythonPath ? { PYTHONPATH: config.pythonPath } : {}),
|
|
419
|
+
HSE_MODEL_GATEWAY_URL: lease.endpoint,
|
|
420
|
+
HSE_MODEL_GATEWAY_TOKEN: lease.token,
|
|
421
|
+
HSE_MODEL_GATEWAY_PROVIDER: lease.candidateProvider,
|
|
422
|
+
HSE_MODEL_GATEWAY_INFO: JSON.stringify(lease.modelInfo),
|
|
423
|
+
HSE_MODEL_GATEWAY_PROTOCOL: lease.protocol,
|
|
424
|
+
},
|
|
425
|
+
})
|
|
426
|
+
} catch (error) {
|
|
427
|
+
throw await explainHarborFailure(error, jobDir)
|
|
428
|
+
}
|
|
243
429
|
const summary = JSON.parse(await readFile(path.join(jobDir, 'evaluation-summary.json'), 'utf8'))
|
|
244
430
|
return {
|
|
245
431
|
manifest,
|
|
@@ -255,6 +441,128 @@ export async function runEvaluation(config, args, modelRuntime) {
|
|
|
255
441
|
}
|
|
256
442
|
}
|
|
257
443
|
|
|
444
|
+
/**
|
|
445
|
+
* Materialize an immutable Historical Generation Batch before Harbor creates
|
|
446
|
+
* its Job, then run a deterministic Observation Adapter. This path never
|
|
447
|
+
* snapshots or executes a Candidate.
|
|
448
|
+
*/
|
|
449
|
+
export async function runHistoricalEvaluation(config, args, modelRuntime) {
|
|
450
|
+
const projectRoot = path.resolve(config.projectRoot)
|
|
451
|
+
const batchPath = resolveWithin(projectRoot, args.batchPath, 'batchPath')
|
|
452
|
+
const batchDir = resolveWithin(projectRoot, args.batchDir ?? path.dirname(batchPath), 'batchDir')
|
|
453
|
+
const output = resolveWithin(projectRoot, path.join(batchDir, 'dataset'), 'historicalDataset')
|
|
454
|
+
if (!args.judgeBinding?.provider || !args.judgeBinding?.model) {
|
|
455
|
+
throw new Error('Historical Judge model binding is required before materialization')
|
|
456
|
+
}
|
|
457
|
+
const materialized = await cliJson(config, [
|
|
458
|
+
'historical', 'materialize',
|
|
459
|
+
'--project-root', projectRoot,
|
|
460
|
+
'--batch', batchPath,
|
|
461
|
+
'--output', output,
|
|
462
|
+
'--judge-provider', args.judgeBinding.provider,
|
|
463
|
+
'--judge-model', args.judgeBinding.model,
|
|
464
|
+
...(args.judgeBinding.reasoning_effort === undefined
|
|
465
|
+
? []
|
|
466
|
+
: ['--judge-reasoning-effort', args.judgeBinding.reasoning_effort]),
|
|
467
|
+
])
|
|
468
|
+
const dataset = resolveWithin(projectRoot, materialized.dataset_path ?? output, 'historicalDataset')
|
|
469
|
+
const stack = resolveWithin(projectRoot, materialized.stack_path, 'historicalStack')
|
|
470
|
+
const batch = JSON.parse(await readFile(batchPath, 'utf8'))
|
|
471
|
+
const jobs = resolveWithin(projectRoot, config.jobsDir, 'jobsDir')
|
|
472
|
+
const jobName = args.jobName ?? makeHistoricalJobName(batch)
|
|
473
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$/.test(jobName)) {
|
|
474
|
+
throw new Error('jobName contains unsupported characters')
|
|
475
|
+
}
|
|
476
|
+
const agentImportPath = config.historicalAgentImportPath
|
|
477
|
+
?? 'harbor_dsh_evolution.session_agent:SessionObservationAgent'
|
|
478
|
+
const pluginImportPath = config.historicalPluginImportPath
|
|
479
|
+
?? 'dsh-historical-evaluation'
|
|
480
|
+
const harborArgs = [
|
|
481
|
+
'run', '-y', '-p', dataset,
|
|
482
|
+
'-a', agentImportPath,
|
|
483
|
+
'--job-name', jobName,
|
|
484
|
+
'--jobs-dir', jobs,
|
|
485
|
+
'--plugin', pluginImportPath,
|
|
486
|
+
'--plugin-kwarg', `batch_path=${batchPath}`,
|
|
487
|
+
'--plugin-kwarg', `dataset_path=${dataset}`,
|
|
488
|
+
'--plugin-kwarg', `stack_path=${stack}`,
|
|
489
|
+
'--plugin-kwarg', `project_root=${projectRoot}`,
|
|
490
|
+
'--plugin-kwarg', 'mode=diagnostic',
|
|
491
|
+
]
|
|
492
|
+
const lease = await modelRuntime.openLease(args.judgeBinding, {
|
|
493
|
+
candidateDigest: batch.digest,
|
|
494
|
+
jobName,
|
|
495
|
+
})
|
|
496
|
+
const jobDir = path.join(jobs, jobName)
|
|
497
|
+
try {
|
|
498
|
+
let processResult
|
|
499
|
+
try {
|
|
500
|
+
processResult = await runProcess(config.harborBin, harborArgs, {
|
|
501
|
+
cwd: projectRoot,
|
|
502
|
+
timeoutMs: config.timeoutMs,
|
|
503
|
+
env: {
|
|
504
|
+
...process.env,
|
|
505
|
+
...(config.pythonPath ? { PYTHONPATH: config.pythonPath } : {}),
|
|
506
|
+
HSE_JUDGE_GATEWAY_URL: lease.endpoint,
|
|
507
|
+
HSE_JUDGE_GATEWAY_TOKEN: lease.token,
|
|
508
|
+
HSE_JUDGE_GATEWAY_PROVIDER: lease.candidateProvider,
|
|
509
|
+
HSE_JUDGE_GATEWAY_INFO: JSON.stringify({
|
|
510
|
+
protocol: lease.protocol,
|
|
511
|
+
candidate_digest: batch.digest,
|
|
512
|
+
job: jobName,
|
|
513
|
+
binding: {
|
|
514
|
+
provider: args.judgeBinding.provider,
|
|
515
|
+
model: args.judgeBinding.model,
|
|
516
|
+
...(args.judgeBinding.reasoning_effort === undefined
|
|
517
|
+
? {}
|
|
518
|
+
: { reasoning_effort: args.judgeBinding.reasoning_effort }),
|
|
519
|
+
},
|
|
520
|
+
model_info: lease.modelInfo,
|
|
521
|
+
}),
|
|
522
|
+
HSE_JUDGE_GATEWAY_PROTOCOL: lease.protocol,
|
|
523
|
+
},
|
|
524
|
+
})
|
|
525
|
+
} catch (error) {
|
|
526
|
+
throw await explainHarborFailure(error, jobDir)
|
|
527
|
+
}
|
|
528
|
+
let summary
|
|
529
|
+
let completion
|
|
530
|
+
try {
|
|
531
|
+
summary = JSON.parse(await readFile(path.join(jobDir, 'evaluation-summary.json'), 'utf8'))
|
|
532
|
+
completion = JSON.parse(await readFile(path.join(jobDir, 'historical-evaluation-complete.json'), 'utf8'))
|
|
533
|
+
} catch (error) {
|
|
534
|
+
throw new Error(
|
|
535
|
+
`HISTORICAL_JOB_INCOMPLETE: Harbor exited successfully but the Historical plugin did not write its summary/completion sentinel (${error.message})`,
|
|
536
|
+
)
|
|
537
|
+
}
|
|
538
|
+
assertHistoricalCompletion(summary, completion, {
|
|
539
|
+
jobName,
|
|
540
|
+
batchDigest: batch.digest,
|
|
541
|
+
batchId: batch.batch_id,
|
|
542
|
+
recordCount: Array.isArray(batch.records) ? batch.records.length : undefined,
|
|
543
|
+
})
|
|
544
|
+
return {
|
|
545
|
+
judgeModelBinding: {
|
|
546
|
+
provider: args.judgeBinding.provider,
|
|
547
|
+
model: args.judgeBinding.model,
|
|
548
|
+
...(args.judgeBinding.reasoning_effort === undefined
|
|
549
|
+
? {}
|
|
550
|
+
: { reasoning_effort: args.judgeBinding.reasoning_effort }),
|
|
551
|
+
},
|
|
552
|
+
job: path.relative(projectRoot, jobDir).split(path.sep).join('/'),
|
|
553
|
+
summary,
|
|
554
|
+
completion,
|
|
555
|
+
materialized: {
|
|
556
|
+
dataset: path.relative(projectRoot, dataset).split(path.sep).join('/'),
|
|
557
|
+
stack: path.relative(projectRoot, stack).split(path.sep).join('/'),
|
|
558
|
+
},
|
|
559
|
+
process: { code: processResult.code },
|
|
560
|
+
}
|
|
561
|
+
} finally {
|
|
562
|
+
await lease.close()
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
|
|
258
566
|
export async function readEvaluation(config, args) {
|
|
259
567
|
const jobDir = resolveWithin(config.projectRoot, args.jobPath, 'jobPath')
|
|
260
568
|
return JSON.parse(await readFile(path.join(jobDir, 'evaluation-summary.json'), 'utf8'))
|
package/lib/model-runtime.js
CHANGED
|
@@ -74,26 +74,71 @@ export class CandidateModelRuntime {
|
|
|
74
74
|
this.config = config
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
-
async
|
|
77
|
+
async resolveCurrent() {
|
|
78
|
+
const inherited = this.ctx.agentDefaultModel.currentSelection()
|
|
79
|
+
return this.resolve({
|
|
80
|
+
candidateProvider: inherited.provider,
|
|
81
|
+
candidateModel: inherited.model,
|
|
82
|
+
candidateReasoningEffort: inherited.reasoningEffort,
|
|
83
|
+
}, undefined, { ignoreConfigured: true })
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async currentBinding() {
|
|
87
|
+
const binding = await this.resolveCurrent()
|
|
88
|
+
return {
|
|
89
|
+
schema_version: 1,
|
|
90
|
+
source: 'skill-agent-default',
|
|
91
|
+
provider: binding.provider,
|
|
92
|
+
model: binding.model,
|
|
93
|
+
...(binding.reasoning_effort === undefined
|
|
94
|
+
? {}
|
|
95
|
+
: { reasoning_effort: binding.reasoning_effort }),
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
async resolve(args = {}, pinnedBinding, { ignoreConfigured = false } = {}) {
|
|
78
100
|
const explicitProvider = nonBlank(args.candidateProvider)
|
|
79
101
|
const explicitModel = nonBlank(args.candidateModel)
|
|
80
102
|
if (Boolean(explicitProvider) !== Boolean(explicitModel)) {
|
|
81
103
|
throw new Error('candidateProvider and candidateModel must be supplied together')
|
|
82
104
|
}
|
|
83
|
-
const configuredProvider = nonBlank(this.config.candidateProvider)
|
|
84
|
-
const configuredModel = nonBlank(this.config.candidateModel)
|
|
105
|
+
const configuredProvider = ignoreConfigured ? undefined : nonBlank(this.config.candidateProvider)
|
|
106
|
+
const configuredModel = ignoreConfigured ? undefined : nonBlank(this.config.candidateModel)
|
|
85
107
|
if (Boolean(configuredProvider) !== Boolean(configuredModel)) {
|
|
86
108
|
throw new Error('Harbor candidateProvider and candidateModel configuration must be supplied together')
|
|
87
109
|
}
|
|
88
110
|
|
|
111
|
+
const pinnedProvider = nonBlank(pinnedBinding?.provider)
|
|
112
|
+
const pinnedModel = nonBlank(pinnedBinding?.model)
|
|
113
|
+
if (Boolean(pinnedProvider) !== Boolean(pinnedModel)) {
|
|
114
|
+
throw new Error('Candidate model-binding.json requires provider and model')
|
|
115
|
+
}
|
|
116
|
+
const pinnedReasoning = nonBlank(pinnedBinding?.reasoning_effort)
|
|
117
|
+
if (pinnedProvider && explicitProvider && (
|
|
118
|
+
explicitProvider !== pinnedProvider
|
|
119
|
+
|| explicitModel !== pinnedModel
|
|
120
|
+
|| (nonBlank(args.candidateReasoningEffort) ?? undefined) !== pinnedReasoning
|
|
121
|
+
)) {
|
|
122
|
+
throw new Error('CANDIDATE_MODEL_BINDING_CONFLICT: explicit Job model arguments do not match model-binding.json; create a new Candidate for a different model identity')
|
|
123
|
+
}
|
|
124
|
+
if (pinnedProvider && configuredProvider && (
|
|
125
|
+
configuredProvider !== pinnedProvider
|
|
126
|
+
|| configuredModel !== pinnedModel
|
|
127
|
+
|| (nonBlank(this.config.candidateReasoningEffort) ?? undefined) !== pinnedReasoning
|
|
128
|
+
)) {
|
|
129
|
+
throw new Error('CANDIDATE_MODEL_BINDING_CONFLICT: Plugin model configuration does not match model-binding.json; create a new Candidate or remove the global override')
|
|
130
|
+
}
|
|
131
|
+
|
|
89
132
|
const inherited = this.ctx.agentDefaultModel.currentSelection()
|
|
90
|
-
const provider = explicitProvider ?? configuredProvider ?? inherited.provider
|
|
91
|
-
const model = explicitModel ?? configuredModel ?? inherited.model
|
|
133
|
+
const provider = pinnedProvider ?? explicitProvider ?? configuredProvider ?? inherited.provider
|
|
134
|
+
const model = pinnedModel ?? explicitModel ?? configuredModel ?? inherited.model
|
|
92
135
|
const explicitReasoning = nonBlank(args.candidateReasoningEffort)
|
|
93
|
-
const configuredReasoning = nonBlank(this.config.candidateReasoningEffort)
|
|
136
|
+
const configuredReasoning = ignoreConfigured ? undefined : nonBlank(this.config.candidateReasoningEffort)
|
|
94
137
|
const canInheritReasoning = provider === inherited.provider && model === inherited.model
|
|
95
|
-
const reasoningEffort =
|
|
96
|
-
|
|
138
|
+
const reasoningEffort = pinnedProvider
|
|
139
|
+
? pinnedReasoning
|
|
140
|
+
: explicitReasoning ?? configuredReasoning
|
|
141
|
+
?? (canInheritReasoning ? inherited.reasoningEffort : undefined)
|
|
97
142
|
|
|
98
143
|
if (!this.ctx.llm.listProviders().some(item => item.id === provider)) {
|
|
99
144
|
throw new Error(`Candidate model provider "${provider}" is not registered in DeepSeek Harness`)
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs'
|
|
2
|
+
|
|
3
|
+
const packageJson = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'))
|
|
4
|
+
|
|
5
|
+
export const DSH_RUNTIME_VERSION = packageJson.harborEvolution.dshRuntimeVersion
|
|
6
|
+
export const CANDIDATE_ACP_PACKAGE = packageJson.harborEvolution.candidateAcpPackage
|
|
7
|
+
export const RUNTIME_POLICY = packageJson.harborEvolution.runtimePolicy
|