dsh-harbor-evolution 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -6
- package/index.js +69 -7
- package/lib/candidate.js +29 -2
- package/lib/client.js +321 -0
- package/lib/dashboard.js +266 -0
- package/lib/evolution.js +105 -26
- package/lib/service.js +67 -0
- package/lib/setup.js +7 -1
- package/lib/web.js +76 -0
- package/package.json +23 -4
- package/skills/evolve-agent-with-harbor/SKILL.md +105 -78
- package/skills/evolve-agent-with-harbor/references/initialization.md +89 -83
package/lib/dashboard.js
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import { access, constants, lstat, readdir, readFile, stat } from 'node:fs/promises'
|
|
2
|
+
import path from 'node:path'
|
|
3
|
+
|
|
4
|
+
import { resolveWithin } from './evolution.js'
|
|
5
|
+
|
|
6
|
+
const SUMMARY_NAME = 'evaluation-summary.json'
|
|
7
|
+
const MAX_JOBS = 50
|
|
8
|
+
const MAX_JSON_BYTES = 2 * 1024 * 1024
|
|
9
|
+
const MAX_TRIAL_LIMIT = 100
|
|
10
|
+
const jsonCache = new Map()
|
|
11
|
+
const SENSITIVE_KEY = /authorization|cookie|token|api[_-]?key|secret|password|request[_-]?headers/i
|
|
12
|
+
|
|
13
|
+
function safeSegment(value, label) {
|
|
14
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(String(value ?? ''))) throw new Error(`${label} is invalid`)
|
|
15
|
+
return String(value)
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function redact(value, depth = 0) {
|
|
19
|
+
if (depth > 10) return '[TRUNCATED depth]'
|
|
20
|
+
if (Array.isArray(value)) return value.slice(0, 10_000).map(item => redact(item, depth + 1))
|
|
21
|
+
if (value && typeof value === 'object') {
|
|
22
|
+
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, SENSITIVE_KEY.test(key) ? '[REDACTED]' : redact(item, depth + 1)]))
|
|
23
|
+
}
|
|
24
|
+
if (typeof value === 'string' && value.length > 8_000) return `${value.slice(0, 8_000)}\n[TRUNCATED ${value.length - 8_000} chars]`
|
|
25
|
+
return value
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
async function readJson(file, { maxBytes = MAX_JSON_BYTES } = {}) {
|
|
29
|
+
try {
|
|
30
|
+
const details = await lstat(file)
|
|
31
|
+
if (details.isSymbolicLink()) return { __readError: `${path.basename(file)} may not be a symlink` }
|
|
32
|
+
if (!details.isFile()) return { __readError: `${path.basename(file)} is not a file` }
|
|
33
|
+
if (details.size > maxBytes) return { __readError: `${path.basename(file)} exceeds ${maxBytes} bytes` }
|
|
34
|
+
const cached = jsonCache.get(file)
|
|
35
|
+
const identity = `${details.mtimeMs}:${details.size}`
|
|
36
|
+
if (cached?.identity === identity) return cached.value
|
|
37
|
+
const value = redact(JSON.parse(await readFile(file, 'utf8')))
|
|
38
|
+
jsonCache.set(file, { identity, value })
|
|
39
|
+
return value
|
|
40
|
+
} catch (error) {
|
|
41
|
+
if (error.code === 'ENOENT') return undefined
|
|
42
|
+
if (error instanceof SyntaxError) return { __readError: `invalid JSON in ${path.basename(file)}` }
|
|
43
|
+
throw error
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async function directoryCheck(directory, { optional = false } = {}) {
|
|
48
|
+
try {
|
|
49
|
+
const details = await lstat(directory)
|
|
50
|
+
if (details.isSymbolicLink() || !details.isDirectory()) return { status: 'error', detail: 'not a safe directory' }
|
|
51
|
+
await access(directory, constants.R_OK)
|
|
52
|
+
return { status: 'ok', detail: 'readable' }
|
|
53
|
+
} catch (error) {
|
|
54
|
+
if (optional && error.code === 'ENOENT') return { status: 'warning', detail: 'not created yet' }
|
|
55
|
+
return { status: 'error', detail: error.code === 'ENOENT' ? 'not found' : 'not readable' }
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async function fileCheck(file) {
|
|
60
|
+
try {
|
|
61
|
+
const details = await lstat(file)
|
|
62
|
+
return details.isFile() && !details.isSymbolicLink()
|
|
63
|
+
? { status: 'ok', detail: path.basename(file) }
|
|
64
|
+
: { status: 'error', detail: 'not a safe file' }
|
|
65
|
+
} catch (error) {
|
|
66
|
+
return { status: 'error', detail: error.code === 'ENOENT' ? 'not found' : 'not readable' }
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async function executableCheck(command) {
|
|
71
|
+
if (!command) return { status: 'error', detail: 'not configured' }
|
|
72
|
+
if (!path.isAbsolute(command)) return { status: 'ok', detail: `${command} (PATH)` }
|
|
73
|
+
try {
|
|
74
|
+
await access(command, constants.X_OK)
|
|
75
|
+
return { status: 'ok', detail: path.basename(command) }
|
|
76
|
+
} catch (error) {
|
|
77
|
+
return { status: 'error', detail: error.code === 'ENOENT' ? 'not found' : 'not executable' }
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function jobStatus(summary) {
|
|
82
|
+
if (!summary) return 'pending'
|
|
83
|
+
if (summary.__readError) return 'failed'
|
|
84
|
+
if (Number(summary.n_trials ?? 0) > 0 && Number(summary.n_exceptions ?? 0) >= Number(summary.n_trials ?? 0)) return 'failed'
|
|
85
|
+
if (Number(summary.n_exceptions ?? 0) > 0) return 'partial'
|
|
86
|
+
return 'completed'
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function primaryMetric(summary, contract) {
|
|
90
|
+
const name = contract?.primary_metric
|
|
91
|
+
if (name && typeof summary?.metrics?.[name] === 'number') return { name, value: summary.metrics[name] }
|
|
92
|
+
const entry = Object.entries(summary?.metrics ?? {}).find(([, value]) => typeof value === 'number')
|
|
93
|
+
return entry ? { name: entry[0], value: entry[1] } : undefined
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
async function readJob(jobsDir, entry, details) {
|
|
97
|
+
const directory = path.join(jobsDir, entry.name)
|
|
98
|
+
const [summary, context, promotion, contract] = await Promise.all([
|
|
99
|
+
readJson(path.join(directory, SUMMARY_NAME)),
|
|
100
|
+
readJson(path.join(directory, 'evaluation-context.json')),
|
|
101
|
+
readJson(path.join(directory, 'promotion-report.json')),
|
|
102
|
+
readJson(path.join(directory, 'evaluation-contract.json')),
|
|
103
|
+
])
|
|
104
|
+
const evaluationContext = summary?.evaluation_context ?? context
|
|
105
|
+
if (evaluationContext?.schema_version !== 2) return undefined
|
|
106
|
+
return {
|
|
107
|
+
name: entry.name,
|
|
108
|
+
updatedAt: details.mtime.toISOString(),
|
|
109
|
+
status: jobStatus(summary),
|
|
110
|
+
mode: summary?.mode,
|
|
111
|
+
nTrials: Number(summary?.n_trials ?? 0),
|
|
112
|
+
nExceptions: Number(summary?.n_exceptions ?? 0),
|
|
113
|
+
primaryMetric: primaryMetric(summary, contract),
|
|
114
|
+
metrics: summary?.metrics ?? {},
|
|
115
|
+
candidate: summary?.candidate,
|
|
116
|
+
evaluationContext,
|
|
117
|
+
artifactValidation: summary?.artifact_validation,
|
|
118
|
+
promotion: promotion ? { decision: promotion.decision, reasons: promotion.reasons ?? [], baselineJob: promotion.baseline_job } : undefined,
|
|
119
|
+
readError: summary?.__readError,
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
async function listJobs(jobsDir) {
|
|
124
|
+
let entries
|
|
125
|
+
try {
|
|
126
|
+
entries = await readdir(jobsDir, { withFileTypes: true })
|
|
127
|
+
} catch (error) {
|
|
128
|
+
if (error.code === 'ENOENT') return []
|
|
129
|
+
throw error
|
|
130
|
+
}
|
|
131
|
+
const directories = entries.filter(entry => entry.isDirectory() && !entry.isSymbolicLink())
|
|
132
|
+
const recent = await Promise.all(directories.map(async entry => ({ entry, details: await stat(path.join(jobsDir, entry.name)) })))
|
|
133
|
+
recent.sort((left, right) => right.details.mtimeMs - left.details.mtimeMs)
|
|
134
|
+
const jobs = await Promise.all(recent.map(({ entry, details }) => readJob(jobsDir, entry, details)))
|
|
135
|
+
return jobs.filter(Boolean).slice(0, MAX_JOBS)
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function jobsDirectory(config) {
|
|
139
|
+
return resolveWithin(path.resolve(config.projectRoot), config.jobsDir, 'jobsDir')
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function jobDirectory(config, job) {
|
|
143
|
+
return path.join(jobsDirectory(config), safeSegment(job, 'job'))
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export async function readDashboardSnapshot(config, metadata = {}) {
|
|
147
|
+
const projectRoot = path.resolve(config.projectRoot)
|
|
148
|
+
const jobsDir = jobsDirectory(config)
|
|
149
|
+
const [jobs, projectRootCheck, jobsDirCheck, harborCheck, harborDshCheck, stackCheck] = await Promise.all([
|
|
150
|
+
listJobs(jobsDir),
|
|
151
|
+
directoryCheck(projectRoot),
|
|
152
|
+
directoryCheck(jobsDir, { optional: true }),
|
|
153
|
+
executableCheck(config.harborBin),
|
|
154
|
+
executableCheck(config.harborDshBin),
|
|
155
|
+
fileCheck(path.join(projectRoot, '.harbor', 'evaluation-stack.yml')),
|
|
156
|
+
])
|
|
157
|
+
const counts = jobs.reduce((result, job) => ({ ...result, [job.status]: (result[job.status] ?? 0) + 1 }), {})
|
|
158
|
+
const latestMetric = jobs.find(job => job.primaryMetric)?.primaryMetric
|
|
159
|
+
return {
|
|
160
|
+
schemaVersion: 2,
|
|
161
|
+
generatedAt: new Date().toISOString(),
|
|
162
|
+
pluginVersion: metadata.pluginVersion ?? 'development',
|
|
163
|
+
config: { jobsDir: config.jobsDir, dshVersion: config.dshVersion, agentImportPath: config.agentImportPath, pluginImportPath: config.pluginImportPath },
|
|
164
|
+
checks: { projectRoot: projectRootCheck, jobsDir: jobsDirCheck, harbor: harborCheck, harborDsh: harborDshCheck, evaluationStack: stackCheck },
|
|
165
|
+
overview: {
|
|
166
|
+
totalJobs: jobs.length,
|
|
167
|
+
completedJobs: (counts.completed ?? 0) + (counts.partial ?? 0),
|
|
168
|
+
activeJobs: counts.pending ?? 0,
|
|
169
|
+
failedJobs: counts.failed ?? 0,
|
|
170
|
+
latestMetric,
|
|
171
|
+
},
|
|
172
|
+
jobs,
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const DETAIL_ARTIFACTS = {
|
|
177
|
+
summary: 'evaluation-summary.json',
|
|
178
|
+
candidate: 'candidate-manifest.json',
|
|
179
|
+
dataset: 'dataset-manifest.json',
|
|
180
|
+
stack: 'evaluation-stack-manifest.json',
|
|
181
|
+
context: 'evaluation-context.json',
|
|
182
|
+
contract: 'evaluation-contract.json',
|
|
183
|
+
doctor: 'architecture-doctor.json',
|
|
184
|
+
population: 'population-report.json',
|
|
185
|
+
optimization: 'optimization-report.json',
|
|
186
|
+
promotion: 'promotion-report.json',
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function schemaIssue(key, value) {
|
|
190
|
+
if (value === undefined) return undefined
|
|
191
|
+
if (value?.__readError) return value.__readError
|
|
192
|
+
if (!isObject(value)) return 'artifact must be an object'
|
|
193
|
+
const version = { summary: 2, candidate: 1, dataset: 1, stack: 1, context: 2, contract: 1, doctor: 1, population: 1, optimization: 1, promotion: 2 }[key]
|
|
194
|
+
if (value.schema_version !== version) return `schema_version must be ${version}`
|
|
195
|
+
const required = {
|
|
196
|
+
summary: ['job', 'candidate', 'evaluation_context', 'metrics'],
|
|
197
|
+
candidate: ['candidate_id', 'version', 'digest'],
|
|
198
|
+
dataset: ['dataset_id', 'version', 'source_digest', 'tasks'],
|
|
199
|
+
stack: ['stack_id', 'version', 'digest', 'comparison_digest', 'components', 'judge'],
|
|
200
|
+
context: ['digest', 'full_digest', 'candidate', 'dataset', 'evaluation_stack', 'runtime'],
|
|
201
|
+
contract: ['contract_id', 'version', 'primary_metric', 'metrics'],
|
|
202
|
+
doctor: ['promotion_ready', 'findings'],
|
|
203
|
+
population: ['population_size', 'groups', 'metrics'],
|
|
204
|
+
optimization: ['hypotheses'],
|
|
205
|
+
promotion: ['decision', 'reasons', 'policy_digest'],
|
|
206
|
+
}[key] ?? []
|
|
207
|
+
const missing = required.filter(field => value[field] === undefined)
|
|
208
|
+
return missing.length ? `missing fields: ${missing.join(', ')}` : undefined
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function isObject(value) {
|
|
212
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
export async function readJobDetail(config, args) {
|
|
216
|
+
const job = safeSegment(args.job, 'job')
|
|
217
|
+
const directory = jobDirectory(config, job)
|
|
218
|
+
const check = await directoryCheck(directory)
|
|
219
|
+
if (check.status !== 'ok') throw new Error('Job not found')
|
|
220
|
+
const values = await Promise.all(Object.values(DETAIL_ARTIFACTS).map(name => readJson(path.join(directory, name))))
|
|
221
|
+
const artifacts = Object.fromEntries(Object.keys(DETAIL_ARTIFACTS).map((key, index) => [key, values[index]]))
|
|
222
|
+
if (artifacts.summary && !artifacts.summary.__readError) {
|
|
223
|
+
const { trials: _trials, ...lightSummary } = artifacts.summary
|
|
224
|
+
artifacts.summary = lightSummary
|
|
225
|
+
}
|
|
226
|
+
const validation = Object.fromEntries(Object.entries(artifacts).map(([key, value]) => {
|
|
227
|
+
const issue = schemaIssue(key, value)
|
|
228
|
+
return [key, value === undefined ? { status: 'missing' } : issue ? { status: 'invalid', error: issue } : { status: 'valid' }]
|
|
229
|
+
}))
|
|
230
|
+
if (validation.context.status !== 'valid') throw new Error('Job is not a Context v2 evaluation')
|
|
231
|
+
return { schemaVersion: 1, job, artifacts, validation }
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
export async function readTrialsPage(config, args) {
|
|
235
|
+
const job = safeSegment(args.job, 'job')
|
|
236
|
+
const offset = Math.max(0, Number.parseInt(args.offset ?? 0, 10) || 0)
|
|
237
|
+
const limit = Math.min(MAX_TRIAL_LIMIT, Math.max(1, Number.parseInt(args.limit ?? 50, 10) || 50))
|
|
238
|
+
const query = String(args.query ?? '').trim().toLowerCase()
|
|
239
|
+
const status = String(args.status ?? '')
|
|
240
|
+
const summary = await readJson(path.join(jobDirectory(config, job), SUMMARY_NAME))
|
|
241
|
+
if (!summary || summary.__readError) throw new Error('Job summary is unavailable')
|
|
242
|
+
let trials = Array.isArray(summary.trials) ? summary.trials : []
|
|
243
|
+
if (query) trials = trials.filter(trial => `${trial.id ?? ''} ${trial.name ?? ''}`.toLowerCase().includes(query))
|
|
244
|
+
if (status) trials = trials.filter(trial => (trial.exception ? 'infrastructure-error' : 'assessed') === status)
|
|
245
|
+
const items = trials.slice(offset, offset + limit).map(trial => ({
|
|
246
|
+
id: trial.id ?? trial.name,
|
|
247
|
+
name: trial.name,
|
|
248
|
+
status: trial.exception ? 'infrastructure-error' : 'assessed',
|
|
249
|
+
rewards: trial.rewards ?? {},
|
|
250
|
+
exception: trial.exception ? { type: trial.exception.type, classification: trial.exception.classification } : undefined,
|
|
251
|
+
}))
|
|
252
|
+
return { schemaVersion: 1, job, offset, limit, total: trials.length, items, hasMore: offset + items.length < trials.length }
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
function assessmentName(id) {
|
|
256
|
+
return `${String(id).replace(/[^A-Za-z0-9._-]+/g, '-').replace(/^[.-]+|[.-]+$/g, '') || 'trial'}.json`
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
export async function readTrialDetail(config, args) {
|
|
260
|
+
const job = safeSegment(args.job, 'job')
|
|
261
|
+
const trial = safeSegment(args.trial, 'trial')
|
|
262
|
+
const directory = jobDirectory(config, job)
|
|
263
|
+
const assessment = await readJson(path.join(directory, 'trial-assessments', assessmentName(trial)))
|
|
264
|
+
if (!assessment || assessment.__readError) throw new Error('Trial assessment not found')
|
|
265
|
+
return { schemaVersion: 1, job, trial, assessment }
|
|
266
|
+
}
|
package/lib/evolution.js
CHANGED
|
@@ -23,47 +23,135 @@ export async function snapshot(config, args) {
|
|
|
23
23
|
}
|
|
24
24
|
|
|
25
25
|
function slug(value) {
|
|
26
|
-
return String(value)
|
|
27
|
-
.toLowerCase()
|
|
28
|
-
.replace(/[^a-z0-9._-]+/g, '-')
|
|
29
|
-
.replace(/^[._-]+|[._-]+$/g, '') || 'candidate'
|
|
26
|
+
return String(value).toLowerCase().replace(/[^a-z0-9._-]+/g, '-').replace(/^[._-]+|[._-]+$/g, '') || 'candidate'
|
|
30
27
|
}
|
|
31
28
|
|
|
32
29
|
export function makeJobName(manifest, now = new Date()) {
|
|
33
30
|
const timestamp = now.toISOString().replace(/\.\d{3}Z$/, 'Z').replace(/[-:]/g, '')
|
|
34
31
|
const suffix = `${timestamp}-${manifest.digest.slice(7, 15)}`
|
|
35
|
-
const
|
|
36
|
-
const identity = `${slug(manifest.candidate_id)}-${slug(manifest.version)}`.slice(0, available)
|
|
32
|
+
const identity = `${slug(manifest.candidate_id)}-${slug(manifest.version)}`.slice(0, 99 - suffix.length)
|
|
37
33
|
return `${identity}-${suffix}`
|
|
38
34
|
}
|
|
39
35
|
|
|
36
|
+
async function cliJson(config, args, { allowedExitCodes = [0] } = {}) {
|
|
37
|
+
const result = await runProcess(config.harborDshBin, args, {
|
|
38
|
+
cwd: config.projectRoot,
|
|
39
|
+
timeoutMs: config.timeoutMs,
|
|
40
|
+
allowedExitCodes,
|
|
41
|
+
env: { ...process.env, ...(config.pythonPath ? { PYTHONPATH: config.pythonPath } : {}) },
|
|
42
|
+
})
|
|
43
|
+
try {
|
|
44
|
+
return JSON.parse(result.stdout)
|
|
45
|
+
} catch {
|
|
46
|
+
throw new Error(`harbor-dsh returned invalid JSON for ${args.slice(0, 2).join(' ')}`)
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function strictInputs(config, args) {
|
|
51
|
+
const projectRoot = path.resolve(config.projectRoot)
|
|
52
|
+
const candidate = resolveWithin(projectRoot, args.candidatePath, 'candidatePath')
|
|
53
|
+
const dataset = resolveWithin(projectRoot, args.datasetPath, 'datasetPath')
|
|
54
|
+
const stack = resolveWithin(projectRoot, args.stackPath, 'stackPath')
|
|
55
|
+
const jobs = resolveWithin(projectRoot, config.jobsDir, 'jobsDir')
|
|
56
|
+
const mode = args.mode
|
|
57
|
+
if (!['diagnostic', 'promotion-eligible'].includes(mode)) throw new Error('mode must be diagnostic or promotion-eligible')
|
|
58
|
+
const policy = args.policyPath ? resolveWithin(projectRoot, args.policyPath, 'policyPath') : undefined
|
|
59
|
+
if (mode === 'promotion-eligible' && !policy) throw new Error('promotion-eligible mode requires policyPath')
|
|
60
|
+
return { projectRoot, candidate, dataset, stack, jobs, mode, policy }
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export async function validateDataset(config, args) {
|
|
64
|
+
const dataset = resolveWithin(config.projectRoot, args.datasetPath, 'datasetPath')
|
|
65
|
+
return cliJson(config, ['dataset', 'validate', dataset, '--project-root', config.projectRoot], { allowedExitCodes: [0, 2] })
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export async function initializeProject(config, args) {
|
|
69
|
+
const dataset = resolveWithin(config.projectRoot, args.datasetPath, 'datasetPath')
|
|
70
|
+
return cliJson(config, [
|
|
71
|
+
'init', '--project-root', config.projectRoot, '--dataset', dataset,
|
|
72
|
+
'--stack-id', args.stackId, '--stack-version', args.stackVersion,
|
|
73
|
+
'--dataset-id', args.datasetId, '--dataset-version', args.datasetVersion,
|
|
74
|
+
'--contract-id', args.contractId, '--contract-version', args.contractVersion,
|
|
75
|
+
'--primary-metric', args.primaryMetric, '--primary-direction', args.primaryDirection,
|
|
76
|
+
'--judge-provider', args.judgeProvider, '--judge-model', args.judgeModel, '--judge-version', args.judgeVersion,
|
|
77
|
+
'--policy-id', args.policyId, '--policy-version', args.policyVersion,
|
|
78
|
+
'--min-improvement', String(args.minImprovement),
|
|
79
|
+
])
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export async function runDoctor(config, args) {
|
|
83
|
+
const inputs = strictInputs(config, { ...args, mode: args.mode ?? 'diagnostic' })
|
|
84
|
+
const command = ['doctor', '--architecture', '--project-root', inputs.projectRoot, '--stack', inputs.stack, '--dataset', inputs.dataset]
|
|
85
|
+
if (args.candidatePath) command.push('--candidate', inputs.candidate)
|
|
86
|
+
if (inputs.policy) command.push('--policy', inputs.policy)
|
|
87
|
+
return cliJson(config, command, { allowedExitCodes: [0, 2] })
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export async function previewContext(config, args) {
|
|
91
|
+
const manifest = await snapshot(config, args)
|
|
92
|
+
const inputs = strictInputs(config, args)
|
|
93
|
+
const preview = await cliJson(config, [
|
|
94
|
+
'context', 'preview',
|
|
95
|
+
'--project-root', inputs.projectRoot,
|
|
96
|
+
'--candidate', inputs.candidate,
|
|
97
|
+
'--dataset', inputs.dataset,
|
|
98
|
+
'--stack', inputs.stack,
|
|
99
|
+
'--jobs-dir', inputs.jobs,
|
|
100
|
+
'--mode', inputs.mode,
|
|
101
|
+
])
|
|
102
|
+
return { manifest, ...preview }
|
|
103
|
+
}
|
|
104
|
+
|
|
40
105
|
export async function runEvaluation(config, args) {
|
|
41
106
|
const manifest = await snapshot(config, args)
|
|
42
|
-
const
|
|
43
|
-
const
|
|
44
|
-
|
|
107
|
+
const inputs = strictInputs(config, args)
|
|
108
|
+
const datasetValidation = await validateDataset(config, args)
|
|
109
|
+
if (!datasetValidation.valid) {
|
|
110
|
+
throw new Error(`Dataset validation failed: ${datasetValidation.findings.map(item => item.code).join(', ')}`)
|
|
111
|
+
}
|
|
112
|
+
const doctor = await runDoctor(config, args)
|
|
113
|
+
if (inputs.mode === 'promotion-eligible' && !doctor.promotion_ready) {
|
|
114
|
+
throw new Error(`Architecture Doctor blocked promotion-eligible Job: ${doctor.findings.filter(item => item.level === 'error').map(item => item.code).join(', ')}`)
|
|
115
|
+
}
|
|
116
|
+
const preview = await cliJson(config, [
|
|
117
|
+
'context', 'preview', '--project-root', inputs.projectRoot,
|
|
118
|
+
'--candidate', inputs.candidate, '--dataset', inputs.dataset,
|
|
119
|
+
'--stack', inputs.stack, '--jobs-dir', inputs.jobs, '--mode', inputs.mode,
|
|
120
|
+
])
|
|
45
121
|
const jobName = args.jobName ?? makeJobName(manifest)
|
|
46
122
|
if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$/.test(jobName)) throw new Error('jobName contains unsupported characters')
|
|
47
123
|
|
|
48
124
|
const harborArgs = [
|
|
49
|
-
'run', '-p', dataset,
|
|
125
|
+
'run', '-p', inputs.dataset,
|
|
50
126
|
'-a', config.agentImportPath,
|
|
51
|
-
'--ak', `candidate_path=${
|
|
127
|
+
'--ak', `candidate_path=${inputs.candidate}`,
|
|
52
128
|
'--ak', `candidate_version=${manifest.version}`,
|
|
53
129
|
'--ak', `candidate_digest=${manifest.digest}`,
|
|
54
130
|
'--job-name', jobName,
|
|
55
|
-
'--jobs-dir',
|
|
131
|
+
'--jobs-dir', inputs.jobs,
|
|
56
132
|
'--plugin', config.pluginImportPath,
|
|
57
|
-
'--plugin-kwarg', `candidate_manifest=${path.join(
|
|
133
|
+
'--plugin-kwarg', `candidate_manifest=${path.join(inputs.candidate, MANIFEST_NAME)}`,
|
|
134
|
+
'--plugin-kwarg', `dataset_path=${inputs.dataset}`,
|
|
135
|
+
'--plugin-kwarg', `stack_path=${inputs.stack}`,
|
|
136
|
+
'--plugin-kwarg', `project_root=${inputs.projectRoot}`,
|
|
137
|
+
'--plugin-kwarg', `mode=${inputs.mode}`,
|
|
58
138
|
]
|
|
59
|
-
|
|
139
|
+
if (inputs.policy) harborArgs.push('--plugin-kwarg', `policy_path=${inputs.policy}`)
|
|
140
|
+
const processResult = await runProcess(config.harborBin, harborArgs, {
|
|
60
141
|
cwd: config.projectRoot,
|
|
61
142
|
timeoutMs: config.timeoutMs,
|
|
62
143
|
env: { ...process.env, ...(config.pythonPath ? { PYTHONPATH: config.pythonPath } : {}) },
|
|
63
144
|
})
|
|
64
|
-
const jobDir = path.join(
|
|
145
|
+
const jobDir = path.join(inputs.jobs, jobName)
|
|
65
146
|
const summary = JSON.parse(await readFile(path.join(jobDir, 'evaluation-summary.json'), 'utf8'))
|
|
66
|
-
return {
|
|
147
|
+
return {
|
|
148
|
+
manifest,
|
|
149
|
+
job: path.relative(inputs.projectRoot, jobDir),
|
|
150
|
+
summary,
|
|
151
|
+
doctor,
|
|
152
|
+
contextPreview: preview,
|
|
153
|
+
process: { code: processResult.code },
|
|
154
|
+
}
|
|
67
155
|
}
|
|
68
156
|
|
|
69
157
|
export async function readEvaluation(config, args) {
|
|
@@ -75,14 +163,5 @@ export async function compareCandidates(config, args) {
|
|
|
75
163
|
const baseline = resolveWithin(config.projectRoot, args.baselineJob, 'baselineJob')
|
|
76
164
|
const candidate = resolveWithin(config.projectRoot, args.candidateJob, 'candidateJob')
|
|
77
165
|
const policy = resolveWithin(config.projectRoot, args.policyPath, 'policyPath')
|
|
78
|
-
|
|
79
|
-
'promote', baseline, candidate, '--policy', policy,
|
|
80
|
-
], {
|
|
81
|
-
cwd: config.projectRoot,
|
|
82
|
-
timeoutMs: config.timeoutMs,
|
|
83
|
-
// harbor-dsh uses exit 1 for a valid REJECT decision. The report remains
|
|
84
|
-
// model-readable; infrastructure failures still reject the subprocess.
|
|
85
|
-
allowedExitCodes: [0, 1],
|
|
86
|
-
})
|
|
87
|
-
return JSON.parse(result.stdout)
|
|
166
|
+
return cliJson(config, ['promote', baseline, candidate, '--policy', policy], { allowedExitCodes: [0, 1] })
|
|
88
167
|
}
|
package/lib/service.js
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { readDashboardSnapshot, readJobDetail, readTrialDetail, readTrialsPage } from './dashboard.js'
|
|
2
|
+
import {
|
|
3
|
+
compareCandidates,
|
|
4
|
+
initializeProject,
|
|
5
|
+
previewContext,
|
|
6
|
+
readEvaluation,
|
|
7
|
+
runDoctor,
|
|
8
|
+
runEvaluation,
|
|
9
|
+
snapshot,
|
|
10
|
+
validateDataset,
|
|
11
|
+
} from './evolution.js'
|
|
12
|
+
|
|
13
|
+
/** One Host-side boundary shared by Agent tools and the Web dashboard. */
|
|
14
|
+
export class EvolutionService {
|
|
15
|
+
constructor(config, metadata = {}) {
|
|
16
|
+
this.config = config
|
|
17
|
+
this.metadata = metadata
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
snapshot(args) {
|
|
21
|
+
return snapshot(this.config, args)
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
initialize(args) {
|
|
25
|
+
return initializeProject(this.config, args)
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
run(args) {
|
|
29
|
+
return runEvaluation(this.config, args)
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
result(args) {
|
|
33
|
+
return readEvaluation(this.config, args)
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
compare(args) {
|
|
37
|
+
return compareCandidates(this.config, args)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
doctor(args) {
|
|
41
|
+
return runDoctor(this.config, args)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
validateDataset(args) {
|
|
45
|
+
return validateDataset(this.config, args)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
previewContext(args) {
|
|
49
|
+
return previewContext(this.config, args)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
dashboard() {
|
|
53
|
+
return readDashboardSnapshot(this.config, this.metadata)
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
job(args) {
|
|
57
|
+
return readJobDetail(this.config, args)
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
trials(args) {
|
|
61
|
+
return readTrialsPage(this.config, args)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
trial(args) {
|
|
65
|
+
return readTrialDetail(this.config, args)
|
|
66
|
+
}
|
|
67
|
+
}
|
package/lib/setup.js
CHANGED
|
@@ -247,6 +247,10 @@ export async function setupIntegration(raw = {}, dependencies = {}) {
|
|
|
247
247
|
// the complete locked graph there so runtime dependencies and host peers
|
|
248
248
|
// do not disappear behind the profile's `link:` entry.
|
|
249
249
|
await run('npm', ['ci', '--ignore-scripts'], { cwd: localPluginDir })
|
|
250
|
+
// The browser half is generated from source and embeds its visual asset.
|
|
251
|
+
// Build explicitly because the locked install above intentionally skips
|
|
252
|
+
// lifecycle scripts for deterministic source-checkout setup.
|
|
253
|
+
await run('npm', ['run', 'build'], { cwd: localPluginDir })
|
|
250
254
|
}
|
|
251
255
|
|
|
252
256
|
progress('2/4 Installing the Harbor Python runtime...')
|
|
@@ -310,7 +314,9 @@ export function renderSetupResult(result) {
|
|
|
310
314
|
`cd ${shellQuote(result.projectRoot)}`,
|
|
311
315
|
startCommand,
|
|
312
316
|
'',
|
|
313
|
-
|
|
317
|
+
result.profile === 'web'
|
|
318
|
+
? 'Open the Harbor tab, or invoke: /evolve-agent-with-harbor'
|
|
319
|
+
: 'Then invoke: /evolve-agent-with-harbor',
|
|
314
320
|
)
|
|
315
321
|
return lines.join('\n')
|
|
316
322
|
}
|
package/lib/web.js
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
export const DASHBOARD_ROUTE = '/_dsh/harbor-evolution/dashboard'
|
|
2
|
+
export const JOB_ROUTE = '/_dsh/harbor-evolution/job'
|
|
3
|
+
export const TRIALS_ROUTE = '/_dsh/harbor-evolution/trials'
|
|
4
|
+
export const TRIAL_ROUTE = '/_dsh/harbor-evolution/trial'
|
|
5
|
+
|
|
6
|
+
function sendJson(response, status, body) {
|
|
7
|
+
response.writeHead(status, {
|
|
8
|
+
'cache-control': 'no-store',
|
|
9
|
+
'content-type': 'application/json; charset=utf-8',
|
|
10
|
+
'x-content-type-options': 'nosniff',
|
|
11
|
+
})
|
|
12
|
+
response.end(JSON.stringify(body))
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function isSameOriginRequest(request) {
|
|
16
|
+
const fetchSite = request.headers['sec-fetch-site']
|
|
17
|
+
if (fetchSite && fetchSite !== 'same-origin' && fetchSite !== 'none') return false
|
|
18
|
+
const origin = request.headers.origin
|
|
19
|
+
if (!origin) {
|
|
20
|
+
if (fetchSite === 'same-origin' || fetchSite === 'none') return true
|
|
21
|
+
const address = request.socket?.remoteAddress ?? ''
|
|
22
|
+
return address === '::1' || address === '127.0.0.1' || address.startsWith('127.') || address.startsWith('::ffff:127.')
|
|
23
|
+
}
|
|
24
|
+
const host = request.headers.host
|
|
25
|
+
if (!host) return false
|
|
26
|
+
try {
|
|
27
|
+
const parsed = new URL(origin)
|
|
28
|
+
return ['http:', 'https:'].includes(parsed.protocol) && parsed.host === host
|
|
29
|
+
} catch {
|
|
30
|
+
return false
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function safeError(error) {
|
|
35
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
36
|
+
return message.replace(/(?:\/[A-Za-z0-9._ -]+){2,}/g, '[local path]').replace(/[A-Za-z]:\\[^\s]+/g, '[local path]')
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function createApiHandler(load, code = 'request-failed') {
|
|
40
|
+
return (request, response) => {
|
|
41
|
+
if (request.method !== 'GET') {
|
|
42
|
+
response.writeHead(405, { allow: 'GET' })
|
|
43
|
+
response.end()
|
|
44
|
+
return
|
|
45
|
+
}
|
|
46
|
+
if (!isSameOriginRequest(request)) {
|
|
47
|
+
sendJson(response, 403, { ok: false, error: { code: 'forbidden', message: 'same-origin request required' } })
|
|
48
|
+
return
|
|
49
|
+
}
|
|
50
|
+
const url = new URL(request.url ?? '/', 'http://localhost')
|
|
51
|
+
const args = Object.fromEntries(url.searchParams)
|
|
52
|
+
Promise.resolve(load(args)).then(
|
|
53
|
+
value => sendJson(response, 200, { ok: true, value }),
|
|
54
|
+
error => sendJson(response, 500, { ok: false, error: { code, message: safeError(error) } }),
|
|
55
|
+
)
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function createDashboardHandler(service) {
|
|
60
|
+
return createApiHandler(() => service.dashboard(), 'dashboard-unavailable')
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function installDashboardWeb(ctx, service) {
|
|
64
|
+
if (typeof ctx.inject !== 'function') return
|
|
65
|
+
ctx.inject(['webServer'], (webCtx) => {
|
|
66
|
+
const routes = [
|
|
67
|
+
[DASHBOARD_ROUTE, createDashboardHandler(service)],
|
|
68
|
+
[JOB_ROUTE, createApiHandler(args => service.job(args), 'job-unavailable')],
|
|
69
|
+
[TRIALS_ROUTE, createApiHandler(args => service.trials(args), 'trials-unavailable')],
|
|
70
|
+
[TRIAL_ROUTE, createApiHandler(args => service.trial(args), 'trial-unavailable')],
|
|
71
|
+
]
|
|
72
|
+
for (const [route, handler] of routes) {
|
|
73
|
+
webCtx.effect(() => webCtx.webServer.register({ kind: 'exact', path: route, handler }), `harbor-evolution: ${route}`)
|
|
74
|
+
}
|
|
75
|
+
})
|
|
76
|
+
}
|
package/package.json
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-harbor-evolution",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "DeepSeek Harness plugin and bundled Skill for safely evolving Cordis Candidates with Harbor.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.js",
|
|
7
|
-
"exports":
|
|
7
|
+
"exports": {
|
|
8
|
+
".": "./index.js",
|
|
9
|
+
"./client": "./lib/client.js",
|
|
10
|
+
"./cordis.patch.yml": "./cordis.patch.yml",
|
|
11
|
+
"./package.json": "./package.json"
|
|
12
|
+
},
|
|
8
13
|
"bin": {
|
|
9
14
|
"dsh-harbor": "bin/dsh-harbor.mjs"
|
|
10
15
|
},
|
|
@@ -18,12 +23,24 @@
|
|
|
18
23
|
"LICENSE"
|
|
19
24
|
],
|
|
20
25
|
"scripts": {
|
|
26
|
+
"build": "node scripts/build-client.mjs",
|
|
27
|
+
"prepack": "npm run build",
|
|
21
28
|
"test": "node --test",
|
|
22
|
-
"check": "node --check index.js && node --check bin/dsh-harbor.mjs && node --test"
|
|
29
|
+
"check": "npm run build && node --check index.js && node --check bin/dsh-harbor.mjs && node --test"
|
|
23
30
|
},
|
|
24
31
|
"dsh": {
|
|
25
32
|
"bundle": {
|
|
26
33
|
"patch": "./cordis.patch.yml"
|
|
34
|
+
},
|
|
35
|
+
"client": {
|
|
36
|
+
"inject": [
|
|
37
|
+
"@deepseek-ai/dsh-client-runtime",
|
|
38
|
+
"@deepseek-ai/dsh-client-locale",
|
|
39
|
+
"@deepseek-ai/dsh-client-ui-conversation",
|
|
40
|
+
"@deepseek-ai/dsh-client-ui-tool",
|
|
41
|
+
"@deepseek-ai/dsh-client-ui-settings"
|
|
42
|
+
],
|
|
43
|
+
"platform": "web"
|
|
27
44
|
}
|
|
28
45
|
},
|
|
29
46
|
"peerDependencies": {
|
|
@@ -37,7 +54,9 @@
|
|
|
37
54
|
"@deepseek-ai/cordis": "4.0.1",
|
|
38
55
|
"@deepseek-ai/dsh-skill": "0.1.0-rc.6",
|
|
39
56
|
"@deepseek-ai/dsh-tools": "0.1.0-rc.6",
|
|
40
|
-
"@deepseek-ai/schemastery": "3.18.1"
|
|
57
|
+
"@deepseek-ai/schemastery": "3.18.1",
|
|
58
|
+
"esbuild": "0.28.2",
|
|
59
|
+
"react": "18.3.1"
|
|
41
60
|
},
|
|
42
61
|
"engines": {
|
|
43
62
|
"node": ">=22"
|