dsh-harbor-evolution 0.7.3 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -3
- package/index.js +40 -0
- package/lib/client.js +96 -17
- package/lib/dashboard.js +167 -15
- package/lib/evolution.js +189 -0
- package/lib/model-runtime.js +11 -7
- package/lib/session-diagnostic.js +320 -0
- package/lib/session-materializer.js +194 -0
- package/lib/session-projection.js +161 -0
- package/lib/session-redaction.js +311 -0
- package/lib/session-selection.js +294 -0
- package/lib/setup.js +9 -3
- package/package.json +7 -2
- package/schemas/dsh-session-observation.schema.json +69 -0
- package/schemas/evaluation-result-v2.schema.json +45 -0
- package/schemas/historical-evaluation-context.schema.json +66 -0
- package/schemas/historical-evaluation-summary.schema.json +49 -0
- package/schemas/historical-generation-batch.schema.json +76 -0
- package/skills/evolve-agent-with-harbor/SKILL.md +47 -7
- package/skills/evolve-agent-with-harbor/evals/evals.json +9 -6
- package/skills/evolve-agent-with-harbor/references/evaluator-upgrade.md +3 -1
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
import { canonicalDigest } from './session-selection.js'
|
|
2
|
+
|
|
3
|
+
const MAX_MESSAGE_CHARS = 4_000
|
|
4
|
+
const MAX_TRANSCRIPT_MESSAGES = 80
|
|
5
|
+
const MAX_OBSERVATION_BYTES = 512 * 1024
|
|
6
|
+
|
|
7
|
+
const SECRET_PATTERNS = [
|
|
8
|
+
/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/gi,
|
|
9
|
+
/\b(?:authorization\s*[:=]\s*(?:bearer\s+)?|bearer\s+)[A-Za-z0-9._~+/=-]{8,}/gi,
|
|
10
|
+
/\b(?:api[_-]?key|access[_-]?token|refresh[_-]?token|secret|password|passwd)\s*[:=]\s*["']?[^\s,"'}\]]{4,}/gi,
|
|
11
|
+
/\b(?:sk|rk|pk)-[A-Za-z0-9_-]{12,}\b/g,
|
|
12
|
+
/\bgh[opusr]_[A-Za-z0-9]{20,}\b/g,
|
|
13
|
+
/\bAKIA[0-9A-Z]{16}\b/g,
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
const ABSOLUTE_PATH = /(?:[A-Za-z]:\\(?:[^\s<>:"|?*]+\\)+[^\s<>:"|?*]*|\/(?:Users|home|private|tmp|var|etc|opt|Volumes|workspace)(?:\/[A-Za-z0-9._ @+-]+)+)/g
|
|
17
|
+
|
|
18
|
+
function replaceCanaries(value, canaries) {
|
|
19
|
+
let text = value
|
|
20
|
+
let replacements = 0
|
|
21
|
+
for (const canary of canaries) {
|
|
22
|
+
if (!canary || !text.includes(canary)) continue
|
|
23
|
+
const pieces = text.split(canary)
|
|
24
|
+
replacements += pieces.length - 1
|
|
25
|
+
text = pieces.join('[REDACTED_SESSION_ID]')
|
|
26
|
+
}
|
|
27
|
+
return { text, replacements }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function replaceSecrets(value, canaries = []) {
|
|
31
|
+
const canaryResult = replaceCanaries(value, canaries)
|
|
32
|
+
value = canaryResult.text
|
|
33
|
+
let text = value
|
|
34
|
+
let replacements = canaryResult.replacements
|
|
35
|
+
for (const pattern of SECRET_PATTERNS) {
|
|
36
|
+
text = text.replace(pattern, () => {
|
|
37
|
+
replacements += 1
|
|
38
|
+
return '[REDACTED_SECRET]'
|
|
39
|
+
})
|
|
40
|
+
}
|
|
41
|
+
text = text.replace(ABSOLUTE_PATH, () => {
|
|
42
|
+
replacements += 1
|
|
43
|
+
return '[REDACTED_PATH]'
|
|
44
|
+
})
|
|
45
|
+
return { text, replacements }
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function sanitizeText(value, maxChars = MAX_MESSAGE_CHARS, canaries = []) {
|
|
49
|
+
const input = String(value ?? '')
|
|
50
|
+
const redacted = replaceSecrets(input, canaries)
|
|
51
|
+
const truncated = redacted.text.length > maxChars
|
|
52
|
+
return {
|
|
53
|
+
text: truncated ? `${redacted.text.slice(0, maxChars)}\n[TRUNCATED]` : redacted.text,
|
|
54
|
+
replacements: redacted.replacements,
|
|
55
|
+
truncated,
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function isoTime(value) {
|
|
60
|
+
return Number.isSafeInteger(value) && value > 0 ? new Date(value).toISOString() : null
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function appendOrigin(event) {
|
|
64
|
+
return event?.surfaceOp === undefined || event.surfaceOp === 'append'
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function visibleContent(message, report, canaries) {
|
|
68
|
+
const content = []
|
|
69
|
+
for (const block of Array.isArray(message?.content) ? message.content : []) {
|
|
70
|
+
if (block?.type !== 'text' || typeof block.text !== 'string') continue
|
|
71
|
+
const sanitized = sanitizeText(block.text, MAX_MESSAGE_CHARS, canaries)
|
|
72
|
+
report.replacements += sanitized.replacements
|
|
73
|
+
if (sanitized.truncated) report.truncations += 1
|
|
74
|
+
if (sanitized.text.trim()) content.push({ type: 'text', text: sanitized.text })
|
|
75
|
+
}
|
|
76
|
+
return content
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function messageRef(message) {
|
|
80
|
+
return canonicalDigest(
|
|
81
|
+
{ id: typeof message?.id === 'string' ? message.id : null },
|
|
82
|
+
'harbor-dsh-session-message-ref-v1',
|
|
83
|
+
)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function sanitizeIdentity(value, report, canaries) {
|
|
87
|
+
const sanitized = sanitizeText(value, 160, canaries)
|
|
88
|
+
report.replacements += sanitized.replacements
|
|
89
|
+
if (sanitized.truncated) report.truncations += 1
|
|
90
|
+
return sanitized.text
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function modelSegments(selected, report, canaries) {
|
|
94
|
+
return selected.index.modelSegments.map(segment => ({
|
|
95
|
+
from_seq: segment.from_seq,
|
|
96
|
+
through_seq: segment.through_seq,
|
|
97
|
+
provider: sanitizeIdentity(segment.provider, report, canaries),
|
|
98
|
+
model: sanitizeIdentity(segment.model, report, canaries),
|
|
99
|
+
...(segment.reasoning_effort
|
|
100
|
+
? { reasoning_effort: sanitizeIdentity(segment.reasoning_effort, report, canaries) }
|
|
101
|
+
: {}),
|
|
102
|
+
}))
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function toolEvidence(events) {
|
|
106
|
+
const results = new Map()
|
|
107
|
+
for (const event of events) {
|
|
108
|
+
if (event?.type !== 'tool/result') continue
|
|
109
|
+
results.set(event.data?.message?.source?.callId, event)
|
|
110
|
+
}
|
|
111
|
+
const tools = []
|
|
112
|
+
for (const event of events) {
|
|
113
|
+
if (event?.type !== 'tool/call') continue
|
|
114
|
+
const result = results.get(event.data?.callId)
|
|
115
|
+
tools.push({
|
|
116
|
+
event_seq: event.seq,
|
|
117
|
+
name: String(event.data?.name ?? 'unknown').slice(0, 160),
|
|
118
|
+
outcome: result?.data?.error || result?.data?.message?.content?.[0]?.isError ? 'error' : result ? 'success' : 'unknown',
|
|
119
|
+
error_code: typeof result?.data?.error?.code === 'string'
|
|
120
|
+
? result.data.error.code.slice(0, 160)
|
|
121
|
+
: null,
|
|
122
|
+
result_summary: result ? 'Tool completed; payload intentionally omitted.' : 'No matching tool result observed.',
|
|
123
|
+
truncated: true,
|
|
124
|
+
})
|
|
125
|
+
}
|
|
126
|
+
return tools.slice(0, 200)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function turnEvidence(events) {
|
|
130
|
+
const starts = new Map()
|
|
131
|
+
const turns = []
|
|
132
|
+
for (const event of events) {
|
|
133
|
+
if (event?.type === 'turn/start') starts.set(event.data?.turn, event.time)
|
|
134
|
+
if (event?.type === 'turn/end') {
|
|
135
|
+
turns.push({
|
|
136
|
+
turn: event.data?.turn,
|
|
137
|
+
reason: event.data?.reason?.kind ?? 'unknown',
|
|
138
|
+
started_at: isoTime(starts.get(event.data?.turn)),
|
|
139
|
+
ended_at: isoTime(event.time),
|
|
140
|
+
})
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
return turns
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function usageEvidence(events) {
|
|
147
|
+
let inputTokens = 0
|
|
148
|
+
let outputTokens = 0
|
|
149
|
+
let reported = false
|
|
150
|
+
for (const event of events) {
|
|
151
|
+
if (event?.type !== 'assistant/message' || !event.data?.usage) continue
|
|
152
|
+
const usage = event.data.usage
|
|
153
|
+
if (Number.isFinite(usage.inputTokens)) inputTokens += usage.inputTokens
|
|
154
|
+
if (Number.isFinite(usage.outputTokens)) outputTokens += usage.outputTokens
|
|
155
|
+
reported = true
|
|
156
|
+
}
|
|
157
|
+
return { input_tokens: inputTokens, output_tokens: outputTokens, reported }
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function sanitizeFeedback(items, report, canaries) {
|
|
161
|
+
const output = []
|
|
162
|
+
for (const item of Array.isArray(items) ? items : []) {
|
|
163
|
+
if (!['positive', 'negative'].includes(item?.rating)) continue
|
|
164
|
+
const note = sanitizeText(item.note ?? '', 1_000, canaries)
|
|
165
|
+
report.replacements += note.replacements
|
|
166
|
+
if (note.truncated) report.truncations += 1
|
|
167
|
+
output.push({
|
|
168
|
+
message_ref: canonicalDigest({ id: item.messageId ?? null }, 'harbor-dsh-feedback-message-ref-v1'),
|
|
169
|
+
rating: item.rating,
|
|
170
|
+
...(note.text.trim() ? { note: note.text } : {}),
|
|
171
|
+
updated_at: isoTime(item.updatedAt),
|
|
172
|
+
})
|
|
173
|
+
}
|
|
174
|
+
return output.slice(0, 100)
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function assertNoSecret(value, canaries = []) {
|
|
178
|
+
const serialized = JSON.stringify(value)
|
|
179
|
+
for (const canary of canaries) {
|
|
180
|
+
if (canary.length >= 8 && serialized.includes(canary)) {
|
|
181
|
+
throw new Error('SESSION_REDACTION_FAILED: a raw Session id survived the redaction pipeline')
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
for (const pattern of SECRET_PATTERNS) {
|
|
185
|
+
pattern.lastIndex = 0
|
|
186
|
+
if (pattern.test(serialized)) {
|
|
187
|
+
throw new Error('SESSION_REDACTION_FAILED: a credential-shaped value survived the redaction pipeline')
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
ABSOLUTE_PATH.lastIndex = 0
|
|
191
|
+
if (ABSOLUTE_PATH.test(serialized)) {
|
|
192
|
+
throw new Error('SESSION_REDACTION_FAILED: an absolute local path survived the redaction pipeline')
|
|
193
|
+
}
|
|
194
|
+
if (Buffer.byteLength(serialized) > MAX_OBSERVATION_BYTES) {
|
|
195
|
+
throw new Error(`SESSION_OBSERVATION_TOO_LARGE: redacted observation exceeds ${MAX_OBSERVATION_BYTES} bytes`)
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
const policyWithoutDigest = {
|
|
200
|
+
id: 'dsh-session-default-redaction',
|
|
201
|
+
version: '1.0.0',
|
|
202
|
+
projection: 'direct-human-and-assembled-assistant-text',
|
|
203
|
+
tool_payloads: 'omit',
|
|
204
|
+
reasoning: 'omit',
|
|
205
|
+
attachments: 'omit',
|
|
206
|
+
credentials: 'redact-and-fail-closed',
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
export const DEFAULT_REDACTION_POLICY = Object.freeze({
|
|
210
|
+
...policyWithoutDigest,
|
|
211
|
+
digest: canonicalDigest(policyWithoutDigest, 'harbor-dsh-session-redaction-policy-v1'),
|
|
212
|
+
})
|
|
213
|
+
|
|
214
|
+
export function buildSessionObservation(selected, feedbackItems = []) {
|
|
215
|
+
const report = { replacements: 0, truncations: 0, omitted_blocks: 0 }
|
|
216
|
+
const canaries = [String(selected.rawSessionId ?? '')].filter(Boolean)
|
|
217
|
+
const visibleTranscript = []
|
|
218
|
+
for (const event of selected.events) {
|
|
219
|
+
if (!appendOrigin(event)) continue
|
|
220
|
+
let role
|
|
221
|
+
let message
|
|
222
|
+
if (event?.type === 'user/message' && event.data?.source?.kind === 'user') {
|
|
223
|
+
role = 'user'
|
|
224
|
+
message = event.data
|
|
225
|
+
} else if (event?.type === 'assistant/message') {
|
|
226
|
+
role = 'assistant'
|
|
227
|
+
message = event.data?.message
|
|
228
|
+
} else {
|
|
229
|
+
continue
|
|
230
|
+
}
|
|
231
|
+
const content = visibleContent(message, report, canaries)
|
|
232
|
+
const originalBlocks = Array.isArray(message?.content) ? message.content.length : 0
|
|
233
|
+
report.omitted_blocks += Math.max(0, originalBlocks - content.length)
|
|
234
|
+
if (!content.length) continue
|
|
235
|
+
if (visibleTranscript.length >= MAX_TRANSCRIPT_MESSAGES) {
|
|
236
|
+
report.truncations += 1
|
|
237
|
+
break
|
|
238
|
+
}
|
|
239
|
+
visibleTranscript.push({
|
|
240
|
+
event_seq: event.seq,
|
|
241
|
+
message_ref: messageRef(message),
|
|
242
|
+
role,
|
|
243
|
+
content,
|
|
244
|
+
time: isoTime(event.time),
|
|
245
|
+
})
|
|
246
|
+
}
|
|
247
|
+
const initialGoal = visibleTranscript.find(message => message.role === 'user')?.content
|
|
248
|
+
?.map(block => block.text).join('\n') ?? ''
|
|
249
|
+
const sanitizedTitle = sanitizeText(
|
|
250
|
+
initialGoal.split('\n').find(Boolean) ?? 'Historical DSH Session',
|
|
251
|
+
120,
|
|
252
|
+
canaries,
|
|
253
|
+
)
|
|
254
|
+
report.replacements += sanitizedTitle.replacements
|
|
255
|
+
if (sanitizedTitle.truncated) report.truncations += 1
|
|
256
|
+
|
|
257
|
+
const agentPreset = selected.index.effectiveAgentPreset ?? selected.header.agentPreset
|
|
258
|
+
const observation = {
|
|
259
|
+
schema_version: 1,
|
|
260
|
+
protocol: 'dsh-session-observation/v1',
|
|
261
|
+
record_kind: 'dsh-session',
|
|
262
|
+
execution_mode: 'observe-existing',
|
|
263
|
+
trial_id: selected.trialId,
|
|
264
|
+
source: {
|
|
265
|
+
ref: selected.sourceRef,
|
|
266
|
+
captured_through_seq: selected.capturedThroughSeq,
|
|
267
|
+
source_digest: selected.sourceDigest,
|
|
268
|
+
created_at: isoTime(selected.header.createdAt),
|
|
269
|
+
last_activity_at: isoTime(selected.index.lastActivityAt),
|
|
270
|
+
last_turn_reason: selected.index.lastTurnReason,
|
|
271
|
+
session_format_version: selected.header.version,
|
|
272
|
+
},
|
|
273
|
+
generator: {
|
|
274
|
+
agent_preset: agentPreset
|
|
275
|
+
? sanitizeIdentity(agentPreset, report, canaries)
|
|
276
|
+
: null,
|
|
277
|
+
model_segments: modelSegments(selected, report, canaries),
|
|
278
|
+
},
|
|
279
|
+
task: {
|
|
280
|
+
title: sanitizedTitle.text || 'Historical DSH Session',
|
|
281
|
+
initial_user_goal: initialGoal,
|
|
282
|
+
turn_count: selected.index.turnCount,
|
|
283
|
+
},
|
|
284
|
+
visible_transcript: visibleTranscript,
|
|
285
|
+
execution: {
|
|
286
|
+
tools: toolEvidence(selected.events),
|
|
287
|
+
turns: turnEvidence(selected.events),
|
|
288
|
+
usage: usageEvidence(selected.events),
|
|
289
|
+
},
|
|
290
|
+
feedback: { items: sanitizeFeedback(feedbackItems, report, canaries) },
|
|
291
|
+
completeness: {
|
|
292
|
+
transcript_complete: visibleTranscript.length < MAX_TRANSCRIPT_MESSAGES,
|
|
293
|
+
tool_payloads_complete: false,
|
|
294
|
+
attachments_complete: false,
|
|
295
|
+
truncations: report.truncations ? [`${report.truncations} bounded text projection(s)`] : [],
|
|
296
|
+
},
|
|
297
|
+
redaction: report,
|
|
298
|
+
}
|
|
299
|
+
observation.digest = canonicalDigest(observation, 'harbor-dsh-session-observation-v1')
|
|
300
|
+
assertNoSecret(observation, canaries)
|
|
301
|
+
return observation
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
export function scanForCredentialCanaries(value) {
|
|
305
|
+
try {
|
|
306
|
+
assertNoSecret(value)
|
|
307
|
+
return []
|
|
308
|
+
} catch (error) {
|
|
309
|
+
return [error.message]
|
|
310
|
+
}
|
|
311
|
+
}
|
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
import { createHash, randomBytes } from 'node:crypto'
|
|
2
|
+
import path from 'node:path'
|
|
3
|
+
|
|
4
|
+
import { foldSessionDiagnosticIndex } from './session-projection.js'
|
|
5
|
+
|
|
6
|
+
function canonicalize(value) {
|
|
7
|
+
if (Array.isArray(value)) return value.map(canonicalize)
|
|
8
|
+
if (value && typeof value === 'object') {
|
|
9
|
+
return Object.fromEntries(
|
|
10
|
+
Object.keys(value).sort().map(key => [key, canonicalize(value[key])]),
|
|
11
|
+
)
|
|
12
|
+
}
|
|
13
|
+
return value
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function canonicalDigest(value, namespace) {
|
|
17
|
+
const body = JSON.stringify(canonicalize(value))
|
|
18
|
+
return `sha256:${createHash('sha256').update(namespace).update('\0').update(body).digest('hex')}`
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function sessionHeaderIdentity(header, effectiveAgentPreset) {
|
|
22
|
+
return {
|
|
23
|
+
version: header?.version,
|
|
24
|
+
id: header?.id,
|
|
25
|
+
createdAt: header?.createdAt,
|
|
26
|
+
cwd: header?.cwd,
|
|
27
|
+
parentSession: header?.parentSession,
|
|
28
|
+
seedLength: header?.seedLength,
|
|
29
|
+
origin: header?.origin,
|
|
30
|
+
delegationDepth: header?.delegationDepth,
|
|
31
|
+
...(effectiveAgentPreset === undefined ? {} : { agentPreset: effectiveAgentPreset }),
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function sameProjectRoot(value, projectRoot) {
|
|
36
|
+
if (typeof value !== 'string' || !path.isAbsolute(value)) return false
|
|
37
|
+
return path.resolve(value) === path.resolve(projectRoot)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function isoTime(value) {
|
|
41
|
+
return Number.isSafeInteger(value) && value > 0 ? new Date(value).toISOString() : null
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function safeIdentity(value, rawSessionId) {
|
|
45
|
+
const text = typeof value === 'string' ? value : ''
|
|
46
|
+
if (rawSessionId && text.includes(rawSessionId)) return '[redacted-identity]'
|
|
47
|
+
if (
|
|
48
|
+
/(?:api[_-]?key|token|secret|password|authorization|bearer\s+)/i.test(text)
|
|
49
|
+
|| /(?:^|[\\/])(?:Users|home|private|tmp|var|etc|opt|Volumes)(?:[\\/]|$)/.test(text)
|
|
50
|
+
|| /\b(?:sk|rk|pk)-[A-Za-z0-9_-]{12,}\b/.test(text)
|
|
51
|
+
) return '[redacted-identity]'
|
|
52
|
+
return text.slice(0, 160)
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function mapConcurrent(values, concurrency, mapper) {
|
|
56
|
+
const result = new Array(values.length)
|
|
57
|
+
let cursor = 0
|
|
58
|
+
const workers = Array.from({ length: Math.min(concurrency, values.length) }, async () => {
|
|
59
|
+
while (cursor < values.length) {
|
|
60
|
+
const index = cursor
|
|
61
|
+
cursor += 1
|
|
62
|
+
try {
|
|
63
|
+
result[index] = { status: 'fulfilled', value: await mapper(values[index], index) }
|
|
64
|
+
} catch (reason) {
|
|
65
|
+
result[index] = { status: 'rejected', reason }
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
})
|
|
69
|
+
await Promise.all(workers)
|
|
70
|
+
return result
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function publicSelection(item, index) {
|
|
74
|
+
const agentPreset = item.index.effectiveAgentPreset ?? item.header.agentPreset
|
|
75
|
+
return {
|
|
76
|
+
trialId: item.trialId,
|
|
77
|
+
title: `历史会话 ${index + 1}`,
|
|
78
|
+
createdAt: isoTime(item.header.createdAt),
|
|
79
|
+
lastActivityAt: isoTime(item.index.lastActivityAt),
|
|
80
|
+
turnCount: item.index.turnCount,
|
|
81
|
+
humanMessageCount: item.index.humanMessageCount,
|
|
82
|
+
assistantMessageCount: item.index.assistantMessageCount,
|
|
83
|
+
toolCallCount: item.index.toolCallCount,
|
|
84
|
+
lastTurnReason: item.index.lastTurnReason,
|
|
85
|
+
agentPreset: agentPreset ? safeIdentity(agentPreset, item.rawSessionId) : null,
|
|
86
|
+
modelRoutes: item.index.modelRoutes.map(route => ({
|
|
87
|
+
provider: safeIdentity(route.provider, item.rawSessionId),
|
|
88
|
+
model: safeIdentity(route.model, item.rawSessionId),
|
|
89
|
+
...(route.reasoning_effort ? { reasoning_effort: safeIdentity(route.reasoning_effort, item.rawSessionId) } : {}),
|
|
90
|
+
})),
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export async function selectRecentSessions({
|
|
95
|
+
sessionQuery,
|
|
96
|
+
projectRoot,
|
|
97
|
+
currentSessionId,
|
|
98
|
+
limit = 10,
|
|
99
|
+
maxSessionReads = 100,
|
|
100
|
+
concurrency = 4,
|
|
101
|
+
createdAfter,
|
|
102
|
+
signal,
|
|
103
|
+
}) {
|
|
104
|
+
if (!sessionQuery || typeof sessionQuery.readSession !== 'function') {
|
|
105
|
+
throw new Error('DSH_SESSION_QUERY_UNAVAILABLE: this DSH Profile does not expose the Session Query service')
|
|
106
|
+
}
|
|
107
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 10) {
|
|
108
|
+
throw new Error('SESSION_LIMIT_INVALID: limit must be an integer from 1 to 10')
|
|
109
|
+
}
|
|
110
|
+
if (createdAfter !== undefined && (!Number.isSafeInteger(createdAfter) || createdAfter < 0)) {
|
|
111
|
+
throw new Error('SESSION_CREATED_AFTER_INVALID: createdAfter must be a valid timestamp')
|
|
112
|
+
}
|
|
113
|
+
const listed = typeof sessionQuery.filterSessions === 'function'
|
|
114
|
+
? await sessionQuery.filterSessions([
|
|
115
|
+
{ kind: 'cwd', values: [projectRoot] },
|
|
116
|
+
...(createdAfter === undefined ? [] : [{ kind: 'created-at', from: createdAfter }]),
|
|
117
|
+
], signal)
|
|
118
|
+
: await sessionQuery.listSessions(signal)
|
|
119
|
+
const excludedCounts = {
|
|
120
|
+
outsideWorkspace: 0,
|
|
121
|
+
beforeCreatedAfter: 0,
|
|
122
|
+
currentSession: 0,
|
|
123
|
+
subagent: 0,
|
|
124
|
+
forkOrChild: 0,
|
|
125
|
+
openTurn: 0,
|
|
126
|
+
noDirectHumanInput: 0,
|
|
127
|
+
noAssistantOutput: 0,
|
|
128
|
+
userAborted: 0,
|
|
129
|
+
harborInternal: 0,
|
|
130
|
+
empty: 0,
|
|
131
|
+
unreadable: 0,
|
|
132
|
+
}
|
|
133
|
+
const candidates = []
|
|
134
|
+
for (const record of Array.isArray(listed) ? listed : []) {
|
|
135
|
+
const header = record?.header ?? {}
|
|
136
|
+
if (!sameProjectRoot(header.cwd, projectRoot)) {
|
|
137
|
+
excludedCounts.outsideWorkspace += 1
|
|
138
|
+
} else if (header.id === currentSessionId) {
|
|
139
|
+
excludedCounts.currentSession += 1
|
|
140
|
+
} else if (createdAfter !== undefined && Number(header.createdAt) < createdAfter) {
|
|
141
|
+
excludedCounts.beforeCreatedAfter += 1
|
|
142
|
+
} else if (header.origin === 'subagent') {
|
|
143
|
+
excludedCounts.subagent += 1
|
|
144
|
+
} else if (
|
|
145
|
+
header.parentSession !== undefined
|
|
146
|
+
|| Number(header.seedLength ?? 0) > 0
|
|
147
|
+
|| Number(header.delegationDepth ?? 0) > 0
|
|
148
|
+
) {
|
|
149
|
+
excludedCounts.forkOrChild += 1
|
|
150
|
+
} else {
|
|
151
|
+
candidates.push(record)
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
if (candidates.length > maxSessionReads) {
|
|
155
|
+
throw new Error(
|
|
156
|
+
`SESSION_SELECTION_TOO_EXPENSIVE: ${candidates.length} exact Session reads exceed maxSessionReads=${maxSessionReads}; preview again with createdAfter to narrow the scan`,
|
|
157
|
+
)
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
const snapshots = await mapConcurrent(candidates, concurrency, async record => {
|
|
161
|
+
const snapshot = await sessionQuery.readSession(record.header.id)
|
|
162
|
+
if (snapshot?.session?.id !== record.header.id) {
|
|
163
|
+
throw new Error('Session Query returned a mismatched Session header')
|
|
164
|
+
}
|
|
165
|
+
if (!sameProjectRoot(snapshot.session.cwd, projectRoot)) {
|
|
166
|
+
throw new Error('Session Query changed the Session workspace boundary')
|
|
167
|
+
}
|
|
168
|
+
return snapshot
|
|
169
|
+
})
|
|
170
|
+
const eligible = []
|
|
171
|
+
for (const outcome of snapshots) {
|
|
172
|
+
if (outcome.status === 'rejected') {
|
|
173
|
+
excludedCounts.unreadable += 1
|
|
174
|
+
continue
|
|
175
|
+
}
|
|
176
|
+
const snapshot = outcome.value
|
|
177
|
+
const index = foldSessionDiagnosticIndex(snapshot.events, snapshot.session)
|
|
178
|
+
if (index.lastSeq === null) {
|
|
179
|
+
excludedCounts.empty += 1
|
|
180
|
+
continue
|
|
181
|
+
}
|
|
182
|
+
if (index.openTurn) {
|
|
183
|
+
excludedCounts.openTurn += 1
|
|
184
|
+
continue
|
|
185
|
+
}
|
|
186
|
+
if (index.lastTurnReason === 'aborted') {
|
|
187
|
+
excludedCounts.userAborted += 1
|
|
188
|
+
continue
|
|
189
|
+
}
|
|
190
|
+
if (index.humanMessageCount < 1) {
|
|
191
|
+
excludedCounts.noDirectHumanInput += 1
|
|
192
|
+
continue
|
|
193
|
+
}
|
|
194
|
+
if (index.assistantMessageCount < 1) {
|
|
195
|
+
excludedCounts.noAssistantOutput += 1
|
|
196
|
+
continue
|
|
197
|
+
}
|
|
198
|
+
if (index.hasHarborToolCall) {
|
|
199
|
+
excludedCounts.harborInternal += 1
|
|
200
|
+
continue
|
|
201
|
+
}
|
|
202
|
+
if (/harbor/i.test(String(index.effectiveAgentPreset ?? ''))) {
|
|
203
|
+
excludedCounts.harborInternal += 1
|
|
204
|
+
continue
|
|
205
|
+
}
|
|
206
|
+
const header = sessionHeaderIdentity(snapshot.session, index.effectiveAgentPreset)
|
|
207
|
+
const sourceDigest = canonicalDigest(
|
|
208
|
+
{ session: header, events: snapshot.events },
|
|
209
|
+
'harbor-dsh-session-source-v1',
|
|
210
|
+
)
|
|
211
|
+
const sourceRef = canonicalDigest(
|
|
212
|
+
{ id: snapshot.session.id, header },
|
|
213
|
+
'harbor-dsh-session-source-ref-v1',
|
|
214
|
+
)
|
|
215
|
+
eligible.push({
|
|
216
|
+
rawSessionId: snapshot.session.id,
|
|
217
|
+
header,
|
|
218
|
+
events: snapshot.events,
|
|
219
|
+
index,
|
|
220
|
+
sourceDigest,
|
|
221
|
+
sourceRef,
|
|
222
|
+
capturedThroughSeq: index.lastSeq,
|
|
223
|
+
trialId: `session-${sourceRef.slice('sha256:'.length, 'sha256:'.length + 12)}`,
|
|
224
|
+
})
|
|
225
|
+
}
|
|
226
|
+
eligible.sort((left, right) => (
|
|
227
|
+
right.index.lastActivityAt - left.index.lastActivityAt
|
|
228
|
+
|| left.sourceRef.localeCompare(right.sourceRef)
|
|
229
|
+
))
|
|
230
|
+
const selected = eligible.slice(0, limit)
|
|
231
|
+
return {
|
|
232
|
+
selected,
|
|
233
|
+
publicSelected: selected.map(publicSelection),
|
|
234
|
+
excludedCounts,
|
|
235
|
+
warnings: excludedCounts.unreadable
|
|
236
|
+
? [`${excludedCounts.unreadable} Session(s) could not be read and were excluded.`]
|
|
237
|
+
: [],
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
export function verifySessionSnapshot(expected, snapshot, projectRoot) {
|
|
242
|
+
if (snapshot?.session?.id !== expected.rawSessionId || !sameProjectRoot(snapshot?.session?.cwd, projectRoot)) {
|
|
243
|
+
return false
|
|
244
|
+
}
|
|
245
|
+
const index = foldSessionDiagnosticIndex(snapshot.events, snapshot.session)
|
|
246
|
+
if (index.lastSeq !== expected.capturedThroughSeq || index.openTurn) return false
|
|
247
|
+
const digest = canonicalDigest(
|
|
248
|
+
{ session: sessionHeaderIdentity(snapshot.session, index.effectiveAgentPreset), events: snapshot.events },
|
|
249
|
+
'harbor-dsh-session-source-v1',
|
|
250
|
+
)
|
|
251
|
+
return digest === expected.sourceDigest
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
export class SessionSelectionTokenStore {
|
|
255
|
+
constructor({ ttlMs = 15 * 60 * 1000, now = () => Date.now(), randomToken } = {}) {
|
|
256
|
+
this.ttlMs = ttlMs
|
|
257
|
+
this.now = now
|
|
258
|
+
this.randomToken = randomToken ?? (() => randomBytes(32).toString('base64url'))
|
|
259
|
+
this.tokens = new Map()
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
issue(value) {
|
|
263
|
+
this.purge()
|
|
264
|
+
const token = this.randomToken()
|
|
265
|
+
const expiresAt = this.now() + this.ttlMs
|
|
266
|
+
this.tokens.set(token, { ...value, expiresAt })
|
|
267
|
+
return { token, expiresAt }
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
consume(token, { ownerSessionId, projectRoot }) {
|
|
271
|
+
const stored = this.tokens.get(token)
|
|
272
|
+
if (!stored) throw new Error('SESSION_SELECTION_TOKEN_INVALID: preview again before running the diagnostic')
|
|
273
|
+
// Consume before validation so a stolen or failed token cannot be replayed.
|
|
274
|
+
this.tokens.delete(token)
|
|
275
|
+
if (stored.ownerSessionId !== ownerSessionId) {
|
|
276
|
+
throw new Error('SESSION_SELECTION_TOKEN_OWNER_MISMATCH: the token belongs to another Agent Session')
|
|
277
|
+
}
|
|
278
|
+
if (path.resolve(stored.projectRoot) !== path.resolve(projectRoot)) {
|
|
279
|
+
throw new Error('SESSION_SELECTION_TOKEN_WORKSPACE_MISMATCH: the token belongs to another workspace')
|
|
280
|
+
}
|
|
281
|
+
if (stored.expiresAt <= this.now()) {
|
|
282
|
+
throw new Error('SESSION_SELECTION_TOKEN_EXPIRED: preview again before running the diagnostic')
|
|
283
|
+
}
|
|
284
|
+
this.purge()
|
|
285
|
+
return stored
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
purge() {
|
|
289
|
+
const now = this.now()
|
|
290
|
+
for (const [token, value] of this.tokens) {
|
|
291
|
+
if (value.expiresAt <= now) this.tokens.delete(token)
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
package/lib/setup.js
CHANGED
|
@@ -281,9 +281,7 @@ export async function setupIntegration(raw = {}, dependencies = {}) {
|
|
|
281
281
|
const patchChanged = await writeProfilePatch(config.patchFile, config)
|
|
282
282
|
const harborVersion = await run(config.harborBin, ['--version'], { timeoutMs: 10_000 })
|
|
283
283
|
const plugins = await run(config.harborBin, ['plugins', 'list'], { timeoutMs: 10_000 })
|
|
284
|
-
|
|
285
|
-
throw new Error('Harbor installed, but its dsh-evolution plugin entry point was not discovered')
|
|
286
|
-
}
|
|
284
|
+
verifyHarborPlugins(plugins.stdout)
|
|
287
285
|
await run(config.harborDshBin, ['--help'], { timeoutMs: 10_000 })
|
|
288
286
|
|
|
289
287
|
return {
|
|
@@ -295,6 +293,14 @@ export async function setupIntegration(raw = {}, dependencies = {}) {
|
|
|
295
293
|
}
|
|
296
294
|
}
|
|
297
295
|
|
|
296
|
+
export function verifyHarborPlugins(output) {
|
|
297
|
+
const missing = ['dsh-evolution', 'dsh-historical-evaluation']
|
|
298
|
+
.filter(name => !String(output).includes(name))
|
|
299
|
+
if (missing.length) {
|
|
300
|
+
throw new Error(`Harbor installed, but these plugin entry points were not discovered: ${missing.join(', ')}`)
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
|
|
298
304
|
function shellQuote(value) {
|
|
299
305
|
return `'${String(value).replaceAll("'", `'"'"'`)}'`
|
|
300
306
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-harbor-evolution",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "DeepSeek Harness plugin and
|
|
3
|
+
"version": "0.8.1",
|
|
4
|
+
"description": "DeepSeek Harness plugin and Skill for Harbor Candidate and Historical Session evaluation workflows.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "index.js",
|
|
7
7
|
"exports": {
|
|
@@ -9,6 +9,11 @@
|
|
|
9
9
|
"./client": "./lib/client.js",
|
|
10
10
|
"./cordis.patch.yml": "./cordis.patch.yml",
|
|
11
11
|
"./schemas/evaluation-result.schema.json": "./schemas/evaluation-result.schema.json",
|
|
12
|
+
"./schemas/evaluation-result-v2.schema.json": "./schemas/evaluation-result-v2.schema.json",
|
|
13
|
+
"./schemas/historical-generation-batch.schema.json": "./schemas/historical-generation-batch.schema.json",
|
|
14
|
+
"./schemas/dsh-session-observation.schema.json": "./schemas/dsh-session-observation.schema.json",
|
|
15
|
+
"./schemas/historical-evaluation-context.schema.json": "./schemas/historical-evaluation-context.schema.json",
|
|
16
|
+
"./schemas/historical-evaluation-summary.schema.json": "./schemas/historical-evaluation-summary.schema.json",
|
|
12
17
|
"./schemas/ground-truth.schema.json": "./schemas/ground-truth.schema.json",
|
|
13
18
|
"./schemas/evaluator-observations.schema.json": "./schemas/evaluator-observations.schema.json",
|
|
14
19
|
"./schemas/meta-evaluation-report.schema.json": "./schemas/meta-evaluation-report.schema.json",
|