dsh-harbor-evolution 0.7.3 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -3
- package/index.js +40 -0
- package/lib/client.js +96 -17
- package/lib/dashboard.js +167 -15
- package/lib/evolution.js +189 -0
- package/lib/model-runtime.js +11 -7
- package/lib/session-diagnostic.js +320 -0
- package/lib/session-materializer.js +194 -0
- package/lib/session-projection.js +161 -0
- package/lib/session-redaction.js +311 -0
- package/lib/session-selection.js +294 -0
- package/lib/setup.js +9 -3
- package/package.json +6 -1
- package/schemas/dsh-session-observation.schema.json +69 -0
- package/schemas/evaluation-result-v2.schema.json +45 -0
- package/schemas/historical-evaluation-context.schema.json +66 -0
- package/schemas/historical-evaluation-summary.schema.json +49 -0
- package/schemas/historical-generation-batch.schema.json +76 -0
- package/skills/evolve-agent-with-harbor/SKILL.md +47 -7
- package/skills/evolve-agent-with-harbor/evals/evals.json +9 -6
- package/skills/evolve-agent-with-harbor/references/evaluator-upgrade.md +3 -1
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
import path from 'node:path'
|
|
2
|
+
|
|
3
|
+
import { buildHistoricalGenerationBatch, writePrivateHistoricalBatch } from './session-materializer.js'
|
|
4
|
+
import { buildSessionObservation } from './session-redaction.js'
|
|
5
|
+
import {
|
|
6
|
+
canonicalDigest,
|
|
7
|
+
SessionSelectionTokenStore,
|
|
8
|
+
selectRecentSessions,
|
|
9
|
+
verifySessionSnapshot,
|
|
10
|
+
} from './session-selection.js'
|
|
11
|
+
|
|
12
|
+
function capability(ctx, name) {
|
|
13
|
+
try {
|
|
14
|
+
return typeof ctx?.get === 'function' ? ctx.get(name) ?? ctx[name] : ctx?.[name]
|
|
15
|
+
} catch {
|
|
16
|
+
return ctx?.[name]
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function executionIdentity(exec) {
|
|
21
|
+
const header = exec?.agent?.session?.header
|
|
22
|
+
if (typeof header?.cwd !== 'string' || !path.isAbsolute(header.cwd)) {
|
|
23
|
+
throw new Error('Harbor Session tools require an Agent Session with an absolute working directory')
|
|
24
|
+
}
|
|
25
|
+
if (typeof header.id !== 'string' || !header.id) {
|
|
26
|
+
throw new Error('Harbor Session tools require the calling Agent Session identity')
|
|
27
|
+
}
|
|
28
|
+
return { projectRoot: path.resolve(header.cwd), ownerSessionId: header.id }
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function feedbackItems(result) {
|
|
32
|
+
return result?.ok === true && Array.isArray(result.value?.items) ? result.value.items : []
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
async function readFeedback(service, sessionId) {
|
|
36
|
+
if (!service || typeof service.list !== 'function') return { items: [], available: false, failed: false }
|
|
37
|
+
try {
|
|
38
|
+
const result = await service.list({ sessionId })
|
|
39
|
+
return {
|
|
40
|
+
items: feedbackItems(result),
|
|
41
|
+
available: result?.ok === true,
|
|
42
|
+
failed: result?.ok === false,
|
|
43
|
+
}
|
|
44
|
+
} catch {
|
|
45
|
+
return { items: [], available: true, failed: true }
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function withoutRawEvents(selection) {
|
|
50
|
+
const { events: _events, ...rest } = selection
|
|
51
|
+
return rest
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function parseCreatedAfter(value) {
|
|
55
|
+
if (value === undefined || value === null || value === '') return undefined
|
|
56
|
+
if (typeof value !== 'string') {
|
|
57
|
+
throw new Error('SESSION_CREATED_AFTER_INVALID: createdAfter must be an ISO-8601 string')
|
|
58
|
+
}
|
|
59
|
+
const parsed = Date.parse(value)
|
|
60
|
+
if (!Number.isSafeInteger(parsed) || parsed < 0) {
|
|
61
|
+
throw new Error('SESSION_CREATED_AFTER_INVALID: createdAfter must be a valid ISO-8601 timestamp')
|
|
62
|
+
}
|
|
63
|
+
return parsed
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function feedbackDigest(observation) {
|
|
67
|
+
return canonicalDigest(
|
|
68
|
+
{
|
|
69
|
+
available: observation.available,
|
|
70
|
+
failed: observation.failed,
|
|
71
|
+
items: observation.items,
|
|
72
|
+
},
|
|
73
|
+
'harbor-dsh-session-feedback-snapshot-v1',
|
|
74
|
+
)
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function requestedJudge(args) {
|
|
78
|
+
if (Boolean(args.evaluatorProvider) !== Boolean(args.evaluatorModel)) {
|
|
79
|
+
throw new Error('EVALUATOR_MODEL_INVALID: evaluatorProvider and evaluatorModel must be supplied together')
|
|
80
|
+
}
|
|
81
|
+
if (args.evaluatorReasoningEffort !== undefined && !args.evaluatorProvider) {
|
|
82
|
+
throw new Error('EVALUATOR_MODEL_INVALID: evaluatorReasoningEffort requires an explicit evaluatorProvider and evaluatorModel')
|
|
83
|
+
}
|
|
84
|
+
return {
|
|
85
|
+
candidateProvider: args.evaluatorProvider,
|
|
86
|
+
candidateModel: args.evaluatorModel,
|
|
87
|
+
candidateReasoningEffort: args.evaluatorReasoningEffort,
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async function resolveJudge(modelRuntime, args) {
|
|
92
|
+
const requested = requestedJudge(args)
|
|
93
|
+
if (requested.candidateProvider) return modelRuntime.resolve(requested)
|
|
94
|
+
if (typeof modelRuntime.resolveCurrent === 'function') {
|
|
95
|
+
return modelRuntime.resolveCurrent()
|
|
96
|
+
}
|
|
97
|
+
const current = await modelRuntime.currentBinding()
|
|
98
|
+
return modelRuntime.resolve({
|
|
99
|
+
candidateProvider: current.provider,
|
|
100
|
+
candidateModel: current.model,
|
|
101
|
+
candidateReasoningEffort: current.reasoning_effort,
|
|
102
|
+
})
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function judgeIdentity(binding, selections) {
|
|
106
|
+
if (!binding?.provider || !binding?.model) {
|
|
107
|
+
throw new Error('EVALUATOR_MODEL_INVALID: Judge resolution returned no provider/model identity')
|
|
108
|
+
}
|
|
109
|
+
const route = `${binding.provider}/${binding.model}`
|
|
110
|
+
const generatorRoutes = new Set(
|
|
111
|
+
selections.flatMap(item => item.index.modelRoutes.map(value => `${value.provider}/${value.model}`)),
|
|
112
|
+
)
|
|
113
|
+
return {
|
|
114
|
+
evaluator: { id: 'dsh-session-historical-evaluator', version: '1.0.0' },
|
|
115
|
+
judge: {
|
|
116
|
+
provider: binding.provider,
|
|
117
|
+
model: binding.model,
|
|
118
|
+
...(binding.reasoning_effort === undefined
|
|
119
|
+
? {}
|
|
120
|
+
: { reasoning_effort: binding.reasoning_effort }),
|
|
121
|
+
transport: 'dsh-host-broker',
|
|
122
|
+
protocol: 'dsh-host-model-gateway/v1',
|
|
123
|
+
},
|
|
124
|
+
coupling: generatorRoutes.size === 0
|
|
125
|
+
? 'generator-model-unknown-diagnostic-only'
|
|
126
|
+
: generatorRoutes.has(route)
|
|
127
|
+
? 'same-host-model-diagnostic-only'
|
|
128
|
+
: 'independent-historical-judge',
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export class SessionDiagnosticService {
|
|
133
|
+
constructor({
|
|
134
|
+
ctx,
|
|
135
|
+
config,
|
|
136
|
+
modelRuntime,
|
|
137
|
+
runHistoricalEvaluation,
|
|
138
|
+
tokenStore,
|
|
139
|
+
now = () => new Date(),
|
|
140
|
+
}) {
|
|
141
|
+
this.ctx = ctx
|
|
142
|
+
this.config = config
|
|
143
|
+
this.modelRuntime = modelRuntime
|
|
144
|
+
this.runHistoricalEvaluation = runHistoricalEvaluation
|
|
145
|
+
this.tokens = tokenStore ?? new SessionSelectionTokenStore()
|
|
146
|
+
this.now = now
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async preview(args = {}, exec) {
|
|
150
|
+
const identity = executionIdentity(exec)
|
|
151
|
+
const sessionQuery = capability(this.ctx, 'sessionQuery')
|
|
152
|
+
const limit = args.limit ?? 10
|
|
153
|
+
const createdAfter = parseCreatedAfter(args.createdAfter)
|
|
154
|
+
const result = await selectRecentSessions({
|
|
155
|
+
sessionQuery,
|
|
156
|
+
projectRoot: identity.projectRoot,
|
|
157
|
+
currentSessionId: identity.ownerSessionId,
|
|
158
|
+
limit,
|
|
159
|
+
maxSessionReads: this.config.sessionMaxReads ?? 100,
|
|
160
|
+
concurrency: this.config.sessionReadConcurrency ?? 4,
|
|
161
|
+
createdAfter,
|
|
162
|
+
signal: exec?.signal,
|
|
163
|
+
})
|
|
164
|
+
if (!result.selected.length) {
|
|
165
|
+
throw new Error('NO_ELIGIBLE_SESSIONS: no completed top-level DSH Sessions with direct human input and assistant output were found in this workspace')
|
|
166
|
+
}
|
|
167
|
+
const judgeBinding = await resolveJudge(this.modelRuntime, args)
|
|
168
|
+
const includeFeedback = args.includeFeedback !== false
|
|
169
|
+
const feedback = capability(this.ctx, 'messageFeedback')
|
|
170
|
+
const feedbackObservations = includeFeedback
|
|
171
|
+
? await Promise.all(result.selected.map(item => readFeedback(feedback, item.rawSessionId)))
|
|
172
|
+
: result.selected.map(() => ({ items: [], available: false, failed: false }))
|
|
173
|
+
const feedbackSnapshots = feedbackObservations.map(observation => ({
|
|
174
|
+
available: observation.available,
|
|
175
|
+
failed: observation.failed,
|
|
176
|
+
digest: feedbackDigest(observation),
|
|
177
|
+
}))
|
|
178
|
+
const selected = result.publicSelected.map((item, index) => ({
|
|
179
|
+
...item,
|
|
180
|
+
feedback: {
|
|
181
|
+
available: feedbackObservations[index].available,
|
|
182
|
+
positive: feedbackObservations[index].items.filter(value => value.rating === 'positive').length,
|
|
183
|
+
negative: feedbackObservations[index].items.filter(value => value.rating === 'negative').length,
|
|
184
|
+
},
|
|
185
|
+
}))
|
|
186
|
+
const evaluation = judgeIdentity(judgeBinding, result.selected)
|
|
187
|
+
const issued = this.tokens.issue({
|
|
188
|
+
...identity,
|
|
189
|
+
selection: result.selected.map(withoutRawEvents),
|
|
190
|
+
feedbackSnapshots,
|
|
191
|
+
judgeBinding,
|
|
192
|
+
evaluation,
|
|
193
|
+
parameters: { limit, includeFeedback, createdAfter, scope: 'exact-cwd', order: 'last-activity-desc' },
|
|
194
|
+
})
|
|
195
|
+
const warnings = [...result.warnings]
|
|
196
|
+
if (feedbackObservations.some(item => item.failed)) {
|
|
197
|
+
warnings.push('Some Message Feedback could not be read; the Session sample remains usable without it.')
|
|
198
|
+
}
|
|
199
|
+
warnings.push('Frozen Session observations remain local under .harbor/private and the Harbor jobs directory; review repository ignore and retention policy before committing artifacts.')
|
|
200
|
+
return {
|
|
201
|
+
schema_version: 1,
|
|
202
|
+
capability: 'historical-generation-evaluation',
|
|
203
|
+
jobKind: 'historical-generation-evaluation',
|
|
204
|
+
projectRoot: identity.projectRoot,
|
|
205
|
+
scope: 'exact-cwd',
|
|
206
|
+
order: 'last-activity-desc',
|
|
207
|
+
...(createdAfter === undefined ? {} : { createdAfter: new Date(createdAfter).toISOString() }),
|
|
208
|
+
executionMode: 'observe-existing',
|
|
209
|
+
promotionEligible: false,
|
|
210
|
+
evaluationLevel: 'trial',
|
|
211
|
+
selectionToken: issued.token,
|
|
212
|
+
expiresAt: new Date(issued.expiresAt).toISOString(),
|
|
213
|
+
selected,
|
|
214
|
+
excludedCounts: result.excludedCounts,
|
|
215
|
+
warnings,
|
|
216
|
+
estimatedJudgeRequests: selected.length,
|
|
217
|
+
estimatedMaxBytes: selected.length * 512 * 1024,
|
|
218
|
+
evaluation,
|
|
219
|
+
retention: {
|
|
220
|
+
privateEvidence: '.harbor/private/session-batches',
|
|
221
|
+
jobEvidence: this.config.jobsDir ?? 'jobs',
|
|
222
|
+
vcsPolicy: 'an ignore-all file is created only when .harbor/private/.gitignore is absent; existing private rules and jobs retention/VCS policy remain project-owned',
|
|
223
|
+
},
|
|
224
|
+
confirmation: `Run 1 historical-generation-evaluation Job with ${selected.length} immutable Trial(s) using ${evaluation.evaluator.id}@${evaluation.evaluator.version} and Judge ${evaluation.judge.provider}/${evaluation.judge.model} (${evaluation.coupling}); no Candidate will be executed or promoted.`,
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
async run(args = {}, exec) {
|
|
229
|
+
const identity = executionIdentity(exec)
|
|
230
|
+
if (args.stackPath !== undefined) {
|
|
231
|
+
throw new Error('HISTORICAL_CUSTOM_STACK_UNSUPPORTED: the first release binds the materialized Broker Evaluator and Stack as one immutable unit')
|
|
232
|
+
}
|
|
233
|
+
if (
|
|
234
|
+
args.evaluatorProvider !== undefined
|
|
235
|
+
|| args.evaluatorModel !== undefined
|
|
236
|
+
|| args.evaluatorReasoningEffort !== undefined
|
|
237
|
+
) {
|
|
238
|
+
throw new Error('HISTORICAL_JUDGE_NOT_CONFIRMED: choose the Judge during Preview, then Run with only the confirmed selectionToken')
|
|
239
|
+
}
|
|
240
|
+
const token = String(args.selectionToken ?? '')
|
|
241
|
+
if (!token) throw new Error('selectionToken is required; call harbor_session_diagnostic_preview first')
|
|
242
|
+
const selectedState = this.tokens.consume(token, identity)
|
|
243
|
+
const sessionQuery = capability(this.ctx, 'sessionQuery')
|
|
244
|
+
if (!sessionQuery || typeof sessionQuery.readSession !== 'function') {
|
|
245
|
+
throw new Error('DSH_SESSION_QUERY_UNAVAILABLE: this DSH Profile does not expose the Session Query service')
|
|
246
|
+
}
|
|
247
|
+
const reads = await Promise.allSettled(
|
|
248
|
+
selectedState.selection.map(item => sessionQuery.readSession(item.rawSessionId)),
|
|
249
|
+
)
|
|
250
|
+
if (reads.some(item => item.status === 'rejected')) {
|
|
251
|
+
throw new Error('SESSION_SOURCE_READ_FAILED: at least one selected Session could not be re-read; no Batch was written')
|
|
252
|
+
}
|
|
253
|
+
const snapshots = reads.map(item => item.value)
|
|
254
|
+
if (snapshots.some((snapshot, index) => (
|
|
255
|
+
!verifySessionSnapshot(selectedState.selection[index], snapshot, identity.projectRoot)
|
|
256
|
+
))) {
|
|
257
|
+
throw new Error('SESSION_SAMPLE_CHANGED: at least one selected Session changed after Preview; no Batch was written, preview again')
|
|
258
|
+
}
|
|
259
|
+
const feedback = capability(this.ctx, 'messageFeedback')
|
|
260
|
+
const feedbackObservations = selectedState.parameters.includeFeedback
|
|
261
|
+
? await Promise.all(selectedState.selection.map(item => readFeedback(feedback, item.rawSessionId)))
|
|
262
|
+
: selectedState.selection.map(() => ({ items: [] }))
|
|
263
|
+
if (selectedState.parameters.includeFeedback) {
|
|
264
|
+
for (let index = 0; index < feedbackObservations.length; index += 1) {
|
|
265
|
+
const expected = selectedState.feedbackSnapshots[index]
|
|
266
|
+
const observed = feedbackObservations[index]
|
|
267
|
+
if (!expected || feedbackDigest(observed) !== expected.digest) {
|
|
268
|
+
throw new Error('SESSION_FEEDBACK_CHANGED: Message Feedback changed after Preview; no Batch was written, preview again')
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
const frozenSelections = selectedState.selection.map((item, index) => ({
|
|
273
|
+
...item,
|
|
274
|
+
events: snapshots[index].events,
|
|
275
|
+
}))
|
|
276
|
+
const observations = frozenSelections.map((item, index) => (
|
|
277
|
+
buildSessionObservation(item, feedbackObservations[index].items)
|
|
278
|
+
))
|
|
279
|
+
const judgeBinding = selectedState.judgeBinding
|
|
280
|
+
if (!judgeBinding?.provider || !judgeBinding?.model) {
|
|
281
|
+
throw new Error('HISTORICAL_JUDGE_NOT_CONFIRMED: preview again to freeze a valid Judge identity before writing the Batch')
|
|
282
|
+
}
|
|
283
|
+
const batch = buildHistoricalGenerationBatch({
|
|
284
|
+
projectRoot: identity.projectRoot,
|
|
285
|
+
selections: frozenSelections,
|
|
286
|
+
observations,
|
|
287
|
+
limit: selectedState.parameters.limit,
|
|
288
|
+
createdAfter: selectedState.parameters.createdAfter,
|
|
289
|
+
now: this.now(),
|
|
290
|
+
})
|
|
291
|
+
const written = await writePrivateHistoricalBatch({
|
|
292
|
+
projectRoot: identity.projectRoot,
|
|
293
|
+
batch,
|
|
294
|
+
observations,
|
|
295
|
+
})
|
|
296
|
+
const result = await this.runHistoricalEvaluation(
|
|
297
|
+
{ ...this.config, projectRoot: identity.projectRoot },
|
|
298
|
+
{
|
|
299
|
+
batchPath: written.batchPath,
|
|
300
|
+
batchDir: written.batchDir,
|
|
301
|
+
jobName: args.jobName,
|
|
302
|
+
judgeBinding,
|
|
303
|
+
},
|
|
304
|
+
this.modelRuntime,
|
|
305
|
+
)
|
|
306
|
+
return {
|
|
307
|
+
schema_version: 1,
|
|
308
|
+
jobKind: 'historical-generation-evaluation',
|
|
309
|
+
executionMode: 'observe-existing',
|
|
310
|
+
promotionEligible: false,
|
|
311
|
+
batch: {
|
|
312
|
+
id: batch.batch_id,
|
|
313
|
+
digest: batch.digest,
|
|
314
|
+
recordCount: batch.records.length,
|
|
315
|
+
path: path.relative(identity.projectRoot, written.batchPath).split(path.sep).join('/'),
|
|
316
|
+
},
|
|
317
|
+
...result,
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
}
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import { chmod, lstat, mkdir, mkdtemp, realpath, rename, rm, writeFile } from 'node:fs/promises'
|
|
2
|
+
import path from 'node:path'
|
|
3
|
+
|
|
4
|
+
import { canonicalDigest } from './session-selection.js'
|
|
5
|
+
import { DEFAULT_REDACTION_POLICY } from './session-redaction.js'
|
|
6
|
+
|
|
7
|
+
function isoCompact(now) {
|
|
8
|
+
return now.toISOString().replace(/\.\d{3}Z$/, 'Z').replace(/[-:]/g, '')
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function routeName(route) {
|
|
12
|
+
return `${route.provider}/${route.model}`
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function buildHistoricalGenerationBatch({ projectRoot, selections, observations, limit = 10, createdAfter, now = new Date() }) {
|
|
16
|
+
if (!Array.isArray(selections) || !selections.length || selections.length > 10) {
|
|
17
|
+
throw new Error('HISTORICAL_BATCH_SIZE_INVALID: a batch requires 1 to 10 Session observations')
|
|
18
|
+
}
|
|
19
|
+
if (!Array.isArray(observations) || observations.length !== selections.length) {
|
|
20
|
+
throw new Error('HISTORICAL_BATCH_OBSERVATION_MISMATCH')
|
|
21
|
+
}
|
|
22
|
+
const seed = canonicalDigest(
|
|
23
|
+
observations.map(observation => observation.digest),
|
|
24
|
+
'harbor-dsh-historical-batch-id-v1',
|
|
25
|
+
).slice('sha256:'.length, 'sha256:'.length + 8)
|
|
26
|
+
const batchId = `recent-${isoCompact(now)}-${seed}`.toLowerCase()
|
|
27
|
+
const records = selections.map((selection, index) => {
|
|
28
|
+
const observation = observations[index]
|
|
29
|
+
if (observation.trial_id !== selection.trialId) {
|
|
30
|
+
throw new Error('HISTORICAL_BATCH_TRIAL_ID_MISMATCH')
|
|
31
|
+
}
|
|
32
|
+
return {
|
|
33
|
+
trial_id: selection.trialId,
|
|
34
|
+
record_kind: 'dsh-session',
|
|
35
|
+
source_ref: selection.sourceRef,
|
|
36
|
+
captured_through_seq: selection.capturedThroughSeq,
|
|
37
|
+
source_digest: selection.sourceDigest,
|
|
38
|
+
observation_digest: observation.digest,
|
|
39
|
+
last_activity_at: observation.source.last_activity_at,
|
|
40
|
+
generator: {
|
|
41
|
+
agent_preset: observation.generator.agent_preset,
|
|
42
|
+
model_routes: [...new Map(observation.generator.model_segments.map(segment => {
|
|
43
|
+
const route = {
|
|
44
|
+
provider: segment.provider,
|
|
45
|
+
model: segment.model,
|
|
46
|
+
...(segment.reasoning_effort ? { reasoning_effort: segment.reasoning_effort } : {}),
|
|
47
|
+
}
|
|
48
|
+
return [routeName(route), route]
|
|
49
|
+
})).values()],
|
|
50
|
+
homogeneous: new Set(observation.generator.model_segments.map(routeName)).size <= 1,
|
|
51
|
+
},
|
|
52
|
+
observation_path: `sessions/${selection.trialId}.json`,
|
|
53
|
+
}
|
|
54
|
+
})
|
|
55
|
+
const presets = [...new Set(records.map(record => record.generator.agent_preset).filter(Boolean))].sort()
|
|
56
|
+
const routes = [...new Set(records.flatMap(record => record.generator.model_routes.map(routeName)))].sort()
|
|
57
|
+
const batch = {
|
|
58
|
+
schema_version: 1,
|
|
59
|
+
protocol: 'historical-generation-batch/v1',
|
|
60
|
+
batch_id: batchId,
|
|
61
|
+
created_at: now.toISOString(),
|
|
62
|
+
project: {
|
|
63
|
+
cwd_digest: canonicalDigest({ cwd: path.resolve(projectRoot) }, 'harbor-dsh-project-cwd-v1'),
|
|
64
|
+
},
|
|
65
|
+
selection: {
|
|
66
|
+
scope: 'exact-cwd',
|
|
67
|
+
order: 'last-activity-desc',
|
|
68
|
+
requested_limit: limit,
|
|
69
|
+
selected_count: records.length,
|
|
70
|
+
current_session_excluded: true,
|
|
71
|
+
...(createdAfter === undefined ? {} : { created_after: new Date(createdAfter).toISOString() }),
|
|
72
|
+
},
|
|
73
|
+
source: {
|
|
74
|
+
kind: 'dsh-session',
|
|
75
|
+
adapter: 'dsh-session-query',
|
|
76
|
+
session_format_versions: [...new Set(selections.map(item => item.header.version))].sort(),
|
|
77
|
+
},
|
|
78
|
+
redaction_policy: DEFAULT_REDACTION_POLICY,
|
|
79
|
+
records,
|
|
80
|
+
generator_population: {
|
|
81
|
+
homogeneous: presets.length <= 1 && routes.length <= 1,
|
|
82
|
+
agent_presets: presets,
|
|
83
|
+
model_routes: routes,
|
|
84
|
+
},
|
|
85
|
+
}
|
|
86
|
+
const serialized = JSON.stringify(batch)
|
|
87
|
+
for (const selection of selections) {
|
|
88
|
+
const canary = String(selection.rawSessionId ?? '')
|
|
89
|
+
if (canary.length >= 8 && serialized.includes(canary)) {
|
|
90
|
+
throw new Error('SESSION_REDACTION_FAILED: a raw Session id survived Batch materialization')
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
batch.digest = canonicalDigest(batch, 'harbor-dsh-historical-generation-batch-v1')
|
|
94
|
+
return batch
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async function writePrivateJson(pathname, value) {
|
|
98
|
+
await writeFile(pathname, `${JSON.stringify(value, null, 2)}\n`, {
|
|
99
|
+
encoding: 'utf8',
|
|
100
|
+
mode: 0o600,
|
|
101
|
+
flag: 'wx',
|
|
102
|
+
})
|
|
103
|
+
await chmod(pathname, 0o600)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function unsafePrivateEvidencePath(label) {
|
|
107
|
+
return new Error(
|
|
108
|
+
`PRIVATE_EVIDENCE_PATH_UNSAFE: ${label} must be a real directory inside projectRoot, not a symlink or non-directory`,
|
|
109
|
+
)
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
async function ensureSafeDirectory(directory, {
|
|
113
|
+
parentReal,
|
|
114
|
+
label,
|
|
115
|
+
mode = 0o700,
|
|
116
|
+
enforceMode = true,
|
|
117
|
+
}) {
|
|
118
|
+
try {
|
|
119
|
+
await mkdir(directory, { mode })
|
|
120
|
+
} catch (error) {
|
|
121
|
+
if (error?.code !== 'EEXIST') throw error
|
|
122
|
+
}
|
|
123
|
+
const details = await lstat(directory)
|
|
124
|
+
if (details.isSymbolicLink() || !details.isDirectory()) {
|
|
125
|
+
throw unsafePrivateEvidencePath(label)
|
|
126
|
+
}
|
|
127
|
+
const resolved = await realpath(directory)
|
|
128
|
+
if (path.dirname(resolved) !== parentReal) {
|
|
129
|
+
throw unsafePrivateEvidencePath(label)
|
|
130
|
+
}
|
|
131
|
+
if (enforceMode) await chmod(directory, mode)
|
|
132
|
+
return resolved
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export async function writePrivateHistoricalBatch({ projectRoot, batch, observations }) {
|
|
136
|
+
const resolvedProjectRoot = path.resolve(projectRoot)
|
|
137
|
+
const projectReal = await realpath(resolvedProjectRoot)
|
|
138
|
+
const projectDetails = await lstat(projectReal)
|
|
139
|
+
if (!projectDetails.isDirectory()) {
|
|
140
|
+
throw unsafePrivateEvidencePath('projectRoot')
|
|
141
|
+
}
|
|
142
|
+
const harborRoot = path.join(resolvedProjectRoot, '.harbor')
|
|
143
|
+
const harborReal = await ensureSafeDirectory(harborRoot, {
|
|
144
|
+
parentReal: projectReal,
|
|
145
|
+
label: '.harbor',
|
|
146
|
+
enforceMode: false,
|
|
147
|
+
})
|
|
148
|
+
const privateRoot = path.join(harborRoot, 'private')
|
|
149
|
+
const privateReal = await ensureSafeDirectory(privateRoot, {
|
|
150
|
+
parentReal: harborReal,
|
|
151
|
+
label: '.harbor/private',
|
|
152
|
+
})
|
|
153
|
+
const batchesRoot = path.join(privateRoot, 'session-batches')
|
|
154
|
+
try {
|
|
155
|
+
await writeFile(path.join(privateRoot, '.gitignore'), '*\n!.gitignore\n', {
|
|
156
|
+
encoding: 'utf8', mode: 0o600, flag: 'wx',
|
|
157
|
+
})
|
|
158
|
+
} catch (error) {
|
|
159
|
+
if (error?.code !== 'EEXIST') throw error
|
|
160
|
+
}
|
|
161
|
+
await ensureSafeDirectory(batchesRoot, {
|
|
162
|
+
parentReal: privateReal,
|
|
163
|
+
label: '.harbor/private/session-batches',
|
|
164
|
+
})
|
|
165
|
+
const staging = await mkdtemp(path.join(batchesRoot, '.staging-'))
|
|
166
|
+
const target = path.join(batchesRoot, batch.batch_id)
|
|
167
|
+
try {
|
|
168
|
+
await chmod(staging, 0o700)
|
|
169
|
+
const sessions = path.join(staging, 'sessions')
|
|
170
|
+
await mkdir(sessions, { mode: 0o700 })
|
|
171
|
+
for (const observation of observations) {
|
|
172
|
+
await writePrivateJson(path.join(sessions, `${observation.trial_id}.json`), observation)
|
|
173
|
+
}
|
|
174
|
+
const redactionReport = {
|
|
175
|
+
schema_version: 1,
|
|
176
|
+
policy: batch.redaction_policy,
|
|
177
|
+
sessions: observations.map(observation => ({
|
|
178
|
+
trial_id: observation.trial_id,
|
|
179
|
+
...observation.redaction,
|
|
180
|
+
})),
|
|
181
|
+
}
|
|
182
|
+
await writePrivateJson(path.join(staging, 'session-redaction-report.json'), redactionReport)
|
|
183
|
+
await writePrivateJson(path.join(staging, 'historical-generation-batch.json'), batch)
|
|
184
|
+
await rename(staging, target)
|
|
185
|
+
return {
|
|
186
|
+
batchDir: target,
|
|
187
|
+
batchPath: path.join(target, 'historical-generation-batch.json'),
|
|
188
|
+
redactionReportPath: path.join(target, 'session-redaction-report.json'),
|
|
189
|
+
}
|
|
190
|
+
} catch (error) {
|
|
191
|
+
await rm(staging, { recursive: true, force: true })
|
|
192
|
+
throw error
|
|
193
|
+
}
|
|
194
|
+
}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
function isAppendOrigin(event) {
|
|
2
|
+
return event?.surfaceOp === undefined || event.surfaceOp === 'append'
|
|
3
|
+
}
|
|
4
|
+
|
|
5
|
+
function finiteInteger(value) {
|
|
6
|
+
return Number.isSafeInteger(value) && value >= 0 ? value : undefined
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
function visibleTextBlocks(message) {
|
|
10
|
+
return Array.isArray(message?.content)
|
|
11
|
+
? message.content.filter(block => block?.type === 'text' && typeof block.text === 'string' && block.text.trim())
|
|
12
|
+
: []
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function routeKey(route) {
|
|
16
|
+
return `${route.provider}\u0000${route.model}\u0000${route.reasoning_effort ?? ''}`
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Resolve the composition that actually produced the Session history.
|
|
21
|
+
*
|
|
22
|
+
* DSH freezes the creation-time preset in the Session header, but a blank
|
|
23
|
+
* Session may select another preset before its first turn. That selection is
|
|
24
|
+
* durable evidence and the newest `agent-preset/selected` event wins.
|
|
25
|
+
*/
|
|
26
|
+
export function resolveEffectiveAgentPreset(header, events) {
|
|
27
|
+
for (let index = Array.isArray(events) ? events.length - 1 : -1; index >= 0; index -= 1) {
|
|
28
|
+
const event = events[index]
|
|
29
|
+
if (event?.type === 'agent-preset/selected') {
|
|
30
|
+
return typeof event.data?.agentPreset === 'string'
|
|
31
|
+
? event.data.agentPreset
|
|
32
|
+
: undefined
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
return typeof header?.agentPreset === 'string' ? header.agentPreset : undefined
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Fold only non-content selection facts from a validated DSH Session log.
|
|
40
|
+
* Transcript text deliberately never enters this projection.
|
|
41
|
+
*/
|
|
42
|
+
export function foldSessionDiagnosticIndex(events, header) {
|
|
43
|
+
const openTurns = new Set()
|
|
44
|
+
const seenTurns = new Set()
|
|
45
|
+
const headerRouteEvents = []
|
|
46
|
+
const contextRouteEvents = []
|
|
47
|
+
const assistantRouteEvents = []
|
|
48
|
+
let lastActivityAt = 0
|
|
49
|
+
let lastSeq = null
|
|
50
|
+
let humanMessageCount = 0
|
|
51
|
+
let assistantMessageCount = 0
|
|
52
|
+
let lastTurnReason = null
|
|
53
|
+
let hasHarborToolCall = false
|
|
54
|
+
let toolCallCount = 0
|
|
55
|
+
|
|
56
|
+
for (const event of Array.isArray(events) ? events : []) {
|
|
57
|
+
const seq = finiteInteger(event?.seq)
|
|
58
|
+
const time = finiteInteger(event?.time)
|
|
59
|
+
if (seq !== undefined) lastSeq = lastSeq === null ? seq : Math.max(lastSeq, seq)
|
|
60
|
+
if (time !== undefined) lastActivityAt = Math.max(lastActivityAt, time)
|
|
61
|
+
|
|
62
|
+
if (event?.type === 'turn/start') {
|
|
63
|
+
const turn = finiteInteger(event.data?.turn)
|
|
64
|
+
if (turn !== undefined) {
|
|
65
|
+
openTurns.add(turn)
|
|
66
|
+
seenTurns.add(turn)
|
|
67
|
+
}
|
|
68
|
+
} else if (event?.type === 'turn/end') {
|
|
69
|
+
const turn = finiteInteger(event.data?.turn)
|
|
70
|
+
if (turn !== undefined) {
|
|
71
|
+
openTurns.delete(turn)
|
|
72
|
+
seenTurns.add(turn)
|
|
73
|
+
}
|
|
74
|
+
lastTurnReason = typeof event.data?.reason?.kind === 'string'
|
|
75
|
+
? event.data.reason.kind
|
|
76
|
+
: null
|
|
77
|
+
} else if (event?.type === 'user/message' && isAppendOrigin(event)) {
|
|
78
|
+
if (event.data?.source?.kind === 'user' && visibleTextBlocks(event.data).length) {
|
|
79
|
+
humanMessageCount += 1
|
|
80
|
+
}
|
|
81
|
+
} else if (event?.type === 'assistant/message' && isAppendOrigin(event)) {
|
|
82
|
+
if (visibleTextBlocks(event.data?.message).length) assistantMessageCount += 1
|
|
83
|
+
const source = event.data?.message?.source
|
|
84
|
+
if (source?.kind === 'model' && typeof source.provider === 'string' && typeof source.model === 'string') {
|
|
85
|
+
const route = { provider: source.provider, model: source.model }
|
|
86
|
+
assistantRouteEvents.push({ seq: seq ?? 0, ...route })
|
|
87
|
+
}
|
|
88
|
+
} else if (event?.type === 'tool/call') {
|
|
89
|
+
toolCallCount += 1
|
|
90
|
+
const name = String(event.data?.name ?? '')
|
|
91
|
+
if (name.startsWith('harbor_')) hasHarborToolCall = true
|
|
92
|
+
} else if (event?.type === 'request/context') {
|
|
93
|
+
const { provider, model } = event.data ?? {}
|
|
94
|
+
if (typeof provider === 'string' && typeof model === 'string') {
|
|
95
|
+
const route = { provider, model }
|
|
96
|
+
contextRouteEvents.push({ seq: seq ?? 0, ...route })
|
|
97
|
+
}
|
|
98
|
+
} else if (event?.type === 'request/header') {
|
|
99
|
+
const config = event.data?.header?.config
|
|
100
|
+
if (typeof config?.provider === 'string' && typeof config?.model === 'string') {
|
|
101
|
+
const route = {
|
|
102
|
+
provider: config.provider,
|
|
103
|
+
model: config.model,
|
|
104
|
+
...(typeof config.reasoningEffort === 'string'
|
|
105
|
+
? { reasoning_effort: config.reasoningEffort }
|
|
106
|
+
: {}),
|
|
107
|
+
}
|
|
108
|
+
headerRouteEvents.push({ seq: seq ?? 0, ...route })
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// request/header is the canonical full request identity. Fall back to
|
|
114
|
+
// request/context or assembled assistant provenance only for older logs.
|
|
115
|
+
const routeEvents = headerRouteEvents.length
|
|
116
|
+
? headerRouteEvents
|
|
117
|
+
: contextRouteEvents.length ? contextRouteEvents : assistantRouteEvents
|
|
118
|
+
const modelSegments = []
|
|
119
|
+
for (const route of routeEvents) {
|
|
120
|
+
const previous = modelSegments.at(-1)
|
|
121
|
+
if (previous && routeKey(previous) === routeKey(route)) continue
|
|
122
|
+
modelSegments.push({
|
|
123
|
+
from_seq: route.seq,
|
|
124
|
+
through_seq: lastSeq ?? route.seq,
|
|
125
|
+
provider: route.provider,
|
|
126
|
+
model: route.model,
|
|
127
|
+
...(route.reasoning_effort ? { reasoning_effort: route.reasoning_effort } : {}),
|
|
128
|
+
})
|
|
129
|
+
}
|
|
130
|
+
for (let index = 0; index < modelSegments.length - 1; index += 1) {
|
|
131
|
+
modelSegments[index].through_seq = Math.max(
|
|
132
|
+
modelSegments[index].from_seq,
|
|
133
|
+
modelSegments[index + 1].from_seq - 1,
|
|
134
|
+
)
|
|
135
|
+
}
|
|
136
|
+
const routes = new Map()
|
|
137
|
+
for (const segment of modelSegments) {
|
|
138
|
+
const route = {
|
|
139
|
+
provider: segment.provider,
|
|
140
|
+
model: segment.model,
|
|
141
|
+
...(segment.reasoning_effort ? { reasoning_effort: segment.reasoning_effort } : {}),
|
|
142
|
+
}
|
|
143
|
+
routes.set(routeKey(route), route)
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const effectiveAgentPreset = resolveEffectiveAgentPreset(header, events)
|
|
147
|
+
return {
|
|
148
|
+
lastActivityAt,
|
|
149
|
+
lastSeq,
|
|
150
|
+
openTurn: openTurns.size > 0,
|
|
151
|
+
turnCount: seenTurns.size,
|
|
152
|
+
humanMessageCount,
|
|
153
|
+
assistantMessageCount,
|
|
154
|
+
toolCallCount,
|
|
155
|
+
lastTurnReason,
|
|
156
|
+
hasHarborToolCall,
|
|
157
|
+
...(effectiveAgentPreset === undefined ? {} : { effectiveAgentPreset }),
|
|
158
|
+
modelRoutes: [...routes.values()].sort((left, right) => routeKey(left).localeCompare(routeKey(right))),
|
|
159
|
+
modelSegments,
|
|
160
|
+
}
|
|
161
|
+
}
|