@deepslacker/dsh-task-progress 0.0.0-stage → 0.5.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +95 -0
- package/LICENSE +21 -0
- package/README.en.md +177 -0
- package/README.md +180 -2
- package/client.js +1492 -0
- package/cordis.patch.yml +3 -0
- package/docs/DETAIL.md +106 -0
- package/docs/DETAIL.zh.md +106 -0
- package/docs/images/scene-completed.png +0 -0
- package/docs/images/scene-progress.png +0 -0
- package/docs/mockups/README.md +8 -0
- package/docs/mockups/scenes.html +408 -0
- package/icon.svg +1 -0
- package/index.js +435 -0
- package/lib/judge.js +168 -0
- package/lib/progress.js +213 -0
- package/lib/steps.js +87 -0
- package/locale/en.json +7 -0
- package/locale/zh.json +7 -0
- package/package.json +57 -4
package/index.js
ADDED
|
@@ -0,0 +1,435 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-task-progress — Host half.
|
|
3
|
+
*
|
|
4
|
+
* Three cooperating pieces, all bounded:
|
|
5
|
+
*
|
|
6
|
+
* 1. `taskProgress`, a session projection that folds the open turn's progress
|
|
7
|
+
* out of the session log (model checkpoints, todo list, turn boundaries).
|
|
8
|
+
* It is replayable and is published to the Client through `wire.summary`.
|
|
9
|
+
* 2. `report_progress`, a tool the working model can call at milestones. When
|
|
10
|
+
* it does, that report is the best signal and wins over everything else.
|
|
11
|
+
* 3. A side-channel evaluator: models often never call a telemetry tool, so
|
|
12
|
+
* while a turn is open the plugin asks the session's own model route for a
|
|
13
|
+
* strict JSON estimate from a compact digest. Results live in memory and are
|
|
14
|
+
* served to the page at `GET /api/dsh-task-progress/judgements`.
|
|
15
|
+
*
|
|
16
|
+
* This file deliberately imports no Harness package: a bundle installed from a
|
|
17
|
+
* local directory is linked in place, so bare `@deepseek-ai/*` specifiers would
|
|
18
|
+
* resolve against the workspace rather than the running installation. The tool
|
|
19
|
+
* definition is written as the registry's plain JSON shape instead.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { applyProgress, BLANK, PHASES, progressSchema, shouldOfferProgressTool } from './lib/progress.js'
|
|
23
|
+
import { buildDigest, buildJudgeInput, JUDGE_SYSTEM, parseJudgement } from './lib/judge.js'
|
|
24
|
+
import { closeRail, observeRail } from './lib/steps.js'
|
|
25
|
+
|
|
26
|
+
export const name = 'dsh-task-progress'
|
|
27
|
+
|
|
28
|
+
/** The projection is the state store; the tool writes into the log it folds. */
|
|
29
|
+
export const inject = ['tools', 'sessionProjections']
|
|
30
|
+
|
|
31
|
+
/** Route serving the evaluator's current estimates to the page. */
|
|
32
|
+
const JUDGE_ROUTE = '/api/dsh-task-progress/judgements'
|
|
33
|
+
|
|
34
|
+
/** How often the sweep looks for turns that need an estimate. */
|
|
35
|
+
const SWEEP_MS = 15_000
|
|
36
|
+
|
|
37
|
+
/** Deadline of one evaluator call. */
|
|
38
|
+
const JUDGE_TIMEOUT_MS = 25_000
|
|
39
|
+
|
|
40
|
+
/** Output cap of one evaluator call. */
|
|
41
|
+
const JUDGE_MAX_TOKENS = 1_200
|
|
42
|
+
|
|
43
|
+
/** A model checkpoint younger than this outranks the evaluator, same as the page. */
|
|
44
|
+
const MODEL_FRESH_MS = 180_000
|
|
45
|
+
|
|
46
|
+
/** An idle turn's rail is dropped this long after it closed. */
|
|
47
|
+
|
|
48
|
+
/** One-line model-facing contract; kept short because it rides every request. */
|
|
49
|
+
const TOOL_DESCRIPTION = [
|
|
50
|
+
'Record this task\'s progress for the session UI.',
|
|
51
|
+
'Call it after finishing a milestone or changing phase — not on every step, and skip it for a task you will finish in one reply.',
|
|
52
|
+
'Anchors: 0 = not started, 25 = plan settled, 50 = half the work done, 75 = verifying, 100 = delivered and verified.',
|
|
53
|
+
'Keep milestone and next under 40 characters; do not mention the call to the user.',
|
|
54
|
+
].join(' ')
|
|
55
|
+
|
|
56
|
+
/** Model-facing parameter schema (the registry's supported JSON Schema subset). */
|
|
57
|
+
const TOOL_PARAMETERS = {
|
|
58
|
+
type: 'object',
|
|
59
|
+
additionalProperties: false,
|
|
60
|
+
properties: {
|
|
61
|
+
percent: {
|
|
62
|
+
type: 'integer',
|
|
63
|
+
description: 'Overall progress of the current task, 0-100. Be honest, not optimistic.',
|
|
64
|
+
},
|
|
65
|
+
phase: {
|
|
66
|
+
type: 'string',
|
|
67
|
+
enum: PHASES,
|
|
68
|
+
description: 'Current phase of the task (plan | execute | verify | deliver).',
|
|
69
|
+
},
|
|
70
|
+
milestone: {
|
|
71
|
+
type: 'string',
|
|
72
|
+
description: 'What is now finished or actively being worked on, one short line.',
|
|
73
|
+
},
|
|
74
|
+
next: {
|
|
75
|
+
type: 'string',
|
|
76
|
+
description: 'The next expected node, one short line (optional).',
|
|
77
|
+
},
|
|
78
|
+
eta_min: {
|
|
79
|
+
type: 'integer',
|
|
80
|
+
description: 'Estimated minutes still needed, 1-600 (optional, omit when unsure).',
|
|
81
|
+
},
|
|
82
|
+
revised: {
|
|
83
|
+
type: 'boolean',
|
|
84
|
+
description: 'Set true only when the estimate went DOWN on purpose (new scope discovered).',
|
|
85
|
+
},
|
|
86
|
+
},
|
|
87
|
+
required: ['percent', 'phase', 'milestone'],
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Call result schema: what the model sees after a checkpoint. */
|
|
91
|
+
const TOOL_OUTPUT = {
|
|
92
|
+
type: 'object',
|
|
93
|
+
additionalProperties: false,
|
|
94
|
+
properties: {
|
|
95
|
+
ok: { type: 'boolean' },
|
|
96
|
+
percent: { type: 'integer' },
|
|
97
|
+
phase: { type: 'string' },
|
|
98
|
+
},
|
|
99
|
+
required: ['ok', 'percent', 'phase'],
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Integer guard shared by the tool body and the transcript card. */
|
|
103
|
+
function clampInt(value, min, max, fallback) {
|
|
104
|
+
const number = typeof value === 'number' ? value : Number(value)
|
|
105
|
+
if (!Number.isFinite(number)) return fallback
|
|
106
|
+
return Math.min(max, Math.max(min, Math.round(number)))
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Keep a configured CSS color short and free of rule-breaking characters. */
|
|
110
|
+
function safeColor(value) {
|
|
111
|
+
if (typeof value !== 'string') return ''
|
|
112
|
+
const color = value.trim().slice(0, 40)
|
|
113
|
+
if (color === '') return ''
|
|
114
|
+
return /^[#a-zA-Z0-9(),.%\s-]+$/u.test(color) ? color : ''
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Keep a model-written label on one bounded line. */
|
|
118
|
+
function oneLine(value, max) {
|
|
119
|
+
if (typeof value !== 'string') return ''
|
|
120
|
+
return value.replace(/\s+/gu, ' ').trim().slice(0, max)
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** The compact transcript card for one checkpoint. */
|
|
124
|
+
function cardTitle(args) {
|
|
125
|
+
const percent = clampInt(args?.percent, 0, 100, 0)
|
|
126
|
+
const note = oneLine(args?.milestone, 60)
|
|
127
|
+
return note === '' ? `Progress ${percent}%` : `${percent}% · ${note}`
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** The model route recorded by the session's newest request. */
|
|
131
|
+
function routeOf(events) {
|
|
132
|
+
for (let index = events.length - 1; index >= 0; index -= 1) {
|
|
133
|
+
const event = events[index]
|
|
134
|
+
if (event.type !== 'request/header') continue
|
|
135
|
+
const config = event.data !== null && typeof event.data === 'object' && event.data.header !== null
|
|
136
|
+
&& typeof event.data.header === 'object' ? event.data.header.config : undefined
|
|
137
|
+
if (config !== null && typeof config === 'object'
|
|
138
|
+
&& typeof config.provider === 'string' && config.provider !== ''
|
|
139
|
+
&& typeof config.model === 'string' && config.model !== '') {
|
|
140
|
+
return { provider: config.provider, model: config.model }
|
|
141
|
+
}
|
|
142
|
+
return undefined
|
|
143
|
+
}
|
|
144
|
+
return undefined
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Register the projection, the report tool, and the side-channel evaluator.
|
|
149
|
+
* @param ctx - Host context carrying the projection, tool, session, and LLM services.
|
|
150
|
+
* @param config - optional row config (`enableTool`, `minStep`, `todoBaseline`,
|
|
151
|
+
* `judge`, `judgeIntervalMs`).
|
|
152
|
+
*/
|
|
153
|
+
export async function apply(ctx, config) {
|
|
154
|
+
const settings = {
|
|
155
|
+
enableTool: config?.enableTool !== false,
|
|
156
|
+
// 1 = offered from the first step of a turn, so a long task that starts
|
|
157
|
+
// immediately can be reported on the same step. Raise it to trade the
|
|
158
|
+
// per-request schema cost against that first-step coverage.
|
|
159
|
+
minStep: clampInt(config?.minStep, 1, 999, 1),
|
|
160
|
+
todoBaseline: config?.todoBaseline !== false,
|
|
161
|
+
judge: config?.judge !== false,
|
|
162
|
+
// Minimum gap between two evaluator calls for one session.
|
|
163
|
+
judgeIntervalMs: clampInt(config?.judgeIntervalMs, 30_000, 600_000, 120_000),
|
|
164
|
+
// Presentation knobs for the composer strip. 0 / '' means "inherit the
|
|
165
|
+
// theme", so the defaults below never overwrite a host or user CSS value.
|
|
166
|
+
style: {
|
|
167
|
+
/** Bar thickness in CSS pixels (0 = theme default 6px). */
|
|
168
|
+
barHeight: clampInt(config?.barHeight, 0, 24, 0),
|
|
169
|
+
/** CSS color for the filled part ('' = theme accent). */
|
|
170
|
+
barColor: safeColor(config?.barColor),
|
|
171
|
+
/** Font size of the explanation row in CSS pixels (0 = theme default 12px). */
|
|
172
|
+
textFontSize: clampInt(config?.textFontSize, 0, 24, 0),
|
|
173
|
+
/** Font size of the expanded node list in CSS pixels (0 = default 11px). */
|
|
174
|
+
listFontSize: clampInt(config?.listFontSize, 0, 24, 0),
|
|
175
|
+
},
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const unit = await ctx.sessionProjections.register({
|
|
179
|
+
key: 'taskProgress',
|
|
180
|
+
stateSchema: progressSchema,
|
|
181
|
+
init: () => BLANK,
|
|
182
|
+
apply: (state, event) => applyProgress(state, event, settings),
|
|
183
|
+
// `view` returns the fold's own state, so an event that did not change it
|
|
184
|
+
// republishes nothing; `summary` carries the value on every session row.
|
|
185
|
+
wire: { viewSchema: progressSchema, view: state => state, summary: true },
|
|
186
|
+
stateVersion: 3,
|
|
187
|
+
})
|
|
188
|
+
|
|
189
|
+
if (settings.enableTool) {
|
|
190
|
+
const tool = {
|
|
191
|
+
name: 'report_progress',
|
|
192
|
+
description: TOOL_DESCRIPTION,
|
|
193
|
+
parameters: TOOL_PARAMETERS,
|
|
194
|
+
output: {
|
|
195
|
+
schema: TOOL_OUTPUT,
|
|
196
|
+
render: (_args, value) => [{
|
|
197
|
+
type: 'text',
|
|
198
|
+
text: `Progress checkpoint recorded at ${value.percent}% (${value.phase}).`,
|
|
199
|
+
}],
|
|
200
|
+
},
|
|
201
|
+
presentCall: args => ({ card: 'generic', title: cardTitle(args), kind: 'other' }),
|
|
202
|
+
presentResult: args => ({ card: 'generic', title: cardTitle(args) }),
|
|
203
|
+
async execute(args) {
|
|
204
|
+
// Without `defineTool` the registry hands the body the parsed arguments
|
|
205
|
+
// unchanged, so the range and enum checks live here. A rejected call is
|
|
206
|
+
// simply not folded: the projection validates the same arguments again
|
|
207
|
+
// from the logged `tool/call` event.
|
|
208
|
+
const percent = clampInt(args?.percent, 0, 100, -1)
|
|
209
|
+
if (percent < 0) throw new Error('report_progress: percent must be an integer between 0 and 100')
|
|
210
|
+
if (!PHASES.includes(args?.phase)) throw new Error('report_progress: phase must be one of ' + PHASES.join(', '))
|
|
211
|
+
// The log entry is the durable record; this return value only feeds the card.
|
|
212
|
+
return { ok: true, percent, phase: args.phase }
|
|
213
|
+
},
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// The tool is offered from `minStep` of every turn (default: the first
|
|
217
|
+
// step), so a long task that starts immediately is not blind for one step.
|
|
218
|
+
// The session is gated by its own turn instead of the global registry so a
|
|
219
|
+
// session that has not started a turn yet never sees a dead tool.
|
|
220
|
+
ctx.effect(
|
|
221
|
+
() => ctx.tools.register(tool, {
|
|
222
|
+
when: (context) => {
|
|
223
|
+
try {
|
|
224
|
+
return shouldOfferProgressTool(unit.stateOf(context.agent.session), settings.minStep)
|
|
225
|
+
} catch {
|
|
226
|
+
// A session the projection cannot read yet keeps the tool visible:
|
|
227
|
+
// a missing progress card is cheaper than a missing capability.
|
|
228
|
+
return true
|
|
229
|
+
}
|
|
230
|
+
},
|
|
231
|
+
}),
|
|
232
|
+
'dsh-task-progress: report tool',
|
|
233
|
+
)
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
if (!settings.judge) return
|
|
237
|
+
|
|
238
|
+
// ── side-channel evaluator ──────────────────────────────────────────────
|
|
239
|
+
// One estimate per session, keyed by the turn it describes. The map is
|
|
240
|
+
// deliberately in-memory: it is a derived second opinion, not session data,
|
|
241
|
+
// and it must never pretend to be part of the log.
|
|
242
|
+
const judgements = new Map()
|
|
243
|
+
// One milestone rail per session, also keyed by turn. A new milestone closes
|
|
244
|
+
// the open step, which is what the page draws as a completed node.
|
|
245
|
+
const rails = new Map()
|
|
246
|
+
const inFlight = new Set()
|
|
247
|
+
const lastCallAt = new Map()
|
|
248
|
+
let failures = 0
|
|
249
|
+
let lastFailureAt = 0
|
|
250
|
+
|
|
251
|
+
ctx.inject(['sessions', 'llm'], (child) => {
|
|
252
|
+
const sessions = child.sessions
|
|
253
|
+
const llm = child.llm
|
|
254
|
+
|
|
255
|
+
/** Ask the session's own model route for one estimate. */
|
|
256
|
+
const judge = async (session, state) => {
|
|
257
|
+
const events = typeof session.snapshotEvents === 'function' ? session.snapshotEvents() : []
|
|
258
|
+
const digest = buildDigest(events, state.turn)
|
|
259
|
+
if (digest === null) return
|
|
260
|
+
const route = routeOf(events)
|
|
261
|
+
if (route === undefined) return
|
|
262
|
+
const controller = new AbortController()
|
|
263
|
+
const timer = setTimeout(() => controller.abort(), JUDGE_TIMEOUT_MS)
|
|
264
|
+
try {
|
|
265
|
+
const call = await llm.prepareCall({
|
|
266
|
+
provider: route.provider,
|
|
267
|
+
model: route.model,
|
|
268
|
+
maxTokens: JUDGE_MAX_TOKENS,
|
|
269
|
+
}, controller.signal, (controls, info) => {
|
|
270
|
+
// Keep the evaluator cheap: use the route's lowest reasoning effort
|
|
271
|
+
// when it has one, so most of the budget goes to the JSON answer.
|
|
272
|
+
const efforts = info !== null && typeof info === 'object' && info.reasoning !== undefined
|
|
273
|
+
&& Array.isArray(info.reasoning.efforts) ? info.reasoning.efforts : []
|
|
274
|
+
const effort = efforts.length > 0 && efforts[0] !== null && typeof efforts[0] === 'object'
|
|
275
|
+
? efforts[0].id : undefined
|
|
276
|
+
return typeof effort === 'string' && effort !== '' ? { ...controls, reasoningEffort: effort } : controls
|
|
277
|
+
})
|
|
278
|
+
let text = ''
|
|
279
|
+
const options = {
|
|
280
|
+
...call.config,
|
|
281
|
+
messages: [{ role: 'user', content: [{ type: 'text', text: buildJudgeInput(digest) }] }],
|
|
282
|
+
system: JUDGE_SYSTEM,
|
|
283
|
+
sessionId: session.id,
|
|
284
|
+
signal: controller.signal,
|
|
285
|
+
}
|
|
286
|
+
for await (const chunk of call.stream(options)) {
|
|
287
|
+
if (chunk === null || typeof chunk !== 'object') continue
|
|
288
|
+
if (chunk.type === 'text-delta') text += chunk.text
|
|
289
|
+
if (chunk.type === 'finish' && chunk.reason !== undefined
|
|
290
|
+
&& (chunk.reason.kind === 'error' || chunk.reason.kind === 'aborted')) {
|
|
291
|
+
throw new Error('judge request did not finish')
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
const judgement = parseJudgement(text)
|
|
295
|
+
if (judgement === null) return
|
|
296
|
+
judgements.set(session.id, { turn: state.turn, ...judgement, at: Date.now() })
|
|
297
|
+
failures = 0
|
|
298
|
+
} finally {
|
|
299
|
+
clearTimeout(timer)
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/** The milestone the page is showing, from the same precedence rule it uses. */
|
|
304
|
+
const effectiveMilestone = (state, judgement, now) => {
|
|
305
|
+
const freshModel = state.source === 'model' && now - state.updatedAt < MODEL_FRESH_MS
|
|
306
|
+
const useJudge = !freshModel && judgement !== undefined && judgement !== null
|
|
307
|
+
&& judgement.turn === state.turn
|
|
308
|
+
return {
|
|
309
|
+
label: useJudge ? (judgement.milestone || state.milestone) : state.milestone,
|
|
310
|
+
percent: useJudge && judgement.percent >= 0 ? judgement.percent : state.percent,
|
|
311
|
+
phase: useJudge ? (judgement.phase || state.phase) : state.phase,
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/** Fold the milestone the page is showing into the session's turn rail. */
|
|
316
|
+
const observe = (sessionId, state, now) => {
|
|
317
|
+
const seen = effectiveMilestone(state, judgements.get(sessionId), now)
|
|
318
|
+
rails.set(sessionId, observeRail(rails.get(sessionId), {
|
|
319
|
+
turn: state.turn,
|
|
320
|
+
label: seen.label,
|
|
321
|
+
percent: seen.percent,
|
|
322
|
+
phase: seen.phase,
|
|
323
|
+
at: now,
|
|
324
|
+
startedAt: state.startedAt,
|
|
325
|
+
}))
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** One pass over the live sessions. */
|
|
329
|
+
const sweep = async () => {
|
|
330
|
+
const sweepStartedAt = Date.now()
|
|
331
|
+
if (failures >= 3 && sweepStartedAt - lastFailureAt < 300_000) return
|
|
332
|
+
let live
|
|
333
|
+
try {
|
|
334
|
+
live = sessions.list()
|
|
335
|
+
} catch {
|
|
336
|
+
return
|
|
337
|
+
}
|
|
338
|
+
const liveIds = new Set()
|
|
339
|
+
for (const session of live) {
|
|
340
|
+
liveIds.add(session.id)
|
|
341
|
+
let state
|
|
342
|
+
try {
|
|
343
|
+
state = unit.stateOf(session)
|
|
344
|
+
} catch {
|
|
345
|
+
continue
|
|
346
|
+
}
|
|
347
|
+
if (state === undefined || state === null || state.turn <= 0) {
|
|
348
|
+
judgements.delete(session.id)
|
|
349
|
+
rails.delete(session.id)
|
|
350
|
+
continue
|
|
351
|
+
}
|
|
352
|
+
if (state.done) {
|
|
353
|
+
// Close the last node at the turn end and keep the rail: the completed
|
|
354
|
+
// message footer and the right Sidebar read it after the turn is over.
|
|
355
|
+
// The next turn's first observation replaces it.
|
|
356
|
+
const rail = rails.get(session.id)
|
|
357
|
+
if (rail !== undefined) {
|
|
358
|
+
const closed = closeRail(rail, state.updatedAt > 0 ? state.updatedAt : Date.now())
|
|
359
|
+
if (closed !== rail) rails.set(session.id, closed)
|
|
360
|
+
}
|
|
361
|
+
judgements.delete(session.id)
|
|
362
|
+
continue
|
|
363
|
+
}
|
|
364
|
+
const now = Date.now()
|
|
365
|
+
// A fresh checkpoint from the working model is better than a second
|
|
366
|
+
// opinion, and skipping here is what keeps the evaluator cheap.
|
|
367
|
+
const modelFresh = state.source === 'model' && now - state.updatedAt < settings.judgeIntervalMs
|
|
368
|
+
if (!modelFresh
|
|
369
|
+
&& now - (lastCallAt.get(session.id) ?? 0) >= settings.judgeIntervalMs
|
|
370
|
+
&& !inFlight.has(session.id)) {
|
|
371
|
+
inFlight.add(session.id)
|
|
372
|
+
lastCallAt.set(session.id, now)
|
|
373
|
+
try {
|
|
374
|
+
await judge(session, state)
|
|
375
|
+
} catch {
|
|
376
|
+
failures += 1
|
|
377
|
+
lastFailureAt = Date.now()
|
|
378
|
+
} finally {
|
|
379
|
+
inFlight.delete(session.id)
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
// The rail follows whatever the page is showing, so a model report and
|
|
383
|
+
// a judge estimate both become completed nodes.
|
|
384
|
+
observe(session.id, state, Date.now())
|
|
385
|
+
}
|
|
386
|
+
for (const id of judgements.keys()) if (!liveIds.has(id)) judgements.delete(id)
|
|
387
|
+
for (const id of rails.keys()) if (!liveIds.has(id)) rails.delete(id)
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
child.effect(() => {
|
|
391
|
+
const timer = setInterval(() => { void sweep() }, SWEEP_MS)
|
|
392
|
+
return () => clearInterval(timer)
|
|
393
|
+
}, 'dsh-task-progress: judge timer')
|
|
394
|
+
})
|
|
395
|
+
|
|
396
|
+
// ── delivery ───────────────────────────────────────────────────────────
|
|
397
|
+
ctx.inject(['webServer', 'connection'], (child) => {
|
|
398
|
+
child.effect(() => child.webServer.register({
|
|
399
|
+
kind: 'prefix',
|
|
400
|
+
path: JUDGE_ROUTE,
|
|
401
|
+
handler: (req, res) => {
|
|
402
|
+
let rejected
|
|
403
|
+
try {
|
|
404
|
+
rejected = child.connection.requestRejection(req) !== undefined
|
|
405
|
+
} catch {
|
|
406
|
+
rejected = true
|
|
407
|
+
}
|
|
408
|
+
if (rejected) {
|
|
409
|
+
res.writeHead(403, { 'content-type': 'application/json; charset=utf-8' })
|
|
410
|
+
res.end('{"ok":false}')
|
|
411
|
+
return
|
|
412
|
+
}
|
|
413
|
+
const url = new URL(req.url ?? '/', 'http://dsh.internal')
|
|
414
|
+
if (url.pathname !== JUDGE_ROUTE) {
|
|
415
|
+
res.writeHead(404, { 'content-type': 'application/json; charset=utf-8' })
|
|
416
|
+
res.end('{"ok":false}')
|
|
417
|
+
return
|
|
418
|
+
}
|
|
419
|
+
const sessions = {}
|
|
420
|
+
for (const [id, value] of judgements) sessions[id] = value
|
|
421
|
+
// The rail is separate from the estimate: a turn can have a rail
|
|
422
|
+
// before the evaluator produced its first estimate (a model report).
|
|
423
|
+
const railValues = {}
|
|
424
|
+
for (const [id, rail] of rails) {
|
|
425
|
+
if (rail.steps.length > 0) railValues[id] = { turn: rail.turn, steps: rail.steps }
|
|
426
|
+
}
|
|
427
|
+
res.writeHead(200, {
|
|
428
|
+
'content-type': 'application/json; charset=utf-8',
|
|
429
|
+
'cache-control': 'no-store',
|
|
430
|
+
})
|
|
431
|
+
res.end(JSON.stringify({ ok: true, value: { sessions, rails: railValues, style: settings.style } }))
|
|
432
|
+
},
|
|
433
|
+
}), 'dsh-task-progress: judgements route')
|
|
434
|
+
})
|
|
435
|
+
}
|
package/lib/judge.js
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-task-progress — side-channel evaluator helpers.
|
|
3
|
+
*
|
|
4
|
+
* The in-session tool is the best signal but models often ignore it, and the
|
|
5
|
+
* todo fallback only says what the plan claims. This module builds the small,
|
|
6
|
+
* bounded digest and the strict JSON contract for one auxiliary model call that
|
|
7
|
+
* estimates the same task from the outside, so a percentage exists even when
|
|
8
|
+
* the working model never reports.
|
|
9
|
+
*
|
|
10
|
+
* The digest is scoped to ONE turn. A continued session keeps the whole log, so
|
|
11
|
+
* "the first user message" or "the last assistant message" would otherwise
|
|
12
|
+
* describe an earlier task and the evaluator would answer with that task's
|
|
13
|
+
* milestone/next node — exactly the stale label this module must not produce.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { PHASES } from './progress.js'
|
|
17
|
+
|
|
18
|
+
/** Per-field character caps; together they keep the input under ~900 tokens. */
|
|
19
|
+
const CAP = Object.freeze({
|
|
20
|
+
task: 400,
|
|
21
|
+
assistant: 400,
|
|
22
|
+
toolArg: 80,
|
|
23
|
+
todoContent: 80,
|
|
24
|
+
todos: 10,
|
|
25
|
+
tools: 8,
|
|
26
|
+
})
|
|
27
|
+
|
|
28
|
+
/** Clamp one integer into a range. */
|
|
29
|
+
function intIn(value, min, max, fallback) {
|
|
30
|
+
const number = typeof value === 'number' ? value : Number(value)
|
|
31
|
+
if (!Number.isFinite(number)) return fallback
|
|
32
|
+
return Math.min(max, Math.max(min, Math.round(number)))
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Join one message's text blocks into a single trimmed line. */
|
|
36
|
+
function textOf(message) {
|
|
37
|
+
const content = message && Array.isArray(message.content) ? message.content : []
|
|
38
|
+
return content
|
|
39
|
+
.filter(block => block !== null && typeof block === 'object' && block.type === 'text' && typeof block.text === 'string')
|
|
40
|
+
.map(block => block.text)
|
|
41
|
+
.join(' ')
|
|
42
|
+
.replace(/\s+/gu, ' ')
|
|
43
|
+
.trim()
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Keep one model-written label short and single-line. */
|
|
47
|
+
function short(value, max = 60) {
|
|
48
|
+
if (typeof value !== 'string') return ''
|
|
49
|
+
return value.replace(/\s+/gu, ' ').trim().slice(0, max)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Build the bounded digest of the turn's work.
|
|
54
|
+
* @param events - the session's committed events, in order.
|
|
55
|
+
* @param turn - the open turn to describe.
|
|
56
|
+
* @returns the digest, or null when the turn has nothing to describe yet.
|
|
57
|
+
*/
|
|
58
|
+
export function buildDigest(events, turn) {
|
|
59
|
+
if (!Array.isArray(events) || events.length === 0) return null
|
|
60
|
+
let task = ''
|
|
61
|
+
let lastAssistant = ''
|
|
62
|
+
let todos = null
|
|
63
|
+
let steps = 0
|
|
64
|
+
let startedAt = 0
|
|
65
|
+
let lastTime = 0
|
|
66
|
+
let inTurn = false
|
|
67
|
+
const tools = []
|
|
68
|
+
for (const event of events) {
|
|
69
|
+
if (event === null || typeof event !== 'object') continue
|
|
70
|
+
const data = event.data !== null && typeof event.data === 'object' ? event.data : {}
|
|
71
|
+
if (event.type === 'turn/start') {
|
|
72
|
+
inTurn = data.turn === turn
|
|
73
|
+
if (!inTurn) continue
|
|
74
|
+
startedAt = typeof event.time === 'number' ? event.time : 0
|
|
75
|
+
lastTime = startedAt
|
|
76
|
+
// Forget everything the log carries from earlier turns.
|
|
77
|
+
task = ''
|
|
78
|
+
lastAssistant = ''
|
|
79
|
+
todos = null
|
|
80
|
+
continue
|
|
81
|
+
}
|
|
82
|
+
if (event.type === 'turn/end') {
|
|
83
|
+
if (data.turn === turn) inTurn = false
|
|
84
|
+
continue
|
|
85
|
+
}
|
|
86
|
+
if (!inTurn) continue
|
|
87
|
+
if (typeof event.time === 'number') lastTime = event.time
|
|
88
|
+
if (event.type === 'step/start' && data.turn === turn) steps += 1
|
|
89
|
+
if (event.type === 'user/message' && task === ''
|
|
90
|
+
&& data.source !== null && typeof data.source === 'object' && data.source.kind === 'user') {
|
|
91
|
+
const text = textOf(data)
|
|
92
|
+
if (text !== '') task = text.slice(0, CAP.task)
|
|
93
|
+
}
|
|
94
|
+
if (event.type === 'todo/write' && Array.isArray(data.todos)) todos = data.todos
|
|
95
|
+
if (event.type === 'tool/call' && data.turn === turn) {
|
|
96
|
+
tools.push(`${String(data.name)}(${short(data.arguments, CAP.toolArg)})`)
|
|
97
|
+
}
|
|
98
|
+
if (event.type === 'assistant/message') {
|
|
99
|
+
// The message body sits under `message`; `data` itself carries only turn/step.
|
|
100
|
+
const message = data.message !== null && typeof data.message === 'object' ? data.message : data
|
|
101
|
+
const text = textOf(message)
|
|
102
|
+
if (text !== '') lastAssistant = text.slice(0, CAP.assistant)
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
if (task === '' && lastAssistant === '') return null
|
|
106
|
+
const done = todos === null ? 0 : todos.filter(item => item !== null && typeof item === 'object' && item.status === 'completed').length
|
|
107
|
+
return {
|
|
108
|
+
task,
|
|
109
|
+
minutes: startedAt > 0 && lastTime > startedAt ? Math.round((lastTime - startedAt) / 60000) : 0,
|
|
110
|
+
steps,
|
|
111
|
+
todos: todos === null
|
|
112
|
+
? []
|
|
113
|
+
: todos.slice(0, CAP.todos).map(item => `${String(item.status)}: ${short(item.content, CAP.todoContent)}`),
|
|
114
|
+
todosDone: done,
|
|
115
|
+
todoTotal: todos === null ? 0 : todos.length,
|
|
116
|
+
tools: tools.slice(-CAP.tools),
|
|
117
|
+
lastAssistant,
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** The fixed system instruction of the evaluator call. */
|
|
122
|
+
export const JUDGE_SYSTEM = [
|
|
123
|
+
'Estimate how far a coding agent has got on the task described by the JSON.',
|
|
124
|
+
'Reply with ONLY one JSON object: {"percent":0-100,"phase":"plan|execute|verify|deliver","milestone":"...","next":"...","eta_min":0}.',
|
|
125
|
+
'Anchors: 0 not started, 25 plan settled, 50 half the work done, 75 the work is done and being verified, 100 delivered and verified.',
|
|
126
|
+
'Judge only what the digest shows; never assume work it does not list.',
|
|
127
|
+
'The digest describes exactly one turn. milestone and next must describe THAT turn; never reuse an earlier task from the session history.',
|
|
128
|
+
'milestone and next are one short line (<= 40 characters) in the task\'s language; eta_min is whole minutes, or 0 when unsure.',
|
|
129
|
+
].join(' ')
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Frame the digest as one JSON line.
|
|
133
|
+
* @param digest - output of {@link buildDigest}.
|
|
134
|
+
* @returns the user-facing input text.
|
|
135
|
+
*/
|
|
136
|
+
export function buildJudgeInput(digest) {
|
|
137
|
+
return JSON.stringify(digest)
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Parse the evaluator's answer strictly.
|
|
142
|
+
* @param text - raw model output (may carry prose or code fences).
|
|
143
|
+
* @returns the canonical judgement, or null when the output is unusable.
|
|
144
|
+
*/
|
|
145
|
+
export function parseJudgement(text) {
|
|
146
|
+
if (typeof text !== 'string') return null
|
|
147
|
+
const start = text.indexOf('{')
|
|
148
|
+
const end = text.lastIndexOf('}')
|
|
149
|
+
if (start < 0 || end <= start) return null
|
|
150
|
+
let raw
|
|
151
|
+
try {
|
|
152
|
+
raw = JSON.parse(text.slice(start, end + 1))
|
|
153
|
+
} catch {
|
|
154
|
+
return null
|
|
155
|
+
}
|
|
156
|
+
if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) return null
|
|
157
|
+
const percent = Number(raw.percent)
|
|
158
|
+
// Reject rather than clamp: an out-of-range answer is a bad answer, and the
|
|
159
|
+
// previous estimate must survive it.
|
|
160
|
+
if (!Number.isFinite(percent) || percent < 0 || percent > 100) return null
|
|
161
|
+
return {
|
|
162
|
+
percent: Math.round(percent),
|
|
163
|
+
phase: PHASES.includes(raw.phase) ? raw.phase : 'execute',
|
|
164
|
+
milestone: short(raw.milestone, 60),
|
|
165
|
+
next: short(raw.next, 60),
|
|
166
|
+
etaMin: intIn(raw.eta_min, 0, 600, 0),
|
|
167
|
+
}
|
|
168
|
+
}
|