@kitman-sh/software-engineering 0.0.0-stage → 0.10.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INSTRUCTIONS.md +3 -0
- package/agents/implementer.md +41 -0
- package/agents/investigator.md +20 -0
- package/agents/planner.md +27 -0
- package/agents/releaser.md +22 -0
- package/agents/researcher.md +19 -0
- package/agents/reviewer.md +37 -0
- package/agents/writer.md +17 -0
- package/dist/bundle.js +29301 -0
- package/dist/declarations.json +3126 -0
- package/dist/setup/2a237548a69b0cd53ec3be7f687cfa8c2128335b3a655d34e185f2abce6a3c37.json +1 -0
- package/kitman.config.ts +813 -0
- package/package.json +30 -4
- package/src/build.ts +542 -0
- package/src/change-request-body.ts +95 -0
- package/src/errand.ts +142 -0
- package/src/file-tasks.ts +30 -0
- package/src/firewatch.ts +232 -0
- package/src/hillclimb.ts +407 -0
- package/src/hindsight.ts +242 -0
- package/src/index.ts +49 -0
- package/src/plan.ts +634 -0
- package/src/project-update.ts +88 -0
- package/src/release.ts +406 -0
- package/src/research.ts +225 -0
- package/src/review.ts +397 -0
- package/src/shepherd-output.ts +10 -0
- package/src/shepherd.ts +530 -0
- package/src/sweep.ts +385 -0
- package/src/tidy.ts +286 -0
- package/tsconfig.json +6 -0
- package/README.md +0 -3
package/src/hillclimb.ts
ADDED
|
@@ -0,0 +1,407 @@
|
|
|
1
|
+
import {
|
|
2
|
+
agentWork,
|
|
3
|
+
agentWorkOnce,
|
|
4
|
+
artifact,
|
|
5
|
+
ask,
|
|
6
|
+
call,
|
|
7
|
+
changeRequest,
|
|
8
|
+
ChangeRequestResult,
|
|
9
|
+
createTasks,
|
|
10
|
+
defineLoop,
|
|
11
|
+
isBlocking,
|
|
12
|
+
measure,
|
|
13
|
+
MeasurePlan,
|
|
14
|
+
syncBase,
|
|
15
|
+
verify,
|
|
16
|
+
writeBack,
|
|
17
|
+
type Measurement
|
|
18
|
+
} from '@kitman-sh/sdk/loop'
|
|
19
|
+
import * as Effect from 'effect/Effect'
|
|
20
|
+
import * as Schema from 'effect/Schema'
|
|
21
|
+
import { scopeFindings } from './change-request-body'
|
|
22
|
+
import review from './review'
|
|
23
|
+
|
|
24
|
+
const Input = Schema.Struct({
|
|
25
|
+
/** What to make faster, in words; the journeys that matter most when left out. */
|
|
26
|
+
focus: Schema.optional(Schema.String),
|
|
27
|
+
taskId: Schema.optional(Schema.String),
|
|
28
|
+
repoBindingId: Schema.optional(Schema.String),
|
|
29
|
+
/** The base branch; the repository's default branch when left out. */
|
|
30
|
+
branch: Schema.optional(Schema.String),
|
|
31
|
+
/** Changes tried at most; 5 when left out. */
|
|
32
|
+
cycles: Schema.optional(Schema.Int),
|
|
33
|
+
/** The smallest gain that counts, as a fraction of the base; 0.05 when left out. */
|
|
34
|
+
threshold: Schema.optional(Schema.Finite),
|
|
35
|
+
/** How much worse a guardrail may get, as a fraction of the base; 0.02 when left out. */
|
|
36
|
+
tolerance: Schema.optional(Schema.Finite)
|
|
37
|
+
})
|
|
38
|
+
|
|
39
|
+
const Opportunity = Schema.Struct({
|
|
40
|
+
title: Schema.String,
|
|
41
|
+
detail: Schema.String,
|
|
42
|
+
/** measured: a number shows it; telemetry: production data shows it; hypothesis: reading the code suggests it. */
|
|
43
|
+
evidence: Schema.Literals(['measured', 'telemetry', 'hypothesis']),
|
|
44
|
+
/** Only another repository can make it; offered as a Task at the end, never made here. */
|
|
45
|
+
external: Schema.optional(Schema.Boolean)
|
|
46
|
+
})
|
|
47
|
+
|
|
48
|
+
const Discovery = Schema.Struct({
|
|
49
|
+
stack: Schema.String,
|
|
50
|
+
journeys: Schema.Array(Schema.Struct({ name: Schema.String, objective: Schema.String })),
|
|
51
|
+
telemetry: Schema.Array(
|
|
52
|
+
Schema.Struct({ source: Schema.String, status: Schema.Literals(['used', 'unavailable']), why: Schema.String })
|
|
53
|
+
),
|
|
54
|
+
/** Ranked, most promising first. */
|
|
55
|
+
opportunities: Schema.Array(Opportunity)
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
const Harness = Schema.Struct({
|
|
59
|
+
/** What the benchmark exercises and why it stands for the journeys. */
|
|
60
|
+
summary: Schema.String,
|
|
61
|
+
plan: MeasurePlan
|
|
62
|
+
})
|
|
63
|
+
|
|
64
|
+
const Change = Schema.Struct({
|
|
65
|
+
/** changed: one change is committed; exhausted: nothing left worth trying; unsafe: the gain would cost correctness. */
|
|
66
|
+
status: Schema.Literals(['changed', 'exhausted', 'unsafe']),
|
|
67
|
+
summary: Schema.String
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
const Kept = Schema.Struct({ summary: Schema.String, verdict: Schema.String, reason: Schema.String })
|
|
71
|
+
|
|
72
|
+
const Output = Schema.Struct({
|
|
73
|
+
/** improved: a ChangeRequest holds a gain the final measurement proved; otherwise a findings report only. */
|
|
74
|
+
status: Schema.Literals(['improved', 'no_gain', 'gain_gone', 'checks_failed', 'too_noisy']),
|
|
75
|
+
reason: Schema.String,
|
|
76
|
+
kept: Schema.Array(Kept),
|
|
77
|
+
cycles: Schema.Int,
|
|
78
|
+
reportArtifactId: Schema.String,
|
|
79
|
+
changeRequest: Schema.optional(ChangeRequestResult),
|
|
80
|
+
/** Improvements only another repository can make. */
|
|
81
|
+
external: Schema.Array(Schema.String),
|
|
82
|
+
/** What the ChangeRequest leaves open. */
|
|
83
|
+
unsettled: Schema.optional(Schema.Array(Schema.String))
|
|
84
|
+
})
|
|
85
|
+
|
|
86
|
+
const discoverPrompt = (focus: string | undefined) =>
|
|
87
|
+
[
|
|
88
|
+
`Find where this repository is slow${focus === undefined ? ', for the journeys that matter most' : `: ${focus}`}. Change nothing.`,
|
|
89
|
+
'Read the stack, the user journeys and the objective each has, the docs and earlier performance work. Read whatever telemetry you can reach, read-only and with bounded queries; list each source as used or unavailable, with why.',
|
|
90
|
+
'Rank the opportunities, most promising first, each marked measured, telemetry or hypothesis. Mark one that only another repository can make as external.'
|
|
91
|
+
].join('\n\n')
|
|
92
|
+
|
|
93
|
+
const harnessPrompt = (discovery: typeof Discovery.Type) =>
|
|
94
|
+
[
|
|
95
|
+
'Commit a local benchmark for these journeys, and change no product code:',
|
|
96
|
+
...discovery.journeys.map(journey => `- ${journey.name}: ${journey.objective}`),
|
|
97
|
+
'It runs on this Runner alone: never call a service, database or URL outside it. Each run prints one JSON line of numbers by metric name as its last line; wall time is measured for you as wall_ms. Keep it under a minute a run.',
|
|
98
|
+
'Give the measurement plan: the command, any prepare command, and the metrics with one primary and the rest guardrails. Add a low-variance counter (instructions, allocations, queries) where you can, marked lowVariance.'
|
|
99
|
+
].join('\n\n')
|
|
100
|
+
|
|
101
|
+
const changePrompt = (discovery: typeof Discovery.Type, plan: MeasurePlan, tried: readonly string[], cycle: number) =>
|
|
102
|
+
[
|
|
103
|
+
`Cycle ${cycle}: make one bounded change that should improve ${plan.metrics.find(metric => metric.role === 'primary')?.name ?? 'the primary metric'} without making a guardrail worse, and commit it. Kitman measures it; do not report numbers of your own.`,
|
|
104
|
+
'Opportunities, most promising first:',
|
|
105
|
+
...discovery.opportunities.flatMap(entry =>
|
|
106
|
+
entry.external === true ? [] : [`- ${entry.title} (${entry.evidence}): ${entry.detail}`]
|
|
107
|
+
),
|
|
108
|
+
...(tried.length === 0 ? [] : ['Already tried:', ...tried.map(entry => `- ${entry}`)]),
|
|
109
|
+
'Keep behaviour exactly as it is and run the tests you touch. Answer exhausted when nothing is left worth trying, or unsafe when the gain would cost correctness; either way change nothing.'
|
|
110
|
+
].join('\n\n')
|
|
111
|
+
|
|
112
|
+
const revertPrompt = (start: string, cycle: number) =>
|
|
113
|
+
`Cycle ${cycle}'s change did not prove faster. Revert its commits, and only those: run \`git revert --no-edit ${start}..HEAD\`. Touch no commit at or before ${start}, never reset or force-push, and change nothing else.`
|
|
114
|
+
|
|
115
|
+
const fixPrompt = (findings: readonly { readonly id: string; readonly detail: string }[]) =>
|
|
116
|
+
[
|
|
117
|
+
'The review of the climb found these. Fix each, keep behaviour exactly as it is and keep the gain, and commit. Kitman verifies and measures again; a fix that costs the gain ends the climb.',
|
|
118
|
+
...findings.map(finding => `- ${finding.id}: ${finding.detail}`)
|
|
119
|
+
].join('\n\n')
|
|
120
|
+
|
|
121
|
+
const percent = (fraction: number) => `${(fraction * 100).toFixed(1)}%`
|
|
122
|
+
|
|
123
|
+
const numbers = (measured: Measurement) =>
|
|
124
|
+
[
|
|
125
|
+
'| Metric | Role | Base median | Head median | Change | Noise |',
|
|
126
|
+
'|---|---|---|---|---|---|',
|
|
127
|
+
...measured.metrics.map(
|
|
128
|
+
metric =>
|
|
129
|
+
`| ${metric.name} | ${metric.role} | ${metric.base.median} | ${metric.head.median} | ${percent(metric.change)} | ${percent(metric.noise)} |`
|
|
130
|
+
)
|
|
131
|
+
].join('\n')
|
|
132
|
+
|
|
133
|
+
const report = (input: {
|
|
134
|
+
readonly discovery: typeof Discovery.Type
|
|
135
|
+
readonly harness: typeof Harness.Type
|
|
136
|
+
readonly reason: string
|
|
137
|
+
readonly kept: readonly (typeof Kept.Type)[]
|
|
138
|
+
readonly tried: readonly string[]
|
|
139
|
+
readonly final: Measurement | undefined
|
|
140
|
+
}) =>
|
|
141
|
+
[
|
|
142
|
+
`# Hillclimb findings\n\n${input.reason}`,
|
|
143
|
+
`## Stack\n\n${input.discovery.stack}`,
|
|
144
|
+
`## Telemetry\n\n${input.discovery.telemetry.map(entry => `- ${entry.source}: ${entry.status}, ${entry.why}`).join('\n') || '- None reached'}`,
|
|
145
|
+
`## Benchmark\n\n${input.harness.summary}`,
|
|
146
|
+
`## Tried\n\n${input.tried.map(entry => `- ${entry}`).join('\n') || '- Nothing'}`,
|
|
147
|
+
...(input.kept.length === 0
|
|
148
|
+
? []
|
|
149
|
+
: [`## Kept\n\n${input.kept.map(entry => `- ${entry.summary}: ${entry.reason}`).join('\n')}`]),
|
|
150
|
+
...(input.final === undefined ? [] : [`## Final measurement\n\n${input.final.reason}\n\n${numbers(input.final)}`]),
|
|
151
|
+
`## Opportunities\n\n${input.discovery.opportunities.map(entry => `- ${entry.title} (${entry.evidence}${entry.external === true ? ', external' : ''}): ${entry.detail}`).join('\n')}`
|
|
152
|
+
].join('\n\n')
|
|
153
|
+
|
|
154
|
+
/** Makes the plan's low-variance counter its primary metric, the old primary a guardrail. */
|
|
155
|
+
const steadier = (plan: MeasurePlan): MeasurePlan | undefined => {
|
|
156
|
+
const counter = plan.metrics.find(metric => metric.lowVariance === true && metric.role !== 'primary')
|
|
157
|
+
|
|
158
|
+
return counter === undefined
|
|
159
|
+
? undefined
|
|
160
|
+
: {
|
|
161
|
+
...plan,
|
|
162
|
+
metrics: plan.metrics.map(metric => ({
|
|
163
|
+
...metric,
|
|
164
|
+
role: metric.name === counter.name ? ('primary' as const) : ('guardrail' as const)
|
|
165
|
+
}))
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Climbs one measured step at a time and keeps only the steps Kitman's own measurement proves uphill. It reads the
|
|
171
|
+
* repository and what telemetry it can reach, commits a benchmark, measures the noise floor, then tries one change a
|
|
172
|
+
* cycle: kept on a gain with no guardrail worse, else reverted to the cycle's start. Two unkept cycles in a row end
|
|
173
|
+
* the climb. A ChangeRequest opens only when the final tree still proves the gain over the harness on the base; with
|
|
174
|
+
* no gain the Run ends with a findings report.
|
|
175
|
+
*/
|
|
176
|
+
export default defineLoop({
|
|
177
|
+
id: 'hillclimb',
|
|
178
|
+
version: '1.3.0',
|
|
179
|
+
title: 'Hillclimb',
|
|
180
|
+
outcome: 'performance',
|
|
181
|
+
input: Input,
|
|
182
|
+
output: Output,
|
|
183
|
+
agents: { investigator: 'climber', implementer: 'implementer' },
|
|
184
|
+
checkout: 'write',
|
|
185
|
+
capabilities: ['agent_work', 'verify', 'measure', 'change_request', 'write_back', 'ask', 'create_tasks'],
|
|
186
|
+
calls: () => [review],
|
|
187
|
+
offers: [{ id: 'retry', title: 'Climb again', loop: 'hillclimb' }],
|
|
188
|
+
run: input =>
|
|
189
|
+
Effect.gen(function* () {
|
|
190
|
+
const limits = {
|
|
191
|
+
...(input.threshold === undefined ? undefined : { threshold: input.threshold }),
|
|
192
|
+
...(input.tolerance === undefined ? undefined : { tolerance: input.tolerance })
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
const discovery = yield* agentWork('discover', {
|
|
196
|
+
agent: 'investigator',
|
|
197
|
+
prompt: discoverPrompt(input.focus),
|
|
198
|
+
output: Discovery
|
|
199
|
+
})
|
|
200
|
+
|
|
201
|
+
const external = discovery.opportunities.flatMap(entry => (entry.external === true ? [entry.title] : []))
|
|
202
|
+
|
|
203
|
+
// The harness sits on the base as the climb found it, so every baseline holds the benchmark.
|
|
204
|
+
yield* syncBase('sync:start', {
|
|
205
|
+
agent: 'implementer',
|
|
206
|
+
...(input.branch === undefined ? undefined : { target: input.branch })
|
|
207
|
+
})
|
|
208
|
+
|
|
209
|
+
const harness = yield* agentWorkOnce('harness', {
|
|
210
|
+
agent: 'implementer',
|
|
211
|
+
prompt: harnessPrompt(discovery),
|
|
212
|
+
output: Harness
|
|
213
|
+
})
|
|
214
|
+
|
|
215
|
+
const baseline = (yield* verify('verify:harness', {})).revision
|
|
216
|
+
let plan = harness.plan
|
|
217
|
+
const calibrated = yield* measure('calibrate', { plan, base: baseline, head: baseline, ...limits })
|
|
218
|
+
const kept: (typeof Kept.Type)[] = []
|
|
219
|
+
const tried: string[] = []
|
|
220
|
+
let cycles = 0
|
|
221
|
+
|
|
222
|
+
const ending = (status: typeof Output.Type.status, reason: string, final?: Measurement) =>
|
|
223
|
+
Effect.gen(function* () {
|
|
224
|
+
const stored = yield* artifact('report', {
|
|
225
|
+
kind: 'hillclimb-report',
|
|
226
|
+
title: 'Hillclimb findings',
|
|
227
|
+
text: report({ discovery, harness, reason, kept, tried, final })
|
|
228
|
+
})
|
|
229
|
+
|
|
230
|
+
return { status, reason, kept, cycles, reportArtifactId: stored.artifactId, external }
|
|
231
|
+
})
|
|
232
|
+
|
|
233
|
+
if (calibrated.verdict === 'too_noisy') {
|
|
234
|
+
const switched = steadier(plan)
|
|
235
|
+
|
|
236
|
+
if (switched === undefined)
|
|
237
|
+
return yield* ending('too_noisy', `${calibrated.reason}. ${calibrated.suggestion ?? ''}`.trim())
|
|
238
|
+
|
|
239
|
+
plan = switched
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
let start = baseline
|
|
243
|
+
let unkept = 0
|
|
244
|
+
|
|
245
|
+
for (let cycle = 1; cycle <= (input.cycles ?? 5) && unkept < 2; cycle++) {
|
|
246
|
+
const change = yield* agentWorkOnce(`change:${cycle}`, {
|
|
247
|
+
agent: 'implementer',
|
|
248
|
+
prompt: changePrompt(discovery, plan, tried, cycle),
|
|
249
|
+
output: Change
|
|
250
|
+
})
|
|
251
|
+
|
|
252
|
+
if (change.status !== 'changed') {
|
|
253
|
+
tried.push(`Cycle ${cycle}: ${change.status}, ${change.summary}`)
|
|
254
|
+
break
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
cycles = cycle
|
|
258
|
+
|
|
259
|
+
const head = yield* verify(`verify:${cycle}`, {})
|
|
260
|
+
|
|
261
|
+
const measured = head.passed
|
|
262
|
+
? yield* measure(`measure:${cycle}`, { plan, base: start, head: head.revision, ...limits })
|
|
263
|
+
: undefined
|
|
264
|
+
|
|
265
|
+
if (measured?.verdict === 'gain') {
|
|
266
|
+
kept.push({ summary: change.summary, verdict: measured.verdict, reason: measured.reason })
|
|
267
|
+
tried.push(`Cycle ${cycle}, kept: ${change.summary} (${measured.reason})`)
|
|
268
|
+
start = head.revision
|
|
269
|
+
unkept = 0
|
|
270
|
+
continue
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
tried.push(
|
|
274
|
+
`Cycle ${cycle}, reverted: ${change.summary} (${measured === undefined ? 'its checks failed' : measured.reason})`
|
|
275
|
+
)
|
|
276
|
+
yield* agentWork(`revert:${cycle}`, {
|
|
277
|
+
agent: 'implementer',
|
|
278
|
+
prompt: revertPrompt(start, cycle),
|
|
279
|
+
output: Schema.Struct({ reverted: Schema.Boolean })
|
|
280
|
+
})
|
|
281
|
+
unkept++
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
if (external.length > 0) {
|
|
285
|
+
const file = yield* ask('external', {
|
|
286
|
+
prompt: `Only other repositories can make these improvements: ${external.join('; ')}. File them as Tasks?`,
|
|
287
|
+
options: ['File as Tasks', 'Leave them'],
|
|
288
|
+
recommended: 'File as Tasks',
|
|
289
|
+
recommendationReason:
|
|
290
|
+
'Filed as Tasks, the improvements wait in the repositories that can make them instead of being lost.',
|
|
291
|
+
default: 'Leave them',
|
|
292
|
+
timeout: '1 hour'
|
|
293
|
+
})
|
|
294
|
+
|
|
295
|
+
if (file === 'File as Tasks')
|
|
296
|
+
yield* createTasks('external-tasks', {
|
|
297
|
+
tickets: discovery.opportunities.flatMap((entry, index) =>
|
|
298
|
+
entry.external === true
|
|
299
|
+
? [
|
|
300
|
+
{
|
|
301
|
+
key: `external-${index + 1}`,
|
|
302
|
+
title: entry.title,
|
|
303
|
+
body: entry.detail,
|
|
304
|
+
acceptanceCriteria: [],
|
|
305
|
+
blockedBy: [],
|
|
306
|
+
outcome: 'performance' as const
|
|
307
|
+
}
|
|
308
|
+
]
|
|
309
|
+
: []
|
|
310
|
+
)
|
|
311
|
+
})
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
if (kept.length === 0) return yield* ending('no_gain', 'No change proved faster than the noise allows')
|
|
315
|
+
|
|
316
|
+
const synced = yield* syncBase('sync:final', {
|
|
317
|
+
agent: 'implementer',
|
|
318
|
+
...(input.branch === undefined ? undefined : { target: input.branch })
|
|
319
|
+
})
|
|
320
|
+
|
|
321
|
+
const checked = yield* verify('verify:final', {})
|
|
322
|
+
|
|
323
|
+
if (!checked.passed) {
|
|
324
|
+
const failing = checked.checks.filter(check => check.status === 'failed').map(check => check.name)
|
|
325
|
+
|
|
326
|
+
return yield* ending('checks_failed', `The final tree fails ${failing.join(', ') || 'its checks'}`)
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
let final = yield* measure('measure:final', { plan, base: baseline, head: checked.revision, ...limits })
|
|
330
|
+
|
|
331
|
+
if (final.verdict !== 'gain')
|
|
332
|
+
return yield* ending('gain_gone', `The final tree no longer proves the gain: ${final.reason}`, final)
|
|
333
|
+
|
|
334
|
+
const claims = [
|
|
335
|
+
'Behaviour is preserved: every kept change leaves what the code does exactly as it was.',
|
|
336
|
+
`The measurement is valid: ${harness.summary}`,
|
|
337
|
+
...kept.map(entry => entry.summary)
|
|
338
|
+
]
|
|
339
|
+
|
|
340
|
+
let reviewed = yield* call('review', review, {
|
|
341
|
+
base: baseline,
|
|
342
|
+
head: checked.revision,
|
|
343
|
+
outcome: 'performance',
|
|
344
|
+
claims
|
|
345
|
+
})
|
|
346
|
+
|
|
347
|
+
for (let round = 1; round <= 2 && reviewed.findings.some(isBlocking); round++) {
|
|
348
|
+
const blocking = reviewed.findings.filter(isBlocking)
|
|
349
|
+
|
|
350
|
+
yield* agentWork(`fix:${round}`, {
|
|
351
|
+
agent: 'implementer',
|
|
352
|
+
prompt: fixPrompt(blocking),
|
|
353
|
+
output: Schema.Struct({ summary: Schema.String })
|
|
354
|
+
})
|
|
355
|
+
|
|
356
|
+
const fixed = yield* verify(`verify:fix:${round}`, {})
|
|
357
|
+
|
|
358
|
+
if (!fixed.passed) {
|
|
359
|
+
const failing = fixed.checks.filter(check => check.status === 'failed').map(check => check.name)
|
|
360
|
+
|
|
361
|
+
return yield* ending('checks_failed', `The review's fixes fail ${failing.join(', ') || 'the checks'}`)
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
final = yield* measure(`measure:fix:${round}`, { plan, base: baseline, head: fixed.revision, ...limits })
|
|
365
|
+
|
|
366
|
+
if (final.verdict !== 'gain')
|
|
367
|
+
return yield* ending('gain_gone', `The review's fixes cost the gain: ${final.reason}`, final)
|
|
368
|
+
|
|
369
|
+
reviewed = yield* call(`review:${round + 1}`, review, {
|
|
370
|
+
base: baseline,
|
|
371
|
+
head: fixed.revision,
|
|
372
|
+
outcome: 'performance',
|
|
373
|
+
claims,
|
|
374
|
+
earlier: blocking
|
|
375
|
+
})
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
const unsettled = [
|
|
379
|
+
...[...reviewed.findings, ...scopeFindings(reviewed.scope)].flatMap(finding =>
|
|
380
|
+
isBlocking(finding) ? [`${finding.id}: ${finding.detail}`] : []
|
|
381
|
+
),
|
|
382
|
+
...(synced.status === 'conflict' ? [`Merging the base in conflicts on ${synced.files.join(', ')}`] : [])
|
|
383
|
+
]
|
|
384
|
+
|
|
385
|
+
const caveats =
|
|
386
|
+
synced.status === 'merged'
|
|
387
|
+
? ['The base moved during the climb; the final measurement includes its changes too.']
|
|
388
|
+
: []
|
|
389
|
+
|
|
390
|
+
const opened = yield* changeRequest('change-request', {
|
|
391
|
+
title: `Faster: ${kept.map(entry => entry.summary).join('; ')}`.slice(0, 240),
|
|
392
|
+
body: [
|
|
393
|
+
`${final.reason}, measured by Kitman against the benchmark on the base.`,
|
|
394
|
+
numbers(final),
|
|
395
|
+
`## Kept\n\n${kept.map(entry => `- ${entry.summary}: ${entry.reason}`).join('\n')}`,
|
|
396
|
+
`## Benchmark\n\n${harness.summary}\n\n\`${plan.command}\``,
|
|
397
|
+
...(unsettled.length + caveats.length === 0
|
|
398
|
+
? []
|
|
399
|
+
: [`## Unsettled\n\n${[...unsettled, ...caveats].map(entry => `- ${entry}`).join('\n')}`])
|
|
400
|
+
].join('\n\n')
|
|
401
|
+
})
|
|
402
|
+
|
|
403
|
+
yield* writeBack('write-back', { milestone: 'change_request_opened', changeRequestUrl: opened.url })
|
|
404
|
+
|
|
405
|
+
return { ...(yield* ending('improved', final.reason, final)), changeRequest: opened, unsettled }
|
|
406
|
+
})
|
|
407
|
+
})
|
package/src/hindsight.ts
ADDED
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
import {
|
|
2
|
+
agentWork,
|
|
3
|
+
call,
|
|
4
|
+
defineLoop,
|
|
5
|
+
propose,
|
|
6
|
+
TranscriptPoint,
|
|
7
|
+
transcripts,
|
|
8
|
+
type ProposeRequest,
|
|
9
|
+
type TranscriptsResult
|
|
10
|
+
} from '@kitman-sh/sdk/loop'
|
|
11
|
+
import * as Effect from 'effect/Effect'
|
|
12
|
+
import * as Schema from 'effect/Schema'
|
|
13
|
+
import build from './build'
|
|
14
|
+
import errand from './errand'
|
|
15
|
+
|
|
16
|
+
const Input = Schema.Struct({
|
|
17
|
+
/** How many days back to read; a week when left out. */
|
|
18
|
+
days: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(1), Schema.isLessThanOrEqualTo(31))),
|
|
19
|
+
/** Reads one Team's Runs; every Team's the Run may read when left out. */
|
|
20
|
+
teamId: Schema.optional(Schema.String),
|
|
21
|
+
/** How many findings to start fixing as Errands or Builds; none when left out. */
|
|
22
|
+
improvements: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0), Schema.isLessThanOrEqualTo(5)))
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
const categories = [
|
|
26
|
+
'navigation',
|
|
27
|
+
'checks',
|
|
28
|
+
'standards',
|
|
29
|
+
'instructions',
|
|
30
|
+
'tool_economy',
|
|
31
|
+
'no_ops',
|
|
32
|
+
'information_access',
|
|
33
|
+
'kitman'
|
|
34
|
+
] as const
|
|
35
|
+
|
|
36
|
+
const Category = Schema.Literals(categories)
|
|
37
|
+
|
|
38
|
+
const Finding = Schema.Struct({
|
|
39
|
+
/** A stable kebab-case name for what goes wrong; the same problem gets the same key every week. */
|
|
40
|
+
key: Schema.String.check(Schema.isPattern(/^[a-z0-9]+(?:-[a-z0-9]+)*$/), Schema.isMaxLength(80)),
|
|
41
|
+
category: Category,
|
|
42
|
+
/** Memory teaches agents to avoid it; a Task changes what causes it. */
|
|
43
|
+
kind: Schema.Literals(['memory', 'task']),
|
|
44
|
+
/** The Memory's words, or the Task's title. */
|
|
45
|
+
lesson: Schema.String,
|
|
46
|
+
/** The Task's description. */
|
|
47
|
+
detail: Schema.optional(Schema.String),
|
|
48
|
+
/** How to fix the cause when a Run may start on it: an errand for a small change, a build for a feature. */
|
|
49
|
+
fix: Schema.optional(Schema.Struct({ size: Schema.Literals(['errand', 'build']), words: Schema.String })),
|
|
50
|
+
citations: Schema.Array(TranscriptPoint)
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
type Finding = typeof Finding.Type
|
|
54
|
+
|
|
55
|
+
const Findings = Schema.Struct({ findings: Schema.Array(Finding) })
|
|
56
|
+
|
|
57
|
+
const Proposal = Schema.Struct({
|
|
58
|
+
key: Schema.String,
|
|
59
|
+
category: Category,
|
|
60
|
+
kind: Schema.Literals(['memory', 'task']),
|
|
61
|
+
id: Schema.String,
|
|
62
|
+
/** It matched an earlier proposal or open Task, which it added to instead. */
|
|
63
|
+
matched: Schema.Boolean,
|
|
64
|
+
runIds: Schema.Array(Schema.String)
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
const Output = Schema.Struct({
|
|
68
|
+
/** One line on what the Run found. */
|
|
69
|
+
summary: Schema.String,
|
|
70
|
+
proposals: Schema.Array(Proposal),
|
|
71
|
+
/** Findings left out, and why. */
|
|
72
|
+
leftOut: Schema.Array(Schema.Struct({ key: Schema.String, reason: Schema.String })),
|
|
73
|
+
improvements: Schema.Array(
|
|
74
|
+
Schema.Struct({ key: Schema.String, loop: Schema.Literals(['errand', 'build']), outcome: Schema.String })
|
|
75
|
+
)
|
|
76
|
+
})
|
|
77
|
+
|
|
78
|
+
type ReadRun = TranscriptsResult['runs'][number]
|
|
79
|
+
|
|
80
|
+
const prompt = (runs: readonly ReadRun[], days: number) =>
|
|
81
|
+
[
|
|
82
|
+
`These Runs settled in the last ${days} days, each with its status: a failed one with its failure, one with a retried Step marked retried. Find what keeps going wrong in more than one Run: a failure, or in Runs that completed, a wasted step, a wrong turn, a check run again and again, or work redone. Fix the cause, not the symptom.`,
|
|
83
|
+
'```json',
|
|
84
|
+
JSON.stringify(runs, null, 2),
|
|
85
|
+
'```',
|
|
86
|
+
`Give each finding a category: ${categories.join(', ')}. Use kitman when Kitman itself misbehaved: a Run that stalled, parked or needed rescue for a reason in Kitman, not in the work.`,
|
|
87
|
+
'Give each a stable kebab-case key naming the problem, not the week: the same problem must get the same key next time.',
|
|
88
|
+
"Propose Memory when agents can avoid it by knowing something, or a Task when something has to change; when the change is clear, say how to fix it as an errand (small) or a build (a feature). Cite every Transcript point (runId, stepAttemptId, sequence) it shows in. Cite what happened: tool calls and results, files and answers. An agent's own messages are not evidence. Leave out anything seen in one Run only.",
|
|
89
|
+
"Never quote a person's words into a lesson."
|
|
90
|
+
].join('\n\n')
|
|
91
|
+
|
|
92
|
+
const point = (cited: TranscriptPoint) => `${cited.runId}/${cited.stepAttemptId}/${cited.sequence}`
|
|
93
|
+
|
|
94
|
+
/** The one value every cited Run has, if they share it. */
|
|
95
|
+
const shared = (cited: readonly ReadRun[], field: 'teamId') => {
|
|
96
|
+
const values = new Set(cited.map(run => run[field]))
|
|
97
|
+
const [only] = values
|
|
98
|
+
|
|
99
|
+
return values.size === 1 ? only : undefined
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* What a finding proposes, citing only evidence that was read (not an agent's own messages), or why it is left out
|
|
104
|
+
* when that evidence falls in fewer than two Runs.
|
|
105
|
+
*/
|
|
106
|
+
const proposal = (
|
|
107
|
+
finding: Finding,
|
|
108
|
+
runs: readonly ReadRun[]
|
|
109
|
+
): { readonly request: ProposeRequest } | { readonly request?: undefined; readonly reason: string } => {
|
|
110
|
+
const evidence = new Set(
|
|
111
|
+
runs.flatMap(run =>
|
|
112
|
+
run.entries.flatMap(entry =>
|
|
113
|
+
entry.kind === 'message'
|
|
114
|
+
? []
|
|
115
|
+
: [point({ runId: run.runId, stepAttemptId: entry.stepAttemptId, sequence: entry.sequence })]
|
|
116
|
+
)
|
|
117
|
+
)
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
const citations = [
|
|
121
|
+
...new Map(
|
|
122
|
+
finding.citations.filter(cited => evidence.has(point(cited))).map(cited => [point(cited), cited])
|
|
123
|
+
).values()
|
|
124
|
+
]
|
|
125
|
+
|
|
126
|
+
const runIds = new Set(citations.map(cited => cited.runId))
|
|
127
|
+
const [first, ...rest] = citations
|
|
128
|
+
|
|
129
|
+
if (first === undefined)
|
|
130
|
+
return { reason: 'It cites no evidence that was read: an agent’s own words or a summary are not evidence' }
|
|
131
|
+
|
|
132
|
+
if (runIds.size < 2) return { reason: 'It was seen in one Run only' }
|
|
133
|
+
|
|
134
|
+
const teamId = shared(
|
|
135
|
+
runs.filter(run => runIds.has(run.runId)),
|
|
136
|
+
'teamId'
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
if (finding.kind === 'memory' && teamId !== undefined)
|
|
140
|
+
return { request: { kind: 'memory', teamId, content: finding.lesson, citations: [first, ...rest] } }
|
|
141
|
+
|
|
142
|
+
return {
|
|
143
|
+
request: {
|
|
144
|
+
kind: 'task',
|
|
145
|
+
title: finding.lesson,
|
|
146
|
+
description: finding.detail ?? finding.lesson,
|
|
147
|
+
...(teamId === undefined ? undefined : { teamId }),
|
|
148
|
+
key: finding.key,
|
|
149
|
+
citations: [first, ...rest]
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Reads every Run that settled in the last week (or `days`), of one Team or every Team the person it acts for may read,
|
|
156
|
+
* completed ones too, and proposes Memory and Tasks for what went wrong in at least two, each with a category and a stable key. A key that
|
|
157
|
+
* matches an open Task or a waiting proposal adds the new Runs there instead of a duplicate. With `improvements`, that
|
|
158
|
+
* many findings are started as Errands or Builds. Memory belongs to one Team, so a lesson from Runs in several
|
|
159
|
+
* becomes a Task.
|
|
160
|
+
*/
|
|
161
|
+
export default defineLoop({
|
|
162
|
+
id: 'hindsight',
|
|
163
|
+
version: '1.0.0',
|
|
164
|
+
title: 'Hindsight',
|
|
165
|
+
input: Input,
|
|
166
|
+
output: Output,
|
|
167
|
+
outcome: 'investigation',
|
|
168
|
+
agents: { investigator: 'investigator' },
|
|
169
|
+
checkout: 'none',
|
|
170
|
+
schedule: { rule: { frequency: 'weekly', daysOfWeek: [1], hour: 9, minute: 0 }, timezone: 'UTC' },
|
|
171
|
+
capabilities: ['read_transcripts', 'agent_work', 'propose'],
|
|
172
|
+
calls: () => [errand, build],
|
|
173
|
+
run: input =>
|
|
174
|
+
Effect.gen(function* () {
|
|
175
|
+
const days = input.days ?? 7
|
|
176
|
+
const read = yield* transcripts('read', { days })
|
|
177
|
+
const runs = read.runs.filter(run => input.teamId === undefined || run.teamId === input.teamId)
|
|
178
|
+
|
|
179
|
+
if (runs.length < 2)
|
|
180
|
+
return {
|
|
181
|
+
summary: `Nothing recurred: ${runs.length === 0 ? 'no Run' : 'one Run'} settled in the last ${days} days.`,
|
|
182
|
+
proposals: [],
|
|
183
|
+
leftOut: [],
|
|
184
|
+
improvements: []
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const { findings } = yield* agentWork('find', {
|
|
188
|
+
agent: 'investigator',
|
|
189
|
+
prompt: prompt(runs, days),
|
|
190
|
+
output: Findings
|
|
191
|
+
})
|
|
192
|
+
|
|
193
|
+
const judged = findings.map(finding => ({ finding, judged: proposal(finding, runs) }))
|
|
194
|
+
|
|
195
|
+
const leftOut = judged.flatMap(({ finding, judged }) =>
|
|
196
|
+
judged.request === undefined ? [{ key: finding.key, reason: judged.reason }] : []
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
const kept = judged.flatMap(({ finding, judged }) =>
|
|
200
|
+
judged.request === undefined ? [] : [{ finding, request: judged.request }]
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
const proposals = yield* Effect.forEach(kept, ({ finding, request }) =>
|
|
204
|
+
Effect.map(propose(`propose:${finding.key}`, request), made => ({
|
|
205
|
+
key: finding.key,
|
|
206
|
+
category: finding.category,
|
|
207
|
+
kind: made.kind,
|
|
208
|
+
id: made.id,
|
|
209
|
+
matched: made.matched === true,
|
|
210
|
+
runIds: [...new Set(request.citations.map(cited => cited.runId))]
|
|
211
|
+
}))
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
const fixable = kept.flatMap(({ finding }) =>
|
|
215
|
+
finding.fix === undefined ? [] : [{ ...finding, fix: finding.fix }]
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
const improvements = yield* Effect.forEach(fixable.slice(0, input.improvements ?? 0), finding => {
|
|
219
|
+
const started =
|
|
220
|
+
finding.fix.size === 'errand'
|
|
221
|
+
? Effect.map(call(`improve:${finding.key}`, errand, { words: finding.fix.words }), done => done.status)
|
|
222
|
+
: Effect.map(call(`improve:${finding.key}`, build, { words: finding.fix.words }), built => built.stopped)
|
|
223
|
+
|
|
224
|
+
return started.pipe(
|
|
225
|
+
Effect.catchTag('ChildFailed', failed => Effect.succeed(`failed: ${failed.reason}`)),
|
|
226
|
+
Effect.map(outcome => ({ key: finding.key, loop: finding.fix.size, outcome }))
|
|
227
|
+
)
|
|
228
|
+
})
|
|
229
|
+
|
|
230
|
+
const fresh = proposals.filter(made => !made.matched).length
|
|
231
|
+
|
|
232
|
+
return {
|
|
233
|
+
summary:
|
|
234
|
+
proposals.length === 0
|
|
235
|
+
? `Nothing recurred with evidence in ${runs.length} Runs.`
|
|
236
|
+
: `${proposals.length} recurring ${proposals.length === 1 ? 'problem' : 'problems'}: ${fresh} proposed, ${proposals.length - fresh} added to earlier work.`,
|
|
237
|
+
proposals,
|
|
238
|
+
leftOut,
|
|
239
|
+
improvements
|
|
240
|
+
}
|
|
241
|
+
})
|
|
242
|
+
})
|