@llm4ts/flow 2.30.0 → 2.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/AutonomyContract.d.ts +12 -0
- package/dist/AutonomyContract.d.ts.map +1 -0
- package/dist/AutonomyContract.js +51 -0
- package/dist/AutonomyContract.js.map +1 -0
- package/dist/CarriedNotes.d.ts +10 -0
- package/dist/CarriedNotes.d.ts.map +1 -1
- package/dist/CarriedNotes.js +27 -2
- package/dist/CarriedNotes.js.map +1 -1
- package/dist/Evidence.d.ts +12 -0
- package/dist/Evidence.d.ts.map +1 -0
- package/dist/Evidence.js +30 -0
- package/dist/Evidence.js.map +1 -0
- package/dist/Flow.d.ts +35 -3
- package/dist/Flow.d.ts.map +1 -1
- package/dist/Flow.js +58 -4
- package/dist/Flow.js.map +1 -1
- package/dist/FlowEvents.d.ts +14 -1
- package/dist/FlowEvents.d.ts.map +1 -1
- package/dist/FlowEvents.js +13 -0
- package/dist/FlowEvents.js.map +1 -1
- package/dist/GateTriage.d.ts +81 -0
- package/dist/GateTriage.d.ts.map +1 -0
- package/dist/GateTriage.js +161 -0
- package/dist/GateTriage.js.map +1 -0
- package/dist/Gates.d.ts +35 -0
- package/dist/Gates.d.ts.map +1 -0
- package/dist/Gates.js +63 -0
- package/dist/Gates.js.map +1 -0
- package/dist/OracleGuard.d.ts +53 -0
- package/dist/OracleGuard.d.ts.map +1 -0
- package/dist/OracleGuard.js +165 -0
- package/dist/OracleGuard.js.map +1 -0
- package/dist/Pack.d.ts +7 -0
- package/dist/Pack.d.ts.map +1 -1
- package/dist/Pack.js +9 -0
- package/dist/Pack.js.map +1 -1
- package/dist/Retro.d.ts.map +1 -1
- package/dist/Retro.js +5 -0
- package/dist/Retro.js.map +1 -1
- package/dist/Review.d.ts +68 -2
- package/dist/Review.d.ts.map +1 -1
- package/dist/Review.js +158 -27
- package/dist/Review.js.map +1 -1
- package/dist/Roster.d.ts +2 -0
- package/dist/Roster.d.ts.map +1 -1
- package/dist/Roster.js +3 -1
- package/dist/Roster.js.map +1 -1
- package/dist/RosterSeats.d.ts +2 -0
- package/dist/RosterSeats.d.ts.map +1 -1
- package/dist/RosterSeats.js.map +1 -1
- package/dist/Stories.d.ts +36 -2
- package/dist/Stories.d.ts.map +1 -1
- package/dist/Stories.js +159 -12
- package/dist/Stories.js.map +1 -1
- package/dist/StoryPlan.d.ts +6 -0
- package/dist/StoryPlan.d.ts.map +1 -1
- package/dist/StoryPlan.js +9 -2
- package/dist/StoryPlan.js.map +1 -1
- package/package.json +7 -2
- package/src/AutonomyContract.ts +61 -0
- package/src/CarriedNotes.ts +37 -2
- package/src/Evidence.ts +43 -0
- package/src/Flow.ts +109 -3
- package/src/FlowEvents.ts +13 -0
- package/src/GateTriage.ts +250 -0
- package/src/Gates.ts +104 -0
- package/src/OracleGuard.ts +205 -0
- package/src/Pack.ts +16 -0
- package/src/Retro.ts +7 -0
- package/src/Review.ts +261 -34
- package/src/Roster.ts +3 -1
- package/src/RosterSeats.ts +2 -0
- package/src/Stories.ts +289 -16
- package/src/StoryPlan.ts +11 -1
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
// The oracle guard (ADR 0027, decision 4): a change may not delete a test
|
|
2
|
+
// file, add a skip or focus marker, or lower the passed-test count unless
|
|
3
|
+
// its plan entry says it may. Deterministic, from the diff and the gate
|
|
4
|
+
// output; a violation is a gate failure the fix round can undo.
|
|
5
|
+
import * as Schema from "effect/Schema"
|
|
6
|
+
import { ReviewIssue } from "./Review.ts"
|
|
7
|
+
|
|
8
|
+
export class OracleRules extends Schema.Class<OracleRules>("OracleRules")({
|
|
9
|
+
/** Regex source: a path matching it is a test file. */
|
|
10
|
+
testFiles: Schema.String,
|
|
11
|
+
/** Substrings whose appearance on an added line marks a skipped or focused test. */
|
|
12
|
+
markers: Schema.Array(Schema.String)
|
|
13
|
+
}) {}
|
|
14
|
+
|
|
15
|
+
export const defaultOracleRules = OracleRules.make({
|
|
16
|
+
testFiles:
|
|
17
|
+
"(?:^|/)(?:__tests__|tests?|spec|src/test)/|\\.(?:test|spec)\\.[cm]?[jt]sx?$|_test\\.(?:go|py|rs)$|Tests?\\.(?:java|kt|scala|cs)$|^tests?_.*\\.py$",
|
|
18
|
+
markers: [
|
|
19
|
+
".skip(",
|
|
20
|
+
".only(",
|
|
21
|
+
"xit(",
|
|
22
|
+
"xdescribe(",
|
|
23
|
+
"xtest(",
|
|
24
|
+
"test.todo(",
|
|
25
|
+
"it.todo(",
|
|
26
|
+
"@Ignore",
|
|
27
|
+
"@Disabled",
|
|
28
|
+
"@pytest.mark.skip",
|
|
29
|
+
"#[ignore]"
|
|
30
|
+
]
|
|
31
|
+
})
|
|
32
|
+
|
|
33
|
+
/** The parsed body of a pack's `## Oracle` section: `- tests: <regex>` and `- markers: a, b`. */
|
|
34
|
+
export const oracleRulesFrom = (fields: {
|
|
35
|
+
readonly tests?: string
|
|
36
|
+
readonly markers?: ReadonlyArray<string>
|
|
37
|
+
}): OracleRules =>
|
|
38
|
+
OracleRules.make({
|
|
39
|
+
testFiles: fields.tests ?? defaultOracleRules.testFiles,
|
|
40
|
+
markers: [...defaultOracleRules.markers, ...(fields.markers ?? [])]
|
|
41
|
+
})
|
|
42
|
+
|
|
43
|
+
export interface AddedLine {
|
|
44
|
+
readonly file: string
|
|
45
|
+
readonly line: number
|
|
46
|
+
readonly text: string
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export interface DiffShape {
|
|
50
|
+
/** Files the diff deletes outright. */
|
|
51
|
+
readonly deleted: ReadonlyArray<string>
|
|
52
|
+
/** Lines the diff adds, with the path and the new line number. */
|
|
53
|
+
readonly added: ReadonlyArray<AddedLine>
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const fileHeader = /^diff --git a\/(.+?) b\/(.+)$/u
|
|
57
|
+
const hunkHeader = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/u
|
|
58
|
+
|
|
59
|
+
/** The deletions and added lines of a unified diff; pure, tolerant of noise between files. */
|
|
60
|
+
export const parseUnifiedDiff = (diff: string): DiffShape => {
|
|
61
|
+
const deleted: Array<string> = []
|
|
62
|
+
const added: Array<AddedLine> = []
|
|
63
|
+
let file: string | undefined
|
|
64
|
+
let line = 0
|
|
65
|
+
let inHunk = false
|
|
66
|
+
for (const raw of diff.split(/\r?\n/u)) {
|
|
67
|
+
const header = fileHeader.exec(raw)
|
|
68
|
+
if (header !== null) {
|
|
69
|
+
file = header[2]
|
|
70
|
+
inHunk = false
|
|
71
|
+
continue
|
|
72
|
+
}
|
|
73
|
+
if (file === undefined) continue
|
|
74
|
+
if (raw.startsWith("deleted file mode")) {
|
|
75
|
+
deleted.push(file)
|
|
76
|
+
continue
|
|
77
|
+
}
|
|
78
|
+
const hunk = hunkHeader.exec(raw)
|
|
79
|
+
if (hunk !== null) {
|
|
80
|
+
line = Number.parseInt(hunk[1] ?? "0", 10)
|
|
81
|
+
inHunk = true
|
|
82
|
+
continue
|
|
83
|
+
}
|
|
84
|
+
if (!inHunk) continue
|
|
85
|
+
if (raw.startsWith("+++") || raw.startsWith("---")) continue
|
|
86
|
+
if (raw.startsWith("+")) {
|
|
87
|
+
added.push({ file, line, text: raw.slice(1) })
|
|
88
|
+
line += 1
|
|
89
|
+
} else if (raw.startsWith("-")) {
|
|
90
|
+
// a removed line: the new file's numbering does not advance
|
|
91
|
+
} else if (raw.startsWith("\\")) {
|
|
92
|
+
// ""
|
|
93
|
+
} else {
|
|
94
|
+
line += 1
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return { deleted, added }
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const vitestLike = /\bTests\b[^\n]*?(\d+)\s+passed/u
|
|
101
|
+
const jestLike = /\bTests:\s[^\n]*?(\d+)\s+passed/u
|
|
102
|
+
const pytestLike = /(?:^|\n)[^\n]*?\b(\d+)\s+passed\b[^\n]*\bin\s+[\d.]+s/u
|
|
103
|
+
const cargoLike = /test result:[^\n]*?(\d+)\s+passed/u
|
|
104
|
+
const mochaLike = /(?:^|\n)\s*(\d+)\s+passing\b/u
|
|
105
|
+
const junitLike =
|
|
106
|
+
/Tests run:\s*(\d+),\s*Failures:\s*(\d+),\s*Errors:\s*(\d+)(?:,\s*Skipped:\s*(\d+))?/gu
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* The passed-test count in a test runner's output, when a summary line the
|
|
110
|
+
* guard knows is present (Vitest, Jest, pytest, cargo test, mocha, JUnit
|
|
111
|
+
* via Maven/Gradle); `undefined` otherwise, which the guard reports as an
|
|
112
|
+
* Info rather than a verdict.
|
|
113
|
+
*/
|
|
114
|
+
export const passedCountIn = (output: string): number | undefined => {
|
|
115
|
+
for (const pattern of [vitestLike, jestLike, cargoLike, pytestLike, mochaLike]) {
|
|
116
|
+
const match = pattern.exec(output)
|
|
117
|
+
if (match?.[1] !== undefined) {
|
|
118
|
+
return Number.parseInt(match[1], 10)
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
// Maven prints a line per class and one grand total; the largest run is the total.
|
|
122
|
+
let largestRun = -1
|
|
123
|
+
let passedOfLargest: number | undefined
|
|
124
|
+
junitLike.lastIndex = 0
|
|
125
|
+
for (let match = junitLike.exec(output); match !== null; match = junitLike.exec(output)) {
|
|
126
|
+
const run = Number.parseInt(match[1] ?? "0", 10)
|
|
127
|
+
const failures = Number.parseInt(match[2] ?? "0", 10)
|
|
128
|
+
const errors = Number.parseInt(match[3] ?? "0", 10)
|
|
129
|
+
const skipped = Number.parseInt(match[4] ?? "0", 10)
|
|
130
|
+
if (run >= largestRun) {
|
|
131
|
+
largestRun = run
|
|
132
|
+
passedOfLargest = run - failures - errors - skipped
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
return passedOfLargest
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export interface OracleCounts {
|
|
139
|
+
/** Tests passing on the base, from its baseline; `undefined` when unknown. */
|
|
140
|
+
readonly base: number | undefined
|
|
141
|
+
/** Tests passing on the change, from its test gate; `undefined` when unknown. */
|
|
142
|
+
readonly current: number | undefined
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export const oracleIssuePrefix = "oracle: "
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* The guard's verdict on a change: one Critical per deleted test file, one
|
|
149
|
+
* per added marker (with `file:line`), one for a passed-count drop; empty
|
|
150
|
+
* when `declared` (the plan entry says tests may change). Counts are judged
|
|
151
|
+
* only when both are known; a caller reports the unknown case as an Info.
|
|
152
|
+
*/
|
|
153
|
+
export const checkOracle = (
|
|
154
|
+
diff: DiffShape,
|
|
155
|
+
counts: OracleCounts | undefined,
|
|
156
|
+
rules: OracleRules,
|
|
157
|
+
declared: boolean
|
|
158
|
+
): ReadonlyArray<ReviewIssue> => {
|
|
159
|
+
if (declared) {
|
|
160
|
+
return []
|
|
161
|
+
}
|
|
162
|
+
const testFile = new RegExp(rules.testFiles, "u")
|
|
163
|
+
const issues: Array<ReviewIssue> = []
|
|
164
|
+
for (const path of diff.deleted) {
|
|
165
|
+
if (testFile.test(path)) {
|
|
166
|
+
issues.push(
|
|
167
|
+
ReviewIssue.make({
|
|
168
|
+
severity: "Critical",
|
|
169
|
+
title: `${oracleIssuePrefix}test file deleted: ${path}`,
|
|
170
|
+
description:
|
|
171
|
+
"Restore it. A story may not delete a test unless its plan entry declares `testsChange: true`.",
|
|
172
|
+
file: path
|
|
173
|
+
})
|
|
174
|
+
)
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
for (const line of diff.added) {
|
|
178
|
+
if (!testFile.test(line.file)) continue
|
|
179
|
+
const marker = rules.markers.find((candidate) => line.text.includes(candidate))
|
|
180
|
+
if (marker === undefined) continue
|
|
181
|
+
issues.push(
|
|
182
|
+
ReviewIssue.make({
|
|
183
|
+
severity: "Critical",
|
|
184
|
+
title: `${oracleIssuePrefix}skip or focus marker added: ${marker.trim()}`,
|
|
185
|
+
description: `${line.file}:${line.line}: ${line.text.trim()}\nRemove the marker and make the test pass, or report the test as wrong in your Findings.`,
|
|
186
|
+
file: line.file,
|
|
187
|
+
line: line.line
|
|
188
|
+
})
|
|
189
|
+
)
|
|
190
|
+
}
|
|
191
|
+
if (counts?.base !== undefined && counts.current !== undefined && counts.current < counts.base) {
|
|
192
|
+
issues.push(
|
|
193
|
+
ReviewIssue.make({
|
|
194
|
+
severity: "Critical",
|
|
195
|
+
title: `${oracleIssuePrefix}fewer tests pass than on the base (${counts.current} < ${counts.base})`,
|
|
196
|
+
description:
|
|
197
|
+
"Tests were removed, skipped or no longer discovered. Restore them; declare `testsChange: true` on the story if the change is intended."
|
|
198
|
+
})
|
|
199
|
+
)
|
|
200
|
+
}
|
|
201
|
+
return issues
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
export const isOracleIssue = (issue: ReviewIssue): boolean =>
|
|
205
|
+
issue.title.startsWith(oracleIssuePrefix)
|
package/src/Pack.ts
CHANGED
|
@@ -3,6 +3,7 @@ import * as Schema from "effect/Schema"
|
|
|
3
3
|
import { Dimension } from "@llm4ts/core/eval/Eval"
|
|
4
4
|
import type { ConsolidateRules } from "./Domains.ts"
|
|
5
5
|
import { PlanParseError } from "./FlowError.ts"
|
|
6
|
+
import { oracleRulesFrom, type OracleRules } from "./OracleGuard.ts"
|
|
6
7
|
import { parseReviewer, type Reviewer } from "./Reviewer.ts"
|
|
7
8
|
import { CoverageRule } from "./SpecChecks.ts"
|
|
8
9
|
import type { WorkspaceError, WorkspaceShape } from "./Workspace.ts"
|
|
@@ -67,6 +68,12 @@ export interface Pack {
|
|
|
67
68
|
* fragment as context (`context:`). Absent: every program is its own feature.
|
|
68
69
|
*/
|
|
69
70
|
readonly consolidate: ConsolidateRules | undefined
|
|
71
|
+
/**
|
|
72
|
+
* The `## Oracle` section (ADR 0027): `- tests: <regex>` for what a test
|
|
73
|
+
* file is and `- markers: a, b` for extra skip or focus markers, merged
|
|
74
|
+
* over the defaults. Absent: the defaults.
|
|
75
|
+
*/
|
|
76
|
+
readonly oracle: OracleRules | undefined
|
|
70
77
|
readonly dir: string
|
|
71
78
|
readonly gate: (name: string) => ReadonlyArray<string> | undefined
|
|
72
79
|
readonly prompt: (name: string) => string | undefined
|
|
@@ -266,6 +273,14 @@ export const loadPack = Effect.fn("@llm4ts/flow/Pack.load")(function* (
|
|
|
266
273
|
message: `pack manifest 'spec-schema:' must be 'pagespec' when set, got: ${specSchema}`
|
|
267
274
|
})
|
|
268
275
|
}
|
|
276
|
+
const oracleValues = namedItems(section(manifest.sections, "Oracle"))
|
|
277
|
+
const oracle: OracleRules | undefined =
|
|
278
|
+
section(manifest.sections, "Oracle") === undefined
|
|
279
|
+
? undefined
|
|
280
|
+
: oracleRulesFrom({
|
|
281
|
+
...(oracleValues.tests === undefined ? {} : { tests: oracleValues.tests }),
|
|
282
|
+
markers: commaList(oracleValues.markers)
|
|
283
|
+
})
|
|
269
284
|
const consolidateValues = namedItems(section(manifest.sections, "Consolidate"))
|
|
270
285
|
const consolidate: ConsolidateRules | undefined =
|
|
271
286
|
section(manifest.sections, "Consolidate") === undefined
|
|
@@ -330,6 +345,7 @@ export const loadPack = Effect.fn("@llm4ts/flow/Pack.load")(function* (
|
|
|
330
345
|
featureFiles,
|
|
331
346
|
specSchema,
|
|
332
347
|
consolidate,
|
|
348
|
+
oracle,
|
|
333
349
|
dir: directory,
|
|
334
350
|
gate: (name) => gates[name],
|
|
335
351
|
prompt: (name) => prompts[name],
|
package/src/Retro.ts
CHANGED
|
@@ -461,6 +461,13 @@ export const renderRetroDigest = (inputs: RetroInputs): string => {
|
|
|
461
461
|
case "CapabilityUnenforceable":
|
|
462
462
|
runLines.push(`- capability unenforceable: ${event.detail}`)
|
|
463
463
|
break
|
|
464
|
+
case "EvidenceChecked":
|
|
465
|
+
if (event.unverified > 0) {
|
|
466
|
+
runLines.push(
|
|
467
|
+
`- fabricated status: task "${event.task}"${event.lane === undefined ? "" : ` (story ${event.lane})`} claimed ${event.unverified} command(s) that never ran`
|
|
468
|
+
)
|
|
469
|
+
}
|
|
470
|
+
break
|
|
464
471
|
case "ExecutorExcluded":
|
|
465
472
|
runLines.push(`- executor excluded: ${event.executor} — ${event.reason}`)
|
|
466
473
|
break
|
package/src/Review.ts
CHANGED
|
@@ -1,16 +1,26 @@
|
|
|
1
1
|
import { join } from "node:path"
|
|
2
2
|
import * as Clock from "effect/Clock"
|
|
3
|
+
import * as Duration from "effect/Duration"
|
|
3
4
|
import * as Effect from "effect/Effect"
|
|
5
|
+
import * as Ref from "effect/Ref"
|
|
4
6
|
import * as Schema from "effect/Schema"
|
|
5
7
|
import type { LlmError } from "@llm4ts/core/Errors"
|
|
6
8
|
import type { LlmServiceShape } from "@llm4ts/core/LlmService"
|
|
7
9
|
import type { JsonSchema } from "@llm4ts/core/Models"
|
|
8
|
-
import type { ProcessExecutorShape } from "@llm4ts/core/ProcessExecutor"
|
|
10
|
+
import type { ProcessExecutorShape, ProcessResult } from "@llm4ts/core/ProcessExecutor"
|
|
9
11
|
import { Capabilities } from "@llm4ts/core/Capability"
|
|
10
12
|
import { guarded } from "./CapabilityGuard.ts"
|
|
11
13
|
import type { Chat } from "./Chat.ts"
|
|
12
14
|
import { FlowLlmError, ProcessError, describeFlowError, type FlowError } from "./FlowError.ts"
|
|
13
15
|
import type { PlainFileStoreShape } from "./Persistence.ts"
|
|
16
|
+
import { GateBaseline, isGateIssue, triageGates } from "./GateTriage.ts"
|
|
17
|
+
import {
|
|
18
|
+
type OracleRules,
|
|
19
|
+
checkOracle,
|
|
20
|
+
defaultOracleRules,
|
|
21
|
+
parseUnifiedDiff,
|
|
22
|
+
passedCountIn
|
|
23
|
+
} from "./OracleGuard.ts"
|
|
14
24
|
import { cachedReview, fingerprintOf } from "./ReviewCache.ts"
|
|
15
25
|
import {
|
|
16
26
|
Info,
|
|
@@ -51,7 +61,13 @@ export class ReviewIssue extends Schema.Class<ReviewIssue>("ReviewIssue")({
|
|
|
51
61
|
confidence: Schema.Number.pipe(
|
|
52
62
|
Schema.withConstructorDefault(Effect.succeed(1)),
|
|
53
63
|
Schema.withDecodingDefaultKey(Effect.succeed(1))
|
|
54
|
-
)
|
|
64
|
+
),
|
|
65
|
+
/** A gate failure's triage against the base (ADR 0027); absent for review findings. */
|
|
66
|
+
origin: Schema.optionalKey(Schema.Literals(["new", "base", "flaky"])),
|
|
67
|
+
/** How a gate ended: a red exit, killed at the timeout, or a signal exit. */
|
|
68
|
+
gateClass: Schema.optionalKey(Schema.Literals(["red", "hang", "crash"])),
|
|
69
|
+
/** Where the gate's full output was written, when the caller asked for a log. */
|
|
70
|
+
logPath: Schema.optionalKey(Schema.String)
|
|
55
71
|
}) {}
|
|
56
72
|
|
|
57
73
|
export class ReviewResult extends Schema.Class<ReviewResult>("ReviewResult")({
|
|
@@ -59,7 +75,9 @@ export class ReviewResult extends Schema.Class<ReviewResult>("ReviewResult")({
|
|
|
59
75
|
summary: Schema.String.pipe(
|
|
60
76
|
Schema.withConstructorDefault(Effect.succeed("")),
|
|
61
77
|
Schema.withDecodingDefaultKey(Effect.succeed(""))
|
|
62
|
-
)
|
|
78
|
+
),
|
|
79
|
+
/** A test gate's passed-test count, when its output had a summary the oracle guard reads (ADR 0027). */
|
|
80
|
+
passed: Schema.optionalKey(Schema.Int)
|
|
63
81
|
}) {
|
|
64
82
|
get isClean(): boolean {
|
|
65
83
|
return this.issues.length === 0
|
|
@@ -241,20 +259,165 @@ export const reviewPrompt = (task: string, diff: string): string =>
|
|
|
241
259
|
diff
|
|
242
260
|
].join("\n")
|
|
243
261
|
|
|
244
|
-
export
|
|
245
|
-
|
|
262
|
+
export interface FixPromptOptions {
|
|
263
|
+
/** Characters of a gate's output kept in the prompt, from the end. Default 4000. */
|
|
264
|
+
readonly tailChars?: number
|
|
265
|
+
/** Name the gate log's path (a CLI coder can open it; an API coder cannot). Default false. */
|
|
266
|
+
readonly showPaths?: boolean
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
export const defaultGateTailChars = 4_000
|
|
270
|
+
|
|
271
|
+
const evidenceNote =
|
|
272
|
+
"The gate output above is your only runtime evidence. If you are guessing without it, say `confidence: low` in your Findings."
|
|
273
|
+
|
|
274
|
+
export const fixPrompt = (result: ReviewResult, options: FixPromptOptions = {}): string => {
|
|
275
|
+
const tailChars = options.tailChars ?? defaultGateTailChars
|
|
276
|
+
const lines = result.issues.map((issue) => {
|
|
277
|
+
const gate = isGateIssue(issue)
|
|
278
|
+
const description =
|
|
279
|
+
gate && issue.description.length > tailChars
|
|
280
|
+
? `…${issue.description.slice(-tailChars)}`
|
|
281
|
+
: issue.description
|
|
282
|
+
const path =
|
|
283
|
+
gate && options.showPaths === true && issue.logPath !== undefined
|
|
284
|
+
? ` (full output: ${issue.logPath})`
|
|
285
|
+
: ""
|
|
286
|
+
return `- [${issue.severity}] ${issue.title}: ${description}${path}`
|
|
287
|
+
})
|
|
288
|
+
const hasGate = result.issues.some(isGateIssue)
|
|
289
|
+
return [
|
|
246
290
|
"Address these review findings, then stop:",
|
|
247
|
-
...
|
|
291
|
+
...lines,
|
|
292
|
+
...(hasGate ? ["", evidenceNote] : [])
|
|
248
293
|
].join("\n")
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
export interface GateTriageOptions {
|
|
297
|
+
/** The baseline for the code this change started from; `undefined` means no triage. */
|
|
298
|
+
readonly baseline: Effect.Effect<GateBaseline | undefined, FlowError>
|
|
299
|
+
/** Root prefixes stripped from gate output before comparing (the work dir, the app dir). */
|
|
300
|
+
readonly roots: ReadonlyArray<string>
|
|
301
|
+
/** Re-run the test gate alone to tell a flaky line from a new one; omit to never rerun. */
|
|
302
|
+
readonly rerunTest?: Effect.Effect<ReviewResult, FlowError>
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
const listed = (lines: ReadonlyArray<string>): string => lines.map((line) => ` ${line}`).join("\n")
|
|
306
|
+
|
|
307
|
+
export interface OracleGateOptions {
|
|
308
|
+
/** The change under review as a unified diff (committed and uncommitted). */
|
|
309
|
+
readonly diff: Effect.Effect<string, FlowError>
|
|
310
|
+
readonly rules?: OracleRules
|
|
311
|
+
/** The plan entry says tests may change (`testsChange: true`); the guard then stays silent. */
|
|
312
|
+
readonly declared?: boolean
|
|
313
|
+
/** Tests passing on the base, from its baseline; omitted or `undefined` skips the count check. */
|
|
314
|
+
readonly baseCount?: Effect.Effect<number | undefined, FlowError>
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* The oracle guard (ADR 0027 decision 4) joined to a lint result: deleted
|
|
319
|
+
* test files, added skip or focus markers and a passed-count drop become
|
|
320
|
+
* Critical issues the fix round can undo. When a count is unknown on either
|
|
321
|
+
* side the comparison is skipped and said once.
|
|
322
|
+
*/
|
|
323
|
+
export const withOracle = Effect.fn("@llm4ts/flow/Review.withOracle")(function* (
|
|
324
|
+
lint: ReviewResult,
|
|
325
|
+
oracle: OracleGateOptions | undefined,
|
|
326
|
+
events: FlowEventsShape,
|
|
327
|
+
noted: Ref.Ref<boolean>
|
|
328
|
+
): Effect.fn.Return<ReviewResult, FlowError> {
|
|
329
|
+
if (oracle === undefined || oracle.declared === true) {
|
|
330
|
+
return lint
|
|
331
|
+
}
|
|
332
|
+
const diff = yield* oracle.diff
|
|
333
|
+
const base = oracle.baseCount === undefined ? undefined : yield* oracle.baseCount
|
|
334
|
+
if ((base === undefined || lint.passed === undefined) && !(yield* Ref.get(noted))) {
|
|
335
|
+
yield* Ref.set(noted, true)
|
|
336
|
+
yield* events.publish(
|
|
337
|
+
Info.make({
|
|
338
|
+
message:
|
|
339
|
+
"oracle guard: no passed-test count on the base or the change (no summary line the guard reads), count comparison skipped"
|
|
340
|
+
})
|
|
341
|
+
)
|
|
342
|
+
}
|
|
343
|
+
const issues = checkOracle(
|
|
344
|
+
parseUnifiedDiff(diff),
|
|
345
|
+
{ base, current: lint.passed },
|
|
346
|
+
oracle.rules ?? defaultOracleRules,
|
|
347
|
+
false
|
|
348
|
+
)
|
|
349
|
+
return issues.length === 0
|
|
350
|
+
? lint
|
|
351
|
+
: ReviewResult.make({
|
|
352
|
+
issues: [...lint.issues, ...issues],
|
|
353
|
+
summary: [lint.summary, "oracle guard"].filter((part) => part.length > 0).join("; "),
|
|
354
|
+
...(lint.passed === undefined ? {} : { passed: lint.passed })
|
|
355
|
+
})
|
|
356
|
+
})
|
|
249
357
|
|
|
250
|
-
|
|
251
|
-
|
|
358
|
+
/**
|
|
359
|
+
* Charges a lint result only with what the change caused (ADR 0027).
|
|
360
|
+
* Inherited lines are published once per `reported` set as Info; with
|
|
361
|
+
* `rerunTest`, a new line that is green on one rerun of the test gate is
|
|
362
|
+
* flaky: published, not charged. Without options or a baseline the result
|
|
363
|
+
* is returned as is.
|
|
364
|
+
*/
|
|
365
|
+
export const applyTriage = Effect.fn("@llm4ts/flow/Review.applyTriage")(function* (
|
|
366
|
+
lint: ReviewResult,
|
|
367
|
+
triage: GateTriageOptions | undefined,
|
|
368
|
+
events: FlowEventsShape,
|
|
369
|
+
reported: Ref.Ref<ReadonlySet<string>>
|
|
370
|
+
): Effect.fn.Return<ReviewResult, FlowError> {
|
|
371
|
+
if (triage === undefined || lint.isClean) {
|
|
372
|
+
return lint
|
|
373
|
+
}
|
|
374
|
+
const baseline = yield* triage.baseline
|
|
375
|
+
if (baseline === undefined) {
|
|
376
|
+
return lint
|
|
377
|
+
}
|
|
378
|
+
let triaged = triageGates(lint, baseline, triage.roots)
|
|
379
|
+
const seen = yield* Ref.get(reported)
|
|
380
|
+
const unseen = triaged.inherited.filter((line) => !seen.has(line))
|
|
381
|
+
if (unseen.length > 0) {
|
|
382
|
+
yield* Ref.update(reported, (set) => new Set([...set, ...unseen]))
|
|
383
|
+
yield* events.publish(
|
|
384
|
+
Info.make({
|
|
385
|
+
message: `${unseen.length} gate failure(s) inherited from the base, not charged to this change:\n${listed(unseen)}`
|
|
386
|
+
})
|
|
387
|
+
)
|
|
388
|
+
}
|
|
389
|
+
if (!triaged.blocking.isClean && triaged.newLines.length > 0 && triage.rerunTest !== undefined) {
|
|
390
|
+
const again = yield* triage.rerunTest
|
|
391
|
+
const stillRed = new Set(triageGates(again, baseline, triage.roots).newLines)
|
|
392
|
+
const flaky = triaged.newLines.filter((line) => !stillRed.has(line))
|
|
393
|
+
if (flaky.length > 0) {
|
|
394
|
+
yield* events.publish(
|
|
395
|
+
Info.make({
|
|
396
|
+
message: `${flaky.length} gate failure(s) flaky (red once, green on rerun), not charged:\n${listed(flaky)}`
|
|
397
|
+
})
|
|
398
|
+
)
|
|
399
|
+
triaged = triageGates(
|
|
400
|
+
again,
|
|
401
|
+
GateBaseline.make({ ...baseline, failingLines: [...baseline.failingLines, ...flaky] }),
|
|
402
|
+
triage.roots
|
|
403
|
+
)
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
return triaged.blocking
|
|
407
|
+
})
|
|
408
|
+
|
|
409
|
+
export const mergeReviewResults = (results: ReadonlyArray<ReviewResult>): ReviewResult => {
|
|
410
|
+
const counts = results.flatMap((result) => (result.passed === undefined ? [] : [result.passed]))
|
|
411
|
+
return ReviewResult.make({
|
|
252
412
|
issues: results.flatMap((result) => result.issues),
|
|
253
413
|
summary: results
|
|
254
414
|
.map((result) => result.summary)
|
|
255
415
|
.filter((summary) => summary.length > 0)
|
|
256
|
-
.join("; ")
|
|
416
|
+
.join("; "),
|
|
417
|
+
// Several gates may count tests (unit and e2e): the sum is what the change must keep.
|
|
418
|
+
...(counts.length === 0 ? {} : { passed: counts.reduce((sum, value) => sum + value, 0) })
|
|
257
419
|
})
|
|
420
|
+
}
|
|
258
421
|
|
|
259
422
|
const processProblem = (
|
|
260
423
|
stdout: ReadonlyArray<string>,
|
|
@@ -265,57 +428,107 @@ const processProblem = (
|
|
|
265
428
|
return detail.length === 0 ? `process exited with code ${exitCode}` : detail
|
|
266
429
|
}
|
|
267
430
|
|
|
431
|
+
export interface LintCommandOptions {
|
|
432
|
+
/** Kill the gate after this long; the issue is then `gateClass: "hang"`. Default: never. */
|
|
433
|
+
readonly timeout?: Duration.Duration
|
|
434
|
+
/** Write the gate's whole stdout and stderr here; the issue carries `logPath`. */
|
|
435
|
+
readonly log?: { readonly files: PlainFileStoreShape; readonly path: string }
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
const gateClassOf = (exitCode: number): "red" | "crash" => (exitCode >= 128 ? "crash" : "red")
|
|
439
|
+
|
|
268
440
|
export const lintCommand = Effect.fn("@llm4ts/flow/Review.lintCommand")(function* (
|
|
269
441
|
process: ProcessExecutorShape,
|
|
270
442
|
events: FlowEventsShape,
|
|
271
443
|
command: ReadonlyArray<string>,
|
|
272
|
-
workDir: string
|
|
444
|
+
workDir: string,
|
|
445
|
+
options: LintCommandOptions = {}
|
|
273
446
|
): Effect.fn.Return<ReviewResult, FlowError> {
|
|
274
447
|
const executable = command[0]
|
|
275
448
|
if (executable === undefined) {
|
|
276
449
|
return ReviewResult.make({ issues: [], summary: "" })
|
|
277
450
|
}
|
|
451
|
+
const label = command.join(" ")
|
|
278
452
|
const started = yield* Clock.currentTimeMillis
|
|
453
|
+
const run = guarded(
|
|
454
|
+
Capabilities.Exec(executable),
|
|
455
|
+
`lint: ${label}`,
|
|
456
|
+
events,
|
|
457
|
+
process.run(command, workDir, {}).pipe(
|
|
458
|
+
Effect.mapError((cause) =>
|
|
459
|
+
ProcessError.make({
|
|
460
|
+
message: label,
|
|
461
|
+
detail: cause.message
|
|
462
|
+
})
|
|
463
|
+
)
|
|
464
|
+
)
|
|
465
|
+
)
|
|
466
|
+
// Interrupting `run` interrupts the child through the executor's scope;
|
|
467
|
+
// `undefined` here means the gate never exited (ADR 0027: a hang is a
|
|
468
|
+
// gate failure that says so, never a stuck run).
|
|
469
|
+
const bounded: Effect.Effect<ProcessResult | undefined, FlowError> =
|
|
470
|
+
options.timeout === undefined
|
|
471
|
+
? run
|
|
472
|
+
: run.pipe(
|
|
473
|
+
Effect.timeoutOption(options.timeout),
|
|
474
|
+
Effect.map((option) => (option._tag === "Some" ? option.value : undefined))
|
|
475
|
+
)
|
|
279
476
|
// A TOOL span (ADR 0026) and a Timed event: the command as configured and
|
|
280
477
|
// its exit code, never its output.
|
|
281
478
|
const result = yield* withKindSpan(
|
|
282
|
-
`gate ${
|
|
283
|
-
{ kind: "TOOL", attributes: { [attr.gateCommand]:
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
events,
|
|
288
|
-
process.run(command, workDir, {}).pipe(
|
|
289
|
-
Effect.mapError((cause) =>
|
|
290
|
-
ProcessError.make({
|
|
291
|
-
message: command.join(" "),
|
|
292
|
-
detail: cause.message
|
|
293
|
-
})
|
|
294
|
-
)
|
|
479
|
+
`gate ${label}`,
|
|
480
|
+
{ kind: "TOOL", attributes: { [attr.gateCommand]: label } },
|
|
481
|
+
bounded.pipe(
|
|
482
|
+
Effect.tap((ran) =>
|
|
483
|
+
ran === undefined ? Effect.void : Effect.annotateCurrentSpan(attr.gateExit, ran.exitCode)
|
|
295
484
|
)
|
|
296
|
-
)
|
|
485
|
+
)
|
|
297
486
|
)
|
|
487
|
+
const failed = result === undefined || result.exitCode !== 0
|
|
298
488
|
yield* events.publish(
|
|
299
489
|
Timed.make({
|
|
300
490
|
kind: "gate",
|
|
301
|
-
label
|
|
491
|
+
label,
|
|
302
492
|
ms: (yield* Clock.currentTimeMillis) - started,
|
|
303
|
-
exitCode: result.exitCode,
|
|
304
|
-
...(
|
|
493
|
+
...(result === undefined ? {} : { exitCode: result.exitCode }),
|
|
494
|
+
...(failed ? { failed: true } : {})
|
|
305
495
|
})
|
|
306
496
|
)
|
|
307
|
-
|
|
308
|
-
|
|
497
|
+
let logPath: string | undefined
|
|
498
|
+
if (options.log !== undefined) {
|
|
499
|
+
const output =
|
|
500
|
+
result === undefined ? "" : [...result.stdout, ...result.stderr].join("\n").trim()
|
|
501
|
+
yield* options.log.files.writeAtomic(options.log.path, output)
|
|
502
|
+
logPath = options.log.path
|
|
309
503
|
}
|
|
504
|
+
const passed =
|
|
505
|
+
result === undefined
|
|
506
|
+
? undefined
|
|
507
|
+
: passedCountIn([...result.stdout, ...result.stderr].join("\n"))
|
|
508
|
+
if (result !== undefined && result.exitCode === 0) {
|
|
509
|
+
return ReviewResult.make({
|
|
510
|
+
issues: [],
|
|
511
|
+
summary: "lint passed",
|
|
512
|
+
...(passed === undefined ? {} : { passed })
|
|
513
|
+
})
|
|
514
|
+
}
|
|
515
|
+
const seconds =
|
|
516
|
+
options.timeout === undefined ? 0 : Math.round(Duration.toSeconds(options.timeout))
|
|
310
517
|
return ReviewResult.make({
|
|
311
518
|
issues: [
|
|
312
519
|
ReviewIssue.make({
|
|
313
520
|
severity: "Critical",
|
|
314
|
-
title: `lint failed: ${
|
|
315
|
-
description:
|
|
521
|
+
title: `lint failed: ${label}`,
|
|
522
|
+
description:
|
|
523
|
+
result === undefined
|
|
524
|
+
? `gate killed: no exit after ${seconds} seconds (LLM4TS_GATE_TIMEOUT)`
|
|
525
|
+
: processProblem(result.stdout, result.stderr, result.exitCode),
|
|
526
|
+
gateClass: result === undefined ? "hang" : gateClassOf(result.exitCode),
|
|
527
|
+
...(logPath === undefined ? {} : { logPath })
|
|
316
528
|
})
|
|
317
529
|
],
|
|
318
|
-
summary: "lint failed"
|
|
530
|
+
summary: "lint failed",
|
|
531
|
+
...(passed === undefined ? {} : { passed })
|
|
319
532
|
})
|
|
320
533
|
})
|
|
321
534
|
|
|
@@ -350,6 +563,12 @@ export interface ReviewAndFixOptions {
|
|
|
350
563
|
result: ReviewResult,
|
|
351
564
|
settled: boolean
|
|
352
565
|
) => Effect.Effect<void, FlowError>
|
|
566
|
+
/** Charge the lint gate only with failures the change caused (ADR 0027). */
|
|
567
|
+
readonly triage?: GateTriageOptions
|
|
568
|
+
/** How gate output reaches the coder in the fix prompt. */
|
|
569
|
+
readonly fix?: FixPromptOptions
|
|
570
|
+
/** Fail the round when the change deletes or skips tests (ADR 0027 decision 4). */
|
|
571
|
+
readonly oracle?: OracleGateOptions
|
|
353
572
|
}
|
|
354
573
|
|
|
355
574
|
export interface ReviewCacheLocation {
|
|
@@ -493,13 +712,21 @@ export const reviewAndFixLoop = Effect.fn("@llm4ts/flow/Review.reviewAndFixLoop"
|
|
|
493
712
|
)
|
|
494
713
|
)
|
|
495
714
|
const format = options.format ?? Effect.void
|
|
715
|
+
const reported = yield* Ref.make<ReadonlySet<string>>(new Set())
|
|
716
|
+
const oracleNoted = yield* Ref.make(false)
|
|
496
717
|
|
|
497
718
|
const reviewOnce = (
|
|
498
719
|
round: number,
|
|
499
720
|
previous: ReviewResult | undefined
|
|
500
721
|
): Effect.Effect<ReviewResult, FlowError> =>
|
|
501
722
|
Effect.gen(function* () {
|
|
502
|
-
const
|
|
723
|
+
const lintRaw = yield* options.lint ?? Effect.succeed(ReviewResult.make({ issues: [] }))
|
|
724
|
+
const lint = yield* withOracle(
|
|
725
|
+
yield* applyTriage(lintRaw, options.triage, options.events, reported),
|
|
726
|
+
options.oracle,
|
|
727
|
+
options.events,
|
|
728
|
+
oracleNoted
|
|
729
|
+
)
|
|
503
730
|
if (!lint.isClean) {
|
|
504
731
|
return lint
|
|
505
732
|
}
|
|
@@ -585,7 +812,7 @@ export const reviewAndFixLoop = Effect.fn("@llm4ts/flow/Review.reviewAndFixLoop"
|
|
|
585
812
|
if (settled) {
|
|
586
813
|
return result
|
|
587
814
|
}
|
|
588
|
-
yield* options.coder.ask(fixPrompt(result))
|
|
815
|
+
yield* options.coder.ask(fixPrompt(result, options.fix))
|
|
589
816
|
return yield* loop(round + 1, result)
|
|
590
817
|
})
|
|
591
818
|
|
package/src/Roster.ts
CHANGED
|
@@ -63,7 +63,9 @@ export class ExecutorSpec extends Schema.Class<ExecutorSpec>("ExecutorSpec")({
|
|
|
63
63
|
health: Schema.optionalKey(Schema.String),
|
|
64
64
|
cooldown: Schema.optionalKey(Cooldown),
|
|
65
65
|
/** In a repository roster: removes the user roster's entry with this id. */
|
|
66
|
-
disabled: Schema.optionalKey(Schema.Boolean)
|
|
66
|
+
disabled: Schema.optionalKey(Schema.Boolean),
|
|
67
|
+
/** The autonomy contract profile this executor's coder prompts carry (ADR 0027); default `full`. */
|
|
68
|
+
contract: Schema.optionalKey(Schema.Literals(["full", "minimal", "off"]))
|
|
67
69
|
}) {}
|
|
68
70
|
|
|
69
71
|
export class RosterDocument extends Schema.Class<RosterDocument>("RosterDocument")({
|
package/src/RosterSeats.ts
CHANGED
|
@@ -44,6 +44,8 @@ export interface RosterView {
|
|
|
44
44
|
readonly executor: Effect.Effect<string | undefined>
|
|
45
45
|
/** Every executor that held this context's coder, in order (handovers). */
|
|
46
46
|
readonly history: Effect.Effect<ReadonlyArray<string>>
|
|
47
|
+
/** The autonomy contract profile an executor's roster entry asks for (ADR 0027), when it does. */
|
|
48
|
+
readonly contractOf?: (executor: string) => "full" | "minimal" | "off" | undefined
|
|
47
49
|
}
|
|
48
50
|
|
|
49
51
|
/** A roster failure seen through an `LlmServiceShape`, whose failures are `LlmError`s. */
|