@llm4ts/flow 2.31.0 → 2.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Approval.d.ts +1 -1
- package/dist/Approval.d.ts.map +1 -1
- package/dist/Artifacts.d.ts +2 -2
- package/dist/Artifacts.d.ts.map +1 -1
- package/dist/BenchReport.d.ts +2 -2
- package/dist/BenchReport.d.ts.map +1 -1
- package/dist/Chat.d.ts +5 -2
- package/dist/Chat.d.ts.map +1 -1
- package/dist/Chat.js +7 -4
- package/dist/Chat.js.map +1 -1
- package/dist/CostLedger.d.ts +2 -2
- package/dist/CostLedger.d.ts.map +1 -1
- package/dist/Equiv.d.ts +3 -3
- package/dist/Equiv.d.ts.map +1 -1
- package/dist/Flow.d.ts +16 -1
- package/dist/Flow.d.ts.map +1 -1
- package/dist/Flow.js +29 -3
- package/dist/Flow.js.map +1 -1
- package/dist/FlowError.d.ts +15 -2
- package/dist/FlowError.d.ts.map +1 -1
- package/dist/FlowError.js +15 -0
- package/dist/FlowError.js.map +1 -1
- package/dist/FlowEvents.d.ts +18 -5
- package/dist/FlowEvents.d.ts.map +1 -1
- package/dist/FlowEvents.js +30 -9
- package/dist/FlowEvents.js.map +1 -1
- package/dist/Gates.d.ts +1 -1
- package/dist/Gates.d.ts.map +1 -1
- package/dist/Judgment.d.ts +2 -2
- package/dist/Judgment.d.ts.map +1 -1
- package/dist/Landing.d.ts +1 -1
- package/dist/Landing.d.ts.map +1 -1
- package/dist/NodePreflight.d.ts +1 -1
- package/dist/NodePreflight.d.ts.map +1 -1
- package/dist/Pack.d.ts +9 -1
- package/dist/Pack.d.ts.map +1 -1
- package/dist/Pack.js +26 -2
- package/dist/Pack.js.map +1 -1
- package/dist/Persistence.d.ts +2 -2
- package/dist/Persistence.d.ts.map +1 -1
- package/dist/PlanExecution.d.ts +1 -1
- package/dist/PlanExecution.d.ts.map +1 -1
- package/dist/ProgramJudge.d.ts +1 -1
- package/dist/ProgramJudge.d.ts.map +1 -1
- package/dist/ProgramJudge.js +6 -3
- package/dist/ProgramJudge.js.map +1 -1
- package/dist/Replay.d.ts +2 -2
- package/dist/Replay.d.ts.map +1 -1
- package/dist/Retro.d.ts +51 -4
- package/dist/Retro.d.ts.map +1 -1
- package/dist/Retro.js +196 -11
- package/dist/Retro.js.map +1 -1
- package/dist/Review.d.ts +38 -6
- package/dist/Review.d.ts.map +1 -1
- package/dist/Review.js +147 -8
- package/dist/Review.js.map +1 -1
- package/dist/ReviewCache.d.ts +1 -1
- package/dist/ReviewCache.d.ts.map +1 -1
- package/dist/Reviewer.d.ts +2 -0
- package/dist/Reviewer.d.ts.map +1 -1
- package/dist/Reviewer.js +3 -1
- package/dist/Reviewer.js.map +1 -1
- package/dist/Roster.d.ts +19 -10
- package/dist/Roster.d.ts.map +1 -1
- package/dist/Roster.js +51 -17
- package/dist/Roster.js.map +1 -1
- package/dist/RosterSeats.d.ts +5 -4
- package/dist/RosterSeats.d.ts.map +1 -1
- package/dist/RosterSeats.js +7 -4
- package/dist/RosterSeats.js.map +1 -1
- package/dist/Stall.d.ts +29 -0
- package/dist/Stall.d.ts.map +1 -0
- package/dist/Stall.js +49 -0
- package/dist/Stall.js.map +1 -0
- package/dist/Stories.d.ts +8 -1
- package/dist/Stories.d.ts.map +1 -1
- package/dist/Stories.js +7 -1
- package/dist/Stories.js.map +1 -1
- package/package.json +3 -2
- package/src/Chat.ts +15 -7
- package/src/Flow.ts +48 -3
- package/src/FlowError.ts +16 -0
- package/src/FlowEvents.ts +34 -11
- package/src/Pack.ts +34 -1
- package/src/ProgramJudge.ts +3 -0
- package/src/Retro.ts +229 -4
- package/src/Review.ts +202 -8
- package/src/Reviewer.ts +3 -1
- package/src/Roster.ts +86 -39
- package/src/RosterSeats.ts +11 -6
- package/src/Stall.ts +85 -0
- package/src/Stories.ts +14 -0
package/src/Retro.ts
CHANGED
|
@@ -22,7 +22,7 @@ import { FlowEvent, Info, type FlowEventsShape } from "./FlowEvents.ts"
|
|
|
22
22
|
import type { TraceLine } from "./FlowRecorder.ts"
|
|
23
23
|
import { Plan, Task, parsePlan } from "./Plan.ts"
|
|
24
24
|
import { loadVersioned, makePlanStore, type PlainFileStoreShape } from "./Persistence.ts"
|
|
25
|
-
import { ReviewResult } from "./Review.ts"
|
|
25
|
+
import { ReviewResult, repoReviewRulesPath } from "./Review.ts"
|
|
26
26
|
import {
|
|
27
27
|
EpicReport,
|
|
28
28
|
EpicReportVersion,
|
|
@@ -478,6 +478,7 @@ export const renderRetroDigest = (inputs: RetroInputs): string => {
|
|
|
478
478
|
break
|
|
479
479
|
}
|
|
480
480
|
}
|
|
481
|
+
const signatures = signaturesOf(inputs, trace.events)
|
|
481
482
|
const header = [
|
|
482
483
|
`# Retro digest — run ${inputs.runId}, epic ${inputs.plan.epicId}`,
|
|
483
484
|
"",
|
|
@@ -492,6 +493,13 @@ export const renderRetroDigest = (inputs: RetroInputs): string => {
|
|
|
492
493
|
: "present"
|
|
493
494
|
}`,
|
|
494
495
|
...(retries.length === 0 ? [] : [`- retries across the run: ${countBy(retries)}`]),
|
|
496
|
+
...(signatures.length === 0
|
|
497
|
+
? []
|
|
498
|
+
: [
|
|
499
|
+
"",
|
|
500
|
+
"## Signatures (patterns across stories: a rule, not a task)",
|
|
501
|
+
...signatures.map((line) => `- ${line}`)
|
|
502
|
+
]),
|
|
495
503
|
...(runLines.length === 0
|
|
496
504
|
? []
|
|
497
505
|
: [
|
|
@@ -548,9 +556,33 @@ export const RetroStory = Schema.Struct({
|
|
|
548
556
|
})
|
|
549
557
|
export type RetroStory = typeof RetroStory.Type
|
|
550
558
|
|
|
559
|
+
export const RuleEditOp = Schema.Literals(["append-rule", "replace-section"])
|
|
560
|
+
export type RuleEditOp = typeof RuleEditOp.Type
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* An edit to the loop rather than to a story (ADR 0027 decision 12): a line
|
|
564
|
+
* appended to a rules file, or a section replaced. Structured, never a
|
|
565
|
+
* diff; the report renders it as one for the human.
|
|
566
|
+
*/
|
|
567
|
+
export const RuleEdit = Schema.Struct({
|
|
568
|
+
/** Repo-relative: `.llm4ts/review-rules.md`, a pack's `reviewers/<name>.md`, `pack.md`, `lessons.md`, or a `patterns/pitfalls-*.md` card. */
|
|
569
|
+
target: Schema.String,
|
|
570
|
+
op: RuleEditOp,
|
|
571
|
+
/** `append-rule`: the rule, one line, without a leading dash. */
|
|
572
|
+
line: Schema.optionalKey(Schema.String),
|
|
573
|
+
/** `replace-section`: the `## heading` (without the hashes) and its new body. */
|
|
574
|
+
heading: Schema.optionalKey(Schema.String),
|
|
575
|
+
body: Schema.optionalKey(Schema.String),
|
|
576
|
+
/** The signature in the digest this answers. */
|
|
577
|
+
why: Schema.String
|
|
578
|
+
})
|
|
579
|
+
export type RuleEdit = typeof RuleEdit.Type
|
|
580
|
+
|
|
551
581
|
export class RetroProposal extends Schema.Class<RetroProposal>("RetroProposal")({
|
|
552
582
|
summary: Schema.String,
|
|
553
583
|
stories: Schema.Array(RetroStory),
|
|
584
|
+
/** Edits to the rules the next run reviews with; absent in proposals written before ADR 0027. */
|
|
585
|
+
rules: Schema.optionalKey(Schema.Array(RuleEdit)),
|
|
554
586
|
runAdvice: Schema.Array(Schema.Struct({ finding: Schema.String, evidence: Schema.String })),
|
|
555
587
|
libraryAdvice: Schema.Array(
|
|
556
588
|
Schema.Struct({ title: Schema.String, evidence: Schema.String, suggestion: Schema.String })
|
|
@@ -593,6 +625,21 @@ export const retroProposalJsonSchema: JsonSchema = {
|
|
|
593
625
|
required: ["id", "diagnosis", "kind"]
|
|
594
626
|
}
|
|
595
627
|
},
|
|
628
|
+
rules: {
|
|
629
|
+
type: "array",
|
|
630
|
+
items: {
|
|
631
|
+
type: "object",
|
|
632
|
+
properties: {
|
|
633
|
+
target: { type: "string" },
|
|
634
|
+
op: { type: "string", enum: ["append-rule", "replace-section"] },
|
|
635
|
+
line: { type: "string" },
|
|
636
|
+
heading: { type: "string" },
|
|
637
|
+
body: { type: "string" },
|
|
638
|
+
why: { type: "string" }
|
|
639
|
+
},
|
|
640
|
+
required: ["target", "op", "why"]
|
|
641
|
+
}
|
|
642
|
+
},
|
|
596
643
|
runAdvice: {
|
|
597
644
|
type: "array",
|
|
598
645
|
items: {
|
|
@@ -638,6 +685,14 @@ export const retroPrompt = (digest: string, storiesWithPlan: ReadonlyArray<strin
|
|
|
638
685
|
" no edit covers). Give the `feedback` text a planner should take.",
|
|
639
686
|
"- `none`: nothing to change (an outage, a dependency that will unblock it). Give `why`.",
|
|
640
687
|
"",
|
|
688
|
+
"`rules`: when the digest's Signatures section shows a pattern that repeats across stories",
|
|
689
|
+
"(the same reviewer finding, gaming such as stubs or skipped tests, fabricated status),",
|
|
690
|
+
"propose an edit to the loop rather than a task: `append-rule` adds one line to a rules",
|
|
691
|
+
"file, `replace-section` rewrites one `## heading` of it. Targets are repo-relative:",
|
|
692
|
+
"`.llm4ts/review-rules.md` (every run reads it as a review lens), a pack's",
|
|
693
|
+
"`reviewers/<name>.md`, `pack.md`, `lessons.md`, or a `patterns/pitfalls-*.md` card. Name",
|
|
694
|
+
"the signature in `why`. Nothing here is applied unreviewed.",
|
|
695
|
+
"",
|
|
641
696
|
"`runAdvice`: findings about the environment or configuration (a seat on an old CLI, Node",
|
|
642
697
|
"mismatch, gates missing, transcripts off), each with its evidence line.",
|
|
643
698
|
"`libraryAdvice`: only where the digest shows llm4ts itself — the flow, a seat's harness",
|
|
@@ -757,7 +812,96 @@ export const validateRetroProposal = (
|
|
|
757
812
|
break
|
|
758
813
|
}
|
|
759
814
|
}
|
|
760
|
-
|
|
815
|
+
const rules: Array<RuleEdit> = []
|
|
816
|
+
for (const edit of proposal.rules ?? []) {
|
|
817
|
+
if (!allowedRuleTarget(edit.target)) {
|
|
818
|
+
dropped.push(`rule edit: ${edit.target} is not a rules file a retro may edit`)
|
|
819
|
+
continue
|
|
820
|
+
}
|
|
821
|
+
if (edit.op === "append-rule" && (edit.line ?? "").trim().length === 0) {
|
|
822
|
+
dropped.push(`rule edit: append-rule to ${edit.target} with no line`)
|
|
823
|
+
continue
|
|
824
|
+
}
|
|
825
|
+
if (
|
|
826
|
+
edit.op === "replace-section" &&
|
|
827
|
+
((edit.heading ?? "").trim().length === 0 || (edit.body ?? "").trim().length === 0)
|
|
828
|
+
) {
|
|
829
|
+
dropped.push(`rule edit: replace-section in ${edit.target} with no heading or body`)
|
|
830
|
+
continue
|
|
831
|
+
}
|
|
832
|
+
rules.push(edit)
|
|
833
|
+
}
|
|
834
|
+
return {
|
|
835
|
+
proposal: RetroProposal.make({
|
|
836
|
+
...proposal,
|
|
837
|
+
stories,
|
|
838
|
+
...(rules.length === 0 ? {} : { rules })
|
|
839
|
+
}),
|
|
840
|
+
dropped
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
/** The files a retro may edit: the repository's review rules, a pack's lenses and rules, a pitfall card. */
|
|
845
|
+
export const allowedRuleTarget = (target: string): boolean =>
|
|
846
|
+
!badPath(target) &&
|
|
847
|
+
(target === repoReviewRulesPath ||
|
|
848
|
+
/(?:^|\/)reviewers\/[^/]+\.md$/u.test(target) ||
|
|
849
|
+
/(?:^|\/)pack\.md$/u.test(target) ||
|
|
850
|
+
/(?:^|\/)lessons\.md$/u.test(target) ||
|
|
851
|
+
/(?:^|\/)patterns\/pitfalls-[^/]+\.md$/u.test(target))
|
|
852
|
+
|
|
853
|
+
const findingLine = /^- \[(Critical|Warning|Info)\] (.+?)(?: \(| — |: |$)/u
|
|
854
|
+
|
|
855
|
+
/**
|
|
856
|
+
* Patterns across stories the digest flags for a rule (ADR 0027 decision 12):
|
|
857
|
+
* gaming (oracle-guard findings), the same reviewer finding in two or more
|
|
858
|
+
* stories, fabricated status in two or more stories.
|
|
859
|
+
*/
|
|
860
|
+
export const signaturesOf = (
|
|
861
|
+
inputs: RetroInputs,
|
|
862
|
+
events: ReadonlyArray<FlowEvent>
|
|
863
|
+
): ReadonlyArray<string> => {
|
|
864
|
+
const gaming: Array<string> = []
|
|
865
|
+
const titles = new Map<string, Set<string>>()
|
|
866
|
+
for (const entry of inputs.stories) {
|
|
867
|
+
const text = entry.findings ?? ""
|
|
868
|
+
if (/^- \[(?:Critical|Warning)\] oracle: /mu.test(text)) {
|
|
869
|
+
gaming.push(entry.story.id)
|
|
870
|
+
}
|
|
871
|
+
for (const raw of text.split(/\r?\n/u)) {
|
|
872
|
+
const match = findingLine.exec(raw.trim())
|
|
873
|
+
const title = match?.[2]?.trim().toLowerCase()
|
|
874
|
+
if (title === undefined || title.startsWith("lint failed") || title.startsWith("oracle:")) {
|
|
875
|
+
continue
|
|
876
|
+
}
|
|
877
|
+
titles.set(title, (titles.get(title) ?? new Set()).add(entry.story.id))
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
const fabricated = new Set<string>()
|
|
881
|
+
for (const event of events) {
|
|
882
|
+
if (event._tag === "EvidenceChecked" && event.unverified > 0 && event.lane !== undefined) {
|
|
883
|
+
fabricated.add(event.lane)
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
const lines: Array<string> = []
|
|
887
|
+
if (gaming.length > 0) {
|
|
888
|
+
lines.push(
|
|
889
|
+
`gaming: the oracle guard caught deleted or skipped tests in ${gaming.length} stor${gaming.length === 1 ? "y" : "ies"} (${gaming.join(", ")})`
|
|
890
|
+
)
|
|
891
|
+
}
|
|
892
|
+
for (const [title, stories] of titles) {
|
|
893
|
+
if (stories.size >= 2) {
|
|
894
|
+
lines.push(
|
|
895
|
+
`the same reviewer finding in ${stories.size} stories (${[...stories].join(", ")}): "${title}"`
|
|
896
|
+
)
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
if (fabricated.size >= 2) {
|
|
900
|
+
lines.push(
|
|
901
|
+
`fabricated status in ${fabricated.size} stories (${[...fabricated].join(", ")}): the coder claimed commands that never ran`
|
|
902
|
+
)
|
|
903
|
+
}
|
|
904
|
+
return lines
|
|
761
905
|
}
|
|
762
906
|
|
|
763
907
|
// ---- Report ---------------------------------------------------------------------------------
|
|
@@ -849,6 +993,25 @@ export const renderRetroReport = (inputs: RetroReportInputs): string => {
|
|
|
849
993
|
)
|
|
850
994
|
}
|
|
851
995
|
}
|
|
996
|
+
const rules = proposal.rules ?? []
|
|
997
|
+
if (rules.length > 0) {
|
|
998
|
+
lines.push("## Rule edits", "")
|
|
999
|
+
for (const edit of rules) {
|
|
1000
|
+
lines.push(`### ${edit.target}`, "", `**Why.** ${edit.why.trim()}`, "", "```diff")
|
|
1001
|
+
if (edit.op === "append-rule") {
|
|
1002
|
+
lines.push(`+ - ${(edit.line ?? "").trim()}`)
|
|
1003
|
+
} else {
|
|
1004
|
+
lines.push(
|
|
1005
|
+
` ## ${(edit.heading ?? "").trim()}`,
|
|
1006
|
+
...(edit.body ?? "")
|
|
1007
|
+
.trim()
|
|
1008
|
+
.split(/\r?\n/u)
|
|
1009
|
+
.map((row) => `+ ${row}`)
|
|
1010
|
+
)
|
|
1011
|
+
}
|
|
1012
|
+
lines.push("```", "")
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
852
1015
|
if (proposal.runAdvice.length > 0) {
|
|
853
1016
|
lines.push(
|
|
854
1017
|
"## Run advice",
|
|
@@ -865,7 +1028,7 @@ export const renderRetroReport = (inputs: RetroReportInputs): string => {
|
|
|
865
1028
|
)
|
|
866
1029
|
const refines = proposal.stories.filter((entry) => entry.kind === "refine")
|
|
867
1030
|
lines.push("## Approval", "")
|
|
868
|
-
if (applies.length === 0) {
|
|
1031
|
+
if (applies.length === 0 && rules.length === 0) {
|
|
869
1032
|
lines.push("Nothing here is applied by the next run; the report is for reading.", "")
|
|
870
1033
|
} else {
|
|
871
1034
|
lines.push(
|
|
@@ -876,6 +1039,11 @@ export const renderRetroReport = (inputs: RetroReportInputs): string => {
|
|
|
876
1039
|
? `- ${entry.id}: append ${entry.tasks?.length ?? 0} task(s) to its plan`
|
|
877
1040
|
: `- ${entry.id}: edit its entry and restart it from a fresh worktree`
|
|
878
1041
|
),
|
|
1042
|
+
...rules.map((edit) =>
|
|
1043
|
+
edit.op === "append-rule"
|
|
1044
|
+
? `- ${edit.target}: append one rule`
|
|
1045
|
+
: `- ${edit.target}: replace the "${(edit.heading ?? "").trim()}" section`
|
|
1046
|
+
),
|
|
879
1047
|
""
|
|
880
1048
|
)
|
|
881
1049
|
}
|
|
@@ -963,12 +1131,49 @@ const retroTaskTitle = (runId: string, title: string): string => `Retro ${runId}
|
|
|
963
1131
|
* so the executor's hash check restarts them). Says what it did on the
|
|
964
1132
|
* events. Nothing here fails the run: a retro that no longer fits is skipped.
|
|
965
1133
|
*/
|
|
1134
|
+
export interface ApplyRetroOptions {
|
|
1135
|
+
/** The repository root rule edits resolve against; without it they are reported, not applied. */
|
|
1136
|
+
readonly rulesRoot?: string
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1139
|
+
/** `append-rule`: the file with the rule added once; `replace-section`: with the section rewritten or appended. */
|
|
1140
|
+
export const applyRuleEdit = (existing: string | undefined, edit: RuleEdit): string | undefined => {
|
|
1141
|
+
const current = existing ?? ""
|
|
1142
|
+
if (edit.op === "append-rule") {
|
|
1143
|
+
const line = `- ${(edit.line ?? "").trim()}`
|
|
1144
|
+
if (current.split(/\r?\n/u).some((row) => row.trim() === line)) {
|
|
1145
|
+
return undefined
|
|
1146
|
+
}
|
|
1147
|
+
const base =
|
|
1148
|
+
current.trim().length === 0 && edit.target === repoReviewRulesPath
|
|
1149
|
+
? "# Review rules\n\nRules every review round of this repository applies (ADR 0027).\n\n"
|
|
1150
|
+
: current.length === 0 || current.endsWith("\n")
|
|
1151
|
+
? current
|
|
1152
|
+
: `${current}\n`
|
|
1153
|
+
return `${base}${line}\n`
|
|
1154
|
+
}
|
|
1155
|
+
const heading = `## ${(edit.heading ?? "").trim()}`
|
|
1156
|
+
const body = (edit.body ?? "").trim()
|
|
1157
|
+
const sections = current.split(/^(?=## )/mu)
|
|
1158
|
+
const index = sections.findIndex((section) => section.split(/\r?\n/u)[0]?.trim() === heading)
|
|
1159
|
+
const replacement = `${heading}\n\n${body}\n`
|
|
1160
|
+
if (index < 0) {
|
|
1161
|
+
const base = current.length === 0 || current.endsWith("\n") ? current : `${current}\n`
|
|
1162
|
+
return `${base}${current.trim().length === 0 ? "" : "\n"}${replacement}`
|
|
1163
|
+
}
|
|
1164
|
+
if (sections[index]?.trim() === replacement.trim()) {
|
|
1165
|
+
return undefined
|
|
1166
|
+
}
|
|
1167
|
+
return [...sections.slice(0, index), replacement, ...sections.slice(index + 1)].join("")
|
|
1168
|
+
}
|
|
1169
|
+
|
|
966
1170
|
export const applyApprovedRetros = Effect.fn("@llm4ts/flow/Retro.apply")(function* (
|
|
967
1171
|
files: PlainFileStoreShape,
|
|
968
1172
|
stateDir: string,
|
|
969
1173
|
plan: StoryPlan,
|
|
970
1174
|
events: FlowEventsShape,
|
|
971
|
-
now: number
|
|
1175
|
+
now: number,
|
|
1176
|
+
options: ApplyRetroOptions = {}
|
|
972
1177
|
): Effect.fn.Return<AppliedRetros, FlowError> {
|
|
973
1178
|
const notes: Array<string> = []
|
|
974
1179
|
const say = (message: string): Effect.Effect<void> =>
|
|
@@ -1064,6 +1269,26 @@ export const applyApprovedRetros = Effect.fn("@llm4ts/flow/Retro.apply")(functio
|
|
|
1064
1269
|
break
|
|
1065
1270
|
}
|
|
1066
1271
|
}
|
|
1272
|
+
for (const edit of proposal.rules ?? []) {
|
|
1273
|
+
if (options.rulesRoot === undefined) {
|
|
1274
|
+
yield* say(`${runId}: rule edit to ${edit.target} needs the repository root; not applied`)
|
|
1275
|
+
continue
|
|
1276
|
+
}
|
|
1277
|
+
if (!allowedRuleTarget(edit.target)) {
|
|
1278
|
+
yield* say(`${runId}: rule edit to ${edit.target} is not allowed; skipped`)
|
|
1279
|
+
continue
|
|
1280
|
+
}
|
|
1281
|
+
const path = join(options.rulesRoot, edit.target)
|
|
1282
|
+
const existing = yield* files.read(path).pipe(Effect.catch(() => Effect.succeed(undefined)))
|
|
1283
|
+
const next = applyRuleEdit(existing, edit)
|
|
1284
|
+
if (next === undefined) {
|
|
1285
|
+
continue
|
|
1286
|
+
}
|
|
1287
|
+
yield* files.writeAtomic(path, next)
|
|
1288
|
+
yield* say(
|
|
1289
|
+
`${runId}: ${edit.target} — ${edit.op === "append-rule" ? "one rule appended" : `section "${(edit.heading ?? "").trim()}" replaced`} (${edit.why.trim()})`
|
|
1290
|
+
)
|
|
1291
|
+
}
|
|
1067
1292
|
if (editedIds.length > 0) {
|
|
1068
1293
|
const valid: { readonly plan: StoryPlan; readonly problem: string | undefined } =
|
|
1069
1294
|
yield* validateStoryPlan(edited).pipe(
|
package/src/Review.ts
CHANGED
|
@@ -11,7 +11,13 @@ import type { ProcessExecutorShape, ProcessResult } from "@llm4ts/core/ProcessEx
|
|
|
11
11
|
import { Capabilities } from "@llm4ts/core/Capability"
|
|
12
12
|
import { guarded } from "./CapabilityGuard.ts"
|
|
13
13
|
import type { Chat } from "./Chat.ts"
|
|
14
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
FlowLlmError,
|
|
16
|
+
ProcessError,
|
|
17
|
+
Stalled,
|
|
18
|
+
describeFlowError,
|
|
19
|
+
type FlowError
|
|
20
|
+
} from "./FlowError.ts"
|
|
15
21
|
import type { PlainFileStoreShape } from "./Persistence.ts"
|
|
16
22
|
import { GateBaseline, isGateIssue, triageGates } from "./GateTriage.ts"
|
|
17
23
|
import {
|
|
@@ -23,6 +29,7 @@ import {
|
|
|
23
29
|
} from "./OracleGuard.ts"
|
|
24
30
|
import { cachedReview, fingerprintOf } from "./ReviewCache.ts"
|
|
25
31
|
import {
|
|
32
|
+
ReviewFindingDemoted,
|
|
26
33
|
Info,
|
|
27
34
|
JudgmentObserved,
|
|
28
35
|
ReviewFinding,
|
|
@@ -32,7 +39,7 @@ import {
|
|
|
32
39
|
type FlowEventsShape
|
|
33
40
|
} from "./FlowEvents.ts"
|
|
34
41
|
import { attr, withKindSpan } from "./Spans.ts"
|
|
35
|
-
import { Reviewer } from "./Reviewer.ts"
|
|
42
|
+
import { Reviewer, parseReviewer } from "./Reviewer.ts"
|
|
36
43
|
import { publishUsage } from "./Usage.ts"
|
|
37
44
|
import type { JudgmentShape } from "@llm4ts/core/judgment/Judgment"
|
|
38
45
|
import { truth, type JudgmentResult, type TruthAnswer } from "@llm4ts/core/judgment/Schemas"
|
|
@@ -111,6 +118,40 @@ export const reviewJsonSchema: JsonSchema = {
|
|
|
111
118
|
const reviewer = (name: string, systemPrompt: string, screen: string, files = ".*"): Reviewer =>
|
|
112
119
|
Reviewer.make({ name, systemPrompt, files, screen })
|
|
113
120
|
|
|
121
|
+
/**
|
|
122
|
+
* Rules with teeth (ADR 0027 decision 9), in front of every lens and the
|
|
123
|
+
* judge rubric: what no reviewer accepts whatever its concern. A pack's
|
|
124
|
+
* `## Review rules` extends it; `preamble: off` leaves it out.
|
|
125
|
+
*/
|
|
126
|
+
export const reviewRulesPreamble = [
|
|
127
|
+
"Rules every reviewer applies, whatever its concern:",
|
|
128
|
+
"- A stubbed body, a placeholder return, a TODO where logic belongs, or a function that only",
|
|
129
|
+
" satisfies the type checker is a Critical finding.",
|
|
130
|
+
"- A skipped, deleted, weakened or hard-coded test is a Critical finding.",
|
|
131
|
+
"- A layering workaround (a runtime hook where a signature should change, a cast or `any`",
|
|
132
|
+
" to silence a type, a duplicated type to avoid an import) is a Critical finding.",
|
|
133
|
+
"- If the change needs a paragraph-long comment to justify why a workaround is acceptable,",
|
|
134
|
+
" the code is wrong, not the comment; report it."
|
|
135
|
+
].join("\n")
|
|
136
|
+
|
|
137
|
+
/** The one lens whose job is the Bun port's reviewer brief: find why the diff does not work. */
|
|
138
|
+
export const adversarialReviewer = reviewer(
|
|
139
|
+
"adversarial",
|
|
140
|
+
[
|
|
141
|
+
"Your only job is to find reasons this change does not work. Assume the code is wrong and",
|
|
142
|
+
"look for the evidence: a path that is never taken, an error that is swallowed, a value that",
|
|
143
|
+
"is wrong at a boundary (empty, zero, negative, unicode, concurrent), a promise or effect that",
|
|
144
|
+
"is never awaited or yielded, a resource that leaks, a test that passes without exercising the",
|
|
145
|
+
"change. The diff is the whole subject. Report every concrete way it fails as a finding with",
|
|
146
|
+
"the file and line; an empty list means you looked and found none, not that it is fine."
|
|
147
|
+
].join("\n"),
|
|
148
|
+
"The diff plausibly contains a defect an adversarial reader would find: a wrong boundary, an unhandled path, a leaked resource, a test that does not exercise the change."
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
/** The prompt a lens is asked with: the preamble (unless the lens opts out), then its own rules. */
|
|
152
|
+
export const lensPrompt = (lens: Reviewer): string =>
|
|
153
|
+
lens.preamble === false ? lens.systemPrompt : `${reviewRulesPreamble}\n\n${lens.systemPrompt}`
|
|
154
|
+
|
|
114
155
|
export const correctnessReviewer = reviewer(
|
|
115
156
|
"code-functionality",
|
|
116
157
|
[
|
|
@@ -165,12 +206,14 @@ export const effectReviewer = reviewer(
|
|
|
165
206
|
)
|
|
166
207
|
|
|
167
208
|
export const minimalReviewers: ReadonlyArray<Reviewer> = Object.freeze([
|
|
209
|
+
adversarialReviewer,
|
|
168
210
|
correctnessReviewer,
|
|
169
211
|
readabilityReviewer,
|
|
170
212
|
testReviewer
|
|
171
213
|
])
|
|
172
214
|
|
|
173
215
|
export const allReviewers: ReadonlyArray<Reviewer> = Object.freeze([
|
|
216
|
+
adversarialReviewer,
|
|
174
217
|
correctnessReviewer,
|
|
175
218
|
testReviewer,
|
|
176
219
|
readabilityReviewer,
|
|
@@ -532,6 +575,112 @@ export const lintCommand = Effect.fn("@llm4ts/flow/Review.lintCommand")(function
|
|
|
532
575
|
})
|
|
533
576
|
})
|
|
534
577
|
|
|
578
|
+
const placeKey = (issue: ReviewIssue): string =>
|
|
579
|
+
issue.file === undefined
|
|
580
|
+
? `title:${issue.title.trim().toLowerCase()}`
|
|
581
|
+
: `${issue.file}:${issue.line ?? ""}`
|
|
582
|
+
|
|
583
|
+
/**
|
|
584
|
+
* Independent votes of one lens merged (ADR 0027 decision 7): any Critical
|
|
585
|
+
* blocks; Warnings are unioned, one per place; an Info survives only when
|
|
586
|
+
* more than one vote raised it.
|
|
587
|
+
*/
|
|
588
|
+
export const mergeVotes = (votes: ReadonlyArray<ReviewResult>): ReviewResult => {
|
|
589
|
+
if (votes.length <= 1) {
|
|
590
|
+
return votes[0] ?? ReviewResult.make({ issues: [] })
|
|
591
|
+
}
|
|
592
|
+
const seen = new Map<string, { issue: ReviewIssue; count: number }>()
|
|
593
|
+
for (const vote of votes) {
|
|
594
|
+
for (const issue of vote.issues) {
|
|
595
|
+
const key = `${issue.severity}|${placeKey(issue)}`
|
|
596
|
+
const entry = seen.get(key)
|
|
597
|
+
if (entry === undefined) {
|
|
598
|
+
seen.set(key, { issue, count: 1 })
|
|
599
|
+
} else {
|
|
600
|
+
entry.count += 1
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
const issues = [...seen.values()]
|
|
605
|
+
.filter(({ issue, count }) => issue.severity !== "Info" || count > 1)
|
|
606
|
+
.map(({ issue }) => issue)
|
|
607
|
+
return ReviewResult.make({
|
|
608
|
+
issues,
|
|
609
|
+
summary: votes
|
|
610
|
+
.map((vote) => vote.summary)
|
|
611
|
+
.filter((summary) => summary.length > 0)
|
|
612
|
+
.join(" | ")
|
|
613
|
+
})
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
const inDiff = (file: string, changed: ReadonlyArray<string>): boolean => {
|
|
617
|
+
const wanted = file.replace(/^\.\//u, "")
|
|
618
|
+
return changed.some(
|
|
619
|
+
(path) => path === wanted || path.endsWith(`/${wanted}`) || wanted.endsWith(`/${path}`)
|
|
620
|
+
)
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
/**
|
|
624
|
+
* Findings that cannot be placed move down (ADR 0027 decision 10): a Critical
|
|
625
|
+
* without a file becomes a Warning, a finding whose file is not in the diff
|
|
626
|
+
* becomes an Info. Nothing is dropped; each move is published. With no
|
|
627
|
+
* changed-file list the result is returned as is.
|
|
628
|
+
*/
|
|
629
|
+
export const demoteUnplaced = Effect.fn("@llm4ts/flow/Review.demoteUnplaced")(function* (
|
|
630
|
+
lens: Reviewer,
|
|
631
|
+
result: ReviewResult,
|
|
632
|
+
changed: ReadonlyArray<string>,
|
|
633
|
+
events: FlowEventsShape
|
|
634
|
+
): Effect.fn.Return<ReviewResult> {
|
|
635
|
+
if (changed.length === 0) {
|
|
636
|
+
return result
|
|
637
|
+
}
|
|
638
|
+
const issues: Array<ReviewIssue> = []
|
|
639
|
+
for (const issue of result.issues) {
|
|
640
|
+
let to: ReviewIssue["severity"] = issue.severity
|
|
641
|
+
let reason: string | undefined
|
|
642
|
+
if (issue.file !== undefined && !inDiff(issue.file, changed)) {
|
|
643
|
+
to = "Info"
|
|
644
|
+
reason = `names ${issue.file}, which the diff does not touch`
|
|
645
|
+
} else if (issue.file === undefined && issue.severity === "Critical") {
|
|
646
|
+
to = "Warning"
|
|
647
|
+
reason = "a Critical with no file"
|
|
648
|
+
}
|
|
649
|
+
if (reason !== undefined && to !== issue.severity) {
|
|
650
|
+
yield* events.publish(
|
|
651
|
+
ReviewFindingDemoted.make({
|
|
652
|
+
lens: lens.name,
|
|
653
|
+
title: issue.title,
|
|
654
|
+
from: issue.severity,
|
|
655
|
+
to,
|
|
656
|
+
reason
|
|
657
|
+
})
|
|
658
|
+
)
|
|
659
|
+
issues.push(ReviewIssue.make({ ...issue, severity: to }))
|
|
660
|
+
} else {
|
|
661
|
+
issues.push(issue)
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
return ReviewResult.make({ issues, summary: result.summary })
|
|
665
|
+
})
|
|
666
|
+
|
|
667
|
+
/** The repository's own review rules file, loaded as one extra lens; `undefined` when absent. */
|
|
668
|
+
export const repoReviewRulesPath = ".llm4ts/review-rules.md"
|
|
669
|
+
|
|
670
|
+
export const loadRepoReviewRules = (
|
|
671
|
+
files: PlainFileStoreShape,
|
|
672
|
+
workDir: string
|
|
673
|
+
): Effect.Effect<Reviewer | undefined, FlowError> =>
|
|
674
|
+
files
|
|
675
|
+
.read(join(workDir, repoReviewRulesPath))
|
|
676
|
+
.pipe(
|
|
677
|
+
Effect.map((text) =>
|
|
678
|
+
text === undefined || text.trim().length === 0
|
|
679
|
+
? undefined
|
|
680
|
+
: parseReviewer("repo-rules", text)
|
|
681
|
+
)
|
|
682
|
+
)
|
|
683
|
+
|
|
535
684
|
export interface ReviewAndFixOptions {
|
|
536
685
|
readonly reviewers: ReadonlyArray<Reviewer>
|
|
537
686
|
readonly reviewerService: LlmServiceShape
|
|
@@ -569,6 +718,12 @@ export interface ReviewAndFixOptions {
|
|
|
569
718
|
readonly fix?: FixPromptOptions
|
|
570
719
|
/** Fail the round when the change deletes or skips tests (ADR 0027 decision 4). */
|
|
571
720
|
readonly oracle?: OracleGateOptions
|
|
721
|
+
/** Independent votes of the adversarial lens per round (ADR 0027 decision 7). Default 1. */
|
|
722
|
+
readonly votes?: number
|
|
723
|
+
/** Applies a fix prompt somewhere other than the implementer's chat (a separate fixer). */
|
|
724
|
+
readonly fixWith?: (prompt: string) => Effect.Effect<string, FlowError>
|
|
725
|
+
/** End the task, typed `Stalled`, when a fix round leaves the diff byte-identical (ADR 0027 decision 11). */
|
|
726
|
+
readonly stallOnIdenticalDiff?: boolean
|
|
572
727
|
}
|
|
573
728
|
|
|
574
729
|
export interface ReviewCacheLocation {
|
|
@@ -671,7 +826,7 @@ export const reviewWith = (
|
|
|
671
826
|
taskTitle: string,
|
|
672
827
|
diff: string
|
|
673
828
|
): Effect.Effect<ReviewResult, FlowLlmError> => {
|
|
674
|
-
const prompt = `${lens
|
|
829
|
+
const prompt = `${lensPrompt(lens)}\n\n${reviewPrompt(taskTitle, diff)}`
|
|
675
830
|
// Usage is published per attempt — a schema retry costs real tokens too.
|
|
676
831
|
const attempt = (text: string): Effect.Effect<ReviewResult, LlmError> =>
|
|
677
832
|
service.executeStructuredWithUsage(text, ReviewResult, reviewJsonSchema).pipe(
|
|
@@ -739,16 +894,36 @@ export const reviewAndFixLoop = Effect.fn("@llm4ts/flow/Review.reviewAndFixLoop"
|
|
|
739
894
|
: yield* prescreenReviewers(options.prescreen, options.events, diff, selected)
|
|
740
895
|
const chosen = screened.reviewers
|
|
741
896
|
const cache = options.cache
|
|
742
|
-
const
|
|
897
|
+
const votes = Math.max(1, options.votes ?? 1)
|
|
898
|
+
const reviewVote = (lens: Reviewer, vote: number): Effect.Effect<ReviewResult, FlowError> =>
|
|
743
899
|
cache === undefined
|
|
744
900
|
? reviewWith(options.reviewerService, options.events, lens, options.taskTitle, diff)
|
|
745
901
|
: cachedReview(
|
|
746
902
|
cache.files,
|
|
747
|
-
join(cache.dir, `${fileSlug(lens.name)}.json`),
|
|
748
|
-
fingerprintOf([lens.name, lens
|
|
903
|
+
join(cache.dir, `${fileSlug(lens.name)}${vote === 0 ? "" : `-vote${vote}`}.json`),
|
|
904
|
+
fingerprintOf([lens.name, lensPrompt(lens), String(vote), options.taskTitle, diff]),
|
|
749
905
|
reviewWith(options.reviewerService, options.events, lens, options.taskTitle, diff)
|
|
750
906
|
)
|
|
751
|
-
|
|
907
|
+
// Votes multiply only the adversarial lens: its recall is what independent
|
|
908
|
+
// eyes improve; the concern lenses are cheap scoped questions.
|
|
909
|
+
const review = (lens: Reviewer): Effect.Effect<ReviewResult, FlowError> =>
|
|
910
|
+
lens.name === adversarialReviewer.name && votes > 1
|
|
911
|
+
? Effect.map(
|
|
912
|
+
Effect.forEach(
|
|
913
|
+
Array.from({ length: votes }, (_, index) => index),
|
|
914
|
+
(vote) => reviewVote(lens, vote),
|
|
915
|
+
{ concurrency: "unbounded" }
|
|
916
|
+
),
|
|
917
|
+
mergeVotes
|
|
918
|
+
)
|
|
919
|
+
: reviewVote(lens, 0)
|
|
920
|
+
const run = (lens: Reviewer) =>
|
|
921
|
+
Effect.flatMap(review(lens), (result) =>
|
|
922
|
+
Effect.map(demoteUnplaced(lens, result, files, options.events), (placed) => ({
|
|
923
|
+
lens,
|
|
924
|
+
result: placed
|
|
925
|
+
}))
|
|
926
|
+
)
|
|
752
927
|
const parallelism = options.parallelism ?? 0
|
|
753
928
|
const results =
|
|
754
929
|
parallelism > 0
|
|
@@ -784,12 +959,30 @@ export const reviewAndFixLoop = Effect.fn("@llm4ts/flow/Review.reviewAndFixLoop"
|
|
|
784
959
|
return mergeReviewResults(results.map(({ result }) => result))
|
|
785
960
|
})
|
|
786
961
|
|
|
962
|
+
const lastDiff = yield* Ref.make<string | undefined>(undefined)
|
|
787
963
|
const loop = (
|
|
788
964
|
round: number,
|
|
789
965
|
previous: ReviewResult | undefined
|
|
790
966
|
): Effect.Effect<ReviewResult, FlowError> =>
|
|
791
967
|
Effect.gen(function* () {
|
|
792
968
|
yield* format
|
|
969
|
+
// A fix round that changed nothing is a loop going nowhere (ADR 0027
|
|
970
|
+
// decision 11): the same diff twice in a row ends the task, typed.
|
|
971
|
+
const diffNow = yield* options.currentDiff
|
|
972
|
+
const before = yield* Ref.get(lastDiff)
|
|
973
|
+
if (
|
|
974
|
+
options.stallOnIdenticalDiff === true &&
|
|
975
|
+
round > 1 &&
|
|
976
|
+
before !== undefined &&
|
|
977
|
+
before === diffNow &&
|
|
978
|
+
previous !== undefined
|
|
979
|
+
) {
|
|
980
|
+
return yield* Stalled.make({
|
|
981
|
+
signal: "identical-diff",
|
|
982
|
+
detail: `review round ${round} found the diff unchanged after the fix for ${previous.issues.length} finding(s)`
|
|
983
|
+
})
|
|
984
|
+
}
|
|
985
|
+
yield* Ref.set(lastDiff, diffNow)
|
|
793
986
|
const result = yield* reviewOnce(round, previous)
|
|
794
987
|
const settled = result.isClean || round >= maxRounds
|
|
795
988
|
yield* options.events.publish(
|
|
@@ -812,7 +1005,8 @@ export const reviewAndFixLoop = Effect.fn("@llm4ts/flow/Review.reviewAndFixLoop"
|
|
|
812
1005
|
if (settled) {
|
|
813
1006
|
return result
|
|
814
1007
|
}
|
|
815
|
-
|
|
1008
|
+
const fix = fixPrompt(result, options.fix)
|
|
1009
|
+
yield* options.fixWith === undefined ? options.coder.ask(fix) : options.fixWith(fix)
|
|
816
1010
|
return yield* loop(round + 1, result)
|
|
817
1011
|
})
|
|
818
1012
|
|
package/src/Reviewer.ts
CHANGED
|
@@ -9,7 +9,9 @@ export class Reviewer extends Schema.Class<Reviewer>("Reviewer")({
|
|
|
9
9
|
* decide whether this lens is worth a full pass (ADR 0017). Absent, one is
|
|
10
10
|
* derived from the system prompt.
|
|
11
11
|
*/
|
|
12
|
-
screen: Schema.optionalKey(Schema.String)
|
|
12
|
+
screen: Schema.optionalKey(Schema.String),
|
|
13
|
+
/** False leaves the shared review-rules preamble (ADR 0027) off this lens; default on. */
|
|
14
|
+
preamble: Schema.optionalKey(Schema.Boolean)
|
|
13
15
|
}) {
|
|
14
16
|
/** The screening statement, explicit or derived. */
|
|
15
17
|
get screeningStatement(): string {
|