@llm4ts/flow 2.31.0 → 2.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Approval.d.ts +1 -1
- package/dist/Approval.d.ts.map +1 -1
- package/dist/Artifacts.d.ts +2 -2
- package/dist/Artifacts.d.ts.map +1 -1
- package/dist/BenchReport.d.ts +2 -2
- package/dist/BenchReport.d.ts.map +1 -1
- package/dist/Chat.d.ts +5 -2
- package/dist/Chat.d.ts.map +1 -1
- package/dist/Chat.js +7 -4
- package/dist/Chat.js.map +1 -1
- package/dist/CostLedger.d.ts +2 -2
- package/dist/CostLedger.d.ts.map +1 -1
- package/dist/Equiv.d.ts +3 -3
- package/dist/Equiv.d.ts.map +1 -1
- package/dist/Flow.d.ts +16 -1
- package/dist/Flow.d.ts.map +1 -1
- package/dist/Flow.js +29 -3
- package/dist/Flow.js.map +1 -1
- package/dist/FlowError.d.ts +15 -2
- package/dist/FlowError.d.ts.map +1 -1
- package/dist/FlowError.js +15 -0
- package/dist/FlowError.js.map +1 -1
- package/dist/FlowEvents.d.ts +18 -5
- package/dist/FlowEvents.d.ts.map +1 -1
- package/dist/FlowEvents.js +30 -9
- package/dist/FlowEvents.js.map +1 -1
- package/dist/Gates.d.ts +1 -1
- package/dist/Gates.d.ts.map +1 -1
- package/dist/Judgment.d.ts +2 -2
- package/dist/Judgment.d.ts.map +1 -1
- package/dist/Landing.d.ts +1 -1
- package/dist/Landing.d.ts.map +1 -1
- package/dist/NodePreflight.d.ts +1 -1
- package/dist/NodePreflight.d.ts.map +1 -1
- package/dist/Pack.d.ts +9 -1
- package/dist/Pack.d.ts.map +1 -1
- package/dist/Pack.js +26 -2
- package/dist/Pack.js.map +1 -1
- package/dist/Persistence.d.ts +2 -2
- package/dist/Persistence.d.ts.map +1 -1
- package/dist/PlanExecution.d.ts +1 -1
- package/dist/PlanExecution.d.ts.map +1 -1
- package/dist/ProgramJudge.d.ts +1 -1
- package/dist/ProgramJudge.d.ts.map +1 -1
- package/dist/ProgramJudge.js +6 -3
- package/dist/ProgramJudge.js.map +1 -1
- package/dist/Replay.d.ts +2 -2
- package/dist/Replay.d.ts.map +1 -1
- package/dist/Retro.d.ts +51 -4
- package/dist/Retro.d.ts.map +1 -1
- package/dist/Retro.js +196 -11
- package/dist/Retro.js.map +1 -1
- package/dist/Review.d.ts +38 -6
- package/dist/Review.d.ts.map +1 -1
- package/dist/Review.js +147 -8
- package/dist/Review.js.map +1 -1
- package/dist/ReviewCache.d.ts +1 -1
- package/dist/ReviewCache.d.ts.map +1 -1
- package/dist/Reviewer.d.ts +2 -0
- package/dist/Reviewer.d.ts.map +1 -1
- package/dist/Reviewer.js +3 -1
- package/dist/Reviewer.js.map +1 -1
- package/dist/Roster.d.ts +19 -10
- package/dist/Roster.d.ts.map +1 -1
- package/dist/Roster.js +51 -17
- package/dist/Roster.js.map +1 -1
- package/dist/RosterSeats.d.ts +5 -4
- package/dist/RosterSeats.d.ts.map +1 -1
- package/dist/RosterSeats.js +7 -4
- package/dist/RosterSeats.js.map +1 -1
- package/dist/Stall.d.ts +29 -0
- package/dist/Stall.d.ts.map +1 -0
- package/dist/Stall.js +49 -0
- package/dist/Stall.js.map +1 -0
- package/dist/Stories.d.ts +8 -1
- package/dist/Stories.d.ts.map +1 -1
- package/dist/Stories.js +7 -1
- package/dist/Stories.js.map +1 -1
- package/package.json +3 -2
- package/src/Chat.ts +15 -7
- package/src/Flow.ts +48 -3
- package/src/FlowError.ts +16 -0
- package/src/FlowEvents.ts +34 -11
- package/src/Pack.ts +34 -1
- package/src/ProgramJudge.ts +3 -0
- package/src/Retro.ts +229 -4
- package/src/Review.ts +202 -8
- package/src/Reviewer.ts +3 -1
- package/src/Roster.ts +86 -39
- package/src/RosterSeats.ts +11 -6
- package/src/Stall.ts +85 -0
- package/src/Stories.ts +14 -0
package/src/Chat.ts
CHANGED
|
@@ -3,9 +3,10 @@ import * as Ref from "effect/Ref"
|
|
|
3
3
|
import * as Semaphore from "effect/Semaphore"
|
|
4
4
|
import type { LlmServiceShape } from "@llm4ts/core/LlmService"
|
|
5
5
|
import { Message, type LlmResponse } from "@llm4ts/core/Models"
|
|
6
|
-
import {
|
|
6
|
+
import { collectAny } from "@llm4ts/core/Streaming"
|
|
7
7
|
import { withToolActivity } from "./Activity.ts"
|
|
8
|
-
import { FlowLlmError } from "./FlowError.ts"
|
|
8
|
+
import { FlowLlmError, type Stalled } from "./FlowError.ts"
|
|
9
|
+
import { noRepeats, stallGuard, type StallOptions } from "./Stall.ts"
|
|
9
10
|
import { Info, TokensUsed, type FlowEventsShape } from "./FlowEvents.ts"
|
|
10
11
|
import { isContextOverflow } from "./TransientRetry.ts"
|
|
11
12
|
|
|
@@ -24,6 +25,8 @@ export interface ChatOptions {
|
|
|
24
25
|
readonly events?: FlowEventsShape
|
|
25
26
|
/** Request label for published `TokensUsed` events. Default: "chat". */
|
|
26
27
|
readonly agent?: string
|
|
28
|
+
/** End a turn that repeats one tool call or goes silent (ADR 0027 decision 11). */
|
|
29
|
+
readonly stall?: StallOptions
|
|
27
30
|
}
|
|
28
31
|
|
|
29
32
|
const publishUsage = (options: ChatOptions, response: LlmResponse): Effect.Effect<void> =>
|
|
@@ -38,7 +41,7 @@ const publishUsage = (options: ChatOptions, response: LlmResponse): Effect.Effec
|
|
|
38
41
|
)
|
|
39
42
|
|
|
40
43
|
export interface Chat {
|
|
41
|
-
readonly ask: (prompt: string) => Effect.Effect<string, FlowLlmError>
|
|
44
|
+
readonly ask: (prompt: string) => Effect.Effect<string, FlowLlmError | Stalled>
|
|
42
45
|
readonly messages: Effect.Effect<ReadonlyArray<Message>>
|
|
43
46
|
}
|
|
44
47
|
|
|
@@ -60,15 +63,19 @@ export const makeChat = Effect.fn("@llm4ts/flow/Chat.make")(function* (
|
|
|
60
63
|
): Effect.fn.Return<Chat> {
|
|
61
64
|
const history = yield* Ref.make(initialHistory(options))
|
|
62
65
|
const gate = yield* Semaphore.make(1)
|
|
66
|
+
const repeats = yield* Ref.make(noRepeats)
|
|
63
67
|
|
|
64
68
|
const send = (messages: ReadonlyArray<Message>) => {
|
|
65
69
|
const stream = service.executeStreamWithHistory(messages)
|
|
66
|
-
|
|
70
|
+
const watched = options.events === undefined ? stream : withToolActivity(options.events, stream)
|
|
71
|
+
return collectAny(
|
|
72
|
+
options.stall === undefined ? watched : stallGuard(watched, options.stall, repeats)
|
|
73
|
+
)
|
|
67
74
|
}
|
|
68
75
|
|
|
69
76
|
const askRound = Effect.fn("@llm4ts/flow/Chat.ask")(function* (
|
|
70
77
|
prompt: string
|
|
71
|
-
): Effect.fn.Return<string, FlowLlmError> {
|
|
78
|
+
): Effect.fn.Return<string, FlowLlmError | Stalled> {
|
|
72
79
|
const userTurn = Message.make({ role: "User", content: prompt })
|
|
73
80
|
const earlier = yield* Ref.get(history)
|
|
74
81
|
const full = [...earlier, userTurn]
|
|
@@ -81,7 +88,8 @@ export const makeChat = Effect.fn("@llm4ts/flow/Chat.make")(function* (
|
|
|
81
88
|
// this turn alone, and continue the conversation from there.
|
|
82
89
|
const response = yield* send(full).pipe(
|
|
83
90
|
Effect.catchIf(
|
|
84
|
-
(error) =>
|
|
91
|
+
(error) =>
|
|
92
|
+
error._tag !== "Stalled" && isContextOverflow(error) && full.length > system.length + 1,
|
|
85
93
|
(error) =>
|
|
86
94
|
Effect.gen(function* () {
|
|
87
95
|
messages = [
|
|
@@ -106,7 +114,7 @@ export const makeChat = Effect.fn("@llm4ts/flow/Chat.make")(function* (
|
|
|
106
114
|
return yield* send(messages)
|
|
107
115
|
})
|
|
108
116
|
),
|
|
109
|
-
Effect.mapError(FlowLlmError.from)
|
|
117
|
+
Effect.mapError((error) => (error._tag === "Stalled" ? error : FlowLlmError.from(error)))
|
|
110
118
|
)
|
|
111
119
|
yield* publishUsage(options, response)
|
|
112
120
|
yield* Ref.set(history, [
|
package/src/Flow.ts
CHANGED
|
@@ -44,6 +44,8 @@ import {
|
|
|
44
44
|
} from "./Review.ts"
|
|
45
45
|
import type { OracleRules } from "./OracleGuard.ts"
|
|
46
46
|
import type { Reviewer } from "./Reviewer.ts"
|
|
47
|
+
import { withContract } from "./AutonomyContract.ts"
|
|
48
|
+
import type { StallOptions } from "./Stall.ts"
|
|
47
49
|
|
|
48
50
|
export { publishUsage, structuredAndPublish } from "./Usage.ts"
|
|
49
51
|
|
|
@@ -109,6 +111,18 @@ export interface ImplementPlanOptions {
|
|
|
109
111
|
* `testsChange` is true.
|
|
110
112
|
*/
|
|
111
113
|
readonly oracle?: { readonly rules?: OracleRules; readonly testsChange?: boolean }
|
|
114
|
+
/** Independent votes of the adversarial lens per round (ADR 0027 decision 7). Default 1. */
|
|
115
|
+
readonly votes?: number
|
|
116
|
+
/**
|
|
117
|
+
* Who applies review findings (ADR 0027 decision 8): the implementer's own
|
|
118
|
+
* chat (default), or a fresh chat on the coder seat briefed to apply the
|
|
119
|
+
* findings and nothing else, with the carried notes for orientation.
|
|
120
|
+
*/
|
|
121
|
+
readonly fixer?: "coder" | "separate"
|
|
122
|
+
/** The repository's own review rules (`.llm4ts/review-rules.md`), loaded by the caller, as one extra lens. */
|
|
123
|
+
readonly repoRules?: Effect.Effect<Reviewer | undefined, FlowError>
|
|
124
|
+
/** End a coder turn that repeats one tool call or goes silent (ADR 0027 decision 11). */
|
|
125
|
+
readonly stall?: StallOptions
|
|
112
126
|
/** How gate output reaches the coder in the fix prompt. */
|
|
113
127
|
readonly fix?: FixPromptOptions
|
|
114
128
|
/** Where each review lens's answer is kept, so a rerun over the same diff asks nothing. */
|
|
@@ -211,6 +225,14 @@ const defaultCommitMessage = (plan: Plan, task: Task): string => `${plan.epicId}
|
|
|
211
225
|
const composeSystem = (base: string | undefined, note: string): string =>
|
|
212
226
|
[base, note].filter((part): part is string => part !== undefined && part.length > 0).join("\n\n")
|
|
213
227
|
|
|
228
|
+
/** The separate fixer's brief (ADR 0027 decision 8): the Bun port's fixer, word for word in spirit. */
|
|
229
|
+
export const fixerBrief = [
|
|
230
|
+
"You apply review findings to a change another coder made. Apply the findings. Nothing else.",
|
|
231
|
+
"Surgical edits only: no refactors, no extras, no new tests beyond what a finding asks for.",
|
|
232
|
+
"If a finding is wrong (the reviewer misread the code), skip it and say so in one line.",
|
|
233
|
+
"Then stop."
|
|
234
|
+
].join("\n")
|
|
235
|
+
|
|
214
236
|
/**
|
|
215
237
|
* The task's triage: the explicit one, or one built from `baseline` over the
|
|
216
238
|
* commit the task starts from (HEAD now), or none.
|
|
@@ -258,7 +280,8 @@ export const implementPlanFlow = Effect.fn("@llm4ts/flow/Flow.implementPlan")(fu
|
|
|
258
280
|
sharedCoder = yield* makeChat(context.coder, {
|
|
259
281
|
events: context.events,
|
|
260
282
|
agent: "coder",
|
|
261
|
-
...(options.system === undefined ? {} : { system: options.system })
|
|
283
|
+
...(options.system === undefined ? {} : { system: options.system }),
|
|
284
|
+
...(options.stall === undefined ? {} : { stall: options.stall })
|
|
262
285
|
})
|
|
263
286
|
}
|
|
264
287
|
|
|
@@ -279,7 +302,8 @@ export const implementPlanFlow = Effect.fn("@llm4ts/flow/Flow.implementPlan")(fu
|
|
|
279
302
|
coder = yield* makeChat(context.coder, {
|
|
280
303
|
events: context.events,
|
|
281
304
|
agent: "coder",
|
|
282
|
-
system: composeSystem(options.system, planSoFar.render)
|
|
305
|
+
system: composeSystem(options.system, planSoFar.render),
|
|
306
|
+
...(options.stall === undefined ? {} : { stall: options.stall })
|
|
283
307
|
})
|
|
284
308
|
}
|
|
285
309
|
// `plan.taskPrompt` deliberately reads the frozen `plan` captured at
|
|
@@ -354,6 +378,24 @@ export const implementPlanFlow = Effect.fn("@llm4ts/flow/Flow.implementPlan")(fu
|
|
|
354
378
|
}
|
|
355
379
|
}
|
|
356
380
|
const triage = yield* triageFor(context, options)
|
|
381
|
+
const repoRules = options.repoRules === undefined ? undefined : yield* options.repoRules
|
|
382
|
+
const reviewers = [
|
|
383
|
+
...(options.reviewers ?? minimalReviewers),
|
|
384
|
+
...(repoRules === undefined ? [] : [repoRules])
|
|
385
|
+
]
|
|
386
|
+
const fixWith =
|
|
387
|
+
options.fixer === "separate"
|
|
388
|
+
? (prompt: string): Effect.Effect<string, FlowError> =>
|
|
389
|
+
Effect.flatMap(
|
|
390
|
+
makeChat(context.coder, {
|
|
391
|
+
events: context.events,
|
|
392
|
+
agent: "coder",
|
|
393
|
+
system: withContract(fixerBrief),
|
|
394
|
+
...(options.stall === undefined ? {} : { stall: options.stall })
|
|
395
|
+
}),
|
|
396
|
+
(fixer) => fixer.ask(withNotes(prompt, notes))
|
|
397
|
+
)
|
|
398
|
+
: undefined
|
|
357
399
|
const oracle: OracleGateOptions | undefined =
|
|
358
400
|
options.oracle === undefined
|
|
359
401
|
? undefined
|
|
@@ -368,7 +410,7 @@ export const implementPlanFlow = Effect.fn("@llm4ts/flow/Flow.implementPlan")(fu
|
|
|
368
410
|
: { baseCount: Effect.map(triage.baseline, (base) => base?.passedCount) })
|
|
369
411
|
}
|
|
370
412
|
yield* reviewAndFixLoop({
|
|
371
|
-
reviewers
|
|
413
|
+
reviewers,
|
|
372
414
|
reviewerService: flowReviewer(context),
|
|
373
415
|
coder,
|
|
374
416
|
taskTitle: task.title,
|
|
@@ -379,6 +421,9 @@ export const implementPlanFlow = Effect.fn("@llm4ts/flow/Flow.implementPlan")(fu
|
|
|
379
421
|
...(options.format === undefined ? {} : { format: options.format }),
|
|
380
422
|
...(triage === undefined ? {} : { triage }),
|
|
381
423
|
...(oracle === undefined ? {} : { oracle }),
|
|
424
|
+
...(options.votes === undefined ? {} : { votes: options.votes }),
|
|
425
|
+
...(fixWith === undefined ? {} : { fixWith }),
|
|
426
|
+
...(options.stall === undefined ? {} : { stallOnIdenticalDiff: true }),
|
|
382
427
|
...(options.fix === undefined ? {} : { fix: options.fix }),
|
|
383
428
|
...(options.reviewCache === undefined ? {} : { cache: options.reviewCache }),
|
|
384
429
|
...(options.onReview === undefined
|
package/src/FlowError.ts
CHANGED
|
@@ -267,6 +267,21 @@ export class RosterExhausted extends Schema.TaggedError<RosterExhausted>()("Rost
|
|
|
267
267
|
}
|
|
268
268
|
|
|
269
269
|
/** One story failed; carries the story id so a fail-fast run names its cause. */
|
|
270
|
+
/**
|
|
271
|
+
* A loop that stopped making progress (ADR 0027 decision 11): the review
|
|
272
|
+
* round saw the same diff twice, the coder repeated one tool call too many
|
|
273
|
+
* times, or the seat went silent for too long. A stopping condition the
|
|
274
|
+
* model does not control; the story fails and the retro diagnoses.
|
|
275
|
+
*/
|
|
276
|
+
export class Stalled extends Schema.TaggedError<Stalled>()("Stalled", {
|
|
277
|
+
signal: Schema.Literals(["identical-diff", "repeated-tool-call", "silence"]),
|
|
278
|
+
detail: Schema.String
|
|
279
|
+
}) {
|
|
280
|
+
get message(): string {
|
|
281
|
+
return `stalled (${this.signal}): ${this.detail}`
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
270
285
|
export class StoryFailed extends Schema.TaggedError<StoryFailed>()("StoryFailed", {
|
|
271
286
|
story: Schema.String,
|
|
272
287
|
reason: Schema.String
|
|
@@ -371,6 +386,7 @@ export const FlowError = Schema.Union([
|
|
|
371
386
|
EpicIncomplete,
|
|
372
387
|
LandingFailed,
|
|
373
388
|
StoryFailed,
|
|
389
|
+
Stalled,
|
|
374
390
|
DecisionsInvalid,
|
|
375
391
|
OpenPointsPending,
|
|
376
392
|
ContractConflict,
|
package/src/FlowEvents.ts
CHANGED
|
@@ -171,6 +171,19 @@ export class CapabilityUnenforceable extends Schema.TaggedClass<CapabilityUnenfo
|
|
|
171
171
|
}
|
|
172
172
|
) {}
|
|
173
173
|
|
|
174
|
+
/** A review finding moved down a severity because it could not be placed in the diff (ADR 0027). */
|
|
175
|
+
export class ReviewFindingDemoted extends Schema.TaggedClass<ReviewFindingDemoted>()(
|
|
176
|
+
"ReviewFindingDemoted",
|
|
177
|
+
{
|
|
178
|
+
lens: Schema.String,
|
|
179
|
+
title: Schema.String,
|
|
180
|
+
from: Schema.Literals(["Critical", "Warning", "Info"]),
|
|
181
|
+
to: Schema.Literals(["Critical", "Warning", "Info"]),
|
|
182
|
+
reason: Schema.String,
|
|
183
|
+
lane: Schema.optionalKey(Schema.String)
|
|
184
|
+
}
|
|
185
|
+
) {}
|
|
186
|
+
|
|
174
187
|
/** A task's `verified:` claims against the tool calls its transcript shows (ADR 0027). */
|
|
175
188
|
export class EvidenceChecked extends Schema.TaggedClass<EvidenceChecked>()("EvidenceChecked", {
|
|
176
189
|
task: Schema.String,
|
|
@@ -248,13 +261,15 @@ export class ExecutorLeased extends Schema.TaggedClass<ExecutorLeased>()("Execut
|
|
|
248
261
|
executor: Schema.String,
|
|
249
262
|
role: Schema.String,
|
|
250
263
|
label: Schema.optionalKey(Schema.String),
|
|
251
|
-
/** Took the role on its own context's coder slot: nobody independent could. */
|
|
264
|
+
/** Took the role on its own context's coder slot, no slot of its own free: nobody independent could. */
|
|
252
265
|
borrowed: Schema.optionalKey(Schema.Boolean),
|
|
253
266
|
/**
|
|
254
|
-
*
|
|
255
|
-
*
|
|
267
|
+
* Set when the executor is the context's own coder, so the call is not
|
|
268
|
+
* independent: `nobody` else takes the role (or all are out for the run);
|
|
269
|
+
* `held`: every other executor that could has its slots held by coders;
|
|
270
|
+
* `out`: every other executor that could is out of the round for now.
|
|
256
271
|
*/
|
|
257
|
-
because: Schema.optionalKey(Schema.Literals(["nobody", "held"]))
|
|
272
|
+
because: Schema.optionalKey(Schema.Literals(["nobody", "held", "out"]))
|
|
258
273
|
}) {}
|
|
259
274
|
|
|
260
275
|
export class ExecutorReleased extends Schema.TaggedClass<ExecutorReleased>()("ExecutorReleased", {
|
|
@@ -309,14 +324,21 @@ export const rosterEventMessage = (event: FlowEvent): string | undefined => {
|
|
|
309
324
|
const forLabel = (label: string | undefined): string =>
|
|
310
325
|
label === undefined ? "" : ` for ${label}`
|
|
311
326
|
switch (event._tag) {
|
|
312
|
-
case "ExecutorLeased":
|
|
327
|
+
case "ExecutorLeased": {
|
|
328
|
+
const taken = `roster: ${event.executor} takes ${event.role}${forLabel(event.label)}`
|
|
329
|
+
if (event.because === undefined && event.borrowed !== true) {
|
|
330
|
+
return taken
|
|
331
|
+
}
|
|
332
|
+
const why =
|
|
333
|
+
event.because === "held"
|
|
334
|
+
? `every other executor that takes ${event.role} is held by a coder`
|
|
335
|
+
: event.because === "out"
|
|
336
|
+
? `every other executor that takes ${event.role} is out of the round`
|
|
337
|
+
: `no other executor can take ${event.role}`
|
|
313
338
|
return event.borrowed === true
|
|
314
|
-
?
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
: `no other executor can take ${event.role}`
|
|
318
|
-
})`
|
|
319
|
-
: `roster: ${event.executor} takes ${event.role}${forLabel(event.label)}`
|
|
339
|
+
? `${taken} on its own coder's slot — not independent (${why})`
|
|
340
|
+
: `${taken} on its own slot — not independent (${why})`
|
|
341
|
+
}
|
|
320
342
|
case "ExecutorExcluded":
|
|
321
343
|
return `roster: ${event.executor} out of the round ${event.reason}`
|
|
322
344
|
case "ExecutorResumed":
|
|
@@ -347,6 +369,7 @@ export const FlowEvent = Schema.Union([
|
|
|
347
369
|
CapabilityDeniedEvent,
|
|
348
370
|
CapabilityUnenforceable,
|
|
349
371
|
EvidenceChecked,
|
|
372
|
+
ReviewFindingDemoted,
|
|
350
373
|
Declassified,
|
|
351
374
|
ExecutorLeased,
|
|
352
375
|
ExecutorReleased,
|
package/src/Pack.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { Dimension } from "@llm4ts/core/eval/Eval"
|
|
|
4
4
|
import type { ConsolidateRules } from "./Domains.ts"
|
|
5
5
|
import { PlanParseError } from "./FlowError.ts"
|
|
6
6
|
import { oracleRulesFrom, type OracleRules } from "./OracleGuard.ts"
|
|
7
|
-
import {
|
|
7
|
+
import { Reviewer, parseReviewer } from "./Reviewer.ts"
|
|
8
8
|
import { CoverageRule } from "./SpecChecks.ts"
|
|
9
9
|
import type { WorkspaceError, WorkspaceShape } from "./Workspace.ts"
|
|
10
10
|
|
|
@@ -74,6 +74,11 @@ export interface Pack {
|
|
|
74
74
|
* over the defaults. Absent: the defaults.
|
|
75
75
|
*/
|
|
76
76
|
readonly oracle: OracleRules | undefined
|
|
77
|
+
/**
|
|
78
|
+
* The `## Review rules` section (ADR 0027 decision 9): extra rules every
|
|
79
|
+
* pack lens carries after the shared preamble, or `- preamble: off`.
|
|
80
|
+
*/
|
|
81
|
+
readonly reviewRules: { readonly text: string; readonly preamble: boolean } | undefined
|
|
77
82
|
readonly dir: string
|
|
78
83
|
readonly gate: (name: string) => ReadonlyArray<string> | undefined
|
|
79
84
|
readonly prompt: (name: string) => string | undefined
|
|
@@ -237,9 +242,36 @@ export const loadPack = Effect.fn("@llm4ts/flow/Pack.load")(function* (
|
|
|
237
242
|
const gates = Object.fromEntries(
|
|
238
243
|
Object.entries(gateValues).map(([name, command]) => [name, command.split(/\s+/)])
|
|
239
244
|
)
|
|
245
|
+
const reviewRulesBody = section(manifest.sections, "Review rules")
|
|
246
|
+
const reviewRulesValues = namedItems(reviewRulesBody)
|
|
247
|
+
const reviewRules =
|
|
248
|
+
reviewRulesBody === undefined
|
|
249
|
+
? undefined
|
|
250
|
+
: {
|
|
251
|
+
text: reviewRulesBody
|
|
252
|
+
.split(/\r?\n/)
|
|
253
|
+
.filter((line) => !/^- preamble:/iu.test(line.trim()))
|
|
254
|
+
.join("\n")
|
|
255
|
+
.trim(),
|
|
256
|
+
preamble: reviewRulesValues.preamble?.toLowerCase() !== "off"
|
|
257
|
+
}
|
|
240
258
|
const lenses = Object.entries(lensFiles)
|
|
241
259
|
.sort(([left], [right]) => left.localeCompare(right))
|
|
242
260
|
.map(([name, text]) => parseReviewer(name, text))
|
|
261
|
+
// The pack's own review rules ride on every lens; `preamble: off` leaves
|
|
262
|
+
// the shared preamble out (ADR 0027 decision 9).
|
|
263
|
+
.map((lens) =>
|
|
264
|
+
reviewRules === undefined
|
|
265
|
+
? lens
|
|
266
|
+
: Reviewer.make({
|
|
267
|
+
...lens,
|
|
268
|
+
systemPrompt:
|
|
269
|
+
reviewRules.text.length === 0
|
|
270
|
+
? lens.systemPrompt
|
|
271
|
+
: `${lens.systemPrompt}\n\nPack review rules:\n${reviewRules.text}`,
|
|
272
|
+
...(reviewRules.preamble ? {} : { preamble: false })
|
|
273
|
+
})
|
|
274
|
+
)
|
|
243
275
|
const fields = manifest.fields
|
|
244
276
|
if (fields["programFiles"] !== undefined) {
|
|
245
277
|
return yield* PlanParseError.make({
|
|
@@ -346,6 +378,7 @@ export const loadPack = Effect.fn("@llm4ts/flow/Pack.load")(function* (
|
|
|
346
378
|
specSchema,
|
|
347
379
|
consolidate,
|
|
348
380
|
oracle,
|
|
381
|
+
reviewRules,
|
|
349
382
|
dir: directory,
|
|
350
383
|
gate: (name) => gates[name],
|
|
351
384
|
prompt: (name) => prompts[name],
|
package/src/ProgramJudge.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { reviewRulesPreamble } from "./Review.ts"
|
|
1
2
|
import * as Effect from "effect/Effect"
|
|
2
3
|
import { Sample, type Dimension, type EvalResult } from "@llm4ts/core/eval/Eval"
|
|
3
4
|
import { makeEvaluator, type Evaluator } from "@llm4ts/core/eval/Evaluator"
|
|
@@ -141,7 +142,9 @@ export const withJudgment = (
|
|
|
141
142
|
const join = (root: string, path: string): string =>
|
|
142
143
|
`${root.replace(/[\\/]+$/, "")}/${path.replace(/^[\\/]+/, "")}`
|
|
143
144
|
|
|
145
|
+
/** The shared review rules (ADR 0027 decision 9) precede every rubric the judge scores with. */
|
|
144
146
|
const rubricText = (dimensions: ReadonlyArray<Dimension>): string =>
|
|
147
|
+
`${reviewRulesPreamble}\n\n` +
|
|
145
148
|
dimensions
|
|
146
149
|
.map((dimension) => `${dimension.name} (0..${dimension.maxScore}): ${dimension.rubric}`)
|
|
147
150
|
.join("\n")
|