@llm4ts/shell 2.14.0 → 2.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/flows/epic-stories-board.js +18 -0
- package/flows/epic-stories.js +3 -179
- package/flows/lib/epic-stories.js +201 -2
- package/flows/lib/story-board.js +232 -0
- package/kits/soap-ace/README.md +50 -9
- package/kits/soap-ace/fixtures/demo-bank-soap/RUNBOOK.md +16 -0
- package/kits/soap-ace/flows/lib/soap/Call.js +10 -6
- package/kits/soap-ace/flows/lib/soap/Explore.js +476 -0
- package/kits/soap-ace/flows/soap-explore.js +190 -0
- package/package.json +4 -4
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
// A fork of epic-stories that judges each story with a blackboard ruleset (ADR 0020): the four story dimensions are scored by the Judgment service, decided by the run's judgment policy, and a company-editable bar in flows/lib/story-board.ts says whether the story merges. Same flags and environment as epic-stories.
|
|
2
|
+
//
|
|
3
|
+
// llm4ts run epic-stories-board --repo ~/demo/portal "Add the current account and wire transfers"
|
|
4
|
+
// llm4ts run epic-stories-board --repo ~/demo/portal -- --plan-only "…"
|
|
5
|
+
//
|
|
6
|
+
// The judgment seat is LLM4TS_JUDGMENT_PROVIDER / LLM4TS_JUDGMENT_MODEL (ADR
|
|
7
|
+
// 0017), or one derived from the reasoning seat. Each round's run lands in
|
|
8
|
+
// the trace as a BlackboardRun event and each answer in
|
|
9
|
+
// .llm4ts/judgments/story-board.jsonl. Until this judge is approved,
|
|
10
|
+
// epic-stories keeps the rubric judge.
|
|
11
|
+
import { runFlowMain } from "@llm4ts/runner";
|
|
12
|
+
import { runEpicStories } from "./lib/epic-stories.js";
|
|
13
|
+
import { boardJudgeFactory } from "./lib/story-board.js";
|
|
14
|
+
runFlowMain(runEpicStories({
|
|
15
|
+
storyJudge: boardJudgeFactory,
|
|
16
|
+
judgmentFromEnvironment: true,
|
|
17
|
+
judgmentLog: true
|
|
18
|
+
}));
|
package/flows/epic-stories.js
CHANGED
|
@@ -18,182 +18,6 @@
|
|
|
18
18
|
// (--fail-fast stops instead). The epic branch is left in place; the board
|
|
19
19
|
// and the report under .llm4ts/epics/<epic-id>/ carry ESTIMATED usage
|
|
20
20
|
// figures (ADR 0013).
|
|
21
|
-
import {
|
|
22
|
-
import
|
|
23
|
-
|
|
24
|
-
import { budget, cap } from "@llm4ts/flow/Context";
|
|
25
|
-
import { makeLocalBoardSync } from "@llm4ts/flow/BoardSync";
|
|
26
|
-
import { estimatedUsageOptionsFromEnv, makeEstimatedUsageMeter } from "@llm4ts/flow/EstimatedUsage";
|
|
27
|
-
import { FlowAborted, Info, asReadOnly, nodePlainFileStore, nodeProcessExecutor, resolveFlowInput, runFlowMain, runNode, stage, withModel } from "@llm4ts/runner";
|
|
28
|
-
import { landEpic } from "@llm4ts/flow/Landing";
|
|
29
|
-
import { implementStoriesFlow } from "@llm4ts/flow/Stories";
|
|
30
|
-
import { makeStoryPlanStore, validateStoryPlan } from "@llm4ts/flow/StoryPlan";
|
|
31
|
-
import * as Console from "effect/Console";
|
|
32
|
-
import { awaitServer, chooseEpic, epicsDir, listEpics, renderEpicList, combineTotals, epicIdFor, flagsFromEnvironment, appDirFor, appScripts, gateCommands, gatesIn, inAppDir, generateStoryPlan, httpProbe, judgeStory, localCoderServer, parseEpicArgs, reasonerFromEnvironment, serverHealthUrl, setupAgentEnabled, setupIn, storyCoderFromEnvironment, verifyBlockedOn, worktreeRootFor, worktreeSetupCommand } from "./lib/epic-stories.js";
|
|
33
|
-
/** `LLM4TS_CODER_MODEL` / `LLM4TS_REASONING_MODEL`: pi takes `provider/model` (e.g. `openai-codex/gpt-5.5`). */
|
|
34
|
-
const withOptionalModel = (config, model) => {
|
|
35
|
-
const trimmed = model?.trim();
|
|
36
|
-
return trimmed === undefined || trimmed.length === 0 ? config : withModel(config, trimmed);
|
|
37
|
-
};
|
|
38
|
-
/** Extra CLI flags on top of a preset's own (`LLM4TS_CODER_FLAGS`). */
|
|
39
|
-
const withExtraFlags = (config, flags) => Object.keys(flags).length === 0
|
|
40
|
-
? config
|
|
41
|
-
: CliConnectorConfig.make({ ...config, flags: { ...config.flags, ...flags } });
|
|
42
|
-
const defaultEpic = "Add the retail customer's current account (Conto) with balance and movements, and wire " +
|
|
43
|
-
"transfers (Bonifico) with beneficiary, review, SCA confirmation, and history.";
|
|
44
|
-
const program = Effect.gen(function* () {
|
|
45
|
-
const flags = yield* parseEpicArgs(process.argv.slice(2));
|
|
46
|
-
// The epic is chosen before any seat is resolved: the text given, `--epic`,
|
|
47
|
-
// or the one epic not landed yet (`chooseEpic`); `--list` only lists.
|
|
48
|
-
const given = yield* resolveFlowInput("", flags.rest);
|
|
49
|
-
const files = nodePlainFileStore;
|
|
50
|
-
const epics = yield* listEpics(files, given.workDir);
|
|
51
|
-
if (flags.list) {
|
|
52
|
-
yield* Console.log(renderEpicList(epics));
|
|
53
|
-
return;
|
|
54
|
-
}
|
|
55
|
-
const choice = yield* chooseEpic({
|
|
56
|
-
text: given.prompt,
|
|
57
|
-
epic: flags.epic,
|
|
58
|
-
epics,
|
|
59
|
-
defaultEpic
|
|
60
|
-
});
|
|
61
|
-
const input = {
|
|
62
|
-
...given,
|
|
63
|
-
prompt: choice._tag === "Existing" ? choice.epic.epic : choice.prompt
|
|
64
|
-
};
|
|
65
|
-
const coderFlags = flagsFromEnvironment(process.env.LLM4TS_CODER_FLAGS);
|
|
66
|
-
const reasoning = withExtraFlags(withOptionalModel(yield* reasonerFromEnvironment(process.env), process.env.LLM4TS_REASONING_MODEL), flagsFromEnvironment(process.env.LLM4TS_REASONING_FLAGS));
|
|
67
|
-
const coder = withExtraFlags(withOptionalModel(yield* storyCoderFromEnvironment(process.env), process.env.LLM4TS_CODER_MODEL), coderFlags);
|
|
68
|
-
const localServer = localCoderServer(process.env.LLM4TS_CODER_MODEL, coderFlags, process.env);
|
|
69
|
-
// A new epic's id (and state folder) is derived from its text; an existing
|
|
70
|
-
// one keeps its folder, whatever text is on the command line.
|
|
71
|
-
const epicId = choice._tag === "Existing" ? choice.epic.dir : epicIdFor(input.prompt);
|
|
72
|
-
const stateDir = join(epicsDir(input.workDir), epicId);
|
|
73
|
-
const planPath = join(stateDir, "plan.md");
|
|
74
|
-
const estimateOptions = estimatedUsageOptionsFromEnv(process.env);
|
|
75
|
-
const contextBudget = budget(process.env);
|
|
76
|
-
yield* runNode({
|
|
77
|
-
workDir: input.workDir,
|
|
78
|
-
workspace: input.workspace,
|
|
79
|
-
userPrompt: input.prompt,
|
|
80
|
-
coder,
|
|
81
|
-
reasoning,
|
|
82
|
-
reviewers: [asReadOnly(reasoning)],
|
|
83
|
-
environment: process.env
|
|
84
|
-
}, (context) => Effect.gen(function* () {
|
|
85
|
-
const events = context.events;
|
|
86
|
-
const reasoningMeter = yield* makeEstimatedUsageMeter(context.reasoning, estimateOptions);
|
|
87
|
-
const guidance = yield* Effect.map(files.read(join(input.workDir, "CONTRIBUTING.md")), (text) => cap(text ?? "(no CONTRIBUTING.md in the target repository)", 24_000).text);
|
|
88
|
-
const store = makeStoryPlanStore(files);
|
|
89
|
-
const plan = yield* stage(events, "story plan", store
|
|
90
|
-
.recoverOrCreate(planPath, generateStoryPlan(reasoningMeter.service, events, input.prompt, epicId, guidance))
|
|
91
|
-
.pipe(Effect.flatMap(validateStoryPlan)));
|
|
92
|
-
yield* events.publish(Info.make({
|
|
93
|
-
message: `story plan: ${plan.stories.length} stories at ${planPath} (edit and rerun to re-plan)`
|
|
94
|
-
}));
|
|
95
|
-
if (flags.planOnly) {
|
|
96
|
-
return;
|
|
97
|
-
}
|
|
98
|
-
// Setup and gates run where the application is, in every checkout.
|
|
99
|
-
const appDir = yield* appDirFor(input.workDir, process.env);
|
|
100
|
-
if (appDir !== ".") {
|
|
101
|
-
yield* events.publish(Info.make({ message: `app dir: ${appDir} (setup and gates run there; LLM4TS_APP_DIR)` }));
|
|
102
|
-
}
|
|
103
|
-
const commands = gateCommands(process.env, yield* appScripts(files, join(input.workDir, appDir)));
|
|
104
|
-
yield* events.publish(Info.make({
|
|
105
|
-
message: commands.length === 0
|
|
106
|
-
? "gates: none (the app defines none of typecheck, lint, test, build; set LLM4TS_GATES)"
|
|
107
|
-
: `gates: ${commands.map((command) => command.join(" ")).join(" · ")}`
|
|
108
|
-
}));
|
|
109
|
-
if (flags.land !== undefined) {
|
|
110
|
-
const landed = yield* landEpic(context, {
|
|
111
|
-
plan,
|
|
112
|
-
files,
|
|
113
|
-
stateDir,
|
|
114
|
-
target: flags.land,
|
|
115
|
-
keepWorktrees: flags.keepWorktrees,
|
|
116
|
-
gates: inAppDir(appDir, gatesIn(nodeProcessExecutor, events, commands)),
|
|
117
|
-
system: ["House rules of the target repository (CONTRIBUTING.md):", guidance].join("\n")
|
|
118
|
-
});
|
|
119
|
-
yield* events.publish(Info.make({
|
|
120
|
-
message: `epic ${plan.epicId}: landed on ${landed.target}${landed.conflicts.length === 0 ? "" : ` (${landed.conflicts.length} conflicted file(s) resolved in ${landed.rounds} round(s))`}`
|
|
121
|
-
}));
|
|
122
|
-
return;
|
|
123
|
-
}
|
|
124
|
-
// With a roster, the default is every coder slot it has; the flag caps it.
|
|
125
|
-
const concurrency = flags.concurrency ?? context.roster?.slots("coder") ?? 3;
|
|
126
|
-
if (context.roster === undefined && localServer !== undefined && concurrency > 1) {
|
|
127
|
-
yield* events.publish(Info.make({
|
|
128
|
-
message: `⚠ the coder is served by a local single-model server (${localServer}): it generates one ` +
|
|
129
|
-
`reply at a time, so ${concurrency} parallel coders queue behind each other and a queued ` +
|
|
130
|
-
'request is cut off after a few minutes ("terminated"). Prefer --concurrency 1, or give ' +
|
|
131
|
-
"the server parallel slots."
|
|
132
|
-
}));
|
|
133
|
-
}
|
|
134
|
-
const contextFor = context.contextFor;
|
|
135
|
-
if (contextFor === undefined) {
|
|
136
|
-
return yield* FlowAborted.make({
|
|
137
|
-
message: "this runner cannot rebind seats to a worktree (no contextFor)"
|
|
138
|
-
});
|
|
139
|
-
}
|
|
140
|
-
const appDirNote = appDir === "."
|
|
141
|
-
? []
|
|
142
|
-
: [
|
|
143
|
-
`The application lives in ${appDir}/ — its package.json, sources and tests; the gates run there.`
|
|
144
|
-
];
|
|
145
|
-
const gates = inAppDir(appDir, gatesIn(nodeProcessExecutor, events, commands));
|
|
146
|
-
const setupCommand = worktreeSetupCommand(process.env);
|
|
147
|
-
const healthUrl = serverHealthUrl(localServer, process.env);
|
|
148
|
-
const report = yield* implementStoriesFlow({ ...context, reasoning: reasoningMeter.service }, {
|
|
149
|
-
plan,
|
|
150
|
-
files,
|
|
151
|
-
stateDir,
|
|
152
|
-
worktreeRoot: worktreeRootFor(input.workDir, plan.epicId, process.env),
|
|
153
|
-
board: makeLocalBoardSync(files, stateDir, `Epic: ${plan.epicId}`),
|
|
154
|
-
contextFor: (workDir, contextOptions) => Effect.gen(function* () {
|
|
155
|
-
const rebound = yield* contextFor(workDir, contextOptions);
|
|
156
|
-
const coderMeter = yield* makeEstimatedUsageMeter(rebound.coder, estimateOptions);
|
|
157
|
-
const reviewMeter = yield* makeEstimatedUsageMeter(rebound.reasoning, estimateOptions);
|
|
158
|
-
const seats = {
|
|
159
|
-
context: {
|
|
160
|
-
...rebound,
|
|
161
|
-
coder: coderMeter.service,
|
|
162
|
-
reasoning: reviewMeter.service
|
|
163
|
-
},
|
|
164
|
-
totals: combineTotals(coderMeter.totals, reviewMeter.totals)
|
|
165
|
-
};
|
|
166
|
-
return seats;
|
|
167
|
-
}),
|
|
168
|
-
...(setupCommand === undefined
|
|
169
|
-
? {}
|
|
170
|
-
: {
|
|
171
|
-
setup: inAppDir(appDir, setupIn(nodeProcessExecutor, events, setupCommand)),
|
|
172
|
-
setupAgent: setupAgentEnabled(process.env)
|
|
173
|
-
}),
|
|
174
|
-
gates,
|
|
175
|
-
// With a roster, the judge and the verifier are leased per story,
|
|
176
|
-
// away from the executor coding it (ADR 0019).
|
|
177
|
-
judge: (story, diff, seats) => judgeStory(seats.context.roster?.forRole("judge") ?? reasoningMeter.service, story, diff, contextBudget, plan),
|
|
178
|
-
verifyBlocked: (story, need, workDir, seats) => verifyBlockedOn(seats.context.roster?.forRole("verifier") ?? reasoningMeter.service, events, files, plan)(story, need, workDir),
|
|
179
|
-
// A roster waits for its own executors (the story's next lease
|
|
180
|
-
// does); without one, poll the single coder's engine.
|
|
181
|
-
awaitRecovery: context.roster === undefined
|
|
182
|
-
? awaitServer(healthUrl === undefined ? undefined : httpProbe(healthUrl), events)
|
|
183
|
-
: () => Effect.void,
|
|
184
|
-
system: (story) => Effect.succeed([
|
|
185
|
-
"House rules of the target repository (CONTRIBUTING.md):",
|
|
186
|
-
guidance,
|
|
187
|
-
"",
|
|
188
|
-
`Imitate the exemplar feature before inventing anything. Story id: ${story.id}.`,
|
|
189
|
-
...appDirNote
|
|
190
|
-
].join("\n")),
|
|
191
|
-
concurrency,
|
|
192
|
-
failFast: flags.failFast
|
|
193
|
-
});
|
|
194
|
-
yield* events.publish(Info.make({
|
|
195
|
-
message: `epic ${report.epicId}: ${report.count("done")} done, ${report.count("failed")} failed, ${report.count("waiting")} waiting — report at ${join(stateDir, "report.md")} (usage figures estimated)`
|
|
196
|
-
}));
|
|
197
|
-
}));
|
|
198
|
-
});
|
|
199
|
-
runFlowMain(program);
|
|
21
|
+
import { runFlowMain } from "@llm4ts/runner";
|
|
22
|
+
import { rubricStoryJudge, runEpicStories } from "./lib/epic-stories.js";
|
|
23
|
+
runFlowMain(runEpicStories({ storyJudge: rubricStoryJudge }));
|
|
@@ -18,7 +18,16 @@ import { BlockedVerdict, StoryState, StoryStateVersion } from "@llm4ts/flow/Stor
|
|
|
18
18
|
import { stableHash } from "@llm4ts/flow/Plan";
|
|
19
19
|
import { ReviewIssue } from "@llm4ts/flow/Review";
|
|
20
20
|
import { StoryPlan, dependenciesOf, makeStoryPlanStore, pathsNamedIn } from "@llm4ts/flow/StoryPlan";
|
|
21
|
-
import
|
|
21
|
+
import * as Console from "effect/Console";
|
|
22
|
+
import { CliConnectorConfig } from "@llm4ts/core/ConnectorConfig";
|
|
23
|
+
import { budget } from "@llm4ts/flow/Context";
|
|
24
|
+
import { makeLocalBoardSync } from "@llm4ts/flow/BoardSync";
|
|
25
|
+
import { estimatedUsageOptionsFromEnv, makeEstimatedUsageMeter } from "@llm4ts/flow/EstimatedUsage";
|
|
26
|
+
import { landEpic } from "@llm4ts/flow/Landing";
|
|
27
|
+
import { implementStoriesFlow } from "@llm4ts/flow/Stories";
|
|
28
|
+
import { validateStoryPlan } from "@llm4ts/flow/StoryPlan";
|
|
29
|
+
import { asReadOnly, nodePlainFileStore, nodeProcessExecutor, resolveFlowInput, runNode, stage, withModel } from "@llm4ts/runner";
|
|
30
|
+
import { claude, coderIds, judgmentConnectorFromEnvironment, pi } from "@llm4ts/runner/Connectors";
|
|
22
31
|
import { FlowAborted, FlowLlmError, ReviewResult, ScriptUsage, coderFor, lintCommand, mergeReviewResults } from "@llm4ts/runner";
|
|
23
32
|
export const epicUsage = [
|
|
24
33
|
"epic-stories flags:",
|
|
@@ -307,7 +316,8 @@ export const serverHealthUrl = (server, environment) => {
|
|
|
307
316
|
}
|
|
308
317
|
};
|
|
309
318
|
/** Whether `url` answers 2xx within five seconds (the roster's health check). */
|
|
310
|
-
|
|
319
|
+
import { httpProbe } from "@llm4ts/runner/ExecutorRoster";
|
|
320
|
+
export { httpProbe };
|
|
311
321
|
export const defaultRecoveryTiming = {
|
|
312
322
|
interval: "15 seconds",
|
|
313
323
|
attempts: 60,
|
|
@@ -671,3 +681,192 @@ export const gatesIn = (process, events, commands) => (workDir) => Effect.gen(fu
|
|
|
671
681
|
}
|
|
672
682
|
return mergeReviewResults(results);
|
|
673
683
|
});
|
|
684
|
+
/** Today's story judge: the rubric judge over the four dimensions. */
|
|
685
|
+
export const rubricStoryJudge = (context) => (story, diff, seats) => judgeStory(seats.context.roster?.forRole("judge") ?? context.reasoning, story, diff, context.budget, context.plan);
|
|
686
|
+
/** The runner options the two flags above add; empty when neither is set. */
|
|
687
|
+
export const epicRunnerOptions = Effect.fn("epic-stories.runnerOptions")(function* (options, environment) {
|
|
688
|
+
const judgment = options.judgmentFromEnvironment === true
|
|
689
|
+
? yield* judgmentConnectorFromEnvironment(environment)
|
|
690
|
+
: undefined;
|
|
691
|
+
return {
|
|
692
|
+
...(judgment === undefined ? {} : { judgment }),
|
|
693
|
+
...(options.judgmentLog === true ? { judgmentLog: true } : {})
|
|
694
|
+
};
|
|
695
|
+
});
|
|
696
|
+
/** `LLM4TS_CODER_MODEL` / `LLM4TS_REASONING_MODEL`: pi takes `provider/model` (e.g. `openai-codex/gpt-5.5`). */
|
|
697
|
+
const withOptionalModel = (config, model) => {
|
|
698
|
+
const trimmed = model?.trim();
|
|
699
|
+
return trimmed === undefined || trimmed.length === 0 ? config : withModel(config, trimmed);
|
|
700
|
+
};
|
|
701
|
+
/** Extra CLI flags on top of a preset's own (`LLM4TS_CODER_FLAGS`). */
|
|
702
|
+
const withExtraFlags = (config, flags) => Object.keys(flags).length === 0
|
|
703
|
+
? config
|
|
704
|
+
: CliConnectorConfig.make({ ...config, flags: { ...config.flags, ...flags } });
|
|
705
|
+
const defaultEpic = "Add the retail customer's current account (Conto) with balance and movements, and wire " +
|
|
706
|
+
"transfers (Bonifico) with beneficiary, review, SCA confirmation, and history.";
|
|
707
|
+
export const runEpicStories = (options) => Effect.gen(function* () {
|
|
708
|
+
const flags = yield* parseEpicArgs(process.argv.slice(2));
|
|
709
|
+
// The epic is chosen before any seat is resolved: the text given, `--epic`,
|
|
710
|
+
// or the one epic not landed yet (`chooseEpic`); `--list` only lists.
|
|
711
|
+
const given = yield* resolveFlowInput("", flags.rest);
|
|
712
|
+
const files = nodePlainFileStore;
|
|
713
|
+
const epics = yield* listEpics(files, given.workDir);
|
|
714
|
+
if (flags.list) {
|
|
715
|
+
yield* Console.log(renderEpicList(epics));
|
|
716
|
+
return;
|
|
717
|
+
}
|
|
718
|
+
const choice = yield* chooseEpic({
|
|
719
|
+
text: given.prompt,
|
|
720
|
+
epic: flags.epic,
|
|
721
|
+
epics,
|
|
722
|
+
defaultEpic
|
|
723
|
+
});
|
|
724
|
+
const input = {
|
|
725
|
+
...given,
|
|
726
|
+
prompt: choice._tag === "Existing" ? choice.epic.epic : choice.prompt
|
|
727
|
+
};
|
|
728
|
+
const coderFlags = flagsFromEnvironment(process.env.LLM4TS_CODER_FLAGS);
|
|
729
|
+
const reasoning = withExtraFlags(withOptionalModel(yield* reasonerFromEnvironment(process.env), process.env.LLM4TS_REASONING_MODEL), flagsFromEnvironment(process.env.LLM4TS_REASONING_FLAGS));
|
|
730
|
+
const coder = withExtraFlags(withOptionalModel(yield* storyCoderFromEnvironment(process.env), process.env.LLM4TS_CODER_MODEL), coderFlags);
|
|
731
|
+
const localServer = localCoderServer(process.env.LLM4TS_CODER_MODEL, coderFlags, process.env);
|
|
732
|
+
// A new epic's id (and state folder) is derived from its text; an existing
|
|
733
|
+
// one keeps its folder, whatever text is on the command line.
|
|
734
|
+
const epicId = choice._tag === "Existing" ? choice.epic.dir : epicIdFor(input.prompt);
|
|
735
|
+
const stateDir = join(epicsDir(input.workDir), epicId);
|
|
736
|
+
const planPath = join(stateDir, "plan.md");
|
|
737
|
+
const estimateOptions = estimatedUsageOptionsFromEnv(process.env);
|
|
738
|
+
const contextBudget = budget(process.env);
|
|
739
|
+
const runnerOptions = yield* epicRunnerOptions(options, process.env);
|
|
740
|
+
yield* runNode({
|
|
741
|
+
workDir: input.workDir,
|
|
742
|
+
workspace: input.workspace,
|
|
743
|
+
userPrompt: input.prompt,
|
|
744
|
+
coder,
|
|
745
|
+
reasoning,
|
|
746
|
+
reviewers: [asReadOnly(reasoning)],
|
|
747
|
+
environment: process.env,
|
|
748
|
+
...runnerOptions
|
|
749
|
+
}, (context) => Effect.gen(function* () {
|
|
750
|
+
const events = context.events;
|
|
751
|
+
const reasoningMeter = yield* makeEstimatedUsageMeter(context.reasoning, estimateOptions);
|
|
752
|
+
const guidance = yield* Effect.map(files.read(join(input.workDir, "CONTRIBUTING.md")), (text) => cap(text ?? "(no CONTRIBUTING.md in the target repository)", 24_000).text);
|
|
753
|
+
const store = makeStoryPlanStore(files);
|
|
754
|
+
const plan = yield* stage(events, "story plan", store
|
|
755
|
+
.recoverOrCreate(planPath, generateStoryPlan(reasoningMeter.service, events, input.prompt, epicId, guidance))
|
|
756
|
+
.pipe(Effect.flatMap(validateStoryPlan)));
|
|
757
|
+
yield* events.publish(Info.make({
|
|
758
|
+
message: `story plan: ${plan.stories.length} stories at ${planPath} (edit and rerun to re-plan)`
|
|
759
|
+
}));
|
|
760
|
+
if (flags.planOnly) {
|
|
761
|
+
return;
|
|
762
|
+
}
|
|
763
|
+
// Setup and gates run where the application is, in every checkout.
|
|
764
|
+
const appDir = yield* appDirFor(input.workDir, process.env);
|
|
765
|
+
if (appDir !== ".") {
|
|
766
|
+
yield* events.publish(Info.make({
|
|
767
|
+
message: `app dir: ${appDir} (setup and gates run there; LLM4TS_APP_DIR)`
|
|
768
|
+
}));
|
|
769
|
+
}
|
|
770
|
+
const commands = gateCommands(process.env, yield* appScripts(files, join(input.workDir, appDir)));
|
|
771
|
+
yield* events.publish(Info.make({
|
|
772
|
+
message: commands.length === 0
|
|
773
|
+
? "gates: none (the app defines none of typecheck, lint, test, build; set LLM4TS_GATES)"
|
|
774
|
+
: `gates: ${commands.map((command) => command.join(" ")).join(" · ")}`
|
|
775
|
+
}));
|
|
776
|
+
if (flags.land !== undefined) {
|
|
777
|
+
const landed = yield* landEpic(context, {
|
|
778
|
+
plan,
|
|
779
|
+
files,
|
|
780
|
+
stateDir,
|
|
781
|
+
target: flags.land,
|
|
782
|
+
keepWorktrees: flags.keepWorktrees,
|
|
783
|
+
gates: inAppDir(appDir, gatesIn(nodeProcessExecutor, events, commands)),
|
|
784
|
+
system: ["House rules of the target repository (CONTRIBUTING.md):", guidance].join("\n")
|
|
785
|
+
});
|
|
786
|
+
yield* events.publish(Info.make({
|
|
787
|
+
message: `epic ${plan.epicId}: landed on ${landed.target}${landed.conflicts.length === 0 ? "" : ` (${landed.conflicts.length} conflicted file(s) resolved in ${landed.rounds} round(s))`}`
|
|
788
|
+
}));
|
|
789
|
+
return;
|
|
790
|
+
}
|
|
791
|
+
// With a roster, the default is every coder slot it has; the flag caps it.
|
|
792
|
+
const concurrency = flags.concurrency ?? context.roster?.slots("coder") ?? 3;
|
|
793
|
+
if (context.roster === undefined && localServer !== undefined && concurrency > 1) {
|
|
794
|
+
yield* events.publish(Info.make({
|
|
795
|
+
message: `⚠ the coder is served by a local single-model server (${localServer}): it generates one ` +
|
|
796
|
+
`reply at a time, so ${concurrency} parallel coders queue behind each other and a queued ` +
|
|
797
|
+
'request is cut off after a few minutes ("terminated"). Prefer --concurrency 1, or give ' +
|
|
798
|
+
"the server parallel slots."
|
|
799
|
+
}));
|
|
800
|
+
}
|
|
801
|
+
const contextFor = context.contextFor;
|
|
802
|
+
if (contextFor === undefined) {
|
|
803
|
+
return yield* FlowAborted.make({
|
|
804
|
+
message: "this runner cannot rebind seats to a worktree (no contextFor)"
|
|
805
|
+
});
|
|
806
|
+
}
|
|
807
|
+
const appDirNote = appDir === "."
|
|
808
|
+
? []
|
|
809
|
+
: [
|
|
810
|
+
`The application lives in ${appDir}/ — its package.json, sources and tests; the gates run there.`
|
|
811
|
+
];
|
|
812
|
+
const gates = inAppDir(appDir, gatesIn(nodeProcessExecutor, events, commands));
|
|
813
|
+
const setupCommand = worktreeSetupCommand(process.env);
|
|
814
|
+
const healthUrl = serverHealthUrl(localServer, process.env);
|
|
815
|
+
const report = yield* implementStoriesFlow({ ...context, reasoning: reasoningMeter.service }, {
|
|
816
|
+
plan,
|
|
817
|
+
files,
|
|
818
|
+
stateDir,
|
|
819
|
+
worktreeRoot: worktreeRootFor(input.workDir, plan.epicId, process.env),
|
|
820
|
+
board: makeLocalBoardSync(files, stateDir, `Epic: ${plan.epicId}`),
|
|
821
|
+
contextFor: (workDir, contextOptions) => Effect.gen(function* () {
|
|
822
|
+
const rebound = yield* contextFor(workDir, contextOptions);
|
|
823
|
+
const coderMeter = yield* makeEstimatedUsageMeter(rebound.coder, estimateOptions);
|
|
824
|
+
const reviewMeter = yield* makeEstimatedUsageMeter(rebound.reasoning, estimateOptions);
|
|
825
|
+
const seats = {
|
|
826
|
+
context: {
|
|
827
|
+
...rebound,
|
|
828
|
+
coder: coderMeter.service,
|
|
829
|
+
reasoning: reviewMeter.service
|
|
830
|
+
},
|
|
831
|
+
totals: combineTotals(coderMeter.totals, reviewMeter.totals)
|
|
832
|
+
};
|
|
833
|
+
return seats;
|
|
834
|
+
}),
|
|
835
|
+
...(setupCommand === undefined
|
|
836
|
+
? {}
|
|
837
|
+
: {
|
|
838
|
+
setup: inAppDir(appDir, setupIn(nodeProcessExecutor, events, setupCommand)),
|
|
839
|
+
setupAgent: setupAgentEnabled(process.env)
|
|
840
|
+
}),
|
|
841
|
+
gates,
|
|
842
|
+
// With a roster, the judge and the verifier are leased per story,
|
|
843
|
+
// away from the executor coding it (ADR 0019).
|
|
844
|
+
judge: options.storyJudge({
|
|
845
|
+
plan,
|
|
846
|
+
budget: contextBudget,
|
|
847
|
+
reasoning: reasoningMeter.service,
|
|
848
|
+
events,
|
|
849
|
+
files,
|
|
850
|
+
houseRules: guidance
|
|
851
|
+
}),
|
|
852
|
+
verifyBlocked: (story, need, workDir, seats) => verifyBlockedOn(seats.context.roster?.forRole("verifier") ?? reasoningMeter.service, events, files, plan)(story, need, workDir),
|
|
853
|
+
// A roster waits for its own executors (the story's next lease
|
|
854
|
+
// does); without one, poll the single coder's engine.
|
|
855
|
+
awaitRecovery: context.roster === undefined
|
|
856
|
+
? awaitServer(healthUrl === undefined ? undefined : httpProbe(healthUrl), events)
|
|
857
|
+
: () => Effect.void,
|
|
858
|
+
system: (story) => Effect.succeed([
|
|
859
|
+
"House rules of the target repository (CONTRIBUTING.md):",
|
|
860
|
+
guidance,
|
|
861
|
+
"",
|
|
862
|
+
`Imitate the exemplar feature before inventing anything. Story id: ${story.id}.`,
|
|
863
|
+
...appDirNote
|
|
864
|
+
].join("\n")),
|
|
865
|
+
concurrency,
|
|
866
|
+
failFast: flags.failFast
|
|
867
|
+
});
|
|
868
|
+
yield* events.publish(Info.make({
|
|
869
|
+
message: `epic ${report.epicId}: ${report.count("done")} done, ${report.count("failed")} failed, ${report.count("waiting")} waiting — report at ${join(stateDir, "report.md")} (usage figures estimated)`
|
|
870
|
+
}));
|
|
871
|
+
}));
|
|
872
|
+
});
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
// The per-story "mergeable" decision as a blackboard ruleset (ADR 0020):
|
|
2
|
+
// a judge rule scores the four story dimensions with the Judgment service,
|
|
3
|
+
// decideRule turns each answer into act|caution|hold, and `bar` — the rule a
|
|
4
|
+
// company edits — posts whether the story may merge and the issues the coder
|
|
5
|
+
// gets back. Plugged into implementStoriesFlow's judge seam by boardStoryJudge.
|
|
6
|
+
import * as Effect from "effect/Effect";
|
|
7
|
+
import * as Layer from "effect/Layer";
|
|
8
|
+
import * as Schema from "effect/Schema";
|
|
9
|
+
import { makeKey } from "@llm4ts/core/blackboard/Fact";
|
|
10
|
+
import { all, derive, judge } from "@llm4ts/core/blackboard/Rule";
|
|
11
|
+
import { runRuleset } from "@llm4ts/core/blackboard/Run";
|
|
12
|
+
import { makeRuleset } from "@llm4ts/core/blackboard/Ruleset";
|
|
13
|
+
import { ProviderError } from "@llm4ts/core/Errors";
|
|
14
|
+
import { dimensionQuestion } from "@llm4ts/core/eval/Judge";
|
|
15
|
+
import { Judgment } from "@llm4ts/core/judgment/Judgment";
|
|
16
|
+
import { answerKey, decideRule, decisionKey, publishBlackboardRun, runErrorToFlowError } from "@llm4ts/flow/Blackboard";
|
|
17
|
+
import { cap } from "@llm4ts/flow/Context";
|
|
18
|
+
import { FlowLlmError } from "@llm4ts/flow/FlowError";
|
|
19
|
+
import { JudgmentObserved, publishJudgmentObserved } from "@llm4ts/flow/FlowEvents";
|
|
20
|
+
import { certaintyOf, decide, judgmentOf } from "@llm4ts/flow/Judgment";
|
|
21
|
+
import { ReviewIssue, ReviewResult } from "@llm4ts/flow/Review";
|
|
22
|
+
import { dependenciesOf } from "@llm4ts/flow/StoryPlan";
|
|
23
|
+
import { storyDimensions } from "./epic-stories.js";
|
|
24
|
+
export class StoryBrief extends Schema.Class("StoryBrief")({
|
|
25
|
+
id: Schema.String,
|
|
26
|
+
title: Schema.String,
|
|
27
|
+
description: Schema.String,
|
|
28
|
+
provides: Schema.Array(Schema.String),
|
|
29
|
+
owned: Schema.Array(Schema.String),
|
|
30
|
+
sharedReadOnly: Schema.Array(Schema.String),
|
|
31
|
+
dependencies: Schema.Array(Schema.Struct({ id: Schema.String, provides: Schema.Array(Schema.String) }))
|
|
32
|
+
}) {
|
|
33
|
+
}
|
|
34
|
+
export const storyBriefOf = (story, plan) => StoryBrief.make({
|
|
35
|
+
id: story.id,
|
|
36
|
+
title: story.title,
|
|
37
|
+
description: story.description,
|
|
38
|
+
provides: story.provides,
|
|
39
|
+
owned: story.owned,
|
|
40
|
+
sharedReadOnly: story.sharedReadOnly,
|
|
41
|
+
dependencies: plan === undefined
|
|
42
|
+
? []
|
|
43
|
+
: dependenciesOf(plan, story.id).flatMap((id) => {
|
|
44
|
+
const dependency = plan.story(id);
|
|
45
|
+
return dependency === undefined ? [] : [{ id, provides: dependency.provides }];
|
|
46
|
+
})
|
|
47
|
+
});
|
|
48
|
+
// ---- Facts ------------------------------------------------------------------
|
|
49
|
+
export const storyBrief = makeKey("story.brief", StoryBrief);
|
|
50
|
+
export const storyDiff = makeKey("story.diff", Schema.String);
|
|
51
|
+
export const houseRules = makeKey("story.houseRules", Schema.String);
|
|
52
|
+
export const mergeable = makeKey("story.mergeable", Schema.Boolean);
|
|
53
|
+
export const issues = makeKey("story.issues", Schema.Array(ReviewIssue));
|
|
54
|
+
/** One judged answer and one decision per story dimension, keyed by the dimension's name. */
|
|
55
|
+
const judged = (name) => answerKey(`judge.${name}`);
|
|
56
|
+
const decided = (name) => decisionKey(`decision.${name}`);
|
|
57
|
+
const [provides, scope, houseStyle, tests] = storyDimensions.map((dimension) => ({
|
|
58
|
+
dimension,
|
|
59
|
+
answer: judged(dimension.name),
|
|
60
|
+
decision: decided(dimension.name)
|
|
61
|
+
}));
|
|
62
|
+
if (provides === undefined ||
|
|
63
|
+
scope === undefined ||
|
|
64
|
+
houseStyle === undefined ||
|
|
65
|
+
tests === undefined) {
|
|
66
|
+
throw new Error("story board: storyDimensions must have four entries");
|
|
67
|
+
}
|
|
68
|
+
const dimensions = [provides, scope, houseStyle, tests];
|
|
69
|
+
export const dimensionNames = dimensions.map((d) => d.dimension.name);
|
|
70
|
+
/** Expected level (of 2) a dimension must reach; the company's bar. */
|
|
71
|
+
export const passingScore = 1.5;
|
|
72
|
+
/** The State the judge reads: the same words as the rubric judge, brief then diff. */
|
|
73
|
+
export const stateOf = (brief, diff, rules) => [
|
|
74
|
+
`Story: ${brief.title}`,
|
|
75
|
+
brief.description,
|
|
76
|
+
"",
|
|
77
|
+
`Provides: ${brief.provides.join(", ") || "(none)"}`,
|
|
78
|
+
`Owned paths: ${brief.owned.join(", ")}`,
|
|
79
|
+
`Shared read-only: ${brief.sharedReadOnly.join(", ") || "(none)"}`,
|
|
80
|
+
...(brief.dependencies.length === 0
|
|
81
|
+
? []
|
|
82
|
+
: [
|
|
83
|
+
"",
|
|
84
|
+
"Already merged, from the stories this one depends on (their declared interface):",
|
|
85
|
+
...brief.dependencies.map((d) => `- ${d.id}: ${d.provides.join("; ") || "(nothing declared)"}`),
|
|
86
|
+
"Using these exactly as declared (the same module, the same export) is correct and is",
|
|
87
|
+
"never a house-style or scope problem, even where the module is a fake transport."
|
|
88
|
+
]),
|
|
89
|
+
"",
|
|
90
|
+
"House rules:",
|
|
91
|
+
rules,
|
|
92
|
+
"",
|
|
93
|
+
"Diff:",
|
|
94
|
+
diff
|
|
95
|
+
].join("\n");
|
|
96
|
+
const levelDescription = (answer) => {
|
|
97
|
+
const rounded = String(Math.max(0, Math.min(2, Math.round(answer.score))));
|
|
98
|
+
const legend = answer.legend[rounded];
|
|
99
|
+
return typeof legend === "string" ? legend : JSON.stringify(legend ?? "");
|
|
100
|
+
};
|
|
101
|
+
// ---- The ruleset ------------------------------------------------------------
|
|
102
|
+
export const makeStoryBoard = (policy) => {
|
|
103
|
+
const storyJudge = judge({
|
|
104
|
+
name: "story-judge",
|
|
105
|
+
condition: all(storyBrief, storyDiff, houseRules),
|
|
106
|
+
produces: dimensions.map((d) => d.answer),
|
|
107
|
+
ask: ([brief, diff, rules]) => ({
|
|
108
|
+
state: stateOf(brief, diff, rules),
|
|
109
|
+
questions: Object.fromEntries(dimensions.map((d) => [d.dimension.name, dimensionQuestion(d.dimension)]))
|
|
110
|
+
}),
|
|
111
|
+
post: (result) => dimensions.flatMap((d) => {
|
|
112
|
+
const answer = result.answers[d.dimension.name];
|
|
113
|
+
return answer?.type === "score" ? [d.answer.of(answer)] : [];
|
|
114
|
+
})
|
|
115
|
+
});
|
|
116
|
+
const decisions = dimensions.map((d) => decideRule({
|
|
117
|
+
name: `decide-${d.dimension.name}`,
|
|
118
|
+
answer: d.answer,
|
|
119
|
+
decision: d.decision,
|
|
120
|
+
...(policy === undefined ? {} : { policy })
|
|
121
|
+
}));
|
|
122
|
+
const bar = derive({
|
|
123
|
+
name: "bar",
|
|
124
|
+
condition: all(provides.answer, scope.answer, houseStyle.answer, tests.answer, provides.decision, scope.decision, houseStyle.decision, tests.decision),
|
|
125
|
+
produces: [mergeable, issues],
|
|
126
|
+
derive: ([a1, a2, a3, a4, v1, v2, v3, v4]) => {
|
|
127
|
+
const found = [];
|
|
128
|
+
const answers = [a1, a2, a3, a4];
|
|
129
|
+
const verdicts = [v1, v2, v3, v4];
|
|
130
|
+
dimensions.forEach((d, index) => {
|
|
131
|
+
const answer = answers[index];
|
|
132
|
+
const verdict = verdicts[index];
|
|
133
|
+
if (answer === undefined || verdict === undefined || answer.type !== "score")
|
|
134
|
+
return;
|
|
135
|
+
if (answer.score < passingScore) {
|
|
136
|
+
found.push(ReviewIssue.make({
|
|
137
|
+
severity: "Critical",
|
|
138
|
+
title: `${d.dimension.name} scored ${answer.score.toFixed(1)}`,
|
|
139
|
+
description: levelDescription(answer)
|
|
140
|
+
}));
|
|
141
|
+
}
|
|
142
|
+
else if (verdict !== "act") {
|
|
143
|
+
found.push(ReviewIssue.make({
|
|
144
|
+
severity: "Critical",
|
|
145
|
+
title: `unsure about ${d.dimension.name}`,
|
|
146
|
+
description: `the judge's ${verdict} decision (certainty ${certaintyOf(answer).toFixed(2)}, support ${answer.support.toFixed(2)}) is below the bar for acting on it`
|
|
147
|
+
}));
|
|
148
|
+
}
|
|
149
|
+
});
|
|
150
|
+
return [mergeable.of(found.length === 0), issues.of(found)];
|
|
151
|
+
}
|
|
152
|
+
});
|
|
153
|
+
return makeRuleset({
|
|
154
|
+
name: "story-board",
|
|
155
|
+
imports: [storyBrief, storyDiff, houseRules],
|
|
156
|
+
exports: [mergeable, issues],
|
|
157
|
+
rules: [storyJudge, ...decisions, bar]
|
|
158
|
+
});
|
|
159
|
+
};
|
|
160
|
+
/**
|
|
161
|
+
* The round's error when the bar never fired: which dimensions went unscored
|
|
162
|
+
* and, when the backend itself failed, its message — the story loop reads an
|
|
163
|
+
* outage off that message and waits for recovery instead of failing the story.
|
|
164
|
+
*/
|
|
165
|
+
const unscored = (story, error) => {
|
|
166
|
+
const names = [
|
|
167
|
+
...new Set(error.missing
|
|
168
|
+
.flatMap((m) => m.waitingRules.flatMap((w) => w.missingKeys))
|
|
169
|
+
.filter((key) => key.startsWith("judge."))
|
|
170
|
+
.map((key) => key.slice("judge.".length)))
|
|
171
|
+
];
|
|
172
|
+
const because = error.failures.map((failure) => `${failure.tag}: ${failure.message}`);
|
|
173
|
+
return FlowLlmError.from(ProviderError.make({
|
|
174
|
+
message: `story-board: the judge could not score ${names.join(", ")} for ${story}${because.length === 0 ? "" : ` (${because.join("; ")})`}`
|
|
175
|
+
}));
|
|
176
|
+
};
|
|
177
|
+
/** The seam implementation: the ruleset built once, run per judge round. */
|
|
178
|
+
export const boardStoryJudge = (context) => Effect.map(makeStoryBoard(context.policy).pipe(Effect.mapError(runErrorToFlowError)), (board) => (story, diff, seats) => Effect.gen(function* () {
|
|
179
|
+
const judgment = seats.context.judgment ?? context.judgment;
|
|
180
|
+
const brief = storyBriefOf(story, context.plan);
|
|
181
|
+
const capped = cap(diff, context.budget).text;
|
|
182
|
+
const result = yield* runRuleset(board, [
|
|
183
|
+
storyBrief.of(brief),
|
|
184
|
+
storyDiff.of(capped),
|
|
185
|
+
houseRules.of(context.houseRules)
|
|
186
|
+
]).pipe(Effect.provide(Layer.succeed(Judgment, judgment)), Effect.mapError((error) => error._tag === "ExportsMissing"
|
|
187
|
+
? unscored(story.id, error)
|
|
188
|
+
: runErrorToFlowError(error)));
|
|
189
|
+
yield* publishBlackboardRun(context.events, "story-board", result);
|
|
190
|
+
const isMergeable = yield* result.board.get(mergeable).pipe(Effect.orDie);
|
|
191
|
+
const found = yield* result.board.get(issues).pipe(Effect.orDie);
|
|
192
|
+
const state = stateOf(brief, capped, context.houseRules);
|
|
193
|
+
for (const d of dimensions) {
|
|
194
|
+
yield* observe(context, judgment, d.dimension, state, result, isMergeable, d.answer);
|
|
195
|
+
}
|
|
196
|
+
return ReviewResult.make({
|
|
197
|
+
issues: found.map((issue) => ReviewIssue.make({ ...issue, title: `judge[${story.id}]: ${issue.title}` })),
|
|
198
|
+
summary: `story-board:${story.id}`
|
|
199
|
+
});
|
|
200
|
+
}));
|
|
201
|
+
const observe = (context, judgment, dimension, state, result, isMergeable, key) => Effect.gen(function* () {
|
|
202
|
+
// The bar read every answer, so a missing one after a successful run is a defect.
|
|
203
|
+
const answer = yield* result.board.get(key).pipe(Effect.orDie);
|
|
204
|
+
if (answer.type !== "score")
|
|
205
|
+
return;
|
|
206
|
+
yield* publishJudgmentObserved(context.events, JudgmentObserved.make({
|
|
207
|
+
consumer: "story-board",
|
|
208
|
+
key: dimension.name,
|
|
209
|
+
state,
|
|
210
|
+
question: dimensionQuestion(dimension),
|
|
211
|
+
answer,
|
|
212
|
+
judgmentIdentity: judgment.identity,
|
|
213
|
+
decision: decide(answer, context.policy),
|
|
214
|
+
certainty: certaintyOf(answer),
|
|
215
|
+
support: answer.support,
|
|
216
|
+
origin: answer.origin,
|
|
217
|
+
outcome: {
|
|
218
|
+
_tag: "StoryBoard",
|
|
219
|
+
dimension: dimension.name,
|
|
220
|
+
score: answer.score,
|
|
221
|
+
mergeable: isMergeable
|
|
222
|
+
},
|
|
223
|
+
mode: "act"
|
|
224
|
+
}));
|
|
225
|
+
});
|
|
226
|
+
/**
|
|
227
|
+
* The judge `epic-stories-board` passes to runEpicStories: the run's judgment
|
|
228
|
+
* seat (LLM4TS_JUDGMENT_PROVIDER, or one derived from the reasoning seat),
|
|
229
|
+
* the default policy. Validating six rules is microseconds, so the ruleset
|
|
230
|
+
* is built per call and the factory keeps the plain StoryJudge type.
|
|
231
|
+
*/
|
|
232
|
+
export const boardJudgeFactory = (context) => (story, diff, seats) => Effect.flatMap(boardStoryJudge({ ...context, judgment: judgmentOf(seats.context) }), (judge) => judge(story, diff, seats));
|