@llm4ts/flow 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +41 -0
- package/dist/AzureDevOpsTool.d.ts +60 -0
- package/dist/AzureDevOpsTool.d.ts.map +1 -0
- package/dist/AzureDevOpsTool.js +163 -0
- package/dist/AzureDevOpsTool.js.map +1 -0
- package/dist/Bench.d.ts +133 -0
- package/dist/Bench.d.ts.map +1 -0
- package/dist/Bench.js +272 -0
- package/dist/Bench.js.map +1 -0
- package/dist/BenchReport.d.ts +9 -0
- package/dist/BenchReport.d.ts.map +1 -0
- package/dist/BenchReport.js +360 -0
- package/dist/BenchReport.js.map +1 -0
- package/dist/CapabilityGuard.d.ts +6 -0
- package/dist/CapabilityGuard.d.ts.map +1 -0
- package/dist/CapabilityGuard.js +25 -0
- package/dist/CapabilityGuard.js.map +1 -0
- package/dist/Chat.d.ts +15 -0
- package/dist/Chat.d.ts.map +1 -0
- package/dist/Chat.js +36 -0
- package/dist/Chat.js.map +1 -0
- package/dist/Classified.d.ts +14 -0
- package/dist/Classified.d.ts.map +1 -0
- package/dist/Classified.js +48 -0
- package/dist/Classified.js.map +1 -0
- package/dist/CostLedger.d.ts +55 -0
- package/dist/CostLedger.d.ts.map +1 -0
- package/dist/CostLedger.js +126 -0
- package/dist/CostLedger.js.map +1 -0
- package/dist/CostTracker.d.ts +13 -0
- package/dist/CostTracker.d.ts.map +1 -0
- package/dist/CostTracker.js +116 -0
- package/dist/CostTracker.js.map +1 -0
- package/dist/Equiv.d.ts +163 -0
- package/dist/Equiv.d.ts.map +1 -0
- package/dist/Equiv.js +256 -0
- package/dist/Equiv.js.map +1 -0
- package/dist/EquivReport.d.ts +16 -0
- package/dist/EquivReport.d.ts.map +1 -0
- package/dist/EquivReport.js +77 -0
- package/dist/EquivReport.js.map +1 -0
- package/dist/Flow.d.ts +25 -0
- package/dist/Flow.d.ts.map +1 -0
- package/dist/Flow.js +39 -0
- package/dist/Flow.js.map +1 -0
- package/dist/FlowContext.d.ts +23 -0
- package/dist/FlowContext.d.ts.map +1 -0
- package/dist/FlowContext.js +4 -0
- package/dist/FlowContext.js.map +1 -0
- package/dist/FlowError.d.ts +80 -0
- package/dist/FlowError.d.ts.map +1 -0
- package/dist/FlowError.js +92 -0
- package/dist/FlowError.js.map +1 -0
- package/dist/FlowEvents.d.ts +101 -0
- package/dist/FlowEvents.d.ts.map +1 -0
- package/dist/FlowEvents.js +103 -0
- package/dist/FlowEvents.js.map +1 -0
- package/dist/FlowRecorder.d.ts +26 -0
- package/dist/FlowRecorder.d.ts.map +1 -0
- package/dist/FlowRecorder.js +74 -0
- package/dist/FlowRecorder.js.map +1 -0
- package/dist/GitHubTool.d.ts +53 -0
- package/dist/GitHubTool.d.ts.map +1 -0
- package/dist/GitHubTool.js +191 -0
- package/dist/GitHubTool.js.map +1 -0
- package/dist/GitTool.d.ts +46 -0
- package/dist/GitTool.d.ts.map +1 -0
- package/dist/GitTool.js +97 -0
- package/dist/GitTool.js.map +1 -0
- package/dist/Interaction.d.ts +21 -0
- package/dist/Interaction.d.ts.map +1 -0
- package/dist/Interaction.js +9 -0
- package/dist/Interaction.js.map +1 -0
- package/dist/McpServer.d.ts +18 -0
- package/dist/McpServer.d.ts.map +1 -0
- package/dist/McpServer.js +122 -0
- package/dist/McpServer.js.map +1 -0
- package/dist/Mermaid.d.ts +7 -0
- package/dist/Mermaid.d.ts.map +1 -0
- package/dist/Mermaid.js +98 -0
- package/dist/Mermaid.js.map +1 -0
- package/dist/Pack.d.ts +40 -0
- package/dist/Pack.d.ts.map +1 -0
- package/dist/Pack.js +141 -0
- package/dist/Pack.js.map +1 -0
- package/dist/Package.d.ts +2 -0
- package/dist/Package.d.ts.map +1 -0
- package/dist/Package.js +2 -0
- package/dist/Package.js.map +1 -0
- package/dist/Persistence.d.ts +32 -0
- package/dist/Persistence.d.ts.map +1 -0
- package/dist/Persistence.js +85 -0
- package/dist/Persistence.js.map +1 -0
- package/dist/Plan.d.ts +26 -0
- package/dist/Plan.d.ts.map +1 -0
- package/dist/Plan.js +101 -0
- package/dist/Plan.js.map +1 -0
- package/dist/PlanExecution.d.ts +7 -0
- package/dist/PlanExecution.d.ts.map +1 -0
- package/dist/PlanExecution.js +19 -0
- package/dist/PlanExecution.js.map +1 -0
- package/dist/Planner.d.ts +32 -0
- package/dist/Planner.d.ts.map +1 -0
- package/dist/Planner.js +97 -0
- package/dist/Planner.js.map +1 -0
- package/dist/PrSummary.d.ts +15 -0
- package/dist/PrSummary.d.ts.map +1 -0
- package/dist/PrSummary.js +29 -0
- package/dist/PrSummary.js.map +1 -0
- package/dist/PriceList.d.ts +11 -0
- package/dist/PriceList.d.ts.map +1 -0
- package/dist/PriceList.js +79 -0
- package/dist/PriceList.js.map +1 -0
- package/dist/Provenance.d.ts +27 -0
- package/dist/Provenance.d.ts.map +1 -0
- package/dist/Provenance.js +38 -0
- package/dist/Provenance.js.map +1 -0
- package/dist/Replay.d.ts +29 -0
- package/dist/Replay.d.ts.map +1 -0
- package/dist/Replay.js +181 -0
- package/dist/Replay.js.map +1 -0
- package/dist/Review.d.ts +71 -0
- package/dist/Review.d.ts.map +1 -0
- package/dist/Review.js +204 -0
- package/dist/Review.js.map +1 -0
- package/dist/ReviewCache.d.ts +6 -0
- package/dist/ReviewCache.d.ts.map +1 -0
- package/dist/ReviewCache.js +28 -0
- package/dist/ReviewCache.js.map +1 -0
- package/dist/Reviewer.d.ts +12 -0
- package/dist/Reviewer.d.ts.map +1 -0
- package/dist/Reviewer.js +30 -0
- package/dist/Reviewer.js.map +1 -0
- package/dist/SpecChecks.d.ts +17 -0
- package/dist/SpecChecks.d.ts.map +1 -0
- package/dist/SpecChecks.js +107 -0
- package/dist/SpecChecks.js.map +1 -0
- package/dist/Survey.d.ts +32 -0
- package/dist/Survey.d.ts.map +1 -0
- package/dist/Survey.js +119 -0
- package/dist/Survey.js.map +1 -0
- package/dist/TransientRetry.d.ts +16 -0
- package/dist/TransientRetry.d.ts.map +1 -0
- package/dist/TransientRetry.js +102 -0
- package/dist/TransientRetry.js.map +1 -0
- package/dist/Workspace.d.ts +39 -0
- package/dist/Workspace.d.ts.map +1 -0
- package/dist/Workspace.js +192 -0
- package/dist/Workspace.js.map +1 -0
- package/dist/WorkspaceLayout.d.ts +3 -0
- package/dist/WorkspaceLayout.d.ts.map +1 -0
- package/dist/WorkspaceLayout.js +19 -0
- package/dist/WorkspaceLayout.js.map +1 -0
- package/dist/WorkspaceTools.d.ts +4 -0
- package/dist/WorkspaceTools.d.ts.map +1 -0
- package/dist/WorkspaceTools.js +130 -0
- package/dist/WorkspaceTools.js.map +1 -0
- package/package.json +91 -0
- package/src/AzureDevOpsTool.ts +291 -0
- package/src/Bench.ts +340 -0
- package/src/BenchReport.ts +447 -0
- package/src/CapabilityGuard.ts +43 -0
- package/src/Chat.ts +60 -0
- package/src/Classified.ts +67 -0
- package/src/CostLedger.ts +171 -0
- package/src/CostTracker.ts +170 -0
- package/src/Equiv.ts +378 -0
- package/src/EquivReport.ts +92 -0
- package/src/Flow.ts +72 -0
- package/src/FlowContext.ts +23 -0
- package/src/FlowError.ts +112 -0
- package/src/FlowEvents.ts +147 -0
- package/src/FlowRecorder.ts +116 -0
- package/src/GitHubTool.ts +315 -0
- package/src/GitTool.ts +214 -0
- package/src/Interaction.ts +23 -0
- package/src/McpServer.ts +168 -0
- package/src/Mermaid.ts +112 -0
- package/src/Pack.ts +213 -0
- package/src/Package.ts +1 -0
- package/src/Persistence.ts +198 -0
- package/src/Plan.ts +119 -0
- package/src/PlanExecution.ts +46 -0
- package/src/Planner.ts +134 -0
- package/src/PrSummary.ts +37 -0
- package/src/PriceList.ts +91 -0
- package/src/Provenance.ts +85 -0
- package/src/Replay.ts +265 -0
- package/src/Review.ts +337 -0
- package/src/ReviewCache.ts +45 -0
- package/src/Reviewer.ts +31 -0
- package/src/SpecChecks.ts +143 -0
- package/src/Survey.ts +144 -0
- package/src/TransientRetry.ts +238 -0
- package/src/Workspace.ts +255 -0
- package/src/WorkspaceLayout.ts +22 -0
- package/src/WorkspaceTools.ts +188 -0
package/src/Equiv.ts
ADDED
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
import * as Effect from "effect/Effect"
|
|
2
|
+
import * as Schema from "effect/Schema"
|
|
3
|
+
import { Capabilities } from "@llm4ts/core/Capability"
|
|
4
|
+
import type { ProcessExecutorShape } from "@llm4ts/core/ProcessExecutor"
|
|
5
|
+
import { guarded } from "./CapabilityGuard.ts"
|
|
6
|
+
import {
|
|
7
|
+
PlanParseError,
|
|
8
|
+
ProcessError,
|
|
9
|
+
UnsupportedSchemaVersion,
|
|
10
|
+
type FlowError
|
|
11
|
+
} from "./FlowError.ts"
|
|
12
|
+
import type { FlowEventsShape } from "./FlowEvents.ts"
|
|
13
|
+
import type { ComparisonPolicy } from "./Pack.ts"
|
|
14
|
+
import type { PlainFileStoreShape } from "./Persistence.ts"
|
|
15
|
+
|
|
16
|
+
export const EquivTier = Schema.Literals(["generated", "captured"])
|
|
17
|
+
export type EquivTier = typeof EquivTier.Type
|
|
18
|
+
|
|
19
|
+
export const RecordObservation = Schema.Struct({
|
|
20
|
+
type: Schema.Literal("record"),
|
|
21
|
+
kind: Schema.String,
|
|
22
|
+
fields: Schema.Record(Schema.String, Schema.String)
|
|
23
|
+
})
|
|
24
|
+
export type RecordObservation = typeof RecordObservation.Type
|
|
25
|
+
|
|
26
|
+
export const DbMutationObservation = Schema.Struct({
|
|
27
|
+
type: Schema.Literal("db"),
|
|
28
|
+
table: Schema.String,
|
|
29
|
+
op: Schema.String,
|
|
30
|
+
key: Schema.Record(Schema.String, Schema.String),
|
|
31
|
+
set: Schema.Record(Schema.String, Schema.String)
|
|
32
|
+
})
|
|
33
|
+
export type DbMutationObservation = typeof DbMutationObservation.Type
|
|
34
|
+
|
|
35
|
+
export const MessageObservation = Schema.Struct({
|
|
36
|
+
type: Schema.Literal("message"),
|
|
37
|
+
topic: Schema.String,
|
|
38
|
+
key: Schema.String,
|
|
39
|
+
fields: Schema.Record(Schema.String, Schema.String)
|
|
40
|
+
})
|
|
41
|
+
export type MessageObservation = typeof MessageObservation.Type
|
|
42
|
+
|
|
43
|
+
export const EquivObservation = Schema.Union([
|
|
44
|
+
RecordObservation,
|
|
45
|
+
DbMutationObservation,
|
|
46
|
+
MessageObservation
|
|
47
|
+
])
|
|
48
|
+
export type EquivObservation = typeof EquivObservation.Type
|
|
49
|
+
|
|
50
|
+
export const Observations = Object.freeze({
|
|
51
|
+
record: (kind: string, fields: Readonly<Record<string, string>>): RecordObservation => ({
|
|
52
|
+
type: "record",
|
|
53
|
+
kind,
|
|
54
|
+
fields
|
|
55
|
+
}),
|
|
56
|
+
dbMutation: (
|
|
57
|
+
table: string,
|
|
58
|
+
op: string,
|
|
59
|
+
key: Readonly<Record<string, string>>,
|
|
60
|
+
set: Readonly<Record<string, string>>
|
|
61
|
+
): DbMutationObservation => ({ type: "db", table, op, key, set }),
|
|
62
|
+
message: (
|
|
63
|
+
topic: string,
|
|
64
|
+
key: string,
|
|
65
|
+
fields: Readonly<Record<string, string>>
|
|
66
|
+
): MessageObservation => ({ type: "message", topic, key, fields })
|
|
67
|
+
})
|
|
68
|
+
|
|
69
|
+
export class EquivVector extends Schema.Class<EquivVector>("EquivVector")({
|
|
70
|
+
schemaVersion: Schema.Int,
|
|
71
|
+
program: Schema.String,
|
|
72
|
+
id: Schema.String,
|
|
73
|
+
tier: EquivTier,
|
|
74
|
+
rules: Schema.Array(Schema.String),
|
|
75
|
+
inputs: Schema.Json,
|
|
76
|
+
observations: Schema.Array(EquivObservation)
|
|
77
|
+
}) {}
|
|
78
|
+
|
|
79
|
+
export const CurrentEquivSchema = 1
|
|
80
|
+
|
|
81
|
+
export class MissingObservation extends Schema.TaggedClass<MissingObservation>()("Missing", {
|
|
82
|
+
expected: EquivObservation
|
|
83
|
+
}) {}
|
|
84
|
+
|
|
85
|
+
export class UnexpectedObservation extends Schema.TaggedClass<UnexpectedObservation>()(
|
|
86
|
+
"Unexpected",
|
|
87
|
+
{
|
|
88
|
+
actual: EquivObservation
|
|
89
|
+
}
|
|
90
|
+
) {}
|
|
91
|
+
|
|
92
|
+
export class ObservationFieldDiff extends Schema.TaggedClass<ObservationFieldDiff>()("FieldDiff", {
|
|
93
|
+
at: Schema.String,
|
|
94
|
+
field: Schema.String,
|
|
95
|
+
expected: Schema.String,
|
|
96
|
+
actual: Schema.String
|
|
97
|
+
}) {}
|
|
98
|
+
|
|
99
|
+
export const EquivMismatch = Schema.Union([
|
|
100
|
+
MissingObservation,
|
|
101
|
+
UnexpectedObservation,
|
|
102
|
+
ObservationFieldDiff
|
|
103
|
+
])
|
|
104
|
+
export type EquivMismatch = typeof EquivMismatch.Type
|
|
105
|
+
|
|
106
|
+
const sortedEntries = (
|
|
107
|
+
record: Readonly<Record<string, string>>
|
|
108
|
+
): ReadonlyArray<readonly [string, string]> =>
|
|
109
|
+
Object.entries(record).sort(([left], [right]) => left.localeCompare(right))
|
|
110
|
+
|
|
111
|
+
const identityOf = (observation: EquivObservation): string => {
|
|
112
|
+
switch (observation.type) {
|
|
113
|
+
case "record":
|
|
114
|
+
return `record ${observation.kind}`
|
|
115
|
+
case "message":
|
|
116
|
+
return `message ${observation.topic} ${observation.key}`
|
|
117
|
+
case "db":
|
|
118
|
+
return `db ${observation.table} ${observation.op} ${sortedEntries(observation.key)
|
|
119
|
+
.map(([key, value]) => `${key}=${value}`)
|
|
120
|
+
.join(",")}`
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const fieldsOf = (observation: EquivObservation): Readonly<Record<string, string>> =>
|
|
125
|
+
observation.type === "db" ? observation.set : observation.fields
|
|
126
|
+
|
|
127
|
+
const canonical = (observation: EquivObservation): string =>
|
|
128
|
+
JSON.stringify({
|
|
129
|
+
...observation,
|
|
130
|
+
...(observation.type === "db"
|
|
131
|
+
? {
|
|
132
|
+
key: Object.fromEntries(sortedEntries(observation.key)),
|
|
133
|
+
set: Object.fromEntries(sortedEntries(observation.set))
|
|
134
|
+
}
|
|
135
|
+
: { fields: Object.fromEntries(sortedEntries(observation.fields)) })
|
|
136
|
+
})
|
|
137
|
+
|
|
138
|
+
const scrub = (observation: EquivObservation, ignore: ReadonlySet<string>): EquivObservation => {
|
|
139
|
+
const omit = (fields: Readonly<Record<string, string>>): Readonly<Record<string, string>> =>
|
|
140
|
+
Object.fromEntries(Object.entries(fields).filter(([key]) => !ignore.has(key)))
|
|
141
|
+
switch (observation.type) {
|
|
142
|
+
case "record":
|
|
143
|
+
return Observations.record(observation.kind, omit(observation.fields))
|
|
144
|
+
case "message":
|
|
145
|
+
return Observations.message(observation.topic, observation.key, omit(observation.fields))
|
|
146
|
+
case "db":
|
|
147
|
+
return Observations.dbMutation(
|
|
148
|
+
observation.table,
|
|
149
|
+
observation.op,
|
|
150
|
+
omit(observation.key),
|
|
151
|
+
omit(observation.set)
|
|
152
|
+
)
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
const fieldDiffs = (
|
|
157
|
+
expected: EquivObservation,
|
|
158
|
+
actual: EquivObservation
|
|
159
|
+
): ReadonlyArray<EquivMismatch> => {
|
|
160
|
+
const expectedFields = fieldsOf(expected)
|
|
161
|
+
const actualFields = fieldsOf(actual)
|
|
162
|
+
return [...new Set([...Object.keys(expectedFields), ...Object.keys(actualFields)])]
|
|
163
|
+
.sort()
|
|
164
|
+
.flatMap((field) =>
|
|
165
|
+
expectedFields[field] === actualFields[field]
|
|
166
|
+
? []
|
|
167
|
+
: [
|
|
168
|
+
ObservationFieldDiff.make({
|
|
169
|
+
at: identityOf(expected),
|
|
170
|
+
field,
|
|
171
|
+
expected: expectedFields[field] ?? "(absent)",
|
|
172
|
+
actual: actualFields[field] ?? "(absent)"
|
|
173
|
+
})
|
|
174
|
+
]
|
|
175
|
+
)
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const positionalDiff = (
|
|
179
|
+
expected: ReadonlyArray<EquivObservation>,
|
|
180
|
+
actual: ReadonlyArray<EquivObservation>
|
|
181
|
+
): ReadonlyArray<EquivMismatch> => {
|
|
182
|
+
const result: Array<EquivMismatch> = []
|
|
183
|
+
const maximum = Math.max(expected.length, actual.length)
|
|
184
|
+
for (let index = 0; index < maximum; index += 1) {
|
|
185
|
+
const expectedItem = expected[index]
|
|
186
|
+
const actualItem = actual[index]
|
|
187
|
+
if (expectedItem === undefined && actualItem !== undefined) {
|
|
188
|
+
result.push(UnexpectedObservation.make({ actual: actualItem }))
|
|
189
|
+
} else if (expectedItem !== undefined && actualItem === undefined) {
|
|
190
|
+
result.push(MissingObservation.make({ expected: expectedItem }))
|
|
191
|
+
} else if (expectedItem !== undefined && actualItem !== undefined) {
|
|
192
|
+
if (canonical(expectedItem) === canonical(actualItem)) {
|
|
193
|
+
continue
|
|
194
|
+
}
|
|
195
|
+
if (identityOf(expectedItem) === identityOf(actualItem)) {
|
|
196
|
+
result.push(...fieldDiffs(expectedItem, actualItem))
|
|
197
|
+
} else {
|
|
198
|
+
result.push(MissingObservation.make({ expected: expectedItem }))
|
|
199
|
+
result.push(UnexpectedObservation.make({ actual: actualItem }))
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return result
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
export const diffObservations = (
|
|
207
|
+
expected: ReadonlyArray<EquivObservation>,
|
|
208
|
+
actual: ReadonlyArray<EquivObservation>,
|
|
209
|
+
policy: ComparisonPolicy
|
|
210
|
+
): ReadonlyArray<EquivMismatch> => {
|
|
211
|
+
const expectedScrubbed = expected.map((item) => scrub(item, policy.ignore))
|
|
212
|
+
const actualScrubbed = actual.map((item) => scrub(item, policy.ignore))
|
|
213
|
+
if (policy.ordering === "Ordered") {
|
|
214
|
+
return positionalDiff(expectedScrubbed, actualScrubbed)
|
|
215
|
+
}
|
|
216
|
+
if (policy.ordering === "PerKey") {
|
|
217
|
+
const keys = [
|
|
218
|
+
...new Set([...expectedScrubbed.map(identityOf), ...actualScrubbed.map(identityOf)])
|
|
219
|
+
].sort()
|
|
220
|
+
return keys.flatMap((key) =>
|
|
221
|
+
positionalDiff(
|
|
222
|
+
expectedScrubbed.filter((item) => identityOf(item) === key),
|
|
223
|
+
actualScrubbed.filter((item) => identityOf(item) === key)
|
|
224
|
+
)
|
|
225
|
+
)
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const remainingActual = [...actualScrubbed]
|
|
229
|
+
const unmatchedExpected: Array<EquivObservation> = []
|
|
230
|
+
for (const item of expectedScrubbed) {
|
|
231
|
+
const index = remainingActual.findIndex((candidate) => canonical(candidate) === canonical(item))
|
|
232
|
+
if (index < 0) {
|
|
233
|
+
unmatchedExpected.push(item)
|
|
234
|
+
} else {
|
|
235
|
+
remainingActual.splice(index, 1)
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
const mismatches: Array<EquivMismatch> = []
|
|
239
|
+
const missing: Array<EquivObservation> = []
|
|
240
|
+
for (const item of unmatchedExpected) {
|
|
241
|
+
const index = remainingActual.findIndex(
|
|
242
|
+
(candidate) => identityOf(candidate) === identityOf(item)
|
|
243
|
+
)
|
|
244
|
+
const matching = index < 0 ? undefined : remainingActual[index]
|
|
245
|
+
if (matching === undefined) {
|
|
246
|
+
missing.push(item)
|
|
247
|
+
} else {
|
|
248
|
+
mismatches.push(...fieldDiffs(item, matching))
|
|
249
|
+
remainingActual.splice(index, 1)
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
return [
|
|
253
|
+
...mismatches,
|
|
254
|
+
...missing.map((expectedItem) => MissingObservation.make({ expected: expectedItem })),
|
|
255
|
+
...remainingActual.map((actualItem) => UnexpectedObservation.make({ actual: actualItem }))
|
|
256
|
+
]
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
export class ReplayedObserved extends Schema.TaggedClass<ReplayedObserved>()("Observed", {
|
|
260
|
+
actual: Schema.Array(EquivObservation)
|
|
261
|
+
}) {}
|
|
262
|
+
|
|
263
|
+
export class ReplayedCrashed extends Schema.TaggedClass<ReplayedCrashed>()("Crashed", {
|
|
264
|
+
exitCode: Schema.Int,
|
|
265
|
+
problem: Schema.String
|
|
266
|
+
}) {}
|
|
267
|
+
|
|
268
|
+
export const Replayed = Schema.Union([ReplayedObserved, ReplayedCrashed])
|
|
269
|
+
export type Replayed = typeof Replayed.Type
|
|
270
|
+
|
|
271
|
+
const vectorCodec = Schema.fromJsonString(EquivVector)
|
|
272
|
+
const observationsCodec = Schema.fromJsonString(Schema.Array(EquivObservation))
|
|
273
|
+
|
|
274
|
+
export const writeEquivVectors = Effect.fn("@llm4ts/flow/Equiv.writeVectors")(function* (
|
|
275
|
+
files: PlainFileStoreShape,
|
|
276
|
+
path: string,
|
|
277
|
+
vectors: ReadonlyArray<EquivVector>
|
|
278
|
+
): Effect.fn.Return<void, FlowError> {
|
|
279
|
+
const lines = yield* Effect.forEach(
|
|
280
|
+
vectors,
|
|
281
|
+
(vector) =>
|
|
282
|
+
Schema.encodeEffect(vectorCodec)(vector).pipe(
|
|
283
|
+
Effect.mapError((error) =>
|
|
284
|
+
PlanParseError.make({
|
|
285
|
+
message: `failed to encode equivalence vector: ${String(error)}`
|
|
286
|
+
})
|
|
287
|
+
)
|
|
288
|
+
),
|
|
289
|
+
{ concurrency: 1 }
|
|
290
|
+
)
|
|
291
|
+
yield* files.writeAtomic(path, `${lines.join("\n")}\n`)
|
|
292
|
+
})
|
|
293
|
+
|
|
294
|
+
export const readEquivVectors = Effect.fn("@llm4ts/flow/Equiv.readVectors")(function* (
|
|
295
|
+
files: PlainFileStoreShape,
|
|
296
|
+
path: string
|
|
297
|
+
): Effect.fn.Return<ReadonlyArray<EquivVector>, FlowError> {
|
|
298
|
+
const contents = yield* files.read(path)
|
|
299
|
+
if (contents === undefined) {
|
|
300
|
+
return []
|
|
301
|
+
}
|
|
302
|
+
const vectors = yield* Effect.forEach(
|
|
303
|
+
contents
|
|
304
|
+
.split(/\r?\n/u)
|
|
305
|
+
.map((line) => line.trim())
|
|
306
|
+
.filter((line) => line.length > 0),
|
|
307
|
+
(line, index) =>
|
|
308
|
+
Schema.decodeUnknownEffect(vectorCodec)(line).pipe(
|
|
309
|
+
Effect.mapError((error) =>
|
|
310
|
+
PlanParseError.make({
|
|
311
|
+
message: `malformed vector at ${path}:${index + 1}: ${String(error)}`
|
|
312
|
+
})
|
|
313
|
+
)
|
|
314
|
+
),
|
|
315
|
+
{ concurrency: 1 }
|
|
316
|
+
)
|
|
317
|
+
for (const vector of vectors) {
|
|
318
|
+
if (vector.schemaVersion !== CurrentEquivSchema) {
|
|
319
|
+
return yield* UnsupportedSchemaVersion.make({
|
|
320
|
+
path,
|
|
321
|
+
expected: CurrentEquivSchema,
|
|
322
|
+
actual: vector.schemaVersion
|
|
323
|
+
})
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
return vectors
|
|
327
|
+
})
|
|
328
|
+
|
|
329
|
+
export const replayEquivVector = Effect.fn("@llm4ts/flow/Equiv.replayVector")(function* (
|
|
330
|
+
process: ProcessExecutorShape,
|
|
331
|
+
events: FlowEventsShape,
|
|
332
|
+
command: ReadonlyArray<string>,
|
|
333
|
+
root: string,
|
|
334
|
+
vector: EquivVector
|
|
335
|
+
): Effect.fn.Return<Replayed, FlowError> {
|
|
336
|
+
const executable = command[0]
|
|
337
|
+
if (executable === undefined) {
|
|
338
|
+
return yield* ProcessError.make({
|
|
339
|
+
message: "replay",
|
|
340
|
+
detail: "empty replay command"
|
|
341
|
+
})
|
|
342
|
+
}
|
|
343
|
+
const input = yield* Schema.encodeEffect(vectorCodec)(vector).pipe(
|
|
344
|
+
Effect.mapError((error) =>
|
|
345
|
+
PlanParseError.make({
|
|
346
|
+
message: `failed to encode vector ${vector.program}/${vector.id}: ${String(error)}`
|
|
347
|
+
})
|
|
348
|
+
)
|
|
349
|
+
)
|
|
350
|
+
const result = yield* guarded(
|
|
351
|
+
Capabilities.Exec(executable),
|
|
352
|
+
"equivalence replay",
|
|
353
|
+
events,
|
|
354
|
+
process.runWithStdin(command, root, {}, input).pipe(
|
|
355
|
+
Effect.mapError((error) =>
|
|
356
|
+
ProcessError.make({
|
|
357
|
+
message: "equivalence replay",
|
|
358
|
+
detail: error.message
|
|
359
|
+
})
|
|
360
|
+
)
|
|
361
|
+
)
|
|
362
|
+
)
|
|
363
|
+
if (result.exitCode !== 0) {
|
|
364
|
+
return ReplayedCrashed.make({
|
|
365
|
+
exitCode: result.exitCode,
|
|
366
|
+
problem: [...result.stderr, ...result.stdout].join("\n")
|
|
367
|
+
})
|
|
368
|
+
}
|
|
369
|
+
const output = result.stdout.join("\n")
|
|
370
|
+
const actual = yield* Schema.decodeUnknownEffect(observationsCodec)(output).pipe(
|
|
371
|
+
Effect.mapError((error) =>
|
|
372
|
+
PlanParseError.make({
|
|
373
|
+
message: `replay output for ${vector.program}/${vector.id} is not an observation array: ${String(error)}`
|
|
374
|
+
})
|
|
375
|
+
)
|
|
376
|
+
)
|
|
377
|
+
return ReplayedObserved.make({ actual })
|
|
378
|
+
})
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import * as Schema from "effect/Schema"
|
|
2
|
+
import { EquivMismatch, type EquivObservation, EquivVector } from "./Equiv.ts"
|
|
3
|
+
|
|
4
|
+
export class VectorVerdict extends Schema.Class<VectorVerdict>("VectorVerdict")({
|
|
5
|
+
vector: EquivVector,
|
|
6
|
+
mismatches: Schema.Array(EquivMismatch)
|
|
7
|
+
}) {
|
|
8
|
+
get passed(): boolean {
|
|
9
|
+
return this.mismatches.length === 0
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface EquivVerdict {
|
|
14
|
+
readonly vector: EquivVector
|
|
15
|
+
readonly mismatches: ReadonlyArray<EquivMismatch>
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const observation = (value: EquivObservation): string => {
|
|
19
|
+
const pairs = (fields: Readonly<Record<string, string>>): string =>
|
|
20
|
+
Object.entries(fields)
|
|
21
|
+
.sort(([left], [right]) => left.localeCompare(right))
|
|
22
|
+
.map(([key, item]) => `${key}=${item}`)
|
|
23
|
+
.join(", ")
|
|
24
|
+
switch (value.type) {
|
|
25
|
+
case "record":
|
|
26
|
+
return `record ${value.kind} ${pairs(value.fields)}`
|
|
27
|
+
case "message":
|
|
28
|
+
return `message ${value.topic} ${value.key} ${pairs(value.fields)}`
|
|
29
|
+
case "db":
|
|
30
|
+
return `db ${value.table} ${value.op} ${pairs(value.key)} → ${pairs(value.set)}`
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const describe = (mismatch: EquivMismatch): string => {
|
|
35
|
+
switch (mismatch._tag) {
|
|
36
|
+
case "FieldDiff":
|
|
37
|
+
return `- ${mismatch.at}: \`${mismatch.field}\` expected ${mismatch.expected}, actual ${mismatch.actual}`
|
|
38
|
+
case "Missing":
|
|
39
|
+
return `- missing: ${observation(mismatch.expected)}`
|
|
40
|
+
case "Unexpected":
|
|
41
|
+
return `- unexpected: ${observation(mismatch.actual)}`
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export const renderEquivReport = (
|
|
46
|
+
verdicts: ReadonlyArray<EquivVerdict>,
|
|
47
|
+
allRules: ReadonlyArray<string>
|
|
48
|
+
): string => {
|
|
49
|
+
const rows = allRules.map((rule) => {
|
|
50
|
+
const exercising = verdicts.filter(({ vector }) => vector.rules.includes(rule))
|
|
51
|
+
const generated = exercising.filter(({ vector }) => vector.tier === "generated")
|
|
52
|
+
const captured = exercising.filter(({ vector }) => vector.tier === "captured")
|
|
53
|
+
const cell = (values: ReadonlyArray<EquivVerdict>): string =>
|
|
54
|
+
values.length === 0
|
|
55
|
+
? "—"
|
|
56
|
+
: `${values.filter(({ mismatches }) => mismatches.length === 0).length}/${values.length}`
|
|
57
|
+
const status =
|
|
58
|
+
exercising.length === 0
|
|
59
|
+
? "UNEXERCISED"
|
|
60
|
+
: exercising.some(({ mismatches }) => mismatches.length > 0)
|
|
61
|
+
? "FAILING"
|
|
62
|
+
: captured.length > 0
|
|
63
|
+
? "proven (captured)"
|
|
64
|
+
: "proven (generated)"
|
|
65
|
+
return `| ${rule} | ${cell(generated)} | ${cell(captured)} | ${status} |`
|
|
66
|
+
})
|
|
67
|
+
const failing = verdicts.filter(({ mismatches }) => mismatches.length > 0)
|
|
68
|
+
const failures =
|
|
69
|
+
failing.length === 0
|
|
70
|
+
? []
|
|
71
|
+
: [
|
|
72
|
+
"",
|
|
73
|
+
"## Failing vectors",
|
|
74
|
+
...failing.flatMap(({ vector, mismatches }) => [
|
|
75
|
+
"",
|
|
76
|
+
`### ${vector.program} ${vector.id} (${vector.tier})`,
|
|
77
|
+
...mismatches.map(describe)
|
|
78
|
+
])
|
|
79
|
+
]
|
|
80
|
+
return [
|
|
81
|
+
"# Equivalence report",
|
|
82
|
+
"",
|
|
83
|
+
"| Rule | Generated | Captured | Status |",
|
|
84
|
+
"| ---- | --------- | -------- | ------ |",
|
|
85
|
+
...rows,
|
|
86
|
+
...failures,
|
|
87
|
+
"",
|
|
88
|
+
"Generated vectors prove spec conformance; captured vectors prove behavioural equivalence.",
|
|
89
|
+
"The two tiers are reported separately and never summed.",
|
|
90
|
+
""
|
|
91
|
+
].join("\n")
|
|
92
|
+
}
|
package/src/Flow.ts
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import * as Effect from "effect/Effect"
|
|
2
|
+
import type { LlmServiceShape } from "@llm4ts/core/LlmService"
|
|
3
|
+
import { collect } from "@llm4ts/core/Streaming"
|
|
4
|
+
import { makeChat } from "./Chat.ts"
|
|
5
|
+
import type { FlowContextShape } from "./FlowContext.ts"
|
|
6
|
+
import { FlowLlmError, type FlowError } from "./FlowError.ts"
|
|
7
|
+
import { AssistantMessage, type FlowEventsShape } from "./FlowEvents.ts"
|
|
8
|
+
import type { Plan, Task } from "./Plan.ts"
|
|
9
|
+
import type { PlanStoreShape } from "./Persistence.ts"
|
|
10
|
+
import { implementTaskLoop, stage } from "./PlanExecution.ts"
|
|
11
|
+
import { minimalReviewers, reviewAndFixLoop, type ReviewResult } from "./Review.ts"
|
|
12
|
+
import type { Reviewer } from "./Reviewer.ts"
|
|
13
|
+
|
|
14
|
+
export const flowReviewer = (context: FlowContextShape): LlmServiceShape =>
|
|
15
|
+
context.reviewers[0] ?? context.reasoning
|
|
16
|
+
|
|
17
|
+
export const completeAndPublish = Effect.fn("@llm4ts/flow/Flow.completeAndPublish")(function* (
|
|
18
|
+
service: LlmServiceShape,
|
|
19
|
+
events: FlowEventsShape,
|
|
20
|
+
prompt: string
|
|
21
|
+
): Effect.fn.Return<string, FlowLlmError> {
|
|
22
|
+
const response = yield* collect(service.executeStream(prompt)).pipe(
|
|
23
|
+
Effect.mapError(FlowLlmError.from)
|
|
24
|
+
)
|
|
25
|
+
yield* events.publish(AssistantMessage.make({ text: response.content }))
|
|
26
|
+
return response.content
|
|
27
|
+
})
|
|
28
|
+
|
|
29
|
+
export interface ImplementPlanOptions {
|
|
30
|
+
readonly store: PlanStoreShape
|
|
31
|
+
readonly planPath: string
|
|
32
|
+
readonly plan: Effect.Effect<Plan, FlowError>
|
|
33
|
+
readonly system?: string
|
|
34
|
+
readonly reviewers?: ReadonlyArray<Reviewer>
|
|
35
|
+
readonly commitMessage?: (plan: Plan, task: Task) => string
|
|
36
|
+
readonly checkoutBranch?: boolean
|
|
37
|
+
readonly maxRounds?: number
|
|
38
|
+
readonly lint?: Effect.Effect<ReviewResult, FlowError>
|
|
39
|
+
readonly format?: Effect.Effect<void, FlowError>
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const defaultCommitMessage = (plan: Plan, task: Task): string => `${plan.epicId}: ${task.title}`
|
|
43
|
+
|
|
44
|
+
export const implementPlanFlow = Effect.fn("@llm4ts/flow/Flow.implementPlan")(function* (
|
|
45
|
+
context: FlowContextShape,
|
|
46
|
+
options: ImplementPlanOptions
|
|
47
|
+
): Effect.fn.Return<Plan, FlowError> {
|
|
48
|
+
const plan = yield* options.store.recoverOrCreate(options.planPath, options.plan)
|
|
49
|
+
if (options.checkoutBranch !== false) {
|
|
50
|
+
yield* stage(context.events, "branch", context.git.checkoutOrCreate(plan.epicId))
|
|
51
|
+
}
|
|
52
|
+
const coder = yield* makeChat(context.coder, {
|
|
53
|
+
...(options.system === undefined ? {} : { system: options.system })
|
|
54
|
+
})
|
|
55
|
+
return yield* implementTaskLoop(options.store, context.events, options.planPath, plan, (task) =>
|
|
56
|
+
Effect.gen(function* () {
|
|
57
|
+
yield* coder.ask(plan.taskPrompt(task))
|
|
58
|
+
yield* reviewAndFixLoop({
|
|
59
|
+
reviewers: options.reviewers ?? minimalReviewers,
|
|
60
|
+
reviewerService: flowReviewer(context),
|
|
61
|
+
coder,
|
|
62
|
+
taskTitle: task.title,
|
|
63
|
+
currentDiff: context.git.diff,
|
|
64
|
+
events: context.events,
|
|
65
|
+
...(options.maxRounds === undefined ? {} : { maxRounds: options.maxRounds }),
|
|
66
|
+
...(options.lint === undefined ? {} : { lint: options.lint }),
|
|
67
|
+
...(options.format === undefined ? {} : { format: options.format })
|
|
68
|
+
})
|
|
69
|
+
yield* context.git.commitAll((options.commitMessage ?? defaultCommitMessage)(plan, task))
|
|
70
|
+
})
|
|
71
|
+
)
|
|
72
|
+
})
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import * as Context from "effect/Context"
|
|
2
|
+
import type { ConnectorCapabilities } from "@llm4ts/core/Models"
|
|
3
|
+
import type { LlmServiceShape } from "@llm4ts/core/LlmService"
|
|
4
|
+
import type { FlowEventsShape } from "./FlowEvents.ts"
|
|
5
|
+
import type { GitToolShape } from "./GitTool.ts"
|
|
6
|
+
import type { GitHubToolShape } from "./GitHubTool.ts"
|
|
7
|
+
|
|
8
|
+
export interface FlowContextShape {
|
|
9
|
+
readonly reasoning: LlmServiceShape
|
|
10
|
+
readonly coder: LlmServiceShape
|
|
11
|
+
readonly git: GitToolShape
|
|
12
|
+
readonly hosting: GitHubToolShape
|
|
13
|
+
readonly events: FlowEventsShape
|
|
14
|
+
readonly reviewers: ReadonlyArray<LlmServiceShape>
|
|
15
|
+
readonly coderCapabilities: ConnectorCapabilities
|
|
16
|
+
readonly userPrompt: string
|
|
17
|
+
readonly workDir: string
|
|
18
|
+
readonly workspace: string
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export class FlowContext extends Context.Service<FlowContext, FlowContextShape>()(
|
|
22
|
+
"@llm4ts/flow/FlowContext"
|
|
23
|
+
) {}
|
package/src/FlowError.ts
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import * as Schema from "effect/Schema"
|
|
2
|
+
import { Capability } from "@llm4ts/core/Capability"
|
|
3
|
+
import { LlmError } from "@llm4ts/core/Errors"
|
|
4
|
+
|
|
5
|
+
export class PersistenceError extends Schema.TaggedErrorClass<PersistenceError>()("Persistence", {
|
|
6
|
+
message: Schema.String,
|
|
7
|
+
cause: Schema.optionalKey(Schema.Defect())
|
|
8
|
+
}) {}
|
|
9
|
+
|
|
10
|
+
export class PlanParseError extends Schema.TaggedErrorClass<PlanParseError>()("PlanParse", {
|
|
11
|
+
message: Schema.String
|
|
12
|
+
}) {}
|
|
13
|
+
|
|
14
|
+
export class UnsupportedSchemaVersion extends Schema.TaggedErrorClass<UnsupportedSchemaVersion>()(
|
|
15
|
+
"UnsupportedSchemaVersion",
|
|
16
|
+
{
|
|
17
|
+
path: Schema.String,
|
|
18
|
+
expected: Schema.Int,
|
|
19
|
+
actual: Schema.Int
|
|
20
|
+
}
|
|
21
|
+
) {
|
|
22
|
+
get message(): string {
|
|
23
|
+
return `unsupported schema version in ${this.path}: expected ${this.expected}, received ${this.actual}`
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export class WorkspacePathError extends Schema.TaggedErrorClass<WorkspacePathError>()(
|
|
28
|
+
"WorkspacePath",
|
|
29
|
+
{
|
|
30
|
+
path: Schema.String,
|
|
31
|
+
message: Schema.String
|
|
32
|
+
}
|
|
33
|
+
) {}
|
|
34
|
+
|
|
35
|
+
export class WorkspaceLimitError extends Schema.TaggedErrorClass<WorkspaceLimitError>()(
|
|
36
|
+
"WorkspaceLimit",
|
|
37
|
+
{
|
|
38
|
+
operation: Schema.String,
|
|
39
|
+
limit: Schema.Int,
|
|
40
|
+
actual: Schema.Int
|
|
41
|
+
}
|
|
42
|
+
) {
|
|
43
|
+
get message(): string {
|
|
44
|
+
return `${this.operation} exceeded limit ${this.limit}; received ${this.actual}`
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export class WorkspaceIoError extends Schema.TaggedErrorClass<WorkspaceIoError>()("WorkspaceIo", {
|
|
49
|
+
operation: Schema.String,
|
|
50
|
+
path: Schema.String,
|
|
51
|
+
message: Schema.String,
|
|
52
|
+
cause: Schema.optionalKey(Schema.Defect())
|
|
53
|
+
}) {}
|
|
54
|
+
|
|
55
|
+
export class FlowAborted extends Schema.TaggedErrorClass<FlowAborted>()("Aborted", {
|
|
56
|
+
message: Schema.String
|
|
57
|
+
}) {}
|
|
58
|
+
|
|
59
|
+
export class ProcessError extends Schema.TaggedErrorClass<ProcessError>()("Process", {
|
|
60
|
+
message: Schema.String,
|
|
61
|
+
detail: Schema.String
|
|
62
|
+
}) {}
|
|
63
|
+
|
|
64
|
+
export class FlowLlmError extends Schema.TaggedErrorClass<FlowLlmError>()("Llm", {
|
|
65
|
+
message: Schema.String,
|
|
66
|
+
cause: Schema.optionalKey(LlmError)
|
|
67
|
+
}) {
|
|
68
|
+
static readonly from = (cause?: typeof LlmError.Type): FlowLlmError =>
|
|
69
|
+
FlowLlmError.make({
|
|
70
|
+
message: cause?.message ?? "LLM request failed",
|
|
71
|
+
...(cause === undefined ? {} : { cause })
|
|
72
|
+
})
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export class FlowCapabilityDenied extends Schema.TaggedErrorClass<FlowCapabilityDenied>()(
|
|
76
|
+
"CapabilityDenied",
|
|
77
|
+
{
|
|
78
|
+
capability: Capability,
|
|
79
|
+
operation: Schema.String
|
|
80
|
+
}
|
|
81
|
+
) {
|
|
82
|
+
get message(): string {
|
|
83
|
+
const capability =
|
|
84
|
+
this.capability._tag === "Exec" ? `Exec(${this.capability.command})` : this.capability._tag
|
|
85
|
+
return `capability ${capability} denied for ${this.operation}`
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export class BudgetExceeded extends Schema.TaggedErrorClass<BudgetExceeded>()("BudgetExceeded", {
|
|
90
|
+
metric: Schema.Literals(["tokens", "costUsd"]),
|
|
91
|
+
limit: Schema.Number,
|
|
92
|
+
actual: Schema.Number
|
|
93
|
+
}) {
|
|
94
|
+
get message(): string {
|
|
95
|
+
return `${this.metric} budget exceeded: limit ${this.limit}, actual ${this.actual}`
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export const FlowError = Schema.Union([
|
|
100
|
+
PersistenceError,
|
|
101
|
+
PlanParseError,
|
|
102
|
+
UnsupportedSchemaVersion,
|
|
103
|
+
WorkspacePathError,
|
|
104
|
+
WorkspaceLimitError,
|
|
105
|
+
WorkspaceIoError,
|
|
106
|
+
FlowAborted,
|
|
107
|
+
ProcessError,
|
|
108
|
+
FlowLlmError,
|
|
109
|
+
FlowCapabilityDenied,
|
|
110
|
+
BudgetExceeded
|
|
111
|
+
])
|
|
112
|
+
export type FlowError = typeof FlowError.Type
|