@llm4ts/shell 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Cli.d.ts +2 -2
- package/dist/Cli.js +3 -3
- package/dist/Cli.js.map +1 -1
- package/dist/FlowCatalog.d.ts.map +1 -1
- package/dist/FlowCatalog.js +6 -2
- package/dist/FlowCatalog.js.map +1 -1
- package/flows/implement.js +29 -0
- package/flows/issue-pr.js +71 -0
- package/flows/judge-suite.js +48 -0
- package/flows/local.js +40 -0
- package/flows/modernize-bench.js +264 -0
- package/flows/modernize-extract.js +455 -0
- package/flows/modernize-implement.js +249 -0
- package/flows/modernize-review.js +237 -0
- package/flows/modernize-seed.js +189 -0
- package/flows/modernize-survey.js +0 -0
- package/flows/modernize-verify.js +435 -0
- package/flows/sdd.js +114 -0
- package/package.json +5 -5
- package/src/Cli.ts +3 -3
- package/src/FlowCatalog.ts +8 -2
- package/flows/implement.ts +0 -38
- package/flows/issue-pr.ts +0 -108
- package/flows/judge-suite.ts +0 -63
- package/flows/local.ts +0 -59
- package/flows/modernize-bench.ts +0 -368
- package/flows/modernize-extract.ts +0 -642
- package/flows/modernize-implement.ts +0 -340
- package/flows/modernize-review.ts +0 -351
- package/flows/modernize-seed.ts +0 -266
- package/flows/modernize-survey.ts +0 -0
- package/flows/modernize-verify.ts +0 -618
- package/flows/sdd.ts +0 -181
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
// Legacy modernization phase 2: seed the target repository from the APPROVED spec pack.
|
|
2
|
+
//
|
|
3
|
+
// Runs rooted at the TARGET repository (`--repo <target>`), reading the spec
|
|
4
|
+
// pack out of the legacy repository (`LLM4TS_LEGACY_REPO`). Deliberately
|
|
5
|
+
// deterministic — no LLM calls:
|
|
6
|
+
//
|
|
7
|
+
// 1. Refuses to run until a human has flipped `- [x] Approved` in the legacy
|
|
8
|
+
// repo's docs/modernization/README.md (extract writes it unchecked).
|
|
9
|
+
// 2. An empty target gets the pack's scaffold; a non-empty one is used as-is.
|
|
10
|
+
// 3. Specs, traceability, mapping, and rules.txt land in the pack's
|
|
11
|
+
// specs-dir; .feature files land in its features-dir. Legacy SOURCE never
|
|
12
|
+
// crosses — that is the clean-room wall the later phases enforce.
|
|
13
|
+
// 4. The proposed plan is re-parsed (a hard validation) and materialized at
|
|
14
|
+
// docs/modernization/plan.md for modernize-implement to resume.
|
|
15
|
+
// 5. The provenance manifest (docs/modernization/provenance.json) records the
|
|
16
|
+
// clean-room receipt: spec file hashes, gate verdict digests, the pack,
|
|
17
|
+
// the llm4ts version, the extraction seats, and LLM4TS_APPROVER when set.
|
|
18
|
+
//
|
|
19
|
+
// Run: LLM4TS_LEGACY_REPO=~/estates/meridian-legacy \
|
|
20
|
+
// modernize-seed --repo ~/services/meridian-transfers
|
|
21
|
+
import { join } from "node:path";
|
|
22
|
+
import * as Effect from "effect/Effect";
|
|
23
|
+
import * as Schema from "effect/Schema";
|
|
24
|
+
import { FlowAborted, PersistenceError } from "@llm4ts/flow/FlowError";
|
|
25
|
+
import { Info } from "@llm4ts/flow/FlowEvents";
|
|
26
|
+
import { packageVersion } from "@llm4ts/flow/Package";
|
|
27
|
+
import { parsePlan } from "@llm4ts/flow/Plan";
|
|
28
|
+
import { stage } from "@llm4ts/flow/PlanExecution";
|
|
29
|
+
import { Provenance, makeProvenanceStore } from "@llm4ts/flow/Provenance";
|
|
30
|
+
import { matchingFiles } from "@llm4ts/flow/SpecChecks";
|
|
31
|
+
import { requireApproval } from "@llm4ts/modernize/Approval";
|
|
32
|
+
import { coderFromEnv } from "@llm4ts/runner/Connectors";
|
|
33
|
+
import { resolveFlowInput } from "@llm4ts/runner/FlowArgs";
|
|
34
|
+
import { runFlowMain, runNode } from "@llm4ts/runner/FlowRunner";
|
|
35
|
+
import { nodePlainFileStore } from "@llm4ts/runner/NodePlainFileStore";
|
|
36
|
+
import { makeNodeWorkspace } from "@llm4ts/runner/NodeWorkspace";
|
|
37
|
+
import { openPack } from "@llm4ts/runner/Packs";
|
|
38
|
+
const ModDir = "docs/modernization";
|
|
39
|
+
const skipped = new Set([".git", "target", "node_modules", "dist"]);
|
|
40
|
+
const Seats = Schema.Record(Schema.String, Schema.String);
|
|
41
|
+
const isLitter = (path) => path.split("/").some((segment) => skipped.has(segment));
|
|
42
|
+
/** Copies every text file under `from` into `into`, skipping build and VCS litter. */
|
|
43
|
+
const copyTree = Effect.fn("modernize-seed.copyTree")(function* (source, target, from, into) {
|
|
44
|
+
const paths = yield* source.discover(`${from}/**`).pipe(Effect.orElseSucceed(() => []));
|
|
45
|
+
let copied = 0;
|
|
46
|
+
for (const path of paths) {
|
|
47
|
+
if (isLitter(path)) {
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
const contents = yield* source.read(path).pipe(Effect.orElseSucceed(() => undefined));
|
|
51
|
+
if (contents === undefined) {
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
yield* target.write(join(into, path.slice(from.length + 1)), contents);
|
|
55
|
+
copied += 1;
|
|
56
|
+
}
|
|
57
|
+
return copied;
|
|
58
|
+
});
|
|
59
|
+
/** Copies one file when it exists; false when the source is absent. */
|
|
60
|
+
const copyFile = Effect.fn("modernize-seed.copyFile")(function* (source, target, from, into) {
|
|
61
|
+
const contents = yield* source.read(from).pipe(Effect.orElseSucceed(() => undefined));
|
|
62
|
+
if (contents === undefined) {
|
|
63
|
+
return false;
|
|
64
|
+
}
|
|
65
|
+
yield* target.write(into, contents);
|
|
66
|
+
return true;
|
|
67
|
+
});
|
|
68
|
+
const program = Effect.gen(function* () {
|
|
69
|
+
const input = yield* resolveFlowInput("Seed the target repository from the approved spec pack");
|
|
70
|
+
const legacyRepo = process.env.LLM4TS_LEGACY_REPO?.trim();
|
|
71
|
+
const files = nodePlainFileStore;
|
|
72
|
+
yield* runNode({
|
|
73
|
+
workDir: input.workDir,
|
|
74
|
+
workspace: input.workspace,
|
|
75
|
+
userPrompt: input.prompt,
|
|
76
|
+
// This phase makes no model call; the seat is wired only because the
|
|
77
|
+
// runner composes one context shape for every flow.
|
|
78
|
+
coder: coderFromEnv(process.env),
|
|
79
|
+
environment: process.env
|
|
80
|
+
}, (context) => Effect.gen(function* () {
|
|
81
|
+
if (legacyRepo === undefined || legacyRepo.length === 0) {
|
|
82
|
+
return yield* FlowAborted.make({
|
|
83
|
+
message: "set LLM4TS_LEGACY_REPO to the legacy repository holding the approved spec pack"
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
const legacy = yield* makeNodeWorkspace(legacyRepo);
|
|
87
|
+
const target = yield* makeNodeWorkspace(input.workDir);
|
|
88
|
+
const opened = yield* stage(context.events, "pack", openPack({
|
|
89
|
+
environment: process.env,
|
|
90
|
+
launchDir: input.workspace,
|
|
91
|
+
flowDir: import.meta.dirname
|
|
92
|
+
}));
|
|
93
|
+
const pack = opened.pack;
|
|
94
|
+
const specPackRoot = join(legacyRepo, ModDir);
|
|
95
|
+
yield* stage(context.events, "approval", requireApproval(files, join(specPackRoot, "README.md")));
|
|
96
|
+
yield* stage(context.events, "scaffold", Effect.gen(function* () {
|
|
97
|
+
const existing = (yield* target.discover().pipe(Effect.orElseSucceed(() => []))).filter((path) => !path.startsWith(".git/"));
|
|
98
|
+
if (existing.length > 0) {
|
|
99
|
+
yield* context.events.publish(Info.make({ message: "target repo is not empty — using it as-is" }));
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
if (pack.scaffold === undefined) {
|
|
103
|
+
return yield* FlowAborted.make({
|
|
104
|
+
message: `target repo is empty and pack '${pack.name}' has no scaffold`
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
// The scaffold path is relative to the pack directory.
|
|
108
|
+
const scaffoldRoot = join(pack.dir, pack.scaffold);
|
|
109
|
+
const scaffold = yield* makeNodeWorkspace(join(opened.workspace.root, scaffoldRoot));
|
|
110
|
+
const paths = (yield* scaffold.discover()).filter((path) => !isLitter(path));
|
|
111
|
+
for (const path of paths) {
|
|
112
|
+
yield* target.write(path, yield* scaffold.read(path));
|
|
113
|
+
}
|
|
114
|
+
yield* context.events.publish(Info.make({ message: `scaffolded ${paths.length} file(s) from ${scaffoldRoot}` }));
|
|
115
|
+
}));
|
|
116
|
+
const seeded = yield* stage(context.events, "seed", Effect.gen(function* () {
|
|
117
|
+
const specs = yield* copyTree(legacy, target, `${ModDir}/specs`, pack.specsDir);
|
|
118
|
+
// A spec pack that contributes no specs means the wrong legacy repo,
|
|
119
|
+
// or an extraction that never wrote any. Seeding an empty target and
|
|
120
|
+
// reporting success would push that discovery minutes downstream.
|
|
121
|
+
if (specs === 0) {
|
|
122
|
+
return yield* FlowAborted.make({
|
|
123
|
+
message: `no specs found under ${legacyRepo}/${ModDir}/specs — ` +
|
|
124
|
+
"check LLM4TS_LEGACY_REPO and that extraction wrote its spec pack"
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
const features = yield* copyTree(legacy, target, `${ModDir}/features`, pack.featuresDir);
|
|
128
|
+
for (const index of ["traceability.md", "mapping.md", "rules.txt"]) {
|
|
129
|
+
yield* copyFile(legacy, target, `${ModDir}/${index}`, join(pack.specsDir, index));
|
|
130
|
+
}
|
|
131
|
+
return specs + features;
|
|
132
|
+
}));
|
|
133
|
+
const plan = yield* stage(context.events, "plan", Effect.gen(function* () {
|
|
134
|
+
const text = yield* files.read(join(specPackRoot, "plan.md"));
|
|
135
|
+
if (text === undefined) {
|
|
136
|
+
return yield* PersistenceError.make({
|
|
137
|
+
message: "spec pack has no plan.md — did extraction clear its gate?"
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
// Re-parsing is the hard validation: a malformed plan fails here,
|
|
141
|
+
// not minutes into the implementation phase.
|
|
142
|
+
const parsed = yield* parsePlan(text);
|
|
143
|
+
yield* target.write(`${ModDir}/plan.md`, parsed.render);
|
|
144
|
+
return parsed;
|
|
145
|
+
}));
|
|
146
|
+
yield* stage(context.events, "provenance", Effect.gen(function* () {
|
|
147
|
+
const provenance = makeProvenanceStore(files);
|
|
148
|
+
const specFiles = yield* matchingFiles(target, `^${pack.specsDir}/.*\\.md$`);
|
|
149
|
+
const specs = yield* provenance.hashFiles(input.workDir, specFiles);
|
|
150
|
+
const gateFiles = yield* matchingFiles(legacy, `^${ModDir}/gate/.*\\.json$`).pipe(Effect.orElseSucceed(() => []));
|
|
151
|
+
const verdicts = yield* provenance.hashFiles(legacyRepo, gateFiles);
|
|
152
|
+
const seatsText = yield* files
|
|
153
|
+
.read(join(specPackRoot, "seats.json"))
|
|
154
|
+
.pipe(Effect.orElseSucceed(() => undefined));
|
|
155
|
+
// A pre-seats spec pack, or a malformed sidecar, degrades to no
|
|
156
|
+
// recorded seats rather than failing the seed.
|
|
157
|
+
const seats = seatsText === undefined
|
|
158
|
+
? {}
|
|
159
|
+
: yield* Effect.try({
|
|
160
|
+
try: () => Schema.decodeUnknownSync(Schema.fromJsonString(Seats))(seatsText),
|
|
161
|
+
catch: () => undefined
|
|
162
|
+
}).pipe(Effect.orElseSucceed(() => ({})));
|
|
163
|
+
const approver = process.env.LLM4TS_APPROVER?.trim();
|
|
164
|
+
yield* provenance.write(join(input.workDir, ModDir, "provenance.json"), Provenance.make({
|
|
165
|
+
schema: 1,
|
|
166
|
+
pack: pack.name,
|
|
167
|
+
llm4tsVersion: packageVersion,
|
|
168
|
+
createdAt: new Date().toISOString(),
|
|
169
|
+
...(approver === undefined || approver.length === 0
|
|
170
|
+
? {}
|
|
171
|
+
: { approvedBy: approver }),
|
|
172
|
+
seats,
|
|
173
|
+
specs,
|
|
174
|
+
gateVerdicts: Object.fromEntries(Object.entries(verdicts).map(([path, hash]) => [
|
|
175
|
+
(path.split("/").at(-1) ?? path).replace(/\.json$/, ""),
|
|
176
|
+
hash
|
|
177
|
+
])),
|
|
178
|
+
fixSpecs: []
|
|
179
|
+
}));
|
|
180
|
+
}));
|
|
181
|
+
yield* stage(context.events, "commit", context.git
|
|
182
|
+
.commitAll(`modernize(${pack.name}): seed specs, features, plan, and provenance (${seeded} file(s))`)
|
|
183
|
+
.pipe(Effect.asVoid));
|
|
184
|
+
yield* context.events.publish(Info.make({
|
|
185
|
+
message: `target seeded (plan ${plan.epicId}) — run modernize-implement with the same --repo`
|
|
186
|
+
}));
|
|
187
|
+
}));
|
|
188
|
+
});
|
|
189
|
+
runFlowMain(program);
|
|
Binary file
|
|
@@ -0,0 +1,435 @@
|
|
|
1
|
+
// Legacy modernization phase 4: prove the implementation equivalent to its spec pack.
|
|
2
|
+
//
|
|
3
|
+
// Runs rooted at the TARGET repository (`--repo <target>`) behind the enforced
|
|
4
|
+
// clean-room wall: equivalence is proven against specs and vectors, never
|
|
5
|
+
// against the original code.
|
|
6
|
+
//
|
|
7
|
+
// 1. Per spec'd program, generate equivalence test vectors from the spec and
|
|
8
|
+
// its BDD scenarios — resumable, one .jsonl per program (delete a file to
|
|
9
|
+
// regenerate just that program).
|
|
10
|
+
// 2. Replay every vector through the pack's `replay:` command (a vector JSON
|
|
11
|
+
// on stdin, the resulting observations as a JSON array on stdout) and diff
|
|
12
|
+
// the observations under the pack's comparison policy.
|
|
13
|
+
// 3. Report rule-by-rule coverage against the frozen rules.txt — the rule
|
|
14
|
+
// universe extraction wrote; the target side never re-enumerates it.
|
|
15
|
+
// 4. Triage the failures into fix specs plus plan tasks appended for
|
|
16
|
+
// modernize-implement, extend the provenance manifest with the report
|
|
17
|
+
// hash, commit, and fail while any vector is red.
|
|
18
|
+
//
|
|
19
|
+
// Run: modernize-verify --repo ~/services/meridian-transfers
|
|
20
|
+
import { join } from "node:path";
|
|
21
|
+
import * as Effect from "effect/Effect";
|
|
22
|
+
import * as Schema from "effect/Schema";
|
|
23
|
+
import { CurrentEquivSchema, EquivVector, Observations, readEquivVectors, replayEquivVector, diffObservations, writeEquivVectors } from "@llm4ts/flow/Equiv";
|
|
24
|
+
import { renderEquivReport, VectorVerdict } from "@llm4ts/flow/EquivReport";
|
|
25
|
+
import { FlowAborted, FlowLlmError, PlanParseError } from "@llm4ts/flow/FlowError";
|
|
26
|
+
import { Info } from "@llm4ts/flow/FlowEvents";
|
|
27
|
+
import { makePlanStore } from "@llm4ts/flow/Persistence";
|
|
28
|
+
import { stage } from "@llm4ts/flow/PlanExecution";
|
|
29
|
+
import { Plan, Task } from "@llm4ts/flow/Plan";
|
|
30
|
+
import { Provenance, makeProvenanceStore } from "@llm4ts/flow/Provenance";
|
|
31
|
+
import { matchingFiles } from "@llm4ts/flow/SpecChecks";
|
|
32
|
+
import { checkWall, wallBreachMessage } from "@llm4ts/flow/Wall";
|
|
33
|
+
import { generateVectorsResumably } from "@llm4ts/modernize/Artifacts";
|
|
34
|
+
import { asReadOnly, coderFromEnv } from "@llm4ts/runner/Connectors";
|
|
35
|
+
import { resolveFlowInput } from "@llm4ts/runner/FlowArgs";
|
|
36
|
+
import { runFlowMain, runNode } from "@llm4ts/runner/FlowRunner";
|
|
37
|
+
import { nodePlainFileStore } from "@llm4ts/runner/NodePlainFileStore";
|
|
38
|
+
import { nodeProcessExecutor } from "@llm4ts/runner/NodeProcessExecutor";
|
|
39
|
+
import { makeNodeWorkspace } from "@llm4ts/runner/NodeWorkspace";
|
|
40
|
+
import { openPack } from "@llm4ts/runner/Packs";
|
|
41
|
+
const ModDir = "docs/modernization";
|
|
42
|
+
/**
|
|
43
|
+
* The model-facing vector shape: flat maps only, so structured output stays
|
|
44
|
+
* schema-simple. `kind` is "record" (an emitted record) or "db" (a mutation).
|
|
45
|
+
*/
|
|
46
|
+
class GeneratedObservation extends Schema.Class("GeneratedObservation")({
|
|
47
|
+
kind: Schema.String,
|
|
48
|
+
channel: Schema.String,
|
|
49
|
+
op: Schema.String,
|
|
50
|
+
key: Schema.Record(Schema.String, Schema.String),
|
|
51
|
+
fields: Schema.Record(Schema.String, Schema.String)
|
|
52
|
+
}) {
|
|
53
|
+
}
|
|
54
|
+
class GeneratedVector extends Schema.Class("GeneratedVector")({
|
|
55
|
+
id: Schema.String,
|
|
56
|
+
rules: Schema.Array(Schema.String),
|
|
57
|
+
inputs: Schema.Record(Schema.String, Schema.String),
|
|
58
|
+
observations: Schema.Array(GeneratedObservation)
|
|
59
|
+
}) {
|
|
60
|
+
}
|
|
61
|
+
class GeneratedVectors extends Schema.Class("GeneratedVectors")({
|
|
62
|
+
vectors: Schema.Array(GeneratedVector)
|
|
63
|
+
}) {
|
|
64
|
+
}
|
|
65
|
+
const generatedVectorsJsonSchema = {
|
|
66
|
+
type: "object",
|
|
67
|
+
properties: {
|
|
68
|
+
vectors: {
|
|
69
|
+
type: "array",
|
|
70
|
+
items: {
|
|
71
|
+
type: "object",
|
|
72
|
+
properties: {
|
|
73
|
+
id: { type: "string" },
|
|
74
|
+
rules: { type: "array", items: { type: "string" } },
|
|
75
|
+
inputs: { type: "object", additionalProperties: { type: "string" } },
|
|
76
|
+
observations: {
|
|
77
|
+
type: "array",
|
|
78
|
+
items: {
|
|
79
|
+
type: "object",
|
|
80
|
+
properties: {
|
|
81
|
+
kind: { type: "string", enum: ["record", "db"] },
|
|
82
|
+
channel: { type: "string" },
|
|
83
|
+
op: { type: "string" },
|
|
84
|
+
key: { type: "object", additionalProperties: { type: "string" } },
|
|
85
|
+
fields: { type: "object", additionalProperties: { type: "string" } }
|
|
86
|
+
},
|
|
87
|
+
required: ["kind", "channel", "op", "key", "fields"]
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
},
|
|
91
|
+
required: ["id", "rules", "inputs", "observations"]
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
},
|
|
95
|
+
required: ["vectors"]
|
|
96
|
+
};
|
|
97
|
+
class FixSpec extends Schema.Class("FixSpec")({
|
|
98
|
+
title: Schema.String,
|
|
99
|
+
spec: Schema.String,
|
|
100
|
+
taskTitle: Schema.String,
|
|
101
|
+
taskDescription: Schema.String
|
|
102
|
+
}) {
|
|
103
|
+
}
|
|
104
|
+
class VerifyOutcome extends Schema.Class("VerifyOutcome")({
|
|
105
|
+
fixes: Schema.Array(FixSpec)
|
|
106
|
+
}) {
|
|
107
|
+
}
|
|
108
|
+
const verifyOutcomeJsonSchema = {
|
|
109
|
+
type: "object",
|
|
110
|
+
properties: {
|
|
111
|
+
fixes: {
|
|
112
|
+
type: "array",
|
|
113
|
+
items: {
|
|
114
|
+
type: "object",
|
|
115
|
+
properties: {
|
|
116
|
+
title: { type: "string" },
|
|
117
|
+
spec: { type: "string" },
|
|
118
|
+
taskTitle: { type: "string" },
|
|
119
|
+
taskDescription: { type: "string" }
|
|
120
|
+
},
|
|
121
|
+
required: ["title", "spec", "taskTitle", "taskDescription"]
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
},
|
|
125
|
+
required: ["fixes"]
|
|
126
|
+
};
|
|
127
|
+
const slug = (title) => title
|
|
128
|
+
.toLowerCase()
|
|
129
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
130
|
+
.replace(/^-|-$/g, "")
|
|
131
|
+
.slice(0, 60);
|
|
132
|
+
const toObservation = (vectorId, generated) => {
|
|
133
|
+
switch (generated.kind.toLowerCase()) {
|
|
134
|
+
case "record":
|
|
135
|
+
return Effect.succeed(Observations.record(generated.channel, generated.fields));
|
|
136
|
+
case "db":
|
|
137
|
+
return Effect.succeed(Observations.dbMutation(generated.channel, generated.op, generated.key, generated.fields));
|
|
138
|
+
default:
|
|
139
|
+
return Effect.fail(PlanParseError.make({
|
|
140
|
+
message: `vector ${vectorId}: unknown observation kind '${generated.kind}' (expected record|db)`
|
|
141
|
+
}));
|
|
142
|
+
}
|
|
143
|
+
};
|
|
144
|
+
const toVector = Effect.fn("modernize-verify.toVector")(function* (program, generated) {
|
|
145
|
+
const observations = [];
|
|
146
|
+
for (const observation of generated.observations) {
|
|
147
|
+
observations.push(yield* toObservation(generated.id, observation));
|
|
148
|
+
}
|
|
149
|
+
return EquivVector.make({
|
|
150
|
+
schemaVersion: CurrentEquivSchema,
|
|
151
|
+
program,
|
|
152
|
+
id: generated.id,
|
|
153
|
+
tier: "generated",
|
|
154
|
+
rules: generated.rules,
|
|
155
|
+
inputs: generated.inputs,
|
|
156
|
+
observations
|
|
157
|
+
});
|
|
158
|
+
});
|
|
159
|
+
const generatePrompt = (pack, program, spec, feature, rules) => [
|
|
160
|
+
pack.prompt("vectors") ?? "",
|
|
161
|
+
"",
|
|
162
|
+
`Generate equivalence test vectors for the program ${program} from its behavioural spec and`,
|
|
163
|
+
"BDD scenarios below. Vectors are the generated tier: they prove spec conformance, so cover",
|
|
164
|
+
"every rule the spec states — normal paths, boundary values (thresholds exactly at/over/under),",
|
|
165
|
+
"and error paths (rejects, insufficient funds, validation order).",
|
|
166
|
+
"",
|
|
167
|
+
"For each vector:",
|
|
168
|
+
'- "id": short kebab-case name describing the case.',
|
|
169
|
+
'- "rules": the source units it exercises, chosen ONLY from this list (use the exact names).',
|
|
170
|
+
rules.join("\n"),
|
|
171
|
+
'- "inputs": a flat string map — the fields the replay harness needs to drive one execution',
|
|
172
|
+
" (amounts as plain decimal strings, dates ISO, codes verbatim from the spec).",
|
|
173
|
+
'- "observations": the EXACT outcomes the spec promises, in order. kind "record" for emitted',
|
|
174
|
+
' records ("channel" names the output, "fields" carries values, "op" and "key" empty); kind',
|
|
175
|
+
' "db" for database mutations ("channel" = table, "op" = insert/update/delete, "key" addresses',
|
|
176
|
+
' the row, "fields" = the values written). Every amount, status, and reason code must come',
|
|
177
|
+
" from the spec — never invent values.",
|
|
178
|
+
"",
|
|
179
|
+
"Aim for 5–8 vectors: the happy path, each boundary, each reject.",
|
|
180
|
+
"",
|
|
181
|
+
"Spec:",
|
|
182
|
+
spec,
|
|
183
|
+
"",
|
|
184
|
+
"Scenarios:",
|
|
185
|
+
feature
|
|
186
|
+
].join("\n");
|
|
187
|
+
const triagePrompt = (pack, failing, specText) => {
|
|
188
|
+
const details = failing
|
|
189
|
+
.map((verdict) => {
|
|
190
|
+
const mismatches = verdict.mismatches
|
|
191
|
+
.map((mismatch) => {
|
|
192
|
+
switch (mismatch._tag) {
|
|
193
|
+
case "FieldDiff":
|
|
194
|
+
return ` - ${mismatch.at}: ${mismatch.field} expected ${mismatch.expected}, actual ${mismatch.actual}`;
|
|
195
|
+
case "Missing":
|
|
196
|
+
return ` - missing: ${JSON.stringify(mismatch.expected)}`;
|
|
197
|
+
case "Unexpected":
|
|
198
|
+
return ` - unexpected: ${JSON.stringify(mismatch.actual)}`;
|
|
199
|
+
}
|
|
200
|
+
})
|
|
201
|
+
.join("\n");
|
|
202
|
+
return `- ${verdict.vector.program} ${verdict.vector.id} (rules: ${verdict.vector.rules.join(", ")})\n${mismatches}`;
|
|
203
|
+
})
|
|
204
|
+
.join("\n");
|
|
205
|
+
return [
|
|
206
|
+
pack.prompt("review") ?? "",
|
|
207
|
+
"",
|
|
208
|
+
"The equivalence harness replayed test vectors against the implementation and found the",
|
|
209
|
+
"mismatches below. For each DISTINCT root cause produce one fix: a short spec document",
|
|
210
|
+
"(Markdown: the rule violated, expected vs actual behaviour, the failing vector ids) and a",
|
|
211
|
+
"plan task (title + description naming the spec rules). Group mismatches sharing a cause.",
|
|
212
|
+
"If a mismatch reveals a wrong or ambiguous SPEC rather than wrong code, say so explicitly",
|
|
213
|
+
"in that fix document — spec gaps go back to extraction, not to the coder.",
|
|
214
|
+
"",
|
|
215
|
+
"Mismatches:",
|
|
216
|
+
details,
|
|
217
|
+
"",
|
|
218
|
+
"Specs under test:",
|
|
219
|
+
specText
|
|
220
|
+
].join("\n");
|
|
221
|
+
};
|
|
222
|
+
/** The spec'd programs: top-level `<NAME>.md` files under the specs dir, indexes aside. */
|
|
223
|
+
const specPrograms = Effect.fn("modernize-verify.specPrograms")(function* (target, specsDir) {
|
|
224
|
+
const paths = yield* matchingFiles(target, `^${specsDir}/[^/]+\\.md$`).pipe(Effect.orElseSucceed(() => []));
|
|
225
|
+
const excluded = new Set(["traceability", "mapping", "README"]);
|
|
226
|
+
return paths
|
|
227
|
+
.map((path) => (path.split("/").at(-1) ?? path).replace(/\.md$/, ""))
|
|
228
|
+
.filter((name) => !excluded.has(name))
|
|
229
|
+
.sort();
|
|
230
|
+
});
|
|
231
|
+
const program = Effect.gen(function* () {
|
|
232
|
+
const input = yield* resolveFlowInput("Prove the implementation equivalent to its spec pack");
|
|
233
|
+
const coder = coderFromEnv(process.env);
|
|
234
|
+
const files = nodePlainFileStore;
|
|
235
|
+
const planPath = join(input.workDir, ModDir, "plan.md");
|
|
236
|
+
const vectorsDir = join(input.workDir, ModDir, "vectors");
|
|
237
|
+
yield* runNode({
|
|
238
|
+
workDir: input.workDir,
|
|
239
|
+
workspace: input.workspace,
|
|
240
|
+
userPrompt: input.prompt,
|
|
241
|
+
coder,
|
|
242
|
+
reasoning: asReadOnly(coder),
|
|
243
|
+
environment: process.env
|
|
244
|
+
}, (context) => Effect.gen(function* () {
|
|
245
|
+
const target = yield* makeNodeWorkspace(input.workDir);
|
|
246
|
+
const { pack } = yield* stage(context.events, "pack", openPack({
|
|
247
|
+
environment: process.env,
|
|
248
|
+
launchDir: input.workspace,
|
|
249
|
+
flowDir: import.meta.dirname
|
|
250
|
+
}));
|
|
251
|
+
yield* stage(context.events, "wall", Effect.gen(function* () {
|
|
252
|
+
if (pack.sources === undefined) {
|
|
253
|
+
return yield* context.events.publish(Info.make({ message: "pack has no sources regex — wall check skipped" }));
|
|
254
|
+
}
|
|
255
|
+
const result = yield* checkWall(target, pack.sources);
|
|
256
|
+
if (result._tag === "Breached") {
|
|
257
|
+
return yield* FlowAborted.make({
|
|
258
|
+
message: wallBreachMessage(result, "Equivalence is proven against specs and vectors, never against the original code.")
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
yield* context.events.publish(Info.make({ message: "clean-room wall: no legacy source in the target workspace" }));
|
|
262
|
+
}));
|
|
263
|
+
const rulesText = (yield* files.read(join(input.workDir, pack.specsDir, "rules.txt"))) ?? "";
|
|
264
|
+
const universe = rulesText
|
|
265
|
+
.split(/\r?\n/)
|
|
266
|
+
.map((line) => line.trim())
|
|
267
|
+
.filter((line) => line.length > 0);
|
|
268
|
+
if (universe.length === 0) {
|
|
269
|
+
yield* context.events.publish(Info.make({
|
|
270
|
+
message: "no rules.txt in the seeded spec pack — the report will use the vectors' own " +
|
|
271
|
+
"rules and cannot flag unexercised ones"
|
|
272
|
+
}));
|
|
273
|
+
}
|
|
274
|
+
const programs = yield* stage(context.events, "programs", specPrograms(target, pack.specsDir));
|
|
275
|
+
if (programs.length === 0) {
|
|
276
|
+
return yield* FlowAborted.make({
|
|
277
|
+
message: `no specs under ${pack.specsDir} — run modernize-seed first`
|
|
278
|
+
});
|
|
279
|
+
}
|
|
280
|
+
// Generated-first and resumable per program: an existing .jsonl is kept.
|
|
281
|
+
yield* stage(context.events, "vectors", Effect.gen(function* () {
|
|
282
|
+
const summary = yield* generateVectorsResumably(files, programs, (name) => Effect.gen(function* () {
|
|
283
|
+
const spec = (yield* files.read(join(input.workDir, pack.specsDir, `${name}.md`))) ?? "";
|
|
284
|
+
const featurePaths = yield* matchingFiles(target, `^${pack.featuresDir}/.*\\.feature$`).pipe(Effect.orElseSucceed(() => []));
|
|
285
|
+
const featurePath = featurePaths.find((path) => (path.split("/").at(-1) ?? "").replace(/\.feature$/, "").toLowerCase() ===
|
|
286
|
+
name.toLowerCase());
|
|
287
|
+
const feature = featurePath === undefined
|
|
288
|
+
? ""
|
|
289
|
+
: yield* target.read(featurePath).pipe(Effect.orElseSucceed(() => ""));
|
|
290
|
+
const generated = yield* context.reasoning
|
|
291
|
+
.executeStructured(generatePrompt(pack, name, spec, feature, universe), GeneratedVectors, generatedVectorsJsonSchema)
|
|
292
|
+
.pipe(Effect.mapError(FlowLlmError.from));
|
|
293
|
+
if (generated.vectors.length === 0) {
|
|
294
|
+
return yield* FlowAborted.make({
|
|
295
|
+
message: `generator produced no vectors for ${name}`
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
const vectors = [];
|
|
299
|
+
for (const candidate of generated.vectors) {
|
|
300
|
+
vectors.push(yield* toVector(name, candidate));
|
|
301
|
+
}
|
|
302
|
+
// generateVectorsResumably persists the returned text, so the
|
|
303
|
+
// encoder writes into a scratch path and hands back its body.
|
|
304
|
+
const scratch = join(vectorsDir, `.${name}.tmp`);
|
|
305
|
+
yield* writeEquivVectors(files, scratch, vectors);
|
|
306
|
+
const encoded = (yield* files.read(scratch)) ?? "";
|
|
307
|
+
yield* files.remove(scratch);
|
|
308
|
+
yield* context.events.publish(Info.make({ message: `${vectors.length} vector(s) generated for ${name}` }));
|
|
309
|
+
return encoded;
|
|
310
|
+
}), vectorsDir);
|
|
311
|
+
if (summary.skipped.length > 0) {
|
|
312
|
+
yield* context.events.publish(Info.make({
|
|
313
|
+
message: `vectors exist for ${summary.skipped.join(", ")} — skipping`
|
|
314
|
+
}));
|
|
315
|
+
}
|
|
316
|
+
}));
|
|
317
|
+
if (pack.replay === undefined) {
|
|
318
|
+
return yield* FlowAborted.make({
|
|
319
|
+
message: `pack '${pack.name}' has no replay: command — add one (reads a vector JSON on ` +
|
|
320
|
+
"stdin, prints the resulting observations as a JSON array on stdout)"
|
|
321
|
+
});
|
|
322
|
+
}
|
|
323
|
+
const replayCommand = pack.replay;
|
|
324
|
+
const verdicts = yield* stage(context.events, "replay", Effect.gen(function* () {
|
|
325
|
+
const vectorFiles = yield* matchingFiles(target, `^${ModDir}/vectors/.*\\.jsonl$`).pipe(Effect.orElseSucceed(() => []));
|
|
326
|
+
const vectors = [];
|
|
327
|
+
for (const path of [...vectorFiles].sort()) {
|
|
328
|
+
vectors.push(...(yield* readEquivVectors(files, join(input.workDir, path))));
|
|
329
|
+
}
|
|
330
|
+
if (vectors.length === 0) {
|
|
331
|
+
return yield* FlowAborted.make({
|
|
332
|
+
message: `no vectors under ${vectorsDir} — nothing to replay`
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
const results = [];
|
|
336
|
+
for (const vector of vectors) {
|
|
337
|
+
const replayed = yield* replayEquivVector(nodeProcessExecutor, context.events, replayCommand, input.workDir, vector);
|
|
338
|
+
if (replayed._tag === "Crashed") {
|
|
339
|
+
yield* context.events.publish(Info.make({
|
|
340
|
+
message: `replay failed for ${vector.program}/${vector.id} ` +
|
|
341
|
+
`(exit ${replayed.exitCode}): ${replayed.problem.slice(0, 300)}`
|
|
342
|
+
}));
|
|
343
|
+
results.push(VectorVerdict.make({
|
|
344
|
+
vector,
|
|
345
|
+
mismatches: vector.observations.map((expected) => ({
|
|
346
|
+
_tag: "Missing",
|
|
347
|
+
expected
|
|
348
|
+
}))
|
|
349
|
+
}));
|
|
350
|
+
}
|
|
351
|
+
else {
|
|
352
|
+
results.push(VectorVerdict.make({
|
|
353
|
+
vector,
|
|
354
|
+
mismatches: diffObservations(vector.observations, replayed.actual, pack.equivalence)
|
|
355
|
+
}));
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
return results;
|
|
359
|
+
}));
|
|
360
|
+
const allRules = universe.length > 0
|
|
361
|
+
? universe
|
|
362
|
+
: [...new Set(verdicts.flatMap((verdict) => verdict.vector.rules))].sort();
|
|
363
|
+
const failing = verdicts.filter((verdict) => !verdict.passed);
|
|
364
|
+
yield* stage(context.events, "report", files.writeAtomic(join(input.workDir, ModDir, "equivalence.md"), renderEquivReport(verdicts, allRules)));
|
|
365
|
+
if (failing.length > 0) {
|
|
366
|
+
yield* stage(context.events, "triage", Effect.gen(function* () {
|
|
367
|
+
const specTexts = [
|
|
368
|
+
(yield* files.read(join(input.workDir, pack.specsDir, "traceability.md"))) ?? ""
|
|
369
|
+
];
|
|
370
|
+
for (const name of programs) {
|
|
371
|
+
specTexts.push((yield* files.read(join(input.workDir, pack.specsDir, `${name}.md`))) ?? "");
|
|
372
|
+
}
|
|
373
|
+
const outcome = yield* context.reasoning
|
|
374
|
+
.executeStructured(triagePrompt(pack, failing, specTexts.join("\n\n")), VerifyOutcome, verifyOutcomeJsonSchema)
|
|
375
|
+
.pipe(Effect.mapError(FlowLlmError.from));
|
|
376
|
+
for (const fix of outcome.fixes) {
|
|
377
|
+
yield* files.writeAtomic(join(input.workDir, pack.specsDir, "fixes", `fix-${slug(fix.title)}.md`), `# ${fix.title}\n\n${fix.spec}\n`);
|
|
378
|
+
}
|
|
379
|
+
if (outcome.fixes.length > 0) {
|
|
380
|
+
const store = makePlanStore(files);
|
|
381
|
+
const plan = yield* store.load(planPath);
|
|
382
|
+
if (plan === undefined) {
|
|
383
|
+
return yield* FlowAborted.make({
|
|
384
|
+
message: `no plan at ${planPath} — run modernize-seed first`
|
|
385
|
+
});
|
|
386
|
+
}
|
|
387
|
+
yield* store.save(planPath, Plan.make({
|
|
388
|
+
...plan,
|
|
389
|
+
tasks: [
|
|
390
|
+
...plan.tasks,
|
|
391
|
+
...outcome.fixes.map((fix) => Task.make({
|
|
392
|
+
title: fix.taskTitle,
|
|
393
|
+
description: fix.taskDescription,
|
|
394
|
+
completed: false
|
|
395
|
+
}))
|
|
396
|
+
]
|
|
397
|
+
}));
|
|
398
|
+
yield* context.events.publish(Info.make({
|
|
399
|
+
message: `${outcome.fixes.length} fix task(s) appended — rerun modernize-implement`
|
|
400
|
+
}));
|
|
401
|
+
}
|
|
402
|
+
}));
|
|
403
|
+
}
|
|
404
|
+
yield* stage(context.events, "provenance", Effect.gen(function* () {
|
|
405
|
+
const manifest = join(input.workDir, ModDir, "provenance.json");
|
|
406
|
+
if ((yield* files.read(manifest)) === undefined) {
|
|
407
|
+
return yield* context.events.publish(Info.make({ message: "no provenance.json — seeded by an older run; skipping" }));
|
|
408
|
+
}
|
|
409
|
+
const provenance = makeProvenanceStore(files);
|
|
410
|
+
const hashes = yield* provenance.hashFiles(input.workDir, [`${ModDir}/equivalence.md`]);
|
|
411
|
+
const report = Object.values(hashes)[0];
|
|
412
|
+
// Spreading into a plain object would not satisfy the schema's
|
|
413
|
+
// encoder — the manifest must stay a Provenance instance.
|
|
414
|
+
yield* provenance.extend(manifest, (current) => report === undefined
|
|
415
|
+
? current
|
|
416
|
+
: Provenance.make({ ...current, equivalenceReport: report }));
|
|
417
|
+
}));
|
|
418
|
+
const generated = verdicts.filter((verdict) => verdict.vector.tier === "generated").length;
|
|
419
|
+
const captured = verdicts.filter((verdict) => verdict.vector.tier === "captured").length;
|
|
420
|
+
const summary = `${verdicts.filter((verdict) => verdict.passed).length}/${verdicts.length} vectors green ` +
|
|
421
|
+
`(${generated} generated, ${captured} captured)`;
|
|
422
|
+
yield* stage(context.events, "commit", context.git
|
|
423
|
+
.commitAll(`modernize(${pack.name}): verify — ${summary}` +
|
|
424
|
+
(failing.length > 0 ? `; ${failing.length} failing triaged` : ""))
|
|
425
|
+
.pipe(Effect.asVoid));
|
|
426
|
+
if (failing.length > 0) {
|
|
427
|
+
return yield* FlowAborted.make({
|
|
428
|
+
message: `equivalence not proven: ${failing.length} failing vector(s) — see ` +
|
|
429
|
+
`${ModDir}/equivalence.md; fix specs filed, rerun modernize-implement`
|
|
430
|
+
});
|
|
431
|
+
}
|
|
432
|
+
yield* context.events.publish(Info.make({ message: summary }));
|
|
433
|
+
}));
|
|
434
|
+
});
|
|
435
|
+
runFlowMain(program);
|