@slowcook-ai/cli 0.7.5 → 0.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +13 -1
- package/dist/cli.js.map +1 -1
- package/dist/commands/brew/agent.d.ts.map +1 -1
- package/dist/commands/brew/agent.js +49 -17
- package/dist/commands/brew/agent.js.map +1 -1
- package/dist/commands/brew/halt.d.ts +13 -3
- package/dist/commands/brew/halt.d.ts.map +1 -1
- package/dist/commands/brew/halt.js +24 -4
- package/dist/commands/brew/halt.js.map +1 -1
- package/dist/commands/brew/prompts.d.ts +1 -1
- package/dist/commands/brew/prompts.d.ts.map +1 -1
- package/dist/commands/brew/prompts.js +12 -0
- package/dist/commands/brew/prompts.js.map +1 -1
- package/dist/commands/init/index.d.ts.map +1 -1
- package/dist/commands/init/index.js +30 -0
- package/dist/commands/init/index.js.map +1 -1
- package/dist/commands/init/plan.d.ts.map +1 -1
- package/dist/commands/init/plan.js +12 -1
- package/dist/commands/init/plan.js.map +1 -1
- package/dist/commands/on-brew-merged/index.d.ts +2 -0
- package/dist/commands/on-brew-merged/index.d.ts.map +1 -0
- package/dist/commands/on-brew-merged/index.js +159 -0
- package/dist/commands/on-brew-merged/index.js.map +1 -0
- package/dist/commands/on-spec-merged/index.d.ts.map +1 -1
- package/dist/commands/on-spec-merged/index.js +18 -0
- package/dist/commands/on-spec-merged/index.js.map +1 -1
- package/dist/commands/on-tests-merged/index.d.ts +2 -0
- package/dist/commands/on-tests-merged/index.d.ts.map +1 -0
- package/dist/commands/on-tests-merged/index.js +171 -0
- package/dist/commands/on-tests-merged/index.js.map +1 -0
- package/dist/commands/testgen/agent.d.ts +48 -9
- package/dist/commands/testgen/agent.d.ts.map +1 -1
- package/dist/commands/testgen/agent.js +253 -49
- package/dist/commands/testgen/agent.js.map +1 -1
- package/dist/commands/testgen/prompts.d.ts.map +1 -1
- package/dist/commands/testgen/prompts.js +98 -3
- package/dist/commands/testgen/prompts.js.map +1 -1
- package/package.json +3 -3
|
@@ -1,10 +1,26 @@
|
|
|
1
|
-
import type { ForgeAdapter } from "@slowcook-ai/core";
|
|
1
|
+
import type { ForgeAdapter, Spec } from "@slowcook-ai/core";
|
|
2
2
|
import type { LlmClient } from "../refine/llm.js";
|
|
3
3
|
export declare const LABEL_TESTS_READY = "tests-ready";
|
|
4
4
|
export declare const LABEL_OVERRIDE_FREEZE = "override-freeze";
|
|
5
5
|
export declare const TESTS_INTEGRATION_DIR = "tests/integration";
|
|
6
6
|
export declare const MANIFESTS_DIR = ".brewing/manifests";
|
|
7
7
|
export declare const MOCK_HELPERS_DIR = "tests/helpers/mocks";
|
|
8
|
+
/**
|
|
9
|
+
* Which artifacts testgen should emit for a given spec. Computed per-spec
|
|
10
|
+
* based on what already exists on disk + whether the spec has `ui_behavior`:
|
|
11
|
+
*
|
|
12
|
+
* - `"full"` — neither handler tests nor UI tests exist; emit both (plus
|
|
13
|
+
* any needed stubs + helpers).
|
|
14
|
+
* - `"handler-only"` — spec has no `ui_behavior`; handler tests don't exist.
|
|
15
|
+
* Emit handler test + handler stubs + helpers. This is 0.7.0 behavior.
|
|
16
|
+
* - `"ui-only"` — handler tests already exist; spec has `ui_behavior`; UI
|
|
17
|
+
* tests are missing. Emit ONLY UI test + UI stubs. Use-case: 0.7.5 adoption
|
|
18
|
+
* on a brownfield story where the backend was built before UI tests existed.
|
|
19
|
+
*
|
|
20
|
+
* Mode is inferred by `collectTargetSpecs`; the LLM is told which mode it's
|
|
21
|
+
* in via the user message so it emits only what's needed.
|
|
22
|
+
*/
|
|
23
|
+
export type TestgenMode = "full" | "handler-only" | "ui-only";
|
|
8
24
|
export interface TestgenContext {
|
|
9
25
|
repoRoot: string;
|
|
10
26
|
forge: ForgeAdapter;
|
|
@@ -42,6 +58,10 @@ export type TestgenOutcome = {
|
|
|
42
58
|
* citing the supersede chain in the PR body for auditability).
|
|
43
59
|
*/
|
|
44
60
|
export declare function runTestgen(ctx: TestgenContext): Promise<TestgenOutcome>;
|
|
61
|
+
export interface TargetSpec {
|
|
62
|
+
spec: Spec;
|
|
63
|
+
mode: TestgenMode;
|
|
64
|
+
}
|
|
45
65
|
export declare function buildProjectContext(repoRoot: string): string;
|
|
46
66
|
/**
|
|
47
67
|
* Phase B2 (0.7.0) testgen output: one test file, zero-or-more route stubs,
|
|
@@ -50,6 +70,7 @@ export declare function buildProjectContext(repoRoot: string): string;
|
|
|
50
70
|
* against existing files, and writes only what's new.
|
|
51
71
|
*/
|
|
52
72
|
export interface TestgenBundle {
|
|
73
|
+
/** Handler test file content. Empty string when mode is `"ui-only"`. */
|
|
53
74
|
testContent: string;
|
|
54
75
|
stubs: Array<{
|
|
55
76
|
path: string;
|
|
@@ -59,23 +80,34 @@ export interface TestgenBundle {
|
|
|
59
80
|
path: string;
|
|
60
81
|
contents: string;
|
|
61
82
|
}>;
|
|
83
|
+
/** UI component test file content (tier-1 UI). Empty string when mode is `"handler-only"`. */
|
|
84
|
+
uiTestContent: string;
|
|
85
|
+
/** UI component stubs — React/TSX files under src/components/ or src/app/**\/*.tsx. */
|
|
86
|
+
uiStubs: Array<{
|
|
87
|
+
path: string;
|
|
88
|
+
contents: string;
|
|
89
|
+
}>;
|
|
62
90
|
}
|
|
63
91
|
/**
|
|
64
92
|
* Parse XML-tagged multi-artifact output into a TestgenBundle.
|
|
65
93
|
*
|
|
66
94
|
* Accepted shape (from the prompt):
|
|
67
|
-
* <test_file>...</test_file>
|
|
68
|
-
* <stub path="src/app/api/foo/route.ts">...</stub>
|
|
69
|
-
* <helper path="tests/helpers/mocks/bar.ts">...</helper>
|
|
95
|
+
* <test_file>...</test_file> — required unless mode="ui-only"
|
|
96
|
+
* <stub path="src/app/api/foo/route.ts">...</stub> (zero or more)
|
|
97
|
+
* <helper path="tests/helpers/mocks/bar.ts">...</helper> (zero or more)
|
|
98
|
+
* <ui_test_file>...</ui_test_file> — required when mode="ui-only" or "full" with UI
|
|
99
|
+
* <ui_stub path="src/components/foo.tsx">...</ui_stub> (zero or more)
|
|
100
|
+
*
|
|
101
|
+
* Mode semantics (0.7.7+):
|
|
102
|
+
* - `"handler-only"` — `<test_file>` required; `<ui_test_file>` ignored if present.
|
|
103
|
+
* - `"ui-only"` — `<ui_test_file>` required; `<test_file>` ignored if present.
|
|
104
|
+
* - `"full"` — both `<test_file>` required AND `<ui_test_file>` required.
|
|
70
105
|
*
|
|
71
106
|
* Tolerant of code-fenced output: if the LLM wraps the whole thing in
|
|
72
107
|
* ```, we strip it. If a block's contents are themselves code-fenced,
|
|
73
|
-
* we strip those too — tier-1 test / helper / stub files are raw TS.
|
|
74
|
-
*
|
|
75
|
-
* Throws if `<test_file>` is missing or empty — that's the one mandatory
|
|
76
|
-
* artifact.
|
|
108
|
+
* we strip those too — tier-1 test / helper / stub / UI files are raw TS/TSX.
|
|
77
109
|
*/
|
|
78
|
-
export declare function parseTestgenBundle(raw: string, storyId: string): TestgenBundle;
|
|
110
|
+
export declare function parseTestgenBundle(raw: string, storyId: string, mode?: TestgenMode): TestgenBundle;
|
|
79
111
|
/**
|
|
80
112
|
* Tier-1 conformance lint. Run on every generated test file before commit.
|
|
81
113
|
* Catches patterns the prompt forbids — inline `vi.mock`, `fetch(...)`,
|
|
@@ -107,5 +139,12 @@ export declare function lintTierOneTest(filePath: string, source: string): TierO
|
|
|
107
139
|
* Robust against typical usage; falls back to a single synthetic entry if
|
|
108
140
|
* nothing recognisable is found.
|
|
109
141
|
*/
|
|
142
|
+
/**
|
|
143
|
+
* Read an existing test file from disk and extract its test IDs. Thin
|
|
144
|
+
* wrapper around extractTestIdsFromFile for the "ui-only" code path where
|
|
145
|
+
* the handler test already exists and we need its IDs to preserve in the
|
|
146
|
+
* rewritten combined manifest.
|
|
147
|
+
*/
|
|
148
|
+
export declare function extractTestIdsFromExistingFile(repoRoot: string, filePath: string): string[];
|
|
110
149
|
export declare function extractTestIdsFromFile(filePath: string, source: string): string[];
|
|
111
150
|
//# sourceMappingURL=agent.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../../../src/commands/testgen/agent.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,YAAY,
|
|
1
|
+
{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../../../src/commands/testgen/agent.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AASlD,eAAO,MAAM,iBAAiB,gBAAgB,CAAC;AAC/C,eAAO,MAAM,qBAAqB,oBAAoB,CAAC;AAEvD,eAAO,MAAM,qBAAqB,sBAAsB,CAAC;AACzD,eAAO,MAAM,aAAa,uBAAuB,CAAC;AAClD,eAAO,MAAM,gBAAgB,wBAAwB,CAAC;AAEtD;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,cAAc,GAAG,SAAS,CAAC;AAE9D,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,YAAY,CAAC;IACpB,GAAG,EAAE,SAAS,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,iFAAiF;IACjF,GAAG,EAAE,OAAO,CAAC;IACb,2EAA2E;IAC3E,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,mCAAmC;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,gCAAgC;IAChC,GAAG,EAAE,IAAI,CAAC;CACX;AAED,MAAM,MAAM,cAAc,GACtB;IAAE,IAAI,EAAE,eAAe,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,eAAe,EAAE,MAAM,EAAE,CAAA;CAAE,GACzG;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC/C;IAAE,IAAI,EAAE,qBAAqB,CAAC;IAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAAC;AAE5E;;;;;GAKG;AACH,wBAAsB,UAAU,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CA8M7E;AAqDD,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,IAAI,CAAC;IACX,IAAI,EAAE,WAAW,CAAC;CACnB;AA0DD,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CA8G5D;AA0ED;;;;;GAKG;AACH,MAAM,WAAW,aAAa;IAC5B,wEAAwE;IACxE,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjD,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACnD,8FAA8F;IAC9F,aAAa,EAAE,MAAM,CAAC;IACtB,uFAAuF;IACvF,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACpD;AAkCD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,kBAAkB,CAChC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM,EACf,IAAI,GAAE,WAA4B,GACjC,aAAa,CAsDf;AASD;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB;AA6DD,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,gBAAgB,EAAE,CAepF;AAED;;;;;;;;;GASG;AACH;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAC5C,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,GACf,MAAM,EAAE,CASV;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CA4BjF"}
|
|
@@ -17,53 +17,75 @@ export const MOCK_HELPERS_DIR = "tests/helpers/mocks";
|
|
|
17
17
|
* citing the supersede chain in the PR body for auditability).
|
|
18
18
|
*/
|
|
19
19
|
export async function runTestgen(ctx) {
|
|
20
|
-
const
|
|
21
|
-
if (
|
|
20
|
+
const targets = collectTargetSpecs(ctx);
|
|
21
|
+
if (targets.length === 0) {
|
|
22
22
|
return { kind: "nothing-to-generate", reason: "no active specs without tests" };
|
|
23
23
|
}
|
|
24
24
|
// Produce per-spec artifacts in a scratch area; commit together at the end.
|
|
25
25
|
const generated = [];
|
|
26
26
|
const toRemove = [];
|
|
27
|
-
for (const spec of
|
|
27
|
+
for (const { spec, mode } of targets) {
|
|
28
28
|
const projectContext = buildProjectContext(ctx.repoRoot);
|
|
29
|
-
const bundle = await generateTestBundle(spec, ctx, projectContext);
|
|
30
|
-
const testPath =
|
|
31
|
-
|
|
32
|
-
//
|
|
33
|
-
// loudly
|
|
34
|
-
//
|
|
35
|
-
//
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
`
|
|
45
|
-
|
|
29
|
+
const bundle = await generateTestBundle(spec, ctx, projectContext, mode);
|
|
30
|
+
const testPath = handlerTestPathFor(spec.story_id);
|
|
31
|
+
const uiTestPath = uiTestPathFor(spec.story_id);
|
|
32
|
+
// Tier-1 conformance gate for handler tests (run only when mode emits one).
|
|
33
|
+
// Halts loudly if the LLM slipped back to tier-0 habits (inline vi.mock,
|
|
34
|
+
// fetch(), etc.) rather than quietly producing HTTP-loopback tests the
|
|
35
|
+
// brewing loop can't ratchet against.
|
|
36
|
+
if (mode !== "ui-only" && bundle.testContent) {
|
|
37
|
+
const violations = lintTierOneTest(testPath, bundle.testContent);
|
|
38
|
+
if (violations.length > 0) {
|
|
39
|
+
const details = violations
|
|
40
|
+
.slice(0, 10)
|
|
41
|
+
.map((v) => ` - line ${v.line}: \`${v.pattern}\` — ${v.reason}`)
|
|
42
|
+
.join("\n");
|
|
43
|
+
const more = violations.length > 10 ? `\n - (+${violations.length - 10} more)` : "";
|
|
44
|
+
throw new Error(`testgen output for story-${spec.story_id} violates tier-1 conventions (${violations.length} issue(s)):\n${details}${more}\n\n` +
|
|
45
|
+
`The LLM emitted patterns banned by docs/plans/0.7-testgen-two-tier.md §4.1-§7.3. ` +
|
|
46
|
+
`Re-run testgen with a different model/seed, or hand-edit the generated file to use project mock helpers.`);
|
|
47
|
+
}
|
|
46
48
|
}
|
|
47
49
|
// De-dupe stubs + helpers: skip anything whose target file exists and
|
|
48
50
|
// isn't a @slowcook-stub (for stubs) or isn't empty (for helpers). This
|
|
49
51
|
// lets testgen re-run safely without clobbering in-progress impls.
|
|
50
52
|
const stubsToWrite = bundle.stubs.filter((s) => shouldWriteStub(ctx.repoRoot, s.path));
|
|
51
53
|
const helpersToWrite = bundle.helpers.filter((h) => shouldWriteHelper(ctx.repoRoot, h.path));
|
|
52
|
-
const
|
|
54
|
+
const uiStubsToWrite = bundle.uiStubs.filter((s) => shouldWriteStub(ctx.repoRoot, s.path));
|
|
55
|
+
// Manifest: combine handler + UI test IDs. When mode is "ui-only" the
|
|
56
|
+
// handler manifest already exists on disk — for simplicity we still
|
|
57
|
+
// rewrite it here with the combined shape, preserving the handler IDs
|
|
58
|
+
// by re-extracting from the existing file.
|
|
59
|
+
const handlerIds = mode === "ui-only"
|
|
60
|
+
? extractTestIdsFromExistingFile(ctx.repoRoot, testPath)
|
|
61
|
+
: extractTestIdsFromFile(testPath, bundle.testContent);
|
|
62
|
+
const uiIds = mode !== "handler-only" && bundle.uiTestContent
|
|
63
|
+
? extractTestIdsFromFile(uiTestPath, bundle.uiTestContent)
|
|
64
|
+
: [];
|
|
65
|
+
const manifestTests = [
|
|
66
|
+
...handlerIds.map((id) => ({ id, file: testPath })),
|
|
67
|
+
...uiIds.map((id) => ({ id, file: uiTestPath })),
|
|
68
|
+
];
|
|
53
69
|
const manifest = buildManifest({
|
|
54
70
|
slowcookVersion: ctx.cliVersion,
|
|
55
71
|
storyId: spec.story_id,
|
|
56
|
-
tests:
|
|
57
|
-
suites: [
|
|
72
|
+
tests: manifestTests,
|
|
73
|
+
suites: [
|
|
74
|
+
{ suite: "backend", command: "npx vitest list", test_count: manifestTests.length },
|
|
75
|
+
],
|
|
58
76
|
now: ctx.now,
|
|
59
77
|
});
|
|
60
78
|
generated.push({
|
|
61
79
|
spec,
|
|
62
|
-
|
|
63
|
-
|
|
80
|
+
mode,
|
|
81
|
+
testPath: mode === "ui-only" ? "" : testPath,
|
|
82
|
+
fileContents: mode === "ui-only" ? "" : bundle.testContent,
|
|
64
83
|
manifest,
|
|
65
84
|
stubs: stubsToWrite,
|
|
66
85
|
helpers: helpersToWrite,
|
|
86
|
+
uiTestPath: mode === "handler-only" ? "" : uiTestPath,
|
|
87
|
+
uiFileContents: mode === "handler-only" ? "" : bundle.uiTestContent,
|
|
88
|
+
uiStubs: uiStubsToWrite,
|
|
67
89
|
});
|
|
68
90
|
for (const superseded of spec.supersedes) {
|
|
69
91
|
toRemove.push(superseded);
|
|
@@ -71,7 +93,9 @@ export async function runTestgen(ctx) {
|
|
|
71
93
|
}
|
|
72
94
|
// Apply to disk: write new, delete superseded
|
|
73
95
|
for (const g of generated) {
|
|
74
|
-
|
|
96
|
+
if (g.testPath && g.fileContents) {
|
|
97
|
+
writeFileAt(ctx.repoRoot, g.testPath, g.fileContents);
|
|
98
|
+
}
|
|
75
99
|
const manifestPath = join(MANIFESTS_DIR, `story-${g.spec.story_id}.json`);
|
|
76
100
|
writeFileAt(ctx.repoRoot, manifestPath, JSON.stringify(g.manifest, null, 2) + "\n");
|
|
77
101
|
for (const stub of g.stubs) {
|
|
@@ -80,6 +104,12 @@ export async function runTestgen(ctx) {
|
|
|
80
104
|
for (const helper of g.helpers) {
|
|
81
105
|
writeFileAt(ctx.repoRoot, helper.path, helper.contents);
|
|
82
106
|
}
|
|
107
|
+
if (g.uiTestPath && g.uiFileContents) {
|
|
108
|
+
writeFileAt(ctx.repoRoot, g.uiTestPath, g.uiFileContents);
|
|
109
|
+
}
|
|
110
|
+
for (const stub of g.uiStubs) {
|
|
111
|
+
writeFileAt(ctx.repoRoot, stub.path, stub.contents);
|
|
112
|
+
}
|
|
83
113
|
}
|
|
84
114
|
const actuallyRemoved = [];
|
|
85
115
|
for (const id of toRemove) {
|
|
@@ -93,12 +123,17 @@ export async function runTestgen(ctx) {
|
|
|
93
123
|
// Git: branch, stage, commit, push
|
|
94
124
|
await ctx.forge.git.createBranch(ctx.branchName);
|
|
95
125
|
for (const g of generated) {
|
|
96
|
-
|
|
126
|
+
if (g.testPath)
|
|
127
|
+
await ctx.forge.git.stage(g.testPath);
|
|
97
128
|
await ctx.forge.git.stage(join(MANIFESTS_DIR, `story-${g.spec.story_id}.json`));
|
|
98
129
|
for (const stub of g.stubs)
|
|
99
130
|
await ctx.forge.git.stage(stub.path);
|
|
100
131
|
for (const helper of g.helpers)
|
|
101
132
|
await ctx.forge.git.stage(helper.path);
|
|
133
|
+
if (g.uiTestPath)
|
|
134
|
+
await ctx.forge.git.stage(g.uiTestPath);
|
|
135
|
+
for (const stub of g.uiStubs)
|
|
136
|
+
await ctx.forge.git.stage(stub.path);
|
|
102
137
|
}
|
|
103
138
|
for (const id of actuallyRemoved) {
|
|
104
139
|
await ctx.forge.git.stage(join(TESTS_INTEGRATION_DIR, `story-${id}.test.ts`));
|
|
@@ -125,6 +160,36 @@ export async function runTestgen(ctx) {
|
|
|
125
160
|
draft: true,
|
|
126
161
|
labels,
|
|
127
162
|
});
|
|
163
|
+
// Post one audit-trail comment per spec's source_issue so the issue
|
|
164
|
+
// thread tells the whole pipeline story (refine posted earlier; brew
|
|
165
|
+
// posts next). Best-effort — don't fail the testgen run on a bad
|
|
166
|
+
// comment post.
|
|
167
|
+
for (const g of generated) {
|
|
168
|
+
const src = g.spec.source_issue?.match(/^#?(\d+)$/)?.[1];
|
|
169
|
+
if (!src)
|
|
170
|
+
continue;
|
|
171
|
+
const testCount = g.manifest.tests.length;
|
|
172
|
+
const fileParts = [];
|
|
173
|
+
if (g.testPath)
|
|
174
|
+
fileParts.push(`\`${g.testPath}\``);
|
|
175
|
+
if (g.uiTestPath)
|
|
176
|
+
fileParts.push(`\`${g.uiTestPath}\``);
|
|
177
|
+
const modeNote = g.mode === "ui-only"
|
|
178
|
+
? " *(UI tests only — handler tests already merged)*"
|
|
179
|
+
: g.mode === "full"
|
|
180
|
+
? " *(handler + UI tests)*"
|
|
181
|
+
: "";
|
|
182
|
+
const body = `### slowcook · tests opened\n\n` +
|
|
183
|
+
`[PR #${pr.number}](${pr.url}) — \`story-${g.spec.story_id}\`, ${testCount} test(s) in ${fileParts.join(" + ")}${modeNote}.\n\n` +
|
|
184
|
+
`Review the test shape + stubs, merge when ready. Merge triggers \`slowcook-brew-auto\`.\n\n` +
|
|
185
|
+
`---\n*Generated by \`slowcook testgen\`.*`;
|
|
186
|
+
try {
|
|
187
|
+
await ctx.forge.createIssueComment(parseInt(src, 10), body);
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
/* best effort — audit trail is nice-to-have */
|
|
191
|
+
}
|
|
192
|
+
}
|
|
128
193
|
return {
|
|
129
194
|
kind: "tests-emitted",
|
|
130
195
|
storyIds,
|
|
@@ -177,25 +242,51 @@ function shouldWriteStub(repoRoot, path) {
|
|
|
177
242
|
function shouldWriteHelper(repoRoot, path) {
|
|
178
243
|
return !existsSync(join(repoRoot, path));
|
|
179
244
|
}
|
|
245
|
+
function handlerTestPathFor(storyId) {
|
|
246
|
+
return join(TESTS_INTEGRATION_DIR, `story-${storyId}.test.ts`);
|
|
247
|
+
}
|
|
248
|
+
function uiTestPathFor(storyId) {
|
|
249
|
+
return join(TESTS_INTEGRATION_DIR, `story-${storyId}-ui.test.tsx`);
|
|
250
|
+
}
|
|
251
|
+
function specHasUiBehavior(spec) {
|
|
252
|
+
return !!spec.ui_behavior && Object.keys(spec.ui_behavior).length > 0;
|
|
253
|
+
}
|
|
180
254
|
function collectTargetSpecs(ctx) {
|
|
181
255
|
const index = readIndex(ctx.repoRoot);
|
|
182
256
|
const all = Object.entries(index.stories)
|
|
183
257
|
.filter(([, entry]) => entry.status === "active")
|
|
184
258
|
.map(([id]) => id);
|
|
185
259
|
const targetIds = ctx.specId ? [ctx.specId] : all;
|
|
186
|
-
const
|
|
260
|
+
const targets = [];
|
|
187
261
|
for (const id of targetIds) {
|
|
188
|
-
const
|
|
189
|
-
|
|
190
|
-
|
|
262
|
+
const handlerTestAbs = join(ctx.repoRoot, handlerTestPathFor(id));
|
|
263
|
+
const uiTestAbs = join(ctx.repoRoot, uiTestPathFor(id));
|
|
264
|
+
const handlerExists = existsSync(handlerTestAbs);
|
|
265
|
+
const uiExists = existsSync(uiTestAbs);
|
|
266
|
+
let spec;
|
|
191
267
|
try {
|
|
192
|
-
|
|
268
|
+
spec = readSpec(ctx.repoRoot, id);
|
|
193
269
|
}
|
|
194
270
|
catch {
|
|
195
271
|
// spec file missing despite being in index — skip
|
|
272
|
+
continue;
|
|
196
273
|
}
|
|
274
|
+
const hasUi = specHasUiBehavior(spec);
|
|
275
|
+
const needsHandler = !handlerExists;
|
|
276
|
+
const needsUi = hasUi && !uiExists;
|
|
277
|
+
if (!needsHandler && !needsUi) {
|
|
278
|
+
// Already has everything — skip unless explicit --spec requests it
|
|
279
|
+
if (!ctx.specId)
|
|
280
|
+
continue;
|
|
281
|
+
// With --spec, we still skip if nothing's missing; there's nothing to do.
|
|
282
|
+
continue;
|
|
283
|
+
}
|
|
284
|
+
const mode = needsHandler && needsUi ? "full" :
|
|
285
|
+
needsHandler ? "handler-only" :
|
|
286
|
+
"ui-only";
|
|
287
|
+
targets.push({ spec, mode });
|
|
197
288
|
}
|
|
198
|
-
return
|
|
289
|
+
return targets;
|
|
199
290
|
}
|
|
200
291
|
export function buildProjectContext(repoRoot) {
|
|
201
292
|
const bits = [];
|
|
@@ -255,6 +346,21 @@ export function buildProjectContext(repoRoot) {
|
|
|
255
346
|
bits.push(`- … (${routes.length - 50} more)`);
|
|
256
347
|
}
|
|
257
348
|
}
|
|
349
|
+
// List existing React components + client-side pages so the LLM knows
|
|
350
|
+
// NOT to emit a <ui_stub> block for a component that already exists.
|
|
351
|
+
// Components live at src/components/**; client pages live at
|
|
352
|
+
// src/app/**/page.tsx (or layout.tsx, though we don't usually brew layouts).
|
|
353
|
+
const componentsDir = join(repoRoot, "src", "components");
|
|
354
|
+
const pagesUnderApp = existsSync(appDir) ? listReactComponents(appDir) : [];
|
|
355
|
+
const libComponents = existsSync(componentsDir) ? listReactComponents(componentsDir) : [];
|
|
356
|
+
const allComponents = [...libComponents, ...pagesUnderApp].sort();
|
|
357
|
+
if (allComponents.length > 0) {
|
|
358
|
+
bits.push(`\n### Existing React component / page files (tsx)\n\nThese already exist — do NOT emit a \`<ui_stub>\` block for any of them. If a UI test imports one of these, assume it's real code and skip stub generation.`);
|
|
359
|
+
for (const c of allComponents.slice(0, 50))
|
|
360
|
+
bits.push(`- \`${c}\``);
|
|
361
|
+
if (allComponents.length > 50)
|
|
362
|
+
bits.push(`- … (${allComponents.length - 50} more)`);
|
|
363
|
+
}
|
|
258
364
|
// List existing mock helpers so the LLM knows which to import. The
|
|
259
365
|
// helper pattern is load-bearing for the future record-and-replay swap
|
|
260
366
|
// (plans/0.7-testgen-two-tier.md §4.3). Helpers NOT listed here will
|
|
@@ -294,6 +400,45 @@ export function buildProjectContext(repoRoot) {
|
|
|
294
400
|
* testgen LLM so it doesn't emit a <stub> for a file that already
|
|
295
401
|
* exists.
|
|
296
402
|
*/
|
|
403
|
+
/**
|
|
404
|
+
* Walk a React directory and return every `.tsx` file's repo-relative
|
|
405
|
+
* path. Used to surface existing components + client pages so testgen
|
|
406
|
+
* doesn't emit <ui_stub> blocks for files that already have real
|
|
407
|
+
* implementations.
|
|
408
|
+
*/
|
|
409
|
+
function listReactComponents(dir) {
|
|
410
|
+
const out = [];
|
|
411
|
+
const walk = (d) => {
|
|
412
|
+
let entries;
|
|
413
|
+
try {
|
|
414
|
+
entries = readdirSync(d);
|
|
415
|
+
}
|
|
416
|
+
catch {
|
|
417
|
+
return;
|
|
418
|
+
}
|
|
419
|
+
for (const name of entries) {
|
|
420
|
+
if (name === "node_modules" || name.startsWith("."))
|
|
421
|
+
continue;
|
|
422
|
+
const full = join(d, name);
|
|
423
|
+
let stat;
|
|
424
|
+
try {
|
|
425
|
+
stat = statSync(full);
|
|
426
|
+
}
|
|
427
|
+
catch {
|
|
428
|
+
continue;
|
|
429
|
+
}
|
|
430
|
+
if (stat.isDirectory())
|
|
431
|
+
walk(full);
|
|
432
|
+
else if (stat.isFile() && name.endsWith(".tsx") && !name.endsWith(".test.tsx")) {
|
|
433
|
+
const srcIdx = full.indexOf("src/");
|
|
434
|
+
const rel = srcIdx >= 0 ? full.slice(srcIdx) : full;
|
|
435
|
+
out.push(rel);
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
};
|
|
439
|
+
walk(dir);
|
|
440
|
+
return out;
|
|
441
|
+
}
|
|
297
442
|
function listAppRouterFiles(appDir) {
|
|
298
443
|
const out = [];
|
|
299
444
|
const walk = (dir) => {
|
|
@@ -326,9 +471,15 @@ function listAppRouterFiles(appDir) {
|
|
|
326
471
|
walk(appDir);
|
|
327
472
|
return out;
|
|
328
473
|
}
|
|
329
|
-
async function generateTestBundle(spec, ctx, projectContext) {
|
|
474
|
+
async function generateTestBundle(spec, ctx, projectContext, mode) {
|
|
330
475
|
const systemPrompt = TESTGEN_SYSTEM(projectContext);
|
|
331
|
-
const
|
|
476
|
+
const modeInstruction = {
|
|
477
|
+
"full": "Generate BOTH the handler tier-1 test bundle (`<test_file>` + any `<stub>` + any `<helper>`) AND the UI tier-1 bundle (`<ui_test_file>` + any `<ui_stub>`). This story has both API and UI scope.",
|
|
478
|
+
"handler-only": "Generate the handler tier-1 test bundle (`<test_file>` + any `<stub>` + any `<helper>`). This story has no `ui_behavior`, so do NOT emit `<ui_test_file>` or `<ui_stub>` blocks.",
|
|
479
|
+
"ui-only": "Handler tests already exist for this story. Emit ONLY the UI tier-1 bundle: `<ui_test_file>` + any `<ui_stub>` blocks needed to make the UI tests collect. Do NOT emit `<test_file>`, `<stub>`, or `<helper>` blocks — those artifacts are already on disk and should not be regenerated.",
|
|
480
|
+
};
|
|
481
|
+
const userMessage = `${modeInstruction[mode]}\n\n` +
|
|
482
|
+
`Here is the spec YAML:\n\n\`\`\`yaml\n${YAML.stringify(spec)}\n\`\`\``;
|
|
332
483
|
const raw = await ctx.llm.complete({
|
|
333
484
|
system: systemPrompt,
|
|
334
485
|
cacheSystem: true,
|
|
@@ -336,34 +487,46 @@ async function generateTestBundle(spec, ctx, projectContext) {
|
|
|
336
487
|
messages: [{ role: "user", content: userMessage }],
|
|
337
488
|
maxTokens: 16384,
|
|
338
489
|
});
|
|
339
|
-
return parseTestgenBundle(raw, spec.story_id);
|
|
490
|
+
return parseTestgenBundle(raw, spec.story_id, mode);
|
|
340
491
|
}
|
|
341
492
|
/**
|
|
342
493
|
* Parse XML-tagged multi-artifact output into a TestgenBundle.
|
|
343
494
|
*
|
|
344
495
|
* Accepted shape (from the prompt):
|
|
345
|
-
* <test_file>...</test_file>
|
|
346
|
-
* <stub path="src/app/api/foo/route.ts">...</stub>
|
|
347
|
-
* <helper path="tests/helpers/mocks/bar.ts">...</helper>
|
|
496
|
+
* <test_file>...</test_file> — required unless mode="ui-only"
|
|
497
|
+
* <stub path="src/app/api/foo/route.ts">...</stub> (zero or more)
|
|
498
|
+
* <helper path="tests/helpers/mocks/bar.ts">...</helper> (zero or more)
|
|
499
|
+
* <ui_test_file>...</ui_test_file> — required when mode="ui-only" or "full" with UI
|
|
500
|
+
* <ui_stub path="src/components/foo.tsx">...</ui_stub> (zero or more)
|
|
501
|
+
*
|
|
502
|
+
* Mode semantics (0.7.7+):
|
|
503
|
+
* - `"handler-only"` — `<test_file>` required; `<ui_test_file>` ignored if present.
|
|
504
|
+
* - `"ui-only"` — `<ui_test_file>` required; `<test_file>` ignored if present.
|
|
505
|
+
* - `"full"` — both `<test_file>` required AND `<ui_test_file>` required.
|
|
348
506
|
*
|
|
349
507
|
* Tolerant of code-fenced output: if the LLM wraps the whole thing in
|
|
350
508
|
* ```, we strip it. If a block's contents are themselves code-fenced,
|
|
351
|
-
* we strip those too — tier-1 test / helper / stub files are raw TS.
|
|
352
|
-
*
|
|
353
|
-
* Throws if `<test_file>` is missing or empty — that's the one mandatory
|
|
354
|
-
* artifact.
|
|
509
|
+
* we strip those too — tier-1 test / helper / stub / UI files are raw TS/TSX.
|
|
355
510
|
*/
|
|
356
|
-
export function parseTestgenBundle(raw, storyId) {
|
|
511
|
+
export function parseTestgenBundle(raw, storyId, mode = "handler-only") {
|
|
357
512
|
const trimmed = raw.trim();
|
|
358
513
|
// Strip outer code fence if the LLM wrapped everything
|
|
359
514
|
const outerFenceMatch = trimmed.match(/^```[a-z]*\s*\n([\s\S]*)\n```$/);
|
|
360
515
|
const body = outerFenceMatch && outerFenceMatch[1] ? outerFenceMatch[1] : trimmed;
|
|
516
|
+
const handlerRequired = mode !== "ui-only";
|
|
517
|
+
const uiRequired = mode !== "handler-only";
|
|
361
518
|
const testMatch = body.match(/<test_file>([\s\S]*?)<\/test_file>/);
|
|
362
|
-
if (!testMatch || !testMatch[1]) {
|
|
363
|
-
throw new Error(`testgen: LLM output for story-${storyId} missing a <test_file> block. ` +
|
|
519
|
+
if (handlerRequired && (!testMatch || !testMatch[1])) {
|
|
520
|
+
throw new Error(`testgen: LLM output for story-${storyId} missing a <test_file> block (mode=${mode}). ` +
|
|
364
521
|
`Got ${body.length} chars starting with: ${body.slice(0, 120)}...`);
|
|
365
522
|
}
|
|
366
|
-
const testContent = stripInnerFence(testMatch[1]);
|
|
523
|
+
const testContent = testMatch && testMatch[1] ? stripInnerFence(testMatch[1]) : "";
|
|
524
|
+
const uiTestMatch = body.match(/<ui_test_file>([\s\S]*?)<\/ui_test_file>/);
|
|
525
|
+
if (uiRequired && (!uiTestMatch || !uiTestMatch[1])) {
|
|
526
|
+
throw new Error(`testgen: LLM output for story-${storyId} missing a <ui_test_file> block (mode=${mode}). ` +
|
|
527
|
+
`Got ${body.length} chars starting with: ${body.slice(0, 120)}...`);
|
|
528
|
+
}
|
|
529
|
+
const uiTestContent = uiTestMatch && uiTestMatch[1] ? stripInnerFence(uiTestMatch[1]) : "";
|
|
367
530
|
const stubs = [];
|
|
368
531
|
const stubRe = /<stub\s+path="([^"]+)">([\s\S]*?)<\/stub>/g;
|
|
369
532
|
let m;
|
|
@@ -381,7 +544,15 @@ export function parseTestgenBundle(raw, storyId) {
|
|
|
381
544
|
if (p && c.trim())
|
|
382
545
|
helpers.push({ path: p, contents: stripInnerFence(c) });
|
|
383
546
|
}
|
|
384
|
-
|
|
547
|
+
const uiStubs = [];
|
|
548
|
+
const uiStubRe = /<ui_stub\s+path="([^"]+)">([\s\S]*?)<\/ui_stub>/g;
|
|
549
|
+
while ((m = uiStubRe.exec(body)) !== null) {
|
|
550
|
+
const p = m[1] ?? "";
|
|
551
|
+
const c = m[2] ?? "";
|
|
552
|
+
if (p && c.trim())
|
|
553
|
+
uiStubs.push({ path: p, contents: stripInnerFence(c) });
|
|
554
|
+
}
|
|
555
|
+
return { testContent, stubs, helpers, uiTestContent, uiStubs };
|
|
385
556
|
}
|
|
386
557
|
function stripInnerFence(raw) {
|
|
387
558
|
const t = raw.trim();
|
|
@@ -468,6 +639,24 @@ export function lintTierOneTest(filePath, source) {
|
|
|
468
639
|
* Robust against typical usage; falls back to a single synthetic entry if
|
|
469
640
|
* nothing recognisable is found.
|
|
470
641
|
*/
|
|
642
|
+
/**
|
|
643
|
+
* Read an existing test file from disk and extract its test IDs. Thin
|
|
644
|
+
* wrapper around extractTestIdsFromFile for the "ui-only" code path where
|
|
645
|
+
* the handler test already exists and we need its IDs to preserve in the
|
|
646
|
+
* rewritten combined manifest.
|
|
647
|
+
*/
|
|
648
|
+
export function extractTestIdsFromExistingFile(repoRoot, filePath) {
|
|
649
|
+
const abs = join(repoRoot, filePath);
|
|
650
|
+
if (!existsSync(abs))
|
|
651
|
+
return [];
|
|
652
|
+
try {
|
|
653
|
+
const source = readFileSync(abs, "utf8");
|
|
654
|
+
return extractTestIdsFromFile(filePath, source);
|
|
655
|
+
}
|
|
656
|
+
catch {
|
|
657
|
+
return [];
|
|
658
|
+
}
|
|
659
|
+
}
|
|
471
660
|
export function extractTestIdsFromFile(filePath, source) {
|
|
472
661
|
// Strip comments and blank string-literal contents so commented-out or
|
|
473
662
|
// string-embedded `describe`/`it` don't register. Offsets are preserved
|
|
@@ -612,7 +801,22 @@ function buildPrBody(args) {
|
|
|
612
801
|
sections.push("## Tests added");
|
|
613
802
|
for (const g of args.generated) {
|
|
614
803
|
const manifestCount = g.manifest.tests.length;
|
|
615
|
-
|
|
804
|
+
const parts = [];
|
|
805
|
+
if (g.testPath)
|
|
806
|
+
parts.push(`\`${g.testPath}\``);
|
|
807
|
+
if (g.uiTestPath)
|
|
808
|
+
parts.push(`\`${g.uiTestPath}\``);
|
|
809
|
+
const modeTag = g.mode === "ui-only" ? " *(ui-only mode — handler tests already present)*" : g.mode === "full" ? " *(handler + UI)*" : "";
|
|
810
|
+
sections.push(`- \`story-${g.spec.story_id}\` — *${g.spec.title}* — ${manifestCount} test(s) in ${parts.join(" + ")}${modeTag}`);
|
|
811
|
+
}
|
|
812
|
+
const allUiStubs = args.generated.flatMap((g) => g.uiStubs);
|
|
813
|
+
if (allUiStubs.length > 0) {
|
|
814
|
+
sections.push("");
|
|
815
|
+
sections.push("## Generated UI stubs (React components / pages)");
|
|
816
|
+
sections.push("Minimal placeholder components so tier-1 UI tests can collect. Each carries an \`@slowcook-stub\` marker on line 1. **Brewing will replace these bodies** with the real component implementation. Reviewer check: correct file path + default export present + \`@slowcook-stub\` marker intact. Signature-wrong = PR-wrong — flag it now.");
|
|
817
|
+
for (const s of allUiStubs) {
|
|
818
|
+
sections.push(`- \`${s.path}\``);
|
|
819
|
+
}
|
|
616
820
|
}
|
|
617
821
|
const allStubs = args.generated.flatMap((g) => g.stubs);
|
|
618
822
|
if (allStubs.length > 0) {
|