@activate-core/test-forge 2.3.1 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -17
- package/dist/cli/index.js +12 -2
- package/dist/core/docs/discover.js +62 -40
- package/dist/core/docs/doc-check.js +257 -0
- package/dist/core/docs/doc-guidance.js +3 -176
- package/dist/core/docs/grounding.js +37 -58
- package/dist/core/docs/reference-text.js +24 -6
- package/dist/core/docs/reference.js +23 -41
- package/dist/core/docs/review.js +17 -40
- package/dist/core/gates/validate.js +78 -0
- package/dist/core/generate/ai-generator.js +138 -87
- package/dist/core/generate/ai-usage.js +9 -1
- package/dist/core/generate/defect-review.js +69 -66
- package/dist/core/generate/doc-diff.js +25 -1
- package/dist/core/generate/harness.js +184 -210
- package/dist/core/generate/journey-memory.js +39 -3
- package/dist/core/generate/run-generate.js +9 -8
- package/dist/core/generate/seed-generator.js +56 -65
- package/dist/core/generate/update-journey.js +3 -1
- package/dist/core/project/init.js +98 -0
- package/dist/core/project/setup.js +15 -20
- package/dist/core/prompts/blocks.js +340 -0
- package/dist/core/run/name-scan.js +14 -6
- package/dist/core/run/run-tests.js +21 -1
- package/package.json +1 -1
- package/reporters/_shared.cjs +32 -1
- package/reporters/jest-reporter.cjs +7 -6
- package/reporters/playwright-reporter.cjs +5 -5
package/README.md
CHANGED
|
@@ -93,7 +93,7 @@ never touched.
|
|
|
93
93
|
|
|
94
94
|
| Command | What it does |
|
|
95
95
|
| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------- |
|
|
96
|
-
| `testforge setup` | One-time setup:
|
|
96
|
+
| `testforge setup` | One-time setup: one call writes the config and the map, then discover drafts the docs, with progress. |
|
|
97
97
|
| `testforge generate` | Writes the tests for each journey doc. A changed doc updates only the tests of the rules that changed. Writes only: it runs nothing. |
|
|
98
98
|
| `testforge update` | Refreshes the map, docs and tests when the **source code** or the **reference** has changed. Skips everything if neither has (no AI calls). Broad by design — for one journey, see below. |
|
|
99
99
|
|
|
@@ -104,6 +104,7 @@ never touched.
|
|
|
104
104
|
| `testforge init` | Inspect the project and write `testforge.config.json` only. |
|
|
105
105
|
| `testforge map` | Rebuild the project map (`testforge.md`) only. |
|
|
106
106
|
| `testforge discover` | Redraft the per-journey docs in `docs/` only. |
|
|
107
|
+
| `testforge discover --check --journey <name>` | Check a doc that already exists (a hand-written one too) and add what it lacks — the baseline cases its controls need, the branches the code shows. Nothing is redrafted. |
|
|
107
108
|
| `testforge env` | Rebuild `.env.testing.example` from the docs only. |
|
|
108
109
|
| `testforge run` | Run the generated tests (`--journey` / `--area` / `--env-file`). First fills the names tests wait for from the code (only where each kind of name lives, only when those lines changed); `--no-scan` skips that. |
|
|
109
110
|
| `testforge bind` | After a feature ships: which test files the code is ready for, which wait for the feature, which use a name the code spells differently. `--accept "doc=code"` takes the code's wording. No AI. |
|
|
@@ -148,20 +149,45 @@ No source changes detected.
|
|
|
148
149
|
|
|
149
150
|
## How it works
|
|
150
151
|
|
|
151
|
-
**setup** prepares the project in
|
|
152
|
+
**setup** prepares the project in two stages:
|
|
152
153
|
|
|
153
|
-
1. **
|
|
154
|
-
live, JS or TS) and writes `testforge.config.json
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
154
|
+
1. **config and map, in one call** — detects the layout (monorepo or single
|
|
155
|
+
app, where tests live, JS or TS) and writes `testforge.config.json`, then
|
|
156
|
+
`testforge.md`: a short list of journeys and a summary of the data layer
|
|
157
|
+
(database, models, key fields). The config is held to a closed list of
|
|
158
|
+
keys, and a framework other than `jest` or `playwright` is refused.
|
|
159
|
+
2. **discover** — writes one doc per journey into `docs/`.
|
|
160
|
+
|
|
161
|
+
**The journey doc.** Every criterion is one line with its own id, its tier and
|
|
162
|
+
the controls it covers:
|
|
163
|
+
|
|
164
|
+
```
|
|
165
|
+
- R12 [spec] [C3] The operator offered first is the top of the ranking. (source: "Offer to the top operator…")
|
|
166
|
+
- R13 [baseline] (must, security) [C3] Another customer's booking id is refused.
|
|
167
|
+
- R14 [observed] [C5] The postcode takes 5 to 8 characters.
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
`[spec]` comes from the reference and quotes it; `[baseline]` is the robustness
|
|
171
|
+
every app owes, which TestForge adds itself; `[observed]` is a rule the code
|
|
172
|
+
fixes where the reference is silent. Depth follows risk: a payment flow gets
|
|
173
|
+
its states, calculations and idempotency, a static page only its controls and
|
|
174
|
+
the baseline cases that fit them. A doc written before this format (no ids)
|
|
175
|
+
is still read: its rules are numbered R1, R2 … in reading order, as before.
|
|
176
|
+
|
|
177
|
+
**discover** drafts a doc in one call, then **checks** it in a second: the
|
|
178
|
+
check answers with a JSON list of fixes (add, edit, retag, remove), and
|
|
179
|
+
TestForge applies them — the doc is never rewritten wholesale, so a sound
|
|
180
|
+
criterion cannot drop out unnoticed. Then, without AI: every `[spec]` quote is
|
|
181
|
+
looked up in the reference (one that is not there is re-tagged `[observed]`),
|
|
182
|
+
and the Coverage table is built from the `[C…]` on each criterion; a control
|
|
183
|
+
nothing covers is handed to the check as a gap. `discover --check` runs the
|
|
184
|
+
same check on a doc that already exists, hand-written or not.
|
|
161
185
|
|
|
162
186
|
**generate** turns each doc into a test. A journey with `Type: api`
|
|
163
|
-
becomes a Jest test
|
|
164
|
-
|
|
187
|
+
becomes a Jest test that sends every request through the testkit's
|
|
188
|
+
`kit.call()` — in-process for a Node server, over HTTP to the running server
|
|
189
|
+
otherwise — and checks stored state only through the testkit; `Type: web`
|
|
190
|
+
becomes a Playwright test. Other areas use the framework set in config. It also
|
|
165
191
|
generates a seed file that provisions the test data each test needs.
|
|
166
192
|
|
|
167
193
|
**generate** hands each file it writes to the project's own ESLint with
|
|
@@ -384,12 +410,16 @@ history, the plan, and the code facts. No later step sends the whole doc again.
|
|
|
384
410
|
element with `kit.find(page, kind, name)`, check outcomes rather than markup,
|
|
385
411
|
and stub a state they cannot create.
|
|
386
412
|
5. **Checks before anything runs.** A file is refused, and written once more
|
|
387
|
-
with
|
|
413
|
+
with its own code and each problem on its line, if it sleeps for a fixed time, skips because an element is
|
|
388
414
|
missing, has no test or no assertion, spends more than `maxTestsPerRule`
|
|
389
415
|
tests on one rule, or names a label found neither in the code nor the doc.
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
416
|
+
Comments outside the policy (the header, one `// oracle:` line per test,
|
|
417
|
+
`// cannot-test:` lines, a `// timeout:` reason, a note on a parametrised
|
|
418
|
+
row) are dropped from the answer, without a second call.
|
|
419
|
+
6. **Every rule, a test that can fail.** Every scenario comes from the doc —
|
|
420
|
+
the edge, security and user cases TestForge adds are written into the doc
|
|
421
|
+
as `[baseline]` criteria first, where a person can read them — and what the
|
|
422
|
+
code has or lacks never removes one. Each test is checked on its own: every planned
|
|
393
423
|
rule has a test, or — only when it needs an outside service the test
|
|
394
424
|
cannot drive (a payment provider's inside, an email inbox, another app) —
|
|
395
425
|
a `// cannot-test: R7 — <the service>` line;
|
|
@@ -421,7 +451,20 @@ words. Nothing found, or the same lines as last time: no AI call. New lines:
|
|
|
421
451
|
one small call without tools picks each value, kept only if the code really
|
|
422
452
|
has it, and printed with what was found. Tests, docs and plans are never
|
|
423
453
|
touched. A test whose name is still blank is reported as "waiting for the
|
|
424
|
-
product", not as a defect
|
|
454
|
+
product", not as a defect — also when the name is asked for at the top of the
|
|
455
|
+
file, outside any test. `--no-scan` skips the look-up.
|
|
456
|
+
|
|
457
|
+
**Before the run, the testkit's packages.** A testkit lists what it needs
|
|
458
|
+
(`REQUIRED_PACKAGES`, a database client say). When the project lacks one,
|
|
459
|
+
`run` stops that area before running anything and prints the `npm i -D` to
|
|
460
|
+
run, instead of every test failing as if the product were broken. A test that
|
|
461
|
+
still hits it is reported as skipped with that reason. The same goes for the
|
|
462
|
+
testkit's sign-in check: it uses an endpoint no journey tests, and when the
|
|
463
|
+
server answers it with an error (a 5xx, a 429) the tests that need that role are
|
|
464
|
+
reported as not started, with the status and the endpoint, not as failing their
|
|
465
|
+
rules. `run` also says when
|
|
466
|
+
`TESTFORGE_KEY` is set but `dashboardUrl` is empty, so a run that will not be
|
|
467
|
+
published is not a surprise.
|
|
425
468
|
|
|
426
469
|
Every run of `generate` ends with its AI cost per step (write, grounding, seed, harness …).
|
|
427
470
|
|
package/dist/cli/index.js
CHANGED
|
@@ -29,7 +29,7 @@ import { envDiff, describeEnvDiff } from '../core/project/env-diff.js';
|
|
|
29
29
|
import { bundleTickets } from '../core/docs/tickets.js';
|
|
30
30
|
import { writeGithubWorkflow } from '../core/run/ci.js';
|
|
31
31
|
import { runTests } from '../core/run/run-tests.js';
|
|
32
|
-
import { runDiscover } from '../core/docs/discover.js';
|
|
32
|
+
import { checkExistingDoc, runDiscover } from '../core/docs/discover.js';
|
|
33
33
|
import { runMap } from '../core/project/map.js';
|
|
34
34
|
import { runInit } from '../core/project/init.js';
|
|
35
35
|
import { runSetup } from '../core/project/setup.js';
|
|
@@ -98,7 +98,7 @@ function printUsage() {
|
|
|
98
98
|
console.log(' testforge generate --journey <name> Generate just ONE journey');
|
|
99
99
|
console.log(' testforge generate --area <area> Generate a whole area (e.g. api)');
|
|
100
100
|
console.log(' testforge discover Draft journey docs (asks: map or reference)');
|
|
101
|
-
console.log(' testforge discover --journey <name> Draft just ONE journey doc', ' testforge discover --journey <name> --reference <x> [--refresh] One journey from a reference: the reference is read once and kept (--refresh reads it again)');
|
|
101
|
+
console.log(' testforge discover --journey <name> Draft just ONE journey doc', ' testforge discover --check --journey <name> Check an existing doc (hand-written too) and add what it lacks; nothing is redrafted', ' testforge discover --journey <name> --reference <x> [--refresh] One journey from a reference: the reference is read once and kept (--refresh reads it again)');
|
|
102
102
|
console.log(' testforge discover --reference <file|url> Base docs on a reference');
|
|
103
103
|
console.log(' testforge discover --tickets <folder> Base docs on exported tickets (one file per ticket); each source names its ticket');
|
|
104
104
|
console.log(' (a reference only writes the journeys it');
|
|
@@ -195,6 +195,16 @@ async function main() {
|
|
|
195
195
|
}
|
|
196
196
|
console.log('\nVerdicts written to testforge/runs/defect-review/. The next run publishes them with each failed test.');
|
|
197
197
|
}
|
|
198
|
+
else if (command === 'discover' && process.argv.includes('--check')) {
|
|
199
|
+
// An existing doc (hand-written or older): add what it lacks, rewrite nothing.
|
|
200
|
+
const journey = flagValue('--journey');
|
|
201
|
+
if (!journey) {
|
|
202
|
+
console.log('Usage: testforge discover --check --journey <name>');
|
|
203
|
+
}
|
|
204
|
+
else {
|
|
205
|
+
console.log(checkExistingDoc('docs', journey, '.', process.argv.includes('--refresh')));
|
|
206
|
+
}
|
|
207
|
+
}
|
|
198
208
|
else if (command === 'discover') {
|
|
199
209
|
console.log('Drafting journey files with AI...');
|
|
200
210
|
console.log('This writes drafts into docs/ from your chosen source.\n');
|
|
@@ -17,12 +17,14 @@
|
|
|
17
17
|
import { existsSync, readdirSync, readFileSync, writeFileSync, mkdirSync, renameSync, } from 'node:fs';
|
|
18
18
|
import { join, relative, sep, basename } from 'node:path';
|
|
19
19
|
import { notAJourneyDoc } from './parser.js';
|
|
20
|
-
import { loadConfig, splitCommand, withEditTools } from '../project/config.js';
|
|
20
|
+
import { loadConfig, splitCommand, withEditTools, withNoTools } from '../project/config.js';
|
|
21
21
|
import { DEEP_DOC_GUIDANCE } from './doc-guidance.js';
|
|
22
22
|
import { promptConfirm } from '../prompt.js';
|
|
23
23
|
import { loadSourceChoice, resolveDocsSource, saveJourneyReferences, saveSourceChoice, } from './source-of-truth.js';
|
|
24
24
|
import { analyzeReference, buildJourneyDraftPrompt, draftDocsFromReference, draftJourneyDoc, DIFF_PATH, } from './reference.js';
|
|
25
|
-
import {
|
|
25
|
+
import { enforceSpecSources, reviewDocs } from './review.js';
|
|
26
|
+
import { applyFixes, buildDocCheckPrompt, coverageGaps, parseFixes, verifySpecQuotes, withCoverage } from './doc-check.js';
|
|
27
|
+
import { readRules } from '../generate/journey-memory.js';
|
|
26
28
|
import { readReference, sectionFor } from './reference-text.js';
|
|
27
29
|
import { sourceExcerpts } from './excerpts.js';
|
|
28
30
|
import { runAiSync } from '../generate/ai-usage.js';
|
|
@@ -67,36 +69,14 @@ export function buildDiscoverPrompt(srcDir, docsDir, journey) {
|
|
|
67
69
|
`shared note that every journey should know about goes under`,
|
|
68
70
|
`"${docsDir}/_reference/" and is never a journey.`,
|
|
69
71
|
``,
|
|
70
|
-
`Each file
|
|
71
|
-
`this header (required):`,
|
|
72
|
+
`Each file starts with this header (required):`,
|
|
72
73
|
``,
|
|
73
74
|
`# Journey: <Name>`,
|
|
74
|
-
``,
|
|
75
75
|
`Type: web (use "web" for a page/UI, "api" for an endpoint)`,
|
|
76
76
|
`Route: <path> (the URL path or endpoint, e.g. /login)`,
|
|
77
|
+
`Coverage: functional, security (add others that apply: payment, session, …)`,
|
|
77
78
|
``,
|
|
78
|
-
`
|
|
79
|
-
`- functional`,
|
|
80
|
-
`- security`,
|
|
81
|
-
` (add more if relevant: performance, accessibility, session,`,
|
|
82
|
-
` race-condition, payment, webhook, data-integrity)`,
|
|
83
|
-
``,
|
|
84
|
-
`Then include whichever of these sections actually apply (skip the`,
|
|
85
|
-
`ones that do not):`,
|
|
86
|
-
``,
|
|
87
|
-
`Fields: each input as: <name> (<type>, required|optional)`,
|
|
88
|
-
`Actions: numbered steps the user/caller takes`,
|
|
89
|
-
`Expected: what should happen on success`,
|
|
90
|
-
`Negative Cases: things that should be rejected or fail safely`,
|
|
91
|
-
`Race Conditions: concurrency risks (e.g. double-submit, duplicate charge)`,
|
|
92
|
-
`Webhooks: any webhooks involved and what must hold (idempotency, signatures)`,
|
|
93
|
-
`Security Notes: specific threats to test for this journey`,
|
|
94
|
-
`Data: which models/collections this journey reads or writes,`,
|
|
95
|
-
` and the key fields/enum values a test would need to`,
|
|
96
|
-
` seed or verify (use the data layer from the map)`,
|
|
97
|
-
``,
|
|
98
|
-
`Also add "Test setup" and "Questions for product (not asserted)" last,`,
|
|
99
|
-
`as described above.`,
|
|
79
|
+
`Then the sections and criterion lines exactly as FORMAT above describes.`,
|
|
100
80
|
``,
|
|
101
81
|
`Rules:`,
|
|
102
82
|
`- Base everything on real evidence in the code and the map. Do not`,
|
|
@@ -314,19 +294,7 @@ async function draftOneJourney(srcDir, docsDir, reference, journey, refresh = fa
|
|
|
314
294
|
const draft = draftJourneyDoc(buildJourneyDraftPrompt(journey, section, excerpts, areas, others));
|
|
315
295
|
if (!draft.markdown)
|
|
316
296
|
throw new Error('The AI answered without a journey doc.');
|
|
317
|
-
|
|
318
|
-
let doc = draft.markdown;
|
|
319
|
-
try {
|
|
320
|
-
const reviewed = draftJourneyDoc(buildLeanReviewPrompt(doc, section, excerpts, others));
|
|
321
|
-
if (reviewed.markdown)
|
|
322
|
-
doc = reviewed.markdown;
|
|
323
|
-
}
|
|
324
|
-
catch (e) {
|
|
325
|
-
console.log(` review failed, kept as drafted (${e.message.split('\n')[0]})`);
|
|
326
|
-
}
|
|
327
|
-
const { markdown, retagged } = enforceSpecSources(doc);
|
|
328
|
-
if (retagged)
|
|
329
|
-
console.log(` ${retagged} [spec] line(s) without a quoted source re-tagged [observed]`);
|
|
297
|
+
const markdown = checkAndFinish(draft.markdown, text, section, excerpts, others);
|
|
330
298
|
const rel = existing ?? join(draft.area && areas.includes(draft.area) ? draft.area : areas[0], `${kebab(docTitle(markdown) ?? journey)}.md`);
|
|
331
299
|
mkdirSync(join(docsDir, rel, '..'), { recursive: true });
|
|
332
300
|
writeFileSync(join(docsDir, rel), markdown, 'utf-8');
|
|
@@ -334,3 +302,57 @@ async function draftOneJourney(srcDir, docsDir, reference, journey, refresh = fa
|
|
|
334
302
|
saveJourneyReferences([rel], reference);
|
|
335
303
|
return `${existing ? 'Rewrote' : 'Wrote'} ${join(docsDir, rel)} from the reference. Please review it, then run \`testforge generate --journey ${kebab(journey)}\`.`;
|
|
336
304
|
}
|
|
305
|
+
/**
|
|
306
|
+
* The doc check and the checks that need no model, in order: the model lists
|
|
307
|
+
* fixes for the doc (with the controls nothing covers yet), the code applies
|
|
308
|
+
* them, every [spec] quote is looked up in the reference, a [spec] with no
|
|
309
|
+
* quote at all is re-tagged, and the Coverage table is rebuilt from the doc.
|
|
310
|
+
*/
|
|
311
|
+
function checkAndFinish(doc, referenceText, section, excerpts, others) {
|
|
312
|
+
let markdown = doc;
|
|
313
|
+
const gaps = coverageGaps(readRules(markdown));
|
|
314
|
+
console.log(`Checking it for missing scenarios (one call, no tools${gaps.length ? `; ${gaps.length} control(s) with no criterion yet` : ''})...`);
|
|
315
|
+
try {
|
|
316
|
+
const { program, args } = withNoTools(splitCommand(loadConfig().aiCommand));
|
|
317
|
+
const out = runAiSync(program, args, { input: buildDocCheckPrompt(markdown, section, excerpts, gaps, others), encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 });
|
|
318
|
+
const fixes = parseFixes(out);
|
|
319
|
+
const res = applyFixes(markdown, fixes);
|
|
320
|
+
markdown = res.markdown;
|
|
321
|
+
console.log(` ${res.applied} fix(es) applied${res.refused.length ? `, ${res.refused.length} refused: ${res.refused.slice(0, 3).join('; ')}${res.refused.length > 3 ? ' …' : ''}` : ''}`);
|
|
322
|
+
}
|
|
323
|
+
catch (e) {
|
|
324
|
+
console.log(` check failed, kept as drafted (${e.message.split('\n')[0]})`);
|
|
325
|
+
}
|
|
326
|
+
const quoted = verifySpecQuotes(markdown, referenceText);
|
|
327
|
+
markdown = quoted.markdown;
|
|
328
|
+
if (quoted.retagged.length)
|
|
329
|
+
console.log(` ${quoted.retagged.length} [spec] quote(s) not found in the reference, re-tagged [observed]: ${quoted.retagged.slice(0, 8).join(', ')}`);
|
|
330
|
+
const sourced = enforceSpecSources(markdown);
|
|
331
|
+
markdown = sourced.markdown;
|
|
332
|
+
if (sourced.retagged)
|
|
333
|
+
console.log(` ${sourced.retagged} [spec] line(s) without a quoted source re-tagged [observed]`);
|
|
334
|
+
return `${withCoverage(markdown).trimEnd()}\n`;
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* Check a doc that already exists — one a person wrote, or one drafted before —
|
|
338
|
+
* and add what it lacks: the baseline items its controls need, the branches the
|
|
339
|
+
* code shows. Nothing is redrafted; the doc's own criteria stay as written.
|
|
340
|
+
*/
|
|
341
|
+
export function checkExistingDoc(docsDir, journey, srcDir, refresh = false) {
|
|
342
|
+
const config = loadConfig();
|
|
343
|
+
const docs = [...snapshotDocs(docsDir)].filter(([p, t]) => !notAJourneyDoc(p, t));
|
|
344
|
+
const hit = docs.find(([p, t]) => kebab(docTitle(t) ?? '') === kebab(journey) || p.replace(/\.md$/, '').endsWith(kebab(journey)));
|
|
345
|
+
if (!hit)
|
|
346
|
+
throw new Error(`No journey doc matches "${journey}" under ${docsDir}/.`);
|
|
347
|
+
const [rel, doc] = hit;
|
|
348
|
+
const stored = loadSourceChoice();
|
|
349
|
+
const reference = stored?.journeys?.[rel] ?? (stored?.mode === 'reference' ? stored.reference : undefined);
|
|
350
|
+
const text = reference ? readReference(reference, refresh) : '';
|
|
351
|
+
const section = text ? sectionFor(text, docTitle(doc) ?? journey) : '';
|
|
352
|
+
const excerpts = sourceExcerpts(section || doc, [srcDir, ...(config.relatedDirs ?? [])]);
|
|
353
|
+
const others = docs.filter(([p]) => p !== rel).map(([p, t]) => `${docTitle(t) ?? p} (${p})`);
|
|
354
|
+
console.log(`\nChecking ${rel}${reference ? ` against ${reference}` : ' (no reference: its own [spec] lines stand)'}...`);
|
|
355
|
+
const markdown = checkAndFinish(doc, text, section, excerpts, others);
|
|
356
|
+
writeFileSync(join(docsDir, rel), markdown, 'utf-8');
|
|
357
|
+
return `Checked ${join(docsDir, rel)}. Review the added criteria, then run \`testforge generate --journey ${kebab(journey)}\`.`;
|
|
358
|
+
}
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* testforge — check a journey doc, and fix it in code.
|
|
3
|
+
*
|
|
4
|
+
* The check used to answer with the whole doc rewritten: thousands of output
|
|
5
|
+
* tokens to change a few lines, and a sound criterion could drop out on the
|
|
6
|
+
* way without anyone noticing. Now the model lists what is wrong or missing
|
|
7
|
+
* as JSON, and the code below applies each fix to the doc as it stands. A fix
|
|
8
|
+
* that names a rule the doc does not have, or a line with no tier, is refused
|
|
9
|
+
* and reported, never guessed at.
|
|
10
|
+
*
|
|
11
|
+
* Three checks need no model at all: the Coverage table is built from the
|
|
12
|
+
* controls each criterion names, a control nothing covers is handed to the
|
|
13
|
+
* check as a gap, and every [spec] quote is looked up in the reference.
|
|
14
|
+
*/
|
|
15
|
+
import { BASELINE, DOC_GUIDANCE, TIERS } from '../prompts/blocks.js';
|
|
16
|
+
import { readRules } from '../generate/journey-memory.js';
|
|
17
|
+
/** The prompt that checks ONE journey doc and answers with fixes, not a new doc. */
|
|
18
|
+
export function buildDocCheckPrompt(doc, reference, excerpts, gaps, otherDocs = []) {
|
|
19
|
+
return [
|
|
20
|
+
`You check ONE journey doc against its reference and code excerpts, and list`,
|
|
21
|
+
`what is wrong or missing. You do not rewrite the doc. You have no tools.`,
|
|
22
|
+
``,
|
|
23
|
+
TIERS,
|
|
24
|
+
``,
|
|
25
|
+
BASELINE,
|
|
26
|
+
``,
|
|
27
|
+
DOC_GUIDANCE,
|
|
28
|
+
``,
|
|
29
|
+
`Check that:`,
|
|
30
|
+
`1. Every interactive element in the excerpts is in Controls.`,
|
|
31
|
+
`2. Every baseline item that fits each control is present, with its level and`,
|
|
32
|
+
` category.`,
|
|
33
|
+
`3. Every combination the reference states follows Method step 2.`,
|
|
34
|
+
`4. Every branch the excerpts show is an [observed] rule or a Question.`,
|
|
35
|
+
`5. Every [observed] value is right: run the pattern, and follow any middleware`,
|
|
36
|
+
` that runs before validation.`,
|
|
37
|
+
`6. Every [spec] quotes a real sentence of the reference.`,
|
|
38
|
+
`7. Nothing states a policy, formula, state or limit that neither the`,
|
|
39
|
+
` reference nor the excerpts show.`,
|
|
40
|
+
...(gaps.length ? [``, `These controls have no criterion yet: ${gaps.join(', ')}. Add the ones that fit them.`] : []),
|
|
41
|
+
...(otherDocs.length ? [``, `Rules owned by other journeys stay with them:`, ...otherDocs.map((d) => ` - ${d}`)] : []),
|
|
42
|
+
``,
|
|
43
|
+
`Output only this JSON:`,
|
|
44
|
+
`{"fixes": [{"action": "add|retag|edit|remove",`,
|
|
45
|
+
` "rule": "R<n>, or empty for add",`,
|
|
46
|
+
` "section": "<section heading>",`,
|
|
47
|
+
` "line": "<the full new criterion line, or empty for remove>",`,
|
|
48
|
+
` "why": "<short reason>"}]}`,
|
|
49
|
+
`If nothing needs changing, output {"fixes": []}.`,
|
|
50
|
+
``,
|
|
51
|
+
`<reference>`,
|
|
52
|
+
reference || '(no separate reference: the doc itself is the reference for [spec])',
|
|
53
|
+
`</reference>`,
|
|
54
|
+
`<source_excerpts>`,
|
|
55
|
+
excerpts || '(none found)',
|
|
56
|
+
`</source_excerpts>`,
|
|
57
|
+
`<draft>`,
|
|
58
|
+
doc,
|
|
59
|
+
`</draft>`,
|
|
60
|
+
].join('\n');
|
|
61
|
+
}
|
|
62
|
+
/** The fixes in a check's answer; anything that is not one is dropped. */
|
|
63
|
+
export function parseFixes(out) {
|
|
64
|
+
const at = out.indexOf('{');
|
|
65
|
+
const end = out.lastIndexOf('}');
|
|
66
|
+
if (at < 0 || end <= at)
|
|
67
|
+
throw new Error('the doc check answered without JSON');
|
|
68
|
+
const json = JSON.parse(out.slice(at, end + 1));
|
|
69
|
+
if (!Array.isArray(json.fixes))
|
|
70
|
+
throw new Error('the doc check answered without a "fixes" list');
|
|
71
|
+
return json.fixes.filter((f) => !!f && typeof f === 'object' && ['add', 'retag', 'edit', 'remove'].includes(f.action));
|
|
72
|
+
}
|
|
73
|
+
const TAGGED = /\[(spec|baseline|observed)\]/i;
|
|
74
|
+
const norm = (t) => t.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
|
|
75
|
+
/** Line index of each rule's first line, by id. */
|
|
76
|
+
function ruleLines(lines, memory) {
|
|
77
|
+
const at = new Map();
|
|
78
|
+
let from = 0;
|
|
79
|
+
for (const r of memory.rules) {
|
|
80
|
+
if (!r.tag)
|
|
81
|
+
continue;
|
|
82
|
+
const first = r.text.split('\n')[0].trim();
|
|
83
|
+
for (let i = from; i < lines.length; i++) {
|
|
84
|
+
const body = lines[i].replace(/^\s*(?:[-*]|\d+(?:\.\d+)*\.?)\s+/, '').replace(/^R\d+\s*[:.]?\s+/, '').trim();
|
|
85
|
+
if (body === first) {
|
|
86
|
+
at.set(r.id, i);
|
|
87
|
+
from = i + 1;
|
|
88
|
+
break;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return at;
|
|
93
|
+
}
|
|
94
|
+
/** The lines that belong to the rule starting at `i`: itself, its indented lines, its "Likely defect" note. */
|
|
95
|
+
function blockEnd(lines, i) {
|
|
96
|
+
const indent = /^(\s*)/.exec(lines[i])[1].length;
|
|
97
|
+
let j = i + 1;
|
|
98
|
+
while (j < lines.length) {
|
|
99
|
+
const l = lines[j];
|
|
100
|
+
if (/^\s*Likely defect:/i.test(l) || (l.trim() && /^(\s*)/.exec(l)[1].length > indent && !/^#/.test(l.trim()))) {
|
|
101
|
+
j += 1;
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
break;
|
|
105
|
+
}
|
|
106
|
+
return j;
|
|
107
|
+
}
|
|
108
|
+
const asBullet = (line, indent = '') => `${indent}- ${line.trim().replace(/^[-*]\s+/, '')}`;
|
|
109
|
+
/**
|
|
110
|
+
* Apply a check's fixes to the doc. Returns the new doc, how many fixes were
|
|
111
|
+
* applied, and why the others were refused.
|
|
112
|
+
*/
|
|
113
|
+
export function applyFixes(markdown, fixes) {
|
|
114
|
+
let lines = markdown.split('\n');
|
|
115
|
+
const refused = [];
|
|
116
|
+
let applied = 0;
|
|
117
|
+
const memory = readRules(markdown);
|
|
118
|
+
const ids = new Set(memory.rules.filter((r) => r.tag).map((r) => r.id));
|
|
119
|
+
let top = Math.max(0, ...memory.rules.map((r) => Number(r.id.slice(1)) || 0));
|
|
120
|
+
// Edits and removals by the line each rule starts on, bottom first, so the
|
|
121
|
+
// indexes above stay right; additions after, by section.
|
|
122
|
+
const where = ruleLines(lines, memory);
|
|
123
|
+
const byRule = fixes.filter((f) => f.action !== 'add');
|
|
124
|
+
const removed = new Set();
|
|
125
|
+
for (const f of byRule.sort((a, b) => (where.get(b.rule ?? '') ?? -1) - (where.get(a.rule ?? '') ?? -1))) {
|
|
126
|
+
const id = (f.rule ?? '').trim().toUpperCase();
|
|
127
|
+
const i = where.get(id);
|
|
128
|
+
if (!id || !ids.has(id) || i === undefined || removed.has(id)) {
|
|
129
|
+
refused.push(`${f.action} ${id || '(no rule)'}: the doc has no such rule`);
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
if (f.action === 'remove') {
|
|
133
|
+
lines.splice(i, blockEnd(lines, i) - i);
|
|
134
|
+
removed.add(id);
|
|
135
|
+
applied += 1;
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
if (!f.line || !TAGGED.test(f.line)) {
|
|
139
|
+
refused.push(`${f.action} ${id}: the new line has no tier`);
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
const indent = /^(\s*)/.exec(lines[i])[1];
|
|
143
|
+
// The rule keeps its id: an edited line names it whether the model wrote it or not.
|
|
144
|
+
const body = f.line.trim().replace(/^[-*]\s+/, '').replace(/^R\d+\s*[:.]?\s+/, '');
|
|
145
|
+
const named = /^\s*[-*]\s+R\d+\s/.test(lines[i]);
|
|
146
|
+
lines[i] = `${indent}- ${named ? `${id} ` : ''}${body}`;
|
|
147
|
+
applied += 1;
|
|
148
|
+
}
|
|
149
|
+
for (const f of fixes.filter((x) => x.action === 'add')) {
|
|
150
|
+
if (!f.line || !TAGGED.test(f.line)) {
|
|
151
|
+
refused.push(`add: the new line has no tier`);
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
const body = f.line.trim().replace(/^[-*]\s+/, '').replace(/^R\d+\s*[:.]?\s+/, '');
|
|
155
|
+
const line = asBullet(`R${++top} ${body}`);
|
|
156
|
+
const want = norm((f.section ?? '').replace(/^#+\s*/, '').replace(/^\d+(?:\.\d+)*\.?\s*/, ''));
|
|
157
|
+
const headings = lines.map((l, i) => ({ l, i })).filter((h) => /^##\s/.test(h.l));
|
|
158
|
+
const hit = want
|
|
159
|
+
? headings.find((h) => { const t = norm(h.l.replace(/^##\s*/, '').replace(/^\d+(?:\.\d+)*\.?\s*/, '')); return t && (t === want || t.includes(want) || want.includes(t)); })
|
|
160
|
+
: undefined;
|
|
161
|
+
const tail = /^##\s+(?:coverage|not requirements|test setup|questions for product)/i;
|
|
162
|
+
if (hit && !tail.test(hit.l) && !/^##\s+controls/i.test(hit.l)) {
|
|
163
|
+
let end = headings.find((h) => h.i > hit.i)?.i ?? lines.length;
|
|
164
|
+
while (end > hit.i + 1 && !lines[end - 1].trim())
|
|
165
|
+
end -= 1;
|
|
166
|
+
lines.splice(end, 0, line);
|
|
167
|
+
}
|
|
168
|
+
else {
|
|
169
|
+
// A section the doc does not have yet: a new numbered one, before the closing sections.
|
|
170
|
+
const n = Math.max(0, ...headings.map((h) => Number(/^##\s+(\d+)/.exec(h.l)?.[1] ?? 0))) + 1;
|
|
171
|
+
const before = headings.find((h) => tail.test(h.l))?.i ?? lines.length;
|
|
172
|
+
const title = (f.section ?? 'Further criteria').replace(/^#+\s*/, '').replace(/^\d+(?:\.\d+)*\.?\s*/, '').trim() || 'Further criteria';
|
|
173
|
+
lines.splice(before, 0, `## ${n}. ${title}`, '', line, '');
|
|
174
|
+
}
|
|
175
|
+
applied += 1;
|
|
176
|
+
}
|
|
177
|
+
lines = lines.join('\n').replace(/\n{3,}/g, '\n\n').split('\n');
|
|
178
|
+
return { markdown: lines.join('\n'), applied, refused };
|
|
179
|
+
}
|
|
180
|
+
/** Does the doc tie its criteria to controls at all? A doc written before [C<ids>] does not. */
|
|
181
|
+
const namesControls = (memory) => memory.rules.some((r) => r.tag && r.controls.length);
|
|
182
|
+
/** Controls no criterion names. A doc whose criteria name no control has no gaps to report: it predates [C<ids>]. */
|
|
183
|
+
export function coverageGaps(memory) {
|
|
184
|
+
if (!namesControls(memory))
|
|
185
|
+
return [];
|
|
186
|
+
const covered = new Set(memory.rules.filter((r) => r.tag).flatMap((r) => r.controls));
|
|
187
|
+
return Object.keys(memory.controls).filter((c) => !covered.has(c));
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* The doc with its "## Coverage" table built from the controls each criterion
|
|
191
|
+
* names, in place of any table the model wrote. A doc with no Controls table
|
|
192
|
+
* is returned as it is.
|
|
193
|
+
*/
|
|
194
|
+
export function withCoverage(markdown) {
|
|
195
|
+
const memory = readRules(markdown);
|
|
196
|
+
const controls = Object.keys(memory.controls);
|
|
197
|
+
// No controls, or criteria that name none (a doc from before [C<ids>]): nothing to build from.
|
|
198
|
+
if (!controls.length || !namesControls(memory))
|
|
199
|
+
return markdown;
|
|
200
|
+
const lines = markdown.split('\n');
|
|
201
|
+
// Drop the old table, whoever wrote it.
|
|
202
|
+
const at = lines.findIndex((l) => /^##\s+coverage\b/i.test(l));
|
|
203
|
+
if (at >= 0) {
|
|
204
|
+
let end = lines.findIndex((l, i) => i > at && /^##\s/.test(l));
|
|
205
|
+
if (end < 0)
|
|
206
|
+
end = lines.length;
|
|
207
|
+
lines.splice(at, end - at);
|
|
208
|
+
}
|
|
209
|
+
const table = [
|
|
210
|
+
'## Coverage',
|
|
211
|
+
'',
|
|
212
|
+
'Built by TestForge from the controls each criterion names.',
|
|
213
|
+
'',
|
|
214
|
+
'| Control | Criteria |',
|
|
215
|
+
'|---|---|',
|
|
216
|
+
...controls.map((c) => {
|
|
217
|
+
const rs = memory.rules.filter((r) => r.tag && r.controls.includes(c)).map((r) => r.id);
|
|
218
|
+
return `| ${c} ${memory.controls[c] ?? ''} | ${rs.length ? rs.join(', ') : 'none yet'} |`;
|
|
219
|
+
}),
|
|
220
|
+
'',
|
|
221
|
+
];
|
|
222
|
+
const before = lines.findIndex((l) => /^##\s+(?:not requirements|test setup|questions for product)/i.test(l));
|
|
223
|
+
lines.splice(before >= 0 ? before : lines.length, 0, ...table);
|
|
224
|
+
return lines.join('\n').replace(/\n{3,}/g, '\n\n');
|
|
225
|
+
}
|
|
226
|
+
/** The quoted words of a [spec] line's source, one entry per quote. */
|
|
227
|
+
function quotesOf(line) {
|
|
228
|
+
const src = /\(source:[^"“]*["“]([\s\S]*?)["”]\s*\)/i.exec(line);
|
|
229
|
+
return src ? [src[1]] : [];
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* A [spec] line whose quote is not in the reference is the drafter's reading,
|
|
233
|
+
* not the client's words: it is re-tagged [observed]. A quote with "…" in it
|
|
234
|
+
* is checked piece by piece. Without a reference text nothing is checked.
|
|
235
|
+
*/
|
|
236
|
+
export function verifySpecQuotes(markdown, reference) {
|
|
237
|
+
const ref = ` ${norm(reference)} `;
|
|
238
|
+
const retagged = [];
|
|
239
|
+
if (!reference.trim())
|
|
240
|
+
return { markdown, retagged };
|
|
241
|
+
let inFence = false;
|
|
242
|
+
const out = markdown.split('\n').map((line) => {
|
|
243
|
+
if (/^\s*```/.test(line))
|
|
244
|
+
inFence = !inFence;
|
|
245
|
+
if (inFence || !/\[spec\]/i.test(line))
|
|
246
|
+
return line;
|
|
247
|
+
const quotes = quotesOf(line);
|
|
248
|
+
if (!quotes.length)
|
|
249
|
+
return line; // "(source as above)" and the like: enforceSpecSources handles a missing quote
|
|
250
|
+
const pieces = quotes.flatMap((q) => q.split(/…|\.\.\./)).map(norm).filter((p) => p.split(' ').length >= 3);
|
|
251
|
+
if (pieces.every((p) => ref.includes(` ${p} `)))
|
|
252
|
+
return line;
|
|
253
|
+
retagged.push(/R\d+/.exec(line)?.[0] ?? line.trim().slice(0, 40));
|
|
254
|
+
return line.replace(/\[spec\]/gi, '[observed]');
|
|
255
|
+
});
|
|
256
|
+
return { markdown: out.join('\n'), retagged };
|
|
257
|
+
}
|