@planu/cli 5.2.0 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/config/environment-schema.json +161 -21
- package/dist/config/release-policy.json +0 -8
- package/dist/engine/planu-core.darwin-arm64.node.manifest.json +7 -7
- package/dist/engine/planu-core.darwin-arm64.node.sbom.json +4 -4
- package/dist/engine/planu-core.darwin-x64.node.manifest.json +7 -7
- package/dist/engine/planu-core.darwin-x64.node.sbom.json +4 -4
- package/dist/engine/planu-core.linux-arm64-gnu.node.manifest.json +7 -7
- package/dist/engine/planu-core.linux-arm64-gnu.node.sbom.json +4 -4
- package/dist/engine/planu-core.linux-arm64-musl.node.manifest.json +7 -7
- package/dist/engine/planu-core.linux-arm64-musl.node.sbom.json +4 -4
- package/dist/engine/planu-core.linux-x64-gnu.node.manifest.json +7 -7
- package/dist/engine/planu-core.linux-x64-gnu.node.sbom.json +4 -4
- package/dist/engine/planu-core.linux-x64-musl.node.manifest.json +7 -7
- package/dist/engine/planu-core.linux-x64-musl.node.sbom.json +4 -4
- package/dist/engine/planu-core.win32-arm64-msvc.node.manifest.json +7 -7
- package/dist/engine/planu-core.win32-arm64-msvc.node.sbom.json +4 -4
- package/dist/engine/planu-core.win32-x64-msvc.node.manifest.json +7 -7
- package/dist/engine/planu-core.win32-x64-msvc.node.sbom.json +4 -4
- package/dist/engine/validation/validation-freshness.d.ts +20 -1
- package/dist/engine/validation/validation-freshness.js +181 -21
- package/dist/engine/validator/analyzer.d.ts +5 -2
- package/dist/engine/validator/analyzer.js +81 -23
- package/dist/engine/validator/deep-code-checker.d.ts +10 -0
- package/dist/engine/validator/deep-code-checker.js +33 -0
- package/dist/engine/validator/test-evidence-grounding.d.ts +56 -0
- package/dist/engine/validator/test-evidence-grounding.js +187 -0
- package/dist/engine/validator.js +15 -4
- package/dist/tools/validate.js +6 -2
- package/dist/types/analysis.d.ts +2 -0
- package/dist/types/validation-receipt.d.ts +7 -0
- package/package.json +20 -20
- package/planu-native.json +1 -1
- package/planu-plugin.json +1 -1
|
@@ -50,6 +50,7 @@ async function buildBroadCache(projectPath) {
|
|
|
50
50
|
}
|
|
51
51
|
import { deepCheckCriterion } from './deep-code-checker.js';
|
|
52
52
|
import { ConfigLoader } from '../config-loader.js';
|
|
53
|
+
import { filterValidatedTestPaths, groundNumericLiteral, createProbeCache, groundAgainstDeclaredTests, findBacktickIdentifierEvidence, } from './test-evidence-grounding.js';
|
|
53
54
|
// ---------------------------------------------------------------------------
|
|
54
55
|
// SPEC-209: Drift severity config (loaded from src/config/drift-severity.json)
|
|
55
56
|
// ---------------------------------------------------------------------------
|
|
@@ -155,6 +156,50 @@ export async function scanCodeForSpec(spec, projectPath) {
|
|
|
155
156
|
}
|
|
156
157
|
return { affectedFiles, unexpectedFiles, fileContents };
|
|
157
158
|
}
|
|
159
|
+
/**
|
|
160
|
+
* Search codeState (then a broader project-wide cache) for a component/function
|
|
161
|
+
* identifier. Extracted from checkCriterionEvidence to keep its complexity bounded.
|
|
162
|
+
* This is the pre-existing (legacy, unquoted/single/double-quoted) search path —
|
|
163
|
+
* unchanged by SPEC-1367.
|
|
164
|
+
*/
|
|
165
|
+
async function findComponentEvidence(name, projectPath, codeState, sharedFileContents) {
|
|
166
|
+
// Search in already-loaded codeState contents first (zero extra I/O)
|
|
167
|
+
for (const [, content] of codeState.fileContents) {
|
|
168
|
+
if (content.includes(name)) {
|
|
169
|
+
return { status: 'proven' };
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
// Broader search: reuse sharedFileContents when provided to avoid a
|
|
173
|
+
// second glob+readFile pass. Falls back to a fresh (capped) read only
|
|
174
|
+
// when no shared cache is available.
|
|
175
|
+
const broadContents = sharedFileContents ?? (await buildBroadCache(projectPath));
|
|
176
|
+
for (const [, content] of broadContents) {
|
|
177
|
+
if (content.includes(name)) {
|
|
178
|
+
return { status: 'proven' };
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
return { status: 'missing' };
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Match a criterion's "component/function/class/module/service NAME" shape
|
|
185
|
+
* and ground it, or return null when no such shape is present. Extracted
|
|
186
|
+
* from checkCriterionEvidence to keep its complexity bounded.
|
|
187
|
+
*/
|
|
188
|
+
async function checkComponentMatch(criterion, projectPath, codeState, sharedFileContents, safeTestPaths) {
|
|
189
|
+
// Legacy (pre-existing) quote forms only. Unchanged from pre-SPEC-1367 behavior.
|
|
190
|
+
const legacyComponentMatch = /(?:component|function|class|module|service)\s+["']?(\w+)["']?/i.exec(criterion);
|
|
191
|
+
if (legacyComponentMatch?.[1]) {
|
|
192
|
+
return findComponentEvidence(legacyComponentMatch[1], projectPath, codeState, sharedFileContents);
|
|
193
|
+
}
|
|
194
|
+
// SPEC-1367: identifiers are frequently backtick-quoted in BDD THEN clauses
|
|
195
|
+
// (e.g. "function `classifyValidationWatchEventForTests`"). This quote form
|
|
196
|
+
// is NEW — see findBacktickIdentifierEvidence for its restricted scope.
|
|
197
|
+
const backtickComponentMatch = /(?:component|function|class|module|service)\s+`(\w+)`/i.exec(criterion);
|
|
198
|
+
if (backtickComponentMatch?.[1]) {
|
|
199
|
+
return findBacktickIdentifierEvidence(backtickComponentMatch[1], projectPath, codeState, safeTestPaths);
|
|
200
|
+
}
|
|
201
|
+
return null;
|
|
202
|
+
}
|
|
158
203
|
/**
|
|
159
204
|
* Check whether a single criterion is satisfied in the given code state.
|
|
160
205
|
* When `verifyBlock` is provided it is forwarded to deepCheckCriterion,
|
|
@@ -164,7 +209,15 @@ export async function scanCodeForSpec(spec, projectPath) {
|
|
|
164
209
|
* buildFileContentsCache from criterion-matcher) to avoid a redundant
|
|
165
210
|
* glob+readFile pass when checking multiple criteria in the same call.
|
|
166
211
|
*/
|
|
167
|
-
export async function checkCriterionEvidence(criterion, projectPath, codeState, verifyBlock, sharedFileContents) {
|
|
212
|
+
export async function checkCriterionEvidence(criterion, projectPath, codeState, verifyBlock, sharedFileContents, declaredTestPaths, probeCache) {
|
|
213
|
+
// SPEC-1367 review finding 1: validate declared test paths ONCE, up front —
|
|
214
|
+
// they reach both a readFile call and a spawnSync argv further down. Every
|
|
215
|
+
// grounding path below uses `safeTestPaths`, never the raw parameter, and
|
|
216
|
+
// an empty result here (no declared tests, or all rejected) makes every
|
|
217
|
+
// new grounding path a no-op, per review finding 4.
|
|
218
|
+
const safeTestPaths = declaredTestPaths && declaredTestPaths.length > 0
|
|
219
|
+
? await filterValidatedTestPaths(projectPath, declaredTestPaths)
|
|
220
|
+
: [];
|
|
168
221
|
// Check for file existence criteria
|
|
169
222
|
const fileMatch = /(?:create|add|implement)\s+(?:file\s+)?["']?([^\s"']+\.\w+)["']?/i.exec(criterion);
|
|
170
223
|
if (fileMatch?.[1]) {
|
|
@@ -176,26 +229,10 @@ export async function checkCriterionEvidence(criterion, projectPath, codeState,
|
|
|
176
229
|
return { status: 'missing' };
|
|
177
230
|
}
|
|
178
231
|
}
|
|
179
|
-
// Check for component/function existence in code
|
|
180
|
-
const
|
|
181
|
-
if (
|
|
182
|
-
|
|
183
|
-
// Search in already-loaded codeState contents first (zero extra I/O)
|
|
184
|
-
for (const [, content] of codeState.fileContents) {
|
|
185
|
-
if (content.includes(name)) {
|
|
186
|
-
return { status: 'proven' };
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
// Broader search: reuse sharedFileContents when provided to avoid a
|
|
190
|
-
// second glob+readFile pass. Falls back to a fresh (capped) read only
|
|
191
|
-
// when no shared cache is available.
|
|
192
|
-
const broadContents = sharedFileContents ?? (await buildBroadCache(projectPath));
|
|
193
|
-
for (const [, content] of broadContents) {
|
|
194
|
-
if (content.includes(name)) {
|
|
195
|
-
return { status: 'proven' };
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
return { status: 'missing' };
|
|
232
|
+
// Check for component/function existence in code (legacy + SPEC-1367 backtick forms)
|
|
233
|
+
const componentResult = await checkComponentMatch(criterion, projectPath, codeState, sharedFileContents, safeTestPaths);
|
|
234
|
+
if (componentResult) {
|
|
235
|
+
return componentResult;
|
|
199
236
|
}
|
|
200
237
|
// Check for API endpoint criteria
|
|
201
238
|
const endpointMatch = /(?:endpoint|route|api)\s+(?:for\s+)?["']?(\w+)["']?/i.exec(criterion);
|
|
@@ -206,6 +243,16 @@ export async function checkCriterionEvidence(criterion, projectPath, codeState,
|
|
|
206
243
|
}
|
|
207
244
|
}
|
|
208
245
|
}
|
|
246
|
+
// SPEC-1367 AC1: a declared-test-owned criterion citing a numeric literal
|
|
247
|
+
// (e.g. a timeout) is grounded against its own declared file(s) first — this
|
|
248
|
+
// takes priority over the generic keyword classifier below so the result
|
|
249
|
+
// carries a concrete evidence citation instead of a bare pass/fail.
|
|
250
|
+
if (safeTestPaths.length > 0) {
|
|
251
|
+
const literalGrounded = await groundNumericLiteral(criterion, projectPath, safeTestPaths);
|
|
252
|
+
if (literalGrounded) {
|
|
253
|
+
return literalGrounded;
|
|
254
|
+
}
|
|
255
|
+
}
|
|
209
256
|
// Deep code check: grep/metric/coverage analysis on the actual filesystem.
|
|
210
257
|
// If a verify block is present it is passed explicitly (bypasses keyword inference).
|
|
211
258
|
// For non-manual checks, the deep result is authoritative.
|
|
@@ -214,6 +261,17 @@ export async function checkCriterionEvidence(criterion, projectPath, codeState,
|
|
|
214
261
|
if (deepResult.checkType !== 'manual') {
|
|
215
262
|
return { status: deepResult.passed ? 'proven' : 'missing' };
|
|
216
263
|
}
|
|
264
|
+
// SPEC-1367: ground test-only criteria against their declared TEST file(s)
|
|
265
|
+
// (numeric-literal match, then a bounded evidence probe) before dead-ending
|
|
266
|
+
// at indeterminate/missing.
|
|
267
|
+
if (safeTestPaths.length > 0) {
|
|
268
|
+
// Review finding 2: without a caller-provided per-run cache, fall back to
|
|
269
|
+
// a call-local one — correctness never depends on cross-call reuse.
|
|
270
|
+
const grounded = groundAgainstDeclaredTests(projectPath, safeTestPaths, probeCache ?? createProbeCache());
|
|
271
|
+
if (grounded) {
|
|
272
|
+
return grounded;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
217
275
|
// Affected files alone show scope, not proof. Keep this visible so validate
|
|
218
276
|
// cannot report score 100 for manual/unclassified criteria without evidence.
|
|
219
277
|
if (codeState.affectedFiles.length > 0) {
|
|
@@ -221,8 +279,8 @@ export async function checkCriterionEvidence(criterion, projectPath, codeState,
|
|
|
221
279
|
}
|
|
222
280
|
return { status: 'missing' };
|
|
223
281
|
}
|
|
224
|
-
export async function checkCriterion(criterion, projectPath, codeState, verifyBlock, sharedFileContents) {
|
|
225
|
-
return ((await checkCriterionEvidence(criterion, projectPath, codeState, verifyBlock, sharedFileContents)).status === 'proven');
|
|
282
|
+
export async function checkCriterion(criterion, projectPath, codeState, verifyBlock, sharedFileContents, declaredTestPaths) {
|
|
283
|
+
return ((await checkCriterionEvidence(criterion, projectPath, codeState, verifyBlock, sharedFileContents, declaredTestPaths)).status === 'proven');
|
|
226
284
|
}
|
|
227
285
|
/**
|
|
228
286
|
* Classify the severity of a missing criterion based on its content.
|
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
import type { DeepCheckResult, DeepCheckConfig, DeepCheckType, VerifyBlock } from '../../types/index.js';
|
|
2
2
|
export type { DeepCheckResult, DeepCheckConfig, DeepCheckType };
|
|
3
|
+
/** Strip underscore separators so "45_000" and "45000" compare equal. */
|
|
4
|
+
export declare function normalizeNumericLiteral(literal: string): string;
|
|
5
|
+
/**
|
|
6
|
+
* Build a regex source that matches a numeric literal regardless of any
|
|
7
|
+
* underscore separators present in the target text (e.g. "45000" also
|
|
8
|
+
* matches "45_000", "4_5000", etc. — any grouping).
|
|
9
|
+
*/
|
|
10
|
+
export declare function buildNumericLiteralPattern(literal: string): string;
|
|
11
|
+
/** Extract timeout-shaped numeric literals (>= 3 significant digits) from criterion text. */
|
|
12
|
+
export declare function extractNumericLiterals(text: string): string[];
|
|
3
13
|
/**
|
|
4
14
|
* Classify criterion text into a check type using keyword heuristics.
|
|
5
15
|
* Returns the type and optional config derived from the criterion.
|
|
@@ -7,6 +7,32 @@ const DEFAULT_IGNORE = ['node_modules/**', 'dist/**', 'build/**', '.git/**', 'co
|
|
|
7
7
|
function escapeRegex(input) {
|
|
8
8
|
return input.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
9
9
|
}
|
|
10
|
+
// ─── Numeric literal normalization (SPEC-1367) ───────────────────────────────
|
|
11
|
+
// Grounds criteria whose THEN names an explicit numeric timeout (e.g. "45_000")
|
|
12
|
+
// against test files that may spell the same literal without underscores (or
|
|
13
|
+
// vice versa) — JS numeric separators are cosmetic, so the matcher must not
|
|
14
|
+
// dead-end on them.
|
|
15
|
+
/** Strip underscore separators so "45_000" and "45000" compare equal. */
|
|
16
|
+
export function normalizeNumericLiteral(literal) {
|
|
17
|
+
return literal.replace(/_/g, '');
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Build a regex source that matches a numeric literal regardless of any
|
|
21
|
+
* underscore separators present in the target text (e.g. "45000" also
|
|
22
|
+
* matches "45_000", "4_5000", etc. — any grouping).
|
|
23
|
+
*/
|
|
24
|
+
export function buildNumericLiteralPattern(literal) {
|
|
25
|
+
const digits = normalizeNumericLiteral(literal).split('').join('_?');
|
|
26
|
+
// Digit-boundary guards (review finding 3): without them "45_000" also
|
|
27
|
+
// matches inside "145000" or "45_0001". Reject a digit/underscore
|
|
28
|
+
// immediately before or after the run of matched digits.
|
|
29
|
+
return `(?<![\\d_])${digits}(?![\\d_])`;
|
|
30
|
+
}
|
|
31
|
+
/** Extract timeout-shaped numeric literals (>= 3 significant digits) from criterion text. */
|
|
32
|
+
export function extractNumericLiterals(text) {
|
|
33
|
+
const tokens = text.match(/\d[\d_]*/g) ?? [];
|
|
34
|
+
return [...new Set(tokens.filter((token) => normalizeNumericLiteral(token).length >= 3))];
|
|
35
|
+
}
|
|
10
36
|
// ─── Keyword classification ──────────────────────────────────────────────────
|
|
11
37
|
/**
|
|
12
38
|
* Classify criterion text into a check type using keyword heuristics.
|
|
@@ -22,6 +48,13 @@ export function classifyCriterion(text) {
|
|
|
22
48
|
lower.includes('remove all')) {
|
|
23
49
|
return { type: 'grep_absent', config: buildAbsentConfig(text) };
|
|
24
50
|
}
|
|
51
|
+
// SPEC-1367 review: a global numeric_timeout bucket used to live here,
|
|
52
|
+
// grepping tests/**/*.test.ts for ANY criterion mentioning "timeout" plus a
|
|
53
|
+
// number — including specs with no declared test ownership. That let an
|
|
54
|
+
// unrelated test file's literal prove a criterion it was never scoped to.
|
|
55
|
+
// Numeric-literal grounding is now exclusively declared-test-scoped: see
|
|
56
|
+
// `groundNumericLiteral` in analyzer.ts, which only runs when the criterion
|
|
57
|
+
// has declaredTestPaths. No declared tests -> no numeric grounding at all.
|
|
25
58
|
// metric_max: file-size / line-count thresholds
|
|
26
59
|
if (lower.includes('max lines') ||
|
|
27
60
|
lower.includes('no file exceeds') ||
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { CodeState, CriterionEvidenceResult } from '../../types/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* Parse the spec.md frontmatter `scenarios` block and return, per criterion
|
|
4
|
+
* (matched by normalized scenario title), the declared TEST file paths.
|
|
5
|
+
* Reuses the already-tested frontmatter scenario parser from
|
|
6
|
+
* spec-compliance-runner.ts instead of re-implementing YAML parsing here.
|
|
7
|
+
*/
|
|
8
|
+
export declare function extractDeclaredTestPaths(specContent: string): Map<string, string[]>;
|
|
9
|
+
/**
|
|
10
|
+
* SPEC-1367 review finding 2: the probe cache is created per validate
|
|
11
|
+
* invocation (see validateSpec) and threaded through explicitly — never
|
|
12
|
+
* module-level — so concurrent validate runs share no state and a probe
|
|
13
|
+
* result never outlives the run that produced it. Within one run, two
|
|
14
|
+
* criteria that declare the exact same test file(s) legitimately share one
|
|
15
|
+
* probe result — that is the intended "own declared TEST file" scoping.
|
|
16
|
+
*/
|
|
17
|
+
export declare function createProbeCache(): Map<string, {
|
|
18
|
+
passed: boolean;
|
|
19
|
+
evidence: string;
|
|
20
|
+
}>;
|
|
21
|
+
/**
|
|
22
|
+
* SPEC-1367 review finding 1: declared test paths originate from spec.md
|
|
23
|
+
* frontmatter and are used both as a readFile argument and as spawnSync
|
|
24
|
+
* argv — untrusted input reaching both a filesystem read and a subprocess
|
|
25
|
+
* argument list. Reject flag-injection shapes (leading "-"), require
|
|
26
|
+
* *.test.* naming, and require the path to resolve to a real, non-symlink
|
|
27
|
+
* file contained within the project root (traversal-safe).
|
|
28
|
+
*/
|
|
29
|
+
export declare function filterValidatedTestPaths(projectPath: string, testPaths: string[]): Promise<string[]>;
|
|
30
|
+
/**
|
|
31
|
+
* AC1: static grounding — an explicit numeric literal (e.g. a timeout) named
|
|
32
|
+
* in the criterion is present in one of its declared TEST file(s), tolerating
|
|
33
|
+
* underscore separators in either direction. Runs ahead of the generic
|
|
34
|
+
* keyword classifier so declared-test ownership wins and cites evidence.
|
|
35
|
+
*/
|
|
36
|
+
export declare function groundNumericLiteral(criterion: string, projectPath: string, testPaths: string[]): Promise<CriterionEvidenceResult | null>;
|
|
37
|
+
/**
|
|
38
|
+
* Ground a manual (unclassified) criterion against its declared TEST file(s)
|
|
39
|
+
* via a bounded evidence probe, before it dead-ends as indeterminate. Returns
|
|
40
|
+
* null when grounding is inconclusive, leaving the existing affectedFiles-based
|
|
41
|
+
* fallback to decide the final status.
|
|
42
|
+
*/
|
|
43
|
+
export declare function groundAgainstDeclaredTests(projectPath: string, testPaths: string[], probeCache: Map<string, {
|
|
44
|
+
passed: boolean;
|
|
45
|
+
evidence: string;
|
|
46
|
+
}>): CriterionEvidenceResult | null;
|
|
47
|
+
/**
|
|
48
|
+
* SPEC-1367 review finding 5 / AC4: backtick-quoted identifiers are the NEW
|
|
49
|
+
* quote form this spec adds (the legacy regex never matched a backtick, so
|
|
50
|
+
* this path is additive, not a behavior change for existing specs). Its
|
|
51
|
+
* search never falls back to the unrestricted project-wide scan — only the
|
|
52
|
+
* already-loaded codeState (same trust level as the legacy path) and the
|
|
53
|
+
* criterion's OWN validated declared test file(s).
|
|
54
|
+
*/
|
|
55
|
+
export declare function findBacktickIdentifierEvidence(name: string, projectPath: string, codeState: CodeState, safeTestPaths: string[]): Promise<CriterionEvidenceResult>;
|
|
56
|
+
//# sourceMappingURL=test-evidence-grounding.d.ts.map
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
// Planu — Validator: declared-test-owned criterion grounding (SPEC-1367)
|
|
2
|
+
//
|
|
3
|
+
// Criteria whose only ownership evidence is a declared TEST file (frontmatter
|
|
4
|
+
// `scenarios[].tests`) previously dead-ended at 'manual' -> indeterminate.
|
|
5
|
+
// These helpers ground such criteria by (1) reading the declared file(s) for
|
|
6
|
+
// an explicit numeric-literal match, (2) a backtick-quoted identifier search
|
|
7
|
+
// scoped to those files, and (3) failing both, running a bounded evidence
|
|
8
|
+
// probe scoped to exactly those declared files. Extracted from analyzer.ts
|
|
9
|
+
// to keep that file under the subdirectory line-count budget.
|
|
10
|
+
import { readFile } from 'node:fs/promises';
|
|
11
|
+
import { spawnSync } from 'node:child_process';
|
|
12
|
+
import { join } from 'node:path';
|
|
13
|
+
import { extractNumericLiterals, buildNumericLiteralPattern } from './deep-code-checker.js';
|
|
14
|
+
import { parseFrontmatterScenarios } from './spec-compliance-runner.js';
|
|
15
|
+
import { normalizeCriterionText } from '../criterion-identity.js';
|
|
16
|
+
import { resolveContainedProjectFile } from '../safety/contained-project-file.js';
|
|
17
|
+
import { technologyValue } from '../technology-registry.js';
|
|
18
|
+
/**
|
|
19
|
+
* Parse the spec.md frontmatter `scenarios` block and return, per criterion
|
|
20
|
+
* (matched by normalized scenario title), the declared TEST file paths.
|
|
21
|
+
* Reuses the already-tested frontmatter scenario parser from
|
|
22
|
+
* spec-compliance-runner.ts instead of re-implementing YAML parsing here.
|
|
23
|
+
*/
|
|
24
|
+
export function extractDeclaredTestPaths(specContent) {
|
|
25
|
+
const map = new Map();
|
|
26
|
+
for (const scenario of parseFrontmatterScenarios(specContent)) {
|
|
27
|
+
const paths = (scenario.tests ?? []).map((test) => test.path).filter(Boolean);
|
|
28
|
+
if (paths.length === 0) {
|
|
29
|
+
continue;
|
|
30
|
+
}
|
|
31
|
+
const key = normalizeCriterionText(scenario.title);
|
|
32
|
+
map.set(key, [...(map.get(key) ?? []), ...paths]);
|
|
33
|
+
}
|
|
34
|
+
return map;
|
|
35
|
+
}
|
|
36
|
+
const DEFAULT_PROBE_TIMEOUT_MS = 60_000;
|
|
37
|
+
/**
|
|
38
|
+
* SPEC-1367 review finding 2: the probe cache is created per validate
|
|
39
|
+
* invocation (see validateSpec) and threaded through explicitly — never
|
|
40
|
+
* module-level — so concurrent validate runs share no state and a probe
|
|
41
|
+
* result never outlives the run that produced it. Within one run, two
|
|
42
|
+
* criteria that declare the exact same test file(s) legitimately share one
|
|
43
|
+
* probe result — that is the intended "own declared TEST file" scoping.
|
|
44
|
+
*/
|
|
45
|
+
export function createProbeCache() {
|
|
46
|
+
return new Map();
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* SPEC-1367 review finding 1: declared test paths originate from spec.md
|
|
50
|
+
* frontmatter and are used both as a readFile argument and as spawnSync
|
|
51
|
+
* argv — untrusted input reaching both a filesystem read and a subprocess
|
|
52
|
+
* argument list. Reject flag-injection shapes (leading "-"), require
|
|
53
|
+
* *.test.* naming, and require the path to resolve to a real, non-symlink
|
|
54
|
+
* file contained within the project root (traversal-safe).
|
|
55
|
+
*/
|
|
56
|
+
export async function filterValidatedTestPaths(projectPath, testPaths) {
|
|
57
|
+
const valid = [];
|
|
58
|
+
for (const candidate of testPaths) {
|
|
59
|
+
if (candidate.trim() !== candidate ||
|
|
60
|
+
candidate.length === 0 ||
|
|
61
|
+
candidate.startsWith('-') ||
|
|
62
|
+
!/\.test\.[^/\\]+$/.test(candidate)) {
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
try {
|
|
66
|
+
await resolveContainedProjectFile(projectPath, join(projectPath, candidate));
|
|
67
|
+
valid.push(candidate);
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
// outside the project root, a symlink, or missing on disk — reject
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return valid;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Run ONLY the given declared test file(s) as a bounded evidence probe.
|
|
77
|
+
* Safety contract (SPEC-1367):
|
|
78
|
+
* - refuses to start when already inside a probe (PLANU_VALIDATE_PROBE=1) — no recursion
|
|
79
|
+
* - bounded by `timeoutMs` (default 60s, configurable via PLANU_VALIDATE_PROBE_TIMEOUT_MS)
|
|
80
|
+
* - on timeout or crash the caller must treat the criterion as needs-evidence, never passed
|
|
81
|
+
* - cached per (projectPath, sorted test paths) within the caller-supplied
|
|
82
|
+
* per-run cache so criteria sharing the same declared test file(s) do not
|
|
83
|
+
* re-spawn the runner
|
|
84
|
+
*/
|
|
85
|
+
function runEvidenceProbe(testPaths, projectPath, cache, timeoutMs = DEFAULT_PROBE_TIMEOUT_MS) {
|
|
86
|
+
if (process.env.PLANU_VALIDATE_PROBE === '1') {
|
|
87
|
+
return {
|
|
88
|
+
passed: false,
|
|
89
|
+
evidence: 'Evidence probe skipped: already running inside a validate probe (recursion guard).',
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
const cacheKey = `${projectPath}::${[...testPaths].sort().join(',')}`;
|
|
93
|
+
const cached = cache.get(cacheKey);
|
|
94
|
+
if (cached) {
|
|
95
|
+
return cached;
|
|
96
|
+
}
|
|
97
|
+
const pnpmExecutable = process.platform === 'win32' ? 'pnpm.cmd' : 'pnpm';
|
|
98
|
+
const result = spawnSync(pnpmExecutable, ['exec', technologyValue('technology-vitest-a9127f'), 'run', ...testPaths], {
|
|
99
|
+
cwd: projectPath,
|
|
100
|
+
encoding: 'utf8',
|
|
101
|
+
timeout: timeoutMs,
|
|
102
|
+
env: { ...process.env, PLANU_VALIDATE_PROBE: '1' },
|
|
103
|
+
});
|
|
104
|
+
const outcome = result.error
|
|
105
|
+
? { passed: false, evidence: `Evidence probe crashed: ${result.error.message}` }
|
|
106
|
+
: result.status === 0
|
|
107
|
+
? { passed: true, evidence: (result.stdout || result.stderr || '').slice(-2000) }
|
|
108
|
+
: {
|
|
109
|
+
passed: false,
|
|
110
|
+
evidence: `Evidence probe failed (exit ${String(result.status)}): ${(result.stdout || result.stderr || '').slice(-2000)}`,
|
|
111
|
+
};
|
|
112
|
+
cache.set(cacheKey, outcome);
|
|
113
|
+
return outcome;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* AC1: static grounding — an explicit numeric literal (e.g. a timeout) named
|
|
117
|
+
* in the criterion is present in one of its declared TEST file(s), tolerating
|
|
118
|
+
* underscore separators in either direction. Runs ahead of the generic
|
|
119
|
+
* keyword classifier so declared-test ownership wins and cites evidence.
|
|
120
|
+
*/
|
|
121
|
+
export async function groundNumericLiteral(criterion, projectPath, testPaths) {
|
|
122
|
+
const literals = extractNumericLiterals(criterion);
|
|
123
|
+
if (literals.length === 0) {
|
|
124
|
+
return null;
|
|
125
|
+
}
|
|
126
|
+
for (const testPath of testPaths) {
|
|
127
|
+
let content;
|
|
128
|
+
try {
|
|
129
|
+
content = await readFile(join(projectPath, testPath), 'utf-8');
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
for (const literal of literals) {
|
|
135
|
+
if (new RegExp(buildNumericLiteralPattern(literal)).test(content)) {
|
|
136
|
+
return { status: 'proven', evidence: `${testPath}: matched numeric literal "${literal}"` };
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return null;
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Ground a manual (unclassified) criterion against its declared TEST file(s)
|
|
144
|
+
* via a bounded evidence probe, before it dead-ends as indeterminate. Returns
|
|
145
|
+
* null when grounding is inconclusive, leaving the existing affectedFiles-based
|
|
146
|
+
* fallback to decide the final status.
|
|
147
|
+
*/
|
|
148
|
+
export function groundAgainstDeclaredTests(projectPath, testPaths, probeCache) {
|
|
149
|
+
// AC2: bounded evidence probe — run exactly the declared test file(s).
|
|
150
|
+
// Review finding 7: an invalid override (e.g. "-1", "abc") must fall back
|
|
151
|
+
// to the default, never propagate a non-positive/NaN timeout to spawnSync.
|
|
152
|
+
const overrideMs = Number(process.env.PLANU_VALIDATE_PROBE_TIMEOUT_MS);
|
|
153
|
+
const timeoutMs = Number.isFinite(overrideMs) && overrideMs > 0 ? overrideMs : DEFAULT_PROBE_TIMEOUT_MS;
|
|
154
|
+
const probe = runEvidenceProbe(testPaths, projectPath, probeCache, timeoutMs);
|
|
155
|
+
if (probe.passed) {
|
|
156
|
+
return { status: 'proven', evidence: probe.evidence };
|
|
157
|
+
}
|
|
158
|
+
return null;
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* SPEC-1367 review finding 5 / AC4: backtick-quoted identifiers are the NEW
|
|
162
|
+
* quote form this spec adds (the legacy regex never matched a backtick, so
|
|
163
|
+
* this path is additive, not a behavior change for existing specs). Its
|
|
164
|
+
* search never falls back to the unrestricted project-wide scan — only the
|
|
165
|
+
* already-loaded codeState (same trust level as the legacy path) and the
|
|
166
|
+
* criterion's OWN validated declared test file(s).
|
|
167
|
+
*/
|
|
168
|
+
export async function findBacktickIdentifierEvidence(name, projectPath, codeState, safeTestPaths) {
|
|
169
|
+
for (const [, content] of codeState.fileContents) {
|
|
170
|
+
if (content.includes(name)) {
|
|
171
|
+
return { status: 'proven' };
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
for (const testPath of safeTestPaths) {
|
|
175
|
+
try {
|
|
176
|
+
const content = await readFile(join(projectPath, testPath), 'utf-8');
|
|
177
|
+
if (content.includes(name)) {
|
|
178
|
+
return { status: 'proven', evidence: `${testPath}: found identifier "${name}"` };
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
catch {
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return { status: 'missing' };
|
|
186
|
+
}
|
|
187
|
+
//# sourceMappingURL=test-evidence-grounding.js.map
|
package/dist/engine/validator.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
import { readFile } from 'node:fs/promises';
|
|
5
5
|
import { extractCriteria, extractVerifyBlocks } from './validator/extractors.js';
|
|
6
6
|
import { scanCodeForSpec, checkCriterionEvidence, classifyDriftSeverity, quickQualityCheck, } from './validator/analyzer.js';
|
|
7
|
+
import { extractDeclaredTestPaths, createProbeCache } from './validator/test-evidence-grounding.js';
|
|
7
8
|
import { resolveApplicableDimensions } from './validator/scope-resolver.js';
|
|
8
9
|
import { buildHolisticReportFromFlatScore } from './validator/holistic-report.js';
|
|
9
10
|
import { readEvidenceArtifacts, traceabilityRowHasCurrentCommandEvidence, traceabilityRowHasCurrentTestEvidence, } from './evidence-gates/artifact-reader.js';
|
|
@@ -64,6 +65,10 @@ async function readTraceabilityEvidence(spec, projectPath) {
|
|
|
64
65
|
* normalised over the applicable set only.
|
|
65
66
|
*/
|
|
66
67
|
export async function validateSpec(spec, projectPath) {
|
|
68
|
+
// SPEC-1367 review finding 2: the bounded evidence probe cache lives only
|
|
69
|
+
// for this validate invocation — concurrent runs share no state and a
|
|
70
|
+
// stale probe result can never outlive the file edit that invalidated it.
|
|
71
|
+
const probeCache = createProbeCache();
|
|
67
72
|
// SPEC-730: Resolve applicable dimensions BEFORE running any checks
|
|
68
73
|
const { applicable, skipped } = resolveApplicableDimensions(spec);
|
|
69
74
|
// Emit structured warning when no dimensions apply (malformed frontmatter or future edge case)
|
|
@@ -89,14 +94,17 @@ export async function validateSpec(spec, projectPath) {
|
|
|
89
94
|
const criteria = await extractCriteria(spec);
|
|
90
95
|
const traceabilityEvidence = await readTraceabilityEvidence(spec, projectPath);
|
|
91
96
|
const codeState = await scanCodeForSpec(spec, projectPath);
|
|
92
|
-
// Extract verify blocks
|
|
97
|
+
// Extract verify blocks and declared TEST file ownership from the spec file
|
|
98
|
+
// (forgiving — empty maps if the file is unreadable).
|
|
93
99
|
let verifyBlockMap = new Map();
|
|
100
|
+
let declaredTestsMap = new Map();
|
|
94
101
|
try {
|
|
95
102
|
const specContent = await readFile(spec.specPath, 'utf-8');
|
|
96
103
|
verifyBlockMap = extractVerifyBlocks(specContent);
|
|
104
|
+
declaredTestsMap = extractDeclaredTestPaths(specContent);
|
|
97
105
|
}
|
|
98
106
|
catch {
|
|
99
|
-
// spec file not available — proceed without verify blocks
|
|
107
|
+
// spec file not available — proceed without verify blocks or declared tests
|
|
100
108
|
}
|
|
101
109
|
const matches = [];
|
|
102
110
|
const missing = [];
|
|
@@ -117,7 +125,8 @@ export async function validateSpec(spec, projectPath) {
|
|
|
117
125
|
matches.push(criterion);
|
|
118
126
|
continue;
|
|
119
127
|
}
|
|
120
|
-
const
|
|
128
|
+
const declaredTestPaths = declaredTestsMap.get(criterionIdentity.normalizedText);
|
|
129
|
+
const evidence = await checkCriterionEvidence(criterion, projectPath, codeState, verifyBlock, undefined, declaredTestPaths, probeCache);
|
|
121
130
|
if (evidence.status === 'proven') {
|
|
122
131
|
matches.push(criterion);
|
|
123
132
|
}
|
|
@@ -180,11 +189,13 @@ export async function detectDrift(spec, projectPath, _mode = 'full', threshold =
|
|
|
180
189
|
});
|
|
181
190
|
}
|
|
182
191
|
for (const criterion of validation.indeterminate ?? []) {
|
|
192
|
+
// SPEC-1367: reported as "needs-evidence" — distinct wording from "Missing"
|
|
193
|
+
// even though it scores identically to failing (excluded from `matches`).
|
|
183
194
|
drifts.push({
|
|
184
195
|
file: '',
|
|
185
196
|
specCriterion: criterion,
|
|
186
197
|
expected: 'Proven implementation evidence',
|
|
187
|
-
actual: '
|
|
198
|
+
actual: 'needs-evidence',
|
|
188
199
|
severity: classifyDriftSeverity(criterion),
|
|
189
200
|
autoFixable: false,
|
|
190
201
|
suggestedFix: 'Add explicit validation evidence or implement a recognizable check.',
|
package/dist/tools/validate.js
CHANGED
|
@@ -730,7 +730,9 @@ function buildValidateStructuredContent(args) {
|
|
|
730
730
|
const minimalityBlockingFindings = args.minimalityReport?.findings.filter((finding) => finding.blocksDone) ?? [];
|
|
731
731
|
const blockers = [
|
|
732
732
|
...args.result.missing.map((missing) => `missing: ${missing}`),
|
|
733
|
-
|
|
733
|
+
// SPEC-1367 review finding 6: matches detectDrift's "needs-evidence"
|
|
734
|
+
// wording — display text only, the `indeterminate` field/enum is unchanged.
|
|
735
|
+
...args.indeterminate.map((item) => `needs-evidence: ${item}`),
|
|
734
736
|
...args.qualityGateSummary.failures.map((failure) => `gate: ${failure}`),
|
|
735
737
|
...args.qualityIssues
|
|
736
738
|
.filter((issue) => isBlockingQualitySeverity(issue.severity))
|
|
@@ -872,7 +874,9 @@ function buildCompactValidateText(args) {
|
|
|
872
874
|
const MAX_FAILURES = 5;
|
|
873
875
|
const labelledFailures = [
|
|
874
876
|
...missing.map((criterion) => ({ label: 'missing', criterion })),
|
|
875
|
-
|
|
877
|
+
// SPEC-1367 review finding 6: display label only — matches detectDrift's
|
|
878
|
+
// "needs-evidence" wording; the underlying `indeterminate` array is unchanged.
|
|
879
|
+
...indeterminate.map((criterion) => ({ label: 'needs-evidence', criterion })),
|
|
876
880
|
];
|
|
877
881
|
const shownFailures = labelledFailures.slice(0, MAX_FAILURES);
|
|
878
882
|
const remainingCount = labelledFailures.length - shownFailures.length;
|
package/dist/types/analysis.d.ts
CHANGED
|
@@ -203,6 +203,8 @@ export interface ValidateResult {
|
|
|
203
203
|
export type CriterionEvidenceStatus = 'proven' | 'missing' | 'indeterminate';
|
|
204
204
|
export interface CriterionEvidenceResult {
|
|
205
205
|
status: CriterionEvidenceStatus;
|
|
206
|
+
/** SPEC-1367: citation for how the status was grounded (e.g. probe runner output, matched literal). */
|
|
207
|
+
evidence?: string;
|
|
206
208
|
}
|
|
207
209
|
export interface CodeState {
|
|
208
210
|
affectedFiles: string[];
|
|
@@ -28,6 +28,13 @@ export interface ValidationFreshnessLease {
|
|
|
28
28
|
readonly epoch: string;
|
|
29
29
|
readonly generation: number;
|
|
30
30
|
}
|
|
31
|
+
/**
|
|
32
|
+
* Classification of one raw filesystem watcher event feeding the freshness
|
|
33
|
+
* barrier: a direct relevant/ignored child path, a synthetic root event (the
|
|
34
|
+
* host OS reporting activity on the watched directory itself), or an
|
|
35
|
+
* ambiguous filename-less event that needs authoritative verification.
|
|
36
|
+
*/
|
|
37
|
+
export type ValidationWatchEventClassification = 'direct-relevant' | 'ignored' | 'synthetic' | 'ambiguous';
|
|
31
38
|
export interface ValidationFreshnessWatchState {
|
|
32
39
|
readonly epoch: string;
|
|
33
40
|
readonly expiresAt: number;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@planu/cli",
|
|
3
|
-
"version": "5.
|
|
3
|
+
"version": "5.3.0",
|
|
4
4
|
"description": "Planu — MCP Server for Spec Driven Development with native Rust acceleration for hot paths. Cross-platform (Linux/macOS/Windows, x64/arm64, glibc/musl).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -35,14 +35,14 @@
|
|
|
35
35
|
"packageName": "@planu/core"
|
|
36
36
|
},
|
|
37
37
|
"optionalDependencies": {
|
|
38
|
-
"@planu/core-darwin-arm64": "5.
|
|
39
|
-
"@planu/core-darwin-x64": "5.
|
|
40
|
-
"@planu/core-linux-arm64-gnu": "5.
|
|
41
|
-
"@planu/core-linux-arm64-musl": "5.
|
|
42
|
-
"@planu/core-linux-x64-gnu": "5.
|
|
43
|
-
"@planu/core-linux-x64-musl": "5.
|
|
44
|
-
"@planu/core-win32-arm64-msvc": "5.
|
|
45
|
-
"@planu/core-win32-x64-msvc": "5.
|
|
38
|
+
"@planu/core-darwin-arm64": "5.3.0",
|
|
39
|
+
"@planu/core-darwin-x64": "5.3.0",
|
|
40
|
+
"@planu/core-linux-arm64-gnu": "5.3.0",
|
|
41
|
+
"@planu/core-linux-arm64-musl": "5.3.0",
|
|
42
|
+
"@planu/core-linux-x64-gnu": "5.3.0",
|
|
43
|
+
"@planu/core-linux-x64-musl": "5.3.0",
|
|
44
|
+
"@planu/core-win32-arm64-msvc": "5.3.0",
|
|
45
|
+
"@planu/core-win32-x64-msvc": "5.3.0"
|
|
46
46
|
},
|
|
47
47
|
"engines": {
|
|
48
48
|
"node": ">=24.0.0"
|
|
@@ -161,8 +161,8 @@
|
|
|
161
161
|
"license": "SEE LICENSE IN LICENSE",
|
|
162
162
|
"dependencies": {
|
|
163
163
|
"@anthropic-ai/sdk": "^0.115.0",
|
|
164
|
-
"@hono/node-server": "2.0
|
|
165
|
-
"@modelcontextprotocol/sdk": "^1.
|
|
164
|
+
"@hono/node-server": "2.1.0",
|
|
165
|
+
"@modelcontextprotocol/sdk": "^1.30.0",
|
|
166
166
|
"glob": "^13.0.6",
|
|
167
167
|
"yaml": "^2.9.0",
|
|
168
168
|
"zod": "^4.4.3"
|
|
@@ -173,16 +173,16 @@
|
|
|
173
173
|
"@commitlint/config-conventional": "^21.2.0",
|
|
174
174
|
"@eslint/js": "^10.0.1",
|
|
175
175
|
"@lhci/cli": "0.15.1",
|
|
176
|
-
"@napi-rs/cli": "3.
|
|
176
|
+
"@napi-rs/cli": "3.8.2",
|
|
177
177
|
"@noble/hashes": "2.2.0",
|
|
178
|
-
"@playwright/test": "1.62.
|
|
178
|
+
"@playwright/test": "1.62.1",
|
|
179
179
|
"@scure/base": "2.2.0",
|
|
180
180
|
"@secretlint/secretlint-rule-no-homedir": "^13.0.4",
|
|
181
181
|
"@secretlint/secretlint-rule-preset-recommend": "^13.0.4",
|
|
182
182
|
"@stryker-mutator/core": "^9.6.1",
|
|
183
183
|
"@stryker-mutator/vitest-runner": "^9.6.1",
|
|
184
|
-
"@supabase/supabase-js": "^2.
|
|
185
|
-
"@types/node": "^26.1.
|
|
184
|
+
"@supabase/supabase-js": "^2.112.0",
|
|
185
|
+
"@types/node": "^26.1.2",
|
|
186
186
|
"@types/qrcode": "1.5.6",
|
|
187
187
|
"@typescript/native": "npm:typescript@^7.0.2",
|
|
188
188
|
"@vitejs/plugin-vue": "^6.0.8",
|
|
@@ -196,17 +196,17 @@
|
|
|
196
196
|
"husky": "^9.1.7",
|
|
197
197
|
"javascript-obfuscator": "^5.5.0",
|
|
198
198
|
"jiti": "2.7.0",
|
|
199
|
-
"knip": "^6.
|
|
200
|
-
"lint-staged": "^17.
|
|
199
|
+
"knip": "^6.31.0",
|
|
200
|
+
"lint-staged": "^17.3.0",
|
|
201
201
|
"madge": "^8.0.0",
|
|
202
202
|
"prettier": "^3.9.6",
|
|
203
203
|
"qrcode": "1.5.4",
|
|
204
204
|
"secretlint": "^13.0.4",
|
|
205
205
|
"tsc-alias": "^1.9.1",
|
|
206
|
-
"type-coverage": "^2.
|
|
206
|
+
"type-coverage": "^2.30.1",
|
|
207
207
|
"typescript": "npm:@typescript/typescript6@^6.0.2",
|
|
208
|
-
"typescript-eslint": "^8.
|
|
209
|
-
"vite": "^8.
|
|
208
|
+
"typescript-eslint": "^8.66.0",
|
|
209
|
+
"vite": "^8.2.0",
|
|
210
210
|
"vitest": "^4.1.10",
|
|
211
211
|
"vue": "^3.5.40"
|
|
212
212
|
}
|