@sfqa/scar 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/LICENSE +21 -0
- package/README.md +863 -2
- package/dist/capture.d.ts +20 -0
- package/dist/capture.js +42 -0
- package/dist/cli.d.ts +80 -0
- package/dist/cli.js +622 -0
- package/dist/harvest.d.ts +82 -0
- package/dist/harvest.js +511 -0
- package/dist/hash.d.ts +1 -0
- package/dist/hash.js +10 -0
- package/dist/ignore.d.ts +17 -0
- package/dist/ignore.js +38 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.js +14 -0
- package/dist/local.d.ts +17 -0
- package/dist/local.js +163 -0
- package/dist/new-rule.d.ts +39 -0
- package/dist/new-rule.js +209 -0
- package/dist/prove.d.ts +37 -0
- package/dist/prove.js +158 -0
- package/dist/redact.d.ts +3 -0
- package/dist/redact.js +20 -0
- package/dist/report.d.ts +23 -0
- package/dist/report.js +137 -0
- package/dist/rules.d.ts +38 -0
- package/dist/rules.js +166 -0
- package/dist/scan.d.ts +38 -0
- package/dist/scan.js +404 -0
- package/dist/types.d.ts +77 -0
- package/dist/types.js +11 -0
- package/package.json +59 -4
- package/prompts/extract-rules.md +80 -0
- package/prompts/turn-incident-into-rule.md +82 -0
- package/rules/generic.json +518 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
// packages/scar/src/index.ts - public surface of the scar-tissue reviewer.
|
|
2
|
+
export * from './types.js';
|
|
3
|
+
export * from './rules.js';
|
|
4
|
+
export * from './scan.js';
|
|
5
|
+
export * from './report.js';
|
|
6
|
+
export * from './redact.js';
|
|
7
|
+
export * from './hash.js';
|
|
8
|
+
export * from './ignore.js';
|
|
9
|
+
export * from './capture.js';
|
|
10
|
+
export * from './local.js';
|
|
11
|
+
export * from './harvest.js';
|
|
12
|
+
export * from './cli.js';
|
|
13
|
+
export * from './prove.js';
|
|
14
|
+
export * from './new-rule.js';
|
package/dist/local.d.ts
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export type ReportFormat = 'auto' | 'playwright' | 'junit';
|
|
2
|
+
export interface LocalFailure {
|
|
3
|
+
test: string;
|
|
4
|
+
file: string | null;
|
|
5
|
+
status: string;
|
|
6
|
+
message: string;
|
|
7
|
+
occurrences: number;
|
|
8
|
+
fingerprint: string;
|
|
9
|
+
ignoredAt?: string;
|
|
10
|
+
ignoreReason?: string;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Read one or more report paths (file or directory) and merge duplicate
|
|
14
|
+
* failures. Duplicates are the signal: the same test failing in four runs is a
|
|
15
|
+
* stronger candidate than a one-off.
|
|
16
|
+
*/
|
|
17
|
+
export declare function readLocalReports(paths: readonly string[], format?: ReportFormat, cwd?: string): LocalFailure[];
|
package/dist/local.js
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
// packages/scar/src/local.ts
|
|
2
|
+
// -----------------------------------------------------------------------------
|
|
3
|
+
// Local-run harvesting. Git history shows the failures a team already fixed;
|
|
4
|
+
// a test report shows the failures it is currently living with. This reads the
|
|
5
|
+
// two formats most runners emit - Playwright JSON and JUnit XML - and turns
|
|
6
|
+
// repeated failures into briefs, not rules: a runtime error is not yet a
|
|
7
|
+
// source pattern.
|
|
8
|
+
// -----------------------------------------------------------------------------
|
|
9
|
+
import { readFileSync, readdirSync, statSync } from 'node:fs';
|
|
10
|
+
import { extname, join, resolve } from 'node:path';
|
|
11
|
+
import { fingerprint } from './hash.js';
|
|
12
|
+
import { redactText } from './redact.js';
|
|
13
|
+
const FAILED_STATUSES = new Set(['failed', 'timedOut', 'interrupted', 'error']);
|
|
14
|
+
function readReportFiles(path, output = []) {
|
|
15
|
+
let stat;
|
|
16
|
+
try {
|
|
17
|
+
stat = statSync(path);
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
return output;
|
|
21
|
+
}
|
|
22
|
+
if (stat.isDirectory()) {
|
|
23
|
+
for (const entry of readdirSync(path, { withFileTypes: true })) {
|
|
24
|
+
if (entry.name === 'node_modules' || entry.name === '.git')
|
|
25
|
+
continue;
|
|
26
|
+
readReportFiles(join(path, entry.name), output);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
else if (/\.(json|xml)$/i.test(path)) {
|
|
30
|
+
output.push(path);
|
|
31
|
+
}
|
|
32
|
+
return output;
|
|
33
|
+
}
|
|
34
|
+
/** Very small XML entity decoder - enough for JUnit attributes and text. */
|
|
35
|
+
function decodeXml(value) {
|
|
36
|
+
return value
|
|
37
|
+
.replace(/"/g, '"')
|
|
38
|
+
.replace(/'/g, "'")
|
|
39
|
+
.replace(/</g, '<')
|
|
40
|
+
.replace(/>/g, '>')
|
|
41
|
+
.replace(/ /g, '\n')
|
|
42
|
+
.replace(/&/g, '&');
|
|
43
|
+
}
|
|
44
|
+
function attr(attrs, name) {
|
|
45
|
+
const match = new RegExp(`(?:^|\\s)${name}\\s*=\\s*("([^"]*)"|'([^']*)')`).exec(attrs);
|
|
46
|
+
return match ? decodeXml(match[2] ?? match[3] ?? '') : null;
|
|
47
|
+
}
|
|
48
|
+
function stripTags(value) {
|
|
49
|
+
return value.replace(/<[^>]+>/g, '').trim();
|
|
50
|
+
}
|
|
51
|
+
function parseJUnit(xml) {
|
|
52
|
+
const failures = [];
|
|
53
|
+
const testcase = /<testcase\b([^>]*)>([\s\S]*?)<\/testcase>|<testcase\b([^>]*)\/>/g;
|
|
54
|
+
let match;
|
|
55
|
+
while ((match = testcase.exec(xml)) !== null) {
|
|
56
|
+
const attrs = match[1] ?? match[3] ?? '';
|
|
57
|
+
const body = match[2] ?? '';
|
|
58
|
+
if (!/<(failure|error)\b/.test(body))
|
|
59
|
+
continue;
|
|
60
|
+
const blocking = /<(failure|error)\b([^>]*)>([\s\S]*?)<\/\1>/.exec(body);
|
|
61
|
+
const selfClosing = /<(failure|error)\b([^>]*)\/>/.exec(body);
|
|
62
|
+
const failureAttrs = blocking?.[2] ?? selfClosing?.[2] ?? '';
|
|
63
|
+
const failureBody = blocking?.[3] ?? '';
|
|
64
|
+
const name = attr(attrs, 'name') ?? 'unnamed test';
|
|
65
|
+
const classname = attr(attrs, 'classname') ?? '';
|
|
66
|
+
const test = classname && classname !== name ? `${classname} > ${name}` : name;
|
|
67
|
+
const message = attr(failureAttrs, 'message') ?? (stripTags(failureBody) || 'failed without a message');
|
|
68
|
+
failures.push({
|
|
69
|
+
test,
|
|
70
|
+
file: attr(attrs, 'file'),
|
|
71
|
+
status: 'failed',
|
|
72
|
+
message: redactText(message).trim().slice(0, 500),
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
return failures;
|
|
76
|
+
}
|
|
77
|
+
function walkPlaywright(suite, output) {
|
|
78
|
+
for (const spec of suite.specs ?? []) {
|
|
79
|
+
for (const test of spec.tests ?? []) {
|
|
80
|
+
for (const result of test.results ?? []) {
|
|
81
|
+
if (!result.status || !FAILED_STATUSES.has(result.status))
|
|
82
|
+
continue;
|
|
83
|
+
output.push({
|
|
84
|
+
test: [suite.title, spec.title, test.title].filter(Boolean).join(' > ') || 'unnamed test',
|
|
85
|
+
file: spec.file ?? suite.file ?? null,
|
|
86
|
+
status: result.status,
|
|
87
|
+
message: redactText(result.error?.message ?? 'failed without a message')
|
|
88
|
+
.trim()
|
|
89
|
+
.slice(0, 500),
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
for (const child of suite.suites ?? [])
|
|
95
|
+
walkPlaywright(child, output);
|
|
96
|
+
}
|
|
97
|
+
function parsePlaywright(data) {
|
|
98
|
+
if (!data || typeof data !== 'object')
|
|
99
|
+
return [];
|
|
100
|
+
const suites = data.suites;
|
|
101
|
+
if (!Array.isArray(suites))
|
|
102
|
+
return [];
|
|
103
|
+
const failures = [];
|
|
104
|
+
for (const suite of suites)
|
|
105
|
+
walkPlaywright(suite, failures);
|
|
106
|
+
return failures;
|
|
107
|
+
}
|
|
108
|
+
function parseReport(path, format) {
|
|
109
|
+
let text;
|
|
110
|
+
try {
|
|
111
|
+
text = readFileSync(path, 'utf8');
|
|
112
|
+
}
|
|
113
|
+
catch {
|
|
114
|
+
return [];
|
|
115
|
+
}
|
|
116
|
+
const trimmed = text.trimStart();
|
|
117
|
+
const detected = format !== 'auto'
|
|
118
|
+
? format
|
|
119
|
+
: trimmed.startsWith('<')
|
|
120
|
+
? 'junit'
|
|
121
|
+
: trimmed.startsWith('{')
|
|
122
|
+
? 'playwright'
|
|
123
|
+
: null;
|
|
124
|
+
if (detected === 'junit')
|
|
125
|
+
return parseJUnit(text);
|
|
126
|
+
if (detected === 'playwright') {
|
|
127
|
+
try {
|
|
128
|
+
return parsePlaywright(JSON.parse(text));
|
|
129
|
+
}
|
|
130
|
+
catch {
|
|
131
|
+
return [];
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return [];
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Read one or more report paths (file or directory) and merge duplicate
|
|
138
|
+
* failures. Duplicates are the signal: the same test failing in four runs is a
|
|
139
|
+
* stronger candidate than a one-off.
|
|
140
|
+
*/
|
|
141
|
+
export function readLocalReports(paths, format = 'auto', cwd = process.cwd()) {
|
|
142
|
+
const raw = [];
|
|
143
|
+
for (const path of paths) {
|
|
144
|
+
for (const file of readReportFiles(resolve(cwd, path))) {
|
|
145
|
+
raw.push(...parseReport(file, format));
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
const merged = new Map();
|
|
149
|
+
for (const failure of raw) {
|
|
150
|
+
const key = `${failure.test}\n${failure.file ?? ''}\n${failure.message}`;
|
|
151
|
+
const existing = merged.get(key);
|
|
152
|
+
if (existing) {
|
|
153
|
+
existing.occurrences += 1;
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
156
|
+
merged.set(key, {
|
|
157
|
+
...failure,
|
|
158
|
+
occurrences: 1,
|
|
159
|
+
fingerprint: `test:${fingerprint([failure.test, failure.file ?? '', failure.message])}`,
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
return [...merged.values()];
|
|
163
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { Severity } from './types.js';
|
|
2
|
+
export declare class ScaffoldError extends Error {
|
|
3
|
+
constructor(message: string);
|
|
4
|
+
}
|
|
5
|
+
export interface ScaffoldOptions {
|
|
6
|
+
/** The rule in one imperative sentence: what must not happen again. */
|
|
7
|
+
title: string;
|
|
8
|
+
/** Glob(s) the rule is allowed to fire in. The fixture is placed to match. */
|
|
9
|
+
scope: readonly string[];
|
|
10
|
+
severity?: Severity;
|
|
11
|
+
id?: string;
|
|
12
|
+
rulesPath: string;
|
|
13
|
+
fixtureRoot: string;
|
|
14
|
+
cwd: string;
|
|
15
|
+
}
|
|
16
|
+
export interface ScaffoldResult {
|
|
17
|
+
id: string;
|
|
18
|
+
rulesPath: string;
|
|
19
|
+
positivePath: string;
|
|
20
|
+
negativePath: string;
|
|
21
|
+
/** Fixture paths when --scope was given more than once. */
|
|
22
|
+
otherFixtures: string[];
|
|
23
|
+
createdRuleFile: boolean;
|
|
24
|
+
nextSteps: string[];
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Where the fixture has to live: the literal directory in front of the first
|
|
28
|
+
* wildcard, plus the scope's own extension. A fixture outside this path is never
|
|
29
|
+
* scanned, which is the failure mode people hit when they place it by hand.
|
|
30
|
+
*/
|
|
31
|
+
export declare function fixtureRelativePath(rawScope: string, id: string): string;
|
|
32
|
+
/**
|
|
33
|
+
* People write `--scope test-pages/` when they mean "everything under it". A
|
|
34
|
+
* literal path with no wildcard would scope the rule to one file that does not
|
|
35
|
+
* exist yet, so a trailing slash or an existing directory becomes a subtree.
|
|
36
|
+
*/
|
|
37
|
+
export declare function normaliseScope(entry: string): string;
|
|
38
|
+
export declare function scaffoldRule(options: ScaffoldOptions): ScaffoldResult;
|
|
39
|
+
export declare function formatScaffold(result: ScaffoldResult): string;
|
package/dist/new-rule.js
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
// packages/scar/src/new-rule.ts
|
|
2
|
+
// -----------------------------------------------------------------------------
|
|
3
|
+
// Scaffolding one rule. The two steps people get wrong are mechanical, not
|
|
4
|
+
// thoughtful: picking an id, and putting the fixture where the rule actually
|
|
5
|
+
// looks. This does both, then prints the three things left to a person.
|
|
6
|
+
//
|
|
7
|
+
// It writes a skeleton, never an enforced rule: the pattern starts empty, so the
|
|
8
|
+
// proof reports the rule as a gap until someone supplies a detector.
|
|
9
|
+
// -----------------------------------------------------------------------------
|
|
10
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
11
|
+
import { dirname, join, resolve } from 'node:path';
|
|
12
|
+
import { globToRegex } from './rules.js';
|
|
13
|
+
import { relPosix, walk } from './scan.js';
|
|
14
|
+
const ID_SHAPE = /^[A-Z]+-\d+$/;
|
|
15
|
+
export class ScaffoldError extends Error {
|
|
16
|
+
constructor(message) {
|
|
17
|
+
super(message);
|
|
18
|
+
this.name = 'ScaffoldError';
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
function readRuleSet(path) {
|
|
22
|
+
if (!existsSync(path)) {
|
|
23
|
+
return { ruleSet: { version: 1, description: 'Scar tissue this repository has paid for.', rules: [] }, created: true };
|
|
24
|
+
}
|
|
25
|
+
let raw;
|
|
26
|
+
try {
|
|
27
|
+
raw = JSON.parse(readFileSync(path, 'utf8'));
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
throw new ScaffoldError(`${path} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
|
|
31
|
+
}
|
|
32
|
+
const candidate = raw;
|
|
33
|
+
if (!candidate || typeof candidate !== 'object' || !Array.isArray(candidate.rules)) {
|
|
34
|
+
throw new ScaffoldError(`${path} has no "rules" array`);
|
|
35
|
+
}
|
|
36
|
+
return { ruleSet: candidate, created: false };
|
|
37
|
+
}
|
|
38
|
+
/** Next free id for the prefix already used in this rule set (ST by default). */
|
|
39
|
+
function nextId(rules, requested) {
|
|
40
|
+
if (requested) {
|
|
41
|
+
if (!ID_SHAPE.test(requested))
|
|
42
|
+
throw new ScaffoldError(`--id must look like "ST-020", got "${requested}"`);
|
|
43
|
+
if (rules.some((rule) => rule.id === requested))
|
|
44
|
+
throw new ScaffoldError(`${requested} already exists in this rule set`);
|
|
45
|
+
return requested;
|
|
46
|
+
}
|
|
47
|
+
const used = rules.map((rule) => /^([A-Z]+)-(\d+)$/.exec(rule.id)).filter(Boolean);
|
|
48
|
+
const prefix = used.length > 0 ? used[0][1] : 'ST';
|
|
49
|
+
const highest = used
|
|
50
|
+
.filter((match) => match[1] === prefix)
|
|
51
|
+
.reduce((max, match) => Math.max(max, Number(match[2])), 0);
|
|
52
|
+
return `${prefix}-${String(highest + 1).padStart(3, '0')}`;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Where the fixture has to live: the literal directory in front of the first
|
|
56
|
+
* wildcard, plus the scope's own extension. A fixture outside this path is never
|
|
57
|
+
* scanned, which is the failure mode people hit when they place it by hand.
|
|
58
|
+
*/
|
|
59
|
+
export function fixtureRelativePath(rawScope, id) {
|
|
60
|
+
const scope = rawScope.replace(/^\.\//, '');
|
|
61
|
+
const wildcard = scope.search(/[*?[{]/);
|
|
62
|
+
if (wildcard === -1)
|
|
63
|
+
return scope.replace(/^\/+/, '');
|
|
64
|
+
const literal = scope.slice(0, wildcard);
|
|
65
|
+
const directory = literal.endsWith('/') ? literal.slice(0, -1) : dirname(literal);
|
|
66
|
+
// Prefer the extension the scope names (`features/**/*.feature`), then the
|
|
67
|
+
// one on a literal suffix (`playwright.config.ts`), then TypeScript.
|
|
68
|
+
const extension = /\*\.([a-z0-9]{1,8})(?![a-z0-9])/i.exec(scope)?.[1] ??
|
|
69
|
+
/\.([a-z0-9]{1,8})$/.exec(scope)?.[1] ??
|
|
70
|
+
// `features/**` names no extension, and a .ts file there is never run.
|
|
71
|
+
(/\bfeatures\b/.test(directory) ? 'feature' : 'ts');
|
|
72
|
+
const file = `${id.toLowerCase()}.${extension}`;
|
|
73
|
+
return directory && directory !== '.' ? `${directory}/${file}` : file;
|
|
74
|
+
}
|
|
75
|
+
/** `#` for features and markdown, `//` for code. Keep the fixture valid in its language. */
|
|
76
|
+
function commentFor(path) {
|
|
77
|
+
return /\.(feature|md|ya?ml|sh|py|rb)$/.test(path) ? '#' : '//';
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* People write `--scope test-pages/` when they mean "everything under it". A
|
|
81
|
+
* literal path with no wildcard would scope the rule to one file that does not
|
|
82
|
+
* exist yet, so a trailing slash or an existing directory becomes a subtree.
|
|
83
|
+
*/
|
|
84
|
+
export function normaliseScope(entry) {
|
|
85
|
+
const trimmed = entry.replace(/^\.\//, '').replace(/\/+$/, '');
|
|
86
|
+
if (!trimmed)
|
|
87
|
+
throw new ScaffoldError('--scope cannot be the repository root; name a directory');
|
|
88
|
+
const wildcard = /[*?[{]/.test(trimmed);
|
|
89
|
+
const looksLikeFile = /\.[a-z0-9]{1,8}$/i.test(trimmed);
|
|
90
|
+
if (wildcard || looksLikeFile)
|
|
91
|
+
return trimmed;
|
|
92
|
+
return `${trimmed}/**`;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* A scope that matches nothing today is usually a typo (`test-page/`), and the
|
|
96
|
+
* proof cannot catch it: the fixture is placed to match the same wrong scope, so
|
|
97
|
+
* it fires while the rule never sees a real file. Count what is really there and
|
|
98
|
+
* say so.
|
|
99
|
+
*/
|
|
100
|
+
function filesInScope(cwd, scope, fixtureRoot) {
|
|
101
|
+
const matchers = scope.map(globToRegex);
|
|
102
|
+
const root = resolve(cwd);
|
|
103
|
+
const skipFixture = resolve(cwd, fixtureRoot);
|
|
104
|
+
let count = 0;
|
|
105
|
+
for (const file of walk(root)) {
|
|
106
|
+
if (file.startsWith(skipFixture))
|
|
107
|
+
continue;
|
|
108
|
+
const rel = relPosix(root, file);
|
|
109
|
+
if (matchers.some((matcher) => matcher.test(rel)))
|
|
110
|
+
count++;
|
|
111
|
+
}
|
|
112
|
+
return count;
|
|
113
|
+
}
|
|
114
|
+
export function scaffoldRule(options) {
|
|
115
|
+
const title = options.title.trim();
|
|
116
|
+
if (!title)
|
|
117
|
+
throw new ScaffoldError('--rule needs the rule in one sentence');
|
|
118
|
+
if (options.scope.length === 0) {
|
|
119
|
+
throw new ScaffoldError('--scope is required: the rule needs a place it can fire, and the fixture is placed to match it. Try: --scope "test-pages/"');
|
|
120
|
+
}
|
|
121
|
+
const rulesPath = resolve(options.cwd, options.rulesPath);
|
|
122
|
+
const { ruleSet, created } = readRuleSet(rulesPath);
|
|
123
|
+
const id = nextId(ruleSet.rules, options.id);
|
|
124
|
+
// Running `new` twice with the same sentence is a duplicate, not a second rule.
|
|
125
|
+
const sameSentence = ruleSet.rules.find((rule) => rule.rule.trim().toLowerCase() === title.toLowerCase());
|
|
126
|
+
if (sameSentence) {
|
|
127
|
+
throw new ScaffoldError(`${sameSentence.id} already says that: "${sameSentence.rule}". Edit it with --id ${sameSentence.id}, or write a different rule.`);
|
|
128
|
+
}
|
|
129
|
+
const scope = options.scope.map(normaliseScope);
|
|
130
|
+
const positiveRels = scope.map((entry) => fixtureRelativePath(entry, id));
|
|
131
|
+
const negativeRels = positiveRels.map((rel) => rel);
|
|
132
|
+
const fixtureRoot = resolve(options.cwd, options.fixtureRoot);
|
|
133
|
+
const positivePaths = positiveRels.map((rel) => join(fixtureRoot, 'positive', rel));
|
|
134
|
+
const negativePaths = negativeRels.map((rel) => join(fixtureRoot, 'negative', rel));
|
|
135
|
+
for (const target of [...positivePaths, ...negativePaths]) {
|
|
136
|
+
if (existsSync(target))
|
|
137
|
+
throw new ScaffoldError(`${relPosix(options.cwd, target)} already exists - refusing to overwrite it`);
|
|
138
|
+
}
|
|
139
|
+
const rule = {
|
|
140
|
+
id,
|
|
141
|
+
rule: title,
|
|
142
|
+
context: `Files matching ${scope.join(', ')}. TODO: name the objects or helpers this applies to.`,
|
|
143
|
+
origin: {
|
|
144
|
+
commit: null,
|
|
145
|
+
pr: null,
|
|
146
|
+
summary: 'TODO: what broke, and what it cost. Keep the detail that makes the rule defensible.',
|
|
147
|
+
},
|
|
148
|
+
severity: options.severity ?? 'med',
|
|
149
|
+
detection_pattern: 'TODO: what a reviewer should look for, in concrete syntax.',
|
|
150
|
+
match: { pathInclude: scope, linePatterns: [] },
|
|
151
|
+
recommendation: 'TODO: the fix that worked, specific enough to paste.',
|
|
152
|
+
};
|
|
153
|
+
ruleSet.rules.push(rule);
|
|
154
|
+
mkdirSync(dirname(rulesPath), { recursive: true });
|
|
155
|
+
writeFileSync(rulesPath, `${JSON.stringify(ruleSet, null, 2)}\n`, 'utf8');
|
|
156
|
+
positivePaths.forEach((positivePath, index) => {
|
|
157
|
+
const negativePath = negativePaths[index];
|
|
158
|
+
const marker = commentFor(positiveRels[index]);
|
|
159
|
+
mkdirSync(dirname(positivePath), { recursive: true });
|
|
160
|
+
mkdirSync(dirname(negativePath), { recursive: true });
|
|
161
|
+
writeFileSync(positivePath, `${marker} ${id} positive fixture: the bad line goes here.\n${marker} The rule must fire on it. Delete these comments once it does.\n`, 'utf8');
|
|
162
|
+
writeFileSync(negativePath, `${marker} ${id} clean fixture: the same code with the fix applied.\n${marker} The rule must fire zero times here. Delete these comments once it stays silent.\n`, 'utf8');
|
|
163
|
+
});
|
|
164
|
+
const inScope = filesInScope(options.cwd, scope, options.fixtureRoot);
|
|
165
|
+
const shownRules = relPosix(options.cwd, rulesPath);
|
|
166
|
+
const shownPositive = relPosix(options.cwd, positivePaths[0]);
|
|
167
|
+
const shownNegative = relPosix(options.cwd, negativePaths[0]);
|
|
168
|
+
const scopeHint = scope.join(', ');
|
|
169
|
+
return {
|
|
170
|
+
id,
|
|
171
|
+
rulesPath: shownRules,
|
|
172
|
+
positivePath: shownPositive,
|
|
173
|
+
negativePath: shownNegative,
|
|
174
|
+
otherFixtures: positivePaths
|
|
175
|
+
.slice(1)
|
|
176
|
+
.flatMap((positivePath, index) => [
|
|
177
|
+
relPosix(options.cwd, positivePath),
|
|
178
|
+
relPosix(options.cwd, negativePaths[index + 1]),
|
|
179
|
+
]),
|
|
180
|
+
createdRuleFile: created,
|
|
181
|
+
nextSteps: [
|
|
182
|
+
`open ${shownRules}, replace the TODOs, and put the pattern in match.linePatterns`,
|
|
183
|
+
`not sure of the pattern? prompts/turn-incident-into-rule.md (in the package) drafts it from the incident`,
|
|
184
|
+
`paste the bad line into ${shownPositive}`,
|
|
185
|
+
`paste the same code with the fix into ${shownNegative}`,
|
|
186
|
+
`prove just this rule: sfqa-scar prove --id ${id}`,
|
|
187
|
+
`it is scoped to ${scopeHint}; a fixture outside that path is never scanned`,
|
|
188
|
+
...(inScope === 0
|
|
189
|
+
? [
|
|
190
|
+
`warning: nothing in this repo matches ${scopeHint} yet. If that is a typo, the rule will never fire - delete it with --id ${id} and start again`,
|
|
191
|
+
]
|
|
192
|
+
: []),
|
|
193
|
+
],
|
|
194
|
+
};
|
|
195
|
+
}
|
|
196
|
+
export function formatScaffold(result) {
|
|
197
|
+
const lines = [];
|
|
198
|
+
lines.push(`created ${result.id}${result.createdRuleFile ? ` (new rule file)` : ''}`);
|
|
199
|
+
lines.push('');
|
|
200
|
+
lines.push(` rule ${result.rulesPath}`);
|
|
201
|
+
lines.push(` positive ${result.positivePath}`);
|
|
202
|
+
lines.push(` clean ${result.negativePath}`);
|
|
203
|
+
for (const other of result.otherFixtures)
|
|
204
|
+
lines.push(` also ${other}`);
|
|
205
|
+
lines.push('');
|
|
206
|
+
lines.push('what to do next');
|
|
207
|
+
result.nextSteps.forEach((step, index) => lines.push(` ${index + 1}. ${step}`));
|
|
208
|
+
return lines.join('\n');
|
|
209
|
+
}
|
package/dist/prove.d.ts
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { ScarRule, Severity } from './types.js';
|
|
2
|
+
/** Where `new` writes fixtures and `prove` reads them, unless told otherwise. */
|
|
3
|
+
export declare const DEFAULT_FIXTURE_ROOT = "scar-fixtures";
|
|
4
|
+
export declare class ProveError extends Error {
|
|
5
|
+
constructor(message: string);
|
|
6
|
+
}
|
|
7
|
+
export interface RuleProof {
|
|
8
|
+
id: string;
|
|
9
|
+
severity: Severity;
|
|
10
|
+
/** False when the rule has no machine detector: it cannot fire by design. */
|
|
11
|
+
executable: boolean;
|
|
12
|
+
positive: 'fires' | 'missing' | 'n/a';
|
|
13
|
+
clean: 'clean' | 'leak' | 'n/a';
|
|
14
|
+
pathInclude: string[];
|
|
15
|
+
}
|
|
16
|
+
export interface ProofResult {
|
|
17
|
+
fixtureRoot: string;
|
|
18
|
+
executable: number;
|
|
19
|
+
gaps: number;
|
|
20
|
+
covered: number;
|
|
21
|
+
proofs: RuleProof[];
|
|
22
|
+
/** True only when every executable rule fires and the clean set leaks nothing. */
|
|
23
|
+
passed: boolean;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Run every selected rule against the fixture pair.
|
|
27
|
+
* `ids` limits the run to particular rules, which is what `--id` uses after
|
|
28
|
+
* `new` so a person sees one verdict instead of nineteen.
|
|
29
|
+
*/
|
|
30
|
+
export declare function proveRules(rules: readonly ScarRule[], fixtureRoot: string, ids?: readonly string[]): ProofResult;
|
|
31
|
+
/**
|
|
32
|
+
* The table the proof has printed since v0.1, plus one addition: when a rule
|
|
33
|
+
* has a detector but no fixture made it fire, the scope is printed. Without it
|
|
34
|
+
* the person has to guess which directory the fixture belongs in, and puts it
|
|
35
|
+
* somewhere the rule never looks.
|
|
36
|
+
*/
|
|
37
|
+
export declare function formatProof(result: ProofResult, ruleSet: string, total: number, fixtureRoot?: string): string;
|
package/dist/prove.js
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
// packages/scar/src/prove.ts
|
|
2
|
+
// -----------------------------------------------------------------------------
|
|
3
|
+
// The proof. One fixture that must fire per rule, one clean set that must stay
|
|
4
|
+
// silent. A rule with no detector is reported as a gap, never counted as
|
|
5
|
+
// coverage - a rule that cannot fire is worse than a missing rule, because it
|
|
6
|
+
// buys trust it has not earned.
|
|
7
|
+
//
|
|
8
|
+
// This lives in the package, not only in this repository's harness, so someone
|
|
9
|
+
// who installed @sfqa/scar can prove the rules they write without cloning
|
|
10
|
+
// anything. `sfqa-scar prove` is the command; `npm run validate:scar` is the
|
|
11
|
+
// same code path.
|
|
12
|
+
// -----------------------------------------------------------------------------
|
|
13
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
14
|
+
import { join, resolve } from 'node:path';
|
|
15
|
+
import { compileRule, isExecutable } from './rules.js';
|
|
16
|
+
import { lintContent, relPosix, walk } from './scan.js';
|
|
17
|
+
/** Where `new` writes fixtures and `prove` reads them, unless told otherwise. */
|
|
18
|
+
export const DEFAULT_FIXTURE_ROOT = 'scar-fixtures';
|
|
19
|
+
export class ProveError extends Error {
|
|
20
|
+
constructor(message) {
|
|
21
|
+
super(message);
|
|
22
|
+
this.name = 'ProveError';
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
function readFixtures(dir) {
|
|
26
|
+
if (!existsSync(dir))
|
|
27
|
+
return [];
|
|
28
|
+
return walk(dir).map((file) => ({
|
|
29
|
+
rel: relPosix(dir, file),
|
|
30
|
+
content: readFileSync(file, 'utf8'),
|
|
31
|
+
}));
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Run every selected rule against the fixture pair.
|
|
35
|
+
* `ids` limits the run to particular rules, which is what `--id` uses after
|
|
36
|
+
* `new` so a person sees one verdict instead of nineteen.
|
|
37
|
+
*/
|
|
38
|
+
export function proveRules(rules, fixtureRoot, ids = []) {
|
|
39
|
+
const root = resolve(fixtureRoot);
|
|
40
|
+
const positiveDir = join(root, 'positive');
|
|
41
|
+
const negativeDir = join(root, 'negative');
|
|
42
|
+
const positive = readFixtures(positiveDir);
|
|
43
|
+
const negative = readFixtures(negativeDir);
|
|
44
|
+
if (positive.length === 0 && negative.length === 0) {
|
|
45
|
+
throw new ProveError(`no fixtures under ${fixtureRoot}. Expected ${fixtureRoot}/positive/ and ${fixtureRoot}/negative/. Create the first pair with: sfqa-scar new --rule "what must not happen again"`);
|
|
46
|
+
}
|
|
47
|
+
const selected = ids.length > 0 ? rules.filter((rule) => ids.includes(rule.id)) : [...rules];
|
|
48
|
+
if (ids.length > 0) {
|
|
49
|
+
const known = new Set(rules.map((rule) => rule.id));
|
|
50
|
+
const unknown = ids.filter((id) => !known.has(id));
|
|
51
|
+
if (unknown.length > 0) {
|
|
52
|
+
throw new ProveError(`no rule with id ${unknown.join(', ')} in this rule set`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
const proofs = selected.map((rule) => {
|
|
56
|
+
const executable = isExecutable(rule);
|
|
57
|
+
const pathInclude = rule.match?.pathInclude ?? [];
|
|
58
|
+
if (!executable) {
|
|
59
|
+
return {
|
|
60
|
+
id: rule.id,
|
|
61
|
+
severity: rule.severity,
|
|
62
|
+
executable,
|
|
63
|
+
positive: 'n/a',
|
|
64
|
+
clean: 'n/a',
|
|
65
|
+
pathInclude,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
const compiled = compileRule(rule);
|
|
69
|
+
const fires = positive.some((fixture) => lintContent(fixture.rel, fixture.content, [compiled]).length > 0);
|
|
70
|
+
const leaks = negative.some((fixture) => lintContent(fixture.rel, fixture.content, [compiled]).length > 0);
|
|
71
|
+
return {
|
|
72
|
+
id: rule.id,
|
|
73
|
+
severity: rule.severity,
|
|
74
|
+
executable,
|
|
75
|
+
positive: fires ? 'fires' : 'missing',
|
|
76
|
+
clean: leaks ? 'leak' : 'clean',
|
|
77
|
+
pathInclude,
|
|
78
|
+
};
|
|
79
|
+
});
|
|
80
|
+
const executable = proofs.filter((proof) => proof.executable).length;
|
|
81
|
+
const covered = proofs.filter((proof) => proof.positive === 'fires').length;
|
|
82
|
+
const gaps = selected.length - executable;
|
|
83
|
+
// A gap is reported, never counted as a pass. But a rule set where nothing
|
|
84
|
+
// can fire is not "0 failures" - it is a rule set that proves nothing, and
|
|
85
|
+
// saying PROOF over it would be the exact false confidence this tool exists to
|
|
86
|
+
// remove. So passing requires no missing, no leak, and at least one fire.
|
|
87
|
+
const passed = covered > 0 &&
|
|
88
|
+
proofs.every((proof) => !proof.executable || (proof.positive === 'fires' && proof.clean === 'clean'));
|
|
89
|
+
return { fixtureRoot, executable, gaps, covered, proofs, passed };
|
|
90
|
+
}
|
|
91
|
+
function pad(text, width) {
|
|
92
|
+
return text.length >= width ? text : text + ' '.repeat(width - text.length);
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* The table the proof has printed since v0.1, plus one addition: when a rule
|
|
96
|
+
* has a detector but no fixture made it fire, the scope is printed. Without it
|
|
97
|
+
* the person has to guess which directory the fixture belongs in, and puts it
|
|
98
|
+
* somewhere the rule never looks.
|
|
99
|
+
*/
|
|
100
|
+
export function formatProof(result, ruleSet, total, fixtureRoot = result.fixtureRoot) {
|
|
101
|
+
const lines = [];
|
|
102
|
+
lines.push(`sfqa-scar prove - ${total} rule(s), ${result.executable} executable`);
|
|
103
|
+
lines.push(`rule set: ${ruleSet}`);
|
|
104
|
+
lines.push(`fixtures: ${fixtureRoot}`);
|
|
105
|
+
lines.push('');
|
|
106
|
+
lines.push('rule severity detector positive clean');
|
|
107
|
+
lines.push('------ -------- --------- --------- -------');
|
|
108
|
+
for (const proof of result.proofs) {
|
|
109
|
+
const id = pad(proof.id, 6);
|
|
110
|
+
const severity = pad(proof.severity, 8);
|
|
111
|
+
const detector = pad(proof.executable ? 'yes' : 'NO', 9);
|
|
112
|
+
const positive = pad(proof.positive, 9);
|
|
113
|
+
lines.push(`${id} ${severity} ${detector} ${positive} ${proof.clean}`);
|
|
114
|
+
}
|
|
115
|
+
lines.push('');
|
|
116
|
+
const missing = result.proofs.filter((proof) => proof.positive === 'missing');
|
|
117
|
+
const leaks = result.proofs.filter((proof) => proof.clean === 'leak');
|
|
118
|
+
const gaps = result.proofs.filter((proof) => !proof.executable);
|
|
119
|
+
if (gaps.length > 0) {
|
|
120
|
+
lines.push(`GAPS (no machine detector, cannot fire): ${gaps.map((p) => p.id).join(', ')}`);
|
|
121
|
+
lines.push(' These rules are documentation until a detector is added.');
|
|
122
|
+
for (const gap of gaps) {
|
|
123
|
+
const where = gap.pathInclude.length > 0 ? gap.pathInclude.join(', ') : 'no path scope';
|
|
124
|
+
lines.push(` ${gap.id} is scoped to: ${where}`);
|
|
125
|
+
}
|
|
126
|
+
lines.push('');
|
|
127
|
+
}
|
|
128
|
+
if (missing.length > 0) {
|
|
129
|
+
lines.push(`MISSING (detector exists but no fixture made it fire): ${missing.map((p) => p.id).join(', ')}`);
|
|
130
|
+
for (const proof of missing) {
|
|
131
|
+
const where = proof.pathInclude.length > 0 ? proof.pathInclude.join(', ') : 'any file';
|
|
132
|
+
lines.push(` ${proof.id} is scoped to: ${where}`);
|
|
133
|
+
lines.push(` put a file containing the bad line under ${result.fixtureRoot}/positive/ so it matches that scope`);
|
|
134
|
+
}
|
|
135
|
+
lines.push(' A fixture outside the scope is never scanned by the rule.');
|
|
136
|
+
lines.push('');
|
|
137
|
+
}
|
|
138
|
+
if (leaks.length > 0) {
|
|
139
|
+
lines.push(`FALSE POSITIVES on the clean set: ${leaks.map((p) => p.id).join(', ')}`);
|
|
140
|
+
for (const proof of leaks.filter((p) => p.clean === 'leak')) {
|
|
141
|
+
lines.push(` ${proof.id} matched a file under ${result.fixtureRoot}/negative/ - either the clean fixture still contains the bad shape, or the pattern is too wide`);
|
|
142
|
+
}
|
|
143
|
+
lines.push('');
|
|
144
|
+
}
|
|
145
|
+
if (result.passed) {
|
|
146
|
+
lines.push(`PROOF: ${result.covered}/${result.executable} executable rules fire on a positive fixture and stay silent on the clean set.`);
|
|
147
|
+
lines.push(`Coverage: ${result.covered}/${total} rules enforced today; ${result.gaps} rule(s) have no detector.`);
|
|
148
|
+
}
|
|
149
|
+
else if (result.covered === 0) {
|
|
150
|
+
lines.push('FAIL: no rule fired on a positive fixture, so nothing is proven.');
|
|
151
|
+
lines.push(' The usual cause is an empty detector: a scaffolded rule keeps `linePatterns: []` until you fill it.');
|
|
152
|
+
lines.push(' Add a pattern under match.linePatterns, or delete the rule - an unproved rule is a comment.');
|
|
153
|
+
}
|
|
154
|
+
else {
|
|
155
|
+
lines.push(`FAIL: ${missing.length} missing, ${leaks.length} false positive(s).`);
|
|
156
|
+
}
|
|
157
|
+
return lines.join('\n');
|
|
158
|
+
}
|
package/dist/redact.d.ts
ADDED
package/dist/redact.js
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// packages/scar/src/redact.ts
|
|
2
|
+
// -----------------------------------------------------------------------------
|
|
3
|
+
// Discovery reads failures, and failures sometimes contain the secret that
|
|
4
|
+
// caused them. Nothing from a credential-shaped line reaches stdout, a JSON
|
|
5
|
+
// report or a draft. The name stays; the value is replaced.
|
|
6
|
+
// -----------------------------------------------------------------------------
|
|
7
|
+
const SECRET_NAME = /\b(password|passwd|pwd|secret|token|api[_-]?key|apikey|credential|authorization|bearer|private[_-]?key|access[_-]?key)\b/i;
|
|
8
|
+
export function redactLine(line) {
|
|
9
|
+
if (!SECRET_NAME.test(line))
|
|
10
|
+
return line;
|
|
11
|
+
return line
|
|
12
|
+
.replace(/(['"])([^'"\n]{2,})\1/g, '$1<redacted>$1')
|
|
13
|
+
.replace(/(=\s*)([^\s;,'"]+)/, '$1<redacted>');
|
|
14
|
+
}
|
|
15
|
+
export function redactText(text) {
|
|
16
|
+
return text.split('\n').map(redactLine).join('\n');
|
|
17
|
+
}
|
|
18
|
+
export function isSecretShaped(line) {
|
|
19
|
+
return redactLine(line) !== line;
|
|
20
|
+
}
|