harness-everything 0.47.3 → 0.47.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/CHANGELOG.md +24 -0
- package/behavioral-evals/README.md +27 -0
- package/behavioral-evals/case-validator.js +22 -1
- package/behavioral-evals/cases/baseline-profile-lookup-json-only.yaml +23 -0
- package/behavioral-evals/cases/baseline-tdd-workflow.yaml +4 -0
- package/behavioral-evals/run.js +257 -6
- package/behavioral-evals/transcript-parser.js +43 -0
- package/benchmarks/issue290-pilot/metrics.test.py +60 -2
- package/ci/mechanism-2s-tdd-quality.test.js +149 -6
- package/ci/test-behavioral-case-validation.js +362 -3
- package/code-review/SKILL.md +1 -1
- package/code-review/references/review-playbook.md +9 -0
- package/opencode-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/plugins/harness-everything/.codex-plugin/plugin.json +1 -1
- package/plugins/harness-everything/plugin.json +1 -1
- package/plugins/harness-everything/skills/code-review/SKILL.md +1 -1
- package/plugins/harness-everything/skills/code-review/references/review-playbook.md +9 -0
- package/plugins/harness-everything/skills/tdd/SKILL.md +1 -1
- package/plugins/harness-everything/skills/tdd/fixtures/quality-scenarios.json +10 -2
- package/plugins/harness-everything/skills/tdd/references/quality-model.md +2 -0
- package/plugins/harness-everything/skills/tdd/schemas/quality-evidence.schema.json +21 -0
- package/plugins/harness-everything/skills/tdd/schemas/quality-report.schema.json +2 -0
- package/plugins/harness-everything/skills/tdd/scripts/quality-gate.js +165 -1
- package/plugins/harness-everything/skills/zoom-out/SKILL.md +1 -1
- package/plugins/harness-everything/skills/zoom-out/references/circuit-breaker.md +1 -0
- package/sbom.cdx.json +7 -7
- package/tdd/SKILL.md +1 -1
- package/tdd/fixtures/quality-scenarios.json +10 -2
- package/tdd/references/quality-model.md +2 -0
- package/tdd/schemas/quality-evidence.schema.json +21 -0
- package/tdd/schemas/quality-report.schema.json +2 -0
- package/tdd/scripts/quality-gate.js +165 -1
- package/zoom-out/SKILL.md +1 -1
- package/zoom-out/references/circuit-breaker.md +1 -0
|
@@ -239,6 +239,163 @@ function evaluateSource(source, errors, prefix) {
|
|
|
239
239
|
return { source: clone(source), traced: valid ? 1 : 0, total: 1, percentage: valid ? 100 : 0 };
|
|
240
240
|
}
|
|
241
241
|
|
|
242
|
+
// SOURCE_DEFECT and SOURCE_CONFLICT must quote the assertions they blame and
|
|
243
|
+
// compare each with the observed value from the run. Sources are identified by
|
|
244
|
+
// their declared authoritative path.
|
|
245
|
+
const EXPECT_ASSERTION = /^expect\((.*?)\)\s*\.\s*(?:toEqual|toStrictEqual|toBe)\s*\((.*)\)$/;
|
|
246
|
+
const ASSERT_ASSERTION = /^assert\s*\.\s*(?:equal|strictEqual|deepEqual|deepStrictEqual)\s*\((.*)\)$/;
|
|
247
|
+
|
|
248
|
+
function jsonLiteral(text) {
|
|
249
|
+
const literal = text.trim();
|
|
250
|
+
const single = /^'([^'"\\]*)'$/.exec(literal);
|
|
251
|
+
try {
|
|
252
|
+
return { checkable: true, value: JSON.parse(single ? `"${single[1]}"` : literal) };
|
|
253
|
+
} catch (error) {
|
|
254
|
+
return { checkable: false };
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// Splits an argument list on commas at depth 0, outside string literals.
|
|
259
|
+
// Returns null when brackets or strings are unbalanced. Not a general parser.
|
|
260
|
+
function splitTopLevelArguments(text) {
|
|
261
|
+
const parts = [];
|
|
262
|
+
let depth = 0;
|
|
263
|
+
let quote = null;
|
|
264
|
+
let current = '';
|
|
265
|
+
for (let index = 0; index < text.length; index += 1) {
|
|
266
|
+
const ch = text[index];
|
|
267
|
+
if (quote) {
|
|
268
|
+
current += ch;
|
|
269
|
+
if (ch === '\\') {
|
|
270
|
+
current += text[index + 1] ?? '';
|
|
271
|
+
index += 1;
|
|
272
|
+
} else if (ch === quote) {
|
|
273
|
+
quote = null;
|
|
274
|
+
}
|
|
275
|
+
continue;
|
|
276
|
+
}
|
|
277
|
+
if (ch === '"' || ch === "'" || ch === '`') {
|
|
278
|
+
quote = ch;
|
|
279
|
+
} else if ('([{'.includes(ch)) {
|
|
280
|
+
depth += 1;
|
|
281
|
+
} else if (')]}'.includes(ch)) {
|
|
282
|
+
depth -= 1;
|
|
283
|
+
if (depth < 0) return null;
|
|
284
|
+
} else if (ch === ',' && depth === 0) {
|
|
285
|
+
parts.push(current);
|
|
286
|
+
current = '';
|
|
287
|
+
continue;
|
|
288
|
+
}
|
|
289
|
+
current += ch;
|
|
290
|
+
}
|
|
291
|
+
if (quote !== null || depth !== 0) return null;
|
|
292
|
+
parts.push(current);
|
|
293
|
+
return parts;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// The literal expected value pinned by a quoted assertion, for the recognised
|
|
297
|
+
// shapes only. Anything else (identifiers, constructors, prose, negated or
|
|
298
|
+
// resolved modifiers) is not checkable. One trailing ';' is tolerated.
|
|
299
|
+
function quotedExpectedValue(assertion) {
|
|
300
|
+
if (typeof assertion !== 'string') return { checkable: false };
|
|
301
|
+
const text = assertion.trim().replace(/;$/, '').trim();
|
|
302
|
+
const expectCall = EXPECT_ASSERTION.exec(text);
|
|
303
|
+
if (expectCall) return jsonLiteral(expectCall[2]);
|
|
304
|
+
const assertCall = ASSERT_ASSERTION.exec(text);
|
|
305
|
+
if (!assertCall) return { checkable: false };
|
|
306
|
+
// assert.equal(actual, expected[, message]): the expected value is the second argument.
|
|
307
|
+
const args = splitTopLevelArguments(assertCall[1]);
|
|
308
|
+
if (!args || (args.length !== 2 && args.length !== 3)) return { checkable: false };
|
|
309
|
+
return jsonLiteral(args[1]);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// Returns the sourceEvidence status. VERIFIED means two quoted literals from
|
|
313
|
+
// different sources disagree (SOURCE_CONFLICT only). A quote/observed mismatch
|
|
314
|
+
// alone shows the implementation differs from the source, not that the source
|
|
315
|
+
// is defective, so SOURCE_DEFECT is always REVIEWER_REQUIRED.
|
|
316
|
+
function evaluateSourceEvidence(requirement, declaredSources, errors, prefix) {
|
|
317
|
+
const label = requirement.sourceValidity;
|
|
318
|
+
const evidence = requirement.sourceEvidence;
|
|
319
|
+
if (!Array.isArray(evidence) || evidence.length === 0) {
|
|
320
|
+
errors.push(`${prefix}.sourceEvidence must quote the assertions for ${label}`);
|
|
321
|
+
return undefined;
|
|
322
|
+
}
|
|
323
|
+
const declaredPaths = declaredSources.filter(isObject).map(source => source.path);
|
|
324
|
+
const entries = [];
|
|
325
|
+
const checkable = new Map();
|
|
326
|
+
evidence.forEach((entry, index) => {
|
|
327
|
+
const entryPrefix = `${prefix}.sourceEvidence[${index}]`;
|
|
328
|
+
if (!isObject(entry)) {
|
|
329
|
+
errors.push(`${entryPrefix} must be an object`);
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
if (!declaredPaths.includes(entry.source)) errors.push(`${entryPrefix}.source must name a declared authoritative source path`);
|
|
333
|
+
if (!nonEmpty(entry.assertion)) errors.push(`${entryPrefix}.assertion must quote the exact assertion text`);
|
|
334
|
+
if (!('expected' in entry)) errors.push(`${entryPrefix}.expected is required`);
|
|
335
|
+
if (!('observed' in entry)) errors.push(`${entryPrefix}.observed is required`);
|
|
336
|
+
if (declaredPaths.includes(entry.source) && nonEmpty(entry.assertion) && 'expected' in entry && 'observed' in entry) {
|
|
337
|
+
const quoted = quotedExpectedValue(entry.assertion);
|
|
338
|
+
if (quoted.checkable && !equal(quoted.value, entry.expected)) {
|
|
339
|
+
errors.push(`${entryPrefix}.expected does not match the value in the quoted assertion`);
|
|
340
|
+
return;
|
|
341
|
+
}
|
|
342
|
+
checkable.set(entry, quoted.checkable);
|
|
343
|
+
entries.push(entry);
|
|
344
|
+
}
|
|
345
|
+
});
|
|
346
|
+
let status;
|
|
347
|
+
const mismatched = entries.filter(entry => !equal(entry.expected, entry.observed));
|
|
348
|
+
if (mismatched.length === 0) {
|
|
349
|
+
errors.push(`${prefix}.sourceEvidence must compare a quoted assertion with a failing observed value before blaming the source`);
|
|
350
|
+
} else if (label === 'SOURCE_DEFECT') {
|
|
351
|
+
// No independent corroboration exists for a single source, so the reviewer decides.
|
|
352
|
+
status = 'REVIEWER_REQUIRED';
|
|
353
|
+
}
|
|
354
|
+
if (label === 'SOURCE_CONFLICT') {
|
|
355
|
+
if (new Set(entries.map(entry => entry.source)).size < 2) {
|
|
356
|
+
errors.push(`${prefix}.sourceEvidence must quote assertions from at least two distinct authoritative sources`);
|
|
357
|
+
}
|
|
358
|
+
// A conflict needs two sources constraining the same contract subject;
|
|
359
|
+
// unequal expectations about unrelated targets are not a conflict.
|
|
360
|
+
const subjects = new Map();
|
|
361
|
+
evidence.forEach((entry, index) => {
|
|
362
|
+
if (!entries.includes(entry)) return;
|
|
363
|
+
const entryPrefix = `${prefix}.sourceEvidence[${index}]`;
|
|
364
|
+
if (!nonEmpty(entry.subject)) {
|
|
365
|
+
errors.push(`${entryPrefix}.subject must name the contract target the assertion constrains`);
|
|
366
|
+
} else if (!assertionNamesSubject(entry.assertion, entry.subject.trim())) {
|
|
367
|
+
errors.push(`${entryPrefix}.subject must appear as a whole target in the quoted assertion`);
|
|
368
|
+
} else {
|
|
369
|
+
subjects.set(entry, entry.subject.trim());
|
|
370
|
+
}
|
|
371
|
+
});
|
|
372
|
+
const sameSubjectPairs = [];
|
|
373
|
+
for (const a of subjects.keys()) {
|
|
374
|
+
for (const b of subjects.keys()) {
|
|
375
|
+
if (a !== b && a.source !== b.source && subjects.get(a) === subjects.get(b)) sameSubjectPairs.push([a, b]);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
const disagreeing = sameSubjectPairs.filter(([a, b]) => !equal(a.expected, b.expected));
|
|
379
|
+
if (subjects.size >= 2 && sameSubjectPairs.length === 0) {
|
|
380
|
+
errors.push(`${prefix}.sourceEvidence must quote assertions from two sources about the same subject; unequal expectations about unrelated targets are not a SOURCE_CONFLICT`);
|
|
381
|
+
} else if (sameSubjectPairs.length && disagreeing.length === 0) {
|
|
382
|
+
errors.push(`${prefix}.sourceEvidence expected values must disagree across sources; agreeing expectations with a failing observed value are NON_CONFORMANT`);
|
|
383
|
+
} else if (disagreeing.length) {
|
|
384
|
+
// Only a disagreement between two literal quotes is verified; a quote that
|
|
385
|
+
// cannot be read mechanically goes to the reviewer without an error.
|
|
386
|
+
status = disagreeing.some(([a, b]) => checkable.get(a) && checkable.get(b)) ? 'VERIFIED' : 'REVIEWER_REQUIRED';
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
return status;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
// True when `subject` appears in `assertion` as a whole target: `list.head`
|
|
393
|
+
// matches `expect(list.head)` but not `expect(list.head.value)`.
|
|
394
|
+
function assertionNamesSubject(assertion, subject) {
|
|
395
|
+
const escaped = subject.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
396
|
+
return new RegExp(`(?<![\\w$.])${escaped}(?![\\w$.\\[])`).test(assertion);
|
|
397
|
+
}
|
|
398
|
+
|
|
242
399
|
function scoreDimensions(dimensions) {
|
|
243
400
|
let weighted = 0;
|
|
244
401
|
let appliedWeight = 0;
|
|
@@ -323,6 +480,10 @@ function evaluateRequirement(requirement, index, errors) {
|
|
|
323
480
|
if (requirement.sourceValidity === 'SOURCE_CONFLICT' && declaredSources.some(source => !isObject(source) || source.status !== 'AUTHORITATIVE')) {
|
|
324
481
|
errors.push(`${prefix}.SOURCE_CONFLICT may only identify authoritative sources`);
|
|
325
482
|
}
|
|
483
|
+
let sourceEvidenceStatus;
|
|
484
|
+
if (['SOURCE_DEFECT', 'SOURCE_CONFLICT'].includes(requirement.sourceValidity)) {
|
|
485
|
+
sourceEvidenceStatus = evaluateSourceEvidence(requirement, declaredSources, errors, prefix);
|
|
486
|
+
}
|
|
326
487
|
|
|
327
488
|
const scoring = scoringDetails(dimensions);
|
|
328
489
|
const score = scoreDimensions(dimensions);
|
|
@@ -347,6 +508,8 @@ function evaluateRequirement(requirement, index, errors) {
|
|
|
347
508
|
sourceConformance: requirement.sourceConformance || null,
|
|
348
509
|
sourceValidity: requirement.sourceValidity || null,
|
|
349
510
|
...(requirement.sourceNotes ? { sourceNotes: requirement.sourceNotes } : {}),
|
|
511
|
+
...(requirement.sourceEvidence ? { sourceEvidence: clone(requirement.sourceEvidence) } : {}),
|
|
512
|
+
...(sourceEvidenceStatus ? { sourceEvidenceStatus } : {}),
|
|
350
513
|
dimensions,
|
|
351
514
|
skippedTests,
|
|
352
515
|
flakyTests: Array.isArray(requirement.flakyTests) ? clone(requirement.flakyTests) : [],
|
|
@@ -432,6 +595,7 @@ function printSummary(report) {
|
|
|
432
595
|
console.log('Requirements:');
|
|
433
596
|
for (const requirement of report.requirements) {
|
|
434
597
|
console.log(`- ${requirement.requirementId} [${requirement.testProfile}]: ${requirement.result} ${requirement.score}%`);
|
|
598
|
+
if (requirement.sourceEvidenceStatus) console.log(`- ${requirement.requirementId} sourceEvidence: ${requirement.sourceEvidenceStatus}`);
|
|
435
599
|
}
|
|
436
600
|
console.log('Dimensions:');
|
|
437
601
|
for (const name of DIMENSIONS) {
|
|
@@ -472,4 +636,4 @@ function runCli(argv) {
|
|
|
472
636
|
|
|
473
637
|
if (require.main === module) process.exit(runCli(process.argv));
|
|
474
638
|
|
|
475
|
-
module.exports = { DIMENSIONS, REQUIRED_CLASSES, VERSION, evaluateEvidence, runCli };
|
|
639
|
+
module.exports = { DIMENSIONS, REQUIRED_CLASSES, VERSION, evaluateEvidence, quotedExpectedValue, runCli };
|
|
@@ -33,6 +33,7 @@ flowchart TD
|
|
|
33
33
|
- "Same decision class" means the same acceptance decision under the same trust assumptions, possibly at different sites. "Same directory" or "validation" is too broad. Count a repeated or renamed finding once, and treat missing history as unknown, not zero.
|
|
34
34
|
- Phase 2 here means: list every finding so far by decision class, state the acceptance clause each violates, and inventory every site (callers, state readers, Stop consumers) that makes the same decision. Findings that map to no acceptance clause stay `unresolved` for the owner instead of becoming new fixes.
|
|
35
35
|
- RESUME by using the inventory to choose the smallest justified correction (which may be one shared fix, or no code change), then re-trace one normal end-to-end acceptance path. Unresolved findings need scope adjudication, not speculative repairs.
|
|
36
|
+
- Whoever applies the fixes counts too. Before fixing review round 2 or later on one feature, classify each new finding against the decision classes of earlier rounds. A class that also appeared in the previous round MUST trigger Phase 2 before any edit; do not wait for the reviewer to name the pattern.
|
|
36
37
|
- A one-shot reviewer cannot see the previous round on its own. The packet carries the current acceptance anchor, a link to the previous comparable report with its revision, the prior finding IDs and dispositions, and the fix made since.
|
|
37
38
|
- **Goal Drift**: Execution strays far from the initial task objective.
|
|
38
39
|
- **User Instruction**: The user requests a step back ("zoom out", "step back", "stop and think").
|
package/sbom.cdx.json
CHANGED
|
@@ -2,23 +2,23 @@
|
|
|
2
2
|
"$schema": "https://cyclonedx.org/schema/bom-1.7.schema.json",
|
|
3
3
|
"bomFormat": "CycloneDX",
|
|
4
4
|
"specVersion": "1.7",
|
|
5
|
-
"serialNumber": "urn:uuid:
|
|
5
|
+
"serialNumber": "urn:uuid:7704887c-bf54-5e11-9e00-50ee977f324d",
|
|
6
6
|
"version": 1,
|
|
7
7
|
"metadata": {
|
|
8
|
-
"timestamp": "2026-
|
|
8
|
+
"timestamp": "2026-11-03T02:46:23.147Z",
|
|
9
9
|
"tools": [
|
|
10
10
|
{
|
|
11
11
|
"vendor": "Harness Everything",
|
|
12
12
|
"name": "generate-sbom.js",
|
|
13
|
-
"version": "0.47.
|
|
13
|
+
"version": "0.47.5"
|
|
14
14
|
}
|
|
15
15
|
],
|
|
16
16
|
"component": {
|
|
17
17
|
"type": "application",
|
|
18
|
-
"bom-ref": "pkg:npm/harness-everything@0.47.
|
|
18
|
+
"bom-ref": "pkg:npm/harness-everything@0.47.5",
|
|
19
19
|
"name": "harness-everything",
|
|
20
|
-
"version": "0.47.
|
|
21
|
-
"purl": "pkg:npm/harness-everything@0.47.
|
|
20
|
+
"version": "0.47.5",
|
|
21
|
+
"purl": "pkg:npm/harness-everything@0.47.5"
|
|
22
22
|
}
|
|
23
23
|
},
|
|
24
24
|
"components": [
|
|
@@ -2033,7 +2033,7 @@
|
|
|
2033
2033
|
]
|
|
2034
2034
|
},
|
|
2035
2035
|
{
|
|
2036
|
-
"ref": "pkg:npm/harness-everything@0.47.
|
|
2036
|
+
"ref": "pkg:npm/harness-everything@0.47.5",
|
|
2037
2037
|
"dependsOn": [
|
|
2038
2038
|
"pkg:npm/%40commitlint/cli@19.8.1",
|
|
2039
2039
|
"pkg:npm/%40commitlint/config-conventional@19.8.1",
|
package/tdd/SKILL.md
CHANGED
|
@@ -8,8 +8,16 @@
|
|
|
8
8
|
{ "name": "weighted integration failure", "mutation": "the 0.15 positive dimension scores zero", "expectedScore": 85, "expectedResult": "FAIL" },
|
|
9
9
|
{ "name": "weighted unit failure", "mutation": "unit determinism is N/A and the 0.15 positive dimension scores zero", "expectedScore": 83.33, "expectedResult": "FAIL" },
|
|
10
10
|
{ "name": "NON_CONFORMANT", "mutation": "sourceConformance = NON_CONFORMANT", "expectedResult": "FAIL" },
|
|
11
|
-
{ "name": "SOURCE_DEFECT", "mutation": "sourceValidity = SOURCE_DEFECT and
|
|
12
|
-
{ "name": "
|
|
11
|
+
{ "name": "SOURCE_DEFECT", "mutation": "sourceValidity = SOURCE_DEFECT, sourceNotes is present, and sourceEvidence quotes the defective assertion against the observed value; sourceEvidenceStatus is REVIEWER_REQUIRED, never VERIFIED, because a single-source mismatch does not prove the source is wrong", "expectedResult": "PASS" },
|
|
12
|
+
{ "name": "SOURCE_DEFECT without sourceEvidence", "mutation": "sourceValidity = SOURCE_DEFECT with sourceNotes but no sourceEvidence", "expectedResult": "FAIL" },
|
|
13
|
+
{ "name": "SOURCE_CONFLICT", "mutation": "sourceValidity = SOURCE_CONFLICT with a traced additional source and sourceEvidence quoting both sources", "expectedResult": "FAIL" },
|
|
14
|
+
{ "name": "SOURCE_CONFLICT without sourceEvidence", "mutation": "sourceValidity = SOURCE_CONFLICT with two authoritative sources but no sourceEvidence", "expectedResult": "FAIL" },
|
|
15
|
+
{ "name": "SOURCE_CONFLICT node-versus-value", "mutation": "sourceValidity = SOURCE_CONFLICT citing only expect(list.head.value).toEqual(1) from one spec, observed value is an Element object", "expectedEvidence": "REJECTED", "expectedResult": "FAIL" },
|
|
16
|
+
{ "name": "SOURCE_CONFLICT agreeing expectations", "mutation": "sourceValidity = SOURCE_CONFLICT with two sources expecting the same value and a failing observed value; the mismatch is implementation behaviour", "expectedEvidence": "REJECTED", "expectedResult": "FAIL" },
|
|
17
|
+
{ "name": "SOURCE_CONFLICT two-source evidence", "mutation": "sourceValidity = SOURCE_CONFLICT with two authoritative sources quoting disagreeing expectations for the same subject list.head and a failing observed value; one quote (toBeInstanceOf) is not mechanically checkable, so sourceEvidenceStatus is REVIEWER_REQUIRED and the unresolved conflict gate stays closed", "expectedEvidence": "ACCEPTED", "expectedResult": "FAIL" },
|
|
18
|
+
{ "name": "SOURCE_CONFLICT relabelled expected", "mutation": "sourceValidity = SOURCE_CONFLICT with two authoritative sources quoting the identical assertion expect(list.head).toEqual(1) but expected values 1 and 3; each expected must match its quoted literal, so the second entry is rejected", "expectedEvidence": "REJECTED", "expectedResult": "FAIL" },
|
|
19
|
+
{ "name": "SOURCE_CONFLICT verified literal disagreement", "mutation": "sourceValidity = SOURCE_CONFLICT with two authoritative sources quoting expect(list.head.value).toBe(1) and expect(list.head.value).toBe('1') with matching expected values 1 and '1' and a failing observed value; sourceEvidenceStatus is VERIFIED and the unresolved conflict gate stays closed", "expectedEvidence": "ACCEPTED", "expectedResult": "FAIL" },
|
|
20
|
+
{ "name": "SOURCE_CONFLICT unrelated assertions", "mutation": "sourceValidity = SOURCE_CONFLICT quoting expect(foo).toEqual(1) from one source and expect(bar).toEqual(3) from another; unequal expectations about different subjects", "expectedEvidence": "REJECTED", "expectedResult": "FAIL" },
|
|
13
21
|
{ "name": "malformed input", "mutation": "requirements is not an array", "expectedResult": "FAIL" },
|
|
14
22
|
{ "name": "unexpected input", "mutation": "one unexpectedInput check is FAIL", "expectedResult": "FAIL" },
|
|
15
23
|
{ "name": "error handling", "mutation": "one errorHandling check is SKIPPED", "expectedResult": "FAIL" },
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
This model turns each source requirement into auditable evidence. The authoritative source remains the contract even when it appears wrong. Record a suspected documentation problem as `SOURCE_DEFECT`; record disagreement between traced sources as `SOURCE_CONFLICT`. Never reinterpret either into a passing implementation. If no authoritative source exists, use `INFERRED` and state the inference basis.
|
|
4
4
|
|
|
5
|
+
Before declaring either outcome, `sourceEvidence` quotes the exact conflicting assertions, names each declared source by path, and compares each `expected` value with the `observed` value from the run. A `SOURCE_CONFLICT` needs at least two distinct sources whose expected values disagree about the same `subject` — the contract target each quoted assertion constrains, written as it appears in the quote (`list.head` is not `list.head.value`). Unequal expectations about unrelated targets are not a conflict. If the quoted expectations agree and the run still fails, the mismatch is in the implementation: record `NON_CONFORMANT`, not `SOURCE_CONFLICT`. Each `expected` must match a mechanically readable literal in the quoted assertion (`toEqual`, `toStrictEqual`, `toBe`, or `assert.*` with a JSON or single-quoted literal); otherwise the claim is marked `REVIEWER_REQUIRED` and the gate does not treat it as an established disagreement. `VERIFIED` applies only to a `SOURCE_CONFLICT` whose two quoted literals disagree. A `SOURCE_DEFECT` is always `REVIEWER_REQUIRED`, because a mismatch with the observed value does not prove the source is wrong; the implementation may be the one at fault.
|
|
6
|
+
|
|
5
7
|
## Profile contract
|
|
6
8
|
|
|
7
9
|
Every requirement declares exactly one `testProfile`: `unit` or `integration`. The selected profile defines which case classes are applicable; the common scoring and source rules do not make every test use identical evidence.
|
|
@@ -149,6 +149,18 @@
|
|
|
149
149
|
}
|
|
150
150
|
]
|
|
151
151
|
},
|
|
152
|
+
"sourceEvidence": {
|
|
153
|
+
"type": "object",
|
|
154
|
+
"additionalProperties": false,
|
|
155
|
+
"required": ["source", "assertion", "expected", "observed"],
|
|
156
|
+
"properties": {
|
|
157
|
+
"source": { "type": "string", "minLength": 1 },
|
|
158
|
+
"subject": { "type": "string", "minLength": 1 },
|
|
159
|
+
"assertion": { "type": "string", "minLength": 1 },
|
|
160
|
+
"expected": {},
|
|
161
|
+
"observed": {}
|
|
162
|
+
}
|
|
163
|
+
},
|
|
152
164
|
"requirement": {
|
|
153
165
|
"type": "object",
|
|
154
166
|
"additionalProperties": false,
|
|
@@ -161,6 +173,7 @@
|
|
|
161
173
|
"sourceConformance": { "enum": ["PASS", "NON_CONFORMANT"] },
|
|
162
174
|
"sourceValidity": { "enum": ["VALID", "SOURCE_DEFECT", "SOURCE_CONFLICT"] },
|
|
163
175
|
"sourceNotes": { "type": "string", "minLength": 1 },
|
|
176
|
+
"sourceEvidence": { "type": "array", "items": { "$ref": "#/$defs/sourceEvidence" } },
|
|
164
177
|
"skippedTests": { "type": "array" },
|
|
165
178
|
"flakyTests": { "type": "array" },
|
|
166
179
|
"naDeclarations": { "type": "array" },
|
|
@@ -186,6 +199,14 @@
|
|
|
186
199
|
}
|
|
187
200
|
},
|
|
188
201
|
"allOf": [
|
|
202
|
+
{
|
|
203
|
+
"if": { "required": ["sourceValidity"], "properties": { "sourceValidity": { "enum": ["SOURCE_DEFECT", "SOURCE_CONFLICT"] } } },
|
|
204
|
+
"then": { "required": ["sourceEvidence"], "properties": { "sourceEvidence": { "minItems": 1 } } }
|
|
205
|
+
},
|
|
206
|
+
{
|
|
207
|
+
"if": { "required": ["sourceValidity"], "properties": { "sourceValidity": { "const": "SOURCE_CONFLICT" } } },
|
|
208
|
+
"then": { "properties": { "sourceEvidence": { "items": { "required": ["subject"] } } } }
|
|
209
|
+
},
|
|
189
210
|
{
|
|
190
211
|
"if": {
|
|
191
212
|
"required": ["testProfile"],
|
|
@@ -96,6 +96,8 @@
|
|
|
96
96
|
"sourceConformance": { "enum": ["PASS", "NON_CONFORMANT", null] },
|
|
97
97
|
"sourceValidity": { "enum": ["VALID", "SOURCE_DEFECT", "SOURCE_CONFLICT", null] },
|
|
98
98
|
"sourceNotes": { "type": "string" },
|
|
99
|
+
"sourceEvidence": { "type": "array" },
|
|
100
|
+
"sourceEvidenceStatus": { "enum": ["VERIFIED", "REVIEWER_REQUIRED"], "description": "VERIFIED only for a SOURCE_CONFLICT whose two quoted literals disagree; SOURCE_DEFECT is always REVIEWER_REQUIRED." },
|
|
99
101
|
"dimensions": { "$ref": "#/$defs/dimensions" },
|
|
100
102
|
"skippedTests": { "type": "array" },
|
|
101
103
|
"flakyTests": { "type": "array" },
|
|
@@ -239,6 +239,163 @@ function evaluateSource(source, errors, prefix) {
|
|
|
239
239
|
return { source: clone(source), traced: valid ? 1 : 0, total: 1, percentage: valid ? 100 : 0 };
|
|
240
240
|
}
|
|
241
241
|
|
|
242
|
+
// SOURCE_DEFECT and SOURCE_CONFLICT must quote the assertions they blame and
|
|
243
|
+
// compare each with the observed value from the run. Sources are identified by
|
|
244
|
+
// their declared authoritative path.
|
|
245
|
+
const EXPECT_ASSERTION = /^expect\((.*?)\)\s*\.\s*(?:toEqual|toStrictEqual|toBe)\s*\((.*)\)$/;
|
|
246
|
+
const ASSERT_ASSERTION = /^assert\s*\.\s*(?:equal|strictEqual|deepEqual|deepStrictEqual)\s*\((.*)\)$/;
|
|
247
|
+
|
|
248
|
+
function jsonLiteral(text) {
|
|
249
|
+
const literal = text.trim();
|
|
250
|
+
const single = /^'([^'"\\]*)'$/.exec(literal);
|
|
251
|
+
try {
|
|
252
|
+
return { checkable: true, value: JSON.parse(single ? `"${single[1]}"` : literal) };
|
|
253
|
+
} catch (error) {
|
|
254
|
+
return { checkable: false };
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// Splits an argument list on commas at depth 0, outside string literals.
|
|
259
|
+
// Returns null when brackets or strings are unbalanced. Not a general parser.
|
|
260
|
+
function splitTopLevelArguments(text) {
|
|
261
|
+
const parts = [];
|
|
262
|
+
let depth = 0;
|
|
263
|
+
let quote = null;
|
|
264
|
+
let current = '';
|
|
265
|
+
for (let index = 0; index < text.length; index += 1) {
|
|
266
|
+
const ch = text[index];
|
|
267
|
+
if (quote) {
|
|
268
|
+
current += ch;
|
|
269
|
+
if (ch === '\\') {
|
|
270
|
+
current += text[index + 1] ?? '';
|
|
271
|
+
index += 1;
|
|
272
|
+
} else if (ch === quote) {
|
|
273
|
+
quote = null;
|
|
274
|
+
}
|
|
275
|
+
continue;
|
|
276
|
+
}
|
|
277
|
+
if (ch === '"' || ch === "'" || ch === '`') {
|
|
278
|
+
quote = ch;
|
|
279
|
+
} else if ('([{'.includes(ch)) {
|
|
280
|
+
depth += 1;
|
|
281
|
+
} else if (')]}'.includes(ch)) {
|
|
282
|
+
depth -= 1;
|
|
283
|
+
if (depth < 0) return null;
|
|
284
|
+
} else if (ch === ',' && depth === 0) {
|
|
285
|
+
parts.push(current);
|
|
286
|
+
current = '';
|
|
287
|
+
continue;
|
|
288
|
+
}
|
|
289
|
+
current += ch;
|
|
290
|
+
}
|
|
291
|
+
if (quote !== null || depth !== 0) return null;
|
|
292
|
+
parts.push(current);
|
|
293
|
+
return parts;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// The literal expected value pinned by a quoted assertion, for the recognised
|
|
297
|
+
// shapes only. Anything else (identifiers, constructors, prose, negated or
|
|
298
|
+
// resolved modifiers) is not checkable. One trailing ';' is tolerated.
|
|
299
|
+
function quotedExpectedValue(assertion) {
|
|
300
|
+
if (typeof assertion !== 'string') return { checkable: false };
|
|
301
|
+
const text = assertion.trim().replace(/;$/, '').trim();
|
|
302
|
+
const expectCall = EXPECT_ASSERTION.exec(text);
|
|
303
|
+
if (expectCall) return jsonLiteral(expectCall[2]);
|
|
304
|
+
const assertCall = ASSERT_ASSERTION.exec(text);
|
|
305
|
+
if (!assertCall) return { checkable: false };
|
|
306
|
+
// assert.equal(actual, expected[, message]): the expected value is the second argument.
|
|
307
|
+
const args = splitTopLevelArguments(assertCall[1]);
|
|
308
|
+
if (!args || (args.length !== 2 && args.length !== 3)) return { checkable: false };
|
|
309
|
+
return jsonLiteral(args[1]);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// Returns the sourceEvidence status. VERIFIED means two quoted literals from
|
|
313
|
+
// different sources disagree (SOURCE_CONFLICT only). A quote/observed mismatch
|
|
314
|
+
// alone shows the implementation differs from the source, not that the source
|
|
315
|
+
// is defective, so SOURCE_DEFECT is always REVIEWER_REQUIRED.
|
|
316
|
+
function evaluateSourceEvidence(requirement, declaredSources, errors, prefix) {
|
|
317
|
+
const label = requirement.sourceValidity;
|
|
318
|
+
const evidence = requirement.sourceEvidence;
|
|
319
|
+
if (!Array.isArray(evidence) || evidence.length === 0) {
|
|
320
|
+
errors.push(`${prefix}.sourceEvidence must quote the assertions for ${label}`);
|
|
321
|
+
return undefined;
|
|
322
|
+
}
|
|
323
|
+
const declaredPaths = declaredSources.filter(isObject).map(source => source.path);
|
|
324
|
+
const entries = [];
|
|
325
|
+
const checkable = new Map();
|
|
326
|
+
evidence.forEach((entry, index) => {
|
|
327
|
+
const entryPrefix = `${prefix}.sourceEvidence[${index}]`;
|
|
328
|
+
if (!isObject(entry)) {
|
|
329
|
+
errors.push(`${entryPrefix} must be an object`);
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
if (!declaredPaths.includes(entry.source)) errors.push(`${entryPrefix}.source must name a declared authoritative source path`);
|
|
333
|
+
if (!nonEmpty(entry.assertion)) errors.push(`${entryPrefix}.assertion must quote the exact assertion text`);
|
|
334
|
+
if (!('expected' in entry)) errors.push(`${entryPrefix}.expected is required`);
|
|
335
|
+
if (!('observed' in entry)) errors.push(`${entryPrefix}.observed is required`);
|
|
336
|
+
if (declaredPaths.includes(entry.source) && nonEmpty(entry.assertion) && 'expected' in entry && 'observed' in entry) {
|
|
337
|
+
const quoted = quotedExpectedValue(entry.assertion);
|
|
338
|
+
if (quoted.checkable && !equal(quoted.value, entry.expected)) {
|
|
339
|
+
errors.push(`${entryPrefix}.expected does not match the value in the quoted assertion`);
|
|
340
|
+
return;
|
|
341
|
+
}
|
|
342
|
+
checkable.set(entry, quoted.checkable);
|
|
343
|
+
entries.push(entry);
|
|
344
|
+
}
|
|
345
|
+
});
|
|
346
|
+
let status;
|
|
347
|
+
const mismatched = entries.filter(entry => !equal(entry.expected, entry.observed));
|
|
348
|
+
if (mismatched.length === 0) {
|
|
349
|
+
errors.push(`${prefix}.sourceEvidence must compare a quoted assertion with a failing observed value before blaming the source`);
|
|
350
|
+
} else if (label === 'SOURCE_DEFECT') {
|
|
351
|
+
// No independent corroboration exists for a single source, so the reviewer decides.
|
|
352
|
+
status = 'REVIEWER_REQUIRED';
|
|
353
|
+
}
|
|
354
|
+
if (label === 'SOURCE_CONFLICT') {
|
|
355
|
+
if (new Set(entries.map(entry => entry.source)).size < 2) {
|
|
356
|
+
errors.push(`${prefix}.sourceEvidence must quote assertions from at least two distinct authoritative sources`);
|
|
357
|
+
}
|
|
358
|
+
// A conflict needs two sources constraining the same contract subject;
|
|
359
|
+
// unequal expectations about unrelated targets are not a conflict.
|
|
360
|
+
const subjects = new Map();
|
|
361
|
+
evidence.forEach((entry, index) => {
|
|
362
|
+
if (!entries.includes(entry)) return;
|
|
363
|
+
const entryPrefix = `${prefix}.sourceEvidence[${index}]`;
|
|
364
|
+
if (!nonEmpty(entry.subject)) {
|
|
365
|
+
errors.push(`${entryPrefix}.subject must name the contract target the assertion constrains`);
|
|
366
|
+
} else if (!assertionNamesSubject(entry.assertion, entry.subject.trim())) {
|
|
367
|
+
errors.push(`${entryPrefix}.subject must appear as a whole target in the quoted assertion`);
|
|
368
|
+
} else {
|
|
369
|
+
subjects.set(entry, entry.subject.trim());
|
|
370
|
+
}
|
|
371
|
+
});
|
|
372
|
+
const sameSubjectPairs = [];
|
|
373
|
+
for (const a of subjects.keys()) {
|
|
374
|
+
for (const b of subjects.keys()) {
|
|
375
|
+
if (a !== b && a.source !== b.source && subjects.get(a) === subjects.get(b)) sameSubjectPairs.push([a, b]);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
const disagreeing = sameSubjectPairs.filter(([a, b]) => !equal(a.expected, b.expected));
|
|
379
|
+
if (subjects.size >= 2 && sameSubjectPairs.length === 0) {
|
|
380
|
+
errors.push(`${prefix}.sourceEvidence must quote assertions from two sources about the same subject; unequal expectations about unrelated targets are not a SOURCE_CONFLICT`);
|
|
381
|
+
} else if (sameSubjectPairs.length && disagreeing.length === 0) {
|
|
382
|
+
errors.push(`${prefix}.sourceEvidence expected values must disagree across sources; agreeing expectations with a failing observed value are NON_CONFORMANT`);
|
|
383
|
+
} else if (disagreeing.length) {
|
|
384
|
+
// Only a disagreement between two literal quotes is verified; a quote that
|
|
385
|
+
// cannot be read mechanically goes to the reviewer without an error.
|
|
386
|
+
status = disagreeing.some(([a, b]) => checkable.get(a) && checkable.get(b)) ? 'VERIFIED' : 'REVIEWER_REQUIRED';
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
return status;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
// True when `subject` appears in `assertion` as a whole target: `list.head`
|
|
393
|
+
// matches `expect(list.head)` but not `expect(list.head.value)`.
|
|
394
|
+
function assertionNamesSubject(assertion, subject) {
|
|
395
|
+
const escaped = subject.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
396
|
+
return new RegExp(`(?<![\\w$.])${escaped}(?![\\w$.\\[])`).test(assertion);
|
|
397
|
+
}
|
|
398
|
+
|
|
242
399
|
function scoreDimensions(dimensions) {
|
|
243
400
|
let weighted = 0;
|
|
244
401
|
let appliedWeight = 0;
|
|
@@ -323,6 +480,10 @@ function evaluateRequirement(requirement, index, errors) {
|
|
|
323
480
|
if (requirement.sourceValidity === 'SOURCE_CONFLICT' && declaredSources.some(source => !isObject(source) || source.status !== 'AUTHORITATIVE')) {
|
|
324
481
|
errors.push(`${prefix}.SOURCE_CONFLICT may only identify authoritative sources`);
|
|
325
482
|
}
|
|
483
|
+
let sourceEvidenceStatus;
|
|
484
|
+
if (['SOURCE_DEFECT', 'SOURCE_CONFLICT'].includes(requirement.sourceValidity)) {
|
|
485
|
+
sourceEvidenceStatus = evaluateSourceEvidence(requirement, declaredSources, errors, prefix);
|
|
486
|
+
}
|
|
326
487
|
|
|
327
488
|
const scoring = scoringDetails(dimensions);
|
|
328
489
|
const score = scoreDimensions(dimensions);
|
|
@@ -347,6 +508,8 @@ function evaluateRequirement(requirement, index, errors) {
|
|
|
347
508
|
sourceConformance: requirement.sourceConformance || null,
|
|
348
509
|
sourceValidity: requirement.sourceValidity || null,
|
|
349
510
|
...(requirement.sourceNotes ? { sourceNotes: requirement.sourceNotes } : {}),
|
|
511
|
+
...(requirement.sourceEvidence ? { sourceEvidence: clone(requirement.sourceEvidence) } : {}),
|
|
512
|
+
...(sourceEvidenceStatus ? { sourceEvidenceStatus } : {}),
|
|
350
513
|
dimensions,
|
|
351
514
|
skippedTests,
|
|
352
515
|
flakyTests: Array.isArray(requirement.flakyTests) ? clone(requirement.flakyTests) : [],
|
|
@@ -432,6 +595,7 @@ function printSummary(report) {
|
|
|
432
595
|
console.log('Requirements:');
|
|
433
596
|
for (const requirement of report.requirements) {
|
|
434
597
|
console.log(`- ${requirement.requirementId} [${requirement.testProfile}]: ${requirement.result} ${requirement.score}%`);
|
|
598
|
+
if (requirement.sourceEvidenceStatus) console.log(`- ${requirement.requirementId} sourceEvidence: ${requirement.sourceEvidenceStatus}`);
|
|
435
599
|
}
|
|
436
600
|
console.log('Dimensions:');
|
|
437
601
|
for (const name of DIMENSIONS) {
|
|
@@ -472,4 +636,4 @@ function runCli(argv) {
|
|
|
472
636
|
|
|
473
637
|
if (require.main === module) process.exit(runCli(process.argv));
|
|
474
638
|
|
|
475
|
-
module.exports = { DIMENSIONS, REQUIRED_CLASSES, VERSION, evaluateEvidence, runCli };
|
|
639
|
+
module.exports = { DIMENSIONS, REQUIRED_CLASSES, VERSION, evaluateEvidence, quotedExpectedValue, runCli };
|
package/zoom-out/SKILL.md
CHANGED
|
@@ -33,6 +33,7 @@ flowchart TD
|
|
|
33
33
|
- "Same decision class" means the same acceptance decision under the same trust assumptions, possibly at different sites. "Same directory" or "validation" is too broad. Count a repeated or renamed finding once, and treat missing history as unknown, not zero.
|
|
34
34
|
- Phase 2 here means: list every finding so far by decision class, state the acceptance clause each violates, and inventory every site (callers, state readers, Stop consumers) that makes the same decision. Findings that map to no acceptance clause stay `unresolved` for the owner instead of becoming new fixes.
|
|
35
35
|
- RESUME by using the inventory to choose the smallest justified correction (which may be one shared fix, or no code change), then re-trace one normal end-to-end acceptance path. Unresolved findings need scope adjudication, not speculative repairs.
|
|
36
|
+
- Whoever applies the fixes counts too. Before fixing review round 2 or later on one feature, classify each new finding against the decision classes of earlier rounds. A class that also appeared in the previous round MUST trigger Phase 2 before any edit; do not wait for the reviewer to name the pattern.
|
|
36
37
|
- A one-shot reviewer cannot see the previous round on its own. The packet carries the current acceptance anchor, a link to the previous comparable report with its revision, the prior finding IDs and dispositions, and the fix made since.
|
|
37
38
|
- **Goal Drift**: Execution strays far from the initial task objective.
|
|
38
39
|
- **User Instruction**: The user requests a step back ("zoom out", "step back", "stop and think").
|