@produtype/core 0.7.1 → 0.8.0

package/dist/expectations/evaluateExpectations.js

@@ -287,19 +287,51 @@ function evidenceQualityFor(detectors) {
     }
     return 'weak';
 }
-function confidenceFor(status, profileMode, evidenceQuality) {
-    if (profileMode !== 'observed-only' && profileMode !== 'auto' && status !== 'unknown') {
-        return 'high';
-    }
-    if (profileMode === 'auto') {
-        return evidenceQuality === 'strong' ? 'medium' : 'low';
-    }
+/**
+ * How sure the analyzer is about *this* finding.
+ *
+ * It used to fold in how the profile was chosen: on `auto` — which is what the hosted
+ * product uses by default — every finding came out low or medium however strong its
+ * evidence. Two different uncertainties were being multiplied into one number, and the
+ * profile's own uncertainty is already reported separately as `inferenceConfidence`.
+ *
+ * An independent review of a real report put it plainly: eleven findings of
+ * thirty-two carried "no direct evidence captured" at low confidence and weak
+ * evidence, and still arrived as critical or high.
+ */
+function confidenceFor(status, evidenceQuality) {
+    // Nothing was determined, so there is nothing to be confident about.
+    if (status === 'unknown')
+        return 'low';
     if (evidenceQuality === 'strong')
         return 'high';
     if (evidenceQuality === 'medium')
         return 'medium';
     return 'low';
 }
+/**
+ * Severity says how bad it would be if true. Confidence says whether it is.
+ *
+ * A reader treats `critical` as "stop and fix this", and spending that word on a claim
+ * the analyzer itself is unsure of is how a tool becomes a checklist nobody trusts.
+ * The claim stays, at the weight the evidence supports.
+ */
+function severityForConfidence(severity, confidence) {
+    if (confidence !== 'low')
+        return severity;
+    /**
+     * `critical` only. Not a ceiling at `medium`, which was the first attempt and was
+     * wrong for a reason worth writing down: a missing capability cannot carry direct
+     * evidence — there is no line to point at for something that is not there — so every
+     * absence reads as low confidence, and capping them all at `medium` would leave the
+     * report unable to say anything is serious.
+     *
+     * What it can stop doing is shouting. `critical` is read as "stop and fix this
+     * before anything else", and the analyzer should not spend that word on a claim it
+     * could not evidence.
+     */
+    return severity === 'critical' ? 'high' : severity;
+}
 /**
  * Which capabilities each declaration makes required.
  *
@@ -390,10 +422,11 @@ function evaluateExpectedCapabilities(args) {
             effectiveImportance = 'not_applicable';
         }
         const findingId = toFindingId(cap, effectiveImportance);
-        const severity = severityFor(cap, status, effectiveImportance);
+        const claimedSeverity = severityFor(cap, status, effectiveImportance);
         const detectorEvidence = evidenceFor(args.analysis, cap).flatMap((d) => d.evidence);
         const quality = evidenceQualityFor(evidenceFor(args.analysis, cap));
-        const confidence = confidenceFor(status, args.requestedProfile, quality);
+        const confidence = confidenceFor(status, quality);
+        const severity = severityForConfidence(claimedSeverity, confidence);
         const evaluation = {
             capabilityId: cap.id,
             title: cap.title,

package/dist/report/categoryScores.js

@@ -52,7 +52,16 @@ function buildCategoryScores(findings) {
             findingCount: actionable.length,
             criticalCount: actionable.filter((finding) => finding.severity === 'critical').length,
             highCount: actionable.filter((finding) => finding.severity === 'high').length,
-            notAssessed: categoryFindings.length === 0,
+            /**
+             * "I don't know" is not an assessment.
+             *
+             * This was `categoryFindings.length === 0`, so a single `info` finding with
+             * status `unknown` — the analyzer recording that it could not tell — made the
+             * category count as assessed, with zero actionable findings and a score of 100.
+             * A Django school platform with no payments anywhere was reported as having
+             * "no issues found in taking payments", as a strength.
+             */
+            notAssessed: categoryFindings.every((finding) => finding.status === 'unknown'),
         };
     });
 }

package/dist/report/executiveSummary.js

@@ -83,8 +83,19 @@ function buildVerdict(args) {
 }
 function buildStrengths(categoryScores, findings) {
     const passed = findings.filter((finding) => finding.status === 'passed');
+    /**
+     * A strength is something verified, not something absent.
+     *
+     * Filtering on "assessed, scoring well, nothing open" let a category qualify on the
+     * strength of having nothing to say about it. Requiring a passed check means the
+     * report only calls something a strength when it watched it work.
+     */
+    const passedCategories = new Set(passed.map((finding) => finding.category));
     const strongCategories = categoryScores
-        .filter((entry) => !entry.notAssessed && entry.score >= 80 && entry.findingCount === 0)
+        .filter((entry) => !entry.notAssessed
+        && entry.score >= 80
+        && entry.findingCount === 0
+        && passedCategories.has(entry.category))
         .map((entry) => CATEGORY_LABEL[entry.category]);
     const strengths = strongCategories.slice(0, 3).map((label) => `no issues found in ${label}`);
     if (strengths.length === 0 && passed.length > 0) {

package/dist/rules/rules.js

@@ -46,6 +46,23 @@ function evidenceQualityFor(evidence) {
     }
     return 'weak';
 }
+/** Severity is capped by how sure the analyzer is. See the note where it is used. */
+function severityForConfidence(severity, confidence) {
+    if (confidence !== 'low')
+        return severity;
+    /**
+     * `critical` only. Not a ceiling at `medium`, which was the first attempt and was
+     * wrong for a reason worth writing down: a missing capability cannot carry direct
+     * evidence — there is no line to point at for something that is not there — so every
+     * absence reads as low confidence, and capping them all at `medium` would leave the
+     * report unable to say anything is serious.
+     *
+     * What it can stop doing is shouting. `critical` is read as "stop and fix this
+     * before anything else", and the analyzer should not spend that word on a claim it
+     * could not evidence.
+     */
+    return severity === 'critical' ? 'high' : severity;
+}
 function confidenceFor(status, evidenceQuality) {
     if (status === 'passed' && evidenceQuality === 'strong') {
         return 'high';
@@ -60,16 +77,24 @@ function confidenceFor(status, evidenceQuality) {
 }
 function mkFinding(args) {
     const evidenceQuality = evidenceQualityFor(args.evidence);
+    const confidence = confidenceFor(args.status, evidenceQuality);
     return {
         id: args.id,
         title: args.title,
         category: args.category,
         status: args.status,
-        severity: args.severity,
+        /**
+         * The same rule the expectations follow: severity says how bad it would be if true,
+         * confidence says whether it is, and a reader treats `critical` as "stop and fix
+         * this". Spending that word on something the analyzer is unsure of is how a report
+         * stops being believed — and it was doing it on both sides of the report, not only
+         * in the expectation half.
+         */
+        severity: severityForConfidence(args.severity, confidence),
         description: args.description,
         recommendation: args.recommendation,
         evidence: detectorEvidence(args.evidence),
-        confidence: confidenceFor(args.status, evidenceQuality),
+        confidence,
         evidenceQuality,
     };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@produtype/core",
-  "version": "0.7.1",
+  "version": "0.8.0",
   "description": "Deterministic CLI and library that analyzes a web application repository and reports how far it is from production-ready for the kind of product it is meant to be.",
   "license": "MIT",
   "bin": {