@dogfood-lab/verify 1.9.0 → 1.11.0

package/validators/steps.js

@@ -72,6 +72,51 @@ export function validateStepResults(scenarioResult) {
     }
   }
 
+  // F-88fb37ff: mirror of the pass-direction check above. A scenario cannot
+  // claim "fail"/"blocked" while every reported step says otherwise either —
+  // computeVerdict() (validators/verdict.js) trusts scenario_results[].verdict
+  // verbatim and never re-derives it from step_results, so without this check
+  // a self-reported "blocked" verdict backed by zero failing/blocked steps
+  // sailed through with no rejection reason at all.
+  //
+  // F-e42e8f80 (wave 20, amends F-88fb37ff): the original gate required at
+  // least one step to ACTIVELY report fail/blocked, which wrongly rejected
+  // the common honest shape "the scenario was blocked before any step could
+  // run, so every step reports 'skip'" — 'skip' and 'partial' are NEUTRAL
+  // (no evidence either way), not "evidence of no failure." Only 'pass' is an
+  // ACTIVE CONTRADICTION of a fail/blocked verdict (the step ran and claimed
+  // success). The gate now fires only when EVERY reported step actively says
+  // "pass" — a single skip/partial/fail/blocked step is enough to keep a
+  // fail/blocked verdict internally consistent.
+  //
+  // F-cc198701 (wave 22, confirming audit of F-e42e8f80): this condition and
+  // its validateRequiredSteps mirror below enumerated only 2 of the 4 legal
+  // values in scenario_results[].verdict's own schema enum (dogfood-record-
+  // submission.schema.json: ["pass","fail","blocked","partial"]) — 'partial'
+  // was never checked in either direction. A submitter could self-report
+  // verdict:'partial' with EVERY step actively 'fail' (strictly worse,
+  // more self-contradictory evidence than the fail/blocked direction already
+  // rejects) and sail through with zero rejection reason, because
+  // computeVerdict() never re-derives the verdict from step_results either —
+  // the ONLY guard against a dishonest self-report was this exact check, and
+  // it had a verdict-enum-shaped hole. Widened to also fire for 'partial',
+  // reusing the identical "all present steps actively pass" bar: 'partial'
+  // backed by zero non-pass evidence is exactly as self-contradictory as
+  // 'blocked'/'fail' backed by zero non-pass evidence. Deliberately does NOT
+  // touch the pass-direction check above (line ~59) — 'partial' backed by
+  // SOME failing steps is not inherently contradictory the way 'partial'
+  // backed by all-pass steps is.
+  if (verdict === 'fail' || verdict === 'blocked' || verdict === 'partial') {
+    const allStepsActivelyPass = step_results.every(
+      s => s != null && s.status === 'pass'
+    );
+    if (allStepsActivelyPass) {
+      errors.push(
+        `scenario verdict is "${verdict}" but no step reports status fail/blocked`
+      );
+    }
+  }
+
   return errors;
 }
 
@@ -136,5 +181,37 @@ export function validateRequiredSteps(scenarioResult, requiredSteps) {
     }
   }
 
+  // F-88fb37ff: mirror of the pass-direction block above, scoped to REQUIRED
+  // steps (the sibling check in validateStepResults enforces the same rule
+  // over ALL reported steps regardless of which are required).
+  //
+  // F-e42e8f80 (wave 20, amends F-88fb37ff): "no required step reports
+  // fail/blocked" was too narrow a bar — a required step honestly reporting
+  // 'skip' (never ran because the scenario was blocked upstream) or 'partial'
+  // is NEUTRAL, not an active contradiction, and must not force a rejection.
+  // The gate now fires only when every PRESENT required step actively says
+  // "pass" — mirroring validateStepResults' all-pass bar. A required step
+  // that is simply MISSING is "not an active pass" exactly like skip/partial
+  // would be, so it never by itself forces this check to fire (`result !=
+  // null && result.status === 'pass'` is false for a missing step too); its
+  // absence is already rejected unconditionally by the [step-results-present]
+  // loop above, regardless of verdict, so this check deliberately does not
+  // pile a second, verdict-specific error onto the exact same gap.
+  //
+  // F-cc198701 (wave 22): mirrors the identical widening in
+  // validateStepResults above — 'partial' was the one legal scenario-verdict
+  // enum value this scoped-to-required-steps check never enumerated either.
+  if ((verdict === 'fail' || verdict === 'blocked' || verdict === 'partial') && requiredSteps.length > 0) {
+    const allPresentRequiredStepsActivelyPass = requiredSteps.every(stepId => {
+      const result = resultMap.get(stepId);
+      return result != null && result.status === 'pass';
+    });
+    if (allPresentRequiredStepsActivelyPass) {
+      errors.push(
+        `[step-verdict-consistent] scenario verdict is "${verdict}" but no required step reports status fail/blocked`
+      );
+    }
+  }
+
   return errors;
 }

package/validators/verdict.js

@@ -6,7 +6,15 @@
  * Verdict severity (highest to lowest): fail > blocked > partial > pass
  */
 
-const VERDICT_RANK = { fail: 0, blocked: 1, partial: 2, pass: 3 };
+// F-937733ee: family sibling of F-2965699b/F-7ce07baa. Object.create(null)
+// removes the prototype chain entirely — `VERDICT_RANK['constructor']` is
+// `undefined`, not `Object.prototype.constructor` — so the `== null` guards
+// below fire correctly for every Object.prototype key, not just for a
+// literal typo. Currently unreachable in production (index.js:326 only ever
+// passes schema-enum-valid verdicts into `scenarioResults`), but sealing it
+// here keeps a future caller that skips the schema gate from silently
+// reopening the class.
+const VERDICT_RANK = Object.assign(Object.create(null), { fail: 0, blocked: 1, partial: 2, pass: 3 });
 
 /**
  * Compute the verified verdict.
@@ -60,6 +68,16 @@ export function computeVerdict(proposed, context) {
     downgrade_reasons.push('policy validation failed');
   }
 
+  // F-a0a4d806: non-empty rejection reasons are a fail floor. verify() sets
+  // status from reasons.length > 0, but overall_verdict.verified previously
+  // ignored the already-plumbed `reasons` argument — a steps[...] or
+  // repo:mismatch rejection could leave verified:'pass' next to
+  // status:'rejected'. Treat reasons as evidence, not a dual signal.
+  if (Array.isArray(reasons) && reasons.length > 0) {
+    floorVerdict = 'fail';
+    downgrade_reasons.push('non-empty rejection reasons force fail');
+  }
+
   // The verified verdict is the worse of proposed and floor
   // (we never upgrade, so if proposed is worse than floor, keep proposed)
   if (proposed && VERDICT_RANK[proposed] == null) {