@dogfood-lab/verify 1.8.0 → 1.10.0

package/validators/repo-binding.test.js

@@ -29,6 +29,19 @@ const schemaPath = require.resolve(
 const submissionSchema = JSON.parse(readFileSync(schemaPath, 'utf-8'));
 const PROVIDER_ENUM = submissionSchema.properties.source.properties.provider.enum;
 
+/**
+ * F-a2fb624f: sibling table to provenance-registry.test.js's — the pre-fix
+ * tripwire here ('returns null for a provider with no parser') only probed
+ * 'bitbucket', a NON-prototype key, so it gave zero coverage for the actual
+ * F-7ce07baa defect class (every Object.prototype own-property name, which
+ * `RUN_URL_PARSERS[provider]` resolved as a truthy inherited method before
+ * the Object.hasOwn fix).
+ */
+const OBJECT_PROTOTYPE_KEYS = [
+  'constructor', 'toString', 'valueOf', '__proto__',
+  'hasOwnProperty', 'isPrototypeOf', 'propertyIsEnumerable', 'toLocaleString',
+];
+
 describe('repo-binding guard coverage (verify-B-001)', () => {
   it('exposes the provider enum it is meant to cover (sanity)', () => {
     assert.ok(Array.isArray(PROVIDER_ENUM) && PROVIDER_ENUM.length > 0);
@@ -115,4 +128,31 @@ describe('repo-binding guard coverage (verify-B-001)', () => {
   it('returns null for a provider with no parser (no throw)', () => {
     assert.equal(parseRunUrlRepo('bitbucket', 'https://bitbucket.org/a/b/pipelines/1'), null);
   });
+
+  describe('F-7ce07baa: parseRunUrlRepo never resolves an inherited Object.prototype key', () => {
+    for (const key of OBJECT_PROTOTYPE_KEYS) {
+      it(`returns null (not the inherited method) for provider="${key}"`, () => {
+        // Deletion/emptiness proof: revert parseRunUrlRepo to a bare
+        // `RUN_URL_PARSERS[provider]` lookup and this goes red for every key
+        // here — each currently resolves a truthy inherited Object.prototype
+        // value the `if (!parser)` guard cannot catch, and `parser(runUrl)`
+        // then calls an unrelated built-in method.
+        assert.equal(
+          parseRunUrlRepo(key, 'https://github.com/acme/widget/actions/runs/1'),
+          null,
+          `provider="${key}" must resolve to null, not an inherited Object.prototype member`
+        );
+      });
+    }
+
+    it('a non-string provider (object, number, null, undefined) never throws and returns null', () => {
+      for (const bad of [{}, 42, null, undefined, ['github']]) {
+        assert.equal(
+          parseRunUrlRepo(bad, 'https://github.com/acme/widget/actions/runs/1'),
+          null,
+          `provider=${JSON.stringify(bad)} must resolve to null`
+        );
+      }
+    });
+  });
 });

package/validators/steps.js

@@ -10,8 +10,9 @@
  * Validate step results for a single scenario result.
  *
  * Note: Without access to the source repo's scenario definition, we validate
- * structural integrity. The full required_steps check is done by policy
- * evaluation when scenario definitions are available.
+ * structural integrity. The full required_steps check is done by
+ * validateRequiredSteps below, which verify() runs per scenario_result when
+ * the caller supplies loaded scenario definitions (options.scenarios).
  *
  * @param {object} scenarioResult - A single scenario_results[] item
  * @returns {string[]} Array of error messages (empty if valid)
@@ -41,6 +42,10 @@ export function validateStepResults(scenarioResult) {
   const seenIds = new Set();
   for (const step of step_results) {
     if (step == null || typeof step !== 'object') continue;
+    // F-70338558: a step without a string step_id is already reported as
+    // malformed above; letting `undefined` into the dedupe Set made two
+    // malformed steps emit a spurious `duplicate step_id: undefined`.
+    if (typeof step.step_id !== 'string') continue;
     if (seenIds.has(step.step_id)) {
       errors.push(`duplicate step_id: ${step.step_id}`);
     }
@@ -67,12 +72,63 @@ export function validateStepResults(scenarioResult) {
     }
   }
 
+  // F-88fb37ff: mirror of the pass-direction check above. A scenario cannot
+  // claim "fail"/"blocked" while every reported step says otherwise either —
+  // computeVerdict() (validators/verdict.js) trusts scenario_results[].verdict
+  // verbatim and never re-derives it from step_results, so without this check
+  // a self-reported "blocked" verdict backed by zero failing/blocked steps
+  // sailed through with no rejection reason at all.
+  //
+  // F-e42e8f80 (wave 20, amends F-88fb37ff): the original gate required at
+  // least one step to ACTIVELY report fail/blocked, which wrongly rejected
+  // the common honest shape "the scenario was blocked before any step could
+  // run, so every step reports 'skip'" — 'skip' and 'partial' are NEUTRAL
+  // (no evidence either way), not "evidence of no failure." Only 'pass' is an
+  // ACTIVE CONTRADICTION of a fail/blocked verdict (the step ran and claimed
+  // success). The gate now fires only when EVERY reported step actively says
+  // "pass" — a single skip/partial/fail/blocked step is enough to keep a
+  // fail/blocked verdict internally consistent.
+  //
+  // F-cc198701 (wave 22, confirming audit of F-e42e8f80): this condition and
+  // its validateRequiredSteps mirror below enumerated only 2 of the 4 legal
+  // values in scenario_results[].verdict's own schema enum (dogfood-record-
+  // submission.schema.json: ["pass","fail","blocked","partial"]) — 'partial'
+  // was never checked in either direction. A submitter could self-report
+  // verdict:'partial' with EVERY step actively 'fail' (strictly worse,
+  // more self-contradictory evidence than the fail/blocked direction already
+  // rejects) and sail through with zero rejection reason, because
+  // computeVerdict() never re-derives the verdict from step_results either —
+  // the ONLY guard against a dishonest self-report was this exact check, and
+  // it had a verdict-enum-shaped hole. Widened to also fire for 'partial',
+  // reusing the identical "all present steps actively pass" bar: 'partial'
+  // backed by zero non-pass evidence is exactly as self-contradictory as
+  // 'blocked'/'fail' backed by zero non-pass evidence. Deliberately does NOT
+  // touch the pass-direction check above (line ~59) — 'partial' backed by
+  // SOME failing steps is not inherently contradictory the way 'partial'
+  // backed by all-pass steps is.
+  if (verdict === 'fail' || verdict === 'blocked' || verdict === 'partial') {
+    const allStepsActivelyPass = step_results.every(
+      s => s != null && s.status === 'pass'
+    );
+    if (allStepsActivelyPass) {
+      errors.push(
+        `scenario verdict is "${verdict}" but no step reports status fail/blocked`
+      );
+    }
+  }
+
   return errors;
 }
 
 /**
  * Validate step results against a scenario definition's required_steps.
- * Used when the scenario definition is available (policy evaluation phase).
+ *
+ * F-3bfc2885: this is the REAL enforcement behind the `step-results-present`
+ * and `step-verdict-consistent` global reject rules (policies/global-policy.yaml,
+ * allowlisted in KNOWN_REJECT_RULE_IDS). verify() calls it per scenario_result
+ * when the caller supplies loaded scenario definitions (options.scenarios).
+ * Each error is tagged with the `[rule-id]` it enforces so operators can pivot
+ * from a rejection reason straight to the policy rule.
  *
  * @param {object} scenarioResult - A single scenario_results[] item
  * @param {string[]} requiredSteps - Step IDs from scenario definition's success_criteria.required_steps
@@ -82,29 +138,80 @@ export function validateRequiredSteps(scenarioResult, requiredSteps) {
   const errors = [];
   const { step_results, verdict } = scenarioResult;
 
-  if (!step_results) return ['step_results missing'];
+  if (!step_results) return ['[step-results-present] step_results missing'];
 
-  const resultMap = new Map(step_results.map(s => [s.step_id, s]));
+  // F-ad98b5ac: build the map defensively — the same floor the sibling
+  // validateStepResults applies (its verify-B-004 guard). A null / non-object /
+  // string-step_id-less element must not throw a TypeError here: that TypeError
+  // escapes to runValidator('steps', ...) which mislabels it as an operational
+  // VALIDATOR_FAULT_STEPS, inverting a submission-bad signal into an ops fault
+  // (the F-efe4f893 family inversion). Structural malformed-step *reporting*
+  // stays validateStepResults' job; this only stops the map build from throwing.
+  const resultMap = new Map();
+  for (const s of step_results) {
+    if (s != null && typeof s === 'object' && typeof s.step_id === 'string') {
+      resultMap.set(s.step_id, s);
+    }
+  }
 
   // Every required step must have a matching step_result
   for (const stepId of requiredSteps) {
     const result = resultMap.get(stepId);
     if (!result) {
-      errors.push(`required step "${stepId}" has no matching step_result`);
+      errors.push(`[step-results-present] required step "${stepId}" has no matching step_result`);
     }
   }
 
-  // A scenario cannot be "pass" if any required step is fail/blocked
+  // A scenario cannot be "pass" if any required step is fail/blocked — or
+  // absent entirely: validateStepResults' structural check only sees REPORTED
+  // steps, so a submitter who silently omits a failing required step would
+  // otherwise keep a "pass" verdict consistent (F-3bfc2885 sibling case).
   if (verdict === 'pass') {
     for (const stepId of requiredSteps) {
       const result = resultMap.get(stepId);
-      if (result && (result.status === 'fail' || result.status === 'blocked')) {
+      if (!result) {
         errors.push(
-          `scenario verdict is "pass" but required step "${stepId}" has status "${result.status}"`
+          `[step-verdict-consistent] scenario verdict is "pass" but required step "${stepId}" has no step_result`
+        );
+      } else if (result.status === 'fail' || result.status === 'blocked') {
+        errors.push(
+          `[step-verdict-consistent] scenario verdict is "pass" but required step "${stepId}" has status "${result.status}"`
         );
       }
     }
   }
 
+  // F-88fb37ff: mirror of the pass-direction block above, scoped to REQUIRED
+  // steps (the sibling check in validateStepResults enforces the same rule
+  // over ALL reported steps regardless of which are required).
+  //
+  // F-e42e8f80 (wave 20, amends F-88fb37ff): "no required step reports
+  // fail/blocked" was too narrow a bar — a required step honestly reporting
+  // 'skip' (never ran because the scenario was blocked upstream) or 'partial'
+  // is NEUTRAL, not an active contradiction, and must not force a rejection.
+  // The gate now fires only when every PRESENT required step actively says
+  // "pass" — mirroring validateStepResults' all-pass bar. A required step
+  // that is simply MISSING is "not an active pass" exactly like skip/partial
+  // would be, so it never by itself forces this check to fire (`result !=
+  // null && result.status === 'pass'` is false for a missing step too); its
+  // absence is already rejected unconditionally by the [step-results-present]
+  // loop above, regardless of verdict, so this check deliberately does not
+  // pile a second, verdict-specific error onto the exact same gap.
+  //
+  // F-cc198701 (wave 22): mirrors the identical widening in
+  // validateStepResults above — 'partial' was the one legal scenario-verdict
+  // enum value this scoped-to-required-steps check never enumerated either.
+  if ((verdict === 'fail' || verdict === 'blocked' || verdict === 'partial') && requiredSteps.length > 0) {
+    const allPresentRequiredStepsActivelyPass = requiredSteps.every(stepId => {
+      const result = resultMap.get(stepId);
+      return result != null && result.status === 'pass';
+    });
+    if (allPresentRequiredStepsActivelyPass) {
+      errors.push(
+        `[step-verdict-consistent] scenario verdict is "${verdict}" but no required step reports status fail/blocked`
+      );
+    }
+  }
+
   return errors;
 }

package/validators/verdict.js

@@ -6,7 +6,15 @@
  * Verdict severity (highest to lowest): fail > blocked > partial > pass
  */
 
-const VERDICT_RANK = { fail: 0, blocked: 1, partial: 2, pass: 3 };
+// F-937733ee: family sibling of F-2965699b/F-7ce07baa. Object.create(null)
+// removes the prototype chain entirely — `VERDICT_RANK['constructor']` is
+// `undefined`, not `Object.prototype.constructor` — so the `== null` guards
+// below fire correctly for every Object.prototype key, not just for a
+// literal typo. Currently unreachable in production (index.js:326 only ever
+// passes schema-enum-valid verdicts into `scenarioResults`), but sealing it
+// here keeps a future caller that skips the schema gate from silently
+// reopening the class.
+const VERDICT_RANK = Object.assign(Object.create(null), { fail: 0, blocked: 1, partial: 2, pass: 3 });
 
 /**
  * Compute the verified verdict.

package/validators/w4-f-8e72d0de-timeout-retry.test.js

@@ -0,0 +1,118 @@
+/**
+ * w4-f-8e72d0de-timeout-retry.test.js
+ *
+ * F-8e72d0de (wave 4): both provenance adapters threw IMMEDIATELY on an
+ * AbortError (per-request timeout) while retrying transport rejects and
+ * 429/5xx within the PROVENANCE_RETRIES budget. A timeout is at least as
+ * transient as a connection refusal — a single slow response failed the
+ * whole confirmation when one retry might have confirmed the run. Contrast:
+ * the scenario fetcher in ingest/load-context.js already retried timeouts.
+ *
+ * Contract (both adapters — the documented peer discipline):
+ *   - an AbortError is retried within the same budget as a transport reject;
+ *   - on exhaustion the timeout error still throws (operational,
+ *     `provenance-fault:` at the verify() boundary).
+ */
+
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+
+import { githubProvenance, gitlabProvenance } from './provenance.js';
+
+const RUN_HEAD = 'c5d6c4e0000000000000000000000000deadbeef';
+
+const GH_SOURCE = {
+  provider: 'github',
+  workflow: 'dogfood.yml',
+  provider_run_id: '9123456789',
+  run_url: 'https://github.com/acme/widget/actions/runs/9123456789'
+};
+
+const GL_SOURCE = {
+  provider: 'gitlab',
+  workflow: '.gitlab-ci.yml',
+  provider_run_id: '424242',
+  run_url: 'https://gitlab.com/acme/widget/-/pipelines/424242',
+  repo: 'acme/widget'
+};
+
+function abortError() {
+  const e = new Error('This operation was aborted');
+  e.name = 'AbortError';
+  return e;
+}
+
+const noSleep = async () => {};
+
+describe('F-8e72d0de — githubProvenance retries per-request timeouts', () => {
+  it('a single timeout followed by a 200 confirms the run', async () => {
+    let calls = 0;
+    const adapter = githubProvenance('token', {
+      timeoutMs: 1000, retries: 2, sleepImpl: noSleep,
+      fetchImpl: async () => {
+        calls++;
+        if (calls === 1) throw abortError();
+        return {
+          ok: true,
+          status: 200,
+          json: async () => ({
+            id: 9123456789,
+            status: 'completed',
+            head_sha: RUN_HEAD,
+            repository: { full_name: 'acme/widget' }
+          })
+        };
+      }
+    });
+    const ok = await adapter.confirm(GH_SOURCE, { refCommitSha: RUN_HEAD });
+    assert.equal(ok, true, 'one slow response must not fail the confirmation');
+    assert.equal(calls, 2);
+  });
+
+  it('exhausting every attempt on timeouts still throws the timeout error', async () => {
+    let calls = 0;
+    const adapter = githubProvenance('token', {
+      timeoutMs: 1000, retries: 2, sleepImpl: noSleep,
+      fetchImpl: async () => { calls++; throw abortError(); }
+    });
+    await assert.rejects(
+      () => adapter.confirm(GH_SOURCE),
+      /provenance: GitHub API timeout after 1000ms/
+    );
+    assert.equal(calls, 3, 'retries=2 → 3 total attempts before the operational throw');
+  });
+});
+
+describe('F-8e72d0de — gitlabProvenance mirrors the timeout-retry discipline', () => {
+  it('a single timeout followed by a 200 confirms the pipeline', async () => {
+    let calls = 0;
+    const adapter = gitlabProvenance('token', {
+      timeoutMs: 1000, retries: 2, sleepImpl: noSleep,
+      fetchImpl: async () => {
+        calls++;
+        if (calls === 1) throw abortError();
+        return {
+          ok: true,
+          status: 200,
+          json: async () => ({ id: 424242, status: 'success', sha: RUN_HEAD })
+        };
+      }
+    });
+    const ok = await adapter.confirm(GL_SOURCE, { refCommitSha: RUN_HEAD });
+    assert.equal(ok, true);
+    assert.equal(calls, 2);
+  });
+
+  it('exhausting every attempt on timeouts still throws the timeout error', async () => {
+    let calls = 0;
+    const adapter = gitlabProvenance('token', {
+      timeoutMs: 1000, retries: 1, sleepImpl: noSleep,
+      fetchImpl: async () => { calls++; throw abortError(); }
+    });
+    await assert.rejects(
+      () => adapter.confirm(GL_SOURCE),
+      /provenance: GitLab API timeout after 1000ms/
+    );
+    assert.equal(calls, 2);
+  });
+});