@dogfood-lab/verify 1.7.0 → 1.9.0

package/validators/steps.js

@@ -10,8 +10,9 @@
  * Validate step results for a single scenario result.
  *
  * Note: Without access to the source repo's scenario definition, we validate
- * structural integrity. The full required_steps check is done by policy
- * evaluation when scenario definitions are available.
+ * structural integrity. The full required_steps check is done by
+ * validateRequiredSteps below, which verify() runs per scenario_result when
+ * the caller supplies loaded scenario definitions (options.scenarios).
  *
  * @param {object} scenarioResult - A single scenario_results[] item
  * @returns {string[]} Array of error messages (empty if valid)
@@ -41,6 +42,10 @@ export function validateStepResults(scenarioResult) {
   const seenIds = new Set();
   for (const step of step_results) {
     if (step == null || typeof step !== 'object') continue;
+    // F-70338558: a step without a string step_id is already reported as
+    // malformed above; letting `undefined` into the dedupe Set made two
+    // malformed steps emit a spurious `duplicate step_id: undefined`.
+    if (typeof step.step_id !== 'string') continue;
     if (seenIds.has(step.step_id)) {
       errors.push(`duplicate step_id: ${step.step_id}`);
     }
@@ -72,7 +77,13 @@ export function validateStepResults(scenarioResult) {
 
 /**
  * Validate step results against a scenario definition's required_steps.
- * Used when the scenario definition is available (policy evaluation phase).
+ *
+ * F-3bfc2885: this is the REAL enforcement behind the `step-results-present`
+ * and `step-verdict-consistent` global reject rules (policies/global-policy.yaml,
+ * allowlisted in KNOWN_REJECT_RULE_IDS). verify() calls it per scenario_result
+ * when the caller supplies loaded scenario definitions (options.scenarios).
+ * Each error is tagged with the `[rule-id]` it enforces so operators can pivot
+ * from a rejection reason straight to the policy rule.
  *
  * @param {object} scenarioResult - A single scenario_results[] item
  * @param {string[]} requiredSteps - Step IDs from scenario definition's success_criteria.required_steps
@@ -82,25 +93,44 @@ export function validateRequiredSteps(scenarioResult, requiredSteps) {
   const errors = [];
   const { step_results, verdict } = scenarioResult;
 
-  if (!step_results) return ['step_results missing'];
+  if (!step_results) return ['[step-results-present] step_results missing'];
 
-  const resultMap = new Map(step_results.map(s => [s.step_id, s]));
+  // F-ad98b5ac: build the map defensively — the same floor the sibling
+  // validateStepResults applies (its verify-B-004 guard). A null / non-object /
+  // string-step_id-less element must not throw a TypeError here: that TypeError
+  // escapes to runValidator('steps', ...) which mislabels it as an operational
+  // VALIDATOR_FAULT_STEPS, inverting a submission-bad signal into an ops fault
+  // (the F-efe4f893 family inversion). Structural malformed-step *reporting*
+  // stays validateStepResults' job; this only stops the map build from throwing.
+  const resultMap = new Map();
+  for (const s of step_results) {
+    if (s != null && typeof s === 'object' && typeof s.step_id === 'string') {
+      resultMap.set(s.step_id, s);
+    }
+  }
 
   // Every required step must have a matching step_result
   for (const stepId of requiredSteps) {
     const result = resultMap.get(stepId);
     if (!result) {
-      errors.push(`required step "${stepId}" has no matching step_result`);
+      errors.push(`[step-results-present] required step "${stepId}" has no matching step_result`);
     }
   }
 
-  // A scenario cannot be "pass" if any required step is fail/blocked
+  // A scenario cannot be "pass" if any required step is fail/blocked — or
+  // absent entirely: validateStepResults' structural check only sees REPORTED
+  // steps, so a submitter who silently omits a failing required step would
+  // otherwise keep a "pass" verdict consistent (F-3bfc2885 sibling case).
   if (verdict === 'pass') {
     for (const stepId of requiredSteps) {
       const result = resultMap.get(stepId);
-      if (result && (result.status === 'fail' || result.status === 'blocked')) {
+      if (!result) {
+        errors.push(
+          `[step-verdict-consistent] scenario verdict is "pass" but required step "${stepId}" has no step_result`
+        );
+      } else if (result.status === 'fail' || result.status === 'blocked') {
         errors.push(
-          `scenario verdict is "pass" but required step "${stepId}" has status "${result.status}"`
+          `[step-verdict-consistent] scenario verdict is "pass" but required step "${stepId}" has status "${result.status}"`
         );
       }
     }

package/validators/w4-f-8e72d0de-timeout-retry.test.js

@@ -0,0 +1,118 @@
+/**
+ * w4-f-8e72d0de-timeout-retry.test.js
+ *
+ * F-8e72d0de (wave 4): both provenance adapters threw IMMEDIATELY on an
+ * AbortError (per-request timeout) while retrying transport rejects and
+ * 429/5xx within the PROVENANCE_RETRIES budget. A timeout is at least as
+ * transient as a connection refusal — a single slow response failed the
+ * whole confirmation when one retry might have confirmed the run. Contrast:
+ * the scenario fetcher in ingest/load-context.js already retried timeouts.
+ *
+ * Contract (both adapters — the documented peer discipline):
+ *   - an AbortError is retried within the same budget as a transport reject;
+ *   - on exhaustion the timeout error still throws (operational,
+ *     `provenance-fault:` at the verify() boundary).
+ */
+
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+
+import { githubProvenance, gitlabProvenance } from './provenance.js';
+
+const RUN_HEAD = 'c5d6c4e0000000000000000000000000deadbeef';
+
+const GH_SOURCE = {
+  provider: 'github',
+  workflow: 'dogfood.yml',
+  provider_run_id: '9123456789',
+  run_url: 'https://github.com/acme/widget/actions/runs/9123456789'
+};
+
+const GL_SOURCE = {
+  provider: 'gitlab',
+  workflow: '.gitlab-ci.yml',
+  provider_run_id: '424242',
+  run_url: 'https://gitlab.com/acme/widget/-/pipelines/424242',
+  repo: 'acme/widget'
+};
+
+function abortError() {
+  const e = new Error('This operation was aborted');
+  e.name = 'AbortError';
+  return e;
+}
+
+const noSleep = async () => {};
+
+describe('F-8e72d0de — githubProvenance retries per-request timeouts', () => {
+  it('a single timeout followed by a 200 confirms the run', async () => {
+    let calls = 0;
+    const adapter = githubProvenance('token', {
+      timeoutMs: 1000, retries: 2, sleepImpl: noSleep,
+      fetchImpl: async () => {
+        calls++;
+        if (calls === 1) throw abortError();
+        return {
+          ok: true,
+          status: 200,
+          json: async () => ({
+            id: 9123456789,
+            status: 'completed',
+            head_sha: RUN_HEAD,
+            repository: { full_name: 'acme/widget' }
+          })
+        };
+      }
+    });
+    const ok = await adapter.confirm(GH_SOURCE, { refCommitSha: RUN_HEAD });
+    assert.equal(ok, true, 'one slow response must not fail the confirmation');
+    assert.equal(calls, 2);
+  });
+
+  it('exhausting every attempt on timeouts still throws the timeout error', async () => {
+    let calls = 0;
+    const adapter = githubProvenance('token', {
+      timeoutMs: 1000, retries: 2, sleepImpl: noSleep,
+      fetchImpl: async () => { calls++; throw abortError(); }
+    });
+    await assert.rejects(
+      () => adapter.confirm(GH_SOURCE),
+      /provenance: GitHub API timeout after 1000ms/
+    );
+    assert.equal(calls, 3, 'retries=2 → 3 total attempts before the operational throw');
+  });
+});
+
+describe('F-8e72d0de — gitlabProvenance mirrors the timeout-retry discipline', () => {
+  it('a single timeout followed by a 200 confirms the pipeline', async () => {
+    let calls = 0;
+    const adapter = gitlabProvenance('token', {
+      timeoutMs: 1000, retries: 2, sleepImpl: noSleep,
+      fetchImpl: async () => {
+        calls++;
+        if (calls === 1) throw abortError();
+        return {
+          ok: true,
+          status: 200,
+          json: async () => ({ id: 424242, status: 'success', sha: RUN_HEAD })
+        };
+      }
+    });
+    const ok = await adapter.confirm(GL_SOURCE, { refCommitSha: RUN_HEAD });
+    assert.equal(ok, true);
+    assert.equal(calls, 2);
+  });
+
+  it('exhausting every attempt on timeouts still throws the timeout error', async () => {
+    let calls = 0;
+    const adapter = gitlabProvenance('token', {
+      timeoutMs: 1000, retries: 1, sleepImpl: noSleep,
+      fetchImpl: async () => { calls++; throw abortError(); }
+    });
+    await assert.rejects(
+      () => adapter.confirm(GL_SOURCE),
+      /provenance: GitLab API timeout after 1000ms/
+    );
+    assert.equal(calls, 2);
+  });
+});