@onlineapps/conn-orch-validator 3.3.2 → 4.0.0

package/src/CookbookTestRunner.js

@@ -6,6 +6,23 @@ const crypto = require('crypto');
 const MockMQClient = require('./mocks/MockMQClient');
 const MockRegistry = require('./mocks/MockRegistry');
 const { resolveHeaders } = require('./utils/resolveHeaders');
+const { checkCookbookFormatVersion } = require('./utils/cookbookFormat');
+const { getTestNamespace } = require('./utils/testNamespace');
+const { describeStepFailure } = require('./utils/stepFailure');
+
+/**
+ * The step's stopwatch, spelled out. A single total hid the fact that
+ * biz-hello's `count_stub` spent 2264 ms loading sequelize and ~0 ms doing its
+ * work; the reader of the log had no way to tell those apart, and neither did
+ * `expect.duration.max`.
+ *
+ * @param {Object} result - a step result carrying the three timings
+ * @returns {string} e.g. `2ms handler + 402ms module load = 404ms total`
+ */
+function describeStepTiming(result) {
+  return `${result.handlerDurationMs}ms handler + ${result.setupDurationMs}ms module load`
+    + ` = ${result.duration}ms total`;
+}
 
 /**
  * CookbookTestRunner — executes cookbook tests offline with mocked infrastructure.
@@ -17,7 +34,8 @@ const { resolveHeaders } = require('./utils/resolveHeaders');
  *
  * The handler dispatch builds a minimal real ctx:
  *   { logger, tenant_id, workspace_id, correlation_id, db: null, cache: null,
- *     httpClient: null, secrets: null, stream: null, abortSignal: null }
+ *     httpClient: null, secrets: null, state: null, stream: null,
+ *     abortSignal: null }
  * matching the shape ServiceWrapper.ContextBuilder produces in production
  * (see api/docs/architecture/biz-service-invocation-model.md §5). Handlers
  * that need DB access import sequelize directly from their own
@@ -78,8 +96,18 @@ class CookbookTestRunner {
       ? this.loadCookbook(cookbook)
       : cookbook;
 
-    // Validate cookbook format
-    this.validateCookbook(cookbookData);
+    // Validate cookbook format. When the cookbook came from a file, the file
+    // path is appended to whatever the validation rejected — the reader of a
+    // boot failure otherwise learns that "a" cookbook is malformed but not
+    // which one, and a service can carry a dozen of them.
+    try {
+      this.validateCookbook(cookbookData);
+    } catch (error) {
+      if (typeof cookbook === 'string') {
+        throw new Error(`${error.message} (cookbook file: ${cookbook})`, { cause: error });
+      }
+      throw error;
+    }
 
     // Get test configuration
     const testConfig = cookbookData.test || { mode: 'production' };
@@ -94,9 +122,14 @@ class CookbookTestRunner {
     const stepsArray = this.normalizeSteps(cookbookData.steps);
 
     // Execute steps
+    const cookbookName = cookbookData.description || 'Unnamed';
     const stepResults = [];
     for (const step of stepsArray) {
       const stepResult = await this.executeStep(step, testConfig);
+      // Which cookbook this step came from. Without it a failed step in the
+      // aggregate is an id with no file behind it, and a service can carry a
+      // dozen cookbooks (utils/stepFailure.js).
+      stepResult.cookbook = cookbookName;
       stepResults.push(stepResult);
 
       // If step failed and has expect, fail immediately
@@ -137,6 +170,21 @@ class CookbookTestRunner {
    * @returns {Object} Aggregate results
    */
   async runCookbooks(cookbooksDir) {
+    // Every run reports ITS OWN run. `this.results` only ever accumulated, and
+    // `ValidationOrchestrator` builds one runner in its constructor while
+    // `ServiceWrapper._ensureValidationProof` reuses one orchestrator across
+    // six startup retries — so a single transient failure on attempt 1 stayed
+    // in `failed` for the whole process, and step 4's verdict
+    // (`success: result.failed === 0`) could never come back true. Measured on
+    // biz-hello 2026-08-29: 7/8 → 15/16 → 23/24, then `Validation FAILED` in
+    // 104–221 ms while every step in the same log said PASSED — a healthy
+    // service reporting itself broken (automation-gates.md §5).
+    //
+    // resetResults() had a unit test and no production caller at all
+    // (change-discipline.md § "Removing something removes its declaration").
+    // This is that caller.
+    this.resetResults();
+
     const files = fs.readdirSync(cookbooksDir).filter(f => f.endsWith('.json'));
 
     this.logger.info(`Found ${files.length} cookbook(s) in ${cookbooksDir}`);
@@ -171,36 +219,49 @@ class CookbookTestRunner {
       id: step.id,
       operation: step.operation,
       passed: false,
+      // Three separate numbers, because they answer different questions.
+      // `duration` is the wall clock of the whole step. `setupDurationMs` is
+      // what the RUNNER spent getting to the handler — reading operations.json
+      // and require()ing the module, a one-off cost per process. Only
+      // `handlerDurationMs` is the handler's own runtime, and only that one is
+      // what `expect.duration.max` asserts about (see validateExpectations).
       duration: 0,
+      setupDurationMs: 0,
+      handlerDurationMs: 0,
       request: null,
       response: null,
       expected: step.expect || null,
       actual: null,
       error: null,
+      errorStack: null,
       validationErrors: []
     };
 
     try {
+      // The namespace is a safety boundary, so it is resolved before anything
+      // else happens — before the operation is even looked up. utils/testNamespace
+      // is the single owner of that decision: it reads the TESTING_* env contract
+      // (docs/biz/40-cookbooks/test-env-vars.md +
+      // docs/biz/60-templates/env-conventions.md — the TESTING_ prefix marks
+      // values that legitimately ship to production, because this runner fires at
+      // every service startup, prod included) AND it returns a namespace only for
+      // the non-production environment classes — 96 CI, 97 TESTING, 98 DEVEL,
+      // 99 VALIDATION (api/docs/standards/tenant-allocation.md). Production
+      // tenants, 100 LIVE and every customer from 101 up, are refused.
+      //
+      // That refusal is the runtime half of deploy-contract R8: R8 rejects a
+      // per-service override before it ships, this refuses the accident that
+      // ships anyway. Reading process.env here instead would route the startup
+      // probe straight around the guard — which is precisely how the 2026-08
+      // biz-property incident wrote into the live tenant at every restart.
+      // Values arrive already validated as integers, so ctx matches the
+      // production ContextBuilder for handlers checking Number.isInteger().
+      const { tenant_id: validationTenantId, workspace_id: validationWorkspaceId } = getTestNamespace();
+
       // Resolve operation spec from operations.json — always returns
       // { modulePath, exportName } (v3 handler dispatch is the only mode).
       const spec = await this.resolveOperation(step.service, step.operation);
 
-      // Env-var contract (docs/biz/40-cookbooks/test-env-vars.md +
-      // docs/biz/60-templates/env-conventions.md): TESTING_* prefix marks
-      // values that legitimately ship to production because Tier-1 cookbook
-      // runner fires at every service startup, including in prod.
-      const rawTenantId = process.env.TESTING_TENANT_ID;
-      const rawWorkspaceId = process.env.TESTING_WORKSPACE_ID;
-      if (!rawTenantId || !rawWorkspaceId) {
-        throw new Error('[CookbookTestRunner] Missing required environment variables TESTING_TENANT_ID and/or TESTING_WORKSPACE_ID');
-      }
-      // Coerce numeric env vars to integers so ctx matches production
-      // ContextBuilder (which receives already-typed values from the MQ envelope).
-      // Handlers that check `Number.isInteger(ctx.tenant_id)` would otherwise
-      // reject the string form and every workspace-scoped cookbook would fail.
-      const validationTenantId = /^\d+$/.test(rawTenantId) ? parseInt(rawTenantId, 10) : rawTenantId;
-      const validationWorkspaceId = /^\d+$/.test(rawWorkspaceId) ? parseInt(rawWorkspaceId, 10) : rawWorkspaceId;
-
       await this._dispatchViaHandler({
         step,
         spec,
@@ -211,14 +272,26 @@ class CookbookTestRunner {
         startTime
       });
 
-      this.logger.info(`Step ${step.id}: ${result.passed ? 'PASSED' : 'FAILED'} (${result.duration}ms)`);
+      if (result.passed) {
+        this.logger.info(`Step ${step.id}: PASSED (${describeStepTiming(result)})`);
+      } else {
+        // A verdict without its reason is a silent gate (automation-gates.md
+        // §5). Until 2026-08-29 this line said only "FAILED (1101ms)" and the
+        // reason — held in result.validationErrors or result.error — reached
+        // nobody, which made a failed biz-hello boot undiagnosable from logs.
+        this.logger.error(`Step ${step.id}: FAILED (${describeStepTiming(result)}) — ${describeStepFailure(result)}`);
+      }
 
     } catch (error) {
       result.error = error.message;
+      result.errorStack = error.stack;
       result.duration = Date.now() - startTime;
+      // Nothing reached the handler, so all of it was setup.
+      result.setupDurationMs = result.duration;
+      result.handlerDurationMs = 0;
       result.passed = false;
 
-      this.logger.error(`Step ${step.id}: ERROR - ${error.message}`);
+      this.logger.error(`Step ${step.id}: ERROR - ${error.message}\n${error.stack}`);
     }
 
     return result;
@@ -252,6 +325,14 @@ class CookbookTestRunner {
       cache: null,
       httpClient: null,
       secrets: null,
+      // Persistent Redis state (conn-base-state), the slot production builds
+      // in ContextBuilder. Null here, and null is the only honest value:
+      // Tier-1 validation runs in ServiceWrapper phase 0.2, before
+      // _initializeState() instantiates the connector, so there is nothing to
+      // inject — same situation as db and cache. Declaring the slot keeps a
+      // handler that reads ctx.state from receiving `undefined`, which is a
+      // different failure from the documented `null`.
+      state: null,
       stream: null,
       abortSignal: null,
       headers: {
@@ -298,6 +379,12 @@ class CookbookTestRunner {
       if (timeoutId.unref) timeoutId.unref();
     });
 
+    // Everything up to here was the runner getting ready: the namespace, the
+    // operations.json read, require.resolve and the require() of the handler
+    // module. That is a one-off cost per process and it is not the handler's.
+    const handlerStart = Date.now();
+    result.setupDurationMs = handlerStart - startTime;
+
     let handlerError = null;
     let handlerOutput = null;
     try {
@@ -311,7 +398,9 @@ class CookbookTestRunner {
       clearTimeout(timeoutId);
     }
 
-    result.duration = Date.now() - startTime;
+    const finished = Date.now();
+    result.handlerDurationMs = finished - handlerStart;
+    result.duration = finished - startTime;
 
     if (handlerError) {
       // Synthesize an HTTP-like response so existing expect validators
@@ -327,6 +416,9 @@ class CookbookTestRunner {
       };
       result.actual = result.response.data;
       result.error = { code: errorCode, message: handlerError.message };
+      // The stack is the half of a thrown error that says WHERE. Kept on the
+      // result so the step's failure line can print it (utils/stepFailure.js).
+      result.errorStack = handlerError.stack;
     } else {
       result.response = {
         status: 200,
@@ -368,10 +460,23 @@ class CookbookTestRunner {
       }
     }
 
-    // Validate duration
+    // Validate duration — of the HANDLER.
+    //
+    // Until 2026-08-29 this compared `result.duration`, the whole step, whose
+    // dominant term for any handler with a heavy module is the one-off
+    // require(). biz-hello's `count_stub` is a stub returning four constants;
+    // its module pulls in sequelize, measured at 2264 ms cold inside the
+    // container against a 1000 ms bar, while every other cookbook handler in
+    // the same service loaded in 5–25 ms. So the boot verdict of a healthy
+    // service depended on filesystem cache warmth — a gate that is not
+    // predictable is not a gate (automation-gates.md §1.1).
+    //
+    // The load cost is not swept away: it is `setupDurationMs`, it is printed
+    // on the step's log line, and it stays inside `duration`.
     if (expect.duration && expect.duration.max) {
-      if (result.duration > expect.duration.max) {
-        errors.push(`Duration ${result.duration}ms exceeds max ${expect.duration.max}ms`);
+      if (result.handlerDurationMs > expect.duration.max) {
+        errors.push(`Handler duration ${result.handlerDurationMs}ms exceeds max ${expect.duration.max}ms`
+          + ` (module load ${result.setupDurationMs}ms is the runner's own cost and is not counted)`);
       }
     }
 
@@ -573,7 +678,13 @@ class CookbookTestRunner {
     }
 
     const content = fs.readFileSync(cookbookPath, 'utf8');
-    return JSON.parse(content);
+    try {
+      return JSON.parse(content);
+    } catch (error) {
+      throw new Error(`[CookbookTestRunner] Cookbook file is not valid JSON - Expected a JSON object per `
+        + `api/docs/biz/40-cookbooks/format.md. Fix: repair the JSON syntax in ${cookbookPath} (${error.message})`,
+      { cause: error });
+    }
   }
 
   /**
@@ -605,8 +716,26 @@ class CookbookTestRunner {
   /**
    * Validate cookbook format
    * Supports both V1 (steps as array) and V2 (steps as object) formats
+   *
+   * The format-version check is FIRST and unconditional. Until 2026-08-27 it
+   * lived only in `CookbookTestUtils.validateCookbook`, reachable exclusively
+   * through the optional `tests/bootstrap/service-readiness.test.js` wrapper
+   * that 3 of 8 biz repos happen to have — so the rule from
+   * api/docs/biz/40-cookbooks/format.md § "Required fields" was, in practice,
+   * opt-in per repo, and 4 repos ship cookbooks with no `version` field at all.
+   * A check a repo can decline is not a check (automation-gates §1.5).
    */
   validateCookbook(cookbook) {
+    if (!cookbook || typeof cookbook !== 'object' || Array.isArray(cookbook)) {
+      throw new Error('[CookbookTestRunner] Cookbook is not a JSON object - Expected an object with "version" and "steps". '
+        + 'Fix: check the cookbook file content (api/docs/biz/40-cookbooks/format.md § Basic structure).');
+    }
+
+    const versionProblem = checkCookbookFormatVersion(cookbook.version);
+    if (versionProblem) {
+      throw new Error(`[CookbookTestRunner] ${versionProblem}`);
+    }
+
     if (!cookbook.steps || (typeof cookbook.steps !== 'object' && !Array.isArray(cookbook.steps))) {
       throw new Error('Cookbook must have steps (array or object)');
     }

package/src/CookbookTestUtils.js

@@ -1,5 +1,7 @@
 'use strict';
 
+const { MIN_COOKBOOK_FORMAT_VERSION, checkCookbookFormatVersion } = require('./utils/cookbookFormat');
+
 /**
  * CookbookTestUtils - Utilities for cookbook testing
  */
@@ -11,7 +13,7 @@ class CookbookTestUtils {
     return {
       workflow_id: options.workflow_id || `test-workflow-${Date.now()}`,
       cookbook: options.cookbook || {
-        version: '1.0.0',
+        version: MIN_COOKBOOK_FORMAT_VERSION,
         steps: options.steps || [
           {
             id: 'step1',
@@ -51,7 +53,7 @@ class CookbookTestUtils {
     }
 
     return {
-      version: '1.0.0',
+      version: MIN_COOKBOOK_FORMAT_VERSION,
       api_input: options.api_input || { test: 'data' },
       steps
     };
@@ -59,6 +61,11 @@ class CookbookTestUtils {
 
   /**
    * Validate cookbook structure (basic validation)
+   *
+   * The format-version rule is the shared one (`utils/cookbookFormat`), the
+   * same rule the Tier-1 `CookbookTestRunner` enforces — one fact, one owner.
+   * Presence alone used to be enough here, so a cookbook stuck on an outdated
+   * format passed the readiness wrapper unnoticed.
    */
   static validateCookbook(cookbook) {
     const errors = [];
@@ -68,8 +75,9 @@ class CookbookTestUtils {
       return { valid: false, errors };
     }
 
-    if (!cookbook.version) {
-      errors.push('Version is required');
+    const versionProblem = checkCookbookFormatVersion(cookbook.version);
+    if (versionProblem) {
+      errors.push(versionProblem);
     }
 
     if (!cookbook.steps || !Array.isArray(cookbook.steps)) {

package/src/ServiceReadinessValidator.js

@@ -21,6 +21,17 @@ const CookbookTestUtils = require('./CookbookTestUtils');
  * previously spent on `health` fold into `operations` so total stays
  * at 100.
  *
+ * 2026-08-22: the `url` the probe consumed is gone too. It outlived the
+ * probe by five months as an accepted-and-ignored argument echoed into
+ * `results.serviceUrl` and printed as a `URL:` line — a report field
+ * naming an endpoint that does not exist. The only caller that still
+ * passed one was Tier-1's readiness step, and that step is gone
+ * (see ValidationOrchestrator: its verdict was a strict function of
+ * steps 2 and 3). What is left has exactly one consumer:
+ * `helpers/createServiceReadinessTests`, and through it the
+ * `tests/bootstrap/` suites of the biz repos — which pass name, version,
+ * operations, testCookbook and registry, and never passed a url.
+ *
  * @see /api/docs/architecture/biz-service-invocation-model.md §5.3
  * @see /api/docs/biz/40-cookbooks/test-runner-flow.md (input probe contract)
  * @see /api/docs/biz/80-decisions/0005-no-http-in-biz-containers.md
@@ -55,7 +66,6 @@ class ServiceReadinessValidator {
    */
   async validateReadiness(service) {
     const {
-      url,
       operations,
       registry,
       testCookbook
@@ -64,7 +74,6 @@ class ServiceReadinessValidator {
     const results = {
       timestamp: Date.now(),
       serviceName: service.name,
-      serviceUrl: url,
       checks: {},
       score: 0,
       maxScore: 100,
@@ -73,14 +82,29 @@ class ServiceReadinessValidator {
       warnings: []
     };
 
+    /**
+     * Record what a check contributed. `createServiceReadinessTests` prints
+     * `checkResult.score` per check; nothing ever assigned it, so every biz
+     * bootstrap run printed "0 points" beside checks that had just been
+     * awarded their full weight, under a total that contradicted them.
+     */
+    const award = (name, failureMessage) => {
+      const { weight, required } = this.checks[name];
+      if (results.checks[name].passed) {
+        results.checks[name].score = weight;
+        results.score += weight;
+      } else {
+        results.checks[name].score = 0;
+        if (required) {
+          results.errors.push(failureMessage);
+        }
+      }
+    };
+
     // 1. Validate operations.json structure
     if (operations) {
       results.checks.operations = await this.checkOperationsCompliance(operations);
-      if (results.checks.operations.passed) {
-        results.score += this.checks.operations.weight;
-      } else if (this.checks.operations.required) {
-        results.errors.push('Operations validation failed');
-      }
+      award('operations', 'Operations validation failed');
     } else {
       results.errors.push('No operations.json provided - required for all services');
       results.checks.operations = { passed: false, error: 'Missing operations.json' };
@@ -101,11 +125,7 @@ class ServiceReadinessValidator {
       results.checks.cookbook = await this.checkCookbookExecution(
         testCookbook
       );
-      if (results.checks.cookbook.passed) {
-        results.score += this.checks.cookbook.weight;
-      } else if (this.checks.cookbook.required) {
-        results.errors.push('Cookbook validation failed');
-      }
+      award('cookbook', 'Cookbook validation failed');
     }
 
     // 5. Verify registry compatibility
@@ -114,11 +134,7 @@ class ServiceReadinessValidator {
         service,
         registry
       );
-      if (results.checks.registry.passed) {
-        results.score += this.checks.registry.weight;
-      } else if (this.checks.registry.required) {
-        results.errors.push('Registry compatibility check failed');
-      }
+      award('registry', 'Registry compatibility check failed');
     }
 
     // Calculate readiness
@@ -292,7 +308,8 @@ class ServiceReadinessValidator {
 
     report.push('=== Service Readiness Report ===');
     report.push(`Service: ${results.serviceName}`);
-    report.push(`URL: ${results.serviceUrl}`);
+    // No URL line: biz services expose no endpoint (ADR 0005). Printing one
+    // named an address nothing listened on.
     report.push(`Score: ${results.score}/${results.maxScore}`);
     report.push(`Status: ${results.ready ? 'READY' : 'NOT READY'}`);
     report.push('');