@onlineapps/conn-orch-validator 3.3.2 → 4.0.1

package/src/utils/envContract.js

@@ -0,0 +1,417 @@
+'use strict';
+
+/**
+ * Environment contract (F16) — one declaration, two consumers.
+ *
+ * The platform already had an ENFORCEMENT engine for environment variables
+ * (@onlineapps/runtime-config schemas throw on a missing required key) but no
+ * DECLARATION: nothing said which variables a service needs, so nothing could
+ * tell a missing one from a variable nobody reads any more. The 2026-08 audit
+ * measured 56 variables across 8 services; 21 were enforced by ConfigLoader,
+ * 5 by the connector contract, and the rest by nothing at all.
+ *
+ * So the service declares its environment in the file where it already declares
+ * its connectors and its database — `config/service/integration-contract.json`:
+ *
+ *   "env": {
+ *     "required": [{ "name": "SECRETS_MASTER_KEY", "why": "decrypts per-tenant secrets" }],
+ *     "optional": [{ "name": "SES_REGION",         "why": "AWS region when SES is live" }]
+ *   }
+ *
+ * `why` is mandatory. A required declaration nobody can justify is exactly the
+ * one nobody dares delete (change-discipline.md § Removing something removes
+ * its declaration), and a gate must be able to say what the fix is
+ * (automation-gates.md §3).
+ *
+ * Two consumers read that one declaration:
+ *
+ *   CI   — COMPLETENESS: everything the repo reads is declared or covered
+ *          (`verifyEnvCompleteness`, folded into `biz-ci-gate verify-contract`).
+ *   boot — PRESENCE: everything declared required is actually set
+ *          (`verifyEnvPresence`, Tier-1 phase 0.2, before any connector opens —
+ *          the fix for SECRETS_MASTER_KEY failing after MQ registration).
+ *
+ * ## What the declaration must NOT repeat
+ *
+ * Two mechanisms already cover part of the environment, and duplicating them
+ * would create a second owner for the same fact:
+ *
+ *   M1  `${VAR}` placeholders in `config/service/*.json` — ConfigLoader
+ *       substitutes them and fails on a missing required key.
+ *   M2  the endpoint variables of the connectors the contract declares
+ *       required — checked by `utils/connectorContract.js`.
+ *
+ * A name covered by either is REJECTED in the env block.
+ *
+ * ## Measurability boundary — stated, not implied
+ *
+ * The completeness scan reads the service repository only, and only literal
+ * accesses. It cannot see:
+ *
+ *   - dynamic access (`process.env[key]`) — counted and reported, never guessed;
+ *   - environment read inside an installed library (`node_modules`), e.g.
+ *     `SECRETS_MASTER_KEY` in @onlineapps/service-wrapper. Those names are
+ *     declared by hand until libraries export their own env schemas (variant D
+ *     of the F16 design, api/shared/TODO.md §5);
+ *   - anything outside `src/`, `index.js`, `scripts/` and `config/service/*.json`.
+ *
+ * The scan therefore proves one direction only: what it CAN see is declared.
+ * It never claims the declaration is complete.
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { CONNECTORS } = require('./connectorContract');
+
+/** POSIX-ish environment variable name, as every platform template writes them. */
+const ENV_NAME = /^[A-Z][A-Z0-9_]*$/;
+
+/** Directories and files the completeness scan reads, relative to the service root. */
+const SCAN_ROOTS = ['src', 'index.js', 'scripts'];
+
+/** Config files whose `${VAR}` placeholders are M1 coverage. */
+const CONFIG_GLOB_DIR = path.join('config', 'service');
+
+const SCANNED_EXTENSIONS = new Set(['.js', '.cjs', '.mjs']);
+
+const ENV_SCAN_SCOPE = 'config/service/*.json ${VAR}; literal process.env.X / env.X / '
+  + "requireEnv('X') in src/, index.js, scripts/";
+
+const ENV_SCAN_BLIND_SPOTS = 'dynamic access (process.env[key]) and environment read inside '
+  + 'installed libraries (node_modules)';
+
+const ENV_LIST_KEYS = ['required', 'optional'];
+const ENV_ITEM_KEYS = ['name', 'why'];
+
+/**
+ * Validate and normalize the `env` block of an integration contract.
+ *
+ * @param {object|undefined|null} rawEnv the contract's `env` value
+ * @param {object} options
+ * @param {Map<string,string>} [options.coverage] name -> what already covers it (M1/M2).
+ *   Mandatory whenever a block is present: without it redundancy cannot be
+ *   decided, and a check that silently degrades to half of itself is a false
+ *   guarantee (automation-gates.md §5).
+ * @returns {{required: Array<{name: string, why: string}>, optional: Array<{name: string, why: string}>}|null}
+ */
+function normalizeEnvDeclaration(rawEnv, options = {}) {
+  // Absent means "not adopted yet" — a legitimate state until every repo carries
+  // the block. Null rather than {} so callers can say so out loud instead of
+  // reporting an empty declaration as a satisfied one.
+  if (rawEnv === undefined || rawEnv === null) return null;
+
+  if (typeof rawEnv !== 'object' || Array.isArray(rawEnv)) {
+    throw new Error('[BizCiGate] Invalid env block - Expected an object with "required" and/or "optional" arrays. '
+      + 'Fix: "env": { "required": [{ "name": "DB_TIMEOUT_MS", "why": "..." }] }, or omit the key entirely.');
+  }
+
+  const coverage = options.coverage;
+  if (!(coverage instanceof Map)) {
+    throw new Error('[BizCiGate] Cannot validate the env block - no M1/M2 coverage set was supplied. '
+      + 'Fix: load the contract through loadAndValidateIntegrationContract(serviceRoot), which collects '
+      + 'the config placeholders and connector variables the declaration must not repeat.');
+  }
+
+  for (const key of Object.keys(rawEnv)) {
+    if (!ENV_LIST_KEYS.includes(key)) {
+      throw new Error(`[BizCiGate] Unknown env key - "${key}" is not part of the env block. `
+        + 'Fix: the block has exactly two keys, "required" and "optional".');
+    }
+  }
+
+  if (rawEnv.required === undefined && rawEnv.optional === undefined) {
+    throw new Error('[BizCiGate] Empty env block - Neither env.required nor env.optional is declared. '
+      + 'Fix: declare at least one variable, or omit the "env" key entirely — an empty block claims '
+      + 'the service reads no environment, which the completeness check will contradict.');
+  }
+
+  const declaredIn = new Map();
+  const normalized = { required: [], optional: [] };
+
+  for (const listKey of ENV_LIST_KEYS) {
+    const list = rawEnv[listKey];
+    if (list === undefined) continue;
+
+    if (!Array.isArray(list)) {
+      throw new Error(`[BizCiGate] Invalid env.${listKey} - Expected an array of `
+        + '{ "name": "...", "why": "..." } items. Fix: use [] or omit the key.');
+    }
+
+    list.forEach((item, index) => {
+      const at = `env.${listKey}[${index}]`;
+
+      if (!item || typeof item !== 'object' || Array.isArray(item)) {
+        throw new Error(`[BizCiGate] Invalid ${at} - Expected {"name": "...", "why": "..."}. `
+          + 'Fix: a bare string does not say why the variable exists.');
+      }
+
+      for (const key of Object.keys(item)) {
+        if (!ENV_ITEM_KEYS.includes(key)) {
+          throw new Error(`[BizCiGate] Unknown key in ${at} - "${key}". `
+            + 'Fix: an item has exactly two keys, "name" and "why".');
+        }
+      }
+
+      if (typeof item.name !== 'string' || !ENV_NAME.test(item.name)) {
+        throw new Error(`[BizCiGate] Invalid ${at}.name - "${item.name}" is not a valid environment `
+          + 'variable name. Fix: use SCREAMING_SNAKE_CASE exactly as the env file writes it, e.g. "DB_HOST".');
+      }
+
+      if (typeof item.why !== 'string' || item.why.trim() === '') {
+        throw new Error(`[BizCiGate] Missing ${at}.why - "${item.name}" is declared without a reason. `
+          + 'Fix: state in one sentence what the service does with it — a declaration nobody can justify '
+          + 'is the one nobody dares delete.');
+      }
+
+      const previous = declaredIn.get(item.name);
+      if (previous) {
+        throw new Error(`[BizCiGate] Duplicate env declaration - "${item.name}" is declared twice `
+          + `(env.${previous}, env.${listKey}). Fix: declare each variable exactly once.`);
+      }
+
+      const coveredBy = coverage.get(item.name);
+      if (coveredBy) {
+        throw new Error(`[BizCiGate] Redundant env declaration - "${item.name}" is already covered by `
+          + `${coveredBy}, remove the duplicate declaration. Fix: the env block declares what NO other `
+          + 'mechanism already enforces; two owners of the same fact drift apart.');
+      }
+
+      declaredIn.set(item.name, listKey);
+      normalized[listKey].push({ name: item.name, why: item.why });
+    });
+  }
+
+  return normalized;
+}
+
+/**
+ * Collect the names M1 (config placeholders) and M2 (required connectors)
+ * already cover — the set the env block must not repeat.
+ *
+ * @param {object} args
+ * @param {string} args.serviceRoot
+ * @param {object} args.requiredConnectors db/redis/mq/minio booleans
+ * @returns {Map<string,string>} name -> what covers it
+ */
+function collectEnvCoverage({ serviceRoot, requiredConnectors }) {
+  const coverage = new Map();
+
+  for (const [file, names] of collectConfigPlaceholders(serviceRoot)) {
+    for (const name of names) {
+      if (!ENV_NAME.test(name)) continue; // e.g. ${npm_package_version} — not an env contract concern
+      if (!coverage.has(name)) coverage.set(name, `${file} (\${${name}})`);
+    }
+  }
+
+  for (const [connector, spec] of Object.entries(CONNECTORS)) {
+    if (requiredConnectors?.[connector] !== true) continue;
+    for (const name of spec.env) {
+      if (!coverage.has(name)) coverage.set(name, `requiredConnectors.${connector}`);
+    }
+  }
+
+  return coverage;
+}
+
+/** `${VAR}` placeholders per config/service/*.json file, as [relativePath, Set<name>]. */
+function collectConfigPlaceholders(serviceRoot) {
+  const configDir = path.join(serviceRoot, CONFIG_GLOB_DIR);
+  if (!fs.existsSync(configDir)) return [];
+
+  const entries = [];
+  for (const fileName of fs.readdirSync(configDir).sort()) {
+    if (!fileName.endsWith('.json')) continue;
+    const filePath = path.join(configDir, fileName);
+    if (!fs.statSync(filePath).isFile()) continue;
+
+    const content = fs.readFileSync(filePath, 'utf8');
+    const names = new Set();
+    for (const match of content.matchAll(/\$\{([A-Za-z_][A-Za-z0-9_]*)/g)) {
+      names.add(match[1]);
+    }
+    entries.push([path.posix.join('config', 'service', fileName), names]);
+  }
+
+  return entries;
+}
+
+function listSourceFiles(serviceRoot) {
+  const files = [];
+  const stack = SCAN_ROOTS.map((entry) => path.join(serviceRoot, entry));
+
+  while (stack.length > 0) {
+    const current = stack.pop();
+    if (!fs.existsSync(current)) continue;
+
+    const stat = fs.statSync(current);
+    if (stat.isDirectory()) {
+      for (const entry of fs.readdirSync(current)) {
+        if (entry === 'node_modules' || entry === '.git') continue;
+        stack.push(path.join(current, entry));
+      }
+      continue;
+    }
+
+    if (stat.isFile() && SCANNED_EXTENSIONS.has(path.extname(current))) {
+      files.push(current);
+    }
+  }
+
+  return files.sort();
+}
+
+/**
+ * `process.env.X` / `env.X` where X is an environment-shaped name.
+ *
+ * `env.X` is deliberate, not sloppy: architecture principle 1 makes injected
+ * configuration the norm, so a service that obeys DI reads `env.SES_REGION`
+ * from an injected object and never writes `process.env` at all. Measured
+ * 2026-08-27 across the eight biz repos: `process.env.` finds 9 names,
+ * `env.` finds 22 — including all thirteen of the emailer provider variables
+ * whose absence is defect D1. Restricting the pattern to SCREAMING_SNAKE_CASE
+ * on a receiver literally named `env` produced no false positive in that run.
+ */
+const READ_PATTERN = /(?<![A-Za-z0-9_$.])(?:process\.)?env\??\.([A-Z][A-Z0-9_]*)/g;
+const REQUIRE_ENV_PATTERN = /requireEnv\(\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]/g;
+const BRACKET_LITERAL_PATTERN = /(?:process\.)?env\[\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*\]/g;
+const DYNAMIC_PATTERN = /(?:process\.)?env\[\s*(?!['"])/g;
+
+/** A write (`process.env.X = …`) is not a read requirement. */
+function isAssignment(content, matchEnd) {
+  const rest = content.slice(matchEnd, matchEnd + 4);
+  return /^\s*=(?!=)/.test(rest);
+}
+
+function lineOf(content, index) {
+  return content.slice(0, index).split('\n').length;
+}
+
+/**
+ * Scan the service repository for environment names it reads.
+ *
+ * @param {object} args
+ * @param {string} args.serviceRoot
+ * @returns {{reads: Map<string, string[]>, dynamicSites: Array<{file: string, line: number}>, filesScanned: number}}
+ */
+function collectEnvReads({ serviceRoot }) {
+  const reads = new Map();
+  const dynamicSites = [];
+  const files = listSourceFiles(serviceRoot);
+
+  const record = (name, location) => {
+    if (!ENV_NAME.test(name)) return;
+    const existing = reads.get(name);
+    if (existing) {
+      if (!existing.includes(location)) existing.push(location);
+      return;
+    }
+    reads.set(name, [location]);
+  };
+
+  for (const filePath of files) {
+    const content = fs.readFileSync(filePath, 'utf8');
+    const relative = path.relative(serviceRoot, filePath).split(path.sep).join('/');
+
+    for (const match of content.matchAll(READ_PATTERN)) {
+      if (isAssignment(content, match.index + match[0].length)) continue;
+      record(match[1], `${relative}:${lineOf(content, match.index)}`);
+    }
+
+    for (const pattern of [BRACKET_LITERAL_PATTERN, REQUIRE_ENV_PATTERN]) {
+      for (const match of content.matchAll(pattern)) {
+        record(match[1], `${relative}:${lineOf(content, match.index)}`);
+      }
+    }
+
+    for (const match of content.matchAll(DYNAMIC_PATTERN)) {
+      dynamicSites.push({ file: relative, line: lineOf(content, match.index) });
+    }
+  }
+
+  return { reads, dynamicSites, filesScanned: files.length };
+}
+
+/**
+ * COMPLETENESS: every environment name the repository visibly reads is either
+ * declared in the env block or covered by M1/M2.
+ *
+ * @param {object} args
+ * @param {string} args.serviceRoot
+ * @param {object} args.contract normalized integration contract
+ * @returns {{ok: boolean, violations: Array<{name: string, sources: string[]}>, declaredCount: number,
+ *            coveredCount: number, readCount: number, filesScanned: number,
+ *            dynamicSites: Array<{file: string, line: number}>}}
+ */
+function verifyEnvCompleteness({ serviceRoot, contract }) {
+  const coverage = collectEnvCoverage({
+    serviceRoot,
+    requiredConnectors: contract.requiredConnectors
+  });
+
+  const declared = new Set();
+  for (const listKey of ENV_LIST_KEYS) {
+    for (const item of contract.env?.[listKey] ?? []) declared.add(item.name);
+  }
+
+  const { reads, dynamicSites, filesScanned } = collectEnvReads({ serviceRoot });
+
+  const violations = [];
+  for (const [name, sources] of reads) {
+    if (coverage.has(name) || declared.has(name)) continue;
+    violations.push({ name, sources });
+  }
+  violations.sort((a, b) => a.name.localeCompare(b.name));
+
+  return {
+    ok: violations.length === 0,
+    violations,
+    declaredCount: declared.size,
+    coveredCount: coverage.size,
+    readCount: reads.size,
+    filesScanned,
+    dynamicSites
+  };
+}
+
+/**
+ * PRESENCE: every name declared required is set in the environment.
+ *
+ * Runs in Tier-1 phase 0.2, before the wrapper opens a connector — a missing
+ * key must fail where its name is known, not several layers deeper (defect D2:
+ * SECRETS_MASTER_KEY failed in ServiceWrapper.js:916, after MQ registration).
+ *
+ * @param {object} args
+ * @param {object|null} args.declaration normalized env declaration, null when absent
+ * @param {object} args.env environment to check against
+ * @returns {{valid: boolean, skipped?: boolean, checked: string[], errors: string[]}}
+ */
+function verifyEnvPresence({ declaration, env }) {
+  if (declaration === null || declaration === undefined) {
+    return { valid: true, skipped: true, checked: [], errors: [] };
+  }
+
+  const checked = [];
+  const errors = [];
+
+  for (const item of declaration.required) {
+    checked.push(item.name);
+    const value = env?.[item.name];
+    if (typeof value === 'string' && value.trim() !== '') continue;
+
+    errors.push(`[EnvContract] Missing environment variable - ${item.name} is required `
+      + `(${item.why}). Fix: set ${item.name} in env-active/*.env`);
+  }
+
+  return { valid: errors.length === 0, checked, errors };
+}
+
+module.exports = {
+  ENV_NAME,
+  ENV_SCAN_SCOPE,
+  ENV_SCAN_BLIND_SPOTS,
+  normalizeEnvDeclaration,
+  collectEnvCoverage,
+  collectEnvReads,
+  verifyEnvCompleteness,
+  verifyEnvPresence
+};

package/src/utils/installContract.js

@@ -0,0 +1,142 @@
+'use strict';
+
+/**
+ * Installation contract for a biz service repository.
+ *
+ *   SETUP_DOCS   every repo carries docs/setup/{INSTALL,PLATFORM_MATRIX,VALIDATION}.md
+ *   SQL_PACKAGE  a repo WITH a database carries the BASELINE / SEED SQL tree
+ *   SQL_HEADERS  every SQL package file declares its dataset class and safety
+ *
+ * Rules: ADR 0006 (api/docs/biz/80-decisions/) — database and test-data contract.
+ *
+ * Whether the SQL half applies is DERIVED from the service's own
+ * integration-contract.json database block, never declared a second time here.
+ * The eight shell copies this replaces each carried a hand-maintained
+ * REPO_HAS_DB literal, and three of them (converter, hello-service, ingest) said
+ * "false" while their contract declared a database — so the SQL half checked
+ * nothing in those repos while still printing PASS. One fact, one owner: the
+ * contract says whether there is a database, and this decides what that costs.
+ *
+ * Pure module: reads the repository, returns a structured result. It renders
+ * nothing and exits nothing — the CLI owns presentation, this owns the rules.
+ *
+ * Every violation is reported, not just the first, so a repo can be fixed in
+ * one pass.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const SETUP_DOCS = [
+  'docs/setup/INSTALL.md',
+  'docs/setup/PLATFORM_MATRIX.md',
+  'docs/setup/VALIDATION.md'
+];
+
+const SQL_DIRECTORIES = [
+  'migrations/BASELINE',
+  'migrations/SEED/production_like',
+  'migrations/SEED/test_only'
+];
+
+const REQUIRED_HEADERS = [
+  '-- Dataset-Class:',
+  '-- Target-DB:',
+  '-- Safe-For-Production:',
+  '-- Idempotency:'
+];
+
+const TEST_ONLY_DIRECTORY = 'migrations/SEED/test_only';
+
+/** Sorted so the report order is the directory order, not the filesystem's. */
+function listSqlFiles(absoluteDir) {
+  return fs.readdirSync(absoluteDir)
+    .filter((name) => name.endsWith('.sql'))
+    .sort();
+}
+
+function checkSetupDocs(serviceRoot, add) {
+  for (const relativePath of SETUP_DOCS) {
+    if (!fs.existsSync(path.join(serviceRoot, relativePath))) {
+      add('SETUP_DOCS', `Missing required file - ${relativePath}. `
+        + `Fix: add ${relativePath} to the repository.`);
+    }
+  }
+}
+
+function checkSqlHeaders(serviceRoot, relativeDir, add) {
+  for (const name of listSqlFiles(path.join(serviceRoot, relativeDir))) {
+    const relativeFile = `${relativeDir}/${name}`;
+    const content = fs.readFileSync(path.join(serviceRoot, relativeFile), 'utf8');
+    const lines = content.split('\n');
+
+    for (const header of REQUIRED_HEADERS) {
+      if (!lines.some((line) => line.startsWith(header))) {
+        add('SQL_HEADERS', `Missing "${header}" header - ${relativeFile}. `
+          + `Fix: add the "${header}" header line to the file.`);
+      }
+    }
+
+    if (relativeDir !== TEST_ONLY_DIRECTORY) continue;
+
+    // Test data must be unmistakable at the file level: a TEST_ONLY package that
+    // reads as production-safe is the one mistake that reaches a real database.
+    if (!lines.some((line) => line.startsWith('-- Dataset-Class: TEST_ONLY'))) {
+      add('SQL_HEADERS', `Wrong Dataset-Class - ${relativeFile} must declare `
+        + '"-- Dataset-Class: TEST_ONLY". Fix: correct the Dataset-Class header.');
+    }
+    if (!lines.some((line) => line.startsWith('-- Safe-For-Production: no'))) {
+      add('SQL_HEADERS', `Wrong Safe-For-Production - ${relativeFile} must declare `
+        + '"-- Safe-For-Production: no". Fix: correct the Safe-For-Production header.');
+    }
+  }
+}
+
+function checkSqlPackage(serviceRoot, add) {
+  for (const relativeDir of SQL_DIRECTORIES) {
+    const absoluteDir = path.join(serviceRoot, relativeDir);
+
+    if (!fs.existsSync(absoluteDir)) {
+      add('SQL_PACKAGE', `Missing required directory - ${relativeDir}. `
+        + `Fix: create ${relativeDir} with its SQL package files.`);
+      continue;
+    }
+
+    // An existing but empty directory is the shape the shell version passed on:
+    // its `for sql in dir/*.sql` loop saw the unexpanded glob and moved on.
+    if (listSqlFiles(absoluteDir).length === 0) {
+      add('SQL_PACKAGE', `No SQL package files - ${relativeDir} contains no .sql file. `
+        + 'Fix: add the SQL package files, or remove the database declaration from the contract.');
+      continue;
+    }
+
+    checkSqlHeaders(serviceRoot, relativeDir, add);
+  }
+}
+
+/**
+ * @param {string} serviceRoot repository root to inspect
+ * @param {object|null} database normalized contract database block; null means
+ *   the service declares no database, and the SQL half does not apply
+ * @returns {{service: string, ok: boolean, databaseChecked: boolean, violations: Array<{requirement: string, message: string}>}}
+ */
+function verifyInstallContract(serviceRoot, database) {
+  const violations = [];
+  const add = (requirement, message) => violations.push({ requirement, message });
+
+  checkSetupDocs(serviceRoot, add);
+
+  const databaseChecked = database !== null && database !== undefined;
+  if (databaseChecked) {
+    checkSqlPackage(serviceRoot, add);
+  }
+
+  return {
+    service: path.basename(path.resolve(serviceRoot)),
+    ok: violations.length === 0,
+    databaseChecked,
+    violations
+  };
+}
+
+module.exports = { verifyInstallContract };