@caoscompanybr/merlin 3.5.0

package/.merlin-core/core/process/executor.js

@@ -0,0 +1,231 @@
+/**
+ * Process Executor -- local permission-free execution of certified processes (Constitution XIV).
+ *
+ * The local realization of "the agent runs the entire process free, start to stop":
+ *   - refuses any process that is not `status: certified` with a VALID signature;
+ *   - per step, asks the ApprovalEngine under a certification context — autonomous
+ *     non-absolute steps auto-approve (CERTIFIED stake) and run; absolute steps,
+ *     non-autonomous steps, and steps whose pre-push secret scan fails HALT;
+ *   - records the run outcome into the process's report_back frontmatter.
+ *
+ * Default mode is DRY-RUN (no commands executed) — this is also the rehearsal vehicle
+ * for certification gates G3/G4. Pass {dryRun:false} to actually execute autonomous steps.
+ *
+ * @module core/process/executor
+ * @version 3.3.1
+ */
+
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const { execSync } = require("child_process");
+const { loadProcess } = require("./process-file");
+const { verify } = require("./signature");
+const { scanForSecrets, gitleaksStaged } = require("./secret-scan");
+const {
+  setFrontmatterScalars,
+} = require("../../tools/commands/certify-process");
+const { ApprovalEngine } = require("../approval/approval-engine");
+
+const PROJECT_ROOT = path.resolve(__dirname, "..", "..", "..");
+
+/** A step touches the outside world via git push — the canonical CERTIFIED action. */
+function stepRequiresPush(step) {
+  return /\bgit\s+push\b/.test(step.command || "");
+}
+
+/**
+ * Decide what happens to a single step (pure of side effects — used by runProcess and tests).
+ *
+ * @param {ApprovalEngine} engine
+ * @param {object} step
+ * @param {boolean} signatureValid - whether the process signature verified
+ * @param {object} ctx - { processContent }
+ * @returns {Promise<{id,stake,approved,action,requiresPush,secretFindings}>}
+ *   action ∈ run | halt-absolute | halt-approval | block-secret
+ */
+async function decideStep(engine, step, signatureValid, ctx = {}) {
+  const requiresPush = stepRequiresPush(step);
+  let secretScanPassed = true;
+  let secretFindings = [];
+  if (requiresPush) {
+    secretFindings = scanForSecrets(
+      `${ctx.processContent || ""}\n${step.command || ""}`,
+    );
+    secretScanPassed = secretFindings.length === 0;
+  }
+
+  // certification.verified folds in step.autonomous: a non-autonomous step is, by
+  // definition, NOT pre-approved for free execution even within a certified process.
+  const certification = {
+    verified: signatureValid && step.autonomous === true,
+    absolute: step.safety === "absolute",
+    requiresPush,
+    secretScanPassed,
+  };
+
+  const decision = await engine.requestApproval(
+    `process.step.${step.id || "anon"}`,
+    { certification, description: step.description },
+  );
+
+  let action;
+  if (decision.approved) action = "run";
+  else if (certification.absolute) action = "halt-absolute";
+  else if (requiresPush && !secretScanPassed) action = "block-secret";
+  else action = "halt-approval";
+
+  return {
+    id: step.id,
+    stake: decision.stake,
+    approved: decision.approved,
+    action,
+    requiresPush,
+    secretFindings: secretFindings.map((f) => f.sample),
+  };
+}
+
+/** Resolve a slug or path to a process file. */
+function resolveProcessPath(fileOrSlug, projectRoot) {
+  if (path.isAbsolute(fileOrSlug) && fs.existsSync(fileOrSlug))
+    return fileOrSlug;
+  const asRel = path.join(projectRoot, fileOrSlug);
+  if (fs.existsSync(asRel)) return asRel;
+  const dir = path.join(
+    projectRoot,
+    ".merlin-core",
+    "thoughts",
+    "shared",
+    "processes",
+  );
+  if (fs.existsSync(dir)) {
+    const hit = fs
+      .readdirSync(dir)
+      .find((f) => f.endsWith(".md") && f.includes(fileOrSlug));
+    if (hit) return path.join(dir, hit);
+  }
+  return asRel; // let the caller's existsSync fail with a clear message
+}
+
+/**
+ * Run a certified process locally.
+ *
+ * @param {string} fileOrSlug
+ * @param {{dryRun?: boolean, projectRoot?: string}} [opts]
+ * @returns {Promise<object>} run summary
+ */
+async function runProcess(fileOrSlug, opts = {}) {
+  const dryRun = opts.dryRun !== false; // default DRY-RUN
+  const projectRoot = opts.projectRoot || PROJECT_ROOT;
+  const filePath = resolveProcessPath(fileOrSlug, projectRoot);
+
+  if (!fs.existsSync(filePath)) {
+    return {
+      ran: false,
+      refused: true,
+      reason: `process not found: ${fileOrSlug}`,
+    };
+  }
+  const { content, frontmatter, definition } = loadProcess(filePath);
+
+  // Gate 1: must be certified
+  if (frontmatter.status !== "certified") {
+    return {
+      ran: false,
+      refused: true,
+      reason: `status is '${frontmatter.status || "draft"}', not certified — run /certify-process first`,
+    };
+  }
+  if (
+    !definition ||
+    !Array.isArray(definition.steps) ||
+    !definition.steps.length
+  ) {
+    return {
+      ran: false,
+      refused: true,
+      reason: "no runnable steps in definition",
+    };
+  }
+
+  // Gate 2: signature must verify (tamper detection)
+  const signatureValid = verify(definition.steps, frontmatter.signature);
+  if (!signatureValid) {
+    return {
+      ran: false,
+      refused: true,
+      reason:
+        "signature mismatch — file edited since certification; re-certify required",
+    };
+  }
+
+  const engine = new ApprovalEngine(projectRoot);
+  const steps = [];
+  let status = "ok";
+
+  for (const step of definition.steps) {
+    const d = await decideStep(engine, step, signatureValid, {
+      processContent: content,
+    });
+
+    if (d.action !== "run") {
+      // Autonomous run halts at the first gate (absolute / non-autonomous / secret block).
+      status = d.action === "block-secret" ? "failed" : "halted-for-approval";
+      steps.push({ ...d, executed: false });
+      break;
+    }
+
+    if (dryRun) {
+      steps.push({ ...d, simulated: true });
+      continue;
+    }
+
+    // Live: a push step gets a final gitleaks staged scan immediately before running.
+    if (d.requiresPush) {
+      const gl = gitleaksStaged(projectRoot);
+      if (gl.available && gl.clean === false) {
+        status = "failed";
+        steps.push({
+          ...d,
+          executed: false,
+          error: "gitleaks blocked push (secret in staged diff)",
+        });
+        break;
+      }
+    }
+    try {
+      execSync(step.command, { cwd: projectRoot, stdio: "pipe" });
+      steps.push({ ...d, executed: true });
+    } catch (err) {
+      const failMsg = String(err.message || "").slice(0, 300);
+      steps.push({ ...d, executed: false, error: failMsg });
+      if ((step.on_failure || "halt") === "halt") {
+        status = "failed";
+        break;
+      }
+    }
+  }
+
+  // Record outcome (skip in dry-run so rehearsals don't mutate the file).
+  if (!dryRun) {
+    try {
+      const updated = setFrontmatterScalars(content, {
+        last_run_status: status,
+        last_run_at: new Date().toISOString(),
+      });
+      fs.writeFileSync(filePath, updated);
+    } catch {
+      /* non-fatal: report_back write is best-effort */
+    }
+  }
+
+  return { ran: true, dryRun, refused: false, status, steps, path: filePath };
+}
+
+module.exports = {
+  runProcess,
+  decideStep,
+  stepRequiresPush,
+  resolveProcessPath,
+};

package/.merlin-core/core/process/process-file.js

@@ -0,0 +1,50 @@
+/**
+ * Process File loader -- shared parsing for PROCESSO thoughts.
+ *
+ * A PROCESSO file has FLAT frontmatter (metadata + flattened trigger + cert scalars,
+ * parsed by the indexer's naive frontmatter parser) plus a fenced ```json body block
+ * holding the structured runbook (`steps` + `success_criteria`). The naive YAML parser
+ * cannot handle block-lists/nested maps, so the runbook lives in the JSON block and is
+ * parsed here with JSON.parse — used by both /certify-process (Phase B) and the executor.
+ *
+ * @module core/process/process-file
+ * @version 3.3.1
+ */
+
+"use strict";
+
+const fs = require("fs");
+const { parseFrontmatter } = require("../alkimia/memory/thoughts-indexer");
+
+/**
+ * Extract the canonical machine-readable definition from the FIRST fenced ```json
+ * block in the body.
+ *
+ * @param {string} content - full file content
+ * @returns {{steps: Array, success_criteria: Array}|null} parsed definition, or null
+ */
+function extractDefinition(content) {
+  const match = content.match(/```json\s*\r?\n([\s\S]*?)```/);
+  if (!match) return null;
+  try {
+    return JSON.parse(match[1]);
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Load and parse a PROCESSO file.
+ *
+ * @param {string} filePath
+ * @returns {{path: string, content: string, frontmatter: object, definition: object|null}}
+ */
+function loadProcess(filePath) {
+  const content = fs.readFileSync(filePath, "utf8");
+  const parsed = parseFrontmatter(content);
+  const frontmatter = parsed.frontmatter || parsed || {};
+  const definition = extractDefinition(content);
+  return { path: filePath, content, frontmatter, definition };
+}
+
+module.exports = { extractDefinition, loadProcess };

package/.merlin-core/core/process/secret-scan.js

@@ -0,0 +1,86 @@
+/**
+ * Secret Scan -- reusable secret detection for the certified-process pipeline.
+ *
+ * Enforces the Article XI / Article XIV(c) floor: a certified process is secret-scanned
+ * before certification (gate G5) and before any autonomous outbound git push (executor).
+ * `secrets.expose` can never be auto-approved, so a positive finding HARD-BLOCKS.
+ *
+ * Provides a regex scanner (no external deps) plus a best-effort gitleaks wrapper for
+ * staged-diff scanning when gitleaks is installed (the `secret-safe-commit` skill's tool).
+ *
+ * @module core/process/secret-scan
+ * @version 3.3.1
+ */
+
+"use strict";
+
+const fs = require("fs");
+const { execFileSync } = require("child_process");
+
+// Mirrors core/execution/guardrail-engine.js SECRET_PATTERNS (kept in sync intentionally;
+// Phase D dedupes guardrail-engine to import from here).
+const SECRET_PATTERNS = [
+  /(?:api[_-]?key|secret|password|token|credential)\s*[:=]\s*['"][^'"]{8,}/gi,
+  /(?:AWS|AZURE|GCP)[_A-Z]*(?:KEY|SECRET|TOKEN)\s*[:=]\s*['"][^'"]+/gi,
+  /-----BEGIN (?:RSA |EC )?PRIVATE KEY-----/g,
+];
+
+/**
+ * Scan a string for secret-like content.
+ *
+ * @param {string} content
+ * @returns {Array<{pattern: string, sample: string}>} findings (empty = clean)
+ */
+function scanForSecrets(content) {
+  const findings = [];
+  if (!content || typeof content !== "string") return findings;
+  for (const pattern of SECRET_PATTERNS) {
+    pattern.lastIndex = 0;
+    const m = pattern.exec(content);
+    if (m) {
+      findings.push({
+        pattern: pattern.source,
+        // redact the matched value — never echo the secret itself
+        sample: `${m[0].slice(0, 12)}…[redacted]`,
+      });
+    }
+  }
+  return findings;
+}
+
+/**
+ * Scan a file's contents for secrets.
+ * @param {string} filePath
+ * @returns {Array} findings
+ */
+function scanFile(filePath) {
+  return scanForSecrets(fs.readFileSync(filePath, "utf8"));
+}
+
+/**
+ * Best-effort gitleaks staged-diff scan (used before an autonomous push).
+ * Returns {available:false} when gitleaks isn't installed — callers must then fall back
+ * to scanForSecrets and MUST NOT treat "not available" as "clean" for a real push.
+ *
+ * @param {string} cwd
+ * @returns {{available: boolean, clean?: boolean, output?: string}}
+ */
+function gitleaksStaged(cwd) {
+  try {
+    execFileSync("gitleaks", ["protect", "--staged", "--redact"], {
+      cwd,
+      stdio: "pipe",
+    });
+    return { available: true, clean: true };
+  } catch (err) {
+    if (err && err.code === "ENOENT") return { available: false };
+    // gitleaks ran and exited non-zero → leak found
+    return {
+      available: true,
+      clean: false,
+      output: String(err.stdout || err.message || "").slice(0, 2000),
+    };
+  }
+}
+
+module.exports = { SECRET_PATTERNS, scanForSecrets, scanFile, gitleaksStaged };

package/.merlin-core/core/process/signature.js

@@ -0,0 +1,77 @@
+/**
+ * Process Signature -- tamper-detection for certified processes (Constitution XIV).
+ *
+ * A certified PROCESSO carries a SHA-256 signature over its canonicalized step list.
+ * The executor re-computes the signature at run time and refuses autonomous execution
+ * if it does not match the value recorded at certification — so editing a process file
+ * after it was signed revokes its autonomy until it is re-certified.
+ *
+ * v1 uses an integrity HASH (detects tampering). Authenticity (proving Merlin signed it,
+ * e.g. via Ed25519) is deferred hardening.
+ *
+ * @module core/process/signature
+ * @version 3.3.1
+ */
+
+"use strict";
+
+const crypto = require("crypto");
+
+/**
+ * Deterministically serialize the step list so logically-identical steps hash
+ * identically regardless of key order. Keys within each step are sorted; array
+ * order is preserved (step order is semantically meaningful).
+ *
+ * @param {Array<object>} steps
+ * @returns {string} canonical JSON
+ */
+function canonicalizeSteps(steps) {
+  if (!Array.isArray(steps)) {
+    throw new TypeError("steps must be an array");
+  }
+  const canonical = steps.map((step) => {
+    if (!step || typeof step !== "object") return step;
+    const out = {};
+    for (const key of Object.keys(step).sort()) {
+      out[key] = step[key];
+    }
+    return out;
+  });
+  return JSON.stringify(canonical);
+}
+
+/**
+ * Compute the signature for a step list.
+ *
+ * @param {Array<object>} steps
+ * @returns {string} hex SHA-256 digest
+ */
+function sign(steps) {
+  return crypto
+    .createHash("sha256")
+    .update(canonicalizeSteps(steps))
+    .digest("hex");
+}
+
+/**
+ * Verify a recorded signature against the current step list (constant-time).
+ *
+ * @param {Array<object>} steps
+ * @param {string} signature - the hex digest recorded at certification
+ * @returns {boolean}
+ */
+function verify(steps, signature) {
+  if (!signature || typeof signature !== "string") return false;
+  let expected;
+  try {
+    expected = sign(steps);
+  } catch {
+    return false;
+  }
+  const a = Buffer.from(expected, "utf8");
+  const b = Buffer.from(signature, "utf8");
+  if (a.length !== b.length) return false;
+  return crypto.timingSafeEqual(a, b);
+}
+
+module.exports = { canonicalizeSteps, sign, verify };

package/.merlin-core/core/quality-gates/README.md

@@ -0,0 +1,17 @@
+# Quality Gates System
+
+The quality gates module implements a 3-layer quality assurance system that enforces standards at different stages of development. Layer 1 catches issues at pre-commit time, Layer 2 automates checks on pull requests, and Layer 3 enables human review and approval workflows.
+
+## Contents
+
+- **layer-1-pre-commit/** - Pre-commit hooks and immediate validation rules
+- **layer-2-pr-automation/** - Pull request checks and automated testing
+- **layer-3-human-review/** - Human review workflows and approval gates
+- **gate-evaluator.js** - Core gate evaluation engine
+- **rules.json** - Quality gate rules and policies
+
+## Related
+
+- [Hooks](../../hooks/README.md)
+- [Approval](../approval/README.md)
+- [Execution](../execution/README.md)

package/.merlin-core/core/quality-gates/layer1-precommit.js

@@ -0,0 +1,110 @@
+/**
+ * Quality Gate Layer 1 -- Pre-commit Checks
+ *
+ * Automated checks that run before every commit.
+ * Implements Constitution Article V: Quality First.
+ *
+ * Checks:
+ * - Lint (eslint/prettier)
+ * - Tests (jest/vitest/pytest)
+ * - Type check (tsc)
+ * - Build (if applicable)
+ *
+ * @module core/quality-gates/layer1-precommit
+ * @version 3.0.0
+ */
+
+const { exec } = require("child_process");
+const { promisify } = require("util");
+const execAsync = promisify(exec);
+const fs = require("fs");
+const path = require("path");
+
+class Layer1PrecommitGate {
+  constructor(projectRoot) {
+    this.projectRoot = projectRoot;
+  }
+
+  /**
+   * Run all Layer 1 checks.
+   *
+   * @returns {Promise<{passed: boolean, checks: object[]}>}
+   */
+  async runAll() {
+    const checks = [];
+    const packageJsonPath = path.join(this.projectRoot, "package.json");
+
+    let scripts = {};
+    if (fs.existsSync(packageJsonPath)) {
+      const pkg = JSON.parse(fs.readFileSync(packageJsonPath, "utf8"));
+      scripts = pkg.scripts || {};
+    }
+
+    // Lint
+    if (scripts.lint) {
+      checks.push(this._runCheck("lint", "npm run lint"));
+    }
+
+    // Tests
+    if (scripts.test) {
+      checks.push(this._runCheck("test", "npm test"));
+    }
+
+    // TypeCheck
+    if (scripts.typecheck) {
+      checks.push(this._runCheck("typecheck", "npm run typecheck"));
+    }
+
+    // Build
+    if (scripts.build) {
+      checks.push(this._runCheck("build", "npm run build"));
+    }
+
+    const results = await Promise.all(checks);
+    const passed = results.every((r) => r.passed);
+
+    return { passed, checks: results };
+  }
+
+  /**
+   * Run a single check.
+   */
+  // M1 fix: Use async exec to avoid blocking the event loop
+  async _runCheck(name, command) {
+    try {
+      const { stdout } = await execAsync(command, {
+        cwd: this.projectRoot,
+        timeout: 120000, // 2 min timeout
+      });
+      return { name, passed: true, output: (stdout || "").substring(0, 500) };
+    } catch (error) {
+      return {
+        name,
+        passed: false,
+        output: (error.stderr || error.stdout || error.message || "").substring(
+          0,
+          500,
+        ),
+      };
+    }
+  }
+
+  /**
+   * Format results as a report.
+   */
+  formatReport(results) {
+    const lines = ["## Quality Gate Layer 1: Pre-commit", ""];
+
+    for (const check of results.checks) {
+      const status = check.passed ? "PASS" : "FAIL";
+      lines.push(`- [${check.passed ? "x" : " "}] ${check.name}: ${status}`);
+    }
+
+    lines.push("");
+    lines.push(`**Verdict: ${results.passed ? "PASS" : "FAIL"}**`);
+
+    return lines.join("\n");
+  }
+}
+
+module.exports = { Layer1PrecommitGate };

package/.merlin-core/core/quality-gates/layer2-pr-automation.js

@@ -0,0 +1,116 @@
+/**
+ * Quality Gate Layer 2 -- PR Automation
+ *
+ * Automated checks that run on pull request creation.
+ * Extends Layer 1 with PR-specific validations.
+ *
+ * Checks:
+ * - All Layer 1 checks pass
+ * - PR title follows conventional format
+ * - PR description is present and complete
+ * - Branch is up-to-date with base
+ * - No merge conflicts
+ * - CI pipeline passes
+ * - Load-test regression (optional, present when .ai/load-tests/LATEST.json exists)
+ *
+ * @module core/quality-gates/layer2-pr-automation
+ * @version 3.0.0
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+class Layer2PrAutomationGate {
+  constructor(projectRoot) {
+    this.projectRoot = projectRoot;
+  }
+
+  /**
+   * Validate PR metadata.
+   *
+   * @param {object} pr - PR details
+   * @param {string} pr.title - PR title
+   * @param {string} pr.body - PR description
+   * @param {string} pr.branch - Source branch
+   * @param {string} pr.baseBranch - Target branch
+   * @returns {{passed: boolean, checks: object[]}}
+   */
+  validatePR(pr) {
+    const checks = [];
+
+    // Title format (conventional commits)
+    const titlePattern =
+      /^(feat|fix|docs|style|refactor|test|chore|perf|ci|build|revert)(\(.+\))?: .+/;
+    checks.push({
+      name: "title-format",
+      passed: titlePattern.test(pr.title),
+      message: titlePattern.test(pr.title)
+        ? "Title follows conventional format"
+        : "Title should follow conventional format: type(scope): description",
+    });
+
+    // Description present
+    checks.push({
+      name: "description-present",
+      passed: pr.body && pr.body.length > 20,
+      message:
+        pr.body && pr.body.length > 20
+          ? "Description is present"
+          : "PR needs a meaningful description",
+    });
+
+    // Summary section
+    checks.push({
+      name: "summary-section",
+      passed: pr.body && pr.body.includes("## Summary"),
+      message:
+        pr.body && pr.body.includes("## Summary")
+          ? "Summary section found"
+          : "PR description should include a ## Summary section",
+    });
+
+    // Load-test regression (optional -- present only when run_local.py has executed)
+    const latestRunFile = path.join(
+      this.projectRoot,
+      ".ai",
+      "load-tests",
+      "LATEST.json",
+    );
+    if (fs.existsSync(latestRunFile)) {
+      try {
+        const run = JSON.parse(fs.readFileSync(latestRunFile, "utf8"));
+        const ok = run.threshold_ok !== false && run.regression_ok !== false;
+        checks.push({
+          name: "load-test-regression",
+          passed: ok,
+          message:
+            run.summary || (ok ? "Load test passed" : "Load test failed"),
+        });
+      } catch (err) {
+        checks.push({
+          name: "load-test-regression",
+          passed: false,
+          message: `Could not parse LATEST.json: ${err.message}`,
+        });
+      }
+    }
+
+    const passed = checks.every((c) => c.passed);
+    return { passed, checks };
+  }
+
+  formatReport(results) {
+    const lines = ["## Quality Gate Layer 2: PR Automation", ""];
+    for (const check of results.checks) {
+      const status = check.passed ? "PASS" : "FAIL";
+      lines.push(
+        `- [${check.passed ? "x" : " "}] ${check.name}: ${status} -- ${check.message}`,
+      );
+    }
+    lines.push("");
+    lines.push(`**Verdict: ${results.passed ? "PASS" : "FAIL"}**`);
+    return lines.join("\n");
+  }
+}
+
+module.exports = { Layer2PrAutomationGate };