@mmerterden/multi-agent-pipeline 17.6.0 → 18.0.0

package/pipeline/lib/outbound-gate.mjs

@@ -0,0 +1,175 @@
+#!/usr/bin/env node
+/**
+ * outbound-gate.mjs  -  nothing leaves this machine carrying a credential.
+ *
+ * The repo already has a leak scanner, and it looks at the wrong thing for
+ * this purpose: `smoke-personal-data.sh` audits the FILES IN THE REPO so the
+ * published pipeline stays generic. It never sees a Jira comment, a PR body or
+ * an issue update - text that a run composes at runtime out of command output,
+ * error messages and file excerpts, any of which can contain a token that was
+ * on the machine a second earlier.
+ *
+ * SCOPE, and the part the plan for this got wrong: corporate names are NOT
+ * checked here. A Jira comment on a corporate ticket legitimately contains the
+ * corporate project key, the repo name and the hostname - refusing those would
+ * make the gate fire on every correct publish, and a gate that cries wolf is
+ * turned off. Repo genericity and run-output hygiene are different jobs with
+ * different lists, and only the second one ships.
+ *
+ * WIRED INTO, and the honest limit: the three SHELL publishers -
+ * lib/jira-publish.sh (which is also how `channels` posts to Jira),
+ * lib/post-pr-review.sh (per-finding comments and the request-changes body),
+ * and scripts/update-issue-progress.sh. That is every publish this repo makes
+ * from a script.
+ *
+ * It is NOT every publish. `channels` also writes a Confluence page and a PR
+ * description, and those are composed and posted by the agent following the
+ * skill rather than by a script, so nothing in shell can intercept them. A
+ * gate that claimed four paths while covering three would be worse than one
+ * that covers three and says so.
+ *
+ * WHAT IT NEVER DOES: print the thing it found. A gate that echoes the secret
+ * into a terminal, a log and a CI transcript has moved it somewhere new.
+ * Findings are reported as rule name plus line number, nothing else.
+ *
+ * Usage:
+ *   cat body.md | outbound-gate.mjs                 # exit 0 clean, 7 if blocked
+ *   outbound-gate.mjs --file body.md
+ *   cat body.md | outbound-gate.mjs --json
+ *   cat body.md | outbound-gate.mjs --warn          # report, never block
+ *
+ * Exit codes:
+ *   0  -  nothing found
+ *   7  -  at least one finding; the caller must not publish
+ *   2  -  usage error (no input)
+ */
+
+import { readFileSync } from "node:fs";
+import { invokedDirectly } from "./invoked-directly.mjs";
+
+/**
+ * Each rule is a SHAPE, not a dictionary. A list of known token prefixes goes
+ * stale the moment a provider adds one; a shape keeps working.
+ *
+ * `confidence` separates the two kinds of finding. A GitHub PAT has a fixed
+ * prefix and length and is never a false positive - that blocks. An email
+ * address is a shape that legitimate prose contains, so it is reported and the
+ * caller decides. Mixing them would mean either blocking valid publishes or
+ * letting credentials through, and there is no threshold that does both.
+ */
+export const RULES = [
+  // --- credentials: a match is a match -------------------------------------
+  { name: "github-pat", certain: true, re: /\bgh[pousr]_[A-Za-z0-9]{36,}\b/g },
+  { name: "github-fine-grained-pat", certain: true, re: /\bgithub_pat_[A-Za-z0-9_]{50,}\b/g },
+  { name: "slack-token", certain: true, re: /\bxox[abprs]-[A-Za-z0-9-]{10,}\b/g },
+  { name: "aws-access-key-id", certain: true, re: /\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g },
+  { name: "google-api-key", certain: true, re: /\bAIza[0-9A-Za-z_-]{35}\b/g },
+  { name: "openai-key", certain: true, re: /\bsk-(?:proj-)?[A-Za-z0-9_-]{32,}\b/g },
+  { name: "anthropic-key", certain: true, re: /\bsk-ant-[A-Za-z0-9_-]{32,}\b/g },
+  { name: "figma-token", certain: true, re: /\bfig[a-z]_[A-Za-z0-9_-]{20,}\b/g },
+  { name: "npm-token", certain: true, re: /\bnpm_[A-Za-z0-9]{36}\b/g },
+  {
+    name: "jwt",
+    certain: true,
+    re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g,
+  },
+  { name: "private-key-block", certain: true, re: /-----BEGIN [A-Z ]*PRIVATE KEY-----/g },
+  // A URL carrying credentials. This is the one that leaks by accident most
+  // often: a `git remote -v` pasted into a comment.
+  {
+    name: "url-with-credentials",
+    certain: true,
+    re: /\b[a-z][a-z0-9+.-]*:\/\/[^\s/:@]+:[^\s/@]+@/g,
+  },
+  // `Authorization: Bearer <something long>` copied out of a curl trace.
+  {
+    name: "bearer-header",
+    certain: true,
+    re: /\bAuthorization:\s*(?:Bearer|Basic)\s+[A-Za-z0-9._~+/=-]{16,}/gi,
+  },
+
+  // --- PII and environment: a shape, so reported rather than certain -------
+  {
+    name: "email-address",
+    certain: false,
+    re: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g,
+  },
+  // An absolute path under /Users or /home names the operator.
+  { name: "home-directory-path", certain: false, re: /\/(?:Users|home)\/[A-Za-z0-9_.-]+\//g },
+  {
+    name: "private-ip",
+    certain: false,
+    re: /\b(?:10\.\d{1,3}|192\.168|172\.(?:1[6-9]|2\d|3[01]))\.\d{1,3}\.\d{1,3}\b/g,
+  },
+];
+
+/**
+ * @param {string} text
+ * @returns {{rule: string, certain: boolean, line: number}[]}
+ */
+export function scan(text) {
+  const findings = [];
+  const lines = String(text).split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    for (const rule of RULES) {
+      // `lastIndex` survives between calls on a /g regex and would make every
+      // second line skip its start. Reset per line.
+      rule.re.lastIndex = 0;
+      if (rule.re.test(lines[i])) {
+        findings.push({ rule: rule.name, certain: rule.certain, line: i + 1 });
+      }
+    }
+  }
+  return findings;
+}
+
+function readInput(args) {
+  const fileFlag = args.indexOf("--file");
+  if (fileFlag !== -1) {
+    const p = args[fileFlag + 1];
+    if (!p) {
+      process.stderr.write("outbound-gate: --file needs a path\n");
+      process.exitCode = 2;
+      return null;
+    }
+    return readFileSync(p, "utf-8");
+  }
+  try {
+    return readFileSync(0, "utf-8");
+  } catch {
+    process.stderr.write("outbound-gate: nothing on stdin and no --file\n");
+    process.exitCode = 2;
+    return null;
+  }
+}
+
+function main() {
+  const args = process.argv.slice(2);
+  const text = readInput(args);
+  if (text === null) return;
+
+  const findings = scan(text);
+  const blocking = findings.filter((f) => f.certain);
+
+  if (args.includes("--json")) {
+    process.stdout.write(
+      JSON.stringify({ findings, blocked: blocking.length > 0 }, null, 2) + "\n",
+    );
+  } else if (findings.length) {
+    // Rule name and line only. The value is deliberately absent: writing it
+    // here would put the secret into a terminal, a log and a CI transcript.
+    const shown = findings.map(
+      (f) => `  ${f.certain ? "BLOCK" : "check"}  ${f.rule}  line ${f.line}`,
+    );
+    process.stderr.write(
+      `outbound-gate: ${findings.length} finding(s) in the text about to be published\n` +
+        shown.join("\n") +
+        "\n  (values are not printed - reading them out would move the secret here)\n",
+    );
+  }
+
+  if (blocking.length && !args.includes("--warn")) process.exitCode = 7;
+}
+
+const isDirectRun = invokedDirectly(import.meta.url);
+if (isDirectRun) main();

package/pipeline/lib/plan-todos.sh

@@ -30,6 +30,31 @@
 
 set -euo pipefail
 
+# jq is not optional on this path. Without the guard below a missing binary
+# renders as EMPTY DATA and the work continues on it; see lib/require-jq.sh.
+for _rq in "$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)/require-jq.sh" \
+           "$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/../lib" 2>/dev/null && pwd)/require-jq.sh" \
+           "$HOME/.claude/lib/require-jq.sh" \
+           "$HOME/.copilot/lib/require-jq.sh" \
+           "$HOME/.codex/lib/require-jq.sh"; do
+  [ -f "$_rq" ] || continue
+  # shellcheck source=/dev/null
+  . "$_rq" && break
+done
+unset _rq
+if ! command -v ma_require_jq >/dev/null 2>&1; then
+  # The helper itself is missing, which is an install problem, not a jq one.
+  ma_require_jq() { command -v jq >/dev/null 2>&1 || { echo "jq not found - cannot ${1:-continue}." >&2; return 1; }; }
+fi
+ma_require_jq "read or write the plan" || exit 3
+
+# Run-state path resolution: pipeline/lib/run-paths.sh owns the two layouts
+# (nested <root>/<project>/<id>/ and flat <root>/<id>/) and every id spelling.
+_MA_RP_HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=/dev/null
+. "$_MA_RP_HERE/run-paths.sh" 2>/dev/null || . "$HOME/.claude/lib/run-paths.sh"
+
+
 CMD="${1:-}"
 shift || true
 
@@ -49,12 +74,8 @@ resolve_state() {
   # directory ($HOME/.claude/logs/multi-agent/{project}/{taskId}/) - not the
   # ~/.claude/projects/*/state/ path this used to glob, which no writer in
   # the pipeline ever populates.
-  for candidate in "$HOME"/.claude/logs/multi-agent/*/"$task_id"/agent-state.json; do
-    if [ -f "$candidate" ]; then
-      found="$candidate"
-      break
-    fi
-  done
+  # This globbed ONLY the nested layout; flat-layout runs resolved to nothing.
+  found="$(ma_resolve_run_file "$task_id" agent-state.json 2>/dev/null || true)"
   if [ -z "$found" ]; then
     echo "plan-todos: agent-state.json not found for task=$task_id" >&2
     exit 1

package/pipeline/lib/post-pr-review.sh

@@ -34,6 +34,65 @@
 
 set -euo pipefail
 
+# jq is not optional on this path. Without the guard below a missing binary
+# renders as EMPTY DATA and the work continues on it; see lib/require-jq.sh.
+for _rq in "$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)/require-jq.sh" \
+           "$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/../lib" 2>/dev/null && pwd)/require-jq.sh" \
+           "$HOME/.claude/lib/require-jq.sh" \
+           "$HOME/.copilot/lib/require-jq.sh" \
+           "$HOME/.codex/lib/require-jq.sh"; do
+  [ -f "$_rq" ] || continue
+  # shellcheck source=/dev/null
+  . "$_rq" && break
+done
+unset _rq
+if ! command -v ma_require_jq >/dev/null 2>&1; then
+  # The helper itself is missing, which is an install problem, not a jq one.
+  ma_require_jq() { command -v jq >/dev/null 2>&1 || { echo "jq not found - cannot ${1:-continue}." >&2; return 1; }; }
+fi
+ma_require_jq "parse the review findings" || exit 3
+
+# Outbound leak gate. Every byte below is composed at runtime out of command
+# output, error text and file excerpts, any of which can carry a token that was
+# on this machine a second earlier - and once it is in a comment it is in
+# someone else's database. The repo's own leak scanner looks at FILES IN THE
+# REPO and never sees this text. See lib/outbound-gate.mjs.
+ma_outbound_gate() {
+  local body_file="$1" og=""
+  for c in "$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)/outbound-gate.mjs" \
+           "$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/../lib" 2>/dev/null && pwd)/outbound-gate.mjs" \
+           "$HOME/.claude/lib/outbound-gate.mjs" \
+           "$HOME/.copilot/lib/outbound-gate.mjs" \
+           "$HOME/.codex/lib/outbound-gate.mjs"; do
+    [ -f "$c" ] && { og="$c"; break; }
+  done
+  # Missing gate is NOT an open door: refusing to publish beats publishing
+  # unchecked, and the only way this file is absent is a broken install.
+  if [ -z "$og" ]; then
+    echo "outbound-gate.mjs not found - refusing to publish unchecked text." >&2
+    return 7
+  fi
+  node "$og" --file "$body_file"
+}
+
+# Same gate, for the bodies this file keeps in a variable rather than a file.
+ma_outbound_gate_text() {
+  local tmp
+  tmp=$(mktemp) || return 7
+  printf '%s' "$1" > "$tmp"
+  ma_outbound_gate "$tmp"
+  local rc=$?
+  rm -f "$tmp"
+  return $rc
+}
+
+# Run-state path resolution: pipeline/lib/run-paths.sh owns the two layouts
+# (nested <root>/<project>/<id>/ and flat <root>/<id>/) and every id spelling.
+_MA_RP_HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=/dev/null
+. "$_MA_RP_HERE/run-paths.sh" 2>/dev/null || . "$HOME/.claude/lib/run-paths.sh"
+
+
 # Basic-auth via a curl config fed through process substitution so the
 # credential never appears in argv (argv is visible to `ps` / process audit).
 bb_auth_cfg() { printf 'user = "%s:%s"\n' "$1" "$2"; }
@@ -49,13 +108,10 @@ fi
 # per-project directory ($HOME/.claude/logs/multi-agent/{project}/{taskId}/)
 # - not the ~/.claude/projects/*/state/ path this used to glob, which no
 # writer in the pipeline ever populates.
-AGENT_STATE=""
-for candidate in "$HOME"/.claude/logs/multi-agent/*/"$TASK_ID"/agent-state.json; do
-  if [ -f "$candidate" ]; then
-    AGENT_STATE="$candidate"
-    break
-  fi
-done
+# This globbed ONLY the nested layout. phase-tracker.sh and most Phase 0 paths
+# write flat, so on a real machine the majority of runs resolved to nothing and
+# this script reported "no agent-state.json" for a run whose state existed.
+AGENT_STATE="$(ma_resolve_run_file "$TASK_ID" agent-state.json 2>/dev/null || true)"
 
 if [ -z "$AGENT_STATE" ]; then
   echo "post-pr-review: agent-state.json not found for task=$TASK_ID" >&2
@@ -271,6 +327,13 @@ post_github() {
           continue
         fi
 
+        # A finding's body quotes the diff and the reviewer's prose, so it is
+        # the most likely of the three to carry something from the machine.
+        if ! ma_outbound_gate_text "$body"; then
+          echo "post-pr-review: outbound gate refused the comment for $path:$line - not posted" >&2
+          continue
+        fi
+
         local inline_ok=0
         if [ -n "$head_sha" ] && [ "$line" -gt 0 ] 2>/dev/null; then
           if gh api -X POST "/repos/$org_repo/pulls/$PR_NUM/comments" \
@@ -290,7 +353,13 @@ post_github() {
 
   case "$DECISION" in
     approve)    gh pr review "$PR_NUM" --repo "$org_repo" --approve ;;
-    needs_work) gh pr review "$PR_NUM" --repo "$org_repo" --request-changes --body "$rc_body" ;;
+    needs_work)
+      if ! ma_outbound_gate_text "$rc_body"; then
+        echo "post-pr-review: outbound gate refused the request-changes body; no verdict posted" >&2
+        return 7
+      fi
+      gh pr review "$PR_NUM" --repo "$org_repo" --request-changes --body "$rc_body"
+      ;;
   esac
 }
 

package/pipeline/lib/repo-hygiene.sh

@@ -141,11 +141,16 @@ ma_hygiene_local_gitignore() {
 # rmdir, never rm -rf: it fails harmlessly when anything is left, which is the
 # behaviour we want. An empty .worktrees/ and .pipeline/evidence/ survive a
 # clean run today and read as leftovers to anyone looking at the checkout.
+#
+# "Harmlessly" is only true while the CALLER does not run under `set -e`, and
+# this file is sourced, so that is the caller's choice and not ours. Every
+# rmdir here is therefore explicitly forgiven: a non-empty directory is the
+# ordinary case, and it must not abort a teardown half-way.
 ma_hygiene_prune_empty() {
   local root d
   root=$(ma_hygiene_safe_root "${1:-}") || return 0
   for d in "$root/.pipeline/evidence" "$root/.pipeline" "$root/.worktrees/.archive" "$root/.worktrees" "$root/.multi-agent/refs"; do
-    [ -d "$d" ] && rmdir "$d" 2>/dev/null
+    [ -d "$d" ] && { rmdir "$d" 2>/dev/null || true; }
   done
 
   # .multi-agent survives the loop above because it still holds the .gitignore
@@ -156,8 +161,8 @@ ma_hygiene_prune_empty() {
     local leftover
     leftover=$(ls -A "$root/.multi-agent" 2>/dev/null)
     if [ "$leftover" = ".gitignore" ]; then
-      rm -f "$root/.multi-agent/.gitignore" 2>/dev/null
-      rmdir "$root/.multi-agent" 2>/dev/null
+      rm -f "$root/.multi-agent/.gitignore" 2>/dev/null || true
+      rmdir "$root/.multi-agent" 2>/dev/null || true
     fi
   fi
   return 0

package/pipeline/lib/require-jq.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# require-jq.sh  -  one place to refuse when `jq` is missing.
+#
+# 78 shell files in this repo call `jq`. 46 of them called it with no check at
+# all, and the README described that as helpers which "skip in silence" - which
+# is the defect stated as if it were the design. A bare `jq` on a machine
+# without it does not skip. It writes "command not found" to stderr, produces
+# an empty string on stdout, and the caller carries on with that empty string
+# as if it were the answer.
+#
+# The nine production paths are where that matters:
+#
+#   autopilot-state.sh   an unreadable queue reads as an EMPTY queue, so the
+#                        runner concludes there is no work and goes quiet
+#   jira-publish.sh      posts a comment built from empty fields
+#   post-pr-review.sh    publishes a review whose findings did not parse
+#   update-issue-progress.sh  same, on an issue
+#   _jira-auth.sh        no credential, and the failure surfaces later as 401
+#   figma-mcp-refresh.sh / figma-screenshot.sh   silent no-op on a design fetch
+#   plan-todos.sh        an empty plan looks like a plan with no work in it
+#   search-logs.sh       "no results" instead of "could not search"
+#
+# Every one of those is the same shape: absence of a tool rendered as absence
+# of data. Refusing is the only honest answer, and the exit code is distinct
+# so a caller can tell "jq missing" from "the work failed".
+#
+# Usage:
+#   . "$(dirname "$0")/require-jq.sh"
+#   ma_require_jq "read the autopilot queue" || exit 3
+#
+# It returns rather than exits: this file is SOURCED, and a sourced file that
+# exits kills the caller's shell in ways the caller did not write down.
+
+# shellcheck disable=SC2329  # sourced by other scripts, not called here
+ma_require_jq() {
+  command -v jq >/dev/null 2>&1 && return 0
+  printf 'jq not found - cannot %s.\n' "${1:-continue}" >&2
+  printf '  install it: brew install jq\n' >&2
+  return 1
+}