@cxi-lmai/ci-agent-platform 3.0.0 → 3.1.0

package/payload/ci-templates/scripts/lib/pipeline-common.sh

@@ -9,8 +9,12 @@ set -u
 SCRIPT_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=./usage-capture.sh
 source "$SCRIPT_LIB_DIR/usage-capture.sh"
+# shellcheck source=./usage-capture-omp.sh
+source "$SCRIPT_LIB_DIR/usage-capture-omp.sh"
 # shellcheck source=./platform.sh
 source "$SCRIPT_LIB_DIR/platform.sh"
+# shellcheck source=./failure-notice.sh
+source "$SCRIPT_LIB_DIR/failure-notice.sh"
 
 # --- Defaults ---------------------------------------------------------------
 # Every PIPE_ variable has a default here so a project that overrides nothing
@@ -23,28 +27,87 @@ pipe_defaults() {
   : "${PIPE_CONFIG_PATH:=.claude/pipeline-config.md}"
   : "${PIPE_CONTEXT_DIR:=build/pipeline}"
   : "${PIPE_METRICS_DIR:=$PIPE_CONTEXT_DIR/metrics}"
+  : "${PIPE_AGENT_STDERR:=$PIPE_CONTEXT_DIR/agent-stderr.log}"
   : "${PIPE_TARGET_BRANCH:=main}"
   : "${PIPE_FIX_LOOP_CAP:=2}"
   : "${PIPE_CODER_CAP:=3}"
   : "${PIPE_ISSUE_SCAN:=20}"
-  : "${PIPE_MODEL_TRIAGE:=haiku}"
-  : "${PIPE_MODEL_CODE:=sonnet}"
-  : "${PIPE_MODEL_REVIEW:=sonnet}"
+  : "${PIPE_HARNESS:=claude}"
+  # Model defaults track the active harness. Each PIPE_MODEL_* variable remains
+  # explicitly overridable.
+  if [ "$PIPE_HARNESS" = "omp" ]; then
+    : "${PIPE_MODEL_TRIAGE:=openrouter/anthropic/claude-haiku-4-5}"
+    : "${PIPE_MODEL_CODE:=openrouter/anthropic/claude-sonnet-5-0}"
+    : "${PIPE_MODEL_REVIEW:=openrouter/anthropic/claude-sonnet-5-0}"
+  else
+    : "${PIPE_MODEL_TRIAGE:=haiku}"
+    : "${PIPE_MODEL_CODE:=claude-sonnet-5-0}"
+    : "${PIPE_MODEL_REVIEW:=claude-sonnet-5-0}"
+  fi
   : "${PIPE_COMMIT_TESTFIX:=Fix test errors}"
   : "${PIPE_COMMIT_REVIEWFIX:=Fix review findings}"
   : "${PIPE_COMMIT_COVERAGE:=Add coverage tests}"
+  : "${PIPE_COMMIT_DOCSSYNC:=Update docs per docs-sync findings}"
+  : "${PIPE_COMMIT_AUDIT:=docs: apply convention audit findings}"
+  : "${PIPE_COMMIT_CVEFIX:=Update dependencies to resolve CVEs}"
   : "${PIPE_LABEL_READY:=pipe-ready}"
   : "${PIPE_LABEL_WIP:=pipe-wip}"
   : "${PIPE_LABEL_STUCK:=pipe-stuck}"
   : "${PIPE_LABEL_BLOCKER:=pipe-blocker}"
   : "${PIPE_LABEL_BLOCKED:=pipe-blocked}"
   : "${PIPE_LABEL_DECOMPOSED:=pipe-decomposed}"
+  : "${PIPE_LABEL_AUDIT:=codebase-audit}"
   : "${PIPE_RESULT_REVIEW:=$PIPE_CONTEXT_DIR/code-review-result.txt}"
   : "${PIPE_RESULT_REVIEWFIX:=$PIPE_CONTEXT_DIR/review-fix-result.json}"
+  : "${PIPE_RESULT_DOCSSYNC:=$PIPE_CONTEXT_DIR/docs-sync.json}"
   : "${PIPE_VERIFY_CMD:=}"
   : "${PIPE_COMPILE_CMD:=${PIPE_VERIFY_CMD}}"
   : "${PIPE_TEST_REPORT_GLOB:=}"
+  : "${PIPE_COVERAGE_RATCHET:=0}"
+  : "${PIPE_COVERAGE_REPORT:=}"
+  : "${PIPE_COVERAGE_REPORT_KIND:=jacoco}"
   : "${PIPE_COVERAGE_SIGNAL:=}"
+  # CVE remediation. Off by default: project-local scanners provide the
+  # HIGH/CRITICAL identifier list when PIPE_CVE_FIX=1 enables this job.
+  : "${PIPE_CVE_FIX:=0}"
+  : "${PIPE_DEPENDENCY_MANIFEST:=build.gradle}"
+  : "${PIPE_CVE_SUPPRESSIONS:=owasp-suppressions.xml}"
+  # Issue-driven E2E generation. Off by default; PIPE_VERIFY_CMD is the
+  # configured verification command and must be non-empty before tests run.
+  : "${PIPE_E2E_TEST_GEN:=0}"
+  : "${PIPE_LABEL_TESTREADY:=test-ready}"
+  : "${PIPE_LABEL_E2E_SCOPE:=}"
+  : "${PIPE_E2E_TEST_DIR:=e2e/tests}"
+  : "${PIPE_E2E_BASE_URL:=}"
+  : "${PIPE_E2E_VERIFY_CMD:=}"
+  # Documentation sync. Off by default: the job only runs when the pipeline
+  # sets PIPE_DOCS_SYNC=1. PIPE_DOCS_ROOTS is the comma-separated set of paths
+  # the docs-sync agent may edit and the runner may commit.
+  : "${PIPE_DOCS_SYNC:=0}"
+  : "${PIPE_DOCS_ROOTS:=docs,CLAUDE.md,.claude/memory}"
+  # Codebase audit. Off by default: the job only runs on a schedule that sets
+  # PIPE_CODEBASE_AUDIT=1. Reuses PIPE_DOCS_ROOTS for the optional auto-fix
+  # phase's coder scope.
+  : "${PIPE_CODEBASE_AUDIT:=0}"
+  # Agent architect. Off by default: a scheduled pipeline explicitly enables
+  # this evidence-backed proposal job after a project has reviewed its output.
+  : "${PIPE_AGENT_ARCHITECT:=0}"
+  : "${PIPE_ARCHITECT_WINDOW_DAYS:=7}"
+  : "${PIPE_ARCHITECT_DRY_RUN:=true}"
+  : "${PIPE_LABEL_IMPROVEMENT:=pipe-improvement}"
+  # Metrics aggregation. Off by default: the job only runs on a schedule that
+  # sets PIPE_METRICS_SNAPSHOT=1, and it publishes an artifact unless the
+  # project explicitly opts in to committing generated data to its repository.
+  : "${PIPE_METRICS_SNAPSHOT:=0}"
+  : "${PIPE_METRICS_OUTPUT_DIR:=data/metrics/snapshots}"
+  : "${PIPE_METRICS_WINDOW_DAYS:=1}"
+  : "${PIPE_METRICS_COMMIT_REPO:=false}"
+  : "${PIPE_METRICS_REVIEW_AGENTS:=code-reviewer,review-verdict,review-fix}"
+  # Commit subjects that mark a defect in already-merged work. Keep in sync with
+  # the Metrics Conventions section of the pipeline config.
+  : "${PIPE_METRICS_DEFECT_REVERT:=Reverts !}"
+  : "${PIPE_METRICS_DEFECT_REGRESSION:=Fixes regression from !}"
+  : "${PIPE_METRICS_DEFECT_REINTRODUCE:=Reintroduces work from !}"
   # Spec template path. GitHub projects usually keep it under .github, GitLab
   # under .gitlab, so the default follows the detected platform.
   if [ "$PIPE_PLATFORM" = "github" ]; then
@@ -58,13 +121,21 @@ pipe_defaults() {
   # would pick up result files (E2E finding N-6). A `*` gitignore inside the
   # dir hides everything in it, including itself, from any git add.
   [ -f "$PIPE_CONTEXT_DIR/.gitignore" ] || printf '*\n' > "$PIPE_CONTEXT_DIR/.gitignore"
-  export PIPE_PLATFORM PIPE_CONFIG_PATH PIPE_CONTEXT_DIR PIPE_METRICS_DIR \
+  export PIPE_PLATFORM PIPE_CONFIG_PATH PIPE_CONTEXT_DIR PIPE_METRICS_DIR PIPE_AGENT_STDERR \
     PIPE_TARGET_BRANCH PIPE_FIX_LOOP_CAP PIPE_CODER_CAP PIPE_ISSUE_SCAN PIPE_COMMIT_TESTFIX \
-    PIPE_COMMIT_REVIEWFIX PIPE_COMMIT_COVERAGE PIPE_LABEL_READY PIPE_LABEL_WIP \
-    PIPE_LABEL_STUCK PIPE_LABEL_BLOCKER PIPE_LABEL_BLOCKED PIPE_LABEL_DECOMPOSED \
-    PIPE_RESULT_REVIEW PIPE_RESULT_REVIEWFIX PIPE_VERIFY_CMD PIPE_COMPILE_CMD \
-    PIPE_TEST_REPORT_GLOB PIPE_COVERAGE_SIGNAL PIPE_SPEC_TEMPLATE_PATH \
-    PIPE_MODEL_TRIAGE PIPE_MODEL_CODE PIPE_MODEL_REVIEW
+    PIPE_COMMIT_REVIEWFIX PIPE_COMMIT_COVERAGE PIPE_COMMIT_CVEFIX PIPE_LABEL_READY PIPE_LABEL_WIP \
+    PIPE_LABEL_STUCK PIPE_LABEL_BLOCKER PIPE_LABEL_BLOCKED PIPE_LABEL_DECOMPOSED PIPE_LABEL_AUDIT \
+    PIPE_RESULT_REVIEW PIPE_RESULT_REVIEWFIX PIPE_RESULT_DOCSSYNC PIPE_COMMIT_DOCSSYNC PIPE_COMMIT_AUDIT \
+    PIPE_CVE_FIX PIPE_DEPENDENCY_MANIFEST PIPE_CVE_SUPPRESSIONS PIPE_E2E_TEST_GEN PIPE_LABEL_TESTREADY \
+    PIPE_LABEL_E2E_SCOPE PIPE_E2E_TEST_DIR PIPE_E2E_BASE_URL PIPE_E2E_VERIFY_CMD PIPE_DOCS_SYNC PIPE_DOCS_ROOTS PIPE_VERIFY_CMD PIPE_COMPILE_CMD \
+    PIPE_TEST_REPORT_GLOB PIPE_COVERAGE_RATCHET PIPE_COVERAGE_REPORT PIPE_COVERAGE_REPORT_KIND \
+    PIPE_COVERAGE_SIGNAL PIPE_SPEC_TEMPLATE_PATH PIPE_CODEBASE_AUDIT PIPE_AGENT_ARCHITECT \
+    PIPE_ARCHITECT_WINDOW_DAYS PIPE_ARCHITECT_DRY_RUN PIPE_LABEL_IMPROVEMENT \
+    PIPE_METRICS_SNAPSHOT PIPE_METRICS_OUTPUT_DIR PIPE_METRICS_WINDOW_DAYS \
+    PIPE_METRICS_COMMIT_REPO PIPE_METRICS_REVIEW_AGENTS PIPE_METRICS_DEFECT_REVERT \
+    PIPE_METRICS_DEFECT_REGRESSION PIPE_METRICS_DEFECT_REINTRODUCE \
+    PIPE_METRICS_SNAPSHOT_DATE PIPE_METRICS_RAW_DIR PIPE_METRICS_MRS_FILE \
+    PIPE_HARNESS PIPE_MODEL_TRIAGE PIPE_MODEL_CODE PIPE_MODEL_REVIEW
 }
 
 pipe_log() { echo "[pipeline] $*"; }
@@ -88,6 +159,17 @@ pipe_git_identity() {
 # Checkout the real source branch so commits land on the branch tip, not on the
 # detached MR/PR ref that can fall behind and cause non-fast-forward push errors.
 pipe_checkout_source() {
+  # The MR/PR itself is the authoritative source of the branch name. A
+  # workflow_run or branch pipeline carries no merge-request source ref, and
+  # without one this checkout used to no-op silently, leaving the fix commit on
+  # a detached head that no push could land.
+  if [ -z "${PIPE_SOURCE_BRANCH:-}" ] && [ -n "${PIPE_MR_IID:-}" ]; then
+    # `|| PIPE_SOURCE_BRANCH=""` is the whole error path: an unreachable or
+    # unconfigured API must leave the caller where it was, not abort it.
+    PIPE_SOURCE_BRANCH=$(platform_mr_meta 2>/dev/null | jq -r '.source_branch // ""' 2>/dev/null) \
+      || PIPE_SOURCE_BRANCH=""
+    export PIPE_SOURCE_BRANCH
+  fi
   [ -z "${PIPE_SOURCE_BRANCH:-}" ] && return 0
   git fetch origin "$PIPE_TARGET_BRANCH" "$PIPE_SOURCE_BRANCH" 2>/dev/null || true
   git checkout -B "$PIPE_SOURCE_BRANCH" "origin/$PIPE_SOURCE_BRANCH" 2>/dev/null || true
@@ -106,15 +188,17 @@ pipe_rebase_and_push() {
 }
 
 # --- Cap counting -----------------------------------------------------------
-# Count consecutive commits from the tip back toward the target branch whose
-# subject matches a pattern. The skill also computes this, but the runner needs
-# it to decide whether the agent actually committed this run.
-
+# Count consecutive matching commits from the tip back toward the target branch.
+# Docs-sync commits are transparent: they do not count, but do not interrupt a
+# fix streak. The skills use the same exact-subject rules for their cap checks;
+# the runner needs this count only to detect whether a fix commit landed.
 pipe_count_fix_commits() {
   local pattern="$1" count=0 subject
   while IFS= read -r subject; do
-    if echo "$subject" | grep -qF "$pattern"; then
+    if [ "$subject" = "$pattern" ]; then
       count=$((count + 1))
+    elif [ "$subject" = "$PIPE_COMMIT_DOCSSYNC" ]; then
+      continue
     else
       break
     fi
@@ -131,14 +215,22 @@ pipe_count_fix_commits() {
 
 pipe_agent_secret_var() {
   # Return success when an exported variable looks credential-bearing and is
-  # not explicitly required by the Claude API itself or allowlisted for a
-  # project verification command. The allowlist is comma/space separated.
+  # not explicitly required by the active harness's own auth or allowlisted
+  # for a project verification command. The allowlist is comma/space
+  # separated.
   local name="$1" allow=" ${PIPE_AGENT_ENV_ALLOWLIST:-} "
   allow=" ${allow//,/ } "
-  # The Claude process must keep its own credential regardless of which of the
-  # three supported auth variables the project uses.
-  case "$name" in
-    ANTHROPIC_API_KEY|ANTHROPIC_AUTH_TOKEN|CLAUDE_CODE_OAUTH_TOKEN) return 1 ;;
+  # Preserve only the active harness's built-in model credentials. Credentials
+  # for the inactive harness fall through to the generic secret-name check.
+  case "${PIPE_HARNESS:-claude}" in
+    omp)
+      [ "$name" = "OPENROUTER_API_KEY" ] && return 1
+      ;;
+    *)
+      case "$name" in
+        ANTHROPIC_API_KEY|ANTHROPIC_AUTH_TOKEN|CLAUDE_CODE_OAUTH_TOKEN) return 1 ;;
+      esac
+      ;;
   esac
   [[ "$allow" == *" $name "* ]] && return 1
   case "$name" in
@@ -213,6 +305,115 @@ pipe_run_claude() {
   rm -f "$stdin_file"
 }
 
+# --- omp invocation -----------------------------------------------------
+# Runs `omp -p "/<skill>"` wrapped in usage-capture, the omp equivalent of
+# pipe_run_claude. Same non-root/su-user dance (unverified whether omp
+# tolerates root under --yolo; the same-user fallback Claude Code needs is
+# always safe, so it ships unconditionally either way).
+
+pipe_translate_tools_to_omp() {
+  # $1 = Claude Code-style comma-separated tool list, e.g.
+  # "Agent,Read,Write,Edit,Glob,Grep,Bash". Echoes the omp-equivalent
+  # comma-separated list, deduplicated and order-preserving. Tokens with no
+  # omp equivalent (LS, NotebookRead, Skill, KillShell, BashOutput) are
+  # dropped; mcp__* and any unrecognized token pass through unchanged. See
+  # the translation table in
+  # docs/superpowers/specs/2026-08-26-omp-openrouter-harness-design.md.
+  local tools="$1" tok mapped seen="," result=""
+  IFS=',' read -ra tokens <<< "$tools"
+  for tok in "${tokens[@]}"; do
+    case "$tok" in
+      Agent) mapped=task ;;
+      Bash) mapped=bash ;;
+      Edit|MultiEdit) mapped=edit ;;
+      Glob) mapped=glob ;;
+      Grep) mapped="grep" ;;
+      Read|WebFetch) mapped="read" ;;
+      Write) mapped="write" ;;
+      TodoWrite) mapped=todo ;;
+      WebSearch) mapped=web_search ;;
+      LS|NotebookRead|Skill|KillShell|BashOutput) continue ;;
+      *) mapped="$tok" ;;
+    esac
+    case "$seen" in *",$mapped,"*) continue ;; esac
+    seen="$seen$mapped,"
+    result="$result,$mapped"
+  done
+  echo "${result#,}"
+}
+
+pipe_run_omp() {
+  # $1 = agent label for metrics, $2 = skill invocation (e.g. "/review-mr"),
+  # $3 = allowed tools in Claude Code naming (optional, translated
+  # internally), $4 = model for the main loop (optional, e.g.
+  # $PIPE_MODEL_REVIEW; subagents keep the models from their own
+  # frontmatter), stdin = extra prompt (optional, usually empty).
+  local label="$1" skill="$2" tools="${3:-Agent,Read,Write,Edit,Glob,Grep,Bash}" model="${4:-}"
+  local omp_tools; omp_tools=$(pipe_translate_tools_to_omp "$tools")
+  local stdin_file; stdin_file=$(mktemp)
+  cat > "$stdin_file" || true
+
+  if [ "$(id -u)" -eq 0 ]; then
+    local runuser="pipelinebot"
+    id "$runuser" &>/dev/null || useradd -m -s /bin/bash "$runuser"
+    chown -R "$runuser:$runuser" "$PWD" "$PIPE_CONTEXT_DIR" "$stdin_file" 2>/dev/null || true
+    local runner; runner=$(mktemp)
+    {
+      printf '#!/bin/bash\nset -e\nexport HOME=/home/%s\n' "$runuser"
+      # Re-export the runtime environment the skill and usage-capture need.
+      local v
+      for v in $(compgen -v | grep -E '^(PIPE_|CI_|GH_|GITHUB_|OPENROUTER_)'); do
+        pipe_agent_secret_var "$v" && continue
+        printf 'export %s=%q\n' "$v" "${!v}"
+      done
+      # `su -m` preserves the parent environment, so explicitly remove every
+      # detected credential after exporting the safe runtime values above.
+      for v in $(compgen -e); do
+        pipe_agent_secret_var "$v" && printf 'unset %q\n' "$v"
+      done
+      printf 'cd %q\n' "$PWD"
+      printf 'source %q\n' "$SCRIPT_LIB_DIR/usage-capture-omp.sh"
+      printf 'export PIPE_CAPTURE_MODEL=%q\n' "$model"
+      if [ -n "$model" ]; then
+        printf 'capture_omp %q -- --yolo --no-session --no-title --model %q --tools %q -p %q < %q\n' \
+          "$label" "$model" "$omp_tools" "$skill" "$stdin_file"
+      else
+        printf 'capture_omp %q -- --yolo --no-session --no-title --tools %q -p %q < %q\n' \
+          "$label" "$omp_tools" "$skill" "$stdin_file"
+      fi
+    } > "$runner"
+    chmod +x "$runner"
+    chown "$runuser:$runuser" "$runner"
+    su -m "$runuser" -s /bin/bash "$runner" || true
+    chown -R root:root "$PWD" 2>/dev/null || true
+    rm -f "$runner"
+  else
+    (
+      pipe_scrub_agent_secrets
+      export PIPE_CAPTURE_MODEL="$model"
+      if [ -n "$model" ]; then
+        capture_omp "$label" -- --yolo --no-session --no-title --model "$model" --tools "$omp_tools" -p "$skill" < "$stdin_file"
+      else
+        capture_omp "$label" -- --yolo --no-session --no-title --tools "$omp_tools" -p "$skill" < "$stdin_file"
+      fi
+    ) || true
+  fi
+  rm -f "$stdin_file"
+}
+
+# --- Harness dispatch ---------------------------------------------------
+# Single entry point every job script calls. Same signature/contract as
+# pipe_run_claude/pipe_run_omp themselves: $1 label, $2 skill, $3 tools
+# (optional, Claude Code naming, translated internally when the target is
+# omp), $4 model (optional).
+
+pipe_run_agent() {
+  case "${PIPE_HARNESS:-claude}" in
+    omp) pipe_run_omp "$@" ;;
+    *) pipe_run_claude "$@" ;;
+  esac
+}
+
 # --- dotenv -----------------------------------------------------------------
 
 # Load KEY=VALUE lines from a marker file into the current shell.
@@ -240,23 +441,24 @@ pipe_get_env() {
 pipe_apply_escalation() {
   local pm_env="$PIPE_CONTEXT_DIR/postmortem.env"
   local pm_body="$PIPE_CONTEXT_DIR/postmortem.md"
-  local add remove category diag
+  local add remove category comment_file
   add=$(pipe_get_env "$pm_env" ADD_LABELS);       add=${add:-$PIPE_LABEL_STUCK}
   remove=$(pipe_get_env "$pm_env" REMOVE_LABELS); remove=${remove:-$PIPE_LABEL_WIP}
   category=$(pipe_get_env "$pm_env" FAILURE_CATEGORY); category=${category:-${1:-other}}
 
   platform_set_labels "$add" "$remove"
 
-  if [ -s "$pm_body" ]; then
-    diag=$(cat "$pm_body")
-  else
-    diag="Automated fixes were exhausted after $PIPE_FIX_LOOP_CAP attempts. A human needs to take over."
-  fi
-  platform_post_comment "🤖 **Auto-fix stuck after $PIPE_FIX_LOOP_CAP attempts. Diagnostic analysis:**
-
-$diag
-
-Human intervention needed."
+  comment_file="$PIPE_CONTEXT_DIR/auto-fix-stuck.md"
+  {
+    printf '🤖 **Auto-fix stuck after %s attempts. Diagnostic analysis:**\n\n' "$PIPE_FIX_LOOP_CAP"
+    if [ -s "$pm_body" ]; then
+      cat "$pm_body"
+    else
+      printf 'Automated fixes were exhausted after %s attempts. A human needs to take over.' "$PIPE_FIX_LOOP_CAP"
+    fi
+    printf '\n\nHuman intervention needed.\n'
+  } > "$comment_file"
+  platform_post_comment_file "$comment_file"
 
   pipe_metric_event postmortem dev_stuck \
     "$(jq -n --arg c "$category" --arg ctx "${2:-unknown}" '{failure_category: $c, context: $ctx}')"