@cxi-lmai/ci-agent-platform 3.0.1 → 3.1.0

package/payload/ci-templates/scripts/lib/platform.sh

@@ -8,7 +8,8 @@
 #   Common (both platforms):
 #     PIPE_MR_IID          MR iid (GitLab) or PR number (GitHub)
 #     PIPE_TARGET_BRANCH   branch the MR/PR targets
-#     PIPE_SOURCE_BRANCH   the MR/PR source branch
+#     PIPE_SOURCE_BRANCH   the MR/PR source branch (optional: platform_mr_meta
+#                          reports it as `source_branch` when the runtime does not)
 #     PIPE_HEAD_SHA        current head commit under review
 #     PIPE_PROJECT_PATH    namespace/project (for git push URL)
 #     PIPE_SERVER_HOST     git host (for git push URL)
@@ -58,7 +59,10 @@ _gl_get() {
     "$PIPE_API_URL/projects/$PIPE_PROJECT_ID/merge_requests/$PIPE_MR_IID$1"
 }
 
-_gl_mr_meta()   { _gl_get "" | jq -r '{title: (.title // ""), description: (.description // ""), labels: ((.labels // []) | join(","))}'; }
+# `source_branch` is the branch the MR/PR was opened from. It comes from the API
+# rather than the CI runtime because branch and workflow_run pipelines carry no
+# merge-request source ref, and a runner that pushes needs it.
+_gl_mr_meta()   { _gl_get "" | jq -r '{title: (.title // ""), description: (.description // ""), labels: ((.labels // []) | join(",")), source_branch: (.source_branch // "")}'; }
 _gl_mr_labels() { _gl_get "" | jq -r '((.labels // []) | join(","))'; }
 # Callers take the LAST matching note, so the array must be chronological AND
 # must contain the newest notes. One asc page would return the oldest 100 on a
@@ -66,11 +70,12 @@ _gl_mr_labels() { _gl_get "" | jq -r '((.labels // []) | join(","))'; }
 _gl_notes()     { _gl_get "/notes?per_page=100&sort=desc" | jq -r '[.[] | {author: .author.username, body: .body}] | reverse'; }
 _gl_issue()     { curl -s -H "PRIVATE-TOKEN: $PIPE_TOKEN" "$PIPE_API_URL/projects/$PIPE_PROJECT_ID/issues/$1" | jq -r '{iid: .iid, title: (.title // ""), description: (.description // "")}'; }
 
-_gl_post_comment() {
+_gl_post_comment_file() {
+  # $1 = body file under PIPE_CONTEXT_DIR.
   curl --fail-with-body -s -X POST \
     -H "PRIVATE-TOKEN: $PIPE_TOKEN" \
     "$PIPE_API_URL/projects/$PIPE_PROJECT_ID/merge_requests/$PIPE_MR_IID/notes" \
-    --data-urlencode "body=$1" > /dev/null
+    --data-urlencode "body@$1" > /dev/null
 }
 
 _gl_set_labels() {
@@ -102,15 +107,22 @@ _gl_compute_diff_base() {
 
 _gh_get() { gh api "repos/$PIPE_REPO/$1"; }
 
-_gh_mr_meta()   { _gh_get "pulls/$PIPE_MR_IID" | jq -r '{title: (.title // ""), description: (.body // ""), labels: ((.labels // []) | map(.name) | join(","))}'; }
+_gh_mr_meta()   { _gh_get "pulls/$PIPE_MR_IID" | jq -r '{title: (.title // ""), description: (.body // ""), labels: ((.labels // []) | map(.name) | join(",")), source_branch: (.head.ref // "")}'; }
 _gh_mr_labels() { _gh_get "pulls/$PIPE_MR_IID" | jq -r '((.labels // []) | map(.name) | join(","))'; }
 # The per-issue comments endpoint has no sort param, so paginate all pages to
 # guarantee the newest comments are present (callers take the last match).
 _gh_notes()     { gh api --paginate "repos/$PIPE_REPO/issues/$PIPE_MR_IID/comments?per_page=100" --jq '.[] | {author: .user.login, body: .body}' | jq -s '.'; }
 _gh_issue()     { _gh_get "issues/$1" | jq -r '{iid: .number, title: (.title // ""), description: (.body // "")}'; }
 
-_gh_post_comment() {
-  gh api -X POST "repos/$PIPE_REPO/issues/$PIPE_MR_IID/comments" -f "body=$1" > /dev/null
+_gh_post_comment_file() {
+  # $1 = body file under PIPE_CONTEXT_DIR.
+  local payload status=0
+  payload=$(mktemp "$PIPE_CONTEXT_DIR/github-comment.XXXXXX.json")
+  jq -n --rawfile body "$1" '{body:$body}' > "$payload"
+  gh api -X POST "repos/$PIPE_REPO/issues/$PIPE_MR_IID/comments" \
+    -H "Content-Type: application/json" --input "$payload" > /dev/null || status=$?
+  rm -f "$payload"
+  return "$status"
 }
 
 _gh_set_labels() {
@@ -135,6 +147,179 @@ _gh_compute_diff_base() {
   fi
 }
 
+# --- Pipeline and artifact helpers ------------------------------------------
+
+_gl_completed_pipelines() {
+  local updated_after
+  updated_after=$(date -u -d "$1 days ago" '+%Y-%m-%dT%H:%M:%SZ') || return 1
+  curl -s --fail -H "PRIVATE-TOKEN: $PIPE_TOKEN" \
+    "$PIPE_API_URL/projects/$PIPE_PROJECT_ID/pipelines?scope=finished&updated_after=$updated_after&per_page=100" |
+    jq -c '[.[] | {id, status: (.status // ""), ref: (.ref // ""), updated_at: (.updated_at // "")}]'
+}
+
+_gl_pipeline_artifacts() {
+  curl -s --fail -H "PRIVATE-TOKEN: $PIPE_TOKEN" \
+    "$PIPE_API_URL/projects/$PIPE_PROJECT_ID/pipelines/$1/jobs?per_page=100" |
+    jq -c '[.[] | {job_id: .id, job_name: (.name // ""), artifact_available: (((.artifacts_file // {}) | .filename) != null)}]'
+}
+
+_gl_download_artifact() {
+  curl -sSL --fail -H "PRIVATE-TOKEN: $PIPE_TOKEN" \
+    "$PIPE_API_URL/projects/$PIPE_PROJECT_ID/jobs/$1/artifacts" -o "$2"
+}
+
+_gl_latest_success_coverage() {
+  curl -s --fail -H "PRIVATE-TOKEN: $PIPE_TOKEN" \
+    "$PIPE_API_URL/projects/$PIPE_PROJECT_ID/pipelines?ref=$1&status=success&per_page=1" |
+    jq -r '.[0].coverage // ""'
+}
+
+_gl_merge_requests_updated_since() {
+  local page response page_size results
+  page=1
+  results='[]'
+  while :; do
+    response=$(curl -s --fail -H "PRIVATE-TOKEN: $PIPE_TOKEN" \
+      "$PIPE_API_URL/projects/$PIPE_PROJECT_ID/merge_requests?updated_after=$1&per_page=100&page=$page&scope=all") || return 1
+    page_size=$(jq -r 'length' <<< "$response") || return 1
+    results=$(jq -c --argjson results "$results" \
+      '$results + [.[] | {iid, title: (.title // ""), state: (.state // ""), author: (.author.username // ""), labels: (.labels // []), created_at: (.created_at // ""), merged_at: .merged_at, target_branch: (.target_branch // ""), description: (.description // "")}]' <<< "$response") || return 1
+    [ "$page_size" -lt 100 ] && break
+    page=$((page + 1))
+  done
+  printf '%s\n' "$results"
+}
+
+# Merged MRs targeting a branch, newest first, filtered to iid > since_iid.
+# Used by the codebase-audit job to find work since the last audit.
+_gl_merged_mrs() {
+  # $1=target_branch $2=since_iid $3=limit. One page, newest merged first;
+  # matches the source script's own non-paginated fetch. Captured into a
+  # variable (not piped straight into jq) so a curl failure propagates: a
+  # pipeline's exit status is jq's, not curl's, without `set -o pipefail`.
+  local response
+  response=$(curl -s --fail -H "PRIVATE-TOKEN: $PIPE_TOKEN" \
+    "$PIPE_API_URL/projects/$PIPE_PROJECT_ID/merge_requests?target_branch=$1&state=merged&per_page=$3&order_by=created_at&sort=desc") || return 1
+  jq -c --argjson since "$2" '[.[] | select(.iid > $since) | {iid, title: (.title // ""), created_at: (.created_at // "")}]' <<< "$response"
+}
+
+_gl_mr_changes() {
+  # $1=iid $2=limit. Changed file paths for one merged MR, capped at limit.
+  local response
+  response=$(curl -s --fail -H "PRIVATE-TOKEN: $PIPE_TOKEN" \
+    "$PIPE_API_URL/projects/$PIPE_PROJECT_ID/merge_requests/$1/changes?per_page=$2") || return 1
+  jq -r --argjson limit "$2" '[.changes[]?.new_path] | .[0:$limit][]' <<< "$response" 2>/dev/null
+}
+
+# $1=iid $2=limit $3=sort("asc"|"desc"). Normalized, non-system MR notes.
+_gl_mr_notes() {
+  local response
+  [[ "$2" =~ ^[1-9][0-9]*$ ]] || return 1
+  [[ "$3" = "asc" || "$3" = "desc" ]] || return 1
+  response=$(curl -s --fail -H "PRIVATE-TOKEN: $PIPE_TOKEN" \
+    "$PIPE_API_URL/projects/$PIPE_PROJECT_ID/merge_requests/$1/notes?per_page=$2&sort=$3") || return 1
+  jq --argjson limit "$2" '[.[] | select(.system != true)
+    | {author: (.author.username // ""), body: (.body // "")}] | .[0:$limit]' <<< "$response"
+}
+
+# These APIs return downloadable binary responses as well as JSON. Use curl
+# rather than gh so artifact downloads remain a silent, status-only operation.
+_gh_pipeline_api() {
+  local token="${GH_TOKEN:-${GITHUB_TOKEN:-}}"
+  curl -s --fail \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer $token" \
+    "${GITHUB_API_URL:-https://api.github.com}/repos/$PIPE_REPO/$1"
+}
+
+_gh_completed_pipelines() {
+  local updated_after page response page_size results
+  updated_after=$(date -u -d "$1 days ago" '+%Y-%m-%dT%H:%M:%SZ') || return 1
+  page=1
+  results='[]'
+  while :; do
+    response=$(_gh_pipeline_api "actions/runs?status=completed&per_page=100&page=$page") || return 1
+    page_size=$(jq -r '(.workflow_runs // []) | length' <<< "$response") || return 1
+    results=$(jq -c --argjson results "$results" --arg updated_after "$updated_after" \
+      '$results + [.workflow_runs[] | select((.updated_at // "") >= $updated_after) | {id, status: (.conclusion // .status // ""), ref: (.head_branch // ""), updated_at: (.updated_at // "")}]' <<< "$response") || return 1
+    [ "$page_size" -lt 100 ] && break
+    page=$((page + 1))
+  done
+  printf '%s\n' "$results"
+}
+
+_gh_pipeline_artifacts() {
+  _gh_pipeline_api "actions/runs/$1/artifacts?per_page=100" |
+    jq -c '[.artifacts[] | {job_id: .id, job_name: (.name // ""), artifact_available: ((.expired // true) | not)}]'
+}
+
+_gh_download_artifact() {
+  local token="${GH_TOKEN:-${GITHUB_TOKEN:-}}"
+  curl -sSL --fail \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer $token" \
+    "${GITHUB_API_URL:-https://api.github.com}/repos/$PIPE_REPO/actions/artifacts/$1/zip" -o "$2"
+}
+
+_gh_latest_success_coverage() {
+  _gh_pipeline_api "actions/runs?branch=$1&status=success&per_page=1" |
+    jq -r '.workflow_runs[0].coverage // ""'
+}
+
+# The pull-request API has no updated-since filter, so the newest-first listing
+# is walked until a page carries nothing inside the window.
+_gh_merge_requests_updated_since() {
+  local page response page_size kept kept_size results
+  page=1
+  results='[]'
+  while :; do
+    response=$(_gh_pipeline_api "pulls?state=all&sort=updated&direction=desc&per_page=100&page=$page") || return 1
+    page_size=$(jq -r 'length' <<< "$response") || return 1
+    kept=$(jq -c --arg since "$1" \
+      '[.[] | select((.updated_at // "") >= $since) | {iid: .number, title: (.title // ""), state: (if .merged_at then "merged" else (.state // "") end), author: (.user.login // ""), labels: [(.labels // [])[].name], created_at: (.created_at // ""), merged_at: .merged_at, target_branch: (.base.ref // ""), description: (.body // "")}]' <<< "$response") || return 1
+    kept_size=$(jq -r 'length' <<< "$kept") || return 1
+    results=$(jq -nc --argjson results "$results" --argjson kept "$kept" '$results + $kept') || return 1
+    [ "$kept_size" -lt "$page_size" ] && break
+    [ "$page_size" -lt 100 ] && break
+    page=$((page + 1))
+  done
+  printf '%s\n' "$results"
+}
+
+# Merged PRs targeting a branch, newest first, filtered to number > since_iid.
+# GitHub's closed-PR listing has no server-side merged-only filter, so
+# merged_at != null is applied client-side.
+_gh_merged_mrs() {
+  # $1=target_branch $2=since_iid $3=limit.
+  local response
+  response=$(_gh_pipeline_api "pulls?base=$1&state=closed&per_page=$3&sort=created&direction=desc") || return 1
+  jq -c --argjson since "$2" '[.[] | select(.merged_at != null and .number > $since) | {iid: .number, title: (.title // ""), created_at: (.created_at // "")}]' <<< "$response"
+}
+
+_gh_mr_changes() {
+  # $1=iid $2=limit. Changed file paths for one merged PR, capped at limit.
+  local response
+  response=$(_gh_pipeline_api "pulls/$1/files?per_page=$2") || return 1
+  jq -r --argjson limit "$2" '[.[]?.filename] | .[0:$limit][]' <<< "$response" 2>/dev/null
+}
+
+# $1=iid $2=limit $3=sort("asc"|"desc"). GitHub returns comments oldest
+# first, so descend requests paginate before reversing and slicing to preserve
+# real newest-first semantics rather than reversing an arbitrary first page.
+_gh_mr_notes() {
+  local response filter
+  [[ "$2" =~ ^[1-9][0-9]*$ ]] || return 1
+  [[ "$3" = "asc" || "$3" = "desc" ]] || return 1
+  if [ "$3" = "asc" ]; then
+    response=$(gh api "repos/$PIPE_REPO/issues/$1/comments?per_page=$2") || return 1
+  else
+    response=$(gh api --paginate "repos/$PIPE_REPO/issues/$1/comments?per_page=100") || return 1
+  fi
+  filter='[.[] | {author: (.user.login // ""), body: (.body // "")}]'
+  [ "$3" = "desc" ] && filter="$filter | reverse"
+  jq --argjson limit "$2" "$filter | .[0:\$limit]" <<< "$response"
+}
+
 # --- Bot identity -------------------------------------------------------------
 # The scripts filter MR/PR comments by the bot's username to recognise their own
 # previous output. The token already knows whose it is, so an empty
@@ -155,12 +340,84 @@ platform_resolve_bot_user() {
 
 # --- Public dispatch --------------------------------------------------------
 
-platform_mr_meta()      { case "$PIPE_PLATFORM" in gitlab) _gl_mr_meta ;; github) _gh_mr_meta ;; esac; }
-platform_mr_labels()    { case "$PIPE_PLATFORM" in gitlab) _gl_mr_labels ;; github) _gh_mr_labels ;; esac; }
-platform_notes()        { case "$PIPE_PLATFORM" in gitlab) _gl_notes ;; github) _gh_notes ;; esac; }
-platform_issue()        { case "$PIPE_PLATFORM" in gitlab) _gl_issue "$1" ;; github) _gh_issue "$1" ;; esac; }
-platform_post_comment() { case "$PIPE_PLATFORM" in gitlab) _gl_post_comment "$1" ;; github) _gh_post_comment "$1" ;; esac; }
-platform_set_labels()   { case "$PIPE_PLATFORM" in gitlab) _gl_set_labels "$1" "$2" ;; github) _gh_set_labels "$1" "$2" ;; esac; }
+platform_mr_meta()           { case "$PIPE_PLATFORM" in gitlab) _gl_mr_meta ;; github) _gh_mr_meta ;; esac; }
+platform_mr_labels()         { case "$PIPE_PLATFORM" in gitlab) _gl_mr_labels ;; github) _gh_mr_labels ;; esac; }
+platform_notes()             { case "$PIPE_PLATFORM" in gitlab) _gl_notes ;; github) _gh_notes ;; esac; }
+platform_issue()             { case "$PIPE_PLATFORM" in gitlab) _gl_issue "$1" ;; github) _gh_issue "$1" ;; esac; }
+platform_post_comment_file() { case "$PIPE_PLATFORM" in gitlab) _gl_post_comment_file "$1" ;; github) _gh_post_comment_file "$1" ;; esac; }
+platform_set_labels()        { case "$PIPE_PLATFORM" in gitlab) _gl_set_labels "$1" "$2" ;; github) _gh_set_labels "$1" "$2" ;; esac; }
+
+platform_completed_pipelines() {
+  [ "$#" -eq 1 ] || return 1
+  case "$PIPE_PLATFORM" in
+    gitlab) _gl_completed_pipelines "$1" ;;
+    github) _gh_completed_pipelines "$1" ;;
+    *) return 1 ;;
+  esac
+}
+
+platform_pipeline_artifacts() {
+  [ "$#" -eq 1 ] || return 1
+  case "$PIPE_PLATFORM" in
+    gitlab) _gl_pipeline_artifacts "$1" ;;
+    github) _gh_pipeline_artifacts "$1" ;;
+    *) return 1 ;;
+  esac
+}
+
+platform_download_artifact() {
+  [ "$#" -eq 2 ] || return 1
+  case "$PIPE_PLATFORM" in
+    gitlab) _gl_download_artifact "$1" "$2" ;;
+    github) _gh_download_artifact "$1" "$2" ;;
+    *) return 1 ;;
+  esac
+}
+
+platform_latest_success_coverage() {
+  [ "$#" -eq 1 ] || return 1
+  case "$PIPE_PLATFORM" in
+    gitlab) _gl_latest_success_coverage "$1" ;;
+    github) _gh_latest_success_coverage "$1" ;;
+    *) return 1 ;;
+  esac
+}
+
+platform_merge_requests_updated_since() {
+  [ "$#" -eq 1 ] || return 1
+  case "$PIPE_PLATFORM" in
+    gitlab) _gl_merge_requests_updated_since "$1" ;;
+    github) _gh_merge_requests_updated_since "$1" ;;
+    *) return 1 ;;
+  esac
+}
+
+platform_merged_mrs() {
+  [ "$#" -eq 3 ] || return 1
+  case "$PIPE_PLATFORM" in
+    gitlab) _gl_merged_mrs "$1" "$2" "$3" ;;
+    github) _gh_merged_mrs "$1" "$2" "$3" ;;
+    *) return 1 ;;
+  esac
+}
+
+platform_mr_changes() {
+  [ "$#" -eq 2 ] || return 1
+  case "$PIPE_PLATFORM" in
+    gitlab) _gl_mr_changes "$1" "$2" ;;
+    github) _gh_mr_changes "$1" "$2" ;;
+    *) return 1 ;;
+  esac
+}
+
+platform_mr_notes() {
+  [ "$#" -eq 3 ] || return 1
+  case "$PIPE_PLATFORM" in
+    gitlab) _gl_mr_notes "$1" "$2" "$3" ;;
+    github) _gh_mr_notes "$1" "$2" "$3" ;;
+    *) return 1 ;;
+  esac
+}
 
 # Sets PIPE_DIFF_BASE and PIPE_IS_INCREMENTAL. Call before building the diff.
 platform_compute_diff_base() {

package/payload/ci-templates/scripts/lib/usage-capture-omp.sh

@@ -110,7 +110,13 @@ capture_omp() {
       omp_exit_code: $exit_code
     }' > "$record_file"
 
-  # If omp failed, forward stderr so debugging still works.
+  # If the agent failed, forward stderr so debugging still works, and keep a
+  # durable copy: the failure detector runs after the invocation returns and
+  # cannot read a temp file the wrapper already deleted.
+  if [ -n "${PIPE_AGENT_STDERR:-}" ]; then
+    mkdir -p "$(dirname "$PIPE_AGENT_STDERR")" 2>/dev/null || true
+    cp "$stderr_file" "$PIPE_AGENT_STDERR" 2>/dev/null || true
+  fi
   [ "$exit_code" -ne 0 ] && cat "$stderr_file" >&2
   rm -f "$stream_file" "$stderr_file"
   return "$exit_code"

package/payload/ci-templates/scripts/lib/usage-capture.sh

@@ -103,7 +103,13 @@ capture_claude() {
       claude_exit_code: $exit_code
     }' > "$record_file"
 
-  # If Claude failed, forward stderr so debugging still works
+  # If the agent failed, forward stderr so debugging still works, and keep a
+  # durable copy: the failure detector runs after the invocation returns and
+  # cannot read a temp file the wrapper already deleted.
+  if [ -n "${PIPE_AGENT_STDERR:-}" ]; then
+    mkdir -p "$(dirname "$PIPE_AGENT_STDERR")" 2>/dev/null || true
+    cp "$stderr_file" "$PIPE_AGENT_STDERR" 2>/dev/null || true
+  fi
   [ "$exit_code" -ne 0 ] && cat "$stderr_file" >&2
   rm -f "$stream_file" "$stderr_file"
   return "$exit_code"