@cxi-lmai/ci-agent-platform 3.0.0 → 3.1.0

package/payload/ci-templates/scripts/lib/issue-loop.sh

@@ -52,6 +52,27 @@ _gli_issues_by_label() {
                   milestone_id: (.milestone.id // null)}]'
 }
 
+# $1=primary label $2=optional secondary label $3=state $4=limit. Unlike the
+# older single-label helpers this preserves fetch failure so callers never
+# mistake a forge outage for an empty work queue.
+_gli_issues_by_labels() {
+  local primary="$1" secondary="$2" state="$3" limit="$4" labels response state_param=""
+  [[ "$limit" =~ ^[1-9][0-9]*$ ]] || return 1
+  case "$state" in
+    all|'') ;;
+    opened|closed) state_param="&state=$state" ;;
+    *) return 1 ;;
+  esac
+  labels=$(printf '%s' "$primary${secondary:+,$secondary}" | jq -sRr @uri)
+  response=$(curl -s --fail -H "PRIVATE-TOKEN: $PIPE_TOKEN" \
+    "$(_gli_api)/issues?labels=$labels${state_param}&per_page=$limit") || return 1
+  jq --arg primary "$primary" --arg secondary "$secondary" --argjson limit "$limit" \
+    '[.[] | select((.labels // []) | index($primary))
+           | select($secondary == "" or ((.labels // []) | index($secondary)))
+           | {iid, title: (.title // ""), description: (.description // ""),
+              labels: ((.labels // []) | join(","))}] | .[0:$limit]' <<< "$response"
+}
+
 _gli_issues_closed_by_label() {
   # $1=label. Closed issues still carrying a lifecycle label (janitor input).
   curl -s -H "PRIVATE-TOKEN: $PIPE_TOKEN" \
@@ -59,6 +80,21 @@ _gli_issues_closed_by_label() {
     | jq '[.[] | {iid}]'
 }
 
+_gli_issues_by_label_created() {
+  # $1=label $2=state("all"|"opened"|"closed") $3=limit. Companion to
+  # _gli_issues_by_label that preserves created_at (dropped there); the
+  # codebase-audit job needs it to find the most recently filed audit issue.
+  # Captured into a variable (not piped straight into jq) so a fetch failure
+  # propagates via `return 1` instead of being swallowed as an empty result
+  # indistinguishable from "no audit issue exists yet" (review round 2 finding).
+  local state_param="" response
+  [ -n "$2" ] && [ "$2" != "all" ] && state_param="&state=$2"
+  response=$(curl -s --fail -H "PRIVATE-TOKEN: $PIPE_TOKEN" \
+    "$(_gli_api)/issues?labels=$1${state_param}&order_by=created_at&sort=desc&per_page=$3") || return 1
+  jq '[.[] | {iid, title: (.title // ""), description: (.description // ""),
+                labels: ((.labels // []) | join(",")), created_at: (.created_at // "")}]' <<< "$response"
+}
+
 _gli_issue_notes() {
   # $1=iid. Non-system notes, newest first. The caller (orchestrate.sh) caps
   # the total volume and adds a visible truncation marker, so the newest and
@@ -113,12 +149,15 @@ _gli_children_state() {
   echo "$total $open"
 }
 
-_gli_create_issue() {
-  # $1=title $2=desc $3=labels(comma). Prints new iid or empty.
-  curl -s -X POST -H "PRIVATE-TOKEN: $PIPE_TOKEN" -H "Content-Type: application/json" \
-    "$(_gli_api)/issues" \
-    -d "$(jq -n --arg t "$1" --arg b "$2" --arg l "$3" '{title:$t,description:$b,labels:$l}')" \
-    | jq -r '.iid // empty'
+_gli_create_issue_file() {
+  # $1=title $2=description file $3=labels(comma). Prints new iid or empty.
+  local payload result
+  payload=$(mktemp "$PIPE_CONTEXT_DIR/gitlab-issue.XXXXXX.json")
+  jq -n --arg t "$1" --rawfile b "$2" --arg l "$3" '{title:$t,description:$b,labels:$l}' > "$payload"
+  result=$(curl -s -X POST -H "PRIVATE-TOKEN: $PIPE_TOKEN" -H "Content-Type: application/json" \
+    "$(_gli_api)/issues" --data-binary "@$payload" | jq -r '.iid // empty')
+  rm -f "$payload"
+  printf '%s\n' "$result"
 }
 
 _gli_link_blocked_by() {
@@ -140,9 +179,10 @@ _gli_close_issue() {
     "$(_gli_api)/issues/$1" -d '{"state_event":"close"}' > /dev/null || true
 }
 
-_gli_issue_comment() {
+_gli_issue_comment_file() {
+  # $1=iid $2=comment file.
   curl -s -X POST -H "PRIVATE-TOKEN: $PIPE_TOKEN" "$(_gli_api)/issues/$1/notes" \
-    --data-urlencode "body=$2" > /dev/null
+    --data-urlencode "body@$2" > /dev/null
 }
 
 _gli_code_trigger() {
@@ -164,13 +204,16 @@ _gli_code_trigger() {
   }
 }
 
-_gli_mr_create() {
-  # $1=source $2=target $3=title $4=desc $5=labels(comma). Prints web_url.
-  curl --fail-with-body -s -X POST -H "PRIVATE-TOKEN: $PIPE_TOKEN" -H "Content-Type: application/json" \
-    "$(_gli_api)/merge_requests" \
-    -d "$(jq -n --arg s "$1" --arg t "$2" --arg ti "$3" --arg d "$4" --arg l "$5" \
-        '{source_branch:$s, target_branch:$t, title:$ti, description:$d, labels:$l, remove_source_branch:true}')" \
-    | jq -r '.web_url // empty'
+_gli_mr_create_file() {
+  # $1=source $2=target $3=title $4=description file $5=labels(comma). Prints web_url.
+  local payload result
+  payload=$(mktemp "$PIPE_CONTEXT_DIR/gitlab-mr.XXXXXX.json")
+  jq -n --arg s "$1" --arg t "$2" --arg ti "$3" --rawfile d "$4" --arg l "$5" \
+    '{source_branch:$s, target_branch:$t, title:$ti, description:$d, labels:$l, remove_source_branch:true}' > "$payload"
+  result=$(curl --fail-with-body -s -X POST -H "PRIVATE-TOKEN: $PIPE_TOKEN" -H "Content-Type: application/json" \
+    "$(_gli_api)/merge_requests" --data-binary "@$payload" | jq -r '.web_url // empty')
+  rm -f "$payload"
+  printf '%s\n' "$result"
 }
 
 # =============================================================================
@@ -194,6 +237,28 @@ _ghi_issues_by_label() {
                milestone_id: (.milestone.number // null)}]'
 }
 
+# $1=primary label $2=optional secondary label $3=state $4=limit. GitHub's
+# shared issues endpoint returns pull requests too, so filter those before the
+# requested slice; otherwise a recent PR can starve an issue queue.
+_ghi_issues_by_labels() {
+  local primary="$1" secondary="$2" state="$3" limit="$4" gh_state response
+  [[ "$limit" =~ ^[1-9][0-9]*$ ]] || return 1
+  case "$state" in
+    opened|'') gh_state=open ;;
+    closed) gh_state=closed ;;
+    all) gh_state=all ;;
+    *) return 1 ;;
+  esac
+  response=$(gh api --paginate "repos/$PIPE_REPO/issues?labels=$primary&state=$gh_state&per_page=100") || return 1
+  jq -s 'add // []' <<< "$response" \
+    | jq --arg primary "$primary" --arg secondary "$secondary" --argjson limit "$limit" \
+      '[.[] | select(has("pull_request") | not)
+             | select(((.labels // []) | map(.name) | index($primary)))
+             | select($secondary == "" or ((.labels // []) | map(.name) | index($secondary)))
+             | {iid: .number, title: (.title // ""), description: (.body // ""),
+                labels: ((.labels // []) | map(.name) | join(","))}] | .[0:$limit]'
+}
+
 _ghi_issues_closed_by_label() {
   # $1=label. Closed issues still carrying a lifecycle label (janitor input).
   # PRs share the issues endpoint; keep issues only, closed PR labels stay
@@ -202,6 +267,29 @@ _ghi_issues_closed_by_label() {
     | jq '[.[] | select(has("pull_request") | not) | {iid: .number}]'
 }
 
+_ghi_issues_by_label_created() {
+  # $1=label $2=state("all"|"opened"|"closed") $3=limit. GitHub's shared
+  # issues/PRs endpoint has no server-side "issues only" filter, so applying
+  # $3 as per_page before the pull_request filter can empty the result: once
+  # this job's own auto-fix PR carries the same label, it is newer than the
+  # audit issues and a limit of 1 would fetch only that PR, filter it away,
+  # and return nothing (review round 1 finding). Fetch a full page (GitHub's
+  # per-page max) and slice to the requested limit after the PR filter.
+  # Captured into a variable (not piped straight into jq) so a fetch failure
+  # propagates via `return 1` instead of being swallowed as an empty result
+  # indistinguishable from "no audit issue exists yet" (review round 2 finding).
+  local gh_state response
+  case "$2" in
+    closed) gh_state=closed ;;
+    all) gh_state=all ;;
+    *) gh_state=open ;;
+  esac
+  response=$(gh api "repos/$PIPE_REPO/issues?labels=$1&state=$gh_state&sort=created&direction=desc&per_page=100") || return 1
+  jq --argjson limit "$3" '[.[] | select(has("pull_request") | not)
+          | {iid: .number, title: (.title // ""), description: (.body // ""),
+             labels: ((.labels // []) | map(.name) | join(",")), created_at: (.created_at // "")}] | .[0:$limit]' <<< "$response"
+}
+
 _ghi_issue_notes() {
   # $1=iid. Comments newest first (GitHub returns them oldest first, so reverse).
   # The caller (orchestrate.sh) caps the total volume and adds a visible
@@ -231,21 +319,32 @@ _ghi_link_blocked_by() {
   # dependent issue body after all local decomposition keys have been mapped.
   # The marker is visible and deliberately idempotent because the release poll
   # parses the same "Blocked by #N" convention.
-  local blocked="$1" blocker="$2" body marker
-  body=$(_ghi_issue_body "$blocked")
+  local blocked="$1" blocker="$2" body_file payload marker
+  body_file=$(mktemp "$PIPE_CONTEXT_DIR/github-issue-body.XXXXXX")
+  _ghi_issue_body "$blocked" > "$body_file"
   marker="Blocked by #$blocker"
-  printf '%s\n' "$body" | grep -qiE "(^|[[:space:]])Blocked by #${blocker}([^0-9]|$)" && return 0
+  grep -qiE "(^|[[:space:]])Blocked by #${blocker}([^0-9]|$)" "$body_file" && {
+    rm -f "$body_file"
+    return 0
+  }
+  payload=$(mktemp "$PIPE_CONTEXT_DIR/github-issue.XXXXXX.json")
+  jq -n --rawfile body "$body_file" --arg marker "$marker" \
+    '{body: $body + (if $body == "" then "" else "\n\n" end) + $marker}' > "$payload"
   gh api -X PATCH "repos/$PIPE_REPO/issues/$blocked" \
-    -f "body=${body}${body:+$'\n\n'}$marker" > /dev/null
+    -H "Content-Type: application/json" --input "$payload" > /dev/null
+  rm -f "$body_file" "$payload"
 }
 
-_ghi_create_issue() {
-  # $1=title $2=desc $3=labels(comma). Prints new number.
-  local args=() l
-  IFS=',' read -ra _labels <<< "$3"
-  for l in "${_labels[@]}"; do [ -n "$l" ] && args+=(-f "labels[]=$l"); done
-  gh api -X POST "repos/$PIPE_REPO/issues" -f "title=$1" -f "body=$2" "${args[@]}" \
-    --jq '.number // empty'
+_ghi_create_issue_file() {
+  # $1=title $2=description file $3=labels(comma). Prints new number.
+  local payload result
+  payload=$(mktemp "$PIPE_CONTEXT_DIR/github-issue.XXXXXX.json")
+  jq -n --arg t "$1" --rawfile b "$2" --arg l "$3" \
+    '{title:$t, body:$b, labels: ($l | split(",") | map(select(length > 0)))}' > "$payload"
+  result=$(gh api -X POST "repos/$PIPE_REPO/issues" \
+    -H "Content-Type: application/json" --input "$payload" --jq '.number // empty')
+  rm -f "$payload"
+  printf '%s\n' "$result"
 }
 
 _ghi_set_issue_labels() {
@@ -260,8 +359,15 @@ _ghi_close_issue() {
   gh api -X PATCH "repos/$PIPE_REPO/issues/$1" -f "state=closed" > /dev/null 2>&1 || true
 }
 
-_ghi_issue_comment() {
-  gh api -X POST "repos/$PIPE_REPO/issues/$1/comments" -f "body=$2" > /dev/null
+_ghi_issue_comment_file() {
+  # $1=iid $2=comment file.
+  local payload status=0
+  payload=$(mktemp "$PIPE_CONTEXT_DIR/github-comment.XXXXXX.json")
+  jq -n --rawfile body "$2" '{body:$body}' > "$payload"
+  gh api -X POST "repos/$PIPE_REPO/issues/$1/comments" \
+    -H "Content-Type: application/json" --input "$payload" > /dev/null || status=$?
+  rm -f "$payload"
+  return "$status"
 }
 
 _ghi_code_trigger() {
@@ -278,21 +384,27 @@ _ghi_code_trigger() {
   }
 }
 
-_ghi_pr_create() {
-  # $1=source $2=target $3=title $4=desc $5=labels(comma). Prints html_url.
+_ghi_pr_create_file() {
+  # $1=source $2=target $3=title $4=description file $5=labels(comma). Prints html_url.
   # Created as a draft and flipped to ready only after the labels are on
   # (finding P5): GitHub cannot attach labels at PR creation, so a non-draft
   # PR emits an `opened` + `labeled` event pair and the review workflow runs
   # twice. Draft PRs do not trigger the review job, which makes the single
   # `ready_for_review` event the only trigger. Falls back to a plain PR when
   # the plan does not support drafts on this repo (free plan, private repo).
-  local url num
+  local url num payload
+  payload=$(mktemp "$PIPE_CONTEXT_DIR/github-pr.XXXXXX.json")
+  jq -n --arg h "$1" --arg b "$2" --arg t "$3" --rawfile d "$4" \
+    '{head:$h, base:$b, title:$t, body:$d, draft:true}' > "$payload"
   url=$(gh api -X POST "repos/$PIPE_REPO/pulls" \
-    -f "head=$1" -f "base=$2" -f "title=$3" -f "body=$4" -F "draft=true" --jq '.html_url // empty' 2>/dev/null || true)
+    -H "Content-Type: application/json" --input "$payload" --jq '.html_url // empty' 2>/dev/null || true)
   if [ -z "$url" ]; then
+    jq -n --arg h "$1" --arg b "$2" --arg t "$3" --rawfile d "$4" \
+      '{head:$h, base:$b, title:$t, body:$d}' > "$payload"
     url=$(gh api -X POST "repos/$PIPE_REPO/pulls" \
-      -f "head=$1" -f "base=$2" -f "title=$3" -f "body=$4" --jq '.html_url // empty' 2>/dev/null || true)
+      -H "Content-Type: application/json" --input "$payload" --jq '.html_url // empty' 2>/dev/null || true)
   fi
+  rm -f "$payload"
   num=$(echo "$url" | grep -oE '[0-9]+$' || true)
   if [ -n "$num" ] && [ -n "$5" ]; then
     # Labels go through the issues API (a PR is an issue on GitHub).
@@ -309,17 +421,19 @@ _ghi_pr_create() {
 
 issue_ensure_label()       { case "$PIPE_PLATFORM" in gitlab) _gli_ensure_label "$1" "$2" "$3" ;; github) _ghi_ensure_label "$1" "$2" "$3" ;; esac; }
 issues_by_label()          { case "$PIPE_PLATFORM" in gitlab) _gli_issues_by_label "$1" ;; github) _ghi_issues_by_label "$1" ;; esac; }
+issues_by_labels()         { case "$PIPE_PLATFORM" in gitlab) _gli_issues_by_labels "$1" "$2" "$3" "$4" ;; github) _ghi_issues_by_labels "$1" "$2" "$3" "$4" ;; esac; }
 issues_closed_by_label()   { case "$PIPE_PLATFORM" in gitlab) _gli_issues_closed_by_label "$1" ;; github) _ghi_issues_closed_by_label "$1" ;; esac; }
+issues_by_label_created()  { case "$PIPE_PLATFORM" in gitlab) _gli_issues_by_label_created "$1" "$2" "$3" ;; github) _ghi_issues_by_label_created "$1" "$2" "$3" ;; esac; }
 issue_notes()              { case "$PIPE_PLATFORM" in gitlab) _gli_issue_notes "$1" ;; github) _ghi_issue_notes "$1" ;; esac; }
 issue_body()               { case "$PIPE_PLATFORM" in gitlab) _gli_issue_body "$1" ;; github) _ghi_issue_body "$1" ;; esac; }
 issue_meta()               { case "$PIPE_PLATFORM" in gitlab) _gli_issue_meta "$1" ;; github) _ghi_issue_meta "$1" ;; esac; }
 issue_state()              { case "$PIPE_PLATFORM" in gitlab) _gli_issue_state "$1" ;; github) _ghi_issue_state "$1" ;; esac; }
-issue_create()             { case "$PIPE_PLATFORM" in gitlab) _gli_create_issue "$1" "$2" "$3" ;; github) _ghi_create_issue "$1" "$2" "$3" ;; esac; }
+issue_create_file()        { case "$PIPE_PLATFORM" in gitlab) _gli_create_issue_file "$1" "$2" "$3" ;; github) _ghi_create_issue_file "$1" "$2" "$3" ;; esac; }
 issue_set_labels()         { case "$PIPE_PLATFORM" in gitlab) _gli_set_issue_labels "$1" "$2" "$3" ;; github) _ghi_set_issue_labels "$1" "$2" "$3" ;; esac; }
 issue_close()              { case "$PIPE_PLATFORM" in gitlab) _gli_close_issue "$1" ;; github) _ghi_close_issue "$1" ;; esac; }
-issue_comment()            { case "$PIPE_PLATFORM" in gitlab) _gli_issue_comment "$1" "$2" ;; github) _ghi_issue_comment "$1" "$2" ;; esac; }
+issue_comment_file()       { case "$PIPE_PLATFORM" in gitlab) _gli_issue_comment_file "$1" "$2" ;; github) _ghi_issue_comment_file "$1" "$2" ;; esac; }
 code_trigger()             { case "$PIPE_PLATFORM" in gitlab) _gli_code_trigger "$1" "$2" ;; github) _ghi_code_trigger "$1" "$2" ;; esac; }
-mr_create()                { case "$PIPE_PLATFORM" in gitlab) _gli_mr_create "$1" "$2" "$3" "$4" "$5" ;; github) _ghi_pr_create "$1" "$2" "$3" "$4" "$5" ;; esac; }
+mr_create_file()           { case "$PIPE_PLATFORM" in gitlab) _gli_mr_create_file "$1" "$2" "$3" "$4" "$5" ;; github) _ghi_pr_create_file "$1" "$2" "$3" "$4" "$5" ;; esac; }
 issue_link_blocked_by()    { case "$PIPE_PLATFORM" in gitlab) _gli_link_blocked_by "$1" "$2" ;; github) _ghi_link_blocked_by "$1" "$2" ;; esac; }
 
 # Count how many issues that block $1 are still open. GitLab uses native links,
@@ -376,13 +490,13 @@ issue_children_state() {
 # templates/spec-issue.template.md. If you rename a section
 # header or change a placeholder token in the template, update both lists here.
 DECOMP_REJECT_REASON=""
-pipe_validate_decomposition() {
+pipe_validate_decomposition_file() {
   # Prints the first problem to stdout on failure (empty on success). Sets
   # DECOMP_REJECT_REASON and returns 1 when invalid, 0 when valid.
-  local json="$1"
+  local json_file="$1"
   DECOMP_REJECT_REASON=""
   local reason
-  reason=$(printf '%s' "$json" | jq -r '
+  reason=$(jq -r '
     def required: ["## 1. Outcome","## 2. Context","## 3. Acceptance Criteria",
                    "## 4. Scope Envelope","## 5. Existing Code Anchors","## 6. Risk Flags",
                    "## 7. Test Plan","## 8. Out of Scope"];
@@ -394,6 +508,7 @@ pipe_validate_decomposition() {
     | if ($conf != "high" and $conf != "low") then "confidence must be high or low, got: \($conf)"
       elif (($subs | length) < 2 or ($subs | length) > 5) then "sub_issues count must be 2-5, got: \($subs | length)"
       elif ($keys | any(. == null or . == "")) then "every sub_issue needs a non-empty key"
+      elif ($keys | length) != ($keys | unique | length) then "sub_issue keys must be unique"
       else
         ( [ $subs[] | (.depends_on // [])[] as $d | select(($keys | index($d)) == null) | $d ] | first ) as $unknown
         | if $unknown != null then "depends_on references unknown key: \($unknown)"
@@ -420,7 +535,7 @@ pipe_validate_decomposition() {
               end
           end
       end
-  ' 2>/dev/null || echo "invalid JSON")
+  ' "$json_file" 2>/dev/null || echo "invalid JSON")
   if [ -n "$reason" ]; then
     # shellcheck disable=SC2034  # read by orchestrate.sh once this returns 1
     DECOMP_REJECT_REASON="$reason"