@cxi-lmai/ci-agent-platform 3.1.2 → 3.1.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cxi-lmai/ci-agent-platform",
-  "version": "3.1.2",
+  "version": "3.1.4",
   "description": "Autonomous dev pipeline on plain GitLab CI or GitHub Actions, driven by Claude Code. A labeled issue goes in, an open merge request comes out.",
   "keywords": [
     "claude",

package/payload/ci-templates/claude-pipeline.gitlab-ci.yml

@@ -79,6 +79,18 @@ variables:
   PIPE_ORCHESTRATE: "0"                          # set to "1" in the schedule to run orchestrate
   PIPE_SPEC_TEMPLATE_PATH: ".gitlab/issue_templates/Spec.md"
 
+  # --- Agent deadline (omp harness) -----------------------------------------
+  # A job `timeout:` is a hard kill — the runner SIGKILLs the process tree and
+  # the runner script never posts its comment, metrics or marker file. So
+  # `pipe_run_omp` bounds the agent itself with `omp --max-time`, derived from
+  # GitLab's own CI_JOB_TIMEOUT minus the time already spent minus the reserve
+  # below, which leaves room for the post-agent steps. An overrun then takes
+  # the normal degraded path (failure-notice comment + metrics) instead of
+  # dying silently. Set PIPE_AGENT_MAX_TIME (e.g. "20m") to override the
+  # derivation for every job; GitHub Actions exposes no job-timeout variable,
+  # so there the flag is omitted and timeout-minutes stays the only bound.
+  PIPE_AGENT_TIME_RESERVE: "180"        # seconds of the job budget kept for the post-agent steps
+
   # --- Build / verify --------------------------------------------------------
   PIPE_VERIFY_CMD: ""                   # compile + unit tests, run after a fix
   PIPE_COMPILE_CMD: ""                  # compile only (defaults to PIPE_VERIFY_CMD)
@@ -212,11 +224,15 @@ variables:
 
 # =============================================================================
 # review: run the automated review on a non-draft MR that changes code.
+# The 45m budget matches `code` and is not generous: the skill fans out to the
+# code-reviewer plus up to three specialist subagents, and observed runs on a
+# real project spread from 1m to over 20m with diff size. It was 20m until
+# three consecutive cap-outs on large MRs were measured in production.
 # =============================================================================
 review:
   extends: .claude-base
   stage: review
-  timeout: 20m
+  timeout: 45m
   script:
     - bash "$PIPE_SCRIPTS_DIR/review.sh"
   artifacts:

package/payload/ci-templates/scripts/lib/issue-loop.sh

@@ -120,6 +120,35 @@ _gli_issue_state() {
   curl -s -H "PRIVATE-TOKEN: $PIPE_TOKEN" "$(_gli_api)/issues/$1" | jq -r '.state // "opened"'
 }
 
+_gli_merged_closer() {
+  # $1=iid. Prints the newest merged MR that closes the issue as compact JSON,
+  # or nothing. A later reopen invalidates an earlier merge.
+  local response closer events merged_at
+  response=$(curl -s --fail -H "PRIVATE-TOKEN: $PIPE_TOKEN" \
+    "$(_gli_api)/issues/$1/closed_by?per_page=100") || return 1
+  closer=$(jq -c --arg branch "$PIPE_TARGET_BRANCH" '
+    [.[] | select(.state == "merged" and .merged_at != null and .target_branch == $branch)]
+    | max_by(.merged_at)
+    | if . == null then empty else
+        {kind: "MR",
+         reference: (.references.relative // .reference // ("!" + (.iid | tostring))),
+         url: .web_url,
+         merged_at: .merged_at}
+      end
+  ' <<< "$response") || return 1
+  [ -n "$closer" ] || return 0
+
+  events=$(curl -s --fail -H "PRIVATE-TOKEN: $PIPE_TOKEN" \
+    "$(_gli_api)/issues/$1/resource_state_events?per_page=100") || return 1
+  merged_at=$(jq -r '.merged_at' <<< "$closer")
+  if jq -e --arg merged_at "$merged_at" \
+    'any(.[]; .state == "opened" and (.created_at // "") > $merged_at)' \
+    <<< "$events" >/dev/null; then
+    return 0
+  fi
+  printf '%s\n' "$closer"
+}
+
 _gli_blockers_open_count() {
   # $1=iid. Count of open issues that block this one (native links).
   curl -s -H "PRIVATE-TOKEN: $PIPE_TOKEN" "$(_gli_api)/issues/$1/links" \
@@ -168,8 +197,9 @@ _gli_link_blocked_by() {
 }
 
 _gli_set_issue_labels() {
-  # $1=iid $2=add(comma) $3=remove(comma). Returns HTTP-independent (best effort).
-  curl -s -X PUT -H "PRIVATE-TOKEN: $PIPE_TOKEN" -H "Content-Type: application/json" \
+  # $1=iid $2=add(comma) $3=remove(comma). HTTP failures must stop dispatch.
+  curl --fail-with-body -s -X PUT \
+    -H "PRIVATE-TOKEN: $PIPE_TOKEN" -H "Content-Type: application/json" \
     "$(_gli_api)/issues/$1" \
     -d "$(jq -n --arg a "$2" --arg r "$3" '{add_labels:$a, remove_labels:$r}')" > /dev/null
 }
@@ -310,10 +340,57 @@ _ghi_issue_meta() {
 
 _ghi_issue_state() {
   # Prints "opened" or "closed" (normalised to the GitLab vocabulary).
-  local s; s=$(gh api "repos/$PIPE_REPO/issues/$1" --jq '.state // "open"' 2>/dev/null || echo open)
+  local s
+  s=$(gh api "repos/$PIPE_REPO/issues/$1" --jq '.state // "open"' 2>/dev/null) || return 1
   [ "$s" = "closed" ] && echo closed || echo opened
 }
 
+_ghi_merged_closer() {
+  # $1=iid. GitHub's GraphQL closing-reference connection is exact for both
+  # keyword and manually linked PRs. One query also carries reopen events.
+  local owner="${PIPE_REPO%%/*}" repo="${PIPE_REPO#*/}" response
+  response=$(gh api graphql \
+    -f "owner=$owner" -f "name=$repo" -F "number=$1" \
+    -f query='
+      query($owner: String!, $name: String!, $number: Int!) {
+        repository(owner: $owner, name: $name) {
+          issue(number: $number) {
+            closedByPullRequestsReferences(first: 100, includeClosedPrs: true) {
+              nodes {
+                number
+                url
+                merged
+                mergedAt
+                baseRefName
+              }
+            }
+            timelineItems(last: 100, itemTypes: [REOPENED_EVENT]) {
+              nodes {
+                ... on ReopenedEvent {
+                  createdAt
+                }
+              }
+            }
+          }
+        }
+      }
+    ') || return 1
+  jq -c --arg branch "$PIPE_TARGET_BRANCH" '
+    .data.repository.issue as $issue
+    | ([$issue.closedByPullRequestsReferences.nodes[]
+        | select(.merged == true and .mergedAt != null and .baseRefName == $branch)]
+       | max_by(.mergedAt)) as $closer
+    | ([$issue.timelineItems.nodes[]?.createdAt] | max // "") as $reopened_at
+    | if $closer != null and ($reopened_at == "" or $closer.mergedAt > $reopened_at)
+      then {kind: "PR",
+            reference: ("#" + ($closer.number | tostring)),
+            url: $closer.url,
+            merged_at: $closer.mergedAt}
+      else empty
+      end
+  ' <<< "$response"
+}
+
 _ghi_link_blocked_by() {
   # GitHub has no native blocking link. Persist the real issue number in the
   # dependent issue body after all local decomposition keys have been mapped.
@@ -351,8 +428,14 @@ _ghi_set_issue_labels() {
   # $1=iid $2=add(comma) $3=remove(comma)
   local l
   local IFS=','
-  for l in $2; do [ -n "$l" ] && gh api -X POST "repos/$PIPE_REPO/issues/$1/labels" -f "labels[]=$l" > /dev/null 2>&1 || true; done
-  for l in $3; do [ -n "$l" ] && gh api -X DELETE "repos/$PIPE_REPO/issues/$1/labels/$l" > /dev/null 2>&1 || true; done
+  for l in $2; do
+    [ -z "$l" ] || gh api -X POST "repos/$PIPE_REPO/issues/$1/labels" \
+      -f "labels[]=$l" > /dev/null 2>&1 || return 1
+  done
+  for l in $3; do
+    [ -z "$l" ] || gh api -X DELETE "repos/$PIPE_REPO/issues/$1/labels/$l" \
+      > /dev/null 2>&1 || return 1
+  done
 }
 
 _ghi_close_issue() {
@@ -428,6 +511,7 @@ issue_notes()              { case "$PIPE_PLATFORM" in gitlab) _gli_issue_notes "
 issue_body()               { case "$PIPE_PLATFORM" in gitlab) _gli_issue_body "$1" ;; github) _ghi_issue_body "$1" ;; esac; }
 issue_meta()               { case "$PIPE_PLATFORM" in gitlab) _gli_issue_meta "$1" ;; github) _ghi_issue_meta "$1" ;; esac; }
 issue_state()              { case "$PIPE_PLATFORM" in gitlab) _gli_issue_state "$1" ;; github) _ghi_issue_state "$1" ;; esac; }
+issue_merged_closer()      { case "$PIPE_PLATFORM" in gitlab) _gli_merged_closer "$1" ;; github) _ghi_merged_closer "$1" ;; esac; }
 issue_create_file()        { case "$PIPE_PLATFORM" in gitlab) _gli_create_issue_file "$1" "$2" "$3" ;; github) _ghi_create_issue_file "$1" "$2" "$3" ;; esac; }
 issue_set_labels()         { case "$PIPE_PLATFORM" in gitlab) _gli_set_issue_labels "$1" "$2" "$3" ;; github) _ghi_set_issue_labels "$1" "$2" "$3" ;; esac; }
 issue_close()              { case "$PIPE_PLATFORM" in gitlab) _gli_close_issue "$1" ;; github) _ghi_close_issue "$1" ;; esac; }
@@ -442,12 +526,13 @@ issue_blockers_open_count() {
   case "$PIPE_PLATFORM" in
     gitlab) _gli_blockers_open_count "$1" ;;
     github)
-      local body refs n open=0
-      body=$(_ghi_issue_body "$1")
+      local body refs n state open=0
+      body=$(_ghi_issue_body "$1") || return 1
       # Match "Blocked by #12" / "blocked by #12, #13" case-insensitively.
       refs=$(printf '%s' "$body" | grep -ioE 'blocked by[^\n]*' | grep -oE '#[0-9]+' | grep -oE '[0-9]+' | sort -u || true)
       for n in $refs; do
-        [ "$(_ghi_issue_state "$n")" = "opened" ] && open=$((open + 1))
+        state=$(_ghi_issue_state "$n") || return 1
+        [ "$state" = "opened" ] && open=$((open + 1))
       done
       echo "$open"
       ;;

package/payload/ci-templates/scripts/lib/pipeline-common.sh

@@ -369,6 +369,49 @@ pipe_translate_tools_to_omp() {
   echo "${result#,}"
 }
 
+# --- Agent deadline -----------------------------------------------------
+# A CI job timeout is a hard kill: the runner SIGKILLs the whole process tree,
+# so the runner script never reaches the steps that post the comment, write
+# the metrics record and emit the marker file. An agent that overruns
+# therefore leaves a red job with no output at all (observed in production:
+# three review timeouts in one week, each at the cap with the agent mid-run).
+# Bounding the agent itself with `omp --max-time` turns an overrun into the
+# platform's normal degraded path — failure-notice comment plus metrics —
+# instead of a hard kill.
+
+pipe_agent_max_time() {
+  # Echoes the --max-time value to pass to omp, or nothing when no deadline
+  # can be derived (GitHub Actions exposes no CI_JOB_TIMEOUT equivalent;
+  # there the workflow's timeout-minutes stays the only bound). An explicit
+  # PIPE_AGENT_MAX_TIME wins and is passed through verbatim, so a project can
+  # set "20m" or "1h" by hand. Otherwise the budget is the job timeout, minus
+  # the time already spent (image pull, checkout, before_script, API prep),
+  # minus PIPE_AGENT_TIME_RESERVE for the post-agent steps.
+  if [ -n "${PIPE_AGENT_MAX_TIME:-}" ]; then
+    printf '%s' "$PIPE_AGENT_MAX_TIME"
+    return 0
+  fi
+  local budget="${CI_JOB_TIMEOUT:-}"
+  case "$budget" in ''|*[!0-9]*) return 0 ;; esac
+  local reserve="${PIPE_AGENT_TIME_RESERVE:-180}" elapsed=0 started now
+  # Guard the emptiness explicitly: `date -d ""` does not fail, it silently
+  # resolves to midnight today, which would read as a many-hour-old job.
+  if [ -n "${CI_JOB_STARTED_AT:-}" ]; then
+    started=$(date -d "$CI_JOB_STARTED_AT" +%s 2>/dev/null || true)
+    if [ -n "$started" ]; then
+      now=$(date +%s)
+      elapsed=$(( now - started ))
+      [ "$elapsed" -lt 0 ] && elapsed=0
+    fi
+  fi
+  local left=$(( budget - elapsed - reserve ))
+  # A job already out of budget gets no flag: let the agent start and be cut
+  # by the runner exactly as it was before this function existed, rather than
+  # be killed one second in with a nonsensical deadline.
+  [ "$left" -lt 60 ] && return 0
+  printf '%s' "$left"
+}
+
 pipe_run_omp() {
   # $1 = agent label for metrics, $2 = skill invocation (e.g. "/review-mr"),
   # $3 = allowed tools in Claude Code naming (optional, translated
@@ -377,6 +420,14 @@ pipe_run_omp() {
   # frontmatter), stdin = extra prompt (optional, usually empty).
   local label="$1" skill="$2" tools="${3:-Agent,Read,Write,Edit,Glob,Grep,Bash}" model="${4:-}"
   local omp_tools; omp_tools=$(pipe_translate_tools_to_omp "$tools")
+  # Bound the run below the job's own hard kill (see pipe_agent_max_time).
+  local max_time; max_time=$(pipe_agent_max_time)
+  local -a max_time_args=()
+  local max_time_txt=""
+  if [ -n "$max_time" ]; then
+    max_time_args=(--max-time "$max_time")
+    max_time_txt=$(printf ' --max-time %q' "$max_time")
+  fi
   pipe_require_harness omp
   local stdin_file; stdin_file=$(mktemp)
   cat > "$stdin_file" || true
@@ -402,12 +453,14 @@ pipe_run_omp() {
       printf 'cd %q\n' "$PWD"
       printf 'source %q\n' "$SCRIPT_LIB_DIR/usage-capture-omp.sh"
       printf 'export PIPE_CAPTURE_MODEL=%q\n' "$model"
+      # The %s after `--` is the optional " --max-time <d>" fragment, already
+      # shell-quoted; it collapses to nothing when no deadline was derived.
       if [ -n "$model" ]; then
-        printf 'capture_omp %q -- --yolo --no-session --no-title --model %q --tools %q -p %q < %q\n' \
-          "$label" "$model" "$omp_tools" "$skill" "$stdin_file"
+        printf 'capture_omp %q --%s --yolo --no-session --no-title --model %q --tools %q -p %q < %q\n' \
+          "$label" "$max_time_txt" "$model" "$omp_tools" "$skill" "$stdin_file"
       else
-        printf 'capture_omp %q -- --yolo --no-session --no-title --tools %q -p %q < %q\n' \
-          "$label" "$omp_tools" "$skill" "$stdin_file"
+        printf 'capture_omp %q --%s --yolo --no-session --no-title --tools %q -p %q < %q\n' \
+          "$label" "$max_time_txt" "$omp_tools" "$skill" "$stdin_file"
       fi
     } > "$runner"
     chmod +x "$runner"
@@ -420,9 +473,9 @@ pipe_run_omp() {
       pipe_scrub_agent_secrets
       export PIPE_CAPTURE_MODEL="$model"
       if [ -n "$model" ]; then
-        capture_omp "$label" -- --yolo --no-session --no-title --model "$model" --tools "$omp_tools" -p "$skill" < "$stdin_file"
+        capture_omp "$label" -- "${max_time_args[@]}" --yolo --no-session --no-title --model "$model" --tools "$omp_tools" -p "$skill" < "$stdin_file"
       else
-        capture_omp "$label" -- --yolo --no-session --no-title --tools "$omp_tools" -p "$skill" < "$stdin_file"
+        capture_omp "$label" -- "${max_time_args[@]}" --yolo --no-session --no-title --tools "$omp_tools" -p "$skill" < "$stdin_file"
       fi
     ) || true
   fi

package/payload/ci-templates/scripts/metrics-snapshot.sh

@@ -186,9 +186,13 @@ MRS_VIEW=$(jq -c \
         elif ($body | test($reintroduce; "i")) then "reintroduces"
         else null end
       ),
+      # match/2 with "g", not scan/2: a Debian bookworm based runner image
+      # ships jq 1.6, which has no scan/2 and fails the whole program with
+      # a compile error.
       references: [
         $body
-        | scan("(?:" + $revert + "|" + $regression + "|" + $reintroduce + ")[[:space:]]*[!#]?([0-9]+)"; "i")[]?
+        | match("(?:" + $revert + "|" + $regression + "|" + $reintroduce + ")[[:space:]]*[!#]?([0-9]+)"; "gi")
+        | .captures[].string
         | select(. != null)
         | tonumber
       ]

package/payload/ci-templates/scripts/orchestrate.sh

@@ -87,6 +87,100 @@ emit_decision_metric() {
         '{issue_iid: $i, verdict: $v, decision_type: $dt, reason: $r} + $x')"
 }
 
+dispatch_restore_ready() {
+  # $1=iid. Best-effort recovery back to ready after a mid-dispatch failure.
+  # Nothing further can be attempted from this runner if this also fails, so
+  # it logs loudly and meters a terminal, distinctly countable label-error.
+  if issue_set_labels "$1" "$PIPE_LABEL_READY" "$PIPE_LABEL_WIP"; then
+    return 0
+  fi
+  pipe_log "  ERROR: could not restore $PIPE_LABEL_READY on #$1; lifecycle labels may be inconsistent, manual check required"
+  emit_decision_metric "$1" skipped label-error "Failed to restore ready after a dispatch-time error"
+  return 1
+}
+
+dispatch_queued_issue() {
+  # $1=iid $2=branch. Claim atomically, then re-check repository state at the
+  # last possible point before spending a coder invocation. Every subsequent
+  # label transition is checked: a silently failed transition is exactly how
+  # an issue gets stuck in wip or loses its ready label forever.
+  local iid="$1" branch="$2" open closer kind reference url reason comment_file
+
+  if ! issue_set_labels "$iid" "$PIPE_LABEL_WIP" "$PIPE_LABEL_READY"; then
+    pipe_log "  ERROR: could not claim #$iid, skipping to avoid a double trigger"
+    emit_decision_metric "$iid" skipped claim-failed "Could not atomically claim the ready issue"
+    return 0
+  fi
+
+  if ! open=$(issue_blockers_open_count "$iid"); then
+    reason="Could not verify whether issue prerequisites are still open"
+    pipe_log "  ERROR: $reason for #$iid; restoring $PIPE_LABEL_READY"
+    dispatch_restore_ready "$iid" || return 0
+    emit_decision_metric "$iid" deferred blocker-validation-error "$reason"
+    return 0
+  fi
+  if [ "${open:-1}" != "0" ]; then
+    reason="Issue has ${open:-unknown} open prerequisite(s)"
+    if issue_set_labels "$iid" "$PIPE_LABEL_BLOCKED" "$PIPE_LABEL_WIP,$PIPE_LABEL_READY"; then
+      pipe_log "  #$iid returned to $PIPE_LABEL_BLOCKED: $reason"
+      emit_decision_metric "$iid" deferred blocked-prerequisite "$reason"
+    else
+      pipe_log "  ERROR: could not move #$iid to $PIPE_LABEL_BLOCKED, restoring $PIPE_LABEL_READY: $reason"
+      dispatch_restore_ready "$iid" || return 0
+      emit_decision_metric "$iid" deferred blocked-transition-error "$reason"
+    fi
+    return 0
+  fi
+
+  if ! closer=$(issue_merged_closer "$iid"); then
+    reason="Could not verify whether a closing MR/PR has already merged"
+    pipe_log "  ERROR: $reason for #$iid; restoring $PIPE_LABEL_READY"
+    dispatch_restore_ready "$iid" || return 0
+    emit_decision_metric "$iid" deferred validation-error "$reason"
+    return 0
+  fi
+
+  if [ -n "$closer" ]; then
+    kind=$(jq -r '.kind' <<< "$closer")
+    reference=$(jq -r '.reference' <<< "$closer")
+    url=$(jq -r '.url' <<< "$closer")
+    reason="$kind $reference already merged into $PIPE_TARGET_BRANCH"
+    comment_file="$PIPE_CONTEXT_DIR/orchestrator-already-merged-$iid.md"
+    {
+      printf '**Dispatch skipped:** %s %s has already merged into `%s`.\n\n' \
+        "$kind" "$reference" "$PIPE_TARGET_BRANCH"
+      printf '%s\n\n' "$url"
+      printf 'The pipeline removed `%s`; apply it again only if more work is required.\n' \
+        "$PIPE_LABEL_READY"
+    } > "$comment_file"
+    # Comment before the final label removal: if the comment never lands,
+    # the ready label must survive so the skip is retried and the change is
+    # named next time, instead of silently vanishing off the board.
+    if ! issue_comment_file "$iid" "$comment_file"; then
+      pipe_log "  ERROR: could not comment on #$iid before skipping; restoring $PIPE_LABEL_READY for retry"
+      dispatch_restore_ready "$iid" || return 0
+      emit_decision_metric "$iid" deferred comment-error "$reason"
+      return 0
+    fi
+    pipe_log "  skipping coder for #$iid: $reason ($url)"
+    if ! issue_set_labels "$iid" "" "$PIPE_LABEL_WIP,$PIPE_LABEL_READY"; then
+      pipe_log "  ERROR: could not clear lifecycle labels on #$iid after commenting; restoring $PIPE_LABEL_READY"
+      dispatch_restore_ready "$iid"
+    fi
+    emit_decision_metric "$iid" skipped already-merged "$reason" \
+      "$(jq -n --arg kind "$kind" --arg reference "$reference" --arg url "$url" \
+          '{closer_kind: $kind, closer_reference: $reference, closer_url: $url}')"
+    return 0
+  fi
+
+  if code_trigger "$iid" "$branch"; then
+    pipe_log "  triggered coder for #$iid on $branch"
+  else
+    pipe_log "  ERROR: trigger failed for #$iid, reverting label"
+    dispatch_restore_ready "$iid"
+  fi
+}
+
 while IFS= read -r ISSUE; do
   [ "${#QUEUE[@]}" -ge "$PIPE_CODER_CAP" ] && { pipe_log "Coder cap ($PIPE_CODER_CAP) reached, deferring the rest"; break; }
 
@@ -297,16 +391,5 @@ pipe_log "Triggering ${#QUEUE[@]} coder run(s)"
 for ITEM in "${QUEUE[@]}"; do
   IID="${ITEM%%:*}"
   BRANCH="${ITEM#*:}"
-  # Atomic claim: relabel ready -> wip first, so a concurrent orchestrator does
-  # not double-trigger. Revert on trigger failure.
-  if issue_set_labels "$IID" "$PIPE_LABEL_WIP" "$PIPE_LABEL_READY"; then
-    if code_trigger "$IID" "$BRANCH"; then
-      pipe_log "  triggered coder for #$IID on $BRANCH"
-    else
-      pipe_log "  ERROR: trigger failed for #$IID, reverting label"
-      issue_set_labels "$IID" "$PIPE_LABEL_READY" "$PIPE_LABEL_WIP"
-    fi
-  else
-    pipe_log "  ERROR: could not claim #$IID, skipping to avoid a double trigger"
-  fi
+  dispatch_queued_issue "$IID" "$BRANCH"
 done