@cxi-lmai/ci-agent-platform 3.0.0 → 3.1.0

package/payload/ci-templates/scripts/docs-sync.sh

@@ -0,0 +1,225 @@
+#!/bin/bash
+# Runner for the opt-in `docs-sync` job. Finds documentation gaps AND acts on
+# them in a single pass: on a wip MR/PR the skill applies the documentation
+# edits and this script commits and pushes them, on any other MR/PR the skill
+# reports the gaps and this script posts them as an advisory comment.
+#
+# The runner is mechanical: it classifies the MR/PR, computes the changed files
+# and the full diff, runs the /docs-sync skill, posts the verdict, and commits.
+# Every judgement about what needs documenting lives in the skill and in the
+# docs-sync agent it spawns.
+#
+# The diff is always base..head, never an incremental slice: a gap introduced by
+# an earlier push must still be found on a later one.
+set -e
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck source=lib/pipeline-common.sh
+source "$SCRIPT_DIR/lib/pipeline-common.sh"
+pipe_defaults
+
+DOCS_MARKER="**Automated Documentation Sync**"
+CONTEXT_FILE="$PIPE_CONTEXT_DIR/docs-sync-context.md"
+FILES_FILE="$PIPE_CONTEXT_DIR/docs-sync-files.txt"
+DIFF_FILE="$PIPE_CONTEXT_DIR/docs-sync-diff.patch"
+DOCS_ENV="$PIPE_CONTEXT_DIR/docs-sync.env"
+COMMENT_FILE="$PIPE_CONTEXT_DIR/docs-sync-comment.md"
+
+# Documentation roots the agent may edit and this script may commit. Comma
+# separated in PIPE_DOCS_ROOTS; a root is a directory or a single file.
+# The split is a bash field split, not `tr ',' '\n'` piped into `read`: that
+# pipeline emits no trailing newline, so `read` silently drops the LAST root.
+# Surrounding whitespace is trimmed per element so "docs, CLAUDE.md" works.
+DOCS_ROOTS=()
+IFS=',' read -ra DOCS_ROOTS_RAW <<< "$PIPE_DOCS_ROOTS"
+for root in "${DOCS_ROOTS_RAW[@]}"; do
+  root="${root#"${root%%[![:space:]]*}"}"
+  root="${root%"${root##*[![:space:]]}"}"
+  if [ -n "$root" ]; then
+    DOCS_ROOTS+=("$root")
+  fi
+done
+if [ "${#DOCS_ROOTS[@]}" -eq 0 ]; then
+  pipe_log "PIPE_DOCS_ROOTS is empty, nothing to sync"
+  exit 0
+fi
+
+# True when the path is one of the documentation roots or sits under one.
+docs_root_path() {
+  local root
+  for root in "${DOCS_ROOTS[@]}"; do
+    [ "$1" = "$root" ] && return 0
+    case "$1" in "$root"/*) return 0 ;; esac
+  done
+  return 1
+}
+
+# 1. Fetch MR/PR meta (token work). The source branch comes from the same read:
+#    it is what an applied documentation commit is pushed to, and the CI runtime
+#    does not always provide it.
+META=$(platform_mr_meta)
+MR_TITLE=$(echo "$META"  | jq -r '.title // ""')
+MR_LABELS=$(echo "$META" | jq -r '.labels // ""')
+META_SOURCE_BRANCH=$(echo "$META" | jq -r '.source_branch // ""')
+if [ -z "${PIPE_SOURCE_BRANCH:-}" ] && [ -n "$META_SOURCE_BRANCH" ]; then
+  PIPE_SOURCE_BRANCH="$META_SOURCE_BRANCH"
+  export PIPE_SOURCE_BRANCH
+fi
+
+# 2. Classify. Only an autonomous (wip) MR/PR gets its documentation edited;
+#    a human-authored one gets a report.
+DOCS_MODE=report
+if echo "$MR_LABELS" | grep -q "$PIPE_LABEL_WIP"; then
+  DOCS_MODE=apply
+fi
+
+# 3. Full-MR diff base. platform_compute_diff_base would hand back the previous
+#    push head on a re-run, which is the wrong scope for this job, so the base
+#    is always the merge base with the target branch.
+git fetch origin "$PIPE_TARGET_BRANCH" 2>/dev/null || true
+DIFF_BASE=$(git merge-base "origin/$PIPE_TARGET_BRANCH" "$PIPE_HEAD_SHA" 2>/dev/null || echo "origin/$PIPE_TARGET_BRANCH")
+pipe_log "mode=$DOCS_MODE diff base=$DIFF_BASE head=$PIPE_HEAD_SHA"
+
+git diff --name-only "$DIFF_BASE" "$PIPE_HEAD_SHA" > "$FILES_FILE" 2>/dev/null || : > "$FILES_FILE"
+if [ ! -s "$FILES_FILE" ]; then
+  pipe_log "No changed files in $DIFF_BASE..$PIPE_HEAD_SHA, skipping docs-sync"
+  exit 0
+fi
+
+# 4. Skip when the MR/PR only touches documentation: there is no new behaviour
+#    to document, and the author is already editing the docs.
+CODE_CHANGED=false
+while IFS= read -r file; do
+  [ -n "$file" ] || continue
+  if ! docs_root_path "$file"; then
+    CODE_CHANGED=true
+    break
+  fi
+done < "$FILES_FILE"
+if [ "$CODE_CHANGED" = "false" ]; then
+  pipe_log "Only documentation paths changed, skipping docs-sync"
+  exit 0
+fi
+
+git diff "$DIFF_BASE" "$PIPE_HEAD_SHA" > "$DIFF_FILE" 2>/dev/null || : > "$DIFF_FILE"
+
+# 5. In apply mode, check out the real source branch so the edits and the commit
+#    land on its tip. Loop guard: at most one consecutive automated docs commit
+#    per branch streak, otherwise apply -> pipeline -> apply would never settle.
+#    The guard degrades this run to report mode instead of skipping it, so the
+#    MR/PR still gets a verdict.
+if [ "$DOCS_MODE" = "apply" ]; then
+  pipe_git_identity
+  pipe_checkout_source
+  if [ "$(pipe_count_fix_commits "$PIPE_COMMIT_DOCSSYNC")" -gt 0 ]; then
+    pipe_log "Tip is already an automated docs commit, degrading to report mode"
+    DOCS_MODE=report
+  fi
+fi
+
+# 6. Write the context file the skill reads. Unbounded text (the diff, the file
+#    list) stays in its own file, never in argv or the environment.
+{
+  echo "# Documentation sync context"
+  echo
+  echo "Mode: $DOCS_MODE"
+  echo "Title: $MR_TITLE"
+  echo "Documentation roots: $PIPE_DOCS_ROOTS"
+  echo "Changed files: $FILES_FILE"
+  echo "Full diff: $DIFF_FILE"
+  echo "Diff range: $DIFF_BASE..$PIPE_HEAD_SHA"
+} > "$CONTEXT_FILE"
+
+# 7. Run the skill. It reads the context, spawns the docs-sync agent in the
+#    requested mode, writes $PIPE_RESULT_DOCSSYNC, the comment body, and the
+#    docs-sync.env marker this script reads back.
+pipe_run_agent docs-sync "/docs-sync" "Agent,Read,Write,Edit,Glob,Grep,Bash" "$PIPE_MODEL_REVIEW" < /dev/null
+
+# 8. Read the marker. DOCS_COMMENT_PATH carries the body because a comment is
+#    unbounded text. A result file written by a model can be malformed JSON, so
+#    the fallback extraction is defensive: an unparseable or comment-less result
+#    is treated exactly like a missing one.
+DOCS_HAS_GAPS=$(pipe_get_env "$DOCS_ENV" DOCS_HAS_GAPS); DOCS_HAS_GAPS=${DOCS_HAS_GAPS:-false}
+DOCS_CHANGED=$(pipe_get_env "$DOCS_ENV" DOCS_CHANGED);   DOCS_CHANGED=${DOCS_CHANGED:-false}
+DOCS_BODY=$(pipe_get_env "$DOCS_ENV" DOCS_COMMENT_PATH); DOCS_BODY=${DOCS_BODY:-$PIPE_CONTEXT_DIR/docs-sync-body.md}
+if [ ! -s "$DOCS_BODY" ] && [ -f "$PIPE_RESULT_DOCSSYNC" ]; then
+  RESULT_BODY="$PIPE_CONTEXT_DIR/docs-sync-result-body.md"
+  jq -re '.comment // empty' "$PIPE_RESULT_DOCSSYNC" > "$RESULT_BODY" 2>/dev/null || true
+  if [ -s "$RESULT_BODY" ]; then
+    DOCS_BODY="$RESULT_BODY"
+  fi
+fi
+
+DOCS_FAILED=false
+if [ -s "$DOCS_BODY" ]; then
+  {
+    printf '%s\n\n' "$DOCS_MARKER"
+    cat "$DOCS_BODY"
+  } > "$COMMENT_FILE"
+else
+  DOCS_FAILED=true
+  {
+    printf '%s\n\n⚠️ ' "$DOCS_MARKER"
+    pipe_failure_notice_files "documentation sync" \
+      "check by hand whether these changes need updates under $PIPE_DOCS_ROOTS" \
+      "$PIPE_AGENT_STDERR" "$PIPE_RESULT_DOCSSYNC"
+  } > "$COMMENT_FILE"
+fi
+
+# 9. Always post the verdict. Metrics reflect the runner's actual commit
+# decision, not the skill's informational DOCS_CHANGED marker.
+DOCS_COMMITTED=false
+emit_docs_metric() {
+  pipe_metric_event docs-sync docs_sync_gaps \
+    "$(jq -n --arg m "$DOCS_MODE" \
+        --argjson g "$([ "$DOCS_HAS_GAPS" = "true" ] && echo true || echo false)" \
+        --argjson c "$([ "$DOCS_COMMITTED" = "true" ] && echo true || echo false)" \
+        --argjson f "$([ "$DOCS_FAILED" = "true" ] && echo true || echo false)" \
+        '{mode: $m, has_gaps: $g, changed: $c, failed: $f}')"
+}
+
+platform_post_comment_file "$COMMENT_FILE"
+
+# 10. Commit and push what the agent applied. A failed agent applied nothing, so
+#     the notice posted above is the whole output and the exit is 0: an
+#     automation outage is not a documentation gap.
+if [ "$DOCS_FAILED" = "true" ]; then
+  emit_docs_metric
+  pipe_log "Agent did not complete, manual-review notice posted, nothing to commit"
+  exit 0
+fi
+if [ "$DOCS_MODE" != "apply" ]; then
+  emit_docs_metric
+  exit 0
+fi
+if [ "$DOCS_CHANGED" != "true" ]; then
+  pipe_log "Skill reported no documentation edits, checking the staged diff anyway"
+fi
+
+# Stage each root on its own: one missing root must not stop the others from
+# being staged, which a single `git add` of the whole set would do.
+for root in "${DOCS_ROOTS[@]}"; do
+  [ -e "$root" ] || continue
+  git add -- "$root" || true
+done
+if git diff --cached --quiet; then
+  emit_docs_metric
+  pipe_log "No documentation changes staged, nothing to commit"
+  exit 0
+fi
+
+git commit -m "$PIPE_COMMIT_DOCSSYNC"
+DOCS_COMMITTED=true
+emit_docs_metric
+
+# The agent may only touch the documentation roots, but an edit outside them
+# would leave the working tree dirty and abort the rebase below before it
+# starts, after which `git rebase --abort` fails too. Discard out-of-scope
+# edits so the tree is clean, enforcing the documented boundary.
+if ! git diff --quiet; then
+  pipe_log "Discarding out-of-scope working-tree changes left outside $PIPE_DOCS_ROOTS:"
+  git diff --name-only
+  git checkout -- .
+fi
+
+pipe_rebase_and_push
+pipe_log "Pushed the docs-sync commit, a new pipeline will run"

package/payload/ci-templates/scripts/e2e-test-gen.sh

@@ -0,0 +1,201 @@
+#!/bin/bash
+# Runner for the opt-in, issue-driven half of E2E test generation. The runner
+# owns forge mutations and git; the skill may only write a new test file.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# shellcheck source=lib/pipeline-common.sh
+source "$SCRIPT_DIR/lib/pipeline-common.sh"
+# shellcheck source=lib/issue-loop.sh
+source "$SCRIPT_DIR/lib/issue-loop.sh"
+pipe_defaults
+
+E2E_ENV="$PIPE_CONTEXT_DIR/e2e-gen.env"
+ISSUES_FILE="$PIPE_CONTEXT_DIR/e2e-gen-issues.json"
+MR_BODY="$PIPE_CONTEXT_DIR/e2e-gen-mr.md"
+GENERATED=0
+CREDIT_ABORT=0
+MR_CREATED=0
+
+write_marker() {
+  printf 'E2E_GENERATED_COUNT=%s\nE2E_CREDIT_ABORT=%s\nE2E_MR_CREATED=%s\n' \
+    "$GENERATED" "$CREDIT_ABORT" "$MR_CREATED" > "$E2E_ENV"
+}
+
+# Persist an explicit zero marker even for an intentionally inactive schedule.
+write_marker
+if [ "$PIPE_E2E_TEST_GEN" != "1" ]; then
+  pipe_log "E2E test generation is disabled"
+  exit 0
+fi
+
+case "$PIPE_E2E_TEST_DIR" in
+  ''|/*|..|../*|*/..|*/../*)
+    pipe_log "Invalid project-relative E2E test directory: $PIPE_E2E_TEST_DIR"
+    exit 0
+    ;;
+esac
+
+# E2E verification is intentionally separate from PIPE_VERIFY_CMD: projects
+# configure a command that accepts one generated E2E spec as its final argument.
+if [ -z "$PIPE_E2E_VERIFY_CMD" ]; then
+  pipe_log "PIPE_E2E_VERIFY_CMD is required to verify generated E2E tests; leaving issues unchanged"
+  exit 0
+fi
+if [ -z "$PIPE_E2E_BASE_URL" ]; then
+  pipe_log "PIPE_E2E_BASE_URL is required for generated E2E tests; leaving issues unchanged"
+  exit 0
+fi
+
+if ! issues_by_labels "$PIPE_LABEL_TESTREADY" "$PIPE_LABEL_E2E_SCOPE" opened "$PIPE_ISSUE_SCAN" > "$ISSUES_FILE"; then
+  pipe_log "Could not list test-ready issues"
+  exit 1
+fi
+ISSUE_COUNT=$(jq 'length' "$ISSUES_FILE")
+if [ "$ISSUE_COUNT" = "0" ]; then
+  pipe_log "No issues need E2E tests"
+  exit 0
+fi
+
+BRANCH="e2e-test-gen-$(date -u +%Y%m%d)"
+pipe_git_identity
+git fetch origin "$PIPE_TARGET_BRANCH" >/dev/null 2>&1 || {
+  pipe_log "Could not fetch target branch $PIPE_TARGET_BRANCH"
+  exit 1
+}
+# A branch of this daily name may contain a human amendment. Never reuse or
+# force-push it; a later scheduled run can safely make a fresh daily branch.
+git fetch origin "$BRANCH" >/dev/null 2>&1 || true
+if git show-ref --verify --quiet "refs/remotes/origin/$BRANCH"; then
+  pipe_log "Branch $BRANCH already exists; refusing to overwrite possible human work"
+  exit 0
+fi
+git checkout -b "$BRANCH" "origin/$PIPE_TARGET_BRANCH"
+
+mark_stuck() {
+  issue_set_labels "$1" "$PIPE_LABEL_STUCK" "$PIPE_LABEL_TESTREADY" || true
+}
+
+credit_abort() {
+  local iid="$1" body="$PIPE_CONTEXT_DIR/e2e-gen-credit-$1.md"
+  pipe_failure_notice_files "E2E test generation" \
+    "wait for credits to be restored, then let a later schedule retry this issue" \
+    "$PIPE_AGENT_STDERR" > "$body"
+  issue_comment_file "$iid" "$body" || pipe_log "Could not post credit-exhaustion notice for issue #$iid"
+  CREDIT_ABORT=1
+  write_marker
+}
+
+while IFS= read -r ISSUE; do
+  IID=$(jq -r '.iid' <<< "$ISSUE")
+  TITLE_FILE="$PIPE_CONTEXT_DIR/e2e-gen-title-$IID.txt"
+  DESC_FILE="$PIPE_CONTEXT_DIR/e2e-gen-description-$IID.txt"
+  CONTEXT_FILE="$PIPE_CONTEXT_DIR/e2e-gen-issue-$IID.md"
+  BEFORE_FILE="$PIPE_CONTEXT_DIR/e2e-gen-before-$IID.txt"
+  NEW_FILE="$PIPE_CONTEXT_DIR/e2e-gen-new-$IID.txt"
+
+  jq -r '(.title // "") | .[0:500]' <<< "$ISSUE" > "$TITLE_FILE"
+  jq -r '(.description // "(no description)") | .[0:3000]' <<< "$ISSUE" > "$DESC_FILE"
+  {
+    printf '# E2E test generation context\n\n'
+    printf 'Mode: generate\n'
+    printf 'Issue IID: %s\n' "$IID"
+    printf 'Issue Title File: %s\n' "$TITLE_FILE"
+    printf 'Issue Description File: %s\n' "$DESC_FILE"
+    printf 'Test Directory: %s\n' "$PIPE_E2E_TEST_DIR"
+    printf 'Base URL: %s\n' "$PIPE_E2E_BASE_URL"
+    printf 'Project Configuration: %s\n' "$PIPE_CONFIG_PATH"
+    printf 'Verification Command: %s\n' "$PIPE_E2E_VERIFY_CMD"
+  } > "$CONTEXT_FILE"
+
+  git ls-files --others --exclude-standard -- "$PIPE_E2E_TEST_DIR" | sort > "$BEFORE_FILE"
+  : > "$PIPE_AGENT_STDERR"
+  pipe_run_agent e2e-test-writer "/e2e-test-gen" "Agent,Read,Write,Edit,Glob,Grep" "$PIPE_MODEL_CODE" < /dev/null
+  # Discard all tracked edits: accepted work is strictly newly-created test files.
+  git reset --hard HEAD >/dev/null
+  if pipe_is_credit_exhaustion_files "$PIPE_AGENT_STDERR"; then
+    git ls-files --others --exclude-standard -- "$PIPE_E2E_TEST_DIR" | sort > "$PIPE_CONTEXT_DIR/e2e-gen-credit-after-$IID.txt"
+    comm -13 "$BEFORE_FILE" "$PIPE_CONTEXT_DIR/e2e-gen-credit-after-$IID.txt" | while IFS= read -r rejected; do
+      rm -f -- "$rejected"
+    done
+    credit_abort "$IID"
+    break
+  fi
+
+  git ls-files --others --exclude-standard -- "$PIPE_E2E_TEST_DIR" | sort > "$PIPE_CONTEXT_DIR/e2e-gen-after-$IID.txt"
+  comm -13 "$BEFORE_FILE" "$PIPE_CONTEXT_DIR/e2e-gen-after-$IID.txt" > "$NEW_FILE"
+  if [ ! -s "$NEW_FILE" ]; then
+    pipe_log "No new E2E test file generated for issue #$IID"
+    mark_stuck "$IID"
+    continue
+  fi
+
+  TEST_PASSED=true
+  while IFS= read -r SPEC; do
+    [ -n "$SPEC" ] || continue
+    if ! bash -c "$PIPE_E2E_VERIFY_CMD \"\$1\"" e2e-test-gen "$SPEC"; then
+      pipe_log "Generated E2E test $SPEC failed verification; attempting one repair"
+      {
+        printf '# E2E test generation context\n\n'
+        printf 'Mode: repair\n'
+        printf 'Generated Test: %s\n' "$SPEC"
+        printf 'Test Directory: %s\n' "$PIPE_E2E_TEST_DIR"
+        printf 'Base URL: %s\n' "$PIPE_E2E_BASE_URL"
+        printf 'Project Configuration: %s\n' "$PIPE_CONFIG_PATH"
+        printf 'Verification Command: %s\n' "$PIPE_E2E_VERIFY_CMD"
+      } > "$CONTEXT_FILE"
+      : > "$PIPE_AGENT_STDERR"
+      pipe_run_agent e2e-test-writer "/e2e-test-gen" "Agent,Read,Write,Edit,Glob,Grep" "$PIPE_MODEL_CODE" < /dev/null
+      git reset --hard HEAD >/dev/null
+      if pipe_is_credit_exhaustion_files "$PIPE_AGENT_STDERR"; then
+        while IFS= read -r rejected; do rm -f -- "$rejected"; done < "$NEW_FILE"
+        credit_abort "$IID"
+        break 2
+      fi
+      if ! bash -c "$PIPE_E2E_VERIFY_CMD \"\$1\"" e2e-test-gen "$SPEC"; then
+        TEST_PASSED=false
+        break
+      fi
+    fi
+  done < "$NEW_FILE"
+
+  if [ "$TEST_PASSED" = true ]; then
+    while IFS= read -r SPEC; do [ -n "$SPEC" ] && git add -- "$SPEC"; done < "$NEW_FILE"
+    if git diff --cached --quiet; then
+      pipe_log "Generated files for issue #$IID were not stageable"
+      mark_stuck "$IID"
+      continue
+    fi
+    git commit -m "test: add E2E coverage for issue #$IID"
+    GENERATED=$((GENERATED + 1))
+    issue_set_labels "$IID" "" "$PIPE_LABEL_TESTREADY" || true
+  else
+    while IFS= read -r rejected; do rm -f -- "$rejected"; done < "$NEW_FILE"
+    mark_stuck "$IID"
+  fi
+  write_marker
+done < <(jq -c '.[]' "$ISSUES_FILE")
+
+if [ "$GENERATED" = "0" ]; then
+  write_marker
+  exit 0
+fi
+
+# This is a new branch by construction. A concurrent branch creation makes this
+# ordinary push fail rather than overwriting the other author's ref.
+if ! git push origin "HEAD:refs/heads/$BRANCH"; then
+  pipe_log "Could not push $BRANCH safely; no MR/PR was created"
+  write_marker
+  exit 1
+fi
+{
+  printf '# Automated E2E test generation\n\n'
+  printf 'Generated and verified E2E tests for %s issue(s).\n' "$GENERATED"
+} > "$MR_BODY"
+MR_URL=$(mr_create_file "$BRANCH" "$PIPE_TARGET_BRANCH" "test: generated E2E coverage" "$MR_BODY" "" || true)
+if [ -n "$MR_URL" ]; then
+  MR_CREATED=1
+else
+  pipe_log "Push succeeded but MR/PR creation failed"
+fi
+write_marker

package/payload/ci-templates/scripts/lib/failure-notice.sh

@@ -0,0 +1,104 @@
+# shellcheck shell=bash
+# Source-only library. Failure notices distinguish two classes: model credit or
+# quota exhaustion, which cannot succeed until the balance is restored, and all
+# other agent failures, which may be transient or require manual follow-up.
+#
+# Both supported harness wordings are covered here: one reports that a credit
+# balance is too low, while the other reports that a request requires more
+# credits. Notices intentionally keep that implementation detail generic.
+
+set -u
+
+PIPE_CREDIT_EXHAUSTION_PATTERN='credit balance is too low|insufficient credits?|out of credits?|credits? exhausted|quota exceeded|requires more credits|payment required|402'
+
+pipe_is_credit_exhaustion() {
+  local text
+  for text in "$@"; do
+    [ -n "$text" ] && printf '%s\n' "$text" | grep -qiE "$PIPE_CREDIT_EXHAUSTION_PATTERN" && return 0
+  done
+  return 1
+}
+
+pipe_failure_notice() {
+  local task="$1" manual_action="$2"
+  shift 2
+
+  if pipe_is_credit_exhaustion "$@"; then
+    printf '**Automated %s could not run because the model credit balance is exhausted.**\n\n' "$task"
+    printf 'This is an automation outage, not a finding about your changes.\n\n'
+    printf 'Retrying the pipeline will not help until the balance is topped up.\n\n'
+    printf 'Please %s. Automation resumes on later pushes.\n' "$manual_action"
+  else
+    printf '**Automated %s failed to complete.**\n\n' "$task"
+    printf 'No result was produced. Check the CI job log for details.\n\n'
+    printf 'The failure may be transient, so retrying the pipeline is appropriate. If it continues, please %s.\n' "$manual_action"
+  fi
+}
+
+pipe_failure_log_notice() {
+  local task="$1" manual_action="$2"
+  shift 2
+
+  if pipe_is_credit_exhaustion "$@"; then
+    printf 'CREDIT EXHAUSTED: Automated %s could not run because the model credit balance is exhausted.\n' "$task"
+    printf 'Retrying will not help until the balance is topped up. Please %s.\n' "$manual_action"
+  else
+    printf 'FAILED: Automated %s failed to complete.\n' "$task"
+    printf 'Check stderr, retry if the failure is transient, and if it persists please %s.\n' "$manual_action"
+  fi
+}
+
+pipe_is_credit_exhaustion_files() {
+  local file
+  for file in "$@"; do
+    [ -s "$file" ] && grep -qiE "$PIPE_CREDIT_EXHAUSTION_PATTERN" "$file" && return 0
+  done
+  return 1
+}
+
+pipe_failure_notice_files() {
+  local task="$1" manual_action="$2"
+  shift 2
+
+  if pipe_is_credit_exhaustion_files "$@"; then
+    printf '**Automated %s could not run because the model credit balance is exhausted.**\n\n' "$task"
+    printf 'This is an automation outage, not a finding about your changes.\n\n'
+    printf 'Retrying the pipeline will not help until the balance is topped up.\n\n'
+    printf 'Please %s. Automation resumes on later pushes.\n' "$manual_action"
+  else
+    printf '**Automated %s failed to complete.**\n\n' "$task"
+    printf 'No result was produced. Check the CI job log for details.\n\n'
+    printf 'The failure may be transient, so retrying the pipeline is appropriate. If it continues, please %s.\n' "$manual_action"
+  fi
+}
+
+pipe_failure_log_notice_files() {
+  local task="$1" manual_action="$2"
+  shift 2
+
+  if pipe_is_credit_exhaustion_files "$@"; then
+    printf 'CREDIT EXHAUSTED: Automated %s could not run because the model credit balance is exhausted.\n' "$task"
+    printf 'Retrying will not help until the balance is topped up. Please %s.\n' "$manual_action"
+  else
+    printf 'FAILED: Automated %s failed to complete.\n' "$task"
+    printf 'Check stderr, retry if the failure is transient, and if it persists please %s.\n' "$manual_action"
+  fi
+}
+
+pipe_agent_failed() {
+  PIPE_AGENT_STDERR_TEXT=''
+  export PIPE_AGENT_STDERR_TEXT
+
+  if [ -f "${PIPE_AGENT_STDERR:-}" ]; then
+    PIPE_AGENT_STDERR_TEXT=$(cat "$PIPE_AGENT_STDERR" 2>/dev/null) || PIPE_AGENT_STDERR_TEXT=''
+    export PIPE_AGENT_STDERR_TEXT
+  fi
+
+  if [ -z "${1:-}" ]; then
+    [ -n "$PIPE_AGENT_STDERR_TEXT" ]
+  elif [ -n "$PIPE_AGENT_STDERR_TEXT" ]; then
+    return 1
+  else
+    return 0
+  fi
+}