@resolveio/server-lib 22.3.60 → 22.3.62

package/src/util/ai-runner-qa-tools.ts

@@ -0,0 +1,803 @@
+import { buildRunnerProcessJanitorShellLibrary } from './runner-process-janitor';
+
+export interface ResolveIORunnerQaToolBundleOptions {
+	mode?: 'support' | 'runner';
+	qaClientPort?: number | string;
+	defaultUsername?: string;
+	defaultPassword?: string;
+	jobId?: string;
+	ownerId?: string;
+	runnerToken?: string;
+	toolsBinPath?: string;
+	browserslistPath?: string;
+	mongodbBinaryCachePath?: string;
+	tmpRoot?: string;
+	homeRoot?: string;
+}
+
+function shellDoubleQuote(value: string): string {
+	return String(value || '').replace(/["\\$`]/g, '\\$&');
+}
+
+function normalizePort(value: number | string | undefined, fallback: string): string {
+	const parsed = Number.parseInt(String(value || ''), 10);
+	return Number.isFinite(parsed) && parsed > 0 ? String(parsed) : fallback;
+}
+
+function envVar(mode: 'support' | 'runner', suffix: string): string {
+	return mode === 'support' ? `RESOLVEIO_SUPPORT_QA_${suffix}` : `RESOLVEIO_RUNNER_QA_${suffix}`;
+}
+
+export function buildResolveIORunnerQaEnvScript(options: ResolveIORunnerQaToolBundleOptions = {}): string {
+	const mode = options.mode || 'runner';
+	const altMode = mode === 'support' ? 'runner' : 'support';
+	const tmpRoot = options.tmpRoot || (mode === 'support' ? '/tmp/resolveio-support-qa' : '/tmp/resolveio-ai-runner-qa');
+	const homeRoot = options.homeRoot || `${tmpRoot}/home`;
+	const defaultPort = normalizePort(options.qaClientPort, '4200');
+	const username = shellDoubleQuote(options.defaultUsername || 'admin');
+	const password = shellDoubleQuote(options.defaultPassword || '');
+	const jobId = shellDoubleQuote(options.jobId || '');
+	const ownerId = shellDoubleQuote(options.ownerId || '');
+	const runnerToken = shellDoubleQuote(options.runnerToken || '');
+	const toolsBinPath = options.toolsBinPath || '';
+	const browserslistPath = options.browserslistPath || '';
+	const mongodbBinaryCachePath = options.mongodbBinaryCachePath || '';
+	const clientPortVar = envVar(mode, 'CLIENT_PORT');
+	const altClientPortVar = envVar(altMode, 'CLIENT_PORT');
+	const clientUrlVar = envVar(mode, 'CLIENT_URL');
+	const altClientUrlVar = envVar(altMode, 'CLIENT_URL');
+	const serverUrlVar = envVar(mode, 'SERVER_URL');
+	const altServerUrlVar = envVar(altMode, 'SERVER_URL');
+	const usernameVar = envVar(mode, 'USERNAME');
+	const altUsernameVar = envVar(altMode, 'USERNAME');
+	const passwordVar = envVar(mode, 'PASSWORD');
+	const altPasswordVar = envVar(altMode, 'PASSWORD');
+	const viewportWidthVar = envVar(mode, 'VIEWPORT_WIDTH');
+	const altViewportWidthVar = envVar(altMode, 'VIEWPORT_WIDTH');
+	const viewportHeightVar = envVar(mode, 'VIEWPORT_HEIGHT');
+	const altViewportHeightVar = envVar(altMode, 'VIEWPORT_HEIGHT');
+	const timeoutVar = envVar(mode, 'ANGULAR_STARTUP_TIMEOUT_SECONDS');
+	const altTimeoutVar = envVar(altMode, 'ANGULAR_STARTUP_TIMEOUT_SECONDS');
+	const prebundleVar = envVar(mode, 'ANGULAR_PREBUNDLE');
+	const altPrebundleVar = envVar(altMode, 'ANGULAR_PREBUNDLE');
+	const reuseVar = envVar(mode, 'REUSE_RUNNING');
+	const altReuseVar = envVar(altMode, 'REUSE_RUNNING');
+	const keepaliveVar = envVar(mode, 'KEEPALIVE');
+	const altKeepaliveVar = envVar(altMode, 'KEEPALIVE');
+	const browserLoopVar = mode === 'support' ? 'RESOLVEIO_SUPPORT_QA_BROWSER' : 'RESOLVEIO_RUNNER_QA_BROWSER';
+	return [
+		`export ${mode === 'support' ? 'RESOLVEIO_SUPPORT_QA_TMP' : 'RESOLVEIO_RUNNER_QA_TMP'}="${'${' + (mode === 'support' ? 'RESOLVEIO_SUPPORT_QA_TMP' : 'RESOLVEIO_RUNNER_QA_TMP') + ':-' + tmpRoot + '}'}"`,
+		`export ${mode === 'support' ? 'RESOLVEIO_SUPPORT_QA_HOME' : 'RESOLVEIO_RUNNER_QA_HOME'}="${'${' + (mode === 'support' ? 'RESOLVEIO_SUPPORT_QA_HOME' : 'RESOLVEIO_RUNNER_QA_HOME') + ':-' + homeRoot + '}'}"`,
+		`RESOLVEIO_QA_TMP="${'${' + (mode === 'support' ? 'RESOLVEIO_SUPPORT_QA_TMP' : 'RESOLVEIO_RUNNER_QA_TMP') + '}'}"`,
+		`RESOLVEIO_QA_HOME="${'${' + (mode === 'support' ? 'RESOLVEIO_SUPPORT_QA_HOME' : 'RESOLVEIO_RUNNER_QA_HOME') + '}'}"`,
+		'mkdir -p "$RESOLVEIO_QA_HOME/.nvm" "$RESOLVEIO_QA_TMP/npm-cache" "$RESOLVEIO_QA_TMP/mongodb-binaries" "$RESOLVEIO_QA_TMP/mongodb-memory-server-core"',
+		'if [ ! -f "$RESOLVEIO_QA_HOME/.nvm/nvm.sh" ]; then',
+		'  cat > "$RESOLVEIO_QA_HOME/.nvm/nvm.sh" <<\'RESOLVEIO_NVM_SHIM\'',
+		'nvm() {',
+		'  case "$1" in',
+		'    use|install|alias) return 0 ;;',
+		'    current) node -v 2>/dev/null || true; return 0 ;;',
+		'    *) return 0 ;;',
+		'  esac',
+		'}',
+		'RESOLVEIO_NVM_SHIM',
+		'fi',
+		'export HOME="$RESOLVEIO_QA_HOME"',
+		'export NVM_DIR="$RESOLVEIO_QA_HOME/.nvm"',
+		toolsBinPath ? `export PATH="${toolsBinPath}:$PATH"` : '',
+		'export NODE_ENV=development',
+		'# Local QA must start the application HTTP server even when launched from a dedicated support-manager worker.',
+		'export IS_WORKERS_ENABLED=false',
+		'export IS_WORKER_INSTANCE=false',
+		'export DISABLE_WORKER_SERVER_CONNECTION=false',
+		'export SUPPORT_CODEX_MANAGER_PROCESS_ONLY=false',
+		'export SUPPORT_AUTO_MANAGER_PROCESS_ONLY=false',
+		'export AI_ASSISTANT_CODEX_MANAGER_PROCESS_ONLY=false',
+		browserslistPath ? `export BROWSERSLIST_CONFIG="${browserslistPath}"` : '',
+		`export ${clientPortVar}="${'${' + clientPortVar + ':-${' + altClientPortVar + ':-' + defaultPort + '}}'}"`,
+		`export ${altClientPortVar}="${'${' + altClientPortVar + ':-${' + clientPortVar + '}}'}"`,
+		`export ${clientUrlVar}="${'${' + clientUrlVar + ':-${' + altClientUrlVar + ':-http://localhost:${' + clientPortVar + '}}}'}"`,
+		`export ${altClientUrlVar}="${'${' + altClientUrlVar + ':-${' + clientUrlVar + '}}'}"`,
+		`export ${serverUrlVar}="${'${' + serverUrlVar + ':-${' + altServerUrlVar + ':-http://localhost:8080}}'}"`,
+		`export ${altServerUrlVar}="${'${' + altServerUrlVar + ':-${' + serverUrlVar + '}}'}"`,
+		'export ADDITIONAL_ALLOWED_ORIGINS="${ADDITIONAL_ALLOWED_ORIGINS:-$' + clientUrlVar + '}"',
+		`export ${viewportWidthVar}="${'${' + viewportWidthVar + ':-${' + altViewportWidthVar + ':-1920}}'}"`,
+		`export ${altViewportWidthVar}="${'${' + altViewportWidthVar + ':-${' + viewportWidthVar + '}}'}"`,
+		`export ${viewportHeightVar}="${'${' + viewportHeightVar + ':-${' + altViewportHeightVar + ':-1080}}'}"`,
+		`export ${altViewportHeightVar}="${'${' + altViewportHeightVar + ':-${' + viewportHeightVar + '}}'}"`,
+		`export ${timeoutVar}="${'${' + timeoutVar + ':-${' + altTimeoutVar + ':-900}}'}"`,
+		`export ${altTimeoutVar}="${'${' + altTimeoutVar + ':-${' + timeoutVar + '}}'}"`,
+		`export ${prebundleVar}="${'${' + prebundleVar + ':-${' + altPrebundleVar + ':-false}}'}"`,
+		`export ${altPrebundleVar}="${'${' + altPrebundleVar + ':-${' + prebundleVar + '}}'}"`,
+		`export ${reuseVar}="${'${' + reuseVar + ':-${' + altReuseVar + ':-true}}'}"`,
+		`export ${altReuseVar}="${'${' + altReuseVar + ':-${' + reuseVar + '}}'}"`,
+		`export ${keepaliveVar}="${'${' + keepaliveVar + ':-${' + altKeepaliveVar + ':-false}}'}"`,
+		`export ${altKeepaliveVar}="${'${' + altKeepaliveVar + ':-${' + keepaliveVar + '}}'}"`,
+		`export ${usernameVar}="${'${' + usernameVar + ':-${' + altUsernameVar + ':-' + username + '}}'}"`,
+		`export ${altUsernameVar}="${'${' + altUsernameVar + ':-${' + usernameVar + '}}'}"`,
+		`export ${passwordVar}="${'${' + passwordVar + ':-${' + altPasswordVar + ':-' + password + '}}'}"`,
+		`export ${altPasswordVar}="${'${' + altPasswordVar + ':-${' + passwordVar + '}}'}"`,
+		`export ${envVar(mode, 'JOB_ID')}="${'${' + envVar(mode, 'JOB_ID') + ':-${' + envVar(altMode, 'JOB_ID') + ':-' + jobId + '}}'}"`,
+		`export ${envVar(altMode, 'JOB_ID')}="${'${' + envVar(altMode, 'JOB_ID') + ':-${' + envVar(mode, 'JOB_ID') + '}}'}"`,
+		`export ${envVar(mode, 'OWNER_ID')}="${'${' + envVar(mode, 'OWNER_ID') + ':-${' + envVar(altMode, 'OWNER_ID') + ':-' + ownerId + '}}'}"`,
+		`export ${envVar(altMode, 'OWNER_ID')}="${'${' + envVar(altMode, 'OWNER_ID') + ':-${' + envVar(mode, 'OWNER_ID') + '}}'}"`,
+		`export ${envVar(mode, 'RUNNER_TOKEN')}="${'${' + envVar(mode, 'RUNNER_TOKEN') + ':-${' + envVar(altMode, 'RUNNER_TOKEN') + ':-' + runnerToken + '}}'}"`,
+		`export ${envVar(altMode, 'RUNNER_TOKEN')}="${'${' + envVar(altMode, 'RUNNER_TOKEN') + ':-${' + envVar(mode, 'RUNNER_TOKEN') + '}}'}"`,
+		'export PUPPETEER_CACHE_DIR="${PUPPETEER_CACHE_DIR:-/var/lib/resolveio/puppeteer}"',
+		'if [ ! -d "$PUPPETEER_CACHE_DIR" ] || [ ! -w "$PUPPETEER_CACHE_DIR" ]; then',
+		'  export PUPPETEER_CACHE_DIR="$RESOLVEIO_QA_TMP/puppeteer"',
+		'  mkdir -p "$PUPPETEER_CACHE_DIR"',
+		'fi',
+		'export PLAYWRIGHT_BROWSERS_PATH="${PLAYWRIGHT_BROWSERS_PATH:-$RESOLVEIO_QA_HOME/.cache/ms-playwright}"',
+		'if [ -n "${PUPPETEER_EXECUTABLE_PATH:-}" ] && [ -x "$PUPPETEER_EXECUTABLE_PATH" ]; then',
+		'  export CHROME_BIN="${CHROME_BIN:-$PUPPETEER_EXECUTABLE_PATH}"',
+		'elif [ -n "${CHROME_BIN:-}" ] && [ -x "$CHROME_BIN" ]; then',
+		'  export PUPPETEER_EXECUTABLE_PATH="$CHROME_BIN"',
+		'else',
+		`  for ${browserLoopVar} in "$PUPPETEER_CACHE_DIR"/chrome-headless-shell/linux-*/chrome-headless-shell-linux64/chrome-headless-shell "$PUPPETEER_CACHE_DIR"/chrome/linux-*/chrome-linux64/chrome "$PLAYWRIGHT_BROWSERS_PATH"/chromium_headless_shell-*/chrome-headless-shell-linux64/chrome-headless-shell "$PLAYWRIGHT_BROWSERS_PATH"/chromium-*/chrome-linux64/chrome /usr/bin/google-chrome-stable /usr/bin/google-chrome /usr/bin/chromium /usr/bin/chromium-browser; do`,
+		`    if [ -x "$${browserLoopVar}" ]; then`,
+		`      export PUPPETEER_EXECUTABLE_PATH="$${browserLoopVar}"`,
+		`      export CHROME_BIN="$${browserLoopVar}"`,
+		'      break',
+		'    fi',
+		'  done',
+		'fi',
+		`unset ${browserLoopVar}`,
+		'export NPM_CONFIG_CACHE="$RESOLVEIO_QA_TMP/npm-cache"',
+		'export NPM_CONFIG_PREFER_OFFLINE=true',
+		'export NPM_CONFIG_PRODUCTION=false',
+		'export npm_config_production=false',
+		'export RESOLVEIO_SUPPORT_MONGOMS_PACKAGE_ROOT="$RESOLVEIO_QA_TMP/mongodb-memory-server-core"',
+		'export MONGOMS_DOWNLOAD_DIR="$RESOLVEIO_QA_TMP/mongodb-binaries"',
+		mongodbBinaryCachePath ? `export RESOLVEIO_SUPPORT_SHARED_MONGOMS_DOWNLOAD_DIR="${mongodbBinaryCachePath}"` : '',
+		'export MONGOMS_VERSION="${MONGOMS_VERSION:-7.0.14}"',
+		''
+	].filter((line) => line !== '').join('\n');
+}
+
+export function buildResolveIORunnerLocalQaScript(): string {
+	return [
+		'#!/usr/bin/env bash',
+		'set -u',
+		'TOOLS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"',
+		'source "$TOOLS_DIR/env.sh"',
+		'PROJECT_ROOT="${1:-$(pwd)}"',
+		'PROJECT_ROOT="$(cd "$PROJECT_ROOT" && pwd)"',
+		'ARTIFACT_DIR="$PROJECT_ROOT/qa-artifacts"',
+		'mkdir -p "$ARTIFACT_DIR"',
+		'CLIENT_URL="${RESOLVEIO_RUNNER_QA_CLIENT_URL:-${RESOLVEIO_SUPPORT_QA_CLIENT_URL:-http://localhost:${RESOLVEIO_RUNNER_QA_CLIENT_PORT:-${RESOLVEIO_SUPPORT_QA_CLIENT_PORT:-4200}}}}"',
+		'SERVER_URL="${RESOLVEIO_RUNNER_QA_SERVER_URL:-${RESOLVEIO_SUPPORT_QA_SERVER_URL:-http://localhost:8080}}"',
+		'CLIENT_PORT="${RESOLVEIO_RUNNER_QA_CLIENT_PORT:-${RESOLVEIO_SUPPORT_QA_CLIENT_PORT:-4200}}"',
+		'SERVER_PORT="${RESOLVEIO_RUNNER_QA_SERVER_PORT:-${RESOLVEIO_SUPPORT_QA_SERVER_PORT:-8080}}"',
+		'MONGO_PORT="${RESOLVEIO_RUNNER_QA_MONGO_PORT:-${RESOLVEIO_SUPPORT_QA_MONGO_PORT:-3001}}"',
+		'INSPECT_PORT="${RESOLVEIO_RUNNER_QA_INSPECT_PORT:-${RESOLVEIO_SUPPORT_QA_INSPECT_PORT:-9229}}"',
+		'STARTUP_TIMEOUT="${RESOLVEIO_RUNNER_QA_ANGULAR_STARTUP_TIMEOUT_SECONDS:-${RESOLVEIO_SUPPORT_QA_ANGULAR_STARTUP_TIMEOUT_SECONDS:-900}}"',
+		'ANGULAR_PREBUNDLE="${RESOLVEIO_RUNNER_QA_ANGULAR_PREBUNDLE:-${RESOLVEIO_SUPPORT_QA_ANGULAR_PREBUNDLE:-false}}"',
+		'REUSE_RUNNING="${RESOLVEIO_RUNNER_QA_REUSE_RUNNING:-${RESOLVEIO_SUPPORT_QA_REUSE_RUNNING:-true}}"',
+		'if [[ "$(basename "$TOOLS_DIR")" == *support* ]]; then',
+		'  KEEPALIVE="${RESOLVEIO_SUPPORT_QA_KEEPALIVE:-${RESOLVEIO_RUNNER_QA_KEEPALIVE:-false}}"',
+		'else',
+		'  KEEPALIVE="${RESOLVEIO_RUNNER_QA_KEEPALIVE:-${RESOLVEIO_SUPPORT_QA_KEEPALIVE:-false}}"',
+		'fi',
+		'case "${KEEPALIVE,,}" in',
+		'  true|1|yes|on) exec >> "$ARTIFACT_DIR/runner.log" 2>&1 ;;',
+		'esac',
+		'SERVER_STABLE_SECONDS="${RESOLVEIO_RUNNER_QA_SERVER_STABLE_SECONDS:-${RESOLVEIO_SUPPORT_QA_SERVER_STABLE_SECONDS:-20}}"',
+		'LOCK_DIR="$ARTIFACT_DIR/.qa.lock"',
+		'SERVER_PID=""',
+		'CLIENT_PID=""',
+		'SERVER_REQUIRED=0',
+		'RUNNER_REUSED_READY=0',
+		'ANGULAR_PREBUNDLE_ARGS=()',
+		'REPO_ROOT="$(git -C "$PROJECT_ROOT" rev-parse --show-toplevel 2>/dev/null || echo "$PROJECT_ROOT")"',
+		buildRunnerProcessJanitorShellLibrary({ mode: 'support' }),
+		'case "${ANGULAR_PREBUNDLE,,}" in',
+		'  false|0|no|off) ANGULAR_PREBUNDLE_ARGS=(--prebundle=false) ;;',
+		'esac',
+		'kill_tree() {',
+		'  local pid="$1"',
+		'  [ -n "$pid" ] || return 0',
+		'  for child in $(pgrep -P "$pid" 2>/dev/null || true); do kill_tree "$child"; done',
+		'  kill "$pid" >/dev/null 2>&1 || true',
+		'  sleep 1',
+		'  kill -0 "$pid" >/dev/null 2>&1 && kill -9 "$pid" >/dev/null 2>&1 || true',
+		'}',
+		'kill_port_listeners() {',
+		'  local port="$1"',
+		'  [ -n "$port" ] || return 0',
+		'  local pids=""',
+		'  if command -v lsof >/dev/null 2>&1; then pids="$pids $(janitor_bounded 3 lsof -ti tcp:"$port")"; fi',
+		'  if command -v fuser >/dev/null 2>&1; then pids="$pids $(janitor_bounded 3 fuser -n tcp "$port")"; fi',
+		'  if command -v ss >/dev/null 2>&1; then pids="$pids $(janitor_bounded 3 ss -ltnp "sport = :$port" | sed -n \'s/.*pid=\\([0-9][0-9]*\\).*/\\1/p\' || true)"; fi',
+		'  for pid in $pids; do',
+		'    [ "$pid" = "$$" ] && continue',
+		'    kill_tree "$pid"',
+		'  done',
+		'  sleep 1',
+		'}',
+		'kill_env_marked_processes() {',
+		'  [ -d /proc ] || return 0',
+		'  local job_id="${RESOLVEIO_RUNNER_QA_JOB_ID:-${RESOLVEIO_SUPPORT_QA_JOB_ID:-}}"',
+		'  local owner_id="${RESOLVEIO_RUNNER_QA_OWNER_ID:-${RESOLVEIO_SUPPORT_QA_OWNER_ID:-}}"',
+		'  local token="${RESOLVEIO_RUNNER_QA_RUNNER_TOKEN:-${RESOLVEIO_SUPPORT_QA_RUNNER_TOKEN:-}}"',
+		'  [ -n "$job_id$owner_id$token" ] || return 0',
+		'  for env_file in /proc/[0-9]*/environ; do',
+		'    pid="${env_file#/proc/}"',
+		'    pid="${pid%/environ}"',
+		'    skip_cleanup_pid "$pid" && continue',
+		'    [ "$pid" = "$$" ] && continue',
+		'    [ -r "$env_file" ] || continue',
+		'    env_text="$(tr "\\0" "\\n" < "$env_file" 2>/dev/null || true)"',
+		'    [ -n "$env_text" ] || continue',
+		'    matched=0',
+		'    [ -n "$job_id" ] && echo "$env_text" | grep -Eq "^(RESOLVEIO_RUNNER_QA_JOB_ID|RESOLVEIO_SUPPORT_QA_JOB_ID)=$job_id$" && matched=1',
+		'    [ "$matched" = "0" ] && [ -n "$owner_id" ] && echo "$env_text" | grep -Eq "^(RESOLVEIO_RUNNER_QA_OWNER_ID|RESOLVEIO_SUPPORT_QA_OWNER_ID)=$owner_id$" && matched=1',
+		'    [ "$matched" = "0" ] && [ -n "$token" ] && echo "$env_text" | grep -Eq "^(RESOLVEIO_RUNNER_QA_RUNNER_TOKEN|RESOLVEIO_SUPPORT_QA_RUNNER_TOKEN)=$token$" && matched=1',
+		'    [ "$matched" = "1" ] && kill_tree "$pid"',
+		'  done',
+		'}',
+		'RUNNER_ANCESTOR_PIDS=" $$ ${PPID:-} "',
+		'ancestor_pid="${PPID:-}"',
+		'while [ -n "$ancestor_pid" ] && [ "$ancestor_pid" != "0" ] && [ "$ancestor_pid" != "1" ]; do',
+		'  ancestor_pid="$(ps -o ppid= -p "$ancestor_pid" 2>/dev/null | tr -d " " || true)"',
+		'  [ -n "$ancestor_pid" ] && RUNNER_ANCESTOR_PIDS="$RUNNER_ANCESTOR_PIDS $ancestor_pid "',
+		'done',
+		'skip_cleanup_pid() {',
+		'  case "$RUNNER_ANCESTOR_PIDS" in *" $1 "*) return 0 ;; esac',
+		'  return 1',
+		'}',
+		'cleanup_project_processes() {',
+		'  for pid_file in "$ARTIFACT_DIR/server.pid" "$ARTIFACT_DIR/client.pid"; do',
+		'    [ -f "$pid_file" ] || continue',
+		'    pid="$(cat "$pid_file" 2>/dev/null || true)"',
+		'    kill_tree "$pid"',
+		'    rm -f "$pid_file"',
+		'  done',
+		'  for pass in 1 2 3; do',
+		'    for port in "$CLIENT_PORT" "$SERVER_PORT" "$MONGO_PORT" "$INSPECT_PORT"; do kill_port_listeners "$port"; done',
+		'    kill_env_marked_processes',
+		'    for pid in $(ps -eo pid=,args= | awk -v root="$PROJECT_ROOT" \'index($0, root) && $0 !~ /awk -v root=/ && $0 !~ /ps -eo pid=,args=/ && $0 !~ /run-local-qa\\.sh|stop-local-qa\\.sh|bugfix-comparison-qa\\.sh/ && $0 ~ /(ng serve|node .*node_modules\\/\\.bin\\/ng|esbuild|npm run client|start_client\\.sh|npm run server|start_server\\.sh|nodemon|node .*tmp\\/index\\.js|mongod|mongodb-binaries\\/mongod)/ {print $1}\' 2>/dev/null || true); do',
+		'      skip_cleanup_pid "$pid" && continue',
+		'      kill_tree "$pid"',
+		'    done',
+		'    if [ -d /proc ]; then',
+		'      for proc_cwd in /proc/[0-9]*/cwd; do',
+		'        pid="${proc_cwd#/proc/}"',
+		'        pid="${pid%/cwd}"',
+		'        skip_cleanup_pid "$pid" && continue',
+		'        cwd="$(readlink -f "$proc_cwd" 2>/dev/null || true)"',
+		'        case "$cwd" in',
+		'          "$PROJECT_ROOT"|"$PROJECT_ROOT"/*) kill_tree "$pid" ;;',
+		'        esac',
+		'      done',
+		'    fi',
+		'    sleep 1',
+		'  done',
+		'  local wait_until=$((SECONDS + 60))',
+		'  while [ "$SECONDS" -lt "$wait_until" ]; do',
+		'    local found=0',
+		'    for port in "$CLIENT_PORT" "$SERVER_PORT" "$MONGO_PORT" "$INSPECT_PORT"; do',
+		'      if command -v lsof >/dev/null 2>&1 && [ -n "$(janitor_bounded 3 lsof -ti tcp:"$port")" ]; then found=1; fi',
+		'    done',
+		'    if [ -d /proc ]; then',
+		'      job_id="${RESOLVEIO_RUNNER_QA_JOB_ID:-${RESOLVEIO_SUPPORT_QA_JOB_ID:-}}"',
+		'      owner_id="${RESOLVEIO_RUNNER_QA_OWNER_ID:-${RESOLVEIO_SUPPORT_QA_OWNER_ID:-}}"',
+		'      token="${RESOLVEIO_RUNNER_QA_RUNNER_TOKEN:-${RESOLVEIO_SUPPORT_QA_RUNNER_TOKEN:-}}"',
+		'      for env_file in /proc/[0-9]*/environ; do',
+		'        pid="${env_file#/proc/}"',
+		'        pid="${pid%/environ}"',
+		'        skip_cleanup_pid "$pid" && continue',
+		'        [ -r "$env_file" ] || continue',
+		'        env_text="$(tr "\\0" "\\n" < "$env_file" 2>/dev/null || true)"',
+		'        [ -n "$job_id" ] && echo "$env_text" | grep -Eq "^(RESOLVEIO_RUNNER_QA_JOB_ID|RESOLVEIO_SUPPORT_QA_JOB_ID)=$job_id$" && found=1',
+		'        [ -n "$owner_id" ] && echo "$env_text" | grep -Eq "^(RESOLVEIO_RUNNER_QA_OWNER_ID|RESOLVEIO_SUPPORT_QA_OWNER_ID)=$owner_id$" && found=1',
+		'        [ -n "$token" ] && echo "$env_text" | grep -Eq "^(RESOLVEIO_RUNNER_QA_RUNNER_TOKEN|RESOLVEIO_SUPPORT_QA_RUNNER_TOKEN)=$token$" && found=1',
+		'      done',
+		'    fi',
+		'    for pid in $(ps -eo pid=,args= | awk -v root="$PROJECT_ROOT" \'index($0, root) && $0 !~ /awk -v root=/ && $0 !~ /ps -eo pid=,args=/ && $0 !~ /run-local-qa\\.sh|stop-local-qa\\.sh|bugfix-comparison-qa\\.sh/ && $0 ~ /(ng serve|node .*node_modules\\/\\.bin\\/ng|esbuild|npm run client|start_client\\.sh|npm run server|start_server\\.sh|nodemon|node .*tmp\\/index\\.js|mongod|mongodb-binaries\\/mongod)/ {print $1}\' 2>/dev/null || true); do',
+		'      skip_cleanup_pid "$pid" && continue',
+		'      found=1',
+		'    done',
+		'    [ "$found" = "0" ] && break',
+		'    sleep 2',
+		'  done',
+		'}',
+		'cleanup() {',
+		'  [ "${RUNNER_REUSED_READY:-0}" = "1" ] && return 0',
+		'  janitor_stop_heartbeat',
+		'  rm -f "$ARTIFACT_DIR/heartbeat.pid"',
+		'  janitor_update_manifest_status cleanup_started',
+		'  kill_tree "$SERVER_PID"',
+		'  kill_tree "$CLIENT_PID"',
+		'  cleanup_project_processes',
+		'  janitor_update_manifest_status cleanup_complete',
+		'  janitor_write_cleanup_status complete 0',
+		'  rm -rf "$LOCK_DIR" >/dev/null 2>&1 || true',
+		'}',
+		'trap cleanup EXIT',
+		'detach_keepalive() {',
+		'  janitor_update_manifest_status ready_keepalive',
+		'  janitor_write_cleanup_status ready_keepalive 0',
+		'  rm -rf "$LOCK_DIR" >/dev/null 2>&1 || true',
+		'  trap - EXIT',
+		'  disown >/dev/null 2>&1 || true',
+		'  echo "ResolveIO AI runner QA keepalive detached at $CLIENT_URL; stop with $TOOLS_DIR/stop-local-qa.sh $PROJECT_ROOT."',
+		'  exit 0',
+		'}',
+		'probe_url() { node - "$1" <<\'RESOLVEIO_PROBE\'',
+		'const http = require("http");',
+		'const https = require("https");',
+		'const url = process.argv[2];',
+		'const mod = /^https:/i.test(url) ? https : http;',
+		'const req = mod.get(url, { timeout: 2500 }, (res) => { res.resume(); process.exit(res.statusCode && res.statusCode < 500 ? 0 : 1); });',
+		'req.on("timeout", () => req.destroy(new Error("timeout")));',
+		'req.on("error", () => process.exit(1));',
+		'RESOLVEIO_PROBE',
+		'}',
+		'truthy() { case "${1,,}" in true|1|yes|on) return 0 ;; *) return 1 ;; esac; }',
+		'reuse_running_app_if_ready() {',
+		'  truthy "$REUSE_RUNNING" || return 1',
+		'  [ -d "$PROJECT_ROOT/server" ] && SERVER_REQUIRED=1 || SERVER_REQUIRED=0',
+		'  probe_url "$CLIENT_URL" || return 1',
+		'  if [ "$SERVER_REQUIRED" = "1" ] && ! probe_url "$SERVER_URL"; then return 1; fi',
+		'  echo "ResolveIO AI runner QA reusing already-ready local app at $CLIENT_URL"',
+		'  return 0',
+		'}',
+		'log_has_fatal() {',
+		'  local file="$1"',
+		'  [ -f "$file" ] || return 1',
+		'  grep -Eiq "Unhandled Rejection|Cannot read properties of undefined|TypeError|ReferenceError|EADDRINUSE|app crashed|Failed to compile|Error: Cannot find module|NG[0-9]{4}|TS[0-9]{4}" "$file"',
+		'}',
+		'prepare_angular_cache_dirs() {',
+		'  [ -d "$PROJECT_ROOT" ] || return 0',
+		'  mkdir -p "$PROJECT_ROOT/.angular/cache" 2>/dev/null || true',
+		'  local version=""',
+		'  for pkg in "$PROJECT_ROOT/node_modules/@angular/build/package.json" "$PROJECT_ROOT/node_modules/@angular/cli/package.json" "$PROJECT_ROOT/node_modules/@angular-devkit/build-angular/package.json"; do',
+		'    if [ -f "$pkg" ]; then',
+		'      version="$(node -e "try{console.log(require(process.argv[1]).version||\\\"\\\")}catch(e){}" "$pkg" 2>/dev/null | head -1)"',
+		'      [ -n "$version" ] && break',
+		'    fi',
+		'  done',
+		'  [ -n "$version" ] || return 0',
+		'  node - "$PROJECT_ROOT/angular.json" "$(basename "$PROJECT_ROOT")" <<\'RESOLVEIO_ANGULAR_CACHE_PROJECTS\' | while IFS= read -r project; do',
+		'const fs = require("fs");',
+		'const path = process.argv[2];',
+		'const fallback = process.argv[3];',
+		'const names = new Set([fallback].filter(Boolean));',
+		'try {',
+		'  const parsed = JSON.parse(fs.readFileSync(path, "utf8"));',
+		'  if (parsed.defaultProject) names.add(String(parsed.defaultProject));',
+		'  if (parsed.projects && typeof parsed.projects === "object") Object.keys(parsed.projects).forEach((name) => names.add(String(name)));',
+		'} catch (error) {}',
+		'for (const name of names) console.log(name.replace(/[^A-Za-z0-9._-]/g, "_"));',
+		'RESOLVEIO_ANGULAR_CACHE_PROJECTS',
+		'    [ -n "$project" ] || continue',
+		'    mkdir -p "$PROJECT_ROOT/.angular/cache/$version/$project/vite" 2>/dev/null || true',
+		'  done',
+		'}',
+		'server_has_started() {',
+		'  local file="$ARTIFACT_DIR/server.log"',
+		'  [ -f "$file" ] || return 1',
+		'  grep -Eiq "nodemon.*starting|node .*tmp/index\\.js|Running as Worker|Standalone Node Reaper|listening on|server listening|Server listening|app listening|App listening|Finished .default." "$file"',
+		'}',
+		'wait_for_server_ready() {',
+		'  [ "$SERVER_REQUIRED" = "1" ] || return 0',
+		'  local end=$((SECONDS + STARTUP_TIMEOUT))',
+		'  while [ "$SECONDS" -lt "$end" ]; do',
+		'    if log_has_fatal "$ARTIFACT_DIR/server.log"; then return 4; fi',
+		'    if server_has_started; then',
+		'      local stable_until=$((SECONDS + SERVER_STABLE_SECONDS))',
+		'      while [ "$SECONDS" -lt "$stable_until" ]; do if log_has_fatal "$ARTIFACT_DIR/server.log"; then return 4; fi; sleep 2; done',
+		'      return 0',
+		'    fi',
+		'    if probe_url "$SERVER_URL"; then',
+		'      local stable_until=$((SECONDS + SERVER_STABLE_SECONDS))',
+		'      while [ "$SECONDS" -lt "$stable_until" ]; do if log_has_fatal "$ARTIFACT_DIR/server.log"; then return 4; fi; sleep 2; done',
+		'      return 0',
+		'    fi',
+		'    sleep 5',
+		'  done',
+		'  return 4',
+		'}',
+		'wait_for_client() {',
+		'  local end=$((SECONDS + STARTUP_TIMEOUT))',
+		'  while [ "$SECONDS" -lt "$end" ]; do',
+		'    if [ -n "$CLIENT_PID" ] && ! kill -0 "$CLIENT_PID" >/dev/null 2>&1; then return 2; fi',
+		'    if log_has_fatal "$ARTIFACT_DIR/client.log"; then return 3; fi',
+		'    if log_has_fatal "$ARTIFACT_DIR/server.log"; then return 4; fi',
+		'    if probe_url "$CLIENT_URL"; then wait_for_server_ready || return $?; return 0; fi',
+		'    sleep 5',
+		'  done',
+		'  return 1',
+		'}',
+		'if reuse_running_app_if_ready; then RUNNER_REUSED_READY=1; exit 0; fi',
+		'if ! mkdir "$LOCK_DIR" 2>/dev/null; then',
+		'  if janitor_lock_is_live; then',
+		'    echo "ResolveIO AI runner QA lock is already held by a live runner for $PROJECT_ROOT; refusing duplicate startup." | tee "$ARTIFACT_DIR/runner.log"',
+		'    exit 6',
+		'  fi',
+		'  echo "ResolveIO AI runner QA lock is stale for $PROJECT_ROOT; cleaning scoped local QA processes before retrying." | tee "$ARTIFACT_DIR/runner.log"',
+		'  cleanup_project_processes',
+		'  rm -rf "$LOCK_DIR" >/dev/null 2>&1 || true',
+		'  if ! mkdir "$LOCK_DIR" 2>/dev/null; then',
+		'    echo "ResolveIO AI runner QA lock is still held for $PROJECT_ROOT after cleanup. Stop the existing QA runner before starting another." | tee -a "$ARTIFACT_DIR/runner.log"',
+		'    exit 6',
+		'  fi',
+		'fi',
+		'janitor_write_lock',
+		'cleanup_project_processes',
+		': > "$ARTIFACT_DIR/server.log"',
+		': > "$ARTIFACT_DIR/client.log"',
+		': > "$ARTIFACT_DIR/runner.log"',
+		'janitor_write_manifest',
+		'janitor_start_heartbeat',
+		'echo "$RUNNER_JANITOR_HEARTBEAT_PID" > "$ARTIFACT_DIR/heartbeat.pid"',
+		'janitor_check_resources || exit $?',
+		'if [ -d "$PROJECT_ROOT/server" ]; then',
+		'  SERVER_REQUIRED=1',
+		'  if node -e "const p=require(process.argv[1]); process.exit(p.scripts&&p.scripts.server?0:1)" "$PROJECT_ROOT/server/package.json" >/dev/null 2>&1; then',
+		'    (cd "$PROJECT_ROOT/server" && source "$TOOLS_DIR/env.sh" && npm run server 2>&1 | tee "$ARTIFACT_DIR/server.log") &',
+		'  elif [ -x "$PROJECT_ROOT/server/start_server.sh" ]; then',
+		'    (cd "$PROJECT_ROOT/server" && source "$TOOLS_DIR/env.sh" && ./start_server.sh 2>&1 | tee "$ARTIFACT_DIR/server.log") &',
+		'  else',
+		'    echo "ResolveIO AI runner QA cannot find npm server script or start_server.sh for $PROJECT_ROOT/server" | tee "$ARTIFACT_DIR/server.log"',
+		'    exit 5',
+		'  fi',
+		'  SERVER_PID=$!',
+		'  wait_for_server_ready',
+		'  SERVER_RESULT=$?',
+		'  if [ "$SERVER_RESULT" != "0" ]; then echo "ResolveIO AI runner QA server startup fatal error. See $ARTIFACT_DIR/server.log"; exit "$SERVER_RESULT"; fi',
+		'fi',
+		'CLIENT_HOST="${RESOLVEIO_RUNNER_QA_CLIENT_HOST:-${RESOLVEIO_SUPPORT_QA_CLIENT_HOST:-0.0.0.0}}"',
+		'if [ -x "$PROJECT_ROOT/node_modules/.bin/ng" ]; then',
+		'  prepare_angular_cache_dirs',
+		'  (cd "$PROJECT_ROOT" && source "$TOOLS_DIR/env.sh" && node --max_old_space_size=8048 ./node_modules/.bin/ng serve --watch --configuration local --host "$CLIENT_HOST" --port "$CLIENT_PORT" "${ANGULAR_PREBUNDLE_ARGS[@]}" 2>&1 | tee "$ARTIFACT_DIR/client.log") &',
+		'elif [ -x "$PROJECT_ROOT/start_client.sh" ]; then',
+		'  (cd "$PROJECT_ROOT" && source "$TOOLS_DIR/env.sh" && ./start_client.sh 2>&1 | tee "$ARTIFACT_DIR/client.log") &',
+		'elif node -e "const p=require(process.argv[1]); process.exit(p.scripts&&p.scripts.client?0:1)" "$PROJECT_ROOT/package.json" >/dev/null 2>&1; then',
+		'  (cd "$PROJECT_ROOT" && source "$TOOLS_DIR/env.sh" && npm run client 2>&1 | tee "$ARTIFACT_DIR/client.log") &',
+		'else',
+		'  echo "ResolveIO AI runner QA cannot find Angular CLI, start_client.sh, or npm client script for $PROJECT_ROOT" | tee "$ARTIFACT_DIR/client.log"',
+		'  exit 5',
+		'fi',
+		'CLIENT_PID=$!',
+		'wait_for_client',
+		'RESULT=$?',
+		'case "$RESULT" in',
+		'  0)',
+		'    echo "ResolveIO AI runner QA local app ready at $CLIENT_URL";',
+		'    echo "$SERVER_PID" > "$ARTIFACT_DIR/server.pid";',
+		'    echo "$CLIENT_PID" > "$ARTIFACT_DIR/client.pid";',
+		'    if truthy "$KEEPALIVE"; then detach_keepalive; fi;',
+		'    exit 0',
+		'    ;;',
+		'  2) echo "ResolveIO AI runner QA client process exited before $CLIENT_URL became ready. See $ARTIFACT_DIR/client.log"; exit 2 ;;',
+		'  3) echo "ResolveIO AI runner QA client startup fatal error. See $ARTIFACT_DIR/client.log"; exit 3 ;;',
+		'  4) echo "ResolveIO AI runner QA server startup fatal error. See $ARTIFACT_DIR/server.log"; exit 4 ;;',
+		'  *) echo "ResolveIO AI runner QA local app did not become ready at $CLIENT_URL within ${STARTUP_TIMEOUT}s. See $ARTIFACT_DIR/client.log and $ARTIFACT_DIR/server.log"; exit 1 ;;',
+		'esac',
+		''
+	].join('\n');
+}
+
+export function buildResolveIORunnerLocalQaStopperScript(): string {
+	return [
+		'#!/usr/bin/env bash',
+		'set -u',
+		'TOOLS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"',
+		'source "$TOOLS_DIR/env.sh"',
+		'PROJECT_ROOT="${1:-$(pwd)}"',
+		'PROJECT_ROOT="$(cd "$PROJECT_ROOT" && pwd)"',
+		'ARTIFACT_DIR="$PROJECT_ROOT/qa-artifacts"',
+		'CLIENT_PORT="${RESOLVEIO_RUNNER_QA_CLIENT_PORT:-${RESOLVEIO_SUPPORT_QA_CLIENT_PORT:-4200}}"',
+		'SERVER_PORT="${RESOLVEIO_RUNNER_QA_SERVER_PORT:-${RESOLVEIO_SUPPORT_QA_SERVER_PORT:-8080}}"',
+		'MONGO_PORT="${RESOLVEIO_RUNNER_QA_MONGO_PORT:-${RESOLVEIO_SUPPORT_QA_MONGO_PORT:-3001}}"',
+		'INSPECT_PORT="${RESOLVEIO_RUNNER_QA_INSPECT_PORT:-${RESOLVEIO_SUPPORT_QA_INSPECT_PORT:-9229}}"',
+		'REPO_ROOT="$(git -C "$PROJECT_ROOT" rev-parse --show-toplevel 2>/dev/null || echo "$PROJECT_ROOT")"',
+		buildRunnerProcessJanitorShellLibrary({ mode: 'support' }),
+		'STOP_LOCK_DIR="$ARTIFACT_DIR/.qa.stop.lock"',
+		'if ! mkdir "$STOP_LOCK_DIR" 2>/dev/null; then',
+		'  lock_pid="$(cat "$STOP_LOCK_DIR/pid" 2>/dev/null || true)"',
+		'  if [ -n "$lock_pid" ] && kill -0 "$lock_pid" >/dev/null 2>&1; then',
+		'    echo "ResolveIO AI runner QA cleanup already active for $PROJECT_ROOT; refusing duplicate stopper."',
+		'    exit 0',
+		'  fi',
+		'  rm -rf "$STOP_LOCK_DIR" >/dev/null 2>&1 || true',
+		'  mkdir "$STOP_LOCK_DIR" 2>/dev/null || { echo "ResolveIO AI runner QA cleanup lock still active for $PROJECT_ROOT."; exit 0; }',
+		'fi',
+		'echo "$$" > "$STOP_LOCK_DIR/pid" 2>/dev/null || true',
+		'trap \'rm -rf "$STOP_LOCK_DIR" >/dev/null 2>&1 || true\' EXIT',
+		'kill_tree() {',
+		'  local pid="$1"',
+		'  [ -n "$pid" ] || return 0',
+		'  kill -0 "$pid" >/dev/null 2>&1 || return 0',
+		'  for child in $(pgrep -P "$pid" 2>/dev/null || true); do kill_tree "$child"; done',
+		'  kill "$pid" >/dev/null 2>&1 || true',
+		'  sleep 1',
+		'  kill -0 "$pid" >/dev/null 2>&1 && kill -9 "$pid" >/dev/null 2>&1 || true',
+		'}',
+		'kill_port_listeners() {',
+		'  local port="$1"',
+		'  [ -n "$port" ] || return 0',
+		'  local pids=""',
+		'  if command -v lsof >/dev/null 2>&1; then pids="$pids $(janitor_bounded 3 lsof -ti tcp:"$port")"; fi',
+		'  if command -v fuser >/dev/null 2>&1; then pids="$pids $(janitor_bounded 3 fuser -n tcp "$port")"; fi',
+		'  if command -v ss >/dev/null 2>&1; then pids="$pids $(janitor_bounded 3 ss -ltnp "sport = :$port" | sed -n \'s/.*pid=\\([0-9][0-9]*\\).*/\\1/p\' || true)"; fi',
+		'  for pid in $pids; do',
+		'    [ "$pid" = "$$" ] && continue',
+		'    kill_tree "$pid"',
+		'  done',
+		'  sleep 1',
+		'}',
+		'kill_env_marked_processes() {',
+		'  [ -d /proc ] || return 0',
+		'  local job_id="${RESOLVEIO_RUNNER_QA_JOB_ID:-${RESOLVEIO_SUPPORT_QA_JOB_ID:-}}"',
+		'  local owner_id="${RESOLVEIO_RUNNER_QA_OWNER_ID:-${RESOLVEIO_SUPPORT_QA_OWNER_ID:-}}"',
+		'  local token="${RESOLVEIO_RUNNER_QA_RUNNER_TOKEN:-${RESOLVEIO_SUPPORT_QA_RUNNER_TOKEN:-}}"',
+		'  [ -n "$job_id$owner_id$token" ] || return 0',
+		'  for env_file in /proc/[0-9]*/environ; do',
+		'    pid="${env_file#/proc/}"',
+		'    pid="${pid%/environ}"',
+		'    skip_cleanup_pid "$pid" && continue',
+		'    [ "$pid" = "$$" ] && continue',
+		'    [ -r "$env_file" ] || continue',
+		'    env_text="$(tr "\\0" "\\n" < "$env_file" 2>/dev/null || true)"',
+		'    [ -n "$env_text" ] || continue',
+		'    matched=0',
+		'    [ -n "$job_id" ] && echo "$env_text" | grep -Eq "^(RESOLVEIO_RUNNER_QA_JOB_ID|RESOLVEIO_SUPPORT_QA_JOB_ID)=$job_id$" && matched=1',
+		'    [ "$matched" = "0" ] && [ -n "$owner_id" ] && echo "$env_text" | grep -Eq "^(RESOLVEIO_RUNNER_QA_OWNER_ID|RESOLVEIO_SUPPORT_QA_OWNER_ID)=$owner_id$" && matched=1',
+		'    [ "$matched" = "0" ] && [ -n "$token" ] && echo "$env_text" | grep -Eq "^(RESOLVEIO_RUNNER_QA_RUNNER_TOKEN|RESOLVEIO_SUPPORT_QA_RUNNER_TOKEN)=$token$" && matched=1',
+		'    [ "$matched" = "1" ] && kill_tree "$pid" && killed_count=$((killed_count + 1))',
+		'  done',
+		'}',
+		'RUNNER_ANCESTOR_PIDS=" $$ ${PPID:-} "',
+		'ancestor_pid="${PPID:-}"',
+		'while [ -n "$ancestor_pid" ] && [ "$ancestor_pid" != "0" ] && [ "$ancestor_pid" != "1" ]; do',
+		'  ancestor_pid="$(ps -o ppid= -p "$ancestor_pid" 2>/dev/null | tr -d " " || true)"',
+		'  [ -n "$ancestor_pid" ] && RUNNER_ANCESTOR_PIDS="$RUNNER_ANCESTOR_PIDS $ancestor_pid "',
+		'done',
+		'skip_cleanup_pid() {',
+		'  case "$RUNNER_ANCESTOR_PIDS" in *" $1 "*) return 0 ;; esac',
+		'  return 1',
+		'}',
+		'janitor_update_manifest_status cleanup_started',
+		'killed_count=0',
+'for pid_file in "$ARTIFACT_DIR/heartbeat.pid" "$ARTIFACT_DIR/server.pid" "$ARTIFACT_DIR/client.pid"; do',
+		'  [ -f "$pid_file" ] || continue',
+		'  pid="$(cat "$pid_file" 2>/dev/null || true)"',
+		'  kill_tree "$pid"',
+		'  killed_count=$((killed_count + 1))',
+		'  rm -f "$pid_file"',
+		'done',
+		'for pass in 1 2 3; do',
+		'  for port in "$CLIENT_PORT" "$SERVER_PORT" "$MONGO_PORT" "$INSPECT_PORT"; do kill_port_listeners "$port"; done',
+		'  kill_env_marked_processes',
+		'  for pid in $(ps -eo pid=,args= | awk -v root="$PROJECT_ROOT" \'index($0, root) && $0 !~ /awk -v root=/ && $0 !~ /ps -eo pid=,args=/ && $0 !~ /run-local-qa\\.sh|stop-local-qa\\.sh|bugfix-comparison-qa\\.sh/ && $0 ~ /(ng serve|node .*node_modules\\/\\.bin\\/ng|esbuild|npm run client|start_client\\.sh|npm run server|start_server\\.sh|nodemon|node .*tmp\\/index\\.js|mongod|mongodb-binaries\\/mongod)/ {print $1}\' 2>/dev/null || true); do',
+		'    skip_cleanup_pid "$pid" && continue',
+		'    kill_tree "$pid"',
+		'    killed_count=$((killed_count + 1))',
+		'  done',
+		'  if [ -d /proc ]; then',
+		'    for proc_cwd in /proc/[0-9]*/cwd; do',
+		'      pid="${proc_cwd#/proc/}"',
+		'      pid="${pid%/cwd}"',
+		'      skip_cleanup_pid "$pid" && continue',
+		'      cwd="$(readlink -f "$proc_cwd" 2>/dev/null || true)"',
+		'      case "$cwd" in',
+		'        "$PROJECT_ROOT"|"$PROJECT_ROOT"/*) kill_tree "$pid" ;;',
+		'      esac',
+		'    done',
+		'  fi',
+		'  sleep 1',
+		'done',
+		'wait_until=$((SECONDS + 60))',
+	'while [ "$SECONDS" -lt "$wait_until" ]; do',
+		'  found=0',
+		'  for port in "$CLIENT_PORT" "$SERVER_PORT" "$MONGO_PORT" "$INSPECT_PORT"; do',
+		'    if command -v lsof >/dev/null 2>&1 && [ -n "$(janitor_bounded 3 lsof -ti tcp:"$port")" ]; then found=1; fi',
+		'  done',
+		'  if [ -d /proc ]; then',
+		'    job_id="${RESOLVEIO_RUNNER_QA_JOB_ID:-${RESOLVEIO_SUPPORT_QA_JOB_ID:-}}"',
+		'    owner_id="${RESOLVEIO_RUNNER_QA_OWNER_ID:-${RESOLVEIO_SUPPORT_QA_OWNER_ID:-}}"',
+		'    token="${RESOLVEIO_RUNNER_QA_RUNNER_TOKEN:-${RESOLVEIO_SUPPORT_QA_RUNNER_TOKEN:-}}"',
+		'    for env_file in /proc/[0-9]*/environ; do',
+		'      pid="${env_file#/proc/}"',
+		'      pid="${pid%/environ}"',
+		'      skip_cleanup_pid "$pid" && continue',
+		'      [ -r "$env_file" ] || continue',
+		'      env_text="$(tr "\\0" "\\n" < "$env_file" 2>/dev/null || true)"',
+		'      [ -n "$job_id" ] && echo "$env_text" | grep -Eq "^(RESOLVEIO_RUNNER_QA_JOB_ID|RESOLVEIO_SUPPORT_QA_JOB_ID)=$job_id$" && found=1',
+		'      [ -n "$owner_id" ] && echo "$env_text" | grep -Eq "^(RESOLVEIO_RUNNER_QA_OWNER_ID|RESOLVEIO_SUPPORT_QA_OWNER_ID)=$owner_id$" && found=1',
+		'      [ -n "$token" ] && echo "$env_text" | grep -Eq "^(RESOLVEIO_RUNNER_QA_RUNNER_TOKEN|RESOLVEIO_SUPPORT_QA_RUNNER_TOKEN)=$token$" && found=1',
+		'    done',
+		'  fi',
+		'  for pid in $(ps -eo pid=,args= | awk -v root="$PROJECT_ROOT" \'index($0, root) && $0 !~ /awk -v root=/ && $0 !~ /ps -eo pid=,args=/ && $0 !~ /run-local-qa\\.sh|stop-local-qa\\.sh|bugfix-comparison-qa\\.sh/ && $0 ~ /(ng serve|node .*node_modules\\/\\.bin\\/ng|esbuild|npm run client|start_client\\.sh|npm run server|start_server\\.sh|nodemon|node .*tmp\\/index\\.js|mongod|mongodb-binaries\\/mongod)/ {print $1}\' 2>/dev/null || true); do',
+		'    skip_cleanup_pid "$pid" && continue',
+		'    found=1',
+		'  done',
+		'  [ "$found" = "0" ] && break',
+		'  sleep 2',
+		'done',
+		'rm -rf "$ARTIFACT_DIR/.qa.lock" >/dev/null 2>&1 || true',
+		'janitor_update_manifest_status stopped',
+		'janitor_write_cleanup_status stopped "$killed_count"',
+		'echo "ResolveIO AI runner QA local app stopped for $PROJECT_ROOT"',
+		''
+	].join('\n');
+}
+
+export function buildResolveIORunnerBugfixComparisonQaScript(): string {
+	return [
+		'#!/usr/bin/env bash',
+		'set -u',
+		'TOOLS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"',
+		'source "$TOOLS_DIR/env.sh"',
+		'PROJECT_ROOT="${1:-}"',
+		'if [ -z "$PROJECT_ROOT" ]; then',
+		'  echo "Usage: $0 <project-root> [baseline-ref] -- <qa-command...>" >&2',
+		'  exit 2',
+		'fi',
+		'shift || true',
+		'DEFAULT_BASELINE_REF="${RESOLVEIO_RUNNER_QA_BASELINE_REF:-${RESOLVEIO_SUPPORT_QA_BASELINE_REF:-origin/master}}"',
+		'if [ "${1:-}" = "--" ]; then',
+		'  BASELINE_REF="$DEFAULT_BASELINE_REF"',
+		'else',
+		'  BASELINE_REF="${1:-$DEFAULT_BASELINE_REF}"',
+		'  if [ "$#" -gt 0 ]; then shift || true; fi',
+		'fi',
+		'if [ "${1:-}" = "--" ]; then shift; fi',
+		'if [ "$#" -eq 0 ]; then',
+		'  echo "Usage: $0 <project-root> [baseline-ref] -- <qa-command...>" >&2',
+		'  exit 2',
+		'fi',
+		'PROJECT_ROOT="$(cd "$PROJECT_ROOT" && pwd)"',
+		'REPO_ROOT="$(git -C "$PROJECT_ROOT" rev-parse --show-toplevel)"',
+		'PROJECT_REL="$(git -C "$REPO_ROOT" ls-files --full-name "$PROJECT_ROOT" 2>/dev/null | head -1 | xargs dirname 2>/dev/null || true)"',
+		'if [ -z "$PROJECT_REL" ] || [ "$PROJECT_REL" = "." ]; then',
+		'  PROJECT_REL="$(node - "$REPO_ROOT" "$PROJECT_ROOT" <<\'RESOLVEIO_REL\'',
+		'const path = require("path");',
+		'console.log(path.relative(process.argv[2], process.argv[3]) || ".");',
+		'RESOLVEIO_REL',
+		'  )"',
+		'fi',
+		'ARTIFACT_DIR="$PROJECT_ROOT/qa-artifacts"',
+		'mkdir -p "$ARTIFACT_DIR/baseline" "$ARTIFACT_DIR/candidate"',
+		'RESULT_JSON="$ARTIFACT_DIR/bugfix-comparison-result.json"',
+		'CANDIDATE_PATCH="$ARTIFACT_DIR/bugfix-candidate.patch"',
+		'ORIGINAL_HEAD="$(git -C "$REPO_ROOT" rev-parse HEAD)"',
+		'ORIGINAL_BRANCH="$(git -C "$REPO_ROOT" symbolic-ref --short HEAD 2>/dev/null || true)"',
+		'QA_COMMAND=("$@")',
+		'STOPPER="$TOOLS_DIR/stop-local-qa.sh"',
+		'CURRENT_PHASE=""',
+		'stop_local_qa() {',
+		'  "$STOPPER" "$PROJECT_ROOT" >/dev/null 2>&1 || true',
+		'}',
+		'write_json() {',
+		'  node - "$RESULT_JSON" "$@" <<\'RESOLVEIO_WRITE_JSON\'',
+		'const fs = require("fs");',
+		'const [path, status, baselineExit, candidateExit, baselineRef, originalHead, projectRel, note] = process.argv.slice(2);',
+		'const payload = {',
+		'  status,',
+		'  baseline_ref: baselineRef,',
+		'  original_head: originalHead,',
+		'  project: projectRel,',
+		'  baseline_exit_code: Number(baselineExit),',
+		'  candidate_exit_code: Number(candidateExit),',
+		'  code_switch_proven: Number(baselineExit) !== 0 && Number(candidateExit) === 0,',
+		'  candidate_passed: Number(candidateExit) === 0,',
+		'  baseline_failed: Number(baselineExit) !== 0,',
+		'  note,',
+		'  artifact_dirs: { baseline: "qa-artifacts/baseline", candidate: "qa-artifacts/candidate" },',
+		'  created_at: new Date().toISOString()',
+		'};',
+		'fs.writeFileSync(path, JSON.stringify(payload, null, 2));',
+		'console.log(JSON.stringify(payload, null, 2));',
+		'RESOLVEIO_WRITE_JSON',
+		'}',
+		'snapshot_phase_artifacts() {',
+		'  local phase="$1"',
+		'  mkdir -p "$ARTIFACT_DIR/$phase"',
+		'  find "$ARTIFACT_DIR" -maxdepth 1 -type f \\( -name "*.png" -o -name "*.jpg" -o -name "*.jpeg" -o -name "*.webp" -o -name "*.json" -o -name "*.txt" -o -name "*.log" -o -name "*.zip" \\) -print0 2>/dev/null | while IFS= read -r -d "" file; do',
+		'    cp "$file" "$ARTIFACT_DIR/$phase/$(basename "$file")" 2>/dev/null || true',
+		'  done',
+		'}',
+		'capture_candidate_patch() {',
+		'  : > "$CANDIDATE_PATCH"',
+		'  git -C "$REPO_ROOT" diff --binary -- "$PROJECT_REL" >> "$CANDIDATE_PATCH" || true',
+		'  git -C "$REPO_ROOT" diff --binary --cached -- "$PROJECT_REL" >> "$CANDIDATE_PATCH" || true',
+		'}',
+		'checkout_project_from_ref() {',
+		'  local ref="$1"',
+		'  git -C "$REPO_ROOT" checkout "$ref" -- "$PROJECT_REL"',
+		'}',
+		'restore_candidate() {',
+		'  stop_local_qa',
+		'  checkout_project_from_ref "$ORIGINAL_HEAD" || true',
+		'  if [ -s "$CANDIDATE_PATCH" ]; then',
+		'    git -C "$REPO_ROOT" apply --whitespace=nowarn "$CANDIDATE_PATCH" || true',
+		'  fi',
+		'}',
+		'trap restore_candidate EXIT',
+		'run_phase() {',
+		'  local phase="$1"',
+		'  CURRENT_PHASE="$phase"',
+		'  stop_local_qa',
+		'  export RESOLVEIO_RUNNER_QA_PHASE="$phase"',
+		'  export RESOLVEIO_SUPPORT_QA_PHASE="$phase"',
+		'  export RESOLVEIO_RUNNER_QA_ARTIFACT_DIR="$ARTIFACT_DIR/$phase"',
+		'  export RESOLVEIO_SUPPORT_QA_ARTIFACT_DIR="$ARTIFACT_DIR/$phase"',
+		'  mkdir -p "$ARTIFACT_DIR/$phase"',
+		'  find "$ARTIFACT_DIR/$phase" -maxdepth 1 -type f \\( -name "*.png" -o -name "*.jpg" -o -name "*.jpeg" -o -name "*.webp" -o -name "*.json" -o -name "*.txt" -o -name "*.log" -o -name "*.zip" \\) -delete 2>/dev/null || true',
+		'  find "$ARTIFACT_DIR" -maxdepth 1 -type f \\( -name "*.png" -o -name "*.jpg" -o -name "*.jpeg" -o -name "*.webp" -o -name "*proof.json" -o -name "auth-bootstrap-result.json" \\) -delete 2>/dev/null || true',
+		'  set +e',
+		'  (cd "$REPO_ROOT" && "${QA_COMMAND[@]}") 2>&1 | tee "$ARTIFACT_DIR/$phase/qa-command.log"',
+		'  local rc="${PIPESTATUS[0]}"',
+		'  set +e',
+		'  snapshot_phase_artifacts "$phase"',
+		'  stop_local_qa',
+		'  return "$rc"',
+		'}',
+		'echo "ResolveIO bugfix comparison QA preserving candidate diff for $PROJECT_REL"',
+		'capture_candidate_patch',
+		'stop_local_qa',
+		'echo "ResolveIO bugfix comparison QA baseline phase: $BASELINE_REF"',
+		'checkout_project_from_ref "$BASELINE_REF"',
+		'run_phase baseline',
+		'BASELINE_EXIT="$?"',
+		'echo "ResolveIO bugfix comparison QA candidate phase: restored workspace candidate"',
+		'restore_candidate',
+		'run_phase candidate',
+		'CANDIDATE_EXIT="$?"',
+		'if [ "$CANDIDATE_EXIT" = "0" ] && [ "$BASELINE_EXIT" != "0" ]; then',
+		'  STATUS="pass"',
+		'  NOTE="Baseline failed and candidate passed for the same QA command."',
+		'elif [ "$CANDIDATE_EXIT" = "0" ]; then',
+		'  STATUS="inconclusive_baseline_also_passed"',
+		'  NOTE="Candidate passed, but baseline also passed; the code switch did not prove the bug fix."',
+		'else',
+		'  STATUS="fail_candidate_failed"',
+		'  NOTE="Candidate failed the QA command."',
+		'fi',
+		'write_json "$STATUS" "$BASELINE_EXIT" "$CANDIDATE_EXIT" "$BASELINE_REF" "$ORIGINAL_HEAD" "$PROJECT_REL" "$NOTE"',
+		'[ "$CANDIDATE_EXIT" = "0" ] || exit "$CANDIDATE_EXIT"',
+		'[ "$STATUS" = "pass" ] || exit 8',
+		''
+	].join('\n');
+}
+
+export function buildResolveIORunnerQaToolsReadme(options: ResolveIORunnerQaToolBundleOptions = {}): string {
+	const mode = options.mode || 'runner';
+	const toolsDir = mode === 'support' ? '.resolveio-support-tools' : '.resolveio-ai-runner-tools';
+	const port = normalizePort(options.qaClientPort, '4200');
+	const clientUrlVar = envVar(mode, 'CLIENT_URL');
+	const keepaliveVar = envVar(mode, 'KEEPALIVE');
+	const usernameVar = envVar(mode, 'USERNAME');
+	const passwordVar = envVar(mode, 'PASSWORD');
+	return [
+		`# ResolveIO ${mode === 'support' ? 'Support' : 'AI Runner'} Local QA Tools`,
+		'',
+		'These scripts are generated by `@resolveio/server-lib` and are shared by support tickets, AICoder app-builder runs, and AI-terminal runs.',
+		'',
+		'```bash',
+		`source ${toolsDir}/env.sh`,
+		`${toolsDir}/run-local-qa.sh <project-root>`,
+		`node ${toolsDir}/qa-auth-bootstrap.js <project-root> /target-route`,
+		`${toolsDir}/bugfix-comparison-qa.sh <project-root> origin/master -- bash -lc '<same QA command>'`,
+		'```',
+		'',
+		`This workspace reserves Angular QA client port ${port}; use \`$${clientUrlVar}\` instead of assuming 4200 is free.`,
+		'The local QA runner starts server/client, polls the reserved client URL, writes `qa-artifacts/server.log` and `qa-artifacts/client.log`, and fails fast on fatal startup/runtime errors.',
+		'The shared auth bootstrap first opens the exact localhost client origin, logs out any visible stale session, clears service workers/cache/IndexedDB/local/session storage, then calls `/login` and `/accessToken`, seeds `refreshToken`, `accessToken`, `user`, and `lastURL`, and writes `qa-artifacts/auth-bootstrap-result.json` plus a ready/failure screenshot.',
+		`For browser clickthrough work, start the runner once with \`${keepaliveVar}=true ${toolsDir}/run-local-qa.sh <project-root>\`; it detaches after the app is ready, and later calls should reuse \`$${clientUrlVar}\` for all login/upload/screenshot retries. Do not restart Angular for auth failures.`,
+		'Do not run `npm run build-dev`, `ng build`, or another Angular compile while keepalive `ng serve` is running. If a full Angular build is required after browser QA, first run the staged `stop-local-qa.sh`, then build, then restart `run-local-qa.sh` for final browser proof.',
+		'Use desktop screenshots at 1920x1080 by default unless the task is explicitly mobile/responsive. Every screenshot must have a customer-facing caption.',
+		'For import/export/form-submit/data workflows, prove before/action/after with representative data and a concrete row/count/value assertion.',
+		'For bug fixes, use `bugfix-comparison-qa.sh` so baseline/master runs the exact same repro before the candidate/PR run. A passing candidate is not enough unless the comparison result shows baseline failed or the report explicitly explains why the baseline failure could not be reproduced.',
+		`Use \`$${usernameVar}\` and \`$${passwordVar}\` for the local fixture admin account unless ticket/app-specific credentials are provided.`,
+		'The env file reuses `/var/lib/resolveio/puppeteer`, npm cache, worker-safe Browserslist settings, and Angular cache prep so QA should not download a browser or rebuild cold caches unnecessarily.',
+		mode === 'support'
+			? 'Support workspaces also stage local `mongod` and `mongosh` wrappers so app server scripts can start MongoDB when the worker image does not have MongoDB installed.'
+			: '',
+		''
+	].filter((line) => line !== '').join('\n');
+}