@indigoai-us/hq-cli 5.345.113 → 5.345.115
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/assets/scaffold/core/scripts/hq-delegate-send.sh +5 -1
- package/dist/commands/bot-jobs.d.ts +1 -11
- package/dist/commands/bot-jobs.js +13 -34
- package/dist/commands/capacity.js +2 -0
- package/dist/commands/core-audit-log-native.js +1 -1
- package/dist/commands/core-checkpoint-native.d.ts +27 -0
- package/dist/commands/core-checkpoint-native.js +85 -2
- package/dist/commands/core-dm-bind-native.js +1 -1
- package/dist/commands/core-git-pack-extension-check-native.js +1 -1
- package/dist/commands/core-job-native-utils.d.ts +2 -2
- package/dist/commands/core-job-native-utils.js +3 -11
- package/dist/commands/core-job-notify-native.js +1 -2
- package/dist/commands/core-job-probe-native.d.ts +2 -3
- package/dist/commands/core-job-probe-native.js +14 -22
- package/dist/commands/core-job-remediate-native.js +1 -2
- package/dist/commands/core-job-run-native.js +58 -54
- package/dist/commands/core-jobs-native.d.ts +1 -3
- package/dist/commands/core-jobs-native.js +16 -271
- package/dist/commands/core-jobs-schedule.d.ts +7 -0
- package/dist/commands/core-jobs-schedule.js +243 -0
- package/dist/commands/core-native-utilities.d.ts +2 -3
- package/dist/commands/core-native-utilities.js +2 -19
- package/dist/commands/core-pin-native.js +1 -1
- package/dist/commands/core-refresh-vault-access-native.js +1 -1
- package/dist/commands/core-resolve-company-command.js +2 -17
- package/dist/commands/core-work-mesh-bind-trusted-native.js +1 -1
- package/dist/commands/core.js +1 -17
- package/dist/commands/lanes.js +1 -0
- package/dist/fast-scaffold.d.ts +1 -1
- package/dist/fast-scaffold.js +0 -38
- package/dist/lib/capacity/lane-login.js +73 -6
- package/dist/lib/capacity/route.d.ts +1 -1
- package/dist/lib/capacity/route.js +5 -1
- package/dist/lib/capacity/sign-in.d.ts +12 -0
- package/dist/lib/capacity/sign-in.js +65 -16
- package/dist/lib/daemon/catalog.d.ts +0 -2
- package/dist/lib/daemon/catalog.js +3 -6
- package/dist/lib/daemon/run.js +0 -7
- package/dist/utils/resident-package-preload.js +2 -7
- package/package.json +1 -1
- package/assets/scaffold/core/scripts/audit-log.sh +0 -384
- package/assets/scaffold/core/scripts/git-pack-extension-check.sh +0 -164
- package/assets/scaffold/core/scripts/hq-dm-bind.sh +0 -590
- package/assets/scaffold/core/scripts/hq-job-notify.sh +0 -389
- package/assets/scaffold/core/scripts/hq-job-probe.sh +0 -865
- package/assets/scaffold/core/scripts/hq-job-remediate.sh +0 -359
- package/assets/scaffold/core/scripts/hq-job-run.sh +0 -1025
- package/assets/scaffold/core/scripts/hq-pin.sh +0 -118
- package/assets/scaffold/core/scripts/jobs-schedule-parse.sh +0 -310
- package/assets/scaffold/core/scripts/jobs-validate.sh +0 -482
- package/assets/scaffold/core/scripts/outpost-jobs-reconcile.sh +0 -836
- package/assets/scaffold/core/scripts/refresh-vault-access.sh +0 -188
- package/assets/scaffold/core/scripts/resolve-company.sh +0 -331
- package/assets/scaffold/core/scripts/work-mesh-live-bind-trusted.sh +0 -184
- package/dist/commands/core-job-post-exit-assets.d.ts +0 -8
- package/dist/commands/core-job-post-exit-assets.js +0 -37
- package/dist/commands/core-native-jobs-flag.d.ts +0 -4
- package/dist/commands/core-native-jobs-flag.js +0 -7
- package/dist/commands/core-native-utilities-flag.d.ts +0 -4
- package/dist/commands/core-native-utilities-flag.js +0 -7
|
@@ -1,1025 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env bash
|
|
2
|
-
# hq-core: public
|
|
3
|
-
# hq-job-run.sh — serialized Outpost scheduled-job executor (US-004 + US-006).
|
|
4
|
-
#
|
|
5
|
-
# Invoked by systemd user units (hq-job-{id}.service) or on demand:
|
|
6
|
-
# core/scripts/hq-job-run.sh --job-id <id>
|
|
7
|
-
# core/scripts/hq-job-run.sh <job.yaml>
|
|
8
|
-
#
|
|
9
|
-
# Behavior:
|
|
10
|
-
# - Global file lock (~/.hq/jobs/run.lock) — serialized-only (US-001)
|
|
11
|
-
# - Enforces job timeout_seconds (default 600; schema 60–14400)
|
|
12
|
-
# - Credentials only via `hq secrets exec` (requirements.secrets[]) or
|
|
13
|
-
# the box's cached CLI auth — never env vars / unit-file secrets
|
|
14
|
-
# - Trust model (7A): injects ONLY requirements.secrets[] — no static prompt scan
|
|
15
|
-
# - Appends stdout/stderr + exit to ~/.hq/jobs/logs/{id}/
|
|
16
|
-
# - Emits a compute-meter attribution marker (scheduled vs interactive)
|
|
17
|
-
# - Classifies failures (auth|secrets|timeout|agent_error|infra), writes
|
|
18
|
-
# failure_class to hq-pro jobs status (kind=run), bounded self-remediate,
|
|
19
|
-
# then dm-notify via hq-job-notify.sh (collapse 1/24h; never create-thread)
|
|
20
|
-
# - Missing runtime CLI auth: classify auth, notify re-login, exit 0 (unit OK)
|
|
21
|
-
#
|
|
22
|
-
# Runtime dispatch (US-001 spike flags):
|
|
23
|
-
# claude → claude -p … --output-format text --permission-mode bypassPermissions
|
|
24
|
-
# codex → codex exec -s workspace-write --skip-git-repo-check …
|
|
25
|
-
#
|
|
26
|
-
# Env / overrides (tests):
|
|
27
|
-
# HQ_ROOT, HOME, HQ_JOB_LOCK, HQ_JOB_LOG_DIR, HQ_JOB_METER_DIR
|
|
28
|
-
# HQ_JOB_TIMEOUT_BIN, HQ_JOB_FLOCK_BIN, HQ_JOB_NOW
|
|
29
|
-
# HQ_JOB_PROBE_CLAUDE_CRED, HQ_JOB_PROBE_CODEX_AUTH (auth paths; shared with probe)
|
|
30
|
-
# HQ_JOB_RUN_SKIP_EXEC=1 — acquire lock + write marker/logs without calling CLI
|
|
31
|
-
# HQ_JOB_NOTIFY_BIN, HQ_JOB_REMEDIATE_BIN, HQ_JOB_RUN_NO_NOTIFY=1, HQ_JOB_RUN_NO_INGEST=1
|
|
32
|
-
# HQ_JOB_RUN_CURL, HQ_PRO_API_URL, OUTPOST_USER_ID, OUTPOST_INSTANCE_TOKEN
|
|
33
|
-
#
|
|
34
|
-
# Exit: 0 success or intentional auth/cwd skip; 1 agent/job failure; 2 usage/config;
|
|
35
|
-
# 124 timeout (GNU timeout convention when available).
|
|
36
|
-
|
|
37
|
-
set -euo pipefail
|
|
38
|
-
|
|
39
|
-
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
40
|
-
HQ_ROOT="${HQ_ROOT:-$(cd "$SCRIPT_DIR/../.." && pwd)}"
|
|
41
|
-
# shellcheck disable=SC1091
|
|
42
|
-
. "$SCRIPT_DIR/hook-lib.sh"
|
|
43
|
-
NOTIFY_BIN="${HQ_JOB_NOTIFY_BIN:-$SCRIPT_DIR/hq-job-notify.sh}"
|
|
44
|
-
REMEDIATE_BIN="${HQ_JOB_REMEDIATE_BIN:-$SCRIPT_DIR/hq-job-remediate.sh}"
|
|
45
|
-
AUTH_HEADER_FILE=""
|
|
46
|
-
|
|
47
|
-
cleanup_auth_header_file() {
|
|
48
|
-
if [ -n "${AUTH_HEADER_FILE:-}" ]; then
|
|
49
|
-
rm -f -- "$AUTH_HEADER_FILE" 2>/dev/null || true
|
|
50
|
-
AUTH_HEADER_FILE=""
|
|
51
|
-
fi
|
|
52
|
-
}
|
|
53
|
-
trap cleanup_auth_header_file EXIT
|
|
54
|
-
|
|
55
|
-
JOB_TARGET=""
|
|
56
|
-
JOB_ID_ARG=""
|
|
57
|
-
|
|
58
|
-
usage() {
|
|
59
|
-
sed -n '3,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
die() {
|
|
63
|
-
echo "hq-job-run: $*" >&2
|
|
64
|
-
exit 2
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
log() {
|
|
68
|
-
echo "hq-job-run: $*" >&2
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
while [ $# -gt 0 ]; do
|
|
72
|
-
case "$1" in
|
|
73
|
-
--hq-root)
|
|
74
|
-
HQ_ROOT="${2:-}"
|
|
75
|
-
[ -n "$HQ_ROOT" ] || die "--hq-root requires a directory"
|
|
76
|
-
shift 2
|
|
77
|
-
;;
|
|
78
|
-
--job-id)
|
|
79
|
-
JOB_ID_ARG="${2:-}"
|
|
80
|
-
[ -n "$JOB_ID_ARG" ] || die "--job-id requires a value"
|
|
81
|
-
shift 2
|
|
82
|
-
;;
|
|
83
|
-
-h|--help)
|
|
84
|
-
usage
|
|
85
|
-
exit 0
|
|
86
|
-
;;
|
|
87
|
-
--)
|
|
88
|
-
shift
|
|
89
|
-
break
|
|
90
|
-
;;
|
|
91
|
-
-*)
|
|
92
|
-
die "unknown option: $1"
|
|
93
|
-
;;
|
|
94
|
-
*)
|
|
95
|
-
if [ -n "$JOB_TARGET" ]; then
|
|
96
|
-
die "unexpected argument: $1"
|
|
97
|
-
fi
|
|
98
|
-
JOB_TARGET="$1"
|
|
99
|
-
shift
|
|
100
|
-
;;
|
|
101
|
-
esac
|
|
102
|
-
done
|
|
103
|
-
|
|
104
|
-
command -v yq >/dev/null 2>&1 || die "yq is required (mikefarah/yq)"
|
|
105
|
-
command -v jq >/dev/null 2>&1 || die "jq is required"
|
|
106
|
-
|
|
107
|
-
iso_now() {
|
|
108
|
-
if [ -n "${HQ_JOB_NOW:-}" ]; then
|
|
109
|
-
printf '%s' "$HQ_JOB_NOW"
|
|
110
|
-
return
|
|
111
|
-
fi
|
|
112
|
-
date -u +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null \
|
|
113
|
-
|| date -u +"%Y-%m-%dT%H:%M:%S+00:00"
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
epoch_now() {
|
|
117
|
-
date -u +%s 2>/dev/null || date +%s
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
claude_cred_path() {
|
|
121
|
-
printf '%s' "${HQ_JOB_PROBE_CLAUDE_CRED:-${HOME}/.claude/.credentials.json}"
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
codex_auth_path() {
|
|
125
|
-
printf '%s' "${HQ_JOB_PROBE_CODEX_AUTH:-${HOME}/.codex/auth.json}"
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
find_job_file_by_id() {
|
|
129
|
-
local id="$1" f found=""
|
|
130
|
-
while IFS= read -r f; do
|
|
131
|
-
[ -n "$f" ] || continue
|
|
132
|
-
if [ "$(yq -r '.id // ""' "$f" 2>/dev/null)" = "$id" ]; then
|
|
133
|
-
if [ -n "$found" ]; then
|
|
134
|
-
die "duplicate job id '$id' in registry: $found and $f"
|
|
135
|
-
fi
|
|
136
|
-
found="$f"
|
|
137
|
-
fi
|
|
138
|
-
done < <(
|
|
139
|
-
{
|
|
140
|
-
find "$HQ_ROOT/personal/jobs" -maxdepth 1 \( -name '*.yaml' -o -name '*.yml' \) -type f 2>/dev/null || true
|
|
141
|
-
find "$HQ_ROOT/companies" -path '*/jobs/*' -maxdepth 3 \( -name '*.yaml' -o -name '*.yml' \) -type f 2>/dev/null || true
|
|
142
|
-
} | grep -Ev '\.conflict-|/\.conflicts/' | sort
|
|
143
|
-
)
|
|
144
|
-
[ -n "$found" ] || return 1
|
|
145
|
-
printf '%s' "$found"
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
resolve_job_file() {
|
|
149
|
-
if [ -n "$JOB_ID_ARG" ]; then
|
|
150
|
-
find_job_file_by_id "$JOB_ID_ARG" || die "job id not found: $JOB_ID_ARG"
|
|
151
|
-
return
|
|
152
|
-
fi
|
|
153
|
-
if [ -z "$JOB_TARGET" ]; then
|
|
154
|
-
die "usage: hq-job-run.sh --job-id <id> | <job.yaml>"
|
|
155
|
-
fi
|
|
156
|
-
if [ -f "$JOB_TARGET" ]; then
|
|
157
|
-
printf '%s' "$JOB_TARGET"
|
|
158
|
-
return
|
|
159
|
-
fi
|
|
160
|
-
if [ -f "$HQ_ROOT/$JOB_TARGET" ]; then
|
|
161
|
-
printf '%s' "$HQ_ROOT/$JOB_TARGET"
|
|
162
|
-
return
|
|
163
|
-
fi
|
|
164
|
-
find_job_file_by_id "$JOB_TARGET" || die "job not found: $JOB_TARGET"
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
check_runtime_auth() {
|
|
168
|
-
local runtime="$1"
|
|
169
|
-
case "$runtime" in
|
|
170
|
-
claude)
|
|
171
|
-
if ! command -v claude >/dev/null 2>&1; then
|
|
172
|
-
printf 'no_cli'
|
|
173
|
-
return 1
|
|
174
|
-
fi
|
|
175
|
-
if [ -s "$(claude_cred_path)" ]; then
|
|
176
|
-
printf 'ok'
|
|
177
|
-
return 0
|
|
178
|
-
fi
|
|
179
|
-
printf 'missing'
|
|
180
|
-
return 1
|
|
181
|
-
;;
|
|
182
|
-
codex)
|
|
183
|
-
if ! command -v codex >/dev/null 2>&1; then
|
|
184
|
-
printf 'no_cli'
|
|
185
|
-
return 1
|
|
186
|
-
fi
|
|
187
|
-
if [ -s "$(codex_auth_path)" ]; then
|
|
188
|
-
printf 'ok'
|
|
189
|
-
return 0
|
|
190
|
-
fi
|
|
191
|
-
printf 'missing'
|
|
192
|
-
return 1
|
|
193
|
-
;;
|
|
194
|
-
*)
|
|
195
|
-
printf 'unknown_runtime'
|
|
196
|
-
return 1
|
|
197
|
-
;;
|
|
198
|
-
esac
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
build_prompt() {
|
|
202
|
-
local job_file="$1"
|
|
203
|
-
local prompt skill
|
|
204
|
-
prompt="$(yq -r '.exec.prompt // ""' "$job_file")"
|
|
205
|
-
skill="$(yq -r '.exec.skill // ""' "$job_file")"
|
|
206
|
-
if [ -n "$prompt" ] && [ "$prompt" != "null" ]; then
|
|
207
|
-
printf '%s' "$prompt"
|
|
208
|
-
return
|
|
209
|
-
fi
|
|
210
|
-
if [ -n "$skill" ] && [ "$skill" != "null" ]; then
|
|
211
|
-
local args_json
|
|
212
|
-
args_json="$(yq -o=json -I=0 '.exec.args // {}' "$job_file" 2>/dev/null || echo '{}')"
|
|
213
|
-
printf 'Execute the HQ skill "%s" unattended with these args (JSON): %s. Follow the skill instructions exactly and produce a concise completion summary.' \
|
|
214
|
-
"$skill" "$args_json"
|
|
215
|
-
return
|
|
216
|
-
fi
|
|
217
|
-
die "job missing exec.prompt and exec.skill: $job_file"
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
append_meter_marker() {
|
|
221
|
-
local job_id="$1" runtime="$2" started="$3" ended="$4" exit_status="$5" duration="$6" skip_reason="${7:-}"
|
|
222
|
-
local dir="${HQ_JOB_METER_DIR:-${HOME}/.hq/jobs/meters}"
|
|
223
|
-
mkdir -p "$dir"
|
|
224
|
-
local line
|
|
225
|
-
line="$(jq -nc \
|
|
226
|
-
--arg schema "hq.outpost.job-run/v1" \
|
|
227
|
-
--arg kind "scheduled-job-run" \
|
|
228
|
-
--arg job_id "$job_id" \
|
|
229
|
-
--arg runtime "$runtime" \
|
|
230
|
-
--arg started_at "$started" \
|
|
231
|
-
--arg ended_at "$ended" \
|
|
232
|
-
--argjson exit_status "$exit_status" \
|
|
233
|
-
--argjson duration_seconds "$duration" \
|
|
234
|
-
--arg outpost_id "${OUTPOST_ID:-}" \
|
|
235
|
-
--arg skip_reason "$skip_reason" \
|
|
236
|
-
'{
|
|
237
|
-
schema: $schema,
|
|
238
|
-
kind: $kind,
|
|
239
|
-
job_id: $job_id,
|
|
240
|
-
runtime: $runtime,
|
|
241
|
-
started_at: $started_at,
|
|
242
|
-
ended_at: $ended_at,
|
|
243
|
-
exit_status: $exit_status,
|
|
244
|
-
duration_seconds: $duration_seconds
|
|
245
|
-
}
|
|
246
|
-
+ (if $outpost_id != "" then {outpost_id: $outpost_id} else {} end)
|
|
247
|
-
+ (if $skip_reason != "" then {skip_reason: $skip_reason} else {} end)')"
|
|
248
|
-
printf '%s\n' "$line" >>"$dir/runs.jsonl"
|
|
249
|
-
chmod 600 "$dir/runs.jsonl" 2>/dev/null || true
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
api_base() {
|
|
253
|
-
local base
|
|
254
|
-
base="${HQ_PRO_API_URL:-${HQ_API_URL:-${HQ_VAULT_API_URL:-https://hqapi.hq.computer}}}"
|
|
255
|
-
base="${base%/}"
|
|
256
|
-
printf '%s' "$base"
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
resolve_box_identity() {
|
|
260
|
-
local xtrace_was_set=0
|
|
261
|
-
case "$-" in
|
|
262
|
-
*x*) xtrace_was_set=1; set +x ;;
|
|
263
|
-
esac
|
|
264
|
-
|
|
265
|
-
if [ -n "${OUTPOST_USER_ID:-}" ] && [ -n "${OUTPOST_INSTANCE_TOKEN:-}" ]; then
|
|
266
|
-
[ "$xtrace_was_set" -eq 0 ] || set -x
|
|
267
|
-
return 0
|
|
268
|
-
fi
|
|
269
|
-
local envf="/etc/outpost/codex-identity.env"
|
|
270
|
-
if [ -f "$envf" ]; then
|
|
271
|
-
# shellcheck disable=SC1090
|
|
272
|
-
. "$envf" 2>/dev/null || true
|
|
273
|
-
if [ -n "${OUTPOST_USER_ID:-}" ] && [ -n "${OUTPOST_INSTANCE_TOKEN:-}" ]; then
|
|
274
|
-
[ "$xtrace_was_set" -eq 0 ] || set -x
|
|
275
|
-
return 0
|
|
276
|
-
fi
|
|
277
|
-
fi
|
|
278
|
-
local runner="${HQ_JOB_PROBE_RUNNER:-/usr/local/bin/outpost-runner.sh}"
|
|
279
|
-
if [ -f "$runner" ]; then
|
|
280
|
-
OUTPOST_USER_ID="${OUTPOST_USER_ID:-$(sed -n 's/^USER_ID="\(.*\)"$/\1/p' "$runner" 2>/dev/null | head -1)}"
|
|
281
|
-
OUTPOST_INSTANCE_TOKEN="${OUTPOST_INSTANCE_TOKEN:-$(sed -n 's/^INSTANCE_TOKEN="\(.*\)"$/\1/p' "$runner" 2>/dev/null | head -1)}"
|
|
282
|
-
OUTPOST_ID="${OUTPOST_ID:-$(sed -n 's/^OUTPOST_ID="\(.*\)"$/\1/p' "$runner" 2>/dev/null | head -1)}"
|
|
283
|
-
fi
|
|
284
|
-
if [ -n "${OUTPOST_USER_ID:-}" ] && [ -n "${OUTPOST_INSTANCE_TOKEN:-}" ]; then
|
|
285
|
-
[ "$xtrace_was_set" -eq 0 ] || set -x
|
|
286
|
-
return 0
|
|
287
|
-
fi
|
|
288
|
-
[ "$xtrace_was_set" -eq 0 ] || set -x
|
|
289
|
-
return 1
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
create_auth_header_file() {
|
|
293
|
-
local xtrace_was_set=0 write_rc=0
|
|
294
|
-
AUTH_HEADER_FILE="$(mktemp "${TMPDIR:-/tmp}/hq-job-run-auth-header.XXXXXX")"
|
|
295
|
-
chmod 600 "$AUTH_HEADER_FILE"
|
|
296
|
-
case "$-" in
|
|
297
|
-
*x*) xtrace_was_set=1; set +x ;;
|
|
298
|
-
esac
|
|
299
|
-
if printf 'x-outpost-instance-token: %s\n' "$OUTPOST_INSTANCE_TOKEN" >"$AUTH_HEADER_FILE"; then
|
|
300
|
-
write_rc=0
|
|
301
|
-
else
|
|
302
|
-
write_rc=$?
|
|
303
|
-
fi
|
|
304
|
-
[ "$xtrace_was_set" -eq 0 ] || set -x
|
|
305
|
-
return "$write_rc"
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
# Classify exit + log tail → auth|secrets|timeout|agent_error|infra
|
|
309
|
-
classify_failure() {
|
|
310
|
-
local exit_code="$1" log_file="$2" auth_hint="${3:-}"
|
|
311
|
-
if [ -n "$auth_hint" ] && [ "$auth_hint" != "ok" ]; then
|
|
312
|
-
printf 'auth'
|
|
313
|
-
return
|
|
314
|
-
fi
|
|
315
|
-
if [ "$exit_code" -eq 124 ] || [ "$exit_code" -eq 137 ]; then
|
|
316
|
-
printf 'timeout'
|
|
317
|
-
return
|
|
318
|
-
fi
|
|
319
|
-
local tail_txt=""
|
|
320
|
-
if [ -n "$log_file" ] && [ -f "$log_file" ]; then
|
|
321
|
-
tail_txt="$(tail -n 80 "$log_file" 2>/dev/null || true)"
|
|
322
|
-
fi
|
|
323
|
-
local lower
|
|
324
|
-
lower="$(printf '%s' "$tail_txt" | tr '[:upper:]' '[:lower:]')"
|
|
325
|
-
|
|
326
|
-
if printf '%s' "$lower" | grep -E \
|
|
327
|
-
'not logged in|please log in|authentication (failed|required)|unauthorized|401|invalid.*(token|credential)|credentials? (missing|expired|invalid)|device.?code|login required' >/dev/null; then
|
|
328
|
-
printf 'auth'
|
|
329
|
-
return
|
|
330
|
-
fi
|
|
331
|
-
if printf '%s' "$lower" | grep -E \
|
|
332
|
-
'secret.*(not found|missing|denied|not visible|not shared)|hq secrets.*(fail|error|denied)|missing vault|access denied.*secret' >/dev/null; then
|
|
333
|
-
printf 'secrets'
|
|
334
|
-
return
|
|
335
|
-
fi
|
|
336
|
-
# Avoid matching job metadata like timeout_seconds=N in the log header.
|
|
337
|
-
if printf '%s' "$lower" | grep -E \
|
|
338
|
-
'(^|[^_])timed out|command timed out|deadline exceeded|killed by timeout|exit[=:][[:space:]]*124\b' >/dev/null; then
|
|
339
|
-
printf 'timeout'
|
|
340
|
-
return
|
|
341
|
-
fi
|
|
342
|
-
if printf '%s' "$lower" | grep -E \
|
|
343
|
-
'connection (refused|reset|timed out)|network is unreachable|temporary failure|econnreset|enotfound|no space left|i/o error|transport error|dns (fail|error)|http[/ ]*(502|503|504)|statusingesterror|api unreachable' >/dev/null; then
|
|
344
|
-
printf 'infra'
|
|
345
|
-
return
|
|
346
|
-
fi
|
|
347
|
-
if [ "$exit_code" -eq 0 ]; then
|
|
348
|
-
printf ''
|
|
349
|
-
return
|
|
350
|
-
fi
|
|
351
|
-
printf 'agent_error'
|
|
352
|
-
}
|
|
353
|
-
|
|
354
|
-
next_action_for_class() {
|
|
355
|
-
local fc="$1" runtime="${2:-}"
|
|
356
|
-
case "$fc" in
|
|
357
|
-
auth)
|
|
358
|
-
printf 're-login on Outpost (device code) for %s' "${runtime:-the runtime CLI}"
|
|
359
|
-
;;
|
|
360
|
-
secrets)
|
|
361
|
-
printf 'share the missing secret to this Outpost (hq secrets share KEY) then re-probe'
|
|
362
|
-
;;
|
|
363
|
-
timeout)
|
|
364
|
-
printf 'raise timeout_seconds or split the job work'
|
|
365
|
-
;;
|
|
366
|
-
infra)
|
|
367
|
-
printf 'check Outpost disk/network; job will self-retry once on transient infra'
|
|
368
|
-
;;
|
|
369
|
-
agent_error)
|
|
370
|
-
printf 'inspect the job log and fix the prompt/skill'
|
|
371
|
-
;;
|
|
372
|
-
*)
|
|
373
|
-
printf ''
|
|
374
|
-
;;
|
|
375
|
-
esac
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
# POST kind=run ingest. Never fails the job run.
|
|
379
|
-
ingest_run_status() {
|
|
380
|
-
local job_id="$1" run_at="$2" run_id="$3" last_exit="$4" failure_class="$5" next_actions_json="$6"
|
|
381
|
-
if [ "${HQ_JOB_RUN_NO_INGEST:-}" = "1" ]; then
|
|
382
|
-
log "ingest skipped (HQ_JOB_RUN_NO_INGEST=1)"
|
|
383
|
-
return 0
|
|
384
|
-
fi
|
|
385
|
-
if ! resolve_box_identity; then
|
|
386
|
-
log "run status ingest skipped — missing Outpost identity"
|
|
387
|
-
return 0
|
|
388
|
-
fi
|
|
389
|
-
local payload fc_json
|
|
390
|
-
if [ -z "$failure_class" ] || [ "$failure_class" = "null" ]; then
|
|
391
|
-
fc_json='null'
|
|
392
|
-
else
|
|
393
|
-
fc_json="$(jq -nc --arg f "$failure_class" '$f')"
|
|
394
|
-
fi
|
|
395
|
-
payload="$(jq -nc \
|
|
396
|
-
--arg userId "$OUTPOST_USER_ID" \
|
|
397
|
-
--arg jobId "$job_id" \
|
|
398
|
-
--arg run_at "$run_at" \
|
|
399
|
-
--arg event_id "$run_id" \
|
|
400
|
-
--arg run_id "$run_id" \
|
|
401
|
-
--argjson last_exit "$last_exit" \
|
|
402
|
-
--argjson failure_class "$fc_json" \
|
|
403
|
-
--argjson next_actions "$next_actions_json" \
|
|
404
|
-
--arg outpostId "${OUTPOST_ID:-}" \
|
|
405
|
-
'{
|
|
406
|
-
userId: $userId,
|
|
407
|
-
jobId: $jobId,
|
|
408
|
-
kind: "run",
|
|
409
|
-
run_at: $run_at,
|
|
410
|
-
event_id: $event_id,
|
|
411
|
-
run_id: $run_id,
|
|
412
|
-
last_exit: $last_exit,
|
|
413
|
-
failure_class: $failure_class,
|
|
414
|
-
next_actions: $next_actions
|
|
415
|
-
} + (if $outpostId != "" then {outpostId: $outpostId} else {} end)')"
|
|
416
|
-
|
|
417
|
-
local curl_bin="${HQ_JOB_RUN_CURL:-curl}"
|
|
418
|
-
local url tmp_body tmp_hdr http=0
|
|
419
|
-
url="$(api_base)/outpost/internal/jobs-status"
|
|
420
|
-
if ! command -v "$curl_bin" >/dev/null 2>&1; then
|
|
421
|
-
log "run status ingest skipped — curl not found"
|
|
422
|
-
return 0
|
|
423
|
-
fi
|
|
424
|
-
tmp_body="$(mktemp "${TMPDIR:-/tmp}/hq-job-run-ingest-body.XXXXXX")"
|
|
425
|
-
tmp_hdr="$(mktemp "${TMPDIR:-/tmp}/hq-job-run-ingest-hdr.XXXXXX")"
|
|
426
|
-
create_auth_header_file
|
|
427
|
-
set +e
|
|
428
|
-
"$curl_bin" -sS -m 20 -X POST "$url" \
|
|
429
|
-
-H "content-type: application/json" \
|
|
430
|
-
-H "@${AUTH_HEADER_FILE}" \
|
|
431
|
-
--data "$payload" \
|
|
432
|
-
-D "$tmp_hdr" \
|
|
433
|
-
-o "$tmp_body"
|
|
434
|
-
local curl_rc=$?
|
|
435
|
-
set -e
|
|
436
|
-
if [ "$curl_rc" -eq 0 ]; then
|
|
437
|
-
http="$(awk 'BEGIN{c=0} /^HTTP\//{c=$2} END{print c+0}' "$tmp_hdr" 2>/dev/null || echo 0)"
|
|
438
|
-
fi
|
|
439
|
-
rm -f "$tmp_body" "$tmp_hdr" "$AUTH_HEADER_FILE"
|
|
440
|
-
AUTH_HEADER_FILE=""
|
|
441
|
-
if [ "$http" -ge 200 ] && [ "$http" -lt 300 ]; then
|
|
442
|
-
log "run status ingested job=$job_id failure_class=${failure_class:-null} http=$http"
|
|
443
|
-
else
|
|
444
|
-
log "run status ingest failed job=$job_id http=$http (logged; not failing run)"
|
|
445
|
-
fi
|
|
446
|
-
return 0
|
|
447
|
-
}
|
|
448
|
-
|
|
449
|
-
call_notify() {
|
|
450
|
-
local job_id="$1" job_name="$2" outcome="$3" summary="$4" duration="$5" log_path="$6"
|
|
451
|
-
local failure_class="${7:-}" next_action="${8:-}" owner="${9:-}" notify_mode="${10:-profile}"
|
|
452
|
-
if [ "${HQ_JOB_RUN_NO_NOTIFY:-}" = "1" ]; then
|
|
453
|
-
log "notify skipped (HQ_JOB_RUN_NO_NOTIFY=1)"
|
|
454
|
-
return 0
|
|
455
|
-
fi
|
|
456
|
-
if [ ! -x "$NOTIFY_BIN" ] && [ -f "$NOTIFY_BIN" ]; then
|
|
457
|
-
chmod +x "$NOTIFY_BIN" 2>/dev/null || true
|
|
458
|
-
fi
|
|
459
|
-
if [ ! -f "$NOTIFY_BIN" ]; then
|
|
460
|
-
log "notify skipped — missing $NOTIFY_BIN"
|
|
461
|
-
return 0
|
|
462
|
-
fi
|
|
463
|
-
set +e
|
|
464
|
-
bash "$NOTIFY_BIN" \
|
|
465
|
-
--hq-root "$HQ_ROOT" \
|
|
466
|
-
--job-id "$job_id" \
|
|
467
|
-
--job-name "$job_name" \
|
|
468
|
-
--outcome "$outcome" \
|
|
469
|
-
--summary "$summary" \
|
|
470
|
-
--duration "$duration" \
|
|
471
|
-
--log-path "$log_path" \
|
|
472
|
-
--failure-class "$failure_class" \
|
|
473
|
-
--next-action "$next_action" \
|
|
474
|
-
--owner "$owner" \
|
|
475
|
-
--notify "$notify_mode" \
|
|
476
|
-
>>"$log_path" 2>&1
|
|
477
|
-
local nrc=$?
|
|
478
|
-
set -e
|
|
479
|
-
if [ "$nrc" -ne 0 ]; then
|
|
480
|
-
log "notify helper exited $nrc (ignored)"
|
|
481
|
-
fi
|
|
482
|
-
return 0
|
|
483
|
-
}
|
|
484
|
-
|
|
485
|
-
call_remediate() {
|
|
486
|
-
local failure_class="$1" job_id="$2" company="$3"
|
|
487
|
-
if [ ! -x "$REMEDIATE_BIN" ] && [ -f "$REMEDIATE_BIN" ]; then
|
|
488
|
-
chmod +x "$REMEDIATE_BIN" 2>/dev/null || true
|
|
489
|
-
fi
|
|
490
|
-
if [ ! -f "$REMEDIATE_BIN" ]; then
|
|
491
|
-
echo '{"retry_recommended":false,"attempted":false,"notes":"remediate missing"}'
|
|
492
|
-
return 0
|
|
493
|
-
fi
|
|
494
|
-
local out
|
|
495
|
-
set +e
|
|
496
|
-
out="$(
|
|
497
|
-
HQ_JOB_REMEDIATE_LOG="${LOG_FILE:-/dev/null}" \
|
|
498
|
-
bash "$REMEDIATE_BIN" \
|
|
499
|
-
--hq-root "$HQ_ROOT" \
|
|
500
|
-
--failure-class "$failure_class" \
|
|
501
|
-
--job-id "$job_id" \
|
|
502
|
-
--company "$company" 2>>"${LOG_FILE:-/dev/null}"
|
|
503
|
-
)"
|
|
504
|
-
local rrc=$?
|
|
505
|
-
set -e
|
|
506
|
-
if [ "$rrc" -ne 0 ] || [ -z "$out" ]; then
|
|
507
|
-
echo '{"retry_recommended":false,"attempted":false,"notes":"remediate failed"}'
|
|
508
|
-
return 0
|
|
509
|
-
fi
|
|
510
|
-
printf '%s' "$out"
|
|
511
|
-
}
|
|
512
|
-
|
|
513
|
-
# --- resolve job ---
|
|
514
|
-
JOB_FILE="$(resolve_job_file)"
|
|
515
|
-
[ -f "$JOB_FILE" ] || die "job file missing: $JOB_FILE"
|
|
516
|
-
|
|
517
|
-
JOB_ID="$(yq -r '.id // ""' "$JOB_FILE")"
|
|
518
|
-
JOB_NAME="$(yq -r '.name // .id // ""' "$JOB_FILE")"
|
|
519
|
-
RUNTIME="$(yq -r '.runtime // ""' "$JOB_FILE")"
|
|
520
|
-
TIMEOUT_SEC="$(yq -r '.timeout_seconds // 600' "$JOB_FILE")"
|
|
521
|
-
CWD_REL="$(yq -r '.cwd // .requirements.cwd // ""' "$JOB_FILE")"
|
|
522
|
-
COMPANY="$(yq -r '.requirements.company // ""' "$JOB_FILE")"
|
|
523
|
-
OWNER="$(yq -r '.owner // ""' "$JOB_FILE")"
|
|
524
|
-
NOTIFY_MODE="$(yq -r '.notify // "profile"' "$JOB_FILE")"
|
|
525
|
-
ENABLED="$(yq -o=json '.' "$JOB_FILE" | jq -r 'if has("enabled") then (.enabled|tostring) else "true" end')"
|
|
526
|
-
# exec.surface overrides profile execution_surface; default headless.
|
|
527
|
-
SURFACE="$(yq -r '.exec.surface // ""' "$JOB_FILE")"
|
|
528
|
-
if [ -z "$SURFACE" ] || [ "$SURFACE" = "null" ]; then
|
|
529
|
-
ALERTS_FILE="${HQ_JOB_ALERTS_FILE:-${HQ_ROOT}/personal/settings/schedule-alerts.yaml}"
|
|
530
|
-
if [ -f "$ALERTS_FILE" ]; then
|
|
531
|
-
SURFACE="$(yq -r '.execution_surface // ""' "$ALERTS_FILE" 2>/dev/null || true)"
|
|
532
|
-
fi
|
|
533
|
-
fi
|
|
534
|
-
case "$SURFACE" in
|
|
535
|
-
remote) SURFACE="remote" ;;
|
|
536
|
-
*) SURFACE="headless" ;;
|
|
537
|
-
esac
|
|
538
|
-
# hq dm accepts personUid recipients; do not replace a job owner with this Outpost's identity.
|
|
539
|
-
|
|
540
|
-
[ -n "$JOB_ID" ] && [ "$JOB_ID" != "null" ] || die "job missing id: $JOB_FILE"
|
|
541
|
-
[ -n "$RUNTIME" ] && [ "$RUNTIME" != "null" ] || die "job missing runtime: $JOB_FILE"
|
|
542
|
-
[ -n "$JOB_NAME" ] || JOB_NAME="$JOB_ID"
|
|
543
|
-
[ -n "$NOTIFY_MODE" ] && [ "$NOTIFY_MODE" != "null" ] || NOTIFY_MODE="profile"
|
|
544
|
-
case "$TIMEOUT_SEC" in
|
|
545
|
-
''|*[!0-9]*) TIMEOUT_SEC=600 ;;
|
|
546
|
-
esac
|
|
547
|
-
if [ "$TIMEOUT_SEC" -lt 60 ]; then TIMEOUT_SEC=60; fi
|
|
548
|
-
if [ "$TIMEOUT_SEC" -gt 14400 ]; then TIMEOUT_SEC=14400; fi
|
|
549
|
-
|
|
550
|
-
if [ "$ENABLED" = "false" ] || [ "$ENABLED" = "False" ] || [ "$ENABLED" = "0" ]; then
|
|
551
|
-
log "skip job $JOB_ID — enabled=false"
|
|
552
|
-
exit 0
|
|
553
|
-
fi
|
|
554
|
-
|
|
555
|
-
LOG_BASE="${HQ_JOB_LOG_DIR:-${HOME}/.hq/jobs/logs}"
|
|
556
|
-
LOG_DIR="$LOG_BASE/$JOB_ID"
|
|
557
|
-
mkdir -p "$LOG_DIR"
|
|
558
|
-
LOCK_PATH="${HQ_JOB_LOCK:-${HOME}/.hq/jobs/run.lock}"
|
|
559
|
-
mkdir -p "$(dirname "$LOCK_PATH")"
|
|
560
|
-
|
|
561
|
-
FLOCK_BIN="${HQ_JOB_FLOCK_BIN:-}"
|
|
562
|
-
if [ -z "$FLOCK_BIN" ]; then
|
|
563
|
-
command -v flock >/dev/null 2>&1 || die "Outpost/Linux requires util-linux flock(1) for serialized job runs"
|
|
564
|
-
FLOCK_BIN="$(command -v flock)"
|
|
565
|
-
fi
|
|
566
|
-
TIMEOUT_BIN="${HQ_JOB_TIMEOUT_BIN:-}"
|
|
567
|
-
if [ -z "$TIMEOUT_BIN" ]; then
|
|
568
|
-
if command -v timeout >/dev/null 2>&1; then
|
|
569
|
-
TIMEOUT_BIN="timeout"
|
|
570
|
-
elif command -v gtimeout >/dev/null 2>&1; then
|
|
571
|
-
TIMEOUT_BIN="gtimeout"
|
|
572
|
-
else
|
|
573
|
-
TIMEOUT_BIN=""
|
|
574
|
-
fi
|
|
575
|
-
fi
|
|
576
|
-
|
|
577
|
-
RUN_STAMP="$(date -u +%Y%m%dT%H%M%SZ 2>/dev/null || date +%Y%m%d%H%M%S)"
|
|
578
|
-
LOG_FILE="$LOG_DIR/${RUN_STAMP}.log"
|
|
579
|
-
STATUS_FILE="$LOG_DIR/${RUN_STAMP}.status"
|
|
580
|
-
|
|
581
|
-
# Serialize all scheduled runs on this box.
|
|
582
|
-
exec 9>"$LOCK_PATH"
|
|
583
|
-
if ! "$FLOCK_BIN" -x 9; then
|
|
584
|
-
die "failed to acquire global run lock: $LOCK_PATH"
|
|
585
|
-
fi
|
|
586
|
-
|
|
587
|
-
STARTED_AT="$(iso_now)"
|
|
588
|
-
STARTED_EPOCH="$(epoch_now)"
|
|
589
|
-
SKIP_REASON=""
|
|
590
|
-
EXIT_CODE=0
|
|
591
|
-
|
|
592
|
-
{
|
|
593
|
-
echo "=== hq-job-run $JOB_ID ==="
|
|
594
|
-
echo "started_at=$STARTED_AT"
|
|
595
|
-
echo "runtime=$RUNTIME"
|
|
596
|
-
echo "timeout_seconds=$TIMEOUT_SEC"
|
|
597
|
-
echo "surface=$SURFACE"
|
|
598
|
-
echo "job_file=$JOB_FILE"
|
|
599
|
-
echo "hq_root=$HQ_ROOT"
|
|
600
|
-
} >>"$LOG_FILE"
|
|
601
|
-
|
|
602
|
-
AUTH_STATUS="$(check_runtime_auth "$RUNTIME" || true)"
|
|
603
|
-
if [ "$AUTH_STATUS" != "ok" ]; then
|
|
604
|
-
case "$AUTH_STATUS" in
|
|
605
|
-
no_cli)
|
|
606
|
-
SKIP_REASON="runtime CLI '$RUNTIME' not installed on PATH — skip (re-login/install on Outpost)"
|
|
607
|
-
;;
|
|
608
|
-
missing)
|
|
609
|
-
SKIP_REASON="runtime CLI '$RUNTIME' not authenticated on Outpost (cached credentials missing) — skip"
|
|
610
|
-
;;
|
|
611
|
-
*)
|
|
612
|
-
SKIP_REASON="unsupported or unknown runtime '$RUNTIME' — skip"
|
|
613
|
-
;;
|
|
614
|
-
esac
|
|
615
|
-
{
|
|
616
|
-
echo "skip_reason=$SKIP_REASON"
|
|
617
|
-
echo "auth_status=$AUTH_STATUS"
|
|
618
|
-
echo "failure_class=auth"
|
|
619
|
-
} >>"$LOG_FILE"
|
|
620
|
-
log "$SKIP_REASON"
|
|
621
|
-
ENDED_AT="$(iso_now)"
|
|
622
|
-
ENDED_EPOCH="$(epoch_now)"
|
|
623
|
-
DUR=$((ENDED_EPOCH - STARTED_EPOCH))
|
|
624
|
-
[ "$DUR" -ge 0 ] || DUR=0
|
|
625
|
-
printf 'exit=0\nskip_reason=%s\nfailure_class=auth\nstarted_at=%s\nended_at=%s\n' \
|
|
626
|
-
"$SKIP_REASON" "$STARTED_AT" "$ENDED_AT" >"$STATUS_FILE"
|
|
627
|
-
append_meter_marker "$JOB_ID" "$RUNTIME" "$STARTED_AT" "$ENDED_AT" 0 "$DUR" "$SKIP_REASON"
|
|
628
|
-
|
|
629
|
-
# Bounded remediate: Cognito refresh only — never device-login loop.
|
|
630
|
-
REM_OUT="$(call_remediate auth "$JOB_ID" "$COMPANY")"
|
|
631
|
-
echo "remediate=$REM_OUT" >>"$LOG_FILE"
|
|
632
|
-
NEXT_ACT="$(printf '%s' "$REM_OUT" | jq -r '.next_action // empty')"
|
|
633
|
-
[ -n "$NEXT_ACT" ] || NEXT_ACT="$(next_action_for_class auth "$RUNTIME")"
|
|
634
|
-
NA_JSON="$(jq -nc --arg a "$NEXT_ACT" 'if $a == "" then [] else [$a] end')"
|
|
635
|
-
RUN_ID="run-${JOB_ID}-${RUN_STAMP}-authskip"
|
|
636
|
-
ingest_run_status "$JOB_ID" "$ENDED_AT" "$RUN_ID" 1 "auth" "$NA_JSON"
|
|
637
|
-
call_notify "$JOB_ID" "$JOB_NAME" "failed" "$SKIP_REASON" "$DUR" "$LOG_FILE" \
|
|
638
|
-
"auth" "$NEXT_ACT" "$OWNER" "$NOTIFY_MODE"
|
|
639
|
-
exit 0
|
|
640
|
-
fi
|
|
641
|
-
|
|
642
|
-
# Working directory — must resolve under HQ_ROOT (no absolute / no .. escape)
|
|
643
|
-
RUN_CWD="$HQ_ROOT"
|
|
644
|
-
if [ -n "$CWD_REL" ] && [ "$CWD_REL" != "null" ]; then
|
|
645
|
-
case "$CWD_REL" in
|
|
646
|
-
/*)
|
|
647
|
-
SKIP_REASON="cwd absolute not allowed: $CWD_REL — skip"
|
|
648
|
-
log "$SKIP_REASON"
|
|
649
|
-
{
|
|
650
|
-
echo "skip_reason=$SKIP_REASON"
|
|
651
|
-
echo "failure_class=infra"
|
|
652
|
-
} >>"$LOG_FILE"
|
|
653
|
-
ENDED_AT="$(iso_now)"
|
|
654
|
-
ENDED_EPOCH="$(epoch_now)"
|
|
655
|
-
DUR=$((ENDED_EPOCH - STARTED_EPOCH))
|
|
656
|
-
[ "$DUR" -ge 0 ] || DUR=0
|
|
657
|
-
printf 'exit=0\nskip_reason=%s\nfailure_class=infra\nstarted_at=%s\nended_at=%s\n' \
|
|
658
|
-
"$SKIP_REASON" "$STARTED_AT" "$ENDED_AT" >"$STATUS_FILE"
|
|
659
|
-
append_meter_marker "$JOB_ID" "$RUNTIME" "$STARTED_AT" "$ENDED_AT" 0 "$DUR" "$SKIP_REASON"
|
|
660
|
-
NEXT_ACT="fix job cwd to an HQ-root-relative path (no absolute, no '..')"
|
|
661
|
-
NA_JSON="$(jq -nc --arg a "$NEXT_ACT" '[$a]')"
|
|
662
|
-
RUN_ID="run-${JOB_ID}-${RUN_STAMP}-cwd"
|
|
663
|
-
ingest_run_status "$JOB_ID" "$ENDED_AT" "$RUN_ID" 1 "infra" "$NA_JSON"
|
|
664
|
-
call_notify "$JOB_ID" "$JOB_NAME" "blocked" "$SKIP_REASON" "$DUR" "$LOG_FILE" \
|
|
665
|
-
"infra" "$NEXT_ACT" "$OWNER" "$NOTIFY_MODE"
|
|
666
|
-
exit 0
|
|
667
|
-
;;
|
|
668
|
-
esac
|
|
669
|
-
_cwd_part=""
|
|
670
|
-
_old_ifs="$IFS"
|
|
671
|
-
IFS='/'
|
|
672
|
-
# shellcheck disable=SC2086
|
|
673
|
-
for _cwd_part in $CWD_REL; do
|
|
674
|
-
if [ "$_cwd_part" = ".." ]; then
|
|
675
|
-
IFS="$_old_ifs"
|
|
676
|
-
SKIP_REASON="cwd contains '..': $CWD_REL — skip"
|
|
677
|
-
log "$SKIP_REASON"
|
|
678
|
-
{
|
|
679
|
-
echo "skip_reason=$SKIP_REASON"
|
|
680
|
-
echo "failure_class=infra"
|
|
681
|
-
} >>"$LOG_FILE"
|
|
682
|
-
ENDED_AT="$(iso_now)"
|
|
683
|
-
ENDED_EPOCH="$(epoch_now)"
|
|
684
|
-
DUR=$((ENDED_EPOCH - STARTED_EPOCH))
|
|
685
|
-
[ "$DUR" -ge 0 ] || DUR=0
|
|
686
|
-
printf 'exit=0\nskip_reason=%s\nfailure_class=infra\nstarted_at=%s\nended_at=%s\n' \
|
|
687
|
-
"$SKIP_REASON" "$STARTED_AT" "$ENDED_AT" >"$STATUS_FILE"
|
|
688
|
-
append_meter_marker "$JOB_ID" "$RUNTIME" "$STARTED_AT" "$ENDED_AT" 0 "$DUR" "$SKIP_REASON"
|
|
689
|
-
NEXT_ACT="fix job cwd to an HQ-root-relative path without '..'"
|
|
690
|
-
NA_JSON="$(jq -nc --arg a "$NEXT_ACT" '[$a]')"
|
|
691
|
-
RUN_ID="run-${JOB_ID}-${RUN_STAMP}-cwd"
|
|
692
|
-
ingest_run_status "$JOB_ID" "$ENDED_AT" "$RUN_ID" 1 "infra" "$NA_JSON"
|
|
693
|
-
call_notify "$JOB_ID" "$JOB_NAME" "blocked" "$SKIP_REASON" "$DUR" "$LOG_FILE" \
|
|
694
|
-
"infra" "$NEXT_ACT" "$OWNER" "$NOTIFY_MODE"
|
|
695
|
-
exit 0
|
|
696
|
-
fi
|
|
697
|
-
done
|
|
698
|
-
IFS="$_old_ifs"
|
|
699
|
-
HQ_CANON="$(realpath -m "$HQ_ROOT" 2>/dev/null || realpath "$HQ_ROOT" 2>/dev/null || true)"
|
|
700
|
-
RUN_CWD="$(realpath -m "$HQ_ROOT/$CWD_REL" 2>/dev/null || true)"
|
|
701
|
-
if [ -z "$RUN_CWD" ] && [ -d "$HQ_ROOT/$CWD_REL" ]; then
|
|
702
|
-
RUN_CWD="$(realpath "$HQ_ROOT/$CWD_REL" 2>/dev/null || true)"
|
|
703
|
-
fi
|
|
704
|
-
if [ -z "$HQ_CANON" ] || [ -z "$RUN_CWD" ]; then
|
|
705
|
-
HQ_CANON="$(hq_normpath "$HQ_ROOT")"
|
|
706
|
-
RUN_CWD="$(hq_normpath "$HQ_ROOT/$CWD_REL")"
|
|
707
|
-
fi
|
|
708
|
-
case "$RUN_CWD" in
|
|
709
|
-
"$HQ_CANON"|"$HQ_CANON"/*) ;;
|
|
710
|
-
*)
|
|
711
|
-
SKIP_REASON="cwd escapes HQ_ROOT: $CWD_REL → $RUN_CWD — skip"
|
|
712
|
-
log "$SKIP_REASON"
|
|
713
|
-
{
|
|
714
|
-
echo "skip_reason=$SKIP_REASON"
|
|
715
|
-
echo "failure_class=infra"
|
|
716
|
-
} >>"$LOG_FILE"
|
|
717
|
-
ENDED_AT="$(iso_now)"
|
|
718
|
-
ENDED_EPOCH="$(epoch_now)"
|
|
719
|
-
DUR=$((ENDED_EPOCH - STARTED_EPOCH))
|
|
720
|
-
[ "$DUR" -ge 0 ] || DUR=0
|
|
721
|
-
printf 'exit=0\nskip_reason=%s\nfailure_class=infra\nstarted_at=%s\nended_at=%s\n' \
|
|
722
|
-
"$SKIP_REASON" "$STARTED_AT" "$ENDED_AT" >"$STATUS_FILE"
|
|
723
|
-
append_meter_marker "$JOB_ID" "$RUNTIME" "$STARTED_AT" "$ENDED_AT" 0 "$DUR" "$SKIP_REASON"
|
|
724
|
-
NEXT_ACT="fix job cwd to stay under HQ root"
|
|
725
|
-
NA_JSON="$(jq -nc --arg a "$NEXT_ACT" '[$a]')"
|
|
726
|
-
RUN_ID="run-${JOB_ID}-${RUN_STAMP}-cwd"
|
|
727
|
-
ingest_run_status "$JOB_ID" "$ENDED_AT" "$RUN_ID" 1 "infra" "$NA_JSON"
|
|
728
|
-
call_notify "$JOB_ID" "$JOB_NAME" "blocked" "$SKIP_REASON" "$DUR" "$LOG_FILE" \
|
|
729
|
-
"infra" "$NEXT_ACT" "$OWNER" "$NOTIFY_MODE"
|
|
730
|
-
exit 0
|
|
731
|
-
;;
|
|
732
|
-
esac
|
|
733
|
-
fi
|
|
734
|
-
if [ ! -d "$RUN_CWD" ]; then
|
|
735
|
-
SKIP_REASON="cwd missing: $RUN_CWD — skip"
|
|
736
|
-
log "$SKIP_REASON"
|
|
737
|
-
{
|
|
738
|
-
echo "skip_reason=$SKIP_REASON"
|
|
739
|
-
echo "failure_class=infra"
|
|
740
|
-
} >>"$LOG_FILE"
|
|
741
|
-
ENDED_AT="$(iso_now)"
|
|
742
|
-
ENDED_EPOCH="$(epoch_now)"
|
|
743
|
-
DUR=$((ENDED_EPOCH - STARTED_EPOCH))
|
|
744
|
-
[ "$DUR" -ge 0 ] || DUR=0
|
|
745
|
-
printf 'exit=0\nskip_reason=%s\nfailure_class=infra\nstarted_at=%s\nended_at=%s\n' \
|
|
746
|
-
"$SKIP_REASON" "$STARTED_AT" "$ENDED_AT" >"$STATUS_FILE"
|
|
747
|
-
append_meter_marker "$JOB_ID" "$RUNTIME" "$STARTED_AT" "$ENDED_AT" 0 "$DUR" "$SKIP_REASON"
|
|
748
|
-
NEXT_ACT="create missing cwd on Outpost: $RUN_CWD"
|
|
749
|
-
NA_JSON="$(jq -nc --arg a "$NEXT_ACT" '[$a]')"
|
|
750
|
-
RUN_ID="run-${JOB_ID}-${RUN_STAMP}-cwd"
|
|
751
|
-
ingest_run_status "$JOB_ID" "$ENDED_AT" "$RUN_ID" 1 "infra" "$NA_JSON"
|
|
752
|
-
call_notify "$JOB_ID" "$JOB_NAME" "blocked" "$SKIP_REASON" "$DUR" "$LOG_FILE" \
|
|
753
|
-
"infra" "$NEXT_ACT" "$OWNER" "$NOTIFY_MODE"
|
|
754
|
-
exit 0
|
|
755
|
-
fi
|
|
756
|
-
|
|
757
|
-
PROMPT_TEXT="$(build_prompt "$JOB_FILE")"
|
|
758
|
-
|
|
759
|
-
# Build secret name list for hq secrets exec allowlist injection.
|
|
760
|
-
SECRET_NAMES=()
|
|
761
|
-
SECRET_COUNT="$(yq -r '.requirements.secrets // [] | length' "$JOB_FILE")"
|
|
762
|
-
if [ "$SECRET_COUNT" != "null" ] && [ "${SECRET_COUNT:-0}" -gt 0 ] 2>/dev/null; then
|
|
763
|
-
i=0
|
|
764
|
-
while [ "$i" -lt "$SECRET_COUNT" ]; do
|
|
765
|
-
sn="$(yq -r ".requirements.secrets[$i]" "$JOB_FILE")"
|
|
766
|
-
i=$((i + 1))
|
|
767
|
-
[ -n "$sn" ] && [ "$sn" != "null" ] || continue
|
|
768
|
-
SECRET_NAMES+=("$sn")
|
|
769
|
-
done
|
|
770
|
-
fi
|
|
771
|
-
|
|
772
|
-
# Claude remote surface: TTY + --remote-control so the run appears in Desktop.
|
|
773
|
-
# Auto-dismisses known startup dialogs; copies pane output into LOG_FILE.
|
|
774
|
-
# Launcher is a temp script (not nested shell quoting) to keep prompts/safe.
|
|
775
|
-
run_claude_remote() {
|
|
776
|
-
local cmd_rc=0
|
|
777
|
-
if ! command -v tmux >/dev/null 2>&1; then
|
|
778
|
-
echo "remote surface requested but tmux missing — falling back to headless" >>"$LOG_FILE"
|
|
779
|
-
SURFACE="headless"
|
|
780
|
-
return 99
|
|
781
|
-
fi
|
|
782
|
-
local sess="hq-job-${JOB_ID}-${RUN_STAMP}"
|
|
783
|
-
local rc_name="schedule/${JOB_ID}"
|
|
784
|
-
local deadline=$((TIMEOUT_SEC + 30))
|
|
785
|
-
local i=0
|
|
786
|
-
local pane=""
|
|
787
|
-
local exit_file="$LOG_DIR/${RUN_STAMP}.remote-exit"
|
|
788
|
-
local done_file="$LOG_DIR/${RUN_STAMP}.remote-done"
|
|
789
|
-
local launcher="$LOG_DIR/${RUN_STAMP}.remote-launch.sh"
|
|
790
|
-
# Completion is a sentinel file — not pane text (prompt instructions also
|
|
791
|
-
# contain marker words and caused false early teardown).
|
|
792
|
-
rm -f "$done_file"
|
|
793
|
-
local remote_prompt
|
|
794
|
-
remote_prompt="${PROMPT_TEXT}
|
|
795
|
-
|
|
796
|
-
When ALL work above is fully finished (success or failure), create this exact empty file with a shell tool (and do not create it earlier):
|
|
797
|
-
touch $(printf %q "$done_file")
|
|
798
|
-
Then stop."
|
|
799
|
-
echo "surface=remote remote_name=$rc_name tmux_session=$sess done_file=$done_file" >>"$LOG_FILE"
|
|
800
|
-
{
|
|
801
|
-
echo '#!/usr/bin/env bash'
|
|
802
|
-
echo 'set -uo pipefail'
|
|
803
|
-
printf 'cd %q\n' "$RUN_CWD"
|
|
804
|
-
printf 'claude --remote-control %q --permission-mode bypassPermissions --dangerously-skip-permissions %q\n' "$rc_name" "$remote_prompt"
|
|
805
|
-
echo 'rc=$?'
|
|
806
|
-
printf 'echo EXIT=$rc > %q\n' "$exit_file"
|
|
807
|
-
echo 'sleep 15'
|
|
808
|
-
echo 'exit "$rc"'
|
|
809
|
-
} >"$launcher"
|
|
810
|
-
chmod 700 "$launcher"
|
|
811
|
-
tmux kill-session -t "$sess" 2>/dev/null || true
|
|
812
|
-
tmux new-session -d -s "$sess" -x 120 -y 40 "$launcher"
|
|
813
|
-
while [ "$i" -lt "$deadline" ]; do
|
|
814
|
-
sleep 2
|
|
815
|
-
i=$((i + 2))
|
|
816
|
-
pane="$(tmux capture-pane -t "$sess" -p -S -120 2>/dev/null || true)"
|
|
817
|
-
# Startup / tool-permission dialogs (remote TTY can still prompt)
|
|
818
|
-
if printf '%s\n' "$pane" | grep -q 'Yes, I accept'; then
|
|
819
|
-
tmux send-keys -t "$sess" "2" Enter
|
|
820
|
-
continue
|
|
821
|
-
fi
|
|
822
|
-
if printf '%s\n' "$pane" | grep -q 'Try the new fullscreen renderer'; then
|
|
823
|
-
tmux send-keys -t "$sess" "2" Enter
|
|
824
|
-
continue
|
|
825
|
-
fi
|
|
826
|
-
if printf '%s\n' "$pane" | grep -q 'Do you want to proceed'; then
|
|
827
|
-
# "2" = Yes, and don't ask again for similar commands
|
|
828
|
-
tmux send-keys -t "$sess" "2" Enter
|
|
829
|
-
echo "auto-accepted tool permission prompt" >>"$LOG_FILE"
|
|
830
|
-
continue
|
|
831
|
-
fi
|
|
832
|
-
if [ -f "$done_file" ]; then
|
|
833
|
-
echo "remote done sentinel present" >>"$LOG_FILE"
|
|
834
|
-
sleep 2
|
|
835
|
-
pane="$(tmux capture-pane -t "$sess" -p -S -300 2>/dev/null || true)"
|
|
836
|
-
cmd_rc=0
|
|
837
|
-
break
|
|
838
|
-
fi
|
|
839
|
-
if [ -f "$exit_file" ]; then
|
|
840
|
-
cmd_rc="$(tr -dc '0-9' <"$exit_file" | head -c 3)"
|
|
841
|
-
[ -n "$cmd_rc" ] || cmd_rc=0
|
|
842
|
-
break
|
|
843
|
-
fi
|
|
844
|
-
done
|
|
845
|
-
if [ "$i" -ge "$deadline" ]; then
|
|
846
|
-
echo "remote surface timed out after ${TIMEOUT_SEC}s" >>"$LOG_FILE"
|
|
847
|
-
cmd_rc=124
|
|
848
|
-
fi
|
|
849
|
-
{
|
|
850
|
-
echo "=== remote pane capture ==="
|
|
851
|
-
tmux capture-pane -t "$sess" -p -S -300 2>/dev/null || true
|
|
852
|
-
} >>"$LOG_FILE"
|
|
853
|
-
# End the Remote Control process so Desktop moves schedule/<job-id> to Archived.
|
|
854
|
-
echo "tearing down remote session ${rc_name} (review under Desktop Archived)" >>"$LOG_FILE"
|
|
855
|
-
tmux kill-session -t "$sess" 2>/dev/null || true
|
|
856
|
-
rm -f "$launcher"
|
|
857
|
-
return "$cmd_rc"
|
|
858
|
-
}
|
|
859
|
-
|
|
860
|
-
run_agent() {
|
|
861
|
-
local cmd_rc=0
|
|
862
|
-
if [ "${HQ_JOB_RUN_SKIP_EXEC:-}" = "1" ]; then
|
|
863
|
-
echo "(HQ_JOB_RUN_SKIP_EXEC=1 — not invoking runtime CLI)" >>"$LOG_FILE"
|
|
864
|
-
return 0
|
|
865
|
-
fi
|
|
866
|
-
|
|
867
|
-
if [ "$RUNTIME" = "claude" ] && [ "$SURFACE" = "remote" ]; then
|
|
868
|
-
set +e
|
|
869
|
-
run_claude_remote
|
|
870
|
-
cmd_rc=$?
|
|
871
|
-
set -e
|
|
872
|
-
if [ "$cmd_rc" -ne 99 ]; then
|
|
873
|
-
return "$cmd_rc"
|
|
874
|
-
fi
|
|
875
|
-
# 99 = fall through to headless
|
|
876
|
-
fi
|
|
877
|
-
if [ "$RUNTIME" = "codex" ] && [ "$SURFACE" = "remote" ]; then
|
|
878
|
-
echo "surface=remote ignored for codex — using headless" >>"$LOG_FILE"
|
|
879
|
-
fi
|
|
880
|
-
|
|
881
|
-
local -a agent_cmd
|
|
882
|
-
case "$RUNTIME" in
|
|
883
|
-
claude)
|
|
884
|
-
agent_cmd=(claude -p "$PROMPT_TEXT" --output-format text --permission-mode bypassPermissions)
|
|
885
|
-
;;
|
|
886
|
-
codex)
|
|
887
|
-
# Spike-validated flags on codex-cli 0.147 (NOT --ask-for-approval never).
|
|
888
|
-
agent_cmd=(codex exec -s workspace-write --skip-git-repo-check -C "$RUN_CWD" "$PROMPT_TEXT")
|
|
889
|
-
;;
|
|
890
|
-
*)
|
|
891
|
-
echo "unknown runtime: $RUNTIME" >>"$LOG_FILE"
|
|
892
|
-
return 2
|
|
893
|
-
;;
|
|
894
|
-
esac
|
|
895
|
-
|
|
896
|
-
local -a wrap_cmd=()
|
|
897
|
-
if [ -n "$TIMEOUT_BIN" ]; then
|
|
898
|
-
wrap_cmd=("$TIMEOUT_BIN" --signal=TERM --kill-after=15s "${TIMEOUT_SEC}s")
|
|
899
|
-
fi
|
|
900
|
-
|
|
901
|
-
# Inject only confirmed requirements.secrets[] via hq secrets exec.
|
|
902
|
-
# CLI shape: hq secrets [--company S|--personal] exec --only K1,K2 -- <cmd>
|
|
903
|
-
if [ "${#SECRET_NAMES[@]}" -gt 0 ] && command -v hq >/dev/null 2>&1; then
|
|
904
|
-
local only_csv=""
|
|
905
|
-
local sn
|
|
906
|
-
for sn in "${SECRET_NAMES[@]}"; do
|
|
907
|
-
if [ -z "$only_csv" ]; then
|
|
908
|
-
only_csv="$sn"
|
|
909
|
-
else
|
|
910
|
-
only_csv="${only_csv},${sn}"
|
|
911
|
-
fi
|
|
912
|
-
done
|
|
913
|
-
local -a secrets_cmd=(hq secrets)
|
|
914
|
-
if [ -n "$COMPANY" ] && [ "$COMPANY" != "null" ]; then
|
|
915
|
-
secrets_cmd+=(--company "$COMPANY")
|
|
916
|
-
else
|
|
917
|
-
secrets_cmd+=(--personal)
|
|
918
|
-
fi
|
|
919
|
-
secrets_cmd+=(exec --only "$only_csv" --script "${BASH_SOURCE[0]}" --)
|
|
920
|
-
(
|
|
921
|
-
cd "$RUN_CWD"
|
|
922
|
-
if [ "${#wrap_cmd[@]}" -gt 0 ]; then
|
|
923
|
-
"${secrets_cmd[@]}" "${wrap_cmd[@]}" "${agent_cmd[@]}"
|
|
924
|
-
else
|
|
925
|
-
"${secrets_cmd[@]}" "${agent_cmd[@]}"
|
|
926
|
-
fi
|
|
927
|
-
) >>"$LOG_FILE" 2>&1 || cmd_rc=$?
|
|
928
|
-
else
|
|
929
|
-
if [ "${#SECRET_NAMES[@]}" -gt 0 ]; then
|
|
930
|
-
echo "warning: requirements.secrets set but hq CLI missing — running without secrets exec" >>"$LOG_FILE"
|
|
931
|
-
fi
|
|
932
|
-
(
|
|
933
|
-
cd "$RUN_CWD"
|
|
934
|
-
if [ "${#wrap_cmd[@]}" -gt 0 ]; then
|
|
935
|
-
"${wrap_cmd[@]}" "${agent_cmd[@]}"
|
|
936
|
-
else
|
|
937
|
-
"${agent_cmd[@]}"
|
|
938
|
-
fi
|
|
939
|
-
) >>"$LOG_FILE" 2>&1 || cmd_rc=$?
|
|
940
|
-
fi
|
|
941
|
-
return "$cmd_rc"
|
|
942
|
-
}
|
|
943
|
-
|
|
944
|
-
RETRIED=0
|
|
945
|
-
FAILURE_CLASS=""
|
|
946
|
-
NEXT_ACT=""
|
|
947
|
-
|
|
948
|
-
set +e
|
|
949
|
-
run_agent
|
|
950
|
-
EXIT_CODE=$?
|
|
951
|
-
set -e
|
|
952
|
-
|
|
953
|
-
finalize_timestamps() {
|
|
954
|
-
ENDED_AT="$(iso_now)"
|
|
955
|
-
ENDED_EPOCH="$(epoch_now)"
|
|
956
|
-
DUR=$((ENDED_EPOCH - STARTED_EPOCH))
|
|
957
|
-
[ "$DUR" -ge 0 ] || DUR=0
|
|
958
|
-
}
|
|
959
|
-
|
|
960
|
-
if [ "$EXIT_CODE" -ne 0 ]; then
|
|
961
|
-
FAILURE_CLASS="$(classify_failure "$EXIT_CODE" "$LOG_FILE" "")"
|
|
962
|
-
echo "failure_class=$FAILURE_CLASS" >>"$LOG_FILE"
|
|
963
|
-
REM_OUT="$(call_remediate "$FAILURE_CLASS" "$JOB_ID" "$COMPANY")"
|
|
964
|
-
echo "remediate=$REM_OUT" >>"$LOG_FILE"
|
|
965
|
-
NEXT_ACT="$(printf '%s' "$REM_OUT" | jq -r '.next_action // empty')"
|
|
966
|
-
[ -n "$NEXT_ACT" ] || NEXT_ACT="$(next_action_for_class "$FAILURE_CLASS" "$RUNTIME")"
|
|
967
|
-
RETRY_REC="$(printf '%s' "$REM_OUT" | jq -r '.retry_recommended // false')"
|
|
968
|
-
if [ "$RETRY_REC" = "true" ] && [ "$RETRIED" -eq 0 ]; then
|
|
969
|
-
RETRIED=1
|
|
970
|
-
echo "remediate_retry=1" >>"$LOG_FILE"
|
|
971
|
-
log "self-remediate recommended retry for $FAILURE_CLASS — retrying once"
|
|
972
|
-
set +e
|
|
973
|
-
run_agent
|
|
974
|
-
EXIT_CODE=$?
|
|
975
|
-
set -e
|
|
976
|
-
if [ "$EXIT_CODE" -ne 0 ]; then
|
|
977
|
-
FAILURE_CLASS="$(classify_failure "$EXIT_CODE" "$LOG_FILE" "")"
|
|
978
|
-
echo "failure_class_after_retry=$FAILURE_CLASS" >>"$LOG_FILE"
|
|
979
|
-
NEXT_ACT="$(next_action_for_class "$FAILURE_CLASS" "$RUNTIME")"
|
|
980
|
-
else
|
|
981
|
-
FAILURE_CLASS=""
|
|
982
|
-
NEXT_ACT=""
|
|
983
|
-
fi
|
|
984
|
-
fi
|
|
985
|
-
fi
|
|
986
|
-
|
|
987
|
-
finalize_timestamps
|
|
988
|
-
|
|
989
|
-
{
|
|
990
|
-
echo "ended_at=$ENDED_AT"
|
|
991
|
-
echo "exit=$EXIT_CODE"
|
|
992
|
-
echo "duration_seconds=$DUR"
|
|
993
|
-
echo "retried=$RETRIED"
|
|
994
|
-
[ -n "$FAILURE_CLASS" ] && echo "failure_class=$FAILURE_CLASS"
|
|
995
|
-
} >>"$LOG_FILE"
|
|
996
|
-
|
|
997
|
-
printf 'exit=%s\nstarted_at=%s\nended_at=%s\nduration_seconds=%s\nfailure_class=%s\nretried=%s\n' \
|
|
998
|
-
"$EXIT_CODE" "$STARTED_AT" "$ENDED_AT" "$DUR" "${FAILURE_CLASS:-}" "$RETRIED" >"$STATUS_FILE"
|
|
999
|
-
|
|
1000
|
-
append_meter_marker "$JOB_ID" "$RUNTIME" "$STARTED_AT" "$ENDED_AT" "$EXIT_CODE" "$DUR" ""
|
|
1001
|
-
|
|
1002
|
-
RUN_ID="run-${JOB_ID}-${RUN_STAMP}"
|
|
1003
|
-
if [ "$EXIT_CODE" -eq 0 ]; then
|
|
1004
|
-
NA_JSON='[]'
|
|
1005
|
-
ingest_run_status "$JOB_ID" "$ENDED_AT" "$RUN_ID" 0 "null" "$NA_JSON"
|
|
1006
|
-
SUMMARY="completed ok"
|
|
1007
|
-
[ "$RETRIED" -eq 1 ] && SUMMARY="completed ok after self-remediation retry"
|
|
1008
|
-
call_notify "$JOB_ID" "$JOB_NAME" "ok" "$SUMMARY" "$DUR" "$LOG_FILE" \
|
|
1009
|
-
"" "" "$OWNER" "$NOTIFY_MODE"
|
|
1010
|
-
log "job $JOB_ID completed ok (${DUR}s)"
|
|
1011
|
-
exit 0
|
|
1012
|
-
fi
|
|
1013
|
-
|
|
1014
|
-
NA_JSON="$(jq -nc --arg a "$NEXT_ACT" 'if $a == "" then [] else [$a] end')"
|
|
1015
|
-
# Cap last_exit to 0–255 for ingest; preserve real exit for process.
|
|
1016
|
-
INGEST_EXIT="$EXIT_CODE"
|
|
1017
|
-
if [ "$INGEST_EXIT" -lt 0 ]; then INGEST_EXIT=1; fi
|
|
1018
|
-
if [ "$INGEST_EXIT" -gt 255 ]; then INGEST_EXIT=255; fi
|
|
1019
|
-
ingest_run_status "$JOB_ID" "$ENDED_AT" "$RUN_ID" "$INGEST_EXIT" "$FAILURE_CLASS" "$NA_JSON"
|
|
1020
|
-
SUMMARY="exit=$EXIT_CODE class=$FAILURE_CLASS"
|
|
1021
|
-
call_notify "$JOB_ID" "$JOB_NAME" "failed" "$SUMMARY" "$DUR" "$LOG_FILE" \
|
|
1022
|
-
"$FAILURE_CLASS" "$NEXT_ACT" "$OWNER" "$NOTIFY_MODE"
|
|
1023
|
-
|
|
1024
|
-
log "job $JOB_ID failed exit=$EXIT_CODE class=$FAILURE_CLASS (${DUR}s) log=$LOG_FILE"
|
|
1025
|
-
exit "$EXIT_CODE"
|