@indigoai-us/hq-cli 5.345.114 → 5.345.116

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/CHANGELOG.md +13 -0
  2. package/assets/scaffold/core/scripts/hq-delegate-send.sh +5 -1
  3. package/dist/commands/bot-jobs.d.ts +1 -11
  4. package/dist/commands/bot-jobs.js +13 -34
  5. package/dist/commands/core-audit-log-native.js +1 -1
  6. package/dist/commands/core-checkpoint-native.d.ts +27 -0
  7. package/dist/commands/core-checkpoint-native.js +85 -2
  8. package/dist/commands/core-dm-bind-native.js +1 -1
  9. package/dist/commands/core-git-pack-extension-check-native.js +1 -1
  10. package/dist/commands/core-job-native-utils.d.ts +2 -2
  11. package/dist/commands/core-job-native-utils.js +3 -11
  12. package/dist/commands/core-job-notify-native.js +1 -2
  13. package/dist/commands/core-job-probe-native.d.ts +2 -3
  14. package/dist/commands/core-job-probe-native.js +14 -22
  15. package/dist/commands/core-job-remediate-native.js +1 -2
  16. package/dist/commands/core-job-run-native.js +58 -54
  17. package/dist/commands/core-jobs-native.d.ts +1 -3
  18. package/dist/commands/core-jobs-native.js +16 -271
  19. package/dist/commands/core-jobs-schedule.d.ts +7 -0
  20. package/dist/commands/core-jobs-schedule.js +243 -0
  21. package/dist/commands/core-native-utilities.d.ts +2 -3
  22. package/dist/commands/core-native-utilities.js +2 -19
  23. package/dist/commands/core-pin-native.js +1 -1
  24. package/dist/commands/core-refresh-vault-access-native.js +1 -1
  25. package/dist/commands/core-resolve-company-command.js +2 -17
  26. package/dist/commands/core-work-mesh-bind-trusted-native.js +1 -1
  27. package/dist/commands/core.js +1 -17
  28. package/dist/fast-scaffold.d.ts +1 -1
  29. package/dist/fast-scaffold.js +0 -38
  30. package/dist/lib/daemon/catalog.d.ts +0 -2
  31. package/dist/lib/daemon/catalog.js +3 -6
  32. package/dist/lib/daemon/run.js +0 -7
  33. package/dist/lib/scan-packages/index.js +6 -0
  34. package/dist/lib/workers-registry/index.d.ts +16 -0
  35. package/dist/lib/workers-registry/index.js +1 -1
  36. package/dist/utils/resident-package-preload.js +2 -7
  37. package/package.json +1 -1
  38. package/assets/scaffold/core/scripts/audit-log.sh +0 -384
  39. package/assets/scaffold/core/scripts/git-pack-extension-check.sh +0 -164
  40. package/assets/scaffold/core/scripts/hq-dm-bind.sh +0 -590
  41. package/assets/scaffold/core/scripts/hq-job-notify.sh +0 -389
  42. package/assets/scaffold/core/scripts/hq-job-probe.sh +0 -865
  43. package/assets/scaffold/core/scripts/hq-job-remediate.sh +0 -359
  44. package/assets/scaffold/core/scripts/hq-job-run.sh +0 -1025
  45. package/assets/scaffold/core/scripts/hq-pin.sh +0 -118
  46. package/assets/scaffold/core/scripts/jobs-schedule-parse.sh +0 -310
  47. package/assets/scaffold/core/scripts/jobs-validate.sh +0 -482
  48. package/assets/scaffold/core/scripts/outpost-jobs-reconcile.sh +0 -836
  49. package/assets/scaffold/core/scripts/refresh-vault-access.sh +0 -188
  50. package/assets/scaffold/core/scripts/resolve-company.sh +0 -331
  51. package/assets/scaffold/core/scripts/work-mesh-live-bind-trusted.sh +0 -184
  52. package/dist/commands/core-job-post-exit-assets.d.ts +0 -8
  53. package/dist/commands/core-job-post-exit-assets.js +0 -37
  54. package/dist/commands/core-native-jobs-flag.d.ts +0 -4
  55. package/dist/commands/core-native-jobs-flag.js +0 -7
  56. package/dist/commands/core-native-utilities-flag.d.ts +0 -4
  57. package/dist/commands/core-native-utilities-flag.js +0 -7
@@ -1,1025 +0,0 @@
1
- #!/usr/bin/env bash
2
- # hq-core: public
3
- # hq-job-run.sh — serialized Outpost scheduled-job executor (US-004 + US-006).
4
- #
5
- # Invoked by systemd user units (hq-job-{id}.service) or on demand:
6
- # core/scripts/hq-job-run.sh --job-id <id>
7
- # core/scripts/hq-job-run.sh <job.yaml>
8
- #
9
- # Behavior:
10
- # - Global file lock (~/.hq/jobs/run.lock) — serialized-only (US-001)
11
- # - Enforces job timeout_seconds (default 600; schema 60–14400)
12
- # - Credentials only via `hq secrets exec` (requirements.secrets[]) or
13
- # the box's cached CLI auth — never env vars / unit-file secrets
14
- # - Trust model (7A): injects ONLY requirements.secrets[] — no static prompt scan
15
- # - Appends stdout/stderr + exit to ~/.hq/jobs/logs/{id}/
16
- # - Emits a compute-meter attribution marker (scheduled vs interactive)
17
- # - Classifies failures (auth|secrets|timeout|agent_error|infra), writes
18
- # failure_class to hq-pro jobs status (kind=run), bounded self-remediate,
19
- # then dm-notify via hq-job-notify.sh (collapse 1/24h; never create-thread)
20
- # - Missing runtime CLI auth: classify auth, notify re-login, exit 0 (unit OK)
21
- #
22
- # Runtime dispatch (US-001 spike flags):
23
- # claude → claude -p … --output-format text --permission-mode bypassPermissions
24
- # codex → codex exec -s workspace-write --skip-git-repo-check …
25
- #
26
- # Env / overrides (tests):
27
- # HQ_ROOT, HOME, HQ_JOB_LOCK, HQ_JOB_LOG_DIR, HQ_JOB_METER_DIR
28
- # HQ_JOB_TIMEOUT_BIN, HQ_JOB_FLOCK_BIN, HQ_JOB_NOW
29
- # HQ_JOB_PROBE_CLAUDE_CRED, HQ_JOB_PROBE_CODEX_AUTH (auth paths; shared with probe)
30
- # HQ_JOB_RUN_SKIP_EXEC=1 — acquire lock + write marker/logs without calling CLI
31
- # HQ_JOB_NOTIFY_BIN, HQ_JOB_REMEDIATE_BIN, HQ_JOB_RUN_NO_NOTIFY=1, HQ_JOB_RUN_NO_INGEST=1
32
- # HQ_JOB_RUN_CURL, HQ_PRO_API_URL, OUTPOST_USER_ID, OUTPOST_INSTANCE_TOKEN
33
- #
34
- # Exit: 0 success or intentional auth/cwd skip; 1 agent/job failure; 2 usage/config;
35
- # 124 timeout (GNU timeout convention when available).
36
-
37
- set -euo pipefail
38
-
39
- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
40
- HQ_ROOT="${HQ_ROOT:-$(cd "$SCRIPT_DIR/../.." && pwd)}"
41
- # shellcheck disable=SC1091
42
- . "$SCRIPT_DIR/hook-lib.sh"
43
- NOTIFY_BIN="${HQ_JOB_NOTIFY_BIN:-$SCRIPT_DIR/hq-job-notify.sh}"
44
- REMEDIATE_BIN="${HQ_JOB_REMEDIATE_BIN:-$SCRIPT_DIR/hq-job-remediate.sh}"
45
- AUTH_HEADER_FILE=""
46
-
47
- cleanup_auth_header_file() {
48
- if [ -n "${AUTH_HEADER_FILE:-}" ]; then
49
- rm -f -- "$AUTH_HEADER_FILE" 2>/dev/null || true
50
- AUTH_HEADER_FILE=""
51
- fi
52
- }
53
- trap cleanup_auth_header_file EXIT
54
-
55
- JOB_TARGET=""
56
- JOB_ID_ARG=""
57
-
58
- usage() {
59
- sed -n '3,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
60
- }
61
-
62
- die() {
63
- echo "hq-job-run: $*" >&2
64
- exit 2
65
- }
66
-
67
- log() {
68
- echo "hq-job-run: $*" >&2
69
- }
70
-
71
- while [ $# -gt 0 ]; do
72
- case "$1" in
73
- --hq-root)
74
- HQ_ROOT="${2:-}"
75
- [ -n "$HQ_ROOT" ] || die "--hq-root requires a directory"
76
- shift 2
77
- ;;
78
- --job-id)
79
- JOB_ID_ARG="${2:-}"
80
- [ -n "$JOB_ID_ARG" ] || die "--job-id requires a value"
81
- shift 2
82
- ;;
83
- -h|--help)
84
- usage
85
- exit 0
86
- ;;
87
- --)
88
- shift
89
- break
90
- ;;
91
- -*)
92
- die "unknown option: $1"
93
- ;;
94
- *)
95
- if [ -n "$JOB_TARGET" ]; then
96
- die "unexpected argument: $1"
97
- fi
98
- JOB_TARGET="$1"
99
- shift
100
- ;;
101
- esac
102
- done
103
-
104
- command -v yq >/dev/null 2>&1 || die "yq is required (mikefarah/yq)"
105
- command -v jq >/dev/null 2>&1 || die "jq is required"
106
-
107
- iso_now() {
108
- if [ -n "${HQ_JOB_NOW:-}" ]; then
109
- printf '%s' "$HQ_JOB_NOW"
110
- return
111
- fi
112
- date -u +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null \
113
- || date -u +"%Y-%m-%dT%H:%M:%S+00:00"
114
- }
115
-
116
- epoch_now() {
117
- date -u +%s 2>/dev/null || date +%s
118
- }
119
-
120
- claude_cred_path() {
121
- printf '%s' "${HQ_JOB_PROBE_CLAUDE_CRED:-${HOME}/.claude/.credentials.json}"
122
- }
123
-
124
- codex_auth_path() {
125
- printf '%s' "${HQ_JOB_PROBE_CODEX_AUTH:-${HOME}/.codex/auth.json}"
126
- }
127
-
128
- find_job_file_by_id() {
129
- local id="$1" f found=""
130
- while IFS= read -r f; do
131
- [ -n "$f" ] || continue
132
- if [ "$(yq -r '.id // ""' "$f" 2>/dev/null)" = "$id" ]; then
133
- if [ -n "$found" ]; then
134
- die "duplicate job id '$id' in registry: $found and $f"
135
- fi
136
- found="$f"
137
- fi
138
- done < <(
139
- {
140
- find "$HQ_ROOT/personal/jobs" -maxdepth 1 \( -name '*.yaml' -o -name '*.yml' \) -type f 2>/dev/null || true
141
- find "$HQ_ROOT/companies" -path '*/jobs/*' -maxdepth 3 \( -name '*.yaml' -o -name '*.yml' \) -type f 2>/dev/null || true
142
- } | grep -Ev '\.conflict-|/\.conflicts/' | sort
143
- )
144
- [ -n "$found" ] || return 1
145
- printf '%s' "$found"
146
- }
147
-
148
- resolve_job_file() {
149
- if [ -n "$JOB_ID_ARG" ]; then
150
- find_job_file_by_id "$JOB_ID_ARG" || die "job id not found: $JOB_ID_ARG"
151
- return
152
- fi
153
- if [ -z "$JOB_TARGET" ]; then
154
- die "usage: hq-job-run.sh --job-id <id> | <job.yaml>"
155
- fi
156
- if [ -f "$JOB_TARGET" ]; then
157
- printf '%s' "$JOB_TARGET"
158
- return
159
- fi
160
- if [ -f "$HQ_ROOT/$JOB_TARGET" ]; then
161
- printf '%s' "$HQ_ROOT/$JOB_TARGET"
162
- return
163
- fi
164
- find_job_file_by_id "$JOB_TARGET" || die "job not found: $JOB_TARGET"
165
- }
166
-
167
- check_runtime_auth() {
168
- local runtime="$1"
169
- case "$runtime" in
170
- claude)
171
- if ! command -v claude >/dev/null 2>&1; then
172
- printf 'no_cli'
173
- return 1
174
- fi
175
- if [ -s "$(claude_cred_path)" ]; then
176
- printf 'ok'
177
- return 0
178
- fi
179
- printf 'missing'
180
- return 1
181
- ;;
182
- codex)
183
- if ! command -v codex >/dev/null 2>&1; then
184
- printf 'no_cli'
185
- return 1
186
- fi
187
- if [ -s "$(codex_auth_path)" ]; then
188
- printf 'ok'
189
- return 0
190
- fi
191
- printf 'missing'
192
- return 1
193
- ;;
194
- *)
195
- printf 'unknown_runtime'
196
- return 1
197
- ;;
198
- esac
199
- }
200
-
201
- build_prompt() {
202
- local job_file="$1"
203
- local prompt skill
204
- prompt="$(yq -r '.exec.prompt // ""' "$job_file")"
205
- skill="$(yq -r '.exec.skill // ""' "$job_file")"
206
- if [ -n "$prompt" ] && [ "$prompt" != "null" ]; then
207
- printf '%s' "$prompt"
208
- return
209
- fi
210
- if [ -n "$skill" ] && [ "$skill" != "null" ]; then
211
- local args_json
212
- args_json="$(yq -o=json -I=0 '.exec.args // {}' "$job_file" 2>/dev/null || echo '{}')"
213
- printf 'Execute the HQ skill "%s" unattended with these args (JSON): %s. Follow the skill instructions exactly and produce a concise completion summary.' \
214
- "$skill" "$args_json"
215
- return
216
- fi
217
- die "job missing exec.prompt and exec.skill: $job_file"
218
- }
219
-
220
- append_meter_marker() {
221
- local job_id="$1" runtime="$2" started="$3" ended="$4" exit_status="$5" duration="$6" skip_reason="${7:-}"
222
- local dir="${HQ_JOB_METER_DIR:-${HOME}/.hq/jobs/meters}"
223
- mkdir -p "$dir"
224
- local line
225
- line="$(jq -nc \
226
- --arg schema "hq.outpost.job-run/v1" \
227
- --arg kind "scheduled-job-run" \
228
- --arg job_id "$job_id" \
229
- --arg runtime "$runtime" \
230
- --arg started_at "$started" \
231
- --arg ended_at "$ended" \
232
- --argjson exit_status "$exit_status" \
233
- --argjson duration_seconds "$duration" \
234
- --arg outpost_id "${OUTPOST_ID:-}" \
235
- --arg skip_reason "$skip_reason" \
236
- '{
237
- schema: $schema,
238
- kind: $kind,
239
- job_id: $job_id,
240
- runtime: $runtime,
241
- started_at: $started_at,
242
- ended_at: $ended_at,
243
- exit_status: $exit_status,
244
- duration_seconds: $duration_seconds
245
- }
246
- + (if $outpost_id != "" then {outpost_id: $outpost_id} else {} end)
247
- + (if $skip_reason != "" then {skip_reason: $skip_reason} else {} end)')"
248
- printf '%s\n' "$line" >>"$dir/runs.jsonl"
249
- chmod 600 "$dir/runs.jsonl" 2>/dev/null || true
250
- }
251
-
252
- api_base() {
253
- local base
254
- base="${HQ_PRO_API_URL:-${HQ_API_URL:-${HQ_VAULT_API_URL:-https://hqapi.hq.computer}}}"
255
- base="${base%/}"
256
- printf '%s' "$base"
257
- }
258
-
259
- resolve_box_identity() {
260
- local xtrace_was_set=0
261
- case "$-" in
262
- *x*) xtrace_was_set=1; set +x ;;
263
- esac
264
-
265
- if [ -n "${OUTPOST_USER_ID:-}" ] && [ -n "${OUTPOST_INSTANCE_TOKEN:-}" ]; then
266
- [ "$xtrace_was_set" -eq 0 ] || set -x
267
- return 0
268
- fi
269
- local envf="/etc/outpost/codex-identity.env"
270
- if [ -f "$envf" ]; then
271
- # shellcheck disable=SC1090
272
- . "$envf" 2>/dev/null || true
273
- if [ -n "${OUTPOST_USER_ID:-}" ] && [ -n "${OUTPOST_INSTANCE_TOKEN:-}" ]; then
274
- [ "$xtrace_was_set" -eq 0 ] || set -x
275
- return 0
276
- fi
277
- fi
278
- local runner="${HQ_JOB_PROBE_RUNNER:-/usr/local/bin/outpost-runner.sh}"
279
- if [ -f "$runner" ]; then
280
- OUTPOST_USER_ID="${OUTPOST_USER_ID:-$(sed -n 's/^USER_ID="\(.*\)"$/\1/p' "$runner" 2>/dev/null | head -1)}"
281
- OUTPOST_INSTANCE_TOKEN="${OUTPOST_INSTANCE_TOKEN:-$(sed -n 's/^INSTANCE_TOKEN="\(.*\)"$/\1/p' "$runner" 2>/dev/null | head -1)}"
282
- OUTPOST_ID="${OUTPOST_ID:-$(sed -n 's/^OUTPOST_ID="\(.*\)"$/\1/p' "$runner" 2>/dev/null | head -1)}"
283
- fi
284
- if [ -n "${OUTPOST_USER_ID:-}" ] && [ -n "${OUTPOST_INSTANCE_TOKEN:-}" ]; then
285
- [ "$xtrace_was_set" -eq 0 ] || set -x
286
- return 0
287
- fi
288
- [ "$xtrace_was_set" -eq 0 ] || set -x
289
- return 1
290
- }
291
-
292
- create_auth_header_file() {
293
- local xtrace_was_set=0 write_rc=0
294
- AUTH_HEADER_FILE="$(mktemp "${TMPDIR:-/tmp}/hq-job-run-auth-header.XXXXXX")"
295
- chmod 600 "$AUTH_HEADER_FILE"
296
- case "$-" in
297
- *x*) xtrace_was_set=1; set +x ;;
298
- esac
299
- if printf 'x-outpost-instance-token: %s\n' "$OUTPOST_INSTANCE_TOKEN" >"$AUTH_HEADER_FILE"; then
300
- write_rc=0
301
- else
302
- write_rc=$?
303
- fi
304
- [ "$xtrace_was_set" -eq 0 ] || set -x
305
- return "$write_rc"
306
- }
307
-
308
- # Classify exit + log tail → auth|secrets|timeout|agent_error|infra
309
- classify_failure() {
310
- local exit_code="$1" log_file="$2" auth_hint="${3:-}"
311
- if [ -n "$auth_hint" ] && [ "$auth_hint" != "ok" ]; then
312
- printf 'auth'
313
- return
314
- fi
315
- if [ "$exit_code" -eq 124 ] || [ "$exit_code" -eq 137 ]; then
316
- printf 'timeout'
317
- return
318
- fi
319
- local tail_txt=""
320
- if [ -n "$log_file" ] && [ -f "$log_file" ]; then
321
- tail_txt="$(tail -n 80 "$log_file" 2>/dev/null || true)"
322
- fi
323
- local lower
324
- lower="$(printf '%s' "$tail_txt" | tr '[:upper:]' '[:lower:]')"
325
-
326
- if printf '%s' "$lower" | grep -E \
327
- 'not logged in|please log in|authentication (failed|required)|unauthorized|401|invalid.*(token|credential)|credentials? (missing|expired|invalid)|device.?code|login required' >/dev/null; then
328
- printf 'auth'
329
- return
330
- fi
331
- if printf '%s' "$lower" | grep -E \
332
- 'secret.*(not found|missing|denied|not visible|not shared)|hq secrets.*(fail|error|denied)|missing vault|access denied.*secret' >/dev/null; then
333
- printf 'secrets'
334
- return
335
- fi
336
- # Avoid matching job metadata like timeout_seconds=N in the log header.
337
- if printf '%s' "$lower" | grep -E \
338
- '(^|[^_])timed out|command timed out|deadline exceeded|killed by timeout|exit[=:][[:space:]]*124\b' >/dev/null; then
339
- printf 'timeout'
340
- return
341
- fi
342
- if printf '%s' "$lower" | grep -E \
343
- 'connection (refused|reset|timed out)|network is unreachable|temporary failure|econnreset|enotfound|no space left|i/o error|transport error|dns (fail|error)|http[/ ]*(502|503|504)|statusingesterror|api unreachable' >/dev/null; then
344
- printf 'infra'
345
- return
346
- fi
347
- if [ "$exit_code" -eq 0 ]; then
348
- printf ''
349
- return
350
- fi
351
- printf 'agent_error'
352
- }
353
-
354
- next_action_for_class() {
355
- local fc="$1" runtime="${2:-}"
356
- case "$fc" in
357
- auth)
358
- printf 're-login on Outpost (device code) for %s' "${runtime:-the runtime CLI}"
359
- ;;
360
- secrets)
361
- printf 'share the missing secret to this Outpost (hq secrets share KEY) then re-probe'
362
- ;;
363
- timeout)
364
- printf 'raise timeout_seconds or split the job work'
365
- ;;
366
- infra)
367
- printf 'check Outpost disk/network; job will self-retry once on transient infra'
368
- ;;
369
- agent_error)
370
- printf 'inspect the job log and fix the prompt/skill'
371
- ;;
372
- *)
373
- printf ''
374
- ;;
375
- esac
376
- }
377
-
378
- # POST kind=run ingest. Never fails the job run.
379
- ingest_run_status() {
380
- local job_id="$1" run_at="$2" run_id="$3" last_exit="$4" failure_class="$5" next_actions_json="$6"
381
- if [ "${HQ_JOB_RUN_NO_INGEST:-}" = "1" ]; then
382
- log "ingest skipped (HQ_JOB_RUN_NO_INGEST=1)"
383
- return 0
384
- fi
385
- if ! resolve_box_identity; then
386
- log "run status ingest skipped — missing Outpost identity"
387
- return 0
388
- fi
389
- local payload fc_json
390
- if [ -z "$failure_class" ] || [ "$failure_class" = "null" ]; then
391
- fc_json='null'
392
- else
393
- fc_json="$(jq -nc --arg f "$failure_class" '$f')"
394
- fi
395
- payload="$(jq -nc \
396
- --arg userId "$OUTPOST_USER_ID" \
397
- --arg jobId "$job_id" \
398
- --arg run_at "$run_at" \
399
- --arg event_id "$run_id" \
400
- --arg run_id "$run_id" \
401
- --argjson last_exit "$last_exit" \
402
- --argjson failure_class "$fc_json" \
403
- --argjson next_actions "$next_actions_json" \
404
- --arg outpostId "${OUTPOST_ID:-}" \
405
- '{
406
- userId: $userId,
407
- jobId: $jobId,
408
- kind: "run",
409
- run_at: $run_at,
410
- event_id: $event_id,
411
- run_id: $run_id,
412
- last_exit: $last_exit,
413
- failure_class: $failure_class,
414
- next_actions: $next_actions
415
- } + (if $outpostId != "" then {outpostId: $outpostId} else {} end)')"
416
-
417
- local curl_bin="${HQ_JOB_RUN_CURL:-curl}"
418
- local url tmp_body tmp_hdr http=0
419
- url="$(api_base)/outpost/internal/jobs-status"
420
- if ! command -v "$curl_bin" >/dev/null 2>&1; then
421
- log "run status ingest skipped — curl not found"
422
- return 0
423
- fi
424
- tmp_body="$(mktemp "${TMPDIR:-/tmp}/hq-job-run-ingest-body.XXXXXX")"
425
- tmp_hdr="$(mktemp "${TMPDIR:-/tmp}/hq-job-run-ingest-hdr.XXXXXX")"
426
- create_auth_header_file
427
- set +e
428
- "$curl_bin" -sS -m 20 -X POST "$url" \
429
- -H "content-type: application/json" \
430
- -H "@${AUTH_HEADER_FILE}" \
431
- --data "$payload" \
432
- -D "$tmp_hdr" \
433
- -o "$tmp_body"
434
- local curl_rc=$?
435
- set -e
436
- if [ "$curl_rc" -eq 0 ]; then
437
- http="$(awk 'BEGIN{c=0} /^HTTP\//{c=$2} END{print c+0}' "$tmp_hdr" 2>/dev/null || echo 0)"
438
- fi
439
- rm -f "$tmp_body" "$tmp_hdr" "$AUTH_HEADER_FILE"
440
- AUTH_HEADER_FILE=""
441
- if [ "$http" -ge 200 ] && [ "$http" -lt 300 ]; then
442
- log "run status ingested job=$job_id failure_class=${failure_class:-null} http=$http"
443
- else
444
- log "run status ingest failed job=$job_id http=$http (logged; not failing run)"
445
- fi
446
- return 0
447
- }
448
-
449
- call_notify() {
450
- local job_id="$1" job_name="$2" outcome="$3" summary="$4" duration="$5" log_path="$6"
451
- local failure_class="${7:-}" next_action="${8:-}" owner="${9:-}" notify_mode="${10:-profile}"
452
- if [ "${HQ_JOB_RUN_NO_NOTIFY:-}" = "1" ]; then
453
- log "notify skipped (HQ_JOB_RUN_NO_NOTIFY=1)"
454
- return 0
455
- fi
456
- if [ ! -x "$NOTIFY_BIN" ] && [ -f "$NOTIFY_BIN" ]; then
457
- chmod +x "$NOTIFY_BIN" 2>/dev/null || true
458
- fi
459
- if [ ! -f "$NOTIFY_BIN" ]; then
460
- log "notify skipped — missing $NOTIFY_BIN"
461
- return 0
462
- fi
463
- set +e
464
- bash "$NOTIFY_BIN" \
465
- --hq-root "$HQ_ROOT" \
466
- --job-id "$job_id" \
467
- --job-name "$job_name" \
468
- --outcome "$outcome" \
469
- --summary "$summary" \
470
- --duration "$duration" \
471
- --log-path "$log_path" \
472
- --failure-class "$failure_class" \
473
- --next-action "$next_action" \
474
- --owner "$owner" \
475
- --notify "$notify_mode" \
476
- >>"$log_path" 2>&1
477
- local nrc=$?
478
- set -e
479
- if [ "$nrc" -ne 0 ]; then
480
- log "notify helper exited $nrc (ignored)"
481
- fi
482
- return 0
483
- }
484
-
485
- call_remediate() {
486
- local failure_class="$1" job_id="$2" company="$3"
487
- if [ ! -x "$REMEDIATE_BIN" ] && [ -f "$REMEDIATE_BIN" ]; then
488
- chmod +x "$REMEDIATE_BIN" 2>/dev/null || true
489
- fi
490
- if [ ! -f "$REMEDIATE_BIN" ]; then
491
- echo '{"retry_recommended":false,"attempted":false,"notes":"remediate missing"}'
492
- return 0
493
- fi
494
- local out
495
- set +e
496
- out="$(
497
- HQ_JOB_REMEDIATE_LOG="${LOG_FILE:-/dev/null}" \
498
- bash "$REMEDIATE_BIN" \
499
- --hq-root "$HQ_ROOT" \
500
- --failure-class "$failure_class" \
501
- --job-id "$job_id" \
502
- --company "$company" 2>>"${LOG_FILE:-/dev/null}"
503
- )"
504
- local rrc=$?
505
- set -e
506
- if [ "$rrc" -ne 0 ] || [ -z "$out" ]; then
507
- echo '{"retry_recommended":false,"attempted":false,"notes":"remediate failed"}'
508
- return 0
509
- fi
510
- printf '%s' "$out"
511
- }
512
-
513
- # --- resolve job ---
514
- JOB_FILE="$(resolve_job_file)"
515
- [ -f "$JOB_FILE" ] || die "job file missing: $JOB_FILE"
516
-
517
- JOB_ID="$(yq -r '.id // ""' "$JOB_FILE")"
518
- JOB_NAME="$(yq -r '.name // .id // ""' "$JOB_FILE")"
519
- RUNTIME="$(yq -r '.runtime // ""' "$JOB_FILE")"
520
- TIMEOUT_SEC="$(yq -r '.timeout_seconds // 600' "$JOB_FILE")"
521
- CWD_REL="$(yq -r '.cwd // .requirements.cwd // ""' "$JOB_FILE")"
522
- COMPANY="$(yq -r '.requirements.company // ""' "$JOB_FILE")"
523
- OWNER="$(yq -r '.owner // ""' "$JOB_FILE")"
524
- NOTIFY_MODE="$(yq -r '.notify // "profile"' "$JOB_FILE")"
525
- ENABLED="$(yq -o=json '.' "$JOB_FILE" | jq -r 'if has("enabled") then (.enabled|tostring) else "true" end')"
526
- # exec.surface overrides profile execution_surface; default headless.
527
- SURFACE="$(yq -r '.exec.surface // ""' "$JOB_FILE")"
528
- if [ -z "$SURFACE" ] || [ "$SURFACE" = "null" ]; then
529
- ALERTS_FILE="${HQ_JOB_ALERTS_FILE:-${HQ_ROOT}/personal/settings/schedule-alerts.yaml}"
530
- if [ -f "$ALERTS_FILE" ]; then
531
- SURFACE="$(yq -r '.execution_surface // ""' "$ALERTS_FILE" 2>/dev/null || true)"
532
- fi
533
- fi
534
- case "$SURFACE" in
535
- remote) SURFACE="remote" ;;
536
- *) SURFACE="headless" ;;
537
- esac
538
- # hq dm accepts personUid recipients; do not replace a job owner with this Outpost's identity.
539
-
540
- [ -n "$JOB_ID" ] && [ "$JOB_ID" != "null" ] || die "job missing id: $JOB_FILE"
541
- [ -n "$RUNTIME" ] && [ "$RUNTIME" != "null" ] || die "job missing runtime: $JOB_FILE"
542
- [ -n "$JOB_NAME" ] || JOB_NAME="$JOB_ID"
543
- [ -n "$NOTIFY_MODE" ] && [ "$NOTIFY_MODE" != "null" ] || NOTIFY_MODE="profile"
544
- case "$TIMEOUT_SEC" in
545
- ''|*[!0-9]*) TIMEOUT_SEC=600 ;;
546
- esac
547
- if [ "$TIMEOUT_SEC" -lt 60 ]; then TIMEOUT_SEC=60; fi
548
- if [ "$TIMEOUT_SEC" -gt 14400 ]; then TIMEOUT_SEC=14400; fi
549
-
550
- if [ "$ENABLED" = "false" ] || [ "$ENABLED" = "False" ] || [ "$ENABLED" = "0" ]; then
551
- log "skip job $JOB_ID — enabled=false"
552
- exit 0
553
- fi
554
-
555
- LOG_BASE="${HQ_JOB_LOG_DIR:-${HOME}/.hq/jobs/logs}"
556
- LOG_DIR="$LOG_BASE/$JOB_ID"
557
- mkdir -p "$LOG_DIR"
558
- LOCK_PATH="${HQ_JOB_LOCK:-${HOME}/.hq/jobs/run.lock}"
559
- mkdir -p "$(dirname "$LOCK_PATH")"
560
-
561
- FLOCK_BIN="${HQ_JOB_FLOCK_BIN:-}"
562
- if [ -z "$FLOCK_BIN" ]; then
563
- command -v flock >/dev/null 2>&1 || die "Outpost/Linux requires util-linux flock(1) for serialized job runs"
564
- FLOCK_BIN="$(command -v flock)"
565
- fi
566
- TIMEOUT_BIN="${HQ_JOB_TIMEOUT_BIN:-}"
567
- if [ -z "$TIMEOUT_BIN" ]; then
568
- if command -v timeout >/dev/null 2>&1; then
569
- TIMEOUT_BIN="timeout"
570
- elif command -v gtimeout >/dev/null 2>&1; then
571
- TIMEOUT_BIN="gtimeout"
572
- else
573
- TIMEOUT_BIN=""
574
- fi
575
- fi
576
-
577
- RUN_STAMP="$(date -u +%Y%m%dT%H%M%SZ 2>/dev/null || date +%Y%m%d%H%M%S)"
578
- LOG_FILE="$LOG_DIR/${RUN_STAMP}.log"
579
- STATUS_FILE="$LOG_DIR/${RUN_STAMP}.status"
580
-
581
- # Serialize all scheduled runs on this box.
582
- exec 9>"$LOCK_PATH"
583
- if ! "$FLOCK_BIN" -x 9; then
584
- die "failed to acquire global run lock: $LOCK_PATH"
585
- fi
586
-
587
- STARTED_AT="$(iso_now)"
588
- STARTED_EPOCH="$(epoch_now)"
589
- SKIP_REASON=""
590
- EXIT_CODE=0
591
-
592
- {
593
- echo "=== hq-job-run $JOB_ID ==="
594
- echo "started_at=$STARTED_AT"
595
- echo "runtime=$RUNTIME"
596
- echo "timeout_seconds=$TIMEOUT_SEC"
597
- echo "surface=$SURFACE"
598
- echo "job_file=$JOB_FILE"
599
- echo "hq_root=$HQ_ROOT"
600
- } >>"$LOG_FILE"
601
-
602
- AUTH_STATUS="$(check_runtime_auth "$RUNTIME" || true)"
603
- if [ "$AUTH_STATUS" != "ok" ]; then
604
- case "$AUTH_STATUS" in
605
- no_cli)
606
- SKIP_REASON="runtime CLI '$RUNTIME' not installed on PATH — skip (re-login/install on Outpost)"
607
- ;;
608
- missing)
609
- SKIP_REASON="runtime CLI '$RUNTIME' not authenticated on Outpost (cached credentials missing) — skip"
610
- ;;
611
- *)
612
- SKIP_REASON="unsupported or unknown runtime '$RUNTIME' — skip"
613
- ;;
614
- esac
615
- {
616
- echo "skip_reason=$SKIP_REASON"
617
- echo "auth_status=$AUTH_STATUS"
618
- echo "failure_class=auth"
619
- } >>"$LOG_FILE"
620
- log "$SKIP_REASON"
621
- ENDED_AT="$(iso_now)"
622
- ENDED_EPOCH="$(epoch_now)"
623
- DUR=$((ENDED_EPOCH - STARTED_EPOCH))
624
- [ "$DUR" -ge 0 ] || DUR=0
625
- printf 'exit=0\nskip_reason=%s\nfailure_class=auth\nstarted_at=%s\nended_at=%s\n' \
626
- "$SKIP_REASON" "$STARTED_AT" "$ENDED_AT" >"$STATUS_FILE"
627
- append_meter_marker "$JOB_ID" "$RUNTIME" "$STARTED_AT" "$ENDED_AT" 0 "$DUR" "$SKIP_REASON"
628
-
629
- # Bounded remediate: Cognito refresh only — never device-login loop.
630
- REM_OUT="$(call_remediate auth "$JOB_ID" "$COMPANY")"
631
- echo "remediate=$REM_OUT" >>"$LOG_FILE"
632
- NEXT_ACT="$(printf '%s' "$REM_OUT" | jq -r '.next_action // empty')"
633
- [ -n "$NEXT_ACT" ] || NEXT_ACT="$(next_action_for_class auth "$RUNTIME")"
634
- NA_JSON="$(jq -nc --arg a "$NEXT_ACT" 'if $a == "" then [] else [$a] end')"
635
- RUN_ID="run-${JOB_ID}-${RUN_STAMP}-authskip"
636
- ingest_run_status "$JOB_ID" "$ENDED_AT" "$RUN_ID" 1 "auth" "$NA_JSON"
637
- call_notify "$JOB_ID" "$JOB_NAME" "failed" "$SKIP_REASON" "$DUR" "$LOG_FILE" \
638
- "auth" "$NEXT_ACT" "$OWNER" "$NOTIFY_MODE"
639
- exit 0
640
- fi
641
-
642
- # Working directory — must resolve under HQ_ROOT (no absolute / no .. escape)
643
- RUN_CWD="$HQ_ROOT"
644
- if [ -n "$CWD_REL" ] && [ "$CWD_REL" != "null" ]; then
645
- case "$CWD_REL" in
646
- /*)
647
- SKIP_REASON="cwd absolute not allowed: $CWD_REL — skip"
648
- log "$SKIP_REASON"
649
- {
650
- echo "skip_reason=$SKIP_REASON"
651
- echo "failure_class=infra"
652
- } >>"$LOG_FILE"
653
- ENDED_AT="$(iso_now)"
654
- ENDED_EPOCH="$(epoch_now)"
655
- DUR=$((ENDED_EPOCH - STARTED_EPOCH))
656
- [ "$DUR" -ge 0 ] || DUR=0
657
- printf 'exit=0\nskip_reason=%s\nfailure_class=infra\nstarted_at=%s\nended_at=%s\n' \
658
- "$SKIP_REASON" "$STARTED_AT" "$ENDED_AT" >"$STATUS_FILE"
659
- append_meter_marker "$JOB_ID" "$RUNTIME" "$STARTED_AT" "$ENDED_AT" 0 "$DUR" "$SKIP_REASON"
660
- NEXT_ACT="fix job cwd to an HQ-root-relative path (no absolute, no '..')"
661
- NA_JSON="$(jq -nc --arg a "$NEXT_ACT" '[$a]')"
662
- RUN_ID="run-${JOB_ID}-${RUN_STAMP}-cwd"
663
- ingest_run_status "$JOB_ID" "$ENDED_AT" "$RUN_ID" 1 "infra" "$NA_JSON"
664
- call_notify "$JOB_ID" "$JOB_NAME" "blocked" "$SKIP_REASON" "$DUR" "$LOG_FILE" \
665
- "infra" "$NEXT_ACT" "$OWNER" "$NOTIFY_MODE"
666
- exit 0
667
- ;;
668
- esac
669
- _cwd_part=""
670
- _old_ifs="$IFS"
671
- IFS='/'
672
- # shellcheck disable=SC2086
673
- for _cwd_part in $CWD_REL; do
674
- if [ "$_cwd_part" = ".." ]; then
675
- IFS="$_old_ifs"
676
- SKIP_REASON="cwd contains '..': $CWD_REL — skip"
677
- log "$SKIP_REASON"
678
- {
679
- echo "skip_reason=$SKIP_REASON"
680
- echo "failure_class=infra"
681
- } >>"$LOG_FILE"
682
- ENDED_AT="$(iso_now)"
683
- ENDED_EPOCH="$(epoch_now)"
684
- DUR=$((ENDED_EPOCH - STARTED_EPOCH))
685
- [ "$DUR" -ge 0 ] || DUR=0
686
- printf 'exit=0\nskip_reason=%s\nfailure_class=infra\nstarted_at=%s\nended_at=%s\n' \
687
- "$SKIP_REASON" "$STARTED_AT" "$ENDED_AT" >"$STATUS_FILE"
688
- append_meter_marker "$JOB_ID" "$RUNTIME" "$STARTED_AT" "$ENDED_AT" 0 "$DUR" "$SKIP_REASON"
689
- NEXT_ACT="fix job cwd to an HQ-root-relative path without '..'"
690
- NA_JSON="$(jq -nc --arg a "$NEXT_ACT" '[$a]')"
691
- RUN_ID="run-${JOB_ID}-${RUN_STAMP}-cwd"
692
- ingest_run_status "$JOB_ID" "$ENDED_AT" "$RUN_ID" 1 "infra" "$NA_JSON"
693
- call_notify "$JOB_ID" "$JOB_NAME" "blocked" "$SKIP_REASON" "$DUR" "$LOG_FILE" \
694
- "infra" "$NEXT_ACT" "$OWNER" "$NOTIFY_MODE"
695
- exit 0
696
- fi
697
- done
698
- IFS="$_old_ifs"
699
- HQ_CANON="$(realpath -m "$HQ_ROOT" 2>/dev/null || realpath "$HQ_ROOT" 2>/dev/null || true)"
700
- RUN_CWD="$(realpath -m "$HQ_ROOT/$CWD_REL" 2>/dev/null || true)"
701
- if [ -z "$RUN_CWD" ] && [ -d "$HQ_ROOT/$CWD_REL" ]; then
702
- RUN_CWD="$(realpath "$HQ_ROOT/$CWD_REL" 2>/dev/null || true)"
703
- fi
704
- if [ -z "$HQ_CANON" ] || [ -z "$RUN_CWD" ]; then
705
- HQ_CANON="$(hq_normpath "$HQ_ROOT")"
706
- RUN_CWD="$(hq_normpath "$HQ_ROOT/$CWD_REL")"
707
- fi
708
- case "$RUN_CWD" in
709
- "$HQ_CANON"|"$HQ_CANON"/*) ;;
710
- *)
711
- SKIP_REASON="cwd escapes HQ_ROOT: $CWD_REL → $RUN_CWD — skip"
712
- log "$SKIP_REASON"
713
- {
714
- echo "skip_reason=$SKIP_REASON"
715
- echo "failure_class=infra"
716
- } >>"$LOG_FILE"
717
- ENDED_AT="$(iso_now)"
718
- ENDED_EPOCH="$(epoch_now)"
719
- DUR=$((ENDED_EPOCH - STARTED_EPOCH))
720
- [ "$DUR" -ge 0 ] || DUR=0
721
- printf 'exit=0\nskip_reason=%s\nfailure_class=infra\nstarted_at=%s\nended_at=%s\n' \
722
- "$SKIP_REASON" "$STARTED_AT" "$ENDED_AT" >"$STATUS_FILE"
723
- append_meter_marker "$JOB_ID" "$RUNTIME" "$STARTED_AT" "$ENDED_AT" 0 "$DUR" "$SKIP_REASON"
724
- NEXT_ACT="fix job cwd to stay under HQ root"
725
- NA_JSON="$(jq -nc --arg a "$NEXT_ACT" '[$a]')"
726
- RUN_ID="run-${JOB_ID}-${RUN_STAMP}-cwd"
727
- ingest_run_status "$JOB_ID" "$ENDED_AT" "$RUN_ID" 1 "infra" "$NA_JSON"
728
- call_notify "$JOB_ID" "$JOB_NAME" "blocked" "$SKIP_REASON" "$DUR" "$LOG_FILE" \
729
- "infra" "$NEXT_ACT" "$OWNER" "$NOTIFY_MODE"
730
- exit 0
731
- ;;
732
- esac
733
- fi
734
- if [ ! -d "$RUN_CWD" ]; then
735
- SKIP_REASON="cwd missing: $RUN_CWD — skip"
736
- log "$SKIP_REASON"
737
- {
738
- echo "skip_reason=$SKIP_REASON"
739
- echo "failure_class=infra"
740
- } >>"$LOG_FILE"
741
- ENDED_AT="$(iso_now)"
742
- ENDED_EPOCH="$(epoch_now)"
743
- DUR=$((ENDED_EPOCH - STARTED_EPOCH))
744
- [ "$DUR" -ge 0 ] || DUR=0
745
- printf 'exit=0\nskip_reason=%s\nfailure_class=infra\nstarted_at=%s\nended_at=%s\n' \
746
- "$SKIP_REASON" "$STARTED_AT" "$ENDED_AT" >"$STATUS_FILE"
747
- append_meter_marker "$JOB_ID" "$RUNTIME" "$STARTED_AT" "$ENDED_AT" 0 "$DUR" "$SKIP_REASON"
748
- NEXT_ACT="create missing cwd on Outpost: $RUN_CWD"
749
- NA_JSON="$(jq -nc --arg a "$NEXT_ACT" '[$a]')"
750
- RUN_ID="run-${JOB_ID}-${RUN_STAMP}-cwd"
751
- ingest_run_status "$JOB_ID" "$ENDED_AT" "$RUN_ID" 1 "infra" "$NA_JSON"
752
- call_notify "$JOB_ID" "$JOB_NAME" "blocked" "$SKIP_REASON" "$DUR" "$LOG_FILE" \
753
- "infra" "$NEXT_ACT" "$OWNER" "$NOTIFY_MODE"
754
- exit 0
755
- fi
756
-
757
- PROMPT_TEXT="$(build_prompt "$JOB_FILE")"
758
-
759
- # Build secret name list for hq secrets exec allowlist injection.
760
- SECRET_NAMES=()
761
- SECRET_COUNT="$(yq -r '.requirements.secrets // [] | length' "$JOB_FILE")"
762
- if [ "$SECRET_COUNT" != "null" ] && [ "${SECRET_COUNT:-0}" -gt 0 ] 2>/dev/null; then
763
- i=0
764
- while [ "$i" -lt "$SECRET_COUNT" ]; do
765
- sn="$(yq -r ".requirements.secrets[$i]" "$JOB_FILE")"
766
- i=$((i + 1))
767
- [ -n "$sn" ] && [ "$sn" != "null" ] || continue
768
- SECRET_NAMES+=("$sn")
769
- done
770
- fi
771
-
772
- # Claude remote surface: TTY + --remote-control so the run appears in Desktop.
773
- # Auto-dismisses known startup dialogs; copies pane output into LOG_FILE.
774
- # Launcher is a temp script (not nested shell quoting) to keep prompts/safe.
775
- run_claude_remote() {
776
- local cmd_rc=0
777
- if ! command -v tmux >/dev/null 2>&1; then
778
- echo "remote surface requested but tmux missing — falling back to headless" >>"$LOG_FILE"
779
- SURFACE="headless"
780
- return 99
781
- fi
782
- local sess="hq-job-${JOB_ID}-${RUN_STAMP}"
783
- local rc_name="schedule/${JOB_ID}"
784
- local deadline=$((TIMEOUT_SEC + 30))
785
- local i=0
786
- local pane=""
787
- local exit_file="$LOG_DIR/${RUN_STAMP}.remote-exit"
788
- local done_file="$LOG_DIR/${RUN_STAMP}.remote-done"
789
- local launcher="$LOG_DIR/${RUN_STAMP}.remote-launch.sh"
790
- # Completion is a sentinel file — not pane text (prompt instructions also
791
- # contain marker words and caused false early teardown).
792
- rm -f "$done_file"
793
- local remote_prompt
794
- remote_prompt="${PROMPT_TEXT}
795
-
796
- When ALL work above is fully finished (success or failure), create this exact empty file with a shell tool (and do not create it earlier):
797
- touch $(printf %q "$done_file")
798
- Then stop."
799
- echo "surface=remote remote_name=$rc_name tmux_session=$sess done_file=$done_file" >>"$LOG_FILE"
800
- {
801
- echo '#!/usr/bin/env bash'
802
- echo 'set -uo pipefail'
803
- printf 'cd %q\n' "$RUN_CWD"
804
- printf 'claude --remote-control %q --permission-mode bypassPermissions --dangerously-skip-permissions %q\n' "$rc_name" "$remote_prompt"
805
- echo 'rc=$?'
806
- printf 'echo EXIT=$rc > %q\n' "$exit_file"
807
- echo 'sleep 15'
808
- echo 'exit "$rc"'
809
- } >"$launcher"
810
- chmod 700 "$launcher"
811
- tmux kill-session -t "$sess" 2>/dev/null || true
812
- tmux new-session -d -s "$sess" -x 120 -y 40 "$launcher"
813
- while [ "$i" -lt "$deadline" ]; do
814
- sleep 2
815
- i=$((i + 2))
816
- pane="$(tmux capture-pane -t "$sess" -p -S -120 2>/dev/null || true)"
817
- # Startup / tool-permission dialogs (remote TTY can still prompt)
818
- if printf '%s\n' "$pane" | grep -q 'Yes, I accept'; then
819
- tmux send-keys -t "$sess" "2" Enter
820
- continue
821
- fi
822
- if printf '%s\n' "$pane" | grep -q 'Try the new fullscreen renderer'; then
823
- tmux send-keys -t "$sess" "2" Enter
824
- continue
825
- fi
826
- if printf '%s\n' "$pane" | grep -q 'Do you want to proceed'; then
827
- # "2" = Yes, and don't ask again for similar commands
828
- tmux send-keys -t "$sess" "2" Enter
829
- echo "auto-accepted tool permission prompt" >>"$LOG_FILE"
830
- continue
831
- fi
832
- if [ -f "$done_file" ]; then
833
- echo "remote done sentinel present" >>"$LOG_FILE"
834
- sleep 2
835
- pane="$(tmux capture-pane -t "$sess" -p -S -300 2>/dev/null || true)"
836
- cmd_rc=0
837
- break
838
- fi
839
- if [ -f "$exit_file" ]; then
840
- cmd_rc="$(tr -dc '0-9' <"$exit_file" | head -c 3)"
841
- [ -n "$cmd_rc" ] || cmd_rc=0
842
- break
843
- fi
844
- done
845
- if [ "$i" -ge "$deadline" ]; then
846
- echo "remote surface timed out after ${TIMEOUT_SEC}s" >>"$LOG_FILE"
847
- cmd_rc=124
848
- fi
849
- {
850
- echo "=== remote pane capture ==="
851
- tmux capture-pane -t "$sess" -p -S -300 2>/dev/null || true
852
- } >>"$LOG_FILE"
853
- # End the Remote Control process so Desktop moves schedule/<job-id> to Archived.
854
- echo "tearing down remote session ${rc_name} (review under Desktop Archived)" >>"$LOG_FILE"
855
- tmux kill-session -t "$sess" 2>/dev/null || true
856
- rm -f "$launcher"
857
- return "$cmd_rc"
858
- }
859
-
860
- run_agent() {
861
- local cmd_rc=0
862
- if [ "${HQ_JOB_RUN_SKIP_EXEC:-}" = "1" ]; then
863
- echo "(HQ_JOB_RUN_SKIP_EXEC=1 — not invoking runtime CLI)" >>"$LOG_FILE"
864
- return 0
865
- fi
866
-
867
- if [ "$RUNTIME" = "claude" ] && [ "$SURFACE" = "remote" ]; then
868
- set +e
869
- run_claude_remote
870
- cmd_rc=$?
871
- set -e
872
- if [ "$cmd_rc" -ne 99 ]; then
873
- return "$cmd_rc"
874
- fi
875
- # 99 = fall through to headless
876
- fi
877
- if [ "$RUNTIME" = "codex" ] && [ "$SURFACE" = "remote" ]; then
878
- echo "surface=remote ignored for codex — using headless" >>"$LOG_FILE"
879
- fi
880
-
881
- local -a agent_cmd
882
- case "$RUNTIME" in
883
- claude)
884
- agent_cmd=(claude -p "$PROMPT_TEXT" --output-format text --permission-mode bypassPermissions)
885
- ;;
886
- codex)
887
- # Spike-validated flags on codex-cli 0.147 (NOT --ask-for-approval never).
888
- agent_cmd=(codex exec -s workspace-write --skip-git-repo-check -C "$RUN_CWD" "$PROMPT_TEXT")
889
- ;;
890
- *)
891
- echo "unknown runtime: $RUNTIME" >>"$LOG_FILE"
892
- return 2
893
- ;;
894
- esac
895
-
896
- local -a wrap_cmd=()
897
- if [ -n "$TIMEOUT_BIN" ]; then
898
- wrap_cmd=("$TIMEOUT_BIN" --signal=TERM --kill-after=15s "${TIMEOUT_SEC}s")
899
- fi
900
-
901
- # Inject only confirmed requirements.secrets[] via hq secrets exec.
902
- # CLI shape: hq secrets [--company S|--personal] exec --only K1,K2 -- <cmd>
903
- if [ "${#SECRET_NAMES[@]}" -gt 0 ] && command -v hq >/dev/null 2>&1; then
904
- local only_csv=""
905
- local sn
906
- for sn in "${SECRET_NAMES[@]}"; do
907
- if [ -z "$only_csv" ]; then
908
- only_csv="$sn"
909
- else
910
- only_csv="${only_csv},${sn}"
911
- fi
912
- done
913
- local -a secrets_cmd=(hq secrets)
914
- if [ -n "$COMPANY" ] && [ "$COMPANY" != "null" ]; then
915
- secrets_cmd+=(--company "$COMPANY")
916
- else
917
- secrets_cmd+=(--personal)
918
- fi
919
- secrets_cmd+=(exec --only "$only_csv" --script "${BASH_SOURCE[0]}" --)
920
- (
921
- cd "$RUN_CWD"
922
- if [ "${#wrap_cmd[@]}" -gt 0 ]; then
923
- "${secrets_cmd[@]}" "${wrap_cmd[@]}" "${agent_cmd[@]}"
924
- else
925
- "${secrets_cmd[@]}" "${agent_cmd[@]}"
926
- fi
927
- ) >>"$LOG_FILE" 2>&1 || cmd_rc=$?
928
- else
929
- if [ "${#SECRET_NAMES[@]}" -gt 0 ]; then
930
- echo "warning: requirements.secrets set but hq CLI missing — running without secrets exec" >>"$LOG_FILE"
931
- fi
932
- (
933
- cd "$RUN_CWD"
934
- if [ "${#wrap_cmd[@]}" -gt 0 ]; then
935
- "${wrap_cmd[@]}" "${agent_cmd[@]}"
936
- else
937
- "${agent_cmd[@]}"
938
- fi
939
- ) >>"$LOG_FILE" 2>&1 || cmd_rc=$?
940
- fi
941
- return "$cmd_rc"
942
- }
943
-
944
- RETRIED=0
945
- FAILURE_CLASS=""
946
- NEXT_ACT=""
947
-
948
- set +e
949
- run_agent
950
- EXIT_CODE=$?
951
- set -e
952
-
953
- finalize_timestamps() {
954
- ENDED_AT="$(iso_now)"
955
- ENDED_EPOCH="$(epoch_now)"
956
- DUR=$((ENDED_EPOCH - STARTED_EPOCH))
957
- [ "$DUR" -ge 0 ] || DUR=0
958
- }
959
-
960
- if [ "$EXIT_CODE" -ne 0 ]; then
961
- FAILURE_CLASS="$(classify_failure "$EXIT_CODE" "$LOG_FILE" "")"
962
- echo "failure_class=$FAILURE_CLASS" >>"$LOG_FILE"
963
- REM_OUT="$(call_remediate "$FAILURE_CLASS" "$JOB_ID" "$COMPANY")"
964
- echo "remediate=$REM_OUT" >>"$LOG_FILE"
965
- NEXT_ACT="$(printf '%s' "$REM_OUT" | jq -r '.next_action // empty')"
966
- [ -n "$NEXT_ACT" ] || NEXT_ACT="$(next_action_for_class "$FAILURE_CLASS" "$RUNTIME")"
967
- RETRY_REC="$(printf '%s' "$REM_OUT" | jq -r '.retry_recommended // false')"
968
- if [ "$RETRY_REC" = "true" ] && [ "$RETRIED" -eq 0 ]; then
969
- RETRIED=1
970
- echo "remediate_retry=1" >>"$LOG_FILE"
971
- log "self-remediate recommended retry for $FAILURE_CLASS — retrying once"
972
- set +e
973
- run_agent
974
- EXIT_CODE=$?
975
- set -e
976
- if [ "$EXIT_CODE" -ne 0 ]; then
977
- FAILURE_CLASS="$(classify_failure "$EXIT_CODE" "$LOG_FILE" "")"
978
- echo "failure_class_after_retry=$FAILURE_CLASS" >>"$LOG_FILE"
979
- NEXT_ACT="$(next_action_for_class "$FAILURE_CLASS" "$RUNTIME")"
980
- else
981
- FAILURE_CLASS=""
982
- NEXT_ACT=""
983
- fi
984
- fi
985
- fi
986
-
987
- finalize_timestamps
988
-
989
- {
990
- echo "ended_at=$ENDED_AT"
991
- echo "exit=$EXIT_CODE"
992
- echo "duration_seconds=$DUR"
993
- echo "retried=$RETRIED"
994
- [ -n "$FAILURE_CLASS" ] && echo "failure_class=$FAILURE_CLASS"
995
- } >>"$LOG_FILE"
996
-
997
- printf 'exit=%s\nstarted_at=%s\nended_at=%s\nduration_seconds=%s\nfailure_class=%s\nretried=%s\n' \
998
- "$EXIT_CODE" "$STARTED_AT" "$ENDED_AT" "$DUR" "${FAILURE_CLASS:-}" "$RETRIED" >"$STATUS_FILE"
999
-
1000
- append_meter_marker "$JOB_ID" "$RUNTIME" "$STARTED_AT" "$ENDED_AT" "$EXIT_CODE" "$DUR" ""
1001
-
1002
- RUN_ID="run-${JOB_ID}-${RUN_STAMP}"
1003
- if [ "$EXIT_CODE" -eq 0 ]; then
1004
- NA_JSON='[]'
1005
- ingest_run_status "$JOB_ID" "$ENDED_AT" "$RUN_ID" 0 "null" "$NA_JSON"
1006
- SUMMARY="completed ok"
1007
- [ "$RETRIED" -eq 1 ] && SUMMARY="completed ok after self-remediation retry"
1008
- call_notify "$JOB_ID" "$JOB_NAME" "ok" "$SUMMARY" "$DUR" "$LOG_FILE" \
1009
- "" "" "$OWNER" "$NOTIFY_MODE"
1010
- log "job $JOB_ID completed ok (${DUR}s)"
1011
- exit 0
1012
- fi
1013
-
1014
- NA_JSON="$(jq -nc --arg a "$NEXT_ACT" 'if $a == "" then [] else [$a] end')"
1015
- # Cap last_exit to 0–255 for ingest; preserve real exit for process.
1016
- INGEST_EXIT="$EXIT_CODE"
1017
- if [ "$INGEST_EXIT" -lt 0 ]; then INGEST_EXIT=1; fi
1018
- if [ "$INGEST_EXIT" -gt 255 ]; then INGEST_EXIT=255; fi
1019
- ingest_run_status "$JOB_ID" "$ENDED_AT" "$RUN_ID" "$INGEST_EXIT" "$FAILURE_CLASS" "$NA_JSON"
1020
- SUMMARY="exit=$EXIT_CODE class=$FAILURE_CLASS"
1021
- call_notify "$JOB_ID" "$JOB_NAME" "failed" "$SUMMARY" "$DUR" "$LOG_FILE" \
1022
- "$FAILURE_CLASS" "$NEXT_ACT" "$OWNER" "$NOTIFY_MODE"
1023
-
1024
- log "job $JOB_ID failed exit=$EXIT_CODE class=$FAILURE_CLASS (${DUR}s) log=$LOG_FILE"
1025
- exit "$EXIT_CODE"