@indigoai-us/hq-cli 5.294.0 → 5.296.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/dist/command-catalog.generated.d.ts +70 -9
- package/dist/command-catalog.generated.js +88 -12
- package/dist/commands/auth.d.ts +1 -0
- package/dist/commands/auth.js +76 -1
- package/dist/commands/core-job-native.d.ts +7 -0
- package/dist/commands/core-job-native.js +20 -0
- package/dist/commands/core-job-probe-native.d.ts +45 -0
- package/dist/commands/core-job-probe-native.js +564 -0
- package/dist/commands/core-job-remote-launcher.d.ts +7 -0
- package/dist/commands/core-job-remote-launcher.js +51 -0
- package/dist/commands/core-job-run-native.d.ts +47 -0
- package/dist/commands/core-job-run-native.js +1215 -0
- package/dist/commands/core.js +1 -1
- package/dist/commands/doctor.js +0 -74
- package/dist/commands/lanes-message.js +24 -5
- package/dist/commands/lanes-resume.js +1 -3
- package/dist/commands/lanes-shared.d.ts +1 -2
- package/dist/commands/lanes-stories.d.ts +1 -0
- package/dist/commands/lanes-stories.js +40 -1
- package/dist/commands/lanes.d.ts +1 -9
- package/dist/commands/lanes.js +197 -174
- package/dist/commands/run.js +5 -0
- package/dist/fast-scaffold.d.ts +0 -5
- package/dist/fast-scaffold.js +0 -10
- package/dist/index.js +1 -10
- package/dist/lib/core-utils/sentry-report.d.ts +0 -19
- package/dist/lib/core-utils/sentry-report.js +8 -61
- package/dist/lib/doctor/checks/claude-wiring.js +9 -0
- package/dist/lib/lanes/flag-keys.d.ts +2 -0
- package/dist/lib/lanes/flag-keys.js +2 -0
- package/dist/lib/lanes/manager-status-gate.d.ts +0 -17
- package/dist/lib/lanes/manager-status-gate.js +2 -43
- package/dist/lib/lanes/mesh-events.js +3 -0
- package/dist/lib/lanes/outpost-dispatch.d.ts +2 -0
- package/dist/lib/lanes/outpost-dispatch.js +7 -1
- package/dist/lib/lanes/outpost-hire-cache.d.ts +0 -1
- package/dist/lib/lanes/outpost-hire-cache.js +0 -1
- package/dist/lib/lanes/outpost-snapshot-memo.d.ts +6 -0
- package/dist/lib/lanes/outpost-snapshot-memo.js +14 -0
- package/dist/lib/lanes/reconcile.d.ts +2 -3
- package/dist/lib/lanes/reconcile.js +67 -57
- package/dist/lib/lanes/spawn.d.ts +0 -2
- package/dist/lib/lanes/spawn.js +57 -5
- package/dist/lib/lanes/store.js +54 -0
- package/dist/lib/lanes/stories.d.ts +7 -1
- package/dist/lib/lanes/stories.js +16 -0
- package/dist/lib/lanes/types.d.ts +30 -1
- package/dist/lib/lanes/types.js +2 -2
- package/dist/lib/lanes/watcher.d.ts +4 -3
- package/dist/lib/lanes/watcher.js +122 -56
- package/dist/lib/mesh/live/daemon/run.js +18 -1
- package/dist/run/env-graph-guard.d.ts +2 -2
- package/dist/run/env-graph-guard.js +5 -2
- package/dist/run/hq-plugin.js +6 -5
- package/dist/utils/cognito-session.d.ts +6 -0
- package/dist/utils/cognito-session.js +9 -0
- package/dist/utils/self-update.js +31 -0
- package/dist/utils/version-gate.js +22 -3
- package/package.json +2 -2
- package/dist/fast-lanes-list-route.d.ts +0 -8
- package/dist/fast-lanes-list-route.js +0 -32
- package/dist/fast-lanes-list.d.ts +0 -3
- package/dist/fast-lanes-list.js +0 -224
- package/dist/lib/doctor/client-health-sentry-flag.d.ts +0 -23
- package/dist/lib/doctor/client-health-sentry-flag.js +0 -99
- package/dist/lib/doctor/client-health-sentry.d.ts +0 -71
- package/dist/lib/doctor/client-health-sentry.js +0 -286
- package/dist/lib/lanes/outpost-snapshot-cache.d.ts +0 -26
- package/dist/lib/lanes/outpost-snapshot-cache.js +0 -215
- package/dist/lib/lanes/outpost-snapshot-refresh.d.ts +0 -21
- package/dist/lib/lanes/outpost-snapshot-refresh.js +0 -329
|
@@ -0,0 +1,1215 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as os from "node:os";
|
|
3
|
+
import * as path from "node:path";
|
|
4
|
+
import { spawn } from "node:child_process";
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
6
|
+
import { JSON_SCHEMA, load } from "js-yaml";
|
|
7
|
+
import { serializeJqJson } from "../lib/core-utils/jq-json-serializer.js";
|
|
8
|
+
import { resolveLiveRoot, resolveBundledAsset } from "../utils/hq-roots.js";
|
|
9
|
+
import { runBundledScript } from "../utils/run-bundled-script.js";
|
|
10
|
+
import { describeCoreCommand } from "./core-command-catalog.js";
|
|
11
|
+
import { coreNativeJobsEnabled } from "./core-native-jobs-flag.js";
|
|
12
|
+
const RUN_ASSET = "core/scripts/hq-job-run.sh";
|
|
13
|
+
const RUNNER_NAME = "hq-job-run";
|
|
14
|
+
export const OUTPOST_IDENTITY_FILE = "/etc/outpost/codex-identity.env";
|
|
15
|
+
export const OUTPOST_RUNNER_FILE = "/usr/local/bin/outpost-runner.sh";
|
|
16
|
+
const REMOTE_LAUNCHER = fileURLToPath(new URL("./core-job-remote-launcher.js", import.meta.url));
|
|
17
|
+
const USAGE = `hq-job-run.sh — serialized Outpost scheduled-job executor (US-004 + US-006).
|
|
18
|
+
|
|
19
|
+
Invoked by systemd user units (hq-job-{id}.service) or on demand:
|
|
20
|
+
core/scripts/hq-job-run.sh --job-id <id>
|
|
21
|
+
core/scripts/hq-job-run.sh <job.yaml>
|
|
22
|
+
|
|
23
|
+
Behavior:
|
|
24
|
+
- Global file lock (~/.hq/jobs/run.lock) — serialized-only (US-001)
|
|
25
|
+
- Enforces job timeout_seconds (default 600; schema 60–14400)
|
|
26
|
+
- Credentials only via \`hq secrets exec\` (requirements.secrets[]) or
|
|
27
|
+
the box's cached CLI auth — never env vars / unit-file secrets
|
|
28
|
+
- Trust model (7A): injects ONLY requirements.secrets[] — no static prompt scan
|
|
29
|
+
- Appends stdout/stderr + exit to ~/.hq/jobs/logs/{id}/
|
|
30
|
+
- Emits a compute-meter attribution marker (scheduled vs interactive)
|
|
31
|
+
- Classifies failures (auth|secrets|timeout|agent_error|infra), writes
|
|
32
|
+
failure_class to hq-pro jobs status (kind=run), bounded self-remediate,
|
|
33
|
+
then dm-notify via hq-job-notify.sh (collapse 1/24h; never create-thread)
|
|
34
|
+
- Missing runtime CLI auth: classify auth, notify re-login, exit 0 (unit OK)
|
|
35
|
+
|
|
36
|
+
Runtime dispatch (US-001 spike flags):
|
|
37
|
+
claude → claude -p … --output-format text --permission-mode bypassPermissions
|
|
38
|
+
codex → codex exec -s workspace-write --skip-git-repo-check …
|
|
39
|
+
|
|
40
|
+
Env / overrides (tests):
|
|
41
|
+
HQ_ROOT, HOME, HQ_JOB_LOCK, HQ_JOB_LOG_DIR, HQ_JOB_METER_DIR
|
|
42
|
+
HQ_JOB_TIMEOUT_BIN, HQ_JOB_FLOCK_BIN, HQ_JOB_NOW
|
|
43
|
+
HQ_JOB_PROBE_CLAUDE_CRED, HQ_JOB_PROBE_CODEX_AUTH (auth paths; shared with probe)
|
|
44
|
+
HQ_JOB_RUN_SKIP_EXEC=1 — acquire lock + write marker/logs without calling CLI
|
|
45
|
+
HQ_JOB_NOTIFY_BIN, HQ_JOB_REMEDIATE_BIN, HQ_JOB_RUN_NO_NOTIFY=1, HQ_JOB_RUN_NO_INGEST=1
|
|
46
|
+
HQ_JOB_RUN_CURL, HQ_PRO_API_URL, OUTPOST_USER_ID, OUTPOST_INSTANCE_TOKEN
|
|
47
|
+
|
|
48
|
+
Exit: 0 success or intentional auth/cwd skip; 1 agent/job failure; 2 usage/config;
|
|
49
|
+
124 timeout (GNU timeout convention when available).
|
|
50
|
+
|
|
51
|
+
set -euo pipefail
|
|
52
|
+
|
|
53
|
+
SCRIPT_DIR="$(cd "$(dirname "\${BASH_SOURCE[0]}")" && pwd)"
|
|
54
|
+
HQ_ROOT="\${HQ_ROOT:-$(cd "$SCRIPT_DIR/../.." && pwd)}"
|
|
55
|
+
`;
|
|
56
|
+
class JobYamlParseFailure extends Error {
|
|
57
|
+
source;
|
|
58
|
+
parseError;
|
|
59
|
+
constructor(source, parseError) {
|
|
60
|
+
super("job YAML parse failed");
|
|
61
|
+
this.source = source;
|
|
62
|
+
this.parseError = parseError;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
const LOCK_HOLDER = "process.stdout.write('HQ_LOCK_READY\\n'); process.stdin.resume();";
|
|
66
|
+
const SIGNAL_NUMBER = { SIGHUP: 1, SIGINT: 2, SIGQUIT: 3, SIGKILL: 9, SIGTERM: 15 };
|
|
67
|
+
function numericSignal(signal) {
|
|
68
|
+
return signal ? SIGNAL_NUMBER[signal] ?? 0 : 0;
|
|
69
|
+
}
|
|
70
|
+
function appendLog(file, chunk) {
|
|
71
|
+
fs.appendFileSync(file, chunk);
|
|
72
|
+
}
|
|
73
|
+
function appendArgvRecord(file, executable, args) {
|
|
74
|
+
if (!file)
|
|
75
|
+
return;
|
|
76
|
+
const fields = [executable, ...args].map((value) => Buffer.from(`${value}\0`));
|
|
77
|
+
fs.appendFileSync(file, Buffer.concat(fields));
|
|
78
|
+
}
|
|
79
|
+
function signalProcessTree(child, signal, processGroup) {
|
|
80
|
+
if (processGroup && process.platform !== "win32" && child.pid !== undefined && child.pid > 0) {
|
|
81
|
+
try {
|
|
82
|
+
process.kill(-child.pid, signal);
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
catch (error) {
|
|
86
|
+
if (error.code === "ESRCH")
|
|
87
|
+
return;
|
|
88
|
+
child.kill(signal);
|
|
89
|
+
throw error;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
child.kill(signal);
|
|
93
|
+
}
|
|
94
|
+
function runtimeArgv(file, args) {
|
|
95
|
+
if (file === "claude" || file === "codex")
|
|
96
|
+
return { runtime: file, args };
|
|
97
|
+
let command = args;
|
|
98
|
+
if (file === "hq") {
|
|
99
|
+
const separator = args.lastIndexOf("--");
|
|
100
|
+
if (separator < 0)
|
|
101
|
+
return undefined;
|
|
102
|
+
command = args.slice(separator + 1);
|
|
103
|
+
}
|
|
104
|
+
if (command[0] === "timeout" || command[0] === "gtimeout")
|
|
105
|
+
command = command.slice(4);
|
|
106
|
+
const runtime = command[0];
|
|
107
|
+
return runtime === "claude" || runtime === "codex" ? { runtime, args: command.slice(1) } : undefined;
|
|
108
|
+
}
|
|
109
|
+
function pathExecutable(name, env) {
|
|
110
|
+
if (name.includes(path.sep)) {
|
|
111
|
+
try {
|
|
112
|
+
fs.accessSync(name, fs.constants.X_OK);
|
|
113
|
+
return name;
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
return undefined;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const pathValue = env.PATH ?? "";
|
|
120
|
+
for (const directory of pathValue.split(path.delimiter)) {
|
|
121
|
+
const candidate = path.join(directory || ".", name);
|
|
122
|
+
try {
|
|
123
|
+
fs.accessSync(candidate, fs.constants.X_OK);
|
|
124
|
+
return candidate;
|
|
125
|
+
}
|
|
126
|
+
catch { /* keep searching PATH */ }
|
|
127
|
+
}
|
|
128
|
+
return undefined;
|
|
129
|
+
}
|
|
130
|
+
export async function spawnProcess(request) {
|
|
131
|
+
if (request.env.HQ_JOB_RUN_STUBS === "1") {
|
|
132
|
+
const recordFile = request.env.HQ_JOB_RUN_PROCESS_RECORD;
|
|
133
|
+
if (recordFile) {
|
|
134
|
+
appendLog(recordFile, `${JSON.stringify({ file: request.file, args: request.args, cwd: request.cwd })}\n`);
|
|
135
|
+
}
|
|
136
|
+
const nestedAgent = runtimeArgv(request.file, request.args);
|
|
137
|
+
if (nestedAgent)
|
|
138
|
+
appendArgvRecord(request.env.HQ_JOB_RUN_AGENT_ARGV_RECORD, nestedAgent.runtime, nestedAgent.args);
|
|
139
|
+
if (request.file === "bash") {
|
|
140
|
+
const helperArgs = request.args.slice(1);
|
|
141
|
+
const helper = path.basename(request.args[0] ?? "");
|
|
142
|
+
if (helper === "notify.sh" || helper === "hq-job-notify.sh")
|
|
143
|
+
appendArgvRecord(request.env.HQ_JOB_RUN_NOTIFY_ARGV_RECORD, "hq-job-notify.sh", helperArgs);
|
|
144
|
+
if (helper === "remediate.sh" || helper === "hq-job-remediate.sh")
|
|
145
|
+
appendArgvRecord(request.env.HQ_JOB_RUN_REMEDIATE_ARGV_RECORD, "hq-job-remediate.sh", helperArgs);
|
|
146
|
+
}
|
|
147
|
+
if (request.file === "tmux" && request.args[0] === "new-session") {
|
|
148
|
+
const doneFile = request.env.HQ_JOB_RUN_REMOTE_DONE_FILE;
|
|
149
|
+
const exitFile = request.env.HQ_JOB_RUN_REMOTE_EXIT_FILE;
|
|
150
|
+
if (doneFile)
|
|
151
|
+
fs.writeFileSync(doneFile, "");
|
|
152
|
+
if (exitFile)
|
|
153
|
+
fs.writeFileSync(exitFile, `EXIT=${request.env.HQ_JOB_RUN_STUB_EXIT_CODE ?? "0"}\n`);
|
|
154
|
+
}
|
|
155
|
+
const diagnostic = request.env.HQ_JOB_RUN_STUB_DIAGNOSTIC || "";
|
|
156
|
+
if (nestedAgent && request.logPath && diagnostic)
|
|
157
|
+
appendLog(request.logPath, `${diagnostic}\n`);
|
|
158
|
+
const response = request.env.HQ_JOB_RUN_STUB_REMEDIATE_OUTPUT
|
|
159
|
+
?? '{"retry_recommended":false,"attempted":false,"notes":"stubbed"}';
|
|
160
|
+
const code = nestedAgent ? Number(request.env.HQ_JOB_RUN_STUB_EXIT_CODE ?? "0") : 0;
|
|
161
|
+
const helperName = path.basename(request.args[0] ?? "");
|
|
162
|
+
const isRemediation = request.file === "bash" && (helperName === "remediate.sh" || helperName === "hq-job-remediate.sh");
|
|
163
|
+
const stdout = isRemediation ? Buffer.from(response) : Buffer.alloc(0);
|
|
164
|
+
return { code: Number.isInteger(code) ? code : 0, stdout, stderr: Buffer.alloc(0) };
|
|
165
|
+
}
|
|
166
|
+
return await new Promise((resolve, reject) => {
|
|
167
|
+
let settled = false;
|
|
168
|
+
let timedOut = false;
|
|
169
|
+
let forceTimer;
|
|
170
|
+
let timeout;
|
|
171
|
+
const stdout = [];
|
|
172
|
+
const stderr = [];
|
|
173
|
+
let logWriteError;
|
|
174
|
+
let processSignalError;
|
|
175
|
+
const processGroup = Boolean(request.timeoutMs && request.timeoutMs > 0 && process.platform !== "win32");
|
|
176
|
+
const child = spawn(request.file, request.args, {
|
|
177
|
+
cwd: request.cwd,
|
|
178
|
+
env: request.env,
|
|
179
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
180
|
+
detached: processGroup,
|
|
181
|
+
});
|
|
182
|
+
child.once("error", reject);
|
|
183
|
+
child.stdout.on("data", (chunk) => {
|
|
184
|
+
if (request.logPath) {
|
|
185
|
+
try {
|
|
186
|
+
appendLog(request.logPath, chunk);
|
|
187
|
+
}
|
|
188
|
+
catch (error) {
|
|
189
|
+
logWriteError = error;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
else
|
|
193
|
+
stdout.push(Buffer.from(chunk));
|
|
194
|
+
});
|
|
195
|
+
child.stderr.on("data", (chunk) => {
|
|
196
|
+
if (request.logPath) {
|
|
197
|
+
try {
|
|
198
|
+
appendLog(request.logPath, chunk);
|
|
199
|
+
}
|
|
200
|
+
catch (error) {
|
|
201
|
+
logWriteError = error;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
else
|
|
205
|
+
stderr.push(Buffer.from(chunk));
|
|
206
|
+
});
|
|
207
|
+
if (request.timeoutMs && request.timeoutMs > 0) {
|
|
208
|
+
timeout = setTimeout(() => {
|
|
209
|
+
timedOut = true;
|
|
210
|
+
try {
|
|
211
|
+
signalProcessTree(child, "SIGTERM", processGroup);
|
|
212
|
+
}
|
|
213
|
+
catch (error) {
|
|
214
|
+
processSignalError = error;
|
|
215
|
+
}
|
|
216
|
+
forceTimer = setTimeout(() => {
|
|
217
|
+
try {
|
|
218
|
+
signalProcessTree(child, "SIGKILL", processGroup);
|
|
219
|
+
}
|
|
220
|
+
catch (error) {
|
|
221
|
+
processSignalError ??= error;
|
|
222
|
+
}
|
|
223
|
+
}, 15_000);
|
|
224
|
+
}, request.timeoutMs);
|
|
225
|
+
timeout.unref?.();
|
|
226
|
+
}
|
|
227
|
+
child.once("close", (code, signal) => {
|
|
228
|
+
if (settled)
|
|
229
|
+
return;
|
|
230
|
+
settled = true;
|
|
231
|
+
if (timeout)
|
|
232
|
+
clearTimeout(timeout);
|
|
233
|
+
if (forceTimer)
|
|
234
|
+
clearTimeout(forceTimer);
|
|
235
|
+
if (processSignalError) {
|
|
236
|
+
reject(processSignalError);
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
if (logWriteError) {
|
|
240
|
+
reject(logWriteError);
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
resolve({
|
|
244
|
+
code: timedOut ? 124 : code ?? 128 + numericSignal(signal),
|
|
245
|
+
stdout: Buffer.concat(stdout),
|
|
246
|
+
stderr: Buffer.concat(stderr),
|
|
247
|
+
});
|
|
248
|
+
});
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
/** Holds the same kernel flock(2) lock used by the shell companion. */
|
|
252
|
+
export async function acquireFlock(file, env = process.env) {
|
|
253
|
+
const binary = env.HQ_JOB_FLOCK_BIN || pathExecutable("flock", env);
|
|
254
|
+
if (!binary)
|
|
255
|
+
throw new Error("Outpost/Linux requires util-linux flock(1) for serialized job runs");
|
|
256
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
257
|
+
return await new Promise((resolve, reject) => {
|
|
258
|
+
const child = spawn(binary, ["-x", file, process.execPath, "-e", LOCK_HOLDER], {
|
|
259
|
+
env,
|
|
260
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
261
|
+
});
|
|
262
|
+
let stdout = "";
|
|
263
|
+
let stderr = "";
|
|
264
|
+
let released = false;
|
|
265
|
+
child.stdout.setEncoding("utf8");
|
|
266
|
+
child.stderr.setEncoding("utf8");
|
|
267
|
+
child.stdout.on("data", (chunk) => { stdout += chunk; });
|
|
268
|
+
child.stderr.on("data", (chunk) => { stderr += chunk; });
|
|
269
|
+
child.once("error", reject);
|
|
270
|
+
child.once("close", (code) => {
|
|
271
|
+
if (released)
|
|
272
|
+
return;
|
|
273
|
+
released = true;
|
|
274
|
+
if (!stdout.includes("HQ_LOCK_READY\n")) {
|
|
275
|
+
reject(new Error(stderr.trim() || `failed to acquire global run lock: ${file} (flock exit ${code ?? 1})`));
|
|
276
|
+
}
|
|
277
|
+
});
|
|
278
|
+
const readyTimer = setInterval(() => {
|
|
279
|
+
if (!stdout.includes("HQ_LOCK_READY\n"))
|
|
280
|
+
return;
|
|
281
|
+
clearInterval(readyTimer);
|
|
282
|
+
let done = false;
|
|
283
|
+
resolve({
|
|
284
|
+
release: async () => {
|
|
285
|
+
if (done)
|
|
286
|
+
return;
|
|
287
|
+
done = true;
|
|
288
|
+
child.stdin.end();
|
|
289
|
+
await new Promise((complete) => child.once("close", () => complete()));
|
|
290
|
+
released = true;
|
|
291
|
+
},
|
|
292
|
+
});
|
|
293
|
+
}, 5);
|
|
294
|
+
readyTimer.unref?.();
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
function defaultDependencies(env) {
|
|
298
|
+
const stubbed = env.HQ_JOB_RUN_STUBS === "1";
|
|
299
|
+
return {
|
|
300
|
+
env,
|
|
301
|
+
cwd: process.cwd(),
|
|
302
|
+
home: env.HOME || os.homedir(),
|
|
303
|
+
now: () => stubbed && env.HQ_JOB_RUN_STUB_DATE ? new Date(env.HQ_JOB_RUN_STUB_DATE) : new Date(),
|
|
304
|
+
epochNow: () => stubbed && env.HQ_JOB_RUN_STUB_EPOCH
|
|
305
|
+
? Number(env.HQ_JOB_RUN_STUB_EPOCH)
|
|
306
|
+
: Math.floor(Date.now() / 1000),
|
|
307
|
+
readFile: (file) => fs.readFileSync(file),
|
|
308
|
+
stat: (file) => fs.statSync(file),
|
|
309
|
+
mkdir: (directory) => fs.mkdirSync(directory, { recursive: true }),
|
|
310
|
+
writeFile: (file, content) => fs.writeFileSync(file, content),
|
|
311
|
+
appendFile: (file, content) => fs.appendFileSync(file, content),
|
|
312
|
+
chmod: (file, mode) => fs.chmodSync(file, mode),
|
|
313
|
+
remove: (file) => fs.rmSync(file, { force: true }),
|
|
314
|
+
fetch: stubbed
|
|
315
|
+
? async (input, init) => {
|
|
316
|
+
const recordFile = env.HQ_JOB_RUN_FETCH_RECORD;
|
|
317
|
+
if (recordFile)
|
|
318
|
+
appendLog(recordFile, `${JSON.stringify({ input, method: init.method, headers: init.headers, body: init.body })}\n`);
|
|
319
|
+
if (env.HQ_JOB_RUN_REQUEST_BODY_RECORD && init.body !== undefined) {
|
|
320
|
+
appendLog(env.HQ_JOB_RUN_REQUEST_BODY_RECORD, `${String(init.body)}\n`);
|
|
321
|
+
}
|
|
322
|
+
if (env.HQ_JOB_RUN_AUTH_HEADER_RECORD) {
|
|
323
|
+
const token = new Headers(init.headers).get("x-outpost-instance-token") ?? "";
|
|
324
|
+
appendLog(env.HQ_JOB_RUN_AUTH_HEADER_RECORD, `x-outpost-instance-token: ${token}\n`);
|
|
325
|
+
}
|
|
326
|
+
const scenario = env.HQ_JOB_RUN_FETCH_SCENARIO || "200";
|
|
327
|
+
if (scenario === "transport-error")
|
|
328
|
+
throw new Error("synthetic transport error");
|
|
329
|
+
if (scenario === "timeout")
|
|
330
|
+
throw Object.assign(new Error("synthetic timeout"), { name: "AbortError" });
|
|
331
|
+
const status = Number(scenario);
|
|
332
|
+
return new Response("", { status: Number.isInteger(status) && status >= 200 && status <= 599 ? status : 200 });
|
|
333
|
+
}
|
|
334
|
+
: (input, init) => globalThis.fetch(input, init),
|
|
335
|
+
runProcess: spawnProcess,
|
|
336
|
+
acquireLock: stubbed && env.HQ_JOB_RUN_LOCK_REAL !== "1"
|
|
337
|
+
? async (file) => {
|
|
338
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
339
|
+
fs.writeFileSync(file, "");
|
|
340
|
+
return { release: async () => undefined };
|
|
341
|
+
}
|
|
342
|
+
: (file) => acquireFlock(file, env),
|
|
343
|
+
sleep: stubbed ? async () => undefined : (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)),
|
|
344
|
+
scriptPath: resolveBundledAsset(RUN_ASSET),
|
|
345
|
+
};
|
|
346
|
+
}
|
|
347
|
+
function yqText(value, fallback = "") {
|
|
348
|
+
if (value === undefined || value === null)
|
|
349
|
+
return fallback;
|
|
350
|
+
if (typeof value === "string")
|
|
351
|
+
return value;
|
|
352
|
+
if (typeof value === "boolean" || typeof value === "number")
|
|
353
|
+
return String(value);
|
|
354
|
+
return JSON.stringify(value);
|
|
355
|
+
}
|
|
356
|
+
function parseYamlJob(file, deps) {
|
|
357
|
+
const source = deps.readFile(file).toString("utf8");
|
|
358
|
+
let value;
|
|
359
|
+
try {
|
|
360
|
+
value = load(source, { schema: JSON_SCHEMA });
|
|
361
|
+
}
|
|
362
|
+
catch (error) {
|
|
363
|
+
throw new JobYamlParseFailure(source, error);
|
|
364
|
+
}
|
|
365
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
366
|
+
}
|
|
367
|
+
function malformedYamlDiagnostic(file, source, error) {
|
|
368
|
+
if (!error || typeof error !== "object" || !("reason" in error))
|
|
369
|
+
return undefined;
|
|
370
|
+
const reason = String(error.reason ?? "");
|
|
371
|
+
if (reason !== "unexpected end of the stream within a flow collection")
|
|
372
|
+
return undefined;
|
|
373
|
+
const flowSequence = source.lastIndexOf("[") > source.lastIndexOf("]");
|
|
374
|
+
const flowMapping = source.lastIndexOf("{") > source.lastIndexOf("}");
|
|
375
|
+
if (!flowSequence && !flowMapping)
|
|
376
|
+
return undefined;
|
|
377
|
+
const flow = flowSequence ? "flow sequence" : "flow mapping";
|
|
378
|
+
const closer = flowSequence ? "]" : "}";
|
|
379
|
+
const lineCount = Math.max(1, source.split(/\r\n|\n/).length - (source.endsWith("\n") ? 1 : 0));
|
|
380
|
+
return `Error: bad file '${file}': yaml: while parsing a ${flow} at <unknown position>: line ${lineCount}: did not find expected ',' or '${closer}'\n`;
|
|
381
|
+
}
|
|
382
|
+
function yamlReason(error) {
|
|
383
|
+
if (!error || typeof error !== "object" || !("reason" in error))
|
|
384
|
+
return "invalid YAML document";
|
|
385
|
+
const reason = String(error.reason ?? "");
|
|
386
|
+
return reason || "invalid YAML document";
|
|
387
|
+
}
|
|
388
|
+
function findJobFileById(id, root, deps) {
|
|
389
|
+
const files = [];
|
|
390
|
+
const personal = path.join(root, "personal", "jobs");
|
|
391
|
+
try {
|
|
392
|
+
for (const entry of fs.readdirSync(personal, { withFileTypes: true })) {
|
|
393
|
+
if (entry.isFile() && /\.ya?ml$/.test(entry.name))
|
|
394
|
+
files.push(path.join(personal, entry.name));
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
catch { /* The shell find ignores a missing job registry. */ }
|
|
398
|
+
const companies = path.join(root, "companies");
|
|
399
|
+
try {
|
|
400
|
+
for (const company of fs.readdirSync(companies, { withFileTypes: true })) {
|
|
401
|
+
if (!company.isDirectory())
|
|
402
|
+
continue;
|
|
403
|
+
const jobs = path.join(companies, company.name, "jobs");
|
|
404
|
+
try {
|
|
405
|
+
for (const entry of fs.readdirSync(jobs, { withFileTypes: true })) {
|
|
406
|
+
if (entry.isFile() && /\.ya?ml$/.test(entry.name))
|
|
407
|
+
files.push(path.join(jobs, entry.name));
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
catch { /* The shell find ignores a company without jobs. */ }
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
catch { /* The shell find ignores a missing company registry. */ }
|
|
414
|
+
const candidates = files.filter((file) => !/\.conflict-|\/\.conflicts\//.test(file)).sort();
|
|
415
|
+
let found;
|
|
416
|
+
for (const file of candidates) {
|
|
417
|
+
let value;
|
|
418
|
+
try {
|
|
419
|
+
value = load(deps.readFile(file).toString("utf8"), { schema: JSON_SCHEMA });
|
|
420
|
+
}
|
|
421
|
+
catch {
|
|
422
|
+
continue;
|
|
423
|
+
}
|
|
424
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
425
|
+
continue;
|
|
426
|
+
if (yqText(value.id) !== id)
|
|
427
|
+
continue;
|
|
428
|
+
if (found)
|
|
429
|
+
throw new Error(`duplicate job id '${id}' in registry: ${found} and ${file}`);
|
|
430
|
+
found = file;
|
|
431
|
+
}
|
|
432
|
+
return found;
|
|
433
|
+
}
|
|
434
|
+
function parseArgs(argv) {
|
|
435
|
+
let jobTarget = "";
|
|
436
|
+
let jobId = "";
|
|
437
|
+
let hqRoot = "";
|
|
438
|
+
for (let index = 0; index < argv.length;) {
|
|
439
|
+
const argument = argv[index];
|
|
440
|
+
if (argument === "--hq-root" || argument === "--job-id") {
|
|
441
|
+
const value = argv[index + 1] ?? "";
|
|
442
|
+
if (argument === "--hq-root") {
|
|
443
|
+
if (!value)
|
|
444
|
+
return { jobTarget, jobId, hqRoot, error: "--hq-root requires a directory", help: false };
|
|
445
|
+
hqRoot = value;
|
|
446
|
+
}
|
|
447
|
+
else {
|
|
448
|
+
if (!value)
|
|
449
|
+
return { jobTarget, jobId, hqRoot, error: "--job-id requires a value", help: false };
|
|
450
|
+
jobId = value;
|
|
451
|
+
}
|
|
452
|
+
index += 2;
|
|
453
|
+
}
|
|
454
|
+
else if (argument === "-h" || argument === "--help") {
|
|
455
|
+
return { jobTarget, jobId, hqRoot, help: true };
|
|
456
|
+
}
|
|
457
|
+
else if (argument === "--") {
|
|
458
|
+
break;
|
|
459
|
+
}
|
|
460
|
+
else if (argument.startsWith("-")) {
|
|
461
|
+
return { jobTarget, jobId, hqRoot, error: `unknown option: ${argument}`, help: false };
|
|
462
|
+
}
|
|
463
|
+
else {
|
|
464
|
+
if (jobTarget)
|
|
465
|
+
return { jobTarget, jobId, hqRoot, error: `unexpected argument: ${argument}`, help: false };
|
|
466
|
+
jobTarget = argument;
|
|
467
|
+
index += 1;
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
return { jobTarget, jobId, hqRoot, help: false };
|
|
471
|
+
}
|
|
472
|
+
function emitUsage() { process.stdout.write(USAGE); }
|
|
473
|
+
function emitLog(message) { process.stderr.write(`${RUNNER_NAME}: ${message}\n`); }
|
|
474
|
+
function emitDie(message) { process.stderr.write(`${RUNNER_NAME}: ${message}\n`); return 2; }
|
|
475
|
+
function getPath(job, dotted) {
|
|
476
|
+
let cursor = job;
|
|
477
|
+
for (const piece of dotted.split(".")) {
|
|
478
|
+
if (!cursor || typeof cursor !== "object" || Array.isArray(cursor))
|
|
479
|
+
return undefined;
|
|
480
|
+
cursor = cursor[piece];
|
|
481
|
+
}
|
|
482
|
+
return cursor;
|
|
483
|
+
}
|
|
484
|
+
function isPlainRecord(value) {
|
|
485
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
486
|
+
}
|
|
487
|
+
function resolveJobPath(target, jobId, root, deps) {
|
|
488
|
+
if (jobId) {
|
|
489
|
+
const found = findJobFileById(jobId, root, deps);
|
|
490
|
+
if (!found)
|
|
491
|
+
throw new Error(`job id not found: ${jobId}`);
|
|
492
|
+
return found;
|
|
493
|
+
}
|
|
494
|
+
if (!target)
|
|
495
|
+
throw new Error("usage: hq-job-run.sh --job-id <id> | <job.yaml>");
|
|
496
|
+
if (fs.existsSync(target) && fs.statSync(target).isFile())
|
|
497
|
+
return target;
|
|
498
|
+
const underRoot = path.join(root, target);
|
|
499
|
+
if (fs.existsSync(underRoot) && fs.statSync(underRoot).isFile())
|
|
500
|
+
return underRoot;
|
|
501
|
+
const found = findJobFileById(target, root, deps);
|
|
502
|
+
if (!found)
|
|
503
|
+
throw new Error(`job not found: ${target}`);
|
|
504
|
+
return found;
|
|
505
|
+
}
|
|
506
|
+
function compactJson(value) {
|
|
507
|
+
return serializeJqJson(value, "compact").replace(/\n$/, "");
|
|
508
|
+
}
|
|
509
|
+
function buildPrompt(job, file) {
|
|
510
|
+
const prompt = yqText(getPath(job, "exec.prompt"));
|
|
511
|
+
const skill = yqText(getPath(job, "exec.skill"));
|
|
512
|
+
if (prompt && prompt !== "null")
|
|
513
|
+
return prompt;
|
|
514
|
+
if (skill && skill !== "null") {
|
|
515
|
+
let args = getPath(job, "exec.args");
|
|
516
|
+
if (args === undefined || args === null)
|
|
517
|
+
args = {};
|
|
518
|
+
return `Execute the HQ skill "${skill}" unattended with these args (JSON): ${compactJson(args)}. Follow the skill instructions exactly and produce a concise completion summary.`;
|
|
519
|
+
}
|
|
520
|
+
throw new Error(`job missing exec.prompt and exec.skill: ${file}`);
|
|
521
|
+
}
|
|
522
|
+
function jobSecretNames(job) {
|
|
523
|
+
const raw = getPath(job, "requirements.secrets");
|
|
524
|
+
if (raw === undefined || raw === null)
|
|
525
|
+
return [];
|
|
526
|
+
if (Array.isArray(raw))
|
|
527
|
+
return raw.map((value) => yqText(value)).filter((value) => value !== "" && value !== "null");
|
|
528
|
+
if (typeof raw === "string")
|
|
529
|
+
return [...raw].filter((value) => value !== "" && value !== "null");
|
|
530
|
+
return [];
|
|
531
|
+
}
|
|
532
|
+
function isoNow(deps) {
|
|
533
|
+
const override = deps.env.HQ_JOB_NOW;
|
|
534
|
+
return override ? override : deps.now().toISOString().replace(/\.\d{3}Z$/, "Z");
|
|
535
|
+
}
|
|
536
|
+
function runStamp(deps) {
|
|
537
|
+
const value = deps.now();
|
|
538
|
+
return value.toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z");
|
|
539
|
+
}
|
|
540
|
+
function elapsed(start, deps) {
|
|
541
|
+
return Math.max(0, deps.epochNow() - start);
|
|
542
|
+
}
|
|
543
|
+
function appendMeterMarker(jobId, runtime, started, ended, exitStatus, duration, skipReason, deps) {
|
|
544
|
+
const directory = deps.env.HQ_JOB_METER_DIR || path.join(deps.home, ".hq/jobs/meters");
|
|
545
|
+
deps.mkdir(directory);
|
|
546
|
+
const value = {
|
|
547
|
+
schema: "hq.outpost.job-run/v1",
|
|
548
|
+
kind: "scheduled-job-run",
|
|
549
|
+
job_id: jobId,
|
|
550
|
+
runtime,
|
|
551
|
+
started_at: started,
|
|
552
|
+
ended_at: ended,
|
|
553
|
+
exit_status: exitStatus,
|
|
554
|
+
duration_seconds: duration,
|
|
555
|
+
...(deps.env.OUTPOST_ID ? { outpost_id: deps.env.OUTPOST_ID } : {}),
|
|
556
|
+
...(skipReason ? { skip_reason: skipReason } : {}),
|
|
557
|
+
};
|
|
558
|
+
const meter = path.join(directory, "runs.jsonl");
|
|
559
|
+
deps.appendFile(meter, `${compactJson(value)}\n`);
|
|
560
|
+
try {
|
|
561
|
+
deps.chmod(meter, 0o600);
|
|
562
|
+
}
|
|
563
|
+
catch { /* The shell ignores chmod failure. */ }
|
|
564
|
+
}
|
|
565
|
+
function apiBase(env) {
|
|
566
|
+
return (env.HQ_PRO_API_URL || env.HQ_API_URL || env.HQ_VAULT_API_URL || "https://hqapi.hq.computer").replace(/\/$/, "");
|
|
567
|
+
}
|
|
568
|
+
function assignmentValue(value) {
|
|
569
|
+
if (value.length >= 2 && ((value[0] === "'" && value.at(-1) === "'") || (value[0] === '"' && value.at(-1) === '"'))) {
|
|
570
|
+
return value.slice(1, -1);
|
|
571
|
+
}
|
|
572
|
+
return value;
|
|
573
|
+
}
|
|
574
|
+
function parseIdentityEnvironment(contents) {
|
|
575
|
+
const identity = {};
|
|
576
|
+
for (const line of contents.split(/\r?\n/)) {
|
|
577
|
+
const match = /^\s*(?:export\s+)?(OUTPOST_USER_ID|OUTPOST_INSTANCE_TOKEN|OUTPOST_ID)\s*=\s*(.*?)\s*$/.exec(line);
|
|
578
|
+
if (!match)
|
|
579
|
+
continue;
|
|
580
|
+
const value = assignmentValue(match[2] ?? "");
|
|
581
|
+
if (match[1] === "OUTPOST_USER_ID")
|
|
582
|
+
identity.userId = value;
|
|
583
|
+
else if (match[1] === "OUTPOST_INSTANCE_TOKEN")
|
|
584
|
+
identity.instanceToken = value;
|
|
585
|
+
else
|
|
586
|
+
identity.outpostId = value;
|
|
587
|
+
}
|
|
588
|
+
return identity;
|
|
589
|
+
}
|
|
590
|
+
function parseRunnerAssignments(contents) {
|
|
591
|
+
const identity = {};
|
|
592
|
+
for (const line of contents.split(/\r?\n/)) {
|
|
593
|
+
const match = /^(USER_ID|INSTANCE_TOKEN|OUTPOST_ID)="(.*)"$/.exec(line);
|
|
594
|
+
if (!match)
|
|
595
|
+
continue;
|
|
596
|
+
if (match[1] === "USER_ID" && identity.userId === undefined)
|
|
597
|
+
identity.userId = match[2] ?? "";
|
|
598
|
+
else if (match[1] === "INSTANCE_TOKEN" && identity.instanceToken === undefined)
|
|
599
|
+
identity.instanceToken = match[2] ?? "";
|
|
600
|
+
else if (match[1] === "OUTPOST_ID" && identity.outpostId === undefined)
|
|
601
|
+
identity.outpostId = match[2] ?? "";
|
|
602
|
+
}
|
|
603
|
+
return identity;
|
|
604
|
+
}
|
|
605
|
+
function readOptionalIdentity(file, parser, deps) {
|
|
606
|
+
try {
|
|
607
|
+
return parser(deps.readFile(file).toString("utf8"));
|
|
608
|
+
}
|
|
609
|
+
catch {
|
|
610
|
+
return {};
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
function resolveBoxIdentity(env, deps) {
|
|
614
|
+
let userId = env.OUTPOST_USER_ID || "";
|
|
615
|
+
let instanceToken = env.OUTPOST_INSTANCE_TOKEN || "";
|
|
616
|
+
let outpostId = env.OUTPOST_ID || "";
|
|
617
|
+
if (userId && instanceToken)
|
|
618
|
+
return { userId, instanceToken, ...(outpostId ? { outpostId } : {}) };
|
|
619
|
+
const stubIdentityFile = env.HQ_JOB_RUN_STUBS === "1" ? env.HQ_JOB_RUN_STUB_IDENTITY_FILE : undefined;
|
|
620
|
+
const identityFile = readOptionalIdentity(stubIdentityFile || OUTPOST_IDENTITY_FILE, parseIdentityEnvironment, deps);
|
|
621
|
+
if (identityFile.userId !== undefined)
|
|
622
|
+
userId = identityFile.userId;
|
|
623
|
+
if (identityFile.instanceToken !== undefined)
|
|
624
|
+
instanceToken = identityFile.instanceToken;
|
|
625
|
+
if (identityFile.outpostId !== undefined)
|
|
626
|
+
outpostId = identityFile.outpostId;
|
|
627
|
+
if (userId && instanceToken)
|
|
628
|
+
return { userId, instanceToken, ...(outpostId ? { outpostId } : {}) };
|
|
629
|
+
const runnerFile = env.HQ_JOB_PROBE_RUNNER || OUTPOST_RUNNER_FILE;
|
|
630
|
+
const runner = readOptionalIdentity(runnerFile, parseRunnerAssignments, deps);
|
|
631
|
+
userId ||= runner.userId || "";
|
|
632
|
+
instanceToken ||= runner.instanceToken || "";
|
|
633
|
+
outpostId ||= runner.outpostId || "";
|
|
634
|
+
if (!userId || !instanceToken)
|
|
635
|
+
return undefined;
|
|
636
|
+
return { userId, instanceToken, ...(outpostId ? { outpostId } : {}) };
|
|
637
|
+
}
|
|
638
|
+
async function ingestRunStatus(jobId, runAt, runId, exit, failureClass, actions, logFile, deps) {
|
|
639
|
+
if (deps.env.HQ_JOB_RUN_NO_INGEST === "1") {
|
|
640
|
+
emitLog("ingest skipped (HQ_JOB_RUN_NO_INGEST=1)");
|
|
641
|
+
return;
|
|
642
|
+
}
|
|
643
|
+
const identity = resolveBoxIdentity(deps.env, deps);
|
|
644
|
+
if (!identity) {
|
|
645
|
+
emitLog("run status ingest skipped — missing Outpost identity");
|
|
646
|
+
return;
|
|
647
|
+
}
|
|
648
|
+
const payload = {
|
|
649
|
+
userId: identity.userId,
|
|
650
|
+
jobId,
|
|
651
|
+
kind: "run",
|
|
652
|
+
run_at: runAt,
|
|
653
|
+
event_id: runId,
|
|
654
|
+
run_id: runId,
|
|
655
|
+
last_exit: exit,
|
|
656
|
+
failure_class: failureClass ? failureClass : null,
|
|
657
|
+
next_actions: actions,
|
|
658
|
+
...(identity.outpostId ? { outpostId: identity.outpostId } : {}),
|
|
659
|
+
};
|
|
660
|
+
const controller = new AbortController();
|
|
661
|
+
const timeout = setTimeout(() => controller.abort(), 20_000);
|
|
662
|
+
let status = 0;
|
|
663
|
+
try {
|
|
664
|
+
const response = await deps.fetch(`${apiBase(deps.env)}/outpost/internal/jobs-status`, {
|
|
665
|
+
method: "POST",
|
|
666
|
+
redirect: "manual",
|
|
667
|
+
headers: {
|
|
668
|
+
"content-type": "application/json",
|
|
669
|
+
"x-outpost-instance-token": identity.instanceToken,
|
|
670
|
+
},
|
|
671
|
+
body: compactJson(payload),
|
|
672
|
+
signal: controller.signal,
|
|
673
|
+
});
|
|
674
|
+
status = response.status;
|
|
675
|
+
}
|
|
676
|
+
catch {
|
|
677
|
+
status = 0;
|
|
678
|
+
}
|
|
679
|
+
finally {
|
|
680
|
+
clearTimeout(timeout);
|
|
681
|
+
}
|
|
682
|
+
if (status >= 200 && status < 300)
|
|
683
|
+
emitLog(`run status ingested job=${jobId} failure_class=${failureClass || "null"} http=${status}`);
|
|
684
|
+
else
|
|
685
|
+
emitLog(`run status ingest failed job=${jobId} http=${status} (logged; not failing run)`);
|
|
686
|
+
void logFile;
|
|
687
|
+
}
|
|
688
|
+
async function runHelper(binary, args, cwd, env, deps, logPath, timeoutMs) {
|
|
689
|
+
return await deps.runProcess({ file: binary, args, cwd, env, ...(logPath ? { logPath } : {}), ...(timeoutMs ? { timeoutMs } : {}) });
|
|
690
|
+
}
|
|
691
|
+
async function callNotify(jobId, jobName, outcome, summary, duration, logPath, failureClass, nextAction, owner, notifyMode, hqRoot, notifyBin, deps) {
|
|
692
|
+
if (deps.env.HQ_JOB_RUN_NO_NOTIFY === "1") {
|
|
693
|
+
emitLog("notify skipped (HQ_JOB_RUN_NO_NOTIFY=1)");
|
|
694
|
+
return;
|
|
695
|
+
}
|
|
696
|
+
try {
|
|
697
|
+
const info = deps.stat(notifyBin);
|
|
698
|
+
if (!(info.mode & 0o111)) {
|
|
699
|
+
try {
|
|
700
|
+
deps.chmod(notifyBin, info.mode | 0o111);
|
|
701
|
+
}
|
|
702
|
+
catch { /* shell ignores chmod failure */ }
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
catch (error) {
|
|
706
|
+
if (error.code === "ENOENT") {
|
|
707
|
+
emitLog(`notify skipped — missing ${notifyBin}`);
|
|
708
|
+
return;
|
|
709
|
+
}
|
|
710
|
+
throw error;
|
|
711
|
+
}
|
|
712
|
+
const args = [notifyBin,
|
|
713
|
+
"--hq-root", hqRoot,
|
|
714
|
+
"--job-id", jobId,
|
|
715
|
+
"--job-name", jobName,
|
|
716
|
+
"--outcome", outcome,
|
|
717
|
+
"--summary", summary,
|
|
718
|
+
"--duration", String(duration),
|
|
719
|
+
"--log-path", logPath,
|
|
720
|
+
"--failure-class", failureClass,
|
|
721
|
+
"--next-action", nextAction,
|
|
722
|
+
"--owner", owner,
|
|
723
|
+
"--notify", notifyMode,
|
|
724
|
+
];
|
|
725
|
+
const result = await runHelper("bash", args, deps.cwd, { ...deps.env, HQ_ROOT: hqRoot }, deps, logPath);
|
|
726
|
+
if (result.code !== 0)
|
|
727
|
+
emitLog(`notify helper exited ${result.code} (ignored)`);
|
|
728
|
+
}
|
|
729
|
+
async function callRemediate(failureClass, jobId, company, hqRoot, remediateBin, logFile, deps) {
|
|
730
|
+
try {
|
|
731
|
+
const info = deps.stat(remediateBin);
|
|
732
|
+
if (!(info.mode & 0o111)) {
|
|
733
|
+
try {
|
|
734
|
+
deps.chmod(remediateBin, info.mode | 0o111);
|
|
735
|
+
}
|
|
736
|
+
catch { /* shell ignores chmod failure */ }
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
catch (error) {
|
|
740
|
+
if (error.code === "ENOENT")
|
|
741
|
+
return '{"retry_recommended":false,"attempted":false,"notes":"remediate missing"}';
|
|
742
|
+
throw error;
|
|
743
|
+
}
|
|
744
|
+
const args = [remediateBin, "--hq-root", hqRoot, "--failure-class", failureClass, "--job-id", jobId, "--company", company];
|
|
745
|
+
const result = await runHelper("bash", args, deps.cwd, { ...deps.env, HQ_ROOT: hqRoot, HQ_JOB_REMEDIATE_LOG: logFile }, deps);
|
|
746
|
+
if (result.stderr.length)
|
|
747
|
+
deps.appendFile(logFile, result.stderr);
|
|
748
|
+
const output = result.stdout.toString("utf8").replace(/\n+$/, "");
|
|
749
|
+
return result.code !== 0 || !output ? '{"retry_recommended":false,"attempted":false,"notes":"remediate failed"}' : output;
|
|
750
|
+
}
|
|
751
|
+
function nextActionForClass(failureClass, runtime) {
|
|
752
|
+
switch (failureClass) {
|
|
753
|
+
case "auth": return `re-login on Outpost (device code) for ${runtime || "the runtime CLI"}`;
|
|
754
|
+
case "secrets": return "share the missing secret to this Outpost (hq secrets share KEY) then re-probe";
|
|
755
|
+
case "timeout": return "raise timeout_seconds or split the job work";
|
|
756
|
+
case "infra": return "check Outpost disk/network; job will self-retry once on transient infra";
|
|
757
|
+
case "agent_error": return "inspect the job log and fix the prompt/skill";
|
|
758
|
+
default: return "";
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
function classifyFailure(exitCode, logFile, authHint = "", deps) {
|
|
762
|
+
if (authHint && authHint !== "ok")
|
|
763
|
+
return "auth";
|
|
764
|
+
if (exitCode === 124 || exitCode === 137)
|
|
765
|
+
return "timeout";
|
|
766
|
+
let text = "";
|
|
767
|
+
try {
|
|
768
|
+
const lines = deps.readFile(logFile).toString("utf8").split(/\r?\n/);
|
|
769
|
+
if (lines[lines.length - 1] === "")
|
|
770
|
+
lines.pop();
|
|
771
|
+
text = lines.slice(-80).join("\n");
|
|
772
|
+
}
|
|
773
|
+
catch { /* tail ignores a missing log */ }
|
|
774
|
+
const lower = text.toLowerCase();
|
|
775
|
+
if (/not logged in|please log in|authentication (failed|required)|unauthorized|401|invalid.*(token|credential)|credentials? (missing|expired|invalid)|device.?code|login required/.test(lower))
|
|
776
|
+
return "auth";
|
|
777
|
+
if (/secret.*(not found|missing|denied|not visible|not shared)|hq secrets.*(fail|error|denied)|missing vault|access denied.*secret/.test(lower))
|
|
778
|
+
return "secrets";
|
|
779
|
+
if (/(^|[^_])timed out|command timed out|deadline exceeded|killed by timeout|exit[=:][\s]*124\b/.test(lower))
|
|
780
|
+
return "timeout";
|
|
781
|
+
if (/connection (refused|reset|timed out)|network is unreachable|temporary failure|econnreset|enotfound|no space left|i\/o error|transport error|dns (fail|error)|http[/ ]*(502|503|504)|statusingesterror|api unreachable/.test(lower))
|
|
782
|
+
return "infra";
|
|
783
|
+
return exitCode === 0 ? "" : "agent_error";
|
|
784
|
+
}
|
|
785
|
+
function canonicalPath(candidate) {
|
|
786
|
+
const absolute = path.resolve(candidate);
|
|
787
|
+
let cursor = absolute;
|
|
788
|
+
const suffix = [];
|
|
789
|
+
while (!fs.existsSync(cursor)) {
|
|
790
|
+
const parent = path.dirname(cursor);
|
|
791
|
+
if (parent === cursor)
|
|
792
|
+
return absolute;
|
|
793
|
+
suffix.unshift(path.basename(cursor));
|
|
794
|
+
cursor = parent;
|
|
795
|
+
}
|
|
796
|
+
try {
|
|
797
|
+
return path.join(fs.realpathSync(cursor), ...suffix);
|
|
798
|
+
}
|
|
799
|
+
catch {
|
|
800
|
+
return absolute;
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
function cwdSkip(cwdRel, hqRoot) {
|
|
804
|
+
if (!cwdRel || cwdRel === "null")
|
|
805
|
+
return undefined;
|
|
806
|
+
if (cwdRel.startsWith("/"))
|
|
807
|
+
return { reason: `cwd absolute not allowed: ${cwdRel} — skip`, action: "fix job cwd to an HQ-root-relative path (no absolute, no '..')" };
|
|
808
|
+
if (cwdRel.split("/").includes(".."))
|
|
809
|
+
return { reason: `cwd contains '..': ${cwdRel} — skip`, action: "fix job cwd to an HQ-root-relative path without '..'" };
|
|
810
|
+
const root = canonicalPath(hqRoot);
|
|
811
|
+
const runCwd = canonicalPath(path.join(hqRoot, cwdRel));
|
|
812
|
+
if (runCwd !== root && !runCwd.startsWith(`${root}${path.sep}`))
|
|
813
|
+
return { reason: `cwd escapes HQ_ROOT: ${cwdRel} → ${runCwd} — skip`, action: "fix job cwd to stay under HQ root" };
|
|
814
|
+
if (!fs.existsSync(runCwd) || !fs.statSync(runCwd).isDirectory())
|
|
815
|
+
return { reason: `cwd missing: ${runCwd} — skip`, action: `create missing cwd on Outpost: ${runCwd}` };
|
|
816
|
+
return undefined;
|
|
817
|
+
}
|
|
818
|
+
function readAuthStatus(runtime, env, home, deps) {
|
|
819
|
+
if (runtime !== "claude" && runtime !== "codex")
|
|
820
|
+
return "unknown_runtime";
|
|
821
|
+
if (env.HQ_JOB_RUN_STUBS !== "1" && !pathExecutable(runtime, env))
|
|
822
|
+
return "no_cli";
|
|
823
|
+
if (env.HQ_JOB_RUN_STUBS === "1" && env.HQ_JOB_RUN_STUB_MISSING_CLI === runtime)
|
|
824
|
+
return "no_cli";
|
|
825
|
+
const file = runtime === "claude"
|
|
826
|
+
? env.HQ_JOB_PROBE_CLAUDE_CRED || path.join(home, ".claude/.credentials.json")
|
|
827
|
+
: env.HQ_JOB_PROBE_CODEX_AUTH || path.join(home, ".codex/auth.json");
|
|
828
|
+
try {
|
|
829
|
+
return deps.stat(file).size > 0 ? "ok" : "missing";
|
|
830
|
+
}
|
|
831
|
+
catch {
|
|
832
|
+
return "missing";
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
function statusText(exit, started, ended, duration, failureClass, retried) {
|
|
836
|
+
return `exit=${exit}\nstarted_at=${started}\nended_at=${ended}\nduration_seconds=${duration}\nfailure_class=${failureClass}\nretried=${retried}\n`;
|
|
837
|
+
}
|
|
838
|
+
function resolveNumber(value, fallback) {
|
|
839
|
+
const raw = yqText(value, String(fallback));
|
|
840
|
+
if (!/^\d+$/.test(raw))
|
|
841
|
+
return fallback;
|
|
842
|
+
return Math.max(60, Math.min(14_400, Number(raw)));
|
|
843
|
+
}
|
|
844
|
+
function bashPrintfQuote(value) {
|
|
845
|
+
const safe = /^[A-Za-z0-9_@%+=:,./-]+$/;
|
|
846
|
+
if (value.length === 0)
|
|
847
|
+
return "''";
|
|
848
|
+
if (safe.test(value))
|
|
849
|
+
return value;
|
|
850
|
+
const hasControlCharacter = [...value].some((character) => {
|
|
851
|
+
const code = character.codePointAt(0);
|
|
852
|
+
return code < 0x20 || code === 0x7f;
|
|
853
|
+
});
|
|
854
|
+
if (hasControlCharacter) {
|
|
855
|
+
const encoded = [...value].map((character) => {
|
|
856
|
+
if (character === "\\")
|
|
857
|
+
return "\\\\";
|
|
858
|
+
if (character === "'")
|
|
859
|
+
return "\\'";
|
|
860
|
+
if (character === "\n")
|
|
861
|
+
return "\\n";
|
|
862
|
+
if (character === "\r")
|
|
863
|
+
return "\\r";
|
|
864
|
+
if (character === "\t")
|
|
865
|
+
return "\\t";
|
|
866
|
+
if (character === "\v")
|
|
867
|
+
return "\\v";
|
|
868
|
+
if (character === "\f")
|
|
869
|
+
return "\\f";
|
|
870
|
+
const code = character.codePointAt(0);
|
|
871
|
+
if (code < 0x20 || code === 0x7f)
|
|
872
|
+
return `\\x${code.toString(16).padStart(2, "0")}`;
|
|
873
|
+
return character;
|
|
874
|
+
}).join("");
|
|
875
|
+
return `$'${encoded}'`;
|
|
876
|
+
}
|
|
877
|
+
return [...value].map((character) => safe.test(character) ? character : `\\${character}`).join("");
|
|
878
|
+
}
|
|
879
|
+
async function runRemoteClaude(jobId, runtime, prompt, runCwd, timeoutSeconds, stamp, logDir, logFile, deps) {
|
|
880
|
+
if (!pathExecutable("tmux", deps.env) && deps.env.HQ_JOB_RUN_STUBS !== "1") {
|
|
881
|
+
deps.appendFile(logFile, "remote surface requested but tmux missing — falling back to headless\n");
|
|
882
|
+
return 99;
|
|
883
|
+
}
|
|
884
|
+
const session = `hq-job-${jobId}-${stamp}`;
|
|
885
|
+
const remoteName = `schedule/${jobId}`;
|
|
886
|
+
const deadline = timeoutSeconds + 30;
|
|
887
|
+
const exitFile = path.join(logDir, `${stamp}.remote-exit`);
|
|
888
|
+
const doneFile = path.join(logDir, `${stamp}.remote-done`);
|
|
889
|
+
try {
|
|
890
|
+
deps.remove(doneFile);
|
|
891
|
+
}
|
|
892
|
+
catch { /* rm -f */ }
|
|
893
|
+
const remotePrompt = `${prompt}\n\nWhen ALL work above is fully finished (success or failure), create this exact empty file with a shell tool (and do not create it earlier):\n touch ${bashPrintfQuote(doneFile)}\nThen stop.`;
|
|
894
|
+
if (deps.env.HQ_JOB_RUN_STUBS === "1") {
|
|
895
|
+
appendArgvRecord(deps.env.HQ_JOB_RUN_AGENT_ARGV_RECORD, "claude", ["--remote-control", remoteName, "--permission-mode", "bypassPermissions", "--dangerously-skip-permissions", remotePrompt]);
|
|
896
|
+
}
|
|
897
|
+
deps.appendFile(logFile, `surface=remote remote_name=${remoteName} tmux_session=${session} done_file=${doneFile}\n`);
|
|
898
|
+
const remoteArgs = ["--remote-control", remoteName, "--permission-mode", "bypassPermissions", "--dangerously-skip-permissions", remotePrompt];
|
|
899
|
+
const tmuxCommand = [process.execPath, REMOTE_LAUNCHER, exitFile, ...remoteArgs].map(bashPrintfQuote).join(" ");
|
|
900
|
+
const tmuxEnv = deps.env.HQ_JOB_RUN_STUBS === "1"
|
|
901
|
+
? { ...deps.env, HQ_JOB_RUN_REMOTE_DONE_FILE: doneFile, HQ_JOB_RUN_REMOTE_EXIT_FILE: exitFile }
|
|
902
|
+
: deps.env;
|
|
903
|
+
const runTmux = async (args) => await runHelper("tmux", args, deps.cwd, tmuxEnv, deps);
|
|
904
|
+
await runTmux(["kill-session", "-t", session]);
|
|
905
|
+
await runTmux(["new-session", "-d", "-s", session, "-x", "120", "-y", "40", "-c", runCwd, tmuxCommand]);
|
|
906
|
+
let elapsedSeconds = 0;
|
|
907
|
+
let code = 0;
|
|
908
|
+
while (elapsedSeconds < deadline) {
|
|
909
|
+
await deps.sleep(2_000);
|
|
910
|
+
elapsedSeconds += 2;
|
|
911
|
+
let pane = (await runTmux(["capture-pane", "-t", session, "-p", "-S", "-120"])).stdout.toString("utf8");
|
|
912
|
+
if (pane.includes("Yes, I accept")) {
|
|
913
|
+
await runTmux(["send-keys", "-t", session, "2", "Enter"]);
|
|
914
|
+
continue;
|
|
915
|
+
}
|
|
916
|
+
if (pane.includes("Try the new fullscreen renderer")) {
|
|
917
|
+
await runTmux(["send-keys", "-t", session, "2", "Enter"]);
|
|
918
|
+
continue;
|
|
919
|
+
}
|
|
920
|
+
if (pane.includes("Do you want to proceed")) {
|
|
921
|
+
await runTmux(["send-keys", "-t", session, "2", "Enter"]);
|
|
922
|
+
deps.appendFile(logFile, "auto-accepted tool permission prompt\n");
|
|
923
|
+
continue;
|
|
924
|
+
}
|
|
925
|
+
if (fs.existsSync(doneFile)) {
|
|
926
|
+
deps.appendFile(logFile, "remote done sentinel present\n");
|
|
927
|
+
await deps.sleep(2_000);
|
|
928
|
+
pane = (await runTmux(["capture-pane", "-t", session, "-p", "-S", "-300"])).stdout.toString("utf8");
|
|
929
|
+
code = 0;
|
|
930
|
+
break;
|
|
931
|
+
}
|
|
932
|
+
if (fs.existsSync(exitFile)) {
|
|
933
|
+
const raw = deps.readFile(exitFile).toString("utf8").replace(/[^0-9]/g, "").slice(0, 3);
|
|
934
|
+
code = Number(raw || "0");
|
|
935
|
+
break;
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
if (elapsedSeconds >= deadline) {
|
|
939
|
+
deps.appendFile(logFile, `remote surface timed out after ${timeoutSeconds}s\n`);
|
|
940
|
+
code = 124;
|
|
941
|
+
}
|
|
942
|
+
const capture = await runTmux(["capture-pane", "-t", session, "-p", "-S", "-300"]);
|
|
943
|
+
deps.appendFile(logFile, `=== remote pane capture ===\n${capture.stdout.toString("utf8")}`);
|
|
944
|
+
deps.appendFile(logFile, `tearing down remote session ${remoteName} (review under Desktop Archived)\n`);
|
|
945
|
+
await runTmux(["kill-session", "-t", session]);
|
|
946
|
+
return code;
|
|
947
|
+
}
|
|
948
|
+
async function runAgent(runtime, surface, prompt, runCwd, timeoutSeconds, secretNames, company, logFile, stamp, jobId, logDir, deps) {
|
|
949
|
+
if (deps.env.HQ_JOB_RUN_SKIP_EXEC === "1") {
|
|
950
|
+
deps.appendFile(logFile, "(HQ_JOB_RUN_SKIP_EXEC=1 — not invoking runtime CLI)\n");
|
|
951
|
+
return 0;
|
|
952
|
+
}
|
|
953
|
+
if (runtime === "claude" && surface === "remote") {
|
|
954
|
+
const remoteCode = await runRemoteClaude(jobId, runtime, prompt, runCwd, timeoutSeconds, stamp, logDir, logFile, deps);
|
|
955
|
+
if (remoteCode !== 99)
|
|
956
|
+
return remoteCode;
|
|
957
|
+
}
|
|
958
|
+
if (runtime === "codex" && surface === "remote")
|
|
959
|
+
deps.appendFile(logFile, "surface=remote ignored for codex — using headless\n");
|
|
960
|
+
let agentArgs;
|
|
961
|
+
if (runtime === "claude")
|
|
962
|
+
agentArgs = ["-p", prompt, "--output-format", "text", "--permission-mode", "bypassPermissions"];
|
|
963
|
+
else if (runtime === "codex")
|
|
964
|
+
agentArgs = ["exec", "-s", "workspace-write", "--skip-git-repo-check", "-C", runCwd, prompt];
|
|
965
|
+
else {
|
|
966
|
+
deps.appendFile(logFile, `unknown runtime: ${runtime}\n`);
|
|
967
|
+
return 2;
|
|
968
|
+
}
|
|
969
|
+
const wrap = deps.env.HQ_JOB_TIMEOUT_BIN
|
|
970
|
+
|| (pathExecutable("timeout", deps.env) ? "timeout" : pathExecutable("gtimeout", deps.env) ? "gtimeout" : "");
|
|
971
|
+
const timeoutMs = wrap ? timeoutSeconds * 1000 : undefined;
|
|
972
|
+
let file = runtime;
|
|
973
|
+
let args = agentArgs;
|
|
974
|
+
if (secretNames.length > 0 && (deps.env.HQ_JOB_RUN_STUBS === "1" || pathExecutable("hq", deps.env))) {
|
|
975
|
+
const only = secretNames.join(",");
|
|
976
|
+
args = ["secrets", ...(company && company !== "null" ? ["--company", company] : ["--personal"]), "exec", "--only", only, "--script", deps.scriptPath, "--", ...(wrap ? [wrap, "--signal=TERM", "--kill-after=15s", `${timeoutSeconds}s`] : []), runtime, ...agentArgs];
|
|
977
|
+
file = "hq";
|
|
978
|
+
}
|
|
979
|
+
else if (secretNames.length > 0) {
|
|
980
|
+
deps.appendFile(logFile, "warning: requirements.secrets set but hq CLI missing — running without secrets exec\n");
|
|
981
|
+
}
|
|
982
|
+
const result = await runHelper(file, args, runCwd, deps.env, deps, logFile, timeoutMs);
|
|
983
|
+
return result.code;
|
|
984
|
+
}
|
|
985
|
+
async function recordSkip(jobId, jobName, runtime, skipReason, failureClass, action, statusSuffix, notifyOutcome, owner, notifyMode, company, hqRoot, logFile, statusFile, startedAt, startedEpoch, logDir, stamp, notifyBin, remediateBin, deps, authStatus = "") {
|
|
986
|
+
deps.appendFile(logFile, `skip_reason=${skipReason}\n${authStatus ? `auth_status=${authStatus}\n` : ""}failure_class=${failureClass}\n`);
|
|
987
|
+
emitLog(skipReason);
|
|
988
|
+
const endedAt = isoNow(deps);
|
|
989
|
+
const duration = elapsed(startedEpoch, deps);
|
|
990
|
+
deps.writeFile(statusFile, `exit=0\nskip_reason=${skipReason}\nfailure_class=${failureClass}\nstarted_at=${startedAt}\nended_at=${endedAt}\n`);
|
|
991
|
+
appendMeterMarker(jobId, runtime, startedAt, endedAt, 0, duration, skipReason, deps);
|
|
992
|
+
let nextAction = action;
|
|
993
|
+
if (failureClass === "auth") {
|
|
994
|
+
const remediate = await callRemediate("auth", jobId, company, hqRoot, remediateBin, logFile, deps);
|
|
995
|
+
deps.appendFile(logFile, `remediate=${remediate}\n`);
|
|
996
|
+
try {
|
|
997
|
+
nextAction = yqText(JSON.parse(remediate).next_action);
|
|
998
|
+
}
|
|
999
|
+
catch {
|
|
1000
|
+
nextAction = "";
|
|
1001
|
+
}
|
|
1002
|
+
if (!nextAction)
|
|
1003
|
+
nextAction = nextActionForClass("auth", runtime);
|
|
1004
|
+
}
|
|
1005
|
+
const actions = nextAction ? [nextAction] : [];
|
|
1006
|
+
const runId = `run-${jobId}-${stamp}-${statusSuffix}`;
|
|
1007
|
+
await ingestRunStatus(jobId, endedAt, runId, 1, failureClass, actions, logFile, deps);
|
|
1008
|
+
await callNotify(jobId, jobName, notifyOutcome, skipReason, duration, logFile, failureClass, nextAction, owner, notifyMode, hqRoot, notifyBin, deps);
|
|
1009
|
+
void logDir;
|
|
1010
|
+
return 0;
|
|
1011
|
+
}
|
|
1012
|
+
export async function runJob(argv, hqRootOverride, injected) {
|
|
1013
|
+
const env = injected?.env ?? process.env;
|
|
1014
|
+
const deps = { ...defaultDependencies(env), ...injected };
|
|
1015
|
+
const args = parseArgs(argv);
|
|
1016
|
+
if (args.help) {
|
|
1017
|
+
emitUsage();
|
|
1018
|
+
return 0;
|
|
1019
|
+
}
|
|
1020
|
+
if (args.error)
|
|
1021
|
+
return emitDie(args.error);
|
|
1022
|
+
const explicitRoot = hqRootOverride || args.hqRoot || env.HQ_ROOT;
|
|
1023
|
+
const hqRoot = resolveLiveRoot({ hqRoot: explicitRoot });
|
|
1024
|
+
let jobFile;
|
|
1025
|
+
try {
|
|
1026
|
+
jobFile = resolveJobPath(args.jobTarget, args.jobId, hqRoot, deps);
|
|
1027
|
+
}
|
|
1028
|
+
catch (error) {
|
|
1029
|
+
return emitDie(error instanceof Error ? error.message : String(error));
|
|
1030
|
+
}
|
|
1031
|
+
if (!fs.existsSync(jobFile))
|
|
1032
|
+
return emitDie(`job file missing: ${jobFile}`);
|
|
1033
|
+
let job;
|
|
1034
|
+
try {
|
|
1035
|
+
job = parseYamlJob(jobFile, deps);
|
|
1036
|
+
}
|
|
1037
|
+
catch (error) {
|
|
1038
|
+
if (error instanceof JobYamlParseFailure) {
|
|
1039
|
+
const diagnostic = malformedYamlDiagnostic(jobFile, error.source, error.parseError)
|
|
1040
|
+
?? `Error: bad file '${jobFile}': yaml: ${yamlReason(error.parseError)}\n`;
|
|
1041
|
+
process.stderr.write(diagnostic);
|
|
1042
|
+
return 1;
|
|
1043
|
+
}
|
|
1044
|
+
return emitDie(error instanceof Error ? error.message : String(error));
|
|
1045
|
+
}
|
|
1046
|
+
const jobId = yqText(job.id);
|
|
1047
|
+
const runtime = yqText(job.runtime);
|
|
1048
|
+
const jobName = yqText(job.name, jobId) || jobId;
|
|
1049
|
+
const timeoutSeconds = resolveNumber(job.timeout_seconds, 600);
|
|
1050
|
+
const cwdRel = yqText(job.cwd ?? getPath(job, "requirements.cwd"));
|
|
1051
|
+
const company = yqText(getPath(job, "requirements.company"));
|
|
1052
|
+
const owner = yqText(job.owner);
|
|
1053
|
+
const notifyMode = yqText(job.notify, "profile") || "profile";
|
|
1054
|
+
const enabledValue = Object.hasOwn(job, "enabled") ? yqText(job.enabled) : "true";
|
|
1055
|
+
let surface = yqText(getPath(job, "exec.surface"));
|
|
1056
|
+
if (!surface || surface === "null") {
|
|
1057
|
+
const alertsFile = env.HQ_JOB_ALERTS_FILE || path.join(hqRoot, "personal/settings/schedule-alerts.yaml");
|
|
1058
|
+
try {
|
|
1059
|
+
const alerts = load(deps.readFile(alertsFile).toString("utf8"), { schema: JSON_SCHEMA });
|
|
1060
|
+
if (isPlainRecord(alerts))
|
|
1061
|
+
surface = yqText(alerts.execution_surface);
|
|
1062
|
+
}
|
|
1063
|
+
catch { /* Optional alerts file. */ }
|
|
1064
|
+
}
|
|
1065
|
+
surface = surface === "remote" ? "remote" : "headless";
|
|
1066
|
+
if (!jobId || jobId === "null")
|
|
1067
|
+
return emitDie(`job missing id: ${jobFile}`);
|
|
1068
|
+
if (!runtime || runtime === "null")
|
|
1069
|
+
return emitDie(`job missing runtime: ${jobFile}`);
|
|
1070
|
+
if (!jobName) {
|
|
1071
|
+
// yq's `.name // .id // ""` returns the id when name is null or absent.
|
|
1072
|
+
}
|
|
1073
|
+
if (enabledValue === "false" || enabledValue === "False" || enabledValue === "0") {
|
|
1074
|
+
emitLog(`skip job ${jobId} — enabled=false`);
|
|
1075
|
+
return 0;
|
|
1076
|
+
}
|
|
1077
|
+
const logBase = env.HQ_JOB_LOG_DIR || path.join(deps.home, ".hq/jobs/logs");
|
|
1078
|
+
const logDir = path.join(logBase, jobId);
|
|
1079
|
+
deps.mkdir(logDir);
|
|
1080
|
+
const lockPath = env.HQ_JOB_LOCK || path.join(deps.home, ".hq/jobs/run.lock");
|
|
1081
|
+
deps.mkdir(path.dirname(lockPath));
|
|
1082
|
+
const stamp = runStamp(deps);
|
|
1083
|
+
const logFile = path.join(logDir, `${stamp}.log`);
|
|
1084
|
+
const statusFile = path.join(logDir, `${stamp}.status`);
|
|
1085
|
+
let lease;
|
|
1086
|
+
try {
|
|
1087
|
+
lease = await deps.acquireLock(lockPath);
|
|
1088
|
+
}
|
|
1089
|
+
catch (error) {
|
|
1090
|
+
return emitDie(`failed to acquire global run lock: ${lockPath}${error instanceof Error && error.message.includes("flock") ? ` (${error.message})` : ""}`);
|
|
1091
|
+
}
|
|
1092
|
+
try {
|
|
1093
|
+
const startedAt = isoNow(deps);
|
|
1094
|
+
const startedEpoch = deps.epochNow();
|
|
1095
|
+
deps.appendFile(logFile, `=== hq-job-run ${jobId} ===\nstarted_at=${startedAt}\nruntime=${runtime}\ntimeout_seconds=${timeoutSeconds}\nsurface=${surface}\njob_file=${jobFile}\nhq_root=${hqRoot}\n`);
|
|
1096
|
+
const notifyBin = env.HQ_JOB_NOTIFY_BIN || path.join(path.dirname(deps.scriptPath), "hq-job-notify.sh");
|
|
1097
|
+
const remediateBin = env.HQ_JOB_REMEDIATE_BIN || path.join(path.dirname(deps.scriptPath), "hq-job-remediate.sh");
|
|
1098
|
+
const authStatus = readAuthStatus(runtime, env, deps.home, deps);
|
|
1099
|
+
if (authStatus !== "ok") {
|
|
1100
|
+
const reason = authStatus === "no_cli"
|
|
1101
|
+
? `runtime CLI '${runtime}' not installed on PATH — skip (re-login/install on Outpost)`
|
|
1102
|
+
: authStatus === "missing"
|
|
1103
|
+
? `runtime CLI '${runtime}' not authenticated on Outpost (cached credentials missing) — skip`
|
|
1104
|
+
: `unsupported or unknown runtime '${runtime}' — skip`;
|
|
1105
|
+
return await recordSkip(jobId, jobName, runtime, reason, "auth", "", "authskip", "failed", owner, notifyMode, company, hqRoot, logFile, statusFile, startedAt, startedEpoch, logDir, stamp, notifyBin, remediateBin, deps, authStatus);
|
|
1106
|
+
}
|
|
1107
|
+
const cwdProblem = cwdSkip(cwdRel, hqRoot);
|
|
1108
|
+
if (cwdProblem)
|
|
1109
|
+
return await recordSkip(jobId, jobName, runtime, cwdProblem.reason, "infra", cwdProblem.action, "cwd", "blocked", owner, notifyMode, company, hqRoot, logFile, statusFile, startedAt, startedEpoch, logDir, stamp, notifyBin, remediateBin, deps);
|
|
1110
|
+
const runCwd = cwdRel && cwdRel !== "null" ? canonicalPath(path.join(hqRoot, cwdRel)) : hqRoot;
|
|
1111
|
+
let prompt;
|
|
1112
|
+
try {
|
|
1113
|
+
prompt = buildPrompt(job, jobFile);
|
|
1114
|
+
}
|
|
1115
|
+
catch (error) {
|
|
1116
|
+
return emitDie(error instanceof Error ? error.message : String(error));
|
|
1117
|
+
}
|
|
1118
|
+
const secretNames = jobSecretNames(job);
|
|
1119
|
+
let exitCode = await runAgent(runtime, surface, prompt, runCwd, timeoutSeconds, secretNames, company, logFile, stamp, jobId, logDir, deps);
|
|
1120
|
+
let failureClass = "";
|
|
1121
|
+
let nextAction = "";
|
|
1122
|
+
let retried = 0;
|
|
1123
|
+
if (exitCode !== 0) {
|
|
1124
|
+
failureClass = classifyFailure(exitCode, logFile, "", deps);
|
|
1125
|
+
deps.appendFile(logFile, `failure_class=${failureClass}\n`);
|
|
1126
|
+
const remediation = await callRemediate(failureClass, jobId, company, hqRoot, remediateBin, logFile, deps);
|
|
1127
|
+
deps.appendFile(logFile, `remediate=${remediation}\n`);
|
|
1128
|
+
let retryRecommended = false;
|
|
1129
|
+
try {
|
|
1130
|
+
const parsed = JSON.parse(remediation);
|
|
1131
|
+
nextAction = yqText(parsed.next_action);
|
|
1132
|
+
retryRecommended = yqText(parsed.retry_recommended, "false") === "true";
|
|
1133
|
+
}
|
|
1134
|
+
catch { /* Invalid helper output follows the shell's jq path; it cannot recommend retry. */ }
|
|
1135
|
+
if (!nextAction)
|
|
1136
|
+
nextAction = nextActionForClass(failureClass, runtime);
|
|
1137
|
+
if (retryRecommended && retried === 0) {
|
|
1138
|
+
retried = 1;
|
|
1139
|
+
deps.appendFile(logFile, "remediate_retry=1\n");
|
|
1140
|
+
emitLog(`self-remediate recommended retry for ${failureClass} — retrying once`);
|
|
1141
|
+
exitCode = await runAgent(runtime, surface, prompt, runCwd, timeoutSeconds, secretNames, company, logFile, stamp, jobId, logDir, deps);
|
|
1142
|
+
if (exitCode !== 0) {
|
|
1143
|
+
failureClass = classifyFailure(exitCode, logFile, "", deps);
|
|
1144
|
+
deps.appendFile(logFile, `failure_class_after_retry=${failureClass}\n`);
|
|
1145
|
+
nextAction = nextActionForClass(failureClass, runtime);
|
|
1146
|
+
}
|
|
1147
|
+
else {
|
|
1148
|
+
failureClass = "";
|
|
1149
|
+
nextAction = "";
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
const endedAt = isoNow(deps);
|
|
1154
|
+
const endedEpoch = deps.epochNow();
|
|
1155
|
+
const duration = Math.max(0, endedEpoch - startedEpoch);
|
|
1156
|
+
deps.appendFile(logFile, `ended_at=${endedAt}\nexit=${exitCode}\nduration_seconds=${duration}\nretried=${retried}\n${failureClass ? `failure_class=${failureClass}\n` : ""}`);
|
|
1157
|
+
deps.writeFile(statusFile, statusText(exitCode, startedAt, endedAt, duration, failureClass, retried));
|
|
1158
|
+
appendMeterMarker(jobId, runtime, startedAt, endedAt, exitCode, duration, "", deps);
|
|
1159
|
+
const runId = `run-${jobId}-${stamp}`;
|
|
1160
|
+
if (exitCode === 0) {
|
|
1161
|
+
await ingestRunStatus(jobId, endedAt, runId, 0, "", [], logFile, deps);
|
|
1162
|
+
const summary = retried === 1 ? "completed ok after self-remediation retry" : "completed ok";
|
|
1163
|
+
await callNotify(jobId, jobName, "ok", summary, duration, logFile, "", "", owner, notifyMode, hqRoot, notifyBin, deps);
|
|
1164
|
+
emitLog(`job ${jobId} completed ok (${duration}s)`);
|
|
1165
|
+
return 0;
|
|
1166
|
+
}
|
|
1167
|
+
await ingestRunStatus(jobId, endedAt, runId, Math.max(0, Math.min(255, exitCode)), failureClass, nextAction ? [nextAction] : [], logFile, deps);
|
|
1168
|
+
const summary = `exit=${exitCode} class=${failureClass}`;
|
|
1169
|
+
await callNotify(jobId, jobName, "failed", summary, duration, logFile, failureClass, nextAction, owner, notifyMode, hqRoot, notifyBin, deps);
|
|
1170
|
+
emitLog(`job ${jobId} failed exit=${exitCode} class=${failureClass} (${duration}s) log=${logFile}`);
|
|
1171
|
+
return exitCode;
|
|
1172
|
+
}
|
|
1173
|
+
finally {
|
|
1174
|
+
await lease.release();
|
|
1175
|
+
}
|
|
1176
|
+
}
|
|
1177
|
+
export function registerCoreJobRunCommand(job) {
|
|
1178
|
+
const command = job.command("run")
|
|
1179
|
+
.description("Run a scheduled job")
|
|
1180
|
+
.helpOption(false)
|
|
1181
|
+
.allowUnknownOption()
|
|
1182
|
+
.allowExcessArguments(true)
|
|
1183
|
+
.argument("[args...]", "Arguments passed to the scaffold helper")
|
|
1184
|
+
.action(async (args, _options, current) => {
|
|
1185
|
+
const argv = rawArgumentsForCommand(current, args);
|
|
1186
|
+
const root = current.parent?.parent?.opts().hqRoot ?? process.env.HQ_ROOT;
|
|
1187
|
+
if (!await coreNativeJobsEnabled()) {
|
|
1188
|
+
const resolvedRoot = resolveLiveRoot({ hqRoot: root });
|
|
1189
|
+
const result = runBundledScript({ asset: RUN_ASSET, args: argv, cwd: process.cwd(), hqRoot: resolvedRoot });
|
|
1190
|
+
if (result.code !== 0)
|
|
1191
|
+
process.exit(result.code);
|
|
1192
|
+
return;
|
|
1193
|
+
}
|
|
1194
|
+
const result = await runJob(argv, root);
|
|
1195
|
+
if (result !== 0)
|
|
1196
|
+
process.exit(result);
|
|
1197
|
+
});
|
|
1198
|
+
describeCoreCommand(command, { execution: "native", asset: RUN_ASSET, root: "live" });
|
|
1199
|
+
}
|
|
1200
|
+
function rawArgumentsForCommand(command, fallback, raw = process.argv) {
|
|
1201
|
+
const names = [];
|
|
1202
|
+
for (let current = command; current?.parent; current = current.parent)
|
|
1203
|
+
names.unshift(current.name());
|
|
1204
|
+
let cursor = 0;
|
|
1205
|
+
let start = -1;
|
|
1206
|
+
for (const name of names) {
|
|
1207
|
+
const index = raw.indexOf(name, cursor);
|
|
1208
|
+
if (index < 0)
|
|
1209
|
+
return command.args.length ? command.args : fallback;
|
|
1210
|
+
cursor = index + 1;
|
|
1211
|
+
start = cursor;
|
|
1212
|
+
}
|
|
1213
|
+
return start >= 0 ? raw.slice(start) : command.args.length ? command.args : fallback;
|
|
1214
|
+
}
|
|
1215
|
+
//# sourceMappingURL=core-job-run-native.js.map
|