auto-harness-client 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +180 -0
- package/package.json +1 -1
- package/src/cli/args.js +17 -1
- package/src/cli/commands/attach-repository.js +117 -0
- package/src/cli/commands/detach-repository.js +109 -0
- package/src/cli/commands/host-repo-add.js +81 -0
- package/src/cli/commands/host-repo-rm.js +9 -86
- package/src/cli/commands/host-repo.js +7 -2
- package/src/cli/commands/host-smoke-format.js +30 -0
- package/src/cli/commands/host-smoke-poll.js +77 -0
- package/src/cli/commands/host-smoke-provider.js +172 -0
- package/src/cli/commands/host-smoke-repository.js +49 -0
- package/src/cli/commands/host-smoke-session-attempt.js +104 -0
- package/src/cli/commands/host-smoke-teardown.js +164 -0
- package/src/cli/commands/host-smoke.js +151 -0
- package/src/cli/commands/host.js +7 -1
- package/src/cli/commands/parse-worktree-flag.js +30 -0
- package/src/cli/commands/repo-add.js +38 -0
- package/src/cli/commands/repo.js +3 -0
- package/src/cli/commands/session-cancel.js +28 -0
- package/src/cli/commands/session-create.js +117 -0
- package/src/cli/commands/session-get.js +26 -0
- package/src/cli/commands/session-logs.js +65 -0
- package/src/cli/commands/session-target.js +29 -0
- package/src/cli/commands/session.js +22 -0
- package/src/cli/main.js +2 -0
- package/src/cli/session-format.js +15 -0
- package/src/cli/usage.js +17 -0
- package/src/cli/wait-for-session.js +46 -0
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
import { CliUsageError } from "../cli-errors.js";
|
|
2
|
+
import { runHostRepoAdd } from "./host-repo-add.js";
|
|
2
3
|
import { runHostRepoRm } from "./host-repo-rm.js";
|
|
3
4
|
|
|
4
|
-
const USAGE =
|
|
5
|
+
const USAGE = `usage: auto-harness host repo <subcommand> ...
|
|
6
|
+
auto-harness host repo add <hostId> <repositoryId> --path <path> [--worktree <id>=<path>]...
|
|
7
|
+
[--default-branch <branch>] [--dry-run] [--json]
|
|
8
|
+
auto-harness host repo rm <hostId> <repositoryId> [--dry-run] [--json]`;
|
|
5
9
|
|
|
6
|
-
/** Dispatches `host repo <rm>`. */
|
|
10
|
+
/** Dispatches `host repo <add|rm>`. */
|
|
7
11
|
export async function runHostRepo(argv, io) {
|
|
8
12
|
const [action, ...rest] = argv;
|
|
13
|
+
if (action === "add") return runHostRepoAdd(rest, io);
|
|
9
14
|
if (action === "rm") return runHostRepoRm(rest, io);
|
|
10
15
|
throw new CliUsageError(USAGE);
|
|
11
16
|
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/** One progress line per step, as it happens — always stderr, so stdout stays a clean final
|
|
2
|
+
* summary (mirrors `session create --wait`'s status-change lines going to stderr). */
|
|
3
|
+
export function step(io, ok, message) {
|
|
4
|
+
io.stderr.write(`${ok ? "ok" : "FAIL"} ${message}\n`);
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The clean final summary on stdout: `--json` prints the full structured `result` verbatim;
|
|
9
|
+
* otherwise one `PASS`/`FAIL` line per provider (only ever empty when repo create/attach itself
|
|
10
|
+
* failed, in which case `result.setupError` is printed instead), then one overall line.
|
|
11
|
+
*/
|
|
12
|
+
export function printSummary(io, flags, result) {
|
|
13
|
+
if (flags["--json"]) {
|
|
14
|
+
io.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
|
15
|
+
return;
|
|
16
|
+
}
|
|
17
|
+
if (result.setupError) {
|
|
18
|
+
io.stdout.write(`FAIL setup: ${result.setupError}\n`);
|
|
19
|
+
}
|
|
20
|
+
for (const provider of result.providers) {
|
|
21
|
+
const detail = provider.message ? `: ${provider.message}` : "";
|
|
22
|
+
io.stdout.write(`${provider.pass ? "PASS" : "FAIL"} ${provider.provider}${detail}\n`);
|
|
23
|
+
}
|
|
24
|
+
const passed = result.providers.filter((provider) => provider.pass).length;
|
|
25
|
+
if (result.providers.length > 0) {
|
|
26
|
+
io.stdout.write(`${passed}/${result.providers.length} providers passed\n`);
|
|
27
|
+
}
|
|
28
|
+
io.stdout.write(`teardown ${result.teardown.ok ? "ok" : "FAILED"}\n`);
|
|
29
|
+
io.stdout.write(`host smoke ${result.ok ? "PASSED" : "FAILED"} for host ${result.hostId}\n`);
|
|
30
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { waitForSession } from "../wait-for-session.js";
|
|
2
|
+
|
|
3
|
+
/** Thrown from the wrapped `getSession` below to escape `waitForSession`'s loop the instant a
|
|
4
|
+
* usage-limit requeue is observed, rather than waiting out the rest of `timeoutMs`. */
|
|
5
|
+
class UsageLimitSignal extends Error {
|
|
6
|
+
constructor(session) {
|
|
7
|
+
super("usage_limit");
|
|
8
|
+
this.session = session;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/** `client.getSession` wrapped to check *every* poll for `errorCode === "usage_limit"`, not
|
|
13
|
+
* only a status *change* — see `waitForSmokeSession`'s doc comment for why that distinction
|
|
14
|
+
* matters here. */
|
|
15
|
+
function usageLimitWatchedClient(client) {
|
|
16
|
+
return {
|
|
17
|
+
getSession: async (id) => {
|
|
18
|
+
const session = await client.getSession(id);
|
|
19
|
+
if (session.errorCode === "usage_limit") throw new UsageLimitSignal(session);
|
|
20
|
+
return session;
|
|
21
|
+
},
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Wraps `waitForSession` so `host smoke` fails a provider fast when the control plane reports
|
|
27
|
+
* `errorCode: "usage_limit"`, instead of sitting out the whole `--timeout`.
|
|
28
|
+
* `session-transition-planner.ts`'s `planUsageLimit()` *requeues* a usage-limited session
|
|
29
|
+
* (status goes back to `"queued"`, cooldown applied to the account) rather than failing it
|
|
30
|
+
* outright, so a plain wait-for-terminal-status loop would never notice until the timeout
|
|
31
|
+
* elapsed. This is checked on every single poll, not on `waitForSession`'s own `onStatus`
|
|
32
|
+
* (which only fires on a status *change*) — a session that never even reaches `"running"`
|
|
33
|
+
* (requeued immediately, or its account was already cooling down when created) gets
|
|
34
|
+
* `errorCode` set with no status transition to hang the check off of.
|
|
35
|
+
*
|
|
36
|
+
* Throwing from the wrapped `getSession` — rather than from `onStatus` — rides a documented
|
|
37
|
+
* part of `wait-for-session.js`'s own contract ("A `getSession` rejection ... propagates
|
|
38
|
+
* as-is"), instead of relying on the undocumented fact that `onStatus` happens to run outside
|
|
39
|
+
* a try/catch. `onStatus` itself is passed through untouched, still only for status-change
|
|
40
|
+
* narration (e.g. to stderr).
|
|
41
|
+
*
|
|
42
|
+
* Resolves to `{ kind: "usage_limit", session }`, `{ kind: "timeout", session }`, or
|
|
43
|
+
* `{ kind: "terminal", session }` — never rejects for a usage-limit signal; a genuine
|
|
44
|
+
* `getSession` failure (network/API error) still propagates, exactly as `waitForSession` docs.
|
|
45
|
+
*/
|
|
46
|
+
export async function waitForSmokeSession(
|
|
47
|
+
client,
|
|
48
|
+
sessionId,
|
|
49
|
+
{ timeoutMs, intervalMs, sleep, now, onStatus },
|
|
50
|
+
) {
|
|
51
|
+
try {
|
|
52
|
+
const result = await waitForSession(usageLimitWatchedClient(client), sessionId, {
|
|
53
|
+
timeoutMs,
|
|
54
|
+
intervalMs,
|
|
55
|
+
sleep,
|
|
56
|
+
now,
|
|
57
|
+
onStatus,
|
|
58
|
+
});
|
|
59
|
+
return { kind: result.timedOut ? "timeout" : "terminal", session: result.session };
|
|
60
|
+
} catch (error) {
|
|
61
|
+
if (error instanceof UsageLimitSignal) return { kind: "usage_limit", session: error.session };
|
|
62
|
+
throw error;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** One-line diagnosis for a timed-out provider, keyed off the last observed status — see
|
|
67
|
+
* `host-smoke.js`'s usage text and the README for the fuller explanation. */
|
|
68
|
+
export function timeoutHint(lastStatus) {
|
|
69
|
+
if (lastStatus === "queued") {
|
|
70
|
+
return (
|
|
71
|
+
"session stayed queued — check that an online host advertises a ready execution " +
|
|
72
|
+
"profile for this provider's account (HARNESS_EXECUTION_PROFILES), and that something " +
|
|
73
|
+
"is running the scheduler (POST /scheduler/assign)"
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
return `session was still ${lastStatus} when the timeout elapsed`;
|
|
77
|
+
}
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { AutoHarnessError } from "../../index.js";
|
|
2
|
+
import { pathSegment } from "../path-segment.js";
|
|
3
|
+
import { resolveSessionTarget } from "./session-target.js";
|
|
4
|
+
import { step } from "./host-smoke-format.js";
|
|
5
|
+
import { timeoutHint } from "./host-smoke-poll.js";
|
|
6
|
+
import { runSessionAttempts } from "./host-smoke-session-attempt.js";
|
|
7
|
+
|
|
8
|
+
/** Cancels a session smoke created and, either way, removes it from the shared
|
|
9
|
+
* `activeSessionIds` bookkeeping the top-level teardown uses as its safety net — unless the
|
|
10
|
+
* cancel call itself fails for a reason other than "already terminal" (409), in which case the
|
|
11
|
+
* id is deliberately left in the set so `teardownSmoke` retries it at the end of the run. */
|
|
12
|
+
async function cancelForOutcome(client, io, sessionId, activeSessionIds) {
|
|
13
|
+
try {
|
|
14
|
+
await client.cancelSession(sessionId);
|
|
15
|
+
activeSessionIds.delete(sessionId);
|
|
16
|
+
} catch (error) {
|
|
17
|
+
if (error instanceof AutoHarnessError && error.status === 409) {
|
|
18
|
+
activeSessionIds.delete(sessionId); // raced to terminal on its own; nothing to cancel
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
io.stderr.write(
|
|
22
|
+
` session ${sessionId}: cancel failed (${error.message}); teardown will retry it\n`,
|
|
23
|
+
);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
async function fetchStdout(client, sessionId) {
|
|
28
|
+
const page = await client.request(`/sessions/${pathSegment(sessionId, "sessionId")}/logs`);
|
|
29
|
+
return (page.items ?? [])
|
|
30
|
+
.filter((item) => item.stream === "stdout")
|
|
31
|
+
.map((item) => item.content)
|
|
32
|
+
.join("\n");
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Runs one `--provider`'s end-to-end smoke session: resolve the target, then hand off to
|
|
37
|
+
* `runSessionAttempts` (create + wait, with a narrow, bounded retry for one specific
|
|
38
|
+
* "host hasn't caught up with the repo it just attached" failure — a host's cached inventory
|
|
39
|
+
* only refreshes on its own periodic poll, there is no push-on-write, and `host smoke` attaches
|
|
40
|
+
* its own throwaway repository immediately before creating a session against it, which races
|
|
41
|
+
* that poll in any real deployment, not only in a fast test), then check its logs. Always
|
|
42
|
+
* resolves to an outcome object — `{ provider, pass, ... }` — never throws or rejects, so a
|
|
43
|
+
* bad `--provider` value or a mid-poll network blip fails only *this* provider rather than
|
|
44
|
+
* aborting the ones after it; `host-smoke.js`'s loop relies on that to keep going.
|
|
45
|
+
*/
|
|
46
|
+
export async function runProviderSmoke({
|
|
47
|
+
client,
|
|
48
|
+
io,
|
|
49
|
+
repositoryId,
|
|
50
|
+
providerRef,
|
|
51
|
+
marker,
|
|
52
|
+
timeoutSeconds,
|
|
53
|
+
activeSessionIds,
|
|
54
|
+
sleep,
|
|
55
|
+
now,
|
|
56
|
+
intervalMs,
|
|
57
|
+
}) {
|
|
58
|
+
let target;
|
|
59
|
+
try {
|
|
60
|
+
target = await resolveSessionTarget(client, { "--provider": providerRef });
|
|
61
|
+
} catch (error) {
|
|
62
|
+
step(io, false, `provider ${providerRef}: could not resolve target: ${error.message}`);
|
|
63
|
+
return {
|
|
64
|
+
provider: providerRef,
|
|
65
|
+
pass: false,
|
|
66
|
+
reason: "target_resolution_failed",
|
|
67
|
+
message: error.message,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const outcome = await runSessionAttempts({
|
|
72
|
+
client,
|
|
73
|
+
io,
|
|
74
|
+
repositoryId,
|
|
75
|
+
providerRef,
|
|
76
|
+
target,
|
|
77
|
+
marker,
|
|
78
|
+
timeoutSeconds,
|
|
79
|
+
activeSessionIds,
|
|
80
|
+
sleep,
|
|
81
|
+
now,
|
|
82
|
+
intervalMs,
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
if (outcome.kind === "create_failed") {
|
|
86
|
+
step(io, false, `provider ${providerRef}: create session failed: ${outcome.message}`);
|
|
87
|
+
return {
|
|
88
|
+
provider: providerRef,
|
|
89
|
+
pass: false,
|
|
90
|
+
reason: "create_failed",
|
|
91
|
+
message: outcome.message,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
if (outcome.kind === "session_wait_failed") {
|
|
95
|
+
step(io, false, `provider ${providerRef}: session wait failed: ${outcome.message}`);
|
|
96
|
+
return {
|
|
97
|
+
provider: providerRef,
|
|
98
|
+
pass: false,
|
|
99
|
+
reason: "session_wait_failed",
|
|
100
|
+
sessionId: outcome.sessionId,
|
|
101
|
+
message: outcome.message,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
if (outcome.kind === "usage_limit") {
|
|
105
|
+
await cancelForOutcome(client, io, outcome.sessionId, activeSessionIds);
|
|
106
|
+
const message = "provider account hit its usage limit";
|
|
107
|
+
step(io, false, `provider ${providerRef}: ${message}`);
|
|
108
|
+
return {
|
|
109
|
+
provider: providerRef,
|
|
110
|
+
pass: false,
|
|
111
|
+
reason: "usage_limit",
|
|
112
|
+
sessionId: outcome.sessionId,
|
|
113
|
+
message,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
if (outcome.kind === "timeout") {
|
|
117
|
+
await cancelForOutcome(client, io, outcome.sessionId, activeSessionIds);
|
|
118
|
+
const hint = timeoutHint(outcome.session.status);
|
|
119
|
+
step(io, false, `provider ${providerRef}: timed out after ${timeoutSeconds}s (${hint})`);
|
|
120
|
+
return {
|
|
121
|
+
provider: providerRef,
|
|
122
|
+
pass: false,
|
|
123
|
+
reason: "timeout",
|
|
124
|
+
sessionId: outcome.sessionId,
|
|
125
|
+
message: hint,
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// Terminal: already resolved on its own, nothing left for teardown to cancel.
|
|
130
|
+
const { session, sessionId } = outcome;
|
|
131
|
+
if (session.status !== "completed" || session.exitCode !== 0) {
|
|
132
|
+
const detail = session.errorMessage ?? `exitCode=${session.exitCode ?? "n/a"}`;
|
|
133
|
+
step(io, false, `provider ${providerRef}: session ${session.status} (${detail})`);
|
|
134
|
+
return {
|
|
135
|
+
provider: providerRef,
|
|
136
|
+
pass: false,
|
|
137
|
+
reason: "session_failed",
|
|
138
|
+
sessionId,
|
|
139
|
+
status: session.status,
|
|
140
|
+
exitCode: session.exitCode,
|
|
141
|
+
message: detail,
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
let stdout;
|
|
146
|
+
try {
|
|
147
|
+
stdout = await fetchStdout(client, sessionId);
|
|
148
|
+
} catch (error) {
|
|
149
|
+
step(io, false, `provider ${providerRef}: fetching logs failed: ${error.message}`);
|
|
150
|
+
return {
|
|
151
|
+
provider: providerRef,
|
|
152
|
+
pass: false,
|
|
153
|
+
reason: "logs_failed",
|
|
154
|
+
sessionId,
|
|
155
|
+
message: error.message,
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
if (!stdout.includes(marker)) {
|
|
159
|
+
const message = "completed but marker was not found in stdout";
|
|
160
|
+
step(io, false, `provider ${providerRef}: ${message}`);
|
|
161
|
+
return { provider: providerRef, pass: false, reason: "marker_missing", sessionId, message };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
step(io, true, `provider ${providerRef}: PASS`);
|
|
165
|
+
return {
|
|
166
|
+
provider: providerRef,
|
|
167
|
+
pass: true,
|
|
168
|
+
sessionId,
|
|
169
|
+
status: session.status,
|
|
170
|
+
exitCode: session.exitCode,
|
|
171
|
+
};
|
|
172
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Creates a throwaway repository for `host smoke` to attach, run one prompt through, and delete
|
|
3
|
+
* again in its teardown. The daemon dispatches sessions against the host-local path this same
|
|
4
|
+
* run attaches (see `smokeInventoryEntry` below) — it never fetches or dials a repository's
|
|
5
|
+
* `url` — so a syntactically valid but inert HTTPS placeholder under the `example.test`
|
|
6
|
+
* reserved TLD (RFC 2606) is fine here, exactly like `e2e/real-cli/real-cli-helpers.ts` and
|
|
7
|
+
* `e2e/control/orchestration.spec.ts` already use for the same reason. `randomHex(bytes)` is
|
|
8
|
+
* injected (defaults to `node:crypto`'s `randomBytes` in `host-smoke.js`) so tests can assert
|
|
9
|
+
* on a deterministic generated name.
|
|
10
|
+
*/
|
|
11
|
+
export async function createSmokeRepository(client, randomHex) {
|
|
12
|
+
const name = `smoke-${randomHex(6)}`;
|
|
13
|
+
return client.request("/repositories", {
|
|
14
|
+
method: "POST",
|
|
15
|
+
body: JSON.stringify({
|
|
16
|
+
name,
|
|
17
|
+
url: `https://example.test/${name}.git`,
|
|
18
|
+
defaultBranch: "main",
|
|
19
|
+
}),
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* The inventory entry `host smoke` attaches via `attachRepository` — one worktree, named after
|
|
25
|
+
* the throwaway repository (`smoke-<hex>`), under `<repoPath>/.worktrees/<that name>`. Worktree
|
|
26
|
+
* names are one namespace across every host (`services/api/src/control-plane-worktree-names.ts`),
|
|
27
|
+
* so a fixed name would make two concurrent smokes on different hosts, or a leftover smoke
|
|
28
|
+
* worktree anywhere in the fleet, reject this attach. `repoPath` is a path on the HOST, which may be a
|
|
29
|
+
* different machine from wherever this CLI runs, so this never joins it with `node:path` (whose
|
|
30
|
+
* separator would be wrong for a remote host) or checks it exists — see `host-smoke.js`'s usage
|
|
31
|
+
* text and the README for the preconditions this command cannot verify itself: `repoPath` must
|
|
32
|
+
* already be a git repository with a clean `main` checkout, and `.worktrees/` must be gitignored
|
|
33
|
+
* there so the daemon's worktree checkout never collides with tracked files.
|
|
34
|
+
*/
|
|
35
|
+
export function smokeInventoryEntry(repository, repoPath) {
|
|
36
|
+
return {
|
|
37
|
+
id: repository.id,
|
|
38
|
+
path: repoPath,
|
|
39
|
+
defaultBranch: repository.defaultBranch ?? "main",
|
|
40
|
+
worktrees: [
|
|
41
|
+
{
|
|
42
|
+
id: repository.name,
|
|
43
|
+
name: repository.name,
|
|
44
|
+
path: `${repoPath}/.worktrees/${repository.name}`,
|
|
45
|
+
labels: [],
|
|
46
|
+
},
|
|
47
|
+
],
|
|
48
|
+
};
|
|
49
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { step } from "./host-smoke-format.js";
|
|
2
|
+
import { waitForSmokeSession } from "./host-smoke-poll.js";
|
|
3
|
+
|
|
4
|
+
// A host's cached inventory only refreshes on its own periodic poll (there is no
|
|
5
|
+
// push-on-write) — see host-smoke-provider.js's doc comment for the full explanation. Five
|
|
6
|
+
// attempts of doubling backoff (2s, 4s, 8s, 16s, capped at 16s) covers the daemon's own
|
|
7
|
+
// default 15s poll at least once while staying well inside a generous --timeout.
|
|
8
|
+
const SETUP_FAILURE_RETRY_LIMIT = 5;
|
|
9
|
+
const SETUP_FAILURE_RETRY_BASE_MS = 2_000;
|
|
10
|
+
const SETUP_FAILURE_RETRY_MAX_MS = 16_000;
|
|
11
|
+
|
|
12
|
+
function defaultSleep(ms) {
|
|
13
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** True for the one specific `setup_failed` shape thrown by
|
|
17
|
+
* `services/host-daemon/src/worktree-manager.ts` when a host's own cached inventory hasn't
|
|
18
|
+
* caught up with a just-attached repository yet. */
|
|
19
|
+
function isUnknownRepositorySetupFailure(session) {
|
|
20
|
+
return (
|
|
21
|
+
session.status === "failed" &&
|
|
22
|
+
session.errorCode === "setup_failed" &&
|
|
23
|
+
typeof session.errorMessage === "string" &&
|
|
24
|
+
session.errorMessage.startsWith("Unknown repository:")
|
|
25
|
+
);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Creates a session and waits for it, retrying up to `SETUP_FAILURE_RETRY_LIMIT` times — with
|
|
30
|
+
* exponential backoff, bounded by the same overall `timeoutSeconds` deadline as everything
|
|
31
|
+
* else — but only for that one exact, unambiguous failure signature. Every other terminal shape
|
|
32
|
+
* (a real setup/session failure, `usage_limit`, a genuine timeout, a `createSession` rejection)
|
|
33
|
+
* returns immediately, unretried. A `waitForSmokeSession` rejection (a genuine `getSession`
|
|
34
|
+
* failure — network error, 5xx — not a `UsageLimitSignal`, which it already converts) is caught
|
|
35
|
+
* here too: it is not one of the narrow shapes above, and letting it propagate would abort every
|
|
36
|
+
* remaining `--provider` in `host-smoke.js`'s loop and get misreported as a top-level
|
|
37
|
+
* `setupError`. The created session's id is deliberately left in `activeSessionIds` (never
|
|
38
|
+
* removed on this path) so `teardownSmoke`'s own safety net cancels it. Resolves to
|
|
39
|
+
* `{ kind: "create_failed", message }`, `{ kind: "session_wait_failed", sessionId, message }`, or
|
|
40
|
+
* `{ kind, session, sessionId }` where `kind` is `waitForSmokeSession`'s own
|
|
41
|
+
* `"usage_limit" | "timeout" | "terminal"`.
|
|
42
|
+
*/
|
|
43
|
+
export async function runSessionAttempts({
|
|
44
|
+
client,
|
|
45
|
+
io,
|
|
46
|
+
repositoryId,
|
|
47
|
+
providerRef,
|
|
48
|
+
target,
|
|
49
|
+
marker,
|
|
50
|
+
timeoutSeconds,
|
|
51
|
+
activeSessionIds,
|
|
52
|
+
sleep = defaultSleep,
|
|
53
|
+
now = Date.now,
|
|
54
|
+
intervalMs,
|
|
55
|
+
}) {
|
|
56
|
+
const deadline = now() + timeoutSeconds * 1000;
|
|
57
|
+
for (let attempt = 1; attempt <= SETUP_FAILURE_RETRY_LIMIT; attempt += 1) {
|
|
58
|
+
let created;
|
|
59
|
+
try {
|
|
60
|
+
created = await client.createSession({
|
|
61
|
+
repositoryId,
|
|
62
|
+
prompt: `Reply with exactly: ${marker}`,
|
|
63
|
+
target,
|
|
64
|
+
timeout: timeoutSeconds,
|
|
65
|
+
});
|
|
66
|
+
} catch (error) {
|
|
67
|
+
return { kind: "create_failed", message: error.message };
|
|
68
|
+
}
|
|
69
|
+
activeSessionIds.add(created.id);
|
|
70
|
+
step(io, true, `provider ${providerRef}: created session ${created.id}`);
|
|
71
|
+
|
|
72
|
+
let outcome;
|
|
73
|
+
try {
|
|
74
|
+
outcome = await waitForSmokeSession(client, created.id, {
|
|
75
|
+
timeoutMs: Math.max(1, deadline - now()),
|
|
76
|
+
intervalMs,
|
|
77
|
+
sleep,
|
|
78
|
+
now,
|
|
79
|
+
onStatus: (status) => io.stderr.write(` session ${created.id}: ${status}\n`),
|
|
80
|
+
});
|
|
81
|
+
} catch (error) {
|
|
82
|
+
return { kind: "session_wait_failed", sessionId: created.id, message: error.message };
|
|
83
|
+
}
|
|
84
|
+
if (outcome.kind === "terminal") activeSessionIds.delete(created.id);
|
|
85
|
+
|
|
86
|
+
const retryable =
|
|
87
|
+
outcome.kind === "terminal" && isUnknownRepositorySetupFailure(outcome.session);
|
|
88
|
+
const budgetMs = deadline - now();
|
|
89
|
+
if (!retryable || attempt === SETUP_FAILURE_RETRY_LIMIT || budgetMs <= 0) {
|
|
90
|
+
return { ...outcome, sessionId: created.id };
|
|
91
|
+
}
|
|
92
|
+
io.stderr.write(
|
|
93
|
+
` provider ${providerRef}: host has not picked up the newly attached repository yet ` +
|
|
94
|
+
`(attempt ${attempt}/${SETUP_FAILURE_RETRY_LIMIT}); retrying\n`,
|
|
95
|
+
);
|
|
96
|
+
const backoffMs = Math.min(
|
|
97
|
+
SETUP_FAILURE_RETRY_BASE_MS * 2 ** (attempt - 1),
|
|
98
|
+
SETUP_FAILURE_RETRY_MAX_MS,
|
|
99
|
+
);
|
|
100
|
+
await sleep(Math.min(backoffMs, budgetMs));
|
|
101
|
+
}
|
|
102
|
+
/* v8 ignore next 2 -- the loop above always returns before falling out */
|
|
103
|
+
return { kind: "create_failed", message: "unreachable" };
|
|
104
|
+
}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { AutoHarnessError } from "../../index.js";
|
|
2
|
+
import { pathSegment } from "../path-segment.js";
|
|
3
|
+
import { isDependencyConflict } from "./dependency-conflict.js";
|
|
4
|
+
import { detachRepository } from "./detach-repository.js";
|
|
5
|
+
import { step } from "./host-smoke-format.js";
|
|
6
|
+
|
|
7
|
+
// A dependency 409 here almost always means the worktree/host-inventory projection the delete
|
|
8
|
+
// guard reads (see services/api/src/control-plane-delete-guards.ts) hasn't caught up with the
|
|
9
|
+
// detach write this same teardown just made, or a just-cancelled session is still winding down
|
|
10
|
+
// — a few short retries clear both without a long wait. The same budget also covers a delete
|
|
11
|
+
// request that itself fails transiently (5xx, or a network/timeout error below the HTTP layer) —
|
|
12
|
+
// see `isTransientDeleteFailure` below.
|
|
13
|
+
const DELETE_ATTEMPTS = 5;
|
|
14
|
+
|
|
15
|
+
function defaultSleep(ms) {
|
|
16
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** Cancels every session still tracked as active — a per-provider timeout/usage-limit already
|
|
20
|
+
* cancels its own session and untracks it (see `host-smoke-provider.js`); what is left here is
|
|
21
|
+
* only the safety net for a session whose fate an unexpected exception left unresolved. Never
|
|
22
|
+
* throws: a cancel failure is reported as a step, the id is kept (not deleted) in
|
|
23
|
+
* `activeSessionIds`, and it comes back in `uncancelled` for the caller to report — a 409 means
|
|
24
|
+
* the session raced to terminal on its own, which is fine to drop. */
|
|
25
|
+
async function cancelOutstanding(client, io, activeSessionIds) {
|
|
26
|
+
const cancelled = [];
|
|
27
|
+
const uncancelled = [];
|
|
28
|
+
// Safe to delete the current entry mid-iteration (a Set only skips an entry deleted before
|
|
29
|
+
// it is reached); nothing is added to the set during this loop.
|
|
30
|
+
for (const sessionId of activeSessionIds) {
|
|
31
|
+
try {
|
|
32
|
+
await client.cancelSession(sessionId);
|
|
33
|
+
cancelled.push(sessionId);
|
|
34
|
+
activeSessionIds.delete(sessionId);
|
|
35
|
+
step(io, true, `teardown: cancelled session ${sessionId}`);
|
|
36
|
+
} catch (error) {
|
|
37
|
+
if (error instanceof AutoHarnessError && error.status === 409) {
|
|
38
|
+
cancelled.push(sessionId);
|
|
39
|
+
activeSessionIds.delete(sessionId);
|
|
40
|
+
step(io, true, `teardown: session ${sessionId} was already terminal`);
|
|
41
|
+
} else {
|
|
42
|
+
uncancelled.push(sessionId);
|
|
43
|
+
step(io, false, `teardown: cancel session ${sessionId} failed: ${error.message}`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return { cancelled, uncancelled };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** True for a delete failure worth retrying beyond the existing dependency-409 case: an HTTP 5xx
|
|
51
|
+
* (server-side, likely transient) or a rejection that never reached the HTTP layer at all (a
|
|
52
|
+
* network error, or `AutoHarnessRequestTimeoutError`) — neither is `AutoHarnessError`'s own 4xx
|
|
53
|
+
* shape, so both are retried like a 5xx. A 409 is intentionally *not* one of these: it is the
|
|
54
|
+
* server definitively refusing (still has live dependents), not evidence the request itself may
|
|
55
|
+
* have landed. */
|
|
56
|
+
function isTransientDeleteFailure(error) {
|
|
57
|
+
return !(error instanceof AutoHarnessError) || error.status >= 500;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** `DELETE /repositories/<id>`, retrying a dependency 409 or a transient failure
|
|
61
|
+
* (`isTransientDeleteFailure`) a few times with a short backoff (`sleep` injected so tests never
|
|
62
|
+
* really wait). Once a transient failure has actually happened, a later 404 is treated as success
|
|
63
|
+
* — the delete most likely landed and the connection dropped before the response did — rather
|
|
64
|
+
* than as a genuine "nothing to delete" failure, which a 404 with no prior transient failure
|
|
65
|
+
* still is. Any other error, or the last attempt, is returned rather than thrown — teardown must
|
|
66
|
+
* never throw (see its own doc comment). */
|
|
67
|
+
async function deleteRepositoryWithRetry(client, repositoryId, sleep) {
|
|
68
|
+
let sawTransientFailure = false;
|
|
69
|
+
for (let attempt = 1; attempt <= DELETE_ATTEMPTS; attempt += 1) {
|
|
70
|
+
try {
|
|
71
|
+
await client.request(`/repositories/${pathSegment(repositoryId, "repositoryId")}`, {
|
|
72
|
+
method: "DELETE",
|
|
73
|
+
});
|
|
74
|
+
return { ok: true };
|
|
75
|
+
} catch (error) {
|
|
76
|
+
const notFound = error instanceof AutoHarnessError && error.status === 404;
|
|
77
|
+
if (sawTransientFailure && notFound) return { ok: true };
|
|
78
|
+
const transient = isTransientDeleteFailure(error);
|
|
79
|
+
if ((!transient && !isDependencyConflict(error)) || attempt === DELETE_ATTEMPTS) {
|
|
80
|
+
return { ok: false, error };
|
|
81
|
+
}
|
|
82
|
+
if (transient) sawTransientFailure = true;
|
|
83
|
+
await sleep(Math.min(250 * attempt, 2_000));
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
/* v8 ignore next 2 -- the loop above always returns before falling out */
|
|
87
|
+
return { ok: false, error: new Error("unreachable") };
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Runs from the caller's `finally`, no matter which earlier step failed or threw. Cancels any
|
|
92
|
+
* session smoke created that never reached a terminal status on its own, detaches the
|
|
93
|
+
* repository only if it was actually attached (an attach that itself threw never wrote
|
|
94
|
+
* anything), then deletes the repository, retrying a dependency 409 or a transient failure a few
|
|
95
|
+
* times. Never throws — a teardown failure is reported through its own `ok: false` result plus a
|
|
96
|
+
* cleanup hint on stderr, so it always composes safely inside a `finally` (an exception thrown
|
|
97
|
+
* from a `finally` would replace whatever error the `try` was already failing with). `ok` is
|
|
98
|
+
* false whenever the repository is left behind (a failed detach or delete) **or** any session
|
|
99
|
+
* failed to cancel — a still-live session blocks the delete guard, so an uncancelled session and
|
|
100
|
+
* a leftover repository often go together, but each is reported (and each gets its own cleanup
|
|
101
|
+
* command) independently of the other.
|
|
102
|
+
*/
|
|
103
|
+
export async function teardownSmoke({
|
|
104
|
+
client,
|
|
105
|
+
io,
|
|
106
|
+
hostId,
|
|
107
|
+
repositoryId,
|
|
108
|
+
attached,
|
|
109
|
+
activeSessionIds,
|
|
110
|
+
sleep = defaultSleep,
|
|
111
|
+
}) {
|
|
112
|
+
const { cancelled: cancelledSessionIds, uncancelled: uncancelledSessionIds } =
|
|
113
|
+
await cancelOutstanding(client, io, activeSessionIds);
|
|
114
|
+
|
|
115
|
+
let detached = false;
|
|
116
|
+
let detachError;
|
|
117
|
+
if (attached) {
|
|
118
|
+
try {
|
|
119
|
+
await detachRepository(client, hostId, repositoryId);
|
|
120
|
+
detached = true;
|
|
121
|
+
step(io, true, `teardown: detached repository ${repositoryId} from host ${hostId}`);
|
|
122
|
+
} catch (error) {
|
|
123
|
+
detachError = error;
|
|
124
|
+
step(io, false, `teardown: detach repository ${repositoryId} failed: ${error.message}`);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const deleteResult = await deleteRepositoryWithRetry(client, repositoryId, sleep);
|
|
129
|
+
if (deleteResult.ok) {
|
|
130
|
+
step(io, true, `teardown: deleted repository ${repositoryId}`);
|
|
131
|
+
} else {
|
|
132
|
+
step(
|
|
133
|
+
io,
|
|
134
|
+
false,
|
|
135
|
+
`teardown: delete repository ${repositoryId} failed: ${deleteResult.error.message}`,
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const repositoryLeftover = Boolean(detachError) || !deleteResult.ok;
|
|
140
|
+
const ok = !repositoryLeftover && uncancelledSessionIds.length === 0;
|
|
141
|
+
if (!ok) {
|
|
142
|
+
for (const sessionId of uncancelledSessionIds) {
|
|
143
|
+
io.stderr.write(
|
|
144
|
+
`leftover session ${sessionId} — finish cleanup with:\n` +
|
|
145
|
+
` auto-harness session cancel ${sessionId}\n`,
|
|
146
|
+
);
|
|
147
|
+
}
|
|
148
|
+
if (repositoryLeftover) {
|
|
149
|
+
io.stderr.write(
|
|
150
|
+
`leftover repository ${repositoryId} — finish cleanup with:\n` +
|
|
151
|
+
` auto-harness host repo rm ${hostId} ${repositoryId}\n` +
|
|
152
|
+
` auto-harness repo rm ${repositoryId}\n`,
|
|
153
|
+
);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return {
|
|
157
|
+
ok,
|
|
158
|
+
cancelledSessionIds,
|
|
159
|
+
uncancelledSessionIds,
|
|
160
|
+
detached,
|
|
161
|
+
repositoryDeleted: deleteResult.ok,
|
|
162
|
+
...(repositoryLeftover ? { leftoverRepositoryId: repositoryId } : {}),
|
|
163
|
+
};
|
|
164
|
+
}
|