@mnemom/mnemom 0.17.32-next.4 → 0.17.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/agent-fuel.d.ts +9 -0
- package/dist/commands/agent-fuel.js +36 -2
- package/dist/commands/agent-run.d.ts +50 -0
- package/dist/commands/agent-run.js +221 -0
- package/dist/commands/agent-setup.d.ts +8 -3
- package/dist/commands/agent-setup.js +32 -9
- package/dist/commands/agent-share-fixture.d.ts +3 -1
- package/dist/commands/agent-share-fixture.js +11 -7
- package/dist/commands/agent-share-render.d.ts +23 -0
- package/dist/commands/agent-share-render.js +72 -0
- package/dist/commands/agent-share.d.ts +15 -2
- package/dist/commands/agent-share.js +30 -9
- package/dist/commands/agent.d.ts +19 -4
- package/dist/commands/agent.js +33 -13
- package/dist/index.js +20 -0
- package/dist/lib/api.js +17 -1
- package/dist/lib/auth.d.ts +6 -0
- package/dist/lib/auth.js +161 -14
- package/dist/lib/experimental.d.ts +1 -1
- package/dist/lib/experimental.js +13 -0
- package/dist/lib/oauth.d.ts +7 -0
- package/dist/lib/oauth.js +13 -0
- package/dist/lib/runner/bundle.d.ts +129 -0
- package/dist/lib/runner/bundle.js +319 -0
- package/dist/lib/runner/contract-header.d.ts +46 -0
- package/dist/lib/runner/contract-header.js +85 -0
- package/dist/lib/runner/executor-client.d.ts +71 -0
- package/dist/lib/runner/executor-client.js +97 -0
- package/dist/lib/runner/job.d.ts +70 -0
- package/dist/lib/runner/job.js +463 -0
- package/dist/lib/runner/loop.d.ts +45 -0
- package/dist/lib/runner/loop.js +104 -0
- package/dist/lib/runner/oidc.d.ts +38 -0
- package/dist/lib/runner/oidc.js +89 -0
- package/dist/lib/runner/options.d.ts +41 -0
- package/dist/lib/runner/options.js +47 -0
- package/dist/lib/runner/outcome.d.ts +47 -0
- package/dist/lib/runner/outcome.js +89 -0
- package/dist/lib/runner/phases.d.ts +49 -0
- package/dist/lib/runner/phases.js +99 -0
- package/dist/lib/runner/proxy.d.ts +45 -0
- package/dist/lib/runner/proxy.js +153 -0
- package/dist/lib/runner/settings.d.ts +31 -0
- package/dist/lib/runner/settings.js +44 -0
- package/dist/lib/statusline-feed.js +10 -11
- package/dist/lib/statusline.d.ts +6 -0
- package/dist/lib/statusline.js +19 -9
- package/dist/rc-proxy.mjs +69 -5
- package/package.json +2 -1
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The thin runner's executor client (ruling 34.12 (7)): S1's executor routes, the same protocol as
|
|
3
|
+
* C2's host client, plus the two post-claim reads outside S1 (the job, W-L15; the served bundle,
|
|
4
|
+
* W-L12) and the JWKS. Every executor route carries the executor key in X-Mnemom-Api-Key and
|
|
5
|
+
* never Authorization (ruling 33.28 (1)), and so does the job read (api#4165 authorizes the executor
|
|
6
|
+
* holding the lease); the bundle fetch carries the lease key the same way.
|
|
7
|
+
*
|
|
8
|
+
* Nothing here decides an outcome: each call returns its status, body and error code, and the job
|
|
9
|
+
* (job.ts) maps them to 34.12's reasons. Never throws: a network failure is status 0.
|
|
10
|
+
*/
|
|
11
|
+
import { EXECUTOR_KEY_HEADER, routePath } from "../../vendor/durable-contract/index.js";
|
|
12
|
+
export function errorCode(body) {
|
|
13
|
+
if (!body || typeof body !== "object")
|
|
14
|
+
return null;
|
|
15
|
+
const b = body;
|
|
16
|
+
if (b.error && typeof b.error === "object") {
|
|
17
|
+
const c = b.error.code;
|
|
18
|
+
if (typeof c === "string")
|
|
19
|
+
return c;
|
|
20
|
+
}
|
|
21
|
+
if (typeof b.error === "string")
|
|
22
|
+
return b.error;
|
|
23
|
+
return typeof b.code === "string" ? b.code : null;
|
|
24
|
+
}
|
|
25
|
+
export function errorDetails(body) {
|
|
26
|
+
if (!body || typeof body !== "object")
|
|
27
|
+
return {};
|
|
28
|
+
const e = body.error;
|
|
29
|
+
const d = e && typeof e === "object" ? e.details : undefined;
|
|
30
|
+
return d && typeof d === "object" && !Array.isArray(d) ? d : {};
|
|
31
|
+
}
|
|
32
|
+
/** The post-claim job read (W-L15), with the executor key. */
|
|
33
|
+
export const JOB_READ_PATH = (sessionId) => `/v1/executor/sessions/${sessionId}/job`;
|
|
34
|
+
/** The served bundle fetch, per claim (W-L12). */
|
|
35
|
+
export const BUNDLE_FETCH_PATH = (sessionId) => `/v1/executor/sessions/${sessionId}/bundle`;
|
|
36
|
+
export const JWKS_PATH = "/v1/.well-known/jwks.json";
|
|
37
|
+
const SESSION_ID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
38
|
+
export function runnerApi(opts) {
|
|
39
|
+
const base = opts.apiUrl.replace(/\/+$/, "");
|
|
40
|
+
const call = async (path, method, key, body) => {
|
|
41
|
+
const ctl = new AbortController();
|
|
42
|
+
const timer = setTimeout(() => ctl.abort(), opts.timeoutMs ?? 15_000);
|
|
43
|
+
const headers = { accept: "application/json" };
|
|
44
|
+
if (key !== null)
|
|
45
|
+
headers[EXECUTOR_KEY_HEADER] = key;
|
|
46
|
+
if (body !== undefined)
|
|
47
|
+
headers["content-type"] = "application/json";
|
|
48
|
+
try {
|
|
49
|
+
const res = await opts.fetch(`${base}${path}`, {
|
|
50
|
+
method,
|
|
51
|
+
headers,
|
|
52
|
+
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
|
53
|
+
signal: ctl.signal,
|
|
54
|
+
});
|
|
55
|
+
const text = await res.text();
|
|
56
|
+
let parsed = null;
|
|
57
|
+
try {
|
|
58
|
+
parsed = text ? JSON.parse(text) : null;
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
parsed = text;
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
status: res.status,
|
|
65
|
+
body: parsed,
|
|
66
|
+
text,
|
|
67
|
+
code: res.status >= 400 ? errorCode(parsed) : null,
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
catch (e) {
|
|
71
|
+
return { status: 0, body: null, text: "", code: null, networkError: e.message };
|
|
72
|
+
}
|
|
73
|
+
finally {
|
|
74
|
+
clearTimeout(timer);
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
// A session id from the api goes into a path; refuse anything but a lowercase uuid.
|
|
78
|
+
const scoped = (id) => {
|
|
79
|
+
if (!SESSION_ID_RE.test(id))
|
|
80
|
+
throw new RangeError("session id must be a lowercase uuid");
|
|
81
|
+
return id;
|
|
82
|
+
};
|
|
83
|
+
const ex = opts.executorKey;
|
|
84
|
+
return {
|
|
85
|
+
readyList: (limit) => call(`${routePath("exec_ready_list")}?limit=${limit}`, "GET", ex),
|
|
86
|
+
attest: (id, nonce) => call(routePath("exec_attest", scoped(id)), "POST", ex, { nonce }),
|
|
87
|
+
claim: (id, body) => call(routePath("exec_claim", scoped(id)), "POST", ex, body),
|
|
88
|
+
append: (id, body) => call(routePath("exec_append", scoped(id)), "POST", ex, body),
|
|
89
|
+
hello: (id, body) => call(routePath("exec_hello", scoped(id)), "POST", ex, body),
|
|
90
|
+
heartbeat: (body) => call(routePath("exec_heartbeat"), "POST", ex, body),
|
|
91
|
+
release: (id, body) => call(routePath("exec_release", scoped(id)), "POST", ex, body),
|
|
92
|
+
fail: (id, body) => call(routePath("exec_fail", scoped(id)), "POST", ex, body),
|
|
93
|
+
job: (id) => call(JOB_READ_PATH(scoped(id)), "GET", ex),
|
|
94
|
+
bundle: (id, leaseKey) => call(BUNDLE_FETCH_PATH(scoped(id)), "GET", leaseKey),
|
|
95
|
+
jwks: () => call(JWKS_PATH, "GET", null),
|
|
96
|
+
};
|
|
97
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One served-phase job, claim to outcome (ruling 34.12; Doors W-L13, W3.11 and W4.4).
|
|
3
|
+
*
|
|
4
|
+
* attest → claim (needs_key retry) → register the spawn → hello → heartbeats → served bundle
|
|
5
|
+
* (verified, bound to this claim) → job read (contract bytes bound to the claim's seal) → repo →
|
|
6
|
+
* phases through the runner's own proxy → PR → release. Every Mnemom error maps to one of S1's
|
|
7
|
+
* refused reasons and fails closed; under --fail-mode open, only bundle_unavailable,
|
|
8
|
+
* job_unavailable and api_unavailable are ridden through, and only once a verified bundle is held
|
|
9
|
+
* (34.12 (2)). A bundle that fails verification stops the job in every mode.
|
|
10
|
+
*
|
|
11
|
+
* Every side effect is in `JobDeps`, so each arm is tested; `liveJobDeps` (commands/agent-run.ts)
|
|
12
|
+
* is the thin driver.
|
|
13
|
+
*/
|
|
14
|
+
import { type HttpResult, type RunnerApi } from "./executor-client.js";
|
|
15
|
+
import { type OutcomeFields, type RefusedReason } from "./outcome.js";
|
|
16
|
+
import { type PhaseInvocation } from "./phases.js";
|
|
17
|
+
import type { ModelProxy, ProxyState } from "./proxy.js";
|
|
18
|
+
import { type RunnerFailMode } from "./settings.js";
|
|
19
|
+
/** A claimed job's working copy. */
|
|
20
|
+
export interface Workspace {
|
|
21
|
+
dir: string;
|
|
22
|
+
/** A fresh, empty HOME for the phases. */
|
|
23
|
+
home: string;
|
|
24
|
+
branch: string;
|
|
25
|
+
}
|
|
26
|
+
export interface JobDeps {
|
|
27
|
+
api: RunnerApi;
|
|
28
|
+
/** One nonce per runner process, reused on every call (A3 fences on it). */
|
|
29
|
+
nonce: string;
|
|
30
|
+
newSpawnId(): string;
|
|
31
|
+
now(): number;
|
|
32
|
+
sleep(ms: number): Promise<void>;
|
|
33
|
+
log(line: string): void;
|
|
34
|
+
gatewayUrl: string;
|
|
35
|
+
startProxy(state: ProxyState): Promise<ModelProxy>;
|
|
36
|
+
prepareWorkspace(repo: string, sessionId: string): Promise<Workspace>;
|
|
37
|
+
/** Run one phase's process; resolves its exit code. Rejects only when it could not start. */
|
|
38
|
+
runPhase(inv: PhaseInvocation, ws: Workspace, modelUrl: string, signal: AbortSignal): Promise<number>;
|
|
39
|
+
/** Push the job branch and open its PR; resolves the PR URL. */
|
|
40
|
+
openPr(repo: string, ws: Workspace, title: string): Promise<string>;
|
|
41
|
+
enableAutoMerge(prUrl: string): Promise<void>;
|
|
42
|
+
cleanup(ws: Workspace): Promise<void>;
|
|
43
|
+
}
|
|
44
|
+
export interface JobOptions {
|
|
45
|
+
failMode: RunnerFailMode;
|
|
46
|
+
/** Aborted by SIGTERM or SIGINT: release the lease, print no outcome line. */
|
|
47
|
+
stop: AbortSignal;
|
|
48
|
+
}
|
|
49
|
+
/** What one claim attempt came to. */
|
|
50
|
+
export type JobResult = {
|
|
51
|
+
kind: "next";
|
|
52
|
+
} | {
|
|
53
|
+
kind: "outcome";
|
|
54
|
+
outcome: OutcomeFields;
|
|
55
|
+
} | {
|
|
56
|
+
kind: "stopped";
|
|
57
|
+
};
|
|
58
|
+
/** A3's executor auth rejections, on whichever executor route comes first (34.12 (8), (10)). */
|
|
59
|
+
export declare function authRefusal(r: HttpResult): RefusedReason | null;
|
|
60
|
+
/**
|
|
61
|
+
* Run one candidate session to an outcome. Never throws for a Mnemom error: every one is an
|
|
62
|
+
* outcome. Throws only on a runner bug.
|
|
63
|
+
*/
|
|
64
|
+
export declare function runJob(sessionId: string, opts: JobOptions, deps: JobDeps): Promise<JobResult>;
|
|
65
|
+
/**
|
|
66
|
+
* Report a failed or refused job after claim (34.12 (6), (10)). The fail reason vocabulary is the
|
|
67
|
+
* Durable lead's to rule (A3 accepts only transcript_lost today); a refused report is logged and
|
|
68
|
+
* never changes the outcome, because the lease lapses on its own.
|
|
69
|
+
*/
|
|
70
|
+
export declare function reportFail(deps: JobDeps, sessionId: string, epoch: number, kind: "runner_failed" | "runner_refused", reason: string): Promise<void>;
|
|
@@ -0,0 +1,463 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One served-phase job, claim to outcome (ruling 34.12; Doors W-L13, W3.11 and W4.4).
|
|
3
|
+
*
|
|
4
|
+
* attest → claim (needs_key retry) → register the spawn → hello → heartbeats → served bundle
|
|
5
|
+
* (verified, bound to this claim) → job read (contract bytes bound to the claim's seal) → repo →
|
|
6
|
+
* phases through the runner's own proxy → PR → release. Every Mnemom error maps to one of S1's
|
|
7
|
+
* refused reasons and fails closed; under --fail-mode open, only bundle_unavailable,
|
|
8
|
+
* job_unavailable and api_unavailable are ridden through, and only once a verified bundle is held
|
|
9
|
+
* (34.12 (2)). A bundle that fails verification stops the job in every mode.
|
|
10
|
+
*
|
|
11
|
+
* Every side effect is in `JobDeps`, so each arm is tested; `liveJobDeps` (commands/agent-run.ts)
|
|
12
|
+
* is the thin driver.
|
|
13
|
+
*/
|
|
14
|
+
import { RUNNER_OPEN_MODE_TOLERATED, claimNeedsKeyRetry, } from "../../vendor/durable-contract/index.js";
|
|
15
|
+
import { canonicalJson } from "../canonical-json.js";
|
|
16
|
+
import { verifyServedPhaseBundle } from "./bundle.js";
|
|
17
|
+
import { sealedContractHeader } from "./contract-header.js";
|
|
18
|
+
import { errorCode, errorDetails } from "./executor-client.js";
|
|
19
|
+
import { doneOutcome, failedOutcome, phaseReason, refusedOutcome, } from "./outcome.js";
|
|
20
|
+
import { planPhase } from "./phases.js";
|
|
21
|
+
import { resolveMergeStrategy } from "./settings.js";
|
|
22
|
+
const TOLERATED = RUNNER_OPEN_MODE_TOLERATED;
|
|
23
|
+
const OPEN_MODE_RETRIES = 3;
|
|
24
|
+
const POOL_SKIP_INELIGIBLE = new Set(["waiting", "not_pool_member"]);
|
|
25
|
+
/** A3's executor auth rejections, on whichever executor route comes first (34.12 (8), (10)). */
|
|
26
|
+
export function authRefusal(r) {
|
|
27
|
+
if (r.status === 401 && r.code === "durable_key_revoked")
|
|
28
|
+
return "executor_revoked";
|
|
29
|
+
if (r.status === 401 && r.code === "unauthorized")
|
|
30
|
+
return "executor_unauthorized";
|
|
31
|
+
if (r.status === 403 && r.code === "executor_key_required")
|
|
32
|
+
return "executor_unauthorized";
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
/** Any other executor route failing: auth first, then api_unavailable. */
|
|
36
|
+
function routeRefusal(r) {
|
|
37
|
+
return (authRefusal(r) ??
|
|
38
|
+
(r.status === 409 && r.code === "durable_lease_fenced" ? "lease_lost" : "api_unavailable"));
|
|
39
|
+
}
|
|
40
|
+
/** A post-claim read: the bundle fetch with the lease key, the job read with the executor key. */
|
|
41
|
+
function leaseReadRefusal(r, unavailable) {
|
|
42
|
+
if (r.status === 200)
|
|
43
|
+
return null;
|
|
44
|
+
if (r.status === 401 || r.status === 403)
|
|
45
|
+
return "lease_lost";
|
|
46
|
+
return unavailable;
|
|
47
|
+
}
|
|
48
|
+
const ok2xx = (r) => r.status >= 200 && r.status < 300;
|
|
49
|
+
const num = (v) => (Number.isSafeInteger(v) ? v : null);
|
|
50
|
+
const str = (v) => (typeof v === "string" && v !== "" ? v : null);
|
|
51
|
+
async function claimSession(deps, sessionId) {
|
|
52
|
+
const at = await deps.api.attest(sessionId, deps.nonce);
|
|
53
|
+
const atRefusal = authRefusal(at);
|
|
54
|
+
if (atRefusal)
|
|
55
|
+
return { kind: "refused", reason: atRefusal, detail: `attest: ${at.code}` };
|
|
56
|
+
if (at.status === 0 || at.status >= 500) {
|
|
57
|
+
return {
|
|
58
|
+
kind: "refused",
|
|
59
|
+
reason: "api_unavailable",
|
|
60
|
+
detail: `attest: ${at.networkError ?? at.status}`,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
// A 404 attest is a session this executor cannot see (taken, or not its org's): try the next.
|
|
64
|
+
if (at.status === 404)
|
|
65
|
+
return { kind: "next" };
|
|
66
|
+
const body = { nonce: deps.nonce };
|
|
67
|
+
let r = await deps.api.claim(sessionId, body);
|
|
68
|
+
const retry = claimNeedsKeyRetry(body, r.code, false);
|
|
69
|
+
if (retry)
|
|
70
|
+
r = await deps.api.claim(sessionId, retry);
|
|
71
|
+
const auth = authRefusal(r);
|
|
72
|
+
if (auth)
|
|
73
|
+
return { kind: "refused", reason: auth, detail: `claim: ${r.code}` };
|
|
74
|
+
if (r.status === 0)
|
|
75
|
+
return { kind: "refused", reason: "api_unavailable", detail: `claim: ${r.networkError}` };
|
|
76
|
+
if (r.status === 503 && r.code === "executor_lookup_unavailable") {
|
|
77
|
+
return {
|
|
78
|
+
kind: "refused",
|
|
79
|
+
reason: "api_unavailable",
|
|
80
|
+
detail: "claim: executor lookup unavailable",
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
const outcome = r.body?.outcome;
|
|
84
|
+
if (r.status === 409 && (outcome === "lost_race" || r.code === "durable_claim_lost_race"))
|
|
85
|
+
return { kind: "next" };
|
|
86
|
+
if (r.status === 403 && (outcome === "ineligible" || r.code === "durable_claim_ineligible")) {
|
|
87
|
+
const reason = String(errorDetails(r.body).reason ?? "");
|
|
88
|
+
if (POOL_SKIP_INELIGIBLE.has(reason))
|
|
89
|
+
return { kind: "next" };
|
|
90
|
+
return {
|
|
91
|
+
kind: "refused",
|
|
92
|
+
reason: "claim_ineligible",
|
|
93
|
+
detail: `claim ineligible: ${reason || "?"}`,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
if (r.status === 409 && r.code === "durable_claim_needs_key") {
|
|
97
|
+
return {
|
|
98
|
+
kind: "refused",
|
|
99
|
+
reason: "claim_needs_key",
|
|
100
|
+
detail: "claim still needs a key after new_key",
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
if (r.status !== 200) {
|
|
104
|
+
const cause = errorDetails(r.body).cause;
|
|
105
|
+
return {
|
|
106
|
+
kind: "refused",
|
|
107
|
+
reason: "claim_unavailable",
|
|
108
|
+
detail: `claim: ${r.code ?? r.status} ${cause ?? ""}`.trim(),
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
const b = (r.body ?? {});
|
|
112
|
+
const epoch = num(b.epoch);
|
|
113
|
+
const leaseKey = str(b.lease_key);
|
|
114
|
+
const sha = str(b.contract_sha256);
|
|
115
|
+
const version = num(b.contract_version);
|
|
116
|
+
if (epoch === null || leaseKey === null || sha === null || version === null) {
|
|
117
|
+
// A pool claim always answers its contract; a homed one, or one with no lease key, is not a job.
|
|
118
|
+
return {
|
|
119
|
+
kind: "refused",
|
|
120
|
+
reason: "claim_unavailable",
|
|
121
|
+
detail: "claim answer has no lease key or contract",
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
return {
|
|
125
|
+
kind: "claimed",
|
|
126
|
+
claim: {
|
|
127
|
+
epoch,
|
|
128
|
+
leaseKey,
|
|
129
|
+
leaseSeconds: num(b.lease_seconds) ?? 60,
|
|
130
|
+
contractSha256: sha,
|
|
131
|
+
contractVersion: version,
|
|
132
|
+
executorId: str(b.executor_id),
|
|
133
|
+
},
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
function parseJob(body) {
|
|
137
|
+
if (!body || typeof body !== "object")
|
|
138
|
+
return null;
|
|
139
|
+
const b = body;
|
|
140
|
+
const repo = str(b.repo);
|
|
141
|
+
const sha = str(b.contract_sha256);
|
|
142
|
+
const version = num(b.version);
|
|
143
|
+
if (repo === null || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repo))
|
|
144
|
+
return null;
|
|
145
|
+
// owner/name only: a dot-only segment ("..") would walk out of the workspace root.
|
|
146
|
+
if (repo.split("/").some((seg) => /^\.+$/.test(seg)))
|
|
147
|
+
return null;
|
|
148
|
+
if (sha === null || version === null || typeof b.contract !== "string")
|
|
149
|
+
return null;
|
|
150
|
+
return { repo, contract_sha256: sha, version, contract: b.contract };
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Run one candidate session to an outcome. Never throws for a Mnemom error: every one is an
|
|
154
|
+
* outcome. Throws only on a runner bug.
|
|
155
|
+
*/
|
|
156
|
+
export async function runJob(sessionId, opts, deps) {
|
|
157
|
+
const facts = {};
|
|
158
|
+
const claimed = await claimSession(deps, sessionId);
|
|
159
|
+
if (claimed.kind === "next")
|
|
160
|
+
return claimed;
|
|
161
|
+
if (claimed.kind === "refused") {
|
|
162
|
+
deps.log(`refused: ${claimed.detail}`);
|
|
163
|
+
return { kind: "outcome", outcome: refusedOutcome(facts, claimed.reason) };
|
|
164
|
+
}
|
|
165
|
+
const claim = claimed.claim;
|
|
166
|
+
Object.assign(facts, {
|
|
167
|
+
session_id: sessionId,
|
|
168
|
+
lease_epoch: claim.epoch,
|
|
169
|
+
contract_sha256: claim.contractSha256,
|
|
170
|
+
contract_version: claim.contractVersion,
|
|
171
|
+
});
|
|
172
|
+
let bundleHeld = false;
|
|
173
|
+
let leaseLost = false;
|
|
174
|
+
let ws = null;
|
|
175
|
+
let proxy = null;
|
|
176
|
+
let heartbeat = null;
|
|
177
|
+
const phaseAbort = new AbortController();
|
|
178
|
+
const onStop = () => phaseAbort.abort();
|
|
179
|
+
opts.stop.addEventListener("abort", onStop);
|
|
180
|
+
const spawnId = deps.newSpawnId();
|
|
181
|
+
const tolerated = (reason) => opts.failMode === "open" && bundleHeld && TOLERATED.includes(reason);
|
|
182
|
+
// A stop or a fenced lease is honoured between every step, not only between phases.
|
|
183
|
+
const interrupted = async () => {
|
|
184
|
+
if (opts.stop.aborted)
|
|
185
|
+
return await stopJob(deps, sessionId, claim.epoch);
|
|
186
|
+
if (leaseLost)
|
|
187
|
+
return await refuse("lease_lost", "the lease was fenced mid-job");
|
|
188
|
+
return null;
|
|
189
|
+
};
|
|
190
|
+
const refuse = async (reason, detail) => {
|
|
191
|
+
deps.log(`refused (${reason}): ${detail}`);
|
|
192
|
+
await reportFail(deps, sessionId, claim.epoch, "runner_refused", reason);
|
|
193
|
+
return { kind: "outcome", outcome: refusedOutcome(facts, reason) };
|
|
194
|
+
};
|
|
195
|
+
try {
|
|
196
|
+
// Register the spawn, then hello: the session moves to running and the lease is ours.
|
|
197
|
+
const ap = await deps.api.append(sessionId, {
|
|
198
|
+
kind: "executor.spawned",
|
|
199
|
+
detail: { spawn_id: spawnId },
|
|
200
|
+
expected_epoch: claim.epoch,
|
|
201
|
+
});
|
|
202
|
+
if (!ok2xx(ap))
|
|
203
|
+
return await refuse(routeRefusal(ap), `register spawn: ${ap.code ?? ap.status}`);
|
|
204
|
+
const hello = await deps.api.hello(sessionId, {
|
|
205
|
+
epoch: claim.epoch,
|
|
206
|
+
nonce: deps.nonce,
|
|
207
|
+
spawn_id: spawnId,
|
|
208
|
+
});
|
|
209
|
+
if (!ok2xx(hello))
|
|
210
|
+
return await refuse(routeRefusal(hello), `hello: ${hello.code ?? hello.status}`);
|
|
211
|
+
const periodMs = Math.max(1, Math.floor(claim.leaseSeconds / 3)) * 1000;
|
|
212
|
+
heartbeat = setInterval(() => {
|
|
213
|
+
void deps.api
|
|
214
|
+
.heartbeat({
|
|
215
|
+
nonce: deps.nonce,
|
|
216
|
+
entries: [{ session_id: sessionId, epoch: claim.epoch, spawn_id: spawnId }],
|
|
217
|
+
})
|
|
218
|
+
.then((r) => {
|
|
219
|
+
const res = r.body
|
|
220
|
+
?.results?.[0];
|
|
221
|
+
if (r.status === 200 && res && res.extended === false) {
|
|
222
|
+
leaseLost = true;
|
|
223
|
+
deps.log(`heartbeat: lease not extended (${String(res.reason)})`);
|
|
224
|
+
phaseAbort.abort();
|
|
225
|
+
}
|
|
226
|
+
else if (r.status !== 200 && !tolerated("api_unavailable")) {
|
|
227
|
+
leaseLost = true;
|
|
228
|
+
deps.log(`heartbeat failed: ${r.code ?? r.networkError ?? r.status}`);
|
|
229
|
+
phaseAbort.abort();
|
|
230
|
+
}
|
|
231
|
+
});
|
|
232
|
+
}, periodMs);
|
|
233
|
+
// The served bundle: fetched per claim with the lease key, verified, bound to this claim.
|
|
234
|
+
const fetched = await fetchBundle(deps, sessionId, claim);
|
|
235
|
+
if (fetched.kind === "refused")
|
|
236
|
+
return await refuse(fetched.reason, fetched.detail);
|
|
237
|
+
const bundle = fetched.bundle;
|
|
238
|
+
bundleHeld = true;
|
|
239
|
+
facts.merge_strategy = resolveMergeStrategy(bundle);
|
|
240
|
+
const afterBundle = await interrupted();
|
|
241
|
+
if (afterBundle)
|
|
242
|
+
return afterBundle;
|
|
243
|
+
// The job and its contract bytes, bound to the claim's seal.
|
|
244
|
+
let job = null;
|
|
245
|
+
for (let attempt = 0;; attempt++) {
|
|
246
|
+
const jr = await deps.api.job(sessionId);
|
|
247
|
+
// The executor key's own rejection first; any other 401/403 is the lease.
|
|
248
|
+
const refusal = authRefusal(jr) ?? leaseReadRefusal(jr, "job_unavailable");
|
|
249
|
+
if (!refusal) {
|
|
250
|
+
job = parseJob(jr.body);
|
|
251
|
+
if (!job)
|
|
252
|
+
return await refuse("job_unavailable", "job read answered an unreadable job");
|
|
253
|
+
break;
|
|
254
|
+
}
|
|
255
|
+
if (!tolerated(refusal) || attempt >= OPEN_MODE_RETRIES) {
|
|
256
|
+
return await refuse(refusal, `job read: ${jr.code ?? jr.networkError ?? jr.status}`);
|
|
257
|
+
}
|
|
258
|
+
deps.log(`job read: ${refusal}, retrying (fail mode open)`);
|
|
259
|
+
await deps.sleep(1000 * 2 ** attempt);
|
|
260
|
+
const afterSleep = await interrupted();
|
|
261
|
+
if (afterSleep)
|
|
262
|
+
return afterSleep;
|
|
263
|
+
}
|
|
264
|
+
if (job.contract_sha256 !== claim.contractSha256 || job.version !== claim.contractVersion) {
|
|
265
|
+
return await refuse("contract_sha_mismatch", "the job's contract is not the claim's seal");
|
|
266
|
+
}
|
|
267
|
+
const header = sealedContractHeader(Buffer.from(job.contract, "utf8"), claim.contractSha256);
|
|
268
|
+
if (!header.ok)
|
|
269
|
+
return await refuse(`contract_${header.reason}`, header.detail);
|
|
270
|
+
// The working copy, the proxy, then each phase in the bundle's order.
|
|
271
|
+
const phases = [...bundle.phases].sort((a, b) => a.order - b.order);
|
|
272
|
+
try {
|
|
273
|
+
ws = await deps.prepareWorkspace(job.repo, sessionId);
|
|
274
|
+
}
|
|
275
|
+
catch (e) {
|
|
276
|
+
// No working copy: the first phase could not start, as when its process cannot.
|
|
277
|
+
const reason = phaseReason(phases[0].order, phases[0].name);
|
|
278
|
+
deps.log(`preparing the working copy of ${job.repo} failed: ${e.message}`);
|
|
279
|
+
await reportFail(deps, sessionId, claim.epoch, "runner_failed", reason);
|
|
280
|
+
return { kind: "outcome", outcome: failedOutcome(facts, reason) };
|
|
281
|
+
}
|
|
282
|
+
const state = { contractValue: header.value, leaseKey: claim.leaseKey };
|
|
283
|
+
proxy = await deps.startProxy(state);
|
|
284
|
+
let opensPr = false;
|
|
285
|
+
for (const phase of phases) {
|
|
286
|
+
const planned = planPhase(phase);
|
|
287
|
+
if (!planned.ok) {
|
|
288
|
+
deps.log(`phase ${phase.order} ${phase.name}: ${planned.detail}`);
|
|
289
|
+
await reportFail(deps, sessionId, claim.epoch, "runner_failed", phaseReason(phase.order, phase.name));
|
|
290
|
+
return {
|
|
291
|
+
kind: "outcome",
|
|
292
|
+
outcome: failedOutcome(facts, phaseReason(phase.order, phase.name)),
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
opensPr ||= planned.opensPr;
|
|
296
|
+
let amended = false;
|
|
297
|
+
for (;;) {
|
|
298
|
+
let code;
|
|
299
|
+
try {
|
|
300
|
+
code = await deps.runPhase(planned.invocation, ws, proxy.url, phaseAbort.signal);
|
|
301
|
+
}
|
|
302
|
+
catch (e) {
|
|
303
|
+
deps.log(`phase ${phase.order} ${phase.name} did not start: ${e.message}`);
|
|
304
|
+
code = -1;
|
|
305
|
+
}
|
|
306
|
+
const afterPhase = await interrupted();
|
|
307
|
+
if (afterPhase)
|
|
308
|
+
return afterPhase;
|
|
309
|
+
if (proxy.contractMismatch && !amended) {
|
|
310
|
+
// 34.11: the charter moved. Continue only on a higher sealed version with both the job
|
|
311
|
+
// and a fresh bundle bound to it; then the phase runs again under the new seal.
|
|
312
|
+
const next = await amendedSeal(deps, sessionId, claim, bundle);
|
|
313
|
+
if (!next)
|
|
314
|
+
return await refuse("contract_mismatch_unrecoverable", "the amended seal did not check out");
|
|
315
|
+
state.contractValue = next.value;
|
|
316
|
+
Object.assign(claim, { contractSha256: next.sha, contractVersion: next.version });
|
|
317
|
+
Object.assign(facts, { contract_sha256: next.sha, contract_version: next.version });
|
|
318
|
+
proxy.contractMismatch = null;
|
|
319
|
+
amended = true;
|
|
320
|
+
continue;
|
|
321
|
+
}
|
|
322
|
+
if (proxy.contractMismatch) {
|
|
323
|
+
return await refuse("contract_mismatch_unrecoverable", "the gateway refused the amended seal too");
|
|
324
|
+
}
|
|
325
|
+
if (code !== 0) {
|
|
326
|
+
const reason = phaseReason(phase.order, phase.name);
|
|
327
|
+
deps.log(`phase ${phase.order} ${phase.name} exited ${code}`);
|
|
328
|
+
await reportFail(deps, sessionId, claim.epoch, "runner_failed", reason);
|
|
329
|
+
return { kind: "outcome", outcome: failedOutcome(facts, reason) };
|
|
330
|
+
}
|
|
331
|
+
break;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
if (opensPr) {
|
|
335
|
+
try {
|
|
336
|
+
facts.pr_url = await deps.openPr(job.repo, ws, `mnemom: ${bundle.workflow.id} job ${sessionId.slice(0, 8)}`);
|
|
337
|
+
}
|
|
338
|
+
catch (e) {
|
|
339
|
+
deps.log(`opening the PR failed: ${e.message}`);
|
|
340
|
+
await reportFail(deps, sessionId, claim.epoch, "runner_failed", "pr_open_failed");
|
|
341
|
+
return { kind: "outcome", outcome: failedOutcome(facts, "pr_open_failed") };
|
|
342
|
+
}
|
|
343
|
+
if (facts.merge_strategy === "merge_on_green") {
|
|
344
|
+
try {
|
|
345
|
+
await deps.enableAutoMerge(facts.pr_url);
|
|
346
|
+
}
|
|
347
|
+
catch (e) {
|
|
348
|
+
deps.log(`enabling auto-merge failed (the PR stays open): ${e.message}`);
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
await release(deps, sessionId, claim.epoch);
|
|
353
|
+
return { kind: "outcome", outcome: doneOutcome(facts) };
|
|
354
|
+
}
|
|
355
|
+
finally {
|
|
356
|
+
if (heartbeat)
|
|
357
|
+
clearInterval(heartbeat);
|
|
358
|
+
opts.stop.removeEventListener("abort", onStop);
|
|
359
|
+
await proxy?.close();
|
|
360
|
+
// A workspace left behind never changes the outcome; it is logged for the operator.
|
|
361
|
+
if (ws)
|
|
362
|
+
await deps.cleanup(ws).catch((e) => deps.log(`cleanup failed: ${e.message}`));
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
async function fetchBundle(deps, sessionId, claim) {
|
|
366
|
+
const [jwksRes, res] = await Promise.all([
|
|
367
|
+
deps.api.jwks(),
|
|
368
|
+
deps.api.bundle(sessionId, claim.leaseKey),
|
|
369
|
+
]);
|
|
370
|
+
const refusal = leaseReadRefusal(res, "bundle_unavailable");
|
|
371
|
+
if (refusal)
|
|
372
|
+
return {
|
|
373
|
+
kind: "refused",
|
|
374
|
+
reason: refusal,
|
|
375
|
+
detail: `bundle fetch: ${res.code ?? res.networkError ?? res.status}`,
|
|
376
|
+
};
|
|
377
|
+
if (jwksRes.status !== 200) {
|
|
378
|
+
return {
|
|
379
|
+
kind: "refused",
|
|
380
|
+
reason: "bundle_unavailable",
|
|
381
|
+
detail: `jwks: ${jwksRes.networkError ?? jwksRes.status}`,
|
|
382
|
+
};
|
|
383
|
+
}
|
|
384
|
+
if (claim.executorId === null) {
|
|
385
|
+
// No answer named this runner's executor id, so the bundle cannot be bound to the claim.
|
|
386
|
+
return {
|
|
387
|
+
kind: "refused",
|
|
388
|
+
reason: "bundle_binding_mismatch",
|
|
389
|
+
detail: "the claim answer named no executor_id",
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
const v = verifyServedPhaseBundle(res.text, jwksRes.body, {
|
|
393
|
+
org_id: orgOf(res.body),
|
|
394
|
+
session_id: sessionId,
|
|
395
|
+
epoch: claim.epoch,
|
|
396
|
+
executor_id: claim.executorId,
|
|
397
|
+
contract_sha256: claim.contractSha256,
|
|
398
|
+
contract_version: claim.contractVersion,
|
|
399
|
+
}, { now: deps.now });
|
|
400
|
+
if (!v.valid)
|
|
401
|
+
return { kind: "refused", reason: `bundle_${v.reason}`, detail: v.detail };
|
|
402
|
+
return { kind: "ok", bundle: v.bundle };
|
|
403
|
+
}
|
|
404
|
+
/**
|
|
405
|
+
* The org the bundle must name. The runner learns its org only from the server, so the bundle's
|
|
406
|
+
* own org_id is the expectation; what binds it to this runner is the session, epoch, executor and
|
|
407
|
+
* seal, all of which the runner knows independently.
|
|
408
|
+
*/
|
|
409
|
+
function orgOf(body) {
|
|
410
|
+
const o = body?.org_id;
|
|
411
|
+
return typeof o === "string" ? o : "";
|
|
412
|
+
}
|
|
413
|
+
async function amendedSeal(deps, sessionId, claim, running) {
|
|
414
|
+
const jr = await deps.api.job(sessionId);
|
|
415
|
+
const job = jr.status === 200 ? parseJob(jr.body) : null;
|
|
416
|
+
if (!job || job.version <= claim.contractVersion)
|
|
417
|
+
return null;
|
|
418
|
+
const fresh = await fetchBundle(deps, sessionId, {
|
|
419
|
+
...claim,
|
|
420
|
+
contractSha256: job.contract_sha256,
|
|
421
|
+
contractVersion: job.version,
|
|
422
|
+
});
|
|
423
|
+
if (fresh.kind !== "ok")
|
|
424
|
+
return null;
|
|
425
|
+
// An amend reseals the charter, never the work: phases already run were signed as these, so a
|
|
426
|
+
// re-sealed bundle with another workflow or phase list fails closed rather than run stale phases.
|
|
427
|
+
if (!sameWork(fresh.bundle, running)) {
|
|
428
|
+
deps.log("the amended bundle changes the workflow or its phases");
|
|
429
|
+
return null;
|
|
430
|
+
}
|
|
431
|
+
const header = sealedContractHeader(Buffer.from(job.contract, "utf8"), job.contract_sha256);
|
|
432
|
+
if (!header.ok)
|
|
433
|
+
return null;
|
|
434
|
+
return { value: header.value, sha: job.contract_sha256, version: job.version };
|
|
435
|
+
}
|
|
436
|
+
function sameWork(a, b) {
|
|
437
|
+
return (canonicalJson({ workflow: a.workflow, phases: a.phases }) ===
|
|
438
|
+
canonicalJson({ workflow: b.workflow, phases: b.phases }));
|
|
439
|
+
}
|
|
440
|
+
/**
|
|
441
|
+
* Report a failed or refused job after claim (34.12 (6), (10)). The fail reason vocabulary is the
|
|
442
|
+
* Durable lead's to rule (A3 accepts only transcript_lost today); a refused report is logged and
|
|
443
|
+
* never changes the outcome, because the lease lapses on its own.
|
|
444
|
+
*/
|
|
445
|
+
export async function reportFail(deps, sessionId, epoch, kind, reason) {
|
|
446
|
+
const r = await deps.api.fail(sessionId, {
|
|
447
|
+
epoch,
|
|
448
|
+
nonce: deps.nonce,
|
|
449
|
+
reason: kind,
|
|
450
|
+
refs: { reason },
|
|
451
|
+
});
|
|
452
|
+
if (!ok2xx(r))
|
|
453
|
+
deps.log(`exec_fail ${kind}/${reason}: ${errorCode(r.body) ?? r.networkError ?? r.status}`);
|
|
454
|
+
}
|
|
455
|
+
async function release(deps, sessionId, epoch) {
|
|
456
|
+
const r = await deps.api.release(sessionId, { epoch, nonce: deps.nonce });
|
|
457
|
+
if (!ok2xx(r))
|
|
458
|
+
deps.log(`release failed (the lease lapses on its own): ${r.code ?? r.networkError ?? r.status}`);
|
|
459
|
+
}
|
|
460
|
+
async function stopJob(deps, sessionId, epoch) {
|
|
461
|
+
await release(deps, sessionId, epoch);
|
|
462
|
+
return { kind: "stopped" };
|
|
463
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `mnemom agent run`'s claim loop (ruling 34.12 (1), (6), (7), (9); Doors W-L13, W3.11).
|
|
3
|
+
*
|
|
4
|
+
* --once: list the ready pool sessions (exec_ready_list, oldest first), try each in order, and run
|
|
5
|
+
* at most one job. A lost race, or an ineligible waiting / not_pool_member, moves to the next
|
|
6
|
+
* candidate; an empty list, or every candidate lost, ends no_job. --watch: the same, again and
|
|
7
|
+
* again, until SIGTERM or SIGINT. Every job prints exactly one outcome line (none when stopped
|
|
8
|
+
* mid-job), and the process exit code is the --once outcome's, or 0 for --watch stopped.
|
|
9
|
+
*
|
|
10
|
+
* Every side effect is in `JobDeps` (plus `emit`), so each arm is tested; `liveJobDeps`
|
|
11
|
+
* (commands/agent-run.ts) is the thin driver.
|
|
12
|
+
*/
|
|
13
|
+
import type { HttpResult } from "./executor-client.js";
|
|
14
|
+
import { type JobDeps } from "./job.js";
|
|
15
|
+
import { type RefusedReason } from "./outcome.js";
|
|
16
|
+
import type { RunnerFailMode } from "./settings.js";
|
|
17
|
+
export interface LoopOptions {
|
|
18
|
+
failMode: RunnerFailMode;
|
|
19
|
+
stop: AbortSignal;
|
|
20
|
+
/** Print one outcome line (stdout). */
|
|
21
|
+
emit(line: string): void;
|
|
22
|
+
/** --watch: wait between empty polls, ms. */
|
|
23
|
+
idlePollMs?: number;
|
|
24
|
+
}
|
|
25
|
+
export declare const WATCH_IDLE_POLL_MS = 15000;
|
|
26
|
+
/** --watch backs off on a ready-list the api cannot answer, up to this, ms. */
|
|
27
|
+
export declare const WATCH_MAX_BACKOFF_MS: number;
|
|
28
|
+
type ListStep = {
|
|
29
|
+
kind: "ok";
|
|
30
|
+
ids: string[];
|
|
31
|
+
} | {
|
|
32
|
+
kind: "refused";
|
|
33
|
+
reason: RefusedReason;
|
|
34
|
+
detail: string;
|
|
35
|
+
};
|
|
36
|
+
/** The ready list, oldest first as served; an id that is not a lowercase uuid is dropped. */
|
|
37
|
+
export declare function readyIds(r: HttpResult): ListStep;
|
|
38
|
+
/** --once: at most one job. Resolves the process exit code. */
|
|
39
|
+
export declare function runOnce(deps: JobDeps, opts: LoopOptions): Promise<number>;
|
|
40
|
+
/**
|
|
41
|
+
* --watch: one outcome line per job until stopped. A ready list the api cannot answer is not a
|
|
42
|
+
* job: it is logged and retried with backoff. A refused executor key ends the loop (exit 3).
|
|
43
|
+
*/
|
|
44
|
+
export declare function runWatch(deps: JobDeps, opts: LoopOptions): Promise<number>;
|
|
45
|
+
export {};
|