@sema-agent/server 7.56.0 → 7.58.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/README.zh-CN.md +2 -2
- package/USAGE.md +101 -9
- package/dist/approval-ask-audit-store.d.ts +100 -1
- package/dist/approval-ask-audit-store.js +103 -2
- package/dist/approval-card.d.ts +148 -13
- package/dist/approval-card.js +82 -13
- package/dist/approval.d.ts +31 -0
- package/dist/approval.js +18 -0
- package/dist/auto-mode-face.d.ts +111 -0
- package/dist/auto-mode-face.js +99 -0
- package/dist/bench/s1/arms.js +5 -5
- package/dist/bench/s1/live-deps.js +11 -11
- package/dist/bench/s1/run-firm.js +3 -0
- package/dist/bench/s1/runner-ctx.d.ts +4 -0
- package/dist/bench/s1/runner-ctx.js +7 -0
- package/dist/boot/config-center.js +23 -1
- package/dist/boot/coordinators.js +13 -1
- package/dist/boot/parked-revive-gate.d.ts +7 -2
- package/dist/boot/parked-revive-gate.js +12 -1
- package/dist/boot/resolve-spec.d.ts +3 -0
- package/dist/boot/resolve-spec.js +3 -1
- package/dist/boot/runner-deps.d.ts +12 -0
- package/dist/boot/runner-deps.js +28 -3
- package/dist/boot/runtime-caps.d.ts +19 -9
- package/dist/boot/runtime-caps.js +49 -21
- package/dist/boot/session-shell-gate-registry.d.ts +39 -0
- package/dist/boot/session-shell-gate-registry.js +21 -0
- package/dist/boot/stores.js +12 -5
- package/dist/config-catalog.js +14 -5
- package/dist/config-center/http-client.js +7 -11
- package/dist/config-center/read-warnings.d.ts +20 -0
- package/dist/config-center/read-warnings.js +36 -0
- package/dist/config-provider.d.ts +3 -0
- package/dist/config-provider.js +4 -8
- package/dist/config-types.d.ts +41 -10
- package/dist/config.d.ts +21 -0
- package/dist/config.js +40 -5
- package/dist/hooks/hook-runner.js +8 -8
- package/dist/http/route-ctx.d.ts +75 -0
- package/dist/http/routes/a2a-serve.js +5 -3
- package/dist/http/routes/approvals-assistant.js +19 -6
- package/dist/http/routes/capabilities.js +21 -4
- package/dist/http/routes/memory-origin.d.ts +2 -2
- package/dist/http/routes/rules.js +3 -2
- package/dist/http/routes/runs.d.ts +0 -14
- package/dist/http/routes/runs.js +20 -8
- package/dist/http/routes/tasks.js +34 -8
- package/dist/http/routes/workflows.js +19 -4
- package/dist/http/server.d.ts +11 -1
- package/dist/http/server.js +215 -43
- package/dist/http/wire-types.d.ts +9 -4
- package/dist/leader/diffout.js +2 -1
- package/dist/leader/endpoint.js +45 -12
- package/dist/leader/fanout.js +6 -5
- package/dist/leader/leader.js +17 -14
- package/dist/leader/merge.js +17 -12
- package/dist/leader/planner.js +3 -2
- package/dist/leader/repair-oracle.js +6 -4
- package/dist/leader/repair-wire.js +5 -3
- package/dist/leader/wire.d.ts +30 -1
- package/dist/leader/wire.js +36 -19
- package/dist/main.js +43 -7
- package/dist/observability/err-text.d.ts +6 -0
- package/dist/observability/err-text.js +10 -0
- package/dist/observability/fail-open.d.ts +40 -0
- package/dist/observability/fail-open.js +19 -0
- package/dist/observability/metrics.js +2 -2
- package/dist/observability/run-terminal-log.d.ts +120 -0
- package/dist/observability/run-terminal-log.js +360 -0
- package/dist/orchestration/workflow-completion-inbox.d.ts +9 -0
- package/dist/orchestration/workflow-completion-inbox.js +8 -0
- package/dist/permission-rule-vocab.d.ts +40 -0
- package/dist/permission-rule-vocab.js +17 -0
- package/dist/plugins/checkpoint-store-sql.d.ts +14 -0
- package/dist/plugins/checkpoint-store-sql.js +19 -3
- package/dist/plugins/file-run-store.d.ts +3 -34
- package/dist/plugins/file-run-store.js +19 -0
- package/dist/plugins/local-checkpoint-store.d.ts +10 -0
- package/dist/plugins/local-checkpoint-store.js +3 -0
- package/dist/plugins/mailbox-store-sql.d.ts +28 -9
- package/dist/plugins/mailbox-store-sql.js +72 -10
- package/dist/plugins/memory-run-store.d.ts +3 -15
- package/dist/plugins/memory-run-store.js +19 -0
- package/dist/plugins/permission-rule-store-file.js +8 -3
- package/dist/plugins/permission-rule-store-sql.js +16 -10
- package/dist/plugins/remote-scratchpad.js +3 -2
- package/dist/plugins/run-store-sql.d.ts +3 -42
- package/dist/plugins/run-store-sql.js +30 -1
- package/dist/plugins/store-backend.d.ts +1 -1
- package/dist/plugins/store-contracts.d.ts +66 -1
- package/dist/plugins/workflow-run-store-sql.d.ts +5 -0
- package/dist/plugins/workflow-run-store-sql.js +10 -2
- package/dist/rules-consent.js +20 -13
- package/dist/run-cancel-context.d.ts +13 -0
- package/dist/run-cancel-context.js +15 -0
- package/dist/run-local.js +2 -2
- package/dist/runs.d.ts +4 -1
- package/dist/runs.js +41 -10
- package/dist/runtime-caps-resolver.d.ts +133 -17
- package/dist/runtime-caps-resolver.js +41 -3
- package/dist/task-settings.d.ts +57 -3
- package/dist/task-settings.js +78 -5
- package/dist/task-workflow.d.ts +32 -2
- package/dist/task-workflow.js +8 -3
- package/dist/tool-approval.d.ts +26 -3
- package/dist/tool-approval.js +96 -13
- package/dist/trace/core-keyset-guard.d.ts +7 -8
- package/dist/trace/engine-notice-wire.d.ts +1 -1
- package/dist/trace/engine-notice-wire.js +2 -0
- package/dist/trace/ledger-sink.d.ts +13 -4
- package/dist/trace/ledger-sink.js +14 -6
- package/dist/trace/project.d.ts +58 -3
- package/dist/trace/project.js +33 -1
- package/dist/trace/redact.d.ts +22 -11
- package/dist/trace/redact.js +660 -21
- package/dist/trace/sema-provenance.d.ts +35 -0
- package/dist/trace/sema-provenance.js +14 -0
- package/dist/turn-activity.d.ts +32 -2
- package/dist/turn-activity.js +29 -4
- package/package.json +3 -3
package/dist/leader/fanout.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { redactErrorMessage, redactWorkerReportError } from "../observability/run-terminal-log.js";
|
|
1
2
|
class Semaphore {
|
|
2
3
|
max;
|
|
3
4
|
active = 0;
|
|
@@ -62,10 +63,10 @@ export async function fanOut(subtasks, opts) {
|
|
|
62
63
|
return { ...base, status: "timeout", error: "cancelled before start" };
|
|
63
64
|
if (opts.runWorker) {
|
|
64
65
|
try {
|
|
65
|
-
return maybeCancelOnSuspend(await opts.runWorker(st, sharedAC.signal));
|
|
66
|
+
return maybeCancelOnSuspend(redactWorkerReportError(await opts.runWorker(st, sharedAC.signal)));
|
|
66
67
|
}
|
|
67
68
|
catch (e) {
|
|
68
|
-
return { ...base, status: cancelled ? "timeout" : "failed", error: e instanceof Error ? e.message : String(e) };
|
|
69
|
+
return { ...base, status: cancelled ? "timeout" : "failed", error: redactErrorMessage(e instanceof Error ? e.message : String(e)) ?? "" };
|
|
69
70
|
}
|
|
70
71
|
}
|
|
71
72
|
let stream;
|
|
@@ -73,16 +74,16 @@ export async function fanOut(subtasks, opts) {
|
|
|
73
74
|
stream = opts.runner.runTaskStream({ ...st.spec, sessionId: st.sessionId, signal: sharedAC.signal });
|
|
74
75
|
}
|
|
75
76
|
catch (e) {
|
|
76
|
-
return { ...base, status: "failed", error: e instanceof Error ? e.message : String(e) };
|
|
77
|
+
return { ...base, status: "failed", error: redactErrorMessage(e instanceof Error ? e.message : String(e)) ?? "" };
|
|
77
78
|
}
|
|
78
79
|
streams.set(st.workerId, stream);
|
|
79
80
|
let report;
|
|
80
81
|
try {
|
|
81
82
|
const result = await stream.result();
|
|
82
|
-
report = { ...base, status: mapStatus(result.status), stats: result.stats, ...(result.errorMessage ? { error: result.errorMessage } : {}) };
|
|
83
|
+
report = { ...base, status: mapStatus(result.status), stats: result.stats, ...(result.errorMessage ? { error: redactErrorMessage(result.errorMessage) ?? "" } : {}) };
|
|
83
84
|
}
|
|
84
85
|
catch (e) {
|
|
85
|
-
report = { ...base, status: cancelled ? "timeout" : "failed", error: e instanceof Error ? e.message : String(e) };
|
|
86
|
+
report = { ...base, status: cancelled ? "timeout" : "failed", error: redactErrorMessage(e instanceof Error ? e.message : String(e)) ?? "" };
|
|
86
87
|
}
|
|
87
88
|
finally {
|
|
88
89
|
streams.delete(st.workerId);
|
package/dist/leader/leader.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { fanOut, isDurablePauseStatus } from "./fanout.js";
|
|
2
|
+
import { redactErrorMessage, redactHead, redactPatchHead } from "../observability/run-terminal-log.js";
|
|
2
3
|
import { pullDiff } from "./diffout.js";
|
|
3
4
|
import { mergeBranches } from "./merge.js";
|
|
4
5
|
import { Runner, runWithVerification, resumeWithVerification, delimitUntrusted } from "@sema-agent/core";
|
|
@@ -58,8 +59,8 @@ export async function runLeaderTask(task, durableBaseSha, deps) {
|
|
|
58
59
|
deps.logger?.warn?.("worker_failed", {
|
|
59
60
|
workerId: w.workerId, status: r.status, turns: r.stats?.turns ?? 0,
|
|
60
61
|
...(r.errorCode ? { errorCode: r.errorCode } : {}),
|
|
61
|
-
...(r.errorMessage ? { error: r.errorMessage
|
|
62
|
-
...(r.blockedReason ? { blockedReason: r.blockedReason
|
|
62
|
+
...(r.errorMessage ? { error: redactHead(r.errorMessage, 200) } : {}),
|
|
63
|
+
...(r.blockedReason ? { blockedReason: redactHead(r.blockedReason, 200) } : {}),
|
|
63
64
|
});
|
|
64
65
|
}
|
|
65
66
|
return {
|
|
@@ -85,7 +86,7 @@ export async function runLeaderTask(task, durableBaseSha, deps) {
|
|
|
85
86
|
deps.logger?.info?.("worker_resource_resume", { workerId: w.workerId, slice });
|
|
86
87
|
return resumeWithVerification(w.runner, token, { gate: "resource_limit", decision: "continue" }, { ...w.spec, sessionId: w.sessionId, signal }, w.spec.objective ?? "", cfg);
|
|
87
88
|
});
|
|
88
|
-
deps.logger?.info?.("verify_outcome", { workerId: w.workerId, status: vr.status, verdict: vr.verification.verdict, rounds: vr.verification.rounds, finding: (vr.verification.findings[0] ?? ""
|
|
89
|
+
deps.logger?.info?.("verify_outcome", { workerId: w.workerId, status: vr.status, verdict: vr.verification.verdict, rounds: vr.verification.rounds, finding: redactHead(vr.verification.findings[0] ?? "", 220) });
|
|
89
90
|
const pass = vr.status === "completed" && vr.verification.verdict === "PASS";
|
|
90
91
|
const stillSuspended = vr.checkpointToken !== undefined;
|
|
91
92
|
const reportStatus = pass ? "completed" : stillSuspended ? "suspended" : "failed";
|
|
@@ -93,8 +94,8 @@ export async function runLeaderTask(task, durableBaseSha, deps) {
|
|
|
93
94
|
deps.logger?.warn?.("worker_failed", {
|
|
94
95
|
workerId: w.workerId, status: reportStatus, turns: vr.stats?.turns ?? 0, verdict: vr.verification.verdict,
|
|
95
96
|
...(vr.errorCode ? { errorCode: vr.errorCode } : {}),
|
|
96
|
-
...(vr.errorMessage ? { error: vr.errorMessage
|
|
97
|
-
...(vr.blockedReason ? { blockedReason: vr.blockedReason
|
|
97
|
+
...(vr.errorMessage ? { error: redactHead(vr.errorMessage, 200) } : {}),
|
|
98
|
+
...(vr.blockedReason ? { blockedReason: redactHead(vr.blockedReason, 200) } : {}),
|
|
98
99
|
});
|
|
99
100
|
}
|
|
100
101
|
return {
|
|
@@ -105,7 +106,7 @@ export async function runLeaderTask(task, durableBaseSha, deps) {
|
|
|
105
106
|
: {
|
|
106
107
|
error: vr.verification.verdict === "unverified"
|
|
107
108
|
? (vr.errorMessage ?? vr.blockedReason ?? `worker ${vr.status}`)
|
|
108
|
-
: `verify ${vr.verification.verdict}: ${(vr.verification.findings[0] ?? ""
|
|
109
|
+
: `verify ${vr.verification.verdict}: ${redactHead(vr.verification.findings[0] ?? "", 200)}`,
|
|
109
110
|
}),
|
|
110
111
|
...(vr.checkpointGate ? { checkpointGate: vr.checkpointGate } : {}),
|
|
111
112
|
...(vr.errorCode ? { errorCode: vr.errorCode } : {}),
|
|
@@ -127,7 +128,7 @@ export async function runLeaderTask(task, durableBaseSha, deps) {
|
|
|
127
128
|
return { reports: [rr.report], cancelled: false };
|
|
128
129
|
}
|
|
129
130
|
catch (e) {
|
|
130
|
-
deps.logger?.warn?.("repair_loop_dispatch_failed", { workerId: solo.workerId, err: e instanceof Error ? e.message : String(e) });
|
|
131
|
+
deps.logger?.warn?.("repair_loop_dispatch_failed", { workerId: solo.workerId, err: redactHead(e instanceof Error ? e.message : String(e), 2000) });
|
|
131
132
|
return runWave([toSubtask(solo)]);
|
|
132
133
|
}
|
|
133
134
|
};
|
|
@@ -178,14 +179,15 @@ export async function runLeaderTask(task, durableBaseSha, deps) {
|
|
|
178
179
|
r.repairs = attempt;
|
|
179
180
|
if (verdict.ok)
|
|
180
181
|
return;
|
|
182
|
+
const safeVerdictReason = redactErrorMessage(verdict.reason ?? "behaviorally incorrect") ?? "";
|
|
181
183
|
if (attempt === maxR) {
|
|
182
184
|
r.status = "failed";
|
|
183
|
-
r.error = `module gate: ${(verdict.reason ?? "behaviorally incorrect"
|
|
185
|
+
r.error = `module gate: ${redactHead(verdict.reason ?? "behaviorally incorrect", 200)}`;
|
|
184
186
|
return;
|
|
185
187
|
}
|
|
186
|
-
deps.logger?.info?.("module_repair", { workerId: r.workerId, attempt: attempt + 1, reason: verdict.reason });
|
|
188
|
+
deps.logger?.info?.("module_repair", { workerId: r.workerId, attempt: attempt + 1, reason: redactHead(verdict.reason ?? "", 500) });
|
|
187
189
|
try {
|
|
188
|
-
const rr = await w.runner.runTaskStream({ ...sub.spec, objective: deps.moduleGate.repairObjective(sub,
|
|
190
|
+
const rr = await w.runner.runTaskStream({ ...sub.spec, objective: deps.moduleGate.repairObjective(sub, safeVerdictReason), sessionId: `${w.sessionId}-r${attempt + 1}` }).result();
|
|
189
191
|
if (r.stats && rr?.stats)
|
|
190
192
|
r.stats = { ...r.stats, tokens: (r.stats.tokens ?? 0) + (rr.stats.tokens ?? 0), costMicroUsd: (r.stats.costMicroUsd ?? 0) + (rr.stats.costMicroUsd ?? 0) };
|
|
191
193
|
}
|
|
@@ -226,8 +228,9 @@ export async function runLeaderTask(task, durableBaseSha, deps) {
|
|
|
226
228
|
};
|
|
227
229
|
};
|
|
228
230
|
const collapseToSingle = async (trigger, reason, priorMerge) => {
|
|
229
|
-
|
|
230
|
-
|
|
231
|
+
const safeReason = redactHead(reason, 300);
|
|
232
|
+
deps.logger?.warn?.("replan_collapse_to_single", { trigger, reason: safeReason });
|
|
233
|
+
const replan = { trigger, collapsed: safeReason, ...(redispatched ? { redispatched } : {}) };
|
|
231
234
|
const priorDiffs = [];
|
|
232
235
|
await Promise.all(reports.map(async (r) => {
|
|
233
236
|
const w = byId.get(r.workerId);
|
|
@@ -249,8 +252,8 @@ export async function runLeaderTask(task, durableBaseSha, deps) {
|
|
|
249
252
|
task,
|
|
250
253
|
``,
|
|
251
254
|
`--- PRIOR PARALLEL WORK — REUSE IT, do not re-derive from scratch ---`,
|
|
252
|
-
`${priorDiffs.length} worker(s) attempted this task in parallel; their changes could NOT be integrated cleanly (${
|
|
253
|
-
...priorDiffs.map((d) => delimitUntrusted(`PRIOR WORKER '${d.workerId}' (${d.status}) DIFF (untrusted)`, d.patch
|
|
255
|
+
`${priorDiffs.length} worker(s) attempted this task in parallel; their changes could NOT be integrated cleanly (${safeReason.slice(0, 150)}). Each worker's diff over the base is below. COPY the correct, non-overlapping parts verbatim and produce ONE coherent change for the WHOLE task, resolving only the overlaps. Treat these as a strong starting point — fix anything wrong, but do not throw the work away.`,
|
|
256
|
+
...priorDiffs.map((d) => delimitUntrusted(`PRIOR WORKER '${d.workerId}' (${d.status}) DIFF (untrusted)`, redactPatchHead(d.patch, 8000))),
|
|
254
257
|
].join("\n");
|
|
255
258
|
const soloSub = { workerId: "collapse-solo", branch: "worker/collapse-solo", files: ["."], spec: { objective: soloObjective } };
|
|
256
259
|
let solo;
|
package/dist/leader/merge.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { delimitUntrusted } from "@sema-agent/core";
|
|
2
|
+
import { OVERSIZED_DIAG_PLACEHOLDER, redactErrorMessage, redactHead, redactTail } from "../observability/run-terminal-log.js";
|
|
2
3
|
import { applyAndGrade, changedFilesOutOfAllowlist } from "./grader-env-factory.js";
|
|
3
4
|
import { shellQuote as Q } from "../plugins/remote-shell.js";
|
|
4
5
|
export async function mergeBranches(reports, baseSha, deps) {
|
|
@@ -40,7 +41,7 @@ export async function mergeBranches(reports, baseSha, deps) {
|
|
|
40
41
|
conflictResolverCostUsd += cr.costUsd;
|
|
41
42
|
if (!cr.resolved) {
|
|
42
43
|
deps.logger?.warn?.("merge_apply_failed", { workerId: p.workerId, exitCode: ap.exitCode });
|
|
43
|
-
return { ok: false, phase: "apply", reason: `git apply --3way failed for ${p.workerId} (conflict)`, quarantine: [p.workerId], rej: rej.stdout, details: ap.stderr
|
|
44
|
+
return { ok: false, phase: "apply", reason: `git apply --3way failed for ${p.workerId} (conflict)`, quarantine: [p.workerId], rej: rej.stdout, details: redactHead(ap.stderr, 800), ...(conflictResolverCostUsd > 0 ? { conflictResolverCostUsd } : {}) };
|
|
44
45
|
}
|
|
45
46
|
conflictsResolved.push(p.workerId);
|
|
46
47
|
}
|
|
@@ -51,14 +52,18 @@ export async function mergeBranches(reports, baseSha, deps) {
|
|
|
51
52
|
let cost = 0;
|
|
52
53
|
let round = 0;
|
|
53
54
|
for (; round < integ.repair.maxRounds && test.exitCode !== 0; round++) {
|
|
54
|
-
const errorText = (test.stdout + "\n" + test.stderr
|
|
55
|
+
const errorText = redactTail(test.stdout + "\n" + test.stderr, 4000);
|
|
56
|
+
if (errorText.startsWith(OVERSIZED_DIAG_PLACEHOLDER)) {
|
|
57
|
+
deps.logger?.warn?.("integration_repair_skipped_oversized_output", { round: round + 1, outputChars: test.stdout.length + test.stderr.length });
|
|
58
|
+
break;
|
|
59
|
+
}
|
|
55
60
|
deps.logger?.warn?.("integration_repair_round", { round: round + 1, maxRounds: integ.repair.maxRounds });
|
|
56
61
|
try {
|
|
57
62
|
const r = await integ.repair.run({ round: round + 1, errorText, testCmd: deps.testCmd, repoDir: repo });
|
|
58
63
|
cost += r.costUsd ?? 0;
|
|
59
64
|
}
|
|
60
65
|
catch (e) {
|
|
61
|
-
deps.logger?.warn?.("integration_repair_error", { round: round + 1, err: e instanceof Error ? e.message : String(e) });
|
|
66
|
+
deps.logger?.warn?.("integration_repair_error", { round: round + 1, err: redactHead(e instanceof Error ? e.message : String(e), 2000) });
|
|
62
67
|
break;
|
|
63
68
|
}
|
|
64
69
|
repairMeta = { rounds: round + 1, fixed: false, costUsd: cost };
|
|
@@ -69,11 +74,11 @@ export async function mergeBranches(reports, baseSha, deps) {
|
|
|
69
74
|
deps.logger?.info?.("integration_repair_done", { ...repairMeta });
|
|
70
75
|
}
|
|
71
76
|
if (test.exitCode !== 0) {
|
|
72
|
-
return { ok: false, phase: "test", reason: `integration test failed (exit ${test.exitCode})`, details: (test.stdout + "\n" + test.stderr
|
|
77
|
+
return { ok: false, phase: "test", reason: `integration test failed (exit ${test.exitCode})`, details: redactTail(test.stdout + "\n" + test.stderr, 2000), ...(repairMeta ? { repair: repairMeta } : {}), ...(conflictResolverCostUsd > 0 ? { conflictResolverCostUsd } : {}) };
|
|
73
78
|
}
|
|
74
79
|
if (deps.measureCmd) {
|
|
75
80
|
const mr = await env.exec(deps.measureCmd, { cwd: repo });
|
|
76
|
-
measure = mr.exitCode === 0 ? { pass: true } : { pass: false, reason: (mr.stdout + " " + mr.stderr
|
|
81
|
+
measure = mr.exitCode === 0 ? { pass: true } : { pass: false, reason: redactTail(mr.stdout + " " + mr.stderr, 200) };
|
|
77
82
|
}
|
|
78
83
|
phase = "push";
|
|
79
84
|
if (repairMeta)
|
|
@@ -81,11 +86,11 @@ export async function mergeBranches(reports, baseSha, deps) {
|
|
|
81
86
|
const costTail = { ...(repairMeta ? { repair: repairMeta } : {}), ...(conflictResolverCostUsd > 0 ? { conflictResolverCostUsd } : {}) };
|
|
82
87
|
const commit = await env.exec(`git add -A && git -c user.email=leader@local -c user.name=leader commit -m ${Q(`merge: integrate ${mergeable.map((r) => r.workerId).join("+")}`)} --allow-empty`, { cwd: repo });
|
|
83
88
|
if (commit.exitCode !== 0) {
|
|
84
|
-
return { ok: false, phase: "push", reason: `failed to create integration commit (exit ${commit.exitCode})`, details: (commit.stdout + "\n" + commit.stderr
|
|
89
|
+
return { ok: false, phase: "push", reason: `failed to create integration commit (exit ${commit.exitCode})`, details: redactHead(commit.stdout + "\n" + commit.stderr, 800), ...costTail };
|
|
85
90
|
}
|
|
86
91
|
const integrated = await env.exec(`git format-patch --stdout ${Q(integBase)}..HEAD`, { cwd: repo });
|
|
87
92
|
if (integrated.exitCode !== 0) {
|
|
88
|
-
return { ok: false, phase: "push", reason: `failed to produce integrated patch (exit ${integrated.exitCode})`, details: integrated.stderr
|
|
93
|
+
return { ok: false, phase: "push", reason: `failed to produce integrated patch (exit ${integrated.exitCode})`, details: redactHead(integrated.stderr, 800), ...costTail };
|
|
89
94
|
}
|
|
90
95
|
if (!integrated.stdout.trim()) {
|
|
91
96
|
return { ok: false, phase: "push", reason: "integrated patch is empty (nothing applied over base) — refusing to push", quarantine: mergeable.map((r) => r.workerId), ...costTail };
|
|
@@ -148,11 +153,11 @@ async function evaluatePushHold(mergeable, observeMeasure, integratedPatch, deps
|
|
|
148
153
|
return { hold: { reason: `integrated patch touches out-of-allowlist / build-config paths (${offending.slice(0, 5).join(", ")}) — needs_human_oracle (§10.4(ii))`, terminal: "needs_human_oracle", candidatePatch: heldCandidate() }, measure };
|
|
149
154
|
}
|
|
150
155
|
const grade = await applyAndGrade(mdr.grader, integratedPatch, mdr.measureSteps);
|
|
151
|
-
measure = grade.passed ? { pass: true } : { pass: false, reason: (grade.trace ?? "measure failed in out-of-process grader"
|
|
156
|
+
measure = grade.passed ? { pass: true } : { pass: false, reason: redactTail(grade.trace ?? "measure failed in out-of-process grader", 200) };
|
|
152
157
|
deps.logger?.info?.("merge_measure_grader", { passed: grade.passed, applied: grade.applied });
|
|
153
158
|
}
|
|
154
159
|
catch (e) {
|
|
155
|
-
deps.logger?.warn?.("merge_measure_grader_error", { err: e instanceof Error ? e.message : String(e) });
|
|
160
|
+
deps.logger?.warn?.("merge_measure_grader_error", { err: redactErrorMessage(e instanceof Error ? e.message : String(e)) ?? "" });
|
|
156
161
|
return { hold: { reason: `measure grader errored — needs_human_oracle (§10.4)`, terminal: "needs_human_oracle", candidatePatch: heldCandidate() }, measure };
|
|
157
162
|
}
|
|
158
163
|
}
|
|
@@ -160,14 +165,14 @@ async function evaluatePushHold(mergeable, observeMeasure, integratedPatch, deps
|
|
|
160
165
|
const strongOracleUnconfirmed = deps.strongOracleSeeded === true;
|
|
161
166
|
if (measureFailing || strongOracleUnconfirmed) {
|
|
162
167
|
const why = measureFailing ? "measure did not pass" : "hidden held-out oracle seeded but only the compile gate cleared";
|
|
163
|
-
deps.logger?.warn?.("merge_push_held_signal", { why, measureReason: (measure?.reason ?? ""
|
|
168
|
+
deps.logger?.warn?.("merge_push_held_signal", { why, measureReason: redactHead(measure?.reason ?? "", 200) });
|
|
164
169
|
return { hold: { reason: `strong oracle unconfirmed: ${why} — candidate_only, push held (§10.3)`, terminal: "candidate_only", candidatePatch: heldCandidate() }, measure };
|
|
165
170
|
}
|
|
166
171
|
return { measure };
|
|
167
172
|
}
|
|
168
173
|
async function resolveApplyConflict(env, repo, p, rejText, applyStderr, resolver, deps) {
|
|
169
174
|
let costUsd = 0;
|
|
170
|
-
const conflictText = `git apply --3way could not cleanly apply worker '${p.workerId}'.\n--- apply stderr ---\n${applyStderr
|
|
175
|
+
const conflictText = `git apply --3way could not cleanly apply worker '${p.workerId}'.\n--- apply stderr ---\n${redactHead(applyStderr, 1500)}\n--- reject hunks (*.rej) ---\n${redactHead(rejText, 3000)}`;
|
|
171
176
|
for (let round = 0; round < resolver.maxRounds; round++) {
|
|
172
177
|
deps.logger?.warn?.("merge_conflict_repair_round", { workerId: p.workerId, round: round + 1, maxRounds: resolver.maxRounds });
|
|
173
178
|
try {
|
|
@@ -175,7 +180,7 @@ async function resolveApplyConflict(env, repo, p, rejText, applyStderr, resolver
|
|
|
175
180
|
costUsd += r.costUsd ?? 0;
|
|
176
181
|
}
|
|
177
182
|
catch (e) {
|
|
178
|
-
deps.logger?.warn?.("merge_conflict_repair_error", { workerId: p.workerId, err: e instanceof Error ? e.message : String(e) });
|
|
183
|
+
deps.logger?.warn?.("merge_conflict_repair_error", { workerId: p.workerId, err: redactHead(e instanceof Error ? e.message : String(e), 2000) });
|
|
179
184
|
return { resolved: false, costUsd };
|
|
180
185
|
}
|
|
181
186
|
const check = await env
|
package/dist/leader/planner.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { redactErrorMessage, redactHead, redactTail } from "../observability/run-terminal-log.js";
|
|
1
2
|
export function economicGuard(objective, opts = {}) {
|
|
2
3
|
if (opts.fanoutEnabled === false)
|
|
3
4
|
return { single: true, reason: "fan-out disabled by deployment (A6)" };
|
|
@@ -124,13 +125,13 @@ export async function routePlanWithFallback(objective, runRoute, opts = {}) {
|
|
|
124
125
|
catch (e) {
|
|
125
126
|
firstErr ??= e;
|
|
126
127
|
if (!firstRaw)
|
|
127
|
-
firstRaw = raw.length > 600 ? `${raw
|
|
128
|
+
firstRaw = raw.length > 600 ? `${redactHead(raw, 300)}…[${raw.length}ch]…${redactTail(raw, 150)}` : (redactErrorMessage(raw) ?? "");
|
|
128
129
|
}
|
|
129
130
|
}
|
|
130
131
|
const reason = firstErr instanceof Error ? firstErr.message : String(firstErr);
|
|
131
132
|
return {
|
|
132
133
|
subs: singleRoute(objective),
|
|
133
|
-
collapsed: `plan rejected twice → collapse to single: ${reason
|
|
134
|
+
collapsed: `plan rejected twice → collapse to single: ${redactHead(reason, 200)}`,
|
|
134
135
|
rawSample: firstRaw,
|
|
135
136
|
};
|
|
136
137
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { runExecGate, } from "@sema-agent/core";
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
3
|
import { shellQuote as sq } from "../plugins/remote-shell.js";
|
|
4
|
+
import { redactHead, redactTail } from "../observability/run-terminal-log.js";
|
|
4
5
|
const sha = (s) => createHash("sha256").update(s).digest("hex").slice(0, 16);
|
|
5
6
|
function changedFilesOf(patch) {
|
|
6
7
|
const out = new Set();
|
|
@@ -28,8 +29,9 @@ export function mkRepairOracle(cfg) {
|
|
|
28
29
|
return async (graderEnv, _evidence) => {
|
|
29
30
|
const diff = await cfg.pullWorkerDiff();
|
|
30
31
|
if (!diff.ok) {
|
|
31
|
-
|
|
32
|
-
|
|
32
|
+
const safeErr = redactHead(diff.error ?? "", 2000);
|
|
33
|
+
cfg.log?.("repair_oracle_pull_failed", { error: safeErr });
|
|
34
|
+
return { tier: cfg.passTier, passed: false, flaky: false, retries: 0, trace: `pull worker diff failed: ${safeErr}` };
|
|
33
35
|
}
|
|
34
36
|
const diffHash = sha(diff.patch);
|
|
35
37
|
const tampered = cfg.oraclePaths && cfg.oraclePaths.length > 0 ? touchesOraclePath(changedFilesOf(diff.patch), cfg.oraclePaths) : undefined;
|
|
@@ -51,7 +53,7 @@ export function mkRepairOracle(cfg) {
|
|
|
51
53
|
{ command: `git -C ${sq(repo)} apply --whitespace=nowarn ${sq(patchPath)}`, label: "grader-apply" },
|
|
52
54
|
]);
|
|
53
55
|
if (!prep.passed) {
|
|
54
|
-
const trace = prep.steps.map((s) => `${s.label}: ${s.stderr}`).join("\n")
|
|
56
|
+
const trace = redactTail(prep.steps.map((s) => `${s.label}: ${s.stderr}`).join("\n"), 2000);
|
|
55
57
|
cfg.log?.("repair_oracle_prep_failed", { trace });
|
|
56
58
|
cfg.onGraded?.({ diff: diff.patch, hash: diffHash, passed: false });
|
|
57
59
|
return { tier: cfg.passTier, passed: false, flaky: false, retries: 0, trace: `grader prep failed:\n${trace}` };
|
|
@@ -64,7 +66,7 @@ export function mkRepairOracle(cfg) {
|
|
|
64
66
|
const gate = await runExecGate(graderEnv, cfg.oracleSteps);
|
|
65
67
|
verdicts.push(gate.passed);
|
|
66
68
|
if (!gate.passed) {
|
|
67
|
-
lastTrace = gate.steps.map((s) => `${s.label}\n${s.stdout}\n${s.stderr}`).join("\n")
|
|
69
|
+
lastTrace = redactTail(gate.steps.map((s) => `${s.label}\n${s.stdout}\n${s.stderr}`).join("\n"), 4000);
|
|
68
70
|
break;
|
|
69
71
|
}
|
|
70
72
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { runRepairLoop, delimitUntrusted, } from "@sema-agent/core";
|
|
2
2
|
import { mkRepairOracle } from "./repair-oracle.js";
|
|
3
|
+
import { redactHead } from "../observability/run-terminal-log.js";
|
|
3
4
|
function statusForTerminal(t) {
|
|
4
5
|
return t === "fixed" || t === "candidate_only" ? "completed" : "failed";
|
|
5
6
|
}
|
|
@@ -41,12 +42,13 @@ async function runRepairLoopForLeaderInner(solo, implSpec, cfg) {
|
|
|
41
42
|
...(cfg.onAttempt
|
|
42
43
|
? { onAttempt: (i) => cfg.onAttempt({ attempt: i.attempt, terminal: i.terminal, oracleTier: i.oracle.tier, passed: i.oracle.passed, flaky: i.oracle.flaky }) }
|
|
43
44
|
: {}),
|
|
44
|
-
onWarn: (e) => cfg.log?.("repair_loop_warn", { warning: e.message }),
|
|
45
|
+
onWarn: (e) => cfg.log?.("repair_loop_warn", { warning: redactHead(e.message, 2000) }),
|
|
45
46
|
});
|
|
46
47
|
}
|
|
47
48
|
catch (e) {
|
|
48
|
-
|
|
49
|
-
|
|
49
|
+
const safe = redactHead(e instanceof Error ? e.message : String(e), 2000);
|
|
50
|
+
cfg.log?.("repair_loop_threw", { error: safe });
|
|
51
|
+
return { report: { ...base, status: "failed", error: safe }, result: undefined };
|
|
50
52
|
}
|
|
51
53
|
const terminal = result.terminal;
|
|
52
54
|
const report = {
|
package/dist/leader/wire.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type Brain, type Model, type ModelRoles, type ModelPricing, type TaskSpec, type ExecutionEnvFactory, type RemoteExecutionEnv, type ToolResultStore, type SessionStore, type RunnerDeps } from "@sema-agent/core";
|
|
1
|
+
import { type Brain, type Model, type ModelRoles, type ModelPricing, type TaskSpec, type ExecutionEnvFactory, type RemoteExecutionEnv, type ToolResultStore, type SessionStore, type ExecutionEnv, type RunnerDeps } from "@sema-agent/core";
|
|
2
2
|
import type { CheckpointStoreFull } from "../plugins/store-backend.js";
|
|
3
3
|
import { type LeaderDeps, type LeaderResult } from "./leader.js";
|
|
4
4
|
import { type DeploymentPostureSeats } from "../boot/runner-deps.js";
|
|
@@ -146,6 +146,35 @@ export interface LeaderWireConfig {
|
|
|
146
146
|
}
|
|
147
147
|
/** The env surface wire needs (both the E2B and k8s adapters implement core's RemoteExecutionEnv). */
|
|
148
148
|
type WireEnv = Pick<RemoteExecutionEnv, "exec" | "writeFile">;
|
|
149
|
+
/**
|
|
150
|
+
* [ref] —— leader 自铸沙盒的**回收闸**(集成沙盒 / repair-loop grader 各一只)。
|
|
151
|
+
*
|
|
152
|
+
* core 契约:`ExecutionEnv` 基面**没有** `destroy`;只有实现了 `RemoteExecutionEnv` 的适配器才有
|
|
153
|
+
* (`core/remote-env.d.ts:426`),而「这只 env 到底有没有」是**运行期结构事实**,core 自己也只用结构探测
|
|
154
|
+
* `hasDestroy()`(:487,runtask/prepare-task 全部 destroy 站点都先过它)。修前 leader 三处把它写成编译期
|
|
155
|
+
* 断言(`as … & { destroy(): Promise<unknown> }`),于是 TS 对「手里其实是个 **Promise**」完全无感——
|
|
156
|
+
* `env.destroy()` 抛 `env.destroy is not a function`,还把**原始错误盖掉**([ref] 现场)。
|
|
157
|
+
*
|
|
158
|
+
* 🔴 缺席臂 = **fail-closed,当场响亮拒**([ref]:执行车道禁静默兜底;codex r2-[medium] 采纳)。理由:
|
|
159
|
+
* · core 之所以容忍「工厂产物没有 destroy」,是因为工厂**可以**交回一只**共享静态** env(`withWorktreeIsolation`
|
|
160
|
+
* 的非隔离臂就是这一形)——那种 env 本来就不该被谁拆掉。但 leader 这两只是**专用、临时、只属于它自己**
|
|
161
|
+
* 的沙盒:唯一的拆机口就是本闭包(`mergeBranches` / `runRepairLoopForLeader` 在 `finally` 里调它)。
|
|
162
|
+
* 收不动 = 一只计费中的沙盒必然留到后端自己的 timeout,零上行收益 ⇒ 没有「合法兜底」可言。
|
|
163
|
+
* · 静默 no-op 反而比**修前**更松:修前同步工厂路径上,一只没有 destroy 的 env 会当场抛 TypeError。
|
|
164
|
+
* 本闸只是把那个抛**提前到 provision 时刻**,并换成一句点名沙盒与成因的话——不是新增严格度。
|
|
165
|
+
* · 当前三条已接线的 leader 车道交回的 env 都实现了 destroy(e2b `RemoteContainerExecutionEnv`、
|
|
166
|
+
* k8s、host `RemoteHostExecutionEnv`——`hostExecutionEnvFactory` 恒 `new RemoteHostExecutionEnv`),
|
|
167
|
+
* 所以这条臂是**防御性**的:它真响的那天,意思是有人接了一条新工厂车道而 leader 收不动它的沙盒。
|
|
168
|
+
*/
|
|
169
|
+
/** 冲突解决模型的目标文本——纯函数(第十三轮重扫 [medium]:wire 内联时无法钉「补丁先脱」;抽出来直接钉)。
|
|
170
|
+
* 补丁走补丁头窗(头 256KB 扫再切 6000;解决者仍有工作树里的冲突标记),conflictText 由 merge.ts 先脱后切。 */
|
|
171
|
+
export declare function buildConflictResolverObjective(args: {
|
|
172
|
+
workerId: string;
|
|
173
|
+
repoDir: string;
|
|
174
|
+
patch: string;
|
|
175
|
+
conflictText: string;
|
|
176
|
+
}): string;
|
|
177
|
+
export declare function createEnvReaper(env: ExecutionEnv, what: string): () => Promise<void>;
|
|
149
178
|
/**
|
|
150
179
|
* Resume-safe worker staging (the [ref] F2(a) factory-env path). Runs adapter-side before the agent's
|
|
151
180
|
* first workspace-touching call: seed the base repo at the worker branch + plant the upload script (so the
|
package/dist/leader/wire.js
CHANGED
|
@@ -3,7 +3,7 @@ import { execFileSync } from "node:child_process";
|
|
|
3
3
|
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
4
4
|
import { join } from "node:path";
|
|
5
5
|
import { tmpdir } from "node:os";
|
|
6
|
-
import { Runner, combinePolicies, createDurableQuestionPolicy, QUESTION_AWAITS_RESUME, delimitUntrusted, } from "@sema-agent/core";
|
|
6
|
+
import { Runner, combinePolicies, createDurableQuestionPolicy, QUESTION_AWAITS_RESUME, delimitUntrusted, hasDestroy, } from "@sema-agent/core";
|
|
7
7
|
import { RemoteContainerExecutionEnv } from "../plugins/remote-env-e2b.js";
|
|
8
8
|
import { createDurableAskPolicy } from "../approval.js";
|
|
9
9
|
import { presignS3Url } from "../plugins/s3-presign.js";
|
|
@@ -14,6 +14,7 @@ import { attachRepairLoopDeps } from "./repair-wire.js";
|
|
|
14
14
|
import { routePlanWithFallback, validateSubtasks } from "./planner.js";
|
|
15
15
|
import { createEngineNoticeSeat } from "../boot/runner-deps.js";
|
|
16
16
|
import { currentPrincipal } from "../observability/principal-context.js";
|
|
17
|
+
import { redactHead, redactPatchHead } from "../observability/run-terminal-log.js";
|
|
17
18
|
const REPAIR_LOOP_WIRED = true;
|
|
18
19
|
const sh = (env) => async (cmd) => {
|
|
19
20
|
const r = await env.exec(`bash -lc ${JSON.stringify(cmd)}`);
|
|
@@ -31,6 +32,24 @@ const asIntegrationEnv = (env) => ({
|
|
|
31
32
|
writeFile: async (p, c) => { const r = await env.writeFile(p, c); if (!r.ok)
|
|
32
33
|
throw r.error; },
|
|
33
34
|
});
|
|
35
|
+
export function buildConflictResolverObjective(args) {
|
|
36
|
+
const { workerId, repoDir, patch, conflictText } = args;
|
|
37
|
+
return [
|
|
38
|
+
`A parallel worker '${workerId}' ported a module on its own branch, but applying its patch onto the already-integrated tree at ${repoDir} produced a MERGE CONFLICT (git apply --3way). The OTHER workers' patches already applied cleanly — integrate THIS worker's changes too, resolving the overlap.`,
|
|
39
|
+
`Resolve EVERY conflict in the working tree: open each file containing conflict markers (<<<<<<< / ======= / >>>>>>>) and merge BOTH sides' real intent (keep both workers' behaviour — never drop one side just to make it apply). Then apply any rejected hunks recorded in *.rej files by hand, and DELETE every *.rej and *.orig file.`,
|
|
40
|
+
`When done, NO conflict markers and NO *.rej files may remain under ${repoDir}. Verify yourself before finishing: grep -rIlE '^(<<<<<<<|>>>>>>>)' . ; find . -name '*.rej' — BOTH must print nothing.`,
|
|
41
|
+
`HARD RULES: do NOT git commit / git add / git push (the coordinator squashes at the end); do NOT modify build configuration; do NOT call AskUserQuestion — decide autonomously. Preserve both sides' ported semantics.`,
|
|
42
|
+
``,
|
|
43
|
+
delimitUntrusted("CONFLICTING WORKER PATCH (untrusted)", redactPatchHead(patch, 6000)),
|
|
44
|
+
delimitUntrusted("CONFLICT DETAILS (untrusted)", conflictText),
|
|
45
|
+
].join("\n");
|
|
46
|
+
}
|
|
47
|
+
export function createEnvReaper(env, what) {
|
|
48
|
+
if (!hasDestroy(env)) {
|
|
49
|
+
throw new Error(`${what}: the execution-env factory returned an env with no destroy() — the leader OWNS this dedicated sandbox and this closure is its only teardown, so it would leak (and keep billing) until the backend's own timeout reaps it. Deploy a factory whose env implements RemoteExecutionEnv.destroy for the leader lane.`);
|
|
50
|
+
}
|
|
51
|
+
return async () => void (await env.destroy());
|
|
52
|
+
}
|
|
34
53
|
export async function stageWorkerEnv(env, opts) {
|
|
35
54
|
const { seedCmd, repo, branch, uploadScriptPath, uploadScript } = opts;
|
|
36
55
|
const probe = await env.exec(`bash -lc ${JSON.stringify(`if [ -f ${uploadScriptPath} ]; then exit 0; fi; ` +
|
|
@@ -197,15 +216,7 @@ export function createLeaderRunner(cfg) {
|
|
|
197
216
|
}
|
|
198
217
|
try {
|
|
199
218
|
const runner = new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing, ...onNoticeSeat, ...governanceSeat, ...deploymentPostureSeat, ...tracerSeat, ...usageWindowSeat, ...promptSourceSeat, ...onErrorSeat, executionEnv: rawEnv, rootPath: repoDir });
|
|
200
|
-
const objective =
|
|
201
|
-
`A parallel worker '${workerId}' ported a module on its own branch, but applying its patch onto the already-integrated tree at ${repoDir} produced a MERGE CONFLICT (git apply --3way). The OTHER workers' patches already applied cleanly — integrate THIS worker's changes too, resolving the overlap.`,
|
|
202
|
-
`Resolve EVERY conflict in the working tree: open each file containing conflict markers (<<<<<<< / ======= / >>>>>>>) and merge BOTH sides' real intent (keep both workers' behaviour — never drop one side just to make it apply). Then apply any rejected hunks recorded in *.rej files by hand, and DELETE every *.rej and *.orig file.`,
|
|
203
|
-
`When done, NO conflict markers and NO *.rej files may remain under ${repoDir}. Verify yourself before finishing: grep -rIlE '^(<<<<<<<|>>>>>>>)' . ; find . -name '*.rej' — BOTH must print nothing.`,
|
|
204
|
-
`HARD RULES: do NOT git commit / git add / git push (the coordinator squashes at the end); do NOT modify build configuration; do NOT call AskUserQuestion — decide autonomously. Preserve both sides' ported semantics.`,
|
|
205
|
-
``,
|
|
206
|
-
delimitUntrusted("CONFLICTING WORKER PATCH (untrusted)", patch.slice(0, 6000)),
|
|
207
|
-
delimitUntrusted("CONFLICT DETAILS (untrusted)", conflictText),
|
|
208
|
-
].join("\n");
|
|
219
|
+
const objective = buildConflictResolverObjective({ workerId, repoDir, patch, conflictText });
|
|
209
220
|
const res = await runner
|
|
210
221
|
.runTaskStream({
|
|
211
222
|
objective,
|
|
@@ -290,7 +301,7 @@ export function createLeaderRunner(cfg) {
|
|
|
290
301
|
onQuestion: QUESTION_AWAITS_RESUME,
|
|
291
302
|
};
|
|
292
303
|
await d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch((e) => {
|
|
293
|
-
cfg.logger?.warn?.("leader_putctx_failed", { workerId: sub.workerId, err: String(e) });
|
|
304
|
+
cfg.logger?.warn?.("leader_putctx_failed", { workerId: sub.workerId, err: redactHead(e instanceof Error ? e.message : String(e), 2000) });
|
|
294
305
|
});
|
|
295
306
|
}
|
|
296
307
|
let resourceSpec = {};
|
|
@@ -354,7 +365,7 @@ export function createLeaderRunner(cfg) {
|
|
|
354
365
|
brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing, ...onNoticeSeat, ...governanceSeat, ...deploymentPostureSeat, ...tracerSeat, ...usageWindowSeat, ...promptSourceSeat, ...onErrorSeat,
|
|
355
366
|
...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}),
|
|
356
367
|
...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}),
|
|
357
|
-
executionEnvFactory: (ctx) => withStaging(envFactory(ctx), stage, async (e) => {
|
|
368
|
+
executionEnvFactory: async (ctx) => withStaging(await envFactory(ctx), stage, async (e) => {
|
|
358
369
|
await sh(e)(`cd ${repo} && git add -A && git -c user.name='leader-belt' -c user.email='belt@leader' commit -qm 'belt: auto-commit uncommitted worker output' || true; sh ${W}/.leader-upload.sh`);
|
|
359
370
|
}),
|
|
360
371
|
})),
|
|
@@ -380,16 +391,21 @@ export function createLeaderRunner(cfg) {
|
|
|
380
391
|
const provisionIntegrationSandbox = async () => {
|
|
381
392
|
if (cfg.envFactory) {
|
|
382
393
|
const integSessionId = `leader-integ-${Date.now()}`;
|
|
383
|
-
const env = cfg.envFactory({ sessionId: integSessionId, placementRootSessionId: integSessionId });
|
|
394
|
+
const env = await cfg.envFactory({ sessionId: integSessionId, placementRootSessionId: integSessionId });
|
|
395
|
+
const reap = createEnvReaper(env, "leader integration sandbox");
|
|
384
396
|
try {
|
|
385
397
|
await sh(env)(body.seedCmd);
|
|
386
398
|
await injectOracles(env);
|
|
387
399
|
}
|
|
388
400
|
catch (e) {
|
|
389
|
-
await
|
|
401
|
+
await reap().catch(() => { });
|
|
390
402
|
throw e;
|
|
391
403
|
}
|
|
392
|
-
|
|
404
|
+
const rawEnv = env;
|
|
405
|
+
const oracleFiles = body.oracleFiles ?? [];
|
|
406
|
+
const repair = mkRepair(rawEnv, repo, oracleFiles);
|
|
407
|
+
const conflictResolver = mkConflictResolver(rawEnv, repo, oracleFiles);
|
|
408
|
+
return { env: asIntegrationEnv(env), destroy: reap, ...(repair ? { repair } : {}), ...(conflictResolver ? { conflictResolver } : {}) };
|
|
393
409
|
}
|
|
394
410
|
if (!cfg.e2bApiKey)
|
|
395
411
|
throw new Error("leader wire: no envFactory and no e2bApiKey");
|
|
@@ -427,7 +443,7 @@ export function createLeaderRunner(cfg) {
|
|
|
427
443
|
let env;
|
|
428
444
|
if (cfg.envFactory) {
|
|
429
445
|
const graderSessionId = `leader-grader-${Date.now()}`;
|
|
430
|
-
env = cfg.envFactory({ sessionId: graderSessionId, placementRootSessionId: graderSessionId });
|
|
446
|
+
env = await cfg.envFactory({ sessionId: graderSessionId, placementRootSessionId: graderSessionId });
|
|
431
447
|
}
|
|
432
448
|
else {
|
|
433
449
|
if (!cfg.e2bApiKey)
|
|
@@ -439,6 +455,7 @@ export function createLeaderRunner(cfg) {
|
|
|
439
455
|
}
|
|
440
456
|
env = e2b;
|
|
441
457
|
}
|
|
458
|
+
const reapGrader = createEnvReaper(env, "leader repair-loop grader");
|
|
442
459
|
let integBase;
|
|
443
460
|
try {
|
|
444
461
|
await sh(env)(body.seedCmd);
|
|
@@ -446,11 +463,11 @@ export function createLeaderRunner(cfg) {
|
|
|
446
463
|
integBase = await sh(env)(`cd ${repo} && git rev-parse HEAD`);
|
|
447
464
|
}
|
|
448
465
|
catch (e) {
|
|
449
|
-
await
|
|
466
|
+
await reapGrader().catch(() => { });
|
|
450
467
|
throw e;
|
|
451
468
|
}
|
|
452
469
|
const grader = {
|
|
453
|
-
env
|
|
470
|
+
env,
|
|
454
471
|
writeFile: async (p, c) => { const r = await env.writeFile(p, c); if (!r.ok)
|
|
455
472
|
throw r.error; },
|
|
456
473
|
repoDir: repo,
|
|
@@ -475,7 +492,7 @@ export function createLeaderRunner(cfg) {
|
|
|
475
492
|
return {
|
|
476
493
|
runner: solo.runner,
|
|
477
494
|
grader,
|
|
478
|
-
graderDestroy:
|
|
495
|
+
graderDestroy: reapGrader,
|
|
479
496
|
pullWorkerDiff,
|
|
480
497
|
oracleSteps,
|
|
481
498
|
passTier,
|