@sema-agent/server 1.314.0 → 1.316.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/approval-hmac.d.ts +17 -0
- package/dist/approval-hmac.js +27 -0
- package/dist/config-center/apply-effective.d.ts +22 -0
- package/dist/config-center/apply-effective.js +283 -0
- package/dist/config-center/http-client.d.ts +23 -0
- package/dist/config-center/http-client.js +109 -0
- package/dist/config-center/restart-signal.d.ts +12 -0
- package/dist/config-center/restart-signal.js +70 -0
- package/dist/config-center/skills-mcp.d.ts +13 -0
- package/dist/config-center/skills-mcp.js +113 -0
- package/dist/config-center/types.d.ts +143 -0
- package/dist/config-center/types.js +2 -0
- package/dist/fleet/fleet-bus.js +92 -83
- package/dist/hooks/hook-runner.js +18 -9
- package/dist/http/idempotency.d.ts +1 -0
- package/dist/http/idempotency.js +3 -0
- package/dist/http/principal-gate.d.ts +12 -0
- package/dist/http/principal-gate.js +52 -0
- package/dist/http/route-ctx.d.ts +67 -0
- package/dist/http/route-ctx.js +2 -0
- package/dist/http/routes/attachments.d.ts +4 -0
- package/dist/http/routes/attachments.js +106 -0
- package/dist/http/routes/capabilities.d.ts +4 -0
- package/dist/http/routes/capabilities.js +111 -0
- package/dist/http/routes/images.d.ts +20 -0
- package/dist/http/routes/images.js +572 -0
- package/dist/http/routes/memory-policy.d.ts +4 -0
- package/dist/http/routes/memory-policy.js +105 -0
- package/dist/http/routes/observability.d.ts +4 -0
- package/dist/http/routes/observability.js +63 -0
- package/dist/http/routes/session-sync.d.ts +36 -0
- package/dist/http/routes/session-sync.js +513 -0
- package/dist/http/routes/sessions-list.d.ts +4 -0
- package/dist/http/routes/sessions-list.js +117 -0
- package/dist/http/routes/sessions.d.ts +17 -0
- package/dist/http/routes/sessions.js +746 -0
- package/dist/http/send.d.ts +6 -0
- package/dist/http/send.js +19 -0
- package/dist/http/server.d.ts +101 -76
- package/dist/http/server.js +129 -2410
- package/dist/http/sse-log.d.ts +25 -0
- package/dist/http/sse-log.js +59 -0
- package/dist/http/workspace-content.d.ts +3 -0
- package/dist/http/workspace-content.js +20 -0
- package/dist/index.d.ts +1 -1
- package/dist/leader/leader.js +5 -2
- package/dist/leader/wire.js +169 -165
- package/dist/main.js +462 -451
- package/dist/plugins/approval-store-sql.d.ts +52 -0
- package/dist/plugins/approval-store-sql.js +119 -0
- package/dist/plugins/breaker-state-sql.d.ts +29 -0
- package/dist/plugins/breaker-state-sql.js +104 -0
- package/dist/plugins/checkpoint-store-sql.d.ts +67 -0
- package/dist/plugins/checkpoint-store-sql.js +224 -0
- package/dist/plugins/file-snapshot-store-sql.d.ts +38 -0
- package/dist/plugins/file-snapshot-store-sql.js +210 -0
- package/dist/plugins/image-bake-store-sql.d.ts +53 -0
- package/dist/plugins/image-bake-store-sql.js +463 -0
- package/dist/plugins/image-index-sql.d.ts +33 -0
- package/dist/plugins/image-index-sql.js +328 -0
- package/dist/plugins/k8s-bg-scripts.d.ts +19 -0
- package/dist/plugins/k8s-bg-scripts.js +129 -0
- package/dist/plugins/k8s-exec-protocol.d.ts +20 -0
- package/dist/plugins/k8s-exec-protocol.js +87 -0
- package/dist/plugins/outcome-ledger-sql.d.ts +70 -0
- package/dist/plugins/outcome-ledger-sql.js +158 -0
- package/dist/plugins/pg-approval-store.d.ts +1 -27
- package/dist/plugins/pg-approval-store.js +1 -83
- package/dist/plugins/pg-breaker-state.d.ts +1 -19
- package/dist/plugins/pg-breaker-state.js +1 -88
- package/dist/plugins/pg-checkpoint-store.d.ts +1 -33
- package/dist/plugins/pg-checkpoint-store.js +1 -189
- package/dist/plugins/pg-cost-quota.js +3 -143
- package/dist/plugins/pg-file-snapshot-store.d.ts +1 -27
- package/dist/plugins/pg-file-snapshot-store.js +1 -190
- package/dist/plugins/pg-image-bake.d.ts +1 -40
- package/dist/plugins/pg-image-bake.js +1 -420
- package/dist/plugins/pg-image-index.d.ts +1 -21
- package/dist/plugins/pg-image-index.js +1 -217
- package/dist/plugins/pg-outcome-ledger.d.ts +2 -22
- package/dist/plugins/pg-outcome-ledger.js +1 -104
- package/dist/plugins/pg-rate-limiter.d.ts +2 -32
- package/dist/plugins/pg-rate-limiter.js +4 -147
- package/dist/plugins/pg-resume-anchor-store.d.ts +1 -13
- package/dist/plugins/pg-resume-anchor-store.js +1 -22
- package/dist/plugins/pg-run-store.d.ts +1 -75
- package/dist/plugins/pg-run-store.js +1 -287
- package/dist/plugins/pg-session-policy-store.d.ts +1 -11
- package/dist/plugins/pg-session-policy-store.js +1 -90
- package/dist/plugins/pg-tool-result-store.d.ts +1 -15
- package/dist/plugins/pg-tool-result-store.js +1 -52
- package/dist/plugins/pg-workflow-journal-store.d.ts +1 -20
- package/dist/plugins/pg-workflow-journal-store.js +1 -34
- package/dist/plugins/pg-workflow-run-store.d.ts +1 -55
- package/dist/plugins/pg-workflow-run-store.js +1 -220
- package/dist/plugins/remote-env-k8s.d.ts +5 -38
- package/dist/plugins/remote-env-k8s.js +7 -213
- package/dist/plugins/resume-anchor-store-sql.d.ts +23 -0
- package/dist/plugins/resume-anchor-store-sql.js +38 -0
- package/dist/plugins/run-store-sql.d.ts +89 -0
- package/dist/plugins/run-store-sql.js +397 -0
- package/dist/plugins/session-policy-store-sql.d.ts +22 -0
- package/dist/plugins/session-policy-store-sql.js +109 -0
- package/dist/plugins/sql-driver.d.ts +25 -0
- package/dist/plugins/sql-driver.js +59 -0
- package/dist/plugins/tidb-approval-store.d.ts +1 -39
- package/dist/plugins/tidb-approval-store.js +1 -71
- package/dist/plugins/tidb-breaker-state.d.ts +1 -17
- package/dist/plugins/tidb-breaker-state.js +1 -74
- package/dist/plugins/tidb-checkpoint-store.d.ts +1 -49
- package/dist/plugins/tidb-checkpoint-store.js +1 -191
- package/dist/plugins/tidb-file-snapshot-store.d.ts +1 -27
- package/dist/plugins/tidb-file-snapshot-store.js +1 -192
- package/dist/plugins/tidb-image-bake.d.ts +1 -38
- package/dist/plugins/tidb-image-bake.js +1 -348
- package/dist/plugins/tidb-image-index.d.ts +1 -19
- package/dist/plugins/tidb-image-index.js +1 -149
- package/dist/plugins/tidb-outcome-ledger.d.ts +1 -57
- package/dist/plugins/tidb-outcome-ledger.js +1 -97
- package/dist/plugins/tidb-resume-anchor-store.d.ts +1 -13
- package/dist/plugins/tidb-resume-anchor-store.js +1 -22
- package/dist/plugins/tidb-run-store.d.ts +1 -75
- package/dist/plugins/tidb-run-store.js +1 -306
- package/dist/plugins/tidb-session-policy-store.d.ts +1 -11
- package/dist/plugins/tidb-session-policy-store.js +1 -86
- package/dist/plugins/tidb-tool-result-store.d.ts +1 -13
- package/dist/plugins/tidb-tool-result-store.js +1 -38
- package/dist/plugins/tidb-workflow-journal-store.d.ts +1 -20
- package/dist/plugins/tidb-workflow-journal-store.js +1 -34
- package/dist/plugins/tidb-workflow-run-store.d.ts +1 -61
- package/dist/plugins/tidb-workflow-run-store.js +1 -231
- package/dist/plugins/tool-result-store-sql.d.ts +25 -0
- package/dist/plugins/tool-result-store-sql.js +69 -0
- package/dist/plugins/workflow-journal-store-sql.d.ts +30 -0
- package/dist/plugins/workflow-journal-store-sql.js +53 -0
- package/dist/plugins/workflow-run-store-sql.d.ts +85 -0
- package/dist/plugins/workflow-run-store-sql.js +299 -0
- package/dist/plugins/write-behind-counter.d.ts +14 -3
- package/dist/plugins/write-behind-counter.js +39 -12
- package/dist/principal-jwt.d.ts +44 -0
- package/dist/principal-jwt.js +95 -0
- package/dist/security.d.ts +2 -58
- package/dist/security.js +3 -118
- package/dist/sema-registry.d.ts +5 -200
- package/dist/sema-registry.js +4 -567
- package/package.json +1 -1
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { IncomingMessage, ServerResponse } from "node:http";
|
|
2
|
+
export interface SseLogProvider<E extends {
|
|
3
|
+
seq: number;
|
|
4
|
+
}> {
|
|
5
|
+
statusOf(id: string): Promise<{
|
|
6
|
+
status: string;
|
|
7
|
+
updatedAt: string;
|
|
8
|
+
} | null | undefined>;
|
|
9
|
+
getEvents(id: string, after: number): Promise<E[]>;
|
|
10
|
+
retainedFrom(id: string): Promise<number>;
|
|
11
|
+
formatEvent(ev: E): {
|
|
12
|
+
id: number;
|
|
13
|
+
event: string;
|
|
14
|
+
data: unknown;
|
|
15
|
+
};
|
|
16
|
+
staleFrame(): {
|
|
17
|
+
event: string;
|
|
18
|
+
data: unknown;
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export declare function streamSseLog<E extends {
|
|
22
|
+
seq: number;
|
|
23
|
+
}>(req: IncomingMessage, res: ServerResponse, provider: SseLogProvider<E>, id: string, staleMs: number): Promise<void>;
|
|
24
|
+
export declare function sleep(ms: number): Promise<void>;
|
|
25
|
+
//# sourceMappingURL=sse-log.d.ts.map
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { sendJson, sseHeaders } from "./send.js";
|
|
2
|
+
export async function streamSseLog(req, res, provider, id, staleMs) {
|
|
3
|
+
const lastId = Number(req.headers["last-event-id"] ?? new URL(req.url ?? "", "http://x").searchParams.get("from") ?? 0);
|
|
4
|
+
let from = Number.isFinite(lastId) ? lastId : 0;
|
|
5
|
+
if (from > 0) {
|
|
6
|
+
const retainedFrom = await provider.retainedFrom(id);
|
|
7
|
+
if (retainedFrom > from + 1) {
|
|
8
|
+
sendJson(res, 416, { error: "resume point evicted past retention", retainedFrom });
|
|
9
|
+
return;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
sseHeaders(res);
|
|
13
|
+
let closed = false;
|
|
14
|
+
req.on("close", () => {
|
|
15
|
+
closed = true;
|
|
16
|
+
});
|
|
17
|
+
const start = Date.now();
|
|
18
|
+
const MAX_MS = 15 * 60 * 1000;
|
|
19
|
+
let lastBeat = Date.now();
|
|
20
|
+
const write = (events) => {
|
|
21
|
+
for (const ev of events) {
|
|
22
|
+
const f = provider.formatEvent(ev);
|
|
23
|
+
res.write(`id: ${f.id}\nevent: ${f.event}\ndata: ${JSON.stringify(f.data)}\n\n`);
|
|
24
|
+
from = ev.seq;
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
while (!closed) {
|
|
28
|
+
const [events, row] = await Promise.all([provider.getEvents(id, from), provider.statusOf(id)]);
|
|
29
|
+
write(events);
|
|
30
|
+
if (!row)
|
|
31
|
+
break;
|
|
32
|
+
if (row.status !== "running") {
|
|
33
|
+
write(await provider.getEvents(id, from));
|
|
34
|
+
break;
|
|
35
|
+
}
|
|
36
|
+
const stale = Date.now() - new Date(row.updatedAt).getTime() > staleMs;
|
|
37
|
+
if (stale && events.length === 0) {
|
|
38
|
+
const sf = provider.staleFrame();
|
|
39
|
+
res.write(`event: ${sf.event}\ndata: ${JSON.stringify(sf.data)}\n\n`);
|
|
40
|
+
break;
|
|
41
|
+
}
|
|
42
|
+
if (Date.now() - start > MAX_MS) {
|
|
43
|
+
res.write(`event: error\ndata: ${JSON.stringify({ type: "error", code: "STREAM_MAX_DURATION", message: "stream reached its 15-minute cap — reconnect with Last-Event-ID to continue (the run is still active)" })}\n\n`);
|
|
44
|
+
break;
|
|
45
|
+
}
|
|
46
|
+
if (events.length === 0) {
|
|
47
|
+
if (Date.now() - lastBeat > 15_000) {
|
|
48
|
+
res.write(`event: heartbeat\ndata: {}\n\n`);
|
|
49
|
+
lastBeat = Date.now();
|
|
50
|
+
}
|
|
51
|
+
await sleep(250);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
res.end();
|
|
55
|
+
}
|
|
56
|
+
export function sleep(ms) {
|
|
57
|
+
return new Promise((r) => setTimeout(r, ms));
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=sse-log.js.map
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
export function workspaceContentType(relPath) {
|
|
2
|
+
const base = relPath.slice(relPath.lastIndexOf("/") + 1);
|
|
3
|
+
const dot = base.lastIndexOf(".");
|
|
4
|
+
const ext = dot > 0 ? base.slice(dot + 1).toLowerCase() : "";
|
|
5
|
+
const TABLE = {
|
|
6
|
+
txt: "text/plain; charset=utf-8", md: "text/plain; charset=utf-8", log: "text/plain; charset=utf-8",
|
|
7
|
+
json: "application/json; charset=utf-8", csv: "text/csv; charset=utf-8",
|
|
8
|
+
png: "image/png", jpg: "image/jpeg", jpeg: "image/jpeg", gif: "image/gif", webp: "image/webp",
|
|
9
|
+
pdf: "application/pdf",
|
|
10
|
+
};
|
|
11
|
+
return TABLE[ext] ?? "application/octet-stream";
|
|
12
|
+
}
|
|
13
|
+
export function looksBinary(bytes) {
|
|
14
|
+
const n = Math.min(bytes.byteLength, 8192);
|
|
15
|
+
for (let i = 0; i < n; i++)
|
|
16
|
+
if (bytes[i] === 0)
|
|
17
|
+
return true;
|
|
18
|
+
return false;
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=workspace-content.js.map
|
package/dist/index.d.ts
CHANGED
|
@@ -22,5 +22,5 @@ export { createCouncilTool } from "./capabilities/code-review-council.js";
|
|
|
22
22
|
export { loadSkills, skillsForScenario, type LoadedSkill } from "./capabilities/skills.js";
|
|
23
23
|
export { buildScenarios, selectScenario, type Scenario, type ScenarioBundle, type ScenarioDeps, type ScenarioRequest, } from "./capabilities/scenarios.js";
|
|
24
24
|
export { createAuthorizer, principalFrom, memoryScopeFor, memoryEngineBackendFor, HttpError, type AuthContext, type OwnerAwareSessionStore, } from "./security.js";
|
|
25
|
-
export { createHttpServer, type ServiceDeps, type TaskRequestBody, type RequestAuth, } from "./http/server.js";
|
|
25
|
+
export { createHttpServer, type ServiceDeps, type TaskRequestBody, type RequestAuth, type FlatServiceDeps, type ServiceDepGroups, type ServiceCoreDeps, type ServiceStoreDeps, type ServiceCoordinatorDeps, type ServiceSeamDeps, type ServiceObservabilityDeps, type ServiceGovernanceDeps, type ServiceDeploymentDeps, type ServiceKnobDeps, } from "./http/server.js";
|
|
26
26
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/leader/leader.js
CHANGED
|
@@ -161,7 +161,9 @@ export async function runLeaderTask(task, durableBaseSha, deps) {
|
|
|
161
161
|
}));
|
|
162
162
|
return out;
|
|
163
163
|
};
|
|
164
|
-
|
|
164
|
+
const runModuleGate = async () => {
|
|
165
|
+
if (!deps.moduleGate)
|
|
166
|
+
return;
|
|
165
167
|
const subById = new Map(subs.map((s) => [s.workerId, s]));
|
|
166
168
|
const maxR = deps.moduleGate.maxRepairs ?? 1;
|
|
167
169
|
await Promise.all(reports.filter((r) => r.status === "completed").map(async (r) => {
|
|
@@ -194,7 +196,8 @@ export async function runLeaderTask(task, durableBaseSha, deps) {
|
|
|
194
196
|
}
|
|
195
197
|
}
|
|
196
198
|
}));
|
|
197
|
-
}
|
|
199
|
+
};
|
|
200
|
+
await runModuleGate();
|
|
198
201
|
const mergeDeps = {
|
|
199
202
|
pullWorkerDiff: async (r) => workerDiff(byId.get(r.workerId)),
|
|
200
203
|
provisionIntegrationSandbox: deps.provisionIntegrationSandbox,
|
package/dist/leader/wire.js
CHANGED
|
@@ -192,186 +192,190 @@ export function createLeaderRunner(cfg) {
|
|
|
192
192
|
if (!ex.ok)
|
|
193
193
|
throw ex.error;
|
|
194
194
|
};
|
|
195
|
-
const
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
const
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
195
|
+
const plan = async () => {
|
|
196
|
+
if (Array.isArray(body.subtasks) && body.subtasks.length > 0) {
|
|
197
|
+
return validateSubtasks(body.subtasks);
|
|
198
|
+
}
|
|
199
|
+
const planRunner = new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing });
|
|
200
|
+
const runRoute = async (prompt) => {
|
|
201
|
+
const res = await planRunner.runTaskStream({
|
|
202
|
+
objective: prompt,
|
|
203
|
+
sessionId: `leader-route-${Date.now()}`,
|
|
204
|
+
...(cfg.routerModel ? { model: cfg.routerModel } : {}),
|
|
205
|
+
}).result();
|
|
206
|
+
return res.result ?? "";
|
|
207
|
+
};
|
|
208
|
+
const { subs, collapsed, rawSample } = await routePlanWithFallback(body.objective, runRoute, {
|
|
209
|
+
...(body.planContext ? { context: body.planContext } : {}),
|
|
210
|
+
...(cfg.fanoutEnabled === false ? { fanoutEnabled: false } : {}),
|
|
211
|
+
});
|
|
212
|
+
if (collapsed)
|
|
213
|
+
cfg.logger?.warn?.("plan_collapsed_to_single", { reason: collapsed, ...(rawSample ? { rawSample } : {}) });
|
|
214
|
+
return subs;
|
|
215
|
+
};
|
|
216
|
+
const provisionWorker = async (sub) => {
|
|
217
|
+
const sessionId = `leader-${sub.workerId}-${Date.now()}`;
|
|
218
|
+
let durableSpec = {};
|
|
219
|
+
if (cfg.durable) {
|
|
220
|
+
const d = cfg.durable;
|
|
221
|
+
durableSpec = {
|
|
222
|
+
toolPolicy: combinePolicies(createDurableQuestionPolicy(), createDurableAskPolicy({ requireApproval: d.requireApproval, ...(d.deny ? { deny: d.deny } : {}), ...(d.autoBudget ? { autoBudget: d.autoBudget } : {}), ...(d.neverAuto ? { neverAuto: d.neverAuto } : {}) })),
|
|
223
|
+
checkpointStore: d.checkpointStore,
|
|
224
|
+
durableApproval: { scope: "_leader", ...(d.ttlMs ? { ttlMs: d.ttlMs } : {}) },
|
|
225
|
+
onQuestion: QUESTION_AWAITS_RESUME,
|
|
208
226
|
};
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
...(cfg.fanoutEnabled === false ? { fanoutEnabled: false } : {}),
|
|
227
|
+
await d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch((e) => {
|
|
228
|
+
cfg.logger?.warn?.("leader_putctx_failed", { workerId: sub.workerId, err: String(e) });
|
|
212
229
|
});
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
resourceSuspend: { scope: `_leader-resource-${sub.workerId}`, totalBudgetUsd: resourceCfg.totalBudgetUsd },
|
|
241
|
-
maxCostUsd: resourceCfg.sliceMaxCostUsd,
|
|
242
|
-
maxSuspends: resourceCfg.maxSuspends,
|
|
243
|
-
};
|
|
244
|
-
}
|
|
245
|
-
else if (resourceCfg && !repairLoopActive) {
|
|
246
|
-
cfg.logger?.warn?.("leader_resource_suspend_inactive", {
|
|
247
|
-
workerId: sub.workerId,
|
|
248
|
-
reason: !cfg.durable
|
|
249
|
-
? "no durable checkpoint store"
|
|
250
|
-
: !cfg.toolResultStore
|
|
251
|
-
? "no durable tool-result store"
|
|
252
|
-
: "no durable session store",
|
|
253
|
-
});
|
|
254
|
-
}
|
|
255
|
-
const keepCtxWarm = (runner) => {
|
|
256
|
-
const d = cfg.durable;
|
|
257
|
-
if (!d)
|
|
258
|
-
return runner;
|
|
259
|
-
const orig = runner.runTaskStream.bind(runner);
|
|
260
|
-
runner.runTaskStream = (spec, resume) => {
|
|
261
|
-
const stream = orig(spec, resume);
|
|
262
|
-
const iv = setInterval(() => {
|
|
263
|
-
void d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch(() => { });
|
|
264
|
-
}, 45_000);
|
|
265
|
-
iv.unref?.();
|
|
266
|
-
void stream.result().then(() => clearInterval(iv), () => clearInterval(iv));
|
|
267
|
-
return stream;
|
|
268
|
-
};
|
|
230
|
+
}
|
|
231
|
+
let resourceSpec = {};
|
|
232
|
+
const repairLoopActive = repairLoopOn && REPAIR_LOOP_WIRED;
|
|
233
|
+
if (repairLoopActive && resourceCfg) {
|
|
234
|
+
cfg.logger?.warn?.("leader_resource_suspend_disabled_for_repair_loop", { workerId: sub.workerId, reason: "LEADER_REPAIR_LOOP active — resource-suspend mutually exclusive with the repair loop (§10.5)" });
|
|
235
|
+
}
|
|
236
|
+
if (resourceCfg && !repairLoopActive && cfg.durable && cfg.toolResultStore && cfg.sessionStore) {
|
|
237
|
+
resourceSpec = {
|
|
238
|
+
checkpointStore: cfg.durable.checkpointStore,
|
|
239
|
+
resourceSuspend: { scope: `_leader-resource-${sub.workerId}`, totalBudgetUsd: resourceCfg.totalBudgetUsd },
|
|
240
|
+
maxCostUsd: resourceCfg.sliceMaxCostUsd,
|
|
241
|
+
maxSuspends: resourceCfg.maxSuspends,
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
else if (resourceCfg && !repairLoopActive) {
|
|
245
|
+
cfg.logger?.warn?.("leader_resource_suspend_inactive", {
|
|
246
|
+
workerId: sub.workerId,
|
|
247
|
+
reason: !cfg.durable
|
|
248
|
+
? "no durable checkpoint store"
|
|
249
|
+
: !cfg.toolResultStore
|
|
250
|
+
? "no durable tool-result store"
|
|
251
|
+
: "no durable session store",
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
const keepCtxWarm = (runner) => {
|
|
255
|
+
const d = cfg.durable;
|
|
256
|
+
if (!d)
|
|
269
257
|
return runner;
|
|
258
|
+
const orig = runner.runTaskStream.bind(runner);
|
|
259
|
+
runner.runTaskStream = (spec, resume) => {
|
|
260
|
+
const stream = orig(spec, resume);
|
|
261
|
+
const iv = setInterval(() => {
|
|
262
|
+
void d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch(() => { });
|
|
263
|
+
}, 45_000);
|
|
264
|
+
iv.unref?.();
|
|
265
|
+
void stream.result().then(() => clearInterval(iv), () => clearInterval(iv));
|
|
266
|
+
return stream;
|
|
267
|
+
};
|
|
268
|
+
return runner;
|
|
269
|
+
};
|
|
270
|
+
if (cfg.s3 && cfg.envFactory) {
|
|
271
|
+
const s3 = cfg.s3;
|
|
272
|
+
const envFactory = cfg.envFactory;
|
|
273
|
+
const baseSha = body.baseSha;
|
|
274
|
+
const seedCmd = body.seedCmd;
|
|
275
|
+
const key = `${s3.keyPrefix ?? "leader-diff"}/${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}/${sub.workerId}.patch`;
|
|
276
|
+
const ttl = presignTtlSec;
|
|
277
|
+
const base = { endpoint: s3.endpoint, bucket: s3.bucket, accessKey: s3.accessKey, secretKey: s3.secretKey, ...(s3.region ? { region: s3.region } : {}), key, expiresSec: ttl };
|
|
278
|
+
const putUrl = presignS3Url({ ...base, method: "PUT" });
|
|
279
|
+
const getUrl = presignS3Url({ ...base, method: "GET" });
|
|
280
|
+
const script = buildUploadScript({ repoDir: repo, baseSha, putUrl, workDir: W });
|
|
281
|
+
const stage = async (env) => {
|
|
282
|
+
await stageWorkerEnv(env, { seedCmd, repo, branch: sub.branch, uploadScriptPath: `${W}/.leader-upload.sh`, uploadScript: script });
|
|
283
|
+
void sh(env)(`nohup sh -c 'while true; do sleep 90; sh ${W}/.leader-upload.sh >/dev/null 2>&1 || true; done' </dev/null >/dev/null 2>&1 &`).catch(() => { });
|
|
270
284
|
};
|
|
271
|
-
if (cfg.s3 && cfg.envFactory) {
|
|
272
|
-
const s3 = cfg.s3;
|
|
273
|
-
const envFactory = cfg.envFactory;
|
|
274
|
-
const baseSha = body.baseSha;
|
|
275
|
-
const seedCmd = body.seedCmd;
|
|
276
|
-
const key = `${s3.keyPrefix ?? "leader-diff"}/${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}/${sub.workerId}.patch`;
|
|
277
|
-
const ttl = presignTtlSec;
|
|
278
|
-
const base = { endpoint: s3.endpoint, bucket: s3.bucket, accessKey: s3.accessKey, secretKey: s3.secretKey, ...(s3.region ? { region: s3.region } : {}), key, expiresSec: ttl };
|
|
279
|
-
const putUrl = presignS3Url({ ...base, method: "PUT" });
|
|
280
|
-
const getUrl = presignS3Url({ ...base, method: "GET" });
|
|
281
|
-
const script = buildUploadScript({ repoDir: repo, baseSha, putUrl, workDir: W });
|
|
282
|
-
const stage = async (env) => {
|
|
283
|
-
await stageWorkerEnv(env, { seedCmd, repo, branch: sub.branch, uploadScriptPath: `${W}/.leader-upload.sh`, uploadScript: script });
|
|
284
|
-
void sh(env)(`nohup sh -c 'while true; do sleep 90; sh ${W}/.leader-upload.sh >/dev/null 2>&1 || true; done' </dev/null >/dev/null 2>&1 &`).catch(() => { });
|
|
285
|
-
};
|
|
286
|
-
return {
|
|
287
|
-
workerId: sub.workerId, sessionId, branch: sub.branch,
|
|
288
|
-
baseSha,
|
|
289
|
-
runner: keepCtxWarm(new Runner({
|
|
290
|
-
brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing,
|
|
291
|
-
...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}),
|
|
292
|
-
...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}),
|
|
293
|
-
executionEnvFactory: (ctx) => withStaging(envFactory(ctx), stage, async (e) => {
|
|
294
|
-
await sh(e)(`cd ${repo} && git add -A && git -c user.name='leader-belt' -c user.email='belt@leader' commit -qm 'belt: auto-commit uncommitted worker output' || true; sh ${W}/.leader-upload.sh`);
|
|
295
|
-
}),
|
|
296
|
-
})),
|
|
297
|
-
fetchDiff: () => fetchUploadedDiff(getUrl),
|
|
298
|
-
spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec, objective: `${sub.spec.objective}${uploadStepSuffix(W)}` },
|
|
299
|
-
};
|
|
300
|
-
}
|
|
301
|
-
if (!cfg.e2bApiKey)
|
|
302
|
-
throw new Error("leader wire: no envFactory+s3 (factory mode) and no e2bApiKey (static mode)");
|
|
303
|
-
const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
|
|
304
|
-
if (!(await env.connect()).ok)
|
|
305
|
-
throw new Error(`connect failed for ${sub.workerId}`);
|
|
306
|
-
await sh(env)(`${body.seedCmd}; cd ${repo}; git checkout -q -b ${sub.branch}`);
|
|
307
|
-
const baseSha = await sh(env)(`cd ${repo} && git rev-parse HEAD`);
|
|
308
285
|
return {
|
|
309
|
-
workerId: sub.workerId, sessionId, branch: sub.branch,
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
286
|
+
workerId: sub.workerId, sessionId, branch: sub.branch,
|
|
287
|
+
baseSha,
|
|
288
|
+
runner: keepCtxWarm(new Runner({
|
|
289
|
+
brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing,
|
|
290
|
+
...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}),
|
|
291
|
+
...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}),
|
|
292
|
+
executionEnvFactory: (ctx) => withStaging(envFactory(ctx), stage, async (e) => {
|
|
293
|
+
await sh(e)(`cd ${repo} && git add -A && git -c user.name='leader-belt' -c user.email='belt@leader' commit -qm 'belt: auto-commit uncommitted worker output' || true; sh ${W}/.leader-upload.sh`);
|
|
294
|
+
}),
|
|
295
|
+
})),
|
|
296
|
+
fetchDiff: () => fetchUploadedDiff(getUrl),
|
|
297
|
+
spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec, objective: `${sub.spec.objective}${uploadStepSuffix(W)}` },
|
|
314
298
|
};
|
|
315
|
-
}
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
}
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
throw new Error("integ connect failed");
|
|
335
|
-
}
|
|
299
|
+
}
|
|
300
|
+
if (!cfg.e2bApiKey)
|
|
301
|
+
throw new Error("leader wire: no envFactory+s3 (factory mode) and no e2bApiKey (static mode)");
|
|
302
|
+
const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
|
|
303
|
+
if (!(await env.connect()).ok)
|
|
304
|
+
throw new Error(`connect failed for ${sub.workerId}`);
|
|
305
|
+
await sh(env)(`${body.seedCmd}; cd ${repo}; git checkout -q -b ${sub.branch}`);
|
|
306
|
+
const baseSha = await sh(env)(`cd ${repo} && git rev-parse HEAD`);
|
|
307
|
+
return {
|
|
308
|
+
workerId: sub.workerId, sessionId, branch: sub.branch, baseSha,
|
|
309
|
+
runner: keepCtxWarm(new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing, ...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}), ...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}), executionEnv: env })),
|
|
310
|
+
diffEnv: env,
|
|
311
|
+
destroy: () => env.destroy().then(() => { }),
|
|
312
|
+
spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec },
|
|
313
|
+
};
|
|
314
|
+
};
|
|
315
|
+
const provisionIntegrationSandbox = async () => {
|
|
316
|
+
if (cfg.envFactory) {
|
|
317
|
+
const env = cfg.envFactory({ sessionId: `leader-integ-${Date.now()}` });
|
|
336
318
|
try {
|
|
337
319
|
await sh(env)(body.seedCmd);
|
|
338
320
|
await injectOracles(env);
|
|
339
321
|
}
|
|
340
322
|
catch (e) {
|
|
341
|
-
await env.destroy().catch(() => { });
|
|
323
|
+
await Promise.resolve(env.destroy()).catch(() => { });
|
|
342
324
|
throw e;
|
|
343
325
|
}
|
|
344
|
-
return { env: asIntegrationEnv(env), destroy: () => env.destroy().then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
|
|
345
|
-
}
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
326
|
+
return { env: asIntegrationEnv(env), destroy: () => Promise.resolve(env.destroy()).then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
|
|
327
|
+
}
|
|
328
|
+
if (!cfg.e2bApiKey)
|
|
329
|
+
throw new Error("leader wire: no envFactory and no e2bApiKey");
|
|
330
|
+
const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
|
|
331
|
+
if (!(await env.connect()).ok) {
|
|
332
|
+
await env.destroy().catch(() => { });
|
|
333
|
+
throw new Error("integ connect failed");
|
|
334
|
+
}
|
|
335
|
+
try {
|
|
336
|
+
await sh(env)(body.seedCmd);
|
|
337
|
+
await injectOracles(env);
|
|
338
|
+
}
|
|
339
|
+
catch (e) {
|
|
340
|
+
await env.destroy().catch(() => { });
|
|
341
|
+
throw e;
|
|
342
|
+
}
|
|
343
|
+
return { env: asIntegrationEnv(env), destroy: () => env.destroy().then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
|
|
344
|
+
};
|
|
345
|
+
const push = async (integratedPatch, baseSha) => {
|
|
346
|
+
const dir = mkdtempSync(join(tmpdir(), "leader-coord-"));
|
|
347
|
+
const gitEnv = { ...process.env, LC_ALL: "C" };
|
|
348
|
+
try {
|
|
349
|
+
execFileSync("git", ["clone", "-q", body.durableRemote, dir], { env: gitEnv });
|
|
350
|
+
execFileSync("git", ["-C", dir, "config", "user.name", ident.name]);
|
|
351
|
+
execFileSync("git", ["-C", dir, "config", "user.email", ident.email]);
|
|
352
|
+
writeFileSync(join(dir, ".leader.patch"), integratedPatch);
|
|
353
|
+
execFileSync("git", ["-C", dir, "am", "--3way", ".leader.patch"], { env: gitEnv });
|
|
354
|
+
rmSync(join(dir, ".leader.patch"));
|
|
355
|
+
const ref = body.targetRef ?? "refs/heads/main";
|
|
356
|
+
const ls = execFileSync("git", ["-C", dir, "ls-remote", "--refs", "origin", ref], { encoding: "utf8", env: gitEnv });
|
|
357
|
+
const exists = ls.split("\n").some((l) => l.split("\t")[1] === ref);
|
|
358
|
+
if (exists) {
|
|
359
|
+
execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:${baseSha}`, "origin", `HEAD:${ref}`], { env: gitEnv });
|
|
370
360
|
}
|
|
371
|
-
|
|
372
|
-
|
|
361
|
+
else {
|
|
362
|
+
execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:`, "origin", `HEAD:${ref}`], { env: gitEnv });
|
|
373
363
|
}
|
|
374
|
-
|
|
364
|
+
return { ok: true, ref };
|
|
365
|
+
}
|
|
366
|
+
catch (e) {
|
|
367
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
368
|
+
return { ok: false, raced: /stale info|force-with-lease|\[rejected\]|non-fast-forward/.test(msg), error: msg };
|
|
369
|
+
}
|
|
370
|
+
finally {
|
|
371
|
+
rmSync(dir, { recursive: true, force: true });
|
|
372
|
+
}
|
|
373
|
+
};
|
|
374
|
+
const deps = {
|
|
375
|
+
plan,
|
|
376
|
+
provisionWorker,
|
|
377
|
+
provisionIntegrationSandbox,
|
|
378
|
+
push,
|
|
375
379
|
testCmd: body.testCmd,
|
|
376
380
|
repoDir: repo, workerRepoDir: repo,
|
|
377
381
|
timeoutMs: leaderTimeoutMs, maxConcurrency: 4,
|