@sema-agent/server 1.314.0 → 1.316.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/dist/approval-hmac.d.ts +17 -0
  2. package/dist/approval-hmac.js +27 -0
  3. package/dist/config-center/apply-effective.d.ts +22 -0
  4. package/dist/config-center/apply-effective.js +283 -0
  5. package/dist/config-center/http-client.d.ts +23 -0
  6. package/dist/config-center/http-client.js +109 -0
  7. package/dist/config-center/restart-signal.d.ts +12 -0
  8. package/dist/config-center/restart-signal.js +70 -0
  9. package/dist/config-center/skills-mcp.d.ts +13 -0
  10. package/dist/config-center/skills-mcp.js +113 -0
  11. package/dist/config-center/types.d.ts +143 -0
  12. package/dist/config-center/types.js +2 -0
  13. package/dist/fleet/fleet-bus.js +92 -83
  14. package/dist/hooks/hook-runner.js +18 -9
  15. package/dist/http/idempotency.d.ts +1 -0
  16. package/dist/http/idempotency.js +3 -0
  17. package/dist/http/principal-gate.d.ts +12 -0
  18. package/dist/http/principal-gate.js +52 -0
  19. package/dist/http/route-ctx.d.ts +67 -0
  20. package/dist/http/route-ctx.js +2 -0
  21. package/dist/http/routes/attachments.d.ts +4 -0
  22. package/dist/http/routes/attachments.js +106 -0
  23. package/dist/http/routes/capabilities.d.ts +4 -0
  24. package/dist/http/routes/capabilities.js +111 -0
  25. package/dist/http/routes/images.d.ts +20 -0
  26. package/dist/http/routes/images.js +572 -0
  27. package/dist/http/routes/memory-policy.d.ts +4 -0
  28. package/dist/http/routes/memory-policy.js +105 -0
  29. package/dist/http/routes/observability.d.ts +4 -0
  30. package/dist/http/routes/observability.js +63 -0
  31. package/dist/http/routes/session-sync.d.ts +36 -0
  32. package/dist/http/routes/session-sync.js +513 -0
  33. package/dist/http/routes/sessions-list.d.ts +4 -0
  34. package/dist/http/routes/sessions-list.js +117 -0
  35. package/dist/http/routes/sessions.d.ts +17 -0
  36. package/dist/http/routes/sessions.js +746 -0
  37. package/dist/http/send.d.ts +6 -0
  38. package/dist/http/send.js +19 -0
  39. package/dist/http/server.d.ts +101 -76
  40. package/dist/http/server.js +129 -2410
  41. package/dist/http/sse-log.d.ts +25 -0
  42. package/dist/http/sse-log.js +59 -0
  43. package/dist/http/workspace-content.d.ts +3 -0
  44. package/dist/http/workspace-content.js +20 -0
  45. package/dist/index.d.ts +1 -1
  46. package/dist/leader/leader.js +5 -2
  47. package/dist/leader/wire.js +169 -165
  48. package/dist/main.js +462 -451
  49. package/dist/plugins/approval-store-sql.d.ts +52 -0
  50. package/dist/plugins/approval-store-sql.js +119 -0
  51. package/dist/plugins/breaker-state-sql.d.ts +29 -0
  52. package/dist/plugins/breaker-state-sql.js +104 -0
  53. package/dist/plugins/checkpoint-store-sql.d.ts +67 -0
  54. package/dist/plugins/checkpoint-store-sql.js +224 -0
  55. package/dist/plugins/file-snapshot-store-sql.d.ts +38 -0
  56. package/dist/plugins/file-snapshot-store-sql.js +210 -0
  57. package/dist/plugins/image-bake-store-sql.d.ts +53 -0
  58. package/dist/plugins/image-bake-store-sql.js +463 -0
  59. package/dist/plugins/image-index-sql.d.ts +33 -0
  60. package/dist/plugins/image-index-sql.js +328 -0
  61. package/dist/plugins/k8s-bg-scripts.d.ts +19 -0
  62. package/dist/plugins/k8s-bg-scripts.js +129 -0
  63. package/dist/plugins/k8s-exec-protocol.d.ts +20 -0
  64. package/dist/plugins/k8s-exec-protocol.js +87 -0
  65. package/dist/plugins/outcome-ledger-sql.d.ts +70 -0
  66. package/dist/plugins/outcome-ledger-sql.js +158 -0
  67. package/dist/plugins/pg-approval-store.d.ts +1 -27
  68. package/dist/plugins/pg-approval-store.js +1 -83
  69. package/dist/plugins/pg-breaker-state.d.ts +1 -19
  70. package/dist/plugins/pg-breaker-state.js +1 -88
  71. package/dist/plugins/pg-checkpoint-store.d.ts +1 -33
  72. package/dist/plugins/pg-checkpoint-store.js +1 -189
  73. package/dist/plugins/pg-cost-quota.js +3 -143
  74. package/dist/plugins/pg-file-snapshot-store.d.ts +1 -27
  75. package/dist/plugins/pg-file-snapshot-store.js +1 -190
  76. package/dist/plugins/pg-image-bake.d.ts +1 -40
  77. package/dist/plugins/pg-image-bake.js +1 -420
  78. package/dist/plugins/pg-image-index.d.ts +1 -21
  79. package/dist/plugins/pg-image-index.js +1 -217
  80. package/dist/plugins/pg-outcome-ledger.d.ts +2 -22
  81. package/dist/plugins/pg-outcome-ledger.js +1 -104
  82. package/dist/plugins/pg-rate-limiter.d.ts +2 -32
  83. package/dist/plugins/pg-rate-limiter.js +4 -147
  84. package/dist/plugins/pg-resume-anchor-store.d.ts +1 -13
  85. package/dist/plugins/pg-resume-anchor-store.js +1 -22
  86. package/dist/plugins/pg-run-store.d.ts +1 -75
  87. package/dist/plugins/pg-run-store.js +1 -287
  88. package/dist/plugins/pg-session-policy-store.d.ts +1 -11
  89. package/dist/plugins/pg-session-policy-store.js +1 -90
  90. package/dist/plugins/pg-tool-result-store.d.ts +1 -15
  91. package/dist/plugins/pg-tool-result-store.js +1 -52
  92. package/dist/plugins/pg-workflow-journal-store.d.ts +1 -20
  93. package/dist/plugins/pg-workflow-journal-store.js +1 -34
  94. package/dist/plugins/pg-workflow-run-store.d.ts +1 -55
  95. package/dist/plugins/pg-workflow-run-store.js +1 -220
  96. package/dist/plugins/remote-env-k8s.d.ts +5 -38
  97. package/dist/plugins/remote-env-k8s.js +7 -213
  98. package/dist/plugins/resume-anchor-store-sql.d.ts +23 -0
  99. package/dist/plugins/resume-anchor-store-sql.js +38 -0
  100. package/dist/plugins/run-store-sql.d.ts +89 -0
  101. package/dist/plugins/run-store-sql.js +397 -0
  102. package/dist/plugins/session-policy-store-sql.d.ts +22 -0
  103. package/dist/plugins/session-policy-store-sql.js +109 -0
  104. package/dist/plugins/sql-driver.d.ts +25 -0
  105. package/dist/plugins/sql-driver.js +59 -0
  106. package/dist/plugins/tidb-approval-store.d.ts +1 -39
  107. package/dist/plugins/tidb-approval-store.js +1 -71
  108. package/dist/plugins/tidb-breaker-state.d.ts +1 -17
  109. package/dist/plugins/tidb-breaker-state.js +1 -74
  110. package/dist/plugins/tidb-checkpoint-store.d.ts +1 -49
  111. package/dist/plugins/tidb-checkpoint-store.js +1 -191
  112. package/dist/plugins/tidb-file-snapshot-store.d.ts +1 -27
  113. package/dist/plugins/tidb-file-snapshot-store.js +1 -192
  114. package/dist/plugins/tidb-image-bake.d.ts +1 -38
  115. package/dist/plugins/tidb-image-bake.js +1 -348
  116. package/dist/plugins/tidb-image-index.d.ts +1 -19
  117. package/dist/plugins/tidb-image-index.js +1 -149
  118. package/dist/plugins/tidb-outcome-ledger.d.ts +1 -57
  119. package/dist/plugins/tidb-outcome-ledger.js +1 -97
  120. package/dist/plugins/tidb-resume-anchor-store.d.ts +1 -13
  121. package/dist/plugins/tidb-resume-anchor-store.js +1 -22
  122. package/dist/plugins/tidb-run-store.d.ts +1 -75
  123. package/dist/plugins/tidb-run-store.js +1 -306
  124. package/dist/plugins/tidb-session-policy-store.d.ts +1 -11
  125. package/dist/plugins/tidb-session-policy-store.js +1 -86
  126. package/dist/plugins/tidb-tool-result-store.d.ts +1 -13
  127. package/dist/plugins/tidb-tool-result-store.js +1 -38
  128. package/dist/plugins/tidb-workflow-journal-store.d.ts +1 -20
  129. package/dist/plugins/tidb-workflow-journal-store.js +1 -34
  130. package/dist/plugins/tidb-workflow-run-store.d.ts +1 -61
  131. package/dist/plugins/tidb-workflow-run-store.js +1 -231
  132. package/dist/plugins/tool-result-store-sql.d.ts +25 -0
  133. package/dist/plugins/tool-result-store-sql.js +69 -0
  134. package/dist/plugins/workflow-journal-store-sql.d.ts +30 -0
  135. package/dist/plugins/workflow-journal-store-sql.js +53 -0
  136. package/dist/plugins/workflow-run-store-sql.d.ts +85 -0
  137. package/dist/plugins/workflow-run-store-sql.js +299 -0
  138. package/dist/plugins/write-behind-counter.d.ts +14 -3
  139. package/dist/plugins/write-behind-counter.js +39 -12
  140. package/dist/principal-jwt.d.ts +44 -0
  141. package/dist/principal-jwt.js +95 -0
  142. package/dist/security.d.ts +2 -58
  143. package/dist/security.js +3 -118
  144. package/dist/sema-registry.d.ts +5 -200
  145. package/dist/sema-registry.js +4 -567
  146. package/package.json +1 -1
@@ -0,0 +1,25 @@
1
+ import type { IncomingMessage, ServerResponse } from "node:http";
2
+ export interface SseLogProvider<E extends {
3
+ seq: number;
4
+ }> {
5
+ statusOf(id: string): Promise<{
6
+ status: string;
7
+ updatedAt: string;
8
+ } | null | undefined>;
9
+ getEvents(id: string, after: number): Promise<E[]>;
10
+ retainedFrom(id: string): Promise<number>;
11
+ formatEvent(ev: E): {
12
+ id: number;
13
+ event: string;
14
+ data: unknown;
15
+ };
16
+ staleFrame(): {
17
+ event: string;
18
+ data: unknown;
19
+ };
20
+ }
21
+ export declare function streamSseLog<E extends {
22
+ seq: number;
23
+ }>(req: IncomingMessage, res: ServerResponse, provider: SseLogProvider<E>, id: string, staleMs: number): Promise<void>;
24
+ export declare function sleep(ms: number): Promise<void>;
25
+ //# sourceMappingURL=sse-log.d.ts.map
@@ -0,0 +1,59 @@
1
+ import { sendJson, sseHeaders } from "./send.js";
2
+ export async function streamSseLog(req, res, provider, id, staleMs) {
3
+ const lastId = Number(req.headers["last-event-id"] ?? new URL(req.url ?? "", "http://x").searchParams.get("from") ?? 0);
4
+ let from = Number.isFinite(lastId) ? lastId : 0;
5
+ if (from > 0) {
6
+ const retainedFrom = await provider.retainedFrom(id);
7
+ if (retainedFrom > from + 1) {
8
+ sendJson(res, 416, { error: "resume point evicted past retention", retainedFrom });
9
+ return;
10
+ }
11
+ }
12
+ sseHeaders(res);
13
+ let closed = false;
14
+ req.on("close", () => {
15
+ closed = true;
16
+ });
17
+ const start = Date.now();
18
+ const MAX_MS = 15 * 60 * 1000;
19
+ let lastBeat = Date.now();
20
+ const write = (events) => {
21
+ for (const ev of events) {
22
+ const f = provider.formatEvent(ev);
23
+ res.write(`id: ${f.id}\nevent: ${f.event}\ndata: ${JSON.stringify(f.data)}\n\n`);
24
+ from = ev.seq;
25
+ }
26
+ };
27
+ while (!closed) {
28
+ const [events, row] = await Promise.all([provider.getEvents(id, from), provider.statusOf(id)]);
29
+ write(events);
30
+ if (!row)
31
+ break;
32
+ if (row.status !== "running") {
33
+ write(await provider.getEvents(id, from));
34
+ break;
35
+ }
36
+ const stale = Date.now() - new Date(row.updatedAt).getTime() > staleMs;
37
+ if (stale && events.length === 0) {
38
+ const sf = provider.staleFrame();
39
+ res.write(`event: ${sf.event}\ndata: ${JSON.stringify(sf.data)}\n\n`);
40
+ break;
41
+ }
42
+ if (Date.now() - start > MAX_MS) {
43
+ res.write(`event: error\ndata: ${JSON.stringify({ type: "error", code: "STREAM_MAX_DURATION", message: "stream reached its 15-minute cap — reconnect with Last-Event-ID to continue (the run is still active)" })}\n\n`);
44
+ break;
45
+ }
46
+ if (events.length === 0) {
47
+ if (Date.now() - lastBeat > 15_000) {
48
+ res.write(`event: heartbeat\ndata: {}\n\n`);
49
+ lastBeat = Date.now();
50
+ }
51
+ await sleep(250);
52
+ }
53
+ }
54
+ res.end();
55
+ }
56
+ export function sleep(ms) {
57
+ return new Promise((r) => setTimeout(r, ms));
58
+ }
59
+ //# sourceMappingURL=sse-log.js.map
@@ -0,0 +1,3 @@
1
+ export declare function workspaceContentType(relPath: string): string;
2
+ export declare function looksBinary(bytes: Uint8Array): boolean;
3
+ //# sourceMappingURL=workspace-content.d.ts.map
@@ -0,0 +1,20 @@
1
+ export function workspaceContentType(relPath) {
2
+ const base = relPath.slice(relPath.lastIndexOf("/") + 1);
3
+ const dot = base.lastIndexOf(".");
4
+ const ext = dot > 0 ? base.slice(dot + 1).toLowerCase() : "";
5
+ const TABLE = {
6
+ txt: "text/plain; charset=utf-8", md: "text/plain; charset=utf-8", log: "text/plain; charset=utf-8",
7
+ json: "application/json; charset=utf-8", csv: "text/csv; charset=utf-8",
8
+ png: "image/png", jpg: "image/jpeg", jpeg: "image/jpeg", gif: "image/gif", webp: "image/webp",
9
+ pdf: "application/pdf",
10
+ };
11
+ return TABLE[ext] ?? "application/octet-stream";
12
+ }
13
+ export function looksBinary(bytes) {
14
+ const n = Math.min(bytes.byteLength, 8192);
15
+ for (let i = 0; i < n; i++)
16
+ if (bytes[i] === 0)
17
+ return true;
18
+ return false;
19
+ }
20
+ //# sourceMappingURL=workspace-content.js.map
package/dist/index.d.ts CHANGED
@@ -22,5 +22,5 @@ export { createCouncilTool } from "./capabilities/code-review-council.js";
22
22
  export { loadSkills, skillsForScenario, type LoadedSkill } from "./capabilities/skills.js";
23
23
  export { buildScenarios, selectScenario, type Scenario, type ScenarioBundle, type ScenarioDeps, type ScenarioRequest, } from "./capabilities/scenarios.js";
24
24
  export { createAuthorizer, principalFrom, memoryScopeFor, memoryEngineBackendFor, HttpError, type AuthContext, type OwnerAwareSessionStore, } from "./security.js";
25
- export { createHttpServer, type ServiceDeps, type TaskRequestBody, type RequestAuth, } from "./http/server.js";
25
+ export { createHttpServer, type ServiceDeps, type TaskRequestBody, type RequestAuth, type FlatServiceDeps, type ServiceDepGroups, type ServiceCoreDeps, type ServiceStoreDeps, type ServiceCoordinatorDeps, type ServiceSeamDeps, type ServiceObservabilityDeps, type ServiceGovernanceDeps, type ServiceDeploymentDeps, type ServiceKnobDeps, } from "./http/server.js";
26
26
  //# sourceMappingURL=index.d.ts.map
@@ -161,7 +161,9 @@ export async function runLeaderTask(task, durableBaseSha, deps) {
161
161
  }));
162
162
  return out;
163
163
  };
164
- if (deps.moduleGate) {
164
+ const runModuleGate = async () => {
165
+ if (!deps.moduleGate)
166
+ return;
165
167
  const subById = new Map(subs.map((s) => [s.workerId, s]));
166
168
  const maxR = deps.moduleGate.maxRepairs ?? 1;
167
169
  await Promise.all(reports.filter((r) => r.status === "completed").map(async (r) => {
@@ -194,7 +196,8 @@ export async function runLeaderTask(task, durableBaseSha, deps) {
194
196
  }
195
197
  }
196
198
  }));
197
- }
199
+ };
200
+ await runModuleGate();
198
201
  const mergeDeps = {
199
202
  pullWorkerDiff: async (r) => workerDiff(byId.get(r.workerId)),
200
203
  provisionIntegrationSandbox: deps.provisionIntegrationSandbox,
@@ -192,186 +192,190 @@ export function createLeaderRunner(cfg) {
192
192
  if (!ex.ok)
193
193
  throw ex.error;
194
194
  };
195
- const deps = {
196
- plan: async () => {
197
- if (Array.isArray(body.subtasks) && body.subtasks.length > 0) {
198
- return validateSubtasks(body.subtasks);
199
- }
200
- const planRunner = new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing });
201
- const runRoute = async (prompt) => {
202
- const res = await planRunner.runTaskStream({
203
- objective: prompt,
204
- sessionId: `leader-route-${Date.now()}`,
205
- ...(cfg.routerModel ? { model: cfg.routerModel } : {}),
206
- }).result();
207
- return res.result ?? "";
195
+ const plan = async () => {
196
+ if (Array.isArray(body.subtasks) && body.subtasks.length > 0) {
197
+ return validateSubtasks(body.subtasks);
198
+ }
199
+ const planRunner = new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing });
200
+ const runRoute = async (prompt) => {
201
+ const res = await planRunner.runTaskStream({
202
+ objective: prompt,
203
+ sessionId: `leader-route-${Date.now()}`,
204
+ ...(cfg.routerModel ? { model: cfg.routerModel } : {}),
205
+ }).result();
206
+ return res.result ?? "";
207
+ };
208
+ const { subs, collapsed, rawSample } = await routePlanWithFallback(body.objective, runRoute, {
209
+ ...(body.planContext ? { context: body.planContext } : {}),
210
+ ...(cfg.fanoutEnabled === false ? { fanoutEnabled: false } : {}),
211
+ });
212
+ if (collapsed)
213
+ cfg.logger?.warn?.("plan_collapsed_to_single", { reason: collapsed, ...(rawSample ? { rawSample } : {}) });
214
+ return subs;
215
+ };
216
+ const provisionWorker = async (sub) => {
217
+ const sessionId = `leader-${sub.workerId}-${Date.now()}`;
218
+ let durableSpec = {};
219
+ if (cfg.durable) {
220
+ const d = cfg.durable;
221
+ durableSpec = {
222
+ toolPolicy: combinePolicies(createDurableQuestionPolicy(), createDurableAskPolicy({ requireApproval: d.requireApproval, ...(d.deny ? { deny: d.deny } : {}), ...(d.autoBudget ? { autoBudget: d.autoBudget } : {}), ...(d.neverAuto ? { neverAuto: d.neverAuto } : {}) })),
223
+ checkpointStore: d.checkpointStore,
224
+ durableApproval: { scope: "_leader", ...(d.ttlMs ? { ttlMs: d.ttlMs } : {}) },
225
+ onQuestion: QUESTION_AWAITS_RESUME,
208
226
  };
209
- const { subs, collapsed, rawSample } = await routePlanWithFallback(body.objective, runRoute, {
210
- ...(body.planContext ? { context: body.planContext } : {}),
211
- ...(cfg.fanoutEnabled === false ? { fanoutEnabled: false } : {}),
227
+ await d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch((e) => {
228
+ cfg.logger?.warn?.("leader_putctx_failed", { workerId: sub.workerId, err: String(e) });
212
229
  });
213
- if (collapsed)
214
- cfg.logger?.warn?.("plan_collapsed_to_single", { reason: collapsed, ...(rawSample ? { rawSample } : {}) });
215
- return subs;
216
- },
217
- provisionWorker: async (sub) => {
218
- const sessionId = `leader-${sub.workerId}-${Date.now()}`;
219
- let durableSpec = {};
220
- if (cfg.durable) {
221
- const d = cfg.durable;
222
- durableSpec = {
223
- toolPolicy: combinePolicies(createDurableQuestionPolicy(), createDurableAskPolicy({ requireApproval: d.requireApproval, ...(d.deny ? { deny: d.deny } : {}), ...(d.autoBudget ? { autoBudget: d.autoBudget } : {}), ...(d.neverAuto ? { neverAuto: d.neverAuto } : {}) })),
224
- checkpointStore: d.checkpointStore,
225
- durableApproval: { scope: "_leader", ...(d.ttlMs ? { ttlMs: d.ttlMs } : {}) },
226
- onQuestion: QUESTION_AWAITS_RESUME,
227
- };
228
- await d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch((e) => {
229
- cfg.logger?.warn?.("leader_putctx_failed", { workerId: sub.workerId, err: String(e) });
230
- });
231
- }
232
- let resourceSpec = {};
233
- const repairLoopActive = repairLoopOn && REPAIR_LOOP_WIRED;
234
- if (repairLoopActive && resourceCfg) {
235
- cfg.logger?.warn?.("leader_resource_suspend_disabled_for_repair_loop", { workerId: sub.workerId, reason: "LEADER_REPAIR_LOOP active — resource-suspend mutually exclusive with the repair loop (§10.5)" });
236
- }
237
- if (resourceCfg && !repairLoopActive && cfg.durable && cfg.toolResultStore && cfg.sessionStore) {
238
- resourceSpec = {
239
- checkpointStore: cfg.durable.checkpointStore,
240
- resourceSuspend: { scope: `_leader-resource-${sub.workerId}`, totalBudgetUsd: resourceCfg.totalBudgetUsd },
241
- maxCostUsd: resourceCfg.sliceMaxCostUsd,
242
- maxSuspends: resourceCfg.maxSuspends,
243
- };
244
- }
245
- else if (resourceCfg && !repairLoopActive) {
246
- cfg.logger?.warn?.("leader_resource_suspend_inactive", {
247
- workerId: sub.workerId,
248
- reason: !cfg.durable
249
- ? "no durable checkpoint store"
250
- : !cfg.toolResultStore
251
- ? "no durable tool-result store"
252
- : "no durable session store",
253
- });
254
- }
255
- const keepCtxWarm = (runner) => {
256
- const d = cfg.durable;
257
- if (!d)
258
- return runner;
259
- const orig = runner.runTaskStream.bind(runner);
260
- runner.runTaskStream = (spec, resume) => {
261
- const stream = orig(spec, resume);
262
- const iv = setInterval(() => {
263
- void d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch(() => { });
264
- }, 45_000);
265
- iv.unref?.();
266
- void stream.result().then(() => clearInterval(iv), () => clearInterval(iv));
267
- return stream;
268
- };
230
+ }
231
+ let resourceSpec = {};
232
+ const repairLoopActive = repairLoopOn && REPAIR_LOOP_WIRED;
233
+ if (repairLoopActive && resourceCfg) {
234
+ cfg.logger?.warn?.("leader_resource_suspend_disabled_for_repair_loop", { workerId: sub.workerId, reason: "LEADER_REPAIR_LOOP active — resource-suspend mutually exclusive with the repair loop (§10.5)" });
235
+ }
236
+ if (resourceCfg && !repairLoopActive && cfg.durable && cfg.toolResultStore && cfg.sessionStore) {
237
+ resourceSpec = {
238
+ checkpointStore: cfg.durable.checkpointStore,
239
+ resourceSuspend: { scope: `_leader-resource-${sub.workerId}`, totalBudgetUsd: resourceCfg.totalBudgetUsd },
240
+ maxCostUsd: resourceCfg.sliceMaxCostUsd,
241
+ maxSuspends: resourceCfg.maxSuspends,
242
+ };
243
+ }
244
+ else if (resourceCfg && !repairLoopActive) {
245
+ cfg.logger?.warn?.("leader_resource_suspend_inactive", {
246
+ workerId: sub.workerId,
247
+ reason: !cfg.durable
248
+ ? "no durable checkpoint store"
249
+ : !cfg.toolResultStore
250
+ ? "no durable tool-result store"
251
+ : "no durable session store",
252
+ });
253
+ }
254
+ const keepCtxWarm = (runner) => {
255
+ const d = cfg.durable;
256
+ if (!d)
269
257
  return runner;
258
+ const orig = runner.runTaskStream.bind(runner);
259
+ runner.runTaskStream = (spec, resume) => {
260
+ const stream = orig(spec, resume);
261
+ const iv = setInterval(() => {
262
+ void d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch(() => { });
263
+ }, 45_000);
264
+ iv.unref?.();
265
+ void stream.result().then(() => clearInterval(iv), () => clearInterval(iv));
266
+ return stream;
267
+ };
268
+ return runner;
269
+ };
270
+ if (cfg.s3 && cfg.envFactory) {
271
+ const s3 = cfg.s3;
272
+ const envFactory = cfg.envFactory;
273
+ const baseSha = body.baseSha;
274
+ const seedCmd = body.seedCmd;
275
+ const key = `${s3.keyPrefix ?? "leader-diff"}/${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}/${sub.workerId}.patch`;
276
+ const ttl = presignTtlSec;
277
+ const base = { endpoint: s3.endpoint, bucket: s3.bucket, accessKey: s3.accessKey, secretKey: s3.secretKey, ...(s3.region ? { region: s3.region } : {}), key, expiresSec: ttl };
278
+ const putUrl = presignS3Url({ ...base, method: "PUT" });
279
+ const getUrl = presignS3Url({ ...base, method: "GET" });
280
+ const script = buildUploadScript({ repoDir: repo, baseSha, putUrl, workDir: W });
281
+ const stage = async (env) => {
282
+ await stageWorkerEnv(env, { seedCmd, repo, branch: sub.branch, uploadScriptPath: `${W}/.leader-upload.sh`, uploadScript: script });
283
+ void sh(env)(`nohup sh -c 'while true; do sleep 90; sh ${W}/.leader-upload.sh >/dev/null 2>&1 || true; done' </dev/null >/dev/null 2>&1 &`).catch(() => { });
270
284
  };
271
- if (cfg.s3 && cfg.envFactory) {
272
- const s3 = cfg.s3;
273
- const envFactory = cfg.envFactory;
274
- const baseSha = body.baseSha;
275
- const seedCmd = body.seedCmd;
276
- const key = `${s3.keyPrefix ?? "leader-diff"}/${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}/${sub.workerId}.patch`;
277
- const ttl = presignTtlSec;
278
- const base = { endpoint: s3.endpoint, bucket: s3.bucket, accessKey: s3.accessKey, secretKey: s3.secretKey, ...(s3.region ? { region: s3.region } : {}), key, expiresSec: ttl };
279
- const putUrl = presignS3Url({ ...base, method: "PUT" });
280
- const getUrl = presignS3Url({ ...base, method: "GET" });
281
- const script = buildUploadScript({ repoDir: repo, baseSha, putUrl, workDir: W });
282
- const stage = async (env) => {
283
- await stageWorkerEnv(env, { seedCmd, repo, branch: sub.branch, uploadScriptPath: `${W}/.leader-upload.sh`, uploadScript: script });
284
- void sh(env)(`nohup sh -c 'while true; do sleep 90; sh ${W}/.leader-upload.sh >/dev/null 2>&1 || true; done' </dev/null >/dev/null 2>&1 &`).catch(() => { });
285
- };
286
- return {
287
- workerId: sub.workerId, sessionId, branch: sub.branch,
288
- baseSha,
289
- runner: keepCtxWarm(new Runner({
290
- brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing,
291
- ...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}),
292
- ...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}),
293
- executionEnvFactory: (ctx) => withStaging(envFactory(ctx), stage, async (e) => {
294
- await sh(e)(`cd ${repo} && git add -A && git -c user.name='leader-belt' -c user.email='belt@leader' commit -qm 'belt: auto-commit uncommitted worker output' || true; sh ${W}/.leader-upload.sh`);
295
- }),
296
- })),
297
- fetchDiff: () => fetchUploadedDiff(getUrl),
298
- spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec, objective: `${sub.spec.objective}${uploadStepSuffix(W)}` },
299
- };
300
- }
301
- if (!cfg.e2bApiKey)
302
- throw new Error("leader wire: no envFactory+s3 (factory mode) and no e2bApiKey (static mode)");
303
- const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
304
- if (!(await env.connect()).ok)
305
- throw new Error(`connect failed for ${sub.workerId}`);
306
- await sh(env)(`${body.seedCmd}; cd ${repo}; git checkout -q -b ${sub.branch}`);
307
- const baseSha = await sh(env)(`cd ${repo} && git rev-parse HEAD`);
308
285
  return {
309
- workerId: sub.workerId, sessionId, branch: sub.branch, baseSha,
310
- runner: keepCtxWarm(new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing, ...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}), ...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}), executionEnv: env })),
311
- diffEnv: env,
312
- destroy: () => env.destroy().then(() => { }),
313
- spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec },
286
+ workerId: sub.workerId, sessionId, branch: sub.branch,
287
+ baseSha,
288
+ runner: keepCtxWarm(new Runner({
289
+ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing,
290
+ ...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}),
291
+ ...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}),
292
+ executionEnvFactory: (ctx) => withStaging(envFactory(ctx), stage, async (e) => {
293
+ await sh(e)(`cd ${repo} && git add -A && git -c user.name='leader-belt' -c user.email='belt@leader' commit -qm 'belt: auto-commit uncommitted worker output' || true; sh ${W}/.leader-upload.sh`);
294
+ }),
295
+ })),
296
+ fetchDiff: () => fetchUploadedDiff(getUrl),
297
+ spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec, objective: `${sub.spec.objective}${uploadStepSuffix(W)}` },
314
298
  };
315
- },
316
- provisionIntegrationSandbox: async () => {
317
- if (cfg.envFactory) {
318
- const env = cfg.envFactory({ sessionId: `leader-integ-${Date.now()}` });
319
- try {
320
- await sh(env)(body.seedCmd);
321
- await injectOracles(env);
322
- }
323
- catch (e) {
324
- await Promise.resolve(env.destroy()).catch(() => { });
325
- throw e;
326
- }
327
- return { env: asIntegrationEnv(env), destroy: () => Promise.resolve(env.destroy()).then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
328
- }
329
- if (!cfg.e2bApiKey)
330
- throw new Error("leader wire: no envFactory and no e2bApiKey");
331
- const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
332
- if (!(await env.connect()).ok) {
333
- await env.destroy().catch(() => { });
334
- throw new Error("integ connect failed");
335
- }
299
+ }
300
+ if (!cfg.e2bApiKey)
301
+ throw new Error("leader wire: no envFactory+s3 (factory mode) and no e2bApiKey (static mode)");
302
+ const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
303
+ if (!(await env.connect()).ok)
304
+ throw new Error(`connect failed for ${sub.workerId}`);
305
+ await sh(env)(`${body.seedCmd}; cd ${repo}; git checkout -q -b ${sub.branch}`);
306
+ const baseSha = await sh(env)(`cd ${repo} && git rev-parse HEAD`);
307
+ return {
308
+ workerId: sub.workerId, sessionId, branch: sub.branch, baseSha,
309
+ runner: keepCtxWarm(new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing, ...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}), ...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}), executionEnv: env })),
310
+ diffEnv: env,
311
+ destroy: () => env.destroy().then(() => { }),
312
+ spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec },
313
+ };
314
+ };
315
+ const provisionIntegrationSandbox = async () => {
316
+ if (cfg.envFactory) {
317
+ const env = cfg.envFactory({ sessionId: `leader-integ-${Date.now()}` });
336
318
  try {
337
319
  await sh(env)(body.seedCmd);
338
320
  await injectOracles(env);
339
321
  }
340
322
  catch (e) {
341
- await env.destroy().catch(() => { });
323
+ await Promise.resolve(env.destroy()).catch(() => { });
342
324
  throw e;
343
325
  }
344
- return { env: asIntegrationEnv(env), destroy: () => env.destroy().then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
345
- },
346
- push: async (integratedPatch, baseSha) => {
347
- const dir = mkdtempSync(join(tmpdir(), "leader-coord-"));
348
- const gitEnv = { ...process.env, LC_ALL: "C" };
349
- try {
350
- execFileSync("git", ["clone", "-q", body.durableRemote, dir], { env: gitEnv });
351
- execFileSync("git", ["-C", dir, "config", "user.name", ident.name]);
352
- execFileSync("git", ["-C", dir, "config", "user.email", ident.email]);
353
- writeFileSync(join(dir, ".leader.patch"), integratedPatch);
354
- execFileSync("git", ["-C", dir, "am", "--3way", ".leader.patch"], { env: gitEnv });
355
- rmSync(join(dir, ".leader.patch"));
356
- const ref = body.targetRef ?? "refs/heads/main";
357
- const ls = execFileSync("git", ["-C", dir, "ls-remote", "--refs", "origin", ref], { encoding: "utf8", env: gitEnv });
358
- const exists = ls.split("\n").some((l) => l.split("\t")[1] === ref);
359
- if (exists) {
360
- execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:${baseSha}`, "origin", `HEAD:${ref}`], { env: gitEnv });
361
- }
362
- else {
363
- execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:`, "origin", `HEAD:${ref}`], { env: gitEnv });
364
- }
365
- return { ok: true, ref };
366
- }
367
- catch (e) {
368
- const msg = e instanceof Error ? e.message : String(e);
369
- return { ok: false, raced: /stale info|force-with-lease|\[rejected\]|non-fast-forward/.test(msg), error: msg };
326
+ return { env: asIntegrationEnv(env), destroy: () => Promise.resolve(env.destroy()).then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
327
+ }
328
+ if (!cfg.e2bApiKey)
329
+ throw new Error("leader wire: no envFactory and no e2bApiKey");
330
+ const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
331
+ if (!(await env.connect()).ok) {
332
+ await env.destroy().catch(() => { });
333
+ throw new Error("integ connect failed");
334
+ }
335
+ try {
336
+ await sh(env)(body.seedCmd);
337
+ await injectOracles(env);
338
+ }
339
+ catch (e) {
340
+ await env.destroy().catch(() => { });
341
+ throw e;
342
+ }
343
+ return { env: asIntegrationEnv(env), destroy: () => env.destroy().then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
344
+ };
345
+ const push = async (integratedPatch, baseSha) => {
346
+ const dir = mkdtempSync(join(tmpdir(), "leader-coord-"));
347
+ const gitEnv = { ...process.env, LC_ALL: "C" };
348
+ try {
349
+ execFileSync("git", ["clone", "-q", body.durableRemote, dir], { env: gitEnv });
350
+ execFileSync("git", ["-C", dir, "config", "user.name", ident.name]);
351
+ execFileSync("git", ["-C", dir, "config", "user.email", ident.email]);
352
+ writeFileSync(join(dir, ".leader.patch"), integratedPatch);
353
+ execFileSync("git", ["-C", dir, "am", "--3way", ".leader.patch"], { env: gitEnv });
354
+ rmSync(join(dir, ".leader.patch"));
355
+ const ref = body.targetRef ?? "refs/heads/main";
356
+ const ls = execFileSync("git", ["-C", dir, "ls-remote", "--refs", "origin", ref], { encoding: "utf8", env: gitEnv });
357
+ const exists = ls.split("\n").some((l) => l.split("\t")[1] === ref);
358
+ if (exists) {
359
+ execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:${baseSha}`, "origin", `HEAD:${ref}`], { env: gitEnv });
370
360
  }
371
- finally {
372
- rmSync(dir, { recursive: true, force: true });
361
+ else {
362
+ execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:`, "origin", `HEAD:${ref}`], { env: gitEnv });
373
363
  }
374
- },
364
+ return { ok: true, ref };
365
+ }
366
+ catch (e) {
367
+ const msg = e instanceof Error ? e.message : String(e);
368
+ return { ok: false, raced: /stale info|force-with-lease|\[rejected\]|non-fast-forward/.test(msg), error: msg };
369
+ }
370
+ finally {
371
+ rmSync(dir, { recursive: true, force: true });
372
+ }
373
+ };
374
+ const deps = {
375
+ plan,
376
+ provisionWorker,
377
+ provisionIntegrationSandbox,
378
+ push,
375
379
  testCmd: body.testCmd,
376
380
  repoDir: repo, workerRepoDir: repo,
377
381
  timeoutMs: leaderTimeoutMs, maxConcurrency: 4,