@sema-agent/server 1.314.0 → 1.315.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/dist/approval-hmac.d.ts +17 -0
  2. package/dist/approval-hmac.js +27 -0
  3. package/dist/config-center/apply-effective.d.ts +22 -0
  4. package/dist/config-center/apply-effective.js +283 -0
  5. package/dist/config-center/http-client.d.ts +23 -0
  6. package/dist/config-center/http-client.js +109 -0
  7. package/dist/config-center/restart-signal.d.ts +12 -0
  8. package/dist/config-center/restart-signal.js +70 -0
  9. package/dist/config-center/skills-mcp.d.ts +13 -0
  10. package/dist/config-center/skills-mcp.js +113 -0
  11. package/dist/config-center/types.d.ts +143 -0
  12. package/dist/config-center/types.js +2 -0
  13. package/dist/fleet/fleet-bus.js +92 -83
  14. package/dist/hooks/hook-runner.js +18 -9
  15. package/dist/http/server.d.ts +95 -69
  16. package/dist/http/server.js +8 -1
  17. package/dist/index.d.ts +1 -1
  18. package/dist/leader/leader.js +5 -2
  19. package/dist/leader/wire.js +169 -165
  20. package/dist/main.js +462 -451
  21. package/dist/plugins/checkpoint-store-sql.d.ts +67 -0
  22. package/dist/plugins/checkpoint-store-sql.js +224 -0
  23. package/dist/plugins/image-bake-store-sql.d.ts +53 -0
  24. package/dist/plugins/image-bake-store-sql.js +463 -0
  25. package/dist/plugins/k8s-bg-scripts.d.ts +19 -0
  26. package/dist/plugins/k8s-bg-scripts.js +129 -0
  27. package/dist/plugins/k8s-exec-protocol.d.ts +20 -0
  28. package/dist/plugins/k8s-exec-protocol.js +87 -0
  29. package/dist/plugins/pg-checkpoint-store.d.ts +1 -33
  30. package/dist/plugins/pg-checkpoint-store.js +1 -189
  31. package/dist/plugins/pg-cost-quota.js +3 -143
  32. package/dist/plugins/pg-image-bake.d.ts +1 -40
  33. package/dist/plugins/pg-image-bake.js +1 -420
  34. package/dist/plugins/pg-rate-limiter.d.ts +2 -32
  35. package/dist/plugins/pg-rate-limiter.js +4 -147
  36. package/dist/plugins/remote-env-k8s.d.ts +5 -38
  37. package/dist/plugins/remote-env-k8s.js +7 -213
  38. package/dist/plugins/sql-driver.d.ts +25 -0
  39. package/dist/plugins/sql-driver.js +59 -0
  40. package/dist/plugins/tidb-checkpoint-store.d.ts +1 -49
  41. package/dist/plugins/tidb-checkpoint-store.js +1 -191
  42. package/dist/plugins/tidb-image-bake.d.ts +1 -38
  43. package/dist/plugins/tidb-image-bake.js +1 -348
  44. package/dist/plugins/write-behind-counter.d.ts +14 -3
  45. package/dist/plugins/write-behind-counter.js +39 -12
  46. package/dist/principal-jwt.d.ts +44 -0
  47. package/dist/principal-jwt.js +95 -0
  48. package/dist/security.d.ts +2 -58
  49. package/dist/security.js +3 -118
  50. package/dist/sema-registry.d.ts +5 -200
  51. package/dist/sema-registry.js +4 -567
  52. package/package.json +1 -1
@@ -30,58 +30,59 @@ export interface RequestAuth {
30
30
  memoryScope?: string;
31
31
  resolvedProjectId?: string;
32
32
  }
33
- export interface ServiceDeps {
33
+ export interface ServiceCoreDeps {
34
34
  runner: Runner;
35
35
  config: ServiceConfig;
36
- registryJwtVerifier?: {
37
- verify: (bearer: string) => Promise<import("../auth-bridge.js").VerifyOutcome>;
38
- };
39
- hookWakeBus?: {
40
- deliver?: (sessionId: string, text: string) => Promise<boolean>;
41
- };
42
- drainState?: {
43
- draining: boolean;
44
- since?: number;
45
- inflight?: () => number;
46
- lastActivityAt?: () => number;
47
- };
48
- storeDegraded?: boolean;
49
- sessionStoreLabel?: string;
50
- modelReady?: () => boolean;
51
- scenarioDetails?: Record<string, import("../capabilities/scenarios.js").ScenarioDetail>;
52
- authorize?: (ctx: {
53
- req: IncomingMessage;
54
- body: TaskRequestBody;
55
- }) => Promise<RequestAuth>;
56
36
  resolveSpec: (body: TaskRequestBody, req: IncomingMessage | undefined, auth?: RequestAuth, opts?: {
57
37
  leg?: "fresh" | "resume";
58
38
  }) => TaskSpec | Promise<TaskSpec>;
39
+ }
40
+ export interface ServiceStoreDeps {
59
41
  runStore?: RunStore;
60
42
  resumeAnchorStore?: ResumeAnchorStore;
61
- sessionTitler?: {
62
- maybeTitle(sessionId: string, objective: string): void;
63
- };
43
+ approvalStore?: ApprovalStore;
64
44
  approvalExemptionStore?: ApprovalExemptionStore;
45
+ checkpointStore?: CheckpointStoreFull;
65
46
  sessionPolicyStore?: ServiceSessionPolicyStore;
66
47
  fileSnapshotStore?: ServiceFileSnapshotStore;
67
- snapshotBlobSqlCapBytes?: number;
68
- backend?: StoreBackend;
69
- sessionMirrorRuling?: (principal: string | undefined) => Promise<SessionMirrorRuling | undefined>;
70
- approvalStore?: ApprovalStore;
71
- checkpointStore?: CheckpointStoreFull;
72
- backgroundAgentStore?: import("@sema-agent/core").BackgroundAgentStore;
73
- parkedReviveTool?: import("@sema-agent/core").ToolSpec;
74
- parkedKnownAgentTypes?: ReadonlySet<string>;
75
- parkedReviveInheritedGate?: (row: import("@sema-agent/core").BackgroundAgentRecord) => unknown;
76
48
  taskAttachmentStore?: TaskAttachmentStore;
77
- imageIndex?: ImageIndex;
78
- imageBakes?: ImageBake;
79
- leaderEndpoint?: LeaderEndpoint;
49
+ backgroundAgentStore?: import("@sema-agent/core").BackgroundAgentStore;
50
+ sessionStorage?: OwnerAwareSessionStore;
80
51
  workflowRunStore?: WorkflowRunStore;
81
52
  workflowJournalStore?: import("@sema-agent/core").WorkflowJournalStore;
82
- workflowsCapable?: boolean;
53
+ imageIndex?: ImageIndex;
54
+ imageBakes?: ImageBake;
55
+ outcomeSink?: import("../plugins/file-outcome-sink.js").OutcomeSink;
56
+ sendFileLedger?: import("../plugins/send-file-ledger.js").SendFileLedger;
57
+ backend?: StoreBackend;
58
+ }
59
+ export interface ServiceCoordinatorDeps {
60
+ elicitation?: ElicitationCoordinator;
61
+ question?: QuestionCoordinator;
62
+ toolApproval?: ToolApprovalCoordinator;
83
63
  workflowAgentRegistry?: WorkflowAgentRegistry;
84
64
  subagentSteerRegistry?: SubagentSteerRegistry;
65
+ workflowCompletionInbox?: WorkflowCompletionInbox;
66
+ fleetBus?: FleetEventBus;
67
+ sessionWatch?: import("../session-watch.js").SessionWatchRegistry;
68
+ hookWakeBus?: {
69
+ deliver?: (sessionId: string, text: string) => Promise<boolean>;
70
+ };
71
+ sendUserFile?: import("../capabilities/send-user-file-tool.js").SendUserFileEmitter;
72
+ leaderEndpoint?: LeaderEndpoint;
73
+ sessionTitler?: {
74
+ maybeTitle(sessionId: string, objective: string): void;
75
+ };
76
+ }
77
+ export interface ServiceSeamDeps {
78
+ sessionAudit?: (sessionId: string) => Promise<({
79
+ owner: string | null;
80
+ } & Record<string, unknown>) | undefined>;
81
+ purgeSession?: (sessionId: string, owner: string | null) => Promise<{
82
+ deleted: boolean;
83
+ } | {
84
+ active: string;
85
+ }>;
85
86
  subagentTaskOutput?: (handle: string, access: {
86
87
  owner: string;
87
88
  scope: string;
@@ -106,23 +107,10 @@ export interface ServiceDeps {
106
107
  content: string;
107
108
  details: unknown;
108
109
  }>;
109
- workflowCompletionInbox?: WorkflowCompletionInbox;
110
- fleetBus?: FleetEventBus;
111
- sessionAudit?: (sessionId: string) => Promise<({
112
- owner: string | null;
113
- } & Record<string, unknown>) | undefined>;
114
- sessionStorage?: OwnerAwareSessionStore;
115
- purgeSession?: (sessionId: string, owner: string | null) => Promise<{
116
- deleted: boolean;
117
- } | {
118
- active: string;
119
- }>;
120
- instanceId?: string;
121
- logger?: Logger;
122
- metrics?: Metrics;
123
- rateLimiter?: RateGate;
124
- costQuota?: QuotaTracker;
125
- fleetLease?: FleetLeaseManager;
110
+ memoryExport?: (scope: string) => Promise<MemoryEntry[]>;
111
+ memorySync?: (scope: string, syncReq: MemorySyncRequest) => Promise<MemorySyncResponse>;
112
+ sessionMirrorRuling?: (principal: string | undefined) => Promise<SessionMirrorRuling | undefined>;
113
+ instrumentDegenerate?: (result: TaskResult) => void;
126
114
  sideQueryAccounting?: (principal: string | undefined, r: {
127
115
  model: string;
128
116
  family?: "input-includes-cached" | "input-excludes-cached";
@@ -136,30 +124,68 @@ export interface ServiceDeps {
136
124
  };
137
125
  };
138
126
  }) => void;
139
- outcomeSink?: import("../plugins/file-outcome-sink.js").OutcomeSink;
140
- memoryExport?: (scope: string) => Promise<MemoryEntry[]>;
141
- memorySync?: (scope: string, syncReq: MemorySyncRequest) => Promise<MemorySyncResponse>;
142
- instrumentDegenerate?: (result: TaskResult) => void;
143
- planCacheProbe?: PlanCacheProbe;
127
+ parkedReviveTool?: import("@sema-agent/core").ToolSpec;
128
+ parkedKnownAgentTypes?: ReadonlySet<string>;
129
+ parkedReviveInheritedGate?: (row: import("@sema-agent/core").BackgroundAgentRecord) => unknown;
130
+ }
131
+ export interface ServiceObservabilityDeps {
132
+ logger?: Logger;
133
+ metrics?: Metrics;
144
134
  modelUsage?: ModelUsageTracker;
145
135
  promptManifests?: PromptManifestTracker;
146
- elicitation?: ElicitationCoordinator;
147
- question?: QuestionCoordinator;
148
- toolApproval?: ToolApprovalCoordinator;
149
- sendUserFile?: import("../capabilities/send-user-file-tool.js").SendUserFileEmitter;
150
- sendFileLedger?: import("../plugins/send-file-ledger.js").SendFileLedger;
151
- sessionWatch?: import("../session-watch.js").SessionWatchRegistry;
152
- sessionEventsMaxConnections?: number;
153
- sessionEventsHeartbeatMs?: number;
154
- sessionEventsStallMs?: number;
136
+ planCacheProbe?: PlanCacheProbe;
137
+ }
138
+ export interface ServiceGovernanceDeps {
139
+ authorize?: (ctx: {
140
+ req: IncomingMessage;
141
+ body: TaskRequestBody;
142
+ }) => Promise<RequestAuth>;
143
+ registryJwtVerifier?: {
144
+ verify: (bearer: string) => Promise<import("../auth-bridge.js").VerifyOutcome>;
145
+ };
146
+ rateLimiter?: RateGate;
147
+ costQuota?: QuotaTracker;
148
+ fleetLease?: FleetLeaseManager;
149
+ }
150
+ export interface ServiceDeploymentDeps {
151
+ instanceId?: string;
155
152
  capabilities?: Record<string, unknown>;
153
+ scenarioDetails?: Record<string, import("../capabilities/scenarios.js").ScenarioDetail>;
154
+ sessionStoreLabel?: string;
155
+ storeDegraded?: boolean;
156
+ drainState?: {
157
+ draining: boolean;
158
+ since?: number;
159
+ inflight?: () => number;
160
+ lastActivityAt?: () => number;
161
+ };
162
+ modelReady?: () => boolean;
156
163
  restartState?: () => RestartSignal | undefined;
157
164
  planeDeferredState?: () => {
158
165
  version: number;
159
166
  since: number;
160
167
  blocked?: string[];
161
168
  } | undefined;
169
+ workflowsCapable?: boolean;
170
+ }
171
+ export interface ServiceKnobDeps {
172
+ snapshotBlobSqlCapBytes?: number;
173
+ sessionEventsMaxConnections?: number;
174
+ sessionEventsHeartbeatMs?: number;
175
+ sessionEventsStallMs?: number;
176
+ }
177
+ export interface ServiceDepGroups {
178
+ stores?: ServiceStoreDeps;
179
+ coordinators?: ServiceCoordinatorDeps;
180
+ seams?: ServiceSeamDeps;
181
+ observability?: ServiceObservabilityDeps;
182
+ governance?: ServiceGovernanceDeps;
183
+ deployment?: ServiceDeploymentDeps;
184
+ knobs?: ServiceKnobDeps;
162
185
  }
186
+ export type FlatServiceDeps = ServiceCoreDeps & Partial<ServiceStoreDeps & ServiceCoordinatorDeps & ServiceSeamDeps & ServiceObservabilityDeps & ServiceGovernanceDeps & ServiceDeploymentDeps & ServiceKnobDeps>;
187
+ export type ServiceDeps = FlatServiceDeps & ServiceDepGroups;
188
+ export declare function flattenServiceDeps(deps: ServiceDeps): FlatServiceDeps;
163
189
  export type { TaskRequestBody } from "./wire-types.js";
164
190
  export declare const MAX_USER_SKILLS = 10;
165
191
  export declare const MAX_SKILL_CONTENT_CHARS = 1048576;
@@ -175,7 +201,7 @@ export declare function cascadeConfig(ladder: readonly string[], maxCostUsd?: nu
175
201
  export declare function clampVerifyRounds(v: unknown): number;
176
202
  export declare function scopedIdempotencyKey(raw: string | undefined, source: string | null, principal: string | undefined): string | undefined;
177
203
  export declare function isQuestionAnswer(v: unknown): v is QuestionAnswer;
178
- export declare function createHttpServer(deps: ServiceDeps): http.Server & {
204
+ export declare function createHttpServer(rawDeps: ServiceDeps): http.Server & {
179
205
  denyExpiredApprovals: (now: number) => Promise<void>;
180
206
  };
181
207
  export declare function streamApprovals(req: IncomingMessage, res: ServerResponse, cs: Pick<CheckpointStoreFull, "listPending">, scope: string | undefined, pollMs?: number): Promise<void>;
@@ -35,6 +35,12 @@ import { redactSecrets, redactDeep, redactedPreview } from "../trace/redact.js";
35
35
  import { usageSummary, usageSeries, usageBreakdown } from "../usage-analytics.js";
36
36
  import { IdempotencyCache } from "./idempotency.js";
37
37
  import { pgHasUnstorable } from "../plugins/pg-safe-json.js";
38
+ export function flattenServiceDeps(deps) {
39
+ const { stores, coordinators, seams, observability, governance, deployment, knobs, ...flat } = deps;
40
+ if (!(stores || coordinators || seams || observability || governance || deployment || knobs))
41
+ return deps;
42
+ return { ...stores, ...coordinators, ...seams, ...observability, ...governance, ...deployment, ...knobs, ...flat };
43
+ }
38
44
  export const MAX_USER_SKILLS = 10;
39
45
  export const MAX_SKILL_CONTENT_CHARS = 1_048_576;
40
46
  export const MAX_SYSTEM_PROMPT_CHARS = 16_384;
@@ -195,7 +201,8 @@ function bearerPresentedButUnverified(req, config) {
195
201
  return false;
196
202
  return true;
197
203
  }
198
- export function createHttpServer(deps) {
204
+ export function createHttpServer(rawDeps) {
205
+ const deps = flattenServiceDeps(rawDeps);
199
206
  const idemCache = new IdempotencyCache();
200
207
  let sseConnections = 0;
201
208
  let sseOwnerLookups = 0;
package/dist/index.d.ts CHANGED
@@ -22,5 +22,5 @@ export { createCouncilTool } from "./capabilities/code-review-council.js";
22
22
  export { loadSkills, skillsForScenario, type LoadedSkill } from "./capabilities/skills.js";
23
23
  export { buildScenarios, selectScenario, type Scenario, type ScenarioBundle, type ScenarioDeps, type ScenarioRequest, } from "./capabilities/scenarios.js";
24
24
  export { createAuthorizer, principalFrom, memoryScopeFor, memoryEngineBackendFor, HttpError, type AuthContext, type OwnerAwareSessionStore, } from "./security.js";
25
- export { createHttpServer, type ServiceDeps, type TaskRequestBody, type RequestAuth, } from "./http/server.js";
25
+ export { createHttpServer, type ServiceDeps, type TaskRequestBody, type RequestAuth, type FlatServiceDeps, type ServiceDepGroups, type ServiceCoreDeps, type ServiceStoreDeps, type ServiceCoordinatorDeps, type ServiceSeamDeps, type ServiceObservabilityDeps, type ServiceGovernanceDeps, type ServiceDeploymentDeps, type ServiceKnobDeps, } from "./http/server.js";
26
26
  //# sourceMappingURL=index.d.ts.map
@@ -161,7 +161,9 @@ export async function runLeaderTask(task, durableBaseSha, deps) {
161
161
  }));
162
162
  return out;
163
163
  };
164
- if (deps.moduleGate) {
164
+ const runModuleGate = async () => {
165
+ if (!deps.moduleGate)
166
+ return;
165
167
  const subById = new Map(subs.map((s) => [s.workerId, s]));
166
168
  const maxR = deps.moduleGate.maxRepairs ?? 1;
167
169
  await Promise.all(reports.filter((r) => r.status === "completed").map(async (r) => {
@@ -194,7 +196,8 @@ export async function runLeaderTask(task, durableBaseSha, deps) {
194
196
  }
195
197
  }
196
198
  }));
197
- }
199
+ };
200
+ await runModuleGate();
198
201
  const mergeDeps = {
199
202
  pullWorkerDiff: async (r) => workerDiff(byId.get(r.workerId)),
200
203
  provisionIntegrationSandbox: deps.provisionIntegrationSandbox,
@@ -192,186 +192,190 @@ export function createLeaderRunner(cfg) {
192
192
  if (!ex.ok)
193
193
  throw ex.error;
194
194
  };
195
- const deps = {
196
- plan: async () => {
197
- if (Array.isArray(body.subtasks) && body.subtasks.length > 0) {
198
- return validateSubtasks(body.subtasks);
199
- }
200
- const planRunner = new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing });
201
- const runRoute = async (prompt) => {
202
- const res = await planRunner.runTaskStream({
203
- objective: prompt,
204
- sessionId: `leader-route-${Date.now()}`,
205
- ...(cfg.routerModel ? { model: cfg.routerModel } : {}),
206
- }).result();
207
- return res.result ?? "";
195
+ const plan = async () => {
196
+ if (Array.isArray(body.subtasks) && body.subtasks.length > 0) {
197
+ return validateSubtasks(body.subtasks);
198
+ }
199
+ const planRunner = new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing });
200
+ const runRoute = async (prompt) => {
201
+ const res = await planRunner.runTaskStream({
202
+ objective: prompt,
203
+ sessionId: `leader-route-${Date.now()}`,
204
+ ...(cfg.routerModel ? { model: cfg.routerModel } : {}),
205
+ }).result();
206
+ return res.result ?? "";
207
+ };
208
+ const { subs, collapsed, rawSample } = await routePlanWithFallback(body.objective, runRoute, {
209
+ ...(body.planContext ? { context: body.planContext } : {}),
210
+ ...(cfg.fanoutEnabled === false ? { fanoutEnabled: false } : {}),
211
+ });
212
+ if (collapsed)
213
+ cfg.logger?.warn?.("plan_collapsed_to_single", { reason: collapsed, ...(rawSample ? { rawSample } : {}) });
214
+ return subs;
215
+ };
216
+ const provisionWorker = async (sub) => {
217
+ const sessionId = `leader-${sub.workerId}-${Date.now()}`;
218
+ let durableSpec = {};
219
+ if (cfg.durable) {
220
+ const d = cfg.durable;
221
+ durableSpec = {
222
+ toolPolicy: combinePolicies(createDurableQuestionPolicy(), createDurableAskPolicy({ requireApproval: d.requireApproval, ...(d.deny ? { deny: d.deny } : {}), ...(d.autoBudget ? { autoBudget: d.autoBudget } : {}), ...(d.neverAuto ? { neverAuto: d.neverAuto } : {}) })),
223
+ checkpointStore: d.checkpointStore,
224
+ durableApproval: { scope: "_leader", ...(d.ttlMs ? { ttlMs: d.ttlMs } : {}) },
225
+ onQuestion: QUESTION_AWAITS_RESUME,
208
226
  };
209
- const { subs, collapsed, rawSample } = await routePlanWithFallback(body.objective, runRoute, {
210
- ...(body.planContext ? { context: body.planContext } : {}),
211
- ...(cfg.fanoutEnabled === false ? { fanoutEnabled: false } : {}),
227
+ await d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch((e) => {
228
+ cfg.logger?.warn?.("leader_putctx_failed", { workerId: sub.workerId, err: String(e) });
212
229
  });
213
- if (collapsed)
214
- cfg.logger?.warn?.("plan_collapsed_to_single", { reason: collapsed, ...(rawSample ? { rawSample } : {}) });
215
- return subs;
216
- },
217
- provisionWorker: async (sub) => {
218
- const sessionId = `leader-${sub.workerId}-${Date.now()}`;
219
- let durableSpec = {};
220
- if (cfg.durable) {
221
- const d = cfg.durable;
222
- durableSpec = {
223
- toolPolicy: combinePolicies(createDurableQuestionPolicy(), createDurableAskPolicy({ requireApproval: d.requireApproval, ...(d.deny ? { deny: d.deny } : {}), ...(d.autoBudget ? { autoBudget: d.autoBudget } : {}), ...(d.neverAuto ? { neverAuto: d.neverAuto } : {}) })),
224
- checkpointStore: d.checkpointStore,
225
- durableApproval: { scope: "_leader", ...(d.ttlMs ? { ttlMs: d.ttlMs } : {}) },
226
- onQuestion: QUESTION_AWAITS_RESUME,
227
- };
228
- await d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch((e) => {
229
- cfg.logger?.warn?.("leader_putctx_failed", { workerId: sub.workerId, err: String(e) });
230
- });
231
- }
232
- let resourceSpec = {};
233
- const repairLoopActive = repairLoopOn && REPAIR_LOOP_WIRED;
234
- if (repairLoopActive && resourceCfg) {
235
- cfg.logger?.warn?.("leader_resource_suspend_disabled_for_repair_loop", { workerId: sub.workerId, reason: "LEADER_REPAIR_LOOP active — resource-suspend mutually exclusive with the repair loop (§10.5)" });
236
- }
237
- if (resourceCfg && !repairLoopActive && cfg.durable && cfg.toolResultStore && cfg.sessionStore) {
238
- resourceSpec = {
239
- checkpointStore: cfg.durable.checkpointStore,
240
- resourceSuspend: { scope: `_leader-resource-${sub.workerId}`, totalBudgetUsd: resourceCfg.totalBudgetUsd },
241
- maxCostUsd: resourceCfg.sliceMaxCostUsd,
242
- maxSuspends: resourceCfg.maxSuspends,
243
- };
244
- }
245
- else if (resourceCfg && !repairLoopActive) {
246
- cfg.logger?.warn?.("leader_resource_suspend_inactive", {
247
- workerId: sub.workerId,
248
- reason: !cfg.durable
249
- ? "no durable checkpoint store"
250
- : !cfg.toolResultStore
251
- ? "no durable tool-result store"
252
- : "no durable session store",
253
- });
254
- }
255
- const keepCtxWarm = (runner) => {
256
- const d = cfg.durable;
257
- if (!d)
258
- return runner;
259
- const orig = runner.runTaskStream.bind(runner);
260
- runner.runTaskStream = (spec, resume) => {
261
- const stream = orig(spec, resume);
262
- const iv = setInterval(() => {
263
- void d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch(() => { });
264
- }, 45_000);
265
- iv.unref?.();
266
- void stream.result().then(() => clearInterval(iv), () => clearInterval(iv));
267
- return stream;
268
- };
230
+ }
231
+ let resourceSpec = {};
232
+ const repairLoopActive = repairLoopOn && REPAIR_LOOP_WIRED;
233
+ if (repairLoopActive && resourceCfg) {
234
+ cfg.logger?.warn?.("leader_resource_suspend_disabled_for_repair_loop", { workerId: sub.workerId, reason: "LEADER_REPAIR_LOOP active — resource-suspend mutually exclusive with the repair loop (§10.5)" });
235
+ }
236
+ if (resourceCfg && !repairLoopActive && cfg.durable && cfg.toolResultStore && cfg.sessionStore) {
237
+ resourceSpec = {
238
+ checkpointStore: cfg.durable.checkpointStore,
239
+ resourceSuspend: { scope: `_leader-resource-${sub.workerId}`, totalBudgetUsd: resourceCfg.totalBudgetUsd },
240
+ maxCostUsd: resourceCfg.sliceMaxCostUsd,
241
+ maxSuspends: resourceCfg.maxSuspends,
242
+ };
243
+ }
244
+ else if (resourceCfg && !repairLoopActive) {
245
+ cfg.logger?.warn?.("leader_resource_suspend_inactive", {
246
+ workerId: sub.workerId,
247
+ reason: !cfg.durable
248
+ ? "no durable checkpoint store"
249
+ : !cfg.toolResultStore
250
+ ? "no durable tool-result store"
251
+ : "no durable session store",
252
+ });
253
+ }
254
+ const keepCtxWarm = (runner) => {
255
+ const d = cfg.durable;
256
+ if (!d)
269
257
  return runner;
258
+ const orig = runner.runTaskStream.bind(runner);
259
+ runner.runTaskStream = (spec, resume) => {
260
+ const stream = orig(spec, resume);
261
+ const iv = setInterval(() => {
262
+ void d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch(() => { });
263
+ }, 45_000);
264
+ iv.unref?.();
265
+ void stream.result().then(() => clearInterval(iv), () => clearInterval(iv));
266
+ return stream;
267
+ };
268
+ return runner;
269
+ };
270
+ if (cfg.s3 && cfg.envFactory) {
271
+ const s3 = cfg.s3;
272
+ const envFactory = cfg.envFactory;
273
+ const baseSha = body.baseSha;
274
+ const seedCmd = body.seedCmd;
275
+ const key = `${s3.keyPrefix ?? "leader-diff"}/${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}/${sub.workerId}.patch`;
276
+ const ttl = presignTtlSec;
277
+ const base = { endpoint: s3.endpoint, bucket: s3.bucket, accessKey: s3.accessKey, secretKey: s3.secretKey, ...(s3.region ? { region: s3.region } : {}), key, expiresSec: ttl };
278
+ const putUrl = presignS3Url({ ...base, method: "PUT" });
279
+ const getUrl = presignS3Url({ ...base, method: "GET" });
280
+ const script = buildUploadScript({ repoDir: repo, baseSha, putUrl, workDir: W });
281
+ const stage = async (env) => {
282
+ await stageWorkerEnv(env, { seedCmd, repo, branch: sub.branch, uploadScriptPath: `${W}/.leader-upload.sh`, uploadScript: script });
283
+ void sh(env)(`nohup sh -c 'while true; do sleep 90; sh ${W}/.leader-upload.sh >/dev/null 2>&1 || true; done' </dev/null >/dev/null 2>&1 &`).catch(() => { });
270
284
  };
271
- if (cfg.s3 && cfg.envFactory) {
272
- const s3 = cfg.s3;
273
- const envFactory = cfg.envFactory;
274
- const baseSha = body.baseSha;
275
- const seedCmd = body.seedCmd;
276
- const key = `${s3.keyPrefix ?? "leader-diff"}/${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}/${sub.workerId}.patch`;
277
- const ttl = presignTtlSec;
278
- const base = { endpoint: s3.endpoint, bucket: s3.bucket, accessKey: s3.accessKey, secretKey: s3.secretKey, ...(s3.region ? { region: s3.region } : {}), key, expiresSec: ttl };
279
- const putUrl = presignS3Url({ ...base, method: "PUT" });
280
- const getUrl = presignS3Url({ ...base, method: "GET" });
281
- const script = buildUploadScript({ repoDir: repo, baseSha, putUrl, workDir: W });
282
- const stage = async (env) => {
283
- await stageWorkerEnv(env, { seedCmd, repo, branch: sub.branch, uploadScriptPath: `${W}/.leader-upload.sh`, uploadScript: script });
284
- void sh(env)(`nohup sh -c 'while true; do sleep 90; sh ${W}/.leader-upload.sh >/dev/null 2>&1 || true; done' </dev/null >/dev/null 2>&1 &`).catch(() => { });
285
- };
286
- return {
287
- workerId: sub.workerId, sessionId, branch: sub.branch,
288
- baseSha,
289
- runner: keepCtxWarm(new Runner({
290
- brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing,
291
- ...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}),
292
- ...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}),
293
- executionEnvFactory: (ctx) => withStaging(envFactory(ctx), stage, async (e) => {
294
- await sh(e)(`cd ${repo} && git add -A && git -c user.name='leader-belt' -c user.email='belt@leader' commit -qm 'belt: auto-commit uncommitted worker output' || true; sh ${W}/.leader-upload.sh`);
295
- }),
296
- })),
297
- fetchDiff: () => fetchUploadedDiff(getUrl),
298
- spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec, objective: `${sub.spec.objective}${uploadStepSuffix(W)}` },
299
- };
300
- }
301
- if (!cfg.e2bApiKey)
302
- throw new Error("leader wire: no envFactory+s3 (factory mode) and no e2bApiKey (static mode)");
303
- const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
304
- if (!(await env.connect()).ok)
305
- throw new Error(`connect failed for ${sub.workerId}`);
306
- await sh(env)(`${body.seedCmd}; cd ${repo}; git checkout -q -b ${sub.branch}`);
307
- const baseSha = await sh(env)(`cd ${repo} && git rev-parse HEAD`);
308
285
  return {
309
- workerId: sub.workerId, sessionId, branch: sub.branch, baseSha,
310
- runner: keepCtxWarm(new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing, ...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}), ...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}), executionEnv: env })),
311
- diffEnv: env,
312
- destroy: () => env.destroy().then(() => { }),
313
- spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec },
286
+ workerId: sub.workerId, sessionId, branch: sub.branch,
287
+ baseSha,
288
+ runner: keepCtxWarm(new Runner({
289
+ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing,
290
+ ...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}),
291
+ ...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}),
292
+ executionEnvFactory: (ctx) => withStaging(envFactory(ctx), stage, async (e) => {
293
+ await sh(e)(`cd ${repo} && git add -A && git -c user.name='leader-belt' -c user.email='belt@leader' commit -qm 'belt: auto-commit uncommitted worker output' || true; sh ${W}/.leader-upload.sh`);
294
+ }),
295
+ })),
296
+ fetchDiff: () => fetchUploadedDiff(getUrl),
297
+ spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec, objective: `${sub.spec.objective}${uploadStepSuffix(W)}` },
314
298
  };
315
- },
316
- provisionIntegrationSandbox: async () => {
317
- if (cfg.envFactory) {
318
- const env = cfg.envFactory({ sessionId: `leader-integ-${Date.now()}` });
319
- try {
320
- await sh(env)(body.seedCmd);
321
- await injectOracles(env);
322
- }
323
- catch (e) {
324
- await Promise.resolve(env.destroy()).catch(() => { });
325
- throw e;
326
- }
327
- return { env: asIntegrationEnv(env), destroy: () => Promise.resolve(env.destroy()).then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
328
- }
329
- if (!cfg.e2bApiKey)
330
- throw new Error("leader wire: no envFactory and no e2bApiKey");
331
- const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
332
- if (!(await env.connect()).ok) {
333
- await env.destroy().catch(() => { });
334
- throw new Error("integ connect failed");
335
- }
299
+ }
300
+ if (!cfg.e2bApiKey)
301
+ throw new Error("leader wire: no envFactory+s3 (factory mode) and no e2bApiKey (static mode)");
302
+ const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
303
+ if (!(await env.connect()).ok)
304
+ throw new Error(`connect failed for ${sub.workerId}`);
305
+ await sh(env)(`${body.seedCmd}; cd ${repo}; git checkout -q -b ${sub.branch}`);
306
+ const baseSha = await sh(env)(`cd ${repo} && git rev-parse HEAD`);
307
+ return {
308
+ workerId: sub.workerId, sessionId, branch: sub.branch, baseSha,
309
+ runner: keepCtxWarm(new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing, ...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}), ...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}), executionEnv: env })),
310
+ diffEnv: env,
311
+ destroy: () => env.destroy().then(() => { }),
312
+ spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec },
313
+ };
314
+ };
315
+ const provisionIntegrationSandbox = async () => {
316
+ if (cfg.envFactory) {
317
+ const env = cfg.envFactory({ sessionId: `leader-integ-${Date.now()}` });
336
318
  try {
337
319
  await sh(env)(body.seedCmd);
338
320
  await injectOracles(env);
339
321
  }
340
322
  catch (e) {
341
- await env.destroy().catch(() => { });
323
+ await Promise.resolve(env.destroy()).catch(() => { });
342
324
  throw e;
343
325
  }
344
- return { env: asIntegrationEnv(env), destroy: () => env.destroy().then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
345
- },
346
- push: async (integratedPatch, baseSha) => {
347
- const dir = mkdtempSync(join(tmpdir(), "leader-coord-"));
348
- const gitEnv = { ...process.env, LC_ALL: "C" };
349
- try {
350
- execFileSync("git", ["clone", "-q", body.durableRemote, dir], { env: gitEnv });
351
- execFileSync("git", ["-C", dir, "config", "user.name", ident.name]);
352
- execFileSync("git", ["-C", dir, "config", "user.email", ident.email]);
353
- writeFileSync(join(dir, ".leader.patch"), integratedPatch);
354
- execFileSync("git", ["-C", dir, "am", "--3way", ".leader.patch"], { env: gitEnv });
355
- rmSync(join(dir, ".leader.patch"));
356
- const ref = body.targetRef ?? "refs/heads/main";
357
- const ls = execFileSync("git", ["-C", dir, "ls-remote", "--refs", "origin", ref], { encoding: "utf8", env: gitEnv });
358
- const exists = ls.split("\n").some((l) => l.split("\t")[1] === ref);
359
- if (exists) {
360
- execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:${baseSha}`, "origin", `HEAD:${ref}`], { env: gitEnv });
361
- }
362
- else {
363
- execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:`, "origin", `HEAD:${ref}`], { env: gitEnv });
364
- }
365
- return { ok: true, ref };
366
- }
367
- catch (e) {
368
- const msg = e instanceof Error ? e.message : String(e);
369
- return { ok: false, raced: /stale info|force-with-lease|\[rejected\]|non-fast-forward/.test(msg), error: msg };
326
+ return { env: asIntegrationEnv(env), destroy: () => Promise.resolve(env.destroy()).then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
327
+ }
328
+ if (!cfg.e2bApiKey)
329
+ throw new Error("leader wire: no envFactory and no e2bApiKey");
330
+ const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
331
+ if (!(await env.connect()).ok) {
332
+ await env.destroy().catch(() => { });
333
+ throw new Error("integ connect failed");
334
+ }
335
+ try {
336
+ await sh(env)(body.seedCmd);
337
+ await injectOracles(env);
338
+ }
339
+ catch (e) {
340
+ await env.destroy().catch(() => { });
341
+ throw e;
342
+ }
343
+ return { env: asIntegrationEnv(env), destroy: () => env.destroy().then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
344
+ };
345
+ const push = async (integratedPatch, baseSha) => {
346
+ const dir = mkdtempSync(join(tmpdir(), "leader-coord-"));
347
+ const gitEnv = { ...process.env, LC_ALL: "C" };
348
+ try {
349
+ execFileSync("git", ["clone", "-q", body.durableRemote, dir], { env: gitEnv });
350
+ execFileSync("git", ["-C", dir, "config", "user.name", ident.name]);
351
+ execFileSync("git", ["-C", dir, "config", "user.email", ident.email]);
352
+ writeFileSync(join(dir, ".leader.patch"), integratedPatch);
353
+ execFileSync("git", ["-C", dir, "am", "--3way", ".leader.patch"], { env: gitEnv });
354
+ rmSync(join(dir, ".leader.patch"));
355
+ const ref = body.targetRef ?? "refs/heads/main";
356
+ const ls = execFileSync("git", ["-C", dir, "ls-remote", "--refs", "origin", ref], { encoding: "utf8", env: gitEnv });
357
+ const exists = ls.split("\n").some((l) => l.split("\t")[1] === ref);
358
+ if (exists) {
359
+ execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:${baseSha}`, "origin", `HEAD:${ref}`], { env: gitEnv });
370
360
  }
371
- finally {
372
- rmSync(dir, { recursive: true, force: true });
361
+ else {
362
+ execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:`, "origin", `HEAD:${ref}`], { env: gitEnv });
373
363
  }
374
- },
364
+ return { ok: true, ref };
365
+ }
366
+ catch (e) {
367
+ const msg = e instanceof Error ? e.message : String(e);
368
+ return { ok: false, raced: /stale info|force-with-lease|\[rejected\]|non-fast-forward/.test(msg), error: msg };
369
+ }
370
+ finally {
371
+ rmSync(dir, { recursive: true, force: true });
372
+ }
373
+ };
374
+ const deps = {
375
+ plan,
376
+ provisionWorker,
377
+ provisionIntegrationSandbox,
378
+ push,
375
379
  testCmd: body.testCmd,
376
380
  repoDir: repo, workerRepoDir: repo,
377
381
  timeoutMs: leaderTimeoutMs, maxConcurrency: 4,