@sema-agent/server 1.314.0 → 1.315.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/approval-hmac.d.ts +17 -0
- package/dist/approval-hmac.js +27 -0
- package/dist/config-center/apply-effective.d.ts +22 -0
- package/dist/config-center/apply-effective.js +283 -0
- package/dist/config-center/http-client.d.ts +23 -0
- package/dist/config-center/http-client.js +109 -0
- package/dist/config-center/restart-signal.d.ts +12 -0
- package/dist/config-center/restart-signal.js +70 -0
- package/dist/config-center/skills-mcp.d.ts +13 -0
- package/dist/config-center/skills-mcp.js +113 -0
- package/dist/config-center/types.d.ts +143 -0
- package/dist/config-center/types.js +2 -0
- package/dist/fleet/fleet-bus.js +92 -83
- package/dist/hooks/hook-runner.js +18 -9
- package/dist/http/server.d.ts +95 -69
- package/dist/http/server.js +8 -1
- package/dist/index.d.ts +1 -1
- package/dist/leader/leader.js +5 -2
- package/dist/leader/wire.js +169 -165
- package/dist/main.js +462 -451
- package/dist/plugins/checkpoint-store-sql.d.ts +67 -0
- package/dist/plugins/checkpoint-store-sql.js +224 -0
- package/dist/plugins/image-bake-store-sql.d.ts +53 -0
- package/dist/plugins/image-bake-store-sql.js +463 -0
- package/dist/plugins/k8s-bg-scripts.d.ts +19 -0
- package/dist/plugins/k8s-bg-scripts.js +129 -0
- package/dist/plugins/k8s-exec-protocol.d.ts +20 -0
- package/dist/plugins/k8s-exec-protocol.js +87 -0
- package/dist/plugins/pg-checkpoint-store.d.ts +1 -33
- package/dist/plugins/pg-checkpoint-store.js +1 -189
- package/dist/plugins/pg-cost-quota.js +3 -143
- package/dist/plugins/pg-image-bake.d.ts +1 -40
- package/dist/plugins/pg-image-bake.js +1 -420
- package/dist/plugins/pg-rate-limiter.d.ts +2 -32
- package/dist/plugins/pg-rate-limiter.js +4 -147
- package/dist/plugins/remote-env-k8s.d.ts +5 -38
- package/dist/plugins/remote-env-k8s.js +7 -213
- package/dist/plugins/sql-driver.d.ts +25 -0
- package/dist/plugins/sql-driver.js +59 -0
- package/dist/plugins/tidb-checkpoint-store.d.ts +1 -49
- package/dist/plugins/tidb-checkpoint-store.js +1 -191
- package/dist/plugins/tidb-image-bake.d.ts +1 -38
- package/dist/plugins/tidb-image-bake.js +1 -348
- package/dist/plugins/write-behind-counter.d.ts +14 -3
- package/dist/plugins/write-behind-counter.js +39 -12
- package/dist/principal-jwt.d.ts +44 -0
- package/dist/principal-jwt.js +95 -0
- package/dist/security.d.ts +2 -58
- package/dist/security.js +3 -118
- package/dist/sema-registry.d.ts +5 -200
- package/dist/sema-registry.js +4 -567
- package/package.json +1 -1
package/dist/http/server.d.ts
CHANGED
|
@@ -30,58 +30,59 @@ export interface RequestAuth {
|
|
|
30
30
|
memoryScope?: string;
|
|
31
31
|
resolvedProjectId?: string;
|
|
32
32
|
}
|
|
33
|
-
export interface
|
|
33
|
+
export interface ServiceCoreDeps {
|
|
34
34
|
runner: Runner;
|
|
35
35
|
config: ServiceConfig;
|
|
36
|
-
registryJwtVerifier?: {
|
|
37
|
-
verify: (bearer: string) => Promise<import("../auth-bridge.js").VerifyOutcome>;
|
|
38
|
-
};
|
|
39
|
-
hookWakeBus?: {
|
|
40
|
-
deliver?: (sessionId: string, text: string) => Promise<boolean>;
|
|
41
|
-
};
|
|
42
|
-
drainState?: {
|
|
43
|
-
draining: boolean;
|
|
44
|
-
since?: number;
|
|
45
|
-
inflight?: () => number;
|
|
46
|
-
lastActivityAt?: () => number;
|
|
47
|
-
};
|
|
48
|
-
storeDegraded?: boolean;
|
|
49
|
-
sessionStoreLabel?: string;
|
|
50
|
-
modelReady?: () => boolean;
|
|
51
|
-
scenarioDetails?: Record<string, import("../capabilities/scenarios.js").ScenarioDetail>;
|
|
52
|
-
authorize?: (ctx: {
|
|
53
|
-
req: IncomingMessage;
|
|
54
|
-
body: TaskRequestBody;
|
|
55
|
-
}) => Promise<RequestAuth>;
|
|
56
36
|
resolveSpec: (body: TaskRequestBody, req: IncomingMessage | undefined, auth?: RequestAuth, opts?: {
|
|
57
37
|
leg?: "fresh" | "resume";
|
|
58
38
|
}) => TaskSpec | Promise<TaskSpec>;
|
|
39
|
+
}
|
|
40
|
+
export interface ServiceStoreDeps {
|
|
59
41
|
runStore?: RunStore;
|
|
60
42
|
resumeAnchorStore?: ResumeAnchorStore;
|
|
61
|
-
|
|
62
|
-
maybeTitle(sessionId: string, objective: string): void;
|
|
63
|
-
};
|
|
43
|
+
approvalStore?: ApprovalStore;
|
|
64
44
|
approvalExemptionStore?: ApprovalExemptionStore;
|
|
45
|
+
checkpointStore?: CheckpointStoreFull;
|
|
65
46
|
sessionPolicyStore?: ServiceSessionPolicyStore;
|
|
66
47
|
fileSnapshotStore?: ServiceFileSnapshotStore;
|
|
67
|
-
snapshotBlobSqlCapBytes?: number;
|
|
68
|
-
backend?: StoreBackend;
|
|
69
|
-
sessionMirrorRuling?: (principal: string | undefined) => Promise<SessionMirrorRuling | undefined>;
|
|
70
|
-
approvalStore?: ApprovalStore;
|
|
71
|
-
checkpointStore?: CheckpointStoreFull;
|
|
72
|
-
backgroundAgentStore?: import("@sema-agent/core").BackgroundAgentStore;
|
|
73
|
-
parkedReviveTool?: import("@sema-agent/core").ToolSpec;
|
|
74
|
-
parkedKnownAgentTypes?: ReadonlySet<string>;
|
|
75
|
-
parkedReviveInheritedGate?: (row: import("@sema-agent/core").BackgroundAgentRecord) => unknown;
|
|
76
48
|
taskAttachmentStore?: TaskAttachmentStore;
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
leaderEndpoint?: LeaderEndpoint;
|
|
49
|
+
backgroundAgentStore?: import("@sema-agent/core").BackgroundAgentStore;
|
|
50
|
+
sessionStorage?: OwnerAwareSessionStore;
|
|
80
51
|
workflowRunStore?: WorkflowRunStore;
|
|
81
52
|
workflowJournalStore?: import("@sema-agent/core").WorkflowJournalStore;
|
|
82
|
-
|
|
53
|
+
imageIndex?: ImageIndex;
|
|
54
|
+
imageBakes?: ImageBake;
|
|
55
|
+
outcomeSink?: import("../plugins/file-outcome-sink.js").OutcomeSink;
|
|
56
|
+
sendFileLedger?: import("../plugins/send-file-ledger.js").SendFileLedger;
|
|
57
|
+
backend?: StoreBackend;
|
|
58
|
+
}
|
|
59
|
+
export interface ServiceCoordinatorDeps {
|
|
60
|
+
elicitation?: ElicitationCoordinator;
|
|
61
|
+
question?: QuestionCoordinator;
|
|
62
|
+
toolApproval?: ToolApprovalCoordinator;
|
|
83
63
|
workflowAgentRegistry?: WorkflowAgentRegistry;
|
|
84
64
|
subagentSteerRegistry?: SubagentSteerRegistry;
|
|
65
|
+
workflowCompletionInbox?: WorkflowCompletionInbox;
|
|
66
|
+
fleetBus?: FleetEventBus;
|
|
67
|
+
sessionWatch?: import("../session-watch.js").SessionWatchRegistry;
|
|
68
|
+
hookWakeBus?: {
|
|
69
|
+
deliver?: (sessionId: string, text: string) => Promise<boolean>;
|
|
70
|
+
};
|
|
71
|
+
sendUserFile?: import("../capabilities/send-user-file-tool.js").SendUserFileEmitter;
|
|
72
|
+
leaderEndpoint?: LeaderEndpoint;
|
|
73
|
+
sessionTitler?: {
|
|
74
|
+
maybeTitle(sessionId: string, objective: string): void;
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
export interface ServiceSeamDeps {
|
|
78
|
+
sessionAudit?: (sessionId: string) => Promise<({
|
|
79
|
+
owner: string | null;
|
|
80
|
+
} & Record<string, unknown>) | undefined>;
|
|
81
|
+
purgeSession?: (sessionId: string, owner: string | null) => Promise<{
|
|
82
|
+
deleted: boolean;
|
|
83
|
+
} | {
|
|
84
|
+
active: string;
|
|
85
|
+
}>;
|
|
85
86
|
subagentTaskOutput?: (handle: string, access: {
|
|
86
87
|
owner: string;
|
|
87
88
|
scope: string;
|
|
@@ -106,23 +107,10 @@ export interface ServiceDeps {
|
|
|
106
107
|
content: string;
|
|
107
108
|
details: unknown;
|
|
108
109
|
}>;
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
} & Record<string, unknown>) | undefined>;
|
|
114
|
-
sessionStorage?: OwnerAwareSessionStore;
|
|
115
|
-
purgeSession?: (sessionId: string, owner: string | null) => Promise<{
|
|
116
|
-
deleted: boolean;
|
|
117
|
-
} | {
|
|
118
|
-
active: string;
|
|
119
|
-
}>;
|
|
120
|
-
instanceId?: string;
|
|
121
|
-
logger?: Logger;
|
|
122
|
-
metrics?: Metrics;
|
|
123
|
-
rateLimiter?: RateGate;
|
|
124
|
-
costQuota?: QuotaTracker;
|
|
125
|
-
fleetLease?: FleetLeaseManager;
|
|
110
|
+
memoryExport?: (scope: string) => Promise<MemoryEntry[]>;
|
|
111
|
+
memorySync?: (scope: string, syncReq: MemorySyncRequest) => Promise<MemorySyncResponse>;
|
|
112
|
+
sessionMirrorRuling?: (principal: string | undefined) => Promise<SessionMirrorRuling | undefined>;
|
|
113
|
+
instrumentDegenerate?: (result: TaskResult) => void;
|
|
126
114
|
sideQueryAccounting?: (principal: string | undefined, r: {
|
|
127
115
|
model: string;
|
|
128
116
|
family?: "input-includes-cached" | "input-excludes-cached";
|
|
@@ -136,30 +124,68 @@ export interface ServiceDeps {
|
|
|
136
124
|
};
|
|
137
125
|
};
|
|
138
126
|
}) => void;
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
127
|
+
parkedReviveTool?: import("@sema-agent/core").ToolSpec;
|
|
128
|
+
parkedKnownAgentTypes?: ReadonlySet<string>;
|
|
129
|
+
parkedReviveInheritedGate?: (row: import("@sema-agent/core").BackgroundAgentRecord) => unknown;
|
|
130
|
+
}
|
|
131
|
+
export interface ServiceObservabilityDeps {
|
|
132
|
+
logger?: Logger;
|
|
133
|
+
metrics?: Metrics;
|
|
144
134
|
modelUsage?: ModelUsageTracker;
|
|
145
135
|
promptManifests?: PromptManifestTracker;
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
136
|
+
planCacheProbe?: PlanCacheProbe;
|
|
137
|
+
}
|
|
138
|
+
export interface ServiceGovernanceDeps {
|
|
139
|
+
authorize?: (ctx: {
|
|
140
|
+
req: IncomingMessage;
|
|
141
|
+
body: TaskRequestBody;
|
|
142
|
+
}) => Promise<RequestAuth>;
|
|
143
|
+
registryJwtVerifier?: {
|
|
144
|
+
verify: (bearer: string) => Promise<import("../auth-bridge.js").VerifyOutcome>;
|
|
145
|
+
};
|
|
146
|
+
rateLimiter?: RateGate;
|
|
147
|
+
costQuota?: QuotaTracker;
|
|
148
|
+
fleetLease?: FleetLeaseManager;
|
|
149
|
+
}
|
|
150
|
+
export interface ServiceDeploymentDeps {
|
|
151
|
+
instanceId?: string;
|
|
155
152
|
capabilities?: Record<string, unknown>;
|
|
153
|
+
scenarioDetails?: Record<string, import("../capabilities/scenarios.js").ScenarioDetail>;
|
|
154
|
+
sessionStoreLabel?: string;
|
|
155
|
+
storeDegraded?: boolean;
|
|
156
|
+
drainState?: {
|
|
157
|
+
draining: boolean;
|
|
158
|
+
since?: number;
|
|
159
|
+
inflight?: () => number;
|
|
160
|
+
lastActivityAt?: () => number;
|
|
161
|
+
};
|
|
162
|
+
modelReady?: () => boolean;
|
|
156
163
|
restartState?: () => RestartSignal | undefined;
|
|
157
164
|
planeDeferredState?: () => {
|
|
158
165
|
version: number;
|
|
159
166
|
since: number;
|
|
160
167
|
blocked?: string[];
|
|
161
168
|
} | undefined;
|
|
169
|
+
workflowsCapable?: boolean;
|
|
170
|
+
}
|
|
171
|
+
export interface ServiceKnobDeps {
|
|
172
|
+
snapshotBlobSqlCapBytes?: number;
|
|
173
|
+
sessionEventsMaxConnections?: number;
|
|
174
|
+
sessionEventsHeartbeatMs?: number;
|
|
175
|
+
sessionEventsStallMs?: number;
|
|
176
|
+
}
|
|
177
|
+
export interface ServiceDepGroups {
|
|
178
|
+
stores?: ServiceStoreDeps;
|
|
179
|
+
coordinators?: ServiceCoordinatorDeps;
|
|
180
|
+
seams?: ServiceSeamDeps;
|
|
181
|
+
observability?: ServiceObservabilityDeps;
|
|
182
|
+
governance?: ServiceGovernanceDeps;
|
|
183
|
+
deployment?: ServiceDeploymentDeps;
|
|
184
|
+
knobs?: ServiceKnobDeps;
|
|
162
185
|
}
|
|
186
|
+
export type FlatServiceDeps = ServiceCoreDeps & Partial<ServiceStoreDeps & ServiceCoordinatorDeps & ServiceSeamDeps & ServiceObservabilityDeps & ServiceGovernanceDeps & ServiceDeploymentDeps & ServiceKnobDeps>;
|
|
187
|
+
export type ServiceDeps = FlatServiceDeps & ServiceDepGroups;
|
|
188
|
+
export declare function flattenServiceDeps(deps: ServiceDeps): FlatServiceDeps;
|
|
163
189
|
export type { TaskRequestBody } from "./wire-types.js";
|
|
164
190
|
export declare const MAX_USER_SKILLS = 10;
|
|
165
191
|
export declare const MAX_SKILL_CONTENT_CHARS = 1048576;
|
|
@@ -175,7 +201,7 @@ export declare function cascadeConfig(ladder: readonly string[], maxCostUsd?: nu
|
|
|
175
201
|
export declare function clampVerifyRounds(v: unknown): number;
|
|
176
202
|
export declare function scopedIdempotencyKey(raw: string | undefined, source: string | null, principal: string | undefined): string | undefined;
|
|
177
203
|
export declare function isQuestionAnswer(v: unknown): v is QuestionAnswer;
|
|
178
|
-
export declare function createHttpServer(
|
|
204
|
+
export declare function createHttpServer(rawDeps: ServiceDeps): http.Server & {
|
|
179
205
|
denyExpiredApprovals: (now: number) => Promise<void>;
|
|
180
206
|
};
|
|
181
207
|
export declare function streamApprovals(req: IncomingMessage, res: ServerResponse, cs: Pick<CheckpointStoreFull, "listPending">, scope: string | undefined, pollMs?: number): Promise<void>;
|
package/dist/http/server.js
CHANGED
|
@@ -35,6 +35,12 @@ import { redactSecrets, redactDeep, redactedPreview } from "../trace/redact.js";
|
|
|
35
35
|
import { usageSummary, usageSeries, usageBreakdown } from "../usage-analytics.js";
|
|
36
36
|
import { IdempotencyCache } from "./idempotency.js";
|
|
37
37
|
import { pgHasUnstorable } from "../plugins/pg-safe-json.js";
|
|
38
|
+
export function flattenServiceDeps(deps) {
|
|
39
|
+
const { stores, coordinators, seams, observability, governance, deployment, knobs, ...flat } = deps;
|
|
40
|
+
if (!(stores || coordinators || seams || observability || governance || deployment || knobs))
|
|
41
|
+
return deps;
|
|
42
|
+
return { ...stores, ...coordinators, ...seams, ...observability, ...governance, ...deployment, ...knobs, ...flat };
|
|
43
|
+
}
|
|
38
44
|
export const MAX_USER_SKILLS = 10;
|
|
39
45
|
export const MAX_SKILL_CONTENT_CHARS = 1_048_576;
|
|
40
46
|
export const MAX_SYSTEM_PROMPT_CHARS = 16_384;
|
|
@@ -195,7 +201,8 @@ function bearerPresentedButUnverified(req, config) {
|
|
|
195
201
|
return false;
|
|
196
202
|
return true;
|
|
197
203
|
}
|
|
198
|
-
export function createHttpServer(
|
|
204
|
+
export function createHttpServer(rawDeps) {
|
|
205
|
+
const deps = flattenServiceDeps(rawDeps);
|
|
199
206
|
const idemCache = new IdempotencyCache();
|
|
200
207
|
let sseConnections = 0;
|
|
201
208
|
let sseOwnerLookups = 0;
|
package/dist/index.d.ts
CHANGED
|
@@ -22,5 +22,5 @@ export { createCouncilTool } from "./capabilities/code-review-council.js";
|
|
|
22
22
|
export { loadSkills, skillsForScenario, type LoadedSkill } from "./capabilities/skills.js";
|
|
23
23
|
export { buildScenarios, selectScenario, type Scenario, type ScenarioBundle, type ScenarioDeps, type ScenarioRequest, } from "./capabilities/scenarios.js";
|
|
24
24
|
export { createAuthorizer, principalFrom, memoryScopeFor, memoryEngineBackendFor, HttpError, type AuthContext, type OwnerAwareSessionStore, } from "./security.js";
|
|
25
|
-
export { createHttpServer, type ServiceDeps, type TaskRequestBody, type RequestAuth, } from "./http/server.js";
|
|
25
|
+
export { createHttpServer, type ServiceDeps, type TaskRequestBody, type RequestAuth, type FlatServiceDeps, type ServiceDepGroups, type ServiceCoreDeps, type ServiceStoreDeps, type ServiceCoordinatorDeps, type ServiceSeamDeps, type ServiceObservabilityDeps, type ServiceGovernanceDeps, type ServiceDeploymentDeps, type ServiceKnobDeps, } from "./http/server.js";
|
|
26
26
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/leader/leader.js
CHANGED
|
@@ -161,7 +161,9 @@ export async function runLeaderTask(task, durableBaseSha, deps) {
|
|
|
161
161
|
}));
|
|
162
162
|
return out;
|
|
163
163
|
};
|
|
164
|
-
|
|
164
|
+
const runModuleGate = async () => {
|
|
165
|
+
if (!deps.moduleGate)
|
|
166
|
+
return;
|
|
165
167
|
const subById = new Map(subs.map((s) => [s.workerId, s]));
|
|
166
168
|
const maxR = deps.moduleGate.maxRepairs ?? 1;
|
|
167
169
|
await Promise.all(reports.filter((r) => r.status === "completed").map(async (r) => {
|
|
@@ -194,7 +196,8 @@ export async function runLeaderTask(task, durableBaseSha, deps) {
|
|
|
194
196
|
}
|
|
195
197
|
}
|
|
196
198
|
}));
|
|
197
|
-
}
|
|
199
|
+
};
|
|
200
|
+
await runModuleGate();
|
|
198
201
|
const mergeDeps = {
|
|
199
202
|
pullWorkerDiff: async (r) => workerDiff(byId.get(r.workerId)),
|
|
200
203
|
provisionIntegrationSandbox: deps.provisionIntegrationSandbox,
|
package/dist/leader/wire.js
CHANGED
|
@@ -192,186 +192,190 @@ export function createLeaderRunner(cfg) {
|
|
|
192
192
|
if (!ex.ok)
|
|
193
193
|
throw ex.error;
|
|
194
194
|
};
|
|
195
|
-
const
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
const
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
195
|
+
const plan = async () => {
|
|
196
|
+
if (Array.isArray(body.subtasks) && body.subtasks.length > 0) {
|
|
197
|
+
return validateSubtasks(body.subtasks);
|
|
198
|
+
}
|
|
199
|
+
const planRunner = new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing });
|
|
200
|
+
const runRoute = async (prompt) => {
|
|
201
|
+
const res = await planRunner.runTaskStream({
|
|
202
|
+
objective: prompt,
|
|
203
|
+
sessionId: `leader-route-${Date.now()}`,
|
|
204
|
+
...(cfg.routerModel ? { model: cfg.routerModel } : {}),
|
|
205
|
+
}).result();
|
|
206
|
+
return res.result ?? "";
|
|
207
|
+
};
|
|
208
|
+
const { subs, collapsed, rawSample } = await routePlanWithFallback(body.objective, runRoute, {
|
|
209
|
+
...(body.planContext ? { context: body.planContext } : {}),
|
|
210
|
+
...(cfg.fanoutEnabled === false ? { fanoutEnabled: false } : {}),
|
|
211
|
+
});
|
|
212
|
+
if (collapsed)
|
|
213
|
+
cfg.logger?.warn?.("plan_collapsed_to_single", { reason: collapsed, ...(rawSample ? { rawSample } : {}) });
|
|
214
|
+
return subs;
|
|
215
|
+
};
|
|
216
|
+
const provisionWorker = async (sub) => {
|
|
217
|
+
const sessionId = `leader-${sub.workerId}-${Date.now()}`;
|
|
218
|
+
let durableSpec = {};
|
|
219
|
+
if (cfg.durable) {
|
|
220
|
+
const d = cfg.durable;
|
|
221
|
+
durableSpec = {
|
|
222
|
+
toolPolicy: combinePolicies(createDurableQuestionPolicy(), createDurableAskPolicy({ requireApproval: d.requireApproval, ...(d.deny ? { deny: d.deny } : {}), ...(d.autoBudget ? { autoBudget: d.autoBudget } : {}), ...(d.neverAuto ? { neverAuto: d.neverAuto } : {}) })),
|
|
223
|
+
checkpointStore: d.checkpointStore,
|
|
224
|
+
durableApproval: { scope: "_leader", ...(d.ttlMs ? { ttlMs: d.ttlMs } : {}) },
|
|
225
|
+
onQuestion: QUESTION_AWAITS_RESUME,
|
|
208
226
|
};
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
...(cfg.fanoutEnabled === false ? { fanoutEnabled: false } : {}),
|
|
227
|
+
await d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch((e) => {
|
|
228
|
+
cfg.logger?.warn?.("leader_putctx_failed", { workerId: sub.workerId, err: String(e) });
|
|
212
229
|
});
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
resourceSuspend: { scope: `_leader-resource-${sub.workerId}`, totalBudgetUsd: resourceCfg.totalBudgetUsd },
|
|
241
|
-
maxCostUsd: resourceCfg.sliceMaxCostUsd,
|
|
242
|
-
maxSuspends: resourceCfg.maxSuspends,
|
|
243
|
-
};
|
|
244
|
-
}
|
|
245
|
-
else if (resourceCfg && !repairLoopActive) {
|
|
246
|
-
cfg.logger?.warn?.("leader_resource_suspend_inactive", {
|
|
247
|
-
workerId: sub.workerId,
|
|
248
|
-
reason: !cfg.durable
|
|
249
|
-
? "no durable checkpoint store"
|
|
250
|
-
: !cfg.toolResultStore
|
|
251
|
-
? "no durable tool-result store"
|
|
252
|
-
: "no durable session store",
|
|
253
|
-
});
|
|
254
|
-
}
|
|
255
|
-
const keepCtxWarm = (runner) => {
|
|
256
|
-
const d = cfg.durable;
|
|
257
|
-
if (!d)
|
|
258
|
-
return runner;
|
|
259
|
-
const orig = runner.runTaskStream.bind(runner);
|
|
260
|
-
runner.runTaskStream = (spec, resume) => {
|
|
261
|
-
const stream = orig(spec, resume);
|
|
262
|
-
const iv = setInterval(() => {
|
|
263
|
-
void d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch(() => { });
|
|
264
|
-
}, 45_000);
|
|
265
|
-
iv.unref?.();
|
|
266
|
-
void stream.result().then(() => clearInterval(iv), () => clearInterval(iv));
|
|
267
|
-
return stream;
|
|
268
|
-
};
|
|
230
|
+
}
|
|
231
|
+
let resourceSpec = {};
|
|
232
|
+
const repairLoopActive = repairLoopOn && REPAIR_LOOP_WIRED;
|
|
233
|
+
if (repairLoopActive && resourceCfg) {
|
|
234
|
+
cfg.logger?.warn?.("leader_resource_suspend_disabled_for_repair_loop", { workerId: sub.workerId, reason: "LEADER_REPAIR_LOOP active — resource-suspend mutually exclusive with the repair loop (§10.5)" });
|
|
235
|
+
}
|
|
236
|
+
if (resourceCfg && !repairLoopActive && cfg.durable && cfg.toolResultStore && cfg.sessionStore) {
|
|
237
|
+
resourceSpec = {
|
|
238
|
+
checkpointStore: cfg.durable.checkpointStore,
|
|
239
|
+
resourceSuspend: { scope: `_leader-resource-${sub.workerId}`, totalBudgetUsd: resourceCfg.totalBudgetUsd },
|
|
240
|
+
maxCostUsd: resourceCfg.sliceMaxCostUsd,
|
|
241
|
+
maxSuspends: resourceCfg.maxSuspends,
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
else if (resourceCfg && !repairLoopActive) {
|
|
245
|
+
cfg.logger?.warn?.("leader_resource_suspend_inactive", {
|
|
246
|
+
workerId: sub.workerId,
|
|
247
|
+
reason: !cfg.durable
|
|
248
|
+
? "no durable checkpoint store"
|
|
249
|
+
: !cfg.toolResultStore
|
|
250
|
+
? "no durable tool-result store"
|
|
251
|
+
: "no durable session store",
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
const keepCtxWarm = (runner) => {
|
|
255
|
+
const d = cfg.durable;
|
|
256
|
+
if (!d)
|
|
269
257
|
return runner;
|
|
258
|
+
const orig = runner.runTaskStream.bind(runner);
|
|
259
|
+
runner.runTaskStream = (spec, resume) => {
|
|
260
|
+
const stream = orig(spec, resume);
|
|
261
|
+
const iv = setInterval(() => {
|
|
262
|
+
void d.checkpointStore.putCtx(sessionId, { body: { objective: sub.spec.objective } }).catch(() => { });
|
|
263
|
+
}, 45_000);
|
|
264
|
+
iv.unref?.();
|
|
265
|
+
void stream.result().then(() => clearInterval(iv), () => clearInterval(iv));
|
|
266
|
+
return stream;
|
|
267
|
+
};
|
|
268
|
+
return runner;
|
|
269
|
+
};
|
|
270
|
+
if (cfg.s3 && cfg.envFactory) {
|
|
271
|
+
const s3 = cfg.s3;
|
|
272
|
+
const envFactory = cfg.envFactory;
|
|
273
|
+
const baseSha = body.baseSha;
|
|
274
|
+
const seedCmd = body.seedCmd;
|
|
275
|
+
const key = `${s3.keyPrefix ?? "leader-diff"}/${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}/${sub.workerId}.patch`;
|
|
276
|
+
const ttl = presignTtlSec;
|
|
277
|
+
const base = { endpoint: s3.endpoint, bucket: s3.bucket, accessKey: s3.accessKey, secretKey: s3.secretKey, ...(s3.region ? { region: s3.region } : {}), key, expiresSec: ttl };
|
|
278
|
+
const putUrl = presignS3Url({ ...base, method: "PUT" });
|
|
279
|
+
const getUrl = presignS3Url({ ...base, method: "GET" });
|
|
280
|
+
const script = buildUploadScript({ repoDir: repo, baseSha, putUrl, workDir: W });
|
|
281
|
+
const stage = async (env) => {
|
|
282
|
+
await stageWorkerEnv(env, { seedCmd, repo, branch: sub.branch, uploadScriptPath: `${W}/.leader-upload.sh`, uploadScript: script });
|
|
283
|
+
void sh(env)(`nohup sh -c 'while true; do sleep 90; sh ${W}/.leader-upload.sh >/dev/null 2>&1 || true; done' </dev/null >/dev/null 2>&1 &`).catch(() => { });
|
|
270
284
|
};
|
|
271
|
-
if (cfg.s3 && cfg.envFactory) {
|
|
272
|
-
const s3 = cfg.s3;
|
|
273
|
-
const envFactory = cfg.envFactory;
|
|
274
|
-
const baseSha = body.baseSha;
|
|
275
|
-
const seedCmd = body.seedCmd;
|
|
276
|
-
const key = `${s3.keyPrefix ?? "leader-diff"}/${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}/${sub.workerId}.patch`;
|
|
277
|
-
const ttl = presignTtlSec;
|
|
278
|
-
const base = { endpoint: s3.endpoint, bucket: s3.bucket, accessKey: s3.accessKey, secretKey: s3.secretKey, ...(s3.region ? { region: s3.region } : {}), key, expiresSec: ttl };
|
|
279
|
-
const putUrl = presignS3Url({ ...base, method: "PUT" });
|
|
280
|
-
const getUrl = presignS3Url({ ...base, method: "GET" });
|
|
281
|
-
const script = buildUploadScript({ repoDir: repo, baseSha, putUrl, workDir: W });
|
|
282
|
-
const stage = async (env) => {
|
|
283
|
-
await stageWorkerEnv(env, { seedCmd, repo, branch: sub.branch, uploadScriptPath: `${W}/.leader-upload.sh`, uploadScript: script });
|
|
284
|
-
void sh(env)(`nohup sh -c 'while true; do sleep 90; sh ${W}/.leader-upload.sh >/dev/null 2>&1 || true; done' </dev/null >/dev/null 2>&1 &`).catch(() => { });
|
|
285
|
-
};
|
|
286
|
-
return {
|
|
287
|
-
workerId: sub.workerId, sessionId, branch: sub.branch,
|
|
288
|
-
baseSha,
|
|
289
|
-
runner: keepCtxWarm(new Runner({
|
|
290
|
-
brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing,
|
|
291
|
-
...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}),
|
|
292
|
-
...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}),
|
|
293
|
-
executionEnvFactory: (ctx) => withStaging(envFactory(ctx), stage, async (e) => {
|
|
294
|
-
await sh(e)(`cd ${repo} && git add -A && git -c user.name='leader-belt' -c user.email='belt@leader' commit -qm 'belt: auto-commit uncommitted worker output' || true; sh ${W}/.leader-upload.sh`);
|
|
295
|
-
}),
|
|
296
|
-
})),
|
|
297
|
-
fetchDiff: () => fetchUploadedDiff(getUrl),
|
|
298
|
-
spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec, objective: `${sub.spec.objective}${uploadStepSuffix(W)}` },
|
|
299
|
-
};
|
|
300
|
-
}
|
|
301
|
-
if (!cfg.e2bApiKey)
|
|
302
|
-
throw new Error("leader wire: no envFactory+s3 (factory mode) and no e2bApiKey (static mode)");
|
|
303
|
-
const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
|
|
304
|
-
if (!(await env.connect()).ok)
|
|
305
|
-
throw new Error(`connect failed for ${sub.workerId}`);
|
|
306
|
-
await sh(env)(`${body.seedCmd}; cd ${repo}; git checkout -q -b ${sub.branch}`);
|
|
307
|
-
const baseSha = await sh(env)(`cd ${repo} && git rev-parse HEAD`);
|
|
308
285
|
return {
|
|
309
|
-
workerId: sub.workerId, sessionId, branch: sub.branch,
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
286
|
+
workerId: sub.workerId, sessionId, branch: sub.branch,
|
|
287
|
+
baseSha,
|
|
288
|
+
runner: keepCtxWarm(new Runner({
|
|
289
|
+
brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing,
|
|
290
|
+
...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}),
|
|
291
|
+
...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}),
|
|
292
|
+
executionEnvFactory: (ctx) => withStaging(envFactory(ctx), stage, async (e) => {
|
|
293
|
+
await sh(e)(`cd ${repo} && git add -A && git -c user.name='leader-belt' -c user.email='belt@leader' commit -qm 'belt: auto-commit uncommitted worker output' || true; sh ${W}/.leader-upload.sh`);
|
|
294
|
+
}),
|
|
295
|
+
})),
|
|
296
|
+
fetchDiff: () => fetchUploadedDiff(getUrl),
|
|
297
|
+
spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec, objective: `${sub.spec.objective}${uploadStepSuffix(W)}` },
|
|
314
298
|
};
|
|
315
|
-
}
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
}
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
throw new Error("integ connect failed");
|
|
335
|
-
}
|
|
299
|
+
}
|
|
300
|
+
if (!cfg.e2bApiKey)
|
|
301
|
+
throw new Error("leader wire: no envFactory+s3 (factory mode) and no e2bApiKey (static mode)");
|
|
302
|
+
const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
|
|
303
|
+
if (!(await env.connect()).ok)
|
|
304
|
+
throw new Error(`connect failed for ${sub.workerId}`);
|
|
305
|
+
await sh(env)(`${body.seedCmd}; cd ${repo}; git checkout -q -b ${sub.branch}`);
|
|
306
|
+
const baseSha = await sh(env)(`cd ${repo} && git rev-parse HEAD`);
|
|
307
|
+
return {
|
|
308
|
+
workerId: sub.workerId, sessionId, branch: sub.branch, baseSha,
|
|
309
|
+
runner: keepCtxWarm(new Runner({ brain: cfg.brain, models: cfg.models, roles: cfg.roles, pricing: cfg.pricing, ...(cfg.toolResultStore ? { toolResultStore: cfg.toolResultStore } : {}), ...(cfg.sessionStore ? { sessionStore: cfg.sessionStore } : {}), executionEnv: env })),
|
|
310
|
+
diffEnv: env,
|
|
311
|
+
destroy: () => env.destroy().then(() => { }),
|
|
312
|
+
spec: { ...workerLimits, ...sub.spec, ...durableSpec, ...resourceSpec },
|
|
313
|
+
};
|
|
314
|
+
};
|
|
315
|
+
const provisionIntegrationSandbox = async () => {
|
|
316
|
+
if (cfg.envFactory) {
|
|
317
|
+
const env = cfg.envFactory({ sessionId: `leader-integ-${Date.now()}` });
|
|
336
318
|
try {
|
|
337
319
|
await sh(env)(body.seedCmd);
|
|
338
320
|
await injectOracles(env);
|
|
339
321
|
}
|
|
340
322
|
catch (e) {
|
|
341
|
-
await env.destroy().catch(() => { });
|
|
323
|
+
await Promise.resolve(env.destroy()).catch(() => { });
|
|
342
324
|
throw e;
|
|
343
325
|
}
|
|
344
|
-
return { env: asIntegrationEnv(env), destroy: () => env.destroy().then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
|
|
345
|
-
}
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
326
|
+
return { env: asIntegrationEnv(env), destroy: () => Promise.resolve(env.destroy()).then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
|
|
327
|
+
}
|
|
328
|
+
if (!cfg.e2bApiKey)
|
|
329
|
+
throw new Error("leader wire: no envFactory and no e2bApiKey");
|
|
330
|
+
const env = new RemoteContainerExecutionEnv({ apiKey: cfg.e2bApiKey, timeoutMs: leaderTimeoutMs });
|
|
331
|
+
if (!(await env.connect()).ok) {
|
|
332
|
+
await env.destroy().catch(() => { });
|
|
333
|
+
throw new Error("integ connect failed");
|
|
334
|
+
}
|
|
335
|
+
try {
|
|
336
|
+
await sh(env)(body.seedCmd);
|
|
337
|
+
await injectOracles(env);
|
|
338
|
+
}
|
|
339
|
+
catch (e) {
|
|
340
|
+
await env.destroy().catch(() => { });
|
|
341
|
+
throw e;
|
|
342
|
+
}
|
|
343
|
+
return { env: asIntegrationEnv(env), destroy: () => env.destroy().then(() => { }), ...(mkRepair(env, repo, body.oracleFiles ?? []) ? { repair: mkRepair(env, repo, body.oracleFiles ?? []) } : {}), ...(mkConflictResolver(env, repo, body.oracleFiles ?? []) ? { conflictResolver: mkConflictResolver(env, repo, body.oracleFiles ?? []) } : {}) };
|
|
344
|
+
};
|
|
345
|
+
const push = async (integratedPatch, baseSha) => {
|
|
346
|
+
const dir = mkdtempSync(join(tmpdir(), "leader-coord-"));
|
|
347
|
+
const gitEnv = { ...process.env, LC_ALL: "C" };
|
|
348
|
+
try {
|
|
349
|
+
execFileSync("git", ["clone", "-q", body.durableRemote, dir], { env: gitEnv });
|
|
350
|
+
execFileSync("git", ["-C", dir, "config", "user.name", ident.name]);
|
|
351
|
+
execFileSync("git", ["-C", dir, "config", "user.email", ident.email]);
|
|
352
|
+
writeFileSync(join(dir, ".leader.patch"), integratedPatch);
|
|
353
|
+
execFileSync("git", ["-C", dir, "am", "--3way", ".leader.patch"], { env: gitEnv });
|
|
354
|
+
rmSync(join(dir, ".leader.patch"));
|
|
355
|
+
const ref = body.targetRef ?? "refs/heads/main";
|
|
356
|
+
const ls = execFileSync("git", ["-C", dir, "ls-remote", "--refs", "origin", ref], { encoding: "utf8", env: gitEnv });
|
|
357
|
+
const exists = ls.split("\n").some((l) => l.split("\t")[1] === ref);
|
|
358
|
+
if (exists) {
|
|
359
|
+
execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:${baseSha}`, "origin", `HEAD:${ref}`], { env: gitEnv });
|
|
370
360
|
}
|
|
371
|
-
|
|
372
|
-
|
|
361
|
+
else {
|
|
362
|
+
execFileSync("git", ["-C", dir, "push", `--force-with-lease=${ref}:`, "origin", `HEAD:${ref}`], { env: gitEnv });
|
|
373
363
|
}
|
|
374
|
-
|
|
364
|
+
return { ok: true, ref };
|
|
365
|
+
}
|
|
366
|
+
catch (e) {
|
|
367
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
368
|
+
return { ok: false, raced: /stale info|force-with-lease|\[rejected\]|non-fast-forward/.test(msg), error: msg };
|
|
369
|
+
}
|
|
370
|
+
finally {
|
|
371
|
+
rmSync(dir, { recursive: true, force: true });
|
|
372
|
+
}
|
|
373
|
+
};
|
|
374
|
+
const deps = {
|
|
375
|
+
plan,
|
|
376
|
+
provisionWorker,
|
|
377
|
+
provisionIntegrationSandbox,
|
|
378
|
+
push,
|
|
375
379
|
testCmd: body.testCmd,
|
|
376
380
|
repoDir: repo, workerRepoDir: repo,
|
|
377
381
|
timeoutMs: leaderTimeoutMs, maxConcurrency: 4,
|