@abelxiaoxing/cadence 1.8.2 → 1.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +71 -531
- package/agents/design-explorer.md +15 -23
- package/agents/diagnosis-worker.md +12 -23
- package/agents/implementation-worker.md +16 -34
- package/config/.env.example +2 -5
- package/package.json +1 -1
- package/prompts/abel-diagnose.md +5 -5
- package/src/candidate-context.ts +289 -0
- package/src/contracts.ts +12 -22
- package/src/durable-contracts.ts +1 -5
- package/src/durable-workflow.ts +3 -159
- package/src/evidence-draft.ts +1 -1
- package/src/index.ts +12 -16
- package/src/package-candidate-process.ts +206 -0
- package/src/package-candidate.ts +11 -214
- package/src/package-workflow.ts +5 -42
- package/src/packet-runtime.ts +144 -255
- package/src/phase-execution.ts +134 -230
- package/src/startup-card.ts +17 -23
- package/src/subagent-process.ts +476 -0
- package/src/transport-budget.ts +1 -3
- package/src/workflow-policy.ts +9 -18
- package/src/workflow-state-machine.ts +1 -15
- package/config/routes.example.json +0 -39
- package/src/child-budget.ts +0 -58
- package/src/child-model.ts +0 -142
- package/src/child-session.ts +0 -640
- package/src/parent-provider.ts +0 -375
- package/src/route-policy.ts +0 -545
- package/src/submit-tool.ts +0 -950
- package/src/worker-broker.ts +0 -806
package/src/worker-broker.ts
DELETED
|
@@ -1,806 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
inspectRoutePolicy,
|
|
3
|
-
type RouteDialect,
|
|
4
|
-
type RoutePolicy,
|
|
5
|
-
type WorkerRole,
|
|
6
|
-
type WorkerRoutePolicy,
|
|
7
|
-
} from "./route-policy.ts";
|
|
8
|
-
import { TransportTimeout } from "./transport-budget.ts";
|
|
9
|
-
|
|
10
|
-
export const ROUTE_ATTEMPT_BOUNDS = Object.freeze({
|
|
11
|
-
totalMs: 20 * 60_000,
|
|
12
|
-
cooldownMs: 30_000,
|
|
13
|
-
attemptsPerRoute: 2,
|
|
14
|
-
});
|
|
15
|
-
|
|
16
|
-
export type RouteHealthState = "healthy" | "open" | "half-open";
|
|
17
|
-
|
|
18
|
-
export interface RouteRequirements {
|
|
19
|
-
dialects?: RouteDialect[];
|
|
20
|
-
minContextWindow?: number;
|
|
21
|
-
minOutputTokens?: number;
|
|
22
|
-
preferredContextWindow?: number;
|
|
23
|
-
preferredOutputTokens?: number;
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
interface RouteHealth {
|
|
27
|
-
state: RouteHealthState;
|
|
28
|
-
retryAt?: number;
|
|
29
|
-
probeInFlight?: boolean;
|
|
30
|
-
probeExpiresAt?: number;
|
|
31
|
-
lastCode?: string;
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
export interface RouteHealthStore {
|
|
35
|
-
get(fingerprint: string): RouteHealth | undefined;
|
|
36
|
-
set(fingerprint: string, health: RouteHealth): void;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
class MemoryRouteHealthStore implements RouteHealthStore {
|
|
40
|
-
readonly #values = new Map<string, RouteHealth>();
|
|
41
|
-
|
|
42
|
-
get(fingerprint: string): RouteHealth | undefined {
|
|
43
|
-
const value = this.#values.get(fingerprint);
|
|
44
|
-
return value ? structuredClone(value) : undefined;
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
set(fingerprint: string, health: RouteHealth): void {
|
|
48
|
-
this.#values.set(fingerprint, structuredClone(health));
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
export interface WorkerBrokerOptions {
|
|
53
|
-
now?: () => number;
|
|
54
|
-
healthStore?: RouteHealthStore;
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
export interface RunWorkerAttemptInput<T> {
|
|
58
|
-
runId: string;
|
|
59
|
-
taskId?: string;
|
|
60
|
-
operationId: string;
|
|
61
|
-
role: WorkerRole;
|
|
62
|
-
requirements?: RouteRequirements;
|
|
63
|
-
signal?: AbortSignal;
|
|
64
|
-
onActivity?: (event: BrokerActivityUpdate) => void;
|
|
65
|
-
classifyResult?: (value: T) => string | undefined;
|
|
66
|
-
execute(context: RouteAttemptContext): Promise<T>;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
export interface RouteAttemptContext {
|
|
70
|
-
route: WorkerRoutePolicy;
|
|
71
|
-
signal: AbortSignal;
|
|
72
|
-
onRequestStart(): void;
|
|
73
|
-
onHeaders(): void;
|
|
74
|
-
onProgress(): void;
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
export interface BrokerAttemptEvidence {
|
|
78
|
-
routeId: string;
|
|
79
|
-
code: string;
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
export interface BrokerActivityUpdate {
|
|
83
|
-
state: "preparing" | "waiting-first-response" | "running" | "retrying";
|
|
84
|
-
attempt: number;
|
|
85
|
-
maxAttempts: number;
|
|
86
|
-
code?: string;
|
|
87
|
-
wait?: string;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
function emitBrokerActivity(
|
|
91
|
-
observer: ((event: BrokerActivityUpdate) => void) | undefined,
|
|
92
|
-
event: BrokerActivityUpdate,
|
|
93
|
-
): void {
|
|
94
|
-
try {
|
|
95
|
-
observer?.(structuredClone(event));
|
|
96
|
-
} catch {
|
|
97
|
-
// Presentation cannot influence route selection or attempt control.
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
class RouteAttemptStop extends Error {
|
|
102
|
-
readonly code: "cancelled";
|
|
103
|
-
|
|
104
|
-
constructor(code: RouteAttemptStop["code"]) {
|
|
105
|
-
super(code);
|
|
106
|
-
this.name = "RouteAttemptStop";
|
|
107
|
-
this.code = code;
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
function isRole(value: unknown): value is WorkerRole {
|
|
112
|
-
return [
|
|
113
|
-
"design-explorer",
|
|
114
|
-
"implementation-worker",
|
|
115
|
-
"diagnosis-worker",
|
|
116
|
-
].includes(String(value));
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
function validRequirements(value: unknown): value is RouteRequirements {
|
|
120
|
-
if (value === undefined) return true;
|
|
121
|
-
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
122
|
-
const candidate = value as Record<string, unknown>;
|
|
123
|
-
if (
|
|
124
|
-
Object.keys(candidate).some(
|
|
125
|
-
(key) =>
|
|
126
|
-
![
|
|
127
|
-
"dialects",
|
|
128
|
-
"minContextWindow",
|
|
129
|
-
"minOutputTokens",
|
|
130
|
-
"preferredContextWindow",
|
|
131
|
-
"preferredOutputTokens",
|
|
132
|
-
].includes(key),
|
|
133
|
-
)
|
|
134
|
-
) {
|
|
135
|
-
return false;
|
|
136
|
-
}
|
|
137
|
-
return (
|
|
138
|
-
(candidate.dialects === undefined ||
|
|
139
|
-
(Array.isArray(candidate.dialects) &&
|
|
140
|
-
candidate.dialects.length > 0 &&
|
|
141
|
-
candidate.dialects.every((dialect) =>
|
|
142
|
-
[
|
|
143
|
-
"openai-completions",
|
|
144
|
-
"openai-responses",
|
|
145
|
-
"anthropic-messages",
|
|
146
|
-
].includes(String(dialect)),
|
|
147
|
-
))) &&
|
|
148
|
-
(candidate.minContextWindow === undefined ||
|
|
149
|
-
(Number.isSafeInteger(candidate.minContextWindow) &&
|
|
150
|
-
(candidate.minContextWindow as number) > 0)) &&
|
|
151
|
-
(candidate.minOutputTokens === undefined ||
|
|
152
|
-
(Number.isSafeInteger(candidate.minOutputTokens) &&
|
|
153
|
-
(candidate.minOutputTokens as number) > 0)) &&
|
|
154
|
-
[candidate.preferredContextWindow, candidate.preferredOutputTokens].every(
|
|
155
|
-
(value) =>
|
|
156
|
-
value === undefined ||
|
|
157
|
-
(Number.isSafeInteger(value) && (value as number) > 0),
|
|
158
|
-
)
|
|
159
|
-
);
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
function capable(
|
|
163
|
-
route: WorkerRoutePolicy,
|
|
164
|
-
role: WorkerRole,
|
|
165
|
-
requirements: RouteRequirements,
|
|
166
|
-
): boolean {
|
|
167
|
-
const dialects = requirements.dialects ?? [];
|
|
168
|
-
return (
|
|
169
|
-
route.capabilities.roles.includes(role) &&
|
|
170
|
-
dialects.every((dialect) =>
|
|
171
|
-
route.capabilities.dialects.includes(dialect),
|
|
172
|
-
) &&
|
|
173
|
-
route.capabilities.contextWindow >= (requirements.minContextWindow ?? 0) &&
|
|
174
|
-
route.capabilities.maxTokens >= (requirements.minOutputTokens ?? 0)
|
|
175
|
-
);
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
function safeFailureCode(error: unknown): string {
|
|
179
|
-
if (error instanceof RouteAttemptStop || error instanceof TransportTimeout)
|
|
180
|
-
return error.code;
|
|
181
|
-
if (error instanceof Error && error.message === "child-provider-rate-limited")
|
|
182
|
-
return error.message;
|
|
183
|
-
return "transport-failure";
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
export class WorkerBroker {
|
|
187
|
-
readonly #policy: RoutePolicy;
|
|
188
|
-
readonly #now: () => number;
|
|
189
|
-
readonly #healthStore: RouteHealthStore;
|
|
190
|
-
|
|
191
|
-
constructor(policy: RoutePolicy, options: WorkerBrokerOptions = {}) {
|
|
192
|
-
this.#policy = structuredClone(policy);
|
|
193
|
-
this.#now = options.now ?? Date.now;
|
|
194
|
-
this.#healthStore = options.healthStore ?? new MemoryRouteHealthStore();
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
#routeHealth(routeId: string): RouteHealth {
|
|
198
|
-
const route = this.#policy.routes[routeId];
|
|
199
|
-
return (
|
|
200
|
-
(route ? this.#healthStore.get(route.fingerprint) : undefined) ?? {
|
|
201
|
-
state: "healthy",
|
|
202
|
-
}
|
|
203
|
-
);
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
#setRouteHealth(routeId: string, health: RouteHealth): void {
|
|
207
|
-
const route = this.#policy.routes[routeId];
|
|
208
|
-
if (!route) throw new Error("route-not-declared");
|
|
209
|
-
this.#healthStore.set(route.fingerprint, health);
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
markFailure(routeId: string, code: string): void {
|
|
213
|
-
if (!Object.hasOwn(this.#policy.routes, routeId)) {
|
|
214
|
-
throw new Error("route-not-declared");
|
|
215
|
-
}
|
|
216
|
-
this.#setRouteHealth(routeId, {
|
|
217
|
-
state: "open",
|
|
218
|
-
retryAt: this.#now() + ROUTE_ATTEMPT_BOUNDS.cooldownMs,
|
|
219
|
-
lastCode: code,
|
|
220
|
-
});
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
markSuccess(routeId: string): void {
|
|
224
|
-
if (!Object.hasOwn(this.#policy.routes, routeId)) {
|
|
225
|
-
throw new Error("route-not-declared");
|
|
226
|
-
}
|
|
227
|
-
this.#setRouteHealth(routeId, { state: "healthy" });
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
#releaseHalfOpenProbeForRetry(routeId: string, code: string): void {
|
|
231
|
-
const health = this.#routeHealth(routeId);
|
|
232
|
-
if (health.state !== "half-open" || !health.probeInFlight) return;
|
|
233
|
-
this.#setRouteHealth(routeId, {
|
|
234
|
-
state: "open",
|
|
235
|
-
retryAt: this.#now(),
|
|
236
|
-
lastCode: code,
|
|
237
|
-
});
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
select(
|
|
241
|
-
input: RouteRequirements & {
|
|
242
|
-
role: WorkerRole;
|
|
243
|
-
exclude?: readonly string[];
|
|
244
|
-
routeId?: string;
|
|
245
|
-
},
|
|
246
|
-
):
|
|
247
|
-
| { ok: true; route: WorkerRoutePolicy; health: RouteHealthState }
|
|
248
|
-
| { ok: false; code: "endpoint-unavailable" | "route-not-declared" } {
|
|
249
|
-
const requirements: RouteRequirements = {
|
|
250
|
-
...(input.dialects ? { dialects: input.dialects } : {}),
|
|
251
|
-
...(input.minContextWindow === undefined
|
|
252
|
-
? {}
|
|
253
|
-
: { minContextWindow: input.minContextWindow }),
|
|
254
|
-
...(input.minOutputTokens === undefined
|
|
255
|
-
? {}
|
|
256
|
-
: { minOutputTokens: input.minOutputTokens }),
|
|
257
|
-
...(input.preferredContextWindow === undefined
|
|
258
|
-
? {}
|
|
259
|
-
: { preferredContextWindow: input.preferredContextWindow }),
|
|
260
|
-
...(input.preferredOutputTokens === undefined
|
|
261
|
-
? {}
|
|
262
|
-
: { preferredOutputTokens: input.preferredOutputTokens }),
|
|
263
|
-
};
|
|
264
|
-
if (!isRole(input.role) || !validRequirements(requirements)) {
|
|
265
|
-
return { ok: false, code: "endpoint-unavailable" };
|
|
266
|
-
}
|
|
267
|
-
const declared = this.#policy.roles[input.role];
|
|
268
|
-
const candidates = input.routeId
|
|
269
|
-
? declared.includes(input.routeId)
|
|
270
|
-
? [input.routeId]
|
|
271
|
-
: []
|
|
272
|
-
: declared;
|
|
273
|
-
if (input.routeId && candidates.length === 0) {
|
|
274
|
-
return { ok: false, code: "route-not-declared" };
|
|
275
|
-
}
|
|
276
|
-
const excluded = new Set(input.exclude ?? []);
|
|
277
|
-
// Stable preference tiers preserve declared order within each tier. Health,
|
|
278
|
-
// hard minima and explicit pins still determine eligibility.
|
|
279
|
-
const preferred = (routeId: string) => {
|
|
280
|
-
const route = this.#policy.routes[routeId];
|
|
281
|
-
return route &&
|
|
282
|
-
route.capabilities.contextWindow >=
|
|
283
|
-
(requirements.preferredContextWindow ?? 0) &&
|
|
284
|
-
route.capabilities.maxTokens >=
|
|
285
|
-
(requirements.preferredOutputTokens ?? 0)
|
|
286
|
-
? 1
|
|
287
|
-
: 0;
|
|
288
|
-
};
|
|
289
|
-
const ordered = [...candidates].sort(
|
|
290
|
-
(left, right) => preferred(right) - preferred(left),
|
|
291
|
-
);
|
|
292
|
-
for (const routeId of ordered) {
|
|
293
|
-
if (excluded.has(routeId)) continue;
|
|
294
|
-
const route = this.#policy.routes[routeId];
|
|
295
|
-
if (!route || !capable(route, input.role, requirements)) continue;
|
|
296
|
-
const health = this.#routeHealth(routeId);
|
|
297
|
-
if (
|
|
298
|
-
health.state === "open" &&
|
|
299
|
-
health.retryAt !== undefined &&
|
|
300
|
-
health.retryAt > this.#now()
|
|
301
|
-
) {
|
|
302
|
-
continue;
|
|
303
|
-
}
|
|
304
|
-
if (
|
|
305
|
-
(health.state === "open" || health.state === "half-open") &&
|
|
306
|
-
health.probeInFlight &&
|
|
307
|
-
health.probeExpiresAt !== undefined &&
|
|
308
|
-
health.probeExpiresAt > this.#now()
|
|
309
|
-
) {
|
|
310
|
-
continue;
|
|
311
|
-
}
|
|
312
|
-
if (health.state === "open") {
|
|
313
|
-
return { ok: true, route: structuredClone(route), health: "half-open" };
|
|
314
|
-
}
|
|
315
|
-
return {
|
|
316
|
-
ok: true,
|
|
317
|
-
route: structuredClone(route),
|
|
318
|
-
health: health.state,
|
|
319
|
-
};
|
|
320
|
-
}
|
|
321
|
-
return { ok: false, code: "endpoint-unavailable" };
|
|
322
|
-
}
|
|
323
|
-
|
|
324
|
-
async #attempt<T>(
|
|
325
|
-
route: WorkerRoutePolicy,
|
|
326
|
-
execute: (context: RouteAttemptContext) => Promise<T>,
|
|
327
|
-
parentSignal?: AbortSignal,
|
|
328
|
-
onActivity?: (state: "waiting-first-response" | "running") => void,
|
|
329
|
-
): Promise<T> {
|
|
330
|
-
const controller = new AbortController();
|
|
331
|
-
let accepting = true;
|
|
332
|
-
let running = false;
|
|
333
|
-
const onParentAbort = () =>
|
|
334
|
-
controller.abort(new RouteAttemptStop("cancelled"));
|
|
335
|
-
if (parentSignal?.aborted) onParentAbort();
|
|
336
|
-
else parentSignal?.addEventListener("abort", onParentAbort, { once: true });
|
|
337
|
-
const timer = setTimeout(
|
|
338
|
-
() => controller.abort(new TransportTimeout("attempt-timeout")),
|
|
339
|
-
ROUTE_ATTEMPT_BOUNDS.totalMs,
|
|
340
|
-
);
|
|
341
|
-
let removeAbortRace: () => void = () => {};
|
|
342
|
-
try {
|
|
343
|
-
const execution = Promise.resolve().then(() => {
|
|
344
|
-
controller.signal.throwIfAborted();
|
|
345
|
-
return execute({
|
|
346
|
-
route: structuredClone(route),
|
|
347
|
-
signal: controller.signal,
|
|
348
|
-
onRequestStart: () => {
|
|
349
|
-
if (!accepting || controller.signal.aborted) return;
|
|
350
|
-
running = false;
|
|
351
|
-
onActivity?.("waiting-first-response");
|
|
352
|
-
},
|
|
353
|
-
onHeaders: () => {
|
|
354
|
-
if (!accepting || controller.signal.aborted || running) return;
|
|
355
|
-
onActivity?.("waiting-first-response");
|
|
356
|
-
},
|
|
357
|
-
onProgress: () => {
|
|
358
|
-
if (!accepting || controller.signal.aborted) return;
|
|
359
|
-
running = true;
|
|
360
|
-
onActivity?.("running");
|
|
361
|
-
},
|
|
362
|
-
});
|
|
363
|
-
});
|
|
364
|
-
// A timed-out Provider may ignore its AbortSignal and settle later. The
|
|
365
|
-
// broker owns the bounded result, so absorb that late settlement.
|
|
366
|
-
void execution.catch(() => undefined);
|
|
367
|
-
const aborted = new Promise<never>((_resolve, reject) => {
|
|
368
|
-
const onAbort = () =>
|
|
369
|
-
reject(
|
|
370
|
-
controller.signal.reason instanceof Error
|
|
371
|
-
? controller.signal.reason
|
|
372
|
-
: new RouteAttemptStop("cancelled"),
|
|
373
|
-
);
|
|
374
|
-
removeAbortRace = () =>
|
|
375
|
-
controller.signal.removeEventListener("abort", onAbort);
|
|
376
|
-
if (controller.signal.aborted) onAbort();
|
|
377
|
-
else
|
|
378
|
-
controller.signal.addEventListener("abort", onAbort, { once: true });
|
|
379
|
-
});
|
|
380
|
-
return await Promise.race([execution, aborted]);
|
|
381
|
-
} finally {
|
|
382
|
-
removeAbortRace();
|
|
383
|
-
accepting = false;
|
|
384
|
-
clearTimeout(timer);
|
|
385
|
-
parentSignal?.removeEventListener("abort", onParentAbort);
|
|
386
|
-
}
|
|
387
|
-
}
|
|
388
|
-
|
|
389
|
-
async run<T>(input: {
|
|
390
|
-
operationId: string;
|
|
391
|
-
role: WorkerRole;
|
|
392
|
-
requirements?: RouteRequirements;
|
|
393
|
-
signal?: AbortSignal;
|
|
394
|
-
onActivity?: (event: BrokerActivityUpdate) => void;
|
|
395
|
-
classifyResult?: (value: T) => string | undefined;
|
|
396
|
-
execute: (context: RouteAttemptContext) => Promise<T>;
|
|
397
|
-
}): Promise<
|
|
398
|
-
| { ok: true; routeId: string; value: T; attempts: BrokerAttemptEvidence[] }
|
|
399
|
-
| {
|
|
400
|
-
ok: false;
|
|
401
|
-
state: "paused" | "cancelled";
|
|
402
|
-
code: string;
|
|
403
|
-
attempts: BrokerAttemptEvidence[];
|
|
404
|
-
}
|
|
405
|
-
> {
|
|
406
|
-
if (
|
|
407
|
-
typeof input.operationId !== "string" ||
|
|
408
|
-
input.operationId.length === 0 ||
|
|
409
|
-
!isRole(input.role) ||
|
|
410
|
-
!validRequirements(input.requirements)
|
|
411
|
-
) {
|
|
412
|
-
return {
|
|
413
|
-
ok: false,
|
|
414
|
-
state: "paused",
|
|
415
|
-
code: "broker-input-invalid",
|
|
416
|
-
attempts: [],
|
|
417
|
-
};
|
|
418
|
-
}
|
|
419
|
-
const attempted = new Set<string>();
|
|
420
|
-
const evidence: BrokerAttemptEvidence[] = [];
|
|
421
|
-
const capableRouteCount = this.#policy.roles[input.role].filter(
|
|
422
|
-
(routeId) => {
|
|
423
|
-
const route = this.#policy.routes[routeId];
|
|
424
|
-
return route
|
|
425
|
-
? capable(route, input.role, input.requirements ?? {})
|
|
426
|
-
: false;
|
|
427
|
-
},
|
|
428
|
-
).length;
|
|
429
|
-
const maxAttempts = Math.max(
|
|
430
|
-
1,
|
|
431
|
-
capableRouteCount > 1
|
|
432
|
-
? capableRouteCount
|
|
433
|
-
: ROUTE_ATTEMPT_BOUNDS.attemptsPerRoute,
|
|
434
|
-
);
|
|
435
|
-
for (;;) {
|
|
436
|
-
const selection = this.select({
|
|
437
|
-
role: input.role,
|
|
438
|
-
...(input.requirements ?? {}),
|
|
439
|
-
exclude: [...attempted],
|
|
440
|
-
});
|
|
441
|
-
if (!selection.ok) {
|
|
442
|
-
return {
|
|
443
|
-
ok: false,
|
|
444
|
-
state: "paused",
|
|
445
|
-
code: evidence.at(-1)?.code ?? selection.code,
|
|
446
|
-
attempts: evidence,
|
|
447
|
-
};
|
|
448
|
-
}
|
|
449
|
-
const routeId = selection.route.id;
|
|
450
|
-
attempted.add(routeId);
|
|
451
|
-
if (selection.health === "half-open") {
|
|
452
|
-
this.#setRouteHealth(routeId, {
|
|
453
|
-
...this.#routeHealth(routeId),
|
|
454
|
-
state: "half-open",
|
|
455
|
-
probeInFlight: true,
|
|
456
|
-
probeExpiresAt: this.#now() + ROUTE_ATTEMPT_BOUNDS.totalMs,
|
|
457
|
-
});
|
|
458
|
-
}
|
|
459
|
-
const routeAttempt = evidence.filter(
|
|
460
|
-
(item) => item.routeId === routeId,
|
|
461
|
-
).length;
|
|
462
|
-
const attempt = evidence.length + 1;
|
|
463
|
-
emitBrokerActivity(input.onActivity, {
|
|
464
|
-
state: "preparing",
|
|
465
|
-
attempt,
|
|
466
|
-
maxAttempts,
|
|
467
|
-
wait: "worker-setup",
|
|
468
|
-
});
|
|
469
|
-
try {
|
|
470
|
-
const value = await this.#attempt(
|
|
471
|
-
selection.route,
|
|
472
|
-
input.execute,
|
|
473
|
-
input.signal,
|
|
474
|
-
(state) =>
|
|
475
|
-
emitBrokerActivity(input.onActivity, {
|
|
476
|
-
state,
|
|
477
|
-
attempt,
|
|
478
|
-
maxAttempts,
|
|
479
|
-
wait:
|
|
480
|
-
state === "waiting-first-response"
|
|
481
|
-
? "first-response"
|
|
482
|
-
: "worker-progress",
|
|
483
|
-
}),
|
|
484
|
-
);
|
|
485
|
-
const semanticCode = input.classifyResult?.(value);
|
|
486
|
-
if (semanticCode) {
|
|
487
|
-
evidence.push({ routeId, code: semanticCode });
|
|
488
|
-
const routeHasRetry =
|
|
489
|
-
capableRouteCount === 1 &&
|
|
490
|
-
routeAttempt + 1 < ROUTE_ATTEMPT_BOUNDS.attemptsPerRoute;
|
|
491
|
-
if (routeHasRetry) {
|
|
492
|
-
this.#releaseHalfOpenProbeForRetry(routeId, semanticCode);
|
|
493
|
-
attempted.delete(routeId);
|
|
494
|
-
} else {
|
|
495
|
-
this.markFailure(routeId, semanticCode);
|
|
496
|
-
}
|
|
497
|
-
if (attempt < maxAttempts) {
|
|
498
|
-
emitBrokerActivity(input.onActivity, {
|
|
499
|
-
state: "retrying",
|
|
500
|
-
attempt,
|
|
501
|
-
maxAttempts,
|
|
502
|
-
code: semanticCode,
|
|
503
|
-
wait: "bounded-policy",
|
|
504
|
-
});
|
|
505
|
-
continue;
|
|
506
|
-
}
|
|
507
|
-
if (routeHasRetry) this.markFailure(routeId, semanticCode);
|
|
508
|
-
return { ok: true, routeId, value, attempts: evidence };
|
|
509
|
-
}
|
|
510
|
-
this.markSuccess(routeId);
|
|
511
|
-
return { ok: true, routeId, value, attempts: evidence };
|
|
512
|
-
} catch (error) {
|
|
513
|
-
const code = safeFailureCode(error);
|
|
514
|
-
evidence.push({ routeId, code });
|
|
515
|
-
if (code === "cancelled") {
|
|
516
|
-
const health = this.#routeHealth(routeId);
|
|
517
|
-
if (health.state === "half-open" && health.probeInFlight) {
|
|
518
|
-
this.#setRouteHealth(routeId, {
|
|
519
|
-
state: "open",
|
|
520
|
-
retryAt: this.#now(),
|
|
521
|
-
...(health.lastCode ? { lastCode: health.lastCode } : {}),
|
|
522
|
-
});
|
|
523
|
-
}
|
|
524
|
-
return { ok: false, state: "cancelled", code, attempts: evidence };
|
|
525
|
-
}
|
|
526
|
-
const routeHasRetry =
|
|
527
|
-
capableRouteCount === 1 &&
|
|
528
|
-
routeAttempt + 1 < ROUTE_ATTEMPT_BOUNDS.attemptsPerRoute;
|
|
529
|
-
if (routeHasRetry) {
|
|
530
|
-
this.#releaseHalfOpenProbeForRetry(routeId, code);
|
|
531
|
-
attempted.delete(routeId);
|
|
532
|
-
} else {
|
|
533
|
-
this.markFailure(routeId, code);
|
|
534
|
-
}
|
|
535
|
-
if (attempt < maxAttempts) {
|
|
536
|
-
emitBrokerActivity(input.onActivity, {
|
|
537
|
-
state: "retrying",
|
|
538
|
-
attempt,
|
|
539
|
-
maxAttempts,
|
|
540
|
-
code,
|
|
541
|
-
wait: "bounded-policy",
|
|
542
|
-
});
|
|
543
|
-
} else if (routeHasRetry) {
|
|
544
|
-
this.markFailure(routeId, code);
|
|
545
|
-
}
|
|
546
|
-
}
|
|
547
|
-
}
|
|
548
|
-
}
|
|
549
|
-
|
|
550
|
-
rebind(input: Record<string, unknown>):
|
|
551
|
-
| { ok: true; runId: string; route: WorkerRoutePolicy }
|
|
552
|
-
| {
|
|
553
|
-
ok: false;
|
|
554
|
-
code:
|
|
555
|
-
| "approval-boundary"
|
|
556
|
-
| "route-not-declared"
|
|
557
|
-
| "endpoint-unavailable"
|
|
558
|
-
| "route-capability-insufficient";
|
|
559
|
-
} {
|
|
560
|
-
const allowed = new Set(["runId", "role", "routeId", "requirements"]);
|
|
561
|
-
if (Object.keys(input).some((key) => !allowed.has(key))) {
|
|
562
|
-
return { ok: false, code: "approval-boundary" };
|
|
563
|
-
}
|
|
564
|
-
if (
|
|
565
|
-
typeof input.runId !== "string" ||
|
|
566
|
-
input.runId.length === 0 ||
|
|
567
|
-
!isRole(input.role) ||
|
|
568
|
-
typeof input.routeId !== "string" ||
|
|
569
|
-
!validRequirements(input.requirements)
|
|
570
|
-
) {
|
|
571
|
-
return { ok: false, code: "route-capability-insufficient" };
|
|
572
|
-
}
|
|
573
|
-
const selection = this.select({
|
|
574
|
-
role: input.role,
|
|
575
|
-
...(input.requirements ?? {}),
|
|
576
|
-
routeId: input.routeId,
|
|
577
|
-
});
|
|
578
|
-
if (!selection.ok) {
|
|
579
|
-
const route = this.#policy.routes[input.routeId];
|
|
580
|
-
return {
|
|
581
|
-
ok: false,
|
|
582
|
-
code:
|
|
583
|
-
selection.code === "route-not-declared"
|
|
584
|
-
? "route-not-declared"
|
|
585
|
-
: route && capable(route, input.role, input.requirements ?? {})
|
|
586
|
-
? "endpoint-unavailable"
|
|
587
|
-
: "route-capability-insufficient",
|
|
588
|
-
};
|
|
589
|
-
}
|
|
590
|
-
return {
|
|
591
|
-
ok: true,
|
|
592
|
-
runId: input.runId,
|
|
593
|
-
route: structuredClone(selection.route),
|
|
594
|
-
};
|
|
595
|
-
}
|
|
596
|
-
|
|
597
|
-
status(): Record<string, unknown> {
|
|
598
|
-
const health = Object.fromEntries(
|
|
599
|
-
Object.keys(this.#policy.routes).map((routeId) => {
|
|
600
|
-
const value = this.#routeHealth(routeId);
|
|
601
|
-
return [
|
|
602
|
-
routeId,
|
|
603
|
-
{
|
|
604
|
-
state: value.state,
|
|
605
|
-
...(value.retryAt === undefined ? {} : { retryAt: value.retryAt }),
|
|
606
|
-
},
|
|
607
|
-
];
|
|
608
|
-
}),
|
|
609
|
-
);
|
|
610
|
-
return inspectRoutePolicy(
|
|
611
|
-
{
|
|
612
|
-
ok: true,
|
|
613
|
-
source: { kind: "project", path: "<injected>" },
|
|
614
|
-
policy: this.#policy,
|
|
615
|
-
},
|
|
616
|
-
health,
|
|
617
|
-
);
|
|
618
|
-
}
|
|
619
|
-
}
|
|
620
|
-
|
|
621
|
-
/**
|
|
622
|
-
* Adds run/role defaults and independent task bindings to route attempts.
|
|
623
|
-
* All bound brokers consult the same health store.
|
|
624
|
-
* Attempt credentials and provider objects remain disposable and are never
|
|
625
|
-
* retained by this facade.
|
|
626
|
-
*/
|
|
627
|
-
export class RunWorkerBroker {
|
|
628
|
-
#policy: RoutePolicy;
|
|
629
|
-
readonly #options: WorkerBrokerOptions;
|
|
630
|
-
#defaultBroker: WorkerBroker;
|
|
631
|
-
readonly #bindings = new Map<
|
|
632
|
-
string,
|
|
633
|
-
{ routeId: string; fingerprint: string }
|
|
634
|
-
>();
|
|
635
|
-
readonly #boundBrokers = new Map<string, WorkerBroker>();
|
|
636
|
-
|
|
637
|
-
constructor(policy: RoutePolicy, options: WorkerBrokerOptions = {}) {
|
|
638
|
-
this.#policy = structuredClone(policy);
|
|
639
|
-
this.#options = {
|
|
640
|
-
...options,
|
|
641
|
-
healthStore: options.healthStore ?? new MemoryRouteHealthStore(),
|
|
642
|
-
};
|
|
643
|
-
this.#defaultBroker = new WorkerBroker(this.#policy, this.#options);
|
|
644
|
-
}
|
|
645
|
-
|
|
646
|
-
#bindingKey(runId: string, role: WorkerRole, taskId?: string): string {
|
|
647
|
-
return JSON.stringify([runId, role, taskId ?? null]);
|
|
648
|
-
}
|
|
649
|
-
|
|
650
|
-
#brokerFor(runId: string, role: WorkerRole, taskId?: string): WorkerBroker {
|
|
651
|
-
const binding =
|
|
652
|
-
this.#bindings.get(this.#bindingKey(runId, role, taskId)) ??
|
|
653
|
-
this.#bindings.get(this.#bindingKey(runId, role));
|
|
654
|
-
if (!binding) return this.#defaultBroker;
|
|
655
|
-
const routeId = binding.routeId;
|
|
656
|
-
const key = `${role}\0${routeId}`;
|
|
657
|
-
const existing = this.#boundBrokers.get(key);
|
|
658
|
-
if (existing) return existing;
|
|
659
|
-
const ordered = [
|
|
660
|
-
routeId,
|
|
661
|
-
...this.#policy.roles[role].filter((candidate) => candidate !== routeId),
|
|
662
|
-
];
|
|
663
|
-
const policy: RoutePolicy = {
|
|
664
|
-
...structuredClone(this.#policy),
|
|
665
|
-
roles: Object.freeze({
|
|
666
|
-
...structuredClone(this.#policy.roles),
|
|
667
|
-
[role]: Object.freeze(ordered),
|
|
668
|
-
}),
|
|
669
|
-
};
|
|
670
|
-
const broker = new WorkerBroker(policy, this.#options);
|
|
671
|
-
this.#boundBrokers.set(key, broker);
|
|
672
|
-
return broker;
|
|
673
|
-
}
|
|
674
|
-
|
|
675
|
-
async run<T>(input: RunWorkerAttemptInput<T>) {
|
|
676
|
-
if (
|
|
677
|
-
typeof input.runId !== "string" ||
|
|
678
|
-
input.runId.length === 0 ||
|
|
679
|
-
(input.taskId !== undefined &&
|
|
680
|
-
(typeof input.taskId !== "string" || input.taskId.length === 0)) ||
|
|
681
|
-
typeof input.operationId !== "string" ||
|
|
682
|
-
input.operationId.length === 0
|
|
683
|
-
) {
|
|
684
|
-
return {
|
|
685
|
-
ok: false as const,
|
|
686
|
-
state: "paused" as const,
|
|
687
|
-
code: "broker-input-invalid",
|
|
688
|
-
attempts: [],
|
|
689
|
-
};
|
|
690
|
-
}
|
|
691
|
-
const result = await this.#brokerFor(
|
|
692
|
-
input.runId,
|
|
693
|
-
input.role,
|
|
694
|
-
input.taskId,
|
|
695
|
-
).run({
|
|
696
|
-
operationId: input.operationId,
|
|
697
|
-
role: input.role,
|
|
698
|
-
...(input.requirements ? { requirements: input.requirements } : {}),
|
|
699
|
-
...(input.signal ? { signal: input.signal } : {}),
|
|
700
|
-
...(input.onActivity ? { onActivity: input.onActivity } : {}),
|
|
701
|
-
...(input.classifyResult ? { classifyResult: input.classifyResult } : {}),
|
|
702
|
-
execute: input.execute,
|
|
703
|
-
});
|
|
704
|
-
if (result.ok) {
|
|
705
|
-
const route = this.#policy.routes[result.routeId];
|
|
706
|
-
if (!route) throw new Error("broker-route-result-invalid");
|
|
707
|
-
this.#bindings.set(
|
|
708
|
-
this.#bindingKey(input.runId, input.role, input.taskId),
|
|
709
|
-
{
|
|
710
|
-
routeId: route.id,
|
|
711
|
-
fingerprint: route.fingerprint,
|
|
712
|
-
},
|
|
713
|
-
);
|
|
714
|
-
}
|
|
715
|
-
return result;
|
|
716
|
-
}
|
|
717
|
-
|
|
718
|
-
rebind(input: {
|
|
719
|
-
runId: string;
|
|
720
|
-
taskId?: string;
|
|
721
|
-
role: WorkerRole;
|
|
722
|
-
routeId: string;
|
|
723
|
-
requirements?: RouteRequirements;
|
|
724
|
-
}) {
|
|
725
|
-
const { taskId, ...routeInput } = input;
|
|
726
|
-
if (
|
|
727
|
-
taskId !== undefined &&
|
|
728
|
-
(typeof taskId !== "string" || taskId.length === 0)
|
|
729
|
-
) {
|
|
730
|
-
return { ok: false as const, code: "approval-boundary" as const };
|
|
731
|
-
}
|
|
732
|
-
const result = this.#defaultBroker.rebind(routeInput);
|
|
733
|
-
if (result.ok) {
|
|
734
|
-
this.#bindings.set(
|
|
735
|
-
this.#bindingKey(input.runId, input.role, input.taskId),
|
|
736
|
-
{
|
|
737
|
-
routeId: result.route.id,
|
|
738
|
-
fingerprint: result.route.fingerprint,
|
|
739
|
-
},
|
|
740
|
-
);
|
|
741
|
-
}
|
|
742
|
-
return result;
|
|
743
|
-
}
|
|
744
|
-
|
|
745
|
-
resumeBinding(input: {
|
|
746
|
-
runId: string;
|
|
747
|
-
taskId?: string;
|
|
748
|
-
role: WorkerRole;
|
|
749
|
-
routeId: string;
|
|
750
|
-
expectedFingerprint?: string;
|
|
751
|
-
requirements?: RouteRequirements;
|
|
752
|
-
}) {
|
|
753
|
-
const binding = this.#bindings.get(
|
|
754
|
-
this.#bindingKey(input.runId, input.role, input.taskId),
|
|
755
|
-
);
|
|
756
|
-
const route = this.#policy.routes[input.routeId];
|
|
757
|
-
if (!binding) {
|
|
758
|
-
if (
|
|
759
|
-
!route ||
|
|
760
|
-
input.expectedFingerprint === undefined ||
|
|
761
|
-
route.fingerprint !== input.expectedFingerprint
|
|
762
|
-
) {
|
|
763
|
-
return { ok: false as const, code: "route-rebind-required" as const };
|
|
764
|
-
}
|
|
765
|
-
return this.rebind({
|
|
766
|
-
runId: input.runId,
|
|
767
|
-
...(input.taskId === undefined ? {} : { taskId: input.taskId }),
|
|
768
|
-
role: input.role,
|
|
769
|
-
routeId: input.routeId,
|
|
770
|
-
...(input.requirements ? { requirements: input.requirements } : {}),
|
|
771
|
-
});
|
|
772
|
-
}
|
|
773
|
-
if (
|
|
774
|
-
binding.routeId !== input.routeId ||
|
|
775
|
-
!route ||
|
|
776
|
-
route.fingerprint !== binding.fingerprint ||
|
|
777
|
-
(input.expectedFingerprint !== undefined &&
|
|
778
|
-
binding.fingerprint !== input.expectedFingerprint)
|
|
779
|
-
) {
|
|
780
|
-
return { ok: false as const, code: "route-rebind-required" as const };
|
|
781
|
-
}
|
|
782
|
-
return {
|
|
783
|
-
ok: true as const,
|
|
784
|
-
runId: input.runId,
|
|
785
|
-
route: structuredClone(route),
|
|
786
|
-
};
|
|
787
|
-
}
|
|
788
|
-
|
|
789
|
-
updatePolicy(policy: RoutePolicy): void {
|
|
790
|
-
this.#policy = structuredClone(policy);
|
|
791
|
-
this.#defaultBroker = new WorkerBroker(this.#policy, this.#options);
|
|
792
|
-
this.#boundBrokers.clear();
|
|
793
|
-
}
|
|
794
|
-
|
|
795
|
-
binding(
|
|
796
|
-
runId: string,
|
|
797
|
-
role: WorkerRole,
|
|
798
|
-
taskId?: string,
|
|
799
|
-
): string | undefined {
|
|
800
|
-
return this.#bindings.get(this.#bindingKey(runId, role, taskId))?.routeId;
|
|
801
|
-
}
|
|
802
|
-
|
|
803
|
-
status(): Record<string, unknown> {
|
|
804
|
-
return this.#defaultBroker.status();
|
|
805
|
-
}
|
|
806
|
-
}
|