agents-relay 2.0.27 → 2.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dashboard-html.js +3 -1
- package/dist/decision-audit.js +15 -6
- package/dist/execution-routing.js +165 -35
- package/dist/relayd.js +1 -0
- package/dist/router-evals.js +28 -23
- package/package.json +1 -1
package/dist/dashboard-html.js
CHANGED
|
@@ -329,6 +329,8 @@ const displayedEventIds=new Set();
|
|
|
329
329
|
function value(input){return input===null||input===undefined||input===''?'—':String(input);}
|
|
330
330
|
function text(input){return document.createTextNode(value(input));}
|
|
331
331
|
function element(name,className,child){const node=document.createElement(name);if(className)node.className=className;if(child)node.append(child);return node;}
|
|
332
|
+
function yamlScalar(input){if(input===null)return 'null';if(typeof input==='boolean'||typeof input==='number')return String(input);return JSON.stringify(String(input));}
|
|
333
|
+
function yamlDump(input,indent=0){const pad=' '.repeat(indent);if(Array.isArray(input)){if(!input.length)return pad+'[]';return input.map(item=>{if(item&&typeof item==='object'){return pad+'-\\n'+yamlDump(item,indent+2);}return pad+'- '+yamlScalar(item);}).join('\\n');}if(input&&typeof input==='object'){const entries=Object.entries(input);if(!entries.length)return pad+'{}';const lines=[];for(const [key,item] of entries){if(typeof item==='string'&&item.includes('\\n')){lines.push(pad+key+': |');for(const line of item.replace(/\\n$/,'').split('\\n'))lines.push(' '.repeat(indent+2)+line);continue;}if(item&&typeof item==='object'){lines.push(pad+key+':');lines.push(yamlDump(item,indent+2));}else{lines.push(pad+key+': '+yamlScalar(item));}}return lines.join('\\n');}return pad+yamlScalar(input);}
|
|
332
334
|
function field(label,input){const box=element('div','detail-group');box.append(element('p','detail-label',text(label)),element('p','detail-value',text(input)));return box;}
|
|
333
335
|
function markdownField(label,input){const box=element('div','detail-group');box.append(element('p','detail-label',text(label)),markdownValue(input));return box;}
|
|
334
336
|
function rememberSelection(){if(selectedJob)sessionStorage.setItem('agents-relay:selected-job',selectedJob);else sessionStorage.removeItem('agents-relay:selected-job');if(selectedTaskId)sessionStorage.setItem('agents-relay:selected-task',selectedTaskId);else sessionStorage.removeItem('agents-relay:selected-task');}
|
|
@@ -451,7 +453,7 @@ function routeWorker(record){const id=record?.selected?.worker?.id||record?.work
|
|
|
451
453
|
function renderAudit(records){auditLog.replaceChildren();if(!Array.isArray(records)||!records.length){auditLog.append(element('p','empty',text('No schema v5 routing decisions yet.')));return;}for(const record of records){const card=element('article','audit-entry');const head=element('div','audit-entry-head');head.append(element('strong','',text(routeWorker(record)+' · '+value(record?.selected?.model?.model))),element('span','subtle',text(new Date(record.timestamp).toLocaleString())));card.append(head);const correlation=record.correlation||{};card.append(element('div','subtle',text([correlation.job_id,correlation.task_id].filter(Boolean).join(' / ')||'uncorrelated')));const probabilities=record.decision_evidence?.probabilities||{};const gates=(record.gate_results||[]).filter(item=>!item.eligible).map(item=>item.candidate_id+': '+item.rejection_reasons.join(', '));const nl=String.fromCharCode(10);const details=document.createElement('pre');details.textContent='fallback: '+String(Boolean(record.decision_evidence?.fallback))+(Object.keys(probabilities).length?nl+'probabilities: '+JSON.stringify(probabilities,null,2):'')+(gates.length?nl+'rejected: '+gates.join(nl):'');card.append(details);auditLog.append(card);}}
|
|
452
454
|
async function drawAudit(){auditRefresh.disabled=true;try{renderAudit(await loadJSON('/api/decision-audit'));}catch(error){auditLog.replaceChildren(element('p','error-state',text('Audit unavailable: '+error.message)));}finally{auditRefresh.disabled=false;}}
|
|
453
455
|
async function drawRules(){rulesStatus.textContent='';try{const rules=await loadJSON('/api/execution-router-rules');rulesText.value=typeof rules.text==='string'?rules.text:'';rulesMeta.textContent=rules.hash&&rules.hash!=='none'?'Loaded · '+rules.hash.slice(0,12):'No custom rules saved';}catch(error){rulesMeta.textContent='Unavailable';rulesStatus.textContent='Could not load rules: '+error.message;}void drawAudit();}
|
|
454
|
-
async function testRouter(){rulesTest.disabled=true;routerTestResults.replaceChildren(element('p','subtle',text('Calling router with current status…')));try{if(!selectedJob||!selectedTaskId)throw new Error('Select a Job and Task first');const response=await fetch('/api/router-test',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({jobId:selectedJob,taskId:selectedTaskId})});if(!response.ok)throw new Error(response.status+' '+await response.text());const result=await response.json();const selected=result.selected||{};const payload={selected,probabilities:result.probabilities||{},fallback:Boolean(result.fallback),hard_gates:result.gates||[],laya_input:result.decisionInput||null};routerTestResults.replaceChildren(element('strong','',text((selected.worker||'—')+' · '+(selected.model||'—'))));routerTestResults.append(element('div','subtle',text('route: '+(selected.route||'—'))),element('pre','',text(
|
|
456
|
+
async function testRouter(){rulesTest.disabled=true;routerTestResults.replaceChildren(element('p','subtle',text('Calling router with current status…')));try{if(!selectedJob||!selectedTaskId)throw new Error('Select a Job and Task first');const response=await fetch('/api/router-test',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({jobId:selectedJob,taskId:selectedTaskId})});if(!response.ok)throw new Error(response.status+' '+await response.text());const result=await response.json();const selected=result.selected||{};const payload={selected,proposed_fields:result.proposedFields||{},probabilities:result.probabilities||{},fallback:Boolean(result.fallback),hard_gates:result.gates||[],laya_input:result.decisionInput||null};routerTestResults.replaceChildren(element('strong','',text((selected.worker||'—')+' · '+(selected.model||'—'))));routerTestResults.append(element('div','subtle',text('route: '+(selected.route||'—'))),element('pre','',text(yamlDump(payload))));}catch(error){routerTestResults.replaceChildren(element('p','error-state',text('Router test failed: '+error.message)));}finally{rulesTest.disabled=false;}}
|
|
455
457
|
async function saveRules(){rulesSave.disabled=true;rulesStatus.textContent='Saving…';try{const response=await fetch('/api/execution-router-rules',{method:'PUT',headers:{'content-type':'text/plain; charset=utf-8'},body:rulesText.value});if(!response.ok)throw new Error(response.status+' '+await response.text());const rules=await response.json();rulesMeta.textContent='Saved · '+rules.hash.slice(0,12);rulesStatus.textContent='Saved. New routing decisions will load these rules.';}catch(error){rulesStatus.textContent='Save failed: '+error.message;}finally{rulesSave.disabled=false;}}
|
|
456
458
|
async function drawShortcuts(){shortcutsStatus.textContent='';shortcutsText.value=chatShortcutText();}
|
|
457
459
|
function saveShortcuts(){const value=shortcutsText.value.trim();if(value)localStorage.setItem(CHAT_SHORTCUTS_KEY,value);else localStorage.removeItem(CHAT_SHORTCUTS_KEY);shortcutsText.value=chatShortcutText();shortcutsStatus.textContent='Saved.';}
|
package/dist/decision-audit.js
CHANGED
|
@@ -135,12 +135,21 @@ export function makeExecutionDecisionAudit(input) {
|
|
|
135
135
|
fallback: { used: input.routing.decisionEvidence?.fallback ?? false, reason: input.routing.decisionEvidence?.error },
|
|
136
136
|
decision_evidence: input.routing.decisionEvidence,
|
|
137
137
|
decision_input: {
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
138
|
+
type: 'staged_route_decision',
|
|
139
|
+
stages: input.decisionStages.map(stage => ({
|
|
140
|
+
name: stage.name,
|
|
141
|
+
facts: stage.facts,
|
|
142
|
+
contract: {
|
|
143
|
+
type: stage.contract.type,
|
|
144
|
+
question: stage.contract.question,
|
|
145
|
+
choices: [...stage.contract.choices],
|
|
146
|
+
},
|
|
147
|
+
result: {
|
|
148
|
+
choice: stage.result.choice,
|
|
149
|
+
probabilities: stage.result.probabilities ?? null,
|
|
150
|
+
bypassed: stage.result.bypassed ?? false,
|
|
151
|
+
},
|
|
152
|
+
})),
|
|
144
153
|
},
|
|
145
154
|
worker_candidates: input.workers.map(workerAuditCandidate),
|
|
146
155
|
worker_gate_results: input.workerGateEvaluations.map(item => ({
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { chooseWithDecision } from './decision.js';
|
|
2
1
|
import { makeExecutionDecisionAudit } from './decision-audit.js';
|
|
3
2
|
import { discoverModelCandidates } from './model-routing.js';
|
|
4
3
|
import { loadExecutionRouterRules, loadRoutingPolicy, policyAllows } from './routing-policy.js';
|
|
@@ -60,6 +59,65 @@ function modelGateReasons(candidate, constraints, excludedRoutes, policy) {
|
|
|
60
59
|
reasons.push('reasoning_constraint');
|
|
61
60
|
return reasons;
|
|
62
61
|
}
|
|
62
|
+
function codePlan(model) {
|
|
63
|
+
return {
|
|
64
|
+
five_hour: model.resourceWindows?.fiveHour ? {
|
|
65
|
+
remaining_percent: model.resourceWindows.fiveHour.remainingPercent ?? null,
|
|
66
|
+
reset_at: model.resourceWindows.fiveHour.resetAt ?? null,
|
|
67
|
+
reset_in_minutes: model.resourceWindows.fiveHour.resetInMinutes ?? null,
|
|
68
|
+
} : null,
|
|
69
|
+
weekly: model.resourceWindows?.weekly ? {
|
|
70
|
+
remaining_percent: model.resourceWindows.weekly.remainingPercent ?? null,
|
|
71
|
+
reset_at: model.resourceWindows.weekly.resetAt ?? null,
|
|
72
|
+
reset_in_minutes: model.resourceWindows.weekly.resetInMinutes ?? null,
|
|
73
|
+
} : null,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
function workerProviderFacts(routes) {
|
|
77
|
+
const workers = new Map();
|
|
78
|
+
for (const route of routes) {
|
|
79
|
+
const list = workers.get(route.worker.workerId) ?? [];
|
|
80
|
+
list.push(route);
|
|
81
|
+
workers.set(route.worker.workerId, list);
|
|
82
|
+
}
|
|
83
|
+
return [...workers.entries()].map(([workerId, workerRoutes]) => {
|
|
84
|
+
const providers = new Map();
|
|
85
|
+
for (const route of workerRoutes) {
|
|
86
|
+
const list = providers.get(route.model.provider) ?? [];
|
|
87
|
+
list.push(route);
|
|
88
|
+
providers.set(route.model.provider, list);
|
|
89
|
+
}
|
|
90
|
+
return {
|
|
91
|
+
id: workerId,
|
|
92
|
+
adapter: workerRoutes[0].worker.adapter,
|
|
93
|
+
providers: Object.fromEntries([...providers.entries()].map(([provider, providerRoutes]) => {
|
|
94
|
+
const configured = providerRoutes.find(route => !route.model.id.startsWith('catalog:')) ?? providerRoutes[0];
|
|
95
|
+
return [provider, {
|
|
96
|
+
default: { model: configured.model.model, reasoning: configured.model.reasoning ?? null },
|
|
97
|
+
code_plan: codePlan(configured.model),
|
|
98
|
+
}];
|
|
99
|
+
})),
|
|
100
|
+
};
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
async function decideStage(engine, name, facts, question, choices) {
|
|
104
|
+
if (choices.length === 0)
|
|
105
|
+
throw new Error('No choices available for ' + name + ' stage');
|
|
106
|
+
const contract = { type: 'choice', question, choices };
|
|
107
|
+
if (choices.length === 1)
|
|
108
|
+
return { name, facts, contract, result: { choice: choices[0], bypassed: true } };
|
|
109
|
+
const decision = await engine.decide({ facts }, contract);
|
|
110
|
+
if (decision.type !== 'choice')
|
|
111
|
+
throw new Error('Execution router expected a choice result for ' + name);
|
|
112
|
+
return { name, facts, contract, result: { choice: decision.choice, probabilities: decision.probabilities } };
|
|
113
|
+
}
|
|
114
|
+
function stageFacts(baseFacts, selected, candidates) {
|
|
115
|
+
return JSON.parse(JSON.stringify({
|
|
116
|
+
...baseFacts,
|
|
117
|
+
selected,
|
|
118
|
+
...candidates,
|
|
119
|
+
}));
|
|
120
|
+
}
|
|
63
121
|
function routeId(worker, model) {
|
|
64
122
|
return worker.workerId + '::' + model.id + (model.reasoning ? '@' + model.reasoning : '');
|
|
65
123
|
}
|
|
@@ -116,7 +174,7 @@ export async function routeExecution(workers, requirements, options = {}) {
|
|
|
116
174
|
const chatgptRunning = options.chatgptRunning ?? 0;
|
|
117
175
|
const taskFacts = options.taskFacts ?? {};
|
|
118
176
|
const taskShape = executionTaskShape(taskFacts);
|
|
119
|
-
const
|
|
177
|
+
const baseFacts = JSON.parse(JSON.stringify({
|
|
120
178
|
context: {
|
|
121
179
|
job: taskFacts.job ?? null,
|
|
122
180
|
task: taskFacts.task ?? taskFacts,
|
|
@@ -136,36 +194,109 @@ export async function routeExecution(workers, requirements, options = {}) {
|
|
|
136
194
|
chatgpt_running: chatgptRunning,
|
|
137
195
|
preferred_chatgpt_concurrency: preferredChatgptConcurrency,
|
|
138
196
|
},
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
worker: route.worker.adapter,
|
|
142
|
-
provider: route.model.provider,
|
|
143
|
-
model: route.model.model,
|
|
144
|
-
profile: route.model.profile ?? null,
|
|
145
|
-
reasoning: route.model.reasoning ?? null,
|
|
146
|
-
code_plan: {
|
|
147
|
-
five_hour: route.model.resourceWindows?.fiveHour ? {
|
|
148
|
-
remaining_percent: route.model.resourceWindows.fiveHour.remainingPercent ?? null,
|
|
149
|
-
reset_at: route.model.resourceWindows.fiveHour.resetAt ?? null,
|
|
150
|
-
reset_in_minutes: route.model.resourceWindows.fiveHour.resetInMinutes ?? null,
|
|
151
|
-
} : null,
|
|
152
|
-
weekly: route.model.resourceWindows?.weekly ? {
|
|
153
|
-
remaining_percent: route.model.resourceWindows.weekly.remainingPercent ?? null,
|
|
154
|
-
reset_at: route.model.resourceWindows.weekly.resetAt ?? null,
|
|
155
|
-
reset_in_minutes: route.model.resourceWindows.weekly.resetInMinutes ?? null,
|
|
156
|
-
} : null,
|
|
157
|
-
},
|
|
158
|
-
})),
|
|
197
|
+
worker_candidates: workerProviderFacts(routes),
|
|
198
|
+
routing_instruction: 'Interpret user_input_rules directly. Choose among the runtime-provided legal candidates at each stage. Relay does not parse user_input_rules into model or reasoning choices.',
|
|
159
199
|
}));
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
200
|
+
let selected = fallback;
|
|
201
|
+
let evidence = { mode: options.decisionMode ?? 'off', fallback: true };
|
|
202
|
+
const decisionStages = [];
|
|
203
|
+
if ((options.decisionMode ?? 'off') !== 'off' && options.decisionEngine) {
|
|
204
|
+
try {
|
|
205
|
+
const providerKeys = [...new Set(routes.map(route => route.worker.workerId + '::' + route.model.provider))];
|
|
206
|
+
const providerOptions = providerKeys.map(key => {
|
|
207
|
+
const splitAt = key.lastIndexOf('::');
|
|
208
|
+
const worker = key.slice(0, splitAt);
|
|
209
|
+
const provider = key.slice(splitAt + 2);
|
|
210
|
+
const providerRoutes = routes.filter(route => route.worker.workerId === worker && route.model.provider === provider);
|
|
211
|
+
const configured = providerRoutes.find(route => !route.model.id.startsWith('catalog:')) ?? providerRoutes[0];
|
|
212
|
+
return {
|
|
213
|
+
key,
|
|
214
|
+
worker,
|
|
215
|
+
provider,
|
|
216
|
+
default_model: configured.model.model,
|
|
217
|
+
label: 'worker=' + worker + ' provider=' + provider + ' default_model=' + configured.model.model,
|
|
218
|
+
};
|
|
219
|
+
});
|
|
220
|
+
const providerFacts = stageFacts(baseFacts, {}, {
|
|
221
|
+
provider_candidates: providerOptions.map(({ label: _label, key: _key, ...option }) => option),
|
|
222
|
+
});
|
|
223
|
+
const providerStage = await decideStage(options.decisionEngine, 'provider', providerFacts, 'Choose the best worker/provider pair. Interpret user_input_rules directly together with task context, runtime load, provider defaults, and code-plan quota. When user_input_rules explicitly names providers as routing options, prefer those named providers when applicable; treat unmentioned providers as fallbacks unless other facts clearly require them.', providerOptions.map(option => option.label));
|
|
224
|
+
decisionStages.push(providerStage);
|
|
225
|
+
const selectedProvider = providerOptions.find(option => option.label === providerStage.result.choice);
|
|
226
|
+
if (!selectedProvider)
|
|
227
|
+
throw new Error('Laya selected an unknown worker/provider option');
|
|
228
|
+
const workerId = selectedProvider.worker;
|
|
229
|
+
const provider = selectedProvider.provider;
|
|
230
|
+
const providerRoutes = routes.filter(route => route.worker.workerId === workerId && route.model.provider === provider);
|
|
231
|
+
if (providerRoutes.length === 0)
|
|
232
|
+
throw new Error('Laya selected a worker/provider pair with no eligible routes');
|
|
233
|
+
const modelGroups = new Map();
|
|
234
|
+
for (const route of providerRoutes) {
|
|
235
|
+
const group = modelGroups.get(route.model.model) ?? [];
|
|
236
|
+
group.push(route);
|
|
237
|
+
modelGroups.set(route.model.model, group);
|
|
238
|
+
}
|
|
239
|
+
const modelChoices = [...modelGroups.keys()];
|
|
240
|
+
const modelOptions = modelChoices.map(model => {
|
|
241
|
+
const group = modelGroups.get(model) ?? [];
|
|
242
|
+
const configured = group.some(route => !route.model.id.startsWith('catalog:'));
|
|
243
|
+
return {
|
|
244
|
+
model,
|
|
245
|
+
configured_default: configured,
|
|
246
|
+
supported_reasoning: [...new Set(group.map(route => route.model.reasoning).filter((value) => Boolean(value)))],
|
|
247
|
+
label: 'model=' + model + ' configured_default=' + String(configured),
|
|
248
|
+
};
|
|
249
|
+
});
|
|
250
|
+
const modelFacts = stageFacts(baseFacts, { worker: workerId, provider }, {
|
|
251
|
+
model_candidates: modelOptions.map(({ label: _label, ...option }) => option),
|
|
252
|
+
});
|
|
253
|
+
const modelStage = await decideStage(options.decisionEngine, 'model', modelFacts, 'Choose the best model for the selected worker/provider. Interpret user_input_rules directly. Candidate model names come from current runtime/configuration and are not derived from the free-text rules. If user_input_rules explicitly maps a model to the current task type, size, complexity, or other current condition, follow that mapping instead of the provider default.', modelOptions.map(option => option.label));
|
|
254
|
+
decisionStages.push(modelStage);
|
|
255
|
+
const selectedModel = modelOptions.find(option => option.label === modelStage.result.choice);
|
|
256
|
+
if (!selectedModel)
|
|
257
|
+
throw new Error('Laya selected an unknown model option');
|
|
258
|
+
const modelRoutes = modelGroups.get(selectedModel.model) ?? [];
|
|
259
|
+
const reasoningChoices = [...new Set(modelRoutes.map(route => route.model.reasoning ?? 'provider-default'))];
|
|
260
|
+
const reasoningOptions = reasoningChoices.map(reasoning => ({
|
|
261
|
+
reasoning,
|
|
262
|
+
label: 'reasoning=' + reasoning,
|
|
263
|
+
}));
|
|
264
|
+
const reasoningFacts = stageFacts(baseFacts, { worker: workerId, provider, model: selectedModel.model }, {
|
|
265
|
+
reasoning_candidates: reasoningChoices,
|
|
266
|
+
});
|
|
267
|
+
const reasoningStage = await decideStage(options.decisionEngine, 'reasoning', reasoningFacts, 'Choose the reasoning level for the selected model. Interpret user_input_rules directly. If user_input_rules explicitly assigns a reasoning level to the selected model for the current conditions, follow that assignment. Use provider-default only when the model does not expose a specific reasoning level.', reasoningOptions.map(option => option.label));
|
|
268
|
+
decisionStages.push(reasoningStage);
|
|
269
|
+
const selectedReasoning = reasoningOptions.find(option => option.label === reasoningStage.result.choice);
|
|
270
|
+
if (!selectedReasoning)
|
|
271
|
+
throw new Error('Laya selected an unknown reasoning option');
|
|
272
|
+
const reasoning = selectedReasoning.reasoning === 'provider-default' ? null : selectedReasoning.reasoning;
|
|
273
|
+
const proposed = modelRoutes.find(route => (route.model.reasoning ?? null) === reasoning);
|
|
274
|
+
if (!proposed)
|
|
275
|
+
throw new Error('Laya selected a model/reasoning combination with no eligible route');
|
|
276
|
+
selected = proposed;
|
|
277
|
+
evidence = {
|
|
278
|
+
mode: options.decisionMode ?? 'active',
|
|
279
|
+
engine: options.decisionEngine.name,
|
|
280
|
+
proposed: providerStage.result.choice + '/' + selectedModel.model + '@' + selectedReasoning.reasoning,
|
|
281
|
+
proposedFields: {
|
|
282
|
+
worker: selected.worker.workerId,
|
|
283
|
+
provider: selected.model.provider,
|
|
284
|
+
model: selected.model.model,
|
|
285
|
+
reasoning: selected.model.reasoning ?? '',
|
|
286
|
+
},
|
|
287
|
+
fallback: false,
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
catch (error) {
|
|
291
|
+
evidence = {
|
|
292
|
+
mode: options.decisionMode ?? 'active',
|
|
293
|
+
engine: options.decisionEngine.name,
|
|
294
|
+
fallback: true,
|
|
295
|
+
error: error instanceof Error ? error.message : String(error),
|
|
296
|
+
};
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
const selectedBy = evidence.engine && !evidence.fallback ? 'execution-router:laya' : 'execution-router:fallback';
|
|
169
300
|
const routing = {
|
|
170
301
|
provider: selected.model.provider,
|
|
171
302
|
model: selected.model.model,
|
|
@@ -173,7 +304,7 @@ export async function routeExecution(workers, requirements, options = {}) {
|
|
|
173
304
|
reasoning: options.routingConstraints?.reasoning ?? selected.model.reasoning,
|
|
174
305
|
decidedBy: selectedBy,
|
|
175
306
|
decidedAt: new Date().toISOString(),
|
|
176
|
-
decisionEvidence:
|
|
307
|
+
decisionEvidence: evidence,
|
|
177
308
|
};
|
|
178
309
|
const audit = makeExecutionDecisionAudit({
|
|
179
310
|
task: options.auditTask,
|
|
@@ -192,15 +323,14 @@ export async function routeExecution(workers, requirements, options = {}) {
|
|
|
192
323
|
},
|
|
193
324
|
policy,
|
|
194
325
|
routing,
|
|
195
|
-
|
|
196
|
-
decisionContract,
|
|
326
|
+
decisionStages,
|
|
197
327
|
});
|
|
198
328
|
routing.decisionId = audit.decision_id;
|
|
199
329
|
if (options.auditStore)
|
|
200
330
|
await options.auditStore.append(audit);
|
|
201
331
|
return {
|
|
202
332
|
candidate: { ...selected.worker, routing },
|
|
203
|
-
selection: workerSelection(selected.worker, selectedBy,
|
|
333
|
+
selection: workerSelection(selected.worker, selectedBy, evidence),
|
|
204
334
|
model: selected.model,
|
|
205
335
|
audit,
|
|
206
336
|
};
|
package/dist/relayd.js
CHANGED
|
@@ -397,6 +397,7 @@ export async function runDaemon(argv) {
|
|
|
397
397
|
reasoning: result.candidate.routing.reasoning,
|
|
398
398
|
},
|
|
399
399
|
probabilities: result.candidate.routing.decisionEvidence?.probabilities ?? {},
|
|
400
|
+
proposedFields: result.candidate.routing.decisionEvidence?.proposedFields ?? {},
|
|
400
401
|
fallback: result.candidate.routing.decisionEvidence?.fallback ?? false,
|
|
401
402
|
current: {
|
|
402
403
|
chatgptRunning: runtime.workers.chatgpt,
|
package/dist/router-evals.js
CHANGED
|
@@ -12,12 +12,12 @@ export const ROUTER_EVAL_CASES = [
|
|
|
12
12
|
{ id: 'workspace-exhausted', storySize: 'S', task: 'Review a focused issue while Browser Workspace has no free slot.', chatgptAvailable: false, chatgptSlots: 0, expectedWorker: 'codex' },
|
|
13
13
|
{ id: 'large-workspace-available', storySize: 'L', task: 'Implement a large feature even though ChatGPT workspace capacity is healthy.', chatgptAvailable: true, expectedWorker: 'codex' },
|
|
14
14
|
];
|
|
15
|
-
const CODEX_ROUTE = 'codex
|
|
16
|
-
const CHATGPT_ROUTE = 'chatgpt/browser-worker
|
|
15
|
+
const CODEX_ROUTE = 'codex/openai/default';
|
|
16
|
+
const CHATGPT_ROUTE = 'chatgpt/browser-worker/chatgpt/default';
|
|
17
17
|
function workerForRoute(route) {
|
|
18
18
|
if (route.startsWith('chatgpt/'))
|
|
19
19
|
return 'chatgpt';
|
|
20
|
-
if (route.startsWith('codex
|
|
20
|
+
if (route.startsWith('codex/'))
|
|
21
21
|
return 'codex';
|
|
22
22
|
return 'unknown';
|
|
23
23
|
}
|
|
@@ -28,29 +28,31 @@ export async function runRouterEvals(engine, rulesPath) {
|
|
|
28
28
|
const taskShape = executionTaskShape({ storySize: testCase.storySize });
|
|
29
29
|
const available = testCase.chatgptAvailable;
|
|
30
30
|
const choices = available ? [CODEX_ROUTE, CHATGPT_ROUTE] : [CODEX_ROUTE];
|
|
31
|
-
const
|
|
31
|
+
const workerCandidates = [
|
|
32
32
|
{
|
|
33
|
-
id:
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
33
|
+
id: 'codex',
|
|
34
|
+
adapter: 'codex',
|
|
35
|
+
providers: {
|
|
36
|
+
openai: {
|
|
37
|
+
default: { model: 'gpt-5.6-sol', reasoning: 'medium' },
|
|
38
|
+
code_plan: { five_hour: null, weekly: null },
|
|
39
|
+
},
|
|
40
|
+
},
|
|
40
41
|
},
|
|
41
42
|
...(available ? [{
|
|
42
|
-
id:
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
43
|
+
id: 'chatgpt/browser-worker',
|
|
44
|
+
adapter: 'chatgpt',
|
|
45
|
+
providers: {
|
|
46
|
+
chatgpt: {
|
|
47
|
+
default: { model: 'account-defaults', reasoning: null },
|
|
48
|
+
code_plan: { five_hour: null, weekly: null },
|
|
49
|
+
},
|
|
50
|
+
},
|
|
49
51
|
}] : []),
|
|
50
52
|
];
|
|
51
|
-
const facts = {
|
|
53
|
+
const facts = JSON.parse(JSON.stringify({
|
|
52
54
|
context: {
|
|
53
|
-
job: { id: 'router-eval', title: 'Router eval', objective: 'Evaluate worker and
|
|
55
|
+
job: { id: 'router-eval', title: 'Router eval', objective: 'Evaluate worker and provider route selection.' },
|
|
54
56
|
task: { id: testCase.id, kind: 'work', storySize: testCase.storySize, input: testCase.task },
|
|
55
57
|
task_shape: {
|
|
56
58
|
story_size: taskShape.storySize,
|
|
@@ -62,13 +64,16 @@ export async function runRouterEvals(engine, rulesPath) {
|
|
|
62
64
|
},
|
|
63
65
|
user_input_rules: rules.text,
|
|
64
66
|
runtime: { chatgpt_running: testCase.chatgptRunning ?? 0, preferred_chatgpt_concurrency: 10 },
|
|
65
|
-
|
|
66
|
-
|
|
67
|
+
worker_candidates: workerCandidates,
|
|
68
|
+
routing_instruction: 'For each selected provider, use its default model and reasoning unless user_input_rules explicitly provides another model/reasoning option.',
|
|
69
|
+
}));
|
|
67
70
|
const decision = await engine.decide({ facts }, {
|
|
68
71
|
type: 'choice',
|
|
69
|
-
question: 'Choose one
|
|
72
|
+
question: 'Choose one legal route. Each provider default is available. Additional model/reasoning routes exist only when explicitly provided by user_input_rules. Use context, code-plan quota, runtime load, and user_input_rules.',
|
|
70
73
|
choices,
|
|
71
74
|
});
|
|
75
|
+
if (decision.type !== 'choice')
|
|
76
|
+
throw new Error('Router eval expected a single-choice decision');
|
|
72
77
|
const selectedWorker = workerForRoute(decision.choice);
|
|
73
78
|
results.push({
|
|
74
79
|
...testCase,
|