agents-relay 2.0.24 → 2.0.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dashboard-html.js +2 -2
- package/dist/decision-audit.js +4 -3
- package/dist/execution-routing.js +35 -38
- package/dist/model-routing.js +62 -35
- package/dist/router-evals.js +35 -23
- package/dist/worker-routing.js +4 -4
- package/package.json +1 -1
package/dist/dashboard-html.js
CHANGED
|
@@ -177,7 +177,7 @@ export const dashboardHtml = `<!doctype html>
|
|
|
177
177
|
}
|
|
178
178
|
@media (prefers-reduced-motion:reduce) { .skeleton { animation:none; } .usage-fill { transition:none; } }
|
|
179
179
|
|
|
180
|
-
.router-rules-workspace{display:grid;grid-template-columns:minmax(0,1fr) minmax(0,1fr);gap:16px;align-items:start}.router-rules-pane{min-width:0}.audit-log{display:flex;flex-direction:column;gap:8px;max-height:
|
|
180
|
+
.router-rules-workspace{display:grid;grid-template-columns:minmax(0,1fr) minmax(0,1fr);gap:16px;align-items:start}.router-rules-pane{min-width:0;max-height:560px;overflow:auto}.rules-editor .rules-textarea{min-height:420px}.audit-log{display:flex;flex-direction:column;gap:8px;max-height:500px;overflow:auto}.audit-entry{border:1px solid var(--border);border-radius:10px;padding:10px;background:var(--surface)}.audit-entry-head{display:flex;justify-content:space-between;gap:8px;align-items:center}.audit-entry pre{white-space:pre-wrap;word-break:break-word;margin:8px 0 0;font-size:11px}.router-test-panel{margin-top:16px;border-top:1px solid var(--border);padding-top:12px}.router-test-results{margin-top:8px;display:flex;flex-direction:column;gap:6px;max-height:480px;overflow:auto;min-width:0}.router-test-results pre{white-space:pre;overflow:auto;max-width:100%;font-size:11px;line-height:1.45;background:var(--surface);border:1px solid var(--border);border-radius:10px;padding:12px}.router-test-row{display:grid;grid-template-columns:minmax(0,1fr) auto auto;gap:8px;padding:7px 0;border-bottom:1px solid var(--border)}@media(max-width:900px){.router-rules-workspace{grid-template-columns:1fr}.router-rules-pane{max-height:none}.router-test-results{max-height:420px}}
|
|
181
181
|
</style>
|
|
182
182
|
</head>
|
|
183
183
|
<body>
|
|
@@ -235,7 +235,7 @@ export const dashboardHtml = `<!doctype html>
|
|
|
235
235
|
</div>
|
|
236
236
|
</div>
|
|
237
237
|
</section>
|
|
238
|
-
<section id="rules-view" role="tabpanel" aria-labelledby="rules-tab" hidden><div class="panel"><div class="panel-head"><h2>Router Rules</h2><span id="rules-meta" class="subtle"></span></div><div class="panel-body"><div class="router-rules-workspace"><div class="router-rules-pane rules-editor"><h3>Rules Editor</h3><p class="rules-help">Free-text soft policy loaded by the Execution Router for every worker+model routing decision. Impossible routes remain excluded by code; Codex remains the deterministic fallback if decision fails.</p><textarea id="rules-text" class="rules-textarea" spellcheck="true" aria-label="Execution router rules" placeholder="Example: Prefer ZAI glm-5.3-flash high while quota is available. Use ChatGPT for research when appropriate. Codex is fallback only."></textarea><div class="rules-footer"><span id="rules-status" class="subtle" aria-live="polite"></span><div><button id="rules-test" class="rules-save" type="button">Test Router</button> <button id="rules-save" class="rules-save" type="button">Save rules</button></div></div
|
|
238
|
+
<section id="rules-view" role="tabpanel" aria-labelledby="rules-tab" hidden><div class="panel"><div class="panel-head"><h2>Router Rules</h2><span id="rules-meta" class="subtle"></span></div><div class="panel-body"><div class="router-rules-workspace"><div class="router-rules-pane rules-editor"><h3>Rules Editor</h3><p class="rules-help">Free-text soft policy loaded by the Execution Router for every worker+model routing decision. Impossible routes remain excluded by code; Codex remains the deterministic fallback if decision fails.</p><textarea id="rules-text" class="rules-textarea" spellcheck="true" aria-label="Execution router rules" placeholder="Example: Prefer ZAI glm-5.3-flash high while quota is available. Use ChatGPT for research when appropriate. Codex is fallback only."></textarea><div class="rules-footer"><span id="rules-status" class="subtle" aria-live="polite"></span><div><button id="rules-test" class="rules-save" type="button">Test Router</button> <button id="rules-save" class="rules-save" type="button">Save rules</button></div></div></div><div class="router-rules-pane"><div class="panel-head"><h3>Audit Log</h3><button id="audit-refresh" type="button">Refresh</button></div><div id="audit-log" class="audit-log" aria-live="polite"><p class="empty">Loading latest routing decisions…</p></div></div></div><div class="router-test-panel"><h3>Test Router Result</h3><div id="router-test-results" class="router-test-results" aria-live="polite"><p class="empty">Select a Job and Task, then run Test Router.</p></div></div></div></div></section>
|
|
239
239
|
<section id="shortcuts-view" role="tabpanel" aria-labelledby="shortcuts-tab" hidden>
|
|
240
240
|
<div class="panel">
|
|
241
241
|
<div class="panel-head"><h2>ChatGPT Shortcuts</h2><span class="subtle">One per line · %1 = current repo/pr</span></div>
|
package/dist/decision-audit.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { appendFile, mkdir, readFile } from 'node:fs/promises';
|
|
2
2
|
import { dirname } from 'node:path';
|
|
3
3
|
import { randomUUID, createHash } from 'node:crypto';
|
|
4
|
+
import { modelChoiceId } from './model-routing.js';
|
|
4
5
|
export const DECISION_AUDIT_SCHEMA_VERSION = 5;
|
|
5
6
|
export class JsonlDecisionAuditStore {
|
|
6
7
|
path;
|
|
@@ -35,8 +36,8 @@ export function taskAuditFacts(task) {
|
|
|
35
36
|
return { job_id: task.jobId, task_id: task.id, attempt: task.attempt, execution_id: task.executionId, capabilities: [...task.capabilities], input_class: task.input.slice(0, 80).replace(/\s+/g, ' ') };
|
|
36
37
|
}
|
|
37
38
|
export function makeDecisionAudit(input) {
|
|
38
|
-
const candidate = (item) => ({ id: item
|
|
39
|
-
const eligibleIds = new Set(input.eligibleCandidates.map(
|
|
39
|
+
const candidate = (item) => ({ id: modelChoiceId(item), provider: item.provider, model: item.model, profile: item.profile ?? null, reasoning: item.reasoning ?? null, resourceHealthy: item.resourceHealthy ?? null, resourceUsedPercent: item.resourceUsedPercent ?? null, resourceRemainingPercent: item.resourceRemainingPercent ?? null, resourceResetAt: item.resourceResetAt ?? null, resourceResetInMinutes: item.resourceResetInMinutes ?? null, resourceWindows: item.resourceWindows ?? null });
|
|
40
|
+
const eligibleIds = new Set(input.eligibleCandidates.map(modelChoiceId));
|
|
40
41
|
const gate_results = input.candidates.map(item => {
|
|
41
42
|
const reasons = [];
|
|
42
43
|
if (item.resourceHealthy === false)
|
|
@@ -55,7 +56,7 @@ export function makeDecisionAudit(input) {
|
|
|
55
56
|
reasons.push('policy_disabled_provider');
|
|
56
57
|
if (item.profile && input.policy.policy.disabledProfiles?.includes(item.profile))
|
|
57
58
|
reasons.push('policy_disabled_profile');
|
|
58
|
-
return { candidate_id: item
|
|
59
|
+
return { candidate_id: modelChoiceId(item), eligible: eligibleIds.has(modelChoiceId(item)), rejection_reasons: reasons };
|
|
59
60
|
});
|
|
60
61
|
return { record_type: 'decision', schema_version: DECISION_AUDIT_SCHEMA_VERSION, decision_id: decisionAuditId(), prior_decision_id: input.priorDecisionId, timestamp: input.routing.decidedAt, correlation: { job_id: input.task?.jobId, task_id: input.task?.id, execution_id: input.task?.executionId, attempt: input.task?.attempt }, task: taskAuditFacts(input.task), worker: input.worker, contract: input.contract, policy: { path: input.policy.path, hash: input.policy.hash, snapshot: input.policy.snapshot, rules: input.policy.rules }, candidates: input.candidates.map(candidate), gate_results, eligible_candidates: input.eligibleCandidates.map(candidate), selected: candidate(input.selected), engine: input.routing.decidedBy, fallback: { used: input.routing.decisionEvidence?.fallback ?? false, reason: input.routing.decisionEvidence?.error }, decision_evidence: input.routing.decisionEvidence, prior_failure: input.priorFailure };
|
|
61
62
|
}
|
|
@@ -61,7 +61,7 @@ function modelGateReasons(candidate, constraints, excludedRoutes, policy) {
|
|
|
61
61
|
return reasons;
|
|
62
62
|
}
|
|
63
63
|
function routeId(worker, model) {
|
|
64
|
-
return
|
|
64
|
+
return worker.workerId + '::' + model.id + (model.reasoning ? '@' + model.reasoning : '');
|
|
65
65
|
}
|
|
66
66
|
function fallbackRoute(routes) {
|
|
67
67
|
const codex = routes.filter(route => route.worker.adapter === 'codex');
|
|
@@ -111,59 +111,56 @@ export async function routeExecution(workers, requirements, options = {}) {
|
|
|
111
111
|
throw new Error('No executable worker+model route satisfies authoritative constraints');
|
|
112
112
|
const fallback = fallbackRoute(routes);
|
|
113
113
|
const rules = await loadExecutionRouterRules(options.rulesPath);
|
|
114
|
-
const
|
|
114
|
+
const userInputRules = rules.text;
|
|
115
115
|
const preferredChatgptConcurrency = options.preferredChatgptConcurrency ?? DEFAULT_CHATGPT_PREFERRED_CONCURRENCY;
|
|
116
116
|
const chatgptRunning = options.chatgptRunning ?? 0;
|
|
117
117
|
const taskFacts = options.taskFacts ?? {};
|
|
118
118
|
const taskShape = executionTaskShape(taskFacts);
|
|
119
119
|
const facts = JSON.parse(JSON.stringify({
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
120
|
+
context: {
|
|
121
|
+
job: taskFacts.job ?? null,
|
|
122
|
+
task: taskFacts.task ?? taskFacts,
|
|
123
|
+
task_shape: {
|
|
124
|
+
story_size: taskShape.storySize,
|
|
125
|
+
sustained_work: taskShape.sustainedWork,
|
|
126
|
+
preferred_worker: taskShape.preferredWorker,
|
|
127
|
+
reason: taskShape.reason,
|
|
128
|
+
},
|
|
129
|
+
reviewer: taskFacts.reviewer_diversity ? {
|
|
130
|
+
role: taskFacts.role ?? null,
|
|
131
|
+
reviewer_diversity: taskFacts.reviewer_diversity,
|
|
132
|
+
} : null,
|
|
127
133
|
},
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
role: taskFacts.role ?? null,
|
|
131
|
-
reviewer_diversity: taskFacts.reviewer_diversity,
|
|
132
|
-
} : null,
|
|
133
|
-
runtime_load: {
|
|
134
|
+
user_input_rules: userInputRules,
|
|
135
|
+
runtime: {
|
|
134
136
|
chatgpt_running: chatgptRunning,
|
|
135
137
|
preferred_chatgpt_concurrency: preferredChatgptConcurrency,
|
|
136
138
|
},
|
|
137
|
-
|
|
139
|
+
eligible_routes: routes.map(route => ({
|
|
138
140
|
id: route.id,
|
|
139
|
-
worker:
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
remaining_percent: route.model.resourceWindows.weekly.remainingPercent ?? null,
|
|
156
|
-
reset_at: route.model.resourceWindows.weekly.resetAt ?? null,
|
|
157
|
-
reset_in_minutes: route.model.resourceWindows.weekly.resetInMinutes ?? null,
|
|
158
|
-
} : null,
|
|
159
|
-
},
|
|
141
|
+
worker: route.worker.adapter,
|
|
142
|
+
provider: route.model.provider,
|
|
143
|
+
model: route.model.model,
|
|
144
|
+
profile: route.model.profile ?? null,
|
|
145
|
+
reasoning: route.model.reasoning ?? null,
|
|
146
|
+
code_plan: {
|
|
147
|
+
five_hour: route.model.resourceWindows?.fiveHour ? {
|
|
148
|
+
remaining_percent: route.model.resourceWindows.fiveHour.remainingPercent ?? null,
|
|
149
|
+
reset_at: route.model.resourceWindows.fiveHour.resetAt ?? null,
|
|
150
|
+
reset_in_minutes: route.model.resourceWindows.fiveHour.resetInMinutes ?? null,
|
|
151
|
+
} : null,
|
|
152
|
+
weekly: route.model.resourceWindows?.weekly ? {
|
|
153
|
+
remaining_percent: route.model.resourceWindows.weekly.remainingPercent ?? null,
|
|
154
|
+
reset_at: route.model.resourceWindows.weekly.resetAt ?? null,
|
|
155
|
+
reset_in_minutes: route.model.resourceWindows.weekly.resetInMinutes ?? null,
|
|
156
|
+
} : null,
|
|
160
157
|
},
|
|
161
158
|
})),
|
|
162
159
|
}));
|
|
163
160
|
const decisionRequest = { facts };
|
|
164
161
|
const decisionContract = {
|
|
165
162
|
type: 'choice',
|
|
166
|
-
question: 'Choose
|
|
163
|
+
question: 'Choose one eligible complete execution route. Treat context.task_shape.preferred_worker as the default worker-fit signal; user_input_rules and concrete runtime or code-plan resource facts may override it. The choice identifies worker, provider, model, profile, and reasoning together.',
|
|
167
164
|
choices: routes.map(route => route.id),
|
|
168
165
|
};
|
|
169
166
|
const choice = await chooseWithDecision(options.decisionEngine, options.decisionMode ?? 'off', decisionRequest, decisionContract, () => fallback.id);
|
package/dist/model-routing.js
CHANGED
|
@@ -4,7 +4,39 @@ import { readdir, readFile } from 'node:fs/promises';
|
|
|
4
4
|
import { chooseWithDecision } from './decision.js';
|
|
5
5
|
import { usageForProvider } from './worker-routing.js';
|
|
6
6
|
import { makeDecisionAudit } from './decision-audit.js';
|
|
7
|
-
import {
|
|
7
|
+
import { loadRoutingPolicy, policyAllows, policyRank } from './routing-policy.js';
|
|
8
|
+
export function modelChoiceId(candidate) {
|
|
9
|
+
return candidate.id + (candidate.reasoning ? '@' + candidate.reasoning : '');
|
|
10
|
+
}
|
|
11
|
+
async function readCodexModelCatalog(codexHome) {
|
|
12
|
+
try {
|
|
13
|
+
const parsed = JSON.parse(await readFile(join(codexHome, 'models_cache.json'), 'utf8'));
|
|
14
|
+
return Array.isArray(parsed.models)
|
|
15
|
+
? parsed.models.filter(item => typeof item.slug === 'string' && item.visibility === 'list' && item.supported_in_api !== false)
|
|
16
|
+
: [];
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
return [];
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
function catalogCandidates(models, usage) {
|
|
23
|
+
const result = [];
|
|
24
|
+
for (const item of [...models].sort((a, b) => (a.priority ?? 999) - (b.priority ?? 999))) {
|
|
25
|
+
if (!item.slug)
|
|
26
|
+
continue;
|
|
27
|
+
const efforts = (item.supported_reasoning_levels ?? []).map(level => level.effort).filter((value) => Boolean(value));
|
|
28
|
+
const reasoningLevels = efforts.length ? efforts : item.default_reasoning_level ? [item.default_reasoning_level] : [undefined];
|
|
29
|
+
for (const reasoning of reasoningLevels) {
|
|
30
|
+
result.push(withResource({
|
|
31
|
+
id: 'catalog:openai/' + item.slug,
|
|
32
|
+
provider: 'openai',
|
|
33
|
+
model: item.slug,
|
|
34
|
+
reasoning,
|
|
35
|
+
}, usage));
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return result;
|
|
39
|
+
}
|
|
8
40
|
function topLevelCodexConfig(text) {
|
|
9
41
|
const result = {};
|
|
10
42
|
let inSection = false;
|
|
@@ -38,36 +70,15 @@ async function readConfig(path) {
|
|
|
38
70
|
return null;
|
|
39
71
|
}
|
|
40
72
|
}
|
|
41
|
-
function compact(value) {
|
|
42
|
-
return value.toLowerCase().replace(/[^a-z0-9]+/g, '');
|
|
43
|
-
}
|
|
44
|
-
function modelAliases(model) {
|
|
45
|
-
const aliases = new Set([compact(model)]);
|
|
46
|
-
const firstDigit = model.search(/\d/);
|
|
47
|
-
if (firstDigit >= 0)
|
|
48
|
-
aliases.add(compact(model.slice(firstDigit)));
|
|
49
|
-
return [...aliases].filter(Boolean);
|
|
50
|
-
}
|
|
51
|
-
export function nonOpenAiModelAuthorized(provider, model, rulesText) {
|
|
52
|
-
if (provider === 'openai')
|
|
53
|
-
return true;
|
|
54
|
-
const rules = compact(rulesText);
|
|
55
|
-
if (!rules.includes(compact(provider)))
|
|
56
|
-
return false;
|
|
57
|
-
return modelAliases(model).some(alias => alias.length >= 3 && rules.includes(alias));
|
|
58
|
-
}
|
|
59
73
|
function withResource(candidate, usage) {
|
|
60
74
|
const resource = usageForProvider(usage, candidate.provider);
|
|
61
75
|
return { ...candidate, resourceHealthy: resource.healthy, resourceUsedPercent: resource.usedPercent, resourceRemainingPercent: resource.remainingPercent, resourceResetAt: resource.resetAt, resourceResetInMinutes: resource.resetInMinutes, resourceWindows: resource.windows };
|
|
62
76
|
}
|
|
63
|
-
export async function discoverModelCandidates(worker, registered, usage, codexHome = process.env.CODEX_HOME ?? join(homedir(), '.codex'),
|
|
77
|
+
export async function discoverModelCandidates(worker, registered, usage, codexHome = process.env.CODEX_HOME ?? join(homedir(), '.codex'), _rulesPath) {
|
|
64
78
|
const registration = registered.find(item => item.id === worker.workerId);
|
|
65
79
|
if (registration?.runtime.provider && registration.runtime.model) {
|
|
66
|
-
const rules = await loadExecutionRouterRules(rulesPath);
|
|
67
|
-
if (!nonOpenAiModelAuthorized(registration.runtime.provider, registration.runtime.model, rules.text))
|
|
68
|
-
return [];
|
|
69
80
|
return [withResource({
|
|
70
|
-
id:
|
|
81
|
+
id: registration.runtime.provider + '/' + registration.runtime.model,
|
|
71
82
|
provider: registration.runtime.provider,
|
|
72
83
|
model: registration.runtime.model,
|
|
73
84
|
}, usage)];
|
|
@@ -76,12 +87,22 @@ export async function discoverModelCandidates(worker, registered, usage, codexHo
|
|
|
76
87
|
return [withResource({ id: 'chatgpt/account-defaults', provider: 'chatgpt', model: 'account-defaults' }, usage)];
|
|
77
88
|
}
|
|
78
89
|
const base = await readConfig(join(codexHome, 'config.toml')) ?? {};
|
|
79
|
-
const rules = await loadExecutionRouterRules(rulesPath);
|
|
80
90
|
const candidates = [];
|
|
81
91
|
if (base.model) {
|
|
82
92
|
const provider = base.provider ?? 'openai';
|
|
83
|
-
|
|
84
|
-
|
|
93
|
+
candidates.push(withResource({
|
|
94
|
+
id: 'default:' + provider + '/' + base.model,
|
|
95
|
+
provider,
|
|
96
|
+
model: base.model,
|
|
97
|
+
reasoning: base.reasoning,
|
|
98
|
+
}, usage));
|
|
99
|
+
}
|
|
100
|
+
if ((base.provider ?? 'openai') === 'openai') {
|
|
101
|
+
for (const item of catalogCandidates(await readCodexModelCatalog(codexHome), usage)) {
|
|
102
|
+
if (item.model === base.model && item.reasoning === base.reasoning)
|
|
103
|
+
continue;
|
|
104
|
+
candidates.push(item);
|
|
105
|
+
}
|
|
85
106
|
}
|
|
86
107
|
let files = [];
|
|
87
108
|
try {
|
|
@@ -94,13 +115,18 @@ export async function discoverModelCandidates(worker, registered, usage, codexHo
|
|
|
94
115
|
continue;
|
|
95
116
|
const profile = basename(file, '.config.toml');
|
|
96
117
|
const provider = config.provider ?? base.provider ?? 'openai';
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
118
|
+
candidates.push(withResource({
|
|
119
|
+
id: profile + ':' + provider + '/' + config.model,
|
|
120
|
+
provider,
|
|
121
|
+
model: config.model,
|
|
122
|
+
profile,
|
|
123
|
+
reasoning: config.reasoning ?? base.reasoning,
|
|
124
|
+
}, usage));
|
|
100
125
|
}
|
|
101
|
-
if (candidates.length === 0)
|
|
126
|
+
if (candidates.length === 0) {
|
|
102
127
|
candidates.push(withResource({ id: 'default:openai/gpt-5.6-luna', provider: 'openai', model: 'gpt-5.6-luna' }, usage));
|
|
103
|
-
|
|
128
|
+
}
|
|
129
|
+
return candidates.filter((candidate, index, all) => all.findIndex(item => item.id === candidate.id && item.reasoning === candidate.reasoning) === index);
|
|
104
130
|
}
|
|
105
131
|
function sameRoute(candidate, route) {
|
|
106
132
|
return (!route.provider || candidate.provider === route.provider)
|
|
@@ -144,14 +170,15 @@ export async function routeModel(worker, candidates, options = {}) {
|
|
|
144
170
|
: await chooseWithDecision(options.decisionEngine, options.decisionMode ?? 'off', {
|
|
145
171
|
facts: JSON.parse(JSON.stringify({
|
|
146
172
|
task: options.taskFacts ?? {},
|
|
147
|
-
|
|
173
|
+
user_input_rules: policy.rules.text,
|
|
174
|
+
structured_policy: policy.snapshot,
|
|
148
175
|
prior_decision_id: options.priorDecisionId ?? null,
|
|
149
176
|
prior_failure: options.priorFailure ?? null,
|
|
150
177
|
worker: { id: worker.workerId, adapter: worker.adapter, capabilities: [...worker.capabilities] },
|
|
151
178
|
candidates: eligible.map(item => ({ id: item.id, provider: item.provider, model: item.model, profile: item.profile ?? null, reasoning: item.reasoning ?? null, resourceHealthy: item.resourceHealthy ?? null, resourceUsedPercent: item.resourceUsedPercent ?? null, resourceRemainingPercent: item.resourceRemainingPercent ?? null, resourceResetAt: item.resourceResetAt ?? null, resourceResetInMinutes: item.resourceResetInMinutes ?? null, resourceWindows: item.resourceWindows ?? null })),
|
|
152
179
|
})),
|
|
153
|
-
}, { type: 'choice', question: 'Which eligible model profile is best suited to execute this task? Apply hard gates first, then explicit task constraints, then current
|
|
154
|
-
const selected = eligible.find(item => item
|
|
180
|
+
}, { type: 'choice', question: 'Which eligible model profile is best suited to execute this task? Apply hard gates first, then explicit task constraints, then current user input rules and structured policy; consider every quota window and never violate task fit or continuity.', choices: eligible.map(modelChoiceId) }, () => modelChoiceId(deterministic));
|
|
181
|
+
const selected = eligible.find(item => modelChoiceId(item) === choice.choice) ?? deterministic;
|
|
155
182
|
const routing = {
|
|
156
183
|
provider: selected.provider,
|
|
157
184
|
model: selected.model,
|
package/dist/router-evals.js
CHANGED
|
@@ -12,7 +12,7 @@ export const ROUTER_EVAL_CASES = [
|
|
|
12
12
|
{ id: 'workspace-exhausted', storySize: 'S', task: 'Review a focused issue while Browser Workspace has no free slot.', chatgptAvailable: false, chatgptSlots: 0, expectedWorker: 'codex' },
|
|
13
13
|
{ id: 'large-workspace-available', storySize: 'L', task: 'Implement a large feature even though ChatGPT workspace capacity is healthy.', chatgptAvailable: true, expectedWorker: 'codex' },
|
|
14
14
|
];
|
|
15
|
-
const CODEX_ROUTE = 'codex::default:openai/gpt-5.6-sol';
|
|
15
|
+
const CODEX_ROUTE = 'codex::default:openai/gpt-5.6-sol@medium';
|
|
16
16
|
const CHATGPT_ROUTE = 'chatgpt/browser-worker::chatgpt/account-defaults';
|
|
17
17
|
function workerForRoute(route) {
|
|
18
18
|
if (route.startsWith('chatgpt/'))
|
|
@@ -27,34 +27,46 @@ export async function runRouterEvals(engine, rulesPath) {
|
|
|
27
27
|
for (const testCase of ROUTER_EVAL_CASES) {
|
|
28
28
|
const taskShape = executionTaskShape({ storySize: testCase.storySize });
|
|
29
29
|
const available = testCase.chatgptAvailable;
|
|
30
|
-
const slots = testCase.chatgptSlots ?? (available ? 5 : 0);
|
|
31
30
|
const choices = available ? [CODEX_ROUTE, CHATGPT_ROUTE] : [CODEX_ROUTE];
|
|
32
|
-
const
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
31
|
+
const eligibleRoutes = [
|
|
32
|
+
{
|
|
33
|
+
id: CODEX_ROUTE,
|
|
34
|
+
worker: 'codex',
|
|
35
|
+
provider: 'openai',
|
|
36
|
+
model: 'gpt-5.6-sol',
|
|
37
|
+
profile: null,
|
|
38
|
+
reasoning: 'medium',
|
|
39
|
+
code_plan: { five_hour: null, weekly: null },
|
|
38
40
|
},
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
41
|
+
...(available ? [{
|
|
42
|
+
id: CHATGPT_ROUTE,
|
|
43
|
+
worker: 'chatgpt',
|
|
44
|
+
provider: 'chatgpt',
|
|
45
|
+
model: 'account-defaults',
|
|
46
|
+
profile: null,
|
|
47
|
+
reasoning: null,
|
|
48
|
+
code_plan: { five_hour: null, weekly: null },
|
|
49
|
+
}] : []),
|
|
50
|
+
];
|
|
51
|
+
const facts = {
|
|
52
|
+
context: {
|
|
53
|
+
job: { id: 'router-eval', title: 'Router eval', objective: 'Evaluate worker and model route selection.' },
|
|
54
|
+
task: { id: testCase.id, kind: 'work', storySize: testCase.storySize, input: testCase.task },
|
|
55
|
+
task_shape: {
|
|
56
|
+
story_size: taskShape.storySize,
|
|
57
|
+
sustained_work: taskShape.sustainedWork,
|
|
58
|
+
preferred_worker: taskShape.preferredWorker,
|
|
59
|
+
reason: taskShape.reason,
|
|
47
60
|
},
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
],
|
|
61
|
+
reviewer: null,
|
|
62
|
+
},
|
|
63
|
+
user_input_rules: rules.text,
|
|
64
|
+
runtime: { chatgpt_running: testCase.chatgptRunning ?? 0, preferred_chatgpt_concurrency: 10 },
|
|
65
|
+
eligible_routes: eligibleRoutes,
|
|
54
66
|
};
|
|
55
67
|
const decision = await engine.decide({ facts }, {
|
|
56
68
|
type: 'choice',
|
|
57
|
-
question: 'Choose
|
|
69
|
+
question: 'Choose one eligible complete execution route. Treat context.task_shape.preferred_worker as the default worker-fit signal; user_input_rules and concrete runtime or code-plan resource facts may override it. The choice identifies worker, provider, model, profile, and reasoning together.',
|
|
58
70
|
choices,
|
|
59
71
|
});
|
|
60
72
|
const selectedWorker = workerForRoute(decision.choice);
|
package/dist/worker-routing.js
CHANGED
|
@@ -2,7 +2,7 @@ import { chooseWithDecision } from './decision.js';
|
|
|
2
2
|
import { loadExecutionRouterRules } from './routing-policy.js';
|
|
3
3
|
export const DEFAULT_CHATGPT_PREFERRED_CONCURRENCY = 10;
|
|
4
4
|
export const DEFAULT_WORKER_ROUTER_PRINCIPLES = [
|
|
5
|
-
'User-provided
|
|
5
|
+
'User-provided user_input_rules are higher-priority routing guidance than these built-in defaults. Follow user guidance when it conflicts with a default unless doing so would violate an authoritative hard constraint.',
|
|
6
6
|
'Worker Router chooses only the execution worker. Model Router chooses provider/model/profile after worker selection.',
|
|
7
7
|
'Prefer ChatGPT when both workers can execute the task because it does not consume Codex coding-plan tokens and normally has access to a stronger ChatGPT Sol model.',
|
|
8
8
|
'Prefer Codex when the task requires native/local execution or computer-use abilities that are unavailable through ChatGPT plus MacBridge.',
|
|
@@ -141,8 +141,8 @@ export async function routeWorker(candidates, requirements, options = {}) {
|
|
|
141
141
|
const facts = JSON.parse(JSON.stringify({
|
|
142
142
|
task: options.taskFacts ?? {},
|
|
143
143
|
explicit_constraints: { runtime: requirements.runtime ?? null },
|
|
144
|
-
decision_precedence: ['authoritative_hard_constraints', '
|
|
145
|
-
|
|
144
|
+
decision_precedence: ['authoritative_hard_constraints', 'user_input_rules', 'routing_principles', 'resource_and_runtime_preferences'],
|
|
145
|
+
user_input_rules: rules.text,
|
|
146
146
|
routing_principles: DEFAULT_WORKER_ROUTER_PRINCIPLES,
|
|
147
147
|
usage_snapshot: options.usage ?? null,
|
|
148
148
|
runtime_load: {
|
|
@@ -167,7 +167,7 @@ export async function routeWorker(candidates, requirements, options = {}) {
|
|
|
167
167
|
}));
|
|
168
168
|
const choice = await chooseWithDecision(options.decisionEngine, options.decisionMode ?? 'off', { facts }, {
|
|
169
169
|
type: 'choice',
|
|
170
|
-
question: 'Choose the best execution worker for this task. Authoritative hard constraints come first. Then follow
|
|
170
|
+
question: 'Choose the best execution worker for this task. Authoritative hard constraints come first. Then follow user_input_rules ahead of built-in routing principles when they conflict. Use task facts, reviewer-diversity context when present, current usage, and runtime load as supporting evidence. Do not choose a provider or model here.',
|
|
171
171
|
choices: eligible.map(item => item.workerId),
|
|
172
172
|
}, () => deterministic.workerId);
|
|
173
173
|
const selected = eligible.find(item => item.workerId === choice.choice) ?? deterministic;
|