dsh-auto-model-routing 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/README.md +8 -3
- package/README.zh.md +8 -2
- package/docs/feedback-evaluation.md +17 -0
- package/lib/client/index.d.ts +9 -1
- package/lib/client/index.js +93 -11
- package/lib/client/label.d.ts +1 -1
- package/lib/client/label.js +36 -1
- package/lib/client.js +160 -17
- package/lib/endpoints.d.ts +1 -0
- package/lib/endpoints.js +19 -0
- package/lib/feedback.d.ts +18 -0
- package/lib/feedback.js +21 -0
- package/lib/index.js +44 -10
- package/lib/policy-host.d.ts +10 -3
- package/lib/policy-host.js +65 -15
- package/lib/policy-store.d.ts +3 -0
- package/lib/policy-store.js +3 -2
- package/lib/policy-v2.d.ts +3 -2
- package/lib/policy-v2.js +7 -6
- package/lib/settings-host.js +3 -2
- package/lib/subagent-tool.js +3 -3
- package/lib/support.d.ts +25 -0
- package/lib/support.js +40 -0
- package/lib/usage-report.d.ts +19 -0
- package/lib/usage-report.js +62 -6
- package/package.json +17 -4
- package/skills/auto-model-routing-report/SKILL.md +5 -0
package/lib/policy-host.js
CHANGED
|
@@ -5,6 +5,7 @@ import { choosePolicyRoute, mappingVersion, policyEffort, modelFamily, policyPro
|
|
|
5
5
|
import { userInput } from './input.js';
|
|
6
6
|
import { buildState, previousTurn } from './context.js';
|
|
7
7
|
import { isRecord } from './settings-validation.js';
|
|
8
|
+
import { autoSupportedModel, currentMainSelection, mainSupported } from './support.js';
|
|
8
9
|
export function policyHost(ctx, deps) {
|
|
9
10
|
const delegation = new AsyncLocalStorage();
|
|
10
11
|
const rootOf = (agent) => {
|
|
@@ -27,6 +28,13 @@ export function policyHost(ctx, deps) {
|
|
|
27
28
|
const decidedAt = new Map();
|
|
28
29
|
const applied = new Map();
|
|
29
30
|
const latest = new Map();
|
|
31
|
+
/** Fall back to MANUAL when the main model left AUTO's Claude/GPT scope. */
|
|
32
|
+
const lockUnsupportedMain = (agent) => {
|
|
33
|
+
deps.state.setMode(agent.id, 'MANUAL');
|
|
34
|
+
mainStatus.delete(agent.id);
|
|
35
|
+
latest.delete(agent.id);
|
|
36
|
+
applied.delete(agent.id);
|
|
37
|
+
};
|
|
30
38
|
ctx.on('tools/execute', async (exec, next) => {
|
|
31
39
|
if (!['subagent', 'subagent_fork', 'spawn_teammate'].includes(exec.name) || !exec.agent)
|
|
32
40
|
return next();
|
|
@@ -80,6 +88,11 @@ export function policyHost(ctx, deps) {
|
|
|
80
88
|
main.set(agent, { ...old, turn: payload.turn });
|
|
81
89
|
return decision;
|
|
82
90
|
}
|
|
91
|
+
// The main model left AUTO's Claude/GPT scope: judge nothing, rewrite nothing.
|
|
92
|
+
if (!mainSupported(currentMainSelection(ctx, agent))) {
|
|
93
|
+
lockUnsupportedMain(agent);
|
|
94
|
+
return decision;
|
|
95
|
+
}
|
|
83
96
|
applied.delete(agent.id);
|
|
84
97
|
mainStatus.delete(agent.id);
|
|
85
98
|
const startedEpoch = epoch(agent.id);
|
|
@@ -190,17 +203,43 @@ export function policyHost(ctx, deps) {
|
|
|
190
203
|
const runtime = main.get(agent);
|
|
191
204
|
if (!runtime?.active || runtime.turn !== payload.turn || runtime.epoch !== epoch(agent.id) || !enabled(agent.id))
|
|
192
205
|
return config;
|
|
206
|
+
// Authoritative gate: the request's actual model decides, whatever the picker showed earlier.
|
|
207
|
+
if (!autoSupportedModel(config.model)) {
|
|
208
|
+
lockUnsupportedMain(agent);
|
|
209
|
+
return config;
|
|
210
|
+
}
|
|
211
|
+
const unsupported = (effort = '模型不支持调整') => {
|
|
212
|
+
const old = mainStatus.get(agent.id);
|
|
213
|
+
mainStatus.set(agent.id, { ...old, policyVersion, depth: runtime.depth, reason: 'no-effort-support', ...(old?.decision ? { decision: { ...old.decision, status: 'unsupported', model: config.model, provider: config.provider, effort } } : {}) });
|
|
214
|
+
};
|
|
193
215
|
const info = await ctx.llm.resolveModelInfo(config.provider, config.model, payload.signal);
|
|
194
216
|
if (!enabled(agent.id) || runtime.epoch !== epoch(agent.id) || payload.signal.aborted)
|
|
195
217
|
return config;
|
|
196
218
|
const efforts = (info.reasoning?.efforts ?? []).map(e => String(e.id));
|
|
197
219
|
if (!efforts.length) {
|
|
198
|
-
|
|
199
|
-
|
|
220
|
+
unsupported();
|
|
221
|
+
return config;
|
|
222
|
+
}
|
|
223
|
+
let reasoningEffort;
|
|
224
|
+
try {
|
|
225
|
+
reasoningEffort = policyEffort(config.model, runtime.depth, efforts, deps.settings().effortCap, info.reasoning?.defaultEffort ? String(info.reasoning.defaultEffort) : undefined);
|
|
226
|
+
}
|
|
227
|
+
catch (error) {
|
|
228
|
+
// A supported model whose directory cannot produce an automatic effort must not fail the turn.
|
|
229
|
+
ctx.logger.warn(`auto-model-routing: could not map the main reasoning effort: ${String(error)}`);
|
|
230
|
+
unsupported();
|
|
200
231
|
return config;
|
|
201
232
|
}
|
|
202
|
-
const reasoningEffort = policyEffort(config.model, runtime.depth, efforts, deps.settings().effortCap, info.reasoning?.defaultEffort ? String(info.reasoning.defaultEffort) : undefined);
|
|
203
233
|
mainStatus.set(agent.id, { ...mainStatus.get(agent.id), policyVersion, depth: runtime.depth, effort: reasoningEffort, reason: 'jev', ...(mainStatus.get(agent.id)?.decision ? { decision: { ...mainStatus.get(agent.id).decision, effort: reasoningEffort, model: config.model, provider: config.provider } } : {}) });
|
|
234
|
+
if (reasoningEffort !== config.reasoningEffort) {
|
|
235
|
+
try {
|
|
236
|
+
await deps.selectModel({ sessionId: agent.id, provider: config.provider, model: config.model, reasoningEffort });
|
|
237
|
+
}
|
|
238
|
+
catch (error) {
|
|
239
|
+
ctx.logger.warn(`auto-model-routing: could not sync the model selection: ${String(error)}`);
|
|
240
|
+
return config;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
204
243
|
return { ...config, reasoningEffort };
|
|
205
244
|
}
|
|
206
245
|
const record = deps.store.get(agent.id);
|
|
@@ -217,8 +256,29 @@ export function policyHost(ctx, deps) {
|
|
|
217
256
|
validatePolicyRoute(await deps.catalog(), record.route.provider, record.route.model, record.route.reasoningEffort, deps.settings().effortCap);
|
|
218
257
|
return { ...config, ...record.route };
|
|
219
258
|
}, { prepend: true });
|
|
259
|
+
const confirmMainRequest = (session, requestSeq, turn) => {
|
|
260
|
+
const status = mainStatus.get(session.id);
|
|
261
|
+
const actual = session.requestHeader()?.config;
|
|
262
|
+
if (!enabled(session.id) || !status?.decision || !actual)
|
|
263
|
+
return;
|
|
264
|
+
const d = { ...status.decision, status: status.decision.status === 'unsupported' ? 'unsupported' : actual.reasoningEffort === status.effort && actual.provider === status.decision.provider && actual.model === status.decision.model ? 'applied' : 'failed', ...(actual.model ? { model: actual.model } : {}), ...(actual.provider ? { provider: actual.provider } : {}) };
|
|
265
|
+
mainStatus.set(session.id, { ...status, decision: d });
|
|
266
|
+
latest.set(session.id, d);
|
|
267
|
+
if (d.status === 'applied') {
|
|
268
|
+
applied.set(session.id, d);
|
|
269
|
+
requestMetadata.set(session.id, { requestSeq, appliedAt: Date.now(), ...(turn !== undefined ? { requestTurn: turn } : {}) });
|
|
270
|
+
}
|
|
271
|
+
else
|
|
272
|
+
applied.delete(session.id);
|
|
273
|
+
};
|
|
220
274
|
ctx.on('session/event', (session, event) => {
|
|
221
275
|
if (event.type === 'assistant/message') {
|
|
276
|
+
// Unchanged requests reuse the durable header and emit no request/header event.
|
|
277
|
+
if (!applied.has(session.id) && mainStatus.get(session.id)?.decision?.status === 'recommended') {
|
|
278
|
+
const header = session.snapshotEvents().findLast(e => e.type === 'request/header');
|
|
279
|
+
if (header)
|
|
280
|
+
confirmMainRequest(session, header.seq, event.data.turn);
|
|
281
|
+
}
|
|
222
282
|
const d = applied.get(session.id);
|
|
223
283
|
const data = event.data;
|
|
224
284
|
if (d?.status === 'applied' && enabled(session.id) && data.message?.id && d.model) {
|
|
@@ -243,18 +303,7 @@ export function policyHost(ctx, deps) {
|
|
|
243
303
|
requestMetadata.delete(session.id);
|
|
244
304
|
const record = deps.store.get(session.id);
|
|
245
305
|
if (!record?.route) {
|
|
246
|
-
|
|
247
|
-
const actual = event.data.header?.config;
|
|
248
|
-
if (enabled(session.id) && status?.decision && actual) {
|
|
249
|
-
const d = { ...status.decision, status: status.decision.status === 'unsupported' ? 'unsupported' : actual.reasoningEffort === status.effort && actual.provider === status.decision.provider && actual.model === status.decision.model ? 'applied' : 'failed', ...(actual.model ? { model: actual.model } : {}), ...(actual.provider ? { provider: actual.provider } : {}) };
|
|
250
|
-
mainStatus.set(session.id, { ...status, decision: d });
|
|
251
|
-
latest.set(session.id, d);
|
|
252
|
-
if (d.status === 'applied') {
|
|
253
|
-
applied.set(session.id, d);
|
|
254
|
-
const data = event.data;
|
|
255
|
-
requestMetadata.set(session.id, { requestSeq: event.seq, appliedAt: Date.now(), ...(typeof data.turn === 'number' ? { requestTurn: data.turn } : {}) });
|
|
256
|
-
}
|
|
257
|
-
}
|
|
306
|
+
confirmMainRequest(session, event.seq, event.data.turn);
|
|
258
307
|
return;
|
|
259
308
|
}
|
|
260
309
|
const actual = event.data.header?.config;
|
|
@@ -268,6 +317,7 @@ export function policyHost(ctx, deps) {
|
|
|
268
317
|
throw new Error('policy-v2 实际请求配置不匹配');
|
|
269
318
|
});
|
|
270
319
|
return {
|
|
320
|
+
appliedDecision: (id) => enabled(id) ? applied.get(id)?.id : undefined,
|
|
271
321
|
invalidate: (id) => { epochs.set(id, epoch(id) + 1); mainStatus.delete(id); latest.delete(id); applied.delete(id); },
|
|
272
322
|
status: (id) => deps.store.get(id), mainStatus: (id) => enabled(id) ? { ...mainStatus.get(id), ...(latest.get(id) ? { decision: latest.get(id) } : {}) } : undefined,
|
|
273
323
|
};
|
package/lib/policy-store.d.ts
CHANGED
|
@@ -27,6 +27,7 @@ declare const schema: z.ZodObject<{
|
|
|
27
27
|
claude: "claude";
|
|
28
28
|
codex: "codex";
|
|
29
29
|
"dsh-magpie-connect": "dsh-magpie-connect";
|
|
30
|
+
"dsh-magpie-connector": "dsh-magpie-connector";
|
|
30
31
|
}>;
|
|
31
32
|
model: z.ZodOptional<z.ZodString>;
|
|
32
33
|
reasoningEffort: z.ZodOptional<z.ZodString>;
|
|
@@ -57,6 +58,7 @@ declare const schema: z.ZodObject<{
|
|
|
57
58
|
claude: "claude";
|
|
58
59
|
codex: "codex";
|
|
59
60
|
"dsh-magpie-connect": "dsh-magpie-connect";
|
|
61
|
+
"dsh-magpie-connector": "dsh-magpie-connector";
|
|
60
62
|
}>;
|
|
61
63
|
model: z.ZodString;
|
|
62
64
|
reasoningEffort: z.ZodEnum<{
|
|
@@ -71,6 +73,7 @@ declare const schema: z.ZodObject<{
|
|
|
71
73
|
claude: "claude";
|
|
72
74
|
codex: "codex";
|
|
73
75
|
"dsh-magpie-connect": "dsh-magpie-connect";
|
|
76
|
+
"dsh-magpie-connector": "dsh-magpie-connector";
|
|
74
77
|
}>;
|
|
75
78
|
model: z.ZodString;
|
|
76
79
|
reasoningEffort: z.ZodEnum<{
|
package/lib/policy-store.js
CHANGED
|
@@ -2,14 +2,15 @@ import { createHash, randomUUID } from 'node:crypto';
|
|
|
2
2
|
import { mkdirSync, readFileSync, writeFileSync, renameSync, chmodSync } from 'node:fs';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
4
|
import { z } from 'zod';
|
|
5
|
-
|
|
5
|
+
import { POLICY_PROVIDERS } from './policy-v2.js';
|
|
6
|
+
const route = z.object({ provider: z.enum(POLICY_PROVIDERS), model: z.string().min(1).max(128), reasoningEffort: z.enum(['low', 'medium', 'high', 'xhigh']) }).strict();
|
|
6
7
|
const schema = z.object({
|
|
7
8
|
version: z.literal(1), childId: z.string(), parentId: z.string(), rootId: z.string(),
|
|
8
9
|
policyVersion: z.string(), mappingVersion: z.string(),
|
|
9
10
|
status: z.enum(['recommended', 'applied', 'unsupported', 'needs-context', 'failed']),
|
|
10
11
|
path: z.enum(['ordinary', 'teammate', 'fork']),
|
|
11
12
|
family: z.enum(['claude', 'codex']).optional(),
|
|
12
|
-
explicit: z.object({ provider: z.enum(
|
|
13
|
+
explicit: z.object({ provider: z.enum(POLICY_PROVIDERS), model: z.string().optional(), reasoningEffort: z.string().optional() }).strict(),
|
|
13
14
|
decision: z.object({ id: z.number(), source: z.enum(['main', 'subagent']), tier: z.string(), depth: z.string(), capability: z.string().optional(), effort: z.string(), model: z.string().optional(), provider: z.string().optional(), status: z.string(), ms: z.number(), decisionProvider: z.string(), decisionModel: z.string(), confidence: z.number().min(0).max(1).optional(), capabilityConfidence: z.number().min(0).max(1).optional(), probabilities: z.record(z.string(), z.number().min(0).max(1)).optional(), capabilityProbabilities: z.record(z.string(), z.number().min(0).max(1)).optional() }).strict().optional(),
|
|
14
15
|
route: route.optional(), actual: route.optional(), reason: z.string().max(240).optional(),
|
|
15
16
|
}).strict();
|
package/lib/policy-v2.d.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import type { Capability, Depth } from './jev.js';
|
|
2
2
|
import { type ChildProvider } from './effort.js';
|
|
3
3
|
import { type AutomaticEffort, type RoutingCatalog } from './settings-contract.js';
|
|
4
|
-
export declare const
|
|
5
|
-
export type PolicyProvider =
|
|
4
|
+
export declare const POLICY_PROVIDERS: readonly ["claude", "codex", "dsh-magpie-connect", "dsh-magpie-connector"];
|
|
5
|
+
export type PolicyProvider = typeof POLICY_PROVIDERS[number];
|
|
6
|
+
export declare function magpieProvider(value: string): value is 'dsh-magpie-connect' | 'dsh-magpie-connector';
|
|
6
7
|
export declare function policyProvider(value: string): value is PolicyProvider;
|
|
7
8
|
export declare function modelFamily(provider: string, model: string): ChildProvider | undefined;
|
|
8
9
|
export declare function validatePolicyRoute(catalog: RoutingCatalog, provider: string, model: string, effort: string, cap: AutomaticEffort): void;
|
package/lib/policy-v2.js
CHANGED
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
import { targetEffort } from './effort.js';
|
|
2
2
|
import { AUTO_EFFORTS } from './settings-contract.js';
|
|
3
3
|
import { MODEL_ID_PATTERN } from './settings-validation.js';
|
|
4
|
-
export const
|
|
5
|
-
export function
|
|
4
|
+
export const POLICY_PROVIDERS = ['claude', 'codex', 'dsh-magpie-connect', 'dsh-magpie-connector'];
|
|
5
|
+
export function magpieProvider(value) { return value === 'dsh-magpie-connect' || value === 'dsh-magpie-connector'; }
|
|
6
|
+
export function policyProvider(value) { return POLICY_PROVIDERS.some(provider => provider === value); }
|
|
6
7
|
export function modelFamily(provider, model) {
|
|
7
8
|
if (provider === 'claude' || provider === 'codex')
|
|
8
9
|
return provider;
|
|
9
|
-
if (provider
|
|
10
|
+
if (!magpieProvider(provider))
|
|
10
11
|
return undefined;
|
|
11
12
|
const family = model.split('/')[0];
|
|
12
13
|
return family === 'claude' || family === 'codex' ? family : undefined;
|
|
13
14
|
}
|
|
14
15
|
export function validatePolicyRoute(catalog, provider, model, effort, cap) {
|
|
15
16
|
const entry = catalog.models.find(m => m.provider === provider && m.model === model);
|
|
16
|
-
const validId = provider
|
|
17
|
+
const validId = magpieProvider(provider) ? /^[A-Za-z0-9._:-]+(?:\/[A-Za-z0-9._:-]+)+$/.test(model) && model.length <= 128 : MODEL_ID_PATTERN.test(model);
|
|
17
18
|
if (!policyProvider(provider) || !validId || !entry || !AUTO_EFFORTS.includes(effort) || AUTO_EFFORTS.indexOf(effort) > AUTO_EFFORTS.indexOf(cap) || !entry.reasoningEfforts.includes(effort) || catalog.childAllowlist.status !== 'enabled' || !catalog.childAllowlist.allowedModels.some(m => m.provider === provider && m.model === model))
|
|
18
19
|
throw new Error('策略路由不在当前目录/允许列表或 effort 上限内;暂停委派并询问用户');
|
|
19
20
|
}
|
|
@@ -42,10 +43,10 @@ export function policyEffort(model, depth, supported, cap, defaultEffort) {
|
|
|
42
43
|
return AUTO_EFFORTS.indexOf(chosen) > AUTO_EFFORTS.indexOf(cap) ? allowed.at(-1) : chosen;
|
|
43
44
|
}
|
|
44
45
|
export function choosePolicyRoute(provider, capability, depth, catalog, cap, explicit = {}, family) {
|
|
45
|
-
const selectedFamily = provider
|
|
46
|
+
const selectedFamily = magpieProvider(provider) ? family : provider;
|
|
46
47
|
if (!explicit.model && !selectedFamily)
|
|
47
48
|
throw new Error('unsupported:没有已定义的模型能力家族;请显式指定模型');
|
|
48
|
-
const model = explicit.model ?? `${provider
|
|
49
|
+
const model = explicit.model ?? `${magpieProvider(provider) ? `${selectedFamily}/` : ''}${POLICY_MODELS[selectedFamily][capability]}`;
|
|
49
50
|
const entry = catalog.models.find(m => m.provider === provider && m.model === model);
|
|
50
51
|
const reasoningEffort = explicit.reasoningEffort ?? policyEffort(model, depth, entry?.reasoningEfforts ?? [], cap, entry?.defaultEffort);
|
|
51
52
|
validatePolicyRoute(catalog, provider, model, reasoningEffort, cap);
|
package/lib/settings-host.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { credentialRef } from '@deepseek-ai/dsh-credentials';
|
|
2
2
|
import { launchEnvironmentOf } from '@deepseek-ai/dsh-launch-environment';
|
|
3
3
|
import { DEFAULT_SETTINGS, SettingsValidationError } from './settings.js';
|
|
4
|
+
import { magpieProvider } from './policy-v2.js';
|
|
4
5
|
import rubric from './rubric.json' with { type: 'json' };
|
|
5
6
|
import { JevClient } from './jev.js';
|
|
6
7
|
import { DECISION_SERVICES } from './decision-service.js';
|
|
@@ -10,7 +11,7 @@ export function settingsHost(ctx, store) {
|
|
|
10
11
|
const environment = launchEnvironmentOf(ctx);
|
|
11
12
|
const credentials = () => ctx.get('credentials');
|
|
12
13
|
const catalog = async (includeMagpie = store.read().settings.policy === 'policy-v2') => {
|
|
13
|
-
const models = (await Promise.all(ctx.llm.listProviders().filter(p => p.id === 'claude' || p.id === 'codex' || (includeMagpie && p.id
|
|
14
|
+
const models = (await Promise.all(ctx.llm.listProviders().filter(p => p.id === 'claude' || p.id === 'codex' || (includeMagpie && magpieProvider(p.id))).map(async (provider) => {
|
|
14
15
|
try {
|
|
15
16
|
return await Promise.all((await ctx.llm.listModels(provider.id)).map(async (model) => {
|
|
16
17
|
try {
|
|
@@ -31,7 +32,7 @@ export function settingsHost(ctx, store) {
|
|
|
31
32
|
try {
|
|
32
33
|
if (service) {
|
|
33
34
|
const current = service.current();
|
|
34
|
-
return { models, childAllowlist: { status: current.enabled ? 'enabled' : 'disabled', allowedModels: current.allowedModels.filter(r => r.provider === 'claude' || r.provider === 'codex' || (includeMagpie && r.provider
|
|
35
|
+
return { models, childAllowlist: { status: current.enabled ? 'enabled' : 'disabled', allowedModels: current.allowedModels.filter(r => r.provider === 'claude' || r.provider === 'codex' || (includeMagpie && magpieProvider(r.provider))) } };
|
|
35
36
|
}
|
|
36
37
|
}
|
|
37
38
|
catch { /* An unavailable policy must never broaden the allowed routes. */ }
|
package/lib/subagent-tool.js
CHANGED
|
@@ -5,7 +5,7 @@ import { DEFAULT_SETTINGS } from './settings-contract.js';
|
|
|
5
5
|
import { validateChildRoute } from './settings-validation.js';
|
|
6
6
|
import { CATEGORY_HINTS } from './jev.js';
|
|
7
7
|
import { buildState } from './context.js';
|
|
8
|
-
import { choosePolicyRoute, modelFamily } from './policy-v2.js';
|
|
8
|
+
import { choosePolicyRoute, magpieProvider, modelFamily, POLICY_PROVIDERS } from './policy-v2.js';
|
|
9
9
|
import { policyVersion } from './jev.js';
|
|
10
10
|
/** The ordinary session a caller belongs to; undefined when any link is no longer live (treated as MANUAL). */
|
|
11
11
|
function rootOf(caller, agents) {
|
|
@@ -27,7 +27,7 @@ export function routeSubagentTool(jev, deps) {
|
|
|
27
27
|
timeoutMs: 5000,
|
|
28
28
|
parameters: {
|
|
29
29
|
task: { type: 'string', required: true, description: 'The task the sub-agent will do.' },
|
|
30
|
-
provider: { type: 'string', required: true, enum: [
|
|
30
|
+
provider: { type: 'string', required: true, enum: [...POLICY_PROVIDERS], description: 'Exact transport provider. Magpie is available only under policy-v2.' },
|
|
31
31
|
model: { type: 'string', description: 'Sub-agent model already chosen by the user or lead; when given, only reasoning_effort is routed.' },
|
|
32
32
|
reasoning_effort: { type: 'string', description: 'Explicit effort constraint for policy-v2; preserved and validated, never silently replaced.' },
|
|
33
33
|
family: { type: 'string', enum: ['claude', 'codex'], description: 'Magpie automatic capability family; required when no model is explicitly supplied.' },
|
|
@@ -76,7 +76,7 @@ export function routeSubagentTool(jev, deps) {
|
|
|
76
76
|
const route = choosePolicyRoute(args.provider, result.capability ?? 'bounded', result.depth, await deps.catalog(), deps.readSettings().effortCap, explicit, args.family ?? (args.model ? modelFamily(args.provider, args.model) : undefined));
|
|
77
77
|
return { status: 'recommended', policyVersion, ...(result.capability ? { capability: result.capability } : {}), depth: result.depth, provider: route.provider, model: route.model, reasoning_effort: route.reasoningEffort };
|
|
78
78
|
}
|
|
79
|
-
if (args.provider
|
|
79
|
+
if (magpieProvider(args.provider))
|
|
80
80
|
throw new Error('Magpie 路由需显式选择 policy-v2;原策略不变');
|
|
81
81
|
const result = await jev.judge(buildState({ text: args.task }), exec.signal);
|
|
82
82
|
if (result.kind === 'cancelled')
|
package/lib/support.d.ts
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
2
|
+
import type { Agent } from '@deepseek-ai/dsh-agent';
|
|
3
|
+
/** One main-session model selection as the AUTO support gate needs it. */
|
|
4
|
+
export interface MainSelection {
|
|
5
|
+
provider: string;
|
|
6
|
+
model: string;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* AUTO only manages the Claude / GPT model families.
|
|
10
|
+
*
|
|
11
|
+
* The transport prefix (`claude/...`, `codex/...` on Magpie) is not part of the
|
|
12
|
+
* family name, so it is stripped before matching. Every other family — DeepSeek,
|
|
13
|
+
* GLM, Grok, Gemini, Kimi, Qwen, ... — is explicitly out of scope and locks the
|
|
14
|
+
* session to MANUAL instead of being guessed at.
|
|
15
|
+
*/
|
|
16
|
+
export declare function autoSupportedModel(model: string | undefined): boolean;
|
|
17
|
+
/** An unknown selection must never lock a session: only a proven mismatch does. */
|
|
18
|
+
export declare function mainSupported(selection: MainSelection | undefined): boolean;
|
|
19
|
+
/**
|
|
20
|
+
* The main session's current model selection, most-authoritative signal first:
|
|
21
|
+
* the official pending pick (`model/selection` projection), then the last actual
|
|
22
|
+
* request header, then the creation-time agent options. Undefined when none can
|
|
23
|
+
* be read; callers treat that as "unknown, do not lock".
|
|
24
|
+
*/
|
|
25
|
+
export declare function currentMainSelection(ctx: Context, agent: Agent): MainSelection | undefined;
|
package/lib/support.js
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AUTO only manages the Claude / GPT model families.
|
|
3
|
+
*
|
|
4
|
+
* The transport prefix (`claude/...`, `codex/...` on Magpie) is not part of the
|
|
5
|
+
* family name, so it is stripped before matching. Every other family — DeepSeek,
|
|
6
|
+
* GLM, Grok, Gemini, Kimi, Qwen, ... — is explicitly out of scope and locks the
|
|
7
|
+
* session to MANUAL instead of being guessed at.
|
|
8
|
+
*/
|
|
9
|
+
export function autoSupportedModel(model) {
|
|
10
|
+
if (typeof model !== 'string' || model.length === 0)
|
|
11
|
+
return false;
|
|
12
|
+
const bare = model.includes('/') ? model.slice(model.lastIndexOf('/') + 1) : model;
|
|
13
|
+
return /^(?:claude|gpt)-/i.test(bare);
|
|
14
|
+
}
|
|
15
|
+
/** An unknown selection must never lock a session: only a proven mismatch does. */
|
|
16
|
+
export function mainSupported(selection) {
|
|
17
|
+
return selection === undefined || autoSupportedModel(selection.model);
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* The main session's current model selection, most-authoritative signal first:
|
|
21
|
+
* the official pending pick (`model/selection` projection), then the last actual
|
|
22
|
+
* request header, then the creation-time agent options. Undefined when none can
|
|
23
|
+
* be read; callers treat that as "unknown, do not lock".
|
|
24
|
+
*/
|
|
25
|
+
export function currentMainSelection(ctx, agent) {
|
|
26
|
+
try {
|
|
27
|
+
const service = ctx.get('sessionProjections');
|
|
28
|
+
const pending = service?.stateOf(agent.session, 'modelSelection')?.pending;
|
|
29
|
+
if (pending && typeof pending.provider === 'string' && typeof pending.model === 'string')
|
|
30
|
+
return { provider: pending.provider, model: pending.model };
|
|
31
|
+
}
|
|
32
|
+
catch { /* optional projection: fall through to the durable header */ }
|
|
33
|
+
const header = agent.session.requestHeader()?.config;
|
|
34
|
+
if (header && typeof header.provider === 'string' && typeof header.model === 'string')
|
|
35
|
+
return { provider: header.provider, model: header.model };
|
|
36
|
+
const options = agent.options;
|
|
37
|
+
if (typeof options.provider === 'string' && typeof options.model === 'string')
|
|
38
|
+
return { provider: options.provider, model: options.model };
|
|
39
|
+
return undefined;
|
|
40
|
+
}
|
package/lib/usage-report.d.ts
CHANGED
|
@@ -40,6 +40,25 @@ export declare function buildReport(home: string, options?: {
|
|
|
40
40
|
};
|
|
41
41
|
};
|
|
42
42
|
feedback: Record<string, number>;
|
|
43
|
+
implicit: {
|
|
44
|
+
counts: {
|
|
45
|
+
explicit: number;
|
|
46
|
+
negative: number;
|
|
47
|
+
'implicit-ok': number;
|
|
48
|
+
inconclusive: number;
|
|
49
|
+
unobserved: number;
|
|
50
|
+
};
|
|
51
|
+
explicit: {
|
|
52
|
+
conflict: number;
|
|
53
|
+
};
|
|
54
|
+
exposedExplicit: {
|
|
55
|
+
conflict: number;
|
|
56
|
+
};
|
|
57
|
+
negative: {
|
|
58
|
+
'aborted-user': number;
|
|
59
|
+
'manual-switch': number;
|
|
60
|
+
};
|
|
61
|
+
};
|
|
43
62
|
conclusion: string;
|
|
44
63
|
}[];
|
|
45
64
|
}[];
|
package/lib/usage-report.js
CHANGED
|
@@ -18,6 +18,10 @@ const CAVEATS = [
|
|
|
18
18
|
'token-semantics-differ-by-provider',
|
|
19
19
|
'reference-before-auto-effort-only-confounded',
|
|
20
20
|
'mode-history-not-persisted',
|
|
21
|
+
'implicit-ok-not-appropriateness',
|
|
22
|
+
'exposure-not-proof-of-reading',
|
|
23
|
+
'historical-exposure-unknown',
|
|
24
|
+
'exposure-after-continuation-excluded',
|
|
21
25
|
];
|
|
22
26
|
const sorted = (record) => Object.fromEntries(Object.entries(record).sort(([a], [b]) => a.localeCompare(b)));
|
|
23
27
|
const round = (n) => Math.round(n * 100) / 100;
|
|
@@ -122,14 +126,25 @@ function readSession(file, failures) {
|
|
|
122
126
|
}
|
|
123
127
|
const calls = [];
|
|
124
128
|
const seen = new Set();
|
|
129
|
+
const ends = new Map(), userStarts = new Map();
|
|
130
|
+
let startedAt;
|
|
125
131
|
let turn, userTurn;
|
|
126
132
|
let config = {};
|
|
127
133
|
for (const event of events) {
|
|
128
134
|
const data = event.data ?? {};
|
|
129
|
-
if (event.type === 'turn/start')
|
|
135
|
+
if (event.type === 'turn/start') {
|
|
130
136
|
turn = data.turn;
|
|
131
|
-
|
|
137
|
+
startedAt = event.time;
|
|
138
|
+
}
|
|
139
|
+
else if (event.type === 'user/message' && data.source?.kind === 'user' && turn !== undefined) {
|
|
132
140
|
userTurn = turn;
|
|
141
|
+
if (typeof startedAt === 'number')
|
|
142
|
+
userStarts.set(turn, startedAt);
|
|
143
|
+
}
|
|
144
|
+
else if (event.type === 'turn/end' && typeof data.turn === 'number') {
|
|
145
|
+
const reason = data.reason;
|
|
146
|
+
ends.set(data.turn, reason?.kind === 'aborted' ? reason.reason?.kind === 'user' ? 'aborted-user' : 'aborted-other' : ['completed', 'blocked'].includes(reason?.kind) ? reason.kind : 'other');
|
|
147
|
+
}
|
|
133
148
|
else if (event.type === 'request/header' && data.header?.config)
|
|
134
149
|
config = data.header.config;
|
|
135
150
|
else if (event.type === 'assistant/message') {
|
|
@@ -143,10 +158,10 @@ function readSession(file, failures) {
|
|
|
143
158
|
for (const field of FIELDS)
|
|
144
159
|
if (typeof data.usage?.[field] === 'number')
|
|
145
160
|
usage[field] = data.usage[field];
|
|
146
|
-
calls.push({ id, ...(userTurn === undefined ? {} : { userTurn }), time: event.time, usage, ...(config.model ? { model: config.model } : {}), ...(config.reasoningEffort ? { effort: config.reasoningEffort } : {}) });
|
|
161
|
+
calls.push({ id, ...(typeof data.turn === 'number' ? { turn: data.turn } : turn === undefined ? {} : { turn }), ...(userTurn === undefined ? {} : { userTurn }), time: event.time, usage, ...(config.model ? { model: config.model } : {}), ...(config.reasoningEffort ? { effort: config.reasoningEffort } : {}) });
|
|
147
162
|
}
|
|
148
163
|
}
|
|
149
|
-
return { header, calls };
|
|
164
|
+
return { header, calls, ends, userStarts };
|
|
150
165
|
}
|
|
151
166
|
function localMidnight(date) {
|
|
152
167
|
const [y, m, d] = date.split('-').map(Number);
|
|
@@ -208,6 +223,15 @@ export function buildReport(home, options = {}) {
|
|
|
208
223
|
excluded.seeded++;
|
|
209
224
|
continue;
|
|
210
225
|
}
|
|
226
|
+
const nextUserStart = new Map();
|
|
227
|
+
let next;
|
|
228
|
+
const turns = [...new Set([...log.calls.flatMap(call => call.turn === undefined ? [] : [call.turn]), ...log.userStarts.keys()])].sort((a, b) => b - a);
|
|
229
|
+
for (const turn of turns) {
|
|
230
|
+
if (next !== undefined)
|
|
231
|
+
nextUserStart.set(turn, next);
|
|
232
|
+
if (log.userStarts.has(turn))
|
|
233
|
+
next = log.userStarts.get(turn);
|
|
234
|
+
}
|
|
211
235
|
const linked = new Set();
|
|
212
236
|
let v2First;
|
|
213
237
|
for (const call of log.calls) {
|
|
@@ -220,10 +244,12 @@ export function buildReport(home, options = {}) {
|
|
|
220
244
|
const key = JSON.stringify([session, b.decisionId]);
|
|
221
245
|
let d = decisions.get(key);
|
|
222
246
|
if (!d) {
|
|
223
|
-
d = { session, version: b.snapshot ? `${b.snapshot.policyVersion}/${b.snapshot.mappingVersion}` : 'unknown', model: b.model, depth: b.depth, effort: b.effort, calls: [], feedback: new Set() };
|
|
247
|
+
d = { session, id: b.decisionId, version: b.snapshot ? `${b.snapshot.policyVersion}/${b.snapshot.mappingVersion}` : 'unknown', model: b.model, depth: b.depth, effort: b.effort, calls: [], feedback: new Set() };
|
|
224
248
|
decisions.set(key, d);
|
|
225
249
|
}
|
|
226
250
|
d.calls.push(call);
|
|
251
|
+
d.end = call.turn === undefined ? 'unknown' : log.ends.get(call.turn) ?? 'unknown';
|
|
252
|
+
d.continuation = call.turn === undefined ? undefined : nextUserStart.get(call.turn);
|
|
227
253
|
if (b.feedback)
|
|
228
254
|
d.feedback.add(b.feedback);
|
|
229
255
|
}
|
|
@@ -269,6 +295,36 @@ export function buildReport(home, options = {}) {
|
|
|
269
295
|
const feedback = { ...Object.fromEntries(FEEDBACK.map(f => [f, 0])), conflict: 0, unrated: 0 };
|
|
270
296
|
for (const d of list)
|
|
271
297
|
bump(feedback, d.feedback.size === 0 ? 'unrated' : d.feedback.size > 1 ? 'conflict' : [...d.feedback][0]);
|
|
298
|
+
const implicit = {
|
|
299
|
+
counts: { explicit: 0, negative: 0, 'implicit-ok': 0, inconclusive: 0, unobserved: 0 },
|
|
300
|
+
explicit: { ...Object.fromEntries(FEEDBACK.map(f => [f, 0])), conflict: 0 },
|
|
301
|
+
exposedExplicit: { ...Object.fromEntries(FEEDBACK.map(f => [f, 0])), conflict: 0 },
|
|
302
|
+
negative: { 'aborted-user': 0, 'manual-switch': 0 },
|
|
303
|
+
};
|
|
304
|
+
for (const d of list) {
|
|
305
|
+
const facts = book.facts(d.session, d.id);
|
|
306
|
+
const exposed = facts.exposed !== undefined && (d.continuation === undefined || facts.exposed < d.continuation);
|
|
307
|
+
const label = d.feedback.size > 1 ? 'conflict' : [...d.feedback][0];
|
|
308
|
+
if (label) {
|
|
309
|
+
implicit.counts.explicit++;
|
|
310
|
+
bump(implicit.explicit, label);
|
|
311
|
+
if (exposed)
|
|
312
|
+
bump(implicit.exposedExplicit, label);
|
|
313
|
+
}
|
|
314
|
+
else if (d.end === 'aborted-user' || facts['manual-switch'] !== undefined) {
|
|
315
|
+
implicit.counts.negative++;
|
|
316
|
+
if (d.end === 'aborted-user')
|
|
317
|
+
implicit.negative['aborted-user']++;
|
|
318
|
+
if (facts['manual-switch'] !== undefined)
|
|
319
|
+
implicit.negative['manual-switch']++;
|
|
320
|
+
}
|
|
321
|
+
else if (exposed && d.end === 'completed' && d.continuation !== undefined)
|
|
322
|
+
implicit.counts['implicit-ok']++;
|
|
323
|
+
else if (exposed)
|
|
324
|
+
implicit.counts.inconclusive++;
|
|
325
|
+
else
|
|
326
|
+
implicit.counts.unobserved++;
|
|
327
|
+
}
|
|
272
328
|
const byEffort = {};
|
|
273
329
|
for (const d of list) {
|
|
274
330
|
const e = (byEffort[d.effort] ??= { decisions: 0, calls: 0 });
|
|
@@ -278,7 +334,7 @@ export function buildReport(home, options = {}) {
|
|
|
278
334
|
return {
|
|
279
335
|
model, depth, decisions: list.length, sessions, calls: calls.length,
|
|
280
336
|
callsPerDecision: { median: median(list.map(d => d.calls.length)), max: Math.max(...list.map(d => d.calls.length)) },
|
|
281
|
-
tokens: tokens(calls), byEffort: sorted(byEffort), feedback,
|
|
337
|
+
tokens: tokens(calls), byEffort: sorted(byEffort), feedback, implicit,
|
|
282
338
|
conclusion: list.length >= 20 && sessions >= 3 ? 'observed' : 'insufficient-sample',
|
|
283
339
|
};
|
|
284
340
|
}),
|
package/package.json
CHANGED
|
@@ -1,13 +1,26 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-auto-model-routing",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.2",
|
|
4
4
|
"description": "DeepSeek Harness plugin: independent Jev capability and effort decisions, enforced fresh-child routes, decision visibility and allocation feedback",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "prgrmrwy",
|
|
7
|
-
"repository": {
|
|
7
|
+
"repository": {
|
|
8
|
+
"type": "git",
|
|
9
|
+
"url": "git+https://github.com/prgrmrwy/dsh-auto-model-routing.git"
|
|
10
|
+
},
|
|
8
11
|
"homepage": "https://github.com/prgrmrwy/dsh-auto-model-routing#readme",
|
|
9
|
-
"bugs": {
|
|
10
|
-
|
|
12
|
+
"bugs": {
|
|
13
|
+
"url": "https://github.com/prgrmrwy/dsh-auto-model-routing/issues"
|
|
14
|
+
},
|
|
15
|
+
"keywords": [
|
|
16
|
+
"dsh",
|
|
17
|
+
"dsh-plugin",
|
|
18
|
+
"deepseek-harness",
|
|
19
|
+
"reasoning-effort",
|
|
20
|
+
"model-routing",
|
|
21
|
+
"jev",
|
|
22
|
+
"subagent"
|
|
23
|
+
],
|
|
11
24
|
"type": "module",
|
|
12
25
|
"main": "./lib/index.js",
|
|
13
26
|
"types": "./lib/index.d.ts",
|
|
@@ -39,6 +39,11 @@ description: 仅当用户明确要求评估 AUTO 效果、观察 AUTO 期间的
|
|
|
39
39
|
- `token-semantics-differ-by-provider`:不同 provider 的 token 口径可能不同,不要跨模型比较。
|
|
40
40
|
- `reference-before-auto-effort-only-confounded`:`referenceBeforeAuto` 只用于看 effort 起点的变化;它受会话阶段、任务类型、模型切换等因素混杂,不能当作对照组。
|
|
41
41
|
- `mode-history-not-persisted`:会话模式的切换历史没有保存,会话计数只是粗略参考。
|
|
42
|
+
- `implicit-ok-not-appropriateness`:隐式无异议不等于分配合适,不能计作显式好评。
|
|
43
|
+
- `exposure-not-proof-of-reading`:前台可见 3 秒只是曝光,不证明用户读过。
|
|
44
|
+
- `historical-exposure-unknown`:上线前的曝光未知,不回填。
|
|
45
|
+
- `exposure-after-continuation-excluded`:下一用户回合之后才出现的曝光不算,避免翻看旧记录制造认可。
|
|
46
|
+
- `implicit.counts` 按优先级把决策分为显式评价、负向行为、隐式无异议、无定论、未曝光,五类合计为决策数;`implicit.negative` 的两个原因可重叠。`implicit.exposedExplicit` 是有效曝光且有显式评价的参照,不是准确率。
|
|
42
47
|
- 不要自行计算节省百分比,不要说“质量不变”,也不要把“首个记录之前”称为 MANUAL。
|
|
43
48
|
- token 均值的分母是**出现该字段的调用数**(`present`);值为 `null` 表示该字段从未出现,不是 0。
|
|
44
49
|
|