dsh-auto-model-routing 0.3.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/CHANGELOG.md +15 -0
  2. package/README.md +82 -24
  3. package/README.zh.md +82 -24
  4. package/docs/architecture-v2.en.json +26 -0
  5. package/docs/architecture-v2.en.svg +637 -0
  6. package/docs/architecture-v2.zh.json +281 -0
  7. package/docs/architecture-v2.zh.svg +637 -0
  8. package/docs/feedback-evaluation.md +18 -0
  9. package/docs/host-capability-report.md +91 -0
  10. package/docs/jev-policy-v2-validation.md +37 -0
  11. package/docs/magpie-live-acceptance.md +27 -0
  12. package/docs/policy-v2-development.md +68 -0
  13. package/docs/policy-v2-real-evaluation-plan.md +32 -0
  14. package/docs/routing-community-research-2026-10-24.md +98 -0
  15. package/docs/routing-intent-and-gaps.md +56 -0
  16. package/docs/routing-strategy-plan.md +101 -0
  17. package/lib/client/control.d.ts +9 -1
  18. package/lib/client/control.js +16 -1
  19. package/lib/client/index.d.ts +2 -0
  20. package/lib/client/index.js +2 -2
  21. package/lib/client/label.d.ts +4 -0
  22. package/lib/client/label.js +24 -8
  23. package/lib/client/settings-section.js +53 -3
  24. package/lib/client/settings-ui.d.ts +1 -1
  25. package/lib/client/strategy-details.d.ts +4 -0
  26. package/lib/client/strategy-details.js +8 -0
  27. package/lib/client.js +332 -47
  28. package/lib/decision-service.d.ts +39 -0
  29. package/lib/decision-service.js +28 -0
  30. package/lib/endpoints.d.ts +2 -1
  31. package/lib/endpoints.js +22 -1
  32. package/lib/feedback.d.ts +95 -0
  33. package/lib/feedback.js +67 -0
  34. package/lib/hints.js +1 -1
  35. package/lib/index.js +29 -7
  36. package/lib/jev.d.ts +27 -1
  37. package/lib/jev.js +51 -23
  38. package/lib/policy-host.d.ts +68 -0
  39. package/lib/policy-host.js +274 -0
  40. package/lib/policy-store.d.ts +96 -0
  41. package/lib/policy-store.js +60 -0
  42. package/lib/policy-v2-rubric.json +21 -0
  43. package/lib/policy-v2.d.ts +20 -0
  44. package/lib/policy-v2.js +53 -0
  45. package/lib/settings-contract.d.ts +4 -1
  46. package/lib/settings-contract.js +1 -1
  47. package/lib/settings-host.d.ts +2 -2
  48. package/lib/settings-host.js +26 -14
  49. package/lib/settings.js +14 -1
  50. package/lib/subagent-tool.js +32 -5
  51. package/lib/usage-report.d.ts +59 -0
  52. package/lib/usage-report.js +332 -0
  53. package/package.json +3 -3
  54. package/skills/auto-model-routing-report/SKILL.md +50 -0
  55. package/skills/subagent-model-routing/SKILL.md +15 -0
  56. package/lib/selection-projection.d.ts +0 -8
  57. package/lib/selection-projection.js +0 -7
@@ -2,18 +2,20 @@ import { credentialRef } from '@deepseek-ai/dsh-credentials';
2
2
  import { launchEnvironmentOf } from '@deepseek-ai/dsh-launch-environment';
3
3
  import { DEFAULT_SETTINGS, SettingsValidationError } from './settings.js';
4
4
  import rubric from './rubric.json' with { type: 'json' };
5
- const keyRef = credentialRef('OPENROUTER_API_KEY');
5
+ import { JevClient } from './jev.js';
6
+ import { DECISION_SERVICES } from './decision-service.js';
7
+ const keyRef = (provider = 'openrouter') => credentialRef(DECISION_SERVICES[provider].credential);
6
8
  /** Only the fixed credential reference and public catalog metadata cross this adapter. */
7
9
  export function settingsHost(ctx, store) {
8
10
  const environment = launchEnvironmentOf(ctx);
9
11
  const credentials = () => ctx.get('credentials');
10
- const catalog = async () => {
11
- const models = (await Promise.all(ctx.llm.listProviders().filter(p => p.id === 'claude' || p.id === 'codex').map(async (provider) => {
12
+ const catalog = async (includeMagpie = store.read().settings.policy === 'policy-v2') => {
13
+ const models = (await Promise.all(ctx.llm.listProviders().filter(p => p.id === 'claude' || p.id === 'codex' || (includeMagpie && p.id === 'dsh-magpie-connect')).map(async (provider) => {
12
14
  try {
13
15
  return await Promise.all((await ctx.llm.listModels(provider.id)).map(async (model) => {
14
16
  try {
15
17
  const info = await ctx.llm.resolveModelInfo(provider.id, model.id);
16
- return { provider: provider.id, model: model.id, name: model.name, reasoningEfforts: (info.reasoning?.efforts ?? []).map(e => String(e.id)) };
18
+ return { provider: provider.id, model: model.id, name: model.name, reasoningEfforts: (info.reasoning?.efforts ?? []).map(e => String(e.id)), ...(info.reasoning?.defaultEffort ? { defaultEffort: String(info.reasoning.defaultEffort) } : {}) };
17
19
  }
18
20
  catch {
19
21
  return undefined;
@@ -29,23 +31,24 @@ export function settingsHost(ctx, store) {
29
31
  try {
30
32
  if (service) {
31
33
  const current = service.current();
32
- return { models, childAllowlist: { status: current.enabled ? 'enabled' : 'disabled', allowedModels: current.allowedModels.filter(r => r.provider === 'claude' || r.provider === 'codex') } };
34
+ return { models, childAllowlist: { status: current.enabled ? 'enabled' : 'disabled', allowedModels: current.allowedModels.filter(r => r.provider === 'claude' || r.provider === 'codex' || (includeMagpie && r.provider === 'dsh-magpie-connect')) } };
33
35
  }
34
36
  }
35
37
  catch { /* An unavailable policy must never broaden the allowed routes. */ }
36
38
  return { models, childAllowlist: { status: 'unavailable', allowedModels: [] } };
37
39
  };
38
- const keyStatus = async () => {
40
+ const keyStatus = async (provider = store.read().settings.decisionService?.provider ?? 'openrouter') => {
41
+ const ref = keyRef(provider);
39
42
  const service = credentials();
40
43
  if (service) {
41
44
  try {
42
- return { ...await service.describe(keyRef), available: true };
45
+ return { ...await service.describe(ref), available: true };
43
46
  }
44
47
  catch {
45
48
  return { configured: false, writable: false, available: false };
46
49
  }
47
50
  }
48
- return { configured: Boolean(environment.get('OPENROUTER_API_KEY')?.value), writable: false, available: false };
51
+ return { configured: Boolean(environment.get(ref)?.value), writable: false, available: false };
49
52
  };
50
53
  const read = async () => {
51
54
  const current = await catalog();
@@ -55,6 +58,11 @@ export function settingsHost(ctx, store) {
55
58
  const endpoint = async (operation, body) => {
56
59
  if (operation === 'settings')
57
60
  return Response.json(await read());
61
+ if (operation === 'testDecision') {
62
+ const client = new JevClient({ get: async (name) => credentials() ? credentials().resolve(credentialRef(name)) : environment.get(name) }, undefined, 4500, () => store.read().settings.decisionService);
63
+ const result = await client.judgePolicy('Public connection test: sort the numbers [3,1,2] in ascending order.', new AbortController().signal, 'child');
64
+ return Response.json(result);
65
+ }
58
66
  if (operation === 'saveSettings') {
59
67
  try {
60
68
  store.save(body, await catalog());
@@ -69,19 +77,23 @@ export function settingsHost(ctx, store) {
69
77
  const service = credentials();
70
78
  if (!service)
71
79
  return Response.json({ error: '凭据服务不可用' }, { status: 503 });
72
- if (typeof body !== 'object' || body === null || Array.isArray(body) || Object.keys(body).length !== 1 || !('value' in body))
80
+ if (typeof body !== 'object' || body === null || Array.isArray(body) || Object.keys(body).some(k => !['value', 'provider'].includes(k)) || !('value' in body))
73
81
  return Response.json({ error: '无效的凭据请求' }, { status: 400 });
82
+ const provider = 'provider' in body ? body.provider : 'openrouter';
83
+ if (typeof provider !== 'string' || !Object.hasOwn(DECISION_SERVICES, provider))
84
+ return Response.json({ error: '无效的服务商' }, { status: 400 });
85
+ const ref = keyRef(provider);
74
86
  const value = body.value;
75
87
  if (value !== null && (typeof value !== 'string' || !value || value.length > 512 || /\s/.test(value)))
76
88
  return Response.json({ error: 'API key 必须为不含空白的非空字符串,且不超过 512 字符' }, { status: 400 });
77
89
  try {
78
- if (!(await service.describe(keyRef)).writable)
90
+ if (!(await service.describe(ref)).writable)
79
91
  return Response.json({ error: '该 key 来自启动环境,需先从启动环境移除并重启才能在此管理' }, { status: 409 });
80
92
  if (value === null)
81
- await service.unset(keyRef);
93
+ await service.unset(ref);
82
94
  else
83
- await service.set(keyRef, value);
84
- return Response.json({ key: await keyStatus() });
95
+ await service.set(ref, value);
96
+ return Response.json({ key: await keyStatus(provider) });
85
97
  }
86
98
  catch {
87
99
  return Response.json({ error: '无法更新 API key;请检查凭据服务与只读来源' }, { status: 500 });
@@ -91,7 +103,7 @@ export function settingsHost(ctx, store) {
91
103
  catalog, read, endpoint,
92
104
  credential: async (name) => {
93
105
  const service = credentials();
94
- return service ? service.resolve(keyRef) : environment.get(name);
106
+ return service ? service.resolve(credentialRef(name)) : environment.get(name);
95
107
  },
96
108
  };
97
109
  }
package/lib/settings.js CHANGED
@@ -2,11 +2,20 @@ import { chmodSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSy
2
2
  import { dirname } from 'node:path';
3
3
  import { randomUUID } from 'node:crypto';
4
4
  import { z } from 'zod';
5
+ import { resolveDecisionService } from './decision-service.js';
5
6
  import { tiers } from './effort.js';
6
7
  import { AUTO_EFFORTS, DEFAULT_SETTINGS } from './settings-contract.js';
7
8
  import { isRecord, issue, MODEL_ID_PATTERN, validateChildRoute, validateFamilyMainEffort, validateMainRoute } from './settings-validation.js';
8
9
  export { DEFAULT_SETTINGS } from './settings-contract.js';
9
10
  const ordinary = {
11
+ decisionService: z.object({ provider: z.enum(['openrouter', 'typesafe', 'concentrate', 'custom']), model: z.string().regex(/^[A-Za-z0-9._:/~-]{1,128}$/).optional(), endpoint: z.string().max(2048).optional() }).strict().refine(value => { try {
12
+ resolveDecisionService(value);
13
+ return true;
14
+ }
15
+ catch {
16
+ return false;
17
+ } }, 'Invalid decision service'),
18
+ policy: z.enum(['legacy', 'policy-v2']),
10
19
  decisionToast: z.boolean(), defaultMode: z.enum(['MANUAL', 'AUTO']),
11
20
  effortCap: z.enum(AUTO_EFFORTS), jevTimeoutMs: z.number().int().min(1000).max(4500),
12
21
  };
@@ -183,8 +192,12 @@ export class SettingsStore {
183
192
  const parsed = schema.safeParse(raw[key]);
184
193
  if (parsed.success)
185
194
  Object.assign(settings, { [key]: parsed.data });
186
- else
195
+ else {
187
196
  issues.push(...schemaIssues(parsed.error, key));
197
+ // A malformed explicit transport must never fall back to OpenRouter.
198
+ if (key === 'decisionService')
199
+ Object.assign(settings, { decisionService: { provider: 'custom', endpoint: '' } });
200
+ }
188
201
  }
189
202
  for (const kind of ['mainEfforts', 'childRoutes']) {
190
203
  if (!Object.hasOwn(raw, kind))
@@ -5,6 +5,8 @@ import { DEFAULT_SETTINGS } from './settings-contract.js';
5
5
  import { validateChildRoute } from './settings-validation.js';
6
6
  import { CATEGORY_HINTS } from './jev.js';
7
7
  import { buildState } from './context.js';
8
+ import { choosePolicyRoute, modelFamily } from './policy-v2.js';
9
+ import { policyVersion } from './jev.js';
8
10
  /** The ordinary session a caller belongs to; undefined when any link is no longer live (treated as MANUAL). */
9
11
  function rootOf(caller, agents) {
10
12
  let current = caller;
@@ -21,12 +23,15 @@ const MANUAL_TEXT = '主会话为 MANUAL:不判档,按官方默认委派;
21
23
  export function routeSubagentTool(jev, deps) {
22
24
  return defineTool({
23
25
  name: 'route_subagent',
24
- description: 'Call before every sub-agent delegation, whatever the task size. Picks provider/model/reasoning_effort with the Jev decision model; pass provider, model and reasoning_effort explicitly to the sub-agent tool, or the sub-agent inherits this session\'s effort. Sends the task text (redacted, up to 8000 characters) to OpenRouter/TypeSafe only while the main session is AUTO; otherwise returns routed=false: delegate with the official defaults. If the user or lead already chose the sub-agent model, pass it as model: only reasoning_effort is routed and the model is kept. On error: pause delegation and ask the user; never guess a tier.',
26
+ description: 'When policy-v2 is selected, this tool is an optional preview only: use the official fresh creation tool directly for exact-child automatic routing, do not turn preview output into explicit user constraints; fork is unsupported. Under the legacy policy: Call before every sub-agent delegation, whatever the task size. Picks provider/model/reasoning_effort with the Jev decision model; pass provider, model and reasoning_effort explicitly to the sub-agent tool, or the sub-agent inherits this session\'s effort. Sends the task text (redacted, up to 8000 characters) to OpenRouter/TypeSafe only while the main session is AUTO; otherwise returns routed=false: delegate with the official defaults. If the user or lead already chose the sub-agent model, pass it as model: only reasoning_effort is routed and the model is kept. On error: pause delegation and ask the user; never guess a tier.',
25
27
  timeoutMs: 5000,
26
28
  parameters: {
27
29
  task: { type: 'string', required: true, description: 'The task the sub-agent will do.' },
28
- provider: { type: 'string', required: true, enum: ['claude', 'codex'], description: 'Provider family for the sub-agent.' },
30
+ provider: { type: 'string', required: true, enum: ['claude', 'codex', 'dsh-magpie-connect'], description: 'Exact transport provider. Magpie is available only under policy-v2.' },
29
31
  model: { type: 'string', description: 'Sub-agent model already chosen by the user or lead; when given, only reasoning_effort is routed.' },
32
+ reasoning_effort: { type: 'string', description: 'Explicit effort constraint for policy-v2; preserved and validated, never silently replaced.' },
33
+ family: { type: 'string', enum: ['claude', 'codex'], description: 'Magpie automatic capability family; required when no model is explicitly supplied.' },
34
+ path: { type: 'string', enum: ['ordinary', 'fork', 'teammate'], description: 'policy-v2 creation path; fork remains unsupported until real history compatibility is verified.' },
30
35
  },
31
36
  output: {
32
37
  schema: {
@@ -34,6 +39,10 @@ export function routeSubagentTool(jev, deps) {
34
39
  additionalProperties: false,
35
40
  properties: {
36
41
  routed: { type: 'boolean' },
42
+ status: { type: 'string' },
43
+ policyVersion: { type: 'string' },
44
+ capability: { type: 'string' },
45
+ depth: { type: 'string' },
37
46
  reason: { type: 'string' },
38
47
  tier: { type: 'string' },
39
48
  provider: { type: 'string' },
@@ -41,9 +50,12 @@ export function routeSubagentTool(jev, deps) {
41
50
  reasoning_effort: { type: 'string' },
42
51
  },
43
52
  },
44
- render: (_args, v) => [{ type: 'text', text: v.routed === false
45
- ? MANUAL_TEXT
46
- : `Delegate with provider=${v.provider} model=${v.model} reasoning_effort=${v.reasoning_effort} (tier ${v.tier}); label the description [${v.model}·${v.reasoning_effort}].` }],
53
+ render: (_args, v) => [{ type: 'text', text: v.status ? (v.status === 'recommended'
54
+ ? `policy-v2 recommended only (not applied): ${v.provider}/${v.model}/${v.reasoning_effort}. Use the official creation tool; AUTO judges and binds the exact child before its first request. Do not copy this recommendation as a new explicit user constraint. Query the child routing status for applied evidence.`
55
+ : `policy-v2 ${v.status}: ${v.reason}. 暂停委派并询问用户;不将推荐冒充实际应用。`)
56
+ : v.routed === false
57
+ ? MANUAL_TEXT
58
+ : `Delegate with provider=${v.provider} model=${v.model} reasoning_effort=${v.reasoning_effort} (tier ${v.tier}); label the description [${v.model}·${v.reasoning_effort}].` }],
47
59
  },
48
60
  execute: async (args, exec) => {
49
61
  const caller = exec.agent;
@@ -51,6 +63,21 @@ export function routeSubagentTool(jev, deps) {
51
63
  // MANUAL (or disabled, or an unreachable root): no judgement and nothing leaves the machine.
52
64
  if (!root || deps.state.read(root.id).mode !== 'AUTO')
53
65
  return { routed: false, reason: 'manual' };
66
+ if (deps.readSettings?.().policy === 'policy-v2') {
67
+ if (args.path === 'fork')
68
+ return { routed: false, status: 'unsupported', policyVersion, reason: 'fork 真实历史兼容尚未验证;选择 fresh 或由用户关闭试验后继承' };
69
+ const explicit = { ...(args.model ? { model: args.model } : {}), ...(args.reasoning_effort ? { reasoningEffort: args.reasoning_effort } : {}) };
70
+ const result = explicit.model && explicit.reasoningEffort ? { kind: 'ok', depth: 'baseline', capability: 'bounded' }
71
+ : await jev.judgePolicy(buildState({ text: args.task }), exec.signal, 'child', explicit.model ? 'depth' : explicit.reasoningEffort ? 'capability' : 'both');
72
+ if (result.kind !== 'ok')
73
+ throw new Error('policy-v2 Jev 判断失败或取消;暂停委派并询问用户');
74
+ if (result.capability === 'insufficient_context')
75
+ return { routed: false, status: 'needs-context', policyVersion, reason: '请补充任务目标与范围' };
76
+ const route = choosePolicyRoute(args.provider, result.capability ?? 'bounded', result.depth, await deps.catalog(), deps.readSettings().effortCap, explicit, args.family ?? (args.model ? modelFamily(args.provider, args.model) : undefined));
77
+ return { status: 'recommended', policyVersion, ...(result.capability ? { capability: result.capability } : {}), depth: result.depth, provider: route.provider, model: route.model, reasoning_effort: route.reasoningEffort };
78
+ }
79
+ if (args.provider === 'dsh-magpie-connect')
80
+ throw new Error('Magpie 路由需显式选择 policy-v2;原策略不变');
54
81
  const result = await jev.judge(buildState({ text: args.task }), exec.signal);
55
82
  if (result.kind === 'cancelled')
56
83
  throw new Error('route_subagent 已取消。');
@@ -0,0 +1,59 @@
1
+ type Failure = 'decode-failed' | 'tail-incomplete' | 'parse-failed' | 'missing-log';
2
+ export declare function buildReport(home: string, options?: {
3
+ since?: string;
4
+ }): {
5
+ schema: number;
6
+ since: string | null;
7
+ failures: Record<Failure, number>;
8
+ excluded: {
9
+ child: number;
10
+ seeded: number;
11
+ };
12
+ coverage: {
13
+ userTurns: number;
14
+ turnsWithoutRecord: number;
15
+ unlinkedBindings: number;
16
+ autoSessionsWithoutRecord: number;
17
+ };
18
+ groups: {
19
+ version: string;
20
+ subgroups: {
21
+ model: string;
22
+ depth: string;
23
+ decisions: number;
24
+ sessions: number;
25
+ calls: number;
26
+ callsPerDecision: {
27
+ median: number;
28
+ max: number;
29
+ };
30
+ tokens: Record<string, {
31
+ sum: number | null;
32
+ mean: number | null;
33
+ present: number;
34
+ calls: number;
35
+ }>;
36
+ byEffort: {
37
+ [k: string]: {
38
+ decisions: number;
39
+ calls: number;
40
+ };
41
+ };
42
+ feedback: Record<string, number>;
43
+ conclusion: string;
44
+ }[];
45
+ }[];
46
+ referenceBeforeAuto: {
47
+ model: string;
48
+ calls: number;
49
+ sessions: number;
50
+ effort: {
51
+ [k: string]: number;
52
+ };
53
+ autoEffort: {
54
+ [k: string]: number;
55
+ };
56
+ }[];
57
+ caveats: string[];
58
+ };
59
+ export {};
@@ -0,0 +1,332 @@
1
+ // Read-only, on-demand AUTO usage report (change auto-routing-usage-report).
2
+ // Never writes, spawns, or touches the network; never echoes message text, paths or session ids.
3
+ import { createHash } from 'node:crypto';
4
+ import { existsSync, readdirSync, readFileSync } from 'node:fs';
5
+ import { homedir } from 'node:os';
6
+ import { join } from 'node:path';
7
+ import { pathToFileURL } from 'node:url';
8
+ import { parseArgs } from 'node:util';
9
+ import { zstdDecompressSync } from 'node:zlib';
10
+ import { FeedbackBook } from './feedback.js';
11
+ const MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd]);
12
+ const FIELDS = ['inputTokens', 'outputTokens', 'cacheReadTokens', 'cacheWriteTokens', 'reasoningTokens'];
13
+ const FEEDBACK = ['appropriate', 'too-high', 'too-low', 'unknown'];
14
+ const CAVEATS = [
15
+ 'observational-not-controlled',
16
+ 'scope-main-session-assistant-calls-only',
17
+ 'only-applied-decisions-recorded',
18
+ 'token-semantics-differ-by-provider',
19
+ 'reference-before-auto-effort-only-confounded',
20
+ 'mode-history-not-persisted',
21
+ ];
22
+ const sorted = (record) => Object.fromEntries(Object.entries(record).sort(([a], [b]) => a.localeCompare(b)));
23
+ const round = (n) => Math.round(n * 100) / 100;
24
+ const bump = (record, key, by = 1) => { record[key] = (record[key] ?? 0) + by; };
25
+ /** Split concatenated zstd frames; only a failing last segment counts as an unfinished tail. */
26
+ function decode(bytes) {
27
+ const starts = [];
28
+ for (let i = bytes.indexOf(MAGIC); i !== -1; i = bytes.indexOf(MAGIC, i + 1))
29
+ starts.push(i);
30
+ if (starts[0] !== 0)
31
+ return undefined;
32
+ let text = '';
33
+ for (let i = 0; i < starts.length; i++) {
34
+ try {
35
+ text += zstdDecompressSync(bytes.subarray(starts[i], starts[i + 1])).toString('utf8');
36
+ }
37
+ catch {
38
+ if (i === starts.length - 1)
39
+ return { text, tail: true };
40
+ return undefined;
41
+ }
42
+ }
43
+ return { text, tail: false };
44
+ }
45
+ function lines(file, failures) {
46
+ let text;
47
+ try {
48
+ text = readFileSync(file, 'utf8');
49
+ }
50
+ catch {
51
+ return [];
52
+ }
53
+ const out = [];
54
+ for (const line of text.split('\n')) {
55
+ if (!line.trim())
56
+ continue;
57
+ try {
58
+ out.push(JSON.parse(line));
59
+ }
60
+ catch {
61
+ failures['parse-failed']++;
62
+ }
63
+ }
64
+ return out;
65
+ }
66
+ function logPaths(home) {
67
+ const paths = new Map();
68
+ const root = join(home, 'sessions');
69
+ let dirs = [];
70
+ try {
71
+ dirs = readdirSync(root);
72
+ }
73
+ catch {
74
+ return paths;
75
+ }
76
+ for (const dir of dirs) {
77
+ let ids = [];
78
+ try {
79
+ ids = readdirSync(join(root, dir));
80
+ }
81
+ catch {
82
+ continue;
83
+ }
84
+ for (const id of ids) {
85
+ const file = join(root, dir, id, 'session.v4.jsonl.zstd');
86
+ if (existsSync(file))
87
+ paths.set(id, file);
88
+ }
89
+ }
90
+ return paths;
91
+ }
92
+ /** Calls in one session log, with the user turn each belongs to and the config in effect. */
93
+ function readSession(file, failures) {
94
+ let decoded;
95
+ try {
96
+ decoded = decode(readFileSync(file));
97
+ }
98
+ catch {
99
+ decoded = undefined;
100
+ }
101
+ if (!decoded) {
102
+ failures['decode-failed']++;
103
+ return undefined;
104
+ }
105
+ if (decoded.tail)
106
+ failures['tail-incomplete']++;
107
+ const events = [];
108
+ for (const line of decoded.text.split('\n')) {
109
+ if (!line.trim())
110
+ continue;
111
+ try {
112
+ events.push(JSON.parse(line));
113
+ }
114
+ catch {
115
+ failures['parse-failed']++;
116
+ }
117
+ }
118
+ const header = events.shift();
119
+ if (header?.type !== 'session') {
120
+ failures['decode-failed']++;
121
+ return undefined;
122
+ }
123
+ const calls = [];
124
+ const seen = new Set();
125
+ let turn, userTurn;
126
+ let config = {};
127
+ for (const event of events) {
128
+ const data = event.data ?? {};
129
+ if (event.type === 'turn/start')
130
+ turn = data.turn;
131
+ else if (event.type === 'user/message' && data.source?.kind === 'user' && turn !== undefined)
132
+ userTurn = turn;
133
+ else if (event.type === 'request/header' && data.header?.config)
134
+ config = data.header.config;
135
+ else if (event.type === 'assistant/message') {
136
+ const op = event.surfaceOp;
137
+ const append = op === undefined || op === 'append' || (typeof op === 'object' && op?.op === 'append');
138
+ const id = data.message?.id;
139
+ if (!append || typeof id !== 'string' || seen.has(id))
140
+ continue;
141
+ seen.add(id);
142
+ const usage = {};
143
+ for (const field of FIELDS)
144
+ if (typeof data.usage?.[field] === 'number')
145
+ usage[field] = data.usage[field];
146
+ calls.push({ id, ...(userTurn === undefined ? {} : { userTurn }), time: event.time, usage, ...(config.model ? { model: config.model } : {}), ...(config.reasoningEffort ? { effort: config.reasoningEffort } : {}) });
147
+ }
148
+ }
149
+ return { header, calls };
150
+ }
151
+ function localMidnight(date) {
152
+ const [y, m, d] = date.split('-').map(Number);
153
+ const at = new Date(y, m - 1, d);
154
+ const offset = -at.getTimezoneOffset();
155
+ const sign = offset >= 0 ? '+' : '-';
156
+ const hh = String(Math.floor(Math.abs(offset) / 60)).padStart(2, '0'), mm = String(Math.abs(offset) % 60).padStart(2, '0');
157
+ return { ms: at.getTime(), label: `${date}T00:00:00${sign}${hh}:${mm}` };
158
+ }
159
+ function tokens(calls) {
160
+ const out = {};
161
+ for (const field of FIELDS) {
162
+ const values = calls.map(c => c.usage[field]).filter((v) => v !== undefined);
163
+ const sum = values.reduce((a, b) => a + b, 0);
164
+ out[field] = values.length ? { sum, mean: round(sum / values.length), present: values.length, calls: calls.length } : { sum: null, mean: null, present: 0, calls: calls.length };
165
+ }
166
+ return out;
167
+ }
168
+ function median(values) {
169
+ const s = [...values].sort((a, b) => a - b), mid = Math.floor(s.length / 2);
170
+ return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
171
+ }
172
+ export function buildReport(home, options = {}) {
173
+ const failures = { 'decode-failed': 0, 'tail-incomplete': 0, 'parse-failed': 0, 'missing-log': 0 };
174
+ const state = join(home, 'state/auto-model-routing');
175
+ const since = options.since ? localMidnight(options.since) : undefined;
176
+ const after = (time) => !since || time >= since.ms;
177
+ const feedbackFile = join(state, 'allocation-feedback.jsonl');
178
+ lines(feedbackFile, failures); // counts unparsable lines; FeedbackBook skips them silently
179
+ const book = new FeedbackBook(feedbackFile);
180
+ const bindings = book.entries();
181
+ const legacyFirst = new Map();
182
+ for (const line of lines(join(state, 'judgements.jsonl'), failures)) {
183
+ if (typeof line?.sessionId !== 'string' || !Number.isInteger(line.turn) || typeof line.tier !== 'string')
184
+ continue;
185
+ legacyFirst.set(line.sessionId, Math.min(legacyFirst.get(line.sessionId) ?? Infinity, line.turn));
186
+ }
187
+ const ledgerSessions = [...new Set([...bindings.map(b => b.sessionId), ...legacyFirst.keys()])].sort();
188
+ const paths = logPaths(home);
189
+ const excluded = { child: 0, seeded: 0 };
190
+ const decisions = new Map();
191
+ const reference = {};
192
+ const autoEffort = {};
193
+ let unlinkedBindings = 0, userTurns = 0, turnsWithoutRecord = 0;
194
+ for (const session of ledgerSessions) {
195
+ const path = paths.get(session);
196
+ if (!path) {
197
+ failures['missing-log']++;
198
+ continue;
199
+ }
200
+ const log = readSession(path, failures);
201
+ if (!log)
202
+ continue;
203
+ if (log.header.origin) {
204
+ excluded.child++;
205
+ continue;
206
+ }
207
+ if (log.header.isSeeded === true) {
208
+ excluded.seeded++;
209
+ continue;
210
+ }
211
+ const linked = new Set();
212
+ let v2First;
213
+ for (const call of log.calls) {
214
+ const b = book.lookup(session, call.id);
215
+ if (!b)
216
+ continue;
217
+ linked.add(call.id);
218
+ if (call.userTurn !== undefined)
219
+ v2First = Math.min(v2First ?? Infinity, call.userTurn);
220
+ const key = JSON.stringify([session, b.decisionId]);
221
+ let d = decisions.get(key);
222
+ if (!d) {
223
+ d = { session, version: b.snapshot ? `${b.snapshot.policyVersion}/${b.snapshot.mappingVersion}` : 'unknown', model: b.model, depth: b.depth, effort: b.effort, calls: [], feedback: new Set() };
224
+ decisions.set(key, d);
225
+ }
226
+ d.calls.push(call);
227
+ if (b.feedback)
228
+ d.feedback.add(b.feedback);
229
+ }
230
+ unlinkedBindings += bindings.filter(b => b.sessionId === session && !linked.has(b.messageId)).length;
231
+ // Coverage: user turns from the first v2 record on, with calls in range but none linked.
232
+ if (v2First !== undefined) {
233
+ const turns = new Map();
234
+ for (const call of log.calls) {
235
+ if (call.userTurn === undefined || call.userTurn < v2First || !after(call.time))
236
+ continue;
237
+ turns.set(call.userTurn, (turns.get(call.userTurn) ?? false) || linked.has(call.id));
238
+ }
239
+ userTurns += turns.size;
240
+ turnsWithoutRecord += [...turns.values()].filter(v => !v).length;
241
+ }
242
+ // Reference: calls before the earliest record in either ledger (effort only, never tokens).
243
+ const first = Math.min(legacyFirst.get(session) ?? Infinity, v2First ?? Infinity);
244
+ for (const call of log.calls) {
245
+ if (call.userTurn === undefined || call.userTurn >= first || !after(call.time) || !call.model)
246
+ continue;
247
+ const r = (reference[call.model] ??= { calls: 0, sessions: new Set(), effort: {} });
248
+ r.calls++;
249
+ r.sessions.add(session);
250
+ bump(r.effort, call.effort ?? 'unknown');
251
+ }
252
+ }
253
+ const kept = [...decisions.values()].filter(d => after(Math.min(...d.calls.map(c => c.time))));
254
+ for (const d of kept)
255
+ bump((autoEffort[d.model] ??= {}), d.effort, d.calls.length);
256
+ const versions = new Map();
257
+ for (const d of kept) {
258
+ const byVersion = versions.get(d.version) ?? new Map();
259
+ versions.set(d.version, byVersion);
260
+ const key = JSON.stringify([d.model, d.depth]);
261
+ byVersion.set(key, [...(byVersion.get(key) ?? []), d]);
262
+ }
263
+ const groups = [...versions].sort(([a], [b]) => a.localeCompare(b)).map(([version, subs]) => ({
264
+ version,
265
+ subgroups: [...subs].sort(([a], [b]) => a.localeCompare(b)).map(([key, list]) => {
266
+ const [model, depth] = JSON.parse(key);
267
+ const calls = list.flatMap(d => d.calls);
268
+ const sessions = new Set(list.map(d => d.session)).size;
269
+ const feedback = { ...Object.fromEntries(FEEDBACK.map(f => [f, 0])), conflict: 0, unrated: 0 };
270
+ for (const d of list)
271
+ bump(feedback, d.feedback.size === 0 ? 'unrated' : d.feedback.size > 1 ? 'conflict' : [...d.feedback][0]);
272
+ const byEffort = {};
273
+ for (const d of list) {
274
+ const e = (byEffort[d.effort] ??= { decisions: 0, calls: 0 });
275
+ e.decisions++;
276
+ e.calls += d.calls.length;
277
+ }
278
+ return {
279
+ model, depth, decisions: list.length, sessions, calls: calls.length,
280
+ callsPerDecision: { median: median(list.map(d => d.calls.length)), max: Math.max(...list.map(d => d.calls.length)) },
281
+ tokens: tokens(calls), byEffort: sorted(byEffort), feedback,
282
+ conclusion: list.length >= 20 && sessions >= 3 ? 'observed' : 'insufficient-sample',
283
+ };
284
+ }),
285
+ }));
286
+ const recorded = new Set(ledgerSessions.map(s => createHash('sha256').update(s).digest('hex')));
287
+ let autoSessionsWithoutRecord = 0;
288
+ let stateFiles = [];
289
+ try {
290
+ stateFiles = readdirSync(join(state, 'sessions')).filter(f => f.endsWith('.json')).sort();
291
+ }
292
+ catch { /* none */ }
293
+ for (const file of stateFiles) {
294
+ let mode;
295
+ try {
296
+ mode = JSON.parse(readFileSync(join(state, 'sessions', file), 'utf8'))?.mode;
297
+ }
298
+ catch {
299
+ failures['parse-failed']++;
300
+ continue;
301
+ }
302
+ if (mode === 'AUTO' && !recorded.has(file.slice(0, -5)))
303
+ autoSessionsWithoutRecord++;
304
+ }
305
+ return {
306
+ schema: 1,
307
+ since: since?.label ?? null,
308
+ failures, excluded,
309
+ coverage: { userTurns, turnsWithoutRecord, unlinkedBindings, autoSessionsWithoutRecord },
310
+ groups,
311
+ referenceBeforeAuto: Object.keys(reference).sort().map(model => ({
312
+ model, calls: reference[model].calls, sessions: reference[model].sessions.size,
313
+ effort: sorted(reference[model].effort), autoEffort: sorted(autoEffort[model] ?? {}),
314
+ })),
315
+ caveats: CAVEATS,
316
+ };
317
+ }
318
+ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
319
+ try {
320
+ const { values } = parseArgs({ options: { since: { type: 'string' } } });
321
+ if (values.since !== undefined && !/^\d{4}-\d{2}-\d{2}$/.test(values.since)) {
322
+ process.stderr.write('usage-report: --since must be YYYY-MM-DD\n');
323
+ process.exit(2);
324
+ }
325
+ const home = process.env.DSH_HOME ?? join(homedir(), '.dsh');
326
+ process.stdout.write(`${JSON.stringify(buildReport(home, values.since ? { since: values.since } : {}), null, 2)}\n`);
327
+ }
328
+ catch {
329
+ process.stderr.write('usage-report: failed\n');
330
+ process.exit(1);
331
+ }
332
+ }
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "dsh-auto-model-routing",
3
- "version": "0.3.0",
4
- "description": "DeepSeek Harness plugin: use Jev to adjust reasoning effort per turn without changing the selected model, with configurable subagent routing",
3
+ "version": "0.4.1",
4
+ "description": "DeepSeek Harness plugin: independent Jev capability and effort decisions, enforced fresh-child routes, decision visibility and allocation feedback",
5
5
  "license": "MIT",
6
6
  "author": "prgrmrwy",
7
7
  "repository": { "type": "git", "url": "git+https://github.com/prgrmrwy/dsh-auto-model-routing.git" },
@@ -99,7 +99,7 @@
99
99
  "@deepseek-ai/dsh-credentials": "^0.2.0-rc.2",
100
100
  "@deepseek-ai/dsh-llm": "^0.2.0-rc.2",
101
101
  "@deepseek-ai/dsh-session-projection": "^0.2.0-rc.2",
102
- "react": "^19.0.0",
102
+ "react": "^18.3.1 || ^19.0.0",
103
103
  "@deepseek-ai/dsh-launch-environment": "^0.2.0-rc.2",
104
104
  "@deepseek-ai/dsh-skill-filesystem": "^0.2.0-rc.2",
105
105
  "@deepseek-ai/dsh-tools": "^0.2.0-rc.2"