dsh-auto-model-routing 0.4.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +8 -8
- package/README.zh.md +8 -8
- package/docs/feedback-evaluation.md +4 -4
- package/docs/host-capability-report.md +2 -0
- package/docs/jev-policy-v2-validation.md +3 -1
- package/docs/magpie-live-acceptance.md +2 -0
- package/docs/policy-v2-development.md +5 -3
- package/docs/policy-v2-real-evaluation-plan.md +2 -0
- package/lib/client/control.d.ts +2 -2
- package/lib/client/control.js +10 -20
- package/lib/client/index.d.ts +0 -1
- package/lib/client/index.js +9 -6
- package/lib/client/label.d.ts +7 -10
- package/lib/client/label.js +17 -30
- package/lib/client/settings-section.d.ts +2 -2
- package/lib/client/settings-section.js +7 -69
- package/lib/client/strategy-details.js +3 -4
- package/lib/client.js +165 -481
- package/lib/context.d.ts +1 -3
- package/lib/context.js +3 -4
- package/lib/effort.d.ts +2 -10
- package/lib/effort.js +1 -31
- package/lib/endpoints.d.ts +11 -10
- package/lib/endpoints.js +29 -46
- package/lib/feedback.d.ts +60 -7
- package/lib/feedback.js +15 -5
- package/lib/index.d.ts +4 -0
- package/lib/index.js +17 -86
- package/lib/jev.d.ts +17 -29
- package/lib/jev.js +6 -34
- package/lib/policy-host.d.ts +6 -8
- package/lib/policy-host.js +39 -38
- package/lib/policy-store.d.ts +9 -20
- package/lib/policy-store.js +28 -16
- package/lib/routing.d.ts +20 -0
- package/lib/routing.js +51 -0
- package/lib/rubric.json +15 -6
- package/lib/settings-contract.d.ts +5 -21
- package/lib/settings-contract.js +3 -2
- package/lib/settings-host.d.ts +4 -8
- package/lib/settings-host.js +9 -10
- package/lib/settings-validation.d.ts +1 -7
- package/lib/settings-validation.js +0 -51
- package/lib/settings.d.ts +3 -8
- package/lib/settings.js +18 -177
- package/lib/state.d.ts +2 -4
- package/lib/state.js +2 -7
- package/lib/strategies/claude.d.ts +21 -0
- package/lib/strategies/claude.js +8 -0
- package/lib/strategies/codex.d.ts +21 -0
- package/lib/strategies/codex.js +9 -0
- package/lib/strategies/index.d.ts +22 -0
- package/lib/strategies/index.js +85 -0
- package/lib/strategies/types.d.ts +14 -0
- package/lib/strategies/types.js +1 -0
- package/lib/subagent-tool.d.ts +7 -7
- package/lib/subagent-tool.js +33 -76
- package/lib/support.d.ts +7 -8
- package/lib/support.js +9 -13
- package/lib/usage-report.d.ts +2 -2
- package/lib/usage-report.js +25 -17
- package/package.json +3 -4
- package/skills/auto-model-routing-report/SKILL.md +4 -3
- package/skills/subagent-model-routing/SKILL.md +14 -30
- package/lib/client/route-picker.d.ts +0 -21
- package/lib/client/route-picker.js +0 -75
- package/lib/judgements.d.ts +0 -41
- package/lib/judgements.js +0 -55
- package/lib/policy-v2-rubric.json +0 -21
- package/lib/policy-v2.d.ts +0 -21
- package/lib/policy-v2.js +0 -54
- package/lib/router.d.ts +0 -68
- package/lib/router.js +0 -153
- package/scripts/label-report.mjs +0 -50
package/lib/context.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { Tier } from './effort.js';
|
|
2
1
|
export declare function redact(text: string): string;
|
|
3
2
|
interface HistoryMessage {
|
|
4
3
|
role: string;
|
|
@@ -15,9 +14,8 @@ export declare function previousTurn(messages: readonly HistoryMessage[]): {
|
|
|
15
14
|
user?: string;
|
|
16
15
|
assistant?: string;
|
|
17
16
|
};
|
|
18
|
-
/** The Jev `state
|
|
17
|
+
/** The Jev `state`; every section is redacted, then cut. The fixed first line keeps the request bytes stable. */
|
|
19
18
|
export declare function buildState(input: {
|
|
20
|
-
previous?: Tier | undefined;
|
|
21
19
|
prevUser?: string | undefined;
|
|
22
20
|
prevAssistant?: string | undefined;
|
|
23
21
|
text: string;
|
package/lib/context.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
/** Shape-based, best effort (design D4): applied to the whole text before any truncation. */
|
|
1
|
+
/** Shape-based, best effort: applied to the whole text before any truncation. */
|
|
3
2
|
const RULES = [
|
|
4
3
|
[/-----BEGIN [A-Z ]+-----[\s\S]*?-----END [A-Z ]+-----/g, '<redacted>'],
|
|
5
4
|
[/\bBearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, 'Bearer <redacted>'],
|
|
@@ -19,9 +18,9 @@ export function previousTurn(messages) {
|
|
|
19
18
|
return { ...(user ? { user: textOf(user) } : {}), ...(assistant ? { assistant: textOf(assistant) } : {}) };
|
|
20
19
|
}
|
|
21
20
|
const EXPLAIN = 'Earlier conversation is shown so you can tell what the new message asks for. Rate the work the assistant must do next in response to the new message, not the earlier work.';
|
|
22
|
-
/** The Jev `state
|
|
21
|
+
/** The Jev `state`; every section is redacted, then cut. The fixed first line keeps the request bytes stable. */
|
|
23
22
|
export function buildState(input) {
|
|
24
|
-
const lines = [
|
|
23
|
+
const lines = ['Previous user-turn tier: none'];
|
|
25
24
|
const context = input.prevUser !== undefined || input.prevAssistant !== undefined;
|
|
26
25
|
if (context)
|
|
27
26
|
lines.push(EXPLAIN);
|
package/lib/effort.d.ts
CHANGED
|
@@ -1,11 +1,3 @@
|
|
|
1
|
-
export type Tier = 'simple' | 'standard' | 'complex' | 'escalated';
|
|
2
1
|
export type Effort = 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
3
|
-
|
|
4
|
-
export
|
|
5
|
-
/** Target effort before clamping to what the model supports (also used for sub-agents, unclamped). */
|
|
6
|
-
export declare function targetEffort(model: string, tier: Tier, cap?: Effort): Effort;
|
|
7
|
-
/** Undefined when the model supports none of low..xhigh: the request is then left as is. */
|
|
8
|
-
export declare function mapEffort(model: string, tier: Tier, supported: readonly Effort[], cap?: Effort): Effort | undefined;
|
|
9
|
-
export type ChildProvider = 'claude' | 'codex';
|
|
10
|
-
/** Sub-agent models by tier (design D9); the single source for `route_subagent`. Effort comes from `targetEffort`. */
|
|
11
|
-
export declare const CHILD_MODELS: Readonly<Record<ChildProvider, readonly [simple: string, standard: string, complex: string, escalated: string]>>;
|
|
2
|
+
/** Closed set of the reasons the state panel shows for a main request. */
|
|
3
|
+
export type Reason = 'jev' | 'no-effort-support';
|
package/lib/effort.js
CHANGED
|
@@ -1,31 +1 @@
|
|
|
1
|
-
export
|
|
2
|
-
/** Official starting points (design D9; sources in eval/README.md). Unlisted models start at medium. */
|
|
3
|
-
const STARTING_POINTS = {
|
|
4
|
-
'claude-opus-5-5': 'medium',
|
|
5
|
-
'claude-sonnet-5-5': 'medium',
|
|
6
|
-
'claude-haiku-5-5': 'medium',
|
|
7
|
-
'gpt-6.1-sol': 'medium',
|
|
8
|
-
'claude-fable-5-1': 'high',
|
|
9
|
-
'gpt-6-luna': 'high',
|
|
10
|
-
'gpt-6-astra': 'low',
|
|
11
|
-
};
|
|
12
|
-
const OFFSET = { simple: -1, standard: 0, complex: 1, escalated: 2 };
|
|
13
|
-
/** Automatic routing never goes above xhigh: `max` stays a manual choice. */
|
|
14
|
-
const automatic = ['low', 'medium', 'high', 'xhigh'];
|
|
15
|
-
/** Target effort before clamping to what the model supports (also used for sub-agents, unclamped). */
|
|
16
|
-
export function targetEffort(model, tier, cap = 'xhigh') {
|
|
17
|
-
const start = automatic.indexOf(STARTING_POINTS[model] ?? 'medium');
|
|
18
|
-
const ceiling = cap === 'max' ? automatic.length - 1 : automatic.indexOf(cap);
|
|
19
|
-
return automatic[Math.max(0, Math.min(ceiling, start + OFFSET[tier]))];
|
|
20
|
-
}
|
|
21
|
-
/** Undefined when the model supports none of low..xhigh: the request is then left as is. */
|
|
22
|
-
export function mapEffort(model, tier, supported, cap = 'xhigh') {
|
|
23
|
-
const target = targetEffort(model, tier, cap);
|
|
24
|
-
const candidates = automatic.filter(effort => supported.includes(effort) && automatic.indexOf(effort) <= (cap === 'max' ? automatic.length - 1 : automatic.indexOf(cap)));
|
|
25
|
-
return candidates.filter(effort => automatic.indexOf(effort) <= automatic.indexOf(target)).at(-1) ?? candidates[0];
|
|
26
|
-
}
|
|
27
|
-
/** Sub-agent models by tier (design D9); the single source for `route_subagent`. Effort comes from `targetEffort`. */
|
|
28
|
-
export const CHILD_MODELS = {
|
|
29
|
-
claude: ['claude-haiku-5-5', 'claude-sonnet-5-5', 'claude-opus-5-5', 'claude-opus-5-5'],
|
|
30
|
-
codex: ['gpt-6-luna', 'gpt-6.1-sol', 'gpt-6.1-sol', 'gpt-6.1-sol'],
|
|
31
|
-
};
|
|
1
|
+
export {};
|
package/lib/endpoints.d.ts
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import { type FeedbackBook } from './feedback.js';
|
|
2
|
-
import type { AgentRef, Router } from './router.js';
|
|
3
2
|
import type { RoutingState } from './state.js';
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
export interface LabelBook {
|
|
9
|
-
lookup(sessionId: string, messageId: string): TurnRecord | undefined;
|
|
10
|
-
label(sessionId: string, turn: number, label: Tier): void;
|
|
3
|
+
/** Identity of a live session as the endpoints need it. */
|
|
4
|
+
export interface AgentRef {
|
|
5
|
+
id: string;
|
|
6
|
+
origin?: string;
|
|
11
7
|
}
|
|
12
|
-
|
|
13
|
-
|
|
8
|
+
type Resolve = (id: string) => Promise<AgentRef | undefined>;
|
|
9
|
+
/** Validates the session and applies `setMode`; the caller adds the policy host's view to the response. */
|
|
10
|
+
export declare function endpoint(operation: 'state' | 'setMode', body: unknown, resolveAgent: Resolve, state: RoutingState): Promise<Response>;
|
|
11
|
+
export declare function observeEndpoint(body: unknown, resolveAgent: Resolve, book: FeedbackBook, warn: () => void): Promise<Response>;
|
|
12
|
+
/** Query (no `feedback`) or change the allocation feedback of a reply. Unknown keys (e.g. a retired `label`) are rejected. */
|
|
13
|
+
export declare function labelEndpoint(body: unknown, resolveAgent: Resolve, book: FeedbackBook): Promise<Response>;
|
|
14
|
+
export {};
|
package/lib/endpoints.js
CHANGED
|
@@ -1,29 +1,29 @@
|
|
|
1
1
|
import { allocationFeedback } from './feedback.js';
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
const invalid = () => Response.json({ error: 'invalid request' }, { status: 400 });
|
|
3
|
+
const unknown = () => Response.json({ error: 'unknown ordinary session' }, { status: 404 });
|
|
4
|
+
/** Validates the session and applies `setMode`; the caller adds the policy host's view to the response. */
|
|
5
|
+
export async function endpoint(operation, body, resolveAgent, state) {
|
|
4
6
|
if (typeof body !== 'object' || body === null)
|
|
5
|
-
return
|
|
7
|
+
return invalid();
|
|
6
8
|
const { sessionId, mode } = body;
|
|
7
9
|
if (typeof sessionId !== 'string' || !sessionId || (operation === 'setMode' && mode !== 'AUTO' && mode !== 'MANUAL'))
|
|
8
|
-
return
|
|
10
|
+
return invalid();
|
|
9
11
|
const agent = await resolveAgent(sessionId);
|
|
10
12
|
if (!agent || agent.origin === 'subagent')
|
|
11
|
-
return
|
|
13
|
+
return unknown();
|
|
12
14
|
if (operation === 'setMode')
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
const decision = router.judgement(sessionId);
|
|
16
|
-
return Response.json({ ...state.read(sessionId), ...router.status(agent), ...(decision ? { decision } : {}), ...(toastOn() ? { decisionToast: true } : {}) });
|
|
15
|
+
state.setMode(sessionId, mode);
|
|
16
|
+
return Response.json(state.read(sessionId));
|
|
17
17
|
}
|
|
18
18
|
export async function observeEndpoint(body, resolveAgent, book, warn) {
|
|
19
19
|
if (typeof body !== 'object' || body === null)
|
|
20
|
-
return
|
|
20
|
+
return invalid();
|
|
21
21
|
const { sessionId, messageId } = body;
|
|
22
22
|
if (typeof sessionId !== 'string' || !sessionId || typeof messageId !== 'string' || !messageId)
|
|
23
|
-
return
|
|
23
|
+
return invalid();
|
|
24
24
|
const agent = await resolveAgent(sessionId);
|
|
25
25
|
if (!agent || agent.origin === 'subagent')
|
|
26
|
-
return
|
|
26
|
+
return unknown();
|
|
27
27
|
const binding = book.lookup(sessionId, messageId);
|
|
28
28
|
if (binding)
|
|
29
29
|
try {
|
|
@@ -34,48 +34,31 @@ export async function observeEndpoint(body, resolveAgent, book, warn) {
|
|
|
34
34
|
}
|
|
35
35
|
return Response.json({});
|
|
36
36
|
}
|
|
37
|
-
/** Query (no `
|
|
38
|
-
export async function labelEndpoint(body, resolveAgent, book
|
|
39
|
-
if (typeof body !== 'object' || body === null)
|
|
40
|
-
return
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
37
|
+
/** Query (no `feedback`) or change the allocation feedback of a reply. Unknown keys (e.g. a retired `label`) are rejected. */
|
|
38
|
+
export async function labelEndpoint(body, resolveAgent, book) {
|
|
39
|
+
if (typeof body !== 'object' || body === null || Array.isArray(body))
|
|
40
|
+
return invalid();
|
|
41
|
+
if (Object.keys(body).some(key => !['sessionId', 'messageId', 'feedback'].includes(key)))
|
|
42
|
+
return invalid();
|
|
43
|
+
const { sessionId, messageId } = body;
|
|
44
|
+
if (typeof sessionId !== 'string' || !sessionId || typeof messageId !== 'string')
|
|
45
|
+
return invalid();
|
|
44
46
|
const agent = await resolveAgent(sessionId);
|
|
45
47
|
if (!agent || agent.origin === 'subagent')
|
|
46
|
-
return
|
|
48
|
+
return unknown();
|
|
47
49
|
const hasFeedback = Object.hasOwn(body, 'feedback');
|
|
48
50
|
const feedback = body.feedback;
|
|
49
|
-
if (hasFeedback &&
|
|
51
|
+
if (hasFeedback && feedback !== null && !allocationFeedback.includes(feedback))
|
|
50
52
|
return Response.json({ error: 'invalid feedback' }, { status: 400 });
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
return Response.json({ error: 'use allocation feedback' }, { status: 400 });
|
|
55
|
-
if (hasFeedback) {
|
|
56
|
-
try {
|
|
57
|
-
feedbackBook.set(sessionId, messageId, feedback);
|
|
58
|
-
}
|
|
59
|
-
catch {
|
|
60
|
-
return Response.json({ error: 'could not record feedback' }, { status: 500 });
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
return Response.json({ policy: 'policy-v2', ...feedbackBook.lookup(sessionId, messageId) });
|
|
64
|
-
}
|
|
65
|
-
if (hasFeedback)
|
|
66
|
-
return Response.json({ error: 'no applied decision' }, { status: 404 });
|
|
67
|
-
const record = book.lookup(sessionId, messageId);
|
|
68
|
-
if (!record)
|
|
69
|
-
return Response.json({});
|
|
70
|
-
const { turn, ...view } = record;
|
|
71
|
-
if (label !== undefined) {
|
|
53
|
+
if (!book.lookup(sessionId, messageId))
|
|
54
|
+
return hasFeedback ? Response.json({ error: 'no applied decision' }, { status: 404 }) : Response.json({});
|
|
55
|
+
if (hasFeedback) {
|
|
72
56
|
try {
|
|
73
|
-
book.
|
|
57
|
+
book.set(sessionId, messageId, feedback);
|
|
74
58
|
}
|
|
75
59
|
catch {
|
|
76
|
-
return Response.json({ error: 'could not record
|
|
60
|
+
return Response.json({ error: 'could not record feedback' }, { status: 500 });
|
|
77
61
|
}
|
|
78
|
-
view.label = label;
|
|
79
62
|
}
|
|
80
|
-
return Response.json(view);
|
|
63
|
+
return Response.json(book.view(sessionId, messageId));
|
|
81
64
|
}
|
package/lib/feedback.d.ts
CHANGED
|
@@ -10,8 +10,9 @@ declare const schema: z.ZodObject<{
|
|
|
10
10
|
snapshot: z.ZodOptional<z.ZodObject<{
|
|
11
11
|
decidedAt: z.ZodNumber;
|
|
12
12
|
appliedAt: z.ZodNumber;
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
strategy: z.ZodOptional<z.ZodString>;
|
|
14
|
+
policyVersion: z.ZodOptional<z.ZodString>;
|
|
15
|
+
mappingVersion: z.ZodOptional<z.ZodString>;
|
|
15
16
|
decisionProvider: z.ZodString;
|
|
16
17
|
decisionModel: z.ZodString;
|
|
17
18
|
ms: z.ZodNumber;
|
|
@@ -32,7 +33,7 @@ declare const schema: z.ZodObject<{
|
|
|
32
33
|
}, z.core.$strict>;
|
|
33
34
|
declare const factEvent: z.ZodObject<{
|
|
34
35
|
type: z.ZodLiteral<"fact">;
|
|
35
|
-
version: z.ZodLiteral<1
|
|
36
|
+
version: z.ZodOptional<z.ZodLiteral<1>>;
|
|
36
37
|
sessionId: z.ZodString;
|
|
37
38
|
decisionId: z.ZodNumber;
|
|
38
39
|
fact: z.ZodEnum<{
|
|
@@ -60,13 +61,14 @@ export declare class FeedbackBook {
|
|
|
60
61
|
snapshot?: {
|
|
61
62
|
decidedAt: number;
|
|
62
63
|
appliedAt: number;
|
|
63
|
-
policyVersion: string;
|
|
64
|
-
mappingVersion: string;
|
|
65
64
|
decisionProvider: string;
|
|
66
65
|
decisionModel: string;
|
|
67
66
|
ms: number;
|
|
68
67
|
provider: string;
|
|
69
68
|
requestSeq: number;
|
|
69
|
+
strategy?: string | undefined;
|
|
70
|
+
policyVersion?: string | undefined;
|
|
71
|
+
mappingVersion?: string | undefined;
|
|
70
72
|
confidence?: number | undefined;
|
|
71
73
|
probabilities?: Record<string, number> | undefined;
|
|
72
74
|
requestTurn?: number | undefined;
|
|
@@ -75,6 +77,56 @@ export declare class FeedbackBook {
|
|
|
75
77
|
feedback?: "appropriate" | "too-high" | "too-low" | "unknown" | null | undefined;
|
|
76
78
|
feedbackAt?: number | undefined;
|
|
77
79
|
} | undefined;
|
|
80
|
+
/** The binding as the `label` endpoint returns it: historical snapshot keys are left out, nothing is inferred. */
|
|
81
|
+
view(sessionId: string, messageId: string): {
|
|
82
|
+
sessionId: string;
|
|
83
|
+
messageId: string;
|
|
84
|
+
decisionId: number;
|
|
85
|
+
depth: string;
|
|
86
|
+
model: string;
|
|
87
|
+
effort: string;
|
|
88
|
+
snapshot?: {
|
|
89
|
+
decidedAt: number;
|
|
90
|
+
appliedAt: number;
|
|
91
|
+
decisionProvider: string;
|
|
92
|
+
decisionModel: string;
|
|
93
|
+
ms: number;
|
|
94
|
+
provider: string;
|
|
95
|
+
requestSeq: number;
|
|
96
|
+
strategy?: string | undefined;
|
|
97
|
+
policyVersion?: string | undefined;
|
|
98
|
+
mappingVersion?: string | undefined;
|
|
99
|
+
confidence?: number | undefined;
|
|
100
|
+
probabilities?: Record<string, number> | undefined;
|
|
101
|
+
requestTurn?: number | undefined;
|
|
102
|
+
} | undefined;
|
|
103
|
+
boundAt?: number | undefined;
|
|
104
|
+
feedback?: "appropriate" | "too-high" | "too-low" | "unknown" | null | undefined;
|
|
105
|
+
feedbackAt?: number | undefined;
|
|
106
|
+
} | {
|
|
107
|
+
snapshot: {
|
|
108
|
+
decidedAt: number;
|
|
109
|
+
appliedAt: number;
|
|
110
|
+
decisionProvider: string;
|
|
111
|
+
decisionModel: string;
|
|
112
|
+
ms: number;
|
|
113
|
+
provider: string;
|
|
114
|
+
requestSeq: number;
|
|
115
|
+
strategy?: string | undefined;
|
|
116
|
+
confidence?: number | undefined;
|
|
117
|
+
probabilities?: Record<string, number> | undefined;
|
|
118
|
+
requestTurn?: number | undefined;
|
|
119
|
+
};
|
|
120
|
+
sessionId: string;
|
|
121
|
+
messageId: string;
|
|
122
|
+
decisionId: number;
|
|
123
|
+
depth: string;
|
|
124
|
+
model: string;
|
|
125
|
+
effort: string;
|
|
126
|
+
boundAt?: number | undefined;
|
|
127
|
+
feedback?: "appropriate" | "too-high" | "too-low" | "unknown" | null | undefined;
|
|
128
|
+
feedbackAt?: number | undefined;
|
|
129
|
+
} | undefined;
|
|
78
130
|
/** All folded bindings in file order (read-only; used by the usage report). */
|
|
79
131
|
entries(): {
|
|
80
132
|
sessionId: string;
|
|
@@ -86,13 +138,14 @@ export declare class FeedbackBook {
|
|
|
86
138
|
snapshot?: {
|
|
87
139
|
decidedAt: number;
|
|
88
140
|
appliedAt: number;
|
|
89
|
-
policyVersion: string;
|
|
90
|
-
mappingVersion: string;
|
|
91
141
|
decisionProvider: string;
|
|
92
142
|
decisionModel: string;
|
|
93
143
|
ms: number;
|
|
94
144
|
provider: string;
|
|
95
145
|
requestSeq: number;
|
|
146
|
+
strategy?: string | undefined;
|
|
147
|
+
policyVersion?: string | undefined;
|
|
148
|
+
mappingVersion?: string | undefined;
|
|
96
149
|
confidence?: number | undefined;
|
|
97
150
|
probabilities?: Record<string, number> | undefined;
|
|
98
151
|
requestTurn?: number | undefined;
|
package/lib/feedback.js
CHANGED
|
@@ -3,15 +3,17 @@ import { dirname } from 'node:path';
|
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
export const allocationFeedback = ['appropriate', 'too-high', 'too-low', 'unknown'];
|
|
5
5
|
const probability = z.number().min(0).max(1);
|
|
6
|
+
// `policyVersion`/`mappingVersion` exist only on bindings written before 0.5.0; new ones carry `strategy`.
|
|
6
7
|
const snapshot = z.object({
|
|
7
|
-
decidedAt: z.number(), appliedAt: z.number(), policyVersion: z.string(), mappingVersion: z.string(),
|
|
8
|
+
decidedAt: z.number(), appliedAt: z.number(), strategy: z.string().optional(), policyVersion: z.string().optional(), mappingVersion: z.string().optional(),
|
|
8
9
|
decisionProvider: z.string(), decisionModel: z.string(), ms: z.number(),
|
|
9
10
|
confidence: probability.optional(), probabilities: z.record(z.string(), probability).optional(),
|
|
10
11
|
provider: z.string(), requestSeq: z.number(), requestTurn: z.number().optional(),
|
|
11
12
|
}).strict();
|
|
12
13
|
const schema = z.object({ sessionId: z.string(), messageId: z.string(), decisionId: z.number(), depth: z.string(), model: z.string(), effort: z.string(), snapshot: snapshot.optional(), boundAt: z.number().optional(), feedback: z.enum(allocationFeedback).nullable().optional(), feedbackAt: z.number().optional() }).strict();
|
|
13
|
-
|
|
14
|
-
const
|
|
14
|
+
// Older events carried `version` (feedback 2, fact 1); new events have none.
|
|
15
|
+
const feedbackEvent = z.object({ type: z.literal('feedback'), version: z.literal(2).optional(), sessionId: z.string(), messageId: z.string(), decisionId: z.number(), at: z.number(), feedback: z.enum(allocationFeedback).nullable() }).strict();
|
|
16
|
+
const factEvent = z.object({ type: z.literal('fact'), version: z.literal(1).optional(), sessionId: z.string().min(1), decisionId: z.number(), fact: z.enum(['exposed', 'manual-switch']), at: z.number().finite() }).strict();
|
|
15
17
|
/** Immutable decision bindings and separate feedback events; never includes task text or credentials. */
|
|
16
18
|
export class FeedbackBook {
|
|
17
19
|
file;
|
|
@@ -62,6 +64,14 @@ export class FeedbackBook {
|
|
|
62
64
|
}
|
|
63
65
|
key(sessionId, messageId) { return JSON.stringify([sessionId, messageId]); }
|
|
64
66
|
lookup(sessionId, messageId) { const r = this.records.get(this.key(sessionId, messageId)); return r && structuredClone(r); }
|
|
67
|
+
/** The binding as the `label` endpoint returns it: historical snapshot keys are left out, nothing is inferred. */
|
|
68
|
+
view(sessionId, messageId) {
|
|
69
|
+
const r = this.lookup(sessionId, messageId);
|
|
70
|
+
if (!r?.snapshot)
|
|
71
|
+
return r;
|
|
72
|
+
const { policyVersion: _p, mappingVersion: _m, ...snapshot } = r.snapshot;
|
|
73
|
+
return { ...r, snapshot };
|
|
74
|
+
}
|
|
65
75
|
/** All folded bindings in file order (read-only; used by the usage report). */
|
|
66
76
|
entries() { return [...this.records.values()].map(r => structuredClone(r)); }
|
|
67
77
|
bind(record) { if (this.lookup(record.sessionId, record.messageId))
|
|
@@ -70,7 +80,7 @@ export class FeedbackBook {
|
|
|
70
80
|
const r = this.lookup(sessionId, messageId);
|
|
71
81
|
if (!r)
|
|
72
82
|
throw Error('No applied decision');
|
|
73
|
-
const event = feedbackEvent.parse({ type: 'feedback',
|
|
83
|
+
const event = feedbackEvent.parse({ type: 'feedback', sessionId, messageId, decisionId: r.decisionId, at: Date.now(), feedback });
|
|
74
84
|
this.append(event);
|
|
75
85
|
this.records.set(this.key(sessionId, messageId), { ...r, feedback, feedbackAt: event.at });
|
|
76
86
|
}
|
|
@@ -80,7 +90,7 @@ export class FeedbackBook {
|
|
|
80
90
|
const facts = this.factRecords.get(key) ?? {};
|
|
81
91
|
if (facts[fact] !== undefined)
|
|
82
92
|
return;
|
|
83
|
-
const event = factEvent.parse({ type: 'fact',
|
|
93
|
+
const event = factEvent.parse({ type: 'fact', sessionId, decisionId, fact, at: Date.now() });
|
|
84
94
|
this.append(event);
|
|
85
95
|
this.factRecords.set(key, { ...facts, [fact]: event.at });
|
|
86
96
|
}
|
package/lib/index.d.ts
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import type { Context } from '@deepseek-ai/cordis';
|
|
2
|
+
import { type FamilyStrategy } from './strategies/index.js';
|
|
2
3
|
export declare const name = "dsh-auto-model-routing";
|
|
4
|
+
/** Child decision records. The historical directory name is kept on purpose: records are not migrated. */
|
|
5
|
+
export declare const CHILD_DECISIONS_DIR = "state/auto-model-routing/policy-v2";
|
|
3
6
|
export declare const inject: string[];
|
|
4
7
|
/** Test seams only; production uses the defaults. */
|
|
5
8
|
export interface Config {
|
|
6
9
|
home?: string;
|
|
7
10
|
jevBase?: string;
|
|
8
11
|
jevTimeoutMs?: number;
|
|
12
|
+
strategies?: readonly FamilyStrategy[];
|
|
9
13
|
}
|
|
10
14
|
export declare function apply(ctx: Context, config?: Config): void;
|
package/lib/index.js
CHANGED
|
@@ -3,54 +3,41 @@ import { SettingsStore } from './settings.js';
|
|
|
3
3
|
import { settingsHost } from './settings-host.js';
|
|
4
4
|
import { homedir } from 'node:os';
|
|
5
5
|
import { FileStateStore, RoutingState } from './state.js';
|
|
6
|
-
import { Router } from './router.js';
|
|
7
6
|
import { endpoint, labelEndpoint, observeEndpoint } from './endpoints.js';
|
|
8
7
|
import { FeedbackBook } from './feedback.js';
|
|
9
|
-
import {
|
|
10
|
-
import { userInput } from './input.js';
|
|
11
|
-
import { JevClient, rubricVersion } from './jev.js';
|
|
8
|
+
import { JevClient } from './jev.js';
|
|
12
9
|
import { routeSubagentTool } from './subagent-tool.js';
|
|
13
|
-
import { buildState, previousTurn } from './context.js';
|
|
14
10
|
import { policyHost } from './policy-host.js';
|
|
15
11
|
import { PolicyStore } from './policy-store.js';
|
|
16
12
|
import { currentMainSelection, mainSupported } from './support.js';
|
|
13
|
+
import { createRegistry, registry as builtin } from './strategies/index.js';
|
|
17
14
|
export const name = 'dsh-auto-model-routing';
|
|
15
|
+
/** Child decision records. The historical directory name is kept on purpose: records are not migrated. */
|
|
16
|
+
export const CHILD_DECISIONS_DIR = 'state/auto-model-routing/policy-v2';
|
|
18
17
|
export const inject = ['llm', 'agents', 'tools'];
|
|
19
|
-
const knownEfforts = new Set(['low', 'medium', 'high', 'xhigh', 'max']);
|
|
20
18
|
export function apply(ctx, config = {}) {
|
|
21
19
|
const home = config.home ?? process.env.DSH_HOME ?? resolve(homedir(), '.dsh');
|
|
20
|
+
const registry = config.strategies ? createRegistry(config.strategies) : builtin;
|
|
22
21
|
const settings = new SettingsStore(resolve(home, 'plugins/dsh-auto-model-routing/config.json'));
|
|
23
|
-
const hostSettings = settingsHost(ctx, settings);
|
|
22
|
+
const hostSettings = settingsHost(ctx, settings, registry);
|
|
24
23
|
const jev = new JevClient({ get: hostSettings.credential }, config.jevBase, config.jevTimeoutMs ?? (() => settings.read().settings.jevTimeoutMs), config.jevBase ? undefined : () => settings.read().settings.decisionService);
|
|
25
24
|
const stateFiles = new FileStateStore(resolve(home, 'state/auto-model-routing/sessions'));
|
|
26
25
|
const state = new RoutingState(stateFiles);
|
|
27
|
-
const book = new JudgementBook(resolve(home, 'state/auto-model-routing/judgements.jsonl'));
|
|
28
|
-
const router = new Router(state, (sessionId, turn, tier, model, effort) => {
|
|
29
|
-
const line = { sessionId, turn, tier, ...(effort ? { effort } : {}), model, rubricVersion };
|
|
30
|
-
book.judged(line);
|
|
31
|
-
// Recording is best effort: a failed append never touches the request or the mode.
|
|
32
|
-
try {
|
|
33
|
-
book.append(line);
|
|
34
|
-
}
|
|
35
|
-
catch (error) {
|
|
36
|
-
ctx.logger.warn(`auto-model-routing: could not record the judgement: ${String(error)}`);
|
|
37
|
-
}
|
|
38
|
-
}, () => settings.read().settings);
|
|
39
26
|
const feedback = new FeedbackBook(resolve(home, 'state/auto-model-routing/allocation-feedback.jsonl'));
|
|
40
27
|
// Official writer is available only while the session controller is mounted.
|
|
41
28
|
let selectModel;
|
|
42
29
|
const policy = policyHost(ctx, { feedback, jev, state, selectModel: async (request) => { if (!selectModel)
|
|
43
|
-
throw new Error('session controller is unavailable'); await selectModel(request); }, store: new PolicyStore(resolve(home,
|
|
30
|
+
throw new Error('session controller is unavailable'); await selectModel(request); }, store: new PolicyStore(resolve(home, CHILD_DECISIONS_DIR), registry), settings: () => settings.read().settings, catalog: () => hostSettings.catalog(), registry });
|
|
44
31
|
const toastOn = () => settings.read().settings.decisionToast;
|
|
45
|
-
ctx.effect(() => ctx.tools.register(routeSubagentTool(jev, { agents: ctx.agents, state, readSettings: () => settings.read().settings, catalog: hostSettings.catalog,
|
|
32
|
+
ctx.effect(() => ctx.tools.register(routeSubagentTool(jev, { agents: ctx.agents, state, readSettings: () => settings.read().settings, catalog: hostSettings.catalog, registry })));
|
|
46
33
|
ctx.on('agent/created', ({ agent, source }) => {
|
|
47
34
|
// Factory resume announces `resume`; a nonzero initial seed also excludes restored/forked histories.
|
|
48
35
|
if (source !== 'startup' || agent.session.header.origin === 'subagent' || agent.session.firstLiveSeq !== 0 || agent.session.header.isSeeded || agent.session.snapshotEvents().some(event => event.type === 'session/end-seed') || stateFiles.get(agent.id))
|
|
49
36
|
return;
|
|
50
37
|
if (settings.read().settings.defaultMode !== 'AUTO')
|
|
51
38
|
return;
|
|
52
|
-
// A new session whose default main model is outside AUTO's
|
|
53
|
-
if (!mainSupported(currentMainSelection(ctx, agent)))
|
|
39
|
+
// A new session whose default main model is outside AUTO's registered families starts in MANUAL.
|
|
40
|
+
if (!mainSupported(currentMainSelection(ctx, agent), registry))
|
|
54
41
|
return;
|
|
55
42
|
state.setMode(agent.id, 'AUTO');
|
|
56
43
|
return undefined;
|
|
@@ -65,62 +52,6 @@ export function apply(ctx, config = {}) {
|
|
|
65
52
|
}
|
|
66
53
|
return value;
|
|
67
54
|
};
|
|
68
|
-
ctx.on('agent/pre-step', async (payload, next) => {
|
|
69
|
-
const decision = await next();
|
|
70
|
-
// Judge only steps that actually enter, using the messages that enter (a rejected step sends nothing).
|
|
71
|
-
if (settings.read().settings.policy === 'policy-v2' || payload.step !== 1 || decision.kind !== 'enter')
|
|
72
|
-
return decision;
|
|
73
|
-
const agent = ref(payload.agent);
|
|
74
|
-
if (agent.origin !== 'subagent' && userInput(decision.messages).userTurn)
|
|
75
|
-
book.userTurn(agent.id, payload.turn);
|
|
76
|
-
const job = router.begin(agent, payload.turn, decision.messages);
|
|
77
|
-
if (!job)
|
|
78
|
-
return decision;
|
|
79
|
-
// History before this turn's messages enter: the previous user message and assistant reply.
|
|
80
|
-
const prior = previousTurn(payload.agent.session.deriveMessages());
|
|
81
|
-
const result = await jev.judge(buildState({ previous: job.previous, prevUser: prior.user, prevAssistant: prior.assistant, text: job.text }), payload.signal);
|
|
82
|
-
if (result.kind === 'ok') {
|
|
83
|
-
const { kind: _, tier, ...measure } = result;
|
|
84
|
-
router.resolve(agent, payload.turn, tier, measure);
|
|
85
|
-
}
|
|
86
|
-
else if (result.kind === 'failed')
|
|
87
|
-
router.fail(agent, payload.turn, result.category);
|
|
88
|
-
else
|
|
89
|
-
router.cancel(agent, payload.turn);
|
|
90
|
-
return decision;
|
|
91
|
-
});
|
|
92
|
-
// Reply → user turn, for the reply-bar tier button (memory only; empty after a restart).
|
|
93
|
-
ctx.on('session/event', (session, event) => {
|
|
94
|
-
if (event.type !== 'assistant/message')
|
|
95
|
-
return;
|
|
96
|
-
const data = event.data;
|
|
97
|
-
book.assistant(session.id, data.message.id, data.turn);
|
|
98
|
-
});
|
|
99
|
-
// Official selection writer, set while the session controller is available.
|
|
100
|
-
const inflight = new Map();
|
|
101
|
-
ctx.on('agent/request', async (payload, next) => {
|
|
102
|
-
const config = await next();
|
|
103
|
-
if (settings.read().settings.policy === 'policy-v2' || payload.agent.session.header.origin === 'subagent')
|
|
104
|
-
return config;
|
|
105
|
-
const model = router.needsModelInfo(ref(payload.agent)) ? await ctx.llm.resolveModelInfo(config.provider, config.model, payload.signal) : undefined;
|
|
106
|
-
const efforts = (model?.reasoning?.efforts ?? []).map(effort => effort.id).filter(id => knownEfforts.has(id));
|
|
107
|
-
const catalog = model && settings.read().settings.mainEfforts ? await hostSettings.catalog() : undefined;
|
|
108
|
-
const routed = router.request(ref(payload.agent), payload.turn, config, efforts, catalog);
|
|
109
|
-
// AUTO moved the effort away from the official selection: make that selection (and the picker) carry it.
|
|
110
|
-
// `config` is the official selection for this request, so a user pick made since the last sync is caught too.
|
|
111
|
-
// Not awaited and never fatal; `inflight` only stops repeat calls before the first one lands.
|
|
112
|
-
if (selectModel && routed.reasoningEffort && routed.reasoningEffort !== config.reasoningEffort) {
|
|
113
|
-
const key = `${routed.provider}\0${routed.model}\0${routed.reasoningEffort}`;
|
|
114
|
-
if (inflight.get(payload.agent.id) !== key) {
|
|
115
|
-
inflight.set(payload.agent.id, key);
|
|
116
|
-
selectModel({ sessionId: payload.agent.id, provider: routed.provider, model: routed.model, reasoningEffort: routed.reasoningEffort })
|
|
117
|
-
.catch(error => ctx.logger.warn(`auto-model-routing: could not sync the model selection: ${String(error)}`))
|
|
118
|
-
.finally(() => { if (inflight.get(payload.agent.id) === key)
|
|
119
|
-
inflight.delete(payload.agent.id); });
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
return routed;
|
|
123
|
-
}, { prepend: true });
|
|
124
55
|
ctx.inject(['sessionController'], child => {
|
|
125
56
|
selectModel = request => child.sessionController.selectModel(request);
|
|
126
57
|
child.effect(() => () => { selectModel = undefined; });
|
|
@@ -164,12 +95,12 @@ export function apply(ctx, config = {}) {
|
|
|
164
95
|
}
|
|
165
96
|
const response = operation === 'childRouting'
|
|
166
97
|
? (typeof payload?.sessionId === 'string'
|
|
167
|
-
? Response.json(policy.status(payload.sessionId) ?? { status: 'unsupported', reason: '
|
|
98
|
+
? Response.json(policy.status(payload.sessionId) ?? { status: 'unsupported', reason: '没有已绑定的子任务决策' })
|
|
168
99
|
: Response.json({ error: 'invalid request' }, { status: 400 }))
|
|
169
100
|
: operation === 'settings' || operation === 'saveSettings' || operation === 'setKey' || operation === 'testDecision'
|
|
170
101
|
? await hostSettings.endpoint(operation, payload)
|
|
171
102
|
: operation === 'observe' ? await observeEndpoint(payload, resolveAgent, feedback, () => ctx.logger.warn('auto-model-routing: could not record exposure'))
|
|
172
|
-
: operation === 'label' ? await labelEndpoint(payload, resolveAgent,
|
|
103
|
+
: operation === 'label' ? await labelEndpoint(payload, resolveAgent, feedback) : await endpoint(operation, payload, resolveAgent, state);
|
|
173
104
|
if (manualDecision && response.ok) {
|
|
174
105
|
try {
|
|
175
106
|
feedback.fact(manualDecision.sessionId, manualDecision.decisionId, 'manual-switch');
|
|
@@ -178,20 +109,20 @@ export function apply(ctx, config = {}) {
|
|
|
178
109
|
ctx.logger.warn('auto-model-routing: could not record MANUAL switch');
|
|
179
110
|
}
|
|
180
111
|
}
|
|
181
|
-
if ((operation === 'state' || operation === 'setMode') && response.ok
|
|
112
|
+
if ((operation === 'state' || operation === 'setMode') && response.ok) {
|
|
182
113
|
const sessionId = payload.sessionId;
|
|
183
114
|
const agent = await resolveSessionAgent(sessionId).catch(() => undefined);
|
|
184
115
|
const selection = agent ? currentMainSelection(ctx, agent) : undefined;
|
|
185
|
-
const supported = mainSupported(selection);
|
|
186
|
-
// Leaving AUTO is persisted: a pick outside the
|
|
116
|
+
const supported = mainSupported(selection, registry);
|
|
117
|
+
// Leaving AUTO is persisted: a pick outside the registered families must not keep judging turns.
|
|
187
118
|
if (!supported && state.read(sessionId).mode === 'AUTO') {
|
|
188
119
|
state.setMode(sessionId, 'MANUAL');
|
|
189
120
|
policy.invalidate(sessionId);
|
|
190
121
|
}
|
|
191
122
|
const value = await response.json();
|
|
192
123
|
const main = policy.mainStatus(sessionId);
|
|
193
|
-
const {
|
|
194
|
-
const result = { ...
|
|
124
|
+
const { previous: _previous, ...persisted } = value;
|
|
125
|
+
const result = { ...persisted, mode: state.read(sessionId).mode, ...main, ...(toastOn() ? { decisionToast: true } : {}), autoSupport: { supported, ...(selection ? { provider: selection.provider, model: selection.model } : {}) } };
|
|
195
126
|
return Response.json(enveloped ? { type: 'server-response', rpcId: wire.rpcId, result: { ok: true, value: result } } : result);
|
|
196
127
|
}
|
|
197
128
|
if (!enveloped)
|