dsh-auto-model-routing 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +15 -10
- package/README.zh.md +15 -9
- package/docs/feedback-evaluation.md +20 -3
- package/docs/host-capability-report.md +2 -0
- package/docs/jev-policy-v2-validation.md +3 -1
- package/docs/magpie-live-acceptance.md +2 -0
- package/docs/policy-v2-development.md +5 -3
- package/docs/policy-v2-real-evaluation-plan.md +2 -0
- package/lib/client/control.d.ts +2 -2
- package/lib/client/control.js +10 -20
- package/lib/client/index.d.ts +9 -2
- package/lib/client/index.js +97 -12
- package/lib/client/label.d.ts +8 -11
- package/lib/client/label.js +51 -29
- package/lib/client/settings-section.d.ts +2 -2
- package/lib/client/settings-section.js +7 -69
- package/lib/client/strategy-details.js +3 -4
- package/lib/client.js +316 -489
- package/lib/context.d.ts +1 -3
- package/lib/context.js +3 -4
- package/lib/effort.d.ts +2 -10
- package/lib/effort.js +1 -31
- package/lib/endpoints.d.ts +11 -9
- package/lib/endpoints.js +44 -42
- package/lib/feedback.d.ts +77 -6
- package/lib/feedback.js +34 -3
- package/lib/index.d.ts +4 -0
- package/lib/index.js +54 -89
- package/lib/jev.d.ts +17 -29
- package/lib/jev.js +6 -34
- package/lib/policy-host.d.ts +13 -8
- package/lib/policy-host.js +98 -47
- package/lib/policy-store.d.ts +9 -17
- package/lib/policy-store.js +28 -15
- package/lib/routing.d.ts +20 -0
- package/lib/routing.js +51 -0
- package/lib/rubric.json +15 -6
- package/lib/settings-contract.d.ts +5 -21
- package/lib/settings-contract.js +3 -2
- package/lib/settings-host.d.ts +4 -8
- package/lib/settings-host.js +9 -9
- package/lib/settings-validation.d.ts +1 -7
- package/lib/settings-validation.js +0 -51
- package/lib/settings.d.ts +3 -8
- package/lib/settings.js +18 -177
- package/lib/state.d.ts +2 -4
- package/lib/state.js +2 -7
- package/lib/strategies/claude.d.ts +21 -0
- package/lib/strategies/claude.js +8 -0
- package/lib/strategies/codex.d.ts +21 -0
- package/lib/strategies/codex.js +9 -0
- package/lib/strategies/index.d.ts +22 -0
- package/lib/strategies/index.js +85 -0
- package/lib/strategies/types.d.ts +14 -0
- package/lib/strategies/types.js +1 -0
- package/lib/subagent-tool.d.ts +7 -7
- package/lib/subagent-tool.js +33 -76
- package/lib/support.d.ts +24 -0
- package/lib/support.js +36 -0
- package/lib/usage-report.d.ts +21 -2
- package/lib/usage-report.js +86 -22
- package/package.json +19 -7
- package/skills/auto-model-routing-report/SKILL.md +9 -3
- package/skills/subagent-model-routing/SKILL.md +14 -30
- package/lib/client/route-picker.d.ts +0 -21
- package/lib/client/route-picker.js +0 -75
- package/lib/judgements.d.ts +0 -41
- package/lib/judgements.js +0 -55
- package/lib/policy-v2-rubric.json +0 -21
- package/lib/policy-v2.d.ts +0 -20
- package/lib/policy-v2.js +0 -53
- package/lib/router.d.ts +0 -68
- package/lib/router.js +0 -153
- package/scripts/label-report.mjs +0 -50
package/lib/context.d.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { Tier } from './effort.js';
|
|
2
1
|
export declare function redact(text: string): string;
|
|
3
2
|
interface HistoryMessage {
|
|
4
3
|
role: string;
|
|
@@ -15,9 +14,8 @@ export declare function previousTurn(messages: readonly HistoryMessage[]): {
|
|
|
15
14
|
user?: string;
|
|
16
15
|
assistant?: string;
|
|
17
16
|
};
|
|
18
|
-
/** The Jev `state
|
|
17
|
+
/** The Jev `state`; every section is redacted, then cut. The fixed first line keeps the request bytes stable. */
|
|
19
18
|
export declare function buildState(input: {
|
|
20
|
-
previous?: Tier | undefined;
|
|
21
19
|
prevUser?: string | undefined;
|
|
22
20
|
prevAssistant?: string | undefined;
|
|
23
21
|
text: string;
|
package/lib/context.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
/** Shape-based, best effort (design D4): applied to the whole text before any truncation. */
|
|
1
|
+
/** Shape-based, best effort: applied to the whole text before any truncation. */
|
|
3
2
|
const RULES = [
|
|
4
3
|
[/-----BEGIN [A-Z ]+-----[\s\S]*?-----END [A-Z ]+-----/g, '<redacted>'],
|
|
5
4
|
[/\bBearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, 'Bearer <redacted>'],
|
|
@@ -19,9 +18,9 @@ export function previousTurn(messages) {
|
|
|
19
18
|
return { ...(user ? { user: textOf(user) } : {}), ...(assistant ? { assistant: textOf(assistant) } : {}) };
|
|
20
19
|
}
|
|
21
20
|
const EXPLAIN = 'Earlier conversation is shown so you can tell what the new message asks for. Rate the work the assistant must do next in response to the new message, not the earlier work.';
|
|
22
|
-
/** The Jev `state
|
|
21
|
+
/** The Jev `state`; every section is redacted, then cut. The fixed first line keeps the request bytes stable. */
|
|
23
22
|
export function buildState(input) {
|
|
24
|
-
const lines = [
|
|
23
|
+
const lines = ['Previous user-turn tier: none'];
|
|
25
24
|
const context = input.prevUser !== undefined || input.prevAssistant !== undefined;
|
|
26
25
|
if (context)
|
|
27
26
|
lines.push(EXPLAIN);
|
package/lib/effort.d.ts
CHANGED
|
@@ -1,11 +1,3 @@
|
|
|
1
|
-
export type Tier = 'simple' | 'standard' | 'complex' | 'escalated';
|
|
2
1
|
export type Effort = 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
3
|
-
|
|
4
|
-
export
|
|
5
|
-
/** Target effort before clamping to what the model supports (also used for sub-agents, unclamped). */
|
|
6
|
-
export declare function targetEffort(model: string, tier: Tier, cap?: Effort): Effort;
|
|
7
|
-
/** Undefined when the model supports none of low..xhigh: the request is then left as is. */
|
|
8
|
-
export declare function mapEffort(model: string, tier: Tier, supported: readonly Effort[], cap?: Effort): Effort | undefined;
|
|
9
|
-
export type ChildProvider = 'claude' | 'codex';
|
|
10
|
-
/** Sub-agent models by tier (design D9); the single source for `route_subagent`. Effort comes from `targetEffort`. */
|
|
11
|
-
export declare const CHILD_MODELS: Readonly<Record<ChildProvider, readonly [simple: string, standard: string, complex: string, escalated: string]>>;
|
|
2
|
+
/** Closed set of the reasons the state panel shows for a main request. */
|
|
3
|
+
export type Reason = 'jev' | 'no-effort-support';
|
package/lib/effort.js
CHANGED
|
@@ -1,31 +1 @@
|
|
|
1
|
-
export
|
|
2
|
-
/** Official starting points (design D9; sources in eval/README.md). Unlisted models start at medium. */
|
|
3
|
-
const STARTING_POINTS = {
|
|
4
|
-
'claude-opus-5-5': 'medium',
|
|
5
|
-
'claude-sonnet-5-5': 'medium',
|
|
6
|
-
'claude-haiku-5-5': 'medium',
|
|
7
|
-
'gpt-6.1-sol': 'medium',
|
|
8
|
-
'claude-fable-5-1': 'high',
|
|
9
|
-
'gpt-6-luna': 'high',
|
|
10
|
-
'gpt-6-astra': 'low',
|
|
11
|
-
};
|
|
12
|
-
const OFFSET = { simple: -1, standard: 0, complex: 1, escalated: 2 };
|
|
13
|
-
/** Automatic routing never goes above xhigh: `max` stays a manual choice. */
|
|
14
|
-
const automatic = ['low', 'medium', 'high', 'xhigh'];
|
|
15
|
-
/** Target effort before clamping to what the model supports (also used for sub-agents, unclamped). */
|
|
16
|
-
export function targetEffort(model, tier, cap = 'xhigh') {
|
|
17
|
-
const start = automatic.indexOf(STARTING_POINTS[model] ?? 'medium');
|
|
18
|
-
const ceiling = cap === 'max' ? automatic.length - 1 : automatic.indexOf(cap);
|
|
19
|
-
return automatic[Math.max(0, Math.min(ceiling, start + OFFSET[tier]))];
|
|
20
|
-
}
|
|
21
|
-
/** Undefined when the model supports none of low..xhigh: the request is then left as is. */
|
|
22
|
-
export function mapEffort(model, tier, supported, cap = 'xhigh') {
|
|
23
|
-
const target = targetEffort(model, tier, cap);
|
|
24
|
-
const candidates = automatic.filter(effort => supported.includes(effort) && automatic.indexOf(effort) <= (cap === 'max' ? automatic.length - 1 : automatic.indexOf(cap)));
|
|
25
|
-
return candidates.filter(effort => automatic.indexOf(effort) <= automatic.indexOf(target)).at(-1) ?? candidates[0];
|
|
26
|
-
}
|
|
27
|
-
/** Sub-agent models by tier (design D9); the single source for `route_subagent`. Effort comes from `targetEffort`. */
|
|
28
|
-
export const CHILD_MODELS = {
|
|
29
|
-
claude: ['claude-haiku-5-5', 'claude-sonnet-5-5', 'claude-opus-5-5', 'claude-opus-5-5'],
|
|
30
|
-
codex: ['gpt-6-luna', 'gpt-6.1-sol', 'gpt-6.1-sol', 'gpt-6.1-sol'],
|
|
31
|
-
};
|
|
1
|
+
export {};
|
package/lib/endpoints.d.ts
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { type FeedbackBook } from './feedback.js';
|
|
2
|
-
import type { AgentRef, Router } from './router.js';
|
|
3
2
|
import type { RoutingState } from './state.js';
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
lookup(sessionId: string, messageId: string): TurnRecord | undefined;
|
|
9
|
-
label(sessionId: string, turn: number, label: Tier): void;
|
|
3
|
+
/** Identity of a live session as the endpoints need it. */
|
|
4
|
+
export interface AgentRef {
|
|
5
|
+
id: string;
|
|
6
|
+
origin?: string;
|
|
10
7
|
}
|
|
11
|
-
|
|
12
|
-
|
|
8
|
+
type Resolve = (id: string) => Promise<AgentRef | undefined>;
|
|
9
|
+
/** Validates the session and applies `setMode`; the caller adds the policy host's view to the response. */
|
|
10
|
+
export declare function endpoint(operation: 'state' | 'setMode', body: unknown, resolveAgent: Resolve, state: RoutingState): Promise<Response>;
|
|
11
|
+
export declare function observeEndpoint(body: unknown, resolveAgent: Resolve, book: FeedbackBook, warn: () => void): Promise<Response>;
|
|
12
|
+
/** Query (no `feedback`) or change the allocation feedback of a reply. Unknown keys (e.g. a retired `label`) are rejected. */
|
|
13
|
+
export declare function labelEndpoint(body: unknown, resolveAgent: Resolve, book: FeedbackBook): Promise<Response>;
|
|
14
|
+
export {};
|
package/lib/endpoints.js
CHANGED
|
@@ -1,62 +1,64 @@
|
|
|
1
1
|
import { allocationFeedback } from './feedback.js';
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
const invalid = () => Response.json({ error: 'invalid request' }, { status: 400 });
|
|
3
|
+
const unknown = () => Response.json({ error: 'unknown ordinary session' }, { status: 404 });
|
|
4
|
+
/** Validates the session and applies `setMode`; the caller adds the policy host's view to the response. */
|
|
5
|
+
export async function endpoint(operation, body, resolveAgent, state) {
|
|
4
6
|
if (typeof body !== 'object' || body === null)
|
|
5
|
-
return
|
|
7
|
+
return invalid();
|
|
6
8
|
const { sessionId, mode } = body;
|
|
7
9
|
if (typeof sessionId !== 'string' || !sessionId || (operation === 'setMode' && mode !== 'AUTO' && mode !== 'MANUAL'))
|
|
8
|
-
return
|
|
10
|
+
return invalid();
|
|
9
11
|
const agent = await resolveAgent(sessionId);
|
|
10
12
|
if (!agent || agent.origin === 'subagent')
|
|
11
|
-
return
|
|
13
|
+
return unknown();
|
|
12
14
|
if (operation === 'setMode')
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
const decision = router.judgement(sessionId);
|
|
16
|
-
return Response.json({ ...state.read(sessionId), ...router.status(agent), ...(decision ? { decision } : {}), ...(toastOn() ? { decisionToast: true } : {}) });
|
|
15
|
+
state.setMode(sessionId, mode);
|
|
16
|
+
return Response.json(state.read(sessionId));
|
|
17
17
|
}
|
|
18
|
-
|
|
19
|
-
export async function labelEndpoint(body, resolveAgent, book, feedbackBook) {
|
|
18
|
+
export async function observeEndpoint(body, resolveAgent, book, warn) {
|
|
20
19
|
if (typeof body !== 'object' || body === null)
|
|
21
|
-
return
|
|
22
|
-
const { sessionId, messageId
|
|
23
|
-
if (typeof sessionId !== 'string' || !sessionId || typeof messageId !== 'string' ||
|
|
24
|
-
return
|
|
20
|
+
return invalid();
|
|
21
|
+
const { sessionId, messageId } = body;
|
|
22
|
+
if (typeof sessionId !== 'string' || !sessionId || typeof messageId !== 'string' || !messageId)
|
|
23
|
+
return invalid();
|
|
25
24
|
const agent = await resolveAgent(sessionId);
|
|
26
25
|
if (!agent || agent.origin === 'subagent')
|
|
27
|
-
return
|
|
26
|
+
return unknown();
|
|
27
|
+
const binding = book.lookup(sessionId, messageId);
|
|
28
|
+
if (binding)
|
|
29
|
+
try {
|
|
30
|
+
book.fact(sessionId, binding.decisionId, 'exposed');
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
warn();
|
|
34
|
+
}
|
|
35
|
+
return Response.json({});
|
|
36
|
+
}
|
|
37
|
+
/** Query (no `feedback`) or change the allocation feedback of a reply. Unknown keys (e.g. a retired `label`) are rejected. */
|
|
38
|
+
export async function labelEndpoint(body, resolveAgent, book) {
|
|
39
|
+
if (typeof body !== 'object' || body === null || Array.isArray(body))
|
|
40
|
+
return invalid();
|
|
41
|
+
if (Object.keys(body).some(key => !['sessionId', 'messageId', 'feedback'].includes(key)))
|
|
42
|
+
return invalid();
|
|
43
|
+
const { sessionId, messageId } = body;
|
|
44
|
+
if (typeof sessionId !== 'string' || !sessionId || typeof messageId !== 'string')
|
|
45
|
+
return invalid();
|
|
46
|
+
const agent = await resolveAgent(sessionId);
|
|
47
|
+
if (!agent || agent.origin === 'subagent')
|
|
48
|
+
return unknown();
|
|
28
49
|
const hasFeedback = Object.hasOwn(body, 'feedback');
|
|
29
50
|
const feedback = body.feedback;
|
|
30
|
-
if (hasFeedback &&
|
|
51
|
+
if (hasFeedback && feedback !== null && !allocationFeedback.includes(feedback))
|
|
31
52
|
return Response.json({ error: 'invalid feedback' }, { status: 400 });
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
return Response.json({ error: 'use allocation feedback' }, { status: 400 });
|
|
36
|
-
if (hasFeedback) {
|
|
37
|
-
try {
|
|
38
|
-
feedbackBook.set(sessionId, messageId, feedback);
|
|
39
|
-
}
|
|
40
|
-
catch {
|
|
41
|
-
return Response.json({ error: 'could not record feedback' }, { status: 500 });
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
return Response.json({ policy: 'policy-v2', ...feedbackBook.lookup(sessionId, messageId) });
|
|
45
|
-
}
|
|
46
|
-
if (hasFeedback)
|
|
47
|
-
return Response.json({ error: 'no applied decision' }, { status: 404 });
|
|
48
|
-
const record = book.lookup(sessionId, messageId);
|
|
49
|
-
if (!record)
|
|
50
|
-
return Response.json({});
|
|
51
|
-
const { turn, ...view } = record;
|
|
52
|
-
if (label !== undefined) {
|
|
53
|
+
if (!book.lookup(sessionId, messageId))
|
|
54
|
+
return hasFeedback ? Response.json({ error: 'no applied decision' }, { status: 404 }) : Response.json({});
|
|
55
|
+
if (hasFeedback) {
|
|
53
56
|
try {
|
|
54
|
-
book.
|
|
57
|
+
book.set(sessionId, messageId, feedback);
|
|
55
58
|
}
|
|
56
59
|
catch {
|
|
57
|
-
return Response.json({ error: 'could not record
|
|
60
|
+
return Response.json({ error: 'could not record feedback' }, { status: 500 });
|
|
58
61
|
}
|
|
59
|
-
view.label = label;
|
|
60
62
|
}
|
|
61
|
-
return Response.json(view);
|
|
63
|
+
return Response.json(book.view(sessionId, messageId));
|
|
62
64
|
}
|
package/lib/feedback.d.ts
CHANGED
|
@@ -10,8 +10,9 @@ declare const schema: z.ZodObject<{
|
|
|
10
10
|
snapshot: z.ZodOptional<z.ZodObject<{
|
|
11
11
|
decidedAt: z.ZodNumber;
|
|
12
12
|
appliedAt: z.ZodNumber;
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
strategy: z.ZodOptional<z.ZodString>;
|
|
14
|
+
policyVersion: z.ZodOptional<z.ZodString>;
|
|
15
|
+
mappingVersion: z.ZodOptional<z.ZodString>;
|
|
15
16
|
decisionProvider: z.ZodString;
|
|
16
17
|
decisionModel: z.ZodString;
|
|
17
18
|
ms: z.ZodNumber;
|
|
@@ -30,11 +31,24 @@ declare const schema: z.ZodObject<{
|
|
|
30
31
|
}>>>;
|
|
31
32
|
feedbackAt: z.ZodOptional<z.ZodNumber>;
|
|
32
33
|
}, z.core.$strict>;
|
|
34
|
+
declare const factEvent: z.ZodObject<{
|
|
35
|
+
type: z.ZodLiteral<"fact">;
|
|
36
|
+
version: z.ZodOptional<z.ZodLiteral<1>>;
|
|
37
|
+
sessionId: z.ZodString;
|
|
38
|
+
decisionId: z.ZodNumber;
|
|
39
|
+
fact: z.ZodEnum<{
|
|
40
|
+
exposed: "exposed";
|
|
41
|
+
"manual-switch": "manual-switch";
|
|
42
|
+
}>;
|
|
43
|
+
at: z.ZodNumber;
|
|
44
|
+
}, z.core.$strict>;
|
|
45
|
+
export type FeedbackFact = z.infer<typeof factEvent>['fact'];
|
|
33
46
|
export type FeedbackRecord = z.infer<typeof schema>;
|
|
34
47
|
/** Immutable decision bindings and separate feedback events; never includes task text or credentials. */
|
|
35
48
|
export declare class FeedbackBook {
|
|
36
49
|
private file;
|
|
37
50
|
private records;
|
|
51
|
+
private factRecords;
|
|
38
52
|
constructor(file: string);
|
|
39
53
|
private key;
|
|
40
54
|
lookup(sessionId: string, messageId: string): {
|
|
@@ -47,13 +61,41 @@ export declare class FeedbackBook {
|
|
|
47
61
|
snapshot?: {
|
|
48
62
|
decidedAt: number;
|
|
49
63
|
appliedAt: number;
|
|
50
|
-
policyVersion: string;
|
|
51
|
-
mappingVersion: string;
|
|
52
64
|
decisionProvider: string;
|
|
53
65
|
decisionModel: string;
|
|
54
66
|
ms: number;
|
|
55
67
|
provider: string;
|
|
56
68
|
requestSeq: number;
|
|
69
|
+
strategy?: string | undefined;
|
|
70
|
+
policyVersion?: string | undefined;
|
|
71
|
+
mappingVersion?: string | undefined;
|
|
72
|
+
confidence?: number | undefined;
|
|
73
|
+
probabilities?: Record<string, number> | undefined;
|
|
74
|
+
requestTurn?: number | undefined;
|
|
75
|
+
} | undefined;
|
|
76
|
+
boundAt?: number | undefined;
|
|
77
|
+
feedback?: "appropriate" | "too-high" | "too-low" | "unknown" | null | undefined;
|
|
78
|
+
feedbackAt?: number | undefined;
|
|
79
|
+
} | undefined;
|
|
80
|
+
/** The binding as the `label` endpoint returns it: historical snapshot keys are left out, nothing is inferred. */
|
|
81
|
+
view(sessionId: string, messageId: string): {
|
|
82
|
+
sessionId: string;
|
|
83
|
+
messageId: string;
|
|
84
|
+
decisionId: number;
|
|
85
|
+
depth: string;
|
|
86
|
+
model: string;
|
|
87
|
+
effort: string;
|
|
88
|
+
snapshot?: {
|
|
89
|
+
decidedAt: number;
|
|
90
|
+
appliedAt: number;
|
|
91
|
+
decisionProvider: string;
|
|
92
|
+
decisionModel: string;
|
|
93
|
+
ms: number;
|
|
94
|
+
provider: string;
|
|
95
|
+
requestSeq: number;
|
|
96
|
+
strategy?: string | undefined;
|
|
97
|
+
policyVersion?: string | undefined;
|
|
98
|
+
mappingVersion?: string | undefined;
|
|
57
99
|
confidence?: number | undefined;
|
|
58
100
|
probabilities?: Record<string, number> | undefined;
|
|
59
101
|
requestTurn?: number | undefined;
|
|
@@ -61,6 +103,29 @@ export declare class FeedbackBook {
|
|
|
61
103
|
boundAt?: number | undefined;
|
|
62
104
|
feedback?: "appropriate" | "too-high" | "too-low" | "unknown" | null | undefined;
|
|
63
105
|
feedbackAt?: number | undefined;
|
|
106
|
+
} | {
|
|
107
|
+
snapshot: {
|
|
108
|
+
decidedAt: number;
|
|
109
|
+
appliedAt: number;
|
|
110
|
+
decisionProvider: string;
|
|
111
|
+
decisionModel: string;
|
|
112
|
+
ms: number;
|
|
113
|
+
provider: string;
|
|
114
|
+
requestSeq: number;
|
|
115
|
+
strategy?: string | undefined;
|
|
116
|
+
confidence?: number | undefined;
|
|
117
|
+
probabilities?: Record<string, number> | undefined;
|
|
118
|
+
requestTurn?: number | undefined;
|
|
119
|
+
};
|
|
120
|
+
sessionId: string;
|
|
121
|
+
messageId: string;
|
|
122
|
+
decisionId: number;
|
|
123
|
+
depth: string;
|
|
124
|
+
model: string;
|
|
125
|
+
effort: string;
|
|
126
|
+
boundAt?: number | undefined;
|
|
127
|
+
feedback?: "appropriate" | "too-high" | "too-low" | "unknown" | null | undefined;
|
|
128
|
+
feedbackAt?: number | undefined;
|
|
64
129
|
} | undefined;
|
|
65
130
|
/** All folded bindings in file order (read-only; used by the usage report). */
|
|
66
131
|
entries(): {
|
|
@@ -73,13 +138,14 @@ export declare class FeedbackBook {
|
|
|
73
138
|
snapshot?: {
|
|
74
139
|
decidedAt: number;
|
|
75
140
|
appliedAt: number;
|
|
76
|
-
policyVersion: string;
|
|
77
|
-
mappingVersion: string;
|
|
78
141
|
decisionProvider: string;
|
|
79
142
|
decisionModel: string;
|
|
80
143
|
ms: number;
|
|
81
144
|
provider: string;
|
|
82
145
|
requestSeq: number;
|
|
146
|
+
strategy?: string | undefined;
|
|
147
|
+
policyVersion?: string | undefined;
|
|
148
|
+
mappingVersion?: string | undefined;
|
|
83
149
|
confidence?: number | undefined;
|
|
84
150
|
probabilities?: Record<string, number> | undefined;
|
|
85
151
|
requestTurn?: number | undefined;
|
|
@@ -90,6 +156,11 @@ export declare class FeedbackBook {
|
|
|
90
156
|
}[];
|
|
91
157
|
bind(record: FeedbackRecord): void;
|
|
92
158
|
set(sessionId: string, messageId: string, feedback: typeof allocationFeedback[number] | null): void;
|
|
159
|
+
facts(sessionId: string, decisionId: number): {
|
|
160
|
+
exposed?: number;
|
|
161
|
+
"manual-switch"?: number;
|
|
162
|
+
};
|
|
163
|
+
fact(sessionId: string, decisionId: number, fact: FeedbackFact): void;
|
|
93
164
|
private append;
|
|
94
165
|
}
|
|
95
166
|
export {};
|
package/lib/feedback.js
CHANGED
|
@@ -3,18 +3,22 @@ import { dirname } from 'node:path';
|
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
export const allocationFeedback = ['appropriate', 'too-high', 'too-low', 'unknown'];
|
|
5
5
|
const probability = z.number().min(0).max(1);
|
|
6
|
+
// `policyVersion`/`mappingVersion` exist only on bindings written before 0.5.0; new ones carry `strategy`.
|
|
6
7
|
const snapshot = z.object({
|
|
7
|
-
decidedAt: z.number(), appliedAt: z.number(), policyVersion: z.string(), mappingVersion: z.string(),
|
|
8
|
+
decidedAt: z.number(), appliedAt: z.number(), strategy: z.string().optional(), policyVersion: z.string().optional(), mappingVersion: z.string().optional(),
|
|
8
9
|
decisionProvider: z.string(), decisionModel: z.string(), ms: z.number(),
|
|
9
10
|
confidence: probability.optional(), probabilities: z.record(z.string(), probability).optional(),
|
|
10
11
|
provider: z.string(), requestSeq: z.number(), requestTurn: z.number().optional(),
|
|
11
12
|
}).strict();
|
|
12
13
|
const schema = z.object({ sessionId: z.string(), messageId: z.string(), decisionId: z.number(), depth: z.string(), model: z.string(), effort: z.string(), snapshot: snapshot.optional(), boundAt: z.number().optional(), feedback: z.enum(allocationFeedback).nullable().optional(), feedbackAt: z.number().optional() }).strict();
|
|
13
|
-
|
|
14
|
+
// Older events carried `version` (feedback 2, fact 1); new events have none.
|
|
15
|
+
const feedbackEvent = z.object({ type: z.literal('feedback'), version: z.literal(2).optional(), sessionId: z.string(), messageId: z.string(), decisionId: z.number(), at: z.number(), feedback: z.enum(allocationFeedback).nullable() }).strict();
|
|
16
|
+
const factEvent = z.object({ type: z.literal('fact'), version: z.literal(1).optional(), sessionId: z.string().min(1), decisionId: z.number(), fact: z.enum(['exposed', 'manual-switch']), at: z.number().finite() }).strict();
|
|
14
17
|
/** Immutable decision bindings and separate feedback events; never includes task text or credentials. */
|
|
15
18
|
export class FeedbackBook {
|
|
16
19
|
file;
|
|
17
20
|
records = new Map();
|
|
21
|
+
factRecords = new Map();
|
|
18
22
|
constructor(file) {
|
|
19
23
|
this.file = file;
|
|
20
24
|
let text;
|
|
@@ -29,6 +33,15 @@ export class FeedbackBook {
|
|
|
29
33
|
for (const line of text.split('\n'))
|
|
30
34
|
try {
|
|
31
35
|
const raw = JSON.parse(line);
|
|
36
|
+
const fact = factEvent.safeParse(raw);
|
|
37
|
+
if (fact.success) {
|
|
38
|
+
const e = fact.data;
|
|
39
|
+
const key = JSON.stringify([e.sessionId, e.decisionId]);
|
|
40
|
+
const facts = this.factRecords.get(key) ?? {};
|
|
41
|
+
facts[e.fact] ??= e.at;
|
|
42
|
+
this.factRecords.set(key, facts);
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
32
45
|
const event = feedbackEvent.safeParse(raw);
|
|
33
46
|
if (event.success) {
|
|
34
47
|
const e = event.data;
|
|
@@ -51,6 +64,14 @@ export class FeedbackBook {
|
|
|
51
64
|
}
|
|
52
65
|
key(sessionId, messageId) { return JSON.stringify([sessionId, messageId]); }
|
|
53
66
|
lookup(sessionId, messageId) { const r = this.records.get(this.key(sessionId, messageId)); return r && structuredClone(r); }
|
|
67
|
+
/** The binding as the `label` endpoint returns it: historical snapshot keys are left out, nothing is inferred. */
|
|
68
|
+
view(sessionId, messageId) {
|
|
69
|
+
const r = this.lookup(sessionId, messageId);
|
|
70
|
+
if (!r?.snapshot)
|
|
71
|
+
return r;
|
|
72
|
+
const { policyVersion: _p, mappingVersion: _m, ...snapshot } = r.snapshot;
|
|
73
|
+
return { ...r, snapshot };
|
|
74
|
+
}
|
|
54
75
|
/** All folded bindings in file order (read-only; used by the usage report). */
|
|
55
76
|
entries() { return [...this.records.values()].map(r => structuredClone(r)); }
|
|
56
77
|
bind(record) { if (this.lookup(record.sessionId, record.messageId))
|
|
@@ -59,9 +80,19 @@ export class FeedbackBook {
|
|
|
59
80
|
const r = this.lookup(sessionId, messageId);
|
|
60
81
|
if (!r)
|
|
61
82
|
throw Error('No applied decision');
|
|
62
|
-
const event = feedbackEvent.parse({ type: 'feedback',
|
|
83
|
+
const event = feedbackEvent.parse({ type: 'feedback', sessionId, messageId, decisionId: r.decisionId, at: Date.now(), feedback });
|
|
63
84
|
this.append(event);
|
|
64
85
|
this.records.set(this.key(sessionId, messageId), { ...r, feedback, feedbackAt: event.at });
|
|
65
86
|
}
|
|
87
|
+
facts(sessionId, decisionId) { return { ...this.factRecords.get(JSON.stringify([sessionId, decisionId])) }; }
|
|
88
|
+
fact(sessionId, decisionId, fact) {
|
|
89
|
+
const key = JSON.stringify([sessionId, decisionId]);
|
|
90
|
+
const facts = this.factRecords.get(key) ?? {};
|
|
91
|
+
if (facts[fact] !== undefined)
|
|
92
|
+
return;
|
|
93
|
+
const event = factEvent.parse({ type: 'fact', sessionId, decisionId, fact, at: Date.now() });
|
|
94
|
+
this.append(event);
|
|
95
|
+
this.factRecords.set(key, { ...facts, [fact]: event.at });
|
|
96
|
+
}
|
|
66
97
|
append(record) { mkdirSync(dirname(this.file), { recursive: true }); appendFileSync(this.file, JSON.stringify(record) + '\n', { mode: 0o600 }); }
|
|
67
98
|
}
|
package/lib/index.d.ts
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
import type { Context } from '@deepseek-ai/cordis';
|
|
2
|
+
import { type FamilyStrategy } from './strategies/index.js';
|
|
2
3
|
export declare const name = "dsh-auto-model-routing";
|
|
4
|
+
/** Child decision records. The historical directory name is kept on purpose: records are not migrated. */
|
|
5
|
+
export declare const CHILD_DECISIONS_DIR = "state/auto-model-routing/policy-v2";
|
|
3
6
|
export declare const inject: string[];
|
|
4
7
|
/** Test seams only; production uses the defaults. */
|
|
5
8
|
export interface Config {
|
|
6
9
|
home?: string;
|
|
7
10
|
jevBase?: string;
|
|
8
11
|
jevTimeoutMs?: number;
|
|
12
|
+
strategies?: readonly FamilyStrategy[];
|
|
9
13
|
}
|
|
10
14
|
export declare function apply(ctx: Context, config?: Config): void;
|