@danypops/jittor 0.19.3 → 0.21.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -1
- package/src/cli-commands/benchmarks.ts +21 -1
- package/src/constants.ts +29 -0
- package/src/daemon.ts +16 -0
- package/src/index.ts +26 -0
- package/src/observability/model-observation.ts +9 -0
- package/src/observability/task-focus.ts +23 -1
- package/src/observability/usage.ts +12 -0
- package/src/optimization/model-selection/effort.ts +303 -0
- package/src/optimization/model-selection/ranker.ts +3 -1
- package/src/optimization/model-selection/ranking.ts +52 -5
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@danypops/jittor",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.21.2",
|
|
4
4
|
"description": "Just-in-Time Token Optimization Router for Pi -- token and context observability with optimization policies",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -8,10 +8,16 @@
|
|
|
8
8
|
"types": "./src/index.ts",
|
|
9
9
|
"exports": {
|
|
10
10
|
".": "./src/index.ts",
|
|
11
|
+
"./usage": "./src/observability/usage.ts",
|
|
11
12
|
"./package.json": "./package.json"
|
|
12
13
|
},
|
|
13
14
|
"sideEffects": false,
|
|
14
15
|
"keywords": ["llm-router", "token-budget"],
|
|
16
|
+
"zodiac": {
|
|
17
|
+
"integrations": [
|
|
18
|
+
{ "kind": "vehicle-surface", "vehicleName": "jittor", "title": "Jittor" }
|
|
19
|
+
]
|
|
20
|
+
},
|
|
15
21
|
"bin": {
|
|
16
22
|
"jittor": "src/cli.ts"
|
|
17
23
|
},
|
|
@@ -7,7 +7,14 @@ import {
|
|
|
7
7
|
MODEL_RANKING_DEFAULT_RELIABILITY_WEIGHT,
|
|
8
8
|
MODEL_RANKING_MAX_SOURCES,
|
|
9
9
|
} from "../constants.ts";
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
type ModelTaskDomain,
|
|
12
|
+
type ModelTaskEffort,
|
|
13
|
+
type ModelTaskType,
|
|
14
|
+
TASK_DOMAINS,
|
|
15
|
+
TASK_EFFORTS,
|
|
16
|
+
TASK_TYPES,
|
|
17
|
+
} from "../observability/model-observation.ts";
|
|
11
18
|
import type { BenchmarkQuery, BenchmarkQueryResult, BenchmarkRefreshResult } from "../optimization/model-selection/benchmark.ts";
|
|
12
19
|
import type { ModelRecommendationInput } from "../optimization/model-selection/ranker.ts";
|
|
13
20
|
import type { ModelCandidate, ModelRankingResult, ScopeAuthority, UtilityWeights } from "../optimization/model-selection/ranking.ts";
|
|
@@ -34,6 +41,8 @@ function parseBenchmarkArgs(action: string | undefined, args: string[]): Benchma
|
|
|
34
41
|
let scopeAuthority: ScopeAuthority = "available-models";
|
|
35
42
|
let domain: ModelTaskDomain = "general";
|
|
36
43
|
let type: ModelTaskType = "general";
|
|
44
|
+
let effort: ModelTaskEffort = "medium";
|
|
45
|
+
let currentCandidate: ModelCandidate | null = null;
|
|
37
46
|
let budgetPressure = 0;
|
|
38
47
|
let sessionId: string | undefined;
|
|
39
48
|
let sessionSecret: string | undefined;
|
|
@@ -63,6 +72,8 @@ function parseBenchmarkArgs(action: string | undefined, args: string[]): Benchma
|
|
|
63
72
|
"--source",
|
|
64
73
|
"--domain",
|
|
65
74
|
"--type",
|
|
75
|
+
"--effort",
|
|
76
|
+
"--current",
|
|
66
77
|
"--scope",
|
|
67
78
|
"--budget",
|
|
68
79
|
"--weight-quality",
|
|
@@ -98,6 +109,13 @@ function parseBenchmarkArgs(action: string | undefined, args: string[]): Benchma
|
|
|
98
109
|
} else if (argument === "--type") {
|
|
99
110
|
if (!TASK_TYPES.includes(raw as ModelTaskType)) return null;
|
|
100
111
|
type = raw as ModelTaskType;
|
|
112
|
+
} else if (argument === "--effort") {
|
|
113
|
+
if (!TASK_EFFORTS.includes(raw as ModelTaskEffort)) return null;
|
|
114
|
+
effort = raw as ModelTaskEffort;
|
|
115
|
+
} else if (argument === "--current") {
|
|
116
|
+
const candidate = parseCandidate(raw);
|
|
117
|
+
if (!candidate) return null;
|
|
118
|
+
currentCandidate = candidate;
|
|
101
119
|
} else if (argument === "--scope") {
|
|
102
120
|
if (raw !== "exact-session" && raw !== "available-models") return null;
|
|
103
121
|
scopeAuthority = raw;
|
|
@@ -133,6 +151,8 @@ function parseBenchmarkArgs(action: string | undefined, args: string[]): Benchma
|
|
|
133
151
|
scopeAuthority,
|
|
134
152
|
domain,
|
|
135
153
|
type,
|
|
154
|
+
effort,
|
|
155
|
+
currentCandidate,
|
|
136
156
|
budgetPressure,
|
|
137
157
|
weights,
|
|
138
158
|
...(sessionId ? { session_id: sessionId } : {}),
|
package/src/constants.ts
CHANGED
|
@@ -40,7 +40,36 @@ export const MODEL_RANKING_DEFAULT_COST_WEIGHT = 2;
|
|
|
40
40
|
export const MODEL_RANKING_DEFAULT_LATENCY_WEIGHT = 1;
|
|
41
41
|
export const MODEL_RANKING_DEFAULT_CONTEXT_WEIGHT = 1;
|
|
42
42
|
export const MODEL_RANKING_DEFAULT_RELIABILITY_WEIGHT = 2;
|
|
43
|
+
/** A low-effort turn tolerates a cheaper model, so its cost weight is amplified; a high-effort turn tolerates spending more for quality, so its cost weight is dampened. Medium is neutral (matches pre-effort-axis behavior exactly). */
|
|
44
|
+
export const MODEL_RANKING_EFFORT_COST_MULTIPLIER_LOW = 1.5;
|
|
45
|
+
export const MODEL_RANKING_EFFORT_COST_MULTIPLIER_MEDIUM = 1;
|
|
46
|
+
export const MODEL_RANKING_EFFORT_COST_MULTIPLIER_HIGH = 0.5;
|
|
47
|
+
/** Cursor-style uplift gate: a recommendation requires the top candidate to clear a real utility margin over the current model, never just "ranked #1". */
|
|
48
|
+
export const MODEL_RANKING_UPLIFT_MIN_UTILITY_DELTA = 0.05;
|
|
49
|
+
export const MODEL_RANKING_UPLIFT_MIN_CONFIDENCE = 0.5;
|
|
43
50
|
export const MODEL_OBSERVATION_FRESH_MS = 7 * 24 * 60 * 60 * 1_000;
|
|
51
|
+
export const EFFORT_DIMENSION_WEIGHT_TOKEN_COUNT = 0.1;
|
|
52
|
+
export const EFFORT_DIMENSION_WEIGHT_CODE_PRESENCE = 0.2;
|
|
53
|
+
export const EFFORT_DIMENSION_WEIGHT_REASONING_MARKERS = 0.2;
|
|
54
|
+
export const EFFORT_DIMENSION_WEIGHT_TECHNICAL_TERMS = 0.15;
|
|
55
|
+
export const EFFORT_DIMENSION_WEIGHT_SIMPLE_INDICATORS = 0.1;
|
|
56
|
+
export const EFFORT_DIMENSION_WEIGHT_MULTI_STEP_PATTERNS = 0.05;
|
|
57
|
+
export const EFFORT_DIMENSION_WEIGHT_QUESTION_COMPLEXITY = 0.05;
|
|
58
|
+
export const EFFORT_DIMENSION_WEIGHT_TOOL_CALL_MIX = 0.15;
|
|
59
|
+
/** Below this many estimated tokens, a short user message is itself simple-indicator evidence. */
|
|
60
|
+
export const EFFORT_TOKEN_SIMPLE_THRESHOLD = 15;
|
|
61
|
+
/** Above this many estimated tokens, a long user message is itself complexity evidence. */
|
|
62
|
+
export const EFFORT_TOKEN_COMPLEX_THRESHOLD = 400;
|
|
63
|
+
export const EFFORT_LOW_MEDIUM_BOUNDARY = 0.2;
|
|
64
|
+
export const EFFORT_MEDIUM_HIGH_BOUNDARY = 0.45;
|
|
65
|
+
/** 2+ reasoning-marker phrases in the user's own message force "high" regardless of the weighted score -- mirrors LiteLLM's complexity_router reasoning override. */
|
|
66
|
+
export const EFFORT_REASONING_MARKER_OVERRIDE_COUNT = 2;
|
|
67
|
+
/** Prior-turn tool-call counts at or below this are simple-indicator evidence (e.g. a single read, or no tools at all). */
|
|
68
|
+
export const EFFORT_TOOL_CALL_LOW_THRESHOLD = 1;
|
|
69
|
+
/** Prior-turn tool-call counts at or above this are complexity evidence (sustained multi-tool engineering work). */
|
|
70
|
+
export const EFFORT_TOOL_CALL_HIGH_THRESHOLD = 4;
|
|
71
|
+
export const EFFORT_CLASSIFICATION_MAX_TEXT_CHARACTERS = 20_000;
|
|
72
|
+
export const EFFORT_CLASSIFICATION_MAX_TOOL_NAMES = 200;
|
|
44
73
|
export const MAINTENANCE_INTERVAL_MS = 15 * 60 * 1_000;
|
|
45
74
|
export const TELEMETRY_POLL_INTERVAL_MS = 60_000;
|
|
46
75
|
export const TELEMETRY_STALE_AFTER_MS = 120_000;
|
package/src/daemon.ts
CHANGED
|
@@ -137,6 +137,22 @@ export async function startDaemon(
|
|
|
137
137
|
handlePath: paths.handle,
|
|
138
138
|
logger,
|
|
139
139
|
buildApp: () => createApp({ service, token }),
|
|
140
|
+
// Publishes into the shared, cross-package Vehicle Handle Directory
|
|
141
|
+
// (see @danypops/vehicle-server/paths's resolveSharedVehicleHandlePath)
|
|
142
|
+
// -- without this, a discovering caller with no prior knowledge of
|
|
143
|
+
// jittor's own private handlePath (e.g. Zodiac's VehicleSurfaceGateway,
|
|
144
|
+
// resolving purely by vehicleName) can never find a real, running
|
|
145
|
+
// jittor daemon. Papyrus's own hand-rolled daemon.ts already does this
|
|
146
|
+
// explicitly; jittor uses this shared startDaemonKit but was never
|
|
147
|
+
// passing the option that turns the exact same behavior on.
|
|
148
|
+
vehicleName: "jittor",
|
|
149
|
+
tokenPath: paths.token,
|
|
150
|
+
// startDaemon()'s own `env` param was already used for telemetry/
|
|
151
|
+
// benchmark/catalog source detection above, but never forwarded to
|
|
152
|
+
// startDaemonKit itself -- meaning the shared-handle write above would
|
|
153
|
+
// silently always resolve against real process.env regardless of what
|
|
154
|
+
// env a caller (e.g. a test, or a future multi-instance setup) supplied.
|
|
155
|
+
env,
|
|
140
156
|
maintenanceTasks: [
|
|
141
157
|
{
|
|
142
158
|
name: "checkpoint",
|
package/src/index.ts
CHANGED
|
@@ -59,6 +59,23 @@ export {
|
|
|
59
59
|
CONTEXT_TREE_MAX_NODES,
|
|
60
60
|
DATABASE_FILENAME,
|
|
61
61
|
DEFAULT_QUERY_LIMIT,
|
|
62
|
+
EFFORT_CLASSIFICATION_MAX_TEXT_CHARACTERS,
|
|
63
|
+
EFFORT_CLASSIFICATION_MAX_TOOL_NAMES,
|
|
64
|
+
EFFORT_DIMENSION_WEIGHT_CODE_PRESENCE,
|
|
65
|
+
EFFORT_DIMENSION_WEIGHT_MULTI_STEP_PATTERNS,
|
|
66
|
+
EFFORT_DIMENSION_WEIGHT_QUESTION_COMPLEXITY,
|
|
67
|
+
EFFORT_DIMENSION_WEIGHT_REASONING_MARKERS,
|
|
68
|
+
EFFORT_DIMENSION_WEIGHT_SIMPLE_INDICATORS,
|
|
69
|
+
EFFORT_DIMENSION_WEIGHT_TECHNICAL_TERMS,
|
|
70
|
+
EFFORT_DIMENSION_WEIGHT_TOKEN_COUNT,
|
|
71
|
+
EFFORT_DIMENSION_WEIGHT_TOOL_CALL_MIX,
|
|
72
|
+
EFFORT_LOW_MEDIUM_BOUNDARY,
|
|
73
|
+
EFFORT_MEDIUM_HIGH_BOUNDARY,
|
|
74
|
+
EFFORT_REASONING_MARKER_OVERRIDE_COUNT,
|
|
75
|
+
EFFORT_TOKEN_COMPLEX_THRESHOLD,
|
|
76
|
+
EFFORT_TOKEN_SIMPLE_THRESHOLD,
|
|
77
|
+
EFFORT_TOOL_CALL_HIGH_THRESHOLD,
|
|
78
|
+
EFFORT_TOOL_CALL_LOW_THRESHOLD,
|
|
62
79
|
FOOTER_BAR_MAX_WIDTH,
|
|
63
80
|
FOOTER_BAR_MIN_WIDTH,
|
|
64
81
|
FOOTER_COMPACTION_BLINK_HALF_PERIOD_MS,
|
|
@@ -227,15 +244,18 @@ export {
|
|
|
227
244
|
type ModelRunObservation,
|
|
228
245
|
type ModelTaskClassification,
|
|
229
246
|
type ModelTaskDomain,
|
|
247
|
+
type ModelTaskEffort,
|
|
230
248
|
type ModelTaskType,
|
|
231
249
|
modelRunMetrics,
|
|
232
250
|
TASK_DOMAINS,
|
|
251
|
+
TASK_EFFORTS,
|
|
233
252
|
TASK_TYPES,
|
|
234
253
|
validateModelRunObservation,
|
|
235
254
|
} from "./observability/model-observation.ts";
|
|
236
255
|
export type { DistinctScopesFilter, MetricStore, UsageAggregateFilter } from "./observability/store.ts";
|
|
237
256
|
export {
|
|
238
257
|
applyTaskFocusEvent,
|
|
258
|
+
declaredEffortFromTaskFocusEvent,
|
|
239
259
|
type TaskFocusEvent,
|
|
240
260
|
type TaskFocusStatus,
|
|
241
261
|
validateTaskFocusEvent,
|
|
@@ -302,6 +322,12 @@ export {
|
|
|
302
322
|
ModelsDevCatalogSource,
|
|
303
323
|
translateModelsDevCatalog,
|
|
304
324
|
} from "./optimization/model-selection/catalog.ts";
|
|
325
|
+
export {
|
|
326
|
+
classifyEffort,
|
|
327
|
+
type EffortClassification,
|
|
328
|
+
type EffortClassificationInput,
|
|
329
|
+
type EffortClassifierOptions,
|
|
330
|
+
} from "./optimization/model-selection/effort.ts";
|
|
305
331
|
export {
|
|
306
332
|
type ModelCandidate,
|
|
307
333
|
type ModelRankingInput,
|
|
@@ -21,6 +21,15 @@ export const TASK_DOMAINS = ["coding", "design", "math", "general"] as const;
|
|
|
21
21
|
export type ModelTaskDomain = (typeof TASK_DOMAINS)[number];
|
|
22
22
|
export const TASK_TYPES = ["research", "planning", "general"] as const;
|
|
23
23
|
export type ModelTaskType = (typeof TASK_TYPES)[number];
|
|
24
|
+
/**
|
|
25
|
+
* A third, independent axis from domain/type: how much reasoning/engineering effort a turn
|
|
26
|
+
* needs, which drives model-tier selection (a cheap fast model for "low", a frontier model for
|
|
27
|
+
* "high") the way domain/type drive quality-evidence matching. See
|
|
28
|
+
* `optimization/model-selection/effort.ts` for the classifier that derives this from structural
|
|
29
|
+
* signals -- this file only owns the type, not the classification logic.
|
|
30
|
+
*/
|
|
31
|
+
export const TASK_EFFORTS = ["low", "medium", "high"] as const;
|
|
32
|
+
export type ModelTaskEffort = (typeof TASK_EFFORTS)[number];
|
|
24
33
|
export type ExplicitOutcome = "accepted" | "rejected" | "unknown";
|
|
25
34
|
|
|
26
35
|
export interface ModelRunObservation {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { PAPYRUS_TASK_FOCUS_SCHEMA, TASK_FOCUS_EVENT_MAX_AGE_MS, TASK_FOCUS_ID_MAX_LENGTH } from "../constants.ts";
|
|
2
|
+
import { type ModelTaskEffort, TASK_EFFORTS } from "./model-observation.ts";
|
|
2
3
|
|
|
3
4
|
export type TaskFocusStatus = "focused" | "paused" | "unpaused" | "cleared";
|
|
4
5
|
|
|
@@ -8,9 +9,16 @@ export interface TaskFocusEvent {
|
|
|
8
9
|
sessionId?: string;
|
|
9
10
|
status: TaskFocusStatus;
|
|
10
11
|
observedAt: number;
|
|
12
|
+
/**
|
|
13
|
+
* Additive to the papyrus.task-focus/v1 schema: an older Jittor build predates this field, but
|
|
14
|
+
* since it's optional here (never required) and this validator only rejects a field it has
|
|
15
|
+
* never heard of, a coordinated same-session rollout across both packages is what keeps that
|
|
16
|
+
* safe -- see the recorded compatibility decision in the effort-aware Auto mode design Doc.
|
|
17
|
+
*/
|
|
18
|
+
effort?: ModelTaskEffort;
|
|
11
19
|
}
|
|
12
20
|
|
|
13
|
-
const TOP_LEVEL_FIELDS = new Set(["schema", "taskId", "sessionId", "status", "observedAt"]);
|
|
21
|
+
const TOP_LEVEL_FIELDS = new Set(["schema", "taskId", "sessionId", "status", "observedAt", "effort"]);
|
|
14
22
|
const STATUSES = new Set<string>(["focused", "paused", "unpaused", "cleared"]);
|
|
15
23
|
|
|
16
24
|
function record(value: unknown): Record<string, unknown> {
|
|
@@ -48,12 +56,16 @@ export function validateTaskFocusEvent(value: unknown, now = Date.now()): TaskFo
|
|
|
48
56
|
if (taskId === null && status !== "cleared") throw new Error(`task-focus event of status "${status}" requires a taskId`);
|
|
49
57
|
const rawSessionId = input.sessionId;
|
|
50
58
|
const sessionId = rawSessionId === undefined ? undefined : boundedId(rawSessionId, "sessionId");
|
|
59
|
+
const rawEffort = input.effort;
|
|
60
|
+
if (rawEffort !== undefined && !TASK_EFFORTS.includes(rawEffort as ModelTaskEffort))
|
|
61
|
+
throw new Error("task-focus event effort is invalid");
|
|
51
62
|
return {
|
|
52
63
|
schema: PAPYRUS_TASK_FOCUS_SCHEMA,
|
|
53
64
|
taskId,
|
|
54
65
|
status: status as TaskFocusStatus,
|
|
55
66
|
observedAt,
|
|
56
67
|
...(sessionId === undefined ? {} : { sessionId }),
|
|
68
|
+
...(rawEffort === undefined ? {} : { effort: rawEffort as ModelTaskEffort }),
|
|
57
69
|
};
|
|
58
70
|
}
|
|
59
71
|
|
|
@@ -66,3 +78,13 @@ export function validateTaskFocusEvent(value: unknown, now = Date.now()): TaskFo
|
|
|
66
78
|
export function applyTaskFocusEvent(event: TaskFocusEvent): string | null {
|
|
67
79
|
return event.status === "focused" || event.status === "unpaused" ? event.taskId : null;
|
|
68
80
|
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The task-effort analogue of applyTaskFocusEvent: a declared effort is a live "bind-beforehand"
|
|
84
|
+
* routing prior only while its task is actually focused, using the identical focused/unpaused
|
|
85
|
+
* vs. paused/cleared semantics -- a task that declared "high" while paused must not keep pinning
|
|
86
|
+
* routing to "high" once it's no longer the thing being worked on.
|
|
87
|
+
*/
|
|
88
|
+
export function declaredEffortFromTaskFocusEvent(event: TaskFocusEvent): ModelTaskEffort | null {
|
|
89
|
+
return (event.status === "focused" || event.status === "unpaused") && event.effort !== undefined ? event.effort : null;
|
|
90
|
+
}
|
|
@@ -1,3 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Exposed as its own package.json subpath export ("@danypops/jittor/usage")
|
|
3
|
+
* alongside the main barrel: this module and its own single dependency
|
|
4
|
+
* (constants.ts) are the only fully browser/Node-tsc-safe slice of jittor's
|
|
5
|
+
* source tree (pure functions and types, no bun:sqlite, no Bun-only
|
|
6
|
+
* globals) -- importing the whole barrel instead drags in every other
|
|
7
|
+
* module's own raw `.ts` import specifiers, which a consumer's tsc rejects
|
|
8
|
+
* outright (TS5097) unless it enables allowImportingTsExtensions project-
|
|
9
|
+
* wide. A consumer that only needs buildUsageGraph/buildCostGraph/
|
|
10
|
+
* resolveUsageWindow (e.g. Zodiac's own React usage meter) should import
|
|
11
|
+
* this subpath directly, not the main entry.
|
|
12
|
+
*/
|
|
1
13
|
import { MAX_USAGE_BUCKETS, MILLISECONDS_PER_DAY, MILLISECONDS_PER_HOUR } from "../constants.ts";
|
|
2
14
|
|
|
3
15
|
export const USAGE_PERIODS = [
|
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
import {
|
|
2
|
+
EFFORT_CLASSIFICATION_MAX_TEXT_CHARACTERS,
|
|
3
|
+
EFFORT_CLASSIFICATION_MAX_TOOL_NAMES,
|
|
4
|
+
EFFORT_DIMENSION_WEIGHT_CODE_PRESENCE,
|
|
5
|
+
EFFORT_DIMENSION_WEIGHT_MULTI_STEP_PATTERNS,
|
|
6
|
+
EFFORT_DIMENSION_WEIGHT_QUESTION_COMPLEXITY,
|
|
7
|
+
EFFORT_DIMENSION_WEIGHT_REASONING_MARKERS,
|
|
8
|
+
EFFORT_DIMENSION_WEIGHT_SIMPLE_INDICATORS,
|
|
9
|
+
EFFORT_DIMENSION_WEIGHT_TECHNICAL_TERMS,
|
|
10
|
+
EFFORT_DIMENSION_WEIGHT_TOKEN_COUNT,
|
|
11
|
+
EFFORT_DIMENSION_WEIGHT_TOOL_CALL_MIX,
|
|
12
|
+
EFFORT_LOW_MEDIUM_BOUNDARY,
|
|
13
|
+
EFFORT_MEDIUM_HIGH_BOUNDARY,
|
|
14
|
+
EFFORT_REASONING_MARKER_OVERRIDE_COUNT,
|
|
15
|
+
EFFORT_TOKEN_COMPLEX_THRESHOLD,
|
|
16
|
+
EFFORT_TOKEN_SIMPLE_THRESHOLD,
|
|
17
|
+
EFFORT_TOOL_CALL_HIGH_THRESHOLD,
|
|
18
|
+
EFFORT_TOOL_CALL_LOW_THRESHOLD,
|
|
19
|
+
} from "../../constants.ts";
|
|
20
|
+
import type { ModelTaskEffort } from "../../observability/model-observation.ts";
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Structural/lexical effort classifier -- zero network calls, no persisted prompt content,
|
|
24
|
+
* deterministic, bounded, sub-millisecond. Modeled on LiteLLM's real merged `complexity_router`
|
|
25
|
+
* (word-boundary keyword matching, non-greedy ReDoS-safe multi-step regex, a reasoning-marker
|
|
26
|
+
* override), with two deliberate departures recorded here rather than silently made:
|
|
27
|
+
*
|
|
28
|
+
* 1. Only the user's own current-turn message and the prior turn's tool-call mix are scored --
|
|
29
|
+
* never Pi's own system prompt. LiteLLM's design assumes a short, custom system prompt; Pi's
|
|
30
|
+
* real system prompt is a large, fixed agent-harness prompt full of tool descriptions that
|
|
31
|
+
* would permanently saturate the code/technical dimensions if scanned. Scoring only the
|
|
32
|
+
* user's own fresh text avoids that false-signal-forever failure mode.
|
|
33
|
+
* 2. Token count uses a plain char/4 structural estimate by default (matching
|
|
34
|
+
* CONTEXT_ESTIMATE_CHARACTERS_PER_TOKEN elsewhere in this codebase), not a real tokenizer --
|
|
35
|
+
* a real per-provider tokenizer is a replaceable integration per this project's own
|
|
36
|
+
* architecture, not something a turn-boundary classifier should hard-depend on. Pass
|
|
37
|
+
* `estimateTokens` to plug in a better one without changing this module's contract.
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
export interface EffortClassificationInput {
|
|
41
|
+
/** The current turn's own user message. Never retained beyond this call. */
|
|
42
|
+
userText: string;
|
|
43
|
+
/** Tool names actually called during the immediately preceding turn, if any. */
|
|
44
|
+
priorTurnToolNames?: string[];
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface EffortClassifierOptions {
|
|
48
|
+
/** Overrides the default char/4 structural estimate, e.g. with a real per-provider tokenizer. */
|
|
49
|
+
estimateTokens?: (text: string) => number;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface EffortClassification {
|
|
53
|
+
effort: ModelTaskEffort;
|
|
54
|
+
/** The raw weighted score before tier-boundary mapping; not itself bounded to 0..1 (an override can exceed it). */
|
|
55
|
+
score: number;
|
|
56
|
+
/** Human-readable evidence, one entry per dimension that actually contributed -- never a bare label with no reasoning. Never contains the raw input text. */
|
|
57
|
+
signals: string[];
|
|
58
|
+
/** True when the reasoning-marker override forced "high" regardless of the weighted score. */
|
|
59
|
+
override: boolean;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
interface DimensionScore {
|
|
63
|
+
name: string;
|
|
64
|
+
score: number;
|
|
65
|
+
weight: number;
|
|
66
|
+
evidence: string | null;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Single-word keywords use word-boundary matching (avoids "api" matching "capital", "class"
|
|
70
|
+
// matching "classical"); multi-word phrases use substring matching. Mirrors LiteLLM's own
|
|
71
|
+
// complexity_router keyword-matching technique, verified against its real merged source.
|
|
72
|
+
const CODE_KEYWORDS = [
|
|
73
|
+
"function",
|
|
74
|
+
"class",
|
|
75
|
+
"async",
|
|
76
|
+
"await",
|
|
77
|
+
"import",
|
|
78
|
+
"export",
|
|
79
|
+
"api",
|
|
80
|
+
"endpoint",
|
|
81
|
+
"database",
|
|
82
|
+
"query",
|
|
83
|
+
"schema",
|
|
84
|
+
"algorithm",
|
|
85
|
+
"refactor",
|
|
86
|
+
"debug",
|
|
87
|
+
"python",
|
|
88
|
+
"typescript",
|
|
89
|
+
"javascript",
|
|
90
|
+
"docker",
|
|
91
|
+
"kubernetes",
|
|
92
|
+
"git",
|
|
93
|
+
"regex",
|
|
94
|
+
];
|
|
95
|
+
const REASONING_KEYWORDS = [
|
|
96
|
+
"step by step",
|
|
97
|
+
"think through",
|
|
98
|
+
"reason through",
|
|
99
|
+
"analyze this",
|
|
100
|
+
"break down",
|
|
101
|
+
"break it down",
|
|
102
|
+
"explain your reasoning",
|
|
103
|
+
"show your work",
|
|
104
|
+
"chain of thought",
|
|
105
|
+
"think carefully",
|
|
106
|
+
"weigh the options",
|
|
107
|
+
"root cause",
|
|
108
|
+
];
|
|
109
|
+
const TECHNICAL_KEYWORDS = [
|
|
110
|
+
"architecture",
|
|
111
|
+
"distributed",
|
|
112
|
+
"scalable",
|
|
113
|
+
"microservice",
|
|
114
|
+
"encryption",
|
|
115
|
+
"authentication",
|
|
116
|
+
"authorization",
|
|
117
|
+
"performance",
|
|
118
|
+
"latency",
|
|
119
|
+
"throughput",
|
|
120
|
+
"concurrency",
|
|
121
|
+
"orchestration",
|
|
122
|
+
"protocol",
|
|
123
|
+
];
|
|
124
|
+
const SIMPLE_KEYWORDS = [
|
|
125
|
+
"what is",
|
|
126
|
+
"what's",
|
|
127
|
+
"define",
|
|
128
|
+
"who is",
|
|
129
|
+
"who was",
|
|
130
|
+
"when did",
|
|
131
|
+
"when was",
|
|
132
|
+
"how many",
|
|
133
|
+
"hello",
|
|
134
|
+
"hi",
|
|
135
|
+
"thanks",
|
|
136
|
+
"thank you",
|
|
137
|
+
];
|
|
138
|
+
|
|
139
|
+
const MULTI_STEP_PATTERNS = [/first.*?then/i, /step\s*\d/i, /\d+\.\s/, /[a-z]\)\s/i];
|
|
140
|
+
|
|
141
|
+
function boundedText(value: string): string {
|
|
142
|
+
return value.slice(0, EFFORT_CLASSIFICATION_MAX_TEXT_CHARACTERS);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function wordBoundaryOrSubstringMatches(text: string, keyword: string): boolean {
|
|
146
|
+
if (!keyword.includes(" ")) return new RegExp(`\\b${keyword.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`).test(text);
|
|
147
|
+
return text.includes(keyword);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function countMatches(text: string, keywords: string[]): { count: number; matched: string[] } {
|
|
151
|
+
const matched = keywords.filter((keyword) => wordBoundaryOrSubstringMatches(text, keyword));
|
|
152
|
+
return { count: matched.length, matched };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function estimateTokensStructural(text: string): number {
|
|
156
|
+
return Math.ceil(text.length / 4);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function scoreTokenCount(tokens: number): DimensionScore {
|
|
160
|
+
if (tokens < EFFORT_TOKEN_SIMPLE_THRESHOLD)
|
|
161
|
+
return {
|
|
162
|
+
name: "tokenCount",
|
|
163
|
+
score: -1,
|
|
164
|
+
weight: EFFORT_DIMENSION_WEIGHT_TOKEN_COUNT,
|
|
165
|
+
evidence: `short user message (${tokens} est. tokens)`,
|
|
166
|
+
};
|
|
167
|
+
if (tokens > EFFORT_TOKEN_COMPLEX_THRESHOLD)
|
|
168
|
+
return {
|
|
169
|
+
name: "tokenCount",
|
|
170
|
+
score: 1,
|
|
171
|
+
weight: EFFORT_DIMENSION_WEIGHT_TOKEN_COUNT,
|
|
172
|
+
evidence: `long user message (${tokens} est. tokens)`,
|
|
173
|
+
};
|
|
174
|
+
return { name: "tokenCount", score: 0, weight: EFFORT_DIMENSION_WEIGHT_TOKEN_COUNT, evidence: null };
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function scoreKeywordDimension(
|
|
178
|
+
text: string,
|
|
179
|
+
keywords: string[],
|
|
180
|
+
name: string,
|
|
181
|
+
weight: number,
|
|
182
|
+
direction: 1 | -1,
|
|
183
|
+
label: string,
|
|
184
|
+
): { dimension: DimensionScore; count: number } {
|
|
185
|
+
const { count, matched } = countMatches(text, keywords);
|
|
186
|
+
const score = count === 0 ? 0 : count === 1 ? 0.5 * direction : 1 * direction;
|
|
187
|
+
return {
|
|
188
|
+
dimension: { name, score, weight, evidence: count === 0 ? null : `${label} (${matched.slice(0, 3).join(", ")})` },
|
|
189
|
+
count,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function scoreMultiStep(text: string): DimensionScore {
|
|
194
|
+
const hit = MULTI_STEP_PATTERNS.some((pattern) => pattern.test(text));
|
|
195
|
+
return {
|
|
196
|
+
name: "multiStepPatterns",
|
|
197
|
+
score: hit ? 1 : 0,
|
|
198
|
+
weight: EFFORT_DIMENSION_WEIGHT_MULTI_STEP_PATTERNS,
|
|
199
|
+
evidence: hit ? "multi-step structure detected" : null,
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function scoreQuestionComplexity(text: string): DimensionScore {
|
|
204
|
+
const count = (text.match(/\?/g) ?? []).length;
|
|
205
|
+
return {
|
|
206
|
+
name: "questionComplexity",
|
|
207
|
+
score: count > 3 ? 1 : 0,
|
|
208
|
+
weight: EFFORT_DIMENSION_WEIGHT_QUESTION_COMPLEXITY,
|
|
209
|
+
evidence: count > 3 ? `${count} questions in one message` : null,
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Prior-turn tool-call volume as an effort signal: sustained multi-tool engineering work (several
|
|
215
|
+
* distinct tool calls last turn) is real evidence of an in-progress complex task; zero or one
|
|
216
|
+
* tool call is evidence of a light, simple exchange. Bounded, structural, content-free --
|
|
217
|
+
* tool names only, never their arguments or results.
|
|
218
|
+
*/
|
|
219
|
+
function scoreToolCallMix(toolNames: string[]): DimensionScore {
|
|
220
|
+
const distinct = new Set(toolNames.slice(0, EFFORT_CLASSIFICATION_MAX_TOOL_NAMES).map((name) => name.toLowerCase()));
|
|
221
|
+
if (distinct.size <= EFFORT_TOOL_CALL_LOW_THRESHOLD)
|
|
222
|
+
return {
|
|
223
|
+
name: "toolCallMix",
|
|
224
|
+
score: -1,
|
|
225
|
+
weight: EFFORT_DIMENSION_WEIGHT_TOOL_CALL_MIX,
|
|
226
|
+
evidence: distinct.size === 0 ? "no prior tool activity" : "single prior tool call",
|
|
227
|
+
};
|
|
228
|
+
if (distinct.size >= EFFORT_TOOL_CALL_HIGH_THRESHOLD)
|
|
229
|
+
return {
|
|
230
|
+
name: "toolCallMix",
|
|
231
|
+
score: 1,
|
|
232
|
+
weight: EFFORT_DIMENSION_WEIGHT_TOOL_CALL_MIX,
|
|
233
|
+
evidence: `${distinct.size} distinct tools used last turn`,
|
|
234
|
+
};
|
|
235
|
+
return { name: "toolCallMix", score: 0, weight: EFFORT_DIMENSION_WEIGHT_TOOL_CALL_MIX, evidence: null };
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
function tierFor(score: number): ModelTaskEffort {
|
|
239
|
+
if (score < EFFORT_LOW_MEDIUM_BOUNDARY) return "low";
|
|
240
|
+
if (score < EFFORT_MEDIUM_HIGH_BOUNDARY) return "medium";
|
|
241
|
+
return "high";
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
export function classifyEffort(input: EffortClassificationInput, options: EffortClassifierOptions = {}): EffortClassification {
|
|
245
|
+
const userText = boundedText(typeof input.userText === "string" ? input.userText : "");
|
|
246
|
+
const toolNames = Array.isArray(input.priorTurnToolNames) ? input.priorTurnToolNames : [];
|
|
247
|
+
const estimateTokens = options.estimateTokens ?? estimateTokensStructural;
|
|
248
|
+
|
|
249
|
+
const code = scoreKeywordDimension(
|
|
250
|
+
userText,
|
|
251
|
+
CODE_KEYWORDS,
|
|
252
|
+
"codePresence",
|
|
253
|
+
EFFORT_DIMENSION_WEIGHT_CODE_PRESENCE,
|
|
254
|
+
1,
|
|
255
|
+
"code-related terms",
|
|
256
|
+
);
|
|
257
|
+
const reasoning = scoreKeywordDimension(
|
|
258
|
+
userText,
|
|
259
|
+
REASONING_KEYWORDS,
|
|
260
|
+
"reasoningMarkers",
|
|
261
|
+
EFFORT_DIMENSION_WEIGHT_REASONING_MARKERS,
|
|
262
|
+
1,
|
|
263
|
+
"reasoning markers",
|
|
264
|
+
);
|
|
265
|
+
const technical = scoreKeywordDimension(
|
|
266
|
+
userText,
|
|
267
|
+
TECHNICAL_KEYWORDS,
|
|
268
|
+
"technicalTerms",
|
|
269
|
+
EFFORT_DIMENSION_WEIGHT_TECHNICAL_TERMS,
|
|
270
|
+
1,
|
|
271
|
+
"technical terms",
|
|
272
|
+
);
|
|
273
|
+
const simple = scoreKeywordDimension(
|
|
274
|
+
userText,
|
|
275
|
+
SIMPLE_KEYWORDS,
|
|
276
|
+
"simpleIndicators",
|
|
277
|
+
EFFORT_DIMENSION_WEIGHT_SIMPLE_INDICATORS,
|
|
278
|
+
-1,
|
|
279
|
+
"simple-query phrasing",
|
|
280
|
+
);
|
|
281
|
+
|
|
282
|
+
const dimensions: DimensionScore[] = [
|
|
283
|
+
scoreTokenCount(estimateTokens(userText)),
|
|
284
|
+
code.dimension,
|
|
285
|
+
reasoning.dimension,
|
|
286
|
+
technical.dimension,
|
|
287
|
+
simple.dimension,
|
|
288
|
+
scoreMultiStep(userText),
|
|
289
|
+
scoreQuestionComplexity(userText),
|
|
290
|
+
scoreToolCallMix(toolNames),
|
|
291
|
+
];
|
|
292
|
+
|
|
293
|
+
const score = dimensions.reduce((sum, dimension) => sum + dimension.score * dimension.weight, 0);
|
|
294
|
+
const signals = dimensions.filter((dimension) => dimension.evidence !== null).map((dimension) => dimension.evidence as string);
|
|
295
|
+
const override = reasoning.count >= EFFORT_REASONING_MARKER_OVERRIDE_COUNT;
|
|
296
|
+
|
|
297
|
+
return {
|
|
298
|
+
effort: override ? "high" : tierFor(score),
|
|
299
|
+
score,
|
|
300
|
+
signals,
|
|
301
|
+
override,
|
|
302
|
+
};
|
|
303
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { MODEL_AGGREGATE_MAX_ROWS, MODEL_OBSERVATION_FRESH_MS, MODEL_RANKING_MAX_SOURCES } from "../../constants.ts";
|
|
2
|
-
import type { ModelTaskDomain, ModelTaskType } from "../../observability/model-observation.ts";
|
|
2
|
+
import type { ModelTaskDomain, ModelTaskEffort, ModelTaskType } from "../../observability/model-observation.ts";
|
|
3
3
|
import { aggregateModelMetrics } from "../../observability/model-observation.ts";
|
|
4
4
|
import type { MetricStore } from "../../observability/store.ts";
|
|
5
5
|
import { type ModelCandidate, type ModelRankingResult, rankModelCandidates, type ScopeAuthority, type UtilityWeights } from "./ranking.ts";
|
|
@@ -10,6 +10,8 @@ export interface ModelRecommendationInput {
|
|
|
10
10
|
scopeAuthority: ScopeAuthority;
|
|
11
11
|
domain: ModelTaskDomain;
|
|
12
12
|
type: ModelTaskType;
|
|
13
|
+
effort: ModelTaskEffort;
|
|
14
|
+
currentCandidate: ModelCandidate | null;
|
|
13
15
|
budgetPressure: number;
|
|
14
16
|
weights: UtilityWeights;
|
|
15
17
|
sourceIds: string[];
|
|
@@ -1,9 +1,20 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
BENCHMARK_MAX_OBSERVATIONS_PER_SNAPSHOT,
|
|
3
|
+
MAX_DYNAMIC_ROUTES,
|
|
4
|
+
MODEL_AGGREGATE_MAX_ROWS,
|
|
5
|
+
MODEL_RANKING_EFFORT_COST_MULTIPLIER_HIGH,
|
|
6
|
+
MODEL_RANKING_EFFORT_COST_MULTIPLIER_LOW,
|
|
7
|
+
MODEL_RANKING_EFFORT_COST_MULTIPLIER_MEDIUM,
|
|
8
|
+
MODEL_RANKING_UPLIFT_MIN_CONFIDENCE,
|
|
9
|
+
MODEL_RANKING_UPLIFT_MIN_UTILITY_DELTA,
|
|
10
|
+
} from "../../constants.ts";
|
|
2
11
|
import {
|
|
3
12
|
type ModelMetricAggregate,
|
|
4
13
|
type ModelTaskDomain,
|
|
14
|
+
type ModelTaskEffort,
|
|
5
15
|
type ModelTaskType,
|
|
6
16
|
TASK_DOMAINS,
|
|
17
|
+
TASK_EFFORTS,
|
|
7
18
|
TASK_TYPES,
|
|
8
19
|
} from "../../observability/model-observation.ts";
|
|
9
20
|
import { type BenchmarkObservation, normalizeModelIdentity } from "./benchmark.ts";
|
|
@@ -30,6 +41,10 @@ export interface ModelRankingInput {
|
|
|
30
41
|
scopeAuthority: ScopeAuthority;
|
|
31
42
|
domain: ModelTaskDomain;
|
|
32
43
|
type: ModelTaskType;
|
|
44
|
+
/** Predicts how much reasoning/engineering effort this turn needs; shifts the effective cost weight (see MODEL_RANKING_EFFORT_COST_MULTIPLIER_*) the same way budgetPressure already does. */
|
|
45
|
+
effort: ModelTaskEffort;
|
|
46
|
+
/** The model actually in use right now -- the uplift gate's baseline. A recommendation requires a real, evidence-backed improvement over this, never just "ranked #1". Null when there is no active model to compare against (pure ranking still proceeds; there is simply nothing to gate an uplift over, so recommendation stays null). */
|
|
47
|
+
currentCandidate: ModelCandidate | null;
|
|
33
48
|
budgetPressure: number;
|
|
34
49
|
weights: UtilityWeights;
|
|
35
50
|
externalEvidence: BenchmarkObservation[];
|
|
@@ -37,6 +52,12 @@ export interface ModelRankingInput {
|
|
|
37
52
|
now: number;
|
|
38
53
|
}
|
|
39
54
|
|
|
55
|
+
export interface ModelRankingRecommendation {
|
|
56
|
+
candidate: ModelCandidate;
|
|
57
|
+
utilityDelta: number;
|
|
58
|
+
confidence: number;
|
|
59
|
+
}
|
|
60
|
+
|
|
40
61
|
export interface UtilityComponent {
|
|
41
62
|
name: UtilityComponentName;
|
|
42
63
|
score: number | null;
|
|
@@ -71,6 +92,9 @@ export interface ModelRankingResult {
|
|
|
71
92
|
type: ModelTaskType;
|
|
72
93
|
completeness: "complete" | "partial" | "insufficient-evidence";
|
|
73
94
|
ranked: RankedModel[];
|
|
95
|
+
/** A candidate that cleared the uplift gate (real utility margin + confidence over currentCandidate), regardless of scope authority -- the evidence question, independent of whether automation is currently allowed. Null when the current model is already the best choice, or no candidate clears the gate. */
|
|
96
|
+
recommendation: ModelRankingRecommendation | null;
|
|
97
|
+
/** recommendation's candidate, but only once scopeAuthority is "exact-session" -- the governance question layered on top of the evidence question. */
|
|
74
98
|
automaticSelection: ModelCandidate | null;
|
|
75
99
|
}
|
|
76
100
|
|
|
@@ -226,11 +250,19 @@ export function rankModelCandidates(value: ModelRankingInput): ModelRankingResul
|
|
|
226
250
|
throw new Error("scope authority is invalid");
|
|
227
251
|
if (!TASK_DOMAINS.includes(value.domain)) throw new Error("task domain is invalid");
|
|
228
252
|
if (!TASK_TYPES.includes(value.type)) throw new Error("task type is invalid");
|
|
253
|
+
if (!TASK_EFFORTS.includes(value.effort)) throw new Error("task effort is invalid");
|
|
254
|
+
const currentIdentity = value.currentCandidate === null ? null : candidateIdentity(value.currentCandidate);
|
|
229
255
|
if (!Number.isSafeInteger(value.now) || value.now <= 0) throw new Error("ranking time is invalid");
|
|
230
256
|
const budgetPressure = finiteBound(value.budgetPressure, "budget pressure", 0, 2);
|
|
231
257
|
const weights = Object.fromEntries(
|
|
232
258
|
COMPONENTS.map((name) => [name, finiteBound(value.weights[name], `${name} weight`, 0, 10)]),
|
|
233
259
|
) as unknown as UtilityWeights;
|
|
260
|
+
const effortCostMultiplier =
|
|
261
|
+
value.effort === "low"
|
|
262
|
+
? MODEL_RANKING_EFFORT_COST_MULTIPLIER_LOW
|
|
263
|
+
: value.effort === "high"
|
|
264
|
+
? MODEL_RANKING_EFFORT_COST_MULTIPLIER_HIGH
|
|
265
|
+
: MODEL_RANKING_EFFORT_COST_MULTIPLIER_MEDIUM;
|
|
234
266
|
const seen = new Set<string>();
|
|
235
267
|
const candidates = value.candidates
|
|
236
268
|
.map((candidate) => ({ ...candidate }))
|
|
@@ -241,7 +273,7 @@ export function rankModelCandidates(value: ModelRankingInput): ModelRankingResul
|
|
|
241
273
|
return true;
|
|
242
274
|
});
|
|
243
275
|
const raw = candidates.map((candidate) => rawComponents(candidate, value));
|
|
244
|
-
const effectiveWeights: UtilityWeights = { ...weights, cost: weights.cost * (1 + budgetPressure) };
|
|
276
|
+
const effectiveWeights: UtilityWeights = { ...weights, cost: weights.cost * (1 + budgetPressure) * effortCostMultiplier };
|
|
245
277
|
const ranked = candidates
|
|
246
278
|
.map((candidate, index): RankedModel => {
|
|
247
279
|
const source = raw[index]!;
|
|
@@ -277,8 +309,8 @@ export function rankModelCandidates(value: ModelRankingInput): ModelRankingResul
|
|
|
277
309
|
components,
|
|
278
310
|
provenance,
|
|
279
311
|
trace: [
|
|
280
|
-
`domain ${value.domain}, type ${value.type}`,
|
|
281
|
-
`budget pressure ${budgetPressure.toFixed(3)}
|
|
312
|
+
`domain ${value.domain}, type ${value.type}, effort ${value.effort}`,
|
|
313
|
+
`budget pressure ${budgetPressure.toFixed(3)} and effort ${value.effort} make cost weight ${effectiveWeights.cost.toFixed(3)}`,
|
|
282
314
|
`${known.length}/${components.length} utility components have evidence`,
|
|
283
315
|
`scope authority ${value.scopeAuthority}`,
|
|
284
316
|
],
|
|
@@ -292,6 +324,20 @@ export function rankModelCandidates(value: ModelRankingInput): ModelRankingResul
|
|
|
292
324
|
const possibleComponents = ranked.length * COMPONENTS.length;
|
|
293
325
|
const completeness = knownComponents === 0 ? "insufficient-evidence" : knownComponents === possibleComponents ? "complete" : "partial";
|
|
294
326
|
const exact = value.scopeAuthority === "exact-session";
|
|
327
|
+
const top = ranked[0];
|
|
328
|
+
const current = currentIdentity === null ? undefined : ranked.find((item) => item.identity === currentIdentity);
|
|
329
|
+
// Uplift gate (Cursor-style): a recommendation requires a real, evidence-backed improvement
|
|
330
|
+
// over the current model -- never just "ranked #1". Without a located current baseline (it
|
|
331
|
+
// wasn't among the input candidates) there is nothing to uplift over, so no recommendation.
|
|
332
|
+
const recommendation: ModelRankingRecommendation | null =
|
|
333
|
+
top && current && top.identity !== current.identity && top.utility !== null && current.utility !== null
|
|
334
|
+
? (() => {
|
|
335
|
+
const utilityDelta = top.utility! - current.utility!;
|
|
336
|
+
return utilityDelta >= MODEL_RANKING_UPLIFT_MIN_UTILITY_DELTA && top.confidence >= MODEL_RANKING_UPLIFT_MIN_CONFIDENCE
|
|
337
|
+
? { candidate: top.candidate, utilityDelta, confidence: top.confidence }
|
|
338
|
+
: null;
|
|
339
|
+
})()
|
|
340
|
+
: null;
|
|
295
341
|
return {
|
|
296
342
|
scopeAuthority: value.scopeAuthority,
|
|
297
343
|
scopeWarning: exact ? null : "Pi available models are not the exact session scope; automatic selection is disabled",
|
|
@@ -299,6 +345,7 @@ export function rankModelCandidates(value: ModelRankingInput): ModelRankingResul
|
|
|
299
345
|
type: value.type,
|
|
300
346
|
completeness,
|
|
301
347
|
ranked,
|
|
302
|
-
|
|
348
|
+
recommendation,
|
|
349
|
+
automaticSelection: exact && recommendation ? recommendation.candidate : null,
|
|
303
350
|
};
|
|
304
351
|
}
|