lynkr 9.14.12 → 9.14.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/difficulty-anchors.json +12 -2
- package/config/model-capability-seeds.json +4 -0
- package/package.json +3 -3
- package/scripts/validate-intent-anchors.js +2 -2
- package/src/api/router.js +370 -16
- package/src/clients/databricks.js +64 -23
- package/src/clients/openai-format.js +37 -2
- package/src/clients/openrouter-utils.js +20 -0
- package/src/orchestrator/index.js +47 -10
- package/src/prompts/system.js +41 -49
- package/src/routing/affinity-store.js +47 -21
- package/src/routing/cache-switch-cost.js +46 -2
- package/src/routing/complexity-analyzer.js +21 -4
- package/src/routing/index.js +121 -1
- package/src/routing/intent-score.js +48 -13
- package/src/routing/jev-router.js +272 -0
- package/src/routing/model-tiers.js +13 -0
- package/src/routing/passthrough-route.js +286 -0
- package/src/routing/quota-ledger.js +110 -0
- package/src/routing/risk-analyzer.js +17 -1
- package/src/routing/telemetry.js +47 -2
|
@@ -61,12 +61,23 @@ function _cfg() {
|
|
|
61
61
|
* @param {number} [args.outputTokensPerTurn] - est. output tokens per turn.
|
|
62
62
|
* @param {number|null} [args.expectedRemainingTurns] - median remaining
|
|
63
63
|
* turns; falls back to the conservative config default when null.
|
|
64
|
+
* @param {number|null} [args.sessionBurnPressure] - 0..1 quota pressure from
|
|
65
|
+
* quota-ledger.js (sustained expensive serving on a flat subscription).
|
|
66
|
+
* Enables a parallel QUOTA break-even leg: while dollar math prices the
|
|
67
|
+
* rebuild in USD, quota math prices it in ledger units (model weight ×
|
|
68
|
+
* tokens, Haiku = 1.0). Either leg clearing allows the switch, so high
|
|
69
|
+
* burn opens descents the dollar leg alone would hold. Null/0 disables
|
|
70
|
+
* the quota leg entirely — pure dollar behavior, byte-for-byte backward
|
|
71
|
+
* compatible. Upgrades never route through this gate, so pressure can
|
|
72
|
+
* only accelerate descents, never block capability.
|
|
64
73
|
* @param {Object} [args.deps] - test injection: {resolveCacheEconomics}.
|
|
65
74
|
* @returns {{
|
|
66
75
|
* switchAllowed: boolean,
|
|
67
76
|
* reason: string,
|
|
68
77
|
* breakEvenTurns: number|null,
|
|
69
78
|
* expectedRemainingTurns: number,
|
|
79
|
+
* quotaBreakEvenTurns: number|null,
|
|
80
|
+
* burnPressureApplied: number,
|
|
70
81
|
* warmPrefixTokens: number,
|
|
71
82
|
* switchOnceUsd: number|null,
|
|
72
83
|
* stayPerTurnUsd: number|null,
|
|
@@ -81,18 +92,24 @@ function evaluateSwitch({
|
|
|
81
92
|
newTokensPerTurn,
|
|
82
93
|
outputTokensPerTurn,
|
|
83
94
|
expectedRemainingTurns,
|
|
95
|
+
sessionBurnPressure = null,
|
|
84
96
|
deps = {},
|
|
85
97
|
} = {}) {
|
|
86
98
|
const cfg = _cfg();
|
|
87
99
|
const remaining = Number.isFinite(expectedRemainingTurns) && expectedRemainingTurns > 0
|
|
88
100
|
? expectedRemainingTurns
|
|
89
101
|
: cfg.defaultRemainingTurns;
|
|
102
|
+
const burnPressure = Number.isFinite(sessionBurnPressure)
|
|
103
|
+
? Math.max(0, Math.min(1, sessionBurnPressure))
|
|
104
|
+
: 0;
|
|
90
105
|
|
|
91
106
|
const base = {
|
|
92
107
|
switchAllowed: true,
|
|
93
108
|
reason: 'no_cache_state',
|
|
94
109
|
breakEvenTurns: null,
|
|
95
110
|
expectedRemainingTurns: remaining,
|
|
111
|
+
quotaBreakEvenTurns: null,
|
|
112
|
+
burnPressureApplied: burnPressure,
|
|
96
113
|
warmPrefixTokens: cacheState?.warmPrefixTokens ?? 0,
|
|
97
114
|
switchOnceUsd: null,
|
|
98
115
|
stayPerTurnUsd: null,
|
|
@@ -174,19 +191,46 @@ function evaluateSwitch({
|
|
|
174
191
|
const breakEvenTurns = switchOnce / savingsPerTurn;
|
|
175
192
|
const allowed = breakEvenTurns <= remaining;
|
|
176
193
|
|
|
194
|
+
// Quota leg (flat subscriptions): re-run the same break-even in ledger
|
|
195
|
+
// units when the session is burning. Model weight = (input+output $/M) /
|
|
196
|
+
// Haiku baseline — quota burn tracks price ratios, which is exactly what
|
|
197
|
+
// the ledger records. A rebuild that never pays in dollars can still pay
|
|
198
|
+
// in runway when every held turn burns 5x the units a descent would.
|
|
199
|
+
let quotaBreakEvenTurns = null;
|
|
200
|
+
let quotaAllowed = false;
|
|
201
|
+
if (burnPressure > 0) {
|
|
202
|
+
const wCur = (econCur.inputPerM + econCur.outputPerM) / 6;
|
|
203
|
+
const wTgt = (econTgt.inputPerM + econTgt.outputPerM) / 6;
|
|
204
|
+
if (wCur > wTgt) {
|
|
205
|
+
const perTurnTokens = (W + N + O) / 1000;
|
|
206
|
+
const quotaSavingsPerTurn = perTurnTokens * (wCur - wTgt);
|
|
207
|
+
if (quotaSavingsPerTurn > 0) {
|
|
208
|
+
quotaBreakEvenTurns = ((W + N) / 1000) * wTgt / quotaSavingsPerTurn;
|
|
209
|
+
quotaAllowed = quotaBreakEvenTurns <= remaining;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
const finalAllowed = allowed || quotaAllowed;
|
|
214
|
+
|
|
177
215
|
logger.debug({
|
|
178
216
|
current: `${current.provider}:${current.model}`,
|
|
179
217
|
target: `${target.provider}:${target.model}`,
|
|
180
218
|
warmPrefixTokens: W,
|
|
181
219
|
breakEvenTurns: Number(breakEvenTurns.toFixed(2)),
|
|
182
220
|
expectedRemainingTurns: remaining,
|
|
221
|
+
quotaBreakEvenTurns: quotaBreakEvenTurns === null ? null : Number(quotaBreakEvenTurns.toFixed(2)),
|
|
222
|
+
burnPressureApplied: burnPressure,
|
|
183
223
|
allowed,
|
|
224
|
+
quotaAllowed,
|
|
184
225
|
}, '[CacheSwitchCost] break-even evaluated');
|
|
185
226
|
|
|
186
227
|
return {
|
|
187
228
|
...priced,
|
|
188
|
-
|
|
189
|
-
|
|
229
|
+
quotaBreakEvenTurns,
|
|
230
|
+
switchAllowed: finalAllowed,
|
|
231
|
+
reason: finalAllowed
|
|
232
|
+
? (allowed ? 'break_even_cleared' : 'quota_break_even_cleared')
|
|
233
|
+
: 'break_even_blocked',
|
|
190
234
|
breakEvenTurns,
|
|
191
235
|
};
|
|
192
236
|
}
|
|
@@ -104,16 +104,28 @@ const FORCE_REASONING_PATTERNS = [
|
|
|
104
104
|
/\b(prove|proof|formal\s+proof|verify|verification)\b/i,
|
|
105
105
|
/\b(from\s+first\s+principles)\b/i,
|
|
106
106
|
/\b(reason\s+(through|about|from)\s+(the|this))/i,
|
|
107
|
+
// Frontier planning: asking FOR a plan/strategy/design of technical work
|
|
108
|
+
// (not asking about one). Two-sided lookahead — a plan word AND a technical
|
|
109
|
+
// scope word must both be present, so "plan my vacation", "lesson plan"
|
|
110
|
+
// and "business plan" stay quiet while "give me a plan to refactor X",
|
|
111
|
+
// "draft an RFC to redesign Y" and "migration strategy for Z" escalate.
|
|
112
|
+
// Deliberately excludes bare "give me a plan" (nothing technical to plan)
|
|
113
|
+
// and generic nouns (module/api/database) that misfire on non-technical
|
|
114
|
+
// asks. Known overlap: "explain the deployment plan" fires — accepted,
|
|
115
|
+
// deployment planning benefits from the strong model anyway.
|
|
116
|
+
/(?=.*\b(plans?|roadmaps?|strateg(y|ies)|blueprints?|proposals?|rfcs?|design\s+docs?)\b)(?=.*\b(refactor\w*|implement\w*|migrat\w+|architect\w+|redesign\w*|restructur\w+|rewrit\w+|overhaul\w*|codebases?|infrastructure|pipelines?|deploy\w*|systems?)\b)/i,
|
|
107
117
|
];
|
|
108
118
|
|
|
109
119
|
// Force COMPLEX patterns - route to COMPLEX (GLM in config B) regardless of
|
|
110
120
|
// score. Architecture/refactor/code-review are substantial but not
|
|
111
121
|
// security-critical, so GLM is appropriate.
|
|
112
122
|
const FORCE_CLOUD_PATTERNS = [
|
|
113
|
-
/\b(architect
|
|
123
|
+
/\b(architect\w*\s+(review|design|diagram)|system\s+design)\b/i,
|
|
114
124
|
/\b(refactor\s+(entire|whole|all|the\s+entire)|complete\s+rewrite)\b/i,
|
|
115
125
|
/\b(code\s+review|pr\s+review|pull\s+request\s+review)\b/i,
|
|
116
|
-
|
|
126
|
+
// "debug this tricky race" reads naturally but the adjectives rarely sit
|
|
127
|
+
// adjacent to the verb, so up to two words may intervene.
|
|
128
|
+
/\bdebug(?:ging)?\s+(?:\w+\s+){0,2}(complex|difficult|hard|tricky)\b/i,
|
|
117
129
|
/\b(production\s+(issue|bug|incident|outage))\b/i,
|
|
118
130
|
];
|
|
119
131
|
|
|
@@ -655,8 +667,13 @@ function calculateWeightedScore(payload, content) {
|
|
|
655
667
|
const avgLength = content.length / Math.max(sentences.length, 1);
|
|
656
668
|
dimensions.promptComplexity = Math.min(avgLength / 2, 100);
|
|
657
669
|
|
|
658
|
-
// 3. Technical depth (keyword density)
|
|
659
|
-
|
|
670
|
+
// 3. Technical depth (keyword density). NOTE: uses a local global-flag
|
|
671
|
+
// copy — the shared PATTERNS.technical must stay flag-less because .test()
|
|
672
|
+
// callers above depend on stateless matching (a shared /g regex carries
|
|
673
|
+
// lastIndex between calls). String.match with a non-global regex returns
|
|
674
|
+
// at most one hit, which used to cap this dimension at 15 regardless of
|
|
675
|
+
// density and starved COMPLEX-tier scores of their strongest signal.
|
|
676
|
+
const techMatches = (content.match(new RegExp(PATTERNS.technical.source, 'gi')) || []).length;
|
|
660
677
|
dimensions.technicalDepth = Math.min(techMatches * 15, 100);
|
|
661
678
|
|
|
662
679
|
// 4. Domain specificity (how many domains are touched)
|
package/src/routing/index.js
CHANGED
|
@@ -805,6 +805,17 @@ function checkSessionPin(payload, options = {}) {
|
|
|
805
805
|
const guards = _runPinGuards(payload, pin);
|
|
806
806
|
if (!guards.ok) return { serve: false, pin, sessionId, reason: guards.reason };
|
|
807
807
|
|
|
808
|
+
// Tier-less pins (legacy cache-only rows predating the session_cache_state
|
|
809
|
+
// split) must not drive tier decisions: drift math on a null tier never
|
|
810
|
+
// fires, so serving them locks the session with no escalation path.
|
|
811
|
+
// Exempted when ANY tool blocks exist anywhere in the payload: an
|
|
812
|
+
// unresolved tool_use breaks if the provider switches, so provider
|
|
813
|
+
// stickiness wins there (same safety rationale as the tool_history path
|
|
814
|
+
// above, which returns before reaching this check).
|
|
815
|
+
if (!pin.tier && !_payloadHasAnyToolBlocks(payload)) {
|
|
816
|
+
return { serve: false, pin, sessionId, reason: 'tierless_pin' };
|
|
817
|
+
}
|
|
818
|
+
|
|
808
819
|
if (refreshOk) {
|
|
809
820
|
sessionAffinity.setPin(sessionId, pin, {
|
|
810
821
|
messageCount: Math.max(messageCount, pin.messageCount ?? 0),
|
|
@@ -1095,6 +1106,11 @@ async function _determineProviderSmartInner(payload, options = {}) {
|
|
|
1095
1106
|
analysis.score = intent.score;
|
|
1096
1107
|
analysis.scoreMode = intent.mode; // 'anchor' | 'lexical' (clean-text fallback)
|
|
1097
1108
|
analysis.anchorClass = intent.class ?? null;
|
|
1109
|
+
// Jev verdict rides alongside the score so the shortfall floor,
|
|
1110
|
+
// risk corroboration and telemetry can consume it downstream.
|
|
1111
|
+
// Null unless the Jev leg ran (pinned serves, side frames and
|
|
1112
|
+
// classifier skips leave no verdict — same as classifierResult).
|
|
1113
|
+
if (intent.jev && typeof intent.jev === 'object') analysis.jev = intent.jev;
|
|
1098
1114
|
intentScored = true;
|
|
1099
1115
|
logger.debug({
|
|
1100
1116
|
score: intent.score,
|
|
@@ -1213,12 +1229,47 @@ async function _determineProviderSmartInner(payload, options = {}) {
|
|
|
1213
1229
|
// (agentic minTier, context, vision, kNN-ambiguous) for telemetry.
|
|
1214
1230
|
let baseTier = null;
|
|
1215
1231
|
const escalations = [];
|
|
1232
|
+
let _jevOverrode = false;
|
|
1216
1233
|
if (config.modelTiers?.enabled) {
|
|
1217
1234
|
try {
|
|
1218
1235
|
const selector = getModelTierSelector();
|
|
1219
1236
|
tier = selector.getTier(analysis.score);
|
|
1220
1237
|
baseTier = tier;
|
|
1221
1238
|
|
|
1239
|
+
// Jev tier override: a high-confidence measured verdict replaces the
|
|
1240
|
+
// score-band tier in either direction (score bands stay the fallback
|
|
1241
|
+
// for Jev-null/low-confidence). Runs BEFORE agentic floors and risk
|
|
1242
|
+
// lifts so structural guarantees can still lift above the verdict —
|
|
1243
|
+
// never below it. Score is set to the tier midpoint so drift math,
|
|
1244
|
+
// badges and telemetry stay coherent with the decided tier.
|
|
1245
|
+
try {
|
|
1246
|
+
const { jevTierOverride } = require('./jev-router');
|
|
1247
|
+
const _ov = jevTierOverride({
|
|
1248
|
+
baseTier,
|
|
1249
|
+
jevTier: analysis.jev?.tier,
|
|
1250
|
+
jevConfidence: analysis.jev?.confidence,
|
|
1251
|
+
});
|
|
1252
|
+
if (_ov) {
|
|
1253
|
+
escalations.push({
|
|
1254
|
+
source: 'jev_override',
|
|
1255
|
+
fromTier: tier,
|
|
1256
|
+
toTier: _ov.tier,
|
|
1257
|
+
fromModel: null,
|
|
1258
|
+
toModel: null,
|
|
1259
|
+
});
|
|
1260
|
+
logger.debug({
|
|
1261
|
+
fromTier: tier,
|
|
1262
|
+
toTier: _ov.tier,
|
|
1263
|
+
jevConfidence: analysis.jev?.confidence ?? null,
|
|
1264
|
+
}, '[Routing] Jev tier override');
|
|
1265
|
+
tier = _ov.tier;
|
|
1266
|
+
if (typeof _ov.score === 'number') analysis.score = _ov.score;
|
|
1267
|
+
_jevOverrode = true;
|
|
1268
|
+
}
|
|
1269
|
+
} catch (err) {
|
|
1270
|
+
degradation.record('tier_select', err);
|
|
1271
|
+
}
|
|
1272
|
+
|
|
1222
1273
|
// Check if agentic detection requires a higher tier
|
|
1223
1274
|
if (agenticResult?.minTier) {
|
|
1224
1275
|
const agenticTierPriority = TIER_DEFINITIONS[agenticResult.minTier]?.priority || 0;
|
|
@@ -1237,6 +1288,30 @@ async function _determineProviderSmartInner(payload, options = {}) {
|
|
|
1237
1288
|
}
|
|
1238
1289
|
}
|
|
1239
1290
|
|
|
1291
|
+
// Jev risk corroboration (escalate-only): a high Noul on auth /
|
|
1292
|
+
// secrets / production / destructive content lifts SIMPLE/MEDIUM one
|
|
1293
|
+
// band when the keyword analyzer stayed quiet. Never clears a keyword
|
|
1294
|
+
// hit (that path returned far above), never jumps two bands, never
|
|
1295
|
+
// fires without a verdict (fail-soft null skips silently). Mirrors the
|
|
1296
|
+
// kNN-ambiguous escalate pattern with a ledger entry.
|
|
1297
|
+
try {
|
|
1298
|
+
const { jevRiskLift } = require('./jev-router');
|
|
1299
|
+
const _lifted = jevRiskLift({ tier, risky: analysis.jev?.risky, riskLevel: risk?.level });
|
|
1300
|
+
if (_lifted) {
|
|
1301
|
+
escalations.push({
|
|
1302
|
+
source: 'jev_risk',
|
|
1303
|
+
fromTier: tier,
|
|
1304
|
+
toTier: _lifted,
|
|
1305
|
+
fromModel: null,
|
|
1306
|
+
toModel: null,
|
|
1307
|
+
});
|
|
1308
|
+
logger.debug({ fromTier: tier, toTier: _lifted, risky: analysis.jev.risky }, '[Routing] Jev risk corroboration — one-band lift');
|
|
1309
|
+
tier = _lifted;
|
|
1310
|
+
}
|
|
1311
|
+
} catch (err) {
|
|
1312
|
+
degradation.record('tier_select', err);
|
|
1313
|
+
}
|
|
1314
|
+
|
|
1240
1315
|
// Select model for the tier (will be applied after provider selection)
|
|
1241
1316
|
analysis.tier = tier;
|
|
1242
1317
|
} catch (err) {
|
|
@@ -1247,7 +1322,7 @@ async function _determineProviderSmartInner(payload, options = {}) {
|
|
|
1247
1322
|
// Apply routing decision based on tier config (TIER_* env vars take precedence
|
|
1248
1323
|
// but Phase 1.2 lets the cost-optimizer pick a cheaper qualifying model when safe).
|
|
1249
1324
|
let provider;
|
|
1250
|
-
let method = 'tier_config';
|
|
1325
|
+
let method = _jevOverrode ? 'tier_config+jev_override' : 'tier_config';
|
|
1251
1326
|
let costOptimized = false;
|
|
1252
1327
|
|
|
1253
1328
|
const selector = getModelTierSelector();
|
|
@@ -1342,6 +1417,51 @@ async function _determineProviderSmartInner(payload, options = {}) {
|
|
|
1342
1417
|
degradation.record('shortfall', err);
|
|
1343
1418
|
}
|
|
1344
1419
|
|
|
1420
|
+
// Jev shortfall floor: a high-confidence Jev tier is a measured verdict,
|
|
1421
|
+
// not a heuristic guess — cheapest-covering must not demote below it.
|
|
1422
|
+
// Floor target is the stronger of (scored legacy pick, Jev verdict):
|
|
1423
|
+
// score-derived highs are never talked down by the judge, and judged
|
|
1424
|
+
// highs are never talked down by the price optimizer (see jevFloorTarget).
|
|
1425
|
+
// De-escalation below still runs after this, so evidence gates apply like
|
|
1426
|
+
// any other decision.
|
|
1427
|
+
try {
|
|
1428
|
+
const { jevFloorTarget } = require('./jev-router');
|
|
1429
|
+
const _floorTier = shortfallInfo && shortfallInfo.legacy
|
|
1430
|
+
? jevFloorTarget({
|
|
1431
|
+
selectedTier: tier,
|
|
1432
|
+
legacyTier: shortfallInfo.legacy.tier,
|
|
1433
|
+
jevTier: analysis.jev?.tier,
|
|
1434
|
+
jevConfidence: analysis.jev?.confidence,
|
|
1435
|
+
})
|
|
1436
|
+
: null;
|
|
1437
|
+
if (_floorTier) {
|
|
1438
|
+
const _floorSel = selector.selectModel(_floorTier, null);
|
|
1439
|
+
if (_floorSel && _floorSel.provider && _floorSel.model) {
|
|
1440
|
+
const _fromTier = tier;
|
|
1441
|
+
const _fromModel = selectedModel;
|
|
1442
|
+
provider = _floorSel.provider;
|
|
1443
|
+
selectedModel = _floorSel.model;
|
|
1444
|
+
tier = _floorTier;
|
|
1445
|
+
analysis.tier = tier;
|
|
1446
|
+
method = method + '+jev_floor_hold';
|
|
1447
|
+
escalations.push({
|
|
1448
|
+
source: 'jev_floor',
|
|
1449
|
+
fromTier: _fromTier,
|
|
1450
|
+
toTier: tier,
|
|
1451
|
+
fromModel: _fromModel,
|
|
1452
|
+
toModel: selectedModel,
|
|
1453
|
+
});
|
|
1454
|
+
logger.info({
|
|
1455
|
+
from: `${_fromTier}:${_fromModel}`,
|
|
1456
|
+
to: `${tier}:${selectedModel}`,
|
|
1457
|
+
jevConfidence: analysis.jev?.confidence ?? null,
|
|
1458
|
+
}, '[Routing] Jev floor hold — shortfall demotion refused');
|
|
1459
|
+
}
|
|
1460
|
+
}
|
|
1461
|
+
} catch (err) {
|
|
1462
|
+
degradation.record('shortfall', err);
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1345
1465
|
// WS2.3 — evidence-based de-escalation.
|
|
1346
1466
|
//
|
|
1347
1467
|
// The check is intentionally gated by evidence, not a feature flag: the
|
|
@@ -338,9 +338,8 @@ function _reconcile(anchorScore, anchorClass, classifierResult) {
|
|
|
338
338
|
const anchorIdx = TIER_ORDER.indexOf(anchorTier);
|
|
339
339
|
const classifierIdx = TIER_ORDER.indexOf(classifierTier);
|
|
340
340
|
|
|
341
|
-
// Midpoints
|
|
342
|
-
|
|
343
|
-
const TIER_MIDPOINT = { SIMPLE: 10, MEDIUM: 35, COMPLEX: 63, REASONING: 88 };
|
|
341
|
+
// Midpoints live in model-tiers.js (single source of truth).
|
|
342
|
+
const { TIER_MIDPOINT } = require('./model-tiers');
|
|
344
343
|
|
|
345
344
|
if (classifierIdx < anchorIdx) {
|
|
346
345
|
// Classifier says LOWER tier — trust it. Fixes over-routing.
|
|
@@ -410,24 +409,47 @@ async function scoreIntent(payload, opts = {}) {
|
|
|
410
409
|
const [lo, hi] = CLASS_BANDS[cls];
|
|
411
410
|
const anchorScore = Math.max(lo, Math.min(hi, blendScore(sims)));
|
|
412
411
|
|
|
413
|
-
//
|
|
414
|
-
// to keep unit tests hermetic. Runs live
|
|
412
|
+
// Jev routing judge — second opinion. Skipped in tests (opts.skipClassifier)
|
|
413
|
+
// to keep unit tests hermetic (or stubbed via opts.jevFetchFn). Runs live
|
|
414
|
+
// otherwise. Replaces the qwen difficulty classifier in the same result
|
|
415
|
+
// shape ({tier, confidence}) so _reconcile semantics are unchanged;
|
|
416
|
+
// Jev extras (probabilities, risky, model, criteriaHash) ride along
|
|
417
|
+
// for telemetry and the shortfall floor. Fail-soft null preserves the
|
|
418
|
+
// anchor-only path exactly.
|
|
419
|
+
//
|
|
420
|
+
// Hermeticity note: suites asserting deterministic routing scrub
|
|
421
|
+
// TYPESAFE_API_KEY (no key → Jev leg returns null before any fetch),
|
|
422
|
+
// the same posture as the repo's live-Ollama tolerance. Never gate
|
|
423
|
+
// on NODE_ENV — .env pins it to production in every context.
|
|
415
424
|
let classifierResult = null;
|
|
425
|
+
let jevResult = null;
|
|
416
426
|
if (!opts.skipClassifier) {
|
|
417
427
|
try {
|
|
418
|
-
const {
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
428
|
+
const { classifyJev } = require('./jev-router');
|
|
429
|
+
const msgs = Array.isArray(payload?.messages) ? payload.messages : [];
|
|
430
|
+
const toolDefs = Array.isArray(payload?.tools) ? payload.tools : [];
|
|
431
|
+
const toolResults = msgs.filter((m) =>
|
|
432
|
+
m?.role === 'user' && Array.isArray(m.content) && m.content.some((c) => c?.type === 'tool_result')
|
|
433
|
+
).length;
|
|
434
|
+
const jevFetchFn = opts.jevFetchFn;
|
|
435
|
+
jevResult = await classifyJev(text, {
|
|
425
436
|
context: typeof payload?._conversationContext === 'string'
|
|
426
437
|
? payload._conversationContext
|
|
427
438
|
: null,
|
|
439
|
+
signals: {
|
|
440
|
+
message_count: msgs.length,
|
|
441
|
+
tools_attached: toolDefs.length,
|
|
442
|
+
effective_tools: toolDefs.length,
|
|
443
|
+
has_tool_history: toolResults > 0,
|
|
444
|
+
session_turn: Math.max(1, Math.ceil(msgs.length / 2)),
|
|
445
|
+
},
|
|
446
|
+
fetchFn: jevFetchFn,
|
|
428
447
|
});
|
|
448
|
+
if (jevResult) {
|
|
449
|
+
classifierResult = { tier: jevResult.tier, confidence: jevResult.confidence };
|
|
450
|
+
}
|
|
429
451
|
} catch (err) {
|
|
430
|
-
logger.debug({ err: err.message }, '[IntentScore] classifier failed — anchor only');
|
|
452
|
+
logger.debug({ err: err.message }, '[IntentScore] Jev classifier failed — anchor only');
|
|
431
453
|
}
|
|
432
454
|
}
|
|
433
455
|
|
|
@@ -453,6 +475,19 @@ async function scoreIntent(payload, opts = {}) {
|
|
|
453
475
|
classifierTier: classifierResult?.tier ?? null,
|
|
454
476
|
classifierConfidence: classifierResult?.confidence ?? null,
|
|
455
477
|
reconciled,
|
|
478
|
+
// Jev verdict details for telemetry + shortfall floor downstream.
|
|
479
|
+
// Null unless the Jev leg ran (same conditions as classifierResult,
|
|
480
|
+
// plus LRU-cached repeats which still carry the original extras).
|
|
481
|
+
jev: jevResult ? {
|
|
482
|
+
tier: jevResult.tier,
|
|
483
|
+
confidence: jevResult.confidence,
|
|
484
|
+
probabilities: jevResult.probabilities ?? null,
|
|
485
|
+
risky: jevResult.risky ?? null,
|
|
486
|
+
model: jevResult.model ?? null,
|
|
487
|
+
criteriaHash: jevResult.criteriaHash ?? null,
|
|
488
|
+
latencyMs: jevResult.latencyMs ?? null,
|
|
489
|
+
cached: !!jevResult.cached,
|
|
490
|
+
} : null,
|
|
456
491
|
});
|
|
457
492
|
}
|
|
458
493
|
}
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Jev (TypeSafe System One) routing judge.
|
|
3
|
+
*
|
|
4
|
+
* Replaces the qwen-based difficulty classifier as the second opinion in
|
|
5
|
+
* intent scoring: one typed Choice over the four tiers (+ a parallel
|
|
6
|
+
* risk Noul, free — questions share state) instead of JSON coaxed through
|
|
7
|
+
* a thinking block. Native per-option probabilities replace self-reported
|
|
8
|
+
* confidence; 0.5s vs up-to-10s timeout; no local prefill.
|
|
9
|
+
*
|
|
10
|
+
* Design constraints (operator policy):
|
|
11
|
+
* - No new env vars. Endpoint, pinned model, timeout, cache size and the
|
|
12
|
+
* promotion cut are constants below. Only TYPESAFE_API_KEY is read from
|
|
13
|
+
* env (pre-existing); absent key disables the client. Rollback is a
|
|
14
|
+
* revert, not a flag — a hardcoded kill-switch you can only flip with a
|
|
15
|
+
* code edit is theater; the fail-soft nulls are the real safety.
|
|
16
|
+
* - Fail-soft: any failure (no key, timeout, 429/529, malformed body)
|
|
17
|
+
* returns null and callers fall back exactly as if no classifier ran.
|
|
18
|
+
* Single attempt, no retries in the hot path.
|
|
19
|
+
* - Pinned model version (jev-1.13.0 observed live), never jev-latest:
|
|
20
|
+
* routing must not ride alias drift. Re-run the difficulty evals on
|
|
21
|
+
* any bump. Criteria text is hashed into telemetry so prompt tweaks
|
|
22
|
+
* don't silently invalidate history either.
|
|
23
|
+
*
|
|
24
|
+
* Pure except for fetch (injectable) and the LRU cache. Never throws.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
const crypto = require('crypto');
|
|
28
|
+
const logger = require('../logger');
|
|
29
|
+
|
|
30
|
+
const JEV_ENDPOINT = 'https://api.typesafe.ai/v1/systemone';
|
|
31
|
+
const JEV_MODEL = 'jev-1.13.0';
|
|
32
|
+
const JEV_TIMEOUT_MS = 3000;
|
|
33
|
+
const JEV_CACHE_CAPACITY = 500;
|
|
34
|
+
// Promotion cut for letting a Jev tier override the heuristic (Phase 2).
|
|
35
|
+
// Matches the existing upward-gate in intent-score _reconcile (0.8);
|
|
36
|
+
// ratify against shadow agreement-vs-confidence data when available.
|
|
37
|
+
const JEV_PROMOTE_CONFIDENCE = 0.8;
|
|
38
|
+
// Risk corroboration cut: Noul at/above this corroborates upward only
|
|
39
|
+
// (may escalate, never clears a keyword hit).
|
|
40
|
+
const JEV_RISK_CUT = 0.85;
|
|
41
|
+
|
|
42
|
+
const VALID_TIERS = ['SIMPLE', 'MEDIUM', 'COMPLEX', 'REASONING'];
|
|
43
|
+
|
|
44
|
+
// v1 criteria — carries the hard-won disambiguators from the qwen prompt
|
|
45
|
+
// generations (casual follow-ups, arithmetic, verify-file-exists).
|
|
46
|
+
const TIER_CRITERIA_V1 = {
|
|
47
|
+
SIMPLE: 'Greetings, confirmations, casual follow-ups about people, stories or everyday facts, trivial lookups and arithmetic. Any tiny model handles. NOT reasoning: who kills him?, 12+21, prove me wrong lol.',
|
|
48
|
+
MEDIUM: 'One specific mechanical task or focused explanation. A mid-size model suffices. Examples: list the exports from this file, explain this regex, verify the file exists before reading it.',
|
|
49
|
+
COMPLEX: 'Multi-file design, systemic refactor, architecture review, debugging that requires broad code understanding. Needs a strong general model.',
|
|
50
|
+
REASONING: 'Formal proof, correctness verification, security audit, novel algorithm design, reasoning from first principles. Needs a frontier reasoning model.',
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
const TIER_INSTRUCTIONS = 'Which difficulty tier should serve this coding-assistant request? Judge the TASK the model must perform, not the vocabulary.';
|
|
54
|
+
const RISK_INSTRUCTIONS = 'This request involves auth, secrets, credentials, production systems, deployments, or destructive/data-loss operations.';
|
|
55
|
+
|
|
56
|
+
const CRITERIA_HASH = crypto
|
|
57
|
+
.createHash('sha256')
|
|
58
|
+
.update(JSON.stringify(TIER_CRITERIA_V1))
|
|
59
|
+
.digest('hex')
|
|
60
|
+
.slice(0, 12);
|
|
61
|
+
|
|
62
|
+
// Defensive caps (far below the ~32k-token request budget; condensed states
|
|
63
|
+
// never approach them — the guard exists so a pathological upstream payload
|
|
64
|
+
// can't turn a 0.5s judgment into a 30k-token one).
|
|
65
|
+
const MAX_REQUEST_CHARS = 4000;
|
|
66
|
+
const MAX_CONTEXT_CHARS = 500;
|
|
67
|
+
|
|
68
|
+
const _cache = new Map(); // key -> result (LRU-ish: delete+re-set on hit)
|
|
69
|
+
|
|
70
|
+
function _evictIfNeeded() {
|
|
71
|
+
while (_cache.size > JEV_CACHE_CAPACITY) {
|
|
72
|
+
const oldest = _cache.keys().next();
|
|
73
|
+
if (oldest.done) break;
|
|
74
|
+
_cache.delete(oldest.value);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function getApiKey() {
|
|
79
|
+
const raw = process.env.TYPESAFE_API_KEY;
|
|
80
|
+
if (!raw || typeof raw !== 'string') return null;
|
|
81
|
+
const key = raw.trim().replace(/^["']|["']$/g, '').split(/\s/)[0];
|
|
82
|
+
return key || null;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function _stripReminders(s) {
|
|
86
|
+
return typeof s === 'string'
|
|
87
|
+
? s.replace(/<system-reminder>[\s\S]*?<\/system-reminder>/g, '').trim()
|
|
88
|
+
: s;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function _cacheKey(text, context, signals) {
|
|
92
|
+
return crypto
|
|
93
|
+
.createHash('sha256')
|
|
94
|
+
.update(`${text || ''}\0${context || ''}\0${JSON.stringify(signals || {})}\0${CRITERIA_HASH}`)
|
|
95
|
+
.digest('hex');
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Build the Jev state: current ask in full, condensed context, precomputed
|
|
100
|
+
* signals. Signals over transcripts: the counts/flags carry the routing
|
|
101
|
+
* information at ~30 tokens instead of ~6k.
|
|
102
|
+
*/
|
|
103
|
+
function buildJevState({ text, context = null, signals = {} } = {}) {
|
|
104
|
+
let current = _stripReminders(text || '');
|
|
105
|
+
if (current.length > MAX_REQUEST_CHARS) current = current.slice(0, MAX_REQUEST_CHARS);
|
|
106
|
+
let ctx = context ? _stripReminders(context) : null;
|
|
107
|
+
if (ctx && ctx.length > MAX_CONTEXT_CHARS) ctx = ctx.slice(0, MAX_CONTEXT_CHARS);
|
|
108
|
+
return {
|
|
109
|
+
current_request: current,
|
|
110
|
+
conversation_context: ctx || undefined,
|
|
111
|
+
signals: {
|
|
112
|
+
message_count: Number(signals.message_count) || 0,
|
|
113
|
+
tools_attached: Number(signals.tools_attached) || 0,
|
|
114
|
+
effective_tools: Number(signals.effective_tools) || 0,
|
|
115
|
+
has_tool_history: !!signals.has_tool_history,
|
|
116
|
+
session_turn: Number(signals.session_turn) || 0,
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function _normalizeTier(t) {
|
|
122
|
+
if (!t || typeof t !== 'string') return null;
|
|
123
|
+
const up = t.trim().toUpperCase();
|
|
124
|
+
return VALID_TIERS.includes(up) ? up : null;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function _num01(v) {
|
|
128
|
+
const n = Number(v);
|
|
129
|
+
return Number.isFinite(n) ? Math.max(0, Math.min(1, n)) : null;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* One batched evaluation: Choice(tier) + Noul(risky) sharing state.
|
|
134
|
+
* @returns {null | {tier, confidence, probabilities, risky, model, latencyMs, criteriaHash}}
|
|
135
|
+
*/
|
|
136
|
+
async function evaluateJev(state, { fetchFn = null, endpoint = JEV_ENDPOINT, apiKey = null } = {}) {
|
|
137
|
+
const key = apiKey || getApiKey();
|
|
138
|
+
if (!key) return null;
|
|
139
|
+
const fetchImpl = fetchFn || globalThis.fetch;
|
|
140
|
+
if (typeof fetchImpl !== 'function') return null;
|
|
141
|
+
const started = Date.now();
|
|
142
|
+
try {
|
|
143
|
+
const body = {
|
|
144
|
+
state,
|
|
145
|
+
model: JEV_MODEL,
|
|
146
|
+
questions: {
|
|
147
|
+
tier: { type: 'choice', instructions: TIER_INSTRUCTIONS, criteria: TIER_CRITERIA_V1 },
|
|
148
|
+
risky: { type: 'noul', instructions: RISK_INSTRUCTIONS },
|
|
149
|
+
},
|
|
150
|
+
};
|
|
151
|
+
const resp = await fetchImpl(endpoint, {
|
|
152
|
+
method: 'POST',
|
|
153
|
+
headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' },
|
|
154
|
+
body: JSON.stringify(body),
|
|
155
|
+
signal: AbortSignal.timeout(JEV_TIMEOUT_MS),
|
|
156
|
+
});
|
|
157
|
+
if (!resp || !resp.ok) return null;
|
|
158
|
+
const data = await resp.json();
|
|
159
|
+
const t = data?.answers?.tier;
|
|
160
|
+
const tier = _normalizeTier(t?.choice);
|
|
161
|
+
if (!tier) return null;
|
|
162
|
+
return {
|
|
163
|
+
tier,
|
|
164
|
+
confidence: _num01(t?.confidence) ?? 0,
|
|
165
|
+
probabilities: (t?.probabilities && typeof t.probabilities === 'object') ? t.probabilities : null,
|
|
166
|
+
risky: _num01(data?.answers?.risky?.noul),
|
|
167
|
+
model: typeof data?.model === 'string' ? data.model : JEV_MODEL,
|
|
168
|
+
latencyMs: Date.now() - started,
|
|
169
|
+
criteriaHash: CRITERIA_HASH,
|
|
170
|
+
};
|
|
171
|
+
} catch (err) {
|
|
172
|
+
logger.debug({ err: err?.message || String(err) }, '[Jev] evaluation failed — fail-soft null');
|
|
173
|
+
return null;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* LRU-cached classify() matching the qwen classifyDifficulty return shape
|
|
179
|
+
* ({tier, confidence}) plus Jev extras, so intent-score swaps one call.
|
|
180
|
+
*/
|
|
181
|
+
async function classifyJev(text, { context = null, signals = {}, fetchFn = null } = {}) {
|
|
182
|
+
const clean = _stripReminders(text || '');
|
|
183
|
+
if (!clean) return null;
|
|
184
|
+
const cacheKey = _cacheKey(clean, context, signals);
|
|
185
|
+
const hit = _cache.get(cacheKey);
|
|
186
|
+
if (hit) {
|
|
187
|
+
_cache.delete(cacheKey);
|
|
188
|
+
_cache.set(cacheKey, hit);
|
|
189
|
+
return { ...hit, cached: true };
|
|
190
|
+
}
|
|
191
|
+
const state = buildJevState({ text: clean, context, signals });
|
|
192
|
+
const result = await evaluateJev(state, { fetchFn });
|
|
193
|
+
if (result) {
|
|
194
|
+
_cache.set(cacheKey, result);
|
|
195
|
+
_evictIfNeeded();
|
|
196
|
+
}
|
|
197
|
+
return result;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function _clearCache() {
|
|
201
|
+
_cache.clear();
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
const _TIER_PRI = { SIMPLE: 1, MEDIUM: 2, COMPLEX: 3, REASONING: 4 };
|
|
205
|
+
function _pri(tier) {
|
|
206
|
+
return _TIER_PRI[tier] || 0;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
// Tier midpoints live in model-tiers.js (single source of truth, shared
|
|
210
|
+
// with intent-score reconcile) so a decided tier maps to a coherent numeric
|
|
211
|
+
// score for drift math, badges and telemetry.
|
|
212
|
+
const { TIER_MIDPOINT } = require('./model-tiers');
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Jev tier override (pure): when a high-confidence Jev verdict disagrees
|
|
216
|
+
* with the score-derived base tier, the verdict wins either direction.
|
|
217
|
+
* Score bands remain the fallback (Jev null/low-confidence). Force paths
|
|
218
|
+
* already returned upstream; agentic floors and risk lifts apply after.
|
|
219
|
+
* @returns {null | {tier, score}} — score is the tier midpoint for coherence.
|
|
220
|
+
*/
|
|
221
|
+
function jevTierOverride({ baseTier, jevTier, jevConfidence } = {}) {
|
|
222
|
+
if (!baseTier || !_pri(baseTier)) return null;
|
|
223
|
+
if (!jevTier || !_pri(jevTier)) return null;
|
|
224
|
+
if (typeof jevConfidence !== 'number' || jevConfidence < JEV_PROMOTE_CONFIDENCE) return null;
|
|
225
|
+
if (_pri(jevTier) === _pri(baseTier)) return null;
|
|
226
|
+
return { tier: jevTier, score: TIER_MIDPOINT[jevTier] ?? null };
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Shortfall floor target (pure; called by the router after shortfall).
|
|
231
|
+
* A high-confidence Jev tier is a measured verdict — cheapest-covering must
|
|
232
|
+
* not demote below the stronger of (scored legacy pick, Jev verdict).
|
|
233
|
+
* @returns {string|null} floor tier, or null when no floor applies.
|
|
234
|
+
*/
|
|
235
|
+
function jevFloorTarget({ selectedTier, legacyTier, jevTier, jevConfidence } = {}) {
|
|
236
|
+
if (typeof jevConfidence !== 'number' || jevConfidence < JEV_PROMOTE_CONFIDENCE) return null;
|
|
237
|
+
if (!jevTier || !_pri(jevTier)) return null;
|
|
238
|
+
const floor = _pri(legacyTier) >= _pri(jevTier) ? legacyTier : jevTier;
|
|
239
|
+
if (!floor || !( _pri(selectedTier) < _pri(floor))) return null;
|
|
240
|
+
return floor;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* Risk corroboration lift (pure; escalate-only, one band).
|
|
245
|
+
* @returns {string|null} lifted tier, or null when no lift applies.
|
|
246
|
+
*/
|
|
247
|
+
function jevRiskLift({ tier, risky, riskLevel } = {}) {
|
|
248
|
+
if (tier !== 'SIMPLE' && tier !== 'MEDIUM') return null;
|
|
249
|
+
if (typeof risky !== 'number' || risky < JEV_RISK_CUT) return null;
|
|
250
|
+
if (riskLevel === 'high') return null; // keyword path owns high risk
|
|
251
|
+
return tier === 'SIMPLE' ? 'MEDIUM' : 'COMPLEX';
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
module.exports = {
|
|
255
|
+
JEV_ENDPOINT,
|
|
256
|
+
JEV_MODEL,
|
|
257
|
+
JEV_TIMEOUT_MS,
|
|
258
|
+
JEV_PROMOTE_CONFIDENCE,
|
|
259
|
+
JEV_RISK_CUT,
|
|
260
|
+
TIER_CRITERIA_V1,
|
|
261
|
+
CRITERIA_HASH,
|
|
262
|
+
VALID_TIERS,
|
|
263
|
+
getApiKey,
|
|
264
|
+
buildJevState,
|
|
265
|
+
evaluateJev,
|
|
266
|
+
classifyJev,
|
|
267
|
+
jevFloorTarget,
|
|
268
|
+
jevRiskLift,
|
|
269
|
+
jevTierOverride,
|
|
270
|
+
TIER_MIDPOINT,
|
|
271
|
+
_clearCache,
|
|
272
|
+
};
|