@juspay/neurolink 12.18.1 → 12.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +3 -3
- package/README.md +277 -75
- package/dist/browser/neurolink.min.js +396 -396
- package/dist/cli/commands/setup.js +2 -1
- package/dist/cli/factories/commandFactory.d.ts +22 -2
- package/dist/cli/factories/commandFactory.js +14 -2
- package/dist/cli/loop/optionsSchema.js +4 -0
- package/dist/cli/utils/classifierRouterFlags.js +39 -1
- package/dist/constants/enums.d.ts +13 -0
- package/dist/constants/enums.js +14 -0
- package/dist/context/budgetChecker.d.ts +3 -1
- package/dist/context/budgetChecker.js +20 -4
- package/dist/context/contextCompactor.d.ts +43 -3
- package/dist/context/contextCompactor.js +157 -41
- package/dist/context/contextDecision.d.ts +56 -0
- package/dist/context/contextDecision.js +234 -0
- package/dist/core/baseProvider.d.ts +27 -1
- package/dist/core/baseProvider.js +39 -0
- package/dist/core/toolRouting.d.ts +8 -0
- package/dist/core/toolRouting.js +52 -1
- package/dist/core/toolRoutingDecision.d.ts +35 -0
- package/dist/core/toolRoutingDecision.js +194 -0
- package/dist/factories/providerDescriptors.d.ts +15 -0
- package/dist/factories/providerDescriptors.js +57 -1
- package/dist/factories/providerRegistry.js +9 -1
- package/dist/index.d.ts +57 -1
- package/dist/index.js +60 -1
- package/dist/models/manifestRegistry.js +2 -0
- package/dist/models/manifests/typesafe.d.ts +16 -0
- package/dist/models/manifests/typesafe.js +42 -0
- package/dist/neurolink.d.ts +57 -1
- package/dist/neurolink.js +254 -10
- package/dist/observability/exporters/laminarExporter.js +3 -0
- package/dist/observability/exporters/posthogExporter.js +3 -0
- package/dist/observability/utils/spanSerializer.js +4 -0
- package/dist/providers/catalog/schema.d.ts +1 -1
- package/dist/providers/typesafe.d.ts +112 -0
- package/dist/providers/typesafe.js +654 -0
- package/dist/rag/pipeline/RAGPipeline.d.ts +3 -1
- package/dist/rag/pipeline/RAGPipeline.js +65 -4
- package/dist/rag/retrieval/searchDecision.d.ts +27 -0
- package/dist/rag/retrieval/searchDecision.js +124 -0
- package/dist/routing/classifierRouter.d.ts +74 -2
- package/dist/routing/classifierRouter.js +232 -32
- package/dist/routing/classifierStrategies.d.ts +33 -1
- package/dist/routing/classifierStrategies.js +237 -0
- package/dist/routing/index.d.ts +1 -1
- package/dist/routing/index.js +1 -1
- package/dist/routing/modelCatalog.d.ts +79 -0
- package/dist/routing/modelCatalog.js +382 -0
- package/dist/telemetry/tracers.d.ts +1 -0
- package/dist/telemetry/tracers.js +1 -0
- package/dist/types/classifierRouter.d.ts +160 -4
- package/dist/types/cli.d.ts +17 -1
- package/dist/types/config.d.ts +12 -0
- package/dist/types/context.d.ts +55 -1
- package/dist/types/decision.d.ts +200 -0
- package/dist/types/decision.js +21 -0
- package/dist/types/generate.d.ts +20 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +2 -0
- package/dist/types/providers.d.ts +59 -1
- package/dist/types/providers.js +5 -0
- package/dist/types/rag.d.ts +48 -0
- package/dist/types/span.d.ts +11 -0
- package/dist/types/span.js +11 -0
- package/dist/types/stream.d.ts +13 -0
- package/dist/types/toolRouting.d.ts +56 -0
- package/dist/utils/decisionAnswers.d.ts +56 -0
- package/dist/utils/decisionAnswers.js +95 -0
- package/dist/utils/modelChoices.js +13 -1
- package/dist/utils/pricing.js +11 -0
- package/dist/utils/providerConfig.d.ts +4 -0
- package/dist/utils/providerConfig.js +17 -0
- package/docs-site/static/search-index.json +1 -1
- package/package.json +16 -3
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Relevance-driven context compaction.
|
|
3
|
+
*
|
|
4
|
+
* Every stage of the existing compactor is *positional*: prune protects the
|
|
5
|
+
* most recent 40K tokens, truncation drops the oldest half, summarization
|
|
6
|
+
* keeps a trailing ratio. None of them has any notion of what the current
|
|
7
|
+
* request is about, so a fifty-turn conversation that ends with "now rename
|
|
8
|
+
* that variable" keeps the forty turns about database migrations and drops
|
|
9
|
+
* nothing that matters less.
|
|
10
|
+
*
|
|
11
|
+
* A decision model changes the economics of asking. Latency is flat in
|
|
12
|
+
* question count — one question and four hundred questions cost the same
|
|
13
|
+
* round trip — so "is this message needed to answer the current request?" can
|
|
14
|
+
* be asked about *every* message at once, for about $0.00002.
|
|
15
|
+
*
|
|
16
|
+
* What it must never do is lose something the turn needed. The guards below
|
|
17
|
+
* are therefore deliberately conservative, and every one of them is a veto:
|
|
18
|
+
*
|
|
19
|
+
* - only plain user/assistant text is ever eligible; tool calls, tool
|
|
20
|
+
* results, summaries, pinned skills and system messages are untouchable,
|
|
21
|
+
* because dropping half of a tool-call pair produces a malformed request
|
|
22
|
+
* rather than a smaller one
|
|
23
|
+
* - the most recent messages are never eligible, whatever the model says
|
|
24
|
+
* - a message is dropped only on a CONFIDENT no, never on silence or a
|
|
25
|
+
* near-coin-flip
|
|
26
|
+
* - at most a bounded fraction of eligible messages may go in one pass
|
|
27
|
+
*
|
|
28
|
+
* @module context/contextDecision
|
|
29
|
+
*/
|
|
30
|
+
import { decisionKey, gateDecisionBoolean } from "../utils/decisionAnswers.js";
|
|
31
|
+
import { logger } from "../utils/logger.js";
|
|
32
|
+
const MESSAGE_NAMESPACE = "msg";
|
|
33
|
+
/** Messages at the tail that are never eligible, whatever the model says. */
|
|
34
|
+
const DEFAULT_PROTECT_RECENT = 6;
|
|
35
|
+
/** Ceiling on the share of eligible messages one pass may drop. */
|
|
36
|
+
const DEFAULT_MAX_DROP_RATIO = 0.5;
|
|
37
|
+
/**
|
|
38
|
+
* How sure the model must be that a message is irrelevant. High, for the same
|
|
39
|
+
* reason the tool router's bar is high: keeping a useless message costs
|
|
40
|
+
* tokens, losing a needed one costs the answer.
|
|
41
|
+
*/
|
|
42
|
+
const DEFAULT_MIN_DROP_CONFIDENCE = 0.6;
|
|
43
|
+
/** Never ask about more than this many messages in one request. */
|
|
44
|
+
const MAX_QUESTIONS = 300;
|
|
45
|
+
/** Per-message text sent as state. Enough to judge, bounded enough to batch. */
|
|
46
|
+
const MAX_MESSAGE_CHARS = 1200;
|
|
47
|
+
const RELEVANCE_INSTRUCTIONS = "This earlier message contains information the assistant still needs in order to answer the current request correctly. Treat it as needed if it states a requirement, a decision, a constraint, a correction, a name, a number, or a preference that the current request builds on. Treat it as not needed if the current request is about something else entirely, or if the message is small talk, an acknowledgement, or superseded by a later message.";
|
|
48
|
+
/**
|
|
49
|
+
* A message is eligible for relevance-based dropping only if losing it cannot
|
|
50
|
+
* corrupt the request. That rules out far more than it keeps, on purpose.
|
|
51
|
+
*/
|
|
52
|
+
function isEligible(message) {
|
|
53
|
+
if (message.role !== "user" && message.role !== "assistant") {
|
|
54
|
+
return false;
|
|
55
|
+
}
|
|
56
|
+
if (typeof message.content !== "string" || message.content.trim() === "") {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
// A summary already represents messages that were dropped; dropping it
|
|
60
|
+
// discards all of them at once.
|
|
61
|
+
if (message.metadata?.isSummary) {
|
|
62
|
+
return false;
|
|
63
|
+
}
|
|
64
|
+
// Pinned skill instructions are replayed verbatim by design and are already
|
|
65
|
+
// protected from truncation.
|
|
66
|
+
if (message.metadata?.isSkill) {
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
// Assistant turns that carry tool calls pair with a tool_result elsewhere.
|
|
70
|
+
if (message.tool || message.toolCallId || message.events?.length) {
|
|
71
|
+
return false;
|
|
72
|
+
}
|
|
73
|
+
if (message.isTruncationMarker || message.condenseParent) {
|
|
74
|
+
return false;
|
|
75
|
+
}
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Ask which earlier messages the current request still needs, and return the
|
|
80
|
+
* ones to drop.
|
|
81
|
+
*
|
|
82
|
+
* Returns `null` when nothing should change — no decision provider, a failed
|
|
83
|
+
* call, nothing eligible, or no confident drop. A null return is the signal
|
|
84
|
+
* to carry on with positional compaction exactly as before.
|
|
85
|
+
*/
|
|
86
|
+
export async function selectIrrelevantMessages(messages, currentRequest, decide, options) {
|
|
87
|
+
const protectRecent = options?.protectRecent ?? DEFAULT_PROTECT_RECENT;
|
|
88
|
+
const minConfidence = options?.minDropConfidence ?? DEFAULT_MIN_DROP_CONFIDENCE;
|
|
89
|
+
const maxDropRatio = options?.maxDropRatio ?? DEFAULT_MAX_DROP_RATIO;
|
|
90
|
+
const cutoff = Math.max(0, messages.length - protectRecent);
|
|
91
|
+
const eligible = messages
|
|
92
|
+
.map((message, index) => ({ message, index }))
|
|
93
|
+
.filter(({ message, index }) => index < cutoff && isEligible(message))
|
|
94
|
+
.slice(0, MAX_QUESTIONS);
|
|
95
|
+
if (eligible.length === 0 || !currentRequest.trim()) {
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
98
|
+
const questions = {};
|
|
99
|
+
eligible.forEach((entry, position) => {
|
|
100
|
+
questions[decisionKey(MESSAGE_NAMESPACE, position)] = {
|
|
101
|
+
type: "boolean",
|
|
102
|
+
instructions: RELEVANCE_INSTRUCTIONS,
|
|
103
|
+
criteria: {
|
|
104
|
+
true: `The current request depends on this message: "${preview(entry.message)}"`,
|
|
105
|
+
false: "The current request does not depend on this message.",
|
|
106
|
+
},
|
|
107
|
+
};
|
|
108
|
+
});
|
|
109
|
+
const result = await decide({
|
|
110
|
+
state: {
|
|
111
|
+
current_request: currentRequest.slice(0, 4000),
|
|
112
|
+
conversation: eligible.map((entry, position) => ({
|
|
113
|
+
index: position,
|
|
114
|
+
role: entry.message.role,
|
|
115
|
+
text: preview(entry.message),
|
|
116
|
+
})),
|
|
117
|
+
},
|
|
118
|
+
questions,
|
|
119
|
+
timeoutMs: options?.timeoutMs,
|
|
120
|
+
});
|
|
121
|
+
if (!result) {
|
|
122
|
+
return null;
|
|
123
|
+
}
|
|
124
|
+
const dropCandidates = [];
|
|
125
|
+
let answered = 0;
|
|
126
|
+
eligible.forEach((entry, position) => {
|
|
127
|
+
const verdict = gateDecisionBoolean(result.answers, decisionKey(MESSAGE_NAMESPACE, position), { minConfidence });
|
|
128
|
+
if (verdict !== undefined) {
|
|
129
|
+
answered += 1;
|
|
130
|
+
}
|
|
131
|
+
if (verdict === false) {
|
|
132
|
+
dropCandidates.push(entry.index);
|
|
133
|
+
}
|
|
134
|
+
});
|
|
135
|
+
if (answered === 0 || dropCandidates.length === 0) {
|
|
136
|
+
return null;
|
|
137
|
+
}
|
|
138
|
+
// Cap how much one pass may remove. Beyond this the model is more likely to
|
|
139
|
+
// have misread the request than to be right about most of the conversation,
|
|
140
|
+
// and positional compaction is the safer tool for a wholesale reduction.
|
|
141
|
+
//
|
|
142
|
+
// `dropCandidates` is built by walking `eligible` oldest-first, so slicing
|
|
143
|
+
// from the front drops the OLDEST candidates and spares the newest when the
|
|
144
|
+
// cap binds — the same recency assumption every other stage makes.
|
|
145
|
+
const maxDrops = Math.floor(eligible.length * maxDropRatio);
|
|
146
|
+
const droppedIndices = dropCandidates.slice(0, Math.max(0, maxDrops));
|
|
147
|
+
if (droppedIndices.length === 0) {
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
const dropSet = new Set(droppedIndices);
|
|
151
|
+
const kept = messages.filter((_, index) => !dropSet.has(index));
|
|
152
|
+
logger.info("[Compaction] Stage 0 (relevance)", {
|
|
153
|
+
model: result.model,
|
|
154
|
+
latencyMs: result.latencyMs,
|
|
155
|
+
askedCount: eligible.length,
|
|
156
|
+
answeredCount: answered,
|
|
157
|
+
droppedCount: droppedIndices.length,
|
|
158
|
+
cappedFrom: dropCandidates.length,
|
|
159
|
+
});
|
|
160
|
+
return {
|
|
161
|
+
messages: kept,
|
|
162
|
+
droppedIndices,
|
|
163
|
+
askedCount: eligible.length,
|
|
164
|
+
answeredCount: answered,
|
|
165
|
+
model: result.model,
|
|
166
|
+
latencyMs: result.latencyMs,
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Gate a generated summary before it replaces the messages it covers.
|
|
171
|
+
*
|
|
172
|
+
* Stage 3 accepts any non-empty string today, so a summarizer that returned
|
|
173
|
+
* an apology, a refusal, or a truncated fragment silently destroys the
|
|
174
|
+
* conversation it was meant to preserve — and the messages are gone by the
|
|
175
|
+
* time anyone reads the summary.
|
|
176
|
+
*
|
|
177
|
+
* Returns `true` when the summary may be used and `false` only on a confident
|
|
178
|
+
* rejection. Anything else — no decision provider, a failed call, an
|
|
179
|
+
* unanswered or uncertain verdict — returns `true`, because refusing a
|
|
180
|
+
* summary means falling through to truncation, which loses strictly more.
|
|
181
|
+
*/
|
|
182
|
+
export async function summaryPreservesContext(summary, replacedMessages, decide, options) {
|
|
183
|
+
if (!summary.trim() || replacedMessages.length === 0) {
|
|
184
|
+
return true;
|
|
185
|
+
}
|
|
186
|
+
const result = await decide({
|
|
187
|
+
state: {
|
|
188
|
+
summary: summary.slice(0, 12000),
|
|
189
|
+
original_messages: replacedMessages.slice(-60).map((message) => ({
|
|
190
|
+
role: message.role,
|
|
191
|
+
text: preview(message),
|
|
192
|
+
})),
|
|
193
|
+
},
|
|
194
|
+
questions: {
|
|
195
|
+
preserves: {
|
|
196
|
+
type: "boolean",
|
|
197
|
+
instructions: "The summary preserves every decision, requirement, constraint, correction and open question from the original messages. Nothing that a later turn could need has been dropped.",
|
|
198
|
+
},
|
|
199
|
+
is_refusal: {
|
|
200
|
+
type: "boolean",
|
|
201
|
+
instructions: "The summary is not a summary at all — it is an apology, a refusal, an error message, or a request for clarification.",
|
|
202
|
+
},
|
|
203
|
+
},
|
|
204
|
+
timeoutMs: options?.timeoutMs,
|
|
205
|
+
});
|
|
206
|
+
if (!result) {
|
|
207
|
+
return true;
|
|
208
|
+
}
|
|
209
|
+
const minConfidence = options?.minConfidence ?? 0.6;
|
|
210
|
+
const refusal = gateDecisionBoolean(result.answers, "is_refusal", {
|
|
211
|
+
minConfidence,
|
|
212
|
+
});
|
|
213
|
+
if (refusal === true) {
|
|
214
|
+
logger.warn("[Compaction] Summary rejected — not a summary", {
|
|
215
|
+
model: result.model,
|
|
216
|
+
});
|
|
217
|
+
return false;
|
|
218
|
+
}
|
|
219
|
+
const preserves = gateDecisionBoolean(result.answers, "preserves", {
|
|
220
|
+
minConfidence,
|
|
221
|
+
});
|
|
222
|
+
if (preserves === false) {
|
|
223
|
+
logger.warn("[Compaction] Summary rejected — loses required context", {
|
|
224
|
+
model: result.model,
|
|
225
|
+
messageCount: replacedMessages.length,
|
|
226
|
+
});
|
|
227
|
+
return false;
|
|
228
|
+
}
|
|
229
|
+
return true;
|
|
230
|
+
}
|
|
231
|
+
/** Bounded text for one message, used in both state and criteria. */
|
|
232
|
+
function preview(message) {
|
|
233
|
+
return message.content.slice(0, MAX_MESSAGE_CHARS);
|
|
234
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { AIProviderName } from "../constants/enums.js";
|
|
2
2
|
import type { EvaluationData } from "../index.js";
|
|
3
3
|
import type { NeuroLink } from "../neurolink.js";
|
|
4
|
-
import type { UnknownRecord, MiddlewareFactoryOptions, StreamOptions, StreamResult, AIProvider, AnalyticsData, EnhancedGenerateResult, TextGenerationOptions, TextGenerationResult, ValidationSchema, EmbedInput } from "../types/index.js";
|
|
4
|
+
import type { UnknownRecord, MiddlewareFactoryOptions, StreamOptions, StreamResult, AIProvider, AnalyticsData, EnhancedGenerateResult, TextGenerationOptions, TextGenerationResult, ValidationSchema, EmbedInput, DecisionRequest, DecisionResult } from "../types/index.js";
|
|
5
5
|
import { TelemetryHandler } from "./modules/TelemetryHandler.js";
|
|
6
6
|
import type { LanguageModel, ModelMessage, RawUsageObject, Tool, ToolCallRepairFunction, ToolSet } from "../types/index.js";
|
|
7
7
|
export declare abstract class BaseProvider implements AIProvider {
|
|
@@ -358,6 +358,30 @@ export declare abstract class BaseProvider implements AIProvider {
|
|
|
358
358
|
* @returns The default embedding model name, or undefined if not supported
|
|
359
359
|
*/
|
|
360
360
|
protected getDefaultEmbeddingModel(): string | undefined;
|
|
361
|
+
/**
|
|
362
|
+
* Evaluate a state against a batch of typed questions.
|
|
363
|
+
*
|
|
364
|
+
* Default implementation throws, exactly as `embed()` does: `decide` is an
|
|
365
|
+
* optional capability, declared per provider by `"decide"` in its
|
|
366
|
+
* descriptor's `inferenceKinds`. Callers that want a silent no-op on an
|
|
367
|
+
* unsupporting provider should check the descriptor first rather than
|
|
368
|
+
* catching this.
|
|
369
|
+
*
|
|
370
|
+
* Latency on a decision model is flat in question count while concurrent
|
|
371
|
+
* requests queue, so this method is deliberately batch-shaped: one call
|
|
372
|
+
* carries every question a caller needs. There is no single-question
|
|
373
|
+
* convenience wrapper, because it would invite fanning out.
|
|
374
|
+
*
|
|
375
|
+
* @throws Error if the provider does not serve the `decide` inference type
|
|
376
|
+
*/
|
|
377
|
+
decide(_request: DecisionRequest): Promise<DecisionResult>;
|
|
378
|
+
/**
|
|
379
|
+
* Get the default decision model for this provider.
|
|
380
|
+
*
|
|
381
|
+
* Mirrors {@link getDefaultEmbeddingModel}. Returns undefined for providers
|
|
382
|
+
* that do not serve the `decide` inference type.
|
|
383
|
+
*/
|
|
384
|
+
protected getDefaultDecisionModel(): string | undefined;
|
|
361
385
|
/**
|
|
362
386
|
* Create an `experimental_repairToolCall` handler for streamText/generateText.
|
|
363
387
|
* Dynamically reads the tool's JSON schema to repair wrong names and params.
|
|
@@ -441,6 +465,8 @@ export declare abstract class BaseProvider implements AIProvider {
|
|
|
441
465
|
* that call `withTurnTimeout` directly.
|
|
442
466
|
*/
|
|
443
467
|
protected getDescriptorGenerateMs(): number | undefined;
|
|
468
|
+
/** Per-provider `decide` budget from the descriptor, when one is declared. */
|
|
469
|
+
protected getDescriptorDecideMs(): number | undefined;
|
|
444
470
|
protected abstract getAISDKModel(): LanguageModel | Promise<LanguageModel>;
|
|
445
471
|
/**
|
|
446
472
|
* Public handle on this provider's model object.
|
|
@@ -1833,6 +1833,40 @@ export class BaseProvider {
|
|
|
1833
1833
|
return undefined;
|
|
1834
1834
|
}
|
|
1835
1835
|
// ===================
|
|
1836
|
+
// `decide` inference type
|
|
1837
|
+
// ===================
|
|
1838
|
+
/**
|
|
1839
|
+
* Evaluate a state against a batch of typed questions.
|
|
1840
|
+
*
|
|
1841
|
+
* Default implementation throws, exactly as `embed()` does: `decide` is an
|
|
1842
|
+
* optional capability, declared per provider by `"decide"` in its
|
|
1843
|
+
* descriptor's `inferenceKinds`. Callers that want a silent no-op on an
|
|
1844
|
+
* unsupporting provider should check the descriptor first rather than
|
|
1845
|
+
* catching this.
|
|
1846
|
+
*
|
|
1847
|
+
* Latency on a decision model is flat in question count while concurrent
|
|
1848
|
+
* requests queue, so this method is deliberately batch-shaped: one call
|
|
1849
|
+
* carries every question a caller needs. There is no single-question
|
|
1850
|
+
* convenience wrapper, because it would invite fanning out.
|
|
1851
|
+
*
|
|
1852
|
+
* @throws Error if the provider does not serve the `decide` inference type
|
|
1853
|
+
*/
|
|
1854
|
+
async decide(_request) {
|
|
1855
|
+
logger.warn(`decide() called on ${this.providerName}, which does not serve the decide inference type`);
|
|
1856
|
+
throw new Error(`The decide inference type is not supported by the ${this.providerName} provider. ` +
|
|
1857
|
+
`Supported providers: typesafe. A provider declares this capability with ` +
|
|
1858
|
+
`"decide" in its descriptor's inferenceKinds.`);
|
|
1859
|
+
}
|
|
1860
|
+
/**
|
|
1861
|
+
* Get the default decision model for this provider.
|
|
1862
|
+
*
|
|
1863
|
+
* Mirrors {@link getDefaultEmbeddingModel}. Returns undefined for providers
|
|
1864
|
+
* that do not serve the `decide` inference type.
|
|
1865
|
+
*/
|
|
1866
|
+
getDefaultDecisionModel() {
|
|
1867
|
+
return undefined;
|
|
1868
|
+
}
|
|
1869
|
+
// ===================
|
|
1836
1870
|
// ===================
|
|
1837
1871
|
// BZ-665: Schema-driven tool call repair
|
|
1838
1872
|
// ===================
|
|
@@ -1988,6 +2022,11 @@ export class BaseProvider {
|
|
|
1988
2022
|
return PROVIDER_DESCRIPTORS_BY_NAME.get(this.providerName)?.timeouts
|
|
1989
2023
|
?.generateMs;
|
|
1990
2024
|
}
|
|
2025
|
+
/** Per-provider `decide` budget from the descriptor, when one is declared. */
|
|
2026
|
+
getDescriptorDecideMs() {
|
|
2027
|
+
return PROVIDER_DESCRIPTORS_BY_NAME.get(this.providerName)?.timeouts
|
|
2028
|
+
?.decideMs;
|
|
2029
|
+
}
|
|
1991
2030
|
/**
|
|
1992
2031
|
* Public handle on this provider's model object.
|
|
1993
2032
|
*
|
|
@@ -25,6 +25,14 @@
|
|
|
25
25
|
* Tool granularity (ITEM D): when `params.granularity === "tool"`, individual
|
|
26
26
|
* unpicked tools are excluded rather than whole servers. Falls back to "server"
|
|
27
27
|
* if the embedding path is disabled or fails.
|
|
28
|
+
*
|
|
29
|
+
* Decision router: when `params.decideFn` is supplied AND a decision provider
|
|
30
|
+
* is configured, one calibrated yes/no question per server replaces the router
|
|
31
|
+
* LLM call — ~400ms against a 15s budget, and a probability per server rather
|
|
32
|
+
* than a list that cannot express doubt. It runs after the embedding
|
|
33
|
+
* fast-path and before the LLM router, and returning null from it falls
|
|
34
|
+
* through to the LLM path unchanged. See `toolRoutingDecision.ts` for why a
|
|
35
|
+
* server is only ever dropped on a CONFIDENT no.
|
|
28
36
|
*/
|
|
29
37
|
import type { ToolRoutingCatalogEntry, ToolRoutingResolutionParams, ToolRoutingServerDescriptor } from "../types/index.js";
|
|
30
38
|
/**
|
package/dist/core/toolRouting.js
CHANGED
|
@@ -25,11 +25,20 @@
|
|
|
25
25
|
* Tool granularity (ITEM D): when `params.granularity === "tool"`, individual
|
|
26
26
|
* unpicked tools are excluded rather than whole servers. Falls back to "server"
|
|
27
27
|
* if the embedding path is disabled or fails.
|
|
28
|
+
*
|
|
29
|
+
* Decision router: when `params.decideFn` is supplied AND a decision provider
|
|
30
|
+
* is configured, one calibrated yes/no question per server replaces the router
|
|
31
|
+
* LLM call — ~400ms against a 15s budget, and a probability per server rather
|
|
32
|
+
* than a list that cannot express doubt. It runs after the embedding
|
|
33
|
+
* fast-path and before the LLM router, and returning null from it falls
|
|
34
|
+
* through to the LLM path unchanged. See `toolRoutingDecision.ts` for why a
|
|
35
|
+
* server is only ever dropped on a CONFIDENT no.
|
|
28
36
|
*/
|
|
29
37
|
import { z } from "zod";
|
|
30
38
|
import { logger } from "../utils/logger.js";
|
|
31
39
|
import { withTimeout } from "../utils/async/index.js";
|
|
32
40
|
import { selectRelevantToolNames } from "./toolRoutingEmbedding.js";
|
|
41
|
+
import { selectServersByDecision } from "./toolRoutingDecision.js";
|
|
33
42
|
const routerOutputSchema = z.object({
|
|
34
43
|
servers: z.array(z.string()),
|
|
35
44
|
});
|
|
@@ -290,7 +299,7 @@ async function runEmbeddingFastPath(userQuery, routableServers, alwaysIncludeSer
|
|
|
290
299
|
* Returns an empty list on any skip/failure path — see module doc.
|
|
291
300
|
*/
|
|
292
301
|
export async function resolveToolRoutingExclusions(params) {
|
|
293
|
-
const { catalog, alwaysIncludeServerIds, userQuery, routerPromptPrefix, routerModel, timeoutMs, generateFn, emitDecision,
|
|
302
|
+
const { catalog, alwaysIncludeServerIds, userQuery, routerPromptPrefix, routerModel, timeoutMs, generateFn, decideFn, decisionMinDropConfidence, emitDecision,
|
|
294
303
|
// L2 / ITEM D parameters (all optional — omitting reproduces today's behavior)
|
|
295
304
|
embedFn, embeddingConfig, granularity = "server", embeddingVectorCache, } = params;
|
|
296
305
|
const routableServers = catalog.filter((server) => !alwaysIncludeServerIds.includes(server.id));
|
|
@@ -365,6 +374,7 @@ export async function resolveToolRoutingExclusions(params) {
|
|
|
365
374
|
embeddingActivated: true,
|
|
366
375
|
candidateToolCount: embResult.candidateToolCount,
|
|
367
376
|
granularity: embResult.granularity,
|
|
377
|
+
strategy: "embedding",
|
|
368
378
|
});
|
|
369
379
|
return embResult.excludedToolNames;
|
|
370
380
|
}
|
|
@@ -372,6 +382,19 @@ export async function resolveToolRoutingExclusions(params) {
|
|
|
372
382
|
// fall through to the LLM-router path below.
|
|
373
383
|
}
|
|
374
384
|
// -------------------------------------------------------------------------
|
|
385
|
+
// DECISION-MODEL ROUTER — runs when a decision provider is configured
|
|
386
|
+
// -------------------------------------------------------------------------
|
|
387
|
+
// Placed after the embedding fast-path (which is cheaper still and only
|
|
388
|
+
// activates on large catalogs) and before the generative router, which it
|
|
389
|
+
// replaces: one yes/no question per server, ~400ms, calibrated per server.
|
|
390
|
+
// Any null return falls through to the LLM path below unchanged.
|
|
391
|
+
if (decideFn) {
|
|
392
|
+
const decided = await runDecisionRouter(userQuery, routableServers, decideFn, decisionMinDropConfidence, routingStartTime, emitDecision);
|
|
393
|
+
if (decided) {
|
|
394
|
+
return decided;
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
// -------------------------------------------------------------------------
|
|
375
398
|
// EXISTING LLM ROUTER PATH (L3) — runs when embedding is off or fell back
|
|
376
399
|
// -------------------------------------------------------------------------
|
|
377
400
|
const routerPrompt = buildRouterPrompt(userQuery, routableServers, routerPromptPrefix);
|
|
@@ -467,6 +490,7 @@ export async function resolveToolRoutingExclusions(params) {
|
|
|
467
490
|
cacheHit: false,
|
|
468
491
|
durationMs: Date.now() - routingStartTime,
|
|
469
492
|
granularity: "server",
|
|
493
|
+
strategy: "llm",
|
|
470
494
|
});
|
|
471
495
|
return excludedToolNames;
|
|
472
496
|
}
|
|
@@ -493,3 +517,30 @@ export async function resolveToolRoutingExclusions(params) {
|
|
|
493
517
|
return [];
|
|
494
518
|
}
|
|
495
519
|
}
|
|
520
|
+
/**
|
|
521
|
+
* Decision-model routing path, kept out of `resolveToolRoutingExclusions` so
|
|
522
|
+
* that function stays readable as a list of strategies rather than growing a
|
|
523
|
+
* fourth inline block.
|
|
524
|
+
*
|
|
525
|
+
* Returns the exclusion list when the decision router produced one, or `null`
|
|
526
|
+
* to fall through to the generative router unchanged.
|
|
527
|
+
*/
|
|
528
|
+
async function runDecisionRouter(userQuery, routableServers, decideFn, minDropConfidence, routingStartTime, emitDecision) {
|
|
529
|
+
const decided = await selectServersByDecision(userQuery, routableServers, decideFn, { minDropConfidence });
|
|
530
|
+
if (!decided) {
|
|
531
|
+
return null;
|
|
532
|
+
}
|
|
533
|
+
safeEmitDecision(emitDecision, {
|
|
534
|
+
outcome: "applied",
|
|
535
|
+
selectedServerIds: decided.selectedServerIds,
|
|
536
|
+
excludedServerIds: decided.excludedServerIds,
|
|
537
|
+
hallucinatedIds: [],
|
|
538
|
+
excludedToolCount: decided.excludedToolNames.length,
|
|
539
|
+
routableServerCount: routableServers.length,
|
|
540
|
+
cacheHit: false,
|
|
541
|
+
durationMs: Date.now() - routingStartTime,
|
|
542
|
+
granularity: "server",
|
|
543
|
+
strategy: "decision",
|
|
544
|
+
});
|
|
545
|
+
return decided.excludedToolNames;
|
|
546
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool routing by decision model.
|
|
3
|
+
*
|
|
4
|
+
* The shipped router asks a generative model for `{servers: string[]}` on a
|
|
5
|
+
* 15-second budget. That shape cannot express uncertainty: a server is in the
|
|
6
|
+
* list or it is not, and the only recourse for a model that is unsure is to
|
|
7
|
+
* include it. A decision model answers one yes/no question per server and
|
|
8
|
+
* returns a *calibrated probability* for each, in one round trip of ~400ms
|
|
9
|
+
* costing ~$0.00002.
|
|
10
|
+
*
|
|
11
|
+
* The asymmetry that follows is the whole point. Keeping a server that was not
|
|
12
|
+
* needed costs a few hundred tokens of tool definitions. Dropping a server
|
|
13
|
+
* that WAS needed costs the turn — the model cannot call a tool it was never
|
|
14
|
+
* shown, and it has no way to ask for it back. So a server is excluded only on
|
|
15
|
+
* a *confident no*; unanswered, mistyped and near-coin-flip answers all keep
|
|
16
|
+
* it. This is the same reasoning as the classifier's asymmetric upgrade and
|
|
17
|
+
* downgrade bars, applied to a different decision.
|
|
18
|
+
*
|
|
19
|
+
* Fail-open in every direction: returns `null` whenever the caller should
|
|
20
|
+
* carry on as if this module did not exist.
|
|
21
|
+
*
|
|
22
|
+
* @module core/toolRoutingDecision
|
|
23
|
+
*/
|
|
24
|
+
import type { DecisionCallerFn, ToolRoutingCatalogEntry, ToolRoutingDecisionOutcome } from "../types/index.js";
|
|
25
|
+
/**
|
|
26
|
+
* Ask one yes/no question per routable server and return the servers to keep.
|
|
27
|
+
*
|
|
28
|
+
* Returns `null` when the caller should fall through to its existing path:
|
|
29
|
+
* no decision provider, a failed call, fewer than two servers, or an answer
|
|
30
|
+
* set that would change nothing.
|
|
31
|
+
*/
|
|
32
|
+
export declare function selectServersByDecision(userQuery: string, routableServers: ToolRoutingCatalogEntry[], decide: DecisionCallerFn, options?: {
|
|
33
|
+
timeoutMs?: number;
|
|
34
|
+
minDropConfidence?: number;
|
|
35
|
+
}): Promise<ToolRoutingDecisionOutcome | null>;
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool routing by decision model.
|
|
3
|
+
*
|
|
4
|
+
* The shipped router asks a generative model for `{servers: string[]}` on a
|
|
5
|
+
* 15-second budget. That shape cannot express uncertainty: a server is in the
|
|
6
|
+
* list or it is not, and the only recourse for a model that is unsure is to
|
|
7
|
+
* include it. A decision model answers one yes/no question per server and
|
|
8
|
+
* returns a *calibrated probability* for each, in one round trip of ~400ms
|
|
9
|
+
* costing ~$0.00002.
|
|
10
|
+
*
|
|
11
|
+
* The asymmetry that follows is the whole point. Keeping a server that was not
|
|
12
|
+
* needed costs a few hundred tokens of tool definitions. Dropping a server
|
|
13
|
+
* that WAS needed costs the turn — the model cannot call a tool it was never
|
|
14
|
+
* shown, and it has no way to ask for it back. So a server is excluded only on
|
|
15
|
+
* a *confident no*; unanswered, mistyped and near-coin-flip answers all keep
|
|
16
|
+
* it. This is the same reasoning as the classifier's asymmetric upgrade and
|
|
17
|
+
* downgrade bars, applied to a different decision.
|
|
18
|
+
*
|
|
19
|
+
* Fail-open in every direction: returns `null` whenever the caller should
|
|
20
|
+
* carry on as if this module did not exist.
|
|
21
|
+
*
|
|
22
|
+
* @module core/toolRoutingDecision
|
|
23
|
+
*/
|
|
24
|
+
import { decisionKey, gateDecisionBoolean } from "../utils/decisionAnswers.js";
|
|
25
|
+
import { logger } from "../utils/logger.js";
|
|
26
|
+
/** Namespace for the per-server questions in the batch. */
|
|
27
|
+
const SERVER_NAMESPACE = "server";
|
|
28
|
+
/**
|
|
29
|
+
* How confidently the model must answer "no" before a server's tools are
|
|
30
|
+
* withheld. Deliberately high — a wrongly dropped server breaks the turn,
|
|
31
|
+
* while a wrongly kept one costs tokens.
|
|
32
|
+
*/
|
|
33
|
+
const DEFAULT_MIN_DROP_CONFIDENCE = 0.6;
|
|
34
|
+
/**
|
|
35
|
+
* Upper bound on the query text sent as state. The query is untrusted and may
|
|
36
|
+
* attempt injection; the blast radius is bounded (the worst outcome is keeping
|
|
37
|
+
* MORE already-registered tools, since ids are never read back off the wire —
|
|
38
|
+
* answers are matched by position), but an unbounded state would still be
|
|
39
|
+
* billed and could push the request past the model's input ceiling.
|
|
40
|
+
*/
|
|
41
|
+
const MAX_STATE_CHARS = 10000;
|
|
42
|
+
/**
|
|
43
|
+
* Cap on servers asked about in one batch. Latency is flat in question count,
|
|
44
|
+
* so this is purely an input-size guard: state plus all questions must stay
|
|
45
|
+
* under the ~64K request ceiling.
|
|
46
|
+
*/
|
|
47
|
+
const MAX_SERVERS = 200;
|
|
48
|
+
/**
|
|
49
|
+
* The question wording, and why it is this wording.
|
|
50
|
+
*
|
|
51
|
+
* The obvious phrasing — "answering this request will require calling at least
|
|
52
|
+
* one tool from this server", with a `false` criterion reading "this server is
|
|
53
|
+
* unrelated, OR the request needs no tool at all" — was measured against a
|
|
54
|
+
* 10-request × 5-server labelled set. It separated correctly but weakly:
|
|
55
|
+
* unrelated servers averaged p=0.31 and reached 0.80, so at the 0.6 drop bar
|
|
56
|
+
* only 12 of 39 unneeded servers were dropped.
|
|
57
|
+
*
|
|
58
|
+
* Naming the server, asking in the present tense about what carrying out the
|
|
59
|
+
* request *involves*, and splitting the bundled `false` criterion into a single
|
|
60
|
+
* claim moved unrelated servers to a mean of p=0.03 with a maximum of 0.35 —
|
|
61
|
+
* **37 of 39** dropped at the same bar, still with zero wrong drops.
|
|
62
|
+
*
|
|
63
|
+
* The lesson generalises: a decision model reads literally, and an `or` in a
|
|
64
|
+
* criterion is two questions wearing one coat.
|
|
65
|
+
*/
|
|
66
|
+
function serverQuestion(server) {
|
|
67
|
+
const does = uncapitalize(describeServer(server));
|
|
68
|
+
return {
|
|
69
|
+
type: "boolean",
|
|
70
|
+
instructions: `The request needs the "${server.id}" server, which can ${does}.`,
|
|
71
|
+
criteria: {
|
|
72
|
+
true: `Carrying out the request involves ${does}.`,
|
|
73
|
+
false: `The request is about something else; nothing it asks for involves ${does}.`,
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Lowercase only the FIRST character. A plain `toLowerCase()` would turn
|
|
79
|
+
* "Read and write GitHub issues" into "…github issues", and the proper noun is
|
|
80
|
+
* often the strongest signal in the sentence.
|
|
81
|
+
*/
|
|
82
|
+
function uncapitalize(text) {
|
|
83
|
+
// charAt rather than an index: it returns "" for an empty string, so this
|
|
84
|
+
// needs no length guard and no non-null assertion.
|
|
85
|
+
return text.charAt(0).toLowerCase() + text.slice(1);
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Ask one yes/no question per routable server and return the servers to keep.
|
|
89
|
+
*
|
|
90
|
+
* Returns `null` when the caller should fall through to its existing path:
|
|
91
|
+
* no decision provider, a failed call, fewer than two servers, or an answer
|
|
92
|
+
* set that would change nothing.
|
|
93
|
+
*/
|
|
94
|
+
export async function selectServersByDecision(userQuery, routableServers, decide, options) {
|
|
95
|
+
if (!userQuery || routableServers.length < 2) {
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
98
|
+
const servers = routableServers.slice(0, MAX_SERVERS);
|
|
99
|
+
const questions = {};
|
|
100
|
+
servers.forEach((server, index) => {
|
|
101
|
+
questions[decisionKey(SERVER_NAMESPACE, index)] = serverQuestion(server);
|
|
102
|
+
});
|
|
103
|
+
const result = await decide({
|
|
104
|
+
// Structured state rather than a prompt string: the vendor accepts JSON,
|
|
105
|
+
// and naming the fields is what stops the model reading the catalogue as
|
|
106
|
+
// part of the user's request.
|
|
107
|
+
state: {
|
|
108
|
+
request: userQuery.slice(0, MAX_STATE_CHARS),
|
|
109
|
+
available_servers: servers.map((server) => ({
|
|
110
|
+
name: server.id,
|
|
111
|
+
does: describeServer(server),
|
|
112
|
+
tool_count: server.toolNames.length,
|
|
113
|
+
})),
|
|
114
|
+
},
|
|
115
|
+
questions,
|
|
116
|
+
timeoutMs: options?.timeoutMs,
|
|
117
|
+
});
|
|
118
|
+
if (!result) {
|
|
119
|
+
return null;
|
|
120
|
+
}
|
|
121
|
+
const minConfidence = options?.minDropConfidence ?? DEFAULT_MIN_DROP_CONFIDENCE;
|
|
122
|
+
const selectedServerIds = [];
|
|
123
|
+
const excludedServerIds = [];
|
|
124
|
+
let answered = 0;
|
|
125
|
+
servers.forEach((server, index) => {
|
|
126
|
+
const verdict = gateDecisionBoolean(result.answers, decisionKey(SERVER_NAMESPACE, index), { minConfidence });
|
|
127
|
+
if (verdict !== undefined) {
|
|
128
|
+
answered += 1;
|
|
129
|
+
}
|
|
130
|
+
// Only a confident `false` drops a server. `undefined` (unanswered, wrong
|
|
131
|
+
// type, or too close to a coin flip) and `true` both keep it.
|
|
132
|
+
if (verdict === false) {
|
|
133
|
+
excludedServerIds.push(server.id);
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
selectedServerIds.push(server.id);
|
|
137
|
+
}
|
|
138
|
+
});
|
|
139
|
+
// Servers past the cap were never asked about, so they are kept — a server
|
|
140
|
+
// that was not offered to the model must never be dropped by its silence.
|
|
141
|
+
for (const server of routableServers.slice(MAX_SERVERS)) {
|
|
142
|
+
selectedServerIds.push(server.id);
|
|
143
|
+
}
|
|
144
|
+
if (answered === 0) {
|
|
145
|
+
logger.debug("[ToolRouting] Decision router returned no usable answers", {
|
|
146
|
+
routableServerCount: routableServers.length,
|
|
147
|
+
});
|
|
148
|
+
return null;
|
|
149
|
+
}
|
|
150
|
+
if (excludedServerIds.length === 0) {
|
|
151
|
+
// Nothing to exclude is indistinguishable from routing being off, and
|
|
152
|
+
// reporting it as "applied" would make the telemetry claim a narrowing
|
|
153
|
+
// that did not happen.
|
|
154
|
+
return null;
|
|
155
|
+
}
|
|
156
|
+
const excludedSet = new Set(excludedServerIds);
|
|
157
|
+
const excludedToolNames = routableServers
|
|
158
|
+
.filter((server) => excludedSet.has(server.id))
|
|
159
|
+
.flatMap((server) => server.toolNames);
|
|
160
|
+
logger.debug("[ToolRouting] Decision router applied", {
|
|
161
|
+
model: result.model,
|
|
162
|
+
latencyMs: result.latencyMs,
|
|
163
|
+
askedServerCount: servers.length,
|
|
164
|
+
answeredCount: answered,
|
|
165
|
+
keptServerCount: selectedServerIds.length,
|
|
166
|
+
excludedServerCount: excludedServerIds.length,
|
|
167
|
+
excludedToolCount: excludedToolNames.length,
|
|
168
|
+
});
|
|
169
|
+
return {
|
|
170
|
+
selectedServerIds,
|
|
171
|
+
excludedServerIds,
|
|
172
|
+
excludedToolNames,
|
|
173
|
+
answeredCount: answered,
|
|
174
|
+
model: result.model,
|
|
175
|
+
latencyMs: result.latencyMs,
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* One line describing what a server is for. Falls back to naming its tools
|
|
180
|
+
* when the host declared no description, since a bare id ("fs", "gh") carries
|
|
181
|
+
* almost no signal and the tool names usually carry a great deal.
|
|
182
|
+
*/
|
|
183
|
+
function describeServer(server) {
|
|
184
|
+
if (server.description && server.description.trim().length > 0) {
|
|
185
|
+
return server.description;
|
|
186
|
+
}
|
|
187
|
+
const names = server.toolNames
|
|
188
|
+
.map((name) => name.slice(server.id.length + 1))
|
|
189
|
+
.filter((name) => name.length > 0)
|
|
190
|
+
.slice(0, 12);
|
|
191
|
+
return names.length > 0
|
|
192
|
+
? `provides the tools: ${names.join(", ")}`
|
|
193
|
+
: `the "${server.id}" server`;
|
|
194
|
+
}
|