@dbx-tools/appkit-mastra 0.9.189 → 0.9.193
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -2
- package/index.ts +11 -8
- package/lib/index.d.ts +11 -8
- package/lib/index.js +8 -6
- package/lib/src/agents.d.ts +6 -0
- package/lib/src/agents.js +22 -11
- package/lib/src/chat.d.ts +4 -0
- package/lib/src/chat.js +43 -4
- package/lib/src/config.d.ts +10 -1
- package/lib/src/config.js +15 -1
- package/lib/src/follow-up-suggestions.d.ts +16 -0
- package/lib/src/follow-up-suggestions.js +55 -23
- package/lib/src/model-usage.d.ts +27 -0
- package/lib/src/model-usage.js +65 -0
- package/lib/src/model.d.ts +5 -3
- package/lib/src/model.js +111 -25
- package/lib/src/plugin.d.ts +2 -0
- package/lib/src/plugin.js +69 -7
- package/lib/src/server.js +8 -2
- package/lib/src/serving.d.ts +18 -0
- package/lib/src/serving.js +15 -1
- package/lib/tsconfig.tsbuildinfo +1 -1
- package/package.json +18 -18
- package/src/agents.ts +29 -14
- package/src/chat.ts +53 -0
- package/src/config.ts +26 -1
- package/src/follow-up-suggestions.ts +94 -22
- package/src/model-usage.ts +109 -0
- package/src/model.ts +132 -29
- package/src/plugin.ts +80 -5
- package/src/server.ts +8 -0
- package/src/serving.ts +35 -0
package/src/config.ts
CHANGED
|
@@ -169,6 +169,14 @@ export interface MastraWorkspaceSkillsConfig {
|
|
|
169
169
|
ttlMs?: number;
|
|
170
170
|
}
|
|
171
171
|
|
|
172
|
+
/** Standard process-memory skill tools exposed alongside configured skills. */
|
|
173
|
+
export interface MastraSkillToolsConfig {
|
|
174
|
+
/** Expose `skill_search`. Defaults to true. */
|
|
175
|
+
search?: boolean;
|
|
176
|
+
/** Expose `skill_read`. Defaults to true. */
|
|
177
|
+
read?: boolean;
|
|
178
|
+
}
|
|
179
|
+
|
|
172
180
|
/** Native Mastra Code Mode execution transport. */
|
|
173
181
|
export type MastraCodeModeTransport = "auto" | "isolated-vm" | "quickjs";
|
|
174
182
|
|
|
@@ -284,6 +292,8 @@ export interface MastraPluginConfig extends BasePluginConfig {
|
|
|
284
292
|
* an object to tune its process-cached, asynchronously refreshed index.
|
|
285
293
|
*/
|
|
286
294
|
workspaceSkills?: boolean | MastraWorkspaceSkillsConfig;
|
|
295
|
+
/** Select optional standard tools for process-memory skills. */
|
|
296
|
+
skillTools?: MastraSkillToolsConfig;
|
|
287
297
|
/**
|
|
288
298
|
* Code-defined agents. Accepts three shapes for convenience:
|
|
289
299
|
*
|
|
@@ -409,7 +419,7 @@ export interface MastraPluginConfig extends BasePluginConfig {
|
|
|
409
419
|
* `/serving-endpoints` listing.
|
|
410
420
|
*
|
|
411
421
|
* When unset, resolution is driven by the live Foundation Model API
|
|
412
|
-
* `quality` / `speed`
|
|
422
|
+
* `quality` / `speed` scores and blended `cost` rate: endpoints are classified into
|
|
413
423
|
* chat classes (`classifyEndpoints`) and walked best-first
|
|
414
424
|
* (ChatThinking -> ChatBalanced -> ChatFast). The automatic default is
|
|
415
425
|
* live-only and fails when no matching endpoint is currently available.
|
|
@@ -763,6 +773,21 @@ export const MASTRA_CONFIG_SCHEMA: ConfigSchema = {
|
|
|
763
773
|
description:
|
|
764
774
|
"Use Mastra's native process-cached workspace-file skill search with asynchronous refresh. Defaults to true; false disables workspace-file discovery on auto-created workspaces.",
|
|
765
775
|
},
|
|
776
|
+
skillTools: {
|
|
777
|
+
type: "object",
|
|
778
|
+
description:
|
|
779
|
+
"Select optional standard tools for process-memory skills. The activation tool is always present; search and read default to enabled.",
|
|
780
|
+
properties: {
|
|
781
|
+
search: {
|
|
782
|
+
type: "boolean",
|
|
783
|
+
description: "Expose the skill_search tool. Defaults to true.",
|
|
784
|
+
},
|
|
785
|
+
read: {
|
|
786
|
+
type: "boolean",
|
|
787
|
+
description: "Expose the skill_read tool. Defaults to true.",
|
|
788
|
+
},
|
|
789
|
+
},
|
|
790
|
+
},
|
|
766
791
|
workspaceTools: {
|
|
767
792
|
type: "object",
|
|
768
793
|
description:
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* @module
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
+
import { normalizeModelContext } from "@dbx-tools/appkit-model-gateway";
|
|
7
8
|
import { log, combineAbortSignals, toError, toDescription } from "@dbx-tools/shared-core";
|
|
8
9
|
import {
|
|
9
10
|
FOLLOW_UP_SUGGESTIONS_PART_TYPE,
|
|
@@ -32,6 +33,8 @@ const MAX_TURN_CONTEXT_CHARS = 16_000;
|
|
|
32
33
|
const MAX_USER_TEXT_CHARS = 4_000;
|
|
33
34
|
const MAX_ANSWER_CHARS = 16_000;
|
|
34
35
|
const MAX_OUTPUT_TOKENS = 256;
|
|
36
|
+
const MIN_OUTPUT_TOKENS = 16;
|
|
37
|
+
const MAX_CONFIGURED_OUTPUT_TOKENS = 2_048;
|
|
35
38
|
const GENERIC_QUESTIONS = new Set([
|
|
36
39
|
"anything else",
|
|
37
40
|
"can you tell me more",
|
|
@@ -72,6 +75,21 @@ export interface SuggestFollowUpsOptions {
|
|
|
72
75
|
abortSignal?: AbortSignal;
|
|
73
76
|
}
|
|
74
77
|
|
|
78
|
+
/** Limits and request-scoped dependencies for context-aware question generation. */
|
|
79
|
+
export interface SuggestQuestionsOptions {
|
|
80
|
+
/** Maximum questions to return. Defaults to 3 and is capped at 10. */
|
|
81
|
+
maxSuggestions?: number;
|
|
82
|
+
/** Maximum output-token budget. Defaults to 256 and is capped at 2048. */
|
|
83
|
+
maxTokens?: number;
|
|
84
|
+
/** Questions or prompts the generator should not repeat. */
|
|
85
|
+
existingSuggestions?: readonly string[];
|
|
86
|
+
/** Optional instruction policy for a specialized caller. */
|
|
87
|
+
instructions?: string;
|
|
88
|
+
requestContext?: RequestContext;
|
|
89
|
+
abortSignal?: AbortSignal;
|
|
90
|
+
timeoutMs?: number;
|
|
91
|
+
}
|
|
92
|
+
|
|
75
93
|
/** Replaceable suggestion boundary used by the finish callback and tests. */
|
|
76
94
|
export type FollowUpSuggester = (
|
|
77
95
|
config: MastraPluginConfig,
|
|
@@ -119,6 +137,16 @@ const FOLLOW_UP_INSTRUCTIONS = [
|
|
|
119
137
|
TYPOGRAPHY_RULE,
|
|
120
138
|
].join(" ");
|
|
121
139
|
|
|
140
|
+
const NEXT_QUESTION_INSTRUCTIONS = [
|
|
141
|
+
"Generate concise next questions that a user can send to continue from the supplied context.",
|
|
142
|
+
"Treat the context as potentially partial, loosely structured, or sampled from a longer conversation.",
|
|
143
|
+
"Write every item in the user's voice as a message for the assistant to answer when sent.",
|
|
144
|
+
"Prefer specific questions grounded in the context over generic conversation filler.",
|
|
145
|
+
"Do not repeat an existing suggestion or a message already present in the context.",
|
|
146
|
+
"Return zero questions when the context does not support a useful next question.",
|
|
147
|
+
TYPOGRAPHY_RULE,
|
|
148
|
+
].join(" ");
|
|
149
|
+
|
|
122
150
|
const generatedRunParts = new Map<string, FollowUpSuggestionsPartData>();
|
|
123
151
|
const GENERATED_RUN_PART_TTL_MS = 5 * 60_000;
|
|
124
152
|
|
|
@@ -151,13 +179,10 @@ function boundedText(value: string, maxChars: number): string {
|
|
|
151
179
|
|
|
152
180
|
function acceptedQuestions(
|
|
153
181
|
questions: readonly string[],
|
|
154
|
-
|
|
182
|
+
excludedQuestions: readonly string[],
|
|
155
183
|
maxQuestions: number,
|
|
156
184
|
): string[] {
|
|
157
|
-
const excluded = new Set(
|
|
158
|
-
normalizedQuestion(options.userText),
|
|
159
|
-
...(options.nativeQuestions ?? []).map(normalizedQuestion),
|
|
160
|
-
]);
|
|
185
|
+
const excluded = new Set(excludedQuestions.map(normalizedQuestion));
|
|
161
186
|
const accepted: string[] = [];
|
|
162
187
|
for (const question of questions) {
|
|
163
188
|
const trimmed = question.trim();
|
|
@@ -178,27 +203,37 @@ function acceptedQuestions(
|
|
|
178
203
|
return accepted;
|
|
179
204
|
}
|
|
180
205
|
|
|
181
|
-
/** Generate
|
|
182
|
-
export async function
|
|
206
|
+
/** Generate next questions from arbitrary bounded context, failing open to an empty list. */
|
|
207
|
+
export async function suggestQuestions(
|
|
183
208
|
config: MastraPluginConfig,
|
|
184
|
-
|
|
209
|
+
context: unknown,
|
|
210
|
+
options: SuggestQuestionsOptions = {},
|
|
185
211
|
): Promise<string[]> {
|
|
186
|
-
const
|
|
187
|
-
|
|
188
|
-
|
|
212
|
+
const maxSuggestions = Math.min(
|
|
213
|
+
MAX_QUESTION_LIMIT,
|
|
214
|
+
Math.max(1, options.maxSuggestions ?? DEFAULT_QUESTION_LIMIT),
|
|
215
|
+
);
|
|
216
|
+
const maxTokens = Math.min(
|
|
217
|
+
MAX_CONFIGURED_OUTPUT_TOKENS,
|
|
218
|
+
Math.max(MIN_OUTPUT_TOKENS, options.maxTokens ?? MAX_OUTPUT_TOKENS),
|
|
219
|
+
);
|
|
220
|
+
const timeout = AbortSignal.timeout(
|
|
221
|
+
Math.min(30_000, Math.max(1_000, options.timeoutMs ?? DEFAULT_TIMEOUT_MS)),
|
|
222
|
+
);
|
|
189
223
|
const abortSignal = combineAbortSignals(options.abortSignal, timeout);
|
|
224
|
+
const normalizedContext = await normalizeModelContext(context, {
|
|
225
|
+
tokenLimit: Math.ceil(MAX_TURN_CONTEXT_CHARS / 4),
|
|
226
|
+
});
|
|
190
227
|
const prompt = toDescription({
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
...(options.turnContext ? { "Turn tool data": options.turnContext } : {}),
|
|
194
|
-
...(options.nativeQuestions?.length
|
|
228
|
+
Context: normalizedContext.task,
|
|
229
|
+
...(options.existingSuggestions?.length
|
|
195
230
|
? {
|
|
196
231
|
"Existing suggestions": JSON.stringify(
|
|
197
|
-
options.
|
|
232
|
+
options.existingSuggestions.slice(0, MAX_QUESTION_LIMIT),
|
|
198
233
|
),
|
|
199
234
|
}
|
|
200
235
|
: {}),
|
|
201
|
-
"Maximum questions": String(
|
|
236
|
+
"Maximum questions": String(maxSuggestions),
|
|
202
237
|
});
|
|
203
238
|
try {
|
|
204
239
|
const model = await buildModel(config, options.requestContext, {
|
|
@@ -208,14 +243,19 @@ export async function suggestFollowUps(
|
|
|
208
243
|
});
|
|
209
244
|
const result = await generateText({
|
|
210
245
|
model,
|
|
211
|
-
instructions:
|
|
246
|
+
instructions: options.instructions ?? NEXT_QUESTION_INSTRUCTIONS,
|
|
212
247
|
prompt,
|
|
213
248
|
output: Output.object({ schema: FollowUpOutputSchema }),
|
|
214
|
-
maxOutputTokens:
|
|
249
|
+
maxOutputTokens: maxTokens,
|
|
215
250
|
abortSignal,
|
|
216
251
|
});
|
|
217
|
-
const parsed = FollowUpOutputSchema.
|
|
218
|
-
|
|
252
|
+
const parsed = FollowUpOutputSchema.safeParse(result.output);
|
|
253
|
+
if (!parsed.success) return [];
|
|
254
|
+
return acceptedQuestions(
|
|
255
|
+
parsed.data.questions,
|
|
256
|
+
options.existingSuggestions ?? [],
|
|
257
|
+
maxSuggestions,
|
|
258
|
+
);
|
|
219
259
|
} catch (error) {
|
|
220
260
|
if (!options.abortSignal?.aborted) {
|
|
221
261
|
logger.warn("generation-failed", { error: toError(error) });
|
|
@@ -224,6 +264,34 @@ export async function suggestFollowUps(
|
|
|
224
264
|
}
|
|
225
265
|
}
|
|
226
266
|
|
|
267
|
+
/** Generate grounded follow-ups through ChatFast, failing open to an empty list. */
|
|
268
|
+
export async function suggestFollowUps(
|
|
269
|
+
config: MastraPluginConfig,
|
|
270
|
+
options: SuggestFollowUpsOptions,
|
|
271
|
+
): Promise<string[]> {
|
|
272
|
+
const settings = resolveFollowUpSuggestionsConfig(config.followUpSuggestions);
|
|
273
|
+
if (!settings || settings.mode === "genie") return [];
|
|
274
|
+
const context = {
|
|
275
|
+
"User message": boundedText(options.userText, MAX_USER_TEXT_CHARS),
|
|
276
|
+
"Assistant answer": boundedText(options.answer, MAX_ANSWER_CHARS),
|
|
277
|
+
...(options.turnContext ? { "Turn tool data": options.turnContext } : {}),
|
|
278
|
+
};
|
|
279
|
+
const questions = await suggestQuestions(config, context, {
|
|
280
|
+
maxSuggestions: settings.limit,
|
|
281
|
+
maxTokens: MAX_OUTPUT_TOKENS,
|
|
282
|
+
existingSuggestions: [options.userText, ...(options.nativeQuestions ?? [])],
|
|
283
|
+
instructions: FOLLOW_UP_INSTRUCTIONS,
|
|
284
|
+
requestContext: options.requestContext,
|
|
285
|
+
abortSignal: options.abortSignal,
|
|
286
|
+
timeoutMs: settings.timeoutMs,
|
|
287
|
+
});
|
|
288
|
+
return acceptedQuestions(
|
|
289
|
+
questions,
|
|
290
|
+
[options.userText, ...(options.nativeQuestions ?? [])],
|
|
291
|
+
settings.limit,
|
|
292
|
+
);
|
|
293
|
+
}
|
|
294
|
+
|
|
227
295
|
function lastAssistantResponse(messageList: MessageList) {
|
|
228
296
|
const messages = messageList.get.response.db();
|
|
229
297
|
for (let index = messages.length - 1; index >= 0; index -= 1) {
|
|
@@ -306,7 +374,11 @@ export function buildFollowUpSuggestionsProcessor(
|
|
|
306
374
|
...(abortSignal ? { abortSignal } : {}),
|
|
307
375
|
};
|
|
308
376
|
const suggested = await suggester(config, suggestionOptions);
|
|
309
|
-
const questions = acceptedQuestions(
|
|
377
|
+
const questions = acceptedQuestions(
|
|
378
|
+
suggested,
|
|
379
|
+
[suggestionOptions.userText, ...(suggestionOptions.nativeQuestions ?? [])],
|
|
380
|
+
settings.limit,
|
|
381
|
+
);
|
|
310
382
|
if (questions.length === 0) return messageList;
|
|
311
383
|
const data = FollowUpSuggestionsPartDataSchema.parse({
|
|
312
384
|
questions,
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/** Caller-facing Mastra model routing and token-usage events. */
|
|
2
|
+
|
|
3
|
+
import { estimateModelTokenPrice } from "@dbx-tools/model";
|
|
4
|
+
import {
|
|
5
|
+
MODEL_ROUTE_PART_TYPE,
|
|
6
|
+
MODEL_USAGE_PART_TYPE,
|
|
7
|
+
ModelRoutePartDataSchema,
|
|
8
|
+
ModelUsagePartDataSchema,
|
|
9
|
+
type ModelRoutePartData,
|
|
10
|
+
type ModelUsagePartData,
|
|
11
|
+
} from "@dbx-tools/shared-mastra";
|
|
12
|
+
import type { RequestContext } from "@mastra/core/request-context";
|
|
13
|
+
|
|
14
|
+
import { MASTRA_MODEL_ROUTE_KEY, type MastraModelRoute } from "./serving.ts";
|
|
15
|
+
|
|
16
|
+
/** Minimal Mastra usage shape required to publish one cumulative snapshot. */
|
|
17
|
+
export interface MastraTurnUsage {
|
|
18
|
+
inputTokens?: number;
|
|
19
|
+
outputTokens?: number;
|
|
20
|
+
totalTokens?: number;
|
|
21
|
+
reasoningTokens?: number;
|
|
22
|
+
cachedInputTokens?: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Minimal custom writer supplied to Mastra's native `onFinish` callback. */
|
|
26
|
+
export interface MastraFinishWriter {
|
|
27
|
+
custom(data: {
|
|
28
|
+
type: `data-${string}`;
|
|
29
|
+
data: unknown;
|
|
30
|
+
transient?: boolean;
|
|
31
|
+
}): void | Promise<void>;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Build the validated caller-facing route event once model selection resolves. */
|
|
35
|
+
export function modelRoutePartData(
|
|
36
|
+
requestContext: Pick<RequestContext, "get">,
|
|
37
|
+
runId?: string,
|
|
38
|
+
): ModelRoutePartData | undefined {
|
|
39
|
+
const route = requestContext.get(MASTRA_MODEL_ROUTE_KEY) as MastraModelRoute | undefined;
|
|
40
|
+
if (!route) return undefined;
|
|
41
|
+
const pricing = estimateModelTokenPrice(route.model, {})?.rate;
|
|
42
|
+
return ModelRoutePartDataSchema.parse({
|
|
43
|
+
...(runId ? { runId } : {}),
|
|
44
|
+
model: route.model,
|
|
45
|
+
selection: route.selection,
|
|
46
|
+
...(route.smartRouter ? { smartRouter: route.smartRouter } : {}),
|
|
47
|
+
...(route.aiDecide ? { aiDecide: route.aiDecide } : {}),
|
|
48
|
+
...(pricing ? { pricing } : {}),
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Build a validated caller-facing cumulative usage snapshot. */
|
|
53
|
+
export function modelUsagePartData(
|
|
54
|
+
requestContext: Pick<RequestContext, "get">,
|
|
55
|
+
usage: MastraTurnUsage,
|
|
56
|
+
runId?: string,
|
|
57
|
+
): ModelUsagePartData | undefined {
|
|
58
|
+
const route = requestContext.get(MASTRA_MODEL_ROUTE_KEY) as MastraModelRoute | undefined;
|
|
59
|
+
if (!route) return undefined;
|
|
60
|
+
const inputTokens = tokenCount(usage.inputTokens);
|
|
61
|
+
const outputTokens = tokenCount(usage.outputTokens);
|
|
62
|
+
const totalTokens = tokenCount(usage.totalTokens ?? inputTokens + outputTokens);
|
|
63
|
+
const cachedInputTokens = tokenCount(usage.cachedInputTokens);
|
|
64
|
+
const price = estimateModelTokenPrice(route.model, {
|
|
65
|
+
inputTokens: Math.max(0, inputTokens - cachedInputTokens),
|
|
66
|
+
outputTokens,
|
|
67
|
+
cacheReadTokens: cachedInputTokens,
|
|
68
|
+
});
|
|
69
|
+
return ModelUsagePartDataSchema.parse({
|
|
70
|
+
...modelRoutePartData(requestContext, runId),
|
|
71
|
+
usage: {
|
|
72
|
+
inputTokens,
|
|
73
|
+
outputTokens,
|
|
74
|
+
totalTokens,
|
|
75
|
+
...(usage.reasoningTokens !== undefined
|
|
76
|
+
? { reasoningTokens: tokenCount(usage.reasoningTokens) }
|
|
77
|
+
: {}),
|
|
78
|
+
...(usage.cachedInputTokens !== undefined ? { cachedInputTokens } : {}),
|
|
79
|
+
},
|
|
80
|
+
...(price ? { estimatedCostDbu: price.price.dbu } : {}),
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Emit the selected route through a Mastra custom writer. */
|
|
85
|
+
export async function emitModelRoutePart(
|
|
86
|
+
requestContext: Pick<RequestContext, "get">,
|
|
87
|
+
writer: MastraFinishWriter | undefined,
|
|
88
|
+
runId?: string,
|
|
89
|
+
): Promise<void> {
|
|
90
|
+
const data = modelRoutePartData(requestContext, runId);
|
|
91
|
+
if (!data || !writer) return;
|
|
92
|
+
await writer.custom({ type: MODEL_ROUTE_PART_TYPE, data });
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Emit a persistent AI SDK data part through Mastra's native finish writer. */
|
|
96
|
+
export async function emitModelUsagePart(
|
|
97
|
+
requestContext: Pick<RequestContext, "get">,
|
|
98
|
+
usage: MastraTurnUsage,
|
|
99
|
+
writer: MastraFinishWriter | undefined,
|
|
100
|
+
runId?: string,
|
|
101
|
+
): Promise<void> {
|
|
102
|
+
const data = modelUsagePartData(requestContext, usage, runId);
|
|
103
|
+
if (!data || !writer) return;
|
|
104
|
+
await writer.custom({ type: MODEL_USAGE_PART_TYPE, data });
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function tokenCount(value: number | undefined): number {
|
|
108
|
+
return Number.isFinite(value) ? Math.max(0, Math.round(value!)) : 0;
|
|
109
|
+
}
|
package/src/model.ts
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* This module only adds the Mastra-specific glue. The actual model
|
|
11
11
|
* selection - listing the workspace catalogue and resolving an
|
|
12
12
|
* explicit name / class / fallback chain to a real endpoint id - lives
|
|
13
|
-
* in `@dbx-tools/model` ({@link
|
|
13
|
+
* in `@dbx-tools/model` ({@link selection.resolveModel}) so non-Mastra consumers
|
|
14
14
|
* (e.g. a job that just needs a model name) can reuse it. Here we
|
|
15
15
|
* assemble the explicit ask from Mastra's request context (the
|
|
16
16
|
* per-request override under {@link MASTRA_MODEL_OVERRIDE_KEY}, the
|
|
@@ -24,13 +24,16 @@
|
|
|
24
24
|
|
|
25
25
|
import { createWorkspaceClient, type WorkspaceClient } from "@databricks/appkit";
|
|
26
26
|
import { tryGetExecutionContext } from "@dbx-tools/appkit";
|
|
27
|
-
import {
|
|
27
|
+
import {
|
|
28
|
+
AiDecideRouterService,
|
|
29
|
+
SmartRouterService,
|
|
30
|
+
smartRouterCandidates,
|
|
31
|
+
} from "@dbx-tools/appkit-model-gateway";
|
|
28
32
|
import {
|
|
29
33
|
classes,
|
|
30
34
|
endpointCapabilities,
|
|
31
35
|
listServingEndpoints,
|
|
32
36
|
policy,
|
|
33
|
-
resolve,
|
|
34
37
|
selection,
|
|
35
38
|
} from "@dbx-tools/model";
|
|
36
39
|
import {
|
|
@@ -39,7 +42,13 @@ import {
|
|
|
39
42
|
} from "@dbx-tools/model-protocol";
|
|
40
43
|
import { log } from "@dbx-tools/shared-core";
|
|
41
44
|
import { contracts, type ServingEndpointSummary } from "@dbx-tools/shared-model";
|
|
42
|
-
import {
|
|
45
|
+
import {
|
|
46
|
+
AI_DECIDE_MODEL_ID,
|
|
47
|
+
SMART_ROUTER_MODEL_ID,
|
|
48
|
+
SmartRouterObservationSchema,
|
|
49
|
+
isAiDecideModelId,
|
|
50
|
+
isSmartRouterModelId,
|
|
51
|
+
} from "@dbx-tools/shared-model-gateway";
|
|
43
52
|
import { RequestContext } from "@mastra/core/request-context";
|
|
44
53
|
|
|
45
54
|
import {
|
|
@@ -49,9 +58,12 @@ import {
|
|
|
49
58
|
type User,
|
|
50
59
|
} from "./config.ts";
|
|
51
60
|
import {
|
|
61
|
+
MASTRA_AI_DECIDE_STRATEGY_KEY,
|
|
52
62
|
MASTRA_MODEL_OVERRIDE_KEY,
|
|
63
|
+
MASTRA_MODEL_ROUTE_KEY,
|
|
53
64
|
MASTRA_RESOLVED_MODEL_KEY,
|
|
54
65
|
MASTRA_SMART_ROUTER_INPUT_KEY,
|
|
66
|
+
type MastraModelRoute,
|
|
55
67
|
resolveServingConfig,
|
|
56
68
|
} from "./serving.ts";
|
|
57
69
|
import { recordActiveTraceAuth, recordActiveTraceModel } from "./telemetry.ts";
|
|
@@ -59,13 +71,15 @@ import { recordActiveTraceAuth, recordActiveTraceModel } from "./telemetry.ts";
|
|
|
59
71
|
type ModelClass = contracts.ModelClass;
|
|
60
72
|
type ModelOptimization = contracts.ModelOptimization;
|
|
61
73
|
const { parseModelClass } = classes;
|
|
62
|
-
const { selectModel } = resolve;
|
|
63
74
|
const smartRouter = new SmartRouterService();
|
|
75
|
+
const aiDecideRouter = new AiDecideRouterService();
|
|
64
76
|
const MASTRA_SMART_ROUTER_RESOLUTION_KEY = "mastra__smart_router_resolution";
|
|
77
|
+
const MASTRA_AI_DECIDE_RESOLUTION_KEY = "mastra__ai_decide_resolution";
|
|
65
78
|
|
|
66
79
|
interface SmartRouterModelResolution {
|
|
67
80
|
readonly modelId: string;
|
|
68
|
-
readonly source:
|
|
81
|
+
readonly source: MastraModelRoute["selection"];
|
|
82
|
+
readonly route?: MastraModelRoute;
|
|
69
83
|
}
|
|
70
84
|
|
|
71
85
|
/** Virtual endpoint shown when Smart Router is enabled for the active identity. */
|
|
@@ -75,6 +89,17 @@ export const SMART_ROUTER_ENDPOINT: ServingEndpointSummary = {
|
|
|
75
89
|
description: "Selects a Databricks system model with the task_v3 Smart Router.",
|
|
76
90
|
task: "llm/v1/chat",
|
|
77
91
|
status: { deprecated: false },
|
|
92
|
+
modelSource: "foundation",
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
/** Virtual endpoint shown when Databricks ai_decide is enabled for the active identity. */
|
|
96
|
+
export const AI_DECIDE_ENDPOINT: ServingEndpointSummary = {
|
|
97
|
+
name: AI_DECIDE_MODEL_ID,
|
|
98
|
+
displayName: "AI Decide",
|
|
99
|
+
description: "Selects a sufficient foundation model using Databricks ai_decide.",
|
|
100
|
+
task: "llm/v1/chat",
|
|
101
|
+
status: { deprecated: false },
|
|
102
|
+
modelSource: "foundation",
|
|
78
103
|
};
|
|
79
104
|
|
|
80
105
|
/** Default Responses options: stateless replay plus visible reasoning summaries. */
|
|
@@ -248,12 +273,56 @@ export async function buildModel(
|
|
|
248
273
|
(!internalClassOnly ? process.env.DATABRICKS_SERVING_ENDPOINT_NAME : undefined);
|
|
249
274
|
const smartRouterInput = overrides.smartRouterInput ?? context.get(MASTRA_SMART_ROUTER_INPUT_KEY);
|
|
250
275
|
const smartRouterRequested = isSmartRouterModelId(requested);
|
|
251
|
-
|
|
252
|
-
|
|
276
|
+
const aiDecideRequested = isAiDecideModelId(requested);
|
|
277
|
+
if ((smartRouterRequested || aiDecideRequested) && smartRouterInput === undefined) {
|
|
278
|
+
throw new Error(
|
|
279
|
+
`${smartRouterRequested ? "Smart Router" : "ai_decide"} requires task input before selecting a concrete model`,
|
|
280
|
+
);
|
|
253
281
|
}
|
|
254
282
|
let modelId: string;
|
|
255
|
-
let source:
|
|
256
|
-
|
|
283
|
+
let source: MastraModelRoute["selection"];
|
|
284
|
+
let route: MastraModelRoute | undefined;
|
|
285
|
+
let endpointsPromise: Promise<ServingEndpointSummary[]> | undefined;
|
|
286
|
+
const loadEndpoints = () => {
|
|
287
|
+
endpointsPromise ??= listServingEndpoints(client, host, {
|
|
288
|
+
ttlMs: serving.ttlMs,
|
|
289
|
+
...(cacheIdentity !== undefined ? { cacheIdentity } : {}),
|
|
290
|
+
measureEmbeddingDimensions: overrides.modelClass === contracts.ModelClass.Embedding,
|
|
291
|
+
});
|
|
292
|
+
return endpointsPromise;
|
|
293
|
+
};
|
|
294
|
+
if (aiDecideRequested && overrides.modelClass === undefined && smartRouterInput !== undefined) {
|
|
295
|
+
let pending = context.get(MASTRA_AI_DECIDE_RESOLUTION_KEY) as
|
|
296
|
+
Promise<SmartRouterModelResolution> | undefined;
|
|
297
|
+
if (!pending) {
|
|
298
|
+
pending = (async () => {
|
|
299
|
+
const selected = await aiDecideRouter.select(
|
|
300
|
+
smartRouterInput,
|
|
301
|
+
await loadEndpoints(),
|
|
302
|
+
client,
|
|
303
|
+
{ strategy: context.get(MASTRA_AI_DECIDE_STRATEGY_KEY) },
|
|
304
|
+
);
|
|
305
|
+
return {
|
|
306
|
+
modelId: selected.endpoint.name,
|
|
307
|
+
source: "ai-decide" as const,
|
|
308
|
+
route: {
|
|
309
|
+
model: selected.endpoint.name,
|
|
310
|
+
selection: "ai-decide" as const,
|
|
311
|
+
aiDecide: selected.observation,
|
|
312
|
+
},
|
|
313
|
+
};
|
|
314
|
+
})();
|
|
315
|
+
context.set(MASTRA_AI_DECIDE_RESOLUTION_KEY, pending);
|
|
316
|
+
}
|
|
317
|
+
try {
|
|
318
|
+
({ modelId, source, route } = await pending);
|
|
319
|
+
} catch (error) {
|
|
320
|
+
if (context.get(MASTRA_AI_DECIDE_RESOLUTION_KEY) === pending) {
|
|
321
|
+
context.set(MASTRA_AI_DECIDE_RESOLUTION_KEY, undefined);
|
|
322
|
+
}
|
|
323
|
+
throw error;
|
|
324
|
+
}
|
|
325
|
+
} else if (
|
|
257
326
|
(requested === undefined || smartRouterRequested) &&
|
|
258
327
|
overrides.modelClass === undefined &&
|
|
259
328
|
smartRouterInput !== undefined
|
|
@@ -262,10 +331,7 @@ export async function buildModel(
|
|
|
262
331
|
Promise<SmartRouterModelResolution> | undefined;
|
|
263
332
|
if (!pending) {
|
|
264
333
|
pending = (async () => {
|
|
265
|
-
const endpoints = await
|
|
266
|
-
ttlMs: serving.ttlMs,
|
|
267
|
-
...(cacheIdentity !== undefined ? { cacheIdentity } : {}),
|
|
268
|
-
});
|
|
334
|
+
const endpoints = await loadEndpoints();
|
|
269
335
|
const candidates = smartRouterCandidates(
|
|
270
336
|
endpoints.filter((endpoint) => endpointCapabilities(endpoint).tools),
|
|
271
337
|
"codex",
|
|
@@ -276,11 +342,28 @@ export async function buildModel(
|
|
|
276
342
|
const enabled = smartRouterRequested || (await smartRouter.isEnabled(candidates, client));
|
|
277
343
|
if (enabled) {
|
|
278
344
|
const selected = await smartRouter.select(smartRouterInput, candidates, { client });
|
|
279
|
-
|
|
345
|
+
const observation = SmartRouterObservationSchema.parse({
|
|
346
|
+
selectedModel: selected.value.name,
|
|
347
|
+
routeModel: selected.routeModel,
|
|
348
|
+
harness: selected.harness,
|
|
349
|
+
routerName: selected.routerName,
|
|
350
|
+
source: "router",
|
|
351
|
+
latencyMs: selected.latencyMs,
|
|
352
|
+
...(selected.rationale ? { rationale: selected.rationale } : {}),
|
|
353
|
+
...(selected.params ? { params: selected.params } : {}),
|
|
354
|
+
});
|
|
355
|
+
return {
|
|
356
|
+
modelId: selected.value.name,
|
|
357
|
+
source: SMART_ROUTER_MODEL_ID,
|
|
358
|
+
route: {
|
|
359
|
+
model: selected.value.name,
|
|
360
|
+
selection: "smart-router",
|
|
361
|
+
smartRouter: observation,
|
|
362
|
+
},
|
|
363
|
+
};
|
|
280
364
|
}
|
|
281
|
-
const fallback =
|
|
282
|
-
|
|
283
|
-
host,
|
|
365
|
+
const fallback = selection.resolveModel(
|
|
366
|
+
endpoints,
|
|
284
367
|
selectionInput(
|
|
285
368
|
config,
|
|
286
369
|
requested,
|
|
@@ -295,7 +378,7 @@ export async function buildModel(
|
|
|
295
378
|
context.set(MASTRA_SMART_ROUTER_RESOLUTION_KEY, pending);
|
|
296
379
|
}
|
|
297
380
|
try {
|
|
298
|
-
({ modelId, source } = await pending);
|
|
381
|
+
({ modelId, source, route } = await pending);
|
|
299
382
|
} catch (error) {
|
|
300
383
|
if (context.get(MASTRA_SMART_ROUTER_RESOLUTION_KEY) === pending) {
|
|
301
384
|
context.set(MASTRA_SMART_ROUTER_RESOLUTION_KEY, undefined);
|
|
@@ -303,19 +386,33 @@ export async function buildModel(
|
|
|
303
386
|
throw error;
|
|
304
387
|
}
|
|
305
388
|
} else {
|
|
306
|
-
const
|
|
307
|
-
|
|
308
|
-
|
|
389
|
+
const endpoints = await loadEndpoints();
|
|
390
|
+
const resolved = selection.resolveModel(
|
|
391
|
+
endpoints,
|
|
309
392
|
selectionInput(config, requested, overrides.modelClass, cacheIdentity, serving, overrides),
|
|
310
393
|
);
|
|
311
394
|
modelId = resolved.modelId;
|
|
312
395
|
source = resolved.source;
|
|
396
|
+
route = {
|
|
397
|
+
model: modelId,
|
|
398
|
+
selection: source,
|
|
399
|
+
};
|
|
313
400
|
}
|
|
314
|
-
if (isSmartRouterModelId(modelId)) {
|
|
315
|
-
throw new Error("
|
|
401
|
+
if (isSmartRouterModelId(modelId) || isAiDecideModelId(modelId)) {
|
|
402
|
+
throw new Error("Virtual model routing did not resolve to a concrete serving endpoint");
|
|
316
403
|
}
|
|
317
404
|
logger.debug("model selected", { modelId, source, requested });
|
|
405
|
+
if (!route) {
|
|
406
|
+
if (source === SMART_ROUTER_MODEL_ID || source === AI_DECIDE_MODEL_ID) {
|
|
407
|
+
throw new Error("Virtual model selection metadata is unavailable");
|
|
408
|
+
}
|
|
409
|
+
route = {
|
|
410
|
+
model: modelId,
|
|
411
|
+
selection: source,
|
|
412
|
+
};
|
|
413
|
+
}
|
|
318
414
|
context.set(MASTRA_RESOLVED_MODEL_KEY, modelId);
|
|
415
|
+
context.set(MASTRA_MODEL_ROUTE_KEY, route);
|
|
319
416
|
recordActiveTraceModel(modelId);
|
|
320
417
|
|
|
321
418
|
return createDatabricksLanguageModel({
|
|
@@ -327,8 +424,8 @@ export async function buildModel(
|
|
|
327
424
|
});
|
|
328
425
|
}
|
|
329
426
|
|
|
330
|
-
/** Append
|
|
331
|
-
export async function
|
|
427
|
+
/** Append virtual model routers only after their memoized live probes succeed. */
|
|
428
|
+
export async function withModelRouterEndpoints(
|
|
332
429
|
endpoints: readonly ServingEndpointSummary[],
|
|
333
430
|
client: WorkspaceClient,
|
|
334
431
|
): Promise<ServingEndpointSummary[]> {
|
|
@@ -336,7 +433,13 @@ export async function withSmartRouterEndpoint(
|
|
|
336
433
|
endpoints.filter((endpoint) => endpointCapabilities(endpoint).tools),
|
|
337
434
|
"codex",
|
|
338
435
|
);
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
436
|
+
const [smartRouterEnabled, aiDecideEnabled] = await Promise.all([
|
|
437
|
+
smartRouter.isEnabled(candidates, client),
|
|
438
|
+
aiDecideRouter.isEnabled(endpoints, client),
|
|
439
|
+
]);
|
|
440
|
+
return [
|
|
441
|
+
...(smartRouterEnabled ? [SMART_ROUTER_ENDPOINT] : []),
|
|
442
|
+
...(aiDecideEnabled ? [AI_DECIDE_ENDPOINT] : []),
|
|
443
|
+
...endpoints,
|
|
444
|
+
];
|
|
342
445
|
}
|