@dbx-tools/appkit-mastra 0.9.175 → 0.9.177
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -7
- package/index.ts +4 -4
- package/lib/index.d.ts +4 -4
- package/lib/index.js +5 -5
- package/lib/src/agents.js +12 -2
- package/lib/src/chat.d.ts +7 -0
- package/lib/src/chat.js +163 -37
- package/lib/src/config.d.ts +5 -2
- package/lib/src/config.js +6 -2
- package/lib/src/follow-up-suggestions.js +11 -24
- package/lib/src/memory.js +5 -7
- package/lib/src/model.d.ts +12 -6
- package/lib/src/model.js +88 -13
- package/lib/src/plugin.js +4 -3
- package/lib/src/server.d.ts +2 -0
- package/lib/src/server.js +9 -2
- package/lib/src/serving.d.ts +2 -0
- package/lib/src/serving.js +3 -1
- package/lib/src/summarize.d.ts +6 -14
- package/lib/src/summarize.js +7 -15
- package/package.json +18 -16
- package/src/agents.ts +11 -2
- package/src/chat.ts +182 -37
- package/src/config.ts +11 -3
- package/src/follow-up-suggestions.ts +10 -29
- package/src/memory.ts +4 -6
- package/src/model.ts +140 -23
- package/src/plugin.ts +3 -2
- package/src/server.ts +11 -0
- package/src/serving.ts +3 -0
- package/src/summarize.ts +6 -14
package/src/chat.ts
CHANGED
|
@@ -4,16 +4,20 @@
|
|
|
4
4
|
* @module
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import { log, toError, errorMessage, isRecord } from "@dbx-tools/shared-core";
|
|
7
|
+
import { log, toError, errorMessage, isRecord, sleep } from "@dbx-tools/shared-core";
|
|
8
8
|
import {
|
|
9
9
|
MASTRA_ROUTES,
|
|
10
|
+
RESPONSE_COMPLETE_PART_TYPE,
|
|
10
11
|
isStaleMastraResumeError,
|
|
11
12
|
STALE_MASTRA_RESUME_STREAM_TEXT,
|
|
12
13
|
} from "@dbx-tools/shared-mastra";
|
|
13
14
|
import { handleChatStream, toAISdkStream } from "@mastra/ai-sdk";
|
|
14
15
|
import { isEventedAgent } from "@mastra/core/agent/durable";
|
|
16
|
+
import type { EventedAgent } from "@mastra/core/agent/durable";
|
|
15
17
|
import { registerApiRoute } from "@mastra/core/server";
|
|
18
|
+
import type { DataChunkType, MastraStreamTransform } from "@mastra/core/stream";
|
|
16
19
|
import { createUIMessageStreamResponse } from "ai";
|
|
20
|
+
import type { UIMessageChunk } from "ai";
|
|
17
21
|
|
|
18
22
|
import type { MastraPluginConfig } from "./config.ts";
|
|
19
23
|
import {
|
|
@@ -28,6 +32,142 @@ import { activeChatTracingOptions } from "./telemetry.ts";
|
|
|
28
32
|
|
|
29
33
|
const logger = log.logger("mastra/chat");
|
|
30
34
|
|
|
35
|
+
/** Keep a quiet chat stream below the Databricks Apps ingress idle window. */
|
|
36
|
+
export const CHAT_STREAM_KEEPALIVE_MS = 10_000;
|
|
37
|
+
|
|
38
|
+
/** Bound reconnects to producerless durable-run topics before ingress times out. */
|
|
39
|
+
export const CHAT_RUN_OBSERVE_IDLE_TIMEOUT_MS = 15_000;
|
|
40
|
+
|
|
41
|
+
/** Maximum age for a persisted `running` snapshot without a local producer. */
|
|
42
|
+
const CHAT_REMOTE_RUN_FRESHNESS_MS = 60_000;
|
|
43
|
+
|
|
44
|
+
const chatStreamHeartbeat = (): UIMessageChunk => ({
|
|
45
|
+
type: "data-stream-heartbeat",
|
|
46
|
+
data: null,
|
|
47
|
+
transient: true,
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
/** Mark the answer complete while post-turn processors continue before finish. */
|
|
51
|
+
export function buildResponseCompleteTransform(): MastraStreamTransform {
|
|
52
|
+
return () => {
|
|
53
|
+
let emitted = false;
|
|
54
|
+
return new TransformStream({
|
|
55
|
+
transform(chunk, controller) {
|
|
56
|
+
controller.enqueue(chunk);
|
|
57
|
+
if (
|
|
58
|
+
!emitted &&
|
|
59
|
+
chunk.type === "step-finish" &&
|
|
60
|
+
chunk.payload.stepResult.isContinued !== true &&
|
|
61
|
+
(chunk.payload.output.toolCalls?.length ?? 0) === 0
|
|
62
|
+
) {
|
|
63
|
+
emitted = true;
|
|
64
|
+
const part: DataChunkType = {
|
|
65
|
+
type: RESPONSE_COMPLETE_PART_TYPE,
|
|
66
|
+
data: null,
|
|
67
|
+
transient: true,
|
|
68
|
+
};
|
|
69
|
+
controller.enqueue(part as never);
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
});
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function chatStreamTransforms(
|
|
77
|
+
runId: string,
|
|
78
|
+
generatedFollowUps: boolean,
|
|
79
|
+
): readonly MastraStreamTransform[] {
|
|
80
|
+
return [
|
|
81
|
+
buildResponseCompleteTransform(),
|
|
82
|
+
...(generatedFollowUps ? [buildFollowUpSuggestionsTransform(runId)] : []),
|
|
83
|
+
];
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function deferredChatStream(open: () => Promise<ReadableStream<UIMessageChunk>>): Response {
|
|
87
|
+
let reader: ReadableStreamDefaultReader<UIMessageChunk> | undefined;
|
|
88
|
+
let timer: ReturnType<typeof setInterval> | undefined;
|
|
89
|
+
let settled = false;
|
|
90
|
+
|
|
91
|
+
const stream = new ReadableStream<UIMessageChunk>({
|
|
92
|
+
start(controller) {
|
|
93
|
+
const writeHeartbeat = () => {
|
|
94
|
+
if (!settled) controller.enqueue(chatStreamHeartbeat());
|
|
95
|
+
};
|
|
96
|
+
writeHeartbeat();
|
|
97
|
+
timer = setInterval(writeHeartbeat, CHAT_STREAM_KEEPALIVE_MS);
|
|
98
|
+
timer.unref?.();
|
|
99
|
+
|
|
100
|
+
void (async () => {
|
|
101
|
+
try {
|
|
102
|
+
const opened = await open();
|
|
103
|
+
if (settled) {
|
|
104
|
+
await opened.cancel();
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
reader = opened.getReader();
|
|
108
|
+
while (true) {
|
|
109
|
+
const { done, value } = await reader.read();
|
|
110
|
+
if (done) break;
|
|
111
|
+
controller.enqueue(value);
|
|
112
|
+
}
|
|
113
|
+
} catch (error) {
|
|
114
|
+
if (!settled) {
|
|
115
|
+
controller.enqueue({ type: "error", errorText: errorMessage(error) });
|
|
116
|
+
}
|
|
117
|
+
} finally {
|
|
118
|
+
const shouldClose = !settled;
|
|
119
|
+
settled = true;
|
|
120
|
+
if (timer) clearInterval(timer);
|
|
121
|
+
reader?.releaseLock();
|
|
122
|
+
if (shouldClose) controller.close();
|
|
123
|
+
}
|
|
124
|
+
})();
|
|
125
|
+
},
|
|
126
|
+
async cancel(reason) {
|
|
127
|
+
settled = true;
|
|
128
|
+
if (timer) clearInterval(timer);
|
|
129
|
+
await reader?.cancel(reason);
|
|
130
|
+
},
|
|
131
|
+
});
|
|
132
|
+
return createUIMessageStreamResponse({
|
|
133
|
+
stream,
|
|
134
|
+
keepAliveMs: CHAT_STREAM_KEEPALIVE_MS,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
async function backgroundRunAvailable(agent: EventedAgent, runId: string): Promise<boolean> {
|
|
139
|
+
if (agent.runRegistry.has(runId)) return true;
|
|
140
|
+
const active = await agent.listActiveRuns();
|
|
141
|
+
const run = active.runs.find((candidate) => candidate.runId === runId);
|
|
142
|
+
if (!run) return false;
|
|
143
|
+
const updatedAt =
|
|
144
|
+
run.updatedAt instanceof Date ? run.updatedAt.getTime() : Date.parse(String(run.updatedAt));
|
|
145
|
+
return Number.isFinite(updatedAt) && Date.now() - updatedAt <= CHAT_REMOTE_RUN_FRESHNESS_MS;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async function observeBackgroundRun(agent: EventedAgent, runId: string) {
|
|
149
|
+
const available = await Promise.race([
|
|
150
|
+
backgroundRunAvailable(agent, runId),
|
|
151
|
+
sleep(CHAT_RUN_OBSERVE_IDLE_TIMEOUT_MS).then(() => false),
|
|
152
|
+
]);
|
|
153
|
+
if (!available) throw new Error(`Run ${runId} is no longer available`);
|
|
154
|
+
|
|
155
|
+
const pending = agent.observe(runId, {
|
|
156
|
+
idleTimeoutMs: CHAT_RUN_OBSERVE_IDLE_TIMEOUT_MS,
|
|
157
|
+
isAlive: () =>
|
|
158
|
+
Promise.race([
|
|
159
|
+
backgroundRunAvailable(agent, runId),
|
|
160
|
+
sleep(CHAT_RUN_OBSERVE_IDLE_TIMEOUT_MS).then(() => false),
|
|
161
|
+
]),
|
|
162
|
+
});
|
|
163
|
+
return Promise.race([
|
|
164
|
+
pending,
|
|
165
|
+
sleep(CHAT_RUN_OBSERVE_IDLE_TIMEOUT_MS).then(() => {
|
|
166
|
+
throw new Error(`Run ${runId} is no longer available`);
|
|
167
|
+
}),
|
|
168
|
+
]);
|
|
169
|
+
}
|
|
170
|
+
|
|
31
171
|
/** Client request cancellation is detached only for evented background agents. */
|
|
32
172
|
export function agentAbortSignal(agent: unknown, signal: AbortSignal): AbortSignal | undefined {
|
|
33
173
|
return isEventedAgent(agent) ? undefined : signal;
|
|
@@ -56,6 +196,7 @@ function latestUserText(value: unknown): string | undefined {
|
|
|
56
196
|
/** Register detached chat, durable observation, and explicit abort routes. */
|
|
57
197
|
export function agentChatRoutes(config: MastraPluginConfig = {}) {
|
|
58
198
|
const followUps = resolveFollowUpSuggestionsConfig(config.followUpSuggestions);
|
|
199
|
+
const sendReasoning = config.sendReasoning !== false;
|
|
59
200
|
const chatPath = `${MASTRA_ROUTES.chat}/:agentId` as `${string}:agentId`;
|
|
60
201
|
const runPath =
|
|
61
202
|
`${MASTRA_ROUTES.chat}/:agentId${MASTRA_ROUTES.runs}/:runId` as `${string}:agentId${string}:runId`;
|
|
@@ -93,33 +234,35 @@ export function agentChatRoutes(config: MastraPluginConfig = {}) {
|
|
|
93
234
|
requestContext.setRaw(FOLLOW_UP_NATIVE_QUESTIONS_KEY, []);
|
|
94
235
|
requestContext.setRaw(FOLLOW_UP_RUN_ID_KEY, runId);
|
|
95
236
|
const tracingOptions = activeChatTracingOptions(undefined);
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
237
|
+
return deferredChatStream(async () => {
|
|
238
|
+
return handleChatStream({
|
|
239
|
+
mastra,
|
|
240
|
+
agentId,
|
|
241
|
+
...(versionId
|
|
242
|
+
? { agentVersion: { versionId } }
|
|
243
|
+
: status
|
|
244
|
+
? { agentVersion: { status } }
|
|
245
|
+
: {}),
|
|
246
|
+
params: withRequestMemoryScope(
|
|
247
|
+
{
|
|
248
|
+
...params,
|
|
249
|
+
runId,
|
|
250
|
+
requestContext,
|
|
251
|
+
...(tracingOptions ? { tracingOptions } : {}),
|
|
252
|
+
...(abortSignal ? { abortSignal } : {}),
|
|
253
|
+
},
|
|
108
254
|
requestContext,
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
:
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
sendSources: true,
|
|
120
|
-
onError: serializeChatStreamError,
|
|
255
|
+
),
|
|
256
|
+
experimentalTransform: chatStreamTransforms(
|
|
257
|
+
runId,
|
|
258
|
+
Boolean(followUps && followUps.mode !== "genie"),
|
|
259
|
+
),
|
|
260
|
+
version: "v7",
|
|
261
|
+
sendReasoning,
|
|
262
|
+
sendSources: true,
|
|
263
|
+
onError: serializeChatStreamError,
|
|
264
|
+
});
|
|
121
265
|
});
|
|
122
|
-
return createUIMessageStreamResponse({ stream });
|
|
123
266
|
},
|
|
124
267
|
}),
|
|
125
268
|
registerApiRoute(runPath, {
|
|
@@ -136,18 +279,20 @@ export function agentChatRoutes(config: MastraPluginConfig = {}) {
|
|
|
136
279
|
{ status: 409 },
|
|
137
280
|
);
|
|
138
281
|
}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
282
|
+
return deferredChatStream(async () => {
|
|
283
|
+
const observed = await observeBackgroundRun(agent, runId);
|
|
284
|
+
return toAISdkStream(observed.output, {
|
|
285
|
+
from: "agent",
|
|
286
|
+
experimentalTransform: chatStreamTransforms(
|
|
287
|
+
runId,
|
|
288
|
+
Boolean(followUps && followUps.mode !== "genie"),
|
|
289
|
+
),
|
|
290
|
+
version: "v7",
|
|
291
|
+
sendReasoning,
|
|
292
|
+
sendSources: true,
|
|
293
|
+
onError: serializeChatStreamError,
|
|
294
|
+
});
|
|
149
295
|
});
|
|
150
|
-
return createUIMessageStreamResponse({ stream });
|
|
151
296
|
},
|
|
152
297
|
}),
|
|
153
298
|
registerApiRoute(abortPath, {
|
package/src/config.ts
CHANGED
|
@@ -237,6 +237,8 @@ export interface MastraPluginConfig extends BasePluginConfig {
|
|
|
237
237
|
* to `true`.
|
|
238
238
|
*/
|
|
239
239
|
backgroundTurns?: boolean;
|
|
240
|
+
/** Include model reasoning parts in chat UI streams when the provider returns them. Defaults to true. */
|
|
241
|
+
sendReasoning?: boolean;
|
|
240
242
|
/** AI SDK provider-options namespace. Defaults to `"openai"`; no env fallback. */
|
|
241
243
|
providerId?: string;
|
|
242
244
|
/**
|
|
@@ -273,8 +275,9 @@ export interface MastraPluginConfig extends BasePluginConfig {
|
|
|
273
275
|
/** Native Mastra workspace tool availability, approval, and hook configuration. */
|
|
274
276
|
workspaceTools?: WorkspaceToolsConfig;
|
|
275
277
|
/**
|
|
276
|
-
* Use Mastra's native workspace skill manager. Defaults to true. Pass
|
|
277
|
-
*
|
|
278
|
+
* Use Mastra's native workspace-file skill manager. Defaults to true. Pass
|
|
279
|
+
* false to disable workspace-file discovery on auto-created workspaces, or
|
|
280
|
+
* an object to tune its process-cached, asynchronously refreshed index.
|
|
278
281
|
*/
|
|
279
282
|
workspaceSkills?: boolean | MastraWorkspaceSkillsConfig;
|
|
280
283
|
/**
|
|
@@ -703,6 +706,11 @@ export const MASTRA_CONFIG_SCHEMA: ConfigSchema = {
|
|
|
703
706
|
description:
|
|
704
707
|
"Run turns through Mastra evented durable execution so they continue after browser disconnects and remain reconnectable by run id. Defaults to true.",
|
|
705
708
|
},
|
|
709
|
+
sendReasoning: {
|
|
710
|
+
type: "boolean",
|
|
711
|
+
description:
|
|
712
|
+
"Include model reasoning parts in chat UI streams when the provider returns them. Defaults to true.",
|
|
713
|
+
},
|
|
706
714
|
providerId: {
|
|
707
715
|
type: "string",
|
|
708
716
|
description: 'AI SDK provider-options namespace. Defaults to "openai".',
|
|
@@ -741,7 +749,7 @@ export const MASTRA_CONFIG_SCHEMA: ConfigSchema = {
|
|
|
741
749
|
workspaceSkills: {
|
|
742
750
|
type: ["boolean", "object"],
|
|
743
751
|
description:
|
|
744
|
-
"Use Mastra's native
|
|
752
|
+
"Use Mastra's native process-cached workspace-file skill search with asynchronous refresh. Defaults to true; false disables workspace-file discovery on auto-created workspaces.",
|
|
745
753
|
},
|
|
746
754
|
workspaceTools: {
|
|
747
755
|
type: "object",
|
|
@@ -11,13 +11,12 @@ import {
|
|
|
11
11
|
FollowUpSuggestionsPartDataSchema,
|
|
12
12
|
} from "@dbx-tools/shared-mastra";
|
|
13
13
|
import { ModelClass } from "@dbx-tools/shared-model";
|
|
14
|
-
import { Agent } from "@mastra/core/agent";
|
|
15
14
|
import type { MessageList } from "@mastra/core/agent/message-list";
|
|
16
|
-
import type { MastraModelConfig } from "@mastra/core/llm";
|
|
17
15
|
import type { OutputProcessor } from "@mastra/core/processors";
|
|
18
16
|
import type { RequestContext } from "@mastra/core/request-context";
|
|
19
17
|
import type { DataChunkType, LLMStepResult, MastraStreamTransform } from "@mastra/core/stream";
|
|
20
18
|
import { createTool } from "@mastra/core/tools";
|
|
19
|
+
import { generateText, Output } from "ai";
|
|
21
20
|
import { z } from "zod";
|
|
22
21
|
|
|
23
22
|
import type { FollowUpSuggestionsConfig, MastraPluginConfig } from "./config.ts";
|
|
@@ -117,7 +116,6 @@ const FOLLOW_UP_INSTRUCTIONS = [
|
|
|
117
116
|
TYPOGRAPHY_RULE,
|
|
118
117
|
].join(" ");
|
|
119
118
|
|
|
120
|
-
const suggestionAgents = new WeakMap<MastraPluginConfig, Agent>();
|
|
121
119
|
const generatedRunParts = new Map<string, FollowUpSuggestionsPartData>();
|
|
122
120
|
const GENERATED_RUN_PART_TTL_MS = 5 * 60_000;
|
|
123
121
|
|
|
@@ -135,28 +133,6 @@ export function resolveFollowUpSuggestionsConfig(
|
|
|
135
133
|
};
|
|
136
134
|
}
|
|
137
135
|
|
|
138
|
-
function followUpModel(
|
|
139
|
-
config: MastraPluginConfig,
|
|
140
|
-
): (args: { requestContext: RequestContext }) => Promise<MastraModelConfig> {
|
|
141
|
-
return ({ requestContext }) =>
|
|
142
|
-
buildModel(config, requestContext, { modelClass: FOLLOW_UP_MODEL_CLASS });
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
function suggestionAgent(config: MastraPluginConfig): Agent {
|
|
146
|
-
let agent = suggestionAgents.get(config);
|
|
147
|
-
if (!agent) {
|
|
148
|
-
agent = new Agent({
|
|
149
|
-
id: "follow-up-suggestions",
|
|
150
|
-
name: "Follow-up suggestions",
|
|
151
|
-
description: "Generates grounded next questions using the fast chat model.",
|
|
152
|
-
instructions: FOLLOW_UP_INSTRUCTIONS,
|
|
153
|
-
model: followUpModel(config),
|
|
154
|
-
});
|
|
155
|
-
suggestionAgents.set(config, agent);
|
|
156
|
-
}
|
|
157
|
-
return agent;
|
|
158
|
-
}
|
|
159
|
-
|
|
160
136
|
function normalizedQuestion(value: string): string {
|
|
161
137
|
return value
|
|
162
138
|
.trim()
|
|
@@ -255,12 +231,17 @@ export async function suggestFollowUps(
|
|
|
255
231
|
"Maximum questions": String(settings.limit),
|
|
256
232
|
});
|
|
257
233
|
try {
|
|
258
|
-
const
|
|
259
|
-
|
|
260
|
-
|
|
234
|
+
const model = await buildModel(config, options.requestContext, {
|
|
235
|
+
modelClass: FOLLOW_UP_MODEL_CLASS,
|
|
236
|
+
});
|
|
237
|
+
const result = await generateText({
|
|
238
|
+
model,
|
|
239
|
+
instructions: FOLLOW_UP_INSTRUCTIONS,
|
|
240
|
+
prompt,
|
|
241
|
+
output: Output.object({ schema: FollowUpOutputSchema }),
|
|
261
242
|
abortSignal,
|
|
262
243
|
});
|
|
263
|
-
const parsed = FollowUpOutputSchema.parse(result.
|
|
244
|
+
const parsed = FollowUpOutputSchema.parse(result.output);
|
|
264
245
|
return acceptedQuestions(parsed.questions, options, settings.limit);
|
|
265
246
|
} catch (error) {
|
|
266
247
|
if (!options.abortSignal?.aborted) {
|
package/src/memory.ts
CHANGED
|
@@ -257,12 +257,10 @@ export class MemoryBuilder {
|
|
|
257
257
|
...(vector
|
|
258
258
|
? { semanticRecall: { topK: 3, messageRange: 2, scope: "resource" as const } }
|
|
259
259
|
: {}),
|
|
260
|
-
// Auto-name each thread from its opening turn
|
|
261
|
-
//
|
|
262
|
-
//
|
|
263
|
-
//
|
|
264
|
-
// model, so naming a thread never spends the heavyweight model.
|
|
265
|
-
// Only meaningful when storage is on; harmless otherwise.
|
|
260
|
+
// Auto-name each thread from its opening turn using the fast chat tier.
|
|
261
|
+
// Title generation is awaited before Mastra settles the response stream,
|
|
262
|
+
// so reusing the primary agent model can keep the UI busy long after the
|
|
263
|
+
// answer text has finished streaming.
|
|
266
264
|
...(storage
|
|
267
265
|
? {
|
|
268
266
|
generateTitle: {
|
package/src/model.ts
CHANGED
|
@@ -24,19 +24,34 @@
|
|
|
24
24
|
|
|
25
25
|
import { createWorkspaceClient, type WorkspaceClient } from "@databricks/appkit";
|
|
26
26
|
import { tryGetExecutionContext } from "@dbx-tools/appkit";
|
|
27
|
-
import {
|
|
27
|
+
import { SmartRouterService, smartRouterCandidates } from "@dbx-tools/appkit-model-gateway";
|
|
28
|
+
import {
|
|
29
|
+
classes,
|
|
30
|
+
endpointCapabilities,
|
|
31
|
+
listServingEndpoints,
|
|
32
|
+
policy,
|
|
33
|
+
resolve,
|
|
34
|
+
selection,
|
|
35
|
+
} from "@dbx-tools/model";
|
|
28
36
|
import {
|
|
29
37
|
createDatabricksLanguageModel,
|
|
30
38
|
type DatabricksLanguageModel,
|
|
31
39
|
} from "@dbx-tools/model-protocol";
|
|
32
40
|
import { log } from "@dbx-tools/shared-core";
|
|
33
41
|
import { contracts, type ServingEndpointSummary } from "@dbx-tools/shared-model";
|
|
42
|
+
import { SMART_ROUTER_MODEL_ID, isSmartRouterModelId } from "@dbx-tools/shared-model-gateway";
|
|
34
43
|
import { RequestContext } from "@mastra/core/request-context";
|
|
35
44
|
|
|
36
|
-
import {
|
|
45
|
+
import {
|
|
46
|
+
executionContextUserId,
|
|
47
|
+
MASTRA_USER_KEY,
|
|
48
|
+
type MastraPluginConfig,
|
|
49
|
+
type User,
|
|
50
|
+
} from "./config.ts";
|
|
37
51
|
import {
|
|
38
52
|
MASTRA_MODEL_OVERRIDE_KEY,
|
|
39
53
|
MASTRA_RESOLVED_MODEL_KEY,
|
|
54
|
+
MASTRA_SMART_ROUTER_INPUT_KEY,
|
|
40
55
|
resolveServingConfig,
|
|
41
56
|
} from "./serving.ts";
|
|
42
57
|
import { recordActiveTraceAuth, recordActiveTraceModel } from "./telemetry.ts";
|
|
@@ -44,6 +59,22 @@ import { recordActiveTraceAuth, recordActiveTraceModel } from "./telemetry.ts";
|
|
|
44
59
|
type ModelClass = contracts.ModelClass;
|
|
45
60
|
const { parseModelClass } = classes;
|
|
46
61
|
const { selectModel } = resolve;
|
|
62
|
+
const smartRouter = new SmartRouterService();
|
|
63
|
+
const MASTRA_SMART_ROUTER_RESOLUTION_KEY = "mastra__smart_router_resolution";
|
|
64
|
+
|
|
65
|
+
interface SmartRouterModelResolution {
|
|
66
|
+
readonly modelId: string;
|
|
67
|
+
readonly source: string;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Virtual endpoint shown when Smart Router is enabled for the active identity. */
|
|
71
|
+
export const SMART_ROUTER_ENDPOINT: ServingEndpointSummary = {
|
|
72
|
+
name: SMART_ROUTER_MODEL_ID,
|
|
73
|
+
displayName: "Smart Router",
|
|
74
|
+
description: "Selects a Databricks system model with the task_v3 Smart Router.",
|
|
75
|
+
task: "llm/v1/chat",
|
|
76
|
+
status: { deprecated: false },
|
|
77
|
+
};
|
|
47
78
|
|
|
48
79
|
/** Keep Responses turns stateless so tool continuations replay encrypted reasoning inline. */
|
|
49
80
|
export const RESPONSES_PROVIDER_OPTIONS = {
|
|
@@ -66,14 +97,16 @@ export interface BuildModelOverrides {
|
|
|
66
97
|
*/
|
|
67
98
|
modelId?: string;
|
|
68
99
|
/**
|
|
69
|
-
* Chat capability class
|
|
70
|
-
* supplied
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
* available model in the class
|
|
74
|
-
*
|
|
100
|
+
* Chat capability class for internal model work. When no model id is
|
|
101
|
+
* supplied, this bypasses request and environment model overrides so
|
|
102
|
+
* background tasks such as titles and follow-ups cannot inherit the
|
|
103
|
+
* primary agent's Smart Router selection. The live catalogue chooses
|
|
104
|
+
* the top available model in the class, falling back to the class's
|
|
105
|
+
* static list when the workspace has none.
|
|
75
106
|
*/
|
|
76
107
|
modelClass?: ModelClass;
|
|
108
|
+
/** Arbitrary request, messages, JSON, or text used to build Smart Router task context. */
|
|
109
|
+
smartRouterInput?: unknown;
|
|
77
110
|
/** Explicit workspace client for standalone callers such as CLIs. */
|
|
78
111
|
workspaceClient?: WorkspaceClient;
|
|
79
112
|
}
|
|
@@ -82,6 +115,7 @@ export interface BuildModelOverrides {
|
|
|
82
115
|
interface ResolvedModelWorkspaceClient {
|
|
83
116
|
client: WorkspaceClient;
|
|
84
117
|
auth: "obo" | "service-principal";
|
|
118
|
+
cacheIdentity?: string;
|
|
85
119
|
}
|
|
86
120
|
|
|
87
121
|
/** Resolve an OBO workspace client when present, else a service-principal client. */
|
|
@@ -94,6 +128,7 @@ function resolveModelWorkspaceContext(
|
|
|
94
128
|
return {
|
|
95
129
|
client: executionContext.client,
|
|
96
130
|
auth: "isUserContext" in executionContext ? "obo" : "service-principal",
|
|
131
|
+
cacheIdentity: executionContextUserId(executionContext),
|
|
97
132
|
};
|
|
98
133
|
}
|
|
99
134
|
return { client: createWorkspaceClient(), auth: "service-principal" };
|
|
@@ -108,6 +143,7 @@ function selectionInput(
|
|
|
108
143
|
config: MastraPluginConfig,
|
|
109
144
|
requested: string | undefined,
|
|
110
145
|
defaultClass: ModelClass | undefined,
|
|
146
|
+
cacheIdentity: string | undefined,
|
|
111
147
|
serving = resolveServingConfig(config),
|
|
112
148
|
) {
|
|
113
149
|
const requestedClass = requested !== undefined ? parseModelClass(requested) : null;
|
|
@@ -121,6 +157,7 @@ function selectionInput(
|
|
|
121
157
|
fallbacks: serving.fallbacks.slice(),
|
|
122
158
|
liveOnly: requested === undefined,
|
|
123
159
|
ttlMs: serving.ttlMs,
|
|
160
|
+
...(cacheIdentity !== undefined ? { cacheIdentity } : {}),
|
|
124
161
|
};
|
|
125
162
|
}
|
|
126
163
|
|
|
@@ -136,9 +173,14 @@ export function resolveDefaultModelId(
|
|
|
136
173
|
endpoints: readonly ServingEndpointSummary[],
|
|
137
174
|
overrides: BuildModelOverrides = {},
|
|
138
175
|
): string {
|
|
176
|
+
if (endpoints.some((endpoint) => endpoint.name === SMART_ROUTER_MODEL_ID)) {
|
|
177
|
+
return SMART_ROUTER_MODEL_ID;
|
|
178
|
+
}
|
|
139
179
|
const requested = overrides.modelId ?? process.env.DATABRICKS_SERVING_ENDPOINT_NAME;
|
|
140
|
-
return selection.resolveModel(
|
|
141
|
-
|
|
180
|
+
return selection.resolveModel(
|
|
181
|
+
endpoints,
|
|
182
|
+
selectionInput(config, requested, overrides.modelClass, undefined),
|
|
183
|
+
).modelId;
|
|
142
184
|
}
|
|
143
185
|
|
|
144
186
|
/**
|
|
@@ -167,7 +209,7 @@ export async function buildModel(
|
|
|
167
209
|
const resolved = overrides.workspaceClient
|
|
168
210
|
? { client: overrides.workspaceClient, auth: "service-principal" as const }
|
|
169
211
|
: resolveModelWorkspaceContext(context);
|
|
170
|
-
const { client, auth } = resolved;
|
|
212
|
+
const { client, auth, cacheIdentity } = resolved;
|
|
171
213
|
recordActiveTraceAuth(auth);
|
|
172
214
|
const clientConfig = client.config;
|
|
173
215
|
const host = (await clientConfig.getHost()).toString();
|
|
@@ -175,24 +217,85 @@ export async function buildModel(
|
|
|
175
217
|
await clientConfig.authenticate(headers);
|
|
176
218
|
const logger = log.logger(config);
|
|
177
219
|
const serving = resolveServingConfig(config);
|
|
178
|
-
const
|
|
179
|
-
|
|
180
|
-
|
|
220
|
+
const internalClassOnly = overrides.modelClass !== undefined && overrides.modelId === undefined;
|
|
221
|
+
const override =
|
|
222
|
+
serving.allowOverride && !internalClassOnly
|
|
223
|
+
? (context.get(MASTRA_MODEL_OVERRIDE_KEY) as string | undefined)
|
|
224
|
+
: undefined;
|
|
181
225
|
|
|
182
226
|
// The override / agent default / env value can be either a concrete
|
|
183
227
|
// endpoint name or a model class slug ("chat-thinking" /
|
|
184
228
|
// "chat-balanced" / "chat-fast"). A class slug becomes a class intent
|
|
185
229
|
// (let the live catalogue pick the best model in that band); anything
|
|
186
230
|
// else is an explicit name fuzzy-matched against the catalogue. An
|
|
187
|
-
// internal `overrides.modelClass` (e.g.
|
|
188
|
-
//
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
);
|
|
231
|
+
// internal `overrides.modelClass` (e.g. titles, follow-ups, or the chart
|
|
232
|
+
// planner) is authoritative unless that caller also pins `modelId`. It must
|
|
233
|
+
// not inherit the primary agent's request or environment selection.
|
|
234
|
+
const requested =
|
|
235
|
+
override ??
|
|
236
|
+
overrides.modelId ??
|
|
237
|
+
(!internalClassOnly ? process.env.DATABRICKS_SERVING_ENDPOINT_NAME : undefined);
|
|
238
|
+
const smartRouterInput = overrides.smartRouterInput ?? context.get(MASTRA_SMART_ROUTER_INPUT_KEY);
|
|
239
|
+
const smartRouterRequested = isSmartRouterModelId(requested);
|
|
240
|
+
if (smartRouterRequested && smartRouterInput === undefined) {
|
|
241
|
+
throw new Error("Smart Router requires task input before it can resolve a concrete model");
|
|
242
|
+
}
|
|
243
|
+
let modelId: string;
|
|
244
|
+
let source: string;
|
|
245
|
+
if (
|
|
246
|
+
(requested === undefined || smartRouterRequested) &&
|
|
247
|
+
overrides.modelClass === undefined &&
|
|
248
|
+
smartRouterInput !== undefined
|
|
249
|
+
) {
|
|
250
|
+
let pending = context.get(MASTRA_SMART_ROUTER_RESOLUTION_KEY) as
|
|
251
|
+
Promise<SmartRouterModelResolution> | undefined;
|
|
252
|
+
if (!pending) {
|
|
253
|
+
pending = (async () => {
|
|
254
|
+
const endpoints = await listServingEndpoints(client, host, {
|
|
255
|
+
ttlMs: serving.ttlMs,
|
|
256
|
+
...(cacheIdentity !== undefined ? { cacheIdentity } : {}),
|
|
257
|
+
});
|
|
258
|
+
const candidates = smartRouterCandidates(
|
|
259
|
+
endpoints.filter((endpoint) => endpointCapabilities(endpoint).tools),
|
|
260
|
+
"codex",
|
|
261
|
+
);
|
|
262
|
+
// An explicit Smart Router request does not need a synthetic preflight:
|
|
263
|
+
// the real selection is the availability check. Auto mode still probes
|
|
264
|
+
// so it can fall back to the configured model policy when unavailable.
|
|
265
|
+
const enabled = smartRouterRequested || (await smartRouter.isEnabled(candidates, client));
|
|
266
|
+
if (enabled) {
|
|
267
|
+
const selected = await smartRouter.select(smartRouterInput, candidates, { client });
|
|
268
|
+
return { modelId: selected.value.name, source: SMART_ROUTER_MODEL_ID };
|
|
269
|
+
}
|
|
270
|
+
const fallback = await selectModel(
|
|
271
|
+
client,
|
|
272
|
+
host,
|
|
273
|
+
selectionInput(config, requested, overrides.modelClass, cacheIdentity, serving),
|
|
274
|
+
);
|
|
275
|
+
return { modelId: fallback.modelId, source: fallback.source };
|
|
276
|
+
})();
|
|
277
|
+
context.set(MASTRA_SMART_ROUTER_RESOLUTION_KEY, pending);
|
|
278
|
+
}
|
|
279
|
+
try {
|
|
280
|
+
({ modelId, source } = await pending);
|
|
281
|
+
} catch (error) {
|
|
282
|
+
if (context.get(MASTRA_SMART_ROUTER_RESOLUTION_KEY) === pending) {
|
|
283
|
+
context.set(MASTRA_SMART_ROUTER_RESOLUTION_KEY, undefined);
|
|
284
|
+
}
|
|
285
|
+
throw error;
|
|
286
|
+
}
|
|
287
|
+
} else {
|
|
288
|
+
const resolved = await selectModel(
|
|
289
|
+
client,
|
|
290
|
+
host,
|
|
291
|
+
selectionInput(config, requested, overrides.modelClass, cacheIdentity, serving),
|
|
292
|
+
);
|
|
293
|
+
modelId = resolved.modelId;
|
|
294
|
+
source = resolved.source;
|
|
295
|
+
}
|
|
296
|
+
if (isSmartRouterModelId(modelId)) {
|
|
297
|
+
throw new Error("Smart Router did not resolve to a concrete serving endpoint");
|
|
298
|
+
}
|
|
196
299
|
logger.debug("model selected", { modelId, source, requested });
|
|
197
300
|
context.set(MASTRA_RESOLVED_MODEL_KEY, modelId);
|
|
198
301
|
recordActiveTraceModel(modelId);
|
|
@@ -205,3 +308,17 @@ export async function buildModel(
|
|
|
205
308
|
providerName: config.providerId ?? "openai",
|
|
206
309
|
});
|
|
207
310
|
}
|
|
311
|
+
|
|
312
|
+
/** Append the virtual Smart Router endpoint only after its memoized live probe succeeds. */
|
|
313
|
+
export async function withSmartRouterEndpoint(
|
|
314
|
+
endpoints: readonly ServingEndpointSummary[],
|
|
315
|
+
client: WorkspaceClient,
|
|
316
|
+
): Promise<ServingEndpointSummary[]> {
|
|
317
|
+
const candidates = smartRouterCandidates(
|
|
318
|
+
endpoints.filter((endpoint) => endpointCapabilities(endpoint).tools),
|
|
319
|
+
"codex",
|
|
320
|
+
);
|
|
321
|
+
return (await smartRouter.isEnabled(candidates, client))
|
|
322
|
+
? [SMART_ROUTER_ENDPOINT, ...endpoints]
|
|
323
|
+
: [...endpoints];
|
|
324
|
+
}
|
package/src/plugin.ts
CHANGED
|
@@ -142,7 +142,7 @@ import {
|
|
|
142
142
|
mlflowEnabled,
|
|
143
143
|
mlflowExperimentManagerUrl,
|
|
144
144
|
} from "./mlflow.ts";
|
|
145
|
-
import { resolveDefaultModelId } from "./model.ts";
|
|
145
|
+
import { resolveDefaultModelId, withSmartRouterEndpoint } from "./model.ts";
|
|
146
146
|
import { buildObservability, configureOtelPropagation } from "./observability.ts";
|
|
147
147
|
import { provisionRemoteSkills } from "./remote-skills.ts";
|
|
148
148
|
import {
|
|
@@ -1332,10 +1332,11 @@ export class MastraPlugin extends Plugin<MastraPluginConfig> {
|
|
|
1332
1332
|
const cacheIdentity = executionContextUserId(context);
|
|
1333
1333
|
const host = (await client.config.getHost()).toString();
|
|
1334
1334
|
const serving = resolveServingConfig(this.config);
|
|
1335
|
-
|
|
1335
|
+
const endpoints = await modelCatalog.listServingEndpoints(client, host, {
|
|
1336
1336
|
ttlMs: serving.ttlMs,
|
|
1337
1337
|
cacheIdentity,
|
|
1338
1338
|
});
|
|
1339
|
+
return withSmartRouterEndpoint(endpoints, client);
|
|
1339
1340
|
}, modelCatalogueDefaults);
|
|
1340
1341
|
}
|
|
1341
1342
|
|