@dbx-tools/appkit-mastra 0.9.60 → 0.9.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -4
- package/index.ts +6 -3
- package/lib/index.d.ts +6 -3
- package/lib/index.js +5 -3
- package/lib/src/agents.js +26 -3
- package/lib/src/config.d.ts +19 -0
- package/lib/src/config.js +9 -1
- package/lib/src/genie.js +3 -1
- package/lib/src/mlflow.d.ts +28 -4
- package/lib/src/mlflow.js +125 -14
- package/lib/src/server.js +3 -1
- package/lib/src/skill-cache.d.ts +70 -0
- package/lib/src/skill-cache.js +194 -0
- package/lib/src/telemetry.d.ts +41 -0
- package/lib/src/telemetry.js +89 -4
- package/lib/src/workspaces.d.ts +10 -0
- package/lib/src/workspaces.js +84 -24
- package/lib/tsconfig.tsbuildinfo +1 -1
- package/package.json +22 -14
- package/src/agents.ts +29 -2
- package/src/config.ts +30 -0
- package/src/genie.ts +2 -0
- package/src/mlflow.ts +131 -15
- package/src/server.ts +1 -0
- package/src/skill-cache.ts +312 -0
- package/src/telemetry.ts +90 -2
- package/src/workspaces.ts +107 -22
package/src/genie.ts
CHANGED
|
@@ -59,6 +59,7 @@ import { chartPlannerRequestSchema, chartToolOutputSchema, prepareChart } from "
|
|
|
59
59
|
import { MASTRA_USER_KEY, resolveUserKey } from "./config.ts";
|
|
60
60
|
import type { MastraPluginConfig, User } from "./config.ts";
|
|
61
61
|
import { fetchStatementData } from "./statement.ts";
|
|
62
|
+
import { stampGenieToolSpan } from "./telemetry.ts";
|
|
62
63
|
import { safeWriteProgress } from "./writer.ts";
|
|
63
64
|
|
|
64
65
|
const logger = log.logger("mastra/genie");
|
|
@@ -428,6 +429,7 @@ function buildAskGenieTool(opts: {
|
|
|
428
429
|
}),
|
|
429
430
|
execute: async ({ question }, ctxRaw) => {
|
|
430
431
|
const ctx = ctxRaw as ToolExecuteCtx;
|
|
432
|
+
stampGenieToolSpan();
|
|
431
433
|
const { client, requestContext } = requireClient(ctx, toolId);
|
|
432
434
|
const writer = ctx?.writer;
|
|
433
435
|
const toolCallId = ctx?.agent?.toolCallId;
|
package/src/mlflow.ts
CHANGED
|
@@ -13,17 +13,21 @@
|
|
|
13
13
|
*
|
|
14
14
|
* There is no MLflow JS SDK, so this posts to the assessments REST
|
|
15
15
|
* endpoint directly using the OBO-scoped workspace client (the feedback
|
|
16
|
-
* is thus attributed to the signed-in user).
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
16
|
+
* is thus attributed to the signed-in user). UC-backed Apps traces need
|
|
17
|
+
* the experiment's table-prefix URI (`trace:/{catalog}.{schema}.{prefix}/{tr-hex}`)
|
|
18
|
+
* rather than the experiment-store id the UI does not store. That prefix is
|
|
19
|
+
* read from the experiment's `mlflow.experiment.databricksTraceDestinationPath`
|
|
20
|
+
* tag (or the spans-table tag); `MLFLOW_UC_TRACE_PREFIX` remains an optional
|
|
21
|
+
* override. Trace export is asynchronous, so the just-finished trace may not
|
|
22
|
+
* exist in MLflow yet when the user reacts; the log call retries briefly on
|
|
23
|
+
* "not found" before giving up softly.
|
|
20
24
|
*
|
|
21
25
|
* @module
|
|
22
26
|
*/
|
|
23
27
|
|
|
24
28
|
import { ConfigurationError } from "@databricks/appkit";
|
|
25
29
|
import { appkit } from "@dbx-tools/appkit";
|
|
26
|
-
import { asyncUtils, errorUtils, log } from "@dbx-tools/shared-core";
|
|
30
|
+
import { asyncUtils, errorUtils, log, object } from "@dbx-tools/shared-core";
|
|
27
31
|
import { feedback } from "@dbx-tools/shared-mastra";
|
|
28
32
|
import { z } from "zod";
|
|
29
33
|
import { databricksFetch, readResponseJson, readResponseText } from "./rest.ts";
|
|
@@ -33,6 +37,101 @@ const logger = log.logger("mastra/mlflow");
|
|
|
33
37
|
/** Workspace client carried on an AppKit execution context. */
|
|
34
38
|
type WorkspaceClient = appkit.WorkspaceClientLike;
|
|
35
39
|
|
|
40
|
+
/** Experiment tag Databricks stores as `catalog.schema.table_prefix`. */
|
|
41
|
+
const TRACE_DESTINATION_PATH_TAG = "mlflow.experiment.databricksTraceDestinationPath";
|
|
42
|
+
/** Experiment tag naming the OTel spans table (`catalog.schema.{prefix}_otel_spans`). */
|
|
43
|
+
const TRACE_SPAN_STORAGE_TABLE_TAG = "mlflow.experiment.databricksTraceSpanStorageTable";
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Cached UC table prefix resolved from the experiment. `loaded` distinguishes
|
|
47
|
+
* "not fetched yet" from "fetched, experiment has no UC location". Network
|
|
48
|
+
* failures are not cached so a later thumbs click can retry.
|
|
49
|
+
*/
|
|
50
|
+
let loadedUcPrefix: { prefix: string | undefined } | undefined;
|
|
51
|
+
|
|
52
|
+
/** Strip the `trace:/` URI wrapper and trailing slashes from a UC prefix. */
|
|
53
|
+
function normalizeUcPrefix(raw: string | undefined): string | undefined {
|
|
54
|
+
const value = raw
|
|
55
|
+
?.trim()
|
|
56
|
+
.replace(/^trace:\//, "")
|
|
57
|
+
.replace(/\/+$/, "");
|
|
58
|
+
return value || undefined;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Derive `catalog.schema.table_prefix` from a spans table name.
|
|
63
|
+
* Databricks names the table `{prefix}_otel_spans`.
|
|
64
|
+
*/
|
|
65
|
+
function prefixFromSpanTable(table: string | undefined): string | undefined {
|
|
66
|
+
const name = table?.trim();
|
|
67
|
+
if (!name) return undefined;
|
|
68
|
+
if (name.endsWith("_otel_spans")) return name.slice(0, -"_otel_spans".length);
|
|
69
|
+
if (name.endsWith("_spans")) return name.slice(0, -"_spans".length);
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function experimentTagMap(raw: unknown): Map<string, string> {
|
|
74
|
+
const tags = new Map<string, string>();
|
|
75
|
+
if (Array.isArray(raw)) {
|
|
76
|
+
for (const tag of raw) {
|
|
77
|
+
if (!object.isRecord(tag) || typeof tag.key !== "string" || typeof tag.value !== "string") {
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
tags.set(tag.key, tag.value);
|
|
81
|
+
}
|
|
82
|
+
return tags;
|
|
83
|
+
}
|
|
84
|
+
if (!object.isRecord(raw)) return tags;
|
|
85
|
+
for (const [key, value] of Object.entries(raw)) {
|
|
86
|
+
if (typeof value === "string") tags.set(key, value);
|
|
87
|
+
}
|
|
88
|
+
return tags;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* UC table prefix the assessments API needs, read from experiment tags.
|
|
93
|
+
*
|
|
94
|
+
* Prefers the three-part destination path Databricks writes when the
|
|
95
|
+
* experiment is linked to a table-prefix location. Falls back to stripping
|
|
96
|
+
* `_otel_spans` from the spans-table tag. Two-part schema-linked destinations
|
|
97
|
+
* are ignored (they are not a table prefix).
|
|
98
|
+
*/
|
|
99
|
+
export function ucTracePrefixFromExperimentTags(rawTags: unknown): string | undefined {
|
|
100
|
+
const tags = experimentTagMap(rawTags);
|
|
101
|
+
const destination = normalizeUcPrefix(tags.get(TRACE_DESTINATION_PATH_TAG));
|
|
102
|
+
if (destination && destination.split(".").length >= 3) return destination;
|
|
103
|
+
return prefixFromSpanTable(tags.get(TRACE_SPAN_STORAGE_TABLE_TAG));
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** Drop the in-process UC prefix cache (tests). */
|
|
107
|
+
export function resetUcTracePrefixCache(): void {
|
|
108
|
+
loadedUcPrefix = undefined;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Trace id the assessments REST API accepts for this deployment.
|
|
113
|
+
*
|
|
114
|
+
* Databricks Apps persist traces in Unity Catalog. The experiment UI lists
|
|
115
|
+
* those rows, but `POST /api/3.0/mlflow/traces/{id}/assessments` looks up the
|
|
116
|
+
* experiment store unless `{id}` is the UC URI `trace:/{catalog}.{schema}.{prefix}/{tr-hex}`.
|
|
117
|
+
* A bare `tr-<hex>` 404s and the assessment never appears.
|
|
118
|
+
*/
|
|
119
|
+
export function mlflowAssessmentTraceId(traceId: string, prefix?: string): string {
|
|
120
|
+
const resolved = normalizeUcPrefix(prefix);
|
|
121
|
+
return resolved ? `trace:/${resolved}/${traceId}` : traceId;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
async function resolveUcTracePrefix(client: WorkspaceClient): Promise<string | undefined> {
|
|
125
|
+
const fromEnv = normalizeUcPrefix(process.env.MLFLOW_UC_TRACE_PREFIX);
|
|
126
|
+
if (fromEnv) return fromEnv;
|
|
127
|
+
if (loadedUcPrefix) return loadedUcPrefix.prefix;
|
|
128
|
+
const experiment = await fetchExperiment(client);
|
|
129
|
+
if (!experiment) return undefined;
|
|
130
|
+
const prefix = ucTracePrefixFromExperimentTags(experiment.tags);
|
|
131
|
+
loadedUcPrefix = { prefix };
|
|
132
|
+
return prefix;
|
|
133
|
+
}
|
|
134
|
+
|
|
36
135
|
/** Assessments REST path for a trace. `3.0` is the current MLflow API version. */
|
|
37
136
|
const assessmentsPath = (traceId: string): string =>
|
|
38
137
|
`/api/3.0/mlflow/traces/${encodeURIComponent(traceId)}/assessments`;
|
|
@@ -47,6 +146,7 @@ const MlflowExperimentResponseSchema = z
|
|
|
47
146
|
experiment: z
|
|
48
147
|
.object({
|
|
49
148
|
experiment_id: z.string().min(1).describe("Workspace MLflow experiment identifier."),
|
|
149
|
+
tags: z.unknown().optional().describe("Experiment tags, including the UC trace location."),
|
|
50
150
|
})
|
|
51
151
|
.passthrough()
|
|
52
152
|
.describe("Configured MLflow experiment."),
|
|
@@ -226,18 +326,28 @@ async function resolveMlflowExperimentManagerUrl(
|
|
|
226
326
|
return new URL(`/ml/experiments/${encodeURIComponent(experimentId)}`, host).toString();
|
|
227
327
|
}
|
|
228
328
|
|
|
229
|
-
async function
|
|
329
|
+
async function fetchExperiment(
|
|
330
|
+
client: WorkspaceClient,
|
|
331
|
+
): Promise<{ experiment_id: string; tags?: unknown } | undefined> {
|
|
230
332
|
const experiment = configuredExperiment();
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
333
|
+
const path = experiment.id
|
|
334
|
+
? `/api/2.0/mlflow/experiments/get?experiment_id=${encodeURIComponent(experiment.id)}`
|
|
335
|
+
: experiment.name
|
|
336
|
+
? `/api/2.0/mlflow/experiments/get-by-name?experiment_name=${encodeURIComponent(
|
|
337
|
+
experiment.name,
|
|
338
|
+
)}`
|
|
339
|
+
: undefined;
|
|
340
|
+
if (!path) return undefined;
|
|
237
341
|
const response = await databricksFetch(client, path, { method: "GET" });
|
|
238
342
|
if (!response.ok) return undefined;
|
|
239
343
|
const parsed = MlflowExperimentResponseSchema.safeParse(await readResponseJson(response));
|
|
240
|
-
return parsed.success ? parsed.data.experiment
|
|
344
|
+
return parsed.success ? parsed.data.experiment : undefined;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
async function resolveExperimentId(client: WorkspaceClient): Promise<string | undefined> {
|
|
348
|
+
const configured = configuredExperiment();
|
|
349
|
+
if (configured.id) return configured.id;
|
|
350
|
+
return (await fetchExperiment(client))?.experiment_id;
|
|
241
351
|
}
|
|
242
352
|
|
|
243
353
|
/** Parameters for {@link logFeedback}. */
|
|
@@ -272,8 +382,12 @@ export async function logFeedback(
|
|
|
272
382
|
params.name?.trim() ||
|
|
273
383
|
(hasValue ? feedback.DEFAULT_FEEDBACK_NAME : feedback.DEFAULT_COMMENT_NAME);
|
|
274
384
|
const value = hasValue ? params.value : params.comment;
|
|
385
|
+
const assessmentTraceId = mlflowAssessmentTraceId(
|
|
386
|
+
params.traceId,
|
|
387
|
+
await resolveUcTracePrefix(client),
|
|
388
|
+
);
|
|
275
389
|
const assessment: Record<string, unknown> = {
|
|
276
|
-
trace_id:
|
|
390
|
+
trace_id: assessmentTraceId,
|
|
277
391
|
assessment_name: name,
|
|
278
392
|
source: {
|
|
279
393
|
source_type: "HUMAN",
|
|
@@ -287,13 +401,14 @@ export async function logFeedback(
|
|
|
287
401
|
for (let attempt = 0; attempt <= NOT_FOUND_RETRIES; attempt++) {
|
|
288
402
|
let res: Response;
|
|
289
403
|
try {
|
|
290
|
-
res = await databricksFetch(client, assessmentsPath(
|
|
404
|
+
res = await databricksFetch(client, assessmentsPath(assessmentTraceId), {
|
|
291
405
|
method: "POST",
|
|
292
406
|
body,
|
|
293
407
|
});
|
|
294
408
|
} catch (err) {
|
|
295
409
|
logger.warn("feedback request failed", {
|
|
296
410
|
traceId: params.traceId,
|
|
411
|
+
assessmentTraceId,
|
|
297
412
|
error: errorUtils.errorMessage(err),
|
|
298
413
|
});
|
|
299
414
|
return undefined;
|
|
@@ -313,6 +428,7 @@ export async function logFeedback(
|
|
|
313
428
|
}
|
|
314
429
|
logger.warn("feedback not recorded", {
|
|
315
430
|
traceId: params.traceId,
|
|
431
|
+
assessmentTraceId,
|
|
316
432
|
status: res.status,
|
|
317
433
|
body: await readResponseText(res),
|
|
318
434
|
});
|
package/src/server.ts
CHANGED
|
@@ -276,6 +276,7 @@ export class MastraServer extends ExpressMastraServer {
|
|
|
276
276
|
*/
|
|
277
277
|
configureMlflowTraceId(res: express.Response) {
|
|
278
278
|
if (!this.feedbackEnabled || res.headersSent) return;
|
|
279
|
+
if (res.getHeader(feedback.MLFLOW_TRACE_ID_HEADER)) return;
|
|
279
280
|
const traceId = trace.getActiveSpan()?.spanContext().traceId;
|
|
280
281
|
if (!traceId || traceId === INVALID_TRACE_ID) return;
|
|
281
282
|
res.setHeader(feedback.MLFLOW_TRACE_ID_HEADER, `tr-${traceId}`);
|
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AppKit-backed caching for Databricks workspace skill mounts.
|
|
3
|
+
*
|
|
4
|
+
* @module
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { randomUUID } from "node:crypto";
|
|
8
|
+
import { CacheManager } from "@databricks/appkit";
|
|
9
|
+
import { hash, object } from "@dbx-tools/shared-core";
|
|
10
|
+
import type {
|
|
11
|
+
CopyOptions,
|
|
12
|
+
FileContent,
|
|
13
|
+
FileEntry,
|
|
14
|
+
FileStat,
|
|
15
|
+
FileSystem,
|
|
16
|
+
ListOptions,
|
|
17
|
+
MakeDirectoryOptions,
|
|
18
|
+
ReadFileOptions,
|
|
19
|
+
RemoveOptions,
|
|
20
|
+
WriteFileOptions,
|
|
21
|
+
} from "@dbx-tools/shared-fs";
|
|
22
|
+
|
|
23
|
+
import { filesystems, type MastraFileSystemAdapter } from "./filesystems.ts";
|
|
24
|
+
|
|
25
|
+
const CACHE_NAMESPACE = "mastra:workspace-skills";
|
|
26
|
+
const DEFAULT_MAX_SCOPED_MOUNTS = 256;
|
|
27
|
+
|
|
28
|
+
interface CachedFileStat extends Omit<FileStat, "accessedAt" | "createdAt" | "modifiedAt"> {
|
|
29
|
+
accessedAt?: number;
|
|
30
|
+
createdAt?: number;
|
|
31
|
+
modifiedAt?: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
interface ScopedMountEntry {
|
|
35
|
+
adapter: MastraFileSystemAdapter;
|
|
36
|
+
cache: AppKitCachedFileSystem;
|
|
37
|
+
host: string;
|
|
38
|
+
root: string;
|
|
39
|
+
userKey: string;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const scopedMounts = new Map<string, ScopedMountEntry>();
|
|
43
|
+
|
|
44
|
+
/** Default remote skill metadata and content cache lifetime. */
|
|
45
|
+
export const DEFAULT_WORKSPACE_SKILL_CACHE_TTL_MS = 5 * 60 * 1000;
|
|
46
|
+
|
|
47
|
+
/** Options for one AppKit-cached workspace skill filesystem. */
|
|
48
|
+
export interface AppKitCachedFileSystemOptions {
|
|
49
|
+
host: string;
|
|
50
|
+
source: FileSystem;
|
|
51
|
+
ttlMs?: number;
|
|
52
|
+
userKey: string;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Stable Mastra mount plus the identity key used for source reuse. */
|
|
56
|
+
export interface CachedWorkspaceSkillMount {
|
|
57
|
+
cacheKey: string;
|
|
58
|
+
filesystem: MastraFileSystemAdapter;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Cache read-only filesystem operations through AppKit while delegating
|
|
63
|
+
* mutations to the current request's source.
|
|
64
|
+
*/
|
|
65
|
+
export class AppKitCachedFileSystem implements FileSystem<"appkit-cache"> {
|
|
66
|
+
readonly backend = "appkit-cache" as const;
|
|
67
|
+
readonly id: string;
|
|
68
|
+
readonly readOnly: boolean;
|
|
69
|
+
readonly root: string;
|
|
70
|
+
|
|
71
|
+
private source: FileSystem;
|
|
72
|
+
private readonly host: string;
|
|
73
|
+
private readonly ttlSec: number;
|
|
74
|
+
private readonly userKey: string;
|
|
75
|
+
|
|
76
|
+
constructor(options: AppKitCachedFileSystemOptions) {
|
|
77
|
+
this.host = options.host;
|
|
78
|
+
this.source = options.source;
|
|
79
|
+
this.root = options.source.root;
|
|
80
|
+
this.readOnly = options.source.readOnly;
|
|
81
|
+
this.ttlSec = cacheTtlSec(options.ttlMs);
|
|
82
|
+
this.userKey = options.userKey;
|
|
83
|
+
this.id = `appkit-skill-cache-${hash.fnvHash(this.host, this.userKey, this.root)}`;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** Rebind cache misses and writes to the current request's OBO filesystem. */
|
|
87
|
+
bind(source: FileSystem): void {
|
|
88
|
+
if (source.root !== this.root || source.readOnly !== this.readOnly) {
|
|
89
|
+
throw new Error("Cannot rebind a workspace skill cache to a different filesystem scope");
|
|
90
|
+
}
|
|
91
|
+
this.source = source;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async init(): Promise<void> {
|
|
95
|
+
await this.source.init();
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** Keep the shared scope alive when one request releases its workspace view. */
|
|
99
|
+
close(): Promise<void> {
|
|
100
|
+
return Promise.resolve();
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
resolvePath(inputPath: string): string {
|
|
104
|
+
return this.source.resolvePath(inputPath);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
async readFile(inputPath: string): Promise<Uint8Array>;
|
|
108
|
+
async readFile(
|
|
109
|
+
inputPath: string,
|
|
110
|
+
options: ReadFileOptions & { encoding: string },
|
|
111
|
+
): Promise<string>;
|
|
112
|
+
async readFile(inputPath: string, options?: ReadFileOptions): Promise<string | Uint8Array> {
|
|
113
|
+
const value = await this.cached("readFile", inputPath, undefined, async (source) => ({
|
|
114
|
+
base64: Buffer.from(await source.readFile(inputPath)).toString("base64"),
|
|
115
|
+
}));
|
|
116
|
+
const bytes = Buffer.from(value.base64, "base64");
|
|
117
|
+
return options?.encoding ? bytes.toString(options.encoding as BufferEncoding) : bytes;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
async writeFile(
|
|
121
|
+
inputPath: string,
|
|
122
|
+
content: FileContent,
|
|
123
|
+
options?: WriteFileOptions,
|
|
124
|
+
): Promise<void> {
|
|
125
|
+
await this.mutate((source) => source.writeFile(inputPath, content, options));
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
async appendFile(inputPath: string, content: FileContent): Promise<void> {
|
|
129
|
+
await this.mutate((source) => source.appendFile(inputPath, content));
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
async deleteFile(inputPath: string, options?: RemoveOptions): Promise<void> {
|
|
133
|
+
await this.mutate((source) => source.deleteFile(inputPath, options));
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
async copyFile(
|
|
137
|
+
sourcePath: string,
|
|
138
|
+
destinationPath: string,
|
|
139
|
+
options?: CopyOptions,
|
|
140
|
+
): Promise<void> {
|
|
141
|
+
await this.mutate((source) => source.copyFile(sourcePath, destinationPath, options));
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
async moveFile(
|
|
145
|
+
sourcePath: string,
|
|
146
|
+
destinationPath: string,
|
|
147
|
+
options?: CopyOptions,
|
|
148
|
+
): Promise<void> {
|
|
149
|
+
await this.mutate((source) => source.moveFile(sourcePath, destinationPath, options));
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
async mkdir(inputPath: string, options?: MakeDirectoryOptions): Promise<void> {
|
|
153
|
+
await this.mutate((source) => source.mkdir(inputPath, options));
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
async rmdir(inputPath: string, options?: RemoveOptions): Promise<void> {
|
|
157
|
+
await this.mutate((source) => source.rmdir(inputPath, options));
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
async readdir(inputPath: string, options?: ListOptions): Promise<FileEntry[]> {
|
|
161
|
+
return this.cached("readdir", inputPath, options, (source) =>
|
|
162
|
+
source.readdir(inputPath, options),
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
async exists(inputPath: string): Promise<boolean> {
|
|
167
|
+
return this.cached("exists", inputPath, undefined, (source) => source.exists(inputPath));
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
async stat(inputPath: string): Promise<FileStat> {
|
|
171
|
+
const value = await this.cached("stat", inputPath, undefined, async (source) =>
|
|
172
|
+
serializeStat(await source.stat(inputPath)),
|
|
173
|
+
);
|
|
174
|
+
return deserializeStat(value);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** Rotate this scope's cache generation so the next read reloads it. */
|
|
178
|
+
async invalidate(): Promise<void> {
|
|
179
|
+
const cache = await CacheManager.getInstance();
|
|
180
|
+
await cache.set(this.generationKey(cache), randomUUID(), {
|
|
181
|
+
ttl: this.ttlSec * 2,
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
private async cached<T>(
|
|
186
|
+
operation: string,
|
|
187
|
+
inputPath: string,
|
|
188
|
+
options: unknown,
|
|
189
|
+
load: (source: FileSystem) => Promise<T>,
|
|
190
|
+
): Promise<T> {
|
|
191
|
+
const source = this.source;
|
|
192
|
+
const cache = await CacheManager.getInstance();
|
|
193
|
+
const generation = await this.generation(cache);
|
|
194
|
+
return cache.getOrExecute(
|
|
195
|
+
[
|
|
196
|
+
CACHE_NAMESPACE,
|
|
197
|
+
this.host,
|
|
198
|
+
this.root,
|
|
199
|
+
generation,
|
|
200
|
+
operation,
|
|
201
|
+
source.resolvePath(inputPath),
|
|
202
|
+
object.toStableKey(options ?? {}),
|
|
203
|
+
],
|
|
204
|
+
() => load(source),
|
|
205
|
+
this.userKey,
|
|
206
|
+
{ ttl: this.ttlSec },
|
|
207
|
+
);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
private async generation(cache: CacheManager): Promise<string> {
|
|
211
|
+
return cache.getOrExecute(
|
|
212
|
+
[CACHE_NAMESPACE, "generation", this.host, this.root],
|
|
213
|
+
async () => randomUUID(),
|
|
214
|
+
this.userKey,
|
|
215
|
+
{ ttl: this.ttlSec * 2 },
|
|
216
|
+
);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
private generationKey(cache: CacheManager): string {
|
|
220
|
+
return cache.generateKey([CACHE_NAMESPACE, "generation", this.host, this.root], this.userKey);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
private async mutate(run: (source: FileSystem) => Promise<void>): Promise<void> {
|
|
224
|
+
const source = this.source;
|
|
225
|
+
await run(source);
|
|
226
|
+
await this.invalidate();
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** Return one stable Mastra mount per resolved user, host, and source root. */
|
|
231
|
+
export function cachedWorkspaceSkillMount(
|
|
232
|
+
options: AppKitCachedFileSystemOptions,
|
|
233
|
+
): CachedWorkspaceSkillMount {
|
|
234
|
+
const cacheKey = object.toStableKey({
|
|
235
|
+
host: options.host,
|
|
236
|
+
readOnly: options.source.readOnly,
|
|
237
|
+
root: options.source.root,
|
|
238
|
+
userKey: options.userKey,
|
|
239
|
+
});
|
|
240
|
+
const existing = scopedMounts.get(cacheKey);
|
|
241
|
+
if (existing) {
|
|
242
|
+
existing.cache.bind(options.source);
|
|
243
|
+
scopedMounts.delete(cacheKey);
|
|
244
|
+
scopedMounts.set(cacheKey, existing);
|
|
245
|
+
return { cacheKey, filesystem: existing.adapter };
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
const cache = new AppKitCachedFileSystem(options);
|
|
249
|
+
const entry: ScopedMountEntry = {
|
|
250
|
+
adapter: filesystems(cache, {
|
|
251
|
+
id: cache.id,
|
|
252
|
+
readOnly: options.source.readOnly,
|
|
253
|
+
}),
|
|
254
|
+
cache,
|
|
255
|
+
host: options.host,
|
|
256
|
+
root: options.source.root,
|
|
257
|
+
userKey: options.userKey,
|
|
258
|
+
};
|
|
259
|
+
scopedMounts.set(cacheKey, entry);
|
|
260
|
+
trimScopedMounts();
|
|
261
|
+
return { cacheKey, filesystem: entry.adapter };
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/** Invalidate one user's cached workspace skill scope. */
|
|
265
|
+
export async function clearWorkspaceSkillCache(options: {
|
|
266
|
+
host: string;
|
|
267
|
+
root?: string;
|
|
268
|
+
userKey: string;
|
|
269
|
+
}): Promise<void> {
|
|
270
|
+
const matches = [...scopedMounts.entries()].filter(
|
|
271
|
+
([, entry]) =>
|
|
272
|
+
entry.host === options.host &&
|
|
273
|
+
entry.userKey === options.userKey &&
|
|
274
|
+
(options.root === undefined || entry.root === options.root),
|
|
275
|
+
);
|
|
276
|
+
await Promise.all(matches.map(([, entry]) => entry.cache.invalidate()));
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
function cacheTtlSec(ttlMs = DEFAULT_WORKSPACE_SKILL_CACHE_TTL_MS): number {
|
|
280
|
+
if (!Number.isFinite(ttlMs) || ttlMs <= 0) {
|
|
281
|
+
throw new Error("Workspace skill cache ttlMs must be a positive finite number");
|
|
282
|
+
}
|
|
283
|
+
return Math.max(1, Math.ceil(ttlMs / 1000));
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
function serializeStat(value: FileStat): CachedFileStat {
|
|
287
|
+
const { accessedAt, createdAt, modifiedAt, ...stat } = value;
|
|
288
|
+
return {
|
|
289
|
+
...stat,
|
|
290
|
+
...(accessedAt ? { accessedAt: accessedAt.getTime() } : {}),
|
|
291
|
+
...(createdAt ? { createdAt: createdAt.getTime() } : {}),
|
|
292
|
+
...(modifiedAt ? { modifiedAt: modifiedAt.getTime() } : {}),
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
function deserializeStat(value: CachedFileStat): FileStat {
|
|
297
|
+
const { accessedAt, createdAt, modifiedAt, ...stat } = value;
|
|
298
|
+
return {
|
|
299
|
+
...stat,
|
|
300
|
+
...(accessedAt !== undefined ? { accessedAt: new Date(accessedAt) } : {}),
|
|
301
|
+
...(createdAt !== undefined ? { createdAt: new Date(createdAt) } : {}),
|
|
302
|
+
...(modifiedAt !== undefined ? { modifiedAt: new Date(modifiedAt) } : {}),
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function trimScopedMounts(): void {
|
|
307
|
+
while (scopedMounts.size > DEFAULT_MAX_SCOPED_MOUNTS) {
|
|
308
|
+
const oldest = scopedMounts.keys().next().value;
|
|
309
|
+
if (oldest === undefined) return;
|
|
310
|
+
scopedMounts.delete(oldest);
|
|
311
|
+
}
|
|
312
|
+
}
|
package/src/telemetry.ts
CHANGED
|
@@ -20,11 +20,14 @@
|
|
|
20
20
|
|
|
21
21
|
import { StringDecoder } from "node:string_decoder";
|
|
22
22
|
|
|
23
|
-
import { json, log, object } from "@dbx-tools/shared-core";
|
|
23
|
+
import { json, log, object, stringUtils } from "@dbx-tools/shared-core";
|
|
24
|
+
import { feedback, thread } from "@dbx-tools/shared-mastra";
|
|
24
25
|
import { context, SpanKind, trace, type Span } from "@opentelemetry/api";
|
|
25
26
|
import { getRPCMetadata, RPCType } from "@opentelemetry/core";
|
|
26
27
|
import type express from "express";
|
|
27
28
|
|
|
29
|
+
import { USER_EMAIL_HEADER, USER_ID_HEADER } from "./identity.ts";
|
|
30
|
+
|
|
28
31
|
const logger = log.logger("mastra/telemetry");
|
|
29
32
|
|
|
30
33
|
/** Cap on each payload copied onto a span, so one turn cannot bloat the export. */
|
|
@@ -56,6 +59,34 @@ export const CHAT_RESPONSE_ATTR = "appkit.mastra.chat.response";
|
|
|
56
59
|
export const CHAT_IDENTITY_ATTR = "appkit.mastra.identity.mode";
|
|
57
60
|
/** Whether the streamed turn emitted a Genie tool or progress event. */
|
|
58
61
|
export const CHAT_GENIE_USED_ATTR = "appkit.mastra.genie.used";
|
|
62
|
+
/**
|
|
63
|
+
* MLflow span-type attribute. Databricks maps this (or `gen_ai.operation.name`)
|
|
64
|
+
* onto the type chip in the experiment UI. Custom values such as `GENIE` are
|
|
65
|
+
* allowed.
|
|
66
|
+
*/
|
|
67
|
+
export const MLFLOW_SPAN_TYPE_ATTR = "mlflow.spanType";
|
|
68
|
+
/** GenAI semantic-convention operation name, an alternate span-type source. */
|
|
69
|
+
export const GEN_AI_OPERATION_NAME_ATTR = "gen_ai.operation.name";
|
|
70
|
+
/**
|
|
71
|
+
* Root-span user id the MLflow experiment UI's User column reads from OTel
|
|
72
|
+
* traces. Prefer the forwarded email when present so the column is human.
|
|
73
|
+
*/
|
|
74
|
+
export const MLFLOW_USER_ATTR = "user.id";
|
|
75
|
+
/** Root-span session id that enables the MLflow Sessions tab. */
|
|
76
|
+
export const MLFLOW_SESSION_ATTR = "session.id";
|
|
77
|
+
/**
|
|
78
|
+
* Prefix Databricks/MLflow copies onto the unified-view `tags` map. The
|
|
79
|
+
* suffix after this prefix is the tag name shown in the experiment UI.
|
|
80
|
+
*/
|
|
81
|
+
export const MLFLOW_TRACE_TAG_PREFIX = "mlflow.trace.tag.";
|
|
82
|
+
/** Experiment-UI tag marking a turn that called Genie. */
|
|
83
|
+
export const MLFLOW_GENIE_TAG = "genie";
|
|
84
|
+
/** Root-span type for a Mastra agent turn. */
|
|
85
|
+
export const MLFLOW_SPAN_TYPE_AGENT = "AGENT";
|
|
86
|
+
/** Span type for an `ask_genie` tool call. */
|
|
87
|
+
export const MLFLOW_SPAN_TYPE_GENIE = "GENIE";
|
|
88
|
+
/** Attribute key for the `genie` trace tag. */
|
|
89
|
+
export const MLFLOW_GENIE_TAG_ATTR = `${MLFLOW_TRACE_TAG_PREFIX}${MLFLOW_GENIE_TAG}`;
|
|
59
90
|
|
|
60
91
|
/** Root-span metadata known before a chat request is dispatched. */
|
|
61
92
|
export interface ChatTurnTelemetryOptions {
|
|
@@ -68,6 +99,57 @@ interface TraceTarget {
|
|
|
68
99
|
owned: boolean;
|
|
69
100
|
}
|
|
70
101
|
|
|
102
|
+
/** OpenTelemetry's sentinel for "no valid trace" - 32 zero hex chars. */
|
|
103
|
+
const INVALID_TRACE_ID = "0".repeat(32);
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Publish the turn's MLflow trace id before the handler writes the body.
|
|
107
|
+
*
|
|
108
|
+
* The chat UI only shows thumbs when this header is present. Stamping it here
|
|
109
|
+
* (on the same span the middleware just resolved) avoids relying on async
|
|
110
|
+
* Express continuation still sitting inside `context.with()`.
|
|
111
|
+
*/
|
|
112
|
+
/**
|
|
113
|
+
* Mark the active span as a Genie call so the experiment UI types it `GENIE`
|
|
114
|
+
* and tags the trace `genie=true`. Safe no-op when no recording span is active.
|
|
115
|
+
*/
|
|
116
|
+
export function stampGenieToolSpan(span: Span | undefined = trace.getActiveSpan()): void {
|
|
117
|
+
if (!span?.isRecording()) return;
|
|
118
|
+
span.setAttribute(MLFLOW_SPAN_TYPE_ATTR, MLFLOW_SPAN_TYPE_GENIE);
|
|
119
|
+
span.setAttribute(GEN_AI_OPERATION_NAME_ATTR, "execute_tool");
|
|
120
|
+
span.setAttribute(MLFLOW_GENIE_TAG_ATTR, "true");
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function requestHeader(req: express.Request, name: string): string | undefined {
|
|
124
|
+
const fromMethod =
|
|
125
|
+
typeof req.header === "function" ? stringUtils.trimToNull(req.header(name)) : undefined;
|
|
126
|
+
if (fromMethod) return fromMethod;
|
|
127
|
+
const raw = req.headers?.[name.toLowerCase()];
|
|
128
|
+
const value = Array.isArray(raw) ? raw[0] : raw;
|
|
129
|
+
return stringUtils.trimToNull(typeof value === "string" ? value : undefined) ?? undefined;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function requestQuery(req: express.Request, name: string): string | undefined {
|
|
133
|
+
const raw = req.query?.[name];
|
|
134
|
+
const value = Array.isArray(raw) ? raw[0] : raw;
|
|
135
|
+
return stringUtils.trimToNull(typeof value === "string" ? value : undefined) ?? undefined;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function stampMlflowActor(span: Span, req: express.Request): void {
|
|
139
|
+
const user = requestHeader(req, USER_EMAIL_HEADER) ?? requestHeader(req, USER_ID_HEADER);
|
|
140
|
+
if (user) span.setAttribute(MLFLOW_USER_ATTR, user);
|
|
141
|
+
const session =
|
|
142
|
+
requestHeader(req, thread.THREAD_ID_HEADER) ?? requestQuery(req, thread.THREAD_ID_QUERY);
|
|
143
|
+
if (session) span.setAttribute(MLFLOW_SESSION_ATTR, session);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function stampMlflowTraceId(res: express.Response, span: Span): void {
|
|
147
|
+
if (res.headersSent) return;
|
|
148
|
+
const traceId = span.spanContext().traceId;
|
|
149
|
+
if (!traceId || traceId === INVALID_TRACE_ID) return;
|
|
150
|
+
res.setHeader(feedback.MLFLOW_TRACE_ID_HEADER, `tr-${traceId}`);
|
|
151
|
+
}
|
|
152
|
+
|
|
71
153
|
/** Return one raw prompt when the request is exactly one text-only user message. */
|
|
72
154
|
export function textOnlyChatInput(messages: unknown): string | undefined {
|
|
73
155
|
if (typeof messages === "string") return messages || undefined;
|
|
@@ -276,6 +358,10 @@ export function chatTurnTelemetryMiddleware(
|
|
|
276
358
|
next();
|
|
277
359
|
return;
|
|
278
360
|
}
|
|
361
|
+
stampMlflowTraceId(res, target.span);
|
|
362
|
+
stampMlflowActor(target.span, req);
|
|
363
|
+
target.span.setAttribute(MLFLOW_SPAN_TYPE_ATTR, MLFLOW_SPAN_TYPE_AGENT);
|
|
364
|
+
target.span.setAttribute(GEN_AI_OPERATION_NAME_ATTR, "invoke_agent");
|
|
279
365
|
const identity =
|
|
280
366
|
typeof options.identity === "function" ? options.identity(req) : options.identity;
|
|
281
367
|
if (identity) {
|
|
@@ -311,7 +397,9 @@ export function chatTurnTelemetryMiddleware(
|
|
|
311
397
|
fullOutputRecorded = true;
|
|
312
398
|
};
|
|
313
399
|
const recordGenie = (used: boolean): void => {
|
|
314
|
-
if (used)
|
|
400
|
+
if (!used) return;
|
|
401
|
+
target.span.setAttribute(CHAT_GENIE_USED_ATTR, true);
|
|
402
|
+
target.span.setAttribute(MLFLOW_GENIE_TAG_ATTR, "true");
|
|
315
403
|
};
|
|
316
404
|
const endOwnedSpan = (): void => {
|
|
317
405
|
if (!target.owned || ownedSpanEnded) return;
|