@dbx-tools/appkit-mastra 0.9.61 → 0.9.65

package/src/mlflow.ts

@@ -51,7 +51,10 @@ let loadedUcPrefix: { prefix: string | undefined } | undefined;
 
 /** Strip the `trace:/` URI wrapper and trailing slashes from a UC prefix. */
 function normalizeUcPrefix(raw: string | undefined): string | undefined {
-  const value = raw?.trim().replace(/^trace:\//, "").replace(/\/+$/, "");
+  const value = raw
+    ?.trim()
+    .replace(/^trace:\//, "")
+    .replace(/\/+$/, "");
   return value || undefined;
 }
 

package/src/skill-cache.ts

@@ -0,0 +1,312 @@
+/**
+ * AppKit-backed caching for Databricks workspace skill mounts.
+ *
+ * @module
+ */
+
+import { randomUUID } from "node:crypto";
+import { CacheManager } from "@databricks/appkit";
+import { hash, object } from "@dbx-tools/shared-core";
+import type {
+  CopyOptions,
+  FileContent,
+  FileEntry,
+  FileStat,
+  FileSystem,
+  ListOptions,
+  MakeDirectoryOptions,
+  ReadFileOptions,
+  RemoveOptions,
+  WriteFileOptions,
+} from "@dbx-tools/shared-fs";
+
+import { filesystems, type MastraFileSystemAdapter } from "./filesystems.ts";
+
+const CACHE_NAMESPACE = "mastra:workspace-skills";
+const DEFAULT_MAX_SCOPED_MOUNTS = 256;
+
+interface CachedFileStat extends Omit<FileStat, "accessedAt" | "createdAt" | "modifiedAt"> {
+  accessedAt?: number;
+  createdAt?: number;
+  modifiedAt?: number;
+}
+
+interface ScopedMountEntry {
+  adapter: MastraFileSystemAdapter;
+  cache: AppKitCachedFileSystem;
+  host: string;
+  root: string;
+  userKey: string;
+}
+
+const scopedMounts = new Map<string, ScopedMountEntry>();
+
+/** Default remote skill metadata and content cache lifetime. */
+export const DEFAULT_WORKSPACE_SKILL_CACHE_TTL_MS = 5 * 60 * 1000;
+
+/** Options for one AppKit-cached workspace skill filesystem. */
+export interface AppKitCachedFileSystemOptions {
+  host: string;
+  source: FileSystem;
+  ttlMs?: number;
+  userKey: string;
+}
+
+/** Stable Mastra mount plus the identity key used for source reuse. */
+export interface CachedWorkspaceSkillMount {
+  cacheKey: string;
+  filesystem: MastraFileSystemAdapter;
+}
+
+/**
+ * Cache read-only filesystem operations through AppKit while delegating
+ * mutations to the current request's source.
+ */
+export class AppKitCachedFileSystem implements FileSystem<"appkit-cache"> {
+  readonly backend = "appkit-cache" as const;
+  readonly id: string;
+  readonly readOnly: boolean;
+  readonly root: string;
+
+  private source: FileSystem;
+  private readonly host: string;
+  private readonly ttlSec: number;
+  private readonly userKey: string;
+
+  constructor(options: AppKitCachedFileSystemOptions) {
+    this.host = options.host;
+    this.source = options.source;
+    this.root = options.source.root;
+    this.readOnly = options.source.readOnly;
+    this.ttlSec = cacheTtlSec(options.ttlMs);
+    this.userKey = options.userKey;
+    this.id = `appkit-skill-cache-${hash.fnvHash(this.host, this.userKey, this.root)}`;
+  }
+
+  /** Rebind cache misses and writes to the current request's OBO filesystem. */
+  bind(source: FileSystem): void {
+    if (source.root !== this.root || source.readOnly !== this.readOnly) {
+      throw new Error("Cannot rebind a workspace skill cache to a different filesystem scope");
+    }
+    this.source = source;
+  }
+
+  async init(): Promise<void> {
+    await this.source.init();
+  }
+
+  /** Keep the shared scope alive when one request releases its workspace view. */
+  close(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  resolvePath(inputPath: string): string {
+    return this.source.resolvePath(inputPath);
+  }
+
+  async readFile(inputPath: string): Promise<Uint8Array>;
+  async readFile(
+    inputPath: string,
+    options: ReadFileOptions & { encoding: string },
+  ): Promise<string>;
+  async readFile(inputPath: string, options?: ReadFileOptions): Promise<string | Uint8Array> {
+    const value = await this.cached("readFile", inputPath, undefined, async (source) => ({
+      base64: Buffer.from(await source.readFile(inputPath)).toString("base64"),
+    }));
+    const bytes = Buffer.from(value.base64, "base64");
+    return options?.encoding ? bytes.toString(options.encoding as BufferEncoding) : bytes;
+  }
+
+  async writeFile(
+    inputPath: string,
+    content: FileContent,
+    options?: WriteFileOptions,
+  ): Promise<void> {
+    await this.mutate((source) => source.writeFile(inputPath, content, options));
+  }
+
+  async appendFile(inputPath: string, content: FileContent): Promise<void> {
+    await this.mutate((source) => source.appendFile(inputPath, content));
+  }
+
+  async deleteFile(inputPath: string, options?: RemoveOptions): Promise<void> {
+    await this.mutate((source) => source.deleteFile(inputPath, options));
+  }
+
+  async copyFile(
+    sourcePath: string,
+    destinationPath: string,
+    options?: CopyOptions,
+  ): Promise<void> {
+    await this.mutate((source) => source.copyFile(sourcePath, destinationPath, options));
+  }
+
+  async moveFile(
+    sourcePath: string,
+    destinationPath: string,
+    options?: CopyOptions,
+  ): Promise<void> {
+    await this.mutate((source) => source.moveFile(sourcePath, destinationPath, options));
+  }
+
+  async mkdir(inputPath: string, options?: MakeDirectoryOptions): Promise<void> {
+    await this.mutate((source) => source.mkdir(inputPath, options));
+  }
+
+  async rmdir(inputPath: string, options?: RemoveOptions): Promise<void> {
+    await this.mutate((source) => source.rmdir(inputPath, options));
+  }
+
+  async readdir(inputPath: string, options?: ListOptions): Promise<FileEntry[]> {
+    return this.cached("readdir", inputPath, options, (source) =>
+      source.readdir(inputPath, options),
+    );
+  }
+
+  async exists(inputPath: string): Promise<boolean> {
+    return this.cached("exists", inputPath, undefined, (source) => source.exists(inputPath));
+  }
+
+  async stat(inputPath: string): Promise<FileStat> {
+    const value = await this.cached("stat", inputPath, undefined, async (source) =>
+      serializeStat(await source.stat(inputPath)),
+    );
+    return deserializeStat(value);
+  }
+
+  /** Rotate this scope's cache generation so the next read reloads it. */
+  async invalidate(): Promise<void> {
+    const cache = await CacheManager.getInstance();
+    await cache.set(this.generationKey(cache), randomUUID(), {
+      ttl: this.ttlSec * 2,
+    });
+  }
+
+  private async cached<T>(
+    operation: string,
+    inputPath: string,
+    options: unknown,
+    load: (source: FileSystem) => Promise<T>,
+  ): Promise<T> {
+    const source = this.source;
+    const cache = await CacheManager.getInstance();
+    const generation = await this.generation(cache);
+    return cache.getOrExecute(
+      [
+        CACHE_NAMESPACE,
+        this.host,
+        this.root,
+        generation,
+        operation,
+        source.resolvePath(inputPath),
+        object.toStableKey(options ?? {}),
+      ],
+      () => load(source),
+      this.userKey,
+      { ttl: this.ttlSec },
+    );
+  }
+
+  private async generation(cache: CacheManager): Promise<string> {
+    return cache.getOrExecute(
+      [CACHE_NAMESPACE, "generation", this.host, this.root],
+      async () => randomUUID(),
+      this.userKey,
+      { ttl: this.ttlSec * 2 },
+    );
+  }
+
+  private generationKey(cache: CacheManager): string {
+    return cache.generateKey([CACHE_NAMESPACE, "generation", this.host, this.root], this.userKey);
+  }
+
+  private async mutate(run: (source: FileSystem) => Promise<void>): Promise<void> {
+    const source = this.source;
+    await run(source);
+    await this.invalidate();
+  }
+}
+
+/** Return one stable Mastra mount per resolved user, host, and source root. */
+export function cachedWorkspaceSkillMount(
+  options: AppKitCachedFileSystemOptions,
+): CachedWorkspaceSkillMount {
+  const cacheKey = object.toStableKey({
+    host: options.host,
+    readOnly: options.source.readOnly,
+    root: options.source.root,
+    userKey: options.userKey,
+  });
+  const existing = scopedMounts.get(cacheKey);
+  if (existing) {
+    existing.cache.bind(options.source);
+    scopedMounts.delete(cacheKey);
+    scopedMounts.set(cacheKey, existing);
+    return { cacheKey, filesystem: existing.adapter };
+  }
+
+  const cache = new AppKitCachedFileSystem(options);
+  const entry: ScopedMountEntry = {
+    adapter: filesystems(cache, {
+      id: cache.id,
+      readOnly: options.source.readOnly,
+    }),
+    cache,
+    host: options.host,
+    root: options.source.root,
+    userKey: options.userKey,
+  };
+  scopedMounts.set(cacheKey, entry);
+  trimScopedMounts();
+  return { cacheKey, filesystem: entry.adapter };
+}
+
+/** Invalidate one user's cached workspace skill scope. */
+export async function clearWorkspaceSkillCache(options: {
+  host: string;
+  root?: string;
+  userKey: string;
+}): Promise<void> {
+  const matches = [...scopedMounts.entries()].filter(
+    ([, entry]) =>
+      entry.host === options.host &&
+      entry.userKey === options.userKey &&
+      (options.root === undefined || entry.root === options.root),
+  );
+  await Promise.all(matches.map(([, entry]) => entry.cache.invalidate()));
+}
+
+function cacheTtlSec(ttlMs = DEFAULT_WORKSPACE_SKILL_CACHE_TTL_MS): number {
+  if (!Number.isFinite(ttlMs) || ttlMs <= 0) {
+    throw new Error("Workspace skill cache ttlMs must be a positive finite number");
+  }
+  return Math.max(1, Math.ceil(ttlMs / 1000));
+}
+
+function serializeStat(value: FileStat): CachedFileStat {
+  const { accessedAt, createdAt, modifiedAt, ...stat } = value;
+  return {
+    ...stat,
+    ...(accessedAt ? { accessedAt: accessedAt.getTime() } : {}),
+    ...(createdAt ? { createdAt: createdAt.getTime() } : {}),
+    ...(modifiedAt ? { modifiedAt: modifiedAt.getTime() } : {}),
+  };
+}
+
+function deserializeStat(value: CachedFileStat): FileStat {
+  const { accessedAt, createdAt, modifiedAt, ...stat } = value;
+  return {
+    ...stat,
+    ...(accessedAt !== undefined ? { accessedAt: new Date(accessedAt) } : {}),
+    ...(createdAt !== undefined ? { createdAt: new Date(createdAt) } : {}),
+    ...(modifiedAt !== undefined ? { modifiedAt: new Date(modifiedAt) } : {}),
+  };
+}
+
+function trimScopedMounts(): void {
+  while (scopedMounts.size > DEFAULT_MAX_SCOPED_MOUNTS) {
+    const oldest = scopedMounts.keys().next().value;
+    if (oldest === undefined) return;
+    scopedMounts.delete(oldest);
+  }
+}

package/src/workspaces.ts

@@ -34,10 +34,17 @@ import {
   type WorkspaceSandboxResolver,
 } from "@mastra/core/workspace";
 
-import { MASTRA_SCOPES_KEY, MASTRA_USER_EMAIL_KEY, MASTRA_USER_KEY, type User } from "./config.ts";
-import { scratchFilesystem, filesystems } from "./filesystems.ts";
+import {
+  MASTRA_SCOPES_KEY,
+  MASTRA_USER_EMAIL_KEY,
+  MASTRA_USER_KEY,
+  resolveUserKey,
+  type User,
+} from "./config.ts";
+import { scratchFilesystem } from "./filesystems.ts";
 import { MontySandbox } from "./monty-sandbox.ts";
 import { DatabricksSandbox, type DatabricksWorkspaceSandboxOptions } from "./sandbox.ts";
+import { cachedWorkspaceSkillMount, DEFAULT_WORKSPACE_SKILL_CACHE_TTL_MS } from "./skill-cache.ts";
 import { ASSISTANT_SHARED_SKILLS_PATH, userAssistantSkillsPath } from "./skill-paths.ts";
 
 /* ------------------------------ constants ------------------------------ */
@@ -46,6 +53,8 @@ import { ASSISTANT_SHARED_SKILLS_PATH, userAssistantSkillsPath } from "./skill-p
 const WORKSPACE_FILE_SCOPES = ["workspace", "workspace.workspace", "all-apis"] as const;
 
 const logger = log.logger("mastra/workspaces");
+const MAX_SCOPED_WORKSPACE_FILESYSTEMS = 16;
+const scopedWorkspaceFilesystems = new Map<string, WorkspaceFilesystem>();
 
 /* -------------------------------- types -------------------------------- */
 
@@ -93,6 +102,11 @@ export interface WorkspaceMountContribution {
   mounts: Record<string, WorkspaceFilesystem>;
   /** Paths within the composite namespace where `SKILL.md` files are scanned. */
   skillPaths?: string[];
+  /**
+   * Stable identity for this contribution. When every non-empty contribution
+   * supplies one, Mastra reuses the resolved source across requests.
+   */
+  cacheKey?: string;
 }
 
 /** Contributes filesystem mounts (and optional skill paths) for one request. */
@@ -139,6 +153,11 @@ export interface CreateWorkspaceOptions {
   skills?: SkillsResolver;
   /** Forwarded to Mastra when skill discovery is enabled. */
   checkSkillFileMtime?: boolean;
+  /**
+   * AppKit cache TTL for Databricks skill metadata and content.
+   * Defaults to five minutes.
+   */
+  workspaceSkillRefreshTtlMs?: number;
   /** Enable BM25 keyword search over indexed workspace content. */
   bm25?: boolean;
   /**
@@ -220,7 +239,11 @@ export function createWorkspace(options: CreateWorkspaceOptions = {}): Workspace
   const { id, name } = resolveWorkspaceIdentity(options);
   const skillFolders = resolveSkillFolders(options);
   const folderNames = Object.keys(skillFolders);
-  const resolvers = buildMountResolvers(skillFolders, options.mounts);
+  const resolvers = buildMountResolvers(
+    skillFolders,
+    options.mounts,
+    options.workspaceSkillRefreshTtlMs,
+  );
   const extraSkillPaths = options.extraSkillPaths ?? [];
   const skills =
     options.skills ??
@@ -238,6 +261,8 @@ export function createWorkspace(options: CreateWorkspaceOptions = {}): Workspace
     customMountResolvers: options.mounts?.length ?? 0,
     customSkillsResolver: Boolean(options.skills),
     checkSkillFileMtime,
+    workspaceSkillRefreshTtlMs:
+      options.workspaceSkillRefreshTtlMs ?? DEFAULT_WORKSPACE_SKILL_CACHE_TTL_MS,
     bm25,
     extraSkillPaths: extraSkillPaths.length,
     sandbox: sandbox ? sandboxName(options.sandbox) : "disabled",
@@ -372,9 +397,11 @@ function hasWorkspaceFileScope(requestContext: RequestContext | undefined): bool
 async function resolveSkillFolderMounts(
   skillFolders: Record<string, SkillFolderOptions>,
   context: WorkspaceMountContext,
+  workspaceSkillRefreshTtlMs: number,
 ): Promise<WorkspaceMountContribution> {
   const mounts: Record<string, WorkspaceFilesystem> = {};
   const skillPaths: string[] = [];
+  const cacheKeys: string[] = [];
   const requestContext = context.requestContext;
 
   if (!requestContext || !shouldMountSkillFolders(requestContext)) {
@@ -388,12 +415,20 @@ async function resolveSkillFolderMounts(
 
   const user = requestContext.get(MASTRA_USER_KEY) as User | undefined;
   const client = user?.executionContext.client as WorkspaceClient | undefined;
+  const userKey = user ? resolveUserKey(requestContext) : undefined;
+  const host = client ? (await client.config.getHost()).toString() : undefined;
 
   for (const [name, folder] of Object.entries(skillFolders)) {
-    const filesystem = await resolveSkillFolderFilesystem(name, folder, context, client);
-    if (!filesystem) continue;
+    const resolved = await resolveSkillFolderFilesystem(name, folder, context, {
+      client,
+      host,
+      userKey,
+      workspaceSkillRefreshTtlMs,
+    });
+    if (!resolved) continue;
     const mount = folder.mount ?? `/${name}`;
-    mounts[mount] = filesystem;
+    mounts[mount] = resolved.filesystem;
+    if (resolved.cacheKey) cacheKeys.push(resolved.cacheKey);
     if (folder.readable !== false) skillPaths.push(mount);
   }
 
@@ -402,7 +437,13 @@ async function resolveSkillFolderMounts(
     skillPaths,
   });
 
-  return { mounts, skillPaths };
+  return {
+    mounts,
+    skillPaths,
+    ...(cacheKeys.length === Object.keys(mounts).length && cacheKeys.length > 0
+      ? { cacheKey: object.toStableKey(cacheKeys.sort()) }
+      : {}),
+  };
 }
 
 /**
@@ -413,13 +454,19 @@ async function resolveSkillFolderFilesystem(
   name: string,
   folder: SkillFolderOptions,
   context: WorkspaceMountContext,
-  client: WorkspaceClient | undefined,
-): Promise<WorkspaceFilesystem | undefined> {
+  cache: {
+    client: WorkspaceClient | undefined;
+    host: string | undefined;
+    userKey: string | undefined;
+    workspaceSkillRefreshTtlMs: number;
+  },
+): Promise<{ cacheKey?: string; filesystem: WorkspaceFilesystem } | undefined> {
   if (folder.filesystem !== undefined) {
-    return resolveSkillFolderValue(folder.filesystem, context);
+    const filesystem = await resolveSkillFolderValue(folder.filesystem, context);
+    return filesystem ? { filesystem } : undefined;
   }
   // A path mount needs the request's OBO client to reach the workspace.
-  if (folder.path === undefined || !client) {
+  if (folder.path === undefined || !cache.client || !cache.host || !cache.userKey) {
     logger.debug("skill-folder:skipped", {
       name,
       reason: folder.path === undefined ? "no-location" : "missing-obo-client",
@@ -428,7 +475,11 @@ async function resolveSkillFolderFilesystem(
   }
   const root = stringUtils.trimToNull(await resolveSkillFolderValue(folder.path, context));
   if (!root) return undefined;
-  return databricksFilesystem(client, root, folder.writable !== true);
+  return databricksFilesystem(cache.client, root, folder.writable !== true, {
+    host: cache.host,
+    ttlMs: cache.workspaceSkillRefreshTtlMs,
+    userKey: cache.userKey,
+  });
 }
 
 /** Read a {@link SkillFolderValue}, calling it when it is a per-request resolver. */
@@ -464,11 +515,14 @@ function resolveWorkspaceIdentity(options: CreateWorkspaceOptions): {
 function buildMountResolvers(
   skillFolders: Record<string, SkillFolderOptions>,
   mounts: WorkspaceMountResolver[] | undefined,
+  workspaceSkillRefreshTtlMs = DEFAULT_WORKSPACE_SKILL_CACHE_TTL_MS,
 ): WorkspaceMountResolver[] {
   const resolvers: WorkspaceMountResolver[] = [];
   const folderCount = Object.keys(skillFolders).length;
   if (folderCount > 0) {
-    resolvers.push((context) => resolveSkillFolderMounts(skillFolders, context));
+    resolvers.push((context) =>
+      resolveSkillFolderMounts(skillFolders, context, workspaceSkillRefreshTtlMs),
+    );
   }
   if (mounts?.length) {
     resolvers.push(...mounts);
@@ -506,18 +560,23 @@ async function databricksFilesystem(
   client: WorkspaceClient,
   root: string,
   readOnly: boolean = true,
-): Promise<WorkspaceFilesystem> {
+  cache: { host: string; ttlMs: number; userKey: string },
+): Promise<{ cacheKey: string; filesystem: WorkspaceFilesystem } | undefined> {
   const fs = new DatabricksFileSystem({
     client: workspaceClient.toLegacyWorkspaceClient(client),
     root,
     readOnly,
     createRoot: !readOnly,
   });
+  const cached = cachedWorkspaceSkillMount({
+    host: cache.host,
+    source: fs,
+    ttlMs: cache.ttlMs,
+    userKey: cache.userKey,
+  });
   try {
-    await fs.init();
-    if (await fs.exists(".")) {
-      return filesystems(fs, { readOnly });
-    }
+    await cached.filesystem.init();
+    if (await cached.filesystem.exists(".")) return cached;
   } catch (err) {
     logger.debug("databricks-mount:scratch-fallback", {
       root,
@@ -525,7 +584,7 @@ async function databricksFilesystem(
       error: errorUtils.errorMessage(err),
     });
   }
-  return scratchFilesystem();
+  return undefined;
 }
 
 /**
@@ -538,6 +597,8 @@ async function resolveWorkspaceContribution(
 ): Promise<WorkspaceMountContribution> {
   const mounts: Record<string, WorkspaceFilesystem> = {};
   const skillPaths: string[] = [];
+  const cacheKeys: string[] = [];
+  let cacheable = true;
 
   for (const [index, resolver] of resolvers.entries()) {
     const contribution = await resolver(context);
@@ -547,6 +608,10 @@ async function resolveWorkspaceContribution(
       skillPaths: contribution.skillPaths ?? [],
     });
     Object.assign(mounts, contribution.mounts);
+    if (Object.keys(contribution.mounts).length > 0) {
+      if (contribution.cacheKey) cacheKeys.push(contribution.cacheKey);
+      else cacheable = false;
+    }
     if (contribution.skillPaths?.length) {
       skillPaths.push(...contribution.skillPaths);
     }
@@ -557,7 +622,13 @@ async function resolveWorkspaceContribution(
     skillPaths,
   });
 
-  return { mounts, skillPaths };
+  return {
+    mounts,
+    skillPaths,
+    ...(cacheable && cacheKeys.length > 0
+      ? { cacheKey: object.toStableKey(cacheKeys.sort()) }
+      : {}),
+  };
 }
 
 /**
@@ -570,7 +641,7 @@ async function resolveWorkspaceFilesystem(
   resolvers: WorkspaceMountResolver[],
   context: WorkspaceMountContext,
 ): Promise<WorkspaceFilesystem> {
-  const { mounts } = await resolveWorkspaceContribution(resolvers, context);
+  const { cacheKey, mounts } = await resolveWorkspaceContribution(resolvers, context);
   const mountKeys = Object.keys(mounts);
   if (mountKeys.length === 0) {
     logger.debug("filesystem:scratch", {
@@ -579,7 +650,21 @@ async function resolveWorkspaceFilesystem(
     return scratchFilesystem();
   }
   logger.debug("filesystem:composite", { mountKeys });
-  return new CompositeFilesystem({ mounts });
+  if (!cacheKey) return new CompositeFilesystem({ mounts });
+  const existing = scopedWorkspaceFilesystems.get(cacheKey);
+  if (existing) {
+    scopedWorkspaceFilesystems.delete(cacheKey);
+    scopedWorkspaceFilesystems.set(cacheKey, existing);
+    return existing;
+  }
+  const filesystem = new CompositeFilesystem({ mounts });
+  scopedWorkspaceFilesystems.set(cacheKey, filesystem);
+  while (scopedWorkspaceFilesystems.size > MAX_SCOPED_WORKSPACE_FILESYSTEMS) {
+    const oldest = scopedWorkspaceFilesystems.keys().next().value;
+    if (oldest === undefined) break;
+    scopedWorkspaceFilesystems.delete(oldest);
+  }
+  return filesystem;
 }
 
 /**