@dbx-tools/appkit-mastra 0.6.8 → 0.6.10

package/src/plugin.ts

@@ -77,6 +77,7 @@ import type { Pool } from "pg";
 import { buildAgents, FALLBACK_AGENT_ID, type BuiltAgents } from "./agents.ts";
 import { fetchChart } from "./chart.ts";
 import { attributedUserId, MASTRA_CONFIG_SCHEMA, type MastraPluginConfig } from "./config.ts";
+import { provisionDatabricksAITools } from "./databricks-aitools.ts";
 import {
   chartFetchDefaults,
   feedbackWriteDefaults,
@@ -1011,12 +1012,24 @@ export class MastraPlugin extends Plugin<MastraPluginConfig> {
       });
     }
 
+    // Fold Databricks AI Tools skills (`databricks aitools`) in as extra local
+    // skill scan paths: reuse the CLI's installed tree when present, else shell
+    // out to the CLI to materialize a curated set.
+    const aiTools = await provisionDatabricksAITools(this.config.databricksAITools);
+    if (aiTools.localSkillPaths.length > 0) {
+      this.logger.info("databricks ai tools provisioned", {
+        source: aiTools.source,
+        localSkillPaths: aiTools.localSkillPaths.length,
+      });
+    }
+    const extraSkillPaths = [...provisioned.localSkillPaths, ...aiTools.localSkillPaths];
+
     this.built = await buildAgents({
       config: this.config,
       context: this.context,
       memoryBuilder,
       log: this.logger,
-      extraSkillPaths: provisioned.localSkillPaths,
+      extraSkillPaths,
     });
 
     // `mastra.server.apiRoutes` is only honored by Mastra's standalone

package/src/remote-skills.ts

@@ -27,23 +27,24 @@
  * discovered by the built-in Assistant-skills mount. Pass `userEmail` (or an
  * explicit `databricksBasePath`) to target `/Users/<email>/.assistant/skills`
  * instead. When no Databricks client is resolvable at startup, the tree is
- * written to a local temp dir and returned as an extra local skill path for
- * the current process.
+ * written under {@link localFS.tmpFS} and returned as an extra local skill path
+ * for the current process.
  *
  * @module
  */
 
-import { mkdtemp, cp, mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
+import { mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
 import { createRequire } from "node:module";
-import { tmpdir } from "node:os";
 import { join, posix } from "node:path";
+import type { WorkspaceClient } from "@databricks/sdk-experimental";
 import { appkit } from "@dbx-tools/appkit";
 import type { WorkspaceClientLike } from "@dbx-tools/appkit";
-import { spawn } from "@dbx-tools/core";
-import { findFiles } from "@dbx-tools/path";
-import { error, log, net, string } from "@dbx-tools/shared-core";
-
-import { DatabricksWorkspaceFilesystem } from "./filesystems.ts";
+import { exec } from "@dbx-tools/core";
+import { DatabricksFileSystem } from "@dbx-tools/databricks";
+import { localFS, type LocalFileSystem } from "@dbx-tools/fs";
+import { find } from "@dbx-tools/path";
+import { error, hash, log, net, string } from "@dbx-tools/shared-core";
+import type { FileSystem } from "@dbx-tools/shared-fs";
 
 const logger = log.logger("mastra/remote-skills");
 
@@ -64,6 +65,9 @@ const SKILLS_CLI_LAYOUT = [".agents", "skills"] as const;
 /** Cap on a direct-fetch download body, matching the `skills` CLI default. */
 const DEFAULT_MAX_DOWNLOAD_BYTES = 10 * 1024 * 1024;
 
+/** Stable temp directory holding one rebuilt skill tree per remote source. */
+const LOCAL_SKILLS_DIR = "mastra-local-skills";
+
 /* -------------------------------- types -------------------------------- */
 
 /**
@@ -169,8 +173,8 @@ function toSourceOptions(source: RemoteSkillSource): RemoteSkillSourceOptions {
  * Materialize every configured remote skill source at app startup.
  *
  * Writes each resolved `SKILL.md` tree to the Databricks user Assistant skills
- * folder when a writable workspace is available, else to a local temp dir
- * returned in {@link ProvisionedRemoteSkills.localSkillPaths}.
+ * folder when a writable workspace is available, else under
+ * {@link localFS.tmpFS} returned in {@link ProvisionedRemoteSkills.localSkillPaths}.
  */
 export async function provisionRemoteSkills(
   option: RemoteSkillsOption | undefined,
@@ -183,12 +187,17 @@ export async function provisionRemoteSkills(
   const client = options.client ?? appkit.tryGetExecutionContext()?.client;
   const databricksBasePath = resolveDatabricksBasePath(options, client);
   const destination = databricksBasePath
-    ? new DatabricksWorkspaceFilesystem({ client, basePath: databricksBasePath, readOnly: false })
+    ? new DatabricksFileSystem({
+        client: client as WorkspaceClient,
+        root: databricksBasePath,
+        readOnly: false,
+        createRoot: true,
+      })
     : undefined;
 
   const localSkillPaths: string[] = [];
   const skillNames: string[] = [];
-  let staging: string | undefined;
+  let staging: LocalFileSystem | undefined;
 
   try {
     const sources = Array.isArray(options.sources) ? options.sources : [options.sources];
@@ -196,17 +205,17 @@ export async function provisionRemoteSkills(
       const sourceOptions = toSourceOptions(entry);
       const failOnError = sourceOptions.failOnError ?? failDefault;
       try {
-        staging ??= await mkdtemp(join(tmpdir(), "mastra-remote-skills-"));
-        const stagedDir = await stageSource(sourceOptions, staging, options);
+        staging ??= await initializedScratch("mastra-remote-skills");
+        const stagedDir = await stageSource(sourceOptions, staging.root, options);
         const staged = await collectSkillDirs(stagedDir);
         if (staged.length === 0) {
           throw new Error(`no SKILL.md found for source "${sourceOptions.source}"`);
         }
         if (destination && databricksBasePath) {
-          await uploadSkillDirs(destination, staged);
+          await copySkillDirs(destination, staged);
           skillNames.push(...staged.map((dir) => dir.name));
         } else {
-          const localDir = await persistLocally(staged);
+          const localDir = await persistLocally(sourceOptions.source, staged);
           localSkillPaths.push(localDir);
           skillNames.push(...staged.map((dir) => dir.name));
         }
@@ -229,7 +238,9 @@ export async function provisionRemoteSkills(
       }
     }
   } finally {
-    if (staging) await rm(staging, { recursive: true, force: true }).catch(() => undefined);
+    if (staging) {
+      await rm(staging.root, { recursive: true, force: true }).catch(() => undefined);
+    }
   }
 
   return { localSkillPaths, databricksBasePath, skillNames };
@@ -250,15 +261,16 @@ function resolveDatabricksBasePath(
 }
 
 /**
- * Stage one source into a fresh dir under `staging`, preferring the optional
+ * Stage one source into a fresh dir under `stagingRoot`, preferring the optional
  * `skills` CLI and falling back to a direct fetch.
  */
 async function stageSource(
   sourceOptions: RemoteSkillSourceOptions,
-  staging: string,
+  stagingRoot: string,
   options: ProvisionRemoteSkillsOptions,
 ): Promise<string> {
-  const target = await mkdtemp(join(staging, "src-"));
+  const target = join(stagingRoot, `src-${hash.id()}`);
+  await mkdir(target, { recursive: true });
   const viaCli = await stageViaSkillsCli(sourceOptions, target);
   if (viaCli) return viaCli;
   return stageViaFetch(sourceOptions, target, options);
@@ -290,7 +302,7 @@ async function stageViaSkillsCli(
   }
   if (string.parseList(sourceOptions.skills).length === 0) args.push("--skill", "*");
 
-  const result = await spawn(cli.command, args, {
+  const result = await exec.spawn(cli.command, args, {
     cwd: target,
     stdout: "capture",
     stderr: "capture",
@@ -389,28 +401,47 @@ async function collectSkillDirs(root: string): Promise<StagedSkillDir[]> {
   return dirs;
 }
 
-/** Upload each staged skill directory into the Databricks destination tree. */
-async function uploadSkillDirs(
-  destination: DatabricksWorkspaceFilesystem,
-  dirs: StagedSkillDir[],
-): Promise<void> {
-  await destination.init?.();
+/**
+ * Copy every staged skill directory into {@link destination}, preserving the
+ * `<skill>/<relative>` layout. The destination is just a {@link FileSystem},
+ * so the Databricks Assistant tree and the local fallback share one copier.
+ */
+async function copySkillDirs(destination: FileSystem, dirs: StagedSkillDir[]): Promise<void> {
+  await destination.init();
   for (const dir of dirs) {
-    for (const relative of findFiles("**/*", { cwd: dir.absolutePath, nodir: true })) {
+    for (const relative of find.findFiles("**/*", { cwd: dir.absolutePath, nodir: true })) {
       const buffer = await readFile(join(dir.absolutePath, relative));
-      const remotePath = posix.join("/", dir.name, relative.split(/[\\/]/).join("/"));
-      await destination.writeFile(remotePath, buffer, { overwrite: true });
+      const skillPath = posix.join(dir.name, relative.split(/[\\/]/).join("/"));
+      await destination.writeFile(skillPath, buffer, { overwrite: true });
     }
   }
 }
 
-/** Copy staged skill dirs into a persistent local temp dir; return its path. */
-async function persistLocally(dirs: StagedSkillDir[]): Promise<string> {
-  const base = await mkdtemp(join(tmpdir(), "mastra-local-skills-"));
-  for (const dir of dirs) {
-    await cp(dir.absolutePath, join(base, dir.name), { recursive: true });
-  }
-  return base;
+/**
+ * Copy staged skill dirs into a STABLE local tree keyed by the source, and
+ * return its root path.
+ *
+ * A given source resolves to the same content on almost every boot, so each
+ * one owns a directory named for its {@link hash.fnvHash} rather than leaving
+ * a fresh scratch dir behind per restart. Keying on the source (not the
+ * content) is what keeps two different sources from overwriting each other.
+ */
+async function persistLocally(source: string, dirs: StagedSkillDir[]): Promise<string> {
+  const stable = await localFS.rebuildFS(`${LOCAL_SKILLS_DIR}/${hash.fnvHash(source)}`, (scratch) =>
+    copySkillDirs(scratch, dirs),
+  );
+  return stable.root;
+}
+
+/**
+ * A scratch filesystem whose root exists on disk, for the paths that hand a
+ * real directory to `node:fs` or a child process rather than going through
+ * the {@link FileSystem} API.
+ */
+async function initializedScratch(prefix: string): Promise<LocalFileSystem> {
+  const scratch = localFS.scratchFS(prefix);
+  await scratch.init();
+  return scratch;
 }
 
 /** Best-effort existence check. */

package/src/workspaces.ts

@@ -7,10 +7,15 @@
  * contributions merge extra filesystems and skill scan roots; built-in
  * Assistant skill trees are toggled with `assistantSkills` (on by default).
  *
+ * Databricks mounts use `@dbx-tools/databricks` {@link DatabricksFileSystem}
+ * wrapped by {@link filesystems}; missing roots fall back to
+ * {@link scratchFilesystem}.
+ *
  * @module
  */
 
 import type { WorkspaceClient } from "@databricks/sdk-experimental";
+import { DatabricksFileSystem } from "@dbx-tools/databricks";
 import { log, string, token } from "@dbx-tools/shared-core";
 import type { RequestContext } from "@mastra/core/request-context";
 import {
@@ -22,7 +27,7 @@ import {
 } from "@mastra/core/workspace";
 
 import { MASTRA_SCOPES_KEY, MASTRA_USER_EMAIL_KEY, MASTRA_USER_KEY, type User } from "./config.ts";
-import { DatabricksWorkspaceFilesystem, emptyFilesystem } from "./filesystems.ts";
+import { scratchFilesystem, filesystems } from "./filesystems.ts";
 
 /* ------------------------------ constants ------------------------------ */
 
@@ -175,8 +180,10 @@ function hasWorkspaceFileScope(requestContext: RequestContext | undefined): bool
  * set. Returns empty mounts when the OBO user or client is missing.
  * Mastra owns filesystem initialization.
  */
-function resolveAssistantSkillsMounts(context: WorkspaceMountContext): WorkspaceMountContribution {
-  const mounts: Record<string, DatabricksWorkspaceFilesystem> = {};
+async function resolveAssistantSkillsMounts(
+  context: WorkspaceMountContext,
+): Promise<WorkspaceMountContribution> {
+  const mounts: Record<string, WorkspaceFilesystem> = {};
   const requestContext = context.requestContext;
 
   if (!shouldMountAssistantSkills(requestContext)) {
@@ -189,7 +196,7 @@ function resolveAssistantSkillsMounts(context: WorkspaceMountContext): Workspace
   }
 
   const user = requestContext!.get(MASTRA_USER_KEY) as User | undefined;
-  const client = user?.executionContext.client;
+  const client = user?.executionContext.client as WorkspaceClient | undefined;
   if (!client) {
     logger.debug("assistant-skills:skipped", {
       reason: "missing-obo-client",
@@ -198,14 +205,14 @@ function resolveAssistantSkillsMounts(context: WorkspaceMountContext): Workspace
     return { mounts, skillPaths: [] };
   }
 
-  mounts[ASSISTANT_WORKSPACE_SKILLS_MOUNT] = databricksFilesystem(
+  mounts[ASSISTANT_WORKSPACE_SKILLS_MOUNT] = await databricksFilesystem(
     client,
     ASSISTANT_SHARED_SKILLS_PATH,
   );
 
   const email = resolveScopedEmail(requestContext);
   if (email) {
-    mounts[ASSISTANT_USER_SKILLS_MOUNT] = databricksFilesystem(
+    mounts[ASSISTANT_USER_SKILLS_MOUNT] = await databricksFilesystem(
       client,
       userAssistantSkillsPath(email),
       false,
@@ -281,17 +288,35 @@ function resolveScopedEmail(requestContext: RequestContext | undefined): string
   return email?.trim() || undefined;
 }
 
-/** Construct a read-only {@link DatabricksWorkspaceFilesystem} for `basePath`. */
-function databricksFilesystem(
+/**
+ * Wrap a {@link DatabricksFileSystem} as a Mastra filesystem. When the root is
+ * missing (and {@link readOnly} so we will not create it), fall back to
+ * {@link scratchFilesystem} so skill scans still have a writable local mount.
+ */
+async function databricksFilesystem(
   client: WorkspaceClient,
-  basePath: string,
+  root: string,
   readOnly: boolean = true,
-): DatabricksWorkspaceFilesystem {
-  return new DatabricksWorkspaceFilesystem({
+): Promise<WorkspaceFilesystem> {
+  const fs = new DatabricksFileSystem({
     client,
-    basePath,
+    root,
     readOnly,
+    createRoot: !readOnly,
   });
+  try {
+    await fs.init();
+    if (await fs.exists(".")) {
+      return filesystems(fs, { readOnly });
+    }
+  } catch (err) {
+    logger.debug("databricks-mount:scratch-fallback", {
+      root,
+      readOnly,
+      error: err instanceof Error ? err.message : String(err),
+    });
+  }
+  return scratchFilesystem();
 }
 
 /**
@@ -329,8 +354,8 @@ async function resolveWorkspaceContribution(
 /**
  * Dynamic filesystem resolver passed to Mastra {@link Workspace}.
  *
- * Returns a {@link CompositeFilesystem} when any mount resolved; otherwise
- * {@link emptyFilesystem}.
+ * Returns a {@link CompositeFilesystem} when any mount resolved; otherwise a
+ * fresh {@link scratchFilesystem} so Mastra always has a writable local root.
  */
 async function resolveWorkspaceFilesystem(
   resolvers: WorkspaceMountResolver[],
@@ -339,10 +364,10 @@ async function resolveWorkspaceFilesystem(
   const { mounts } = await resolveWorkspaceContribution(resolvers, context);
   const mountKeys = Object.keys(mounts);
   if (mountKeys.length === 0) {
-    logger.debug("filesystem:empty", {
+    logger.debug("filesystem:scratch", {
       hasRequestContext: Boolean(context.requestContext),
     });
-    return emptyFilesystem();
+    return scratchFilesystem();
   }
   logger.debug("filesystem:composite", { mountKeys });
   return new CompositeFilesystem({ mounts });