@dbx-tools/appkit-mastra 0.9.118 → 0.9.119
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -76
- package/index.ts +3 -6
- package/lib/index.d.ts +3 -6
- package/lib/index.js +3 -5
- package/lib/src/agents.js +2 -2
- package/lib/src/config.d.ts +18 -15
- package/lib/src/config.js +4 -4
- package/lib/src/defaults.js +4 -4
- package/lib/src/genie.js +8 -8
- package/lib/src/plugin.js +2 -2
- package/lib/src/remote-skills.d.ts +32 -28
- package/lib/src/remote-skills.js +103 -29
- package/lib/src/workspaces.d.ts +4 -7
- package/lib/src/workspaces.js +55 -52
- package/lib/tsconfig.tsbuildinfo +1 -1
- package/package.json +16 -24
- package/src/agents.ts +1 -1
- package/src/config.ts +25 -18
- package/src/defaults.ts +3 -3
- package/src/genie.ts +9 -9
- package/src/plugin.ts +1 -1
- package/src/remote-skills.ts +158 -38
- package/src/workspaces.ts +58 -74
- package/lib/src/sandbox.d.ts +0 -107
- package/lib/src/sandbox.js +0 -461
- package/src/sandbox.ts +0 -556
package/package.json
CHANGED
|
@@ -34,20 +34,21 @@
|
|
|
34
34
|
},
|
|
35
35
|
"dependencies": {
|
|
36
36
|
"@databricks/appkit": "0.81.0",
|
|
37
|
-
"@dbx-tools/appkit": "^0.9.
|
|
38
|
-
"@dbx-tools/
|
|
39
|
-
"@dbx-tools/
|
|
40
|
-
"@dbx-tools/
|
|
41
|
-
"@dbx-tools/
|
|
42
|
-
"@dbx-tools/
|
|
43
|
-
"@dbx-tools/model
|
|
44
|
-
"@dbx-tools/
|
|
45
|
-
"@dbx-tools/
|
|
46
|
-
"@dbx-tools/
|
|
47
|
-
"@dbx-tools/shared-
|
|
48
|
-
"@dbx-tools/shared-
|
|
49
|
-
"@dbx-tools/shared-
|
|
50
|
-
"@dbx-tools/shared-
|
|
37
|
+
"@dbx-tools/appkit": "^0.9.119",
|
|
38
|
+
"@dbx-tools/cache": "^0.9.119",
|
|
39
|
+
"@dbx-tools/core": "^0.9.119",
|
|
40
|
+
"@dbx-tools/databricks": "^0.9.119",
|
|
41
|
+
"@dbx-tools/fs": "^0.9.119",
|
|
42
|
+
"@dbx-tools/genie": "^0.9.119",
|
|
43
|
+
"@dbx-tools/model": "^0.9.119",
|
|
44
|
+
"@dbx-tools/model-protocol": "^0.9.119",
|
|
45
|
+
"@dbx-tools/path": "^0.9.119",
|
|
46
|
+
"@dbx-tools/postgres": "^0.9.119",
|
|
47
|
+
"@dbx-tools/shared-core": "^0.9.119",
|
|
48
|
+
"@dbx-tools/shared-fs": "^0.9.119",
|
|
49
|
+
"@dbx-tools/shared-genie": "^0.9.119",
|
|
50
|
+
"@dbx-tools/shared-mastra": "^0.9.119",
|
|
51
|
+
"@dbx-tools/shared-model": "^0.9.119",
|
|
51
52
|
"@mastra/ai-sdk": "1.10.5",
|
|
52
53
|
"@mastra/core": "1.71.0",
|
|
53
54
|
"@mastra/express": "1.5.15",
|
|
@@ -68,7 +69,6 @@
|
|
|
68
69
|
"ai": "^7.0.0",
|
|
69
70
|
"express": "^5.1.0",
|
|
70
71
|
"fuse.js": "^7.4.2",
|
|
71
|
-
"lru-cache": "11.5.3",
|
|
72
72
|
"pg": "^8.22.0",
|
|
73
73
|
"yaml": "^2.9.0",
|
|
74
74
|
"zod": "4.3.6"
|
|
@@ -164,10 +164,6 @@
|
|
|
164
164
|
"types": "./lib/src/rest.d.ts",
|
|
165
165
|
"default": "./lib/src/rest.js"
|
|
166
166
|
},
|
|
167
|
-
"./sandbox": {
|
|
168
|
-
"types": "./lib/src/sandbox.d.ts",
|
|
169
|
-
"default": "./lib/src/sandbox.js"
|
|
170
|
-
},
|
|
171
167
|
"./server": {
|
|
172
168
|
"types": "./lib/src/server.d.ts",
|
|
173
169
|
"default": "./lib/src/server.js"
|
|
@@ -223,7 +219,7 @@
|
|
|
223
219
|
"./package.json": "./package.json"
|
|
224
220
|
}
|
|
225
221
|
},
|
|
226
|
-
"version": "0.9.
|
|
222
|
+
"version": "0.9.119",
|
|
227
223
|
"types": "./lib/index.d.ts",
|
|
228
224
|
"dbxToolsConfig": {
|
|
229
225
|
"tags": [
|
|
@@ -316,10 +312,6 @@
|
|
|
316
312
|
"types": "./lib/src/rest.d.ts",
|
|
317
313
|
"default": "./lib/src/rest.js"
|
|
318
314
|
},
|
|
319
|
-
"./sandbox": {
|
|
320
|
-
"types": "./lib/src/sandbox.d.ts",
|
|
321
|
-
"default": "./lib/src/sandbox.js"
|
|
322
|
-
},
|
|
323
315
|
"./server": {
|
|
324
316
|
"types": "./lib/src/server.d.ts",
|
|
325
317
|
"default": "./lib/src/server.js"
|
package/src/agents.ts
CHANGED
|
@@ -767,7 +767,7 @@ export async function buildAgents(opts: {
|
|
|
767
767
|
fallbackToTmp: config.workspaceFallbackToTmp,
|
|
768
768
|
...(config.workspaceTools !== undefined ? { tools: config.workspaceTools } : {}),
|
|
769
769
|
pluginContext: context,
|
|
770
|
-
sandbox: config.sandbox
|
|
770
|
+
sandbox: config.sandbox,
|
|
771
771
|
});
|
|
772
772
|
markDefaultWorkspace(workspace);
|
|
773
773
|
} else if (workspace) {
|
package/src/config.ts
CHANGED
|
@@ -20,14 +20,17 @@ import type { BrandContext } from "@dbx-tools/shared-core";
|
|
|
20
20
|
import type { AgentConfig } from "@mastra/core/agent";
|
|
21
21
|
import type { RequestContext } from "@mastra/core/request-context";
|
|
22
22
|
import { MASTRA_RESOURCE_ID_KEY, MASTRA_THREAD_ID_KEY } from "@mastra/core/request-context";
|
|
23
|
-
import type {
|
|
23
|
+
import type {
|
|
24
|
+
WorkspaceSandbox,
|
|
25
|
+
WorkspaceSandboxResolver,
|
|
26
|
+
WorkspaceToolsConfig,
|
|
27
|
+
} from "@mastra/core/workspace";
|
|
24
28
|
import type { PgVectorConfig, PostgresStoreConfig } from "@mastra/pg";
|
|
25
29
|
|
|
26
30
|
import type { AnyMastraAgentDefinition, MastraTools } from "./agents.ts";
|
|
27
31
|
import type { GenieSpacesConfig } from "./genie.ts";
|
|
28
32
|
import { IDENTITY_MODES, type MastraIdentityMode } from "./identity.ts";
|
|
29
|
-
import type { RemoteSkillsOption } from "./remote-skills.ts";
|
|
30
|
-
import type { DatabricksWorkspaceSandboxOptions } from "./sandbox.ts";
|
|
33
|
+
import type { LocalSkillMount, RemoteSkillsOption } from "./remote-skills.ts";
|
|
31
34
|
|
|
32
35
|
/**
|
|
33
36
|
* `RequestContext` key under which {@link MastraServer} stores the
|
|
@@ -226,20 +229,22 @@ export interface MastraPluginConfig extends BasePluginConfig {
|
|
|
226
229
|
*/
|
|
227
230
|
memory?: boolean | MastraMemoryConfig;
|
|
228
231
|
/**
|
|
229
|
-
* Expose executable tools
|
|
230
|
-
* Code Mode. Defaults on whenever the
|
|
231
|
-
* selects Mastra's isolated-vm transport once during startup and
|
|
232
|
-
* to Mastra QuickJS when the native transport is unavailable.
|
|
232
|
+
* Expose executable tools available to the current request through native
|
|
233
|
+
* Mastra Code Mode. Defaults on whenever the request has executable tools.
|
|
234
|
+
* The plugin selects Mastra's isolated-vm transport once during startup and
|
|
235
|
+
* falls back to Mastra QuickJS when the native transport is unavailable.
|
|
236
|
+
* These transports do not execute through the workspace command sandbox;
|
|
237
|
+
* SSH is used only when an exposed SSH-backed tool is called. Set `false` to
|
|
233
238
|
* opt out.
|
|
234
239
|
*/
|
|
235
240
|
codeMode?: boolean | MastraCodeModeConfig;
|
|
236
241
|
/**
|
|
237
242
|
* Sandbox for auto-created agent workspaces. Defaults to Monty. `false`
|
|
238
|
-
* disables command execution; `true
|
|
239
|
-
*
|
|
240
|
-
* `workspace` keeps that workspace and its sandbox.
|
|
243
|
+
* disables command execution; `true` or `"monty"` selects Monty; an explicit
|
|
244
|
+
* Mastra sandbox or resolver supplies a remote transport. An agent with an
|
|
245
|
+
* explicit custom `workspace` keeps that workspace and its sandbox.
|
|
241
246
|
*/
|
|
242
|
-
sandbox?: boolean | "
|
|
247
|
+
sandbox?: boolean | "monty" | WorkspaceSandbox | WorkspaceSandboxResolver;
|
|
243
248
|
/** Add user-scoped `/tmp` only when no configured workspace path mounts writable. */
|
|
244
249
|
workspaceFallbackToTmp?: boolean;
|
|
245
250
|
/** Native Mastra workspace tool availability, approval, and hook configuration. */
|
|
@@ -634,10 +639,10 @@ export interface MastraPluginConfig extends BasePluginConfig {
|
|
|
634
639
|
* without it, a URL source is fetched directly. A source that resolves
|
|
635
640
|
* through neither fails startup unless `failOnError: false`.
|
|
636
641
|
*
|
|
637
|
-
*
|
|
638
|
-
*
|
|
639
|
-
*
|
|
640
|
-
*
|
|
642
|
+
* AI Tools is cached daily with `@dbx-tools/cache` and served to every
|
|
643
|
+
* auto-created workspace from the shared `/.databricks/skills` memory mount.
|
|
644
|
+
* Other sources are written to the Databricks user's Assistant skills folder
|
|
645
|
+
* when writable, with a read-only local-memory fallback.
|
|
641
646
|
*
|
|
642
647
|
* @example
|
|
643
648
|
* ```ts
|
|
@@ -651,6 +656,8 @@ export interface MastraPluginConfig extends BasePluginConfig {
|
|
|
651
656
|
* ```
|
|
652
657
|
*/
|
|
653
658
|
remoteSkills?: RemoteSkillsOption;
|
|
659
|
+
/** Read-only local skill trees mounted into every default agent workspace. */
|
|
660
|
+
localSkillMounts?: LocalSkillMount[];
|
|
654
661
|
}
|
|
655
662
|
|
|
656
663
|
/**
|
|
@@ -677,7 +684,7 @@ export const MASTRA_CONFIG_SCHEMA: ConfigSchema = {
|
|
|
677
684
|
remoteSkills: {
|
|
678
685
|
type: ["string", "array", "object"],
|
|
679
686
|
description:
|
|
680
|
-
'Remote Agent-Skill sources provisioned at startup: "aitools" for Databricks\' own skill set (
|
|
687
|
+
'Remote Agent-Skill sources provisioned at startup: "aitools" for Databricks\' own skill set (daily @dbx-tools/cache snapshot served from the shared in-memory /.databricks/skills mount), a GitHub owner/repo, a git/GitLab URL, or a direct SKILL.md/archive URL. Non-aitools sources prefer the optional `skills` npm CLI, else a direct fetch; fails startup on error unless failOnError:false. Other sources write to the Databricks user Assistant skills folder or a read-only local-memory fallback.',
|
|
681
688
|
},
|
|
682
689
|
storage: {
|
|
683
690
|
type: ["boolean", "object"],
|
|
@@ -692,12 +699,12 @@ export const MASTRA_CONFIG_SCHEMA: ConfigSchema = {
|
|
|
692
699
|
codeMode: {
|
|
693
700
|
type: ["boolean", "object"],
|
|
694
701
|
description:
|
|
695
|
-
"Expose all executable tools available to the current request
|
|
702
|
+
"Expose all executable tools available to the current request through native Mastra Code Mode for TypeScript batching, aggregation, and arithmetic. The plugin uses Mastra isolated-vm with the existing QuickJS fallback; neither transport runs through the workspace command sandbox, so SSH is used only when an SSH-backed tool is called. False disables it.",
|
|
696
703
|
},
|
|
697
704
|
sandbox: {
|
|
698
705
|
type: ["boolean", "string", "object"],
|
|
699
706
|
description:
|
|
700
|
-
'Command sandbox for auto-created agent workspaces. Defaults to Monty. False disables command execution; true
|
|
707
|
+
'Command sandbox for auto-created agent workspaces. Defaults to Monty. False disables command execution; true or "monty" selects Monty; an explicit sandbox or resolver supplies another transport.',
|
|
701
708
|
},
|
|
702
709
|
workspaceFallbackToTmp: {
|
|
703
710
|
type: "boolean",
|
package/src/defaults.ts
CHANGED
|
@@ -53,9 +53,9 @@ const STATEMENT_CACHE_TTL_SEC = 5 * 60;
|
|
|
53
53
|
export const modelCatalogueDefaults: ExecuteSettings = {
|
|
54
54
|
default: {
|
|
55
55
|
// Cache disabled here because `listServingEndpoints` already memoizes the
|
|
56
|
-
// catalogue
|
|
57
|
-
// `config.modelCacheTtlMs`.
|
|
58
|
-
//
|
|
56
|
+
// catalogue in the shared model cache, keyed by workspace host and trusted
|
|
57
|
+
// credential identity with `config.modelCacheTtlMs`. A second interceptor
|
|
58
|
+
// cache would duplicate it once per user.
|
|
59
59
|
cache: { enabled: false },
|
|
60
60
|
// Retry enabled: listing endpoints is an idempotent GET, and the listing
|
|
61
61
|
// is on the chat UI's load path, so a transient 5xx should not blank the
|
package/src/genie.ts
CHANGED
|
@@ -45,6 +45,7 @@ import {
|
|
|
45
45
|
} from "@databricks/appkit";
|
|
46
46
|
import type { WorkspaceClient } from "@databricks/appkit";
|
|
47
47
|
import { pluginRegistry } from "@dbx-tools/appkit";
|
|
48
|
+
import { createCache } from "@dbx-tools/cache/cache";
|
|
48
49
|
import { chat, space as genieSpace } from "@dbx-tools/genie";
|
|
49
50
|
import { errorUtils, log, stringUtils } from "@dbx-tools/shared-core";
|
|
50
51
|
import { genieModel, type GenieMessage } from "@dbx-tools/shared-genie";
|
|
@@ -52,7 +53,6 @@ import type { StartedEvent } from "@dbx-tools/shared-mastra";
|
|
|
52
53
|
import type { RequestContext } from "@mastra/core/request-context";
|
|
53
54
|
import { MASTRA_THREAD_ID_KEY } from "@mastra/core/request-context";
|
|
54
55
|
import { createTool, type ToolExecutionContext } from "@mastra/core/tools";
|
|
55
|
-
import { LRUCache } from "lru-cache";
|
|
56
56
|
import { z } from "zod";
|
|
57
57
|
|
|
58
58
|
import type { MastraTools } from "./agents.ts";
|
|
@@ -1091,19 +1091,19 @@ const SUGGESTION_LIMIT = 6;
|
|
|
1091
1091
|
const SUGGESTION_CACHE_TTL_MS = 10 * 60_000;
|
|
1092
1092
|
|
|
1093
1093
|
/** Bounded process-local cache of parsed sample questions by Genie space id. */
|
|
1094
|
-
const _suggestionCache =
|
|
1095
|
-
|
|
1094
|
+
const _suggestionCache = createCache({
|
|
1095
|
+
maxEntries: 128,
|
|
1096
1096
|
ttl: SUGGESTION_CACHE_TTL_MS,
|
|
1097
1097
|
});
|
|
1098
1098
|
|
|
1099
1099
|
/** Read a space's cached questions, or `undefined` on miss / expiry. */
|
|
1100
|
-
function readSuggestionCache(spaceId: string): string[] | undefined {
|
|
1101
|
-
return _suggestionCache.get(spaceId);
|
|
1100
|
+
function readSuggestionCache(spaceId: string): Promise<string[] | undefined> {
|
|
1101
|
+
return _suggestionCache.get<string[]>(spaceId);
|
|
1102
1102
|
}
|
|
1103
1103
|
|
|
1104
1104
|
/** Cache a space's parsed questions for {@link SUGGESTION_CACHE_TTL_MS}. */
|
|
1105
|
-
function writeSuggestionCache(spaceId: string, questions: string[]): void {
|
|
1106
|
-
_suggestionCache.set(spaceId, questions);
|
|
1105
|
+
async function writeSuggestionCache(spaceId: string, questions: string[]): Promise<void> {
|
|
1106
|
+
await _suggestionCache.set(spaceId, questions);
|
|
1107
1107
|
}
|
|
1108
1108
|
|
|
1109
1109
|
/**
|
|
@@ -1128,7 +1128,7 @@ export async function collectSpaceSuggestions(opts: {
|
|
|
1128
1128
|
const seen = new Set<string>();
|
|
1129
1129
|
|
|
1130
1130
|
for (const { spaceId } of Object.values(opts.spaces)) {
|
|
1131
|
-
let questions = readSuggestionCache(spaceId);
|
|
1131
|
+
let questions = await readSuggestionCache(spaceId);
|
|
1132
1132
|
if (!questions) {
|
|
1133
1133
|
try {
|
|
1134
1134
|
const space = await genieSpace.getGenieSpace(spaceId, {
|
|
@@ -1136,7 +1136,7 @@ export async function collectSpaceSuggestions(opts: {
|
|
|
1136
1136
|
...(opts.signal ? { context: opts.signal } : {}),
|
|
1137
1137
|
});
|
|
1138
1138
|
questions = genieSpace.genieSampleQuestions(space);
|
|
1139
|
-
writeSuggestionCache(spaceId, questions);
|
|
1139
|
+
await writeSuggestionCache(spaceId, questions);
|
|
1140
1140
|
} catch (err) {
|
|
1141
1141
|
logger.warn("suggestions:fetch-error", {
|
|
1142
1142
|
spaceId,
|
package/src/plugin.ts
CHANGED
|
@@ -1126,7 +1126,7 @@ export class MastraPlugin extends Plugin<MastraPluginConfig> {
|
|
|
1126
1126
|
context: this.context,
|
|
1127
1127
|
memoryBuilder,
|
|
1128
1128
|
log: this.logger,
|
|
1129
|
-
extraSkillMounts: provisioned.localSkillMounts,
|
|
1129
|
+
extraSkillMounts: [...(this.config.localSkillMounts ?? []), ...provisioned.localSkillMounts],
|
|
1130
1130
|
});
|
|
1131
1131
|
|
|
1132
1132
|
// `mastra.server.apiRoutes` is only honored by Mastra's standalone
|
package/src/remote-skills.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* {@link AITOOLS_SOURCE} constant, a GitHub `owner/repo`, any git / GitLab URL,
|
|
6
6
|
* or a direct download URL - and optional per-source policy.
|
|
7
7
|
* {@link provisionRemoteSkills} materializes every source into a local
|
|
8
|
-
* `SKILL.md` tree at app boot and returns read-only
|
|
8
|
+
* `SKILL.md` tree at app boot and returns read-only filesystem backings plus
|
|
9
9
|
* their stable virtual workspace mounts.
|
|
10
10
|
*
|
|
11
11
|
* Resolution per source, in order:
|
|
@@ -26,25 +26,16 @@
|
|
|
26
26
|
* source (or the top-level call) sets `failOnError: false`, in which case it is
|
|
27
27
|
* logged and skipped so one bad source never takes the app down.
|
|
28
28
|
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
* record travels WITH the tree rather than in process memory, so the reuse
|
|
35
|
-
* survives a restart - which is the only way it helps a Databricks App at all.
|
|
36
|
-
* Pass `refreshTtlMs: 0` to download on every boot.
|
|
29
|
+
* AI Tools uses the shared `@dbx-tools/cache` project cache with a one-day TTL,
|
|
30
|
+
* then hydrates one process-memory filesystem mounted at
|
|
31
|
+
* {@link AITOOLS_SKILLS_MOUNT} for every default workspace. Other sources keep
|
|
32
|
+
* a {@link METADATA_FILE} beside their provisioned tree and default to a
|
|
33
|
+
* seven-day refresh window. Pass `refreshTtlMs: 0` to download on every boot.
|
|
37
34
|
*
|
|
38
|
-
*
|
|
39
|
-
* (`/Workspace/.assistant/skills`)
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
* instead. When no Databricks client is resolvable at startup - or the resolved
|
|
43
|
-
* identity cannot WRITE the destination, which is the normal case for a
|
|
44
|
-
* Databricks App service principal against the admin-owned shared tree (see
|
|
45
|
-
* {@link openWritableWorkspace}) - the tree is written under
|
|
46
|
-
* {@link localFS.tmpFS} and exposed through a stable read-only virtual mount
|
|
47
|
-
* for the current process. Storage location degrades; the skills themselves do not.
|
|
35
|
+
* For non-AI-Tools sources, the default destination is the Databricks workspace
|
|
36
|
+
* Assistant skills tree (`/Workspace/.assistant/skills`). Pass `userEmail` or
|
|
37
|
+
* `databricksBasePath` to select another destination. When it is unavailable,
|
|
38
|
+
* the tree is exposed through a stable read-only local mount instead.
|
|
48
39
|
*
|
|
49
40
|
* @module
|
|
50
41
|
*/
|
|
@@ -55,6 +46,7 @@ import { dirname, join, posix } from "node:path";
|
|
|
55
46
|
import type { WorkspaceClient } from "@databricks/appkit";
|
|
56
47
|
import { appkit } from "@dbx-tools/appkit";
|
|
57
48
|
import type { WorkspaceClientLike } from "@dbx-tools/appkit";
|
|
49
|
+
import { flushCache, getOrSet, projectCache } from "@dbx-tools/cache/cache";
|
|
58
50
|
import { exec } from "@dbx-tools/core";
|
|
59
51
|
import { DatabricksFileSystem, workspaceClient } from "@dbx-tools/databricks";
|
|
60
52
|
import { localFS, type LocalFileSystem } from "@dbx-tools/fs";
|
|
@@ -70,7 +62,7 @@ import {
|
|
|
70
62
|
stringUtils,
|
|
71
63
|
} from "@dbx-tools/shared-core";
|
|
72
64
|
import type { OneOrMany } from "@dbx-tools/shared-core";
|
|
73
|
-
import type
|
|
65
|
+
import { MemoryFileSystem, type FileSystem } from "@dbx-tools/shared-fs";
|
|
74
66
|
|
|
75
67
|
import { ORGANIZATION_SKILLS_PATH, personalSkillsPath } from "./skill-paths.ts";
|
|
76
68
|
|
|
@@ -116,9 +108,17 @@ const METADATA_FILE = ".metadata.json";
|
|
|
116
108
|
/** Shape marker for {@link METADATA_FILE}; an unknown version is ignored, not read. */
|
|
117
109
|
const METADATA_VERSION = 1;
|
|
118
110
|
|
|
119
|
-
/** How long a provisioned source is reused before being downloaded again. */
|
|
111
|
+
/** How long a general provisioned source is reused before being downloaded again. */
|
|
120
112
|
const DEFAULT_REFRESH_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
121
113
|
|
|
114
|
+
/** How long the shared Databricks AI Tools snapshot is reused. */
|
|
115
|
+
export const AITOOLS_REFRESH_TTL_MS = 24 * 60 * 60 * 1000;
|
|
116
|
+
|
|
117
|
+
const aiToolsCache = projectCache("mastra-aitools", {
|
|
118
|
+
ttl: AITOOLS_REFRESH_TTL_MS,
|
|
119
|
+
maxEntries: 32,
|
|
120
|
+
});
|
|
121
|
+
|
|
122
122
|
/* ------------------------------ AI Tools ------------------------------ */
|
|
123
123
|
|
|
124
124
|
/**
|
|
@@ -203,14 +203,25 @@ export type RemoteSkillsOption =
|
|
|
203
203
|
/** A source entry with its `source` flattened to a plain string. */
|
|
204
204
|
type NormalizedSource = Omit<RemoteSkillSourceOptions, "source"> & { source: string };
|
|
205
205
|
|
|
206
|
-
/** Read-only local
|
|
207
|
-
export interface
|
|
206
|
+
/** Read-only local directory exposed at a stable virtual workspace path. */
|
|
207
|
+
export interface LocalSkillDirectoryMount {
|
|
208
208
|
/** Physical local directory containing the provisioned `SKILL.md` tree. */
|
|
209
209
|
root: string;
|
|
210
210
|
/** Virtual path agents see inside the composed Mastra workspace. */
|
|
211
211
|
mountPath: string;
|
|
212
212
|
}
|
|
213
213
|
|
|
214
|
+
/** Read-only process-local filesystem exposed at a stable virtual workspace path. */
|
|
215
|
+
export interface MemorySkillMount {
|
|
216
|
+
/** Process-shared filesystem containing the provisioned `SKILL.md` tree. */
|
|
217
|
+
filesystem: FileSystem;
|
|
218
|
+
/** Virtual path agents see inside the composed Mastra workspace. */
|
|
219
|
+
mountPath: string;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/** Local skill backing accepted by every auto-created workspace. */
|
|
223
|
+
export type LocalSkillMount = LocalSkillDirectoryMount | MemorySkillMount;
|
|
224
|
+
|
|
214
225
|
/** Top-level remote-skills provisioning options. */
|
|
215
226
|
export interface ProvisionRemoteSkillsOptions {
|
|
216
227
|
/** Sources to materialize at startup. */
|
|
@@ -233,8 +244,8 @@ export interface ProvisionRemoteSkillsOptions {
|
|
|
233
244
|
maxDownloadBytes?: number;
|
|
234
245
|
/**
|
|
235
246
|
* How long a provisioned tree is reused before its source is downloaded
|
|
236
|
-
* again, in milliseconds.
|
|
237
|
-
* boot. A per-source `refreshTtlMs` wins
|
|
247
|
+
* again, in milliseconds. AI Tools defaults to one day and other sources to
|
|
248
|
+
* seven days; `0` re-downloads on every boot. A per-source `refreshTtlMs` wins.
|
|
238
249
|
*/
|
|
239
250
|
refreshTtlMs?: number;
|
|
240
251
|
}
|
|
@@ -242,9 +253,10 @@ export interface ProvisionRemoteSkillsOptions {
|
|
|
242
253
|
/** What {@link provisionRemoteSkills} resolved. */
|
|
243
254
|
export interface ProvisionedRemoteSkills {
|
|
244
255
|
/**
|
|
245
|
-
* Extra read-only
|
|
246
|
-
*
|
|
247
|
-
*
|
|
256
|
+
* Extra read-only local or process-memory skill mounts to hand Mastra. AI
|
|
257
|
+
* Tools always uses process memory; other sources use this fallback when
|
|
258
|
+
* they could not be written to Databricks. Physical paths remain hidden
|
|
259
|
+
* behind stable virtual workspace paths.
|
|
248
260
|
*/
|
|
249
261
|
localSkillMounts: LocalSkillMount[];
|
|
250
262
|
/** Absolute Databricks destination each source was written to, if any. */
|
|
@@ -257,8 +269,8 @@ export interface ProvisionedRemoteSkills {
|
|
|
257
269
|
}
|
|
258
270
|
|
|
259
271
|
/**
|
|
260
|
-
*
|
|
261
|
-
* {@link METADATA_FILE}.
|
|
272
|
+
* Metadata for one resolved source snapshot. General remote sources store it
|
|
273
|
+
* in {@link METADATA_FILE}; AI Tools stores it with the cache-package value.
|
|
262
274
|
*/
|
|
263
275
|
export interface RemoteSkillCacheEntry {
|
|
264
276
|
/** The source string this entry was downloaded from, for readability. */
|
|
@@ -281,6 +293,20 @@ export interface RemoteSkillsMetadata {
|
|
|
281
293
|
sources: Record<string, RemoteSkillCacheEntry>;
|
|
282
294
|
}
|
|
283
295
|
|
|
296
|
+
interface CachedSkillFile {
|
|
297
|
+
path: string;
|
|
298
|
+
content: string;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
interface CachedAiToolsTree {
|
|
302
|
+
files: CachedSkillFile[];
|
|
303
|
+
record: RemoteSkillCacheEntry;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
interface ResolvedAiToolsTree extends CachedAiToolsTree {
|
|
307
|
+
cached: boolean;
|
|
308
|
+
}
|
|
309
|
+
|
|
284
310
|
/* ------------------------------- helpers ------------------------------- */
|
|
285
311
|
|
|
286
312
|
/** Normalize the `remoteSkills` option into a flat options bag. */
|
|
@@ -381,12 +407,16 @@ function remoteSourceIdentifier(source: string): string {
|
|
|
381
407
|
return `${url.hostname}-${path}`;
|
|
382
408
|
}
|
|
383
409
|
|
|
384
|
-
/** The effective reuse window: per-source,
|
|
410
|
+
/** The effective reuse window: per-source, top-level, then the source default. */
|
|
385
411
|
function resolveRefreshTtl(
|
|
386
412
|
sourceOptions: NormalizedSource,
|
|
387
413
|
options: ProvisionRemoteSkillsOptions,
|
|
388
414
|
): number {
|
|
389
|
-
return
|
|
415
|
+
return (
|
|
416
|
+
sourceOptions.refreshTtlMs ??
|
|
417
|
+
options.refreshTtlMs ??
|
|
418
|
+
(isAiToolsSource(sourceOptions.source) ? AITOOLS_REFRESH_TTL_MS : DEFAULT_REFRESH_TTL_MS)
|
|
419
|
+
);
|
|
390
420
|
}
|
|
391
421
|
|
|
392
422
|
/**
|
|
@@ -465,12 +495,17 @@ export async function provisionRemoteSkills(
|
|
|
465
495
|
if (!options) return empty;
|
|
466
496
|
|
|
467
497
|
const failDefault = options.failOnError !== false;
|
|
498
|
+
const sources = (Array.isArray(options.sources) ? options.sources : [options.sources]).map(
|
|
499
|
+
toSourceOptions,
|
|
500
|
+
);
|
|
468
501
|
const client = options.client ?? appkit.tryGetExecutionContext()?.client;
|
|
469
502
|
// Probed rather than assumed: an identity that cannot write the workspace
|
|
470
503
|
// tree falls back to the local one instead of failing startup. The probe also
|
|
471
504
|
// creates the root, which the first cache READ needs - `copySkillDirs` would
|
|
472
505
|
// otherwise be the first thing to create it, which is too late.
|
|
473
|
-
const requestedBasePath =
|
|
506
|
+
const requestedBasePath = sources.some((source) => !isAiToolsSource(source.source))
|
|
507
|
+
? resolveDatabricksBasePath(options, client)
|
|
508
|
+
: undefined;
|
|
474
509
|
const destination = requestedBasePath
|
|
475
510
|
? await openWritableWorkspace(requestedBasePath, client as WorkspaceClient)
|
|
476
511
|
: undefined;
|
|
@@ -481,15 +516,30 @@ export async function provisionRemoteSkills(
|
|
|
481
516
|
let staging: LocalFileSystem | undefined;
|
|
482
517
|
|
|
483
518
|
try {
|
|
484
|
-
const
|
|
485
|
-
for (const entry of sources) {
|
|
486
|
-
const sourceOptions = toSourceOptions(entry);
|
|
519
|
+
for (const sourceOptions of sources) {
|
|
487
520
|
const failOnError = sourceOptions.failOnError ?? failDefault;
|
|
488
521
|
try {
|
|
489
522
|
// Where this source's bookkeeping lives: the shared Databricks tree, or
|
|
490
523
|
// the source's own stable local dir.
|
|
491
524
|
const key = cacheKey(sourceOptions);
|
|
492
525
|
const mountPath = localSkillMountPath(sourceOptions, key);
|
|
526
|
+
if (isAiToolsSource(sourceOptions.source)) {
|
|
527
|
+
staging ??= await initializedScratch("mastra-remote-skills");
|
|
528
|
+
const tree = await cachedAiToolsTree(sourceOptions, staging.root, options, key);
|
|
529
|
+
localSkillMounts.push({
|
|
530
|
+
filesystem: await memorySkillTree(tree.files, `aitools-${key}`),
|
|
531
|
+
mountPath,
|
|
532
|
+
});
|
|
533
|
+
skillNames.push(...tree.record.skills);
|
|
534
|
+
logger.info("remote skill ready", {
|
|
535
|
+
source: sourceOptions.source,
|
|
536
|
+
destination: mountPath,
|
|
537
|
+
downloadedAt: tree.record.downloadedAt,
|
|
538
|
+
skills: tree.record.skills,
|
|
539
|
+
cached: tree.cached,
|
|
540
|
+
});
|
|
541
|
+
continue;
|
|
542
|
+
}
|
|
493
543
|
const cacheFS = destination ?? localSkillsFS(key);
|
|
494
544
|
const metadata = await readMetadata(cacheFS);
|
|
495
545
|
const cached = metadata?.sources[key];
|
|
@@ -563,17 +613,87 @@ export async function provisionRemoteSkills(
|
|
|
563
613
|
return { localSkillMounts, databricksBasePath, skillNames };
|
|
564
614
|
}
|
|
565
615
|
|
|
616
|
+
async function cachedAiToolsTree(
|
|
617
|
+
sourceOptions: NormalizedSource,
|
|
618
|
+
stagingRoot: string,
|
|
619
|
+
options: ProvisionRemoteSkillsOptions,
|
|
620
|
+
key: string,
|
|
621
|
+
): Promise<ResolvedAiToolsTree> {
|
|
622
|
+
const ttl = resolveRefreshTtl(sourceOptions, options);
|
|
623
|
+
let loaded = false;
|
|
624
|
+
const load = async (): Promise<CachedAiToolsTree> => {
|
|
625
|
+
loaded = true;
|
|
626
|
+
const stagedDir = await stageAiTools(
|
|
627
|
+
sourceOptions,
|
|
628
|
+
join(stagingRoot, `aitools-${hash.id()}`),
|
|
629
|
+
options,
|
|
630
|
+
);
|
|
631
|
+
const staged = await collectSkillDirs(stagedDir);
|
|
632
|
+
if (staged.length === 0) {
|
|
633
|
+
throw new Error(`no SKILL.md found for source "${sourceOptions.source}"`);
|
|
634
|
+
}
|
|
635
|
+
const policy = cachePolicy(sourceOptions);
|
|
636
|
+
return {
|
|
637
|
+
files: await serializeSkillDirs(staged),
|
|
638
|
+
record: {
|
|
639
|
+
source: sourceOptions.source,
|
|
640
|
+
downloadedAt: new Date().toISOString(),
|
|
641
|
+
skills: staged.map((dir) => dir.name),
|
|
642
|
+
...(policy ? { policy } : {}),
|
|
643
|
+
},
|
|
644
|
+
};
|
|
645
|
+
};
|
|
646
|
+
const tree =
|
|
647
|
+
ttl <= 0
|
|
648
|
+
? await load()
|
|
649
|
+
: await getOrSet(aiToolsCache, key, load, {
|
|
650
|
+
ttl,
|
|
651
|
+
throwErrors: true,
|
|
652
|
+
});
|
|
653
|
+
if (!tree) throw new Error(`AI Tools cache returned no tree for ${key}`);
|
|
654
|
+
await flushCache(aiToolsCache);
|
|
655
|
+
return { ...tree, cached: !loaded };
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
async function serializeSkillDirs(dirs: readonly StagedSkillDir[]): Promise<CachedSkillFile[]> {
|
|
659
|
+
const files: CachedSkillFile[] = [];
|
|
660
|
+
for (const dir of dirs) {
|
|
661
|
+
for (const relative of find.findFiles("**/*", { cwd: dir.absolutePath, nodir: true })) {
|
|
662
|
+
files.push({
|
|
663
|
+
path: posix.join(dir.name, relative.split(/[\\/]/).join("/")),
|
|
664
|
+
content: (await readFile(join(dir.absolutePath, relative))).toString("base64"),
|
|
665
|
+
});
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
return files;
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
async function memorySkillTree(
|
|
672
|
+
files: readonly CachedSkillFile[],
|
|
673
|
+
id: string,
|
|
674
|
+
): Promise<MemoryFileSystem> {
|
|
675
|
+
const filesystem = new MemoryFileSystem({ id, root: "/" });
|
|
676
|
+
await filesystem.init();
|
|
677
|
+
for (const file of files) {
|
|
678
|
+
await filesystem.writeFile(file.path, Buffer.from(file.content, "base64"), {
|
|
679
|
+
overwrite: true,
|
|
680
|
+
});
|
|
681
|
+
}
|
|
682
|
+
return filesystem;
|
|
683
|
+
}
|
|
684
|
+
|
|
566
685
|
/** Fail before workspace composition when two sources claim one virtual mount. */
|
|
567
686
|
function assertUniqueLocalSkillMounts(mounts: readonly LocalSkillMount[]): void {
|
|
568
687
|
const rootsByMount = new Map<string, string>();
|
|
569
688
|
for (const mount of mounts) {
|
|
689
|
+
const identity = "root" in mount ? mount.root : mount.filesystem.id;
|
|
570
690
|
const previousRoot = rootsByMount.get(mount.mountPath);
|
|
571
|
-
if (previousRoot && previousRoot !==
|
|
691
|
+
if (previousRoot && previousRoot !== identity) {
|
|
572
692
|
throw new Error(
|
|
573
693
|
`multiple remote skill sources resolve to virtual mount "${mount.mountPath}"; give each source a unique name`,
|
|
574
694
|
);
|
|
575
695
|
}
|
|
576
|
-
rootsByMount.set(mount.mountPath,
|
|
696
|
+
rootsByMount.set(mount.mountPath, identity);
|
|
577
697
|
}
|
|
578
698
|
}
|
|
579
699
|
|