@dbx-tools/appkit-mastra 0.9.153 → 0.9.157
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/index.ts +32 -24
- package/lib/index.d.ts +32 -24
- package/lib/index.js +32 -23
- package/lib/src/agents.d.ts +5 -6
- package/lib/src/agents.js +12 -15
- package/lib/src/chart.js +33 -33
- package/lib/src/chat.js +10 -13
- package/lib/src/config.d.ts +12 -13
- package/lib/src/config.js +4 -4
- package/lib/src/evaluation.js +7 -9
- package/lib/src/feedback-mode.d.ts +1 -1
- package/lib/src/feedback-mode.js +1 -1
- package/lib/src/filesystems.js +6 -6
- package/lib/src/follow-up-suggestions.js +10 -10
- package/lib/src/genie-chat.js +27 -29
- package/lib/src/genie.d.ts +2 -2
- package/lib/src/genie.js +29 -29
- package/lib/src/identity.d.ts +3 -30
- package/lib/src/identity.js +2 -27
- package/lib/src/memory.d.ts +1 -1
- package/lib/src/memory.js +24 -6
- package/lib/src/mlflow.d.ts +4 -4
- package/lib/src/mlflow.js +15 -18
- package/lib/src/model.d.ts +3 -3
- package/lib/src/model.js +6 -6
- package/lib/src/monty-sandbox.js +8 -8
- package/lib/src/observability.d.ts +6 -5
- package/lib/src/observability.js +8 -6
- package/lib/src/plugin.d.ts +1 -1
- package/lib/src/plugin.js +49 -51
- package/lib/src/remote-skills.d.ts +3 -4
- package/lib/src/remote-skills.js +46 -46
- package/lib/src/rest.d.ts +2 -2
- package/lib/src/rest.js +3 -3
- package/lib/src/server.js +17 -20
- package/lib/src/serving.js +10 -11
- package/lib/src/statement.js +7 -7
- package/lib/src/storage-schema.js +4 -4
- package/lib/src/summarize.js +5 -5
- package/lib/src/telemetry.d.ts +0 -6
- package/lib/src/telemetry.js +34 -41
- package/lib/src/validation.js +3 -4
- package/lib/src/workspaces.d.ts +2 -2
- package/lib/src/workspaces.js +34 -36
- package/lib/src/writer.d.ts +2 -2
- package/lib/src/writer.js +3 -3
- package/package.json +17 -305
- package/src/agents.ts +27 -21
- package/src/chart.ts +43 -37
- package/src/chat.ts +10 -12
- package/src/config.ts +14 -15
- package/src/evaluation.ts +6 -8
- package/src/feedback-mode.ts +1 -1
- package/src/filesystems.ts +5 -5
- package/src/follow-up-suggestions.ts +9 -9
- package/src/genie-chat.ts +32 -31
- package/src/genie.ts +29 -29
- package/src/identity.ts +3 -38
- package/src/memory.ts +24 -6
- package/src/mlflow.ts +21 -21
- package/src/model.ts +7 -7
- package/src/monty-sandbox.ts +7 -7
- package/src/observability.ts +8 -6
- package/src/plugin.ts +78 -65
- package/src/remote-skills.ts +63 -55
- package/src/rest.ts +4 -4
- package/src/server.ts +32 -20
- package/src/serving.ts +14 -11
- package/src/statement.ts +10 -6
- package/src/storage-schema.ts +3 -3
- package/src/summarize.ts +4 -4
- package/src/telemetry.ts +49 -55
- package/src/validation.ts +2 -3
- package/src/workspaces.ts +52 -42
- package/src/writer.ts +3 -3
package/src/remote-skills.ts
CHANGED
|
@@ -44,24 +44,32 @@ import { mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"
|
|
|
44
44
|
import { createRequire } from "node:module";
|
|
45
45
|
import { dirname, join, posix } from "node:path";
|
|
46
46
|
import type { WorkspaceClient } from "@databricks/appkit";
|
|
47
|
-
import {
|
|
47
|
+
import { tryGetExecutionContext } from "@dbx-tools/appkit";
|
|
48
48
|
import type { WorkspaceClientLike } from "@dbx-tools/appkit";
|
|
49
|
-
import { flushCache, getOrSet, projectCache } from "@dbx-tools/cache
|
|
50
|
-
import {
|
|
51
|
-
import { DatabricksFileSystem,
|
|
52
|
-
import {
|
|
53
|
-
import {
|
|
49
|
+
import { flushCache, getOrSet, projectCache } from "@dbx-tools/cache";
|
|
50
|
+
import { spawn } from "@dbx-tools/core";
|
|
51
|
+
import { DatabricksFileSystem, toLegacyWorkspaceClient } from "@dbx-tools/databricks";
|
|
52
|
+
import { type LocalFileSystem, tmpFS, rebuildFS, scratchFS } from "@dbx-tools/fs";
|
|
53
|
+
import { findFiles } from "@dbx-tools/path";
|
|
54
54
|
import {
|
|
55
|
-
asyncUtils,
|
|
56
|
-
errorUtils,
|
|
57
55
|
hash,
|
|
58
|
-
json,
|
|
59
56
|
log,
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
57
|
+
mapConcurrent,
|
|
58
|
+
toError,
|
|
59
|
+
errorMessage,
|
|
60
|
+
fnvHash,
|
|
61
|
+
parse,
|
|
62
|
+
type UrlLike,
|
|
63
|
+
urlBuilder,
|
|
64
|
+
isOneOrMany,
|
|
65
|
+
isRecord,
|
|
66
|
+
toDate,
|
|
67
|
+
parseList,
|
|
68
|
+
trimToNull,
|
|
69
|
+
toSlug,
|
|
70
|
+
trimToUndefined,
|
|
71
|
+
OneOrMany,
|
|
63
72
|
} from "@dbx-tools/shared-core";
|
|
64
|
-
import type { OneOrMany } from "@dbx-tools/shared-core";
|
|
65
73
|
import { MemoryFileSystem, type FileSystem } from "@dbx-tools/shared-fs";
|
|
66
74
|
|
|
67
75
|
import { ORGANIZATION_SKILLS_PATH, personalSkillsPath } from "./skill-paths.ts";
|
|
@@ -161,7 +169,7 @@ interface AiToolsManifest {
|
|
|
161
169
|
*/
|
|
162
170
|
export interface RemoteSkillSourceOptions {
|
|
163
171
|
/** `"aitools"`, a GitHub shorthand, a git / GitLab URL, or a download URL. */
|
|
164
|
-
source:
|
|
172
|
+
source: UrlLike | AiToolsSource;
|
|
165
173
|
/** Stable virtual mount name under `/.skills`; generated from the source when omitted. */
|
|
166
174
|
name?: string;
|
|
167
175
|
/** Install only these skill names from the source. */
|
|
@@ -191,7 +199,7 @@ export interface RemoteSkillSourceOptions {
|
|
|
191
199
|
* (string / `URL` / `{ url }`), or a {@link RemoteSkillSourceOptions} bag with
|
|
192
200
|
* per-source policy.
|
|
193
201
|
*/
|
|
194
|
-
export type RemoteSkillSource =
|
|
202
|
+
export type RemoteSkillSource = UrlLike | AiToolsSource | RemoteSkillSourceOptions;
|
|
195
203
|
|
|
196
204
|
/**
|
|
197
205
|
* The workspace `remoteSkills` option: one source, a non-empty list of them, or
|
|
@@ -316,13 +324,13 @@ export function normalizeRemoteSkillsOption(
|
|
|
316
324
|
const bag = isProvisionOptions(option) ? option : undefined;
|
|
317
325
|
const sources = toSourceList(bag ? bag.sources : (option as RemoteSkillSource));
|
|
318
326
|
// An empty list is the same as no configuration at all.
|
|
319
|
-
if (!
|
|
327
|
+
if (!isOneOrMany(sources)) return undefined;
|
|
320
328
|
return { ...bag, sources };
|
|
321
329
|
}
|
|
322
330
|
|
|
323
331
|
/** A `sources`-bearing bag is the top-level options shape, not a single source. */
|
|
324
332
|
function isProvisionOptions(value: unknown): value is ProvisionRemoteSkillsOptions {
|
|
325
|
-
return
|
|
333
|
+
return isRecord(value) && "sources" in value;
|
|
326
334
|
}
|
|
327
335
|
|
|
328
336
|
/** One source or many, as a plain array. A source is never itself an array. */
|
|
@@ -341,7 +349,7 @@ function toSourceOptions(source: RemoteSkillSource): NormalizedSource {
|
|
|
341
349
|
}
|
|
342
350
|
|
|
343
351
|
/** A {@link net.UrlLike} as the plain string the staging paths work with. */
|
|
344
|
-
function toSourceString(source:
|
|
352
|
+
function toSourceString(source: UrlLike): string {
|
|
345
353
|
if (typeof source === "string") return source;
|
|
346
354
|
return source instanceof URL ? source.toString() : source.url;
|
|
347
355
|
}
|
|
@@ -361,11 +369,11 @@ function isAiToolsSource(source: string): boolean {
|
|
|
361
369
|
* not what a success produces.
|
|
362
370
|
*/
|
|
363
371
|
function cachePolicy(sourceOptions: NormalizedSource): RemoteSkillCacheEntry["policy"] {
|
|
364
|
-
const skills =
|
|
372
|
+
const skills = parseList(sourceOptions.skills);
|
|
365
373
|
const policy = {
|
|
366
374
|
...(skills.length > 0 ? { skills: [...skills].sort() } : {}),
|
|
367
375
|
...(sourceOptions.experimental === true ? { experimental: true } : {}),
|
|
368
|
-
...(
|
|
376
|
+
...(trimToNull(sourceOptions.ref) ? { ref: sourceOptions.ref!.trim() } : {}),
|
|
369
377
|
};
|
|
370
378
|
return Object.keys(policy).length > 0 ? policy : undefined;
|
|
371
379
|
}
|
|
@@ -378,7 +386,7 @@ function cachePolicy(sourceOptions: NormalizedSource): RemoteSkillCacheEntry["po
|
|
|
378
386
|
* The skill list is sorted first so a reordered array is still the same key.
|
|
379
387
|
*/
|
|
380
388
|
function cacheKey(sourceOptions: NormalizedSource): string {
|
|
381
|
-
return
|
|
389
|
+
return fnvHash(
|
|
382
390
|
JSON.stringify({ source: sourceOptions.source, policy: cachePolicy(sourceOptions) ?? null }),
|
|
383
391
|
);
|
|
384
392
|
}
|
|
@@ -386,10 +394,10 @@ function cacheKey(sourceOptions: NormalizedSource): string {
|
|
|
386
394
|
/** Stable virtual workspace path for one locally provisioned source. */
|
|
387
395
|
function localSkillMountPath(sourceOptions: NormalizedSource, key: string): string {
|
|
388
396
|
if (isAiToolsSource(sourceOptions.source)) return AITOOLS_SKILLS_MOUNT;
|
|
389
|
-
const configuredName =
|
|
397
|
+
const configuredName = trimToNull(sourceOptions.name);
|
|
390
398
|
const identifier = configuredName
|
|
391
|
-
?
|
|
392
|
-
:
|
|
399
|
+
? toSlug(configuredName)
|
|
400
|
+
: toSlug(remoteSourceIdentifier(sourceOptions.source)) || "source";
|
|
393
401
|
if (!identifier) {
|
|
394
402
|
throw new Error(
|
|
395
403
|
`remote skill source name must contain letters or numbers: "${configuredName}"`,
|
|
@@ -400,7 +408,7 @@ function localSkillMountPath(sourceOptions: NormalizedSource, key: string): stri
|
|
|
400
408
|
|
|
401
409
|
/** Human-readable source portion used by generated virtual mount identifiers. */
|
|
402
410
|
function remoteSourceIdentifier(source: string): string {
|
|
403
|
-
const url =
|
|
411
|
+
const url = urlBuilder(source);
|
|
404
412
|
if (!url) return source;
|
|
405
413
|
const path = url.pathname.replace(/\.(git|md|zip|tar|tgz|gz)$/i, "");
|
|
406
414
|
return `${url.hostname}-${path}`;
|
|
@@ -431,11 +439,11 @@ async function readMetadata(fs: FileSystem): Promise<RemoteSkillsMetadata | unde
|
|
|
431
439
|
try {
|
|
432
440
|
if (!(await fs.exists(METADATA_FILE))) return undefined;
|
|
433
441
|
const raw = await fs.readFile(METADATA_FILE, { encoding: "utf8" });
|
|
434
|
-
const parsed =
|
|
435
|
-
if (!
|
|
436
|
-
return
|
|
442
|
+
const parsed = parse(raw, undefined) as RemoteSkillsMetadata | undefined;
|
|
443
|
+
if (!isRecord(parsed) || parsed.version !== METADATA_VERSION) return undefined;
|
|
444
|
+
return isRecord(parsed.sources) ? parsed : undefined;
|
|
437
445
|
} catch (err) {
|
|
438
|
-
logger.debug("cache:unreadable", { error:
|
|
446
|
+
logger.debug("cache:unreadable", { error: toError(err) });
|
|
439
447
|
return undefined;
|
|
440
448
|
}
|
|
441
449
|
}
|
|
@@ -449,7 +457,7 @@ async function readMetadata(fs: FileSystem): Promise<RemoteSkillsMetadata | unde
|
|
|
449
457
|
*/
|
|
450
458
|
function isFresh(entry: RemoteSkillCacheEntry, ttlMs: number): boolean {
|
|
451
459
|
if (ttlMs <= 0 || !Array.isArray(entry.skills)) return false;
|
|
452
|
-
const downloadedAt =
|
|
460
|
+
const downloadedAt = toDate(entry.downloadedAt);
|
|
453
461
|
if (!downloadedAt) return false;
|
|
454
462
|
const age = Date.now() - downloadedAt.getTime();
|
|
455
463
|
return age >= 0 && age < ttlMs;
|
|
@@ -476,7 +484,7 @@ async function writeMetadata(
|
|
|
476
484
|
|
|
477
485
|
/** The stable local tree a source is rebuilt into, keyed by {@link cacheKey}. */
|
|
478
486
|
function localSkillsFS(key: string): LocalFileSystem {
|
|
479
|
-
return
|
|
487
|
+
return tmpFS(`${LOCAL_SKILLS_DIR}/${key}`);
|
|
480
488
|
}
|
|
481
489
|
|
|
482
490
|
/**
|
|
@@ -501,7 +509,7 @@ export async function provisionRemoteSkills(
|
|
|
501
509
|
const sources = (Array.isArray(options.sources) ? options.sources : [options.sources]).map(
|
|
502
510
|
toSourceOptions,
|
|
503
511
|
);
|
|
504
|
-
const client = options.client ??
|
|
512
|
+
const client = options.client ?? tryGetExecutionContext()?.client;
|
|
505
513
|
// Probed rather than assumed: an identity that cannot write the workspace
|
|
506
514
|
// tree falls back to the local one instead of failing startup. The probe also
|
|
507
515
|
// creates the root, which the first cache READ needs - `copySkillDirs` would
|
|
@@ -612,13 +620,13 @@ export async function provisionRemoteSkills(
|
|
|
612
620
|
} catch (err) {
|
|
613
621
|
if (failOnError) {
|
|
614
622
|
throw new Error(
|
|
615
|
-
`failed to provision remote skill source "${sourceOptions.source}": ${
|
|
616
|
-
{ cause:
|
|
623
|
+
`failed to provision remote skill source "${sourceOptions.source}": ${errorMessage(err)}`,
|
|
624
|
+
{ cause: toError(err) },
|
|
617
625
|
);
|
|
618
626
|
}
|
|
619
627
|
logger.warn("source:skipped", {
|
|
620
628
|
source: sourceOptions.source,
|
|
621
|
-
error:
|
|
629
|
+
error: toError(err),
|
|
622
630
|
});
|
|
623
631
|
}
|
|
624
632
|
}
|
|
@@ -677,7 +685,7 @@ async function cachedAiToolsTree(
|
|
|
677
685
|
async function serializeSkillDirs(dirs: readonly StagedSkillDir[]): Promise<CachedSkillFile[]> {
|
|
678
686
|
const files: CachedSkillFile[] = [];
|
|
679
687
|
for (const dir of dirs) {
|
|
680
|
-
for await (const relative of
|
|
688
|
+
for await (const relative of findFiles("**/*", {
|
|
681
689
|
cwd: dir.absolutePath,
|
|
682
690
|
nodir: true,
|
|
683
691
|
})) {
|
|
@@ -784,7 +792,7 @@ async function openWritableWorkspace(
|
|
|
784
792
|
client: WorkspaceClient,
|
|
785
793
|
): Promise<DatabricksFileSystem | undefined> {
|
|
786
794
|
const fs = new DatabricksFileSystem({
|
|
787
|
-
client:
|
|
795
|
+
client: toLegacyWorkspaceClient(client),
|
|
788
796
|
root: basePath,
|
|
789
797
|
readOnly: false,
|
|
790
798
|
createRoot: true,
|
|
@@ -795,7 +803,7 @@ async function openWritableWorkspace(
|
|
|
795
803
|
} catch (err) {
|
|
796
804
|
logger.warn("destination:unwritable", {
|
|
797
805
|
destination: basePath,
|
|
798
|
-
error:
|
|
806
|
+
error: toError(err),
|
|
799
807
|
});
|
|
800
808
|
return undefined;
|
|
801
809
|
}
|
|
@@ -809,8 +817,8 @@ function resolveDatabricksBasePath(
|
|
|
809
817
|
client: WorkspaceClientLike | undefined,
|
|
810
818
|
): string | undefined {
|
|
811
819
|
if (!client) return undefined;
|
|
812
|
-
if (options.databricksBasePath) return
|
|
813
|
-
const email =
|
|
820
|
+
if (options.databricksBasePath) return trimToUndefined(options.databricksBasePath);
|
|
821
|
+
const email = trimToNull(options.userEmail);
|
|
814
822
|
// A named user targets their personal Assistant tree (the "save a skill"
|
|
815
823
|
// target); otherwise the shared workspace Assistant tree, which the built-in
|
|
816
824
|
// Assistant-skills mount already scans.
|
|
@@ -849,13 +857,13 @@ async function stageAiTools(
|
|
|
849
857
|
options: ProvisionRemoteSkillsOptions,
|
|
850
858
|
): Promise<string> {
|
|
851
859
|
const maxBytes = resolveMaxBytes(sourceOptions, options);
|
|
852
|
-
const ref =
|
|
853
|
-
const manifest =
|
|
860
|
+
const ref = trimToNull(sourceOptions.ref) ?? AITOOLS_REF;
|
|
861
|
+
const manifest = parse(
|
|
854
862
|
(await download(aiToolsRawUrl(ref, "manifest.json"), maxBytes)).toString("utf8"),
|
|
855
863
|
undefined,
|
|
856
864
|
) as AiToolsManifest | undefined;
|
|
857
865
|
|
|
858
|
-
const wanted = new Set(
|
|
866
|
+
const wanted = new Set(parseList(sourceOptions.skills));
|
|
859
867
|
const selected = Object.entries(manifest?.skills ?? {}).filter(([name, entry]) =>
|
|
860
868
|
wanted.size > 0
|
|
861
869
|
? wanted.has(name)
|
|
@@ -871,7 +879,7 @@ async function stageAiTools(
|
|
|
871
879
|
path: join(target, name, ...file.split("/")),
|
|
872
880
|
})),
|
|
873
881
|
);
|
|
874
|
-
await
|
|
882
|
+
await mapConcurrent(
|
|
875
883
|
files,
|
|
876
884
|
async (item) => {
|
|
877
885
|
const body = await download(item.url, maxBytes);
|
|
@@ -911,12 +919,12 @@ async function stageViaSkillsCli(
|
|
|
911
919
|
"--copy",
|
|
912
920
|
"-y",
|
|
913
921
|
];
|
|
914
|
-
for (const skill of
|
|
922
|
+
for (const skill of parseList(sourceOptions.skills)) {
|
|
915
923
|
args.push("--skill", skill);
|
|
916
924
|
}
|
|
917
|
-
if (
|
|
925
|
+
if (parseList(sourceOptions.skills).length === 0) args.push("--skill", "*");
|
|
918
926
|
|
|
919
|
-
const result = await
|
|
927
|
+
const result = await spawn(cli.command, args, {
|
|
920
928
|
cwd: target,
|
|
921
929
|
stdout: "capture",
|
|
922
930
|
stderr: "capture",
|
|
@@ -956,14 +964,14 @@ async function stageViaFetch(
|
|
|
956
964
|
target: string,
|
|
957
965
|
options: ProvisionRemoteSkillsOptions,
|
|
958
966
|
): Promise<string> {
|
|
959
|
-
const url =
|
|
967
|
+
const url = urlBuilder(sourceOptions.source);
|
|
960
968
|
if (!url) {
|
|
961
969
|
throw new Error(
|
|
962
970
|
`source "${sourceOptions.source}" is not a URL and the optional "skills" package is not installed`,
|
|
963
971
|
);
|
|
964
972
|
}
|
|
965
973
|
const buffer = await download(url.toString(), resolveMaxBytes(sourceOptions, options));
|
|
966
|
-
const name =
|
|
974
|
+
const name = toSlug(deriveSkillName(url.toString())) || "remote-skill";
|
|
967
975
|
const skillDir = join(target, name);
|
|
968
976
|
await mkdir(skillDir, { recursive: true });
|
|
969
977
|
await writeFile(join(skillDir, "SKILL.md"), buffer);
|
|
@@ -983,8 +991,8 @@ async function download(url: string, maxBytes: number): Promise<Buffer> {
|
|
|
983
991
|
const response = await fetch(url, {
|
|
984
992
|
signal: AbortSignal.timeout(DEFAULT_DOWNLOAD_TIMEOUT_MS),
|
|
985
993
|
}).catch((err) => {
|
|
986
|
-
throw new Error(`download failed (${url}): ${
|
|
987
|
-
cause:
|
|
994
|
+
throw new Error(`download failed (${url}): ${errorMessage(err)}`, {
|
|
995
|
+
cause: toError(err),
|
|
988
996
|
});
|
|
989
997
|
});
|
|
990
998
|
if (!response.ok) {
|
|
@@ -999,7 +1007,7 @@ async function download(url: string, maxBytes: number): Promise<Buffer> {
|
|
|
999
1007
|
|
|
1000
1008
|
/** Derive a skill directory name from a download URL's last path segment. */
|
|
1001
1009
|
function deriveSkillName(urlString: string): string {
|
|
1002
|
-
const url =
|
|
1010
|
+
const url = urlBuilder(urlString);
|
|
1003
1011
|
const last = url?.pathname.split("/").filter(Boolean).pop() ?? "";
|
|
1004
1012
|
return last.replace(/\.(md|zip|tar|tgz|gz)$/i, "");
|
|
1005
1013
|
}
|
|
@@ -1039,7 +1047,7 @@ async function collectSkillDirs(root: string): Promise<StagedSkillDir[]> {
|
|
|
1039
1047
|
async function copySkillDirs(destination: FileSystem, dirs: StagedSkillDir[]): Promise<void> {
|
|
1040
1048
|
await destination.init();
|
|
1041
1049
|
for (const dir of dirs) {
|
|
1042
|
-
for await (const relative of
|
|
1050
|
+
for await (const relative of findFiles("**/*", {
|
|
1043
1051
|
cwd: dir.absolutePath,
|
|
1044
1052
|
nodir: true,
|
|
1045
1053
|
})) {
|
|
@@ -1070,7 +1078,7 @@ async function persistLocally(
|
|
|
1070
1078
|
dirs: StagedSkillDir[],
|
|
1071
1079
|
record: RemoteSkillCacheEntry,
|
|
1072
1080
|
): Promise<string> {
|
|
1073
|
-
const stable = await
|
|
1081
|
+
const stable = await rebuildFS(`${LOCAL_SKILLS_DIR}/${key}`, async (scratch) => {
|
|
1074
1082
|
await copySkillDirs(scratch, dirs);
|
|
1075
1083
|
await writeMetadata(scratch, key, record);
|
|
1076
1084
|
});
|
|
@@ -1083,7 +1091,7 @@ async function persistLocally(
|
|
|
1083
1091
|
* the {@link FileSystem} API.
|
|
1084
1092
|
*/
|
|
1085
1093
|
async function initializedScratch(prefix: string): Promise<LocalFileSystem> {
|
|
1086
|
-
const scratch =
|
|
1094
|
+
const scratch = scratchFS(prefix);
|
|
1087
1095
|
await scratch.init();
|
|
1088
1096
|
return scratch;
|
|
1089
1097
|
}
|
package/src/rest.ts
CHANGED
|
@@ -10,11 +10,11 @@
|
|
|
10
10
|
* @module
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
13
|
+
import { type WorkspaceClientLike } from "@dbx-tools/appkit";
|
|
14
|
+
import { parse } from "@dbx-tools/shared-core";
|
|
15
15
|
|
|
16
16
|
/** Workspace client carried on an AppKit execution context. */
|
|
17
|
-
type WorkspaceClient =
|
|
17
|
+
type WorkspaceClient = WorkspaceClientLike;
|
|
18
18
|
|
|
19
19
|
/** Request options for {@link databricksFetch}. */
|
|
20
20
|
export interface DatabricksFetchInit {
|
|
@@ -62,5 +62,5 @@ export async function readResponseText(res: Response): Promise<string> {
|
|
|
62
62
|
/** Parse a response body as JSON, returning `{}` on empty / invalid bodies. */
|
|
63
63
|
export async function readResponseJson(res: Response): Promise<unknown> {
|
|
64
64
|
const text = await readResponseText(res);
|
|
65
|
-
return
|
|
65
|
+
return parse<unknown>(text, {});
|
|
66
66
|
}
|
package/src/server.ts
CHANGED
|
@@ -6,9 +6,24 @@
|
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
8
|
import { getExecutionContext } from "@databricks/appkit";
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
|
|
9
|
+
import { requestUserEmail, requestUserId } from "@dbx-tools/appkit";
|
|
10
|
+
import {
|
|
11
|
+
log,
|
|
12
|
+
toError,
|
|
13
|
+
id,
|
|
14
|
+
parseCookies,
|
|
15
|
+
type Logger,
|
|
16
|
+
sequence,
|
|
17
|
+
toIdentifierWithOptions,
|
|
18
|
+
trimToNull,
|
|
19
|
+
getAccessTokenScopes,
|
|
20
|
+
} from "@dbx-tools/shared-core";
|
|
21
|
+
import {
|
|
22
|
+
MLFLOW_TRACE_ID_HEADER,
|
|
23
|
+
THREAD_ID_HEADER,
|
|
24
|
+
THREAD_ID_QUERY,
|
|
25
|
+
isStaleMastraResumeError,
|
|
26
|
+
} from "@dbx-tools/shared-mastra";
|
|
12
27
|
import {
|
|
13
28
|
MASTRA_RESOURCE_ID_KEY,
|
|
14
29
|
MASTRA_THREAD_ID_KEY,
|
|
@@ -34,7 +49,6 @@ import {
|
|
|
34
49
|
FOLLOW_UP_RUN_ID_KEY,
|
|
35
50
|
FOLLOW_UP_USER_TEXT_KEY,
|
|
36
51
|
} from "./follow-up-suggestions.ts";
|
|
37
|
-
import { requestUserEmail, requestUserId } from "./identity.ts";
|
|
38
52
|
import { initializeRequestMlflowTraceInfo } from "./mlflow.ts";
|
|
39
53
|
import { MASTRA_ORIGIN_KEY } from "./request-context.ts";
|
|
40
54
|
import {
|
|
@@ -198,7 +212,7 @@ export async function createRequestContext(
|
|
|
198
212
|
// Stamped AFTER the user so an explicit resource id wins over the ambient
|
|
199
213
|
// identity `stampRequestContextUser` defaulted to.
|
|
200
214
|
if (options.resourceId) requestContext.set(MASTRA_RESOURCE_ID_KEY, options.resourceId);
|
|
201
|
-
requestContext.set(MASTRA_REQUEST_ID_KEY, options.requestId ??
|
|
215
|
+
requestContext.set(MASTRA_REQUEST_ID_KEY, options.requestId ?? id());
|
|
202
216
|
return requestContext;
|
|
203
217
|
}
|
|
204
218
|
|
|
@@ -208,7 +222,7 @@ export async function createRequestContext(
|
|
|
208
222
|
* session cookie (`appkit_<plugin-name>_session_id`).
|
|
209
223
|
*/
|
|
210
224
|
export class MastraServer extends ExpressMastraServer {
|
|
211
|
-
private log:
|
|
225
|
+
private log: Logger;
|
|
212
226
|
/**
|
|
213
227
|
* Whether to stamp the MLflow trace-id header on responses.
|
|
214
228
|
*/
|
|
@@ -299,7 +313,7 @@ export class MastraServer extends ExpressMastraServer {
|
|
|
299
313
|
if (requestContext.get(MASTRA_REQUEST_ID_KEY)) return;
|
|
300
314
|
const headerValue = req.headers["x-request-id"];
|
|
301
315
|
const upstream = Array.isArray(headerValue) ? headerValue[0] : headerValue;
|
|
302
|
-
const requestId = upstream?.trim() ||
|
|
316
|
+
const requestId = upstream?.trim() || id();
|
|
303
317
|
requestContext.set(MASTRA_REQUEST_ID_KEY, requestId);
|
|
304
318
|
res.setHeader("X-Request-Id", requestId);
|
|
305
319
|
}
|
|
@@ -310,9 +324,7 @@ export class MastraServer extends ExpressMastraServer {
|
|
|
310
324
|
*/
|
|
311
325
|
configureRequestContextScopes(req: express.Request, requestContext: RequestContext) {
|
|
312
326
|
const scopes = [
|
|
313
|
-
...
|
|
314
|
-
.sequence(token.getAccessTokenScopes(req), token.getAccessTokenScopes(req, "authorization"))
|
|
315
|
-
.distinct(),
|
|
327
|
+
...sequence(getAccessTokenScopes(req), getAccessTokenScopes(req, "authorization")).distinct(),
|
|
316
328
|
];
|
|
317
329
|
requestContext.set(MASTRA_SCOPES_KEY, scopes);
|
|
318
330
|
}
|
|
@@ -332,10 +344,10 @@ export class MastraServer extends ExpressMastraServer {
|
|
|
332
344
|
*/
|
|
333
345
|
configureMlflowTraceId(res: express.Response) {
|
|
334
346
|
if (!this.mlflowFeedback || res.headersSent) return;
|
|
335
|
-
if (res.getHeader(
|
|
347
|
+
if (res.getHeader(MLFLOW_TRACE_ID_HEADER)) return;
|
|
336
348
|
const traceId = trace.getActiveSpan()?.spanContext().traceId;
|
|
337
349
|
if (!traceId || traceId === INVALID_TRACE_ID) return;
|
|
338
|
-
res.setHeader(
|
|
350
|
+
res.setHeader(MLFLOW_TRACE_ID_HEADER, `tr-${traceId}`);
|
|
339
351
|
}
|
|
340
352
|
|
|
341
353
|
/**
|
|
@@ -367,8 +379,8 @@ export class MastraServer extends ExpressMastraServer {
|
|
|
367
379
|
requestContext.set(MASTRA_THREAD_ID_KEY, requested);
|
|
368
380
|
return;
|
|
369
381
|
}
|
|
370
|
-
const cookies =
|
|
371
|
-
const cookieName =
|
|
382
|
+
const cookies = parseCookies(req.headers.cookie);
|
|
383
|
+
const cookieName = toIdentifierWithOptions(
|
|
372
384
|
{ delimiter: "_", distinct: true },
|
|
373
385
|
"appkit",
|
|
374
386
|
this.config.name!,
|
|
@@ -376,7 +388,7 @@ export class MastraServer extends ExpressMastraServer {
|
|
|
376
388
|
);
|
|
377
389
|
let sessionId = cookies[cookieName];
|
|
378
390
|
if (!sessionId) {
|
|
379
|
-
sessionId =
|
|
391
|
+
sessionId = id();
|
|
380
392
|
res.cookie(cookieName, sessionId, {
|
|
381
393
|
httpOnly: true,
|
|
382
394
|
sameSite: "lax",
|
|
@@ -394,11 +406,11 @@ export class MastraServer extends ExpressMastraServer {
|
|
|
394
406
|
* back to the session cookie.
|
|
395
407
|
*/
|
|
396
408
|
private readRequestedThreadId(req: express.Request): string | null {
|
|
397
|
-
const headerValue = req.headers[
|
|
398
|
-
const queryValue = req.query[
|
|
409
|
+
const headerValue = req.headers[THREAD_ID_HEADER];
|
|
410
|
+
const queryValue = req.query[THREAD_ID_QUERY];
|
|
399
411
|
return (
|
|
400
|
-
|
|
401
|
-
|
|
412
|
+
trimToNull(Array.isArray(headerValue) ? headerValue[0] : headerValue) ??
|
|
413
|
+
trimToNull(Array.isArray(queryValue) ? queryValue[0] : queryValue)
|
|
402
414
|
);
|
|
403
415
|
}
|
|
404
416
|
|
|
@@ -526,7 +538,7 @@ export function attachStaleResumeRecovery(app: express.Express): void {
|
|
|
526
538
|
}
|
|
527
539
|
logger.warn("ignored stale mastra resume", {
|
|
528
540
|
path: req.path,
|
|
529
|
-
error:
|
|
541
|
+
error: toError(error),
|
|
530
542
|
});
|
|
531
543
|
if (res.headersSent) {
|
|
532
544
|
res.end();
|
package/src/serving.ts
CHANGED
|
@@ -11,9 +11,13 @@
|
|
|
11
11
|
* @module
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
14
|
+
import { DEFAULT_MODEL_CACHE_TTL_MS, DEFAULT_FUZZY_THRESHOLD } from "@dbx-tools/model";
|
|
15
|
+
import { firstNonEmpty } from "@dbx-tools/shared-core";
|
|
16
|
+
import {
|
|
17
|
+
MODEL_OVERRIDE_HEADER,
|
|
18
|
+
MODEL_OVERRIDE_QUERY,
|
|
19
|
+
MODEL_OVERRIDE_BODY_FIELDS,
|
|
20
|
+
} from "@dbx-tools/shared-mastra";
|
|
17
21
|
|
|
18
22
|
import type { MastraPluginConfig } from "./config.ts";
|
|
19
23
|
|
|
@@ -55,20 +59,19 @@ export interface ModelOverrideRequest {
|
|
|
55
59
|
export function extractModelOverride(req: ModelOverrideRequest): string | null {
|
|
56
60
|
const headers = req.headers;
|
|
57
61
|
if (headers) {
|
|
58
|
-
const headerVal =
|
|
59
|
-
headers[
|
|
60
|
-
headers[override.MODEL_OVERRIDE_HEADER.toLowerCase()],
|
|
62
|
+
const headerVal = firstNonEmpty(
|
|
63
|
+
headers[MODEL_OVERRIDE_HEADER] ?? headers[MODEL_OVERRIDE_HEADER.toLowerCase()],
|
|
61
64
|
);
|
|
62
65
|
if (headerVal) return headerVal;
|
|
63
66
|
}
|
|
64
67
|
if (req.query) {
|
|
65
|
-
const queryVal =
|
|
68
|
+
const queryVal = firstNonEmpty(req.query[MODEL_OVERRIDE_QUERY]);
|
|
66
69
|
if (queryVal) return queryVal;
|
|
67
70
|
}
|
|
68
71
|
if (req.body && typeof req.body === "object") {
|
|
69
72
|
const record = req.body as Record<string, unknown>;
|
|
70
|
-
for (const field of
|
|
71
|
-
const bodyVal =
|
|
73
|
+
for (const field of MODEL_OVERRIDE_BODY_FIELDS) {
|
|
74
|
+
const bodyVal = firstNonEmpty(record[field]);
|
|
72
75
|
if (bodyVal) return bodyVal;
|
|
73
76
|
}
|
|
74
77
|
}
|
|
@@ -93,8 +96,8 @@ export function resolveServingConfig(config: MastraPluginConfig): {
|
|
|
93
96
|
fallbacks: readonly string[];
|
|
94
97
|
} {
|
|
95
98
|
return {
|
|
96
|
-
ttlMs: config.modelCacheTtlMs ??
|
|
97
|
-
threshold: config.modelFuzzyThreshold ??
|
|
99
|
+
ttlMs: config.modelCacheTtlMs ?? DEFAULT_MODEL_CACHE_TTL_MS,
|
|
100
|
+
threshold: config.modelFuzzyThreshold ?? DEFAULT_FUZZY_THRESHOLD,
|
|
98
101
|
fuzzy: config.modelFuzzyMatch !== false,
|
|
99
102
|
allowOverride: config.modelOverride !== false,
|
|
100
103
|
fallbacks: config.defaultModelFallbacks ?? [],
|
package/src/statement.ts
CHANGED
|
@@ -20,9 +20,13 @@
|
|
|
20
20
|
*/
|
|
21
21
|
|
|
22
22
|
import type { WorkspaceClient } from "@databricks/appkit";
|
|
23
|
-
import {
|
|
24
|
-
import {
|
|
25
|
-
import {
|
|
23
|
+
import { toContext } from "@dbx-tools/appkit";
|
|
24
|
+
import { toNumber } from "@dbx-tools/shared-core";
|
|
25
|
+
import {
|
|
26
|
+
type GenieDatasetData,
|
|
27
|
+
type GenieVisualizationData,
|
|
28
|
+
GenieVisualizationDataSchema,
|
|
29
|
+
} from "@dbx-tools/shared-mastra";
|
|
26
30
|
|
|
27
31
|
/**
|
|
28
32
|
* Hard server-side cap on rows returned by the
|
|
@@ -45,7 +49,7 @@ export const STATEMENT_ROW_CAP = 500;
|
|
|
45
49
|
*/
|
|
46
50
|
function coerceCell(cell: string | null): unknown {
|
|
47
51
|
if (cell === null) return null;
|
|
48
|
-
return
|
|
52
|
+
return toNumber(cell, { separators: false, percent: false }) ?? cell;
|
|
49
53
|
}
|
|
50
54
|
|
|
51
55
|
type StatementResponse = Awaited<ReturnType<WorkspaceClient["statementExecution"]["getStatement"]>>;
|
|
@@ -55,7 +59,7 @@ async function fetchStatementResponse(
|
|
|
55
59
|
statementId: string,
|
|
56
60
|
signal?: AbortSignal,
|
|
57
61
|
): Promise<StatementResponse> {
|
|
58
|
-
const context = signal ?
|
|
62
|
+
const context = signal ? toContext(signal) : undefined;
|
|
59
63
|
return client.statementExecution.getStatement({ statement_id: statementId }, context);
|
|
60
64
|
}
|
|
61
65
|
|
|
@@ -79,7 +83,7 @@ function normalizedStatementData(response: StatementResponse, limit?: number): G
|
|
|
79
83
|
|
|
80
84
|
function visualizationData(response: StatementResponse, limit?: number): GenieVisualizationData {
|
|
81
85
|
const dataArray = (response.result?.data_array ?? []) as Array<Array<string | null>>;
|
|
82
|
-
return
|
|
86
|
+
return GenieVisualizationDataSchema.parse({
|
|
83
87
|
manifest: {
|
|
84
88
|
schema: {
|
|
85
89
|
columns: (response.manifest?.schema?.columns ?? []).map((column) => ({
|
package/src/storage-schema.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* @module
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
import {
|
|
11
|
+
import { toIdentifierWithOptions } from "@dbx-tools/shared-core";
|
|
12
12
|
|
|
13
13
|
const SCHEMA_PREFIX = "mastra_";
|
|
14
14
|
const MAX_PG_IDENTIFIER_LEN = 63;
|
|
@@ -19,7 +19,7 @@ const MAX_PG_IDENTIFIER_LEN = 63;
|
|
|
19
19
|
*/
|
|
20
20
|
export function agentStorageSchemaName(agentId: string): string {
|
|
21
21
|
const maxSlugLen = MAX_PG_IDENTIFIER_LEN - SCHEMA_PREFIX.length;
|
|
22
|
-
const slug =
|
|
22
|
+
const slug = toIdentifierWithOptions(
|
|
23
23
|
{
|
|
24
24
|
delimiter: "_",
|
|
25
25
|
maxLength: maxSlugLen,
|
|
@@ -30,7 +30,7 @@ export function agentStorageSchemaName(agentId: string): string {
|
|
|
30
30
|
);
|
|
31
31
|
const body =
|
|
32
32
|
slug ||
|
|
33
|
-
|
|
33
|
+
toIdentifierWithOptions(
|
|
34
34
|
{
|
|
35
35
|
delimiter: "_",
|
|
36
36
|
maxLength: maxSlugLen,
|
package/src/summarize.ts
CHANGED
|
@@ -19,8 +19,8 @@
|
|
|
19
19
|
* @module
|
|
20
20
|
*/
|
|
21
21
|
|
|
22
|
-
import {
|
|
23
|
-
import {
|
|
22
|
+
import { toDescription } from "@dbx-tools/shared-core";
|
|
23
|
+
import { ModelClass } from "@dbx-tools/shared-model";
|
|
24
24
|
import { Agent } from "@mastra/core/agent";
|
|
25
25
|
import type { MastraModelConfig } from "@mastra/core/llm";
|
|
26
26
|
import type { RequestContext } from "@mastra/core/request-context";
|
|
@@ -32,7 +32,7 @@ import { buildModel } from "./model.ts";
|
|
|
32
32
|
import { TYPOGRAPHY_RULE } from "./style.ts";
|
|
33
33
|
|
|
34
34
|
/** Fast / small chat tier used for both titling and summaries. */
|
|
35
|
-
const SUMMARY_MODEL_CLASS =
|
|
35
|
+
const SUMMARY_MODEL_CLASS = ModelClass.ChatFast;
|
|
36
36
|
|
|
37
37
|
/** System prompt for the summarizer agent (and the `summarize` tool). */
|
|
38
38
|
const SUMMARIZER_INSTRUCTIONS = [
|
|
@@ -136,7 +136,7 @@ export async function summarizeText(
|
|
|
136
136
|
options: SummarizeOptions = {},
|
|
137
137
|
): Promise<string> {
|
|
138
138
|
const { instructions, maxWords, requestContext, abortSignal } = options;
|
|
139
|
-
const prompt =
|
|
139
|
+
const prompt = toDescription({
|
|
140
140
|
...(instructions ? { Guidance: instructions } : {}),
|
|
141
141
|
...(maxWords !== undefined ? { "Max length (words)": String(maxWords) } : {}),
|
|
142
142
|
Text: text,
|