@dbx-tools/appkit-mastra 0.9.93 → 0.9.95

package/src/skill-cache.ts

@@ -1,347 +0,0 @@
-/**
- * User-scoped workspace skill catalogues cached as one semantic record.
- *
- * @module
- */
-
-import { CacheManager } from "@databricks/appkit";
-import { errorUtils, log, object } from "@dbx-tools/shared-core";
-import { posixPath } from "@dbx-tools/shared-fs";
-import type { RequestContext } from "@mastra/core/request-context";
-import { LRUCache } from "lru-cache";
-import { parse as parseYaml } from "yaml";
-import { z } from "zod";
-
-const CACHE_NAMESPACE = "mastra:workspace-skill-catalogue";
-const CATALOGUE_VERSION = 1;
-const MAX_SCOPED_CATALOGUES = 256;
-const logger = log.logger("mastra/skill-cache");
-
-const SkillCatalogueEntrySchema = z
-  .object({
-    name: z.string().describe("Unique skill name from SKILL.md frontmatter."),
-    description: z.string().describe("Skill description from SKILL.md frontmatter."),
-    instructions: z.string().describe("Markdown instructions below SKILL.md frontmatter."),
-    sourceId: z.string().describe("Identity of the mounted source that owns this skill."),
-    sourcePath: z.string().describe("Path to SKILL.md relative to its source root."),
-    signature: z
-      .string()
-      .optional()
-      .describe("Stable metadata signature used to retain unchanged instructions."),
-  })
-  .describe("One parsed skill in a materialized workspace catalogue.");
-
-const SkillCatalogueSchema = z
-  .object({
-    version: z.literal(CATALOGUE_VERSION).describe("Persisted catalogue schema version."),
-    generatedAt: z.string().datetime().describe("Time the catalogue was last refreshed."),
-    roots: z.array(z.string()).describe("Ordered source identities included in the catalogue."),
-    skills: z
-      .array(SkillCatalogueEntrySchema)
-      .describe("Parsed skills after ordered duplicate-name precedence is applied."),
-  })
-  .describe("One user-scoped materialized workspace skill catalogue.");
-
-/** Parsed skill retained in a materialized workspace catalogue. */
-export type SkillCatalogueEntry = z.infer<typeof SkillCatalogueEntrySchema>;
-
-/** Versioned persisted skill catalogue. */
-export type SkillCatalogue = z.infer<typeof SkillCatalogueSchema>;
-
-/** One ordered filesystem root contributing skills to a catalogue. */
-export interface SkillCatalogueSource {
-  /** Stable source identity including its mount precedence. */
-  id: string;
-  /** List one source-relative directory with provider metadata when available. */
-  list(path: string): Promise<SkillCatalogueFileEntry[]>;
-  /** Read one source-relative file fresh from its owning filesystem. */
-  read(path: string): Promise<string>;
-}
-
-/** Minimal directory metadata used to compare one catalogue refresh. */
-export interface SkillCatalogueFileEntry {
-  metadata?: Readonly<Record<string, unknown>>;
-  name: string;
-  size?: number;
-  type: "directory" | "file";
-}
-
-/** Options for one user-scoped workspace skill catalogue. */
-export interface WorkspaceSkillCatalogueOptions {
-  host: string;
-  sources: readonly SkillCatalogueSource[];
-  ttlMs?: number;
-  userKey: string;
-}
-
-/** Request state used to resolve one user-scoped catalogue owner. */
-export interface SkillCatalogueResolveContext {
-  abortSignal?: AbortSignal;
-  requestContext?: RequestContext;
-}
-
-/** Resolve one request's complete skill catalogue. */
-export type SkillCatalogueResolver = (
-  context: SkillCatalogueResolveContext,
-) => Promise<WorkspaceSkillCatalogue>;
-
-interface ParsedSkill {
-  description: string;
-  instructions: string;
-  name: string;
-}
-
-const scopedCatalogues = new LRUCache<string, WorkspaceSkillCatalogue>({
-  max: MAX_SCOPED_CATALOGUES,
-});
-
-/** Default lifetime for one complete workspace skill catalogue. */
-export const DEFAULT_WORKSPACE_SKILL_CACHE_TTL_MS = 5 * 60 * 1000;
-
-/** Materialized catalogue with one AppKit cache record and fresh auxiliary reads. */
-export class WorkspaceSkillCatalogue {
-  readonly cacheKey: string;
-  readonly host: string;
-  readonly userKey: string;
-
-  private lastValid?: SkillCatalogue;
-  private readonly ttlMs: number;
-  private sources: readonly SkillCatalogueSource[];
-
-  constructor(private readonly options: WorkspaceSkillCatalogueOptions) {
-    this.host = options.host;
-    this.userKey = options.userKey;
-    this.sources = options.sources;
-    this.ttlMs = positiveTtl(options.ttlMs);
-    this.cacheKey = object.toStableKey({
-      host: options.host,
-      roots: options.sources.map(({ id }) => id),
-      userKey: options.userKey,
-      version: CATALOGUE_VERSION,
-    });
-  }
-
-  /** Rebind refreshes and lazy reads to the current request's filesystem clients. */
-  bind(sources: readonly SkillCatalogueSource[]): void {
-    const identities = sources.map(({ id }) => id);
-    if (object.toStableKey(identities) !== object.toStableKey(this.sources.map(({ id }) => id))) {
-      throw new Error("Cannot rebind a skill catalogue to a different ordered root set");
-    }
-    this.sources = sources;
-  }
-
-  /** Return the cached catalogue, refreshing all roots once on cache miss. */
-  async get(signal?: AbortSignal): Promise<SkillCatalogue> {
-    const cache = CacheManager.getInstanceSync();
-    try {
-      const catalogue = SkillCatalogueSchema.parse(
-        await cache.getOrExecute(
-          [CACHE_NAMESPACE, CATALOGUE_VERSION, this.options.host, this.rootKey()],
-          () => this.refresh(),
-          this.options.userKey,
-          {
-            ttl: Math.max(1, Math.ceil(this.ttlMs / 1000)),
-            callerSignal: signal,
-          },
-        ),
-      );
-      this.lastValid = catalogue;
-      return catalogue;
-    } catch (error) {
-      if (!this.lastValid) throw error;
-      logger.warn("skill catalogue refresh failed; retaining last valid catalogue", {
-        error: errorUtils.errorMessage(error),
-        roots: this.sources.map(({ id }) => id),
-      });
-      return this.lastValid;
-    }
-  }
-
-  /** Expire this complete catalogue without touching unrelated cache entries. */
-  async invalidate(): Promise<void> {
-    const cache = CacheManager.getInstanceSync();
-    await cache.delete(this.persistentKey(cache));
-  }
-
-  /** Read one selected skill's auxiliary file directly from its current source. */
-  async readSkillFile(
-    skillName: string,
-    inputPath: string,
-    signal?: AbortSignal,
-  ): Promise<string> {
-    signal?.throwIfAborted();
-    const catalogue = await this.get(signal);
-    const skill = catalogue.skills.find(({ name }) => name === skillName);
-    if (!skill) throw new Error(`Unknown skill: ${skillName}`);
-    const normalized = posixPath.normalize(inputPath);
-    if (
-      !normalized.ok ||
-      normalized.path === "/" ||
-      inputPath.split("/").some((segment) => segment === "..")
-    ) {
-      throw new Error(`Invalid skill file path: ${inputPath}`);
-    }
-    const source = this.sources.find(({ id }) => id === skill.sourceId);
-    if (!source) throw new Error(`Skill source is no longer mounted: ${skill.sourceId}`);
-    const parent = skill.sourcePath.split("/").slice(0, -1).join("/");
-    const path = [parent, normalized.path.replace(/^\/+/, "")].filter(Boolean).join("/");
-    return source.read(path);
-  }
-
-  private async refresh(): Promise<SkillCatalogue> {
-    const previous = new Map(
-      this.lastValid?.skills.map((skill) => [`${skill.sourceId}\0${skill.sourcePath}`, skill]) ?? [],
-    );
-    const selected = new Map<string, SkillCatalogueEntry>();
-    for (const source of this.sources) {
-      for (const candidate of await discoverSkillFiles(source)) {
-        const prior = previous.get(`${source.id}\0${candidate.path}`);
-        const signature = fileSignature(candidate.entry);
-        let parsed: ParsedSkill | undefined;
-        if (signature && prior?.signature === signature) {
-          parsed = prior;
-        } else {
-          parsed = parseSkill(
-            await source.read(candidate.path),
-            candidate.path,
-          );
-        }
-        if (!parsed || selected.has(parsed.name)) continue;
-        selected.set(parsed.name, {
-          ...parsed,
-          sourceId: source.id,
-          sourcePath: candidate.path,
-          ...(signature ? { signature } : {}),
-        });
-      }
-    }
-    return SkillCatalogueSchema.parse({
-      version: CATALOGUE_VERSION,
-      generatedAt: new Date().toISOString(),
-      roots: this.sources.map(({ id }) => id),
-      skills: [...selected.values()],
-    });
-  }
-
-  private rootKey(): string {
-    return object.toStableKey(this.sources.map(({ id }) => id));
-  }
-
-  private persistentKey(cache: CacheManager): string {
-    return cache.generateKey(
-      [CACHE_NAMESPACE, CATALOGUE_VERSION, this.options.host, this.rootKey()],
-      this.options.userKey,
-    );
-  }
-}
-
-/** Return one stable catalogue owner for a user, host, and ordered root set. */
-export function workspaceSkillCatalogue(
-  options: WorkspaceSkillCatalogueOptions,
-): WorkspaceSkillCatalogue {
-  const key = object.toStableKey({
-    host: options.host,
-    roots: options.sources.map(({ id }) => id),
-    userKey: options.userKey,
-  });
-  const existing = scopedCatalogues.get(key);
-  if (existing) {
-    existing.bind(options.sources);
-    return existing;
-  }
-  const catalogue = new WorkspaceSkillCatalogue(options);
-  scopedCatalogues.set(key, catalogue);
-  return catalogue;
-}
-
-/** Invalidate matching complete catalogues and retain their stable owners. */
-export async function clearWorkspaceSkillCache(options: {
-  host: string;
-  userKey?: string;
-}): Promise<void> {
-  const matches = [...scopedCatalogues.values()].filter(
-    (catalogue) =>
-      catalogue.host === options.host &&
-      (options.userKey === undefined || catalogue.userKey === options.userKey),
-  );
-  await Promise.all(matches.map((catalogue) => catalogue.invalidate()));
-}
-
-async function discoverSkillFiles(
-  source: SkillCatalogueSource,
-): Promise<Array<{ entry: SkillCatalogueFileEntry; path: string }>> {
-  let rootEntries: SkillCatalogueFileEntry[];
-  try {
-    rootEntries = await source.list(".");
-  } catch (error) {
-    if (errorUtils.errorContext(error).notAccessible) return [];
-    throw error;
-  }
-  const candidates: Array<{ entry: SkillCatalogueFileEntry; path: string }> = [];
-  const rootSkill = rootEntries.find(
-    ({ name, type }) => name.toLowerCase() === "skill.md" && type === "file",
-  );
-  if (rootSkill) candidates.push({ entry: rootSkill, path: rootSkill.name });
-  await Promise.all(
-    rootEntries
-      .filter(({ type }) => type === "directory")
-      .map(async (directory) => {
-        const entries = await source.list(directory.name);
-        const skill = entries.find(
-          ({ name, type }) => name.toLowerCase() === "skill.md" && type === "file",
-        );
-        if (skill) candidates.push({ entry: skill, path: posixPath.join(directory.name, skill.name) });
-      }),
-  );
-  return candidates.sort((left, right) => left.path.localeCompare(right.path));
-}
-
-function parseSkill(source: string, path: string): ParsedSkill | undefined {
-  const match = /^---\s*\r?\n([\s\S]*?)\r?\n---\s*(?:\r?\n|$)([\s\S]*)$/.exec(source);
-  if (!match) {
-    logger.warn("skill skipped (missing YAML frontmatter)", { path });
-    return undefined;
-  }
-  const metadata = parseYaml(match[1]!) as unknown;
-  if (!object.isRecord(metadata)) {
-    logger.warn("skill skipped (invalid YAML frontmatter)", { path });
-    return undefined;
-  }
-  const parsed = z
-    .object({
-      name: z.string().trim().min(1),
-      description: z.string().trim().min(1),
-    })
-    .safeParse(metadata);
-  if (!parsed.success) {
-    logger.warn("skill skipped (invalid metadata)", {
-      path,
-      error: parsed.error.message,
-    });
-    return undefined;
-  }
-  return {
-    name: parsed.data.name,
-    description: parsed.data.description,
-    instructions: match[2]!.trim(),
-  };
-}
-
-function fileSignature(entry: SkillCatalogueFileEntry): string | undefined {
-  const metadata = object.isRecord(entry.metadata) ? entry.metadata : {};
-  const values = {
-    modifiedAt: metadata.modifiedAt,
-    objectId: metadata.objectId,
-    size: entry.size,
-  };
-  return Object.values(values).some((value) => value !== undefined)
-    ? object.toStableKey(values)
-    : undefined;
-}
-
-function positiveTtl(value = DEFAULT_WORKSPACE_SKILL_CACHE_TTL_MS): number {
-  if (!Number.isFinite(value) || value <= 0) {
-    throw new RangeError("Workspace skill catalogue ttlMs must be a positive finite number");
-  }
-  return value;
-}

package/src/skill-search.ts

@@ -1,203 +0,0 @@
-/**
- * On-demand skill discovery backed by a materialized workspace catalogue.
- *
- * @module
- */
-
-import { errorUtils } from "@dbx-tools/shared-core";
-import type { ProcessInputStepArgs, Processor } from "@mastra/core/processors";
-import { MASTRA_THREAD_ID_KEY } from "@mastra/core/request-context";
-import { createTool } from "@mastra/core/tools";
-import Fuse from "fuse.js";
-import { z } from "zod";
-import type { SkillCatalogueResolver } from "./skill-cache.ts";
-
-const DEFAULT_STATE_TTL_MS = 60 * 60 * 1000;
-
-interface ThreadState {
-  lastAccessed: number;
-  skills: Map<string, string>;
-}
-
-/** Configuration for {@link CatalogueSkillSearchProcessor}. */
-export interface CatalogueSkillSearchProcessorOptions {
-  minScore?: number;
-  resolve: SkillCatalogueResolver;
-  topK?: number;
-  ttlMs?: number;
-}
-
-/** Mastra-compatible search, load, and lazy-read tools over one cached catalogue. */
-export class CatalogueSkillSearchProcessor implements Processor<"skill-catalogue-search"> {
-  readonly id = "skill-catalogue-search";
-  readonly name = "Skill Catalogue Search";
-  readonly providesSkillDiscovery = "on-demand" as const;
-
-  private readonly minScore: number;
-  private readonly resolve: SkillCatalogueResolver;
-  private readonly states = new Map<string, ThreadState>();
-  private readonly topK: number;
-  private readonly ttlMs: number;
-
-  constructor(options: CatalogueSkillSearchProcessorOptions) {
-    this.resolve = options.resolve;
-    this.topK = options.topK ?? 5;
-    this.minScore = options.minScore ?? 0.1;
-    this.ttlMs = options.ttlMs ?? DEFAULT_STATE_TTL_MS;
-  }
-
-  async processInputStep(args: ProcessInputStepArgs): Promise<{ tools: Record<string, unknown> }> {
-    const owner = await this.resolve(args);
-    const catalogue = await owner.get(args.abortSignal);
-    const state = this.threadState(args);
-    args.messageList.addSystem(
-      "To discover available skills, call search_skills with a keyword query. " +
-        "To load a skill's instructions, call load_skill with the exact skill name. " +
-        "Use skill_read only after loading a skill when its instructions require a reference, script, or asset.",
-    );
-    for (const [skillName, instructions] of state.skills) {
-      args.messageList.addSystem(`[Skill: ${skillName}]\n\n${instructions}`);
-    }
-    const search = new Fuse(catalogue.skills, {
-      includeScore: true,
-      ignoreLocation: true,
-      threshold: 1,
-      keys: [
-        { name: "name", weight: 0.5 },
-        { name: "description", weight: 0.35 },
-        { name: "instructions", weight: 0.15 },
-      ],
-    });
-    return {
-      tools: {
-        ...(args.tools ?? {}),
-        search_skills: createTool({
-          id: "search_skills",
-          description:
-            "Search for available skills by keyword. Load a useful result with load_skill.",
-          inputSchema: z.object({
-            query: z.string().trim().min(1, "Query is required").describe("Search keywords."),
-          }),
-          outputSchema: z.object({
-            results: z.array(
-              z.object({
-                name: z.string(),
-                description: z.string(),
-                score: z.number(),
-              }),
-            ),
-            message: z.string(),
-          }),
-          execute: async ({ query }) => {
-            const results = search
-              .search(query)
-              .map((result) => ({
-                name: result.item.name,
-                description:
-                  result.item.description.length > 150
-                    ? `${result.item.description.slice(0, 147)}...`
-                    : result.item.description,
-                score: Math.round((1 - (result.score ?? 1)) * 100) / 100,
-              }))
-              .filter(({ score }) => score >= this.minScore)
-              .slice(0, this.topK);
-            return {
-              results,
-              message:
-                results.length === 0
-                  ? `No skills found matching "${query}". Try different keywords.`
-                  : `Found ${results.length} skill(s). Use load_skill with the exact skill name to load its instructions.`,
-            };
-          },
-        }),
-        load_skill: createTool({
-          id: "load_skill",
-          description: "Load a skill's full instructions into the conversation.",
-          inputSchema: z.object({
-            skillName: z.string().describe("Exact skill name returned by search_skills."),
-          }),
-          outputSchema: z.object({
-            success: z.boolean(),
-            message: z.string(),
-            skillName: z.string().optional(),
-          }),
-          execute: async ({ skillName }) => {
-            if (state.skills.has(skillName)) {
-              return {
-                success: true,
-                message: `Skill "${skillName}" is already loaded.`,
-                skillName,
-              };
-            }
-            const skill = catalogue.skills.find(({ name }) => name === skillName);
-            if (!skill) {
-              return {
-                success: false,
-                message: `Skill "${skillName}" not found. Use search_skills to find available skills.`,
-              };
-            }
-            state.skills.set(skillName, skill.instructions);
-            return {
-              success: true,
-              message: `Skill "${skillName}" loaded. Its instructions are now available as context.`,
-              skillName,
-            };
-          },
-        }),
-        skill_read: createTool({
-          id: "skill_read",
-          description:
-            "Read a reference, script, template, or asset from a loaded skill. The path is relative to the skill root.",
-          inputSchema: z.object({
-            skillName: z.string().describe("Exact loaded skill name."),
-            path: z.string().describe("File path relative to the skill root."),
-            startLine: z.number().int().positive().optional(),
-            endLine: z.number().int().positive().optional(),
-          }),
-          execute: async ({ skillName, path, startLine, endLine }) => {
-            if (!state.skills.has(skillName)) {
-              return `Skill "${skillName}" is not loaded. Call load_skill first.`;
-            }
-            try {
-              const source = await owner.readSkillFile(skillName, path, args.abortSignal);
-              return lineRange(source, path, startLine, endLine);
-            } catch (error) {
-              return `Could not read "${path}" from skill "${skillName}": ${errorUtils.errorMessage(error)}`;
-            }
-          },
-        }),
-      },
-    };
-  }
-
-  private threadState(args: ProcessInputStepArgs): ThreadState {
-    const now = Date.now();
-    for (const [key, state] of this.states) {
-      if (now - state.lastAccessed > this.ttlMs) this.states.delete(key);
-    }
-    const key = args.requestContext?.get(MASTRA_THREAD_ID_KEY) ?? "default";
-    const current = this.states.get(String(key)) ?? {
-      lastAccessed: now,
-      skills: new Map<string, string>(),
-    };
-    current.lastAccessed = now;
-    this.states.set(String(key), current);
-    return current;
-  }
-}
-
-function lineRange(
-  source: string,
-  path: string,
-  startLine: number | undefined,
-  endLine: number | undefined,
-): string {
-  if (startLine === undefined && endLine === undefined) return source;
-  const lines = source.split(/\r?\n/);
-  const start = Math.max(1, startLine ?? 1);
-  const end = Math.min(lines.length, endLine ?? lines.length);
-  if (start > end || start > lines.length) {
-    return `File "${path}" has ${lines.length} lines; requested range ${start}-${end} is empty.`;
-  }
-  return `${path} (lines ${start}-${end} of ${lines.length})\n${lines.slice(start - 1, end).join("\n")}`;
-}