@dbx-tools/appkit-mastra 0.1.26 → 0.1.28

package/index.ts

@@ -1,7 +1,8 @@
 /**
- * AppKit Mastra integration: {@link MastraPlugin} / {@link mastra},
- * plugin config types, agent registration helpers, Genie tool
- * builders, and dynamic Model Serving endpoint resolution.
+ * Server-side entry point for the AppKit Mastra integration. Mounts
+ * the plugin via {@link mastra} and re-exports the full server surface
+ * (config, agent wiring, Genie and chart tooling, and dynamic Model
+ * Serving resolution) so apps build agent backends from one import.
  *
  * Client-side consumers should import URL helpers and the
  * {@link MastraClientConfig} type from `@dbx-tools/appkit-mastra-shared`

package/package.json

@@ -9,13 +9,13 @@
     }
   },
   "name": "@dbx-tools/appkit-mastra",
-  "version": "0.1.26",
+  "version": "0.1.28",
   "dependencies": {
     "@databricks/sdk-experimental": "^0.17",
-    "@dbx-tools/appkit-mastra-shared": "0.1.26",
-    "@dbx-tools/genie": "0.1.26",
-    "@dbx-tools/genie-shared": "0.1.26",
-    "@dbx-tools/shared": "0.1.26",
+    "@dbx-tools/appkit-mastra-shared": "0.1.28",
+    "@dbx-tools/genie": "0.1.28",
+    "@dbx-tools/genie-shared": "0.1.28",
+    "@dbx-tools/shared": "0.1.28",
     "@mastra/ai-sdk": "^1",
     "@mastra/core": "^1",
     "@mastra/express": "^1",

package/src/agents.ts

@@ -431,7 +431,7 @@ export async function buildAgents(opts: {
   const style = resolveStyleInstructions(config);
   // Default-on protection against the model copying turn-scoped
   // chartIds from prior assistant tool results into the new
-  // turn's `[[chart:<id>]]` markers. Opt out per-plugin via
+  // turn's `[chart:<id>]` markers. Opt out per-plugin via
   // `config.stripStaleCharts: false`.
   const inputProcessors =
     config.stripStaleCharts === false ? [] : [stripStaleChartsProcessor];

package/src/chart.ts

@@ -20,18 +20,9 @@
  * dataset, custom API). The module has no knowledge of Genie or
  * statement ids; those concerns live in the tools that wrap it.
  *
- * Wire-format schemas (`ChartSchema`, `ChartResultSchema`,
- * {@link ChartTypeSchema}) live in
- * `@dbx-tools/appkit-mastra-shared` so the demo client and any
- * other UI consumer share the exact same shape this module reads
- * and writes.
- *
- * Public surface (everything else is module-private):
- *   - {@link chartPlannerRequestSchema} / {@link ChartPlannerRequest}
- *   - {@link prepareChart} / {@link PrepareChartOptions}
- *   - {@link fetchChart} / {@link FetchChartOptions}
- *   - {@link buildRenderDataTool} (the `render_data` Mastra tool
- *     auto-wired on every agent in `agents.ts`)
+ * Wire-format schemas live in `@dbx-tools/appkit-mastra-shared` so
+ * the demo client and any other UI consumer share the exact same
+ * shape this module reads and writes.
  */
 
 import { CacheManager } from "@databricks/appkit";
@@ -238,13 +229,11 @@ type ChartPlan = z.infer<typeof chartPlanSchema>;
  * sees a planner request, only the resolved {@link Chart}.
  */
 export const chartPlannerRequestSchema = z.object({
-  title: z
-    .string()
-    .describe(
-      stringUtils.toDescription(`
+  title: z.string().describe(
+    stringUtils.toDescription(`
         Concise title shown above the chart (e.g. "Top 10 SKUs by Revenue").
       `),
-    ),
+  ),
   description: z
     .string()
     .optional()
@@ -676,7 +665,7 @@ function planToEchartsOption(
  * an empty placeholder, and kicks off the chart-planner in the
  * background. Returns just the `chartId`; the host UI resolves
  * `[chart:<chartId>]` markers by hitting the plugin's
- * `/charts/:chartId` route.
+ * `/embed/chart/:id` route.
  *
  * For Genie statement results, prefer the Genie agent's
  * `prepare_chart` tool, which accepts a `statement_id` and

package/src/config.ts

@@ -216,7 +216,7 @@ export interface MastraPluginConfig extends BasePluginConfig {
    * processor that strips `chartId` fields from prior assistant
    * tool-invocation results before they reach the model. This
    * prevents the model from reusing turn-scoped chartIds it sees
-   * in memory recall (which would leave `[[chart:<id>]]` markers
+   * in memory recall (which would leave `[chart:<id>]` markers
    * pointing at writer events that no longer exist).
    *
    * Set to `false` to opt out - useful if a non-default agent

package/src/genie.ts

@@ -1,41 +1,19 @@
 /**
  * Genie tools for Mastra.
  *
- * Each configured Genie space is surfaced as a small set of flat
- * Mastra tools the calling agent can drive directly - no inner
- * orchestrator agent. The central agent decomposes user questions,
- * picks which Genie space to ask, and composes the final reply.
- *
- * Per-space tools (suffixed with `_<alias>` for non-default
- * aliases):
- *
- *   - `ask_genie`: drives one `genieEventChat` turn and
- *     forwards every wire event (status, thinking, sql, rows)
- *     through `ctx.writer` for streaming UI updates. Returns
- *     the terminal `GenieMessage` only - rows are NOT fetched
- *     eagerly.
- *   - `get_space_description`: cheap title / description /
- *     warehouse id lookup for grounding.
- *   - `get_space_serialized`: full `GenieSpace` JSON for
- *     column-level grounding when the description isn't enough.
- *
- * Space-agnostic shared tools (registered once, regardless of
- * how many spaces are wired):
- *
- *   - `get_statement`: opt-in lookup of a Genie statement's rows
- *     by `statement_id` (with a row `limit`). The agent calls
- *     this only when it needs to read values to reason about
- *     them; if the data is just being displayed, it embeds a
- *     `[data:<statement_id>]` marker in prose instead and lets
- *     the host UI resolve it.
- *   - `prepare_chart`: mints a short `chartId`, kicks off a
- *     background task that fetches the statement's rows and
- *     runs the chart-planner, and caches the resolved Echarts
- *     spec under the id (1h TTL). Returns the `chartId`
- *     synchronously so the agent embeds `[chart:<chartId>]`
- *     markers in prose without blocking on chart generation;
- *     the host UI fetches the cached chart by id once it's
- *     ready (see {@link fetchChart}).
+ * Surfaces each configured Genie space as a small set of flat Mastra
+ * tools the calling agent drives directly - no inner orchestrator
+ * agent. The central agent decomposes user questions, picks which
+ * space to ask, streams the per-turn wire events (status, thinking,
+ * sql, rows) through `ctx.writer`, and composes the final reply.
+ * Rows are never fetched eagerly: the agent reads a statement's
+ * values only when it needs to reason about them, otherwise it embeds
+ * a `[data:<statement_id>]` marker in prose and lets the host UI
+ * resolve the data. Charts are minted asynchronously and referenced
+ * by `[chart:<chartId>]` markers so prose isn't blocked on chart
+ * generation; the host UI fetches the cached spec by id once ready.
+ * Space description and serialized-space lookups are available for
+ * grounding when the agent needs schema context.
  *
  * Each tool's `execute` pulls the per-request
  * {@link WorkspaceClient} off `ctx.requestContext` (stamped by
@@ -725,9 +703,13 @@ function buildPrepareChartTool(opts: { config: MastraPluginConfig }) {
       `
         To display the chart in your reply, embed
         \`[chart:<chartId>]\` on its own line at the position you
-        want it to appear. The tool returns immediately - do NOT
-        wait or call it again to "check progress"; the chart
-        resolves asynchronously on the host UI's side.
+        want it to appear, using the EXACT \`chartId\` string this
+        call returned. Never construct a chart id yourself (it is
+        not the \`statement_id\` or any variation of it) - only a
+        value returned by this tool resolves to a real chart. The
+        tool returns immediately - do NOT wait or call it again to
+        "check progress"; the chart resolves asynchronously on the
+        host UI's side.
       `,
       `
         Use this only when the data has a story a chart conveys
@@ -842,6 +824,17 @@ export const GENIE_INSTRUCTIONS = stringUtils.toDescription([
               chart should appear. Use a chart when the data has a
               story a visual conveys better than a table (trends,
               rankings, distributions, parts-of-a-whole).
+
+              NEVER invent or hand-build a \`<chartId>\`. A valid
+              \`<chartId>\` is the opaque token a \`prepare_chart\`
+              call returned to you in THIS turn - nothing else. It
+              is NOT a \`statement_id\`, and it is NOT a
+              \`statement_id\` prefix with a label appended (e.g.
+              \`01f1...-region-fill\`). If you have not called
+              \`prepare_chart\` and received an id back, do not write
+              a \`[chart:...]\` marker at all - use \`[data:...]\`
+              instead. A fabricated chart id renders nothing and
+              wastes a request.
             `,
           ],
         },

package/src/intercept.ts

@@ -0,0 +1,206 @@
+/**
+ * Server-side Server-Sent-Events frame interceptor for the native
+ * Mastra agent `/stream` endpoint.
+ *
+ * The Mastra stream (`@mastra/express`) emits one `data: <json>\n\n`
+ * SSE frame per chunk, where the decoded JSON carries a
+ * discriminating `type` (`text-delta`, `reasoning-delta`,
+ * `tool-call`, `step-finish`, ...) and is terminated by a
+ * `data: [DONE]` sentinel. {@link installStreamEventInterceptor} wraps
+ * an Express response and runs a caller-supplied
+ * {@link StreamFrameInterceptor} over each decoded frame so the caller
+ * can keep, rewrite, or drop it; every non-`data:` byte (SSE
+ * keep-alive comments, the `[DONE]` sentinel, blank padding) passes
+ * through verbatim.
+ *
+ * The wrap is inert unless the response turns out to be a streamed
+ * `200 text/event-stream` body, so non-streaming JSON routes and
+ * error responses are never touched.
+ */
+
+import { StringDecoder } from "node:string_decoder";
+
+import type express from "express";
+
+/** SSE event separator used by the Mastra stream wire format. */
+const FRAME_DELIMITER = "\n\n";
+
+/**
+ * Matches a whole SSE frame whose `data:` payload is a JSON object,
+ * capturing it in three parts: the `data:` field prefix (including
+ * whatever inter-token whitespace the producer used), the JSON object
+ * body, and any trailing whitespace. Anchored end-to-end so it only
+ * matches a single self-contained object frame; anything else (SSE
+ * comments, the `[DONE]` sentinel, text/array payloads) fails the
+ * match and is forwarded verbatim without a parse attempt. On rewrite
+ * the captured prefix/suffix are re-emitted as-is so only the body
+ * changes.
+ */
+const DATA_OBJECT_RE = /^(data:\s*)(\{[\s\S]*\})(\s*)$/;
+
+/**
+ * Per-frame interceptor invoked with the `JSON.parse`d object of every
+ * `data: {json}` frame on the Mastra `/stream` response. The return
+ * value decides the frame's fate:
+ *
+ * - `true`: forward the frame's original bytes unchanged (no
+ *   re-serialization).
+ * - `false`: drop the frame entirely.
+ * - `{ replace }`: re-emit the frame with the original `data:` prefix
+ *   and trailing whitespace preserved and only the JSON body replaced
+ *   by `JSON.stringify(replace)`.
+ */
+export type StreamFrameInterceptor = (
+  chunk: unknown,
+) => { replace: unknown } | true | false;
+
+/**
+ * True when `res` is a `200 text/event-stream` body - the only
+ * response shape we intercept. Content-type based so it tracks the
+ * Mastra `/stream` route without hard-coding the path, and naturally
+ * skips the JSON routes (history, models, charts, statements) and any
+ * non-200 error response.
+ */
+function isEventStream(res: express.Response): boolean {
+  if (res.statusCode !== 200) return false;
+  const contentType = String(res.getHeader("content-type") ?? "").toLowerCase();
+  return contentType.includes("text/event-stream");
+}
+
+/**
+ * Run `interceptor` over a single SSE frame. Forwards anything that
+ * isn't a parseable `data: {json}` object frame verbatim (comments,
+ * the `[DONE]` sentinel, blank padding, unparseable / non-object
+ * payloads) so the interceptor only ever sees real chunk objects.
+ * Returns the frame string to emit, or `undefined` to drop the frame.
+ */
+function interceptFrame(
+  frame: string,
+  interceptor: StreamFrameInterceptor,
+): string | undefined {
+  const match = DATA_OBJECT_RE.exec(frame);
+  if (!match) return frame;
+  const [, prefix, body, suffix] = match;
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(body!);
+  } catch {
+    // Forward unparseable payloads untouched - feeding a partial /
+    // malformed frame to the interceptor risks corrupting bytes the
+    // client could still have handled.
+    return frame;
+  }
+  const result = interceptor(parsed);
+  if (result === true) return frame;
+  if (result === false) return undefined;
+  // Preserve the exact captured prefix/suffix so only the JSON body
+  // changes; the framing the client parses stays byte-identical.
+  return `${prefix}${JSON.stringify(result.replace)}${suffix}`;
+}
+
+/**
+ * Wrap `res.write` / `res.end` so each SSE frame on a streamed Mastra
+ * response is run through `interceptor` (keep / rewrite / drop).
+ *
+ * Interception only engages once the response is confirmed to be a
+ * `200 text/event-stream` body (decided lazily on the first write,
+ * after the handler has set status + content-type); any other
+ * response streams through unchanged. A {@link StringDecoder}
+ * reassembles multi-byte UTF-8 sequences that straddle chunk
+ * boundaries, and a running buffer holds the trailing partial frame
+ * until its `\n\n` arrives.
+ */
+export function installStreamEventInterceptor(
+  res: express.Response,
+  interceptor: StreamFrameInterceptor,
+): void {
+  const origWrite = res.write.bind(res);
+  const origEnd = res.end.bind(res);
+  const decoder = new StringDecoder("utf8");
+  let engaged: boolean | undefined;
+  let buffer = "";
+
+  // Append decoded text, emit every whole `\n\n`-delimited frame the
+  // interceptor keeps, and retain any trailing partial frame in
+  // `buffer`. On `final`, the residual buffer is flushed as the last
+  // frame so a stream that doesn't end on a delimiter isn't dropped.
+  const intercept = (text: string, final: boolean): string => {
+    buffer += text;
+
+    const out: string[] = [];
+
+    let idx: number;
+    while ((idx = buffer.indexOf(FRAME_DELIMITER)) !== -1) {
+      const frame = buffer.slice(0, idx);
+      buffer = buffer.slice(idx + FRAME_DELIMITER.length);
+
+      const kept = interceptFrame(frame, interceptor);
+      if (kept !== undefined) {
+        out.push(kept, FRAME_DELIMITER);
+      }
+    }
+
+    if (final && buffer.length > 0) {
+      const kept = interceptFrame(buffer, interceptor);
+      if (kept !== undefined) out.push(kept);
+      buffer = "";
+    }
+
+    return out.length === 0 ? "" : out.join("");
+  };
+
+  const toText = (chunk: unknown): string => {
+    if (typeof chunk === "string") return chunk;
+    if (Buffer.isBuffer(chunk)) return decoder.write(chunk);
+    if (chunk instanceof Uint8Array) {
+      return decoder.write(
+        Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength),
+      );
+    }
+    return "";
+  };
+
+  res.write = function patchedWrite(
+    chunk: unknown,
+    encodingOrCb?: BufferEncoding | ((error?: Error | null) => void),
+    cb?: (error?: Error | null) => void,
+  ): boolean {
+    if (engaged === undefined) engaged = isEventStream(res);
+    if (!engaged) {
+      return origWrite(chunk as never, encodingOrCb as never, cb as never);
+    }
+    const callback = typeof encodingOrCb === "function" ? encodingOrCb : cb;
+    const out = intercept(toText(chunk), false);
+    if (out.length === 0) {
+      // The chunk only advanced a partial frame; nothing to forward
+      // yet. Honor the write callback so backpressure-aware writers
+      // don't stall waiting on an ack that never comes.
+      if (callback) queueMicrotask(() => callback());
+      return true;
+    }
+    return origWrite(out, callback as never);
+  } as typeof res.write;
+
+  res.end = function patchedEnd(
+    chunk?: unknown | (() => void),
+    encodingOrCb?: BufferEncoding | (() => void),
+    cb?: () => void,
+  ): express.Response {
+    if (engaged === undefined) engaged = isEventStream(res);
+    if (!engaged) {
+      return origEnd(chunk as never, encodingOrCb as never, cb as never);
+    }
+    const callback =
+      typeof chunk === "function"
+        ? chunk
+        : typeof encodingOrCb === "function"
+          ? encodingOrCb
+          : cb;
+    const text =
+      (typeof chunk !== "function" && chunk !== undefined ? toText(chunk) : "") +
+      decoder.end();
+    const out = intercept(text, true);
+    if (out.length > 0) origWrite(out);
+    return origEnd(callback as never);
+  } as typeof res.end;
+}