@malloy-publisher/server 0.0.231 → 0.0.233

package/src/mcp/tools/execute_query_tool.spec.ts

@@ -13,7 +13,11 @@ import {
 // compile_tool.spec.ts and reload_package_tool.spec.ts.
 type Handler = (params: Record<string, unknown>) => Promise<{
    isError?: boolean;
-   content: Array<{ resource: { text: string } }>;
+   content: Array<{
+      type?: string;
+      text?: string;
+      resource?: { text: string };
+   }>;
 }>;
 
 function captureHandler(store: Partial<EnvironmentStore>): Handler {
@@ -28,8 +32,8 @@ function captureHandler(store: Partial<EnvironmentStore>): Handler {
    return handler;
 }
 
-function parse(result: { content: Array<{ resource: { text: string } }> }) {
-   return JSON.parse(result.content[0].resource.text);
+function parse(result: { content: Array<{ resource?: { text: string } }> }) {
+   return JSON.parse(result.content[0].resource!.text);
 }
 
 /**
@@ -140,4 +144,20 @@ describe("malloy_executeQuery error classification", () => {
       // one; echoing the name back would undo that.
       expect(parsed.error).not.toContain("salaries");
    });
+
+   it("also states the error in a text block", async () => {
+      // The structured payload rides in an embedded resource block. A client
+      // that renders only text blocks on an isError result shows nothing at
+      // all for it, which is how a real diagnostic surfaces to the agent as a
+      // bare "Unknown error". Every error must say it in plain text too.
+      const handler = captureHandler(
+         storeWhoseQueryThrows(new MalloyError("unexpected '@'", [])),
+      );
+      const result = await handler(args);
+      const parsed = parse(result);
+
+      const textBlock = result.content.find((b) => b.type === "text");
+      expect(textBlock).toBeDefined();
+      expect(textBlock!.text).toContain(parsed.error);
+   });
 });

package/src/mcp/tools/execute_query_tool.ts

@@ -16,6 +16,9 @@ import {
    classifyToolError,
    getModelForQuery,
 } from "../handler_utils";
+import { jsonResource, jsonToolError } from "../tool_response";
+import { buildQueryEnvelope } from "../query_envelope";
+import { bigIntReplacer } from "../../json_utils";
 import { MCP_ERROR_MESSAGES } from "../mcp_constants";
 
 /**
@@ -61,6 +64,26 @@ const executeQueryShape = {
       ),
 };
 
+const EXECUTE_QUERY_DESCRIPTION = `Run a Malloy query against a model and return the rows. Takes either ad-hoc Malloy in query, or a named view/query via queryName (with sourceName for a view).
+
+## Contract rules
+- Check _limit_hit before reporting any total, count, or "top N". True means the server's default row cap cut the result off and more rows exist, so the numbers in front of you are a partial set, not the answer. A query that set its own limit: or top: does not set it, and returning exactly that many rows is a complete answer to what was asked.
+- Never sum or count the returned rows to state a total when _limit_hit or _rows_truncated is set. Aggregate in the query instead.
+- _returned_rows: 0 with _rows_truncated set means one row was too large to send, NOT that nothing matched. Do not report it as an empty result.
+- Use source, view, and field names exactly as malloy_getContext returned them.
+
+## Response
+A JSON object, the same shape Credible's execute_query returns, so a data app behaves the same authored locally and served in production:
+- rows: flat objects keyed by column name, the shape an in-package data app receives.
+- _meta: the Malloy metadata flat rows drop (schema with field types and render tags, annotations, connection_name, query_timezone).
+- _query_row_limit: the cap pushed into the SQL, from the query's own limit: or the server default.
+- _limit_source: "query" when the cap came from the query's own limit:/top:, "server_default" otherwise.
+- _limit_hit: the row count equals that cap AND the cap was the server default.
+- _rows_truncated / _total_rows / _returned_rows: present only when the payload cap dropped rows.
+- warning, renderLogErrors: present only when they apply.
+
+A query with no limit: of its own gets the server default, so a result landing exactly on _query_row_limit is almost never the whole table. Values above 2^53 are returned as JSON strings so their digits survive.`;
+
 // Type inference is handled automatically by the MCP server based on the executeQueryShape
 
 /**
@@ -72,7 +95,7 @@ export function registerExecuteQueryTool(
 ): void {
    mcpServer.tool(
       "malloy_executeQuery",
-      "Executes a Malloy query (either ad-hoc or a named query/view defined in a model) against the specified model and returns the results as JSON.",
+      EXECUTE_QUERY_DESCRIPTION,
       executeQueryShape,
       /** Handles requests for the malloy_executeQuery tool */
       async (params) => {
@@ -120,29 +143,10 @@ export function registerExecuteQueryTool(
 
          // Handle errors during package/model access (e.g., not found, initial compilation)
          if ("error" in modelResult) {
-            // Format error details as structured JSON
-            const errorJson = JSON.stringify(
-               {
-                  error: modelResult.error.message,
-                  suggestions: modelResult.error.suggestions,
-               },
-               null,
-               2,
+            return jsonToolError(
+               "error://executeQuery/modelAccess",
+               modelResult.error,
             );
-            return {
-               isError: true,
-               // Return as application/json nested inside a 'resource' type
-               content: [
-                  {
-                     type: "resource", // Use 'resource' type
-                     resource: {
-                        type: "application/json", // Actual content type
-                        uri: "error://executeQuery/modelAccess", // Placeholder URI
-                        text: errorJson,
-                     },
-                  },
-               ],
-            };
          }
 
          // --- Execute Query ---
@@ -160,123 +164,77 @@ export function registerExecuteQueryTool(
          let querySlot: QuerySlotHandle | null = null;
          try {
             querySlot = tryAcquireQuerySlot("mcp:executeQuery");
-            // If ad-hoc query is provided, use it directly in the 3rd arg
-            if (query) {
-               const { result } = await runWithQueryTimeout(
+            // The two call modes differ only in which arguments carry the
+            // query; everything after the run is identical, so they share one
+            // path rather than two copies that can drift.
+            const { result, compactResult, rowLimit, rowLimitSource } =
+               await runWithQueryTimeout(
                   (abortSignal) =>
-                     model.getQueryResults(
-                        undefined,
-                        undefined,
-                        query,
-                        filterParams,
-                        undefined,
-                        givens as Record<string, GivenValue> | undefined,
-                        abortSignal,
-                     ),
+                     query
+                        ? model.getQueryResults(
+                             undefined,
+                             undefined,
+                             query,
+                             filterParams,
+                             undefined,
+                             givens as Record<string, GivenValue> | undefined,
+                             abortSignal,
+                          )
+                        : model.getQueryResults(
+                             sourceName,
+                             queryName,
+                             undefined,
+                             filterParams,
+                             undefined,
+                             givens as Record<string, GivenValue> | undefined,
+                             abortSignal,
+                          ),
                   getQueryTimeoutMs(),
                );
-               const { validateRenderTags } = await import(
-                  "@malloydata/render-validator"
-               );
-               const renderLogs = validateRenderTags(result);
 
-               const baseUriComponents = {
-                  environment: environmentName,
-                  package: packageName,
-                  resourceType: "models" as const,
-                  resourceName: modelPath,
-               };
-               const resultUri = buildMalloyUri(baseUriComponents, "result");
-               const resultString = JSON.stringify(result, null, 2);
-
-               const content = [
-                  {
-                     type: "resource" as const,
-                     resource: {
-                        type: "application/json",
-                        uri: resultUri,
-                        text: resultString,
-                     },
-                  },
-               ];
-
-               if (renderLogs.length > 0) {
-                  return {
-                     isError: false,
-                     content: [
-                        ...content,
-                        {
-                           type: "text" as const,
-                           text: `Render tag warnings:\n${JSON.stringify(renderLogs, null, 2)}`,
-                        },
-                     ],
-                  };
-               }
-
-               return { isError: false, content };
-            } else if (queryName) {
-               const { result } = await runWithQueryTimeout(
-                  (abortSignal) =>
-                     model.getQueryResults(
-                        sourceName,
-                        queryName,
-                        undefined,
-                        filterParams,
-                        undefined,
-                        givens as Record<string, GivenValue> | undefined,
-                        abortSignal,
-                     ),
-                  getQueryTimeoutMs(),
-               );
-               const { validateRenderTags } = await import(
-                  "@malloydata/render-validator"
-               );
-               const renderLogs = validateRenderTags(result);
+            // Render-tag validation reads the FULL Malloy result: the tags live
+            // in its schema annotations, which the flat rows do not carry. It
+            // runs regardless of which shape is returned.
+            const { validateRenderTags } = await import(
+               "@malloydata/render-validator"
+            );
+            const renderLogs = validateRenderTags(result);
 
-               const baseUriComponents = {
+            const resultUri = buildMalloyUri(
+               {
                   environment: environmentName,
                   package: packageName,
                   resourceType: "models" as const,
                   resourceName: modelPath,
-               };
-               const resultUri = buildMalloyUri(baseUriComponents, "result");
-               const resultString = JSON.stringify(result, null, 2);
-
-               const content = [
-                  {
-                     type: "resource" as const,
-                     resource: {
-                        type: "application/json",
-                        uri: resultUri,
-                        text: resultString,
-                     },
-                  },
-               ];
+               },
+               "result",
+            );
 
-               if (renderLogs.length > 0) {
-                  return {
-                     isError: false,
-                     content: [
-                        ...content,
-                        {
-                           type: "text" as const,
-                           text: `Render tag warnings:\n${JSON.stringify(renderLogs, null, 2)}`,
-                        },
-                     ],
-                  };
-               }
+            const envelope = buildQueryEnvelope(
+               compactResult,
+               rowLimit,
+               result,
+               renderLogs.map((log) => log.message),
+               undefined,
+               rowLimitSource,
+            );
 
-               return { isError: false, content };
-            }
+            // A capped or truncated result, and a broken render tag, are the
+            // things an agent most needs to notice, so they are stated in text
+            // rather than left for a client that parses the payload.
+            const notes = [
+               envelope.warning,
+               envelope.renderLogErrors &&
+                  `Render tag problems: ${envelope.renderLogErrors.join("; ")}`,
+            ].filter(Boolean);
 
-            // If execution reaches this point, something has gone wrong with
-            // the earlier parameter validation logic. Throw an explicit error
-            // so the return type is never 'undefined' from the compiler's
-            // perspective.
-            throw new McpError(
-               ErrorCode.InternalError,
-               "Unreachable executeQuery code path – parameters were not validated correctly.",
-            );
+            return jsonResource(resultUri, envelope, {
+               space: 2,
+               // BigInt reaches here: compactResult is raw driver output and
+               // DuckDB returns count() as one.
+               replacer: bigIntReplacer,
+               text: notes.length > 0 ? notes.join("\n\n") : undefined,
+            });
          } catch (queryError) {
             // Handle query execution errors (syntax errors, invalid queries, etc.)
             logger.error(
@@ -306,29 +264,10 @@ export function registerExecuteQueryTool(
                );
             }
 
-            // Format error details as structured JSON
-            const errorJson = JSON.stringify(
-               {
-                  error: errorDetails.message,
-                  suggestions,
-               },
-               null,
-               2,
-            );
-            return {
-               isError: true,
-               // Return as application/json nested inside a 'resource' type
-               content: [
-                  {
-                     type: "resource", // Use 'resource' type
-                     resource: {
-                        type: "application/json", // Actual content type
-                        uri: "error://executeQuery/queryExecution", // Placeholder URI
-                        text: errorJson,
-                     },
-                  },
-               ],
-            };
+            return jsonToolError("error://executeQuery/queryExecution", {
+               message: errorDetails.message,
+               suggestions,
+            });
          } finally {
             // Release on every exit path — success, error, or
             // unreachable code-path throw. `release()` is idempotent

package/src/mcp/tools/get_context_eval.ts

@@ -1,22 +1,34 @@
 /**
- * Lexical-baseline eval for malloy_getContext.
+ * Retrieval eval for malloy_getContext: lexical baseline vs the optional
+ * embedding-backed semantic mode.
  *
  * Runs a labeled set of plain-English queries against the live MCP endpoint
- * and reports recall@K: whether the expected entity appears in the top-K results.
- * This establishes the lexical (lunr/BM25) baseline so retrieval quality is
- * measurable; an embeddings comparison is deferred to if/when an embedding
- * provider is added (see the EPC eval risk).
+ * and reports recall@K: whether the expected entity appears in the top-K
+ * results. Each case is labeled with the retrieval mode the server reported
+ * (`semantic` / `lexical`; servers without an embedding provider report no
+ * mode and run the lexical baseline).
+ *
+ * A/B: run once against a server started WITHOUT `EMBEDDING_API_KEY`
+ * (lexical baseline), then once against a server started with it, and
+ * compare the two summaries. The cases marked `gap` share no tokens with
+ * their target entity's name or docs, so the lexical run is expected to
+ * miss them and the semantic run to hit them.
  *
  *   bun run packages/server/src/mcp/tools/get_context_eval.ts [K]
  *
- * Targets MCP_URL (default http://localhost:4040/mcp) over the demo
- * environment (EVAL_ENV, default "samples"); start the server first.
+ * Targets MCP_URL (default http://localhost:4040/mcp); start the server
+ * first. Cases cover the bundled `examples` environment (storefront), so
+ * the default config works out of the box, plus the classic ecommerce/faa
+ * cases for a malloy-samples config (`samples` environment); cases whose
+ * environment is not served are skipped, not counted as misses.
  */
 
 interface EvalCase {
+   env: string;
    pkg: string;
    query: string;
    expect: string; // substring expected in a top-K result name (case-insensitive)
+   gap?: boolean; // true = no token overlap; lexical is expected to miss
 }
 
 interface ResultEntity {
@@ -25,34 +37,108 @@ interface ResultEntity {
    source?: string | null;
 }
 
-// Ground truth verified against the demo packages (ecommerce, faa).
+interface GetContextPayload {
+   retrieval?: string;
+   results?: ResultEntity[];
+   error?: string;
+}
+
+// Ground truth verified against the bundled examples (storefront) and the
+// malloy-samples demo packages (ecommerce, faa).
 const CASES: EvalCase[] = [
+   // examples/storefront: served by the default config, works out of the box.
+   {
+      env: "examples",
+      pkg: "storefront",
+      query: "revenue by product category",
+      expect: "category",
+   },
+   {
+      env: "examples",
+      pkg: "storefront",
+      query: "top selling products",
+      expect: "top_products",
+   },
+   {
+      env: "examples",
+      pkg: "storefront",
+      query: "monthly sales trend",
+      expect: "sales_by_month",
+   },
+   // Token-gap cases: no word here appears in the entity's name or #(doc)
+   // text, so a token-based index misses them; embeddings should not.
+   {
+      env: "examples",
+      pkg: "storefront",
+      query: "how much money did we make",
+      expect: "total_sales",
+      gap: true,
+   },
    {
+      env: "examples",
+      pkg: "storefront",
+      query: "refund percentage",
+      expect: "return_rate",
+      gap: true,
+   },
+   // malloy-samples (needs a config serving them as environment "samples").
+   {
+      env: "samples",
       pkg: "ecommerce",
       query: "revenue by product category",
       expect: "category",
    },
-   { pkg: "ecommerce", query: "total sales revenue", expect: "total_sales" },
-   { pkg: "ecommerce", query: "customer state location", expect: "state" },
-   { pkg: "ecommerce", query: "order count", expect: "order_count" },
-   { pkg: "ecommerce", query: "product brand", expect: "brand" },
-   { pkg: "faa", query: "flights by carrier", expect: "carrier" },
-   { pkg: "faa", query: "airport", expect: "airport" },
-   { pkg: "faa", query: "aircraft model", expect: "aircraft" },
-   // Known lexical gap: the field is "dep_delay", which does not share tokens with
-   // "departure delay", so a token-based index misses it. Kept to surface the gap.
-   { pkg: "faa", query: "departure delay", expect: "delay" },
+   {
+      env: "samples",
+      pkg: "ecommerce",
+      query: "total sales revenue",
+      expect: "total_sales",
+   },
+   {
+      env: "samples",
+      pkg: "ecommerce",
+      query: "customer state location",
+      expect: "state",
+   },
+   {
+      env: "samples",
+      pkg: "ecommerce",
+      query: "order count",
+      expect: "order_count",
+   },
+   {
+      env: "samples",
+      pkg: "ecommerce",
+      query: "product brand",
+      expect: "brand",
+   },
+   {
+      env: "samples",
+      pkg: "faa",
+      query: "flights by carrier",
+      expect: "carrier",
+   },
+   { env: "samples", pkg: "faa", query: "airport", expect: "airport" },
+   { env: "samples", pkg: "faa", query: "aircraft model", expect: "aircraft" },
+   // The original lexical-gap case: the field is "dep_delay", which shares
+   // no tokens with "departure delay".
+   {
+      env: "samples",
+      pkg: "faa",
+      query: "departure delay",
+      expect: "delay",
+      gap: true,
+   },
 ];
 
 const ENDPOINT = process.env.MCP_URL || "http://localhost:4040/mcp";
-const EVAL_ENV = process.env.EVAL_ENV || "samples";
 const K = Number(process.argv[2] || 5);
+const WARMUP_ATTEMPTS = 15;
+const WARMUP_DELAY_MS = 2_000;
 
-async function getContext(
-   pkg: string,
-   query: string,
-   limit: number,
-): Promise<ResultEntity[]> {
+async function callGetContext(
+   args: Record<string, unknown>,
+): Promise<GetContextPayload> {
    const res = await fetch(ENDPOINT, {
       method: "POST",
       headers: {
@@ -63,36 +149,91 @@ async function getContext(
          jsonrpc: "2.0",
          id: 1,
          method: "tools/call",
-         params: {
-            name: "malloy_getContext",
-            arguments: {
-               environmentName: EVAL_ENV,
-               packageName: pkg,
-               query,
-               limit,
-            },
-         },
+         params: { name: "malloy_getContext", arguments: args },
       }),
    });
    const text = await res.text();
    const dataLine = text.split("\n").find((l) => l.startsWith("data: "));
-   if (!dataLine) return [];
+   if (!dataLine) return { results: [] };
    const msg = JSON.parse(dataLine.slice(6)) as {
       result?: { content?: { resource?: { text?: string } }[] };
    };
    const payloadText = msg.result?.content?.[0]?.resource?.text;
-   if (!payloadText) return [];
-   const payload = JSON.parse(payloadText) as { results?: ResultEntity[] };
-   return payload.results ?? [];
+   if (!payloadText) return { results: [] };
+   return JSON.parse(payloadText) as GetContextPayload;
+}
+
+/** True when the environment exists on the server (tier-2 listing works). */
+async function environmentServed(env: string): Promise<boolean> {
+   const payload = await callGetContext({ environmentName: env });
+   return payload.error === undefined;
+}
+
+/**
+ * On a semantic-capable server the first query against a package kicks off
+ * the embedding sync and answers lexically; poll until the mode flips (or
+ * give up and report whatever mode the server settles on).
+ */
+async function warmUp(env: string, pkg: string): Promise<string | undefined> {
+   let mode: string | undefined;
+   for (let attempt = 0; attempt < WARMUP_ATTEMPTS; attempt++) {
+      const payload = await callGetContext({
+         environmentName: env,
+         packageName: pkg,
+         query: "warm up",
+         limit: 1,
+      });
+      mode = payload.retrieval;
+      // No marker: the server has no embedding provider configured, so
+      // there is nothing to warm up.
+      if (mode === undefined || mode === "semantic") return mode;
+      await new Promise((resolve) => setTimeout(resolve, WARMUP_DELAY_MS));
+   }
+   return mode;
 }
 
 async function main(): Promise<void> {
    console.log(
-      `get_context lexical baseline, recall@${K} (endpoint ${ENDPOINT})\n`,
+      `get_context retrieval eval, recall@${K} (endpoint ${ENDPOINT})\n`,
    );
+
+   const servedEnvs = new Map<string, boolean>();
+   for (const env of new Set(CASES.map((c) => c.env))) {
+      servedEnvs.set(env, await environmentServed(env));
+   }
+   for (const [env, served] of servedEnvs) {
+      if (!served) {
+         console.log(
+            `  [SKIP] environment "${env}" is not served; skipping its cases`,
+         );
+      }
+   }
+
+   const warmed = new Set<string>();
    let hits = 0;
+   let scored = 0;
+   const modesSeen = new Set<string>();
+
    for (const c of CASES) {
-      const results = await getContext(c.pkg, c.query, K);
+      if (!servedEnvs.get(c.env)) continue;
+
+      const pkgKey = `${c.env}/${c.pkg}`;
+      if (!warmed.has(pkgKey)) {
+         warmed.add(pkgKey);
+         await warmUp(c.env, c.pkg);
+      }
+
+      const payload = await callGetContext({
+         environmentName: c.env,
+         packageName: c.pkg,
+         query: c.query,
+         limit: K,
+      });
+      const results = payload.results ?? [];
+      const mode = payload.retrieval ?? "lexical (no provider)";
+      modesSeen.add(mode);
+
+      scored++;
       const rank = results.findIndex((r) =>
          r.name.toLowerCase().includes(c.expect.toLowerCase()),
       );
@@ -104,20 +245,28 @@ async function main(): Promise<void> {
             .map((r) => `${r.kind}:${r.name}`)
             .join(", ") || "(none)";
       const tag = hit ? `HIT@${rank + 1}` : "MISS  ";
-      console.log(`  [${tag}] ${c.pkg} / "${c.query}" (want ~${c.expect})`);
+      const gap = c.gap ? " [gap]" : "";
+      console.log(
+         `  [${tag}] [${mode}] ${c.env}/${c.pkg} / "${c.query}" (want ~${c.expect})${gap}`,
+      );
       console.log(`            top: ${top}`);
    }
-   const pct = ((hits / CASES.length) * 100).toFixed(0);
-   console.log(`\nrecall@${K}: ${hits}/${CASES.length} (${pct}%)`);
+
+   if (scored === 0) {
+      console.log("\nNo cases ran: no listed environment is served.");
+      process.exit(1);
+   }
+   const pct = ((hits / scored) * 100).toFixed(0);
    console.log(
-      "Misses occur where the field name does not share tokens with the query",
+      `\nrecall@${K}: ${hits}/${scored} (${pct}%)  mode(s): ${[...modesSeen].join(", ")}`,
    );
    console.log(
-      "(e.g. 'departure delay' vs a 'dep_delay' field). Closing that gap is the",
+      "The [gap] cases share no tokens with their target entity, so a lexical",
    );
    console.log(
-      "motivation for an optional embeddings provider; lexical ships as the v1 baseline.",
+      "run misses them by construction; a semantic run (EMBEDDING_API_KEY set)",
    );
+   console.log("is expected to close them. Compare the two summaries.");
 }
 
 main().catch((err) => {