@malloy-publisher/server 0.0.206 → 0.0.208

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.206",
+   "version": "0.0.208",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"
@@ -36,16 +36,16 @@
       "@azure/storage-blob": "^12.26.0",
       "@duckdb/node-api": "1.5.3-r.2",
       "@google-cloud/storage": "^7.16.0",
-      "@malloydata/db-bigquery": "^0.0.405",
-      "@malloydata/db-databricks": "^0.0.405",
-      "@malloydata/db-duckdb": "^0.0.405",
-      "@malloydata/db-mysql": "^0.0.405",
-      "@malloydata/db-postgres": "^0.0.405",
-      "@malloydata/db-snowflake": "^0.0.405",
-      "@malloydata/db-trino": "^0.0.405",
-      "@malloydata/malloy": "^0.0.405",
-      "@malloydata/malloy-sql": "^0.0.405",
-      "@malloydata/render-validator": "^0.0.405",
+      "@malloydata/db-bigquery": "^0.0.415",
+      "@malloydata/db-databricks": "^0.0.415",
+      "@malloydata/db-duckdb": "^0.0.415",
+      "@malloydata/db-mysql": "^0.0.415",
+      "@malloydata/db-postgres": "^0.0.415",
+      "@malloydata/db-snowflake": "^0.0.415",
+      "@malloydata/db-trino": "^0.0.415",
+      "@malloydata/malloy": "^0.0.415",
+      "@malloydata/malloy-sql": "^0.0.415",
+      "@malloydata/render-validator": "^0.0.415",
       "@modelcontextprotocol/sdk": "^1.13.2",
       "@opentelemetry/api": "^1.9.0",
       "@opentelemetry/auto-instrumentations-node": "^0.57.0",
@@ -65,6 +65,7 @@
       "globals": "^15.9.0",
       "handlebars": "^4.7.8",
       "http-proxy-middleware": "^3.0.5",
+      "lunr": "^2.3.6",
       "morgan": "^1.10.0",
       "node-cron": "^3.0.3",
       "recursive-readdir": "^2.2.3",
@@ -80,6 +81,7 @@
       "@types/bun": "^1.2.20",
       "@types/cors": "^2.8.12",
       "@types/express": "^4.17.14",
+      "@types/lunr": "^2.3.7",
       "@types/morgan": "^1.9.9",
       "@types/node": "^24.10.1",
       "@types/node-cron": "^3.0.11",

package/src/health.ts

@@ -74,6 +74,7 @@ export function registerSignalHandlers(
    mcpServer: Server,
    shutdownDrainDurationSeconds: number = 0,
    shutdownGracefulCloseTimeoutSeconds: number = 0,
+   agentMcpServer?: Server,
 ): void {
    // Keep the process alive on SIGTERM — do not close the server.
    // K8s will SIGKILL after terminationGracePeriodSeconds (which cannot be caught).
@@ -96,6 +97,7 @@ export function registerSignalHandlers(
          server,
          mcpServer,
          shutdownGracefulCloseTimeoutSeconds,
+         agentMcpServer,
       );
    });
 }
@@ -114,6 +116,7 @@ export async function performGracefulShutdownAfterDrain(
    server: Server,
    mcpServer: Server,
    shutdownGracefulCloseTimeoutSeconds: number,
+   agentMcpServer?: Server,
 ): Promise<void> {
    const closeServer = (server: Server, name: string) =>
       new Promise<void>((resolve) => {
@@ -134,6 +137,9 @@ export async function performGracefulShutdownAfterDrain(
    await Promise.all([
       closeServer(server, "Main server"),
       closeServer(mcpServer, "MCP server"),
+      ...(agentMcpServer
+         ? [closeServer(agentMcpServer, "Agent MCP server")]
+         : []),
    ]);
 
    try {

package/src/materialization_metrics.ts

@@ -17,9 +17,14 @@ import { metrics, type Counter, type Histogram } from "@opentelemetry/api";
 
 export type MaterializationRound = "round1" | "round2" | "auto";
 export type MaterializationOutcome = "success" | "failed" | "cancelled";
+/** Manifest bind outcome: timeout is split out from generic failure on purpose. */
+export type ManifestBindOutcome = "success" | "failure" | "timeout";
 
 let roundCounter: Counter | null = null;
 let roundDuration: Histogram | null = null;
+let manifestBindCounter: Counter | null = null;
+let sourceBuildDuration: Histogram | null = null;
+let dropTablesCounter: Counter | null = null;
 
 function ensureTelemetry(): { counter: Counter; duration: Histogram } {
    if (roundCounter && roundDuration) {
@@ -59,8 +64,58 @@ export function recordMaterializationRound(
    duration.record(durationMs, { round });
 }
 
+/**
+ * Record a manifest-bind attempt (publisher binding a control-plane manifest to
+ * a package at load). `timeout` is distinguished from `failure` so operators can
+ * tell an unreachable/slow manifest store from a malformed manifest.
+ */
+export function recordManifestBind(outcome: ManifestBindOutcome): void {
+   if (!manifestBindCounter) {
+      manifestBindCounter = metrics
+         .getMeter("publisher")
+         .createCounter("publisher_materialization_manifest_bind_total", {
+            description:
+               "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout').",
+         });
+   }
+   manifestBindCounter.add(1, { outcome });
+}
+
+/** Record the wall-clock duration of building one persist source's table. */
+export function recordSourceBuildDuration(durationMs: number): void {
+   if (!sourceBuildDuration) {
+      sourceBuildDuration = metrics
+         .getMeter("publisher")
+         .createHistogram(
+            "publisher_materialization_source_build_duration_ms",
+            {
+               description:
+                  "Wall-clock duration of building a single persist source.",
+               unit: "ms",
+            },
+         );
+   }
+   sourceBuildDuration.record(durationMs);
+}
+
+/** Record a best-effort physical-table drop on materialization delete. */
+export function recordDropTables(outcome: "success" | "failure"): void {
+   if (!dropTablesCounter) {
+      dropTablesCounter = metrics
+         .getMeter("publisher")
+         .createCounter("publisher_materialization_drop_tables_total", {
+            description:
+               "Physical tables dropped on delete. Label: outcome ('success'|'failure').",
+         });
+   }
+   dropTablesCounter.add(1, { outcome });
+}
+
 /** Visible for tests. Drops cached instruments so a fresh MeterProvider can capture emissions. */
 export function resetMaterializationTelemetryForTesting(): void {
    roundCounter = null;
    roundDuration = null;
+   manifestBindCounter = null;
+   sourceBuildDuration = null;
+   dropTablesCounter = null;
 }

package/src/mcp/agent_server.protocol.spec.ts

@@ -0,0 +1,78 @@
+import { beforeAll, describe, expect, it } from "bun:test";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { initializeAgentMcpServer } from "./agent_server";
+import type { EnvironmentStore } from "../service/environment_store";
+
+/**
+ * End-to-end coverage of the agent MCP server over the real MCP protocol, using
+ * the SDK's in-memory transport (no HTTP, no network, no DuckDB). Exercises tool
+ * registration, the dual-channel prompts, a real tool call (searchDocs over the
+ * bundled index), and the getContext error path.
+ */
+describe("agent MCP server over the MCP protocol (in-memory)", () => {
+   let client: Client;
+
+   beforeAll(async () => {
+      // searchDocs does not touch the store; getContext's error path only needs
+      // getEnvironment to reject, so a throwing stub is sufficient here.
+      const stubStore = {
+         getEnvironment: async (name: string): Promise<never> => {
+            throw new Error(`Environment not found: ${name}`);
+         },
+      } as unknown as EnvironmentStore;
+
+      const server = initializeAgentMcpServer(stubStore);
+      const [clientTransport, serverTransport] =
+         InMemoryTransport.createLinkedPair();
+      await server.connect(serverTransport);
+      client = new Client({ name: "agent-protocol-test", version: "0.0.0" });
+      await client.connect(clientTransport);
+   });
+
+   it("exposes exactly the two agent tools", async () => {
+      const { tools } = await client.listTools();
+      expect(new Set(tools.map((t) => t.name))).toEqual(
+         new Set(["malloy_getContext", "malloy_searchDocs"]),
+      );
+   });
+
+   it("exposes the skill set as dual-channel prompts", async () => {
+      const { prompts } = await client.listPrompts();
+      expect(prompts.length).toBeGreaterThanOrEqual(24);
+      expect(prompts.some((p) => p.name === "malloy-analysis")).toBe(true);
+   });
+
+   it("malloy_searchDocs returns relevant docs over the protocol", async () => {
+      const res = await client.callTool({
+         name: "malloy_searchDocs",
+         arguments: { query: "window functions" },
+      });
+      const content = res.content as Array<{ resource?: { text?: string } }>;
+      const results = JSON.parse(
+         content?.[0]?.resource?.text ?? "[]",
+      ) as Array<{
+         url: string;
+      }>;
+      expect(results.length).toBeGreaterThan(0);
+      expect(results[0].url.length).toBeGreaterThan(0);
+   });
+
+   it("malloy_getContext fails gracefully (isError) for an unknown environment", async () => {
+      const res = await client.callTool({
+         name: "malloy_getContext",
+         arguments: {
+            environmentName: "nope",
+            packageName: "nope",
+            query: "x",
+         },
+      });
+      expect(res.isError).toBe(true);
+   });
+
+   it("a skill prompt returns its body", async () => {
+      const res = await client.getPrompt({ name: "malloy-analysis" });
+      const content = res.messages[0].content as { text?: string };
+      expect((content.text ?? "").length).toBeGreaterThan(200);
+   });
+});

package/src/mcp/agent_server.spec.ts

@@ -0,0 +1,18 @@
+import { describe, expect, it } from "bun:test";
+import { agentServerInfo, initializeAgentMcpServer } from "./agent_server";
+import type { EnvironmentStore } from "../service/environment_store";
+
+describe("agent MCP server", () => {
+   it("advertises a distinct server name from the core MCP server", () => {
+      expect(agentServerInfo.name).toBe("malloy-publisher-agent-mcp-server");
+   });
+
+   it("initializes and registers its tools and skill prompts without throwing", () => {
+      // The store is captured for later handler calls, not used during
+      // registration, so a placeholder is sufficient for this smoke test.
+      const server = initializeAgentMcpServer(
+         {} as unknown as EnvironmentStore,
+      );
+      expect(server).toBeTruthy();
+   });
+});

package/src/mcp/agent_server.ts

@@ -0,0 +1,144 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
+import cors from "cors";
+import express from "express";
+import * as http from "http";
+import { registerHealthEndpoints } from "../health";
+import { logger } from "../logger";
+import { EnvironmentStore } from "../service/environment_store";
+import { registerDocsSearchTool } from "./tools/docs_search_tool";
+import { registerGetContextTool } from "./tools/get_context_tool";
+import skillsBundle from "./skills/skills_bundle.json";
+
+const AGENT_MCP_ENDPOINT = "/mcp";
+
+// Build-time bundle of the agent skills (see skills/build_skills_bundle.ts), exposed
+// as MCP prompts for the dual-channel delivery below.
+const AGENT_SKILLS = (
+   skillsBundle as {
+      skills: { name: string; description: string; body: string }[];
+   }
+).skills;
+
+export const agentServerInfo = {
+   name: "malloy-publisher-agent-mcp-server",
+   version: "0.0.1",
+   displayName: "Malloy Publisher Agent MCP Server",
+   description:
+      "Agent retrieval tools (semantic model context and docs search) for Malloy, served on a separate MCP endpoint from the core Publisher MCP tools.",
+};
+
+/**
+ * Builds an MCP server exposing ONLY the agent retrieval tools (malloy_getContext,
+ * malloy_searchDocs). Kept deliberately separate from the core initializeMcpServer so
+ * these tools and their tests stay isolated from the existing MCP tool surface.
+ */
+export function initializeAgentMcpServer(
+   environmentStore: EnvironmentStore,
+): McpServer {
+   const mcpServer = new McpServer(agentServerInfo);
+   registerGetContextTool(mcpServer, environmentStore);
+   registerDocsSearchTool(mcpServer, environmentStore);
+
+   // Dual-channel: also expose each skill as an MCP prompt, so hosts that ingest MCP
+   // but do not load skill files (e.g. Codex, ChatGPT, Cursor) can pull the same
+   // guidance. Skill-aware hosts (Claude Code/Desktop) use the native skill files.
+   // Note: MCP prompts are on-demand, so always-on "prevention" skills become
+   // on-demand here; see docs/agent-skills/design-principles.md.
+   for (const skill of AGENT_SKILLS) {
+      mcpServer.prompt(skill.name, skill.description, () => ({
+         messages: [
+            {
+               role: "user" as const,
+               content: { type: "text" as const, text: skill.body },
+            },
+         ],
+      }));
+   }
+
+   return mcpServer;
+}
+
+/**
+ * Starts the agent MCP server on its own HTTP listener (a third Publisher listener),
+ * using the same stateless StreamableHTTP transport pattern as the core MCP server.
+ * Returns the http.Server so the caller can manage shutdown.
+ */
+export function startAgentMcpServer(
+   environmentStore: EnvironmentStore,
+   host: string,
+   port: number,
+): http.Server {
+   const agentMcpApp = express();
+   registerHealthEndpoints(agentMcpApp);
+   agentMcpApp.use(AGENT_MCP_ENDPOINT, express.json());
+   agentMcpApp.use(AGENT_MCP_ENDPOINT, cors());
+
+   agentMcpApp.all(AGENT_MCP_ENDPOINT, async (req, res) => {
+      if (req.method !== "POST") {
+         res.setHeader("Allow", "POST");
+         res.status(405).json({
+            jsonrpc: "2.0",
+            error: {
+               code: -32601,
+               message: "Method Not Allowed in Stateless Mode",
+            },
+            id: null,
+         });
+         return;
+      }
+
+      try {
+         const transport = new StreamableHTTPServerTransport({
+            sessionIdGenerator: undefined,
+         });
+         transport.onerror = (err: Error) => {
+            logger.error("[Agent MCP Transport Error]", {
+               error: err instanceof Error ? err.message : String(err),
+            });
+         };
+         const requestMcpServer = initializeAgentMcpServer(environmentStore);
+         await requestMcpServer.connect(transport);
+         res.on("close", () => {
+            transport.close().catch((err) => {
+               logger.error("[Agent MCP Transport Error] close failed", {
+                  error: err instanceof Error ? err.message : String(err),
+               });
+            });
+         });
+         await transport.handleRequest(req, res, req.body);
+      } catch (err) {
+         logger.error("[Agent MCP] request handler error", {
+            error: err instanceof Error ? err.message : String(err),
+         });
+         if (!res.headersSent) {
+            res.status(500).json({
+               jsonrpc: "2.0",
+               error: { code: -32603, message: "Internal server error" },
+               id: null,
+            });
+         }
+      }
+   });
+
+   const agentMcpServer = agentMcpApp.listen(port, host, () => {
+      logger.info(
+         `Agent MCP server listening at http://${host}:${port}${AGENT_MCP_ENDPOINT}`,
+      );
+   });
+
+   // The agent MCP server is auxiliary: a bind failure here (e.g. EADDRINUSE)
+   // must not take down the main API or the core MCP server. Without this
+   // handler the listener's 'error' event would surface as an uncaught
+   // exception and crash the whole process at boot. Log and degrade instead.
+   agentMcpServer.on("error", (err: NodeJS.ErrnoException) => {
+      logger.error("Agent MCP server failed to start; continuing without it", {
+         host,
+         port,
+         code: err.code,
+         error: err.message,
+      });
+   });
+
+   return agentMcpServer;
+}

package/src/mcp/server.ts

@@ -52,6 +52,9 @@ export function initializeMcpServer(
 
    registerTools(mcpServer, environmentStore);
 
+   // Note: malloy_getContext and malloy_searchDocs are intentionally NOT registered
+   // here. They run on a separate, isolated agent MCP server (see mcp/agent_server.ts).
+
    logger.info("[MCP Init] Registering prompt capability...");
    registerPromptCapability(mcpServer, environmentStore);
 

package/src/mcp/skills/build_skills_bundle.spec.ts

@@ -0,0 +1,51 @@
+import { describe, expect, it } from "bun:test";
+import { parseSkill, unquote } from "./build_skills_bundle";
+
+describe("build_skills_bundle parseSkill", () => {
+   it("parses name, description, and body from frontmatter", () => {
+      const md =
+         "---\nname: malloy-queries\ndescription: Query patterns.\n---\n# Body\n\nText.";
+      const s = parseSkill(md, "dir");
+      expect(s.name).toBe("malloy-queries");
+      expect(s.description).toBe("Query patterns.");
+      expect(s.body).toBe("# Body\n\nText.");
+   });
+
+   it("keeps colons in the description (line-based, not split on colon)", () => {
+      const md =
+         "---\nname: x\ndescription: Use when: a thing happens.\n---\nbody";
+      expect(parseSkill(md, "dir").description).toBe(
+         "Use when: a thing happens.",
+      );
+   });
+
+   it("tolerates CRLF line endings", () => {
+      const md = "---\r\nname: x\r\ndescription: d\r\n---\r\nbody line";
+      const s = parseSkill(md, "dir");
+      expect(s.name).toBe("x");
+      expect(s.description).toBe("d");
+      expect(s.body).toBe("body line");
+   });
+
+   it("falls back to the dir name and empty description when frontmatter is missing", () => {
+      const s = parseSkill("# Just a body, no frontmatter", "fallback-name");
+      expect(s.name).toBe("fallback-name");
+      expect(s.description).toBe("");
+      expect(s.body).toBe("# Just a body, no frontmatter");
+   });
+
+   it("strips surrounding quotes from values", () => {
+      const md = "---\nname: \"quoted-name\"\ndescription: 'q'\n---\nb";
+      const s = parseSkill(md, "dir");
+      expect(s.name).toBe("quoted-name");
+      expect(s.description).toBe("q");
+   });
+});
+
+describe("build_skills_bundle unquote", () => {
+   it("removes a single layer of surrounding quotes", () => {
+      expect(unquote('"hi"')).toBe("hi");
+      expect(unquote("'hi'")).toBe("hi");
+      expect(unquote("plain")).toBe("plain");
+   });
+});

package/src/mcp/skills/build_skills_bundle.ts

@@ -0,0 +1,66 @@
+/**
+ * Build a bundled JSON of the agent skills so the agent MCP server can expose them as
+ * MCP prompts (dual-channel): skill-aware hosts read the skill files directly, while
+ * hosts that only ingest MCP get the same guidance through these prompts.
+ *
+ * Committed as a build-time asset so the server has no runtime dependency on the
+ * skills directory layout. Regenerate when the skills change:
+ *
+ *   bun run packages/server/src/mcp/skills/build_skills_bundle.ts <skills-dir> [out.json]
+ *
+ * <skills-dir> points at the repo's top-level skills/ directory.
+ */
+import * as fs from "fs";
+import * as path from "path";
+
+interface SkillEntry {
+   name: string;
+   description: string;
+   body: string;
+}
+
+export function unquote(s: string): string {
+   return s.trim().replace(/^["']|["']$/g, "");
+}
+
+/** Parse a SKILL.md into its frontmatter name/description and Markdown body. */
+export function parseSkill(md: string, dirName: string): SkillEntry {
+   const text = md.replace(/\r\n/g, "\n"); // tolerate CRLF-committed files
+   const fm = text.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
+   const front = fm ? fm[1] : "";
+   const body = (fm ? fm[2] : text).trim();
+   const name = unquote(front.match(/^name:\s*(.+)$/m)?.[1] ?? dirName);
+   const description = unquote(
+      front.match(/^description:\s*(.+)$/m)?.[1] ?? "",
+   );
+   return { name, description, body };
+}
+
+function main(): void {
+   const skillsDir = process.argv[2];
+   const outFile =
+      process.argv[3] || path.join(import.meta.dir, "skills_bundle.json");
+   if (!skillsDir || !fs.existsSync(skillsDir)) {
+      console.error(
+         `skills dir not found: ${skillsDir}\nUsage: bun run build_skills_bundle.ts <skills-dir> [out.json]`,
+      );
+      process.exit(1);
+   }
+
+   const skills: SkillEntry[] = [];
+   for (const entry of fs.readdirSync(skillsDir, { withFileTypes: true })) {
+      if (!entry.isDirectory()) continue;
+      const skillFile = path.join(skillsDir, entry.name, "SKILL.md");
+      if (!fs.existsSync(skillFile)) continue;
+      skills.push(parseSkill(fs.readFileSync(skillFile, "utf8"), entry.name));
+   }
+   skills.sort((a, b) => a.name.localeCompare(b.name));
+
+   fs.writeFileSync(outFile, JSON.stringify({ skills }));
+   console.log(`Wrote ${skills.length} skills to ${outFile}`);
+}
+
+// Only run when invoked directly (bun run ...), not when imported by a test.
+if (import.meta.main) {
+   main();
+}