@pipeshub-ai/mcp 2.1.0 → 2.3.0

package/src/hooks/requestid.ts

@@ -0,0 +1,62 @@
+/*
+ * Stamp an x-request-id trace header onto every outbound SDK request, reusing
+ * the id bound for the current MCP tool/resource/prompt call (see
+ * request-context.ts) so all backend calls from one invocation share it.
+ *
+ * Format mirrors the TypeScript SDK's own RequestIDHook: `mcp-<userId>-<random>`
+ * when the bearer JWT carries a userId claim, else `mcp-<random>`. The userId
+ * is only knowable once a request actually carries a resolved Authorization
+ * header, so it's decoded off the first outbound request of the tool call and
+ * cached on the bound context for every subsequent call to reuse verbatim.
+ */
+
+import { getRequestContextStore, newRequestId } from "./request-context.js";
+import { BeforeRequestContext, BeforeRequestHook } from "./types.js";
+
+const REQUEST_ID_HEADER = "x-request-id";
+
+/** Decodes the JWT payload without verifying the signature. */
+function decodeJwtPayload(token: string): Record<string, unknown> | null {
+  try {
+    const base64Url = token.split(".")[1];
+    if (!base64Url) return null;
+    const base64 = base64Url.replace(/-/g, "+").replace(/_/g, "/");
+    const padded = base64 + "=".repeat((4 - (base64.length % 4)) % 4);
+    const claims = JSON.parse(Buffer.from(padded, "base64").toString("utf8"));
+    return typeof claims === "object" && claims !== null ? claims : null;
+  } catch {
+    return null;
+  }
+}
+
+function userIdFromRequest(request: Request): string | null {
+  const auth = request.headers.get("authorization") ?? "";
+  if (!auth.toLowerCase().startsWith("bearer ")) return null;
+  const token = auth.slice(7).trim();
+  const claims = decodeJwtPayload(token);
+  return (claims?.["userId"] as string) || null;
+}
+
+export class RequestIDHook implements BeforeRequestHook {
+  beforeRequest(_hookCtx: BeforeRequestContext, request: Request): Request {
+    if (request.headers.get(REQUEST_ID_HEADER)) {
+      return request;
+    }
+
+    const store = getRequestContextStore();
+    if (!store) {
+      // No bound tool-call context (shouldn't happen in practice).
+      request.headers.set(REQUEST_ID_HEADER, newRequestId());
+      return request;
+    }
+
+    if (!store.finalId) {
+      const userId = userIdFromRequest(request);
+      store.finalId = userId
+        ? `mcp-${userId}-${store.random}`
+        : `mcp-${store.random}`;
+    }
+    request.headers.set(REQUEST_ID_HEADER, store.finalId);
+    return request;
+  }
+}

package/src/lib/base64.ts

@@ -1,12 +1,15 @@
 /*
  * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ *
+ * Exception: `bytesToBase64` is re-exported from the hand-maintained
+ * `./bytes-to-base64.js` (see that file and `.genignore`). Do not restore
+ * the generated one-liner — it overflows the stack on binary payloads.
  */
 
 import * as z from "zod";
 
-export function bytesToBase64(u8arr: Uint8Array): string {
-  return btoa(String.fromCodePoint(...u8arr));
-}
+export { bytesToBase64 } from "./bytes-to-base64.js";
+import { bytesToBase64 } from "./bytes-to-base64.js";
 
 export function bytesFromBase64(encoded: string): Uint8Array {
   return Uint8Array.from(atob(encoded), (c) => c.charCodeAt(0));

package/src/lib/bytes-to-base64.ts

@@ -0,0 +1,24 @@
+/**
+ * Hand-maintained base64 encoder for binary payloads.
+ *
+ * Speakeasy's generated helper is:
+ *   btoa(String.fromCodePoint(...u8arr))
+ * which passes one argument per byte and throws "Maximum call stack size
+ * exceeded" once the array is larger than the engine's argument limit
+ * (between 64 KB and 128 KB on Node 24). That is smaller than a typical
+ * PDF or image, so binary MCP results fail outright.
+ *
+ * Keep this file out of Speakeasy regeneration (see `.genignore`). The
+ * generated wrappers in `src/lib/base64.ts` and `src/mcp-server/shared.ts`
+ * re-export / call through here.
+ */
+
+const BASE64_CHUNK = 0x8000;
+
+export function bytesToBase64(u8arr: Uint8Array): string {
+  let binary = "";
+  for (let i = 0; i < u8arr.length; i += BASE64_CHUNK) {
+    binary += String.fromCharCode(...u8arr.subarray(i, i + BASE64_CHUNK));
+  }
+  return btoa(binary);
+}

package/src/mcp-server/instructions.ts

@@ -9,14 +9,35 @@ PipesHub is the user's workplace AI platform. It indexes their documents,
 knowledge base content, and connector sources (Drive, Box, Confluence,
 Slack, Jira, Gmail, ...). When in doubt, the answer is in PipesHub.
 
+## Full-document tasks: \`pipeshub_search\` → \`pipeshub_get_record_content\`
+
+\`pipeshub_chat\` answers from a handful of retrieved passages — it never
+reads a whole document. Whenever the task depends on a document's
+COMPLETE content, fetch the document itself:
+
+1. \`pipeshub_search\` with the document's name / topic.
+2. Take the top hit's \`recordId\`.
+3. \`pipeshub_get_record_content\`, and answer from the returned content.
+
+Tasks that need this path — anything where missing part of the document
+could make the answer wrong:
+
+- Summarize / TL;DR / key points / takeaways / action items of a doc.
+- Extract or list ALL of something (dates, owners, requirements, ...).
+- Check whether / where a doc mentions something.
+- Translate, rewrite, outline, review, or reformat a doc.
+- Compare named docs (fetch each \`recordId\`).
+- Any question explicitly scoped to ONE named document — chat retrieval
+  cannot be restricted to a single record.
+
 ## Default tool: \`pipeshub_chat\`
 
 **Use \`pipeshub_chat\` for any question that could plausibly be answered
-by the user's PipesHub-indexed data.** That includes:
+by the user's PipesHub-indexed data** and is not a full-document task:
 
-- Anything about a specific document, file, report, ticket, message, or
-  page (e.g. "what's in the Q4 sales report?", "summarize the langchain
-  doc", "what did Aashil say about onboarding?").
+- A question that may span several documents, or where you don't yet
+  know which record holds the answer (e.g. "what did Aashil say about
+  onboarding?").
 - Anything about company / org policies, processes, decisions, or
   history (e.g. "what's our vacation policy?", "who owns the auth
   service?").
@@ -32,15 +53,17 @@ the user can verify.
 
 ## When to use the other tools
 
-- \`pipeshub_search\` — only when the user asks specifically to **find /
-  locate** a document by name or topic (so you can hand them a list, or
-  resolve a filename to a \`recordId\` for download). For "what does
-  the doc say?" use \`pipeshub_chat\` instead — it does the retrieval
-  internally.
+- \`pipeshub_search\` — locate a document by name or topic and resolve it
+  to a \`recordId\`. To read or summarize one specific document, search,
+  then pass the top hit's \`recordId\` to \`pipeshub_get_record_content\`.
 - \`pipeshub_download_record\` — when the user wants the actual file
   bytes (download, attach, open). Get the \`recordId\` either from
   citations on a prior \`pipeshub_chat\` response or from
   \`pipeshub_search\`.
+- \`pipeshub_get_record_content\` — when you need a record's full parsed
+  content (returned as a single \`content\` string: metadata header plus
+  the document's text) without downloading the original file. Prefer this
+  over download when the question is about what the record says.
 - \`pipeshub_directory\` — people, groups, teams, and \`whoami\` lookups.
   Not for documents.
 - \`pipeshub_sources\` — call once at the start of a session to discover
@@ -48,6 +71,36 @@ the user can verify.
 - \`pipeshub_agents\` — list the org's configured **agents** (specialized
   assistants with their own prompt, tools, and knowledge scope).
 
+## Knowledge Sources
+
+A source (an app connector or a KB) holds record groups (a project, space,
+drive, folder), which hold records (one ticket, page, file). Records nest —
+an epic over its stories, a page over its child pages.
+
+**Every tool returns metadata or content.**
+
+- \`pipeshub_search\` / \`pipeshub_chat\` — content, but only a ranked sample of
+  blocks: never all blocks of a record, never every record that matches, never
+  a complete list of what exists.
+- \`mode:"lookup"\` and \`mode:"navigate"\` — metadata: recordIds, fields, and
+  structure. No document text. \`mode:"content"\` — one whole record.
+
+**Combining them.** Search finds a starting point by wording; navigate on a
+\`recordId\` reveals what search structurally cannot — siblings, children,
+linked records, and the group's total count. For a question that must be
+exhaustive ("all", "how many", "every"), do NOT count search hits or trust a
+chat answer: navigate the record group, which reports the real total. For a
+record's siblings, navigate its parent — take the id from the \`Path\` line.
+
+**A named record** — an issue key, a Confluence / Drive / Slack link, an
+external ID → \`mode:"lookup"\` FIRST. It returns that record's metadata (for a
+ticket: status, assignee, priority, dates) plus its \`recordId\`, which often
+answers the question outright. Never guess a \`recordId\`.
+
+Pass a source's id from \`pipeshub_sources\` as \`apps: ["<id>"]\` to search or
+chat, and as \`nodeId\` to navigate. Navigate and lookup end in a \`Next:\` line
+naming the exact follow-up call — follow it.
+
 ## Agents
 
 Some orgs configure **agents** for specific jobs (e.g. a Slack messenger, a

package/src/mcp-server/server.ts

@@ -2,6 +2,7 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 
 import { PIPESHUB_INSTRUCTIONS } from "./instructions.js";
 import { PipeshubCore } from "../core.js";
+import { bindNewRequestId } from "../hooks/request-context.js";
 import { SDKOptions } from "../lib/config.js";
 import type { ConsoleLogger } from "./console-logger.js";
 import { createRegisterPrompt } from "./prompts.js";
@@ -14,6 +15,7 @@ import { createRegisterTool, registerDynamicTools } from "./tools.js";
 import { tool$pipeshubChat } from "./tools/pipeshubChat.js";
 import { tool$pipeshubSearch } from "./tools/pipeshubSearch.js";
 import { tool$pipeshubDownloadRecord } from "./tools/pipeshubDownloadRecord.js";
+import { tool$pipeshubGetRecordContent } from "./tools/pipeshubGetRecordContent.js";
 import { tool$pipeshubDirectory } from "./tools/pipeshubDirectory.js";
 import { tool$pipeshubSources } from "./tools/pipeshubSources.js";
 import { tool$pipeshubAgents } from "./tools/pipeshubAgents.js";
@@ -40,7 +42,7 @@ export function createMCPServer(deps: {
     },
   );
 
-  const getClient = deps.getSDK || (() =>
+  const resolveClient = deps.getSDK || (() =>
     new PipeshubCore({
       security: deps.security,
       serverURL: deps.serverURL,
@@ -55,6 +57,11 @@ export function createMCPServer(deps: {
         : undefined,
     }));
 
+  const getClient = () => {
+    bindNewRequestId();
+    return resolveClient();
+  };
+
   const scopes = new Set(deps.scopes);
 
   const allowedTools = deps.allowedTools && new Set(deps.allowedTools);
@@ -87,8 +94,9 @@ export function createMCPServer(deps: {
   tool(tool$pipeshubChat);             // 2. ask questions (start + continue)
   tool(tool$pipeshubSearch);           // 3. resolve filename → recordId
   tool(tool$pipeshubDownloadRecord);   // 4. fetch a document by id
-  tool(tool$pipeshubDirectory);        // 5. people / groups / teams / whoami
-  tool(tool$pipeshubAgents);           // 6. discover org agents
+  tool(tool$pipeshubGetRecordContent); // 5. fetch a record's parsed content
+  tool(tool$pipeshubDirectory);        // 6. people / groups / teams / whoami
+  tool(tool$pipeshubAgents);           // 7. discover org agents
 
   // Curated prompt: user-invokable tool-routing guidance.
   prompt(prompt$pipeshubAssistant);

package/src/mcp-server/shared.ts

@@ -1,8 +1,14 @@
 /*
  * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ *
+ * Exception: `bytesToBase64` is imported from the hand-maintained
+ * `../lib/bytes-to-base64.js` (see that file and `.genignore`). Do not
+ * restore the generated one-liner — it overflows the stack on binary
+ * payloads.
  */
 
 import * as z from "zod";
+import { bytesToBase64 } from "../lib/bytes-to-base64.js";
 
 type BinaryData =
   | Uint8Array
@@ -14,9 +20,7 @@ type BinaryData =
 
 const base64Schema = z.string().base64();
 
-export function bytesToBase64(u8arr: Uint8Array): string {
-  return btoa(String.fromCodePoint(...u8arr));
-}
+export { bytesToBase64 };
 
 export async function consumeStream(
   stream: ReadableStream<Uint8Array>,

package/src/mcp-server/tools/_agui.ts

@@ -0,0 +1,138 @@
+// AG-UI frame folding for the chat stream.
+//
+// AG-UI is the only wire protocol the conversation and agent stream routes
+// speak; the legacy `connected` / `answer_chunk` / `complete` / `error`
+// vocabulary was removed with the new agent loop and is never emitted.
+//
+// Kept separate from `pipeshubChat.ts` so the fold is a pure function over
+// frames and can be exercised without a live stream.
+
+import type { SSEFrame } from "./_helpers.js";
+
+export const AGUIEvent = {
+  RUN_FINISHED: "RUN_FINISHED",
+  RUN_ERROR: "RUN_ERROR",
+  CUSTOM: "CUSTOM",
+  TEXT_MESSAGE_CONTENT: "TEXT_MESSAGE_CONTENT",
+} as const;
+
+export type ChatStreamState = {
+  /** Persisted conversation from the terminal frame; null until then. */
+  conversation: any | null;
+  recordsUsed: number | undefined;
+  error: string | null;
+  /** From `CUSTOM{name:"conversation_created"}`, available before the answer. */
+  conversationId: string | null;
+  title: string | null;
+  /**
+   * Answer text accumulated per `messageId`. Universal `agent` mode forwards
+   * child-run text alongside the root answer, so a single buffer would splice
+   * sub-agent output into the middle of the reply.
+   */
+  text: Map<string, string>;
+  /** messageIds seen on frames with no `parentRunId` — i.e. the root run. */
+  rootMessageIds: Set<string>;
+};
+
+export function newChatStreamState(): ChatStreamState {
+  return {
+    conversation: null,
+    recordsUsed: undefined,
+    error: null,
+    conversationId: null,
+    title: null,
+    text: new Map(),
+    rootMessageIds: new Set(),
+  };
+}
+
+/**
+ * Fold one frame into `state`. Returns true when the frame is terminal and the
+ * caller should stop draining the stream.
+ */
+/** The subset of AG-UI payload fields this fold reads. */
+type FrameData = {
+  result?: {
+    conversation?: unknown;
+    recordsUsed?: number;
+    meta?: { recordsUsed?: number };
+  };
+  message?: string;
+  error?: string;
+  name?: string;
+  value?: { conversationId?: string; title?: string };
+  delta?: string;
+  messageId?: string;
+  parentRunId?: string | null;
+};
+
+export function applyAGUIFrame(
+  state: ChatStreamState,
+  frame: SSEFrame,
+): boolean {
+  const data = frame.data;
+  const d: FrameData = data && typeof data === "object"
+    ? data as FrameData
+    : {};
+
+  switch (frame.event) {
+    case AGUIEvent.RUN_FINISHED: {
+      // Only the gateway's own re-emitted frame carries `result` — it swallows
+      // the AI backend's root RUN_FINISHED and re-sends an enriched one after
+      // persisting. Sub-agent RUN_FINISHED frames are forwarded verbatim with
+      // no `result`, so keying off the event name alone would end the stream
+      // as soon as the first sub-agent finished.
+      const result = d.result;
+      if (!result || typeof result !== "object") return false;
+      state.conversation = result.conversation ?? null;
+      state.recordsUsed = result.recordsUsed ?? result.meta?.recordsUsed;
+      return true;
+    }
+
+    case AGUIEvent.RUN_ERROR: {
+      state.error = (typeof data === "string" ? data : null)
+        ?? d.message ?? d.error ?? frame.raw ?? "Stream error";
+      return true;
+    }
+
+    case AGUIEvent.CUSTOM: {
+      if (d.name === "conversation_created") {
+        const value = d.value ?? {};
+        state.conversationId = value.conversationId ?? state.conversationId;
+        state.title = value.title ?? state.title;
+      }
+      return false;
+    }
+
+    case AGUIEvent.TEXT_MESSAGE_CONTENT: {
+      if (typeof d.delta !== "string" || d.delta.length === 0) return false;
+      const id = typeof d.messageId === "string" ? d.messageId : "";
+      state.text.set(id, (state.text.get(id) ?? "") + d.delta);
+      if (d.parentRunId == null) state.rootMessageIds.add(id);
+      return false;
+    }
+
+    // RUN_STARTED, STEP_*, TOOL_CALL_*, REASONING_*, STATE_*,
+    // TEXT_MESSAGE_START/END, HEARTBEAT — lifecycle noise.
+    default:
+      return false;
+  }
+}
+
+/**
+ * Best-effort answer when the stream ended without a terminal frame.
+ *
+ * Prefers the longest root-run buffer; falls back to the longest buffer of any
+ * run so a partial answer still reaches the caller.
+ */
+export function salvagedText(state: ChatStreamState): string | null {
+  const pick = (ids: Iterable<string>): string | null => {
+    let best: string | null = null;
+    for (const id of ids) {
+      const text = state.text.get(id);
+      if (text && (best === null || text.length > best.length)) best = text;
+    }
+    return best;
+  };
+  return pick(state.rootMessageIds) ?? pick(state.text.keys());
+}

package/src/mcp-server/tools/_helpers.ts

@@ -224,17 +224,24 @@ export function trimConversation(conv: any) {
 export function trimCitations(citations: unknown) {
   if (!Array.isArray(citations)) return [];
   return citations.map((c: any) => {
-    const md = c?.metadata ?? {};
+    // Two wire shapes exist. Flat: `{ content, chunkIndex, metadata }`.
+    // Nested (current backend): `{ citationId, citationData: { content,
+    // chunkIndex, metadata } }`. Reading only the flat one silently yields
+    // a citation with every field `undefined` — which JSON.stringify drops,
+    // leaving `{"snippet": null}` and no way to identify the source at all.
+    const cd = c?.citationData ?? c;
+    const md = cd?.metadata ?? c?.metadata ?? {};
+    const content = typeof cd?.content === "string" ? cd.content : c?.content;
     return {
-      recordId: md.recordId ?? c?.recordId,
-      recordName: md.recordName ?? c?.recordName,
-      snippet: typeof c?.content === "string"
-        ? c.content.slice(0, 280)
+      recordId: md.recordId ?? cd?.recordId ?? c?.recordId,
+      recordName: md.recordName ?? cd?.recordName ?? c?.recordName,
+      snippet: typeof content === "string"
+        ? content.slice(0, 280)
         : md.blockText ?? null,
       mimeType: md.mimeType,
       webUrl: md.webUrl,
       pageNum: md.pageNum,
-      chunkIndex: c?.chunkIndex ?? md.chunkIndex,
+      chunkIndex: cd?.chunkIndex ?? c?.chunkIndex ?? md.chunkIndex,
     };
   });
 }