membot 0.5.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/membot.md +25 -10
- package/.cursor/rules/membot.mdc +25 -10
- package/README.md +35 -4
- package/package.json +8 -5
- package/scripts/apply-patches.sh +0 -11
- package/src/cli.ts +2 -2
- package/src/commands/login-page.mustache +50 -0
- package/src/commands/login.ts +83 -0
- package/src/config/schemas.ts +17 -5
- package/src/constants.ts +13 -1
- package/src/context.ts +1 -24
- package/src/db/files.ts +21 -25
- package/src/db/migrations/003-downloader-columns.ts +58 -0
- package/src/db/migrations.ts +2 -1
- package/src/ingest/converter/index.ts +9 -0
- package/src/ingest/converter/xlsx.ts +111 -0
- package/src/ingest/downloaders/browser.ts +180 -0
- package/src/ingest/downloaders/generic-web.ts +81 -0
- package/src/ingest/downloaders/github.ts +178 -0
- package/src/ingest/downloaders/google-docs.ts +56 -0
- package/src/ingest/downloaders/google-shared.ts +86 -0
- package/src/ingest/downloaders/google-sheets.ts +58 -0
- package/src/ingest/downloaders/google-slides.ts +53 -0
- package/src/ingest/downloaders/index.ts +182 -0
- package/src/ingest/downloaders/linear.ts +291 -0
- package/src/ingest/fetcher.ts +104 -129
- package/src/ingest/ingest.ts +43 -70
- package/src/mcp/instructions.ts +4 -2
- package/src/operations/add.ts +6 -4
- package/src/operations/info.ts +4 -6
- package/src/operations/move.ts +2 -3
- package/src/operations/refresh.ts +2 -4
- package/src/operations/remove.ts +23 -2
- package/src/operations/tree.ts +1 -1
- package/src/operations/types.ts +1 -1
- package/src/refresh/runner.ts +59 -114
- package/src/types/text-modules.d.ts +5 -0
- package/patches/@evantahler%2Fmcpx@0.21.4.patch +0 -51
- package/src/commands/mcpx.ts +0 -112
- package/src/ingest/agent-fetcher.ts +0 -639
|
@@ -1,639 +0,0 @@
|
|
|
1
|
-
import Anthropic from "@anthropic-ai/sdk";
|
|
2
|
-
import type {
|
|
3
|
-
Tool as AnthropicTool,
|
|
4
|
-
MessageParam,
|
|
5
|
-
ToolResultBlockParam,
|
|
6
|
-
ToolUseBlock,
|
|
7
|
-
} from "@anthropic-ai/sdk/resources/messages";
|
|
8
|
-
import type { LlmConfig } from "../config/schemas.ts";
|
|
9
|
-
import { HelpfulError } from "../errors.ts";
|
|
10
|
-
import { logger } from "../output/logger.ts";
|
|
11
|
-
import { sha256Hex } from "./local-reader.ts";
|
|
12
|
-
|
|
13
|
-
/** Number of times the agent may iterate. Each turn = one Claude call + tool dispatch. */
|
|
14
|
-
const MAX_TURNS = 10;
|
|
15
|
-
/** Bytes of content shown back to the LLM in the mcp_exec preview. The harness has the full content. */
|
|
16
|
-
const PREVIEW_CHARS = 2_000;
|
|
17
|
-
/** Token budget per Claude call. Should comfortably fit a tool-use response + reasoning. */
|
|
18
|
-
const MAX_RESPONSE_TOKENS = 4_096;
|
|
19
|
-
|
|
20
|
-
/**
|
|
21
|
-
* Outcome shape mirrored from `FetchedRemote` in fetcher.ts. We don't
|
|
22
|
-
* import that type here to avoid a cycle — the fetcher imports us.
|
|
23
|
-
*/
|
|
24
|
-
export interface AgentFetchedRemote {
|
|
25
|
-
bytes: Uint8Array;
|
|
26
|
-
sha256: string;
|
|
27
|
-
mimeType: string;
|
|
28
|
-
fetcher: "mcpx";
|
|
29
|
-
fetcherServer: string;
|
|
30
|
-
fetcherTool: string;
|
|
31
|
-
fetcherArgs: Record<string, unknown>;
|
|
32
|
-
sourceUrl: string;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
/**
|
|
36
|
-
* The slice of mcpx the agent loop needs. Kept minimal so tests can
|
|
37
|
-
* stub it without spinning up a real client.
|
|
38
|
-
*/
|
|
39
|
-
export interface AgentMcpxAdapter {
|
|
40
|
-
search(
|
|
41
|
-
query: string,
|
|
42
|
-
options?: { keywordOnly?: boolean; semanticOnly?: boolean },
|
|
43
|
-
): Promise<{ server: string; tool: string; description?: string; score?: number; matchType?: string }[]>;
|
|
44
|
-
listTools(server?: string): Promise<{ server: string; tool: { name: string; description?: string } }[]>;
|
|
45
|
-
info(
|
|
46
|
-
server: string,
|
|
47
|
-
tool: string,
|
|
48
|
-
): Promise<{ name: string; description?: string; inputSchema?: unknown } | undefined>;
|
|
49
|
-
exec(
|
|
50
|
-
server: string,
|
|
51
|
-
tool: string,
|
|
52
|
-
args?: Record<string, unknown>,
|
|
53
|
-
): Promise<{ isError?: boolean; content?: unknown[] }>;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
export interface AgentFetchOptions {
|
|
57
|
-
url: string;
|
|
58
|
-
mcpx: AgentMcpxAdapter;
|
|
59
|
-
llm: LlmConfig;
|
|
60
|
-
hint?: string;
|
|
61
|
-
/**
|
|
62
|
-
* Optional sublabel callback. Receives compact, human-readable strings
|
|
63
|
-
* describing what the agent is doing each turn (e.g. "mcp_exec
|
|
64
|
-
* linear/list_comments (turn 2)"). Wired to the spinner suffix in TTY
|
|
65
|
-
* mode so users see live progress without `--verbose`.
|
|
66
|
-
*/
|
|
67
|
-
onProgress?: (sublabel: string) => void;
|
|
68
|
-
/** Test seam: inject a pre-built Anthropic client. */
|
|
69
|
-
_testClient?: Anthropic;
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
/**
|
|
73
|
-
* Outcome of the agent loop:
|
|
74
|
-
* - `accepted`: the agent picked a captured mcp_exec result; caller stores it as the new version.
|
|
75
|
-
* - `fallback`: the agent gave up on mcpx (request_http_fallback, no tool calls, max turns); caller does plain HTTP.
|
|
76
|
-
* - HelpfulError thrown: the agent reported an actionable failure (report_failure), or the loop hit a hard error.
|
|
77
|
-
*/
|
|
78
|
-
export type AgentFetchOutcome = { kind: "accepted"; result: AgentFetchedRemote } | { kind: "fallback"; reason: string };
|
|
79
|
-
|
|
80
|
-
const FETCHER_SYSTEM_PROMPT = `You are a content fetcher. Your job is to find the right MCP tool to retrieve the content at the given URL, run it, and tell the harness which result to save.
|
|
81
|
-
|
|
82
|
-
**Important: the harness captures the full result of every mcp_exec call automatically.** You only see a short preview of each result so you can verify it looks reasonable. You do NOT need to read or copy the full content — you just identify which exec call to save.
|
|
83
|
-
|
|
84
|
-
**Format preference: markdown, in order of preference.**
|
|
85
|
-
1. When searching with mcp_search or mcp_list_tools, prefer tools whose names indicate markdown output: anything containing "markdown", "md", "AsMarkdown", "AsMd", "AsDocmd", or similar.
|
|
86
|
-
2. If no markdown-named variant exists, use mcp_info to inspect the tool's input schema for a "format", "mime_type", "output_format", or similar parameter and request "markdown" (or "md") when available.
|
|
87
|
-
3. If neither is possible, run the tool anyway. The membot pipeline will normalize the captured content downstream — markdown-native tools are still preferred because they're cheaper and higher fidelity, but you do not have to find one.
|
|
88
|
-
|
|
89
|
-
Workflow:
|
|
90
|
-
1. Use mcp_search or mcp_list_tools to find the best tool for this URL (e.g., Google Docs tools for docs.google.com, Firecrawl for generic web pages, GitHub tools for github.com). Apply the format preference above.
|
|
91
|
-
2. Use mcp_info to inspect the tool's input schema. **Required before mcp_exec on any tool you haven't called this session.** Many tools want \`document_id\`, \`repo\`, \`page_id\`, etc. — not \`url\`. Extract the right value from the URL.
|
|
92
|
-
3. Call mcp_exec with arguments that conform to the schema.
|
|
93
|
-
4. **Multi-step workflows are expected.** Many providers need a sequence of calls — e.g. Firecrawl: \`scrape\` returns a job id, then \`get_job_status\` polls until done, then the final result has the content; some doc providers need a \`prepare/export\` call before \`download\`; large docs may paginate. Make as many mcp_exec calls as needed. Read each preview to decide the next step.
|
|
94
|
-
5. If the tool errors (input_error / auth_error / "still processing"), read the error, adjust, and retry — or pivot to a different tool.
|
|
95
|
-
6. Once a successful exec preview looks like the FINAL content, call accept_content with the exec_call_id (the tool_use_id of that mcp_exec call) and the actual mime_type the tool returned. Pick the call whose result is the actual content — not an intermediate job id or status response.
|
|
96
|
-
|
|
97
|
-
Terminal tools (call exactly one):
|
|
98
|
-
- accept_content(exec_call_id, mime_type?) — save the content captured from a previous mcp_exec call.
|
|
99
|
-
- request_http_fallback() — fall back to a basic HTTP fetch. Use only when no MCP tool can handle the URL after a genuine attempt.
|
|
100
|
-
- report_failure(message) — surface an actionable message to the user (e.g., "this Google Doc is private — share it with your service account"). Use only when there is a specific next step the user must take.`;
|
|
101
|
-
|
|
102
|
-
const acceptContentTool: AnthropicTool = {
|
|
103
|
-
name: "accept_content",
|
|
104
|
-
description:
|
|
105
|
-
"Save the full content captured by the harness from a previous mcp_exec call. You only need to supply the exec_call_id (the tool_use_id of that mcp_exec call). The harness already has the full content. Do NOT paste content here.",
|
|
106
|
-
input_schema: {
|
|
107
|
-
type: "object" as const,
|
|
108
|
-
properties: {
|
|
109
|
-
exec_call_id: {
|
|
110
|
-
type: "string",
|
|
111
|
-
description:
|
|
112
|
-
"The tool_use_id of the mcp_exec call whose result should be saved (the harness lists captured ids in mcp_exec previews).",
|
|
113
|
-
},
|
|
114
|
-
mime_type: {
|
|
115
|
-
type: "string",
|
|
116
|
-
description:
|
|
117
|
-
"MIME type the source tool returned (e.g. 'text/markdown', 'text/html', 'application/json'). Defaults to text/markdown.",
|
|
118
|
-
},
|
|
119
|
-
},
|
|
120
|
-
required: ["exec_call_id"],
|
|
121
|
-
},
|
|
122
|
-
};
|
|
123
|
-
|
|
124
|
-
const requestHttpFallbackTool: AnthropicTool = {
|
|
125
|
-
name: "request_http_fallback",
|
|
126
|
-
description: "Fall back to a basic HTTP fetch. Use only when no MCP tool can handle the URL after a genuine attempt.",
|
|
127
|
-
input_schema: { type: "object" as const, properties: {}, required: [] },
|
|
128
|
-
};
|
|
129
|
-
|
|
130
|
-
const reportFailureTool: AnthropicTool = {
|
|
131
|
-
name: "report_failure",
|
|
132
|
-
description:
|
|
133
|
-
"Report a fetch failure with an actionable message for the user (e.g., 'this Google Doc is private — share it with your service account'). Use only when there is a clear next step the user must take.",
|
|
134
|
-
input_schema: {
|
|
135
|
-
type: "object" as const,
|
|
136
|
-
properties: {
|
|
137
|
-
message: {
|
|
138
|
-
type: "string",
|
|
139
|
-
description: "Clear, actionable, user-facing message explaining what the user needs to do.",
|
|
140
|
-
},
|
|
141
|
-
},
|
|
142
|
-
required: ["message"],
|
|
143
|
-
},
|
|
144
|
-
};
|
|
145
|
-
|
|
146
|
-
const mcpSearchTool: AnthropicTool = {
|
|
147
|
-
name: "mcp_search",
|
|
148
|
-
description:
|
|
149
|
-
"Search for MCP tools by keyword + semantic similarity over the live mcpx catalog. Returns up to a handful of {server, tool, description, score} entries.",
|
|
150
|
-
input_schema: {
|
|
151
|
-
type: "object" as const,
|
|
152
|
-
properties: {
|
|
153
|
-
query: { type: "string", description: "Search query (e.g. 'fetch google docs as markdown')." },
|
|
154
|
-
},
|
|
155
|
-
required: ["query"],
|
|
156
|
-
},
|
|
157
|
-
};
|
|
158
|
-
|
|
159
|
-
const mcpListToolsTool: AnthropicTool = {
|
|
160
|
-
name: "mcp_list_tools",
|
|
161
|
-
description: "List available tools from configured MCP servers. Optionally filter by server name.",
|
|
162
|
-
input_schema: {
|
|
163
|
-
type: "object" as const,
|
|
164
|
-
properties: {
|
|
165
|
-
server: { type: "string", description: "Optional server name to filter on." },
|
|
166
|
-
},
|
|
167
|
-
required: [],
|
|
168
|
-
},
|
|
169
|
-
};
|
|
170
|
-
|
|
171
|
-
const mcpInfoTool: AnthropicTool = {
|
|
172
|
-
name: "mcp_info",
|
|
173
|
-
description:
|
|
174
|
-
"Get the full schema (name, description, input parameters) for a specific MCP tool. Required before mcp_exec on tools you haven't called this session.",
|
|
175
|
-
input_schema: {
|
|
176
|
-
type: "object" as const,
|
|
177
|
-
properties: {
|
|
178
|
-
server: { type: "string", description: "MCP server name." },
|
|
179
|
-
tool: { type: "string", description: "Tool name on the server." },
|
|
180
|
-
},
|
|
181
|
-
required: ["server", "tool"],
|
|
182
|
-
},
|
|
183
|
-
};
|
|
184
|
-
|
|
185
|
-
const mcpExecTool: AnthropicTool = {
|
|
186
|
-
name: "mcp_exec",
|
|
187
|
-
description:
|
|
188
|
-
"Execute a tool on an MCP server. The full result is captured by the harness keyed by tool_use_id; you receive a short preview to verify the content. To save the result, call accept_content with the exec_call_id.",
|
|
189
|
-
input_schema: {
|
|
190
|
-
type: "object" as const,
|
|
191
|
-
properties: {
|
|
192
|
-
server: { type: "string", description: "MCP server name." },
|
|
193
|
-
tool: { type: "string", description: "Tool name on the server." },
|
|
194
|
-
args: {
|
|
195
|
-
type: "object",
|
|
196
|
-
description: "Arguments object that conforms to the tool's input schema (verify via mcp_info).",
|
|
197
|
-
},
|
|
198
|
-
},
|
|
199
|
-
required: ["server", "tool"],
|
|
200
|
-
},
|
|
201
|
-
};
|
|
202
|
-
|
|
203
|
-
const ALL_TOOLS: AnthropicTool[] = [
|
|
204
|
-
mcpSearchTool,
|
|
205
|
-
mcpListToolsTool,
|
|
206
|
-
mcpInfoTool,
|
|
207
|
-
mcpExecTool,
|
|
208
|
-
acceptContentTool,
|
|
209
|
-
requestHttpFallbackTool,
|
|
210
|
-
reportFailureTool,
|
|
211
|
-
];
|
|
212
|
-
|
|
213
|
-
interface CapturedExec {
|
|
214
|
-
server: string;
|
|
215
|
-
tool: string;
|
|
216
|
-
args: Record<string, unknown>;
|
|
217
|
-
content: string;
|
|
218
|
-
mimeType: string;
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
/**
|
|
222
|
-
* Run the multi-turn fetcher agent. Mirrors botholomew's `runFetcherLoop`.
|
|
223
|
-
*
|
|
224
|
-
* Returns `{ kind: "accepted", result }` when the agent calls `accept_content`
|
|
225
|
-
* on a captured mcp_exec result. Returns `{ kind: "fallback" }` when the agent
|
|
226
|
-
* calls `request_http_fallback`, produces no tool calls, or exhausts MAX_TURNS.
|
|
227
|
-
* Throws `HelpfulError` when the agent calls `report_failure` (the actionable
|
|
228
|
-
* message becomes the error's `message`/`hint`).
|
|
229
|
-
*/
|
|
230
|
-
export async function agentFetch(opts: AgentFetchOptions): Promise<AgentFetchOutcome> {
|
|
231
|
-
if (!opts.llm.anthropic_api_key || opts.llm.anthropic_api_key.trim() === "") {
|
|
232
|
-
throw new HelpfulError({
|
|
233
|
-
kind: "auth_error",
|
|
234
|
-
message: `agentFetch requires ANTHROPIC_API_KEY but llm.anthropic_api_key is empty.`,
|
|
235
|
-
hint: `Set ANTHROPIC_API_KEY in your environment or under llm.anthropic_api_key in ~/.membot/config.json.`,
|
|
236
|
-
});
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
const client = opts._testClient ?? new Anthropic({ apiKey: opts.llm.anthropic_api_key });
|
|
240
|
-
|
|
241
|
-
const userPrompt = opts.hint
|
|
242
|
-
? `Fetch the content at: ${opts.url}\n\nAdditional guidance:\n${opts.hint}`
|
|
243
|
-
: `Fetch the content at: ${opts.url}`;
|
|
244
|
-
const messages: MessageParam[] = [{ role: "user", content: userPrompt }];
|
|
245
|
-
|
|
246
|
-
const captured = new Map<string, CapturedExec>();
|
|
247
|
-
|
|
248
|
-
opts.onProgress?.(`fetching via mcpx agent (turn 1)`);
|
|
249
|
-
|
|
250
|
-
for (let turn = 0; turn < MAX_TURNS; turn++) {
|
|
251
|
-
if (turn > 0) {
|
|
252
|
-
logger.info(`[fetcher] turn ${turn + 1}/${MAX_TURNS}`);
|
|
253
|
-
opts.onProgress?.(`fetching via mcpx agent (turn ${turn + 1})`);
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
const response = await client.messages.create({
|
|
257
|
-
model: opts.llm.converter_model,
|
|
258
|
-
max_tokens: MAX_RESPONSE_TOKENS,
|
|
259
|
-
system: FETCHER_SYSTEM_PROMPT,
|
|
260
|
-
messages,
|
|
261
|
-
tools: ALL_TOOLS,
|
|
262
|
-
});
|
|
263
|
-
|
|
264
|
-
for (const block of response.content) {
|
|
265
|
-
if (block.type === "text" && block.text.trim()) {
|
|
266
|
-
logger.debug(`[fetcher] turn ${turn + 1} reasoning: ${block.text.trim()}`);
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
if (response.stop_reason === "max_tokens") {
|
|
271
|
-
throw new HelpfulError({
|
|
272
|
-
kind: "internal_error",
|
|
273
|
-
message: `Fetcher agent hit max_tokens (${MAX_RESPONSE_TOKENS}) on turn ${turn + 1}.`,
|
|
274
|
-
hint: `The fetched document or the agent's reasoning is too long. Try \`membot add ${opts.url} --fetcher http\` or fetch a more specific section.`,
|
|
275
|
-
});
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
const toolUseBlocks = response.content.filter((b): b is ToolUseBlock => b.type === "tool_use");
|
|
279
|
-
if (toolUseBlocks.length === 0) {
|
|
280
|
-
logger.info(`[fetcher] turn ${turn + 1}: no tool calls — falling back to HTTP`);
|
|
281
|
-
return { kind: "fallback", reason: "agent stopped without selecting an outcome" };
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
messages.push({ role: "assistant", content: response.content });
|
|
285
|
-
|
|
286
|
-
// Log selected tools at info-level so users see what the agent is doing
|
|
287
|
-
// without enabling --verbose. Discovery (search/info/list) stays quiet
|
|
288
|
-
// at info; the actual mcp_exec calls are the high-signal events.
|
|
289
|
-
for (const tu of toolUseBlocks) {
|
|
290
|
-
logToolSelection(tu, turn + 1, opts.onProgress);
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
// Terminal tools — checked in priority order.
|
|
294
|
-
const failureCall = toolUseBlocks.find((b) => b.name === "report_failure");
|
|
295
|
-
if (failureCall) {
|
|
296
|
-
const input = failureCall.input as Partial<{ message: string }>;
|
|
297
|
-
const message =
|
|
298
|
-
typeof input.message === "string" && input.message.trim()
|
|
299
|
-
? input.message.trim()
|
|
300
|
-
: "Fetch failed but the agent did not provide a message.";
|
|
301
|
-
logger.info(`[fetcher] turn ${turn + 1}: report_failure: ${message}`);
|
|
302
|
-
throw new HelpfulError({
|
|
303
|
-
kind: "input_error",
|
|
304
|
-
message: `Fetcher agent reported failure for ${opts.url}: ${message}`,
|
|
305
|
-
hint: message,
|
|
306
|
-
});
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
const fallbackCall = toolUseBlocks.find((b) => b.name === "request_http_fallback");
|
|
310
|
-
if (fallbackCall) {
|
|
311
|
-
logger.info(`[fetcher] turn ${turn + 1}: agent requested HTTP fallback`);
|
|
312
|
-
return { kind: "fallback", reason: "agent requested HTTP fallback" };
|
|
313
|
-
}
|
|
314
|
-
|
|
315
|
-
const acceptCall = toolUseBlocks.find((b) => b.name === "accept_content");
|
|
316
|
-
if (acceptCall) {
|
|
317
|
-
const input = acceptCall.input as Partial<{ exec_call_id: string; mime_type: string }>;
|
|
318
|
-
if (typeof input.exec_call_id !== "string") {
|
|
319
|
-
messages.push({
|
|
320
|
-
role: "user",
|
|
321
|
-
content: [
|
|
322
|
-
{
|
|
323
|
-
type: "tool_result",
|
|
324
|
-
tool_use_id: acceptCall.id,
|
|
325
|
-
content: "Invalid accept_content call: 'exec_call_id' is required.",
|
|
326
|
-
is_error: true,
|
|
327
|
-
},
|
|
328
|
-
],
|
|
329
|
-
});
|
|
330
|
-
continue;
|
|
331
|
-
}
|
|
332
|
-
const cached = captured.get(input.exec_call_id);
|
|
333
|
-
if (!cached) {
|
|
334
|
-
const validIds = [...captured.keys()];
|
|
335
|
-
messages.push({
|
|
336
|
-
role: "user",
|
|
337
|
-
content: [
|
|
338
|
-
{
|
|
339
|
-
type: "tool_result",
|
|
340
|
-
tool_use_id: acceptCall.id,
|
|
341
|
-
content: `No mcp_exec call with id "${input.exec_call_id}" was captured. Captured ids: ${validIds.length ? validIds.join(", ") : "(none yet — run mcp_exec first)"}.`,
|
|
342
|
-
is_error: true,
|
|
343
|
-
},
|
|
344
|
-
],
|
|
345
|
-
});
|
|
346
|
-
continue;
|
|
347
|
-
}
|
|
348
|
-
const claimedMime = (input.mime_type ?? cached.mimeType ?? "text/markdown").trim() || "text/markdown";
|
|
349
|
-
const bytes = new TextEncoder().encode(cached.content);
|
|
350
|
-
logger.info(`[fetcher] accepted: ${cached.server}/${cached.tool} (${bytes.byteLength} bytes, ${claimedMime})`);
|
|
351
|
-
logger.debug(`[fetcher] accepted args: ${truncateJson(cached.args, 500)}`);
|
|
352
|
-
logger.debug(`[fetcher] accepted preview: ${truncate(cached.content, 200)}`);
|
|
353
|
-
opts.onProgress?.(`accepted ${cached.server}/${cached.tool}`);
|
|
354
|
-
return {
|
|
355
|
-
kind: "accepted",
|
|
356
|
-
result: {
|
|
357
|
-
bytes,
|
|
358
|
-
sha256: sha256Hex(bytes),
|
|
359
|
-
mimeType: claimedMime,
|
|
360
|
-
fetcher: "mcpx",
|
|
361
|
-
fetcherServer: cached.server,
|
|
362
|
-
fetcherTool: cached.tool,
|
|
363
|
-
fetcherArgs: cached.args,
|
|
364
|
-
sourceUrl: opts.url,
|
|
365
|
-
},
|
|
366
|
-
};
|
|
367
|
-
}
|
|
368
|
-
|
|
369
|
-
// Discovery / exec tools — execute in parallel, feed results back.
|
|
370
|
-
const toolResults: ToolResultBlockParam[] = await Promise.all(
|
|
371
|
-
toolUseBlocks.map((toolUse) => dispatchAgentTool(toolUse, opts.mcpx, captured)),
|
|
372
|
-
);
|
|
373
|
-
messages.push({ role: "user", content: toolResults });
|
|
374
|
-
}
|
|
375
|
-
|
|
376
|
-
logger.info(`[fetcher] max turns (${MAX_TURNS}) exceeded — falling back to HTTP`);
|
|
377
|
-
return { kind: "fallback", reason: `agent exceeded MAX_TURNS=${MAX_TURNS}` };
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
/**
|
|
381
|
-
* Emit a per-turn line about which tool the agent is about to invoke. mcp_exec
|
|
382
|
-
* is the high-signal event that surfaces *which provider was chosen*, so it
|
|
383
|
-
* goes to info; discovery (search / list / info) stays at debug.
|
|
384
|
-
*/
|
|
385
|
-
function logToolSelection(tu: ToolUseBlock, turn: number, onProgress?: (s: string) => void): void {
|
|
386
|
-
if (tu.name === "mcp_exec") {
|
|
387
|
-
const i = tu.input as Partial<{ server: string; tool: string; args: Record<string, unknown> }>;
|
|
388
|
-
const server = i.server ?? "?";
|
|
389
|
-
const tool = i.tool ?? "?";
|
|
390
|
-
logger.info(`[fetcher] turn ${turn}: mcp_exec ${server}/${tool}`);
|
|
391
|
-
logger.debug(`[fetcher] turn ${turn}: mcp_exec args: ${truncateJson(i.args ?? {}, 500)}`);
|
|
392
|
-
onProgress?.(`mcp_exec ${server}/${tool} (turn ${turn})`);
|
|
393
|
-
} else if (tu.name === "mcp_search") {
|
|
394
|
-
const i = tu.input as Partial<{ query: string }>;
|
|
395
|
-
logger.debug(`[fetcher] turn ${turn}: mcp_search "${i.query ?? ""}"`);
|
|
396
|
-
} else if (tu.name === "mcp_info") {
|
|
397
|
-
const i = tu.input as Partial<{ server: string; tool: string }>;
|
|
398
|
-
logger.debug(`[fetcher] turn ${turn}: mcp_info ${i.server ?? "?"}/${i.tool ?? "?"}`);
|
|
399
|
-
} else if (tu.name === "mcp_list_tools") {
|
|
400
|
-
const i = tu.input as Partial<{ server: string }>;
|
|
401
|
-
logger.debug(`[fetcher] turn ${turn}: mcp_list_tools${i.server ? ` ${i.server}` : ""}`);
|
|
402
|
-
}
|
|
403
|
-
}
|
|
404
|
-
|
|
405
|
-
/** JSON-stringify with a length cap so a giant args payload doesn't bloat logs. */
|
|
406
|
-
function truncateJson(value: unknown, max: number): string {
|
|
407
|
-
let s: string;
|
|
408
|
-
try {
|
|
409
|
-
s = JSON.stringify(value);
|
|
410
|
-
} catch {
|
|
411
|
-
s = String(value);
|
|
412
|
-
}
|
|
413
|
-
return s.length > max ? `${s.slice(0, max)}… (+${s.length - max} chars)` : s;
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
/** Single-line truncation for debug previews; collapses whitespace. */
|
|
417
|
-
function truncate(s: string, max: number): string {
|
|
418
|
-
const oneLine = s.replace(/\s+/g, " ").trim();
|
|
419
|
-
return oneLine.length > max ? `${oneLine.slice(0, max)}… (+${oneLine.length - max} chars)` : oneLine;
|
|
420
|
-
}
|
|
421
|
-
|
|
422
|
-
/** Execute one agent tool call and produce the tool_result block fed back to Claude. */
|
|
423
|
-
async function dispatchAgentTool(
|
|
424
|
-
toolUse: ToolUseBlock,
|
|
425
|
-
mcpx: AgentMcpxAdapter,
|
|
426
|
-
captured: Map<string, CapturedExec>,
|
|
427
|
-
): Promise<ToolResultBlockParam> {
|
|
428
|
-
try {
|
|
429
|
-
switch (toolUse.name) {
|
|
430
|
-
case "mcp_search":
|
|
431
|
-
return await runMcpSearch(toolUse, mcpx);
|
|
432
|
-
case "mcp_list_tools":
|
|
433
|
-
return await runMcpListTools(toolUse, mcpx);
|
|
434
|
-
case "mcp_info":
|
|
435
|
-
return await runMcpInfo(toolUse, mcpx);
|
|
436
|
-
case "mcp_exec":
|
|
437
|
-
return await runMcpExec(toolUse, mcpx, captured);
|
|
438
|
-
default:
|
|
439
|
-
return {
|
|
440
|
-
type: "tool_result",
|
|
441
|
-
tool_use_id: toolUse.id,
|
|
442
|
-
content: `Unknown tool: ${toolUse.name}`,
|
|
443
|
-
is_error: true,
|
|
444
|
-
};
|
|
445
|
-
}
|
|
446
|
-
} catch (err) {
|
|
447
|
-
return {
|
|
448
|
-
type: "tool_result",
|
|
449
|
-
tool_use_id: toolUse.id,
|
|
450
|
-
content: `Error: ${err instanceof Error ? err.message : String(err)}`,
|
|
451
|
-
is_error: true,
|
|
452
|
-
};
|
|
453
|
-
}
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
async function runMcpSearch(toolUse: ToolUseBlock, mcpx: AgentMcpxAdapter): Promise<ToolResultBlockParam> {
|
|
457
|
-
const input = toolUse.input as Partial<{ query: string }>;
|
|
458
|
-
if (typeof input.query !== "string" || !input.query.trim()) {
|
|
459
|
-
return { type: "tool_result", tool_use_id: toolUse.id, content: "mcp_search requires 'query'.", is_error: true };
|
|
460
|
-
}
|
|
461
|
-
try {
|
|
462
|
-
const results = await mcpx.search(input.query);
|
|
463
|
-
const top = results.slice(0, 3).map((r) => `${r.server}/${r.tool}${r.score ? ` (${r.score.toFixed(2)})` : ""}`);
|
|
464
|
-
logger.debug(`[fetcher] mcp_search "${input.query}" → ${top.length ? top.join(", ") : "(no hits)"}`);
|
|
465
|
-
return {
|
|
466
|
-
type: "tool_result",
|
|
467
|
-
tool_use_id: toolUse.id,
|
|
468
|
-
content: JSON.stringify(
|
|
469
|
-
{
|
|
470
|
-
results: results.slice(0, 10).map((r) => ({
|
|
471
|
-
server: r.server,
|
|
472
|
-
tool: r.tool,
|
|
473
|
-
description: r.description ?? "",
|
|
474
|
-
score: r.score ?? 0,
|
|
475
|
-
})),
|
|
476
|
-
hint:
|
|
477
|
-
results.length > 0
|
|
478
|
-
? "Use mcp_info to read the input schema before mcp_exec."
|
|
479
|
-
: "No results. Try broader terms or mcp_list_tools.",
|
|
480
|
-
},
|
|
481
|
-
null,
|
|
482
|
-
2,
|
|
483
|
-
),
|
|
484
|
-
};
|
|
485
|
-
} catch (err) {
|
|
486
|
-
return {
|
|
487
|
-
type: "tool_result",
|
|
488
|
-
tool_use_id: toolUse.id,
|
|
489
|
-
content: `mcp_search failed: ${err instanceof Error ? err.message : String(err)}. Try mcp_list_tools instead.`,
|
|
490
|
-
is_error: true,
|
|
491
|
-
};
|
|
492
|
-
}
|
|
493
|
-
}
|
|
494
|
-
|
|
495
|
-
async function runMcpListTools(toolUse: ToolUseBlock, mcpx: AgentMcpxAdapter): Promise<ToolResultBlockParam> {
|
|
496
|
-
const input = toolUse.input as Partial<{ server: string }>;
|
|
497
|
-
const tools = await mcpx.listTools(input.server);
|
|
498
|
-
const mapped = tools.map((t) => ({ server: t.server, name: t.tool.name, description: t.tool.description ?? "" }));
|
|
499
|
-
return {
|
|
500
|
-
type: "tool_result",
|
|
501
|
-
tool_use_id: toolUse.id,
|
|
502
|
-
content: JSON.stringify(
|
|
503
|
-
{
|
|
504
|
-
tools: mapped,
|
|
505
|
-
hint:
|
|
506
|
-
mapped.length > 0
|
|
507
|
-
? "Use mcp_info on a {server, name} pair before mcp_exec."
|
|
508
|
-
: "No tools. mcpx may not be configured.",
|
|
509
|
-
},
|
|
510
|
-
null,
|
|
511
|
-
2,
|
|
512
|
-
),
|
|
513
|
-
};
|
|
514
|
-
}
|
|
515
|
-
|
|
516
|
-
async function runMcpInfo(toolUse: ToolUseBlock, mcpx: AgentMcpxAdapter): Promise<ToolResultBlockParam> {
|
|
517
|
-
const input = toolUse.input as Partial<{ server: string; tool: string }>;
|
|
518
|
-
if (typeof input.server !== "string" || typeof input.tool !== "string") {
|
|
519
|
-
return {
|
|
520
|
-
type: "tool_result",
|
|
521
|
-
tool_use_id: toolUse.id,
|
|
522
|
-
content: "mcp_info requires 'server' and 'tool'.",
|
|
523
|
-
is_error: true,
|
|
524
|
-
};
|
|
525
|
-
}
|
|
526
|
-
const tool = await mcpx.info(input.server, input.tool);
|
|
527
|
-
if (!tool) {
|
|
528
|
-
return {
|
|
529
|
-
type: "tool_result",
|
|
530
|
-
tool_use_id: toolUse.id,
|
|
531
|
-
content: `Tool "${input.tool}" not found on server "${input.server}". Use mcp_search or mcp_list_tools.`,
|
|
532
|
-
is_error: true,
|
|
533
|
-
};
|
|
534
|
-
}
|
|
535
|
-
return {
|
|
536
|
-
type: "tool_result",
|
|
537
|
-
tool_use_id: toolUse.id,
|
|
538
|
-
content: JSON.stringify(
|
|
539
|
-
{
|
|
540
|
-
name: tool.name,
|
|
541
|
-
description: tool.description ?? "",
|
|
542
|
-
input_schema: tool.inputSchema ?? {},
|
|
543
|
-
hint: `Call mcp_exec with server='${input.server}', tool='${tool.name}', and args matching this schema.`,
|
|
544
|
-
},
|
|
545
|
-
null,
|
|
546
|
-
2,
|
|
547
|
-
),
|
|
548
|
-
};
|
|
549
|
-
}
|
|
550
|
-
|
|
551
|
-
async function runMcpExec(
|
|
552
|
-
toolUse: ToolUseBlock,
|
|
553
|
-
mcpx: AgentMcpxAdapter,
|
|
554
|
-
captured: Map<string, CapturedExec>,
|
|
555
|
-
): Promise<ToolResultBlockParam> {
|
|
556
|
-
const input = toolUse.input as Partial<{ server: string; tool: string; args: Record<string, unknown> }>;
|
|
557
|
-
if (typeof input.server !== "string" || typeof input.tool !== "string") {
|
|
558
|
-
return {
|
|
559
|
-
type: "tool_result",
|
|
560
|
-
tool_use_id: toolUse.id,
|
|
561
|
-
content: "mcp_exec requires 'server' and 'tool'.",
|
|
562
|
-
is_error: true,
|
|
563
|
-
};
|
|
564
|
-
}
|
|
565
|
-
const args = (input.args ?? {}) as Record<string, unknown>;
|
|
566
|
-
|
|
567
|
-
let result: { isError?: boolean; content?: unknown[] };
|
|
568
|
-
try {
|
|
569
|
-
result = await mcpx.exec(input.server, input.tool, args);
|
|
570
|
-
} catch (err) {
|
|
571
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
572
|
-
logger.info(`[fetcher] → ${input.server}/${input.tool} threw: ${truncate(msg, 200)}`);
|
|
573
|
-
return {
|
|
574
|
-
type: "tool_result",
|
|
575
|
-
tool_use_id: toolUse.id,
|
|
576
|
-
content: `mcp_exec ${input.server}/${input.tool} threw: ${msg}. Use mcp_info to verify the schema, then retry — or pivot to a different tool.`,
|
|
577
|
-
is_error: true,
|
|
578
|
-
};
|
|
579
|
-
}
|
|
580
|
-
|
|
581
|
-
const text = extractText(result);
|
|
582
|
-
|
|
583
|
-
if (result.isError === true) {
|
|
584
|
-
logger.info(`[fetcher] → ${input.server}/${input.tool} error: ${truncate(text, 200)}`);
|
|
585
|
-
return {
|
|
586
|
-
type: "tool_result",
|
|
587
|
-
tool_use_id: toolUse.id,
|
|
588
|
-
content: `mcp_exec ${input.server}/${input.tool} returned isError=true: ${text}\n\nUse mcp_info to check the schema, fix the args, and retry — or try a different tool.`,
|
|
589
|
-
is_error: true,
|
|
590
|
-
};
|
|
591
|
-
}
|
|
592
|
-
|
|
593
|
-
if (!text?.trim()) {
|
|
594
|
-
logger.info(`[fetcher] → ${input.server}/${input.tool} empty result`);
|
|
595
|
-
return {
|
|
596
|
-
type: "tool_result",
|
|
597
|
-
tool_use_id: toolUse.id,
|
|
598
|
-
content: `mcp_exec ${input.server}/${input.tool} returned empty content. Try a different tool or different args.`,
|
|
599
|
-
is_error: true,
|
|
600
|
-
};
|
|
601
|
-
}
|
|
602
|
-
|
|
603
|
-
logger.info(`[fetcher] → ${input.server}/${input.tool} ok (${text.length} chars)`);
|
|
604
|
-
captured.set(toolUse.id, { server: input.server, tool: input.tool, args, content: text, mimeType: "text/markdown" });
|
|
605
|
-
const preview =
|
|
606
|
-
text.length > PREVIEW_CHARS
|
|
607
|
-
? `${text.slice(0, PREVIEW_CHARS)}\n\n[... ${text.length - PREVIEW_CHARS} more chars truncated. Full content (${text.length} chars total) is captured by the harness with exec_call_id="${toolUse.id}". Call accept_content with this id to save it.]`
|
|
608
|
-
: `${text}\n\n[Full content (${text.length} chars) captured by the harness with exec_call_id="${toolUse.id}". Call accept_content with this id to save it.]`;
|
|
609
|
-
return { type: "tool_result", tool_use_id: toolUse.id, content: preview };
|
|
610
|
-
}
|
|
611
|
-
|
|
612
|
-
/**
|
|
613
|
-
* Extract a single string out of an MCP CallToolResult envelope. Mirrors
|
|
614
|
-
* the heterogeneous shapes mcpx tools return; tolerates string content,
|
|
615
|
-
* `text` fields, and the array-of-content-blocks shape.
|
|
616
|
-
*/
|
|
617
|
-
function extractText(result: { content?: unknown } | unknown): string {
|
|
618
|
-
if (typeof result === "string") return result;
|
|
619
|
-
if (!result || typeof result !== "object") return "";
|
|
620
|
-
const r = result as Record<string, unknown>;
|
|
621
|
-
if (typeof r.text === "string") return r.text;
|
|
622
|
-
if (typeof r.content === "string") return r.content;
|
|
623
|
-
if (typeof r.markdown === "string") return r.markdown;
|
|
624
|
-
if (Array.isArray(r.content)) {
|
|
625
|
-
const out: string[] = [];
|
|
626
|
-
for (const c of r.content) {
|
|
627
|
-
if (c && typeof c === "object") {
|
|
628
|
-
const inner = c as Record<string, unknown>;
|
|
629
|
-
if (typeof inner.text === "string") out.push(inner.text);
|
|
630
|
-
}
|
|
631
|
-
}
|
|
632
|
-
if (out.length > 0) return out.join("\n\n");
|
|
633
|
-
}
|
|
634
|
-
try {
|
|
635
|
-
return JSON.stringify(result);
|
|
636
|
-
} catch {
|
|
637
|
-
return "";
|
|
638
|
-
}
|
|
639
|
-
}
|