membot 0.0.1 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/membot.md +137 -0
- package/.cursor/rules/membot.mdc +137 -0
- package/README.md +131 -0
- package/package.json +83 -24
- package/patches/@huggingface%2Ftransformers@4.2.0.patch +137 -0
- package/scripts/apply-transformers-patch.sh +35 -0
- package/src/cli.ts +72 -0
- package/src/commands/check-update.ts +69 -0
- package/src/commands/mcpx.ts +112 -0
- package/src/commands/reindex.ts +53 -0
- package/src/commands/serve.ts +58 -0
- package/src/commands/skill.ts +131 -0
- package/src/commands/upgrade.ts +220 -0
- package/src/config/loader.ts +100 -0
- package/src/config/schemas.ts +39 -0
- package/src/constants.ts +42 -0
- package/src/context.ts +80 -0
- package/src/db/blobs.ts +53 -0
- package/src/db/chunks.ts +176 -0
- package/src/db/connection.ts +173 -0
- package/src/db/files.ts +325 -0
- package/src/db/migrations/001-init.ts +63 -0
- package/src/db/migrations/002-fts.ts +12 -0
- package/src/db/migrations.ts +45 -0
- package/src/errors.ts +87 -0
- package/src/ingest/chunker.ts +117 -0
- package/src/ingest/converter/docx.ts +15 -0
- package/src/ingest/converter/html.ts +20 -0
- package/src/ingest/converter/image.ts +71 -0
- package/src/ingest/converter/index.ts +119 -0
- package/src/ingest/converter/llm.ts +66 -0
- package/src/ingest/converter/ocr.ts +51 -0
- package/src/ingest/converter/pdf.ts +38 -0
- package/src/ingest/converter/text.ts +8 -0
- package/src/ingest/describer.ts +72 -0
- package/src/ingest/embedder.ts +98 -0
- package/src/ingest/fetcher.ts +280 -0
- package/src/ingest/ingest.ts +444 -0
- package/src/ingest/local-reader.ts +64 -0
- package/src/ingest/search-text.ts +18 -0
- package/src/ingest/source-resolver.ts +186 -0
- package/src/mcp/instructions.ts +34 -0
- package/src/mcp/server.ts +101 -0
- package/src/mount/commander.ts +174 -0
- package/src/mount/mcp.ts +111 -0
- package/src/mount/zod-to-cli.ts +158 -0
- package/src/operations/add.ts +69 -0
- package/src/operations/diff.ts +105 -0
- package/src/operations/index.ts +38 -0
- package/src/operations/info.ts +95 -0
- package/src/operations/list.ts +87 -0
- package/src/operations/move.ts +83 -0
- package/src/operations/prune.ts +80 -0
- package/src/operations/read.ts +102 -0
- package/src/operations/refresh.ts +72 -0
- package/src/operations/remove.ts +35 -0
- package/src/operations/search.ts +72 -0
- package/src/operations/tree.ts +103 -0
- package/src/operations/types.ts +81 -0
- package/src/operations/versions.ts +78 -0
- package/src/operations/write.ts +77 -0
- package/src/output/formatter.ts +68 -0
- package/src/output/logger.ts +114 -0
- package/src/output/progress.ts +78 -0
- package/src/output/tty.ts +91 -0
- package/src/refresh/runner.ts +296 -0
- package/src/refresh/scheduler.ts +54 -0
- package/src/sdk.ts +27 -0
- package/src/search/hybrid.ts +100 -0
- package/src/search/keyword.ts +62 -0
- package/src/search/semantic.ts +56 -0
- package/src/types/text-modules.d.ts +9 -0
- package/src/update/background.ts +73 -0
- package/src/update/cache.ts +40 -0
- package/src/update/checker.ts +117 -0
- package/.claude/settings.local.json +0 -7
- package/CLAUDE.md +0 -139
- package/docs/plan.md +0 -905
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Server-level instructions sent to the LLM when it connects to membot's
|
|
3
|
+
* MCP server. Frames how the tool surface should be used: discovery →
|
|
4
|
+
* ingest → consume → write, with explicit notes on versioning and refresh.
|
|
5
|
+
*/
|
|
6
|
+
export const SERVER_INSTRUCTIONS = `You have a persistent context store. Files live as versioned markdown rows
|
|
7
|
+
addressed by logical path (e.g. "research/threat-models/llm.md"). The store
|
|
8
|
+
is a hybrid search index: every file is chunked, embedded locally, and
|
|
9
|
+
indexed with BM25 — so prefer membot_search to membot_read+grep for discovery.
|
|
10
|
+
|
|
11
|
+
Workflow:
|
|
12
|
+
1. membot_tree or membot_search to find what already exists before adding new content.
|
|
13
|
+
2. membot_add to ingest a local file, a URL, or a remote document. URLs are
|
|
14
|
+
fetched via mcpx (the chosen invocation is stored so refresh is fast and
|
|
15
|
+
deterministic).
|
|
16
|
+
3. membot_read or membot_search hits to consume content.
|
|
17
|
+
4. membot_write to record agent-authored notes (source_type='inline').
|
|
18
|
+
|
|
19
|
+
Versioning:
|
|
20
|
+
- Every ingest, refresh, or write that changes content creates a NEW
|
|
21
|
+
version_id (a timestamp). Older versions stay queryable via the
|
|
22
|
+
\`version\` parameter on membot_read / membot_info / membot_versions / membot_diff.
|
|
23
|
+
- All other tools default to the current (latest, non-tombstoned) version.
|
|
24
|
+
- membot_delete is a tombstone — history is preserved unless membot_prune runs.
|
|
25
|
+
|
|
26
|
+
Refresh:
|
|
27
|
+
- Each row has source metadata. membot_refresh re-reads the source, hashes
|
|
28
|
+
it, and only re-embeds when bytes changed. Safe to call often.
|
|
29
|
+
- If a file has refresh_frequency_sec set, the daemon refreshes it
|
|
30
|
+
automatically — you do not need to schedule it yourself.
|
|
31
|
+
|
|
32
|
+
When in doubt: search before you read, read before you write, and prefer
|
|
33
|
+
adding the source URL once (with a refresh interval) over copy-pasting
|
|
34
|
+
content that will go stale.`;
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
3
|
+
import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
|
|
4
|
+
import { type AppContext, buildContext, closeContext } from "../context.ts";
|
|
5
|
+
import { mountAsMcpTool } from "../mount/mcp.ts";
|
|
6
|
+
import { OPERATIONS } from "../operations/index.ts";
|
|
7
|
+
import { logger } from "../output/logger.ts";
|
|
8
|
+
import { SERVER_INSTRUCTIONS } from "./instructions.ts";
|
|
9
|
+
|
|
10
|
+
export interface McpServerOptions {
|
|
11
|
+
configFlag?: string;
|
|
12
|
+
httpPort?: number;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Build a fresh `McpServer` instance with every Operation mounted as a
|
|
17
|
+
* tool. The supplied `ctxFactory` is awaited lazily on the first tool
|
|
18
|
+
* invocation — for stdio servers we share one context across the connection;
|
|
19
|
+
* for HTTP servers we'd want one context per session, but for now a single
|
|
20
|
+
* lazy-initialized context is fine.
|
|
21
|
+
*/
|
|
22
|
+
export function buildMcpServer(ctxFactory: () => Promise<AppContext>): McpServer {
|
|
23
|
+
const server = new McpServer({ name: "membot", version: "0.0.1" }, { instructions: SERVER_INSTRUCTIONS });
|
|
24
|
+
|
|
25
|
+
let ctxPromise: Promise<AppContext> | null = null;
|
|
26
|
+
const getCtx = async () => {
|
|
27
|
+
if (!ctxPromise) ctxPromise = ctxFactory();
|
|
28
|
+
return ctxPromise;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
for (const op of OPERATIONS) {
|
|
32
|
+
mountAsMcpTool(server, op, getCtx);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
return server;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Start the MCP server in stdio mode. Used by `membot serve` (default
|
|
40
|
+
* transport) so MCP clients (mcpx, Claude Desktop, etc.) can connect over
|
|
41
|
+
* stdin/stdout.
|
|
42
|
+
*/
|
|
43
|
+
export async function startStdioServer(options: McpServerOptions = {}): Promise<() => Promise<void>> {
|
|
44
|
+
let ctx: AppContext | null = null;
|
|
45
|
+
const server = buildMcpServer(async () => {
|
|
46
|
+
ctx = await buildContext({ configFlag: options.configFlag, json: true });
|
|
47
|
+
return ctx;
|
|
48
|
+
});
|
|
49
|
+
const transport = new StdioServerTransport();
|
|
50
|
+
await server.connect(transport);
|
|
51
|
+
logger.info("membot-mcp: stdio server connected");
|
|
52
|
+
return async () => {
|
|
53
|
+
await server.close();
|
|
54
|
+
if (ctx) await closeContext(ctx);
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Start the MCP server in HTTP (streamable) mode. Used by
|
|
60
|
+
* `membot serve --http <port>` to expose the same tools over HTTP for
|
|
61
|
+
* browser-based or remote clients.
|
|
62
|
+
*/
|
|
63
|
+
export async function startHttpServer(port: number, options: McpServerOptions = {}): Promise<() => Promise<void>> {
|
|
64
|
+
let ctx: AppContext | null = null;
|
|
65
|
+
const server = buildMcpServer(async () => {
|
|
66
|
+
ctx = await buildContext({ configFlag: options.configFlag });
|
|
67
|
+
return ctx;
|
|
68
|
+
});
|
|
69
|
+
const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: () => crypto.randomUUID() });
|
|
70
|
+
await server.connect(transport);
|
|
71
|
+
|
|
72
|
+
const httpServer = Bun.serve({
|
|
73
|
+
port,
|
|
74
|
+
async fetch(req) {
|
|
75
|
+
const url = new URL(req.url);
|
|
76
|
+
if (url.pathname !== "/mcp") return new Response("not found", { status: 404 });
|
|
77
|
+
const body = await req.arrayBuffer();
|
|
78
|
+
const headers: Record<string, string> = {};
|
|
79
|
+
req.headers.forEach((v, k) => {
|
|
80
|
+
headers[k] = v;
|
|
81
|
+
});
|
|
82
|
+
// Adapt Bun's Request → Node-shaped req/res. Streamable HTTP
|
|
83
|
+
// transport expects a Node IncomingMessage / ServerResponse;
|
|
84
|
+
// for now the SDK provides handlers for Web's Request directly
|
|
85
|
+
// in newer versions. Simplest: forward to transport.handleRequest.
|
|
86
|
+
const resp = await transport.handleRequest(
|
|
87
|
+
req as unknown as Parameters<typeof transport.handleRequest>[0],
|
|
88
|
+
undefined as unknown as Parameters<typeof transport.handleRequest>[1],
|
|
89
|
+
body,
|
|
90
|
+
);
|
|
91
|
+
return resp as unknown as Response;
|
|
92
|
+
},
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
logger.info(`membot-mcp: http listening on :${port}/mcp`);
|
|
96
|
+
return async () => {
|
|
97
|
+
httpServer.stop();
|
|
98
|
+
await server.close();
|
|
99
|
+
if (ctx) await closeContext(ctx);
|
|
100
|
+
};
|
|
101
|
+
}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import type { Command } from "commander";
|
|
2
|
+
import type { z } from "zod";
|
|
3
|
+
import { type AppContext, type BuildContextOptions, buildContext, closeContext } from "../context.ts";
|
|
4
|
+
import { asHelpful, HelpfulError, isHelpfulError, mapKindToExit } from "../errors.ts";
|
|
5
|
+
import { composeDescription, defaultCliName, type Operation } from "../operations/types.ts";
|
|
6
|
+
import { colors, renderResult } from "../output/formatter.ts";
|
|
7
|
+
import { logger } from "../output/logger.ts";
|
|
8
|
+
import { isJson } from "../output/tty.ts";
|
|
9
|
+
import { applySchemaToCommand, toKebab } from "./zod-to-cli.ts";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Mount an Operation as a commander subcommand. The command:
|
|
13
|
+
* 1. accepts positional + flag args inferred from the zod input schema
|
|
14
|
+
* 2. validates with the same schema
|
|
15
|
+
* 3. starts a spinner, runs the handler, prints the formatted result
|
|
16
|
+
* 4. catches `HelpfulError` and renders it (color text on TTY, JSON on stderr otherwise)
|
|
17
|
+
*/
|
|
18
|
+
export function mountAsCommanderCommand<I extends z.ZodObject, O extends z.ZodTypeAny>(
|
|
19
|
+
program: Command,
|
|
20
|
+
op: Operation<I, O>,
|
|
21
|
+
getContextOptions: () => BuildContextOptions,
|
|
22
|
+
): void {
|
|
23
|
+
const cmdName = defaultCliName(op);
|
|
24
|
+
const cmd = program.command(cmdName).description(composeDescription(op));
|
|
25
|
+
|
|
26
|
+
applySchemaToCommand(cmd, op.inputSchema, {
|
|
27
|
+
positional: (op.cli?.positional as readonly string[] | undefined) ?? [],
|
|
28
|
+
aliases: op.cli?.aliases as Readonly<Record<string, string>> | undefined,
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
cmd.action(async (...args: unknown[]) => {
|
|
32
|
+
// Commander passes positionals first, then the options object, then the Command instance.
|
|
33
|
+
// The middle option-bag is what we want for flag values.
|
|
34
|
+
let optsObj: Record<string, unknown> = {};
|
|
35
|
+
for (const a of args) {
|
|
36
|
+
if (a && typeof a === "object" && !Array.isArray(a) && a.constructor && a.constructor.name === "Object") {
|
|
37
|
+
optsObj = a as Record<string, unknown>;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
const positionals = args.slice(0, op.cli?.positional?.length ?? 0);
|
|
41
|
+
|
|
42
|
+
const inputObj: Record<string, unknown> = {};
|
|
43
|
+
|
|
44
|
+
const positionalNames = (op.cli?.positional ?? []) as readonly string[];
|
|
45
|
+
positionalNames.forEach((name, i) => {
|
|
46
|
+
if (positionals[i] !== undefined) inputObj[name] = positionals[i];
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
for (const fieldName of Object.keys(op.inputSchema.shape)) {
|
|
50
|
+
if (positionalNames.includes(fieldName)) continue;
|
|
51
|
+
const camel = kebabToCamel(toKebab(fieldName));
|
|
52
|
+
const v = optsObj[camel] ?? optsObj[fieldName];
|
|
53
|
+
if (v !== undefined) inputObj[fieldName] = v;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// stdinField support: read stdin when the field is missing AND stdin is not a TTY.
|
|
57
|
+
if (op.cli?.stdinField && inputObj[op.cli.stdinField as string] === undefined && !process.stdin.isTTY) {
|
|
58
|
+
const stdin = await readStdin();
|
|
59
|
+
if (stdin.length > 0) inputObj[op.cli.stdinField as string] = stdin;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
let ctx: AppContext | null = null;
|
|
63
|
+
try {
|
|
64
|
+
const parsedInput = parseInput(op, inputObj);
|
|
65
|
+
ctx = await buildContext(getContextOptions());
|
|
66
|
+
const result = await op.handler(parsedInput, ctx);
|
|
67
|
+
const validated = parseOutput(op, result);
|
|
68
|
+
process.stdout.write(`${renderResult(validated, { console_formatter: op.console_formatter })}\n`);
|
|
69
|
+
} catch (err) {
|
|
70
|
+
renderCliError(err);
|
|
71
|
+
const exitCode = isHelpfulError(err) ? mapKindToExit(err.kind) : 1;
|
|
72
|
+
if (ctx) await closeContext(ctx);
|
|
73
|
+
process.exit(exitCode);
|
|
74
|
+
}
|
|
75
|
+
if (ctx) await closeContext(ctx);
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Validate the user-supplied input against the operation's zod schema. */
|
|
80
|
+
function parseInput<I extends z.ZodObject, O extends z.ZodTypeAny>(
|
|
81
|
+
op: Operation<I, O>,
|
|
82
|
+
inputObj: Record<string, unknown>,
|
|
83
|
+
): z.infer<I> {
|
|
84
|
+
const result = op.inputSchema.safeParse(inputObj);
|
|
85
|
+
if (!result.success) {
|
|
86
|
+
throw new HelpfulError({
|
|
87
|
+
kind: "input_error",
|
|
88
|
+
message: `invalid arguments to ${op.name}: ${result.error.message}`,
|
|
89
|
+
hint: `Run \`membot ${defaultCliName(op)} --help\` to see expected arguments.`,
|
|
90
|
+
details: result.error.issues,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
return result.data;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Validate the handler's return value against the operation's output schema. */
|
|
97
|
+
function parseOutput<I extends z.ZodObject, O extends z.ZodTypeAny>(op: Operation<I, O>, result: unknown): z.infer<O> {
|
|
98
|
+
const validated = op.outputSchema.safeParse(result);
|
|
99
|
+
if (!validated.success) {
|
|
100
|
+
throw new HelpfulError({
|
|
101
|
+
kind: "internal_error",
|
|
102
|
+
message: `${op.name} produced an output that doesn't match its declared schema: ${validated.error.message}`,
|
|
103
|
+
hint: "This is a membot bug. Re-run with --verbose and report at https://github.com/evantahler/membot/issues.",
|
|
104
|
+
details: validated.error.issues,
|
|
105
|
+
cause: validated.error,
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
return validated.data;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Render an error caught at the mount boundary. Wraps unknown errors via
|
|
113
|
+
* `asHelpful()` so the output shape (kind/message/hint) is uniform regardless
|
|
114
|
+
* of where the throw came from.
|
|
115
|
+
*/
|
|
116
|
+
export function renderCliError(err: unknown): void {
|
|
117
|
+
const helpful = isHelpfulError(err)
|
|
118
|
+
? err
|
|
119
|
+
: asHelpful(
|
|
120
|
+
err,
|
|
121
|
+
"unexpected error",
|
|
122
|
+
"Re-run with --verbose for the underlying message; if it persists this is a bug.",
|
|
123
|
+
"internal_error",
|
|
124
|
+
);
|
|
125
|
+
|
|
126
|
+
if (isJson()) {
|
|
127
|
+
const payload = {
|
|
128
|
+
ok: false,
|
|
129
|
+
error: {
|
|
130
|
+
kind: helpful.kind,
|
|
131
|
+
message: helpful.message,
|
|
132
|
+
hint: helpful.hint,
|
|
133
|
+
details: helpful.details,
|
|
134
|
+
},
|
|
135
|
+
};
|
|
136
|
+
process.stderr.write(`${JSON.stringify(payload)}\n`);
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
logger.error(`✗ ${helpful.message}`);
|
|
141
|
+
logger.writeRaw(` ${colors.yellow("hint:")} ${helpful.hint}\n`);
|
|
142
|
+
if (helpful.details !== undefined) {
|
|
143
|
+
logger.writeRaw(` ${colors.dim(`details: ${formatDetails(helpful.details)}`)}\n`);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function formatDetails(details: unknown): string {
|
|
148
|
+
try {
|
|
149
|
+
return JSON.stringify(details);
|
|
150
|
+
} catch {
|
|
151
|
+
return String(details);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** kebab-case-or-snake_case → camelCase (commander gives us camelCase keys on opts). */
|
|
156
|
+
function kebabToCamel(s: string): string {
|
|
157
|
+
return s.replace(/-([a-z])/g, (_, c) => c.toUpperCase());
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Drain stdin into a single string. Used by operations whose `cli.stdinField` is unset. */
|
|
161
|
+
async function readStdin(): Promise<string> {
|
|
162
|
+
const chunks: Uint8Array[] = [];
|
|
163
|
+
for await (const chunk of process.stdin as AsyncIterable<Uint8Array>) {
|
|
164
|
+
chunks.push(chunk);
|
|
165
|
+
}
|
|
166
|
+
const total = chunks.reduce((n, c) => n + c.byteLength, 0);
|
|
167
|
+
const merged = new Uint8Array(total);
|
|
168
|
+
let offset = 0;
|
|
169
|
+
for (const c of chunks) {
|
|
170
|
+
merged.set(c, offset);
|
|
171
|
+
offset += c.byteLength;
|
|
172
|
+
}
|
|
173
|
+
return new TextDecoder().decode(merged);
|
|
174
|
+
}
|
package/src/mount/mcp.ts
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
|
+
import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
|
|
3
|
+
import type { z } from "zod";
|
|
4
|
+
import type { AppContext } from "../context.ts";
|
|
5
|
+
import { asHelpful, HelpfulError, isHelpfulError } from "../errors.ts";
|
|
6
|
+
import { composeDescription, type Operation } from "../operations/types.ts";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Mount an Operation as an MCP tool on the supplied server. The tool:
|
|
10
|
+
* 1. registers using `op.name` and `op.description`
|
|
11
|
+
* 2. exposes the zod input schema as JSON-Schema (via the SDK helper)
|
|
12
|
+
* 3. validates input + output through the same zod schemas the CLI uses
|
|
13
|
+
* 4. catches HelpfulError and returns it as `isError: true` with the
|
|
14
|
+
* hint placed in BOTH the rendered text and `structuredContent.error`
|
|
15
|
+
*/
|
|
16
|
+
export function mountAsMcpTool<I extends z.ZodObject, O extends z.ZodTypeAny>(
|
|
17
|
+
server: McpServer,
|
|
18
|
+
op: Operation<I, O>,
|
|
19
|
+
getCtx: () => Promise<AppContext>,
|
|
20
|
+
): void {
|
|
21
|
+
server.registerTool(
|
|
22
|
+
op.name,
|
|
23
|
+
{
|
|
24
|
+
description: composeDescription(op),
|
|
25
|
+
inputSchema: op.inputSchema.shape as z.ZodRawShape,
|
|
26
|
+
},
|
|
27
|
+
async (rawInput: unknown): Promise<CallToolResult> => {
|
|
28
|
+
let parsedInput: z.infer<I>;
|
|
29
|
+
try {
|
|
30
|
+
parsedInput = parseInput(op, rawInput);
|
|
31
|
+
} catch (err) {
|
|
32
|
+
return renderMcpError(err);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
let ctx: AppContext;
|
|
36
|
+
try {
|
|
37
|
+
ctx = await getCtx();
|
|
38
|
+
} catch (err) {
|
|
39
|
+
return renderMcpError(err);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
try {
|
|
43
|
+
const result = await op.handler(parsedInput, ctx);
|
|
44
|
+
const validated = parseOutput(op, result);
|
|
45
|
+
return {
|
|
46
|
+
content: [{ type: "text", text: jsonOrText(validated) }],
|
|
47
|
+
structuredContent: validated as Record<string, unknown>,
|
|
48
|
+
};
|
|
49
|
+
} catch (err) {
|
|
50
|
+
return renderMcpError(err);
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Validate the MCP-supplied input against the operation's zod schema. */
|
|
57
|
+
function parseInput<I extends z.ZodObject, O extends z.ZodTypeAny>(op: Operation<I, O>, raw: unknown): z.infer<I> {
|
|
58
|
+
const result = op.inputSchema.safeParse(raw);
|
|
59
|
+
if (!result.success) {
|
|
60
|
+
throw new HelpfulError({
|
|
61
|
+
kind: "input_error",
|
|
62
|
+
message: `invalid input to ${op.name}: ${result.error.message}`,
|
|
63
|
+
hint: `Check the tool's inputSchema. Common issues: missing required fields, wrong types, unknown fields.`,
|
|
64
|
+
details: result.error.issues,
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
return result.data;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Validate the handler's return value against the operation's output schema. */
|
|
71
|
+
function parseOutput<I extends z.ZodObject, O extends z.ZodTypeAny>(op: Operation<I, O>, result: unknown): z.infer<O> {
|
|
72
|
+
const validated = op.outputSchema.safeParse(result);
|
|
73
|
+
if (!validated.success) {
|
|
74
|
+
throw new HelpfulError({
|
|
75
|
+
kind: "internal_error",
|
|
76
|
+
message: `${op.name} produced output that doesn't match its declared schema: ${validated.error.message}`,
|
|
77
|
+
hint: "This is a membot bug. Report at https://github.com/evantahler/membot/issues.",
|
|
78
|
+
details: validated.error.issues,
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
return validated.data;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Render any thrown value as an MCP `isError: true` result. The hint lands
|
|
86
|
+
* in both the human-visible text content and the `structuredContent.error`
|
|
87
|
+
* field so an LLM consuming the tool result gets identical guidance.
|
|
88
|
+
*/
|
|
89
|
+
export function renderMcpError(err: unknown): CallToolResult {
|
|
90
|
+
const helpful = isHelpfulError(err)
|
|
91
|
+
? err
|
|
92
|
+
: asHelpful(err, "unexpected error", "This is a membot bug; check server logs.", "internal_error");
|
|
93
|
+
return {
|
|
94
|
+
isError: true,
|
|
95
|
+
content: [{ type: "text", text: `${helpful.message}\n\nhint: ${helpful.hint}` }],
|
|
96
|
+
structuredContent: {
|
|
97
|
+
error: {
|
|
98
|
+
kind: helpful.kind,
|
|
99
|
+
message: helpful.message,
|
|
100
|
+
hint: helpful.hint,
|
|
101
|
+
details: helpful.details ?? null,
|
|
102
|
+
},
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Serialize an output value to a single text block — JSON for objects, raw for strings. */
|
|
108
|
+
function jsonOrText(value: unknown): string {
|
|
109
|
+
if (typeof value === "string") return value;
|
|
110
|
+
return JSON.stringify(value, null, 2);
|
|
111
|
+
}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import { type Command, Option } from "commander";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import { HelpfulError } from "../errors.ts";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Walk a zod object schema and register its fields onto a commander command.
|
|
7
|
+
* Each field becomes either a positional `<arg>`/`[arg]` or a `--flag`,
|
|
8
|
+
* with descriptions sourced from `.describe()` so the same docstring shows
|
|
9
|
+
* up in `--help` and in the MCP tool's parameter description.
|
|
10
|
+
*/
|
|
11
|
+
export function applySchemaToCommand<S extends z.ZodObject>(
|
|
12
|
+
cmd: Command,
|
|
13
|
+
schema: S,
|
|
14
|
+
options: {
|
|
15
|
+
positional?: readonly string[];
|
|
16
|
+
aliases?: Readonly<Record<string, string>>;
|
|
17
|
+
} = {},
|
|
18
|
+
): void {
|
|
19
|
+
const positional = new Set(options.positional ?? []);
|
|
20
|
+
const aliases = options.aliases ?? {};
|
|
21
|
+
|
|
22
|
+
const shape = schema.shape;
|
|
23
|
+
const positionalOrder = options.positional ?? [];
|
|
24
|
+
|
|
25
|
+
for (const fieldName of positionalOrder) {
|
|
26
|
+
const fieldSchema = shape[fieldName];
|
|
27
|
+
if (!fieldSchema) continue;
|
|
28
|
+
const required = !isOptional(fieldSchema);
|
|
29
|
+
const label = required ? `<${fieldName}>` : `[${fieldName}]`;
|
|
30
|
+
cmd.argument(label, describeOf(fieldSchema));
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
for (const [fieldName, fieldSchemaUnknown] of Object.entries(shape)) {
|
|
34
|
+
if (positional.has(fieldName)) continue;
|
|
35
|
+
const fieldSchema = fieldSchemaUnknown as z.ZodTypeAny;
|
|
36
|
+
const flag = toKebab(fieldName);
|
|
37
|
+
const desc = describeOf(fieldSchema);
|
|
38
|
+
const alias = aliases[fieldName];
|
|
39
|
+
const opt = buildOption(fieldName, flag, desc, fieldSchema, alias);
|
|
40
|
+
cmd.addOption(opt);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Translate a single zod field into a commander Option. Booleans become
|
|
46
|
+
* boolean flags (`--flag` / `--no-flag`); enums become `.choices(...)`;
|
|
47
|
+
* arrays of strings become repeatable flags; everything else becomes a
|
|
48
|
+
* value-taking flag whose argument is parsed as the field's primitive type.
|
|
49
|
+
*/
|
|
50
|
+
function buildOption(
|
|
51
|
+
_fieldName: string,
|
|
52
|
+
flag: string,
|
|
53
|
+
desc: string,
|
|
54
|
+
schema: z.ZodTypeAny,
|
|
55
|
+
alias: string | undefined,
|
|
56
|
+
): Option {
|
|
57
|
+
const inner = unwrap(schema);
|
|
58
|
+
|
|
59
|
+
if (inner instanceof z.ZodBoolean) {
|
|
60
|
+
const longFlag = `--${flag}`;
|
|
61
|
+
const opt = new Option(`${alias ? `${alias}, ` : ""}${longFlag}`, desc);
|
|
62
|
+
const def = defaultOf(schema);
|
|
63
|
+
if (def !== undefined) opt.default(def as boolean);
|
|
64
|
+
return opt;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (inner instanceof z.ZodEnum) {
|
|
68
|
+
const opt = new Option(`${alias ? `${alias}, ` : ""}--${flag} <value>`, desc);
|
|
69
|
+
const enumValues = inner.options as readonly string[];
|
|
70
|
+
opt.choices(enumValues as string[]);
|
|
71
|
+
const def = defaultOf(schema);
|
|
72
|
+
if (def !== undefined) opt.default(def);
|
|
73
|
+
return opt;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
if (inner instanceof z.ZodArray) {
|
|
77
|
+
const opt = new Option(`${alias ? `${alias}, ` : ""}--${flag} <value>`, `${desc} (repeatable)`);
|
|
78
|
+
opt.argParser((val: string, prev: string[] | undefined) => {
|
|
79
|
+
const next = prev ?? [];
|
|
80
|
+
next.push(val);
|
|
81
|
+
return next;
|
|
82
|
+
});
|
|
83
|
+
const def = defaultOf(schema);
|
|
84
|
+
if (def !== undefined) opt.default(def);
|
|
85
|
+
return opt;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
if (inner instanceof z.ZodNumber) {
|
|
89
|
+
const opt = new Option(`${alias ? `${alias}, ` : ""}--${flag} <value>`, desc);
|
|
90
|
+
opt.argParser((v: string) => {
|
|
91
|
+
const n = Number(v);
|
|
92
|
+
if (Number.isNaN(n)) {
|
|
93
|
+
throw new HelpfulError({
|
|
94
|
+
kind: "input_error",
|
|
95
|
+
message: `invalid number for --${flag}: ${JSON.stringify(v)}`,
|
|
96
|
+
hint: `Pass a numeric value, e.g. \`--${flag} 10\`. Run \`membot <command> --help\` to see expected types.`,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
return n;
|
|
100
|
+
});
|
|
101
|
+
const def = defaultOf(schema);
|
|
102
|
+
if (def !== undefined) opt.default(def);
|
|
103
|
+
return opt;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const opt = new Option(`${alias ? `${alias}, ` : ""}--${flag} <value>`, desc);
|
|
107
|
+
const def = defaultOf(schema);
|
|
108
|
+
if (def !== undefined) opt.default(def);
|
|
109
|
+
return opt;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Pull through `.optional()` and `.default()` wrappers to find the underlying schema. */
|
|
113
|
+
function unwrap(schema: z.ZodTypeAny): z.ZodTypeAny {
|
|
114
|
+
let cur: z.ZodTypeAny = schema;
|
|
115
|
+
while (true) {
|
|
116
|
+
if (cur instanceof z.ZodOptional) cur = cur.unwrap() as z.ZodTypeAny;
|
|
117
|
+
else if (cur instanceof z.ZodDefault) cur = cur._def.innerType as z.ZodTypeAny;
|
|
118
|
+
else if (cur instanceof z.ZodNullable) cur = cur.unwrap() as z.ZodTypeAny;
|
|
119
|
+
else break;
|
|
120
|
+
}
|
|
121
|
+
return cur;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** True when the field is optional or defaulted (no value required from the user). */
|
|
125
|
+
function isOptional(schema: z.ZodTypeAny): boolean {
|
|
126
|
+
if (schema instanceof z.ZodOptional) return true;
|
|
127
|
+
if (schema instanceof z.ZodDefault) return true;
|
|
128
|
+
if (schema instanceof z.ZodNullable) return true;
|
|
129
|
+
return false;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Read the description set via `.describe()`, falling back to an empty string. */
|
|
133
|
+
function describeOf(schema: z.ZodTypeAny): string {
|
|
134
|
+
const desc = (schema._def as { description?: string }).description;
|
|
135
|
+
return desc ?? "";
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Read the static `.default()` value off a zod schema, walking through
|
|
140
|
+
* `.optional()` to find the inner default. Returns undefined when no default
|
|
141
|
+
* is set so commander treats the option as truly optional.
|
|
142
|
+
*/
|
|
143
|
+
function defaultOf(schema: z.ZodTypeAny): unknown {
|
|
144
|
+
let cur: z.ZodTypeAny = schema;
|
|
145
|
+
while (cur instanceof z.ZodOptional || cur instanceof z.ZodNullable) {
|
|
146
|
+
cur = cur.unwrap() as z.ZodTypeAny;
|
|
147
|
+
}
|
|
148
|
+
if (cur instanceof z.ZodDefault) {
|
|
149
|
+
const def = cur._def.defaultValue;
|
|
150
|
+
return typeof def === "function" ? def() : def;
|
|
151
|
+
}
|
|
152
|
+
return undefined;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** snake_case → kebab-case for CLI flag names. */
|
|
156
|
+
export function toKebab(name: string): string {
|
|
157
|
+
return name.replaceAll("_", "-");
|
|
158
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { ingest } from "../ingest/ingest.ts";
|
|
3
|
+
import { colors } from "../output/formatter.ts";
|
|
4
|
+
import { defineOperation } from "./types.ts";
|
|
5
|
+
|
|
6
|
+
const FetcherKindEnum = z.enum(["http", "mcpx", "local", "inline"]);
|
|
7
|
+
|
|
8
|
+
export const addOperation = defineOperation({
|
|
9
|
+
name: "membot_add",
|
|
10
|
+
cliName: "add",
|
|
11
|
+
description: `Ingest one or many sources into the store. \`source\` accepts:
|
|
12
|
+
- a local file path
|
|
13
|
+
- a local directory (recursive walk, symlinks followed)
|
|
14
|
+
- a glob pattern (e.g. "docs/**/*.md")
|
|
15
|
+
- a URL (fetched via mcpx if configured, otherwise plain HTTP)
|
|
16
|
+
- "inline:<text>" literal
|
|
17
|
+
PDF, DOCX, HTML, images, and other binaries are converted to markdown — native libraries first, vision/OCR for images, LLM fallback for messy or scanned input. Original bytes are kept in the blobs table; \`membot_read bytes=true\` returns them. Setting \`refresh_frequency\` enables automatic refresh from the daemon. Each ingested file becomes a NEW version under its own logical_path; existing versions stay queryable via membot_versions. Directory/glob ingests stream one file at a time — partial failures do not abort the rest; the response lists per-entry status.`,
|
|
18
|
+
inputSchema: z.object({
|
|
19
|
+
source: z.string().describe("Local path, directory, glob, URL, or `inline:<text>` literal"),
|
|
20
|
+
logical_path: z.string().optional().describe("Destination logical_path (single source) or prefix (directory/glob)"),
|
|
21
|
+
include: z.string().optional().describe("Glob include filter (comma-separated for multiple); default `**/*`"),
|
|
22
|
+
exclude: z.string().optional().describe("Glob exclude filter (comma-separated for multiple)"),
|
|
23
|
+
follow_symlinks: z
|
|
24
|
+
.boolean()
|
|
25
|
+
.default(true)
|
|
26
|
+
.describe("Follow symlinks during directory walks (cycles broken via realpath)"),
|
|
27
|
+
refresh_frequency: z.string().optional().describe("Auto-refresh cadence: 5m | 1h | 24h | 7d. Omit to disable."),
|
|
28
|
+
fetcher_hint: z
|
|
29
|
+
.string()
|
|
30
|
+
.optional()
|
|
31
|
+
.describe("Free-form hint passed to mcpx tool search (e.g. 'firecrawl', 'github', 'google docs', 'http')"),
|
|
32
|
+
change_note: z.string().optional().describe("Free-text note attached to the new version"),
|
|
33
|
+
}),
|
|
34
|
+
outputSchema: z.object({
|
|
35
|
+
ingested: z.array(
|
|
36
|
+
z.object({
|
|
37
|
+
source_path: z.string(),
|
|
38
|
+
logical_path: z.string(),
|
|
39
|
+
version_id: z.string().nullable(),
|
|
40
|
+
status: z.enum(["ok", "failed"]),
|
|
41
|
+
error: z.string().optional(),
|
|
42
|
+
mime_type: z.string().nullable(),
|
|
43
|
+
size_bytes: z.number(),
|
|
44
|
+
fetcher: FetcherKindEnum,
|
|
45
|
+
source_sha256: z.string(),
|
|
46
|
+
}),
|
|
47
|
+
),
|
|
48
|
+
total: z.number(),
|
|
49
|
+
ok: z.number(),
|
|
50
|
+
failed: z.number(),
|
|
51
|
+
}),
|
|
52
|
+
cli: {
|
|
53
|
+
positional: ["source"],
|
|
54
|
+
aliases: { logical_path: "-p", refresh_frequency: "-r", change_note: "-m" },
|
|
55
|
+
},
|
|
56
|
+
console_formatter: (result) => {
|
|
57
|
+
const lines = result.ingested.map((e) => {
|
|
58
|
+
if (e.status === "ok") {
|
|
59
|
+
return `${colors.green("✓")} ${colors.cyan(e.logical_path)} ${colors.dim(`(${e.fetcher}, ${e.size_bytes}B)`)}`;
|
|
60
|
+
}
|
|
61
|
+
return `${colors.red("✗")} ${e.source_path} ${colors.dim(e.error ?? "")}`;
|
|
62
|
+
});
|
|
63
|
+
const summary = result.failed
|
|
64
|
+
? `${colors.green(`added ${result.ok}`)}, ${colors.red(`failed ${result.failed}`)}`
|
|
65
|
+
: colors.green(`added ${result.ok}`);
|
|
66
|
+
return `${lines.join("\n")}\n${summary}`;
|
|
67
|
+
},
|
|
68
|
+
handler: async (input, ctx) => ingest(input, ctx),
|
|
69
|
+
});
|