ctxpkg 0.0.1

package/src/embedder/embedder.ts

@@ -0,0 +1,53 @@
+import { type FeatureExtractionPipeline, pipeline } from '@huggingface/transformers';
+
+// Instruction prefix for query embeddings (mxbai-embed format)
+const QUERY_INSTRUCTION = 'Represent this sentence for searching relevant passages: ';
+
+class EmbedderService {
+  #pipeline?: Promise<FeatureExtractionPipeline>;
+
+  #setup = async () => {
+    const extractor = await pipeline('feature-extraction', 'mixedbread-ai/mxbai-embed-large-v1', {
+      dtype: 'fp32',
+    });
+    return extractor;
+  };
+
+  public getExtractor = async () => {
+    if (!this.#pipeline) {
+      this.#pipeline = this.#setup();
+    }
+    return await this.#pipeline;
+  };
+
+  /**
+   * Create embeddings for documents (no instruction prefix).
+   * Use this when indexing document chunks.
+   */
+  public createDocumentEmbeddings = async (inputs: string[]): Promise<number[][]> => {
+    const extractor = await this.getExtractor();
+    const output = await extractor(inputs, { pooling: 'cls' });
+    return output.tolist();
+  };
+
+  /**
+   * Create embedding for a search query (with instruction prefix).
+   * Use this when searching for relevant documents.
+   */
+  public createQueryEmbedding = async (query: string): Promise<number[]> => {
+    const extractor = await this.getExtractor();
+    const instructedQuery = `${QUERY_INSTRUCTION}${query}`;
+    const output = await extractor([instructedQuery], { pooling: 'cls' });
+    return output.tolist()[0];
+  };
+
+  /**
+   * @deprecated Use createDocumentEmbeddings or createQueryEmbedding instead.
+   * Kept for backwards compatibility.
+   */
+  public createEmbeddings = async (inputs: string[]): Promise<number[][]> => {
+    return this.createDocumentEmbeddings(inputs);
+  };
+}
+
+export { EmbedderService };

package/src/mcp/AGENTS.md

@@ -0,0 +1,264 @@
+# MCP — Agent Guidelines
+
+This document describes the MCP module architecture for AI agents working on this codebase.
+
+## Overview
+
+The MCP module provides [Model Context Protocol](https://modelcontextprotocol.io/) server integration. It creates MCP servers that expose ctxpkg's document tools to AI editors like Cursor, Claude Desktop, and other MCP-compatible clients. The server communicates over stdio transport.
+
+## File Structure
+
+| File | Purpose |
+|------|---------|
+| `mcp.ts` | MCP server creation and stdio runner |
+
+## Architecture
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                     AI Editor / Client                      │
+│              (Cursor, Claude Desktop, etc.)                 │
+└──────────────────────────┬──────────────────────────────────┘
+                           │ stdio (JSON-RPC)
+                           ▼
+┌─────────────────────────────────────────────────────────────┐
+│                      MCP Server                             │
+│  ┌───────────────────────────────────────────────────────┐  │
+│  │              StdioServerTransport                     │  │
+│  └───────────────────────────────────────────────────────┘  │
+│                           │                                 │
+│  ┌───────────────────────▼───────────────────────────────┐  │
+│  │                   McpServer                           │  │
+│  │         (from @modelcontextprotocol/sdk)              │  │
+│  └───────────────────────────────────────────────────────┘  │
+│                           │                                 │
+│  ┌───────────────────────▼───────────────────────────────┐  │
+│  │              Document Tools                           │  │
+│  │  • documents_list_collections                         │  │
+│  │  • documents_search                                   │  │
+│  │  • documents_get_document                             │  │
+│  │  • documents_list_documents                           │  │
+│  │  • documents_get_outline                              │  │
+│  │  • documents_get_section                              │  │
+│  │  • documents_search_batch                             │  │
+│  │  • documents_find_related                             │  │
+│  └───────────────────────────────────────────────────────┘  │
+│                           │                                 │
+│  ┌───────────────────────▼───────────────────────────────┐  │
+│  │                 BackendClient                         │  │
+│  │           (connects to daemon/direct)                 │  │
+│  └───────────────────────────────────────────────────────┘  │
+└─────────────────────────────────────────────────────────────┘
+```
+
+## Usage
+
+### Starting the MCP Server
+
+Via CLI:
+
+```bash
+# Start with all collections
+ctxpkg mcp documents
+
+# Limit to specific collections
+ctxpkg mcp documents -c my-docs langchain-docs
+
+# Custom server name/version
+ctxpkg mcp documents --name my-server --version 2.0.0
+```
+
+### Programmatic Usage
+
+```typescript
+import { createDocumentsMcpServer, runMcpServer } from '#root/mcp/mcp.ts';
+import { createClient } from '#root/client/client.ts';
+
+const client = await createClient({ mode: 'daemon' });
+
+const server = createDocumentsMcpServer({
+  client,
+  name: 'my-mcp-server',
+  version: '1.0.0',
+  aliasMap: new Map([['docs', 'pkg:file://./docs/manifest.json']]),
+});
+
+await runMcpServer(server);
+```
+
+## Editor Configuration
+
+### Cursor
+
+Add to `.cursor/mcp.json`:
+
+```json
+{
+  "mcpServers": {
+    "ctxpkg": {
+      "command": "ctxpkg",
+      "args": ["mcp", "documents"]
+    }
+  }
+}
+```
+
+### Claude Desktop
+
+Add to Claude Desktop config:
+
+```json
+{
+  "mcpServers": {
+    "ctxpkg": {
+      "command": "ctxpkg",
+      "args": ["mcp", "documents"]
+    }
+  }
+}
+```
+
+## Server Modes
+
+### Documents Mode (default)
+
+Exposes all document tools individually. The calling agent decides which tools to use.
+
+```bash
+ctxpkg mcp documents
+```
+
+### Agent Mode
+
+Exposes a single `ask_documents` tool. An internal LangChain agent handles searching and synthesizes a single answer. This reduces token/context costs for the calling agent.
+
+```bash
+ctxpkg mcp agent
+```
+
+Requires LLM configuration:
+```bash
+ctxpkg config set llm.apiKey sk-...
+ctxpkg config set llm.model gpt-4o
+```
+
+## Exposed Tools
+
+### Documents Mode
+
+The MCP server exposes these tools to AI agents:
+
+| Tool | Description |
+|------|-------------|
+| `documents_list_collections` | List available document collections with descriptions and versions |
+| `documents_search` | Semantic search across documents using hybrid vector + keyword matching |
+| `documents_get_document` | Get full document content |
+| `documents_list_documents` | List all documents in a collection (table of contents) |
+| `documents_get_outline` | Get document heading structure without fetching full content |
+| `documents_get_section` | Get a specific section of a document by heading |
+| `documents_search_batch` | Execute multiple search queries in a single call (max 10) |
+| `documents_find_related` | Find content semantically related to a document or chunk |
+
+See `src/tools/documents/` for tool implementation details.
+
+### Agent Mode
+
+| Tool | Description |
+|------|-------------|
+| `ask_documents` | Ask a question with a use case; internal agent searches and synthesizes answer |
+
+The `ask_documents` tool requires both a query and a use case to help the agent determine when sufficient information has been found.
+
+See `src/tools/agent/` and `src/agent/` for implementation details.
+
+## Key Components
+
+### `createDocumentsMcpServer(options)`
+
+Creates an MCP server instance with document tools:
+
+```typescript
+type DocumentsMcpServerOptions = {
+  client: BackendClient;      // Required: backend connection
+  name?: string;              // Server name (default: 'ctxpkg-documents')
+  version?: string;           // Server version (default: '1.0.0')
+  collections?: string[];     // Limit to specific collections
+  aliasMap?: Map<string, string>;  // Project alias → collection ID
+};
+```
+
+### `createAgentMcpServer(options)`
+
+Creates an MCP server with agent mode (single `ask_documents` tool):
+
+```typescript
+type AgentMcpServerOptions = {
+  client: BackendClient;      // Required: backend connection
+  llmConfig: LLMConfig;       // Required: LLM configuration
+  name?: string;              // Server name (default: 'ctxpkg-agent')
+  version?: string;           // Server version (default: '1.0.0')
+  aliasMap?: Map<string, string>;  // Project alias → collection ID
+  maxIterations?: number;     // Max agent iterations (default: 15)
+};
+```
+
+### `runMcpServer(server)`
+
+Connects the server to stdio transport and handles shutdown:
+
+- Creates `StdioServerTransport`
+- Connects server to transport
+- Registers SIGINT/SIGTERM handlers for graceful shutdown
+
+## Adding New MCP Servers
+
+To create an MCP server with different tools:
+
+```typescript
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { registerMcpTools } from '#root/tools/tools.mcp.ts';
+
+const createMyMcpServer = (options: MyOptions) => {
+  const server = new McpServer({
+    name: options.name ?? 'my-server',
+    version: options.version ?? '1.0.0',
+  });
+
+  // Create tool definitions
+  const tools = createMyToolDefinitions(options);
+  
+  // Register on MCP server
+  registerMcpTools(server, tools);
+
+  return server;
+};
+```
+
+Then add a CLI command in `cli.mcp.ts` to start it.
+
+## Key Patterns
+
+### Alias Resolution
+
+Project aliases (from `context.json`) are resolved to collection IDs:
+
+```typescript
+const aliasMap = new Map<string, string>();
+for (const [alias, spec] of Object.entries(projectConfig.collections)) {
+  const collectionId = collectionsService.computeCollectionId(spec, cwd);
+  aliasMap.set(alias, collectionId);
+}
+```
+
+This allows users to search by friendly names like `"langchain"` instead of `"pkg:https://..."`.
+
+### Graceful Shutdown
+
+The server handles shutdown signals to close cleanly:
+
+```typescript
+process.on('SIGINT', async () => {
+  await server.close();
+  process.exit(0);
+});
+```

package/src/mcp/mcp.ts

@@ -0,0 +1,105 @@
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+
+import { createDocumentAgent, type CreateDocumentAgentOptions } from '#root/agent/agent.ts';
+import type { LLMConfig } from '#root/agent/agent.types.ts';
+import type { BackendClient } from '#root/client/client.ts';
+import { createAgentToolDefinitions } from '#root/tools/agent/agent.ts';
+import { createDocumentToolDefinitions } from '#root/tools/documents/documents.ts';
+import { registerMcpTools } from '#root/tools/tools.mcp.ts';
+
+type McpServerOptions = {
+  /** Name of the MCP server */
+  name?: string;
+  /** Version of the MCP server */
+  version?: string;
+};
+
+type DocumentsMcpServerOptions = McpServerOptions & {
+  /** Backend client for accessing the documents service */
+  client: BackendClient;
+  /** Collections to limit searches to (optional, uses cwd and default collections if not specified) */
+  collections?: string[];
+  /** Optional map of project aliases to collection IDs */
+  aliasMap?: Map<string, string>;
+};
+
+type AgentMcpServerOptions = McpServerOptions & {
+  /** Backend client for accessing the documents service */
+  client: BackendClient;
+  /** LLM configuration for the agent */
+  llmConfig: LLMConfig;
+  /** Optional map of project aliases to collection IDs */
+  aliasMap?: Map<string, string>;
+  /** Maximum agent iterations */
+  maxIterations?: number;
+};
+
+/**
+ * Create an MCP server with document tools.
+ */
+const createDocumentsMcpServer = (options: DocumentsMcpServerOptions) => {
+  const { client, aliasMap, name = 'ctxpkg-documents', version = '1.0.0' } = options;
+
+  const server = new McpServer({
+    name,
+    version,
+  });
+
+  // Create and register document tools
+  const documentTools = createDocumentToolDefinitions({ client, aliasMap });
+  registerMcpTools(server, documentTools);
+
+  return server;
+};
+
+/**
+ * Create an MCP server with agent mode (single ask_documents tool).
+ * The internal agent uses LLM to search and synthesize answers.
+ */
+const createAgentMcpServer = (options: AgentMcpServerOptions) => {
+  const { client, aliasMap, llmConfig, maxIterations, name = 'ctxpkg-agent', version = '1.0.0' } = options;
+
+  const server = new McpServer({
+    name,
+    version,
+  });
+
+  // Create document agent
+  const agentOptions: CreateDocumentAgentOptions = {
+    client,
+    llmConfig,
+    aliasMap,
+    maxIterations,
+  };
+  const agent = createDocumentAgent(agentOptions);
+
+  // Create and register agent tools (just ask_documents)
+  const agentTools = createAgentToolDefinitions({ agent });
+  registerMcpTools(server, agentTools);
+
+  return server;
+};
+
+/**
+ * Run an MCP server over stdio transport.
+ * This is the main entry point for running as a standalone MCP server.
+ */
+const runMcpServer = async (server: McpServer) => {
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+
+  // Handle graceful shutdown
+  process.on('SIGINT', async () => {
+    await server.close();
+    process.exit(0);
+  });
+
+  process.on('SIGTERM', async () => {
+    await server.close();
+    process.exit(0);
+  });
+};
+
+export { createAgentMcpServer, createDocumentsMcpServer, runMcpServer };
+export type { AgentMcpServerOptions, DocumentsMcpServerOptions, McpServerOptions };

package/src/tools/AGENTS.md

@@ -0,0 +1,228 @@
+# Tools — Agent Guidelines
+
+This document describes the tools module architecture for AI agents working on this codebase.
+
+## Overview
+
+The tools module provides AI agent tools in a framework-agnostic format. Tools are defined once using a common format and can be adapted to different runtimes (MCP, LangChain). This allows the same tool logic to work across different AI frameworks.
+
+## File Structure
+
+```
+src/tools/
+├── tools.types.ts       # Common tool definition types
+├── tools.mcp.ts         # MCP server adapter
+├── tools.langchain.ts   # LangChain adapter
+├── documents/
+│   └── documents.ts     # Document tools
+├── files/
+│   └── files.ts         # File system tools (legacy)
+└── git/
+    └── git.ts           # Git tools (legacy)
+```
+
+## Architecture
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                    Tool Definitions                         │
+│              (framework-agnostic format)                    │
+│                                                             │
+│   defineTool({                                              │
+│     name: 'tool_name',                                      │
+│     description: '...',                                     │
+│     schema: z.object({...}),                                │
+│     handler: async (input) => {...}                         │
+│   })                                                        │
+└─────────────────────────────────────────────────────────────┘
+                          │
+          ┌───────────────┴───────────────┐
+          ▼                               ▼
+┌─────────────────────┐       ┌─────────────────────┐
+│    tools.mcp.ts     │       │ tools.langchain.ts  │
+│  registerMcpTools() │       │  toLangchainTools() │
+└─────────────────────┘       └─────────────────────┘
+          │                               │
+          ▼                               ▼
+┌─────────────────────┐       ┌─────────────────────┐
+│     MCP Server      │       │   LangChain Agent   │
+└─────────────────────┘       └─────────────────────┘
+```
+
+## Tool Definition Format
+
+Tools use a common format with Zod schemas:
+
+```typescript
+import { defineTool } from '#root/tools/tools.types.ts';
+import * as z from 'zod';
+
+const myTool = defineTool({
+  name: 'my_tool_name',
+  description: 'What the tool does and when to use it',
+  schema: z.object({
+    query: z.string().describe('Parameter description'),
+    limit: z.number().optional().default(10),
+  }),
+  handler: async ({ query, limit }) => {
+    // Tool logic here
+    return { result: 'data' };
+  },
+});
+```
+
+### Key Fields
+
+| Field | Purpose |
+|-------|---------|
+| `name` | Unique identifier (use snake_case with category prefix) |
+| `description` | Help AI understand when/how to use the tool |
+| `schema` | Zod schema for input validation with `.describe()` on fields |
+| `handler` | Async function that executes the tool logic |
+
+## Using Tools
+
+### With MCP Server
+
+```typescript
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { registerMcpTools } from '#root/tools/tools.mcp.ts';
+import { createDocumentToolDefinitions } from '#root/tools/documents/documents.ts';
+
+const server = new McpServer({ name: 'my-server', version: '1.0.0' });
+const tools = createDocumentToolDefinitions({ client });
+
+registerMcpTools(server, tools);
+```
+
+### With LangChain
+
+```typescript
+import { toLangchainTools } from '#root/tools/tools.langchain.ts';
+import { createDocumentToolDefinitions } from '#root/tools/documents/documents.ts';
+
+const definitions = createDocumentToolDefinitions({ client });
+const langchainTools = toLangchainTools(definitions);
+
+// Use with LangChain agent
+const agent = createToolCallingAgent({ tools: Object.values(langchainTools), ... });
+```
+
+## Adding New Tools
+
+### 1. Create Tool Definitions
+
+Create a new file in appropriate category folder:
+
+```typescript
+// src/tools/myfeature/myfeature.ts
+import * as z from 'zod';
+import { defineTool, type ToolDefinitions } from '#root/tools/tools.types.ts';
+
+type MyFeatureToolOptions = {
+  client: BackendClient;
+};
+
+const createMyFeatureToolDefinitions = (options: MyFeatureToolOptions): ToolDefinitions => {
+  const { client } = options;
+
+  const doSomething = defineTool({
+    name: 'myfeature_do_something',
+    description: 'Does something useful. Use this when...',
+    schema: z.object({
+      input: z.string().describe('The input to process'),
+    }),
+    handler: async ({ input }) => {
+      const result = await client.myFeature.process({ input });
+      return result;
+    },
+  });
+
+  return { doSomething };
+};
+
+export { createMyFeatureToolDefinitions };
+```
+
+### 2. Register on MCP Server
+
+In `src/mcp/mcp.ts` or relevant MCP setup:
+
+```typescript
+import { createMyFeatureToolDefinitions } from '#root/tools/myfeature/myfeature.ts';
+
+const tools = createMyFeatureToolDefinitions({ client });
+registerMcpTools(server, tools);
+```
+
+## Tool Categories
+
+### Document Tools (`documents/`)
+
+Tools for searching and retrieving indexed documentation:
+
+- `documents_list_collections` — List available collections with descriptions and versions
+- `documents_search` — Semantic search across documents with hybrid vector + keyword matching
+- `documents_get_document` — Get full document content
+- `documents_list_documents` — List all documents in a collection (table of contents)
+- `documents_get_outline` — Get document heading structure without fetching full content
+- `documents_get_section` — Get a specific section of a document by heading
+- `documents_search_batch` — Execute multiple search queries in a single call
+- `documents_find_related` — Find content semantically related to a document or chunk
+
+### File Tools (`files/`) — Legacy
+
+Direct file system access tools (LangChain format):
+
+- `file_get_content` — Read file content
+- `file_glob_files` — Find files by glob pattern
+- `file_search_multiline` — Search file contents with regex
+- `file_get_stats` — Get file metadata
+
+### Git Tools (`git/`) — Legacy
+
+Git repository tools (LangChain format):
+
+- `git_status` — Repository status
+- `git_get_diff` — File diffs
+- `git_get_log` — Commit history
+
+## Best Practices
+
+### Naming
+
+- Use category prefix: `documents_`, `files_`, `git_`
+- Use snake_case: `documents_list_collections`
+- Be descriptive: `search` → `documents_search`
+
+### Descriptions
+
+Write descriptions that help AI agents understand:
+- What the tool does
+- When to use it
+- What input it expects
+- What output it returns
+
+```typescript
+description: 
+  'Search reference documents using semantic similarity. ' +
+  'Returns the most relevant document chunks for the given query. ' +
+  'Use this to find information in documentation, guides, or other indexed reference materials.',
+```
+
+### Schema Descriptions
+
+Add `.describe()` to all schema fields:
+
+```typescript
+schema: z.object({
+  query: z.string().describe('The search query - describe what information you are looking for'),
+  limit: z.number().optional().default(10).describe('Maximum number of results to return'),
+}),
+```
+
+### Return Values
+
+- Return structured data (objects/arrays) — adapters handle JSON serialization
+- Return helpful error messages as strings
+- Include relevant context in results

package/src/tools/agent/agent.ts

@@ -0,0 +1,45 @@
+import * as z from 'zod';
+
+import type { DocumentAgent } from '#root/agent/agent.ts';
+import { defineTool, type ToolDefinitions } from '#root/tools/tools.types.ts';
+
+type AgentToolOptions = {
+  /** Document agent instance */
+  agent: DocumentAgent;
+};
+
+/**
+ * Creates the ask_documents tool definition for agent mode MCP.
+ * This exposes a single tool that uses an internal agent to search and synthesize.
+ */
+const createAgentToolDefinitions = (options: AgentToolOptions): ToolDefinitions => {
+  const { agent } = options;
+
+  const askDocuments = defineTool({
+    name: 'ask_documents',
+    description:
+      'Ask a question about the indexed documentation. An internal agent will search, ' +
+      'read relevant sections, and synthesize a comprehensive answer. Returns only the ' +
+      'final answer, not intermediate search results. Requires both a query and a use case ' +
+      'to help determine when sufficient information has been found.',
+    schema: z.object({
+      query: z.string().describe('The question to answer. Be specific about what information you need.'),
+      use_case: z
+        .string()
+        .describe(
+          'Why you need this information and how it will be used. ' +
+            'This helps determine when enough information has been found. ' +
+            'Example: "I need to understand authentication flow to implement login in my app"',
+        ),
+    }),
+    handler: async ({ query, use_case }) => {
+      const response = await agent.ask(query, use_case);
+      return response;
+    },
+  });
+
+  return { askDocuments };
+};
+
+export { createAgentToolDefinitions };
+export type { AgentToolOptions };