cursor-recursive-rag 0.2.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/README.md +354 -0
  2. package/bin/cursor-rag.js +18 -0
  3. package/dist/adapters/embeddings/index.d.ts +5 -0
  4. package/dist/adapters/embeddings/index.d.ts.map +1 -0
  5. package/dist/adapters/embeddings/index.js +16 -0
  6. package/dist/adapters/embeddings/index.js.map +1 -0
  7. package/dist/adapters/embeddings/ollama.d.ts +11 -0
  8. package/dist/adapters/embeddings/ollama.d.ts.map +1 -0
  9. package/dist/adapters/embeddings/ollama.js +30 -0
  10. package/dist/adapters/embeddings/ollama.js.map +1 -0
  11. package/dist/adapters/embeddings/openai.d.ts +11 -0
  12. package/dist/adapters/embeddings/openai.d.ts.map +1 -0
  13. package/dist/adapters/embeddings/openai.js +28 -0
  14. package/dist/adapters/embeddings/openai.js.map +1 -0
  15. package/dist/adapters/embeddings/types.d.ts +6 -0
  16. package/dist/adapters/embeddings/types.d.ts.map +1 -0
  17. package/dist/adapters/embeddings/types.js +2 -0
  18. package/dist/adapters/embeddings/types.js.map +1 -0
  19. package/dist/adapters/embeddings/xenova.d.ts +11 -0
  20. package/dist/adapters/embeddings/xenova.d.ts.map +1 -0
  21. package/dist/adapters/embeddings/xenova.js +26 -0
  22. package/dist/adapters/embeddings/xenova.js.map +1 -0
  23. package/dist/adapters/vector/chroma.d.ts +23 -0
  24. package/dist/adapters/vector/chroma.d.ts.map +1 -0
  25. package/dist/adapters/vector/chroma.js +91 -0
  26. package/dist/adapters/vector/chroma.js.map +1 -0
  27. package/dist/adapters/vector/index.d.ts +5 -0
  28. package/dist/adapters/vector/index.d.ts.map +1 -0
  29. package/dist/adapters/vector/index.js +22 -0
  30. package/dist/adapters/vector/index.js.map +1 -0
  31. package/dist/adapters/vector/memory.d.ts +22 -0
  32. package/dist/adapters/vector/memory.d.ts.map +1 -0
  33. package/dist/adapters/vector/memory.js +105 -0
  34. package/dist/adapters/vector/memory.js.map +1 -0
  35. package/dist/adapters/vector/qdrant.d.ts +14 -0
  36. package/dist/adapters/vector/qdrant.d.ts.map +1 -0
  37. package/dist/adapters/vector/qdrant.js +120 -0
  38. package/dist/adapters/vector/qdrant.js.map +1 -0
  39. package/dist/adapters/vector/redis.d.ts +31 -0
  40. package/dist/adapters/vector/redis.d.ts.map +1 -0
  41. package/dist/adapters/vector/redis.js +164 -0
  42. package/dist/adapters/vector/redis.js.map +1 -0
  43. package/dist/adapters/vector/vectorize.d.ts +10 -0
  44. package/dist/adapters/vector/vectorize.d.ts.map +1 -0
  45. package/dist/adapters/vector/vectorize.js +22 -0
  46. package/dist/adapters/vector/vectorize.js.map +1 -0
  47. package/dist/cli/commands/dashboard.d.ts +3 -0
  48. package/dist/cli/commands/dashboard.d.ts.map +1 -0
  49. package/dist/cli/commands/dashboard.js +23 -0
  50. package/dist/cli/commands/dashboard.js.map +1 -0
  51. package/dist/cli/commands/ingest.d.ts +3 -0
  52. package/dist/cli/commands/ingest.d.ts.map +1 -0
  53. package/dist/cli/commands/ingest.js +133 -0
  54. package/dist/cli/commands/ingest.js.map +1 -0
  55. package/dist/cli/commands/search.d.ts +3 -0
  56. package/dist/cli/commands/search.d.ts.map +1 -0
  57. package/dist/cli/commands/search.js +39 -0
  58. package/dist/cli/commands/search.js.map +1 -0
  59. package/dist/cli/commands/setup.d.ts +3 -0
  60. package/dist/cli/commands/setup.d.ts.map +1 -0
  61. package/dist/cli/commands/setup.js +311 -0
  62. package/dist/cli/commands/setup.js.map +1 -0
  63. package/dist/cli/commands/status.d.ts +3 -0
  64. package/dist/cli/commands/status.d.ts.map +1 -0
  65. package/dist/cli/commands/status.js +39 -0
  66. package/dist/cli/commands/status.js.map +1 -0
  67. package/dist/cli/index.d.ts +3 -0
  68. package/dist/cli/index.d.ts.map +1 -0
  69. package/dist/cli/index.js +34 -0
  70. package/dist/cli/index.js.map +1 -0
  71. package/dist/dashboard/public/index.html +819 -0
  72. package/dist/dashboard/server.d.ts +10 -0
  73. package/dist/dashboard/server.d.ts.map +1 -0
  74. package/dist/dashboard/server.js +397 -0
  75. package/dist/dashboard/server.js.map +1 -0
  76. package/dist/integrations/index.d.ts +3 -0
  77. package/dist/integrations/index.d.ts.map +1 -0
  78. package/dist/integrations/index.js +3 -0
  79. package/dist/integrations/index.js.map +1 -0
  80. package/dist/integrations/mcp-gateway.d.ts +91 -0
  81. package/dist/integrations/mcp-gateway.d.ts.map +1 -0
  82. package/dist/integrations/mcp-gateway.js +128 -0
  83. package/dist/integrations/mcp-gateway.js.map +1 -0
  84. package/dist/integrations/openskills.d.ts +67 -0
  85. package/dist/integrations/openskills.d.ts.map +1 -0
  86. package/dist/integrations/openskills.js +223 -0
  87. package/dist/integrations/openskills.js.map +1 -0
  88. package/dist/proxy/index.d.ts +29 -0
  89. package/dist/proxy/index.d.ts.map +1 -0
  90. package/dist/proxy/index.js +124 -0
  91. package/dist/proxy/index.js.map +1 -0
  92. package/dist/server/index.d.ts +2 -0
  93. package/dist/server/index.d.ts.map +1 -0
  94. package/dist/server/index.js +43 -0
  95. package/dist/server/index.js.map +1 -0
  96. package/dist/server/tools/crawl.d.ts +15 -0
  97. package/dist/server/tools/crawl.d.ts.map +1 -0
  98. package/dist/server/tools/crawl.js +97 -0
  99. package/dist/server/tools/crawl.js.map +1 -0
  100. package/dist/server/tools/gateway.d.ts +27 -0
  101. package/dist/server/tools/gateway.d.ts.map +1 -0
  102. package/dist/server/tools/gateway.js +58 -0
  103. package/dist/server/tools/gateway.js.map +1 -0
  104. package/dist/server/tools/index.d.ts +10 -0
  105. package/dist/server/tools/index.d.ts.map +1 -0
  106. package/dist/server/tools/index.js +302 -0
  107. package/dist/server/tools/index.js.map +1 -0
  108. package/dist/server/tools/ingest.d.ts +15 -0
  109. package/dist/server/tools/ingest.d.ts.map +1 -0
  110. package/dist/server/tools/ingest.js +84 -0
  111. package/dist/server/tools/ingest.js.map +1 -0
  112. package/dist/server/tools/list-sources.d.ts +9 -0
  113. package/dist/server/tools/list-sources.d.ts.map +1 -0
  114. package/dist/server/tools/list-sources.js +63 -0
  115. package/dist/server/tools/list-sources.js.map +1 -0
  116. package/dist/server/tools/recursive-query.d.ts +16 -0
  117. package/dist/server/tools/recursive-query.d.ts.map +1 -0
  118. package/dist/server/tools/recursive-query.js +65 -0
  119. package/dist/server/tools/recursive-query.js.map +1 -0
  120. package/dist/server/tools/search.d.ts +15 -0
  121. package/dist/server/tools/search.d.ts.map +1 -0
  122. package/dist/server/tools/search.js +20 -0
  123. package/dist/server/tools/search.js.map +1 -0
  124. package/dist/server/tools/skills.d.ts +30 -0
  125. package/dist/server/tools/skills.d.ts.map +1 -0
  126. package/dist/server/tools/skills.js +85 -0
  127. package/dist/server/tools/skills.js.map +1 -0
  128. package/dist/services/chunker.d.ts +3 -0
  129. package/dist/services/chunker.d.ts.map +1 -0
  130. package/dist/services/chunker.js +75 -0
  131. package/dist/services/chunker.js.map +1 -0
  132. package/dist/services/config.d.ts +8 -0
  133. package/dist/services/config.d.ts.map +1 -0
  134. package/dist/services/config.js +66 -0
  135. package/dist/services/config.js.map +1 -0
  136. package/dist/services/query-decomposer.d.ts +44 -0
  137. package/dist/services/query-decomposer.d.ts.map +1 -0
  138. package/dist/services/query-decomposer.js +79 -0
  139. package/dist/services/query-decomposer.js.map +1 -0
  140. package/dist/types/index.d.ts +93 -0
  141. package/dist/types/index.d.ts.map +1 -0
  142. package/dist/types/index.js +2 -0
  143. package/dist/types/index.js.map +1 -0
  144. package/package.json +56 -0
package/README.md ADDED
@@ -0,0 +1,354 @@
1
+ # cursor-recursive-rag
2
+
3
+ [![Version](https://img.shields.io/badge/version-0.2.0--alpha.1-blue.svg)](https://github.com/garethdaine/cursor-recursive-rag/releases)
4
+ [![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
5
+ [![Node](https://img.shields.io/badge/node-%3E%3D20.0.0-brightgreen.svg)](https://nodejs.org)
6
+
7
+ Recursive RAG MCP server for Cursor IDE with interactive setup wizard and web dashboard. Build a knowledge base from your documentation and codebase, enabling multi-hop retrieval and iterative query refinement.
8
+
9
+ > **Status:** Alpha - Core features working, API may change
10
+
11
+ ## Features
12
+
13
+ - **Recursive Query**: Multi-hop retrieval with query decomposition and iterative refinement
14
+ - **Configurable Vector Stores**: Redis Stack (recommended), Qdrant (local/cloud), ChromaDB, Cloudflare Vectorize
15
+ - **Configurable Embeddings**: Local (Xenova/transformers.js), OpenAI, Ollama
16
+ - **Web Crawling**: Firecrawl integration for documentation ingestion
17
+ - **Rotating Proxy Support**: Optional PacketStream/SmartProxy integration for URL fetching
18
+ - **Web Dashboard**: Real-time monitoring, search, and configuration UI (Tailwind CSS)
19
+ - **Interactive Setup**: Guided configuration wizard
20
+ - **MCP Integration**: Automatic registration with Cursor IDE
21
+ - **MCP Gateway Integration**: Connect to [MCP Gateway](https://github.com/abdullah1854/MCPGateway) for 87+ aggregated tools with token optimization
22
+ - **OpenSkills Integration**: Auto-discover and ingest skills from [OpenSkills](https://github.com/numman-ali/openskills) for semantic search
23
+
24
+ ## Quick Start
25
+
26
+ ```bash
27
+ # Clone and install
28
+ git clone https://github.com/garethdaine/cursor-recursive-rag.git
29
+ cd cursor-recursive-rag
30
+ npm install && npm run build && npm link
31
+
32
+ # Run interactive setup
33
+ cursor-rag setup
34
+
35
+ # Ingest documentation (with Firecrawl)
36
+ cursor-rag ingest https://nextjs.org/docs --crawl --max-pages 200
37
+
38
+ # Ingest a local directory
39
+ cursor-rag ingest ./docs
40
+
41
+ # Check status
42
+ cursor-rag status
43
+
44
+ # Test search from CLI
45
+ cursor-rag search "how to authenticate users"
46
+
47
+ # Start the web dashboard
48
+ cursor-rag dashboard
49
+ ```
50
+
51
+ ## Installation
52
+
53
+ ### From GitHub (recommended)
54
+
55
+ ```bash
56
+ git clone https://github.com/garethdaine/cursor-recursive-rag.git
57
+ cd cursor-recursive-rag
58
+ npm install
59
+ npm run build
60
+ npm link # Makes cursor-rag available globally
61
+ ```
62
+
63
+ ### From npm (coming soon)
64
+
65
+ ```bash
66
+ # Not yet published
67
+ npm install -g cursor-recursive-rag
68
+ ```
69
+
70
+ ## Setup Wizard
71
+
72
+ Run `cursor-rag setup` to configure the system. The wizard will:
73
+
74
+ 1. **Select Vector Store** (Redis recommended):
75
+ - **Redis Stack** (default): Persistent, fast HNSW search, `docker run -p 6379:6379 redis/redis-stack-server`
76
+ - **Qdrant**: Persistent, local Docker or cloud, `docker run -p 6333:6333 qdrant/qdrant`
77
+ - **Memory**: In-process, non-persistent (testing only)
78
+ - **ChromaDB**: Requires separate server, `docker run -p 8000:8000 chromadb/chroma`
79
+ - **Cloudflare Vectorize**: Serverless (not yet implemented)
80
+
81
+ 2. **Select Embedding Model**:
82
+ - **Xenova (local)**: Free, private, ~384 dimensions
83
+ - **Ollama**: Local, configurable models
84
+ - **OpenAI**: High quality, requires API key
85
+
86
+ 3. **API Keys**:
87
+ - OpenAI API key (if using OpenAI embeddings)
88
+ - Firecrawl API key (for web crawling)
89
+ - Qdrant URL/API key (if using Qdrant)
90
+ - Ollama URL/model (if using Ollama)
91
+
92
+ 4. **Auto-register** the MCP server with Cursor IDE
93
+
94
+ ## CLI Commands
95
+
96
+ ### `cursor-rag setup`
97
+
98
+ Interactive setup wizard to configure vector store, embeddings, proxy, and API keys.
99
+
100
+ ```bash
101
+ cursor-rag setup
102
+ cursor-rag setup --vector-store chroma --embeddings xenova
103
+ ```
104
+
105
+ ### `cursor-rag ingest`
106
+
107
+ Add documents to the knowledge base.
108
+
109
+ ```bash
110
+ # Crawl a website (requires Firecrawl API key)
111
+ cursor-rag ingest https://docs.example.com --crawl --max-pages 100 --max-depth 3
112
+
113
+ # Ingest a single URL
114
+ cursor-rag ingest https://example.com/page
115
+
116
+ # Ingest a local file
117
+ cursor-rag ingest ./document.md
118
+
119
+ # Ingest a directory (recursive)
120
+ cursor-rag ingest ./docs
121
+ ```
122
+
123
+ ### `cursor-rag search`
124
+
125
+ Test search from the command line.
126
+
127
+ ```bash
128
+ cursor-rag search "how to set up authentication"
129
+ cursor-rag search "database queries" --top-k 10
130
+ ```
131
+
132
+ ### `cursor-rag status`
133
+
134
+ Show current configuration and knowledge base statistics.
135
+
136
+ ```bash
137
+ cursor-rag status
138
+ ```
139
+
140
+ ### `cursor-rag dashboard`
141
+
142
+ Start the web dashboard for monitoring and configuration.
143
+
144
+ ```bash
145
+ cursor-rag dashboard
146
+ cursor-rag dashboard --port 8080
147
+ ```
148
+
149
+ The dashboard provides:
150
+ - **Overview**: Total chunks, vector store stats, recent activity
151
+ - **Search**: Test queries against your knowledge base
152
+ - **Activity Log**: Real-time view of ingestion and search operations
153
+ - **Settings**: Configure vector store, embeddings, and proxy settings
154
+
155
+ ## Usage in Cursor
156
+
157
+ ### Option 1: Via `@Docs` (Recommended)
158
+
159
+ The dashboard serves your knowledge base as browsable documentation that Cursor can index:
160
+
161
+ 1. Start the dashboard: `cursor-rag dashboard`
162
+ 2. In Cursor, type `@Docs` and select **Add new doc**
163
+ 3. Enter: `http://localhost:3333/docs`
164
+ 4. Now use `@Docs cursor-recursive-rag` in your prompts!
165
+
166
+ ### Option 2: Via MCP Tools (Natural Language)
167
+
168
+ The MCP tools are available to the AI agent in Cursor Chat. Ask questions naturally and the AI will use the appropriate tools:
169
+
170
+ ```
171
+ Search my knowledge base for authentication patterns
172
+
173
+ What sources are indexed in my RAG knowledge base?
174
+
175
+ Crawl and ingest https://docs.example.com into my knowledge base with a max of 50 pages
176
+ ```
177
+
178
+ > **Note:** MCP tools don't use `@` syntax - that's reserved for Cursor's built-in features (`@Codebase`, `@Docs`, `@Files`). MCP tools are called automatically by the AI when relevant to your request.
179
+
180
+ ### Available MCP Tools
181
+
182
+ | Tool | Description |
183
+ |------|-------------|
184
+ | `recursive_query` | Multi-hop retrieval with query decomposition |
185
+ | `search_knowledge` | Direct vector similarity search |
186
+ | `ingest_document` | Add a single document (URL, file, or text) |
187
+ | `crawl_and_ingest` | Crawl website with Firecrawl and index |
188
+ | `list_sources` | List indexed document sources |
189
+
190
+ ## Configuration
191
+
192
+ Configuration is stored in `~/.cursor-rag/config.json`:
193
+
194
+ ```json
195
+ {
196
+ "vectorStore": "chroma",
197
+ "embeddings": "xenova",
198
+ "apiKeys": {
199
+ "firecrawl": "fc-..."
200
+ },
201
+ "proxy": {
202
+ "enabled": true,
203
+ "driver": "packetstream",
204
+ "host": "proxy.packetstream.io",
205
+ "port": 31112,
206
+ "username": "your-username",
207
+ "password": "your-password"
208
+ },
209
+ "dashboard": {
210
+ "enabled": true,
211
+ "port": 3333
212
+ }
213
+ }
214
+ ```
215
+
216
+ ### Proxy Configuration
217
+
218
+ The optional rotating proxy is used for direct URL fetching (not needed when using Firecrawl, which handles proxying internally). Supported providers:
219
+
220
+ - **PacketStream**: Residential proxies with country targeting
221
+ - **SmartProxy**: Datacenter and residential options
222
+
223
+ ### MCP Gateway Integration
224
+
225
+ Connect to [MCP Gateway](https://github.com/abdullah1854/MCPGateway) to access 87+ aggregated tools with token optimization:
226
+
227
+ ```json
228
+ {
229
+ "mcpGateway": {
230
+ "enabled": true,
231
+ "url": "http://localhost:3010",
232
+ "apiKey": "optional-api-key"
233
+ }
234
+ }
235
+ ```
236
+
237
+ **MCP Tools exposed:**
238
+ - `gateway_search_tools` - Search available tools across all backends
239
+ - `gateway_call_tool` - Call any gateway tool with result filtering
240
+ - `gateway_execute_skill` - Execute gateway skills
241
+ - `gateway_health` - Check gateway status
242
+
243
+ ### OpenSkills Integration
244
+
245
+ Connect to [OpenSkills](https://github.com/numman-ali/openskills) for universal skills loading:
246
+
247
+ ```json
248
+ {
249
+ "openSkills": {
250
+ "enabled": true,
251
+ "autoIngestSkills": true
252
+ }
253
+ }
254
+ ```
255
+
256
+ **MCP Tools exposed:**
257
+ - `list_openskills` - List all installed skills
258
+ - `read_openskill` - Read a specific skill's content
259
+ - `ingest_openskills` - Ingest all skills into RAG knowledge base
260
+ - `search_openskills` - Semantic search across ingested skills
261
+
262
+ **Skill Discovery Paths (priority order):**
263
+ 1. `./.agent/skills/` (project universal)
264
+ 2. `~/.agent/skills/` (global universal)
265
+ 3. `./.claude/skills/` (project Claude)
266
+ 4. `~/.claude/skills/` (global Claude)
267
+
268
+ The MCP server is registered in `~/.cursor/mcp.json`:
269
+
270
+ ```json
271
+ {
272
+ "mcpServers": {
273
+ "recursive-rag": {
274
+ "command": "node",
275
+ "args": ["cursor-recursive-rag/dist/server/index.js"],
276
+ "env": {
277
+ "CURSOR_RAG_CONFIG": "/Users/you/.cursor-rag/config.json"
278
+ }
279
+ }
280
+ }
281
+ }
282
+ ```
283
+
284
+ ## Architecture
285
+
286
+ ```
287
+ cursor-recursive-rag/
288
+ ├── src/
289
+ │ ├── cli/ # CLI commands (setup, ingest, search, status)
290
+ │ ├── server/ # MCP server and tool handlers
291
+ │ ├── adapters/
292
+ │ │ ├── vector/ # Vector store adapters (Chroma, Qdrant, Vectorize)
293
+ │ │ └── embeddings/ # Embedding adapters (Xenova, OpenAI, Ollama)
294
+ │ ├── services/ # Chunker, query decomposer, config
295
+ │ └── types/ # TypeScript type definitions
296
+ ├── bin/ # CLI entry point
297
+ ├── dist/ # Compiled JavaScript
298
+ └── package.json
299
+ ```
300
+
301
+ ## Requirements
302
+
303
+ - Node.js >= 20.0.0
304
+ - Cursor IDE (for MCP integration)
305
+ - Optional: Docker (for Qdrant local)
306
+ - Optional: Ollama (for local embeddings)
307
+
308
+ ## API Keys
309
+
310
+ | Service | Purpose | Get Key |
311
+ |---------|---------|---------|
312
+ | Firecrawl | Web crawling | https://www.firecrawl.dev |
313
+ | OpenAI | Embeddings | https://platform.openai.com |
314
+ | Qdrant Cloud | Vector store | https://cloud.qdrant.io |
315
+
316
+ ## Troubleshooting
317
+
318
+ ### "Config file not found"
319
+
320
+ Run `cursor-rag setup` to create the configuration.
321
+
322
+ ### MCP server not showing in Cursor
323
+
324
+ 1. Check `~/.cursor/mcp.json` has the `recursive-rag` entry
325
+ 2. Restart Cursor IDE
326
+ 3. Check the server path is correct
327
+
328
+ ### Firecrawl errors
329
+
330
+ Ensure your Firecrawl API key is valid and starts with `fc-`.
331
+
332
+ ### ChromaDB permission errors
333
+
334
+ The local ChromaDB stores data in `~/.cursor-rag/chroma-data`. Ensure this directory is writable.
335
+
336
+ ## Development
337
+
338
+ ```bash
339
+ # Install dependencies
340
+ npm install
341
+
342
+ # Build
343
+ npm run build
344
+
345
+ # Development mode (watch)
346
+ npm run dev
347
+
348
+ # Link for testing
349
+ npm link
350
+ ```
351
+
352
+ ## License
353
+
354
+ MIT
@@ -0,0 +1,18 @@
1
+ #!/usr/bin/env node
2
+
3
+ import { fileURLToPath } from 'url';
4
+ import { dirname, join } from 'path';
5
+ import { createRequire } from 'module';
6
+
7
+ const __filename = fileURLToPath(import.meta.url);
8
+ const __dirname = dirname(__filename);
9
+
10
+ // Use createRequire to load the compiled CLI
11
+ const require = createRequire(import.meta.url);
12
+ const cliPath = join(__dirname, '../dist/cli/index.js');
13
+
14
+ // Import and run the CLI
15
+ import(cliPath).catch((error) => {
16
+ console.error('Failed to start CLI:', error);
17
+ process.exit(1);
18
+ });
@@ -0,0 +1,5 @@
1
+ import type { Embedder } from './types.js';
2
+ import type { RAGConfig } from '../../types/index.js';
3
+ export { Embedder } from './types.js';
4
+ export declare function createEmbedder(type: string, config: RAGConfig): Promise<Embedder>;
5
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/adapters/embeddings/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAI3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAEtD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,CAWvF"}
@@ -0,0 +1,16 @@
1
+ import { XenovaAdapter } from './xenova.js';
2
+ import { OpenAIAdapter } from './openai.js';
3
+ import { OllamaAdapter } from './ollama.js';
4
+ export async function createEmbedder(type, config) {
5
+ switch (type) {
6
+ case 'xenova':
7
+ return await XenovaAdapter.create(config);
8
+ case 'openai':
9
+ return new OpenAIAdapter(config);
10
+ case 'ollama':
11
+ return new OllamaAdapter(config);
12
+ default:
13
+ throw new Error(`Unknown embedder: ${type}`);
14
+ }
15
+ }
16
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/adapters/embeddings/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAK5C,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,MAAiB;IAClE,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,QAAQ;YACX,OAAO,MAAM,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC5C,KAAK,QAAQ;YACX,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;QACnC,KAAK,QAAQ;YACX,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC;QACnC;YACE,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;IACjD,CAAC;AACH,CAAC"}
@@ -0,0 +1,11 @@
1
+ import type { Embedder } from './types.js';
2
+ import type { RAGConfig } from '../../types/index.js';
3
+ export declare class OllamaAdapter implements Embedder {
4
+ private client;
5
+ private model;
6
+ dimensions: number;
7
+ constructor(config: RAGConfig);
8
+ embed(text: string): Promise<number[]>;
9
+ embedBatch(texts: string[]): Promise<number[][]>;
10
+ }
11
+ //# sourceMappingURL=ollama.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ollama.d.ts","sourceRoot":"","sources":["../../../src/adapters/embeddings/ollama.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAEtD,qBAAa,aAAc,YAAW,QAAQ;IAC5C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;IACtB,UAAU,SAAO;gBAEL,MAAM,EAAE,SAAS;IASvB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAQtC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CASvD"}
@@ -0,0 +1,30 @@
1
+ import { Ollama } from 'ollama';
2
+ export class OllamaAdapter {
3
+ client;
4
+ model;
5
+ dimensions = 768; // nomic-embed-text default dimensions
6
+ constructor(config) {
7
+ const baseUrl = config.apiKeys?.ollama?.baseUrl || 'http://localhost:11434';
8
+ this.model = config.apiKeys?.ollama?.model || 'nomic-embed-text';
9
+ this.client = new Ollama({
10
+ host: baseUrl
11
+ });
12
+ }
13
+ async embed(text) {
14
+ const response = await this.client.embeddings({
15
+ model: this.model,
16
+ prompt: text
17
+ });
18
+ return response.embedding;
19
+ }
20
+ async embedBatch(texts) {
21
+ // Ollama doesn't support batch embeddings, so do sequentially
22
+ const embeddings = [];
23
+ for (const text of texts) {
24
+ const embedding = await this.embed(text);
25
+ embeddings.push(embedding);
26
+ }
27
+ return embeddings;
28
+ }
29
+ }
30
+ //# sourceMappingURL=ollama.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ollama.js","sourceRoot":"","sources":["../../../src/adapters/embeddings/ollama.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAIhC,MAAM,OAAO,aAAa;IAChB,MAAM,CAAS;IACf,KAAK,CAAS;IACtB,UAAU,GAAG,GAAG,CAAC,CAAC,sCAAsC;IAExD,YAAY,MAAiB;QAC3B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,IAAI,wBAAwB,CAAC;QAC5E,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,IAAI,kBAAkB,CAAC;QAEjE,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC;YACvB,IAAI,EAAE,OAAO;SACd,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;YAC5C,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI;SACb,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,SAAS,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,8DAA8D;QAC9D,MAAM,UAAU,GAAe,EAAE,CAAC;QAClC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACzC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7B,CAAC;QACD,OAAO,UAAU,CAAC;IACpB,CAAC;CACF"}
@@ -0,0 +1,11 @@
1
+ import type { Embedder } from './types.js';
2
+ import type { RAGConfig } from '../../types/index.js';
3
+ export declare class OpenAIAdapter implements Embedder {
4
+ private client;
5
+ private model;
6
+ dimensions: number;
7
+ constructor(config: RAGConfig);
8
+ embed(text: string): Promise<number[]>;
9
+ embedBatch(texts: string[]): Promise<number[][]>;
10
+ }
11
+ //# sourceMappingURL=openai.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../../src/adapters/embeddings/openai.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAEtD,qBAAa,aAAc,YAAW,QAAQ;IAC5C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAA4B;IACzC,UAAU,SAAQ;gBAEN,MAAM,EAAE,SAAS;IASvB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAQtC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAOvD"}
@@ -0,0 +1,28 @@
1
+ import OpenAI from 'openai';
2
+ export class OpenAIAdapter {
3
+ client;
4
+ model = 'text-embedding-3-small';
5
+ dimensions = 1536; // text-embedding-3-small dimensions
6
+ constructor(config) {
7
+ const apiKey = config.apiKeys?.openai;
8
+ if (!apiKey) {
9
+ throw new Error('OpenAI API key is required for OpenAI embeddings');
10
+ }
11
+ this.client = new OpenAI({ apiKey });
12
+ }
13
+ async embed(text) {
14
+ const response = await this.client.embeddings.create({
15
+ model: this.model,
16
+ input: text
17
+ });
18
+ return response.data[0].embedding;
19
+ }
20
+ async embedBatch(texts) {
21
+ const response = await this.client.embeddings.create({
22
+ model: this.model,
23
+ input: texts
24
+ });
25
+ return response.data.map(item => item.embedding);
26
+ }
27
+ }
28
+ //# sourceMappingURL=openai.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"openai.js","sourceRoot":"","sources":["../../../src/adapters/embeddings/openai.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAI5B,MAAM,OAAO,aAAa;IAChB,MAAM,CAAS;IACf,KAAK,GAAG,wBAAwB,CAAC;IACzC,UAAU,GAAG,IAAI,CAAC,CAAC,oCAAoC;IAEvD,YAAY,MAAiB;QAC3B,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;YACnD,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;YACnD,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,KAAK;SACb,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACnD,CAAC;CACF"}
@@ -0,0 +1,6 @@
1
+ export interface Embedder {
2
+ embed(text: string): Promise<number[]>;
3
+ embedBatch(texts: string[]): Promise<number[][]>;
4
+ dimensions: number;
5
+ }
6
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/adapters/embeddings/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,QAAQ;IACvB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACvC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACjD,UAAU,EAAE,MAAM,CAAC;CACpB"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/adapters/embeddings/types.ts"],"names":[],"mappings":""}
@@ -0,0 +1,11 @@
1
+ import type { Embedder } from './types.js';
2
+ import type { RAGConfig } from '../../types/index.js';
3
+ export declare class XenovaAdapter implements Embedder {
4
+ private model;
5
+ dimensions: number;
6
+ private constructor();
7
+ static create(config: RAGConfig): Promise<XenovaAdapter>;
8
+ embed(text: string): Promise<number[]>;
9
+ embedBatch(texts: string[]): Promise<number[][]>;
10
+ }
11
+ //# sourceMappingURL=xenova.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"xenova.d.ts","sourceRoot":"","sources":["../../../src/adapters/embeddings/xenova.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAEtD,qBAAa,aAAc,YAAW,QAAQ;IAC5C,OAAO,CAAC,KAAK,CAAM;IACnB,UAAU,SAAO;IAEjB,OAAO;WAIM,MAAM,CAAC,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC;IASxD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAQtC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAKvD"}
@@ -0,0 +1,26 @@
1
+ import { pipeline } from '@xenova/transformers';
2
+ export class XenovaAdapter {
3
+ model;
4
+ dimensions = 384; // all-MiniLM-L6-v2 dimensions
5
+ constructor(model) {
6
+ this.model = model;
7
+ }
8
+ static async create(config) {
9
+ // Initialize the embedding model
10
+ const model = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2');
11
+ return new XenovaAdapter(model);
12
+ }
13
+ async embed(text) {
14
+ const output = await this.model(text, {
15
+ pooling: 'mean',
16
+ normalize: true
17
+ });
18
+ return Array.from(output.data);
19
+ }
20
+ async embedBatch(texts) {
21
+ // Process in parallel
22
+ const embeddings = await Promise.all(texts.map(text => this.embed(text)));
23
+ return embeddings;
24
+ }
25
+ }
26
+ //# sourceMappingURL=xenova.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"xenova.js","sourceRoot":"","sources":["../../../src/adapters/embeddings/xenova.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAIhD,MAAM,OAAO,aAAa;IAChB,KAAK,CAAM;IACnB,UAAU,GAAG,GAAG,CAAC,CAAC,8BAA8B;IAEhD,YAAoB,KAAU;QAC5B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAiB;QACnC,iCAAiC;QACjC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAC1B,oBAAoB,EACpB,yBAAyB,CAC1B,CAAC;QACF,OAAO,IAAI,aAAa,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;YACpC,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QACH,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,sBAAsB;QACtB,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC1E,OAAO,UAAU,CAAC;IACpB,CAAC;CACF"}
@@ -0,0 +1,23 @@
1
+ import type { VectorStore, VectorDocument, SearchResult, SearchOptions } from '../../types/index.js';
2
+ import type { RAGConfig } from '../../types/index.js';
3
+ /**
4
+ * ChromaDB Adapter
5
+ *
6
+ * NOTE: ChromaDB JS client requires a running ChromaDB server.
7
+ * Start one with: docker run -p 8000:8000 chromadb/chroma
8
+ *
9
+ * For serverless local storage, use Qdrant instead.
10
+ */
11
+ export declare class ChromaAdapter implements VectorStore {
12
+ private client;
13
+ private collection;
14
+ private collectionName;
15
+ private serverUrl;
16
+ constructor(config: RAGConfig);
17
+ initialize(): Promise<void>;
18
+ add(docs: VectorDocument[]): Promise<void>;
19
+ search(embedding: number[], options: SearchOptions): Promise<SearchResult[]>;
20
+ delete(ids: string[]): Promise<void>;
21
+ count(): Promise<number>;
22
+ }
23
+ //# sourceMappingURL=chroma.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"chroma.d.ts","sourceRoot":"","sources":["../../../src/adapters/vector/chroma.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrG,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAEtD;;;;;;;GAOG;AACH,qBAAa,aAAc,YAAW,WAAW;IAC/C,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,UAAU,CAAM;IACxB,OAAO,CAAC,cAAc,CAA+B;IACrD,OAAO,CAAC,SAAS,CAAS;gBAEd,MAAM,EAAE,SAAS;IAQvB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAqB3B,GAAG,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB1C,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAgC5E,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAKpC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;CAK/B"}