@copilotkit/pathfinder 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +20 -0
- package/.superpowers/brainstorm/47098-1775507869/content/homepage-mockup.html +324 -0
- package/.superpowers/brainstorm/47098-1775507869/state/server-stopped +1 -0
- package/.superpowers/brainstorm/47098-1775507869/state/server.log +13 -0
- package/.superpowers/brainstorm/47098-1775507869/state/server.pid +1 -0
- package/.superpowers/brainstorm/82141-1775511032/content/migration-v2.html +340 -0
- package/.superpowers/brainstorm/82141-1775511032/content/migration.html +340 -0
- package/.superpowers/brainstorm/82141-1775511032/state/server-stopped +1 -0
- package/.superpowers/brainstorm/82141-1775511032/state/server.log +4 -0
- package/.superpowers/brainstorm/82141-1775511032/state/server.pid +1 -0
- package/CHANGELOG.md +26 -0
- package/LICENSE +21 -0
- package/README.md +284 -0
- package/dist/config.d.ts +32 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +180 -0
- package/dist/config.js.map +1 -0
- package/dist/db/client.d.ts +22 -0
- package/dist/db/client.d.ts.map +1 -0
- package/dist/db/client.js +134 -0
- package/dist/db/client.js.map +1 -0
- package/dist/db/queries.d.ts +51 -0
- package/dist/db/queries.d.ts.map +1 -0
- package/dist/db/queries.js +271 -0
- package/dist/db/queries.js.map +1 -0
- package/dist/db/schema.d.ts +11 -0
- package/dist/db/schema.d.ts.map +1 -0
- package/dist/db/schema.js +63 -0
- package/dist/db/schema.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +366 -0
- package/dist/index.js.map +1 -0
- package/dist/indexing/chunking/code.d.ts +17 -0
- package/dist/indexing/chunking/code.d.ts.map +1 -0
- package/dist/indexing/chunking/code.js +277 -0
- package/dist/indexing/chunking/code.js.map +1 -0
- package/dist/indexing/chunking/index.d.ts +6 -0
- package/dist/indexing/chunking/index.d.ts.map +1 -0
- package/dist/indexing/chunking/index.js +19 -0
- package/dist/indexing/chunking/index.js.map +1 -0
- package/dist/indexing/chunking/markdown.d.ts +16 -0
- package/dist/indexing/chunking/markdown.d.ts.map +1 -0
- package/dist/indexing/chunking/markdown.js +283 -0
- package/dist/indexing/chunking/markdown.js.map +1 -0
- package/dist/indexing/chunking/raw-text.d.ts +11 -0
- package/dist/indexing/chunking/raw-text.d.ts.map +1 -0
- package/dist/indexing/chunking/raw-text.js +59 -0
- package/dist/indexing/chunking/raw-text.js.map +1 -0
- package/dist/indexing/embeddings.d.ts +10 -0
- package/dist/indexing/embeddings.d.ts.map +1 -0
- package/dist/indexing/embeddings.js +78 -0
- package/dist/indexing/embeddings.js.map +1 -0
- package/dist/indexing/orchestrator.d.ts +69 -0
- package/dist/indexing/orchestrator.d.ts.map +1 -0
- package/dist/indexing/orchestrator.js +387 -0
- package/dist/indexing/orchestrator.js.map +1 -0
- package/dist/indexing/source-indexer.d.ts +68 -0
- package/dist/indexing/source-indexer.d.ts.map +1 -0
- package/dist/indexing/source-indexer.js +379 -0
- package/dist/indexing/source-indexer.js.map +1 -0
- package/dist/indexing/url-derivation.d.ts +7 -0
- package/dist/indexing/url-derivation.d.ts.map +1 -0
- package/dist/indexing/url-derivation.js +31 -0
- package/dist/indexing/url-derivation.js.map +1 -0
- package/dist/mcp/server.d.ts +10 -0
- package/dist/mcp/server.d.ts.map +1 -0
- package/dist/mcp/server.js +67 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/mcp/tools/bash-fs.d.ts +19 -0
- package/dist/mcp/tools/bash-fs.d.ts.map +1 -0
- package/dist/mcp/tools/bash-fs.js +134 -0
- package/dist/mcp/tools/bash-fs.js.map +1 -0
- package/dist/mcp/tools/bash-grep.d.ts +29 -0
- package/dist/mcp/tools/bash-grep.d.ts.map +1 -0
- package/dist/mcp/tools/bash-grep.js +153 -0
- package/dist/mcp/tools/bash-grep.js.map +1 -0
- package/dist/mcp/tools/bash-related.d.ts +14 -0
- package/dist/mcp/tools/bash-related.d.ts.map +1 -0
- package/dist/mcp/tools/bash-related.js +54 -0
- package/dist/mcp/tools/bash-related.js.map +1 -0
- package/dist/mcp/tools/bash-session.d.ts +23 -0
- package/dist/mcp/tools/bash-session.d.ts.map +1 -0
- package/dist/mcp/tools/bash-session.js +60 -0
- package/dist/mcp/tools/bash-session.js.map +1 -0
- package/dist/mcp/tools/bash-telemetry.d.ts +26 -0
- package/dist/mcp/tools/bash-telemetry.d.ts.map +1 -0
- package/dist/mcp/tools/bash-telemetry.js +53 -0
- package/dist/mcp/tools/bash-telemetry.js.map +1 -0
- package/dist/mcp/tools/bash-virtual-files.d.ts +3 -0
- package/dist/mcp/tools/bash-virtual-files.d.ts.map +1 -0
- package/dist/mcp/tools/bash-virtual-files.js +65 -0
- package/dist/mcp/tools/bash-virtual-files.js.map +1 -0
- package/dist/mcp/tools/bash.d.ts +25 -0
- package/dist/mcp/tools/bash.d.ts.map +1 -0
- package/dist/mcp/tools/bash.js +140 -0
- package/dist/mcp/tools/bash.js.map +1 -0
- package/dist/mcp/tools/collect.d.ts +13 -0
- package/dist/mcp/tools/collect.d.ts.map +1 -0
- package/dist/mcp/tools/collect.js +56 -0
- package/dist/mcp/tools/collect.js.map +1 -0
- package/dist/mcp/tools/search.d.ts +5 -0
- package/dist/mcp/tools/search.d.ts.map +1 -0
- package/dist/mcp/tools/search.js +68 -0
- package/dist/mcp/tools/search.js.map +1 -0
- package/dist/types.d.ts +1237 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +163 -0
- package/dist/types.js.map +1 -0
- package/dist/webhooks/github.d.ts +12 -0
- package/dist/webhooks/github.d.ts.map +1 -0
- package/dist/webhooks/github.js +117 -0
- package/dist/webhooks/github.js.map +1 -0
- package/package.json +48 -0
package/README.md
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
# Pathfinder
|
|
2
|
+
|
|
3
|
+
A self-hosted MCP server that provides semantic search over your documentation and code. Configure it with a YAML file, deploy with Docker, and give your AI coding agents instant access to your project's knowledge.
|
|
4
|
+
|
|
5
|
+
## How It Works
|
|
6
|
+
|
|
7
|
+
Pathfinder indexes your GitHub repositories — documentation (Markdown/MDX) and source code — into a PostgreSQL vector database using OpenAI embeddings. It exposes configurable search tools via the [Model Context Protocol (MCP)](https://modelcontextprotocol.io), so AI agents like Claude Code can search your docs and code semantically.
|
|
8
|
+
|
|
9
|
+
## Quick Start
|
|
10
|
+
|
|
11
|
+
1. **Clone and configure:**
|
|
12
|
+
```bash
|
|
13
|
+
git clone https://github.com/CopilotKit/pathfinder.git
|
|
14
|
+
cd pathfinder
|
|
15
|
+
cp pathfinder.example.yaml pathfinder.yaml # edit for your project
|
|
16
|
+
cp .env.example .env # add your OPENAI_API_KEY
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
2. **Start the server:**
|
|
20
|
+
```bash
|
|
21
|
+
docker compose up
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
3. **Seed the index:**
|
|
25
|
+
```bash
|
|
26
|
+
docker compose exec app npx tsx scripts/seed-index.ts
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
4. **Connect your AI agent:**
|
|
30
|
+
```json
|
|
31
|
+
{
|
|
32
|
+
"mcpServers": {
|
|
33
|
+
"my-docs": { "url": "http://localhost:3001/mcp" }
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Configuration
|
|
39
|
+
|
|
40
|
+
All configuration lives in `pathfinder.yaml`. See [pathfinder.example.yaml](pathfinder.example.yaml) for a minimal starting point.
|
|
41
|
+
|
|
42
|
+
### Sources
|
|
43
|
+
|
|
44
|
+
Each source defines what to index:
|
|
45
|
+
|
|
46
|
+
```yaml
|
|
47
|
+
sources:
|
|
48
|
+
- name: docs
|
|
49
|
+
type: markdown # Built-in: markdown, code, raw-text
|
|
50
|
+
repo: https://github.com/your-org/your-repo.git
|
|
51
|
+
path: docs/
|
|
52
|
+
base_url: https://docs.your-project.com/
|
|
53
|
+
url_derivation:
|
|
54
|
+
strip_prefix: "docs/"
|
|
55
|
+
strip_suffix: ".md"
|
|
56
|
+
file_patterns: ["**/*.md"]
|
|
57
|
+
chunk:
|
|
58
|
+
target_tokens: 600
|
|
59
|
+
overlap_tokens: 50
|
|
60
|
+
|
|
61
|
+
- name: code
|
|
62
|
+
type: code
|
|
63
|
+
repo: https://github.com/your-org/your-repo.git
|
|
64
|
+
path: "."
|
|
65
|
+
file_patterns: ["**/*.ts", "**/*.py"]
|
|
66
|
+
exclude_patterns: ["**/test/**", "**/*.test.*"]
|
|
67
|
+
chunk:
|
|
68
|
+
target_lines: 80
|
|
69
|
+
overlap_lines: 10
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
### Tools
|
|
73
|
+
|
|
74
|
+
Each tool maps to a source and defines the MCP tool interface:
|
|
75
|
+
|
|
76
|
+
```yaml
|
|
77
|
+
tools:
|
|
78
|
+
- name: search-docs
|
|
79
|
+
description: "Search documentation for relevant information."
|
|
80
|
+
source: docs
|
|
81
|
+
default_limit: 5
|
|
82
|
+
max_limit: 20
|
|
83
|
+
result_format: docs
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
### Collect Tools
|
|
87
|
+
|
|
88
|
+
Collect tools let agents write structured data back to the server. Unlike search tools, they don't query anything — they validate the agent's input against a YAML-defined schema and store it as JSON in the database. Use them to gather signal from agents without writing any code.
|
|
89
|
+
|
|
90
|
+
The first built-in use case is search feedback: agents report whether search results were helpful, what they tried, and what went wrong. This surfaces broken or misleading documentation quickly. But collect tools are generic — you can define any schema for any use case (e.g., broken link reporting, feature requests, error logging).
|
|
91
|
+
|
|
92
|
+
```yaml
|
|
93
|
+
tools:
|
|
94
|
+
- name: submit-feedback
|
|
95
|
+
type: collect
|
|
96
|
+
description: "Submit feedback on whether search results were helpful."
|
|
97
|
+
response: "Feedback recorded. Thank you."
|
|
98
|
+
schema:
|
|
99
|
+
tool_name:
|
|
100
|
+
type: string
|
|
101
|
+
description: "Which search tool was used"
|
|
102
|
+
required: true
|
|
103
|
+
rating:
|
|
104
|
+
type: enum
|
|
105
|
+
values: ["helpful", "not_helpful"]
|
|
106
|
+
description: "Whether the results were helpful"
|
|
107
|
+
required: true
|
|
108
|
+
comment:
|
|
109
|
+
type: string
|
|
110
|
+
description: "What worked or didn't work"
|
|
111
|
+
required: true
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Each field in `schema` supports `type` (`string`, `number`, or `enum`), an optional `description` (shown to the agent), `required` (defaults to false), and `values` (required for `enum` fields). The validated input is written as JSONB to the `collected_data` table along with the tool name and a timestamp.
|
|
115
|
+
|
|
116
|
+
### Bash Tool Options
|
|
117
|
+
|
|
118
|
+
Bash tools expose source files as a read-only virtual filesystem that agents can explore with standard commands (`find`, `grep`, `cat`, `ls`, `head`). Several options control behavior:
|
|
119
|
+
|
|
120
|
+
```yaml
|
|
121
|
+
tools:
|
|
122
|
+
- name: explore-docs
|
|
123
|
+
type: bash
|
|
124
|
+
description: "Explore documentation files"
|
|
125
|
+
sources: [docs]
|
|
126
|
+
bash:
|
|
127
|
+
session_state: true # Persistent CWD across commands (default: false)
|
|
128
|
+
grep_strategy: hybrid # memory | vector | hybrid — enables qmd semantic search (default: memory, no qmd)
|
|
129
|
+
virtual_files: true # Auto-generate INDEX.md, SEARCH_TIPS.md (default: false)
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
- **session_state**: When enabled, `cd` persists across commands within a session. Agents can run `cd /docs` in one tool call and then `ls` or `cat file.md` in the next without repeating the path.
|
|
133
|
+
- **grep_strategy**: Controls whether the `qmd` semantic search command is available. `memory` uses pure in-memory regex only (no `qmd`). `vector` or `hybrid` enable the `qmd` command, which performs semantic search via embeddings plus text `ILIKE`. The `vector` and `hybrid` modes require an `embedding` config block.
|
|
134
|
+
- **virtual_files**: Auto-generates `/INDEX.md` (file listing with descriptions) and `/SEARCH_TIPS.md` (usage guidance) at the root of the virtual filesystem.
|
|
135
|
+
|
|
136
|
+
Agents can also run the `related` command inside bash tools to find semantically similar files across all mounted sources:
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
related /docs/concepts/coagents.mdx
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
This returns a ranked list of files from any source that are semantically related to the given file, useful for discovering cross-references between documentation and code.
|
|
143
|
+
|
|
144
|
+
When `grep_strategy` is set to `vector` or `hybrid`, agents can use the `qmd` command for semantic search:
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
qmd "how do I configure authentication"
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
This performs a 2-pass search (semantic embeddings + text ILIKE) with dedup and filtering, and returns file:line:content results. Standard `grep` is never intercepted — it always works with standard flags as agents expect.
|
|
151
|
+
|
|
152
|
+
**Note:** The virtual filesystem is read-only and shared across all MCP sessions for a given tool. Content refreshes on webhook or server restart.
|
|
153
|
+
|
|
154
|
+
### Built-in Chunker Types
|
|
155
|
+
|
|
156
|
+
| Type | Best For | Splits On |
|
|
157
|
+
|------|----------|-----------|
|
|
158
|
+
| `markdown` | .md, .mdx files | Headings (h2->h3->paragraph->line), preserves code blocks |
|
|
159
|
+
| `code` | Source code files | Blank line boundaries, respects block comments/strings |
|
|
160
|
+
| `raw-text` | Plain text, logs | Paragraph boundaries (double newline) |
|
|
161
|
+
|
|
162
|
+
### Embedding
|
|
163
|
+
|
|
164
|
+
```yaml
|
|
165
|
+
embedding:
|
|
166
|
+
provider: openai
|
|
167
|
+
model: text-embedding-3-small
|
|
168
|
+
dimensions: 1536
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
### Indexing
|
|
172
|
+
|
|
173
|
+
```yaml
|
|
174
|
+
indexing:
|
|
175
|
+
auto_reindex: true # Nightly full reindex
|
|
176
|
+
reindex_hour_utc: 3 # 3 AM UTC
|
|
177
|
+
stale_threshold_hours: 24
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
## Deploying to Production with Docker
|
|
181
|
+
|
|
182
|
+
The simplest way to run in production:
|
|
183
|
+
|
|
184
|
+
1. **Configure:**
|
|
185
|
+
```bash
|
|
186
|
+
cp pathfinder.example.yaml pathfinder.yaml # edit for your project
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
2. **Set environment variables** in `.env`:
|
|
190
|
+
```
|
|
191
|
+
OPENAI_API_KEY=sk-...
|
|
192
|
+
POSTGRES_PASSWORD=your-secure-password
|
|
193
|
+
GITHUB_WEBHOOK_SECRET=your-webhook-secret
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
3. **Deploy:**
|
|
197
|
+
```bash
|
|
198
|
+
docker compose -f docker-compose.prod.yaml up -d
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
4. **Verify:**
|
|
202
|
+
```bash
|
|
203
|
+
curl http://localhost:3001/health | python3 -m json.tool
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
The health endpoint shows index status, chunk counts, and source states.
|
|
207
|
+
|
|
208
|
+
The server automatically indexes on first boot and runs a nightly reindex at the configured hour.
|
|
209
|
+
|
|
210
|
+
### GitHub Webhooks (optional)
|
|
211
|
+
|
|
212
|
+
For real-time re-indexing on push:
|
|
213
|
+
|
|
214
|
+
1. Add webhook config to `pathfinder.yaml`:
|
|
215
|
+
```yaml
|
|
216
|
+
webhook:
|
|
217
|
+
repo_sources:
|
|
218
|
+
"your-org/your-repo": [docs, code]
|
|
219
|
+
path_triggers:
|
|
220
|
+
docs: ["docs/"]
|
|
221
|
+
code: []
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
2. Configure the webhook on GitHub:
|
|
225
|
+
- URL: `https://your-server/webhooks/github`
|
|
226
|
+
- Secret: same as `GITHUB_WEBHOOK_SECRET`
|
|
227
|
+
- Events: Just `push`
|
|
228
|
+
|
|
229
|
+
## Deploying to Railway
|
|
230
|
+
|
|
231
|
+
1. **Run setup:**
|
|
232
|
+
```bash
|
|
233
|
+
./scripts/setup.sh # install deps, build Docker images
|
|
234
|
+
./scripts/deploy.sh # create Railway project, set vars, deploy
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
2. **Set custom domain** in Railway dashboard
|
|
238
|
+
|
|
239
|
+
3. **Configure webhooks:**
|
|
240
|
+
```bash
|
|
241
|
+
./scripts/setup-webhooks.sh
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
See [OPERATIONS.md](OPERATIONS.md) for the full operations runbook.
|
|
245
|
+
|
|
246
|
+
## Development
|
|
247
|
+
|
|
248
|
+
```bash
|
|
249
|
+
# Local setup
|
|
250
|
+
./scripts/setup.sh
|
|
251
|
+
|
|
252
|
+
# Start dev server (hot reload)
|
|
253
|
+
docker compose up
|
|
254
|
+
|
|
255
|
+
# Seed index
|
|
256
|
+
docker compose exec app npx tsx scripts/seed-index.ts
|
|
257
|
+
|
|
258
|
+
# Run unit tests
|
|
259
|
+
npm test
|
|
260
|
+
|
|
261
|
+
# Test search
|
|
262
|
+
docker compose exec app npx tsx scripts/test-search.ts "your query"
|
|
263
|
+
|
|
264
|
+
# Run integration tests
|
|
265
|
+
npx tsx scripts/integration-test.ts
|
|
266
|
+
|
|
267
|
+
# Run path filter tests
|
|
268
|
+
npx tsx scripts/test-path-filter.ts
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
## Environment Variables
|
|
272
|
+
|
|
273
|
+
| Variable | Required | Description |
|
|
274
|
+
|----------|----------|-------------|
|
|
275
|
+
| `OPENAI_API_KEY` | Yes | OpenAI API key for embeddings |
|
|
276
|
+
| `DATABASE_URL` | Yes | PostgreSQL connection string |
|
|
277
|
+
| `GITHUB_WEBHOOK_SECRET` | No | HMAC secret for webhook verification |
|
|
278
|
+
| `GITHUB_TOKEN` | No | GitHub token for private repos |
|
|
279
|
+
| `PATHFINDER_CONFIG` | No | Path to config file (default: `./pathfinder.yaml`) |
|
|
280
|
+
| `PORT` | No | Server port (default: `3001`) |
|
|
281
|
+
|
|
282
|
+
## License
|
|
283
|
+
|
|
284
|
+
MIT — see [LICENSE](LICENSE)
|
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import 'dotenv/config';
|
|
2
|
+
import { type ServerConfig } from './types.js';
|
|
3
|
+
export interface Config {
|
|
4
|
+
databaseUrl: string | undefined;
|
|
5
|
+
openaiApiKey: string;
|
|
6
|
+
githubToken: string;
|
|
7
|
+
githubWebhookSecret: string;
|
|
8
|
+
port: number;
|
|
9
|
+
nodeEnv: string;
|
|
10
|
+
logLevel: string;
|
|
11
|
+
cloneDir: string;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Check whether any search tools are configured (requires embeddings + indexing).
|
|
15
|
+
*/
|
|
16
|
+
export declare function hasSearchTools(): boolean;
|
|
17
|
+
/**
|
|
18
|
+
* Check whether any collect tools are configured (requires database).
|
|
19
|
+
*/
|
|
20
|
+
export declare function hasCollectTools(): boolean;
|
|
21
|
+
/**
|
|
22
|
+
* Check whether any bash tools use vector or hybrid grep (requires embeddings + database).
|
|
23
|
+
*/
|
|
24
|
+
export declare function hasBashSemanticSearch(): boolean;
|
|
25
|
+
/**
|
|
26
|
+
* Get the set of source names that need indexing (only those referenced by search tools).
|
|
27
|
+
*/
|
|
28
|
+
export declare function getIndexableSourceNames(): Set<string>;
|
|
29
|
+
export declare function getConfig(): Config;
|
|
30
|
+
export declare const config: Config;
|
|
31
|
+
export declare function getServerConfig(): ServerConfig;
|
|
32
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAEA,OAAO,eAAe,CAAC;AAIvB,OAAO,EAAsB,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAInE,MAAM,WAAW,MAAM;IACnB,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,wBAAgB,cAAc,IAAI,OAAO,CAExC;AAED;;GAEG;AACH,wBAAgB,eAAe,IAAI,OAAO,CAEzC;AAED;;GAEG;AACH,wBAAgB,qBAAqB,IAAI,OAAO,CAK/C;AAED;;GAEG;AACH,wBAAgB,uBAAuB,IAAI,GAAG,CAAC,MAAM,CAAC,CAGrD;AA0CD,wBAAgB,SAAS,IAAI,MAAM,CAKlC;AAED,eAAO,MAAM,MAAM,QAIjB,CAAC;AA+HH,wBAAgB,eAAe,IAAI,YAAY,CAK9C"}
|
package/dist/config.js
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
// Centralized configuration: env-var secrets + YAML server config.
|
|
2
|
+
import 'dotenv/config';
|
|
3
|
+
import { readFileSync, existsSync } from 'node:fs';
|
|
4
|
+
import { resolve } from 'node:path';
|
|
5
|
+
import { parse as parseYaml } from 'yaml';
|
|
6
|
+
import { ServerConfigSchema } from './types.js';
|
|
7
|
+
/**
|
|
8
|
+
* Check whether any search tools are configured (requires embeddings + indexing).
|
|
9
|
+
*/
|
|
10
|
+
export function hasSearchTools() {
|
|
11
|
+
return getServerConfig().tools.some(t => t.type === 'search');
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Check whether any collect tools are configured (requires database).
|
|
15
|
+
*/
|
|
16
|
+
export function hasCollectTools() {
|
|
17
|
+
return getServerConfig().tools.some(t => t.type === 'collect');
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Check whether any bash tools use vector or hybrid grep (requires embeddings + database).
|
|
21
|
+
*/
|
|
22
|
+
export function hasBashSemanticSearch() {
|
|
23
|
+
return getServerConfig().tools.some(t => t.type === 'bash' &&
|
|
24
|
+
(t.bash?.grep_strategy === 'vector' || t.bash?.grep_strategy === 'hybrid'));
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Get the set of source names that need indexing (only those referenced by search tools).
|
|
28
|
+
*/
|
|
29
|
+
export function getIndexableSourceNames() {
|
|
30
|
+
const searchTools = getServerConfig().tools.filter(t => t.type === 'search');
|
|
31
|
+
return new Set(searchTools.map(t => t.source));
|
|
32
|
+
}
|
|
33
|
+
let cachedConfig = null;
|
|
34
|
+
function parseConfig() {
|
|
35
|
+
const missing = [];
|
|
36
|
+
const needsRag = hasSearchTools();
|
|
37
|
+
const needsDb = needsRag || hasCollectTools() || hasBashSemanticSearch();
|
|
38
|
+
const databaseUrl = process.env.DATABASE_URL;
|
|
39
|
+
if (!databaseUrl && needsDb)
|
|
40
|
+
missing.push('DATABASE_URL');
|
|
41
|
+
const openaiApiKey = process.env.OPENAI_API_KEY;
|
|
42
|
+
if (!openaiApiKey && needsRag)
|
|
43
|
+
missing.push('OPENAI_API_KEY');
|
|
44
|
+
const githubWebhookSecret = process.env.GITHUB_WEBHOOK_SECRET ?? '';
|
|
45
|
+
if (missing.length > 0) {
|
|
46
|
+
throw new Error(`Missing required environment variables: ${missing.join(', ')}. ` +
|
|
47
|
+
`Set them before starting the server.`);
|
|
48
|
+
}
|
|
49
|
+
const port = parseInt(process.env.PORT || '3001', 10);
|
|
50
|
+
if (isNaN(port) || port < 0 || port > 65535) {
|
|
51
|
+
throw new Error(`Invalid PORT value: ${process.env.PORT}. Must be a number between 0 and 65535.`);
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
databaseUrl,
|
|
55
|
+
openaiApiKey: openaiApiKey ?? '',
|
|
56
|
+
githubToken: process.env.GITHUB_TOKEN || '',
|
|
57
|
+
githubWebhookSecret: githubWebhookSecret,
|
|
58
|
+
port,
|
|
59
|
+
nodeEnv: process.env.NODE_ENV || 'development',
|
|
60
|
+
logLevel: process.env.LOG_LEVEL || 'info',
|
|
61
|
+
cloneDir: process.env.CLONE_DIR || '/tmp/mcp-repos',
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
export function getConfig() {
|
|
65
|
+
if (!cachedConfig) {
|
|
66
|
+
cachedConfig = parseConfig();
|
|
67
|
+
}
|
|
68
|
+
return cachedConfig;
|
|
69
|
+
}
|
|
70
|
+
export const config = new Proxy({}, {
|
|
71
|
+
get(_target, prop) {
|
|
72
|
+
return getConfig()[prop];
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
// ── YAML server configuration ─────────────────────────────────────────────────
|
|
76
|
+
let cachedServerConfig = null;
|
|
77
|
+
function resolveConfigPath() {
|
|
78
|
+
// Primary env var
|
|
79
|
+
const pathfinderEnv = process.env.PATHFINDER_CONFIG;
|
|
80
|
+
if (pathfinderEnv) {
|
|
81
|
+
const resolved = resolve(pathfinderEnv);
|
|
82
|
+
if (!existsSync(resolved)) {
|
|
83
|
+
throw new Error(`PATHFINDER_CONFIG points to ${resolved} but file does not exist.`);
|
|
84
|
+
}
|
|
85
|
+
return resolved;
|
|
86
|
+
}
|
|
87
|
+
// Deprecated env var
|
|
88
|
+
const mcpDocsEnv = process.env.MCP_DOCS_CONFIG;
|
|
89
|
+
if (mcpDocsEnv) {
|
|
90
|
+
console.warn('[config] MCP_DOCS_CONFIG is deprecated — use PATHFINDER_CONFIG instead.');
|
|
91
|
+
const resolved = resolve(mcpDocsEnv);
|
|
92
|
+
if (!existsSync(resolved)) {
|
|
93
|
+
throw new Error(`MCP_DOCS_CONFIG points to ${resolved} but file does not exist.`);
|
|
94
|
+
}
|
|
95
|
+
return resolved;
|
|
96
|
+
}
|
|
97
|
+
// Primary config file
|
|
98
|
+
const pathfinderPath = resolve(process.cwd(), 'pathfinder.yaml');
|
|
99
|
+
if (existsSync(pathfinderPath)) {
|
|
100
|
+
return pathfinderPath;
|
|
101
|
+
}
|
|
102
|
+
// Deprecated config file
|
|
103
|
+
const mcpDocsPath = resolve(process.cwd(), 'mcp-docs.yaml');
|
|
104
|
+
if (mcpDocsPath && existsSync(mcpDocsPath)) {
|
|
105
|
+
console.warn('[config] mcp-docs.yaml is deprecated — rename to pathfinder.yaml.');
|
|
106
|
+
return mcpDocsPath;
|
|
107
|
+
}
|
|
108
|
+
throw new Error('No pathfinder.yaml found. Set PATHFINDER_CONFIG env var or place pathfinder.yaml in the working directory.');
|
|
109
|
+
}
|
|
110
|
+
function loadServerConfig() {
|
|
111
|
+
const configPath = resolveConfigPath();
|
|
112
|
+
const raw = readFileSync(configPath, 'utf-8');
|
|
113
|
+
const parsed = parseYaml(raw);
|
|
114
|
+
// Default tool type to 'search' for backwards compatibility
|
|
115
|
+
if (Array.isArray(parsed?.tools)) {
|
|
116
|
+
for (const tool of parsed.tools) {
|
|
117
|
+
if (tool && typeof tool === 'object' && !('type' in tool)) {
|
|
118
|
+
console.warn(`[config] Tool "${tool.name}" has no type field — defaulting to "search". Add "type: search" explicitly to silence this warning.`);
|
|
119
|
+
tool.type = 'search';
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
const result = ServerConfigSchema.safeParse(parsed);
|
|
124
|
+
if (!result.success) {
|
|
125
|
+
const issues = result.error.issues
|
|
126
|
+
.map(i => ` - ${i.path.join('.')}: ${i.message}`)
|
|
127
|
+
.join('\n');
|
|
128
|
+
throw new Error(`Invalid config at ${configPath}:\n${issues}`);
|
|
129
|
+
}
|
|
130
|
+
// Validate source name uniqueness
|
|
131
|
+
const sourceNames = new Set(result.data.sources.map(s => s.name));
|
|
132
|
+
if (sourceNames.size !== result.data.sources.length) {
|
|
133
|
+
throw new Error('Duplicate source names found in sources configuration.');
|
|
134
|
+
}
|
|
135
|
+
// Validate tool name uniqueness
|
|
136
|
+
const toolNames = new Set(result.data.tools.map(t => t.name));
|
|
137
|
+
if (toolNames.size !== result.data.tools.length) {
|
|
138
|
+
throw new Error('Duplicate tool names found in tools configuration.');
|
|
139
|
+
}
|
|
140
|
+
// Cross-validate: every search tool's source must reference an existing source name
|
|
141
|
+
const searchTools = result.data.tools.filter(t => t.type === 'search');
|
|
142
|
+
for (const tool of searchTools) {
|
|
143
|
+
if (!sourceNames.has(tool.source)) {
|
|
144
|
+
throw new Error(`Tool "${tool.name}" references source "${tool.source}" which is not defined in sources.`);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
// Cross-validate: webhook repo_sources and path_triggers must reference valid source names
|
|
148
|
+
if (result.data.webhook) {
|
|
149
|
+
const wh = result.data.webhook;
|
|
150
|
+
for (const [repo, sources] of Object.entries(wh.repo_sources)) {
|
|
151
|
+
for (const src of sources) {
|
|
152
|
+
if (!sourceNames.has(src)) {
|
|
153
|
+
throw new Error(`Webhook repo_sources["${repo}"] references source "${src}" which is not defined in sources.`);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
for (const triggerKey of Object.keys(wh.path_triggers)) {
|
|
158
|
+
if (!sourceNames.has(triggerKey)) {
|
|
159
|
+
throw new Error(`Webhook path_triggers key "${triggerKey}" does not match any defined source name.`);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
// Validate local source paths exist
|
|
164
|
+
for (const source of result.data.sources) {
|
|
165
|
+
if (!source.repo) {
|
|
166
|
+
const resolved = resolve(source.path);
|
|
167
|
+
if (!existsSync(resolved)) {
|
|
168
|
+
throw new Error(`Source "${source.name}" references local path "${source.path}" (resolved to ${resolved}) which does not exist.`);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
return result.data;
|
|
173
|
+
}
|
|
174
|
+
export function getServerConfig() {
|
|
175
|
+
if (!cachedServerConfig) {
|
|
176
|
+
cachedServerConfig = loadServerConfig();
|
|
177
|
+
}
|
|
178
|
+
return cachedServerConfig;
|
|
179
|
+
}
|
|
180
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,mEAAmE;AAEnE,OAAO,eAAe,CAAC;AACvB,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAqB,MAAM,YAAY,CAAC;AAenE;;GAEG;AACH,MAAM,UAAU,cAAc;IAC1B,OAAO,eAAe,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;AAClE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe;IAC3B,OAAO,eAAe,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;AACnE,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB;IACjC,OAAO,eAAe,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CACpC,CAAC,CAAC,IAAI,KAAK,MAAM;QACjB,CAAC,CAAC,CAAC,IAAI,EAAE,aAAa,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,aAAa,KAAK,QAAQ,CAAC,CAC7E,CAAC;AACN,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB;IACnC,MAAM,WAAW,GAAG,eAAe,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;IAC7E,OAAO,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;AACnD,CAAC;AAED,IAAI,YAAY,GAAkB,IAAI,CAAC;AAEvC,SAAS,WAAW;IAChB,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,QAAQ,IAAI,eAAe,EAAE,IAAI,qBAAqB,EAAE,CAAC;IAEzE,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IAC7C,IAAI,CAAC,WAAW,IAAI,OAAO;QAAE,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAE1D,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IAChD,IAAI,CAAC,YAAY,IAAI,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAE9D,MAAM,mBAAmB,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,EAAE,CAAC;IAEpE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CACX,2CAA2C,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;YACjE,sCAAsC,CACzC,CAAC;IACN,CAAC;IAED,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,MAAM,EAAE,EAAE,CAAC,CAAC;IACtD,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,KAAK,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CAAC,uBAAuB,OAAO,CAAC,GAAG,CAAC,IAAI,yCAAyC,CAAC,CAAC;IACtG,CAAC;IAED,OAAO;QACH,WAAW;QACX,YAAY,EAAE,YAAY,IAAI,EAAE;QAChC,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE;QAC3C,mBAAmB,EAAE,mBAAoB;QACzC,IAAI;QACJ,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,aAAa;QAC9C,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM;QACzC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,gBAAgB;KACtD,CAAC;AACN,CAAC;AAED,MAAM,UAAU,SAAS;IACrB,IAAI,CAAC,YAAY,EAAE,CAAC;QAChB,YAAY,GAAG,WAAW,EAAE,CAAC;IACjC,CAAC;IACD,OAAO,YAAY,CAAC;AACxB,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,EAAY,EAAE;IAC1C,GAAG,CAAC,OAAO,EAAE,IAAY;QACrB,OAAO,SAAS,EAAE,CAAC,IAAoB,CAAC,CAAC;IAC7C,CAAC;CACJ,CAAC,CAAC;AAEH,iFAAiF;AAEjF,IAAI,kBAAkB,GAAwB,IAAI,CAAC;AAEnD,SAAS,iBAAiB;IACtB,kBAAkB;IAClB,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IACpD,IAAI,aAAa,EAAE,CAAC;QAChB,MAAM,QAAQ,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;QACxC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,2BAA2B,CAAC,CAAC;QACxF,CAAC;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,qBAAqB;IACrB,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;IAC/C,IAAI,UAAU,EAAE,CAAC;QACb,OAAO,CAAC,IAAI,CAAC,yEAAyE,CAAC,CAAC;QACxF,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,6BAA6B,QAAQ,2BAA2B,CAAC,CAAC;QACtF,CAAC;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,sBAAsB;IACtB,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,iBAAiB,CAAC,CAAC;IACjE,IAAI,UAAU,CAAC,cAAc,CAAC,EAAE,CAAC;QAC7B,OAAO,cAAc,CAAC;IAC1B,CAAC;IAED,yBAAyB;IACzB,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,eAAe,CAAC,CAAC;IAC5D,IAAI,WAAW,IAAI,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QACzC,OAAO,CAAC,IAAI,CAAC,mEAAmE,CAAC,CAAC;QAClF,OAAO,WAAW,CAAC;IACvB,CAAC;IAED,MAAM,IAAI,KAAK,CACX,4GAA4G,CAC/G,CAAC;AACN,CAAC;AAED,SAAS,gBAAgB;IACrB,MAAM,UAAU,GAAG,iBAAiB,EAAE,CAAC;IACvC,MAAM,GAAG,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAE9B,4DAA4D;IAC5D,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,EAAE,CAAC;gBACxD,OAAO,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,IAAI,sGAAsG,CAAC,CAAC;gBAChJ,IAAI,CAAC,IAAI,GAAG,QAAQ,CAAC;YACzB,CAAC;QACL,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACpD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;aAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;aACjD,IAAI,CAAC,IAAI,CAAC,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,qBAAqB,UAAU,MAAM,MAAM,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,kCAAkC;IAClC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAClE,IAAI,WAAW,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;IAC9E,CAAC;IAED,gCAAgC;IAChC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9D,IAAI,SAAS,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAC1E,CAAC;IAED,oFAAoF;IACpF,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;IACvE,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC7B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CACX,SAAS,IAAI,CAAC,IAAI,wBAAwB,IAAI,CAAC,MAAM,oCAAoC,CAC5F,CAAC;QACN,CAAC;IACL,CAAC;IAED,2FAA2F;IAC3F,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACtB,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC;YAC5D,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBACxB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACxB,MAAM,IAAI,KAAK,CACX,yBAAyB,IAAI,yBAAyB,GAAG,oCAAoC,CAChG,CAAC;gBACN,CAAC;YACL,CAAC;QACL,CAAC;QACD,KAAK,MAAM,UAAU,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC;YACrD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,KAAK,CACX,8BAA8B,UAAU,2CAA2C,CACtF,CAAC;YACN,CAAC;QACL,CAAC;IACL,CAAC;IAED,oCAAoC;IACpC,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,KAAK,CACX,WAAW,MAAM,CAAC,IAAI,4BAA4B,MAAM,CAAC,IAAI,kBAAkB,QAAQ,yBAAyB,CACnH,CAAC;YACN,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC;AACvB,CAAC;AAED,MAAM,UAAU,eAAe;IAC3B,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACtB,kBAAkB,GAAG,gBAAgB,EAAE,CAAC;IAC5C,CAAC;IACD,OAAO,kBAAkB,CAAC;AAC9B,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import pg from "pg";
|
|
2
|
+
/**
|
|
3
|
+
* Returns a singleton pg Pool.
|
|
4
|
+
* For standard Postgres URLs, creates a pg.Pool on first call.
|
|
5
|
+
* For PGlite URLs (pglite://...), initializeSchema() must be called first
|
|
6
|
+
* or this will throw — PGlite requires async setup that getPool() cannot do.
|
|
7
|
+
*/
|
|
8
|
+
export declare function getPool(): pg.Pool;
|
|
9
|
+
/**
|
|
10
|
+
* Runs migration (drop old tables) then creates the unified schema.
|
|
11
|
+
* Idempotent — all DDL uses IF NOT EXISTS / IF EXISTS.
|
|
12
|
+
* Also registers the pgvector type so vector columns are handled correctly.
|
|
13
|
+
*
|
|
14
|
+
* When DATABASE_URL starts with "pglite://", uses an in-process PGlite
|
|
15
|
+
* instance instead of connecting to an external PostgreSQL server.
|
|
16
|
+
*/
|
|
17
|
+
export declare function initializeSchema(): Promise<void>;
|
|
18
|
+
/**
|
|
19
|
+
* Close the pool if it was initialized. Safe to call at any time.
|
|
20
|
+
*/
|
|
21
|
+
export declare function closePool(): Promise<void>;
|
|
22
|
+
//# sourceMappingURL=client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/db/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AAOpB;;;;;GAKG;AACH,wBAAgB,OAAO,IAAI,EAAE,CAAC,IAAI,CAsBjC;AAwDD;;;;;;;GAOG;AACH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,CAyCtD;AAED;;GAEG;AACH,wBAAsB,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,CAM/C"}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import pg from "pg";
|
|
2
|
+
import pgvector from "pgvector/pg";
|
|
3
|
+
import { generateSchema, generateMigration } from "./schema.js";
|
|
4
|
+
import { getConfig, getServerConfig } from "../config.js";
|
|
5
|
+
let pool = null;
|
|
6
|
+
/**
|
|
7
|
+
* Returns a singleton pg Pool.
|
|
8
|
+
* For standard Postgres URLs, creates a pg.Pool on first call.
|
|
9
|
+
* For PGlite URLs (pglite://...), initializeSchema() must be called first
|
|
10
|
+
* or this will throw — PGlite requires async setup that getPool() cannot do.
|
|
11
|
+
*/
|
|
12
|
+
export function getPool() {
|
|
13
|
+
if (pool)
|
|
14
|
+
return pool;
|
|
15
|
+
const databaseUrl = getConfig().databaseUrl;
|
|
16
|
+
if (!databaseUrl) {
|
|
17
|
+
throw new Error("DATABASE_URL is not set. Cannot create database pool.");
|
|
18
|
+
}
|
|
19
|
+
if (isPGliteUrl(databaseUrl)) {
|
|
20
|
+
throw new Error("PGlite pool not initialized. Call initializeSchema() first.");
|
|
21
|
+
}
|
|
22
|
+
pool = new pg.Pool({
|
|
23
|
+
connectionString: databaseUrl,
|
|
24
|
+
});
|
|
25
|
+
return pool;
|
|
26
|
+
}
|
|
27
|
+
function isPGliteUrl(url) {
|
|
28
|
+
return !!url && url.startsWith("pglite://");
|
|
29
|
+
}
|
|
30
|
+
function parsePGliteDataDir(url) {
|
|
31
|
+
return url.replace(/^pglite:\/\//, "");
|
|
32
|
+
}
|
|
33
|
+
async function initializePGlite() {
|
|
34
|
+
const databaseUrl = getConfig().databaseUrl;
|
|
35
|
+
if (!databaseUrl) {
|
|
36
|
+
throw new Error("DATABASE_URL is not set. Cannot initialize PGlite.");
|
|
37
|
+
}
|
|
38
|
+
const dataDir = parsePGliteDataDir(databaseUrl);
|
|
39
|
+
const dimensions = getServerConfig().embedding?.dimensions;
|
|
40
|
+
if (!dimensions)
|
|
41
|
+
throw new Error('embedding.dimensions is required for database schema initialization');
|
|
42
|
+
const { PGlite } = await import("@electric-sql/pglite");
|
|
43
|
+
const { vector } = await import("@electric-sql/pglite/vector");
|
|
44
|
+
const db = new PGlite({ dataDir, extensions: { vector } });
|
|
45
|
+
await db.waitReady;
|
|
46
|
+
// Run DDL in a transaction to avoid partial state on failure
|
|
47
|
+
await db.exec('BEGIN');
|
|
48
|
+
try {
|
|
49
|
+
await db.exec(generateMigration());
|
|
50
|
+
await db.exec(generateSchema(dimensions));
|
|
51
|
+
await db.exec('COMMIT');
|
|
52
|
+
}
|
|
53
|
+
catch (err) {
|
|
54
|
+
try {
|
|
55
|
+
await db.exec('ROLLBACK');
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
// ROLLBACK failed — original error is more useful
|
|
59
|
+
}
|
|
60
|
+
throw err;
|
|
61
|
+
}
|
|
62
|
+
// Build a wrapper that duck-types as pg.Pool.
|
|
63
|
+
// Supported pg.Pool surface: query(text, params?), connect() → {query, release}, end().
|
|
64
|
+
// Other pg.Pool methods (e.g. on(), totalCount, idleCount) are NOT implemented —
|
|
65
|
+
// the cast below is intentional since queries.ts only uses the supported subset.
|
|
66
|
+
const wrapper = {
|
|
67
|
+
query: (text, params) => db.query(text, params),
|
|
68
|
+
connect: async () => ({
|
|
69
|
+
query: (text, params) => db.query(text, params),
|
|
70
|
+
release: () => { },
|
|
71
|
+
}),
|
|
72
|
+
end: async () => db.close(),
|
|
73
|
+
};
|
|
74
|
+
pool = wrapper;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Runs migration (drop old tables) then creates the unified schema.
|
|
78
|
+
* Idempotent — all DDL uses IF NOT EXISTS / IF EXISTS.
|
|
79
|
+
* Also registers the pgvector type so vector columns are handled correctly.
|
|
80
|
+
*
|
|
81
|
+
* When DATABASE_URL starts with "pglite://", uses an in-process PGlite
|
|
82
|
+
* instance instead of connecting to an external PostgreSQL server.
|
|
83
|
+
*/
|
|
84
|
+
export async function initializeSchema() {
|
|
85
|
+
const databaseUrl = getConfig().databaseUrl;
|
|
86
|
+
if (!databaseUrl) {
|
|
87
|
+
throw new Error("DATABASE_URL is not set. Cannot initialize database schema.");
|
|
88
|
+
}
|
|
89
|
+
if (isPGliteUrl(databaseUrl)) {
|
|
90
|
+
await initializePGlite();
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
const p = getPool();
|
|
94
|
+
const dimensions = getServerConfig().embedding?.dimensions;
|
|
95
|
+
if (!dimensions)
|
|
96
|
+
throw new Error('embedding.dimensions is required for database schema initialization');
|
|
97
|
+
// Ensure the vector extension exists before registering types
|
|
98
|
+
const setupClient = await p.connect();
|
|
99
|
+
try {
|
|
100
|
+
// Requires superuser or pg_extension_owner — works with the default Docker image
|
|
101
|
+
// but will fail on locked-down setups (e.g. RDS) without explicit grants
|
|
102
|
+
await setupClient.query('CREATE EXTENSION IF NOT EXISTS vector');
|
|
103
|
+
await pgvector.registerType(setupClient);
|
|
104
|
+
}
|
|
105
|
+
finally {
|
|
106
|
+
setupClient.release();
|
|
107
|
+
}
|
|
108
|
+
// Run migration + schema creation atomically
|
|
109
|
+
const migrationClient = await p.connect();
|
|
110
|
+
try {
|
|
111
|
+
await migrationClient.query('BEGIN');
|
|
112
|
+
await migrationClient.query(generateMigration());
|
|
113
|
+
await migrationClient.query(generateSchema(dimensions));
|
|
114
|
+
await migrationClient.query('COMMIT');
|
|
115
|
+
}
|
|
116
|
+
catch (err) {
|
|
117
|
+
await migrationClient.query('ROLLBACK');
|
|
118
|
+
throw err;
|
|
119
|
+
}
|
|
120
|
+
finally {
|
|
121
|
+
migrationClient.release();
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Close the pool if it was initialized. Safe to call at any time.
|
|
126
|
+
*/
|
|
127
|
+
export async function closePool() {
|
|
128
|
+
if (pool) {
|
|
129
|
+
const p = pool;
|
|
130
|
+
pool = null;
|
|
131
|
+
await p.end();
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
//# sourceMappingURL=client.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/db/client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,QAAQ,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAE1D,IAAI,IAAI,GAAmB,IAAI,CAAC;AAEhC;;;;;GAKG;AACH,MAAM,UAAU,OAAO;IACnB,IAAI,IAAI;QAAE,OAAO,IAAI,CAAC;IAEtB,MAAM,WAAW,GAAG,SAAS,EAAE,CAAC,WAAW,CAAC;IAE5C,IAAI,CAAC,WAAW,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACX,uDAAuD,CAC1D,CAAC;IACN,CAAC;IAED,IAAI,WAAW,CAAC,WAAW,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACX,6DAA6D,CAChE,CAAC;IACN,CAAC;IAED,IAAI,GAAG,IAAI,EAAE,CAAC,IAAI,CAAC;QACf,gBAAgB,EAAE,WAAW;KAChC,CAAC,CAAC;IAEH,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,GAAuB;IACxC,OAAO,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,kBAAkB,CAAC,GAAW;IACnC,OAAO,GAAG,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,gBAAgB;IAC3B,MAAM,WAAW,GAAG,SAAS,EAAE,CAAC,WAAW,CAAC;IAC5C,IAAI,CAAC,WAAW,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAC1E,CAAC;IACD,MAAM,OAAO,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAChD,MAAM,UAAU,GAAG,eAAe,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC;IAC3D,IAAI,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IAExG,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;IACxD,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,6BAA6B,CAAC,CAAC;IAE/D,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;IAC3D,MAAM,EAAE,CAAC,SAAS,CAAC;IAEnB,6DAA6D;IAC7D,MAAM,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,IAAI,CAAC;QACD,MAAM,EAAE,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;QACnC,MAAM,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC;QAC1C,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,IAAI,CAAC;YACD,MAAM,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;QAAC,MAAM,CAAC;YACL,kDAAkD;QACtD,CAAC;QACD,MAAM,GAAG,CAAC;IACd,CAAC;IAED,8CAA8C;IAC9C,wFAAwF;IACxF,iFAAiF;IACjF,iFAAiF;IACjF,MAAM,OAAO,GAAG;QACZ,KAAK,EAAE,CAAC,IAAY,EAAE,MAAkB,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC;QACnE,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;YAClB,KAAK,EAAE,CAAC,IAAY,EAAE,MAAkB,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC;YACnE,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC;SACpB,CAAC;QACF,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE;KAC9B,CAAC;IAEF,IAAI,GAAG,OAA6B,CAAC;AACzC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB;IAClC,MAAM,WAAW,GAAG,SAAS,EAAE,CAAC,WAAW,CAAC;IAE5C,IAAI,CAAC,WAAW,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACnF,CAAC;IAED,IAAI,WAAW,CAAC,WAAW,CAAC,EAAE,CAAC;QAC3B,MAAM,gBAAgB,EAAE,CAAC;QACzB,OAAO;IACX,CAAC;IAED,MAAM,CAAC,GAAG,OAAO,EAAE,CAAC;IAEpB,MAAM,UAAU,GAAG,eAAe,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC;IAC3D,IAAI,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IAExG,8DAA8D;IAC9D,MAAM,WAAW,GAAG,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC;IACtC,IAAI,CAAC;QACD,iFAAiF;QACjF,yEAAyE;QACzE,MAAM,WAAW,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;QACjE,MAAM,QAAQ,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;IAC7C,CAAC;YAAS,CAAC;QACP,WAAW,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;IAED,6CAA6C;IAC7C,MAAM,eAAe,GAAG,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC;IAC1C,IAAI,CAAC;QACD,MAAM,eAAe,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACrC,MAAM,eAAe,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAC,CAAC;QACjD,MAAM,eAAe,CAAC,KAAK,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC;QACxD,MAAM,eAAe,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC1C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,MAAM,eAAe,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACxC,MAAM,GAAG,CAAC;IACd,CAAC;YAAS,CAAC;QACP,eAAe,CAAC,OAAO,EAAE,CAAC;IAC9B,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS;IAC3B,IAAI,IAAI,EAAE,CAAC;QACP,MAAM,CAAC,GAAG,IAAI,CAAC;QACf,IAAI,GAAG,IAAI,CAAC;QACZ,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC;IAClB,CAAC;AACL,CAAC"}
|