@gresmcp/mcp 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +18 -0
- package/README.md +219 -0
- package/dist/cli.js +1103 -0
- package/dist/cli.js.map +1 -0
- package/dist/mcp.js +515 -0
- package/dist/mcp.js.map +1 -0
- package/package.json +56 -0
package/dist/cli.js
ADDED
|
@@ -0,0 +1,1103 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
4
|
+
var __esm = (fn, res) => function __init() {
|
|
5
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
6
|
+
};
|
|
7
|
+
var __export = (target, all) => {
|
|
8
|
+
for (var name in all)
|
|
9
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
// src/config.ts
|
|
13
|
+
function databaseUrl() {
|
|
14
|
+
return process.env.GRESMCP_DATABASE_URL || DEFAULT_DATABASE_URL;
|
|
15
|
+
}
|
|
16
|
+
function normalizeUrl(url) {
|
|
17
|
+
return url.replace(/\/+$/, "");
|
|
18
|
+
}
|
|
19
|
+
function ollamaUrl(override) {
|
|
20
|
+
const raw = override || process.env.GRESMCP_OLLAMA_URL || DEFAULT_OLLAMA_URL;
|
|
21
|
+
return normalizeUrl(raw);
|
|
22
|
+
}
|
|
23
|
+
var DEFAULT_DATABASE_URL, DEFAULT_OLLAMA_URL;
|
|
24
|
+
var init_config = __esm({
|
|
25
|
+
"src/config.ts"() {
|
|
26
|
+
"use strict";
|
|
27
|
+
DEFAULT_DATABASE_URL = "postgresql://gres:gres@localhost/gres";
|
|
28
|
+
DEFAULT_OLLAMA_URL = "http://localhost:11434";
|
|
29
|
+
}
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
// src/db.ts
|
|
33
|
+
var db_exports = {};
|
|
34
|
+
__export(db_exports, {
|
|
35
|
+
assertTableName: () => assertTableName,
|
|
36
|
+
closePool: () => closePool,
|
|
37
|
+
contentHash: () => contentHash,
|
|
38
|
+
countEntries: () => countEntries,
|
|
39
|
+
createKs: () => createKs,
|
|
40
|
+
deleteBySource: () => deleteBySource,
|
|
41
|
+
deleteKs: () => deleteKs,
|
|
42
|
+
ensureEntryTable: () => ensureEntryTable,
|
|
43
|
+
entryTableName: () => entryTableName,
|
|
44
|
+
getKs: () => getKs,
|
|
45
|
+
getPool: () => getPool,
|
|
46
|
+
initDb: () => initDb,
|
|
47
|
+
insertChunks: () => insertChunks,
|
|
48
|
+
listKs: () => listKs,
|
|
49
|
+
sourceChunkHashes: () => sourceChunkHashes,
|
|
50
|
+
updateKs: () => updateKs
|
|
51
|
+
});
|
|
52
|
+
import { Pool } from "pg";
|
|
53
|
+
import { toSql } from "pgvector";
|
|
54
|
+
import { createHash } from "crypto";
|
|
55
|
+
function getPool() {
|
|
56
|
+
if (!pool) {
|
|
57
|
+
pool = new Pool({ connectionString: databaseUrl(), max: 5 });
|
|
58
|
+
}
|
|
59
|
+
return pool;
|
|
60
|
+
}
|
|
61
|
+
async function closePool() {
|
|
62
|
+
if (pool) {
|
|
63
|
+
await pool.end();
|
|
64
|
+
pool = void 0;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
function entryTableName(ks) {
|
|
68
|
+
const name = `entry_${ks.id.replace(/-/g, "")}`;
|
|
69
|
+
assertTableName(name);
|
|
70
|
+
return name;
|
|
71
|
+
}
|
|
72
|
+
function assertTableName(name) {
|
|
73
|
+
if (!/^entry_[0-9a-f]{32}$/.test(name)) {
|
|
74
|
+
throw new Error(`Refusing to use unsafe table name: ${name}`);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
function contentHash(content) {
|
|
78
|
+
return createHash("sha256").update(content).digest("hex");
|
|
79
|
+
}
|
|
80
|
+
async function initDb() {
|
|
81
|
+
const p = getPool();
|
|
82
|
+
await p.query(`CREATE EXTENSION IF NOT EXISTS vector`);
|
|
83
|
+
await p.query(`
|
|
84
|
+
CREATE TABLE IF NOT EXISTS ks (
|
|
85
|
+
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
86
|
+
name text NOT NULL UNIQUE,
|
|
87
|
+
description text NOT NULL DEFAULT '',
|
|
88
|
+
embedding_model text NOT NULL,
|
|
89
|
+
embedding_dim integer NOT NULL,
|
|
90
|
+
ollama_url text NOT NULL DEFAULT '${DEFAULT_OLLAMA_URL}',
|
|
91
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
92
|
+
updated_at timestamptz NOT NULL DEFAULT now()
|
|
93
|
+
)
|
|
94
|
+
`);
|
|
95
|
+
}
|
|
96
|
+
async function createKs(input) {
|
|
97
|
+
const res = await getPool().query(
|
|
98
|
+
`INSERT INTO ks (name, description, embedding_model, embedding_dim, ollama_url)
|
|
99
|
+
VALUES ($1, $2, $3, $4, $5)
|
|
100
|
+
RETURNING *`,
|
|
101
|
+
[input.name, input.description ?? "", input.model, input.dim, input.url]
|
|
102
|
+
);
|
|
103
|
+
const ks = res.rows[0];
|
|
104
|
+
await ensureEntryTable(ks);
|
|
105
|
+
return ks;
|
|
106
|
+
}
|
|
107
|
+
async function listKs() {
|
|
108
|
+
const res = await getPool().query(`SELECT * FROM ks ORDER BY name`);
|
|
109
|
+
return res.rows;
|
|
110
|
+
}
|
|
111
|
+
async function getKs(name) {
|
|
112
|
+
const res = await getPool().query(`SELECT * FROM ks WHERE name = $1`, [name]);
|
|
113
|
+
return res.rows[0];
|
|
114
|
+
}
|
|
115
|
+
async function updateKs(id, fields) {
|
|
116
|
+
const sets = [];
|
|
117
|
+
const params = [];
|
|
118
|
+
if (fields.name !== void 0) {
|
|
119
|
+
params.push(fields.name);
|
|
120
|
+
sets.push(`name = $${params.length}`);
|
|
121
|
+
}
|
|
122
|
+
if (fields.description !== void 0) {
|
|
123
|
+
params.push(fields.description);
|
|
124
|
+
sets.push(`description = $${params.length}`);
|
|
125
|
+
}
|
|
126
|
+
if (fields.ollama_url !== void 0) {
|
|
127
|
+
params.push(fields.ollama_url);
|
|
128
|
+
sets.push(`ollama_url = $${params.length}`);
|
|
129
|
+
}
|
|
130
|
+
if (sets.length === 0) {
|
|
131
|
+
const res2 = await getPool().query(`SELECT * FROM ks WHERE id = $1`, [id]);
|
|
132
|
+
return res2.rows[0];
|
|
133
|
+
}
|
|
134
|
+
sets.push(`updated_at = now()`);
|
|
135
|
+
params.push(id);
|
|
136
|
+
const res = await getPool().query(
|
|
137
|
+
`UPDATE ks SET ${sets.join(", ")} WHERE id = $${params.length} RETURNING *`,
|
|
138
|
+
params
|
|
139
|
+
);
|
|
140
|
+
return res.rows[0];
|
|
141
|
+
}
|
|
142
|
+
async function deleteKs(ks) {
|
|
143
|
+
const tbl = entryTableName(ks);
|
|
144
|
+
await getPool().query(`DROP TABLE IF EXISTS ${tbl}`);
|
|
145
|
+
await getPool().query(`DELETE FROM ks WHERE id = $1`, [ks.id]);
|
|
146
|
+
}
|
|
147
|
+
async function ensureEntryTable(ks) {
|
|
148
|
+
const tbl = entryTableName(ks);
|
|
149
|
+
const p = getPool();
|
|
150
|
+
await p.query(`
|
|
151
|
+
CREATE TABLE IF NOT EXISTS ${tbl} (
|
|
152
|
+
id bigserial PRIMARY KEY,
|
|
153
|
+
source text NOT NULL,
|
|
154
|
+
title text NOT NULL DEFAULT '',
|
|
155
|
+
chunk_index integer NOT NULL DEFAULT 0,
|
|
156
|
+
content text NOT NULL,
|
|
157
|
+
content_hash text NOT NULL,
|
|
158
|
+
metadata jsonb NOT NULL DEFAULT '{}',
|
|
159
|
+
tsv tsvector GENERATED ALWAYS AS (to_tsvector('english', content)) STORED,
|
|
160
|
+
embedding vector(${ks.embedding_dim}) NOT NULL,
|
|
161
|
+
created_at timestamptz NOT NULL DEFAULT now()
|
|
162
|
+
)
|
|
163
|
+
`);
|
|
164
|
+
await p.query(`CREATE INDEX IF NOT EXISTS ${tbl}_tsv_idx ON ${tbl} USING gin (tsv)`);
|
|
165
|
+
await p.query(`CREATE INDEX IF NOT EXISTS ${tbl}_emb_idx ON ${tbl} USING hnsw (embedding vector_cosine_ops)`);
|
|
166
|
+
await p.query(`CREATE INDEX IF NOT EXISTS ${tbl}_source_idx ON ${tbl} (source)`);
|
|
167
|
+
}
|
|
168
|
+
async function insertChunks(ks, rows) {
|
|
169
|
+
if (rows.length === 0) return 0;
|
|
170
|
+
const tbl = entryTableName(ks);
|
|
171
|
+
const p = getPool();
|
|
172
|
+
const BATCH = 25;
|
|
173
|
+
let inserted = 0;
|
|
174
|
+
for (let i = 0; i < rows.length; i += BATCH) {
|
|
175
|
+
const batch = rows.slice(i, i + BATCH);
|
|
176
|
+
const values = [];
|
|
177
|
+
const params = [];
|
|
178
|
+
batch.forEach((r, j) => {
|
|
179
|
+
const b = j * 7;
|
|
180
|
+
values.push(`($${b + 1}, $${b + 2}, $${b + 3}, $${b + 4}, $${b + 5}, $${b + 6}, $${b + 7}::vector)`);
|
|
181
|
+
params.push(r.source, r.title, r.chunk_index, r.content, contentHash(r.content), JSON.stringify(r.metadata), toSql(r.embedding));
|
|
182
|
+
});
|
|
183
|
+
const res = await p.query(
|
|
184
|
+
`INSERT INTO ${tbl} (source, title, chunk_index, content, content_hash, metadata, embedding)
|
|
185
|
+
VALUES ${values.join(", ")}`,
|
|
186
|
+
params
|
|
187
|
+
);
|
|
188
|
+
inserted += res.rowCount ?? 0;
|
|
189
|
+
}
|
|
190
|
+
return inserted;
|
|
191
|
+
}
|
|
192
|
+
async function deleteBySource(ks, source) {
|
|
193
|
+
const tbl = entryTableName(ks);
|
|
194
|
+
const res = await getPool().query(`DELETE FROM ${tbl} WHERE source = $1`, [source]);
|
|
195
|
+
return res.rowCount ?? 0;
|
|
196
|
+
}
|
|
197
|
+
async function sourceChunkHashes(ks, source) {
|
|
198
|
+
const tbl = entryTableName(ks);
|
|
199
|
+
const res = await getPool().query(
|
|
200
|
+
`SELECT chunk_index, content_hash FROM ${tbl} WHERE source = $1`,
|
|
201
|
+
[source]
|
|
202
|
+
);
|
|
203
|
+
const map = /* @__PURE__ */ new Map();
|
|
204
|
+
for (const row of res.rows) {
|
|
205
|
+
map.set(Number(row.chunk_index), String(row.content_hash));
|
|
206
|
+
}
|
|
207
|
+
return map;
|
|
208
|
+
}
|
|
209
|
+
async function countEntries(ks) {
|
|
210
|
+
const tbl = entryTableName(ks);
|
|
211
|
+
const res = await getPool().query(`SELECT count(*)::int AS n FROM ${tbl}`);
|
|
212
|
+
return Number(res.rows[0]?.n ?? 0);
|
|
213
|
+
}
|
|
214
|
+
var pool;
|
|
215
|
+
var init_db = __esm({
|
|
216
|
+
"src/db.ts"() {
|
|
217
|
+
"use strict";
|
|
218
|
+
init_config();
|
|
219
|
+
}
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
// src/cli.ts
|
|
223
|
+
init_db();
|
|
224
|
+
import { Command } from "commander";
|
|
225
|
+
import { createInterface } from "readline/promises";
|
|
226
|
+
import { stdin, stdout } from "process";
|
|
227
|
+
|
|
228
|
+
// src/ollama.ts
|
|
229
|
+
init_config();
|
|
230
|
+
var OllamaError = class extends Error {
|
|
231
|
+
constructor(message, status) {
|
|
232
|
+
super(message);
|
|
233
|
+
this.status = status;
|
|
234
|
+
this.name = "OllamaError";
|
|
235
|
+
}
|
|
236
|
+
status;
|
|
237
|
+
};
|
|
238
|
+
async function embed(baseUrl, model, inputs, timeoutMs = 12e4) {
|
|
239
|
+
const url = `${normalizeUrl(baseUrl)}/api/embed`;
|
|
240
|
+
let res;
|
|
241
|
+
try {
|
|
242
|
+
res = await fetch(url, {
|
|
243
|
+
method: "POST",
|
|
244
|
+
headers: { "content-type": "application/json" },
|
|
245
|
+
body: JSON.stringify({ model, input: inputs }),
|
|
246
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
247
|
+
});
|
|
248
|
+
} catch (err) {
|
|
249
|
+
throw new OllamaError(
|
|
250
|
+
`Cannot reach Ollama at ${normalizeUrl(baseUrl)} (${err instanceof Error ? err.message : err})`
|
|
251
|
+
);
|
|
252
|
+
}
|
|
253
|
+
if (!res.ok) {
|
|
254
|
+
const body = await res.text().catch(() => "");
|
|
255
|
+
throw new OllamaError(`Ollama returned ${res.status} for ${url}: ${body.slice(0, 300)}`, res.status);
|
|
256
|
+
}
|
|
257
|
+
const data = await res.json();
|
|
258
|
+
if (data.error) {
|
|
259
|
+
throw new OllamaError(`Ollama error: ${data.error}`);
|
|
260
|
+
}
|
|
261
|
+
if (!Array.isArray(data.embeddings) || data.embeddings.length !== inputs.length) {
|
|
262
|
+
throw new OllamaError(`Unexpected response shape from ${url}`);
|
|
263
|
+
}
|
|
264
|
+
return data.embeddings;
|
|
265
|
+
}
|
|
266
|
+
async function embedBatch(baseUrl, model, inputs, batchSize = 32, onProgress) {
|
|
267
|
+
const out = [];
|
|
268
|
+
let done = 0;
|
|
269
|
+
for (let i = 0; i < inputs.length; i += batchSize) {
|
|
270
|
+
const batch = inputs.slice(i, i + batchSize);
|
|
271
|
+
const embs = await embed(baseUrl, model, batch);
|
|
272
|
+
out.push(...embs);
|
|
273
|
+
done += batch.length;
|
|
274
|
+
onProgress?.(done, inputs.length);
|
|
275
|
+
}
|
|
276
|
+
return out;
|
|
277
|
+
}
|
|
278
|
+
async function probeModel(baseUrl, model) {
|
|
279
|
+
const embs = await embed(baseUrl, model, ["gresmcp dimension probe"], 3e4);
|
|
280
|
+
const first = embs[0];
|
|
281
|
+
if (!first || first.length === 0) {
|
|
282
|
+
throw new OllamaError(`Ollama returned an empty embedding for model ${model}`);
|
|
283
|
+
}
|
|
284
|
+
return { dim: first.length };
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// src/cli.ts
|
|
288
|
+
init_config();
|
|
289
|
+
|
|
290
|
+
// src/ingest.ts
|
|
291
|
+
import { promises as fs } from "fs";
|
|
292
|
+
import path from "path";
|
|
293
|
+
|
|
294
|
+
// src/chunk.ts
|
|
295
|
+
var HEADING_RE = /^(#{1,6})\s+(.+)$/;
|
|
296
|
+
function splitBlocks(text) {
|
|
297
|
+
return text.split(/\n{2,}/).map((b) => b.trim()).filter((b) => b.length > 0);
|
|
298
|
+
}
|
|
299
|
+
function cutPoint(slice) {
|
|
300
|
+
const nl = slice.lastIndexOf("\n");
|
|
301
|
+
if (nl > slice.length * 0.5) return nl + 1;
|
|
302
|
+
const dot = slice.lastIndexOf(". ");
|
|
303
|
+
if (dot > slice.length * 0.5) return dot + 1;
|
|
304
|
+
const sp = slice.lastIndexOf(" ");
|
|
305
|
+
if (sp > slice.length * 0.5) return sp + 1;
|
|
306
|
+
return slice.length;
|
|
307
|
+
}
|
|
308
|
+
function hardSplit(block, maxLen, overlap) {
|
|
309
|
+
const parts = [];
|
|
310
|
+
let pos = 0;
|
|
311
|
+
while (pos < block.length) {
|
|
312
|
+
const end = Math.min(pos + maxLen, block.length);
|
|
313
|
+
let slice = block.slice(pos, end);
|
|
314
|
+
if (end < block.length) {
|
|
315
|
+
const cut = cutPoint(slice);
|
|
316
|
+
slice = slice.slice(0, cut);
|
|
317
|
+
}
|
|
318
|
+
parts.push(slice.trim());
|
|
319
|
+
if (end >= block.length) break;
|
|
320
|
+
const advanced = Math.max(slice.length - overlap, 1);
|
|
321
|
+
pos += advanced;
|
|
322
|
+
}
|
|
323
|
+
return parts.filter((p) => p.length > 0);
|
|
324
|
+
}
|
|
325
|
+
function packBlocks(blocks, maxLen, overlap) {
|
|
326
|
+
const pieces = [];
|
|
327
|
+
let current = "";
|
|
328
|
+
for (const block of blocks) {
|
|
329
|
+
const expanded = block.length > maxLen ? hardSplit(block, maxLen, overlap) : [block];
|
|
330
|
+
for (const part of expanded) {
|
|
331
|
+
if (!current) {
|
|
332
|
+
current = part;
|
|
333
|
+
} else if (current.length + 2 + part.length <= maxLen) {
|
|
334
|
+
current += "\n\n" + part;
|
|
335
|
+
} else {
|
|
336
|
+
pieces.push(current);
|
|
337
|
+
const tail = overlap > 0 && current.length > overlap ? current.slice(current.length - overlap).replace(/^\S+\s+/, "") : "";
|
|
338
|
+
current = tail ? tail + "\n\n" + part : part;
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
if (current) pieces.push(current);
|
|
343
|
+
return pieces;
|
|
344
|
+
}
|
|
345
|
+
function chunkMarkdown(text, opts = {}) {
|
|
346
|
+
const maxLen = opts.maxLen ?? 1200;
|
|
347
|
+
const overlap = Math.min(opts.overlap ?? 180, Math.floor(maxLen / 4));
|
|
348
|
+
const norm = text.replace(/\r\n?/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
|
|
349
|
+
if (!norm) return [];
|
|
350
|
+
const lines = norm.split("\n");
|
|
351
|
+
const sections = [{ path: [], body: [] }];
|
|
352
|
+
for (const line of lines) {
|
|
353
|
+
const m = HEADING_RE.exec(line);
|
|
354
|
+
if (m) {
|
|
355
|
+
const level = m[1].length;
|
|
356
|
+
const path2 = sections[sections.length - 1].path;
|
|
357
|
+
const parent = path2.slice(0, level - 1);
|
|
358
|
+
sections.push({ path: [...parent, m[2].trim()], body: [line] });
|
|
359
|
+
} else {
|
|
360
|
+
sections[sections.length - 1].body.push(line);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
const pieces = [];
|
|
364
|
+
for (const s of sections) {
|
|
365
|
+
const body = s.body.join("\n").trim();
|
|
366
|
+
if (!body) continue;
|
|
367
|
+
const context = s.path.length > 0 ? s.path.join(" > ") + "\n" : "";
|
|
368
|
+
const packed = packBlocks(splitBlocks(body), maxLen, overlap);
|
|
369
|
+
packed.forEach((piece, i) => {
|
|
370
|
+
pieces.push(i === 0 || !context ? piece : context + piece);
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
return pieces.map((content, index) => ({ index, content }));
|
|
374
|
+
}
|
|
375
|
+
function chunkPlain(text, opts = {}) {
|
|
376
|
+
const maxLen = opts.maxLen ?? 1200;
|
|
377
|
+
const overlap = Math.min(opts.overlap ?? 180, Math.floor(maxLen / 4));
|
|
378
|
+
const norm = text.replace(/\r\n?/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
|
|
379
|
+
if (!norm) return [];
|
|
380
|
+
const pieces = packBlocks(splitBlocks(norm), maxLen, overlap);
|
|
381
|
+
return pieces.map((content, index) => ({ index, content }));
|
|
382
|
+
}
|
|
383
|
+
function mdTitle(text) {
|
|
384
|
+
const m = /^#\s+(.+)\s*$/m.exec(text);
|
|
385
|
+
return m ? m[1].trim() : void 0;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
// src/html.ts
|
|
389
|
+
import { JSDOM } from "jsdom";
|
|
390
|
+
import TurndownService from "turndown";
|
|
391
|
+
import gfmModule from "turndown-plugin-gfm";
|
|
392
|
+
var gfm = typeof gfmModule === "function" ? gfmModule : gfmModule.gfm;
|
|
393
|
+
var REMOVE_SELECTORS = "script, style, noscript, template, iframe, svg, nav, footer, aside, form, button";
|
|
394
|
+
var service;
|
|
395
|
+
function turndown() {
|
|
396
|
+
if (!service) {
|
|
397
|
+
service = new TurndownService({
|
|
398
|
+
headingStyle: "atx",
|
|
399
|
+
codeBlockStyle: "fenced",
|
|
400
|
+
bulletListMarker: "-"
|
|
401
|
+
});
|
|
402
|
+
if (typeof gfm === "function") {
|
|
403
|
+
service.use(gfm);
|
|
404
|
+
}
|
|
405
|
+
service.remove(["button"]);
|
|
406
|
+
}
|
|
407
|
+
return service;
|
|
408
|
+
}
|
|
409
|
+
function htmlToMarkdown(html) {
|
|
410
|
+
const dom = new JSDOM(html);
|
|
411
|
+
const doc = dom.window.document;
|
|
412
|
+
doc.querySelectorAll(REMOVE_SELECTORS).forEach((el) => el.remove());
|
|
413
|
+
const title = doc.title?.trim() || doc.querySelector("h1")?.textContent?.trim() || void 0;
|
|
414
|
+
const root = doc.body ?? doc.documentElement;
|
|
415
|
+
let markdown = "";
|
|
416
|
+
try {
|
|
417
|
+
markdown = turndown().turndown(root.innerHTML);
|
|
418
|
+
} catch {
|
|
419
|
+
markdown = root.textContent ?? "";
|
|
420
|
+
}
|
|
421
|
+
markdown = markdown.replace(/\n{3,}/g, "\n\n").trim();
|
|
422
|
+
return { markdown, title: title || void 0 };
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
// src/ingest.ts
|
|
426
|
+
init_db();
|
|
427
|
+
init_config();
|
|
428
|
+
var MD_EXT = /* @__PURE__ */ new Set([".md", ".markdown", ".mdx"]);
|
|
429
|
+
var HTML_EXT = /* @__PURE__ */ new Set([".html", ".htm", ".xhtml"]);
|
|
430
|
+
var TEXT_EXT = /* @__PURE__ */ new Set([
|
|
431
|
+
".txt",
|
|
432
|
+
".text",
|
|
433
|
+
".log",
|
|
434
|
+
".csv",
|
|
435
|
+
".tsv",
|
|
436
|
+
".json",
|
|
437
|
+
".jsonl",
|
|
438
|
+
".yaml",
|
|
439
|
+
".yml",
|
|
440
|
+
".toml",
|
|
441
|
+
".ini",
|
|
442
|
+
".cfg",
|
|
443
|
+
".conf",
|
|
444
|
+
".xml",
|
|
445
|
+
".rst",
|
|
446
|
+
".adoc"
|
|
447
|
+
]);
|
|
448
|
+
var CODE_EXT = /* @__PURE__ */ new Set([
|
|
449
|
+
".js",
|
|
450
|
+
".mjs",
|
|
451
|
+
".cjs",
|
|
452
|
+
".ts",
|
|
453
|
+
".tsx",
|
|
454
|
+
".jsx",
|
|
455
|
+
".py",
|
|
456
|
+
".go",
|
|
457
|
+
".rs",
|
|
458
|
+
".java",
|
|
459
|
+
".kt",
|
|
460
|
+
".kts",
|
|
461
|
+
".rb",
|
|
462
|
+
".php",
|
|
463
|
+
".c",
|
|
464
|
+
".h",
|
|
465
|
+
".cpp",
|
|
466
|
+
".hpp",
|
|
467
|
+
".cc",
|
|
468
|
+
".cs",
|
|
469
|
+
".sh",
|
|
470
|
+
".bash",
|
|
471
|
+
".zsh",
|
|
472
|
+
".ps1",
|
|
473
|
+
".psm1",
|
|
474
|
+
".bat",
|
|
475
|
+
".cmd",
|
|
476
|
+
".sql",
|
|
477
|
+
".css",
|
|
478
|
+
".scss",
|
|
479
|
+
".less",
|
|
480
|
+
".vue",
|
|
481
|
+
".svelte",
|
|
482
|
+
".astro",
|
|
483
|
+
".swift",
|
|
484
|
+
".scala",
|
|
485
|
+
".pl",
|
|
486
|
+
".lua",
|
|
487
|
+
".r",
|
|
488
|
+
".dart",
|
|
489
|
+
".gradle",
|
|
490
|
+
".cmake",
|
|
491
|
+
".dockerfile",
|
|
492
|
+
".makefile",
|
|
493
|
+
".proto",
|
|
494
|
+
".graphql"
|
|
495
|
+
]);
|
|
496
|
+
var SKIP_DIRS = /* @__PURE__ */ new Set([
|
|
497
|
+
"node_modules",
|
|
498
|
+
".git",
|
|
499
|
+
".hg",
|
|
500
|
+
".svn",
|
|
501
|
+
"dist",
|
|
502
|
+
"build",
|
|
503
|
+
"out",
|
|
504
|
+
".next",
|
|
505
|
+
".nuxt",
|
|
506
|
+
".venv",
|
|
507
|
+
"venv",
|
|
508
|
+
"__pycache__",
|
|
509
|
+
".idea",
|
|
510
|
+
".vscode",
|
|
511
|
+
"target",
|
|
512
|
+
"coverage",
|
|
513
|
+
".turbo",
|
|
514
|
+
".cache"
|
|
515
|
+
]);
|
|
516
|
+
function isAllowed(ext) {
|
|
517
|
+
return MD_EXT.has(ext) || HTML_EXT.has(ext) || TEXT_EXT.has(ext) || CODE_EXT.has(ext);
|
|
518
|
+
}
|
|
519
|
+
async function walk(root) {
|
|
520
|
+
const out = [];
|
|
521
|
+
const entries = await fs.readdir(root, { withFileTypes: true });
|
|
522
|
+
for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
|
|
523
|
+
const full = path.join(root, entry.name);
|
|
524
|
+
if (entry.isDirectory()) {
|
|
525
|
+
if (SKIP_DIRS.has(entry.name)) continue;
|
|
526
|
+
out.push(...await walk(full));
|
|
527
|
+
} else if (entry.isFile()) {
|
|
528
|
+
out.push(full);
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
return out;
|
|
532
|
+
}
|
|
533
|
+
async function loadTarget(target, opts) {
|
|
534
|
+
const stat = await fs.stat(target);
|
|
535
|
+
const files = stat.isDirectory() ? await walk(target) : [target];
|
|
536
|
+
const docs = [];
|
|
537
|
+
const skipped = [];
|
|
538
|
+
for (const file of files) {
|
|
539
|
+
const ext = path.extname(file).toLowerCase();
|
|
540
|
+
if (!isAllowed(ext)) {
|
|
541
|
+
skipped.push({ path: file, reason: `unsupported extension '${ext || "(none)"}'` });
|
|
542
|
+
continue;
|
|
543
|
+
}
|
|
544
|
+
let raw;
|
|
545
|
+
try {
|
|
546
|
+
raw = await fs.readFile(file, "utf8");
|
|
547
|
+
} catch {
|
|
548
|
+
skipped.push({ path: file, reason: "unreadable" });
|
|
549
|
+
continue;
|
|
550
|
+
}
|
|
551
|
+
if (raw.includes("\0")) {
|
|
552
|
+
skipped.push({ path: file, reason: "binary content" });
|
|
553
|
+
continue;
|
|
554
|
+
}
|
|
555
|
+
let text = raw;
|
|
556
|
+
let title = "";
|
|
557
|
+
if (HTML_EXT.has(ext)) {
|
|
558
|
+
const conv = htmlToMarkdown(raw);
|
|
559
|
+
text = conv.markdown;
|
|
560
|
+
title = conv.title ?? "";
|
|
561
|
+
} else if (MD_EXT.has(ext)) {
|
|
562
|
+
title = mdTitle(raw) ?? "";
|
|
563
|
+
}
|
|
564
|
+
if (!text.trim()) {
|
|
565
|
+
skipped.push({ path: file, reason: "empty" });
|
|
566
|
+
continue;
|
|
567
|
+
}
|
|
568
|
+
if (!title) title = path.basename(file);
|
|
569
|
+
const chunks = HTML_EXT.has(ext) || MD_EXT.has(ext) ? chunkMarkdown(text, opts) : chunkPlain(text, opts);
|
|
570
|
+
if (chunks.length === 0) {
|
|
571
|
+
skipped.push({ path: file, reason: "no chunks produced" });
|
|
572
|
+
continue;
|
|
573
|
+
}
|
|
574
|
+
docs.push({ source: path.resolve(file), title, chunks });
|
|
575
|
+
}
|
|
576
|
+
return { docs, fileCount: files.length, skipped };
|
|
577
|
+
}
|
|
578
|
+
async function embedAndInsert(ks, pending, opts) {
|
|
579
|
+
if (pending.length === 0 || opts.dryRun) return { inserted: 0 };
|
|
580
|
+
const contents = pending.map((p) => p.chunk.content);
|
|
581
|
+
opts.log?.(`Embedding ${contents.length} chunk(s) with ${ks.embedding_model} at ${opts.url} ...`);
|
|
582
|
+
const embeddings = await embedBatch(opts.url, ks.embedding_model, contents, opts.batchSize, (done, total) => {
|
|
583
|
+
opts.log?.(` embedded ${done}/${total}`);
|
|
584
|
+
});
|
|
585
|
+
const byDoc = /* @__PURE__ */ new Map();
|
|
586
|
+
pending.forEach((p, i) => {
|
|
587
|
+
const list = byDoc.get(p.doc) ?? [];
|
|
588
|
+
list.push({
|
|
589
|
+
source: p.doc.source,
|
|
590
|
+
title: p.doc.title,
|
|
591
|
+
chunk_index: p.chunk.index,
|
|
592
|
+
content: p.chunk.content,
|
|
593
|
+
metadata: p.metadata,
|
|
594
|
+
embedding: embeddings[i]
|
|
595
|
+
});
|
|
596
|
+
byDoc.set(p.doc, list);
|
|
597
|
+
});
|
|
598
|
+
let inserted = 0;
|
|
599
|
+
for (const [, rows] of byDoc) {
|
|
600
|
+
inserted += await insertChunks(ks, rows);
|
|
601
|
+
}
|
|
602
|
+
return { inserted };
|
|
603
|
+
}
|
|
604
|
+
async function feedTarget(ks, target, opts = {}) {
|
|
605
|
+
const loadOpts = { maxLen: opts.maxLen ?? 1200, overlap: opts.overlap ?? 180 };
|
|
606
|
+
const { docs, fileCount, skipped } = await loadTarget(target, loadOpts);
|
|
607
|
+
const log = opts.log ?? ((m) => console.log(m));
|
|
608
|
+
for (const doc of docs) {
|
|
609
|
+
log(`${doc.source}: ${doc.chunks.length} chunk(s)`);
|
|
610
|
+
}
|
|
611
|
+
for (const s of skipped) {
|
|
612
|
+
log(`skipped ${s.path}: ${s.reason}`);
|
|
613
|
+
}
|
|
614
|
+
const meta = { ...opts.metadata };
|
|
615
|
+
if (opts.tags && opts.tags.length > 0) meta.tags = opts.tags;
|
|
616
|
+
let deletedForReplace = 0;
|
|
617
|
+
const pending = [];
|
|
618
|
+
let skippedExisting = 0;
|
|
619
|
+
for (const doc of docs) {
|
|
620
|
+
if (opts.replace) {
|
|
621
|
+
deletedForReplace += await deleteBySource(ks, doc.source);
|
|
622
|
+
}
|
|
623
|
+
const existing = await sourceChunkHashes(ks, doc.source);
|
|
624
|
+
for (const chunk of doc.chunks) {
|
|
625
|
+
if (!opts.replace && existing.get(chunk.index) === contentHash(chunk.content)) {
|
|
626
|
+
skippedExisting++;
|
|
627
|
+
continue;
|
|
628
|
+
}
|
|
629
|
+
pending.push({ doc, chunk, metadata: meta });
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
const { inserted } = await embedAndInsert(ks, pending, {
|
|
633
|
+
url: ollamaUrl(opts.url),
|
|
634
|
+
dryRun: opts.dryRun ?? false,
|
|
635
|
+
batchSize: opts.batchSize ?? 32,
|
|
636
|
+
log
|
|
637
|
+
});
|
|
638
|
+
return {
|
|
639
|
+
documents: docs.length,
|
|
640
|
+
chunks: docs.reduce((n, d) => n + d.chunks.length, 0),
|
|
641
|
+
inserted,
|
|
642
|
+
skippedExisting,
|
|
643
|
+
deletedForReplace,
|
|
644
|
+
skippedFiles: skipped,
|
|
645
|
+
dryRun: opts.dryRun ?? false
|
|
646
|
+
};
|
|
647
|
+
}
|
|
648
|
+
async function feedText(ks, text, opts = {}) {
|
|
649
|
+
const log = opts.log ?? ((m) => console.log(m));
|
|
650
|
+
const source = opts.sourceName?.trim() || "manual";
|
|
651
|
+
const title = opts.title?.trim() || source;
|
|
652
|
+
const chunks = chunkMarkdown(text, { maxLen: opts.maxLen ?? 1200, overlap: opts.overlap ?? 180 });
|
|
653
|
+
const doc = { source, title, chunks };
|
|
654
|
+
log(`${source}: ${chunks.length} chunk(s)`);
|
|
655
|
+
const meta = { ...opts.metadata };
|
|
656
|
+
if (opts.tags && opts.tags.length > 0) meta.tags = opts.tags;
|
|
657
|
+
let deletedForReplace = 0;
|
|
658
|
+
const pending = [];
|
|
659
|
+
let skippedExisting = 0;
|
|
660
|
+
if (opts.replace) {
|
|
661
|
+
deletedForReplace += await deleteBySource(ks, source);
|
|
662
|
+
}
|
|
663
|
+
const existing = await sourceChunkHashes(ks, source);
|
|
664
|
+
for (const chunk of chunks) {
|
|
665
|
+
if (!opts.replace && existing.get(chunk.index) === contentHash(chunk.content)) {
|
|
666
|
+
skippedExisting++;
|
|
667
|
+
continue;
|
|
668
|
+
}
|
|
669
|
+
pending.push({ doc, chunk, metadata: meta });
|
|
670
|
+
}
|
|
671
|
+
const { inserted } = await embedAndInsert(ks, pending, {
|
|
672
|
+
url: ollamaUrl(opts.url),
|
|
673
|
+
dryRun: opts.dryRun ?? false,
|
|
674
|
+
batchSize: opts.batchSize ?? 32,
|
|
675
|
+
log
|
|
676
|
+
});
|
|
677
|
+
return {
|
|
678
|
+
documents: 1,
|
|
679
|
+
chunks: chunks.length,
|
|
680
|
+
inserted,
|
|
681
|
+
skippedExisting,
|
|
682
|
+
deletedForReplace,
|
|
683
|
+
skippedFiles: [],
|
|
684
|
+
dryRun: opts.dryRun ?? false
|
|
685
|
+
};
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
// src/check.ts
|
|
689
|
+
init_config();
|
|
690
|
+
init_db();
|
|
691
|
+
import pg from "pg";
|
|
692
|
+
var MIN_NODE_MAJOR = 20;
|
|
693
|
+
var MIN_PGVECTOR_VERSION = "0.5.0";
|
|
694
|
+
var DB_CONNECT_TIMEOUT_MS = 5e3;
|
|
695
|
+
var OLLAMA_TIMEOUT_MS = 1e4;
|
|
696
|
+
function compareVersions(a, b) {
|
|
697
|
+
const pa = a.split(/[.-]/).map((s) => parseInt(s, 10) || 0);
|
|
698
|
+
const pb = b.split(/[.-]/).map((s) => parseInt(s, 10) || 0);
|
|
699
|
+
const len = Math.max(pa.length, pb.length);
|
|
700
|
+
for (let i = 0; i < len; i++) {
|
|
701
|
+
const d = (pa[i] ?? 0) - (pb[i] ?? 0);
|
|
702
|
+
if (d !== 0) return d < 0 ? -1 : 1;
|
|
703
|
+
}
|
|
704
|
+
return 0;
|
|
705
|
+
}
|
|
706
|
+
function modelMatches(tag, wanted) {
|
|
707
|
+
if (tag === wanted) return true;
|
|
708
|
+
if (wanted.includes(":")) return false;
|
|
709
|
+
return tag === `${wanted}:latest`;
|
|
710
|
+
}
|
|
711
|
+
function errMsg(err) {
|
|
712
|
+
return err instanceof Error ? err.message : String(err);
|
|
713
|
+
}
|
|
714
|
+
function shortVersion(v) {
|
|
715
|
+
return v.split(/\s+/).slice(0, 2).join(" ") || v;
|
|
716
|
+
}
|
|
717
|
+
async function fetchTags(baseUrl) {
|
|
718
|
+
try {
|
|
719
|
+
const res = await fetch(`${normalizeUrl(baseUrl)}/api/tags`, {
|
|
720
|
+
headers: { accept: "application/json" },
|
|
721
|
+
signal: AbortSignal.timeout(OLLAMA_TIMEOUT_MS)
|
|
722
|
+
});
|
|
723
|
+
if (!res.ok) return { error: `HTTP ${res.status}` };
|
|
724
|
+
const data = await res.json();
|
|
725
|
+
const names = (data.models ?? []).map((m) => m.name ?? m.model ?? "").filter(Boolean);
|
|
726
|
+
return { names };
|
|
727
|
+
} catch (err) {
|
|
728
|
+
return { error: errMsg(err) };
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
async function checkOneModel(id, model, baseUrl, names, ks, probe) {
|
|
732
|
+
const scope = ks ? ` (ks '${ks.name}')` : "";
|
|
733
|
+
if (!names.some((n) => modelMatches(n, model))) {
|
|
734
|
+
return {
|
|
735
|
+
id,
|
|
736
|
+
status: "fail",
|
|
737
|
+
detail: `model '${model}' not found at ${baseUrl}${scope} \u2014 run: ollama pull ${model}`
|
|
738
|
+
};
|
|
739
|
+
}
|
|
740
|
+
if (probe) {
|
|
741
|
+
try {
|
|
742
|
+
const { dim } = await probeModel(baseUrl, model);
|
|
743
|
+
if (ks && dim !== ks.embedding_dim) {
|
|
744
|
+
return {
|
|
745
|
+
id,
|
|
746
|
+
status: "fail",
|
|
747
|
+
detail: `model '${model}' serves dim=${dim} but ks '${ks.name}' expects dim=${ks.embedding_dim}`
|
|
748
|
+
};
|
|
749
|
+
}
|
|
750
|
+
return { id, status: "ok", detail: `model '${model}' available at ${baseUrl}${scope} (dim=${dim})` };
|
|
751
|
+
} catch (err) {
|
|
752
|
+
return { id, status: "fail", detail: `probe of model '${model}' failed \u2014 ${errMsg(err)}${scope}` };
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
return { id, status: "ok", detail: `model '${model}' available at ${baseUrl}${scope}` };
|
|
756
|
+
}
|
|
757
|
+
async function runChecks(opts = {}) {
|
|
758
|
+
const results = [];
|
|
759
|
+
const major = Number(process.versions.node.split(".")[0]);
|
|
760
|
+
results.push({
|
|
761
|
+
id: "node",
|
|
762
|
+
status: Number.isFinite(major) && major >= MIN_NODE_MAJOR ? "ok" : "fail",
|
|
763
|
+
detail: `Node ${process.versions.node} (>= ${MIN_NODE_MAJOR} required)`
|
|
764
|
+
});
|
|
765
|
+
let client;
|
|
766
|
+
let dbUp = false;
|
|
767
|
+
let kss = [];
|
|
768
|
+
try {
|
|
769
|
+
try {
|
|
770
|
+
client = new pg.Client({
|
|
771
|
+
connectionString: databaseUrl(),
|
|
772
|
+
connectionTimeoutMillis: DB_CONNECT_TIMEOUT_MS
|
|
773
|
+
});
|
|
774
|
+
client.on("error", () => {
|
|
775
|
+
});
|
|
776
|
+
await client.connect();
|
|
777
|
+
const v = await client.query("SELECT version() AS v");
|
|
778
|
+
dbUp = true;
|
|
779
|
+
results.push({
|
|
780
|
+
id: "postgres",
|
|
781
|
+
status: "ok",
|
|
782
|
+
detail: `PostgreSQL reachable at ${databaseUrl()} (${shortVersion(String(v.rows[0]?.v ?? ""))})`
|
|
783
|
+
});
|
|
784
|
+
} catch (err) {
|
|
785
|
+
results.push({
|
|
786
|
+
id: "postgres",
|
|
787
|
+
status: "fail",
|
|
788
|
+
detail: `PostgreSQL not reachable at ${databaseUrl()} \u2014 ${errMsg(err)}`
|
|
789
|
+
});
|
|
790
|
+
}
|
|
791
|
+
if (dbUp && client) {
|
|
792
|
+
try {
|
|
793
|
+
await client.query(`CREATE EXTENSION IF NOT EXISTS vector`);
|
|
794
|
+
const res = await client.query("SELECT extversion AS v FROM pg_extension WHERE extname = 'vector'");
|
|
795
|
+
const ver = res.rows[0]?.v;
|
|
796
|
+
if (!ver) {
|
|
797
|
+
results.push({
|
|
798
|
+
id: "pgvector",
|
|
799
|
+
status: "fail",
|
|
800
|
+
detail: "pgvector extension not installed \u2014 install it on the PostgreSQL server (https://github.com/pgvector/pgvector)"
|
|
801
|
+
});
|
|
802
|
+
} else if (compareVersions(ver, MIN_PGVECTOR_VERSION) < 0) {
|
|
803
|
+
results.push({
|
|
804
|
+
id: "pgvector",
|
|
805
|
+
status: "warn",
|
|
806
|
+
detail: `pgvector ${ver} installed (>= ${MIN_PGVECTOR_VERSION} required for HNSW indexes; older versions fall back to sequential scans)`
|
|
807
|
+
});
|
|
808
|
+
} else {
|
|
809
|
+
results.push({ id: "pgvector", status: "ok", detail: `pgvector ${ver} installed` });
|
|
810
|
+
}
|
|
811
|
+
} catch (err) {
|
|
812
|
+
results.push({
|
|
813
|
+
id: "pgvector",
|
|
814
|
+
status: "fail",
|
|
815
|
+
detail: `pgvector unavailable \u2014 ${errMsg(err)}`
|
|
816
|
+
});
|
|
817
|
+
}
|
|
818
|
+
} else {
|
|
819
|
+
results.push({ id: "pgvector", status: "skip", detail: "pgvector check skipped \u2014 PostgreSQL unreachable" });
|
|
820
|
+
}
|
|
821
|
+
if (dbUp && client) {
|
|
822
|
+
try {
|
|
823
|
+
const hasKs = await client.query("SELECT to_regclass('ks') IS NOT NULL AS present");
|
|
824
|
+
if (!hasKs.rows[0]?.present) {
|
|
825
|
+
results.push({ id: "schema", status: "fail", detail: "schema missing \u2014 run: gresmcp init" });
|
|
826
|
+
} else {
|
|
827
|
+
const ksRes = await client.query(
|
|
828
|
+
"SELECT id, name, embedding_model, embedding_dim, ollama_url FROM ks ORDER BY name"
|
|
829
|
+
);
|
|
830
|
+
kss = ksRes.rows;
|
|
831
|
+
const missing = [];
|
|
832
|
+
for (const ks of kss) {
|
|
833
|
+
const r = await client.query("SELECT to_regclass($1) IS NOT NULL AS present", [entryTableName(ks)]);
|
|
834
|
+
if (!r.rows[0]?.present) missing.push(ks.name);
|
|
835
|
+
}
|
|
836
|
+
if (missing.length > 0) {
|
|
837
|
+
results.push({
|
|
838
|
+
id: "schema",
|
|
839
|
+
status: "fail",
|
|
840
|
+
detail: `entry table missing for: ${missing.join(", ")} \u2014 run: gresmcp init`
|
|
841
|
+
});
|
|
842
|
+
} else {
|
|
843
|
+
results.push({
|
|
844
|
+
id: "schema",
|
|
845
|
+
status: "ok",
|
|
846
|
+
detail: `schema ready (${kss.length} knowledge source(s))`
|
|
847
|
+
});
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
} catch (err) {
|
|
851
|
+
results.push({ id: "schema", status: "fail", detail: `schema check failed \u2014 ${errMsg(err)}` });
|
|
852
|
+
}
|
|
853
|
+
} else {
|
|
854
|
+
results.push({ id: "schema", status: "skip", detail: "schema check skipped \u2014 PostgreSQL unreachable" });
|
|
855
|
+
}
|
|
856
|
+
} finally {
|
|
857
|
+
if (client) {
|
|
858
|
+
try {
|
|
859
|
+
await client.end();
|
|
860
|
+
} catch {
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
const defaultUrl = ollamaUrl(opts.url);
|
|
865
|
+
const tagCache = /* @__PURE__ */ new Map();
|
|
866
|
+
const getTags = (baseUrl) => {
|
|
867
|
+
const key = normalizeUrl(baseUrl);
|
|
868
|
+
let entry = tagCache.get(key);
|
|
869
|
+
if (!entry) {
|
|
870
|
+
entry = fetchTags(key);
|
|
871
|
+
tagCache.set(key, entry);
|
|
872
|
+
}
|
|
873
|
+
return entry;
|
|
874
|
+
};
|
|
875
|
+
const defaultTags = await getTags(defaultUrl);
|
|
876
|
+
results.push(
|
|
877
|
+
"error" in defaultTags ? { id: "ollama", status: "fail", detail: `Ollama not reachable at ${defaultUrl} \u2014 ${defaultTags.error}` } : { id: "ollama", status: "ok", detail: `Ollama reachable at ${defaultUrl} (${defaultTags.names.length} model(s))` }
|
|
878
|
+
);
|
|
879
|
+
if (opts.model) {
|
|
880
|
+
const tags = await getTags(defaultUrl);
|
|
881
|
+
if ("error" in tags) {
|
|
882
|
+
results.push({
|
|
883
|
+
id: "model",
|
|
884
|
+
status: "skip",
|
|
885
|
+
detail: `Ollama unreachable at ${defaultUrl} \u2014 cannot verify model '${opts.model}'`
|
|
886
|
+
});
|
|
887
|
+
} else {
|
|
888
|
+
results.push(await checkOneModel("model", opts.model, defaultUrl, tags.names, void 0, opts.probe ?? false));
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
for (const ks of kss) {
|
|
892
|
+
const tags = await getTags(ks.ollama_url);
|
|
893
|
+
if ("error" in tags) {
|
|
894
|
+
results.push({
|
|
895
|
+
id: `model:${ks.name}`,
|
|
896
|
+
status: "fail",
|
|
897
|
+
detail: `Ollama not reachable at ${ks.ollama_url} (required by ks '${ks.name}') \u2014 ${tags.error}`
|
|
898
|
+
});
|
|
899
|
+
continue;
|
|
900
|
+
}
|
|
901
|
+
results.push(
|
|
902
|
+
await checkOneModel(`model:${ks.name}`, ks.embedding_model, ks.ollama_url, tags.names, ks, opts.probe ?? false)
|
|
903
|
+
);
|
|
904
|
+
}
|
|
905
|
+
if (!opts.model && kss.length === 0) {
|
|
906
|
+
results.push({
|
|
907
|
+
id: "model",
|
|
908
|
+
status: "warn",
|
|
909
|
+
detail: "no embedding model to verify \u2014 pass --model <model> or create a knowledge source"
|
|
910
|
+
});
|
|
911
|
+
}
|
|
912
|
+
return results;
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
// src/version.ts
|
|
916
|
+
var VERSION = "1.0.0";
|
|
917
|
+
|
|
918
|
+
// src/cli.ts
|
|
919
|
+
async function run(fn) {
|
|
920
|
+
try {
|
|
921
|
+
await fn();
|
|
922
|
+
} catch (err) {
|
|
923
|
+
console.error(`error: ${err instanceof Error ? err.message : err}`);
|
|
924
|
+
process.exitCode = 1;
|
|
925
|
+
} finally {
|
|
926
|
+
await closePool().catch(() => {
|
|
927
|
+
});
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
async function confirm(question) {
|
|
931
|
+
const rl = createInterface({ input: stdin, output: stdout });
|
|
932
|
+
try {
|
|
933
|
+
const answer = (await rl.question(`${question} [y/N] `)).trim().toLowerCase();
|
|
934
|
+
return answer === "y" || answer === "yes";
|
|
935
|
+
} finally {
|
|
936
|
+
rl.close();
|
|
937
|
+
}
|
|
938
|
+
}
|
|
939
|
+
function requireKs(ks, name) {
|
|
940
|
+
if (!ks) throw new Error(`knowledge source '${name}' not found`);
|
|
941
|
+
return ks;
|
|
942
|
+
}
|
|
943
|
+
function parseKeyValue(pairs) {
|
|
944
|
+
const out = {};
|
|
945
|
+
for (const pair of pairs) {
|
|
946
|
+
const idx = pair.indexOf("=");
|
|
947
|
+
if (idx <= 0) throw new Error(`invalid metadata entry '${pair}', expected key=value`);
|
|
948
|
+
out[pair.slice(0, idx)] = pair.slice(idx + 1);
|
|
949
|
+
}
|
|
950
|
+
return out;
|
|
951
|
+
}
|
|
952
|
+
function parseTags(tags) {
|
|
953
|
+
if (!tags) return void 0;
|
|
954
|
+
const list = tags.split(",").map((t) => t.trim()).filter(Boolean);
|
|
955
|
+
return list.length > 0 ? list : void 0;
|
|
956
|
+
}
|
|
957
|
+
var program = new Command();
|
|
958
|
+
program.name("gresmcp").description("Manage Postgres-backed knowledge sources for the gresmcp MCP server").version(VERSION);
|
|
959
|
+
program.command("init").description("Initialize the database schema (extension, ks table; repairs entry tables)").action(
|
|
960
|
+
() => run(async () => {
|
|
961
|
+
await initDb();
|
|
962
|
+
for (const ks of await listKs()) {
|
|
963
|
+
const { ensureEntryTable: ensureEntryTable2 } = await Promise.resolve().then(() => (init_db(), db_exports));
|
|
964
|
+
await ensureEntryTable2(ks);
|
|
965
|
+
}
|
|
966
|
+
console.log("database ready");
|
|
967
|
+
})
|
|
968
|
+
);
|
|
969
|
+
program.command("check").description("Verify environment requirements (Node, Postgres, pgvector, schema, Ollama models)").option("--model <model>", "Verify this Ollama model is available (e.g. nomic-embed-text)").option("--url <url>", "Ollama base URL (default http://localhost:11434)").option("--probe", "Embed a test string with each model to verify dimensions (slower)").option("--json", "Output as JSON").action(
|
|
970
|
+
(opts) => run(async () => {
|
|
971
|
+
const results = await runChecks({ model: opts.model, url: opts.url, probe: opts.probe });
|
|
972
|
+
if (opts.json) {
|
|
973
|
+
console.log(JSON.stringify(results, null, 2));
|
|
974
|
+
} else {
|
|
975
|
+
const symbols = { ok: "\u2714", warn: "\u26A0", fail: "\u2718", skip: "-" };
|
|
976
|
+
for (const r of results) console.log(`${symbols[r.status]} ${r.detail}`);
|
|
977
|
+
const failed = results.filter((r) => r.status === "fail").length;
|
|
978
|
+
const warned = results.filter((r) => r.status === "warn").length;
|
|
979
|
+
if (failed > 0) {
|
|
980
|
+
console.log(`${failed} check(s) failed${warned > 0 ? `, ${warned} warning(s)` : ""}`);
|
|
981
|
+
} else if (warned > 0) {
|
|
982
|
+
console.log(`all checks passed with ${warned} warning(s)`);
|
|
983
|
+
} else {
|
|
984
|
+
console.log("all checks passed");
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
if (results.some((r) => r.status === "fail")) process.exitCode = 1;
|
|
988
|
+
})
|
|
989
|
+
);
|
|
990
|
+
var ksCmd = program.command("ks").description("Manage knowledge sources");
|
|
991
|
+
ksCmd.command("create <name>").description("Create a knowledge source (probes Ollama to validate the model and detect dimensions)").requiredOption("--model <model>", "Ollama embedding model, e.g. nomic-embed-text").option("--url <url>", "Ollama base URL (default http://localhost:11434)").option("--description <description>", "Human-readable description").option("--dim <n>", "Skip the Ollama probe and set the embedding dimension explicitly", parseInt).action(
|
|
992
|
+
(name, opts) => run(async () => {
|
|
993
|
+
await initDb();
|
|
994
|
+
if (await getKs(name)) throw new Error(`knowledge source '${name}' already exists`);
|
|
995
|
+
const url = ollamaUrl(opts.url);
|
|
996
|
+
let dim = opts.dim;
|
|
997
|
+
if (dim === void 0 || !Number.isFinite(dim) || dim <= 0) {
|
|
998
|
+
console.log(`Probing model ${opts.model} at ${url} ...`);
|
|
999
|
+
dim = (await probeModel(url, opts.model)).dim;
|
|
1000
|
+
console.log(`Model OK, embedding dimension: ${dim}`);
|
|
1001
|
+
}
|
|
1002
|
+
const ks = await createKs({
|
|
1003
|
+
name,
|
|
1004
|
+
description: opts.description,
|
|
1005
|
+
model: opts.model,
|
|
1006
|
+
dim,
|
|
1007
|
+
url
|
|
1008
|
+
});
|
|
1009
|
+
console.log(`Created knowledge source '${ks.name}' (dim=${ks.embedding_dim}, model=${ks.embedding_model}, url=${ks.ollama_url})`);
|
|
1010
|
+
})
|
|
1011
|
+
);
|
|
1012
|
+
ksCmd.command("edit <name>").description("Edit a knowledge source (the embedding model is immutable)").option("--url <url>", "New Ollama base URL").option("--rename <new-name>", "Rename the knowledge source").option("--description <description>", "New description").action(
|
|
1013
|
+
(name, opts) => run(async () => {
|
|
1014
|
+
await initDb();
|
|
1015
|
+
const ks = requireKs(await getKs(name), name);
|
|
1016
|
+
if (opts.rename && opts.rename !== name && await getKs(opts.rename)) {
|
|
1017
|
+
throw new Error(`knowledge source '${opts.rename}' already exists`);
|
|
1018
|
+
}
|
|
1019
|
+
const updated = await updateKs(ks.id, {
|
|
1020
|
+
name: opts.rename,
|
|
1021
|
+
description: opts.description,
|
|
1022
|
+
ollama_url: opts.url ? ollamaUrl(opts.url) : void 0
|
|
1023
|
+
});
|
|
1024
|
+
console.log(`Updated '${updated.name}' (model=${updated.embedding_model}, url=${updated.ollama_url}, dim=${updated.embedding_dim})`);
|
|
1025
|
+
})
|
|
1026
|
+
);
|
|
1027
|
+
ksCmd.command("list").description("List knowledge sources").option("--json", "Output as JSON").action(
|
|
1028
|
+
(opts) => run(async () => {
|
|
1029
|
+
await initDb();
|
|
1030
|
+
const all = await listKs();
|
|
1031
|
+
if (opts.json) {
|
|
1032
|
+
console.log(JSON.stringify(all, null, 2));
|
|
1033
|
+
return;
|
|
1034
|
+
}
|
|
1035
|
+
if (all.length === 0) {
|
|
1036
|
+
console.log("no knowledge sources yet; create one with: gresmcp ks create <name> --model <model>");
|
|
1037
|
+
return;
|
|
1038
|
+
}
|
|
1039
|
+
for (const ks of all) {
|
|
1040
|
+
const n = await countEntries(ks);
|
|
1041
|
+
console.log(
|
|
1042
|
+
`${ks.name.padEnd(20)} ${`${ks.embedding_model}@${ks.ollama_url}`.padEnd(50)} dim=${String(ks.embedding_dim).padEnd(6)} entries=${String(n).padEnd(8)} ${ks.description}`
|
|
1043
|
+
);
|
|
1044
|
+
}
|
|
1045
|
+
})
|
|
1046
|
+
);
|
|
1047
|
+
ksCmd.command("delete <name>").description("Delete a knowledge source and all of its entries").option("--yes", "Do not ask for confirmation").action(
|
|
1048
|
+
(name, opts) => run(async () => {
|
|
1049
|
+
await initDb();
|
|
1050
|
+
const ks = requireKs(await getKs(name), name);
|
|
1051
|
+
if (!opts.yes) {
|
|
1052
|
+
const n = await countEntries(ks);
|
|
1053
|
+
const ok = await confirm(`Delete knowledge source '${name}' and its ${n} entry chunk(s)?`);
|
|
1054
|
+
if (!ok) {
|
|
1055
|
+
console.log("aborted");
|
|
1056
|
+
return;
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
await deleteKs(ks);
|
|
1060
|
+
console.log(`Deleted knowledge source '${name}'`);
|
|
1061
|
+
})
|
|
1062
|
+
);
|
|
1063
|
+
program.command("feed <ks>").description("Feed data into a knowledge source (from a file/folder via --path, or inline via --text/--stdin)").option("--path <path>", "File or folder to ingest (text, markdown, code, html)").option("--text <text>", "Inline text entry").option("--stdin", "Read entry text from stdin").option("--title <title>", "Title for manual entries").option("--source-name <name>", "Source name for manual entries (default 'manual')").option("--tags <tags>", "Comma-separated tags").option("--metadata <key=value...>", "Custom metadata entries", ((v, prev) => [...prev, v]), []).option("--url <url>", "Ollama base URL override for this run").option("--replace", "Replace existing chunks of the same source instead of deduplicating").option("--dry-run", "Parse and chunk only; do not embed or write").option("--chunk-size <n>", "Max chunk length in characters", parseInt).option("--overlap <n>", "Chunk overlap in characters", parseInt).action(
|
|
1064
|
+
(name, opts) => run(async () => {
|
|
1065
|
+
await initDb();
|
|
1066
|
+
const ks = requireKs(await getKs(name), name);
|
|
1067
|
+
const modes = [opts.path, opts.text, opts.stdin].filter(Boolean).length;
|
|
1068
|
+
if (modes !== 1) throw new Error("provide exactly one of --path, --text or --stdin");
|
|
1069
|
+
const common = {
|
|
1070
|
+
title: opts.title,
|
|
1071
|
+
url: opts.url,
|
|
1072
|
+
replace: opts.replace,
|
|
1073
|
+
dryRun: opts.dryRun,
|
|
1074
|
+
batchSize: 32,
|
|
1075
|
+
maxLen: opts.chunkSize && opts.chunkSize > 0 ? opts.chunkSize : void 0,
|
|
1076
|
+
overlap: opts.overlap,
|
|
1077
|
+
tags: parseTags(opts.tags),
|
|
1078
|
+
metadata: parseKeyValue(opts.metadata ?? [])
|
|
1079
|
+
};
|
|
1080
|
+
let summary;
|
|
1081
|
+
if (opts.path) {
|
|
1082
|
+
summary = await feedTarget(ks, opts.path, common);
|
|
1083
|
+
} else {
|
|
1084
|
+
const text = opts.stdin ? await readStdin() : opts.text;
|
|
1085
|
+
if (!text.trim()) throw new Error("empty input text");
|
|
1086
|
+
summary = await feedText(ks, text, { ...common, sourceName: opts.sourceName });
|
|
1087
|
+
}
|
|
1088
|
+
if (summary.dryRun) {
|
|
1089
|
+
console.log(`dry run: ${summary.documents} document(s), ${summary.chunks} chunk(s), ${summary.skippedFiles.length} file(s) skipped`);
|
|
1090
|
+
} else {
|
|
1091
|
+
console.log(
|
|
1092
|
+
`fed '${ks.name}': ${summary.inserted} chunk(s) inserted` + (summary.skippedExisting ? `, ${summary.skippedExisting} unchanged chunk(s) skipped` : "") + (summary.deletedForReplace ? `, ${summary.deletedForReplace} old chunk(s) replaced` : "") + `, ${summary.skippedFiles.length} file(s) skipped`
|
|
1093
|
+
);
|
|
1094
|
+
}
|
|
1095
|
+
})
|
|
1096
|
+
);
|
|
1097
|
+
async function readStdin() {
|
|
1098
|
+
const chunks = [];
|
|
1099
|
+
for await (const chunk of stdin) chunks.push(chunk);
|
|
1100
|
+
return Buffer.concat(chunks).toString("utf8");
|
|
1101
|
+
}
|
|
1102
|
+
program.parseAsync(process.argv);
|
|
1103
|
+
//# sourceMappingURL=cli.js.map
|