@farming-labs/docs 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.mjs +52 -4
- package/dist/config-CSywk3ou.mjs +95 -0
- package/dist/index.d.mts +3 -2
- package/dist/index.mjs +4 -1
- package/dist/{init-N0bZQFRd.mjs → init-C7kgy5hD.mjs} +1 -1
- package/dist/mcp-aXyV1jPp.mjs +46 -0
- package/dist/mcp.d.mts +2 -1
- package/dist/mcp.mjs +35 -33
- package/dist/search-BS6C5N1i.mjs +671 -0
- package/dist/search-ChhShKMO.mjs +99 -0
- package/dist/search-KzREATdM.d.mts +21 -0
- package/dist/server.d.mts +3 -2
- package/dist/server.mjs +3 -2
- package/dist/{types-dqnMXLdw.d.mts → types-BAulrjlV.d.mts} +149 -1
- package/dist/{upgrade-BbEyR_JB.mjs → upgrade-J_kkv-ti.mjs} +1 -1
- package/package.json +1 -1
- package/dist/mcp-8rCBy2-U.mjs +0 -93
- /package/dist/{api-reference-wh4_pwG8.mjs → api-reference-DlfH-Y9c.mjs} +0 -0
- /package/dist/{utils-D5Wn7Q5E.mjs → utils-CRhME2g-.mjs} +0 -0
|
@@ -0,0 +1,671 @@
|
|
|
1
|
+
//#region src/search.ts
|
|
2
|
+
const DEFAULT_SEARCH_LIMIT = 10;
|
|
3
|
+
const DEFAULT_MCP_PROTOCOL_VERSION = "2025-11-25";
|
|
4
|
+
const syncedIndexes = /* @__PURE__ */ new Set();
|
|
5
|
+
const ALGOLIA_MAX_RECORD_BYTES = 9500;
|
|
6
|
+
function stripMarkdownText(content) {
|
|
7
|
+
return content.replace(/```[\s\S]*?```/g, "").replace(/~~~[\s\S]*?~~~/g, "").replace(/^(import|export)\s.*$/gm, "").replace(/<[^>]+\/>/g, "").replace(/<\/?[A-Z][^>]*>/g, "").replace(/<\/?[a-z][^>]*>/g, "").replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1").replace(/\[([^\]]+)\]\([^)]+\)/g, "$1").replace(/^#{1,6}\s+/gm, "").replace(/^\|?[\s:-]+(\|[\s:-]+)+\|?\s*$/gm, "").replace(/\|/g, " ").replace(/^[-*+]\s+/gm, "").replace(/(\*{1,3}|_{1,3})(.*?)\1/g, "$2").replace(/`{3,}[^\n]*$/gm, "").replace(/`([^`]+)`/g, "$1").replace(/`+/g, "").replace(/^>\s+/gm, "").replace(/^[-*_]{3,}\s*$/gm, "").replace(/\n{3,}/g, "\n\n").replace(/\s{2,}/g, " ").trim();
|
|
8
|
+
}
|
|
9
|
+
function stripHtml(text) {
|
|
10
|
+
return text.replace(/<[^>]+>/g, "");
|
|
11
|
+
}
|
|
12
|
+
function normalizeMcpSsePayload(body) {
|
|
13
|
+
const payload = body.split("\n").filter((line) => line.startsWith("data: ")).map((line) => line.slice(6).trim()).filter(Boolean).at(-1);
|
|
14
|
+
return payload ? JSON.parse(payload) : null;
|
|
15
|
+
}
|
|
16
|
+
function normalizeWhitespace(value) {
|
|
17
|
+
return value.replace(/\s+/g, " ").trim();
|
|
18
|
+
}
|
|
19
|
+
function makeDocumentId(url, suffix) {
|
|
20
|
+
return `${url}#${suffix}`;
|
|
21
|
+
}
|
|
22
|
+
function slugifyHeading(text) {
|
|
23
|
+
return text.trim().toLowerCase().replace(/[`'"‘’“”]/g, "").replace(/&/g, " and ").replace(/[^a-z0-9\s-]/g, "").replace(/\s+/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "");
|
|
24
|
+
}
|
|
25
|
+
function splitPageIntoSections(page) {
|
|
26
|
+
const lines = (page.rawContent ?? page.content).split("\n");
|
|
27
|
+
const sections = [];
|
|
28
|
+
const headingCounts = /* @__PURE__ */ new Map();
|
|
29
|
+
let currentHeading = "";
|
|
30
|
+
let currentLines = [];
|
|
31
|
+
let index = 0;
|
|
32
|
+
function flush() {
|
|
33
|
+
const content = normalizeWhitespace(stripMarkdownText(currentLines.join("\n").trim()));
|
|
34
|
+
if (!content) return;
|
|
35
|
+
let url = page.url;
|
|
36
|
+
if (currentHeading) {
|
|
37
|
+
const baseSlug = slugifyHeading(currentHeading) || `section-${index}`;
|
|
38
|
+
const seen = headingCounts.get(baseSlug) ?? 0;
|
|
39
|
+
headingCounts.set(baseSlug, seen + 1);
|
|
40
|
+
const slug = seen === 0 ? baseSlug : `${baseSlug}-${seen}`;
|
|
41
|
+
url = `${page.url}#${slug}`;
|
|
42
|
+
}
|
|
43
|
+
sections.push({
|
|
44
|
+
id: makeDocumentId(page.url, currentHeading ? `section-${index}` : "page"),
|
|
45
|
+
url,
|
|
46
|
+
title: page.title,
|
|
47
|
+
section: currentHeading || void 0,
|
|
48
|
+
content,
|
|
49
|
+
description: page.description,
|
|
50
|
+
type: currentHeading ? "heading" : "page",
|
|
51
|
+
locale: page.locale,
|
|
52
|
+
framework: page.framework,
|
|
53
|
+
version: page.version,
|
|
54
|
+
tags: page.tags
|
|
55
|
+
});
|
|
56
|
+
index += 1;
|
|
57
|
+
}
|
|
58
|
+
for (const line of lines) {
|
|
59
|
+
const headingMatch = line.match(/^#{1,6}\s+(.+)$/);
|
|
60
|
+
if (headingMatch) {
|
|
61
|
+
flush();
|
|
62
|
+
currentHeading = normalizeWhitespace(headingMatch[1].replace(/#+$/g, ""));
|
|
63
|
+
currentLines = [];
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
currentLines.push(line);
|
|
67
|
+
}
|
|
68
|
+
flush();
|
|
69
|
+
return sections;
|
|
70
|
+
}
|
|
71
|
+
function buildDocsSearchDocuments(pages, chunking = {}) {
|
|
72
|
+
const strategy = chunking.strategy ?? "section";
|
|
73
|
+
return pages.flatMap((page) => {
|
|
74
|
+
const base = {
|
|
75
|
+
id: makeDocumentId(page.url, "page"),
|
|
76
|
+
url: page.url,
|
|
77
|
+
title: page.title,
|
|
78
|
+
content: normalizeWhitespace(page.content),
|
|
79
|
+
description: page.description,
|
|
80
|
+
type: "page",
|
|
81
|
+
locale: page.locale,
|
|
82
|
+
framework: page.framework,
|
|
83
|
+
version: page.version,
|
|
84
|
+
tags: page.tags
|
|
85
|
+
};
|
|
86
|
+
if (strategy === "page") return [base];
|
|
87
|
+
const sections = splitPageIntoSections(page);
|
|
88
|
+
if (sections.length === 0) return [base];
|
|
89
|
+
return [...base.content ? [base] : [], ...sections];
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
function scoreDocument(query, document) {
|
|
93
|
+
const q = query.toLowerCase().trim();
|
|
94
|
+
if (!q) return 0;
|
|
95
|
+
const words = Array.from(new Set(q.split(/\s+/).filter(Boolean)));
|
|
96
|
+
const title = document.title.toLowerCase();
|
|
97
|
+
const section = document.section?.toLowerCase() ?? "";
|
|
98
|
+
const description = document.description?.toLowerCase() ?? "";
|
|
99
|
+
const content = document.content.toLowerCase();
|
|
100
|
+
const url = document.url.toLowerCase();
|
|
101
|
+
let score = 0;
|
|
102
|
+
if (title === q) score += 120;
|
|
103
|
+
else if (title.startsWith(q)) score += 70;
|
|
104
|
+
else if (title.includes(q)) score += 45;
|
|
105
|
+
if (section === q) score += 80;
|
|
106
|
+
else if (section.startsWith(q)) score += 55;
|
|
107
|
+
else if (section.includes(q)) score += 30;
|
|
108
|
+
if (url.includes(q)) score += 12;
|
|
109
|
+
if (description.includes(q)) score += 18;
|
|
110
|
+
if (content.includes(q)) score += 12;
|
|
111
|
+
let matchedWords = 0;
|
|
112
|
+
for (const word of words) {
|
|
113
|
+
let matched = false;
|
|
114
|
+
if (title === word) {
|
|
115
|
+
score += 28;
|
|
116
|
+
matched = true;
|
|
117
|
+
} else if (title.startsWith(word)) {
|
|
118
|
+
score += 20;
|
|
119
|
+
matched = true;
|
|
120
|
+
} else if (title.includes(word)) {
|
|
121
|
+
score += 12;
|
|
122
|
+
matched = true;
|
|
123
|
+
}
|
|
124
|
+
if (section === word) {
|
|
125
|
+
score += 22;
|
|
126
|
+
matched = true;
|
|
127
|
+
} else if (section.startsWith(word)) {
|
|
128
|
+
score += 16;
|
|
129
|
+
matched = true;
|
|
130
|
+
} else if (section.includes(word)) {
|
|
131
|
+
score += 10;
|
|
132
|
+
matched = true;
|
|
133
|
+
}
|
|
134
|
+
if (description.includes(word)) {
|
|
135
|
+
score += 6;
|
|
136
|
+
matched = true;
|
|
137
|
+
}
|
|
138
|
+
if (content.includes(word)) {
|
|
139
|
+
score += 4;
|
|
140
|
+
matched = true;
|
|
141
|
+
}
|
|
142
|
+
if (matched) matchedWords += 1;
|
|
143
|
+
}
|
|
144
|
+
if (matchedWords === words.length && words.length > 1) score += 20;
|
|
145
|
+
if (document.type === "heading") score += 6;
|
|
146
|
+
return score;
|
|
147
|
+
}
|
|
148
|
+
function buildSnippet(document, query) {
|
|
149
|
+
const q = query.trim().toLowerCase();
|
|
150
|
+
const sources = [normalizeWhitespace(stripMarkdownText(document.content)), normalizeWhitespace(stripMarkdownText(document.description ?? ""))].filter(Boolean);
|
|
151
|
+
for (const source of sources) {
|
|
152
|
+
if (!q) return source.slice(0, 160);
|
|
153
|
+
const idx = source.toLowerCase().indexOf(q);
|
|
154
|
+
if (idx === -1) continue;
|
|
155
|
+
const start = Math.max(0, idx - 48);
|
|
156
|
+
const end = Math.min(source.length, idx + q.length + 96);
|
|
157
|
+
const prefix = start > 0 ? "..." : "";
|
|
158
|
+
const suffix = end < source.length ? "..." : "";
|
|
159
|
+
return `${prefix}${source.slice(start, end).trim()}${suffix}`;
|
|
160
|
+
}
|
|
161
|
+
return sources[0]?.slice(0, 160);
|
|
162
|
+
}
|
|
163
|
+
function cleanSearchResultText(value) {
|
|
164
|
+
if (!value) return void 0;
|
|
165
|
+
return normalizeWhitespace(stripHtml(stripMarkdownText(value))) || void 0;
|
|
166
|
+
}
|
|
167
|
+
function trimTextToBytes(value, maxBytes) {
|
|
168
|
+
if (maxBytes <= 0) return "";
|
|
169
|
+
const encoder = new TextEncoder();
|
|
170
|
+
if (encoder.encode(value).length <= maxBytes) return value;
|
|
171
|
+
let low = 0;
|
|
172
|
+
let high = value.length;
|
|
173
|
+
let best = "";
|
|
174
|
+
while (low <= high) {
|
|
175
|
+
const mid = Math.floor((low + high) / 2);
|
|
176
|
+
const next = `${value.slice(0, mid).trimEnd()}...`;
|
|
177
|
+
if (encoder.encode(next).length <= maxBytes) {
|
|
178
|
+
best = next;
|
|
179
|
+
low = mid + 1;
|
|
180
|
+
} else high = mid - 1;
|
|
181
|
+
}
|
|
182
|
+
return best;
|
|
183
|
+
}
|
|
184
|
+
function buildAlgoliaRecord(document) {
|
|
185
|
+
const record = {
|
|
186
|
+
objectID: document.id,
|
|
187
|
+
id: document.id,
|
|
188
|
+
url: document.url,
|
|
189
|
+
title: document.title,
|
|
190
|
+
section: document.section,
|
|
191
|
+
content: document.content,
|
|
192
|
+
description: document.description,
|
|
193
|
+
type: document.type
|
|
194
|
+
};
|
|
195
|
+
const encoder = new TextEncoder();
|
|
196
|
+
const sizeOf = (value) => encoder.encode(JSON.stringify(value)).length;
|
|
197
|
+
if (sizeOf(record) <= ALGOLIA_MAX_RECORD_BYTES) return record;
|
|
198
|
+
delete record.description;
|
|
199
|
+
if (sizeOf(record) <= ALGOLIA_MAX_RECORD_BYTES) return record;
|
|
200
|
+
const fixedBytes = sizeOf({
|
|
201
|
+
...record,
|
|
202
|
+
content: ""
|
|
203
|
+
});
|
|
204
|
+
const remainingBytes = Math.max(ALGOLIA_MAX_RECORD_BYTES - fixedBytes - 32, 0);
|
|
205
|
+
record.content = trimTextToBytes(document.content, remainingBytes);
|
|
206
|
+
return record;
|
|
207
|
+
}
|
|
208
|
+
function createSimpleSearchAdapter() {
|
|
209
|
+
return {
|
|
210
|
+
name: "simple",
|
|
211
|
+
async search(query, context) {
|
|
212
|
+
const limit = query.limit ?? DEFAULT_SEARCH_LIMIT;
|
|
213
|
+
return context.documents.map((document) => ({
|
|
214
|
+
document,
|
|
215
|
+
score: scoreDocument(query.query, document)
|
|
216
|
+
})).filter((item) => item.score > 0).sort((a, b) => {
|
|
217
|
+
if (b.score !== a.score) return b.score - a.score;
|
|
218
|
+
return a.document.url.localeCompare(b.document.url);
|
|
219
|
+
}).slice(0, limit).map(({ document, score }) => ({
|
|
220
|
+
id: document.id,
|
|
221
|
+
url: document.url,
|
|
222
|
+
content: cleanSearchResultText(document.section ? `${document.title} — ${document.section}` : document.title) ?? (document.section ? `${document.title} — ${document.section}` : document.title),
|
|
223
|
+
description: cleanSearchResultText(buildSnippet(document, query.query) ?? document.description),
|
|
224
|
+
type: document.type,
|
|
225
|
+
score,
|
|
226
|
+
section: document.section
|
|
227
|
+
}));
|
|
228
|
+
}
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
function normalizeDocsSearchConfig(search) {
|
|
232
|
+
if (search === false) return {
|
|
233
|
+
enabled: false,
|
|
234
|
+
provider: "simple",
|
|
235
|
+
maxResults: DEFAULT_SEARCH_LIMIT,
|
|
236
|
+
chunking: { strategy: "section" }
|
|
237
|
+
};
|
|
238
|
+
if (!search || search === true) return {
|
|
239
|
+
enabled: true,
|
|
240
|
+
provider: "simple",
|
|
241
|
+
maxResults: DEFAULT_SEARCH_LIMIT,
|
|
242
|
+
chunking: { strategy: "section" },
|
|
243
|
+
raw: typeof search === "object" ? search : void 0
|
|
244
|
+
};
|
|
245
|
+
const provider = search.provider ?? "simple";
|
|
246
|
+
const maxResults = search.maxResults ?? DEFAULT_SEARCH_LIMIT;
|
|
247
|
+
const chunking = search.chunking ?? { strategy: "section" };
|
|
248
|
+
return {
|
|
249
|
+
enabled: search.enabled ?? true,
|
|
250
|
+
provider,
|
|
251
|
+
maxResults,
|
|
252
|
+
chunking,
|
|
253
|
+
raw: search
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
async function readResponseJson(response) {
|
|
257
|
+
const text = await response.text();
|
|
258
|
+
return text ? JSON.parse(text) : null;
|
|
259
|
+
}
|
|
260
|
+
async function readMcpResponsePayload(response) {
|
|
261
|
+
const text = await response.text();
|
|
262
|
+
if (!text) return null;
|
|
263
|
+
if ((response.headers.get("content-type") ?? "").includes("application/json")) return JSON.parse(text);
|
|
264
|
+
return normalizeMcpSsePayload(text);
|
|
265
|
+
}
|
|
266
|
+
function ensureOk(response, message) {
|
|
267
|
+
if (response.ok) return;
|
|
268
|
+
throw new Error(`${message} (${response.status} ${response.statusText})`);
|
|
269
|
+
}
|
|
270
|
+
function ensureJsonRpcOk(payload, message) {
|
|
271
|
+
if (payload && typeof payload === "object" && "error" in payload && payload.error && typeof payload.error === "object" && "message" in payload.error) throw new Error(`${message}: ${String(payload.error.message)}`);
|
|
272
|
+
}
|
|
273
|
+
function resolveMcpEndpoint(endpoint) {
|
|
274
|
+
if (/^https?:\/\//i.test(endpoint)) return endpoint;
|
|
275
|
+
throw new Error("Relative MCP search endpoints must be resolved before creating the MCP adapter.");
|
|
276
|
+
}
|
|
277
|
+
function isDocsSearchResultType(value) {
|
|
278
|
+
return value === "page" || value === "heading" || value === "text";
|
|
279
|
+
}
|
|
280
|
+
function mapMcpSearchResult(value) {
|
|
281
|
+
if (!value || typeof value !== "object") return null;
|
|
282
|
+
const item = value;
|
|
283
|
+
const section = typeof item.section === "string" ? item.section : void 0;
|
|
284
|
+
const title = typeof item.title === "string" ? item.title : void 0;
|
|
285
|
+
const content = typeof item.content === "string" ? item.content : title ? section ? `${title} — ${section}` : title : void 0;
|
|
286
|
+
const url = typeof item.url === "string" ? item.url : void 0;
|
|
287
|
+
if (!content || !url) return null;
|
|
288
|
+
return {
|
|
289
|
+
id: typeof item.id === "string" ? item.id : typeof item.slug === "string" ? item.slug : url,
|
|
290
|
+
url,
|
|
291
|
+
content: cleanSearchResultText(content) ?? content,
|
|
292
|
+
description: cleanSearchResultText(typeof item.description === "string" ? item.description : typeof item.excerpt === "string" ? item.excerpt : void 0) ?? void 0,
|
|
293
|
+
type: isDocsSearchResultType(item.type) ? item.type : section ? "heading" : "page",
|
|
294
|
+
score: typeof item.score === "number" ? item.score : void 0,
|
|
295
|
+
section
|
|
296
|
+
};
|
|
297
|
+
}
|
|
298
|
+
async function createOllamaEmbedding(text, config) {
|
|
299
|
+
const response = await fetch(`${(config.baseUrl ?? "http://127.0.0.1:11434").replace(/\/$/, "")}/api/embed`, {
|
|
300
|
+
method: "POST",
|
|
301
|
+
headers: { "Content-Type": "application/json" },
|
|
302
|
+
body: JSON.stringify({
|
|
303
|
+
model: config.model,
|
|
304
|
+
input: text
|
|
305
|
+
})
|
|
306
|
+
});
|
|
307
|
+
ensureOk(response, "Failed to create Ollama embedding");
|
|
308
|
+
const payload = await readResponseJson(response);
|
|
309
|
+
if (Array.isArray(payload.embeddings?.[0])) return payload.embeddings[0];
|
|
310
|
+
if (Array.isArray(payload.embedding)) return payload.embedding;
|
|
311
|
+
throw new Error("Ollama embedding response did not include an embedding vector.");
|
|
312
|
+
}
|
|
313
|
+
function getTypesenseSearchBase(config) {
|
|
314
|
+
return config.baseUrl.replace(/\/$/, "");
|
|
315
|
+
}
|
|
316
|
+
async function ensureTypesenseCollection(config, dimensions) {
|
|
317
|
+
const baseUrl = getTypesenseSearchBase(config);
|
|
318
|
+
const headers = {
|
|
319
|
+
"X-TYPESENSE-API-KEY": config.adminApiKey ?? config.apiKey,
|
|
320
|
+
"Content-Type": "application/json"
|
|
321
|
+
};
|
|
322
|
+
const existing = await fetch(`${baseUrl}/collections/${encodeURIComponent(config.collection)}`, { headers });
|
|
323
|
+
if (existing.ok) return;
|
|
324
|
+
if (existing.status !== 404) ensureOk(existing, "Failed to inspect Typesense collection");
|
|
325
|
+
const fields = [
|
|
326
|
+
{
|
|
327
|
+
name: "id",
|
|
328
|
+
type: "string"
|
|
329
|
+
},
|
|
330
|
+
{
|
|
331
|
+
name: "url",
|
|
332
|
+
type: "string"
|
|
333
|
+
},
|
|
334
|
+
{
|
|
335
|
+
name: "title",
|
|
336
|
+
type: "string"
|
|
337
|
+
},
|
|
338
|
+
{
|
|
339
|
+
name: "section",
|
|
340
|
+
type: "string",
|
|
341
|
+
optional: true
|
|
342
|
+
},
|
|
343
|
+
{
|
|
344
|
+
name: "content",
|
|
345
|
+
type: "string"
|
|
346
|
+
},
|
|
347
|
+
{
|
|
348
|
+
name: "description",
|
|
349
|
+
type: "string",
|
|
350
|
+
optional: true
|
|
351
|
+
},
|
|
352
|
+
{
|
|
353
|
+
name: "type",
|
|
354
|
+
type: "string"
|
|
355
|
+
},
|
|
356
|
+
{
|
|
357
|
+
name: "locale",
|
|
358
|
+
type: "string",
|
|
359
|
+
optional: true
|
|
360
|
+
},
|
|
361
|
+
{
|
|
362
|
+
name: "framework",
|
|
363
|
+
type: "string",
|
|
364
|
+
optional: true
|
|
365
|
+
},
|
|
366
|
+
{
|
|
367
|
+
name: "version",
|
|
368
|
+
type: "string",
|
|
369
|
+
optional: true
|
|
370
|
+
},
|
|
371
|
+
{
|
|
372
|
+
name: "tags",
|
|
373
|
+
type: "string[]",
|
|
374
|
+
optional: true
|
|
375
|
+
}
|
|
376
|
+
];
|
|
377
|
+
if (config.embeddings && dimensions) fields.push({
|
|
378
|
+
name: "embedding",
|
|
379
|
+
type: "float[]",
|
|
380
|
+
num_dim: dimensions,
|
|
381
|
+
optional: true
|
|
382
|
+
});
|
|
383
|
+
ensureOk(await fetch(`${baseUrl}/collections`, {
|
|
384
|
+
method: "POST",
|
|
385
|
+
headers,
|
|
386
|
+
body: JSON.stringify({
|
|
387
|
+
name: config.collection,
|
|
388
|
+
fields
|
|
389
|
+
})
|
|
390
|
+
}), "Failed to create Typesense collection");
|
|
391
|
+
}
|
|
392
|
+
function createTypesenseSearchAdapter(config) {
|
|
393
|
+
return {
|
|
394
|
+
name: "typesense",
|
|
395
|
+
async index(context) {
|
|
396
|
+
const adminApiKey = config.adminApiKey ?? config.apiKey;
|
|
397
|
+
const docsForImport = await Promise.all(context.documents.map(async (document) => {
|
|
398
|
+
const next = {
|
|
399
|
+
id: document.id,
|
|
400
|
+
url: document.url,
|
|
401
|
+
title: document.title,
|
|
402
|
+
section: document.section,
|
|
403
|
+
content: document.content,
|
|
404
|
+
description: document.description,
|
|
405
|
+
type: document.type,
|
|
406
|
+
locale: document.locale,
|
|
407
|
+
framework: document.framework,
|
|
408
|
+
version: document.version,
|
|
409
|
+
tags: document.tags
|
|
410
|
+
};
|
|
411
|
+
if (config.mode === "hybrid" && config.embeddings) next.embedding = await createOllamaEmbedding(`${document.title}\n${document.section ?? ""}\n${document.content}`.trim(), config.embeddings);
|
|
412
|
+
return next;
|
|
413
|
+
}));
|
|
414
|
+
if (docsForImport.length === 0) return;
|
|
415
|
+
await ensureTypesenseCollection(config, Array.isArray(docsForImport[0]?.embedding) ? docsForImport[0].embedding.length : void 0);
|
|
416
|
+
ensureOk(await fetch(`${getTypesenseSearchBase(config)}/collections/${encodeURIComponent(config.collection)}/documents/import?action=upsert`, {
|
|
417
|
+
method: "POST",
|
|
418
|
+
headers: {
|
|
419
|
+
"X-TYPESENSE-API-KEY": adminApiKey,
|
|
420
|
+
"Content-Type": "text/plain"
|
|
421
|
+
},
|
|
422
|
+
body: docsForImport.map((document) => JSON.stringify(document)).join("\n")
|
|
423
|
+
}), "Failed to sync documents to Typesense");
|
|
424
|
+
},
|
|
425
|
+
async search(query, _context) {
|
|
426
|
+
const params = new URLSearchParams({
|
|
427
|
+
q: query.query,
|
|
428
|
+
query_by: (config.queryBy ?? [
|
|
429
|
+
"title",
|
|
430
|
+
"section",
|
|
431
|
+
"content",
|
|
432
|
+
"description"
|
|
433
|
+
]).join(","),
|
|
434
|
+
per_page: String(query.limit ?? config.maxResults ?? DEFAULT_SEARCH_LIMIT),
|
|
435
|
+
prioritize_exact_match: "true",
|
|
436
|
+
num_typos: "2",
|
|
437
|
+
highlight_fields: "content,title,section,description"
|
|
438
|
+
});
|
|
439
|
+
if (config.mode === "hybrid" && config.embeddings) {
|
|
440
|
+
const vector = await createOllamaEmbedding(query.query, config.embeddings);
|
|
441
|
+
params.set("vector_query", `embedding:([${vector.join(",")}],k:${Math.max((query.limit ?? 10) * 4, 20)})`);
|
|
442
|
+
}
|
|
443
|
+
const response = await fetch(`${getTypesenseSearchBase(config)}/collections/${encodeURIComponent(config.collection)}/documents/search?${params.toString()}`, { headers: { "X-TYPESENSE-API-KEY": config.apiKey } });
|
|
444
|
+
ensureOk(response, "Typesense search failed");
|
|
445
|
+
return ((await readResponseJson(response)).hits ?? []).map((hit) => {
|
|
446
|
+
const document = hit.document ?? {};
|
|
447
|
+
const section = typeof document.section === "string" ? document.section : void 0;
|
|
448
|
+
const content = typeof document.title === "string" ? section ? `${document.title} — ${section}` : document.title : typeof document.content === "string" ? document.content : "Untitled result";
|
|
449
|
+
const description = hit.highlights?.find((item) => item.field === "content")?.snippet ?? hit.highlights?.find((item) => item.field === "description")?.snippet ?? (typeof document.description === "string" ? document.description : void 0);
|
|
450
|
+
return {
|
|
451
|
+
id: typeof document.id === "string" ? document.id : String(document.url ?? content),
|
|
452
|
+
url: typeof document.url === "string" ? document.url : "/docs",
|
|
453
|
+
content: cleanSearchResultText(content) ?? content,
|
|
454
|
+
description: cleanSearchResultText(description),
|
|
455
|
+
type: typeof document.type === "string" && [
|
|
456
|
+
"page",
|
|
457
|
+
"heading",
|
|
458
|
+
"text"
|
|
459
|
+
].includes(document.type) ? document.type : section ? "heading" : "page",
|
|
460
|
+
score: hit.text_match,
|
|
461
|
+
section
|
|
462
|
+
};
|
|
463
|
+
});
|
|
464
|
+
}
|
|
465
|
+
};
|
|
466
|
+
}
|
|
467
|
+
function resolveSearchRequestConfig(search, requestUrl) {
|
|
468
|
+
if (!search || search === true || typeof search !== "object" || search.provider !== "mcp") return search;
|
|
469
|
+
if (/^https?:\/\//i.test(search.endpoint) || !requestUrl) return search;
|
|
470
|
+
return {
|
|
471
|
+
...search,
|
|
472
|
+
endpoint: new URL(search.endpoint, requestUrl).toString()
|
|
473
|
+
};
|
|
474
|
+
}
|
|
475
|
+
function createMcpSearchAdapter(config) {
|
|
476
|
+
return {
|
|
477
|
+
name: "mcp",
|
|
478
|
+
async search(query) {
|
|
479
|
+
const endpoint = resolveMcpEndpoint(config.endpoint);
|
|
480
|
+
const protocolVersion = config.protocolVersion ?? DEFAULT_MCP_PROTOCOL_VERSION;
|
|
481
|
+
const toolName = config.toolName ?? "search_docs";
|
|
482
|
+
const baseHeaders = config.headers ?? {};
|
|
483
|
+
const initializeResponse = await fetch(endpoint, {
|
|
484
|
+
method: "POST",
|
|
485
|
+
headers: {
|
|
486
|
+
...baseHeaders,
|
|
487
|
+
"Content-Type": "application/json",
|
|
488
|
+
accept: "application/json, text/event-stream",
|
|
489
|
+
"mcp-protocol-version": protocolVersion
|
|
490
|
+
},
|
|
491
|
+
body: JSON.stringify({
|
|
492
|
+
jsonrpc: "2.0",
|
|
493
|
+
id: 1,
|
|
494
|
+
method: "initialize",
|
|
495
|
+
params: {
|
|
496
|
+
protocolVersion,
|
|
497
|
+
capabilities: {},
|
|
498
|
+
clientInfo: {
|
|
499
|
+
name: "@farming-labs/docs-search",
|
|
500
|
+
version: "0.1.2"
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
})
|
|
504
|
+
});
|
|
505
|
+
const initializePayload = await readMcpResponsePayload(initializeResponse);
|
|
506
|
+
ensureOk(initializeResponse, "MCP search initialization failed");
|
|
507
|
+
ensureJsonRpcOk(initializePayload, "MCP search initialization failed");
|
|
508
|
+
const sessionId = initializeResponse.headers.get("mcp-session-id");
|
|
509
|
+
if (!sessionId) throw new Error("MCP search endpoint did not return an mcp-session-id header.");
|
|
510
|
+
try {
|
|
511
|
+
const searchResponse = await fetch(endpoint, {
|
|
512
|
+
method: "POST",
|
|
513
|
+
headers: {
|
|
514
|
+
...baseHeaders,
|
|
515
|
+
"Content-Type": "application/json",
|
|
516
|
+
accept: "application/json, text/event-stream",
|
|
517
|
+
"mcp-protocol-version": protocolVersion,
|
|
518
|
+
"mcp-session-id": sessionId
|
|
519
|
+
},
|
|
520
|
+
body: JSON.stringify({
|
|
521
|
+
jsonrpc: "2.0",
|
|
522
|
+
id: 2,
|
|
523
|
+
method: "tools/call",
|
|
524
|
+
params: {
|
|
525
|
+
name: toolName,
|
|
526
|
+
arguments: {
|
|
527
|
+
query: query.query,
|
|
528
|
+
limit: query.limit ?? config.maxResults ?? DEFAULT_SEARCH_LIMIT,
|
|
529
|
+
locale: query.locale
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
})
|
|
533
|
+
});
|
|
534
|
+
const payload = await readMcpResponsePayload(searchResponse);
|
|
535
|
+
ensureOk(searchResponse, "MCP search request failed");
|
|
536
|
+
ensureJsonRpcOk(payload, "MCP search request failed");
|
|
537
|
+
const resultText = payload && typeof payload === "object" && "result" in payload && payload.result && typeof payload.result === "object" && "content" in payload.result && Array.isArray(payload.result.content) && typeof payload.result.content[0]?.text === "string" ? payload.result.content[0].text : null;
|
|
538
|
+
if (!resultText) return [];
|
|
539
|
+
const parsed = JSON.parse(resultText);
|
|
540
|
+
return (Array.isArray(parsed) ? parsed : Array.isArray(parsed.results) ? parsed.results : Array.isArray(parsed.pages) ? parsed.pages : []).map(mapMcpSearchResult).filter((result) => Boolean(result));
|
|
541
|
+
} finally {
|
|
542
|
+
await fetch(endpoint, {
|
|
543
|
+
method: "DELETE",
|
|
544
|
+
headers: {
|
|
545
|
+
...baseHeaders,
|
|
546
|
+
"mcp-protocol-version": protocolVersion,
|
|
547
|
+
"mcp-session-id": sessionId
|
|
548
|
+
}
|
|
549
|
+
}).catch(() => void 0);
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
};
|
|
553
|
+
}
|
|
554
|
+
function getAlgoliaBase(config) {
|
|
555
|
+
return `https://${config.appId}-dsn.algolia.net`;
|
|
556
|
+
}
|
|
557
|
+
function createAlgoliaSearchAdapter(config) {
|
|
558
|
+
return {
|
|
559
|
+
name: "algolia",
|
|
560
|
+
async index(context) {
|
|
561
|
+
if (!config.adminApiKey) return;
|
|
562
|
+
ensureOk(await fetch(`${getAlgoliaBase(config)}/1/indexes/${encodeURIComponent(config.indexName)}/batch`, {
|
|
563
|
+
method: "POST",
|
|
564
|
+
headers: {
|
|
565
|
+
"Content-Type": "application/json",
|
|
566
|
+
"X-Algolia-Application-Id": config.appId,
|
|
567
|
+
"X-Algolia-API-Key": config.adminApiKey
|
|
568
|
+
},
|
|
569
|
+
body: JSON.stringify({ requests: context.documents.map((document) => ({
|
|
570
|
+
action: "addObject",
|
|
571
|
+
body: buildAlgoliaRecord(document)
|
|
572
|
+
})) })
|
|
573
|
+
}), "Failed to sync documents to Algolia");
|
|
574
|
+
},
|
|
575
|
+
async search(query) {
|
|
576
|
+
const response = await fetch(`${getAlgoliaBase(config)}/1/indexes/${encodeURIComponent(config.indexName)}/query`, {
|
|
577
|
+
method: "POST",
|
|
578
|
+
headers: {
|
|
579
|
+
"Content-Type": "application/json",
|
|
580
|
+
"X-Algolia-Application-Id": config.appId,
|
|
581
|
+
"X-Algolia-API-Key": config.searchApiKey
|
|
582
|
+
},
|
|
583
|
+
body: JSON.stringify({
|
|
584
|
+
query: query.query,
|
|
585
|
+
hitsPerPage: query.limit ?? config.maxResults ?? DEFAULT_SEARCH_LIMIT,
|
|
586
|
+
attributesToSnippet: ["content:20"]
|
|
587
|
+
})
|
|
588
|
+
});
|
|
589
|
+
ensureOk(response, "Algolia search failed");
|
|
590
|
+
return ((await readResponseJson(response)).hits ?? []).map((hit) => {
|
|
591
|
+
const title = typeof hit.title === "string" ? hit.title : "Untitled result";
|
|
592
|
+
const section = typeof hit.section === "string" ? hit.section : void 0;
|
|
593
|
+
return {
|
|
594
|
+
id: hit.objectID ?? String(hit.url ?? title),
|
|
595
|
+
url: typeof hit.url === "string" ? hit.url : "/docs",
|
|
596
|
+
content: cleanSearchResultText(section ? `${title} — ${section}` : title) ?? title,
|
|
597
|
+
description: cleanSearchResultText(hit._snippetResult?.content?.value ?? hit._snippetResult?.description?.value ?? (typeof hit.description === "string" ? hit.description : void 0)),
|
|
598
|
+
type: typeof hit.type === "string" && [
|
|
599
|
+
"page",
|
|
600
|
+
"heading",
|
|
601
|
+
"text"
|
|
602
|
+
].includes(hit.type) ? hit.type : section ? "heading" : "page",
|
|
603
|
+
score: hit._rankingInfo?.nbTypos != null ? 100 - hit._rankingInfo.nbTypos : void 0,
|
|
604
|
+
section
|
|
605
|
+
};
|
|
606
|
+
});
|
|
607
|
+
}
|
|
608
|
+
};
|
|
609
|
+
}
|
|
610
|
+
async function resolveSearchAdapter(search, context) {
|
|
611
|
+
const raw = search.raw;
|
|
612
|
+
if (search.provider === "custom" && raw?.provider === "custom") return typeof raw.adapter === "function" ? await raw.adapter(context) : raw.adapter;
|
|
613
|
+
if (search.provider === "typesense" && raw?.provider === "typesense") return createTypesenseSearchAdapter(raw);
|
|
614
|
+
if (search.provider === "mcp" && raw?.provider === "mcp") return createMcpSearchAdapter(raw);
|
|
615
|
+
if (search.provider === "algolia" && raw?.provider === "algolia") return createAlgoliaSearchAdapter(raw);
|
|
616
|
+
return createSimpleSearchAdapter();
|
|
617
|
+
}
|
|
618
|
+
function shouldSyncOnSearch(search) {
|
|
619
|
+
const raw = search.raw;
|
|
620
|
+
if (search.provider === "algolia" && raw?.provider === "algolia") return (raw.syncOnSearch ?? Boolean(raw.adminApiKey)) && Boolean(raw.adminApiKey);
|
|
621
|
+
if (search.provider === "typesense" && raw?.provider === "typesense") return (raw.syncOnSearch ?? Boolean(raw.adminApiKey)) && Boolean(raw.adminApiKey);
|
|
622
|
+
return false;
|
|
623
|
+
}
|
|
624
|
+
function getSyncKey(search, context) {
|
|
625
|
+
const raw = search.raw;
|
|
626
|
+
if (search.provider === "algolia" && raw?.provider === "algolia") return `algolia:${raw.appId}:${raw.indexName}:${context.locale ?? "__default__"}`;
|
|
627
|
+
if (search.provider === "typesense" && raw?.provider === "typesense") return `typesense:${raw.baseUrl}:${raw.collection}:${context.locale ?? "__default__"}`;
|
|
628
|
+
if (search.provider === "mcp" && raw?.provider === "mcp") return `mcp:${raw.endpoint}:${context.locale ?? "__default__"}`;
|
|
629
|
+
return `${search.provider}:${context.locale ?? "__default__"}`;
|
|
630
|
+
}
|
|
631
|
+
async function maybeSyncSearchIndex(adapter, search, context) {
|
|
632
|
+
if (!shouldSyncOnSearch(search) || typeof adapter.index !== "function") return;
|
|
633
|
+
const syncKey = getSyncKey(search, context);
|
|
634
|
+
if (syncedIndexes.has(syncKey)) return;
|
|
635
|
+
await adapter.index(context);
|
|
636
|
+
syncedIndexes.add(syncKey);
|
|
637
|
+
}
|
|
638
|
+
async function performDocsSearch(options) {
|
|
639
|
+
const search = normalizeDocsSearchConfig(options.search);
|
|
640
|
+
if (!search.enabled) return [];
|
|
641
|
+
const documents = buildDocsSearchDocuments(options.pages, search.chunking);
|
|
642
|
+
const context = {
|
|
643
|
+
pages: options.pages,
|
|
644
|
+
documents,
|
|
645
|
+
locale: options.locale,
|
|
646
|
+
pathname: options.pathname,
|
|
647
|
+
siteTitle: options.siteTitle
|
|
648
|
+
};
|
|
649
|
+
const query = {
|
|
650
|
+
query: options.query,
|
|
651
|
+
limit: options.limit ?? search.maxResults,
|
|
652
|
+
locale: options.locale,
|
|
653
|
+
pathname: options.pathname
|
|
654
|
+
};
|
|
655
|
+
try {
|
|
656
|
+
const adapter = await resolveSearchAdapter(search, context);
|
|
657
|
+
await maybeSyncSearchIndex(adapter, search, context);
|
|
658
|
+
return await adapter.search(query, context);
|
|
659
|
+
} catch {
|
|
660
|
+
return createSimpleSearchAdapter().search(query, context);
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
function createCustomSearchAdapter(adapter) {
|
|
664
|
+
return {
|
|
665
|
+
provider: "custom",
|
|
666
|
+
adapter
|
|
667
|
+
};
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
//#endregion
|
|
671
|
+
export { createSimpleSearchAdapter as a, resolveSearchRequestConfig as c, createMcpSearchAdapter as i, createAlgoliaSearchAdapter as n, createTypesenseSearchAdapter as o, createCustomSearchAdapter as r, performDocsSearch as s, buildDocsSearchDocuments as t };
|