@vercel/geistdocs 1.25.2 → 1.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli.js +8 -0
- package/dist/cli/index.js +6 -2
- package/dist/config.d.ts +5 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/internal/chat-retrieval.d.ts +32 -0
- package/dist/internal/chat-retrieval.d.ts.map +1 -0
- package/dist/internal/chat-retrieval.js +91 -0
- package/dist/internal/mixedbread.d.ts +93 -0
- package/dist/internal/mixedbread.d.ts.map +1 -0
- package/dist/internal/mixedbread.js +286 -0
- package/dist/internal/page-actions-client.d.ts.map +1 -1
- package/dist/internal/page-actions-client.js +1 -19
- package/dist/internal/search-export.d.ts +18 -0
- package/dist/internal/search-export.d.ts.map +1 -0
- package/dist/internal/search-export.js +1 -0
- package/dist/internal/search-sync.d.ts +25 -0
- package/dist/internal/search-sync.d.ts.map +1 -0
- package/dist/internal/search-sync.js +312 -0
- package/dist/routes/chat.d.ts +8 -2
- package/dist/routes/chat.d.ts.map +1 -1
- package/dist/routes/chat.js +53 -15
- package/dist/routes/search-export.d.ts +12 -0
- package/dist/routes/search-export.d.ts.map +1 -0
- package/dist/routes/search-export.js +44 -0
- package/docs/agents.md +4 -3
- package/docs/llms.txt +384 -33
- package/docs/manifest.json +12 -3
- package/docs/pages/ask-ai.md +3 -0
- package/docs/pages/configuration.md +5 -2
- package/docs/pages/deployment.md +30 -5
- package/docs/pages/env.md +13 -3
- package/docs/pages/getting-started.md +29 -9
- package/docs/pages/index.md +3 -2
- package/docs/pages/md.md +1 -1
- package/docs/pages/migration.md +148 -8
- package/docs/pages/mixedbread-retrieval.md +144 -0
- package/docs/sitemap.md +3 -2
- package/package.json +10 -4
- package/template/.env.example +19 -0
- package/template/AGENTS.md +17 -3
- package/template/app/api/search/export/route.ts +8 -0
- package/template/content/docs/ask-ai.mdx +3 -0
- package/template/content/docs/configuration.mdx +5 -2
- package/template/content/docs/deployment.mdx +31 -6
- package/template/content/docs/env.mdx +14 -4
- package/template/content/docs/getting-started.mdx +30 -10
- package/template/content/docs/index.mdx +3 -2
- package/template/content/docs/md.mdx +1 -1
- package/template/content/docs/meta.json +1 -0
- package/template/content/docs/migration.mdx +150 -9
- package/template/content/docs/mixedbread-retrieval.mdx +142 -0
- package/template/geistdocs.tsx +12 -1
- package/template/lib/geistdocs/config.tsx +2 -5
- package/template/package.json +2 -2
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readFile } from "node:fs/promises";
|
|
3
|
+
import { join, resolve, sep } from "node:path";
|
|
4
|
+
import { GEISTDOCS_MIXEDBREAD_SCHEMA_VERSION, mixedbreadMetadataKeys, readGeistdocsMixedbreadMetadata, } from "./mixedbread.js";
|
|
5
|
+
import { GEISTDOCS_SEARCH_EXPORT_VERSION, } from "./search-export.js";
|
|
6
|
+
const SEARCH_EXPORT_ROUTE = "/api/search/export";
|
|
7
|
+
const SEARCH_EXPORT_BODY_PATH = "server/app/api/search/export.body";
|
|
8
|
+
const DEFAULT_CONCURRENCY = 4;
|
|
9
|
+
const DEFAULT_POLL_INTERVAL_MS = 1000;
|
|
10
|
+
const DEFAULT_POLL_TIMEOUT_MS = 180_000;
|
|
11
|
+
const isRecord = (value) => typeof value === "object" && value !== null;
|
|
12
|
+
const isExportDocument = (value) => {
|
|
13
|
+
if (!isRecord(value)) {
|
|
14
|
+
return false;
|
|
15
|
+
}
|
|
16
|
+
return [
|
|
17
|
+
"content",
|
|
18
|
+
"description",
|
|
19
|
+
"id",
|
|
20
|
+
"locale",
|
|
21
|
+
"sourceId",
|
|
22
|
+
"sourceUrl",
|
|
23
|
+
"title",
|
|
24
|
+
"url",
|
|
25
|
+
].every((key) => typeof value[key] === "string");
|
|
26
|
+
};
|
|
27
|
+
export const parseSearchExport = (value) => {
|
|
28
|
+
if (!isRecord(value) ||
|
|
29
|
+
value.version !== GEISTDOCS_SEARCH_EXPORT_VERSION ||
|
|
30
|
+
(value.provider !== "local" && value.provider !== "mixedbread") ||
|
|
31
|
+
!Array.isArray(value.documents) ||
|
|
32
|
+
!value.documents.every(isExportDocument) ||
|
|
33
|
+
(value.siteId !== undefined && typeof value.siteId !== "string")) {
|
|
34
|
+
throw new Error("The Geistdocs search export is invalid or unsupported.");
|
|
35
|
+
}
|
|
36
|
+
const documentIds = new Set();
|
|
37
|
+
for (const document of value.documents) {
|
|
38
|
+
if (documentIds.has(document.id)) {
|
|
39
|
+
throw new Error(`The Geistdocs search export contains duplicate document ID "${document.id}".`);
|
|
40
|
+
}
|
|
41
|
+
documentIds.add(document.id);
|
|
42
|
+
}
|
|
43
|
+
return value;
|
|
44
|
+
};
|
|
45
|
+
export const loadSearchExport = async ({ cwd = process.cwd(), outputDirectory = ".next", } = {}) => {
|
|
46
|
+
const outputRoot = resolve(cwd, outputDirectory);
|
|
47
|
+
const prerenderManifestPath = join(outputRoot, "prerender-manifest.json");
|
|
48
|
+
const prerenderManifest = JSON.parse(await readFile(prerenderManifestPath, "utf8"));
|
|
49
|
+
const route = prerenderManifest.routes?.[SEARCH_EXPORT_ROUTE];
|
|
50
|
+
if (route?.routeType !== "route" ||
|
|
51
|
+
route.compute !== "static" ||
|
|
52
|
+
route.response !== "complete") {
|
|
53
|
+
throw new Error(`${SEARCH_EXPORT_ROUTE} was not statically generated by the latest Next.js build.`);
|
|
54
|
+
}
|
|
55
|
+
const appOutputRoot = resolve(outputRoot, "server/app");
|
|
56
|
+
const bodyPath = resolve(outputRoot, SEARCH_EXPORT_BODY_PATH);
|
|
57
|
+
if (!bodyPath.startsWith(`${appOutputRoot}${sep}`)) {
|
|
58
|
+
throw new Error("The Geistdocs search export path escaped the Next.js output.");
|
|
59
|
+
}
|
|
60
|
+
return parseSearchExport(JSON.parse(await readFile(bodyPath, "utf8")));
|
|
61
|
+
};
|
|
62
|
+
const hashValue = (value) => createHash("sha256").update(value).digest("hex");
|
|
63
|
+
const getDocumentHash = (document) => hashValue(JSON.stringify({
|
|
64
|
+
content: document.content,
|
|
65
|
+
description: document.description,
|
|
66
|
+
locale: document.locale,
|
|
67
|
+
sourceId: document.sourceId,
|
|
68
|
+
sourceUrl: document.sourceUrl,
|
|
69
|
+
title: document.title,
|
|
70
|
+
url: document.url,
|
|
71
|
+
}));
|
|
72
|
+
const createMetadata = ({ document, hash, revision, revisionAncestors, revisionTimestamp, siteId, }) => ({
|
|
73
|
+
[mixedbreadMetadataKeys.contentHash]: hash,
|
|
74
|
+
[mixedbreadMetadataKeys.description]: document.description,
|
|
75
|
+
[mixedbreadMetadataKeys.documentId]: document.id,
|
|
76
|
+
[mixedbreadMetadataKeys.locale]: document.locale,
|
|
77
|
+
...(revision ? { [mixedbreadMetadataKeys.revision]: revision } : {}),
|
|
78
|
+
...(revisionAncestors.length > 0
|
|
79
|
+
? {
|
|
80
|
+
[mixedbreadMetadataKeys.revisionAncestors]: revisionAncestors,
|
|
81
|
+
}
|
|
82
|
+
: {}),
|
|
83
|
+
...(revisionTimestamp === undefined
|
|
84
|
+
? {}
|
|
85
|
+
: { [mixedbreadMetadataKeys.revisionTimestamp]: revisionTimestamp }),
|
|
86
|
+
[mixedbreadMetadataKeys.schemaVersion]: GEISTDOCS_MIXEDBREAD_SCHEMA_VERSION,
|
|
87
|
+
[mixedbreadMetadataKeys.siteId]: siteId,
|
|
88
|
+
[mixedbreadMetadataKeys.sourceId]: document.sourceId,
|
|
89
|
+
[mixedbreadMetadataKeys.sourceUrl]: document.sourceUrl,
|
|
90
|
+
[mixedbreadMetadataKeys.title]: document.title,
|
|
91
|
+
[mixedbreadMetadataKeys.url]: document.url,
|
|
92
|
+
});
|
|
93
|
+
const createDesiredDocument = (document, siteId, revision, revisionAncestors, revisionTimestamp) => {
|
|
94
|
+
const hash = getDocumentHash(document);
|
|
95
|
+
const siteHash = hashValue(siteId).slice(0, 16);
|
|
96
|
+
const documentHash = hashValue(document.id).slice(0, 32);
|
|
97
|
+
return {
|
|
98
|
+
document,
|
|
99
|
+
externalId: `geistdocs/${siteHash}/${documentHash}/${hash.slice(0, 16)}`,
|
|
100
|
+
hash,
|
|
101
|
+
metadata: createMetadata({
|
|
102
|
+
document,
|
|
103
|
+
hash,
|
|
104
|
+
revision,
|
|
105
|
+
revisionAncestors,
|
|
106
|
+
revisionTimestamp,
|
|
107
|
+
siteId,
|
|
108
|
+
}),
|
|
109
|
+
};
|
|
110
|
+
};
|
|
111
|
+
const isFromNewerRevision = (metadata, revision, revisionTimestamp) => Boolean(revision && metadata.revisionAncestors.includes(revision)) ||
|
|
112
|
+
(revisionTimestamp !== undefined &&
|
|
113
|
+
metadata.revisionTimestamp !== undefined &&
|
|
114
|
+
metadata.revisionTimestamp > revisionTimestamp);
|
|
115
|
+
const canDeleteForRevision = ({ currentRevision, currentRevisionAncestors, currentRevisionTimestamp, desired, metadata, }) => {
|
|
116
|
+
if (desired?.hash === metadata.contentHash) {
|
|
117
|
+
return true;
|
|
118
|
+
}
|
|
119
|
+
if (!(currentRevision && metadata.revision)) {
|
|
120
|
+
return false;
|
|
121
|
+
}
|
|
122
|
+
return (metadata.revision === currentRevision ||
|
|
123
|
+
currentRevisionAncestors.includes(metadata.revision) ||
|
|
124
|
+
(currentRevisionTimestamp !== undefined &&
|
|
125
|
+
metadata.revisionTimestamp !== undefined &&
|
|
126
|
+
metadata.revisionTimestamp < currentRevisionTimestamp));
|
|
127
|
+
};
|
|
128
|
+
const defaultSleep = (milliseconds) => new Promise((resolveSleep) => {
|
|
129
|
+
setTimeout(resolveSleep, milliseconds);
|
|
130
|
+
});
|
|
131
|
+
const listAllStoreFiles = async (client) => {
|
|
132
|
+
const files = [];
|
|
133
|
+
const seenCursors = new Set();
|
|
134
|
+
let cursor;
|
|
135
|
+
do {
|
|
136
|
+
const page = await client.listStoreFiles(cursor);
|
|
137
|
+
files.push(...page.files);
|
|
138
|
+
cursor = page.nextCursor;
|
|
139
|
+
if (cursor) {
|
|
140
|
+
if (seenCursors.has(cursor)) {
|
|
141
|
+
throw new Error("Mixedbread file listing repeated a pagination cursor.");
|
|
142
|
+
}
|
|
143
|
+
seenCursors.add(cursor);
|
|
144
|
+
}
|
|
145
|
+
} while (cursor);
|
|
146
|
+
return files;
|
|
147
|
+
};
|
|
148
|
+
const runWithConcurrency = async (inputs, concurrency, processInput) => {
|
|
149
|
+
const results = [];
|
|
150
|
+
const errors = [];
|
|
151
|
+
let nextIndex = 0;
|
|
152
|
+
const worker = async () => {
|
|
153
|
+
while (nextIndex < inputs.length) {
|
|
154
|
+
const index = nextIndex;
|
|
155
|
+
nextIndex += 1;
|
|
156
|
+
try {
|
|
157
|
+
results[index] = await processInput(inputs[index]);
|
|
158
|
+
}
|
|
159
|
+
catch (error) {
|
|
160
|
+
errors.push(error);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
};
|
|
164
|
+
await Promise.all(Array.from({ length: Math.min(Math.max(1, concurrency), inputs.length) }, worker));
|
|
165
|
+
if (errors.length > 0) {
|
|
166
|
+
const details = errors
|
|
167
|
+
.map((error) => (error instanceof Error ? error.message : String(error)))
|
|
168
|
+
.join("\n");
|
|
169
|
+
throw new AggregateError(errors, `Mixedbread search sync did not complete:\n${details}`);
|
|
170
|
+
}
|
|
171
|
+
return results;
|
|
172
|
+
};
|
|
173
|
+
const waitForStoreFile = async ({ client, file, pollIntervalMs, pollTimeoutMs, sleep, }) => {
|
|
174
|
+
let current = file;
|
|
175
|
+
const deadline = Date.now() + pollTimeoutMs;
|
|
176
|
+
while (current.status === "pending" || current.status === "in_progress") {
|
|
177
|
+
if (Date.now() >= deadline) {
|
|
178
|
+
throw new Error(`Mixedbread timed out while indexing file ${file.id}.`);
|
|
179
|
+
}
|
|
180
|
+
await sleep(pollIntervalMs);
|
|
181
|
+
current = await client.retrieveStoreFile(file.id);
|
|
182
|
+
}
|
|
183
|
+
if (current.status !== "completed") {
|
|
184
|
+
throw new Error(`Mixedbread failed to index file ${file.id} (${current.status}).`);
|
|
185
|
+
}
|
|
186
|
+
return current;
|
|
187
|
+
};
|
|
188
|
+
const uploadDocument = async ({ client, desired, pollIntervalMs, pollTimeoutMs, sleep, }) => {
|
|
189
|
+
const fileId = await client.uploadFile({
|
|
190
|
+
content: desired.document.content,
|
|
191
|
+
filename: `${hashValue(desired.document.id).slice(0, 32)}.md`,
|
|
192
|
+
});
|
|
193
|
+
let storeFile;
|
|
194
|
+
try {
|
|
195
|
+
storeFile = await client.createStoreFile({
|
|
196
|
+
externalId: desired.externalId,
|
|
197
|
+
fileId,
|
|
198
|
+
metadata: {
|
|
199
|
+
...desired.metadata,
|
|
200
|
+
[mixedbreadMetadataKeys.rawFileId]: fileId,
|
|
201
|
+
},
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
catch (error) {
|
|
205
|
+
try {
|
|
206
|
+
await client.deleteFile(fileId);
|
|
207
|
+
}
|
|
208
|
+
catch (cleanupError) {
|
|
209
|
+
throw new AggregateError([error, cleanupError], `Mixedbread failed to attach and clean up raw file ${fileId}.`);
|
|
210
|
+
}
|
|
211
|
+
throw error;
|
|
212
|
+
}
|
|
213
|
+
return waitForStoreFile({
|
|
214
|
+
client,
|
|
215
|
+
file: storeFile,
|
|
216
|
+
pollIntervalMs,
|
|
217
|
+
pollTimeoutMs,
|
|
218
|
+
sleep,
|
|
219
|
+
});
|
|
220
|
+
};
|
|
221
|
+
export const syncSearchExport = async ({ client, concurrency = DEFAULT_CONCURRENCY, manifest, pollIntervalMs = DEFAULT_POLL_INTERVAL_MS, pollTimeoutMs = DEFAULT_POLL_TIMEOUT_MS, revision, revisionAncestors = [], revisionTimestamp, sleep = defaultSleep, }) => {
|
|
222
|
+
const validatedManifest = parseSearchExport(manifest);
|
|
223
|
+
if (validatedManifest.provider !== "mixedbread") {
|
|
224
|
+
return {
|
|
225
|
+
deleted: 0,
|
|
226
|
+
skipped: validatedManifest.documents.length,
|
|
227
|
+
uploaded: 0,
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
const siteId = validatedManifest.siteId?.trim();
|
|
231
|
+
if (!siteId) {
|
|
232
|
+
throw new Error("Mixedbread retrieval requires a non-empty siteId in geistdocs.tsx.");
|
|
233
|
+
}
|
|
234
|
+
if (validatedManifest.documents.length === 0) {
|
|
235
|
+
throw new Error("The Geistdocs search export is empty. Refusing to remove indexed documents.");
|
|
236
|
+
}
|
|
237
|
+
const desiredDocuments = validatedManifest.documents.map((document) => createDesiredDocument(document, siteId, revision, revisionAncestors, revisionTimestamp));
|
|
238
|
+
const desiredById = new Map(desiredDocuments.map((desired) => [desired.document.id, desired]));
|
|
239
|
+
const remoteFiles = await listAllStoreFiles(client);
|
|
240
|
+
const ownedFiles = remoteFiles.flatMap((file) => {
|
|
241
|
+
const metadata = readGeistdocsMixedbreadMetadata(file.metadata);
|
|
242
|
+
return metadata?.siteId === siteId ? [{ file, metadata }] : [];
|
|
243
|
+
});
|
|
244
|
+
const existingCurrentFiles = new Map();
|
|
245
|
+
const newerDocumentIds = new Set();
|
|
246
|
+
for (const owned of ownedFiles) {
|
|
247
|
+
if (isFromNewerRevision(owned.metadata, revision, revisionTimestamp)) {
|
|
248
|
+
newerDocumentIds.add(owned.metadata.documentId);
|
|
249
|
+
}
|
|
250
|
+
const desired = desiredById.get(owned.metadata.documentId);
|
|
251
|
+
if (desired &&
|
|
252
|
+
owned.metadata.contentHash === desired.hash &&
|
|
253
|
+
owned.file.status === "completed" &&
|
|
254
|
+
!existingCurrentFiles.has(owned.metadata.documentId)) {
|
|
255
|
+
existingCurrentFiles.set(owned.metadata.documentId, owned);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
const documentsToUpload = desiredDocuments.filter((desired) => !(existingCurrentFiles.has(desired.document.id) ||
|
|
259
|
+
newerDocumentIds.has(desired.document.id)));
|
|
260
|
+
await runWithConcurrency(documentsToUpload, concurrency, (desired) => uploadDocument({
|
|
261
|
+
client,
|
|
262
|
+
desired,
|
|
263
|
+
pollIntervalMs,
|
|
264
|
+
pollTimeoutMs,
|
|
265
|
+
sleep,
|
|
266
|
+
}));
|
|
267
|
+
const latestRemoteFiles = await listAllStoreFiles(client);
|
|
268
|
+
const latestOwnedFiles = latestRemoteFiles.flatMap((file) => {
|
|
269
|
+
const metadata = readGeistdocsMixedbreadMetadata(file.metadata);
|
|
270
|
+
return metadata?.siteId === siteId ? [{ file, metadata }] : [];
|
|
271
|
+
});
|
|
272
|
+
const supersededDocumentIds = new Set(latestOwnedFiles
|
|
273
|
+
.filter(({ metadata }) => isFromNewerRevision(metadata, revision, revisionTimestamp))
|
|
274
|
+
.map(({ metadata }) => metadata.documentId));
|
|
275
|
+
const keepFileIds = new Set();
|
|
276
|
+
for (const desired of desiredDocuments) {
|
|
277
|
+
if (supersededDocumentIds.has(desired.document.id)) {
|
|
278
|
+
continue;
|
|
279
|
+
}
|
|
280
|
+
const currentFiles = latestOwnedFiles
|
|
281
|
+
.filter(({ file, metadata }) => file.status === "completed" &&
|
|
282
|
+
metadata.documentId === desired.document.id &&
|
|
283
|
+
metadata.contentHash === desired.hash)
|
|
284
|
+
.sort((left, right) => left.file.id.localeCompare(right.file.id));
|
|
285
|
+
const survivor = currentFiles[0];
|
|
286
|
+
if (survivor) {
|
|
287
|
+
keepFileIds.add(survivor.file.id);
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
const filesToDelete = latestOwnedFiles.filter(({ file, metadata }) => file.status !== "pending" &&
|
|
291
|
+
file.status !== "in_progress" &&
|
|
292
|
+
!isFromNewerRevision(metadata, revision, revisionTimestamp) &&
|
|
293
|
+
canDeleteForRevision({
|
|
294
|
+
currentRevision: revision,
|
|
295
|
+
currentRevisionAncestors: revisionAncestors,
|
|
296
|
+
currentRevisionTimestamp: revisionTimestamp,
|
|
297
|
+
desired: desiredById.get(metadata.documentId),
|
|
298
|
+
metadata,
|
|
299
|
+
}) &&
|
|
300
|
+
!keepFileIds.has(file.id));
|
|
301
|
+
await runWithConcurrency(filesToDelete, concurrency, async ({ file, metadata }) => {
|
|
302
|
+
// Store files keep the raw file uploaded before attachment. Delete both
|
|
303
|
+
// owned resources so replacements do not accumulate storage.
|
|
304
|
+
await client.deleteFile(metadata.rawFileId ?? file.id);
|
|
305
|
+
await client.deleteStoreFile(file.id);
|
|
306
|
+
});
|
|
307
|
+
return {
|
|
308
|
+
deleted: filesToDelete.length,
|
|
309
|
+
skipped: desiredDocuments.length - documentsToUpload.length,
|
|
310
|
+
uploaded: documentsToUpload.length,
|
|
311
|
+
};
|
|
312
|
+
};
|
package/dist/routes/chat.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type LanguageModel } from "ai";
|
|
2
2
|
import { type GeistdocsConfig } from "../config";
|
|
3
|
+
import { type ChatMixedbreadOptions } from "../internal/chat-retrieval";
|
|
3
4
|
import type { GeistdocsSourceBundle } from "../source";
|
|
4
5
|
type SourceBundle = GeistdocsSourceBundle;
|
|
5
6
|
interface ChatProxyConfig {
|
|
@@ -18,12 +19,17 @@ interface ChatEveAgentOptions {
|
|
|
18
19
|
url?: string;
|
|
19
20
|
}
|
|
20
21
|
interface CreateChatRouteOptions {
|
|
21
|
-
config: Pick<GeistdocsConfig, "ai" | "basePath" | "defaultLanguage" | "translations">;
|
|
22
|
+
config: Pick<GeistdocsConfig, "ai" | "basePath" | "defaultLanguage" | "siteId" | "translations">;
|
|
22
23
|
/**
|
|
23
24
|
* Server-side options for the eve agent responder configured through
|
|
24
25
|
* `config.ai.eveAgent`. Mutually exclusive with `proxy`.
|
|
25
26
|
*/
|
|
26
27
|
eveAgent?: ChatEveAgentOptions;
|
|
28
|
+
/**
|
|
29
|
+
* Server-only Mixedbread overrides. Environment variables are used when
|
|
30
|
+
* these values are omitted.
|
|
31
|
+
*/
|
|
32
|
+
mixedbread?: ChatMixedbreadOptions;
|
|
27
33
|
/**
|
|
28
34
|
* Model used to answer Ask AI requests in the default AI Gateway mode.
|
|
29
35
|
* Accepts any AI SDK `LanguageModel`, including AI Gateway model strings
|
|
@@ -36,7 +42,7 @@ interface CreateChatRouteOptions {
|
|
|
36
42
|
source?: SourceBundle;
|
|
37
43
|
sources?: SourceBundle[];
|
|
38
44
|
}
|
|
39
|
-
export declare const createChatRoute: ({ config, eveAgent, model, proxy, source, sources, }: CreateChatRouteOptions) => {
|
|
45
|
+
export declare const createChatRoute: ({ config, eveAgent, mixedbread, model, proxy, source, sources, }: CreateChatRouteOptions) => {
|
|
40
46
|
POST: (req: Request) => Promise<Response>;
|
|
41
47
|
maxDuration: 800;
|
|
42
48
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/routes/chat.ts"],"names":[],"mappings":"AACA,OAAO,EAIL,KAAK,aAAa,EAQnB,MAAM,IAAI,CAAC;AAGZ,OAAO,EAAE,KAAK,eAAe,EAAiB,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/routes/chat.ts"],"names":[],"mappings":"AACA,OAAO,EAIL,KAAK,aAAa,EAQnB,MAAM,IAAI,CAAC;AAGZ,OAAO,EAAE,KAAK,eAAe,EAAiB,MAAM,WAAW,CAAC;AAChE,OAAO,EAEL,KAAK,qBAAqB,EAG3B,MAAM,4BAA4B,CAAC;AASpC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAEvD,KAAK,YAAY,GAAG,qBAAqB,CAAC;AAyB1C,UAAU,eAAe;IACvB,OAAO,CAAC,EAAE,WAAW,GAAG,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;IACnE,GAAG,EAAE,MAAM,CAAC;CACb;AAED,UAAU,mBAAmB;IAC3B;;;;;OAKG;IACH,OAAO,CAAC,EAAE,WAAW,GAAG,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;IACnE,qEAAqE;IACrE,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,UAAU,sBAAsB;IAC9B,MAAM,EAAE,IAAI,CACV,eAAe,EACf,IAAI,GAAG,UAAU,GAAG,iBAAiB,GAAG,QAAQ,GAAG,cAAc,CAClE,CAAC;IACF;;;OAGG;IACH,QAAQ,CAAC,EAAE,mBAAmB,CAAC;IAC/B;;;OAGG;IACH,UAAU,CAAC,EAAE,qBAAqB,CAAC;IACnC;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;CAC1B;AA82BD,eAAO,MAAM,eAAe,GAAI,kEAQ7B,sBAAsB;gBAcE,OAAO;;CA6JjC,CAAC"}
|
package/dist/routes/chat.js
CHANGED
|
@@ -3,6 +3,7 @@ import { convertToModelMessages, createUIMessageStream, createUIMessageStreamRes
|
|
|
3
3
|
import { initAdvancedSearch } from "fumadocs-core/search/server";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
import { getPublicPath } from "../config";
|
|
6
|
+
import { createChatDocumentSearch, } from "../internal/chat-retrieval";
|
|
6
7
|
import { buildEveAgentTranscript, capLastUserMessageText, getEveAgentSessionHandle, resolveEveAgentUrl, runEveAgentTurn, } from "../internal/eve-agent";
|
|
7
8
|
import { isPageVisibleForSurface } from "../page-visibility";
|
|
8
9
|
const DEFAULT_CHAT_MODEL = "openai/gpt-4.1-mini";
|
|
@@ -95,7 +96,7 @@ const createSearchServer = (sources, lang) => {
|
|
|
95
96
|
}));
|
|
96
97
|
return initAdvancedSearch({ indexes });
|
|
97
98
|
};
|
|
98
|
-
const resolveSearchResult = ({ basePath, lang, sources, url, }) => {
|
|
99
|
+
const resolveSearchResult = ({ basePath, content, lang, sources, url, }) => {
|
|
99
100
|
const result = sources
|
|
100
101
|
.map((source) => source.source.getPageByHref(url, { language: lang }))
|
|
101
102
|
.find((entry) => entry?.page);
|
|
@@ -107,17 +108,23 @@ const resolveSearchResult = ({ basePath, lang, sources, url, }) => {
|
|
|
107
108
|
return {
|
|
108
109
|
title: data.title ?? page.url,
|
|
109
110
|
description: data.description ?? "",
|
|
110
|
-
content: getStructuredContent(page),
|
|
111
|
+
content: content?.trim() || getStructuredContent(page),
|
|
111
112
|
slug: page.url,
|
|
112
113
|
url: getPublicPath(page.url, basePath),
|
|
113
114
|
};
|
|
114
115
|
};
|
|
115
|
-
const
|
|
116
|
+
const searchDocsWithOrama = async ({ basePath, lang, limit, query, sources, }) => {
|
|
116
117
|
const searchServer = createSearchServer(sources, lang);
|
|
117
118
|
const results = await searchServer.search(query);
|
|
118
119
|
const seenSlugs = new Set();
|
|
119
120
|
return results
|
|
120
|
-
.map(({ url }) => resolveSearchResult({
|
|
121
|
+
.map(({ content, type, url }) => resolveSearchResult({
|
|
122
|
+
basePath,
|
|
123
|
+
content: type === "text" ? content : undefined,
|
|
124
|
+
lang,
|
|
125
|
+
sources,
|
|
126
|
+
url,
|
|
127
|
+
}))
|
|
121
128
|
.filter((doc) => Boolean(doc))
|
|
122
129
|
.filter((doc) => {
|
|
123
130
|
if (seenSlugs.has(doc.slug)) {
|
|
@@ -128,6 +135,33 @@ const searchDocs = async ({ basePath, lang, limit, query, sources, }) => {
|
|
|
128
135
|
})
|
|
129
136
|
.slice(0, limit);
|
|
130
137
|
};
|
|
138
|
+
const createDocumentSearch = ({ config, mixedbread, sources, }) => {
|
|
139
|
+
const fallback = ({ lang, limit, query }) => searchDocsWithOrama({
|
|
140
|
+
basePath: config.basePath,
|
|
141
|
+
lang,
|
|
142
|
+
limit,
|
|
143
|
+
query,
|
|
144
|
+
sources,
|
|
145
|
+
});
|
|
146
|
+
return createChatDocumentSearch({
|
|
147
|
+
fallback,
|
|
148
|
+
mixedbread: {
|
|
149
|
+
...mixedbread,
|
|
150
|
+
apiKey: mixedbread?.apiKey ?? process.env.MXBAI_API_KEY,
|
|
151
|
+
storeId: mixedbread?.storeId ?? process.env.MXBAI_STORE_ID,
|
|
152
|
+
},
|
|
153
|
+
onWarning: (message, error) => {
|
|
154
|
+
if (error === undefined) {
|
|
155
|
+
console.warn(message);
|
|
156
|
+
}
|
|
157
|
+
else {
|
|
158
|
+
console.warn(message, error);
|
|
159
|
+
}
|
|
160
|
+
},
|
|
161
|
+
provider: config.ai?.retrieval,
|
|
162
|
+
siteId: config.siteId,
|
|
163
|
+
});
|
|
164
|
+
};
|
|
131
165
|
const formatDocForContext = (doc) => `**${doc.title}**\nURL: ${doc.url}\n${doc.description || ""}\n\n${doc.content.slice(0, MAX_CONTEXT_CHARS)}${doc.content.length > MAX_CONTEXT_CHARS ? "..." : ""}`;
|
|
132
166
|
const getRouteCandidates = ({ basePath, currentRoute, lang, }) => {
|
|
133
167
|
const pathname = normalizePathname(currentRoute);
|
|
@@ -184,7 +218,7 @@ const createSourceChunks = (docs) => docs.map((doc, index) => ({
|
|
|
184
218
|
url: doc.url,
|
|
185
219
|
title: doc.title,
|
|
186
220
|
}));
|
|
187
|
-
const buildProxyMessages = async ({ actualMessages, config, currentRoute, defaultLanguage, languages, pageContext, routeSources, }) => {
|
|
221
|
+
const buildProxyMessages = async ({ actualMessages, config, currentRoute, defaultLanguage, languages, pageContext, routeSources, searchDocuments, }) => {
|
|
188
222
|
if (!isFirstUserMessage(actualMessages)) {
|
|
189
223
|
return { messages: actualMessages, sourceDocs: [] };
|
|
190
224
|
}
|
|
@@ -207,12 +241,10 @@ const buildProxyMessages = async ({ actualMessages, config, currentRoute, defaul
|
|
|
207
241
|
sources: routeSources,
|
|
208
242
|
});
|
|
209
243
|
const relatedDocs = userQuestion.trim()
|
|
210
|
-
? await
|
|
211
|
-
basePath: config.basePath,
|
|
244
|
+
? await searchDocuments({
|
|
212
245
|
lang,
|
|
213
246
|
limit: MAX_PROXY_RELATED_PAGES,
|
|
214
247
|
query: userQuestion,
|
|
215
|
-
sources: routeSources,
|
|
216
248
|
})
|
|
217
249
|
: [];
|
|
218
250
|
const filteredRelatedDocs = relatedDocs.filter((doc) => doc.slug !== currentPageDoc?.slug);
|
|
@@ -378,7 +410,7 @@ You are a helpful assistant specializing in answering questions strictly. If inf
|
|
|
378
410
|
- Be friendly, clear, and specific.`;
|
|
379
411
|
return [basePrompt, config.ai?.prompt].filter(Boolean).join("\n\n");
|
|
380
412
|
};
|
|
381
|
-
const createTools = (writer, sources, defaultLanguage, basePath) => {
|
|
413
|
+
const createTools = (writer, sources, searchDocuments, defaultLanguage, basePath) => {
|
|
382
414
|
const log = (message) => {
|
|
383
415
|
console.log(`[Geistdocs chat] ${message}`);
|
|
384
416
|
};
|
|
@@ -395,12 +427,10 @@ const createTools = (writer, sources, defaultLanguage, basePath) => {
|
|
|
395
427
|
execute: async ({ query, lang }) => {
|
|
396
428
|
try {
|
|
397
429
|
log(`Creating search server for language: ${lang}`);
|
|
398
|
-
const trimmedResults = await
|
|
399
|
-
basePath,
|
|
430
|
+
const trimmedResults = await searchDocuments({
|
|
400
431
|
lang,
|
|
401
432
|
limit: MAX_TOOL_RESULTS,
|
|
402
433
|
query,
|
|
403
|
-
sources,
|
|
404
434
|
});
|
|
405
435
|
if (trimmedResults.length === 0) {
|
|
406
436
|
return `No documentation found for query: "${query}"`;
|
|
@@ -479,7 +509,7 @@ const createChatErrorResponse = (error, status) => new Response(JSON.stringify({
|
|
|
479
509
|
status,
|
|
480
510
|
headers: { "Content-Type": "application/json" },
|
|
481
511
|
});
|
|
482
|
-
const createEveAgentResponse = async ({ actualMessages, agentUrl, config, currentRoute, defaultLanguage, eveAgent, languages, messages, pageContext, req, routeSources, }) => {
|
|
512
|
+
const createEveAgentResponse = async ({ actualMessages, agentUrl, config, currentRoute, defaultLanguage, eveAgent, languages, messages, pageContext, req, routeSources, searchDocuments, }) => {
|
|
483
513
|
const cappedMessages = capLastUserMessageText(actualMessages, MAX_EVE_AGENT_MESSAGE_CHARS);
|
|
484
514
|
const { messages: contextMessages, sourceDocs } = await buildProxyMessages({
|
|
485
515
|
actualMessages: cappedMessages,
|
|
@@ -489,6 +519,7 @@ const createEveAgentResponse = async ({ actualMessages, agentUrl, config, curren
|
|
|
489
519
|
languages,
|
|
490
520
|
pageContext,
|
|
491
521
|
routeSources,
|
|
522
|
+
searchDocuments,
|
|
492
523
|
});
|
|
493
524
|
const lastMessage = contextMessages.at(-1);
|
|
494
525
|
if (lastMessage?.role !== "user") {
|
|
@@ -530,9 +561,14 @@ const resolveSources = (source, sources) => {
|
|
|
530
561
|
}
|
|
531
562
|
return resolvedSources;
|
|
532
563
|
};
|
|
533
|
-
export const createChatRoute = ({ config, eveAgent, model = DEFAULT_CHAT_MODEL, proxy, source, sources, }) => {
|
|
564
|
+
export const createChatRoute = ({ config, eveAgent, mixedbread, model = DEFAULT_CHAT_MODEL, proxy, source, sources, }) => {
|
|
534
565
|
const { defaultLanguage, languages } = getLanguages(config);
|
|
535
566
|
const routeSources = resolveSources(source, sources);
|
|
567
|
+
const searchDocuments = createDocumentSearch({
|
|
568
|
+
config,
|
|
569
|
+
mixedbread,
|
|
570
|
+
sources: routeSources,
|
|
571
|
+
});
|
|
536
572
|
const eveAgentUrl = resolveEveAgentUrl({
|
|
537
573
|
configUrl: config.ai?.eveAgent?.url,
|
|
538
574
|
hasProxy: Boolean(proxy),
|
|
@@ -564,6 +600,7 @@ export const createChatRoute = ({ config, eveAgent, model = DEFAULT_CHAT_MODEL,
|
|
|
564
600
|
pageContext,
|
|
565
601
|
req,
|
|
566
602
|
routeSources,
|
|
603
|
+
searchDocuments,
|
|
567
604
|
});
|
|
568
605
|
}
|
|
569
606
|
if (proxy) {
|
|
@@ -575,6 +612,7 @@ export const createChatRoute = ({ config, eveAgent, model = DEFAULT_CHAT_MODEL,
|
|
|
575
612
|
languages,
|
|
576
613
|
pageContext,
|
|
577
614
|
routeSources,
|
|
615
|
+
searchDocuments,
|
|
578
616
|
});
|
|
579
617
|
const stream = createUIMessageStream({
|
|
580
618
|
originalMessages: messages,
|
|
@@ -630,7 +668,7 @@ export const createChatRoute = ({ config, eveAgent, model = DEFAULT_CHAT_MODEL,
|
|
|
630
668
|
model,
|
|
631
669
|
messages: modelMessages,
|
|
632
670
|
stopWhen: stepCountIs(10),
|
|
633
|
-
tools: createTools(writer, routeSources, defaultLanguage, config.basePath),
|
|
671
|
+
tools: createTools(writer, routeSources, searchDocuments, defaultLanguage, config.basePath),
|
|
634
672
|
system: createSystemPrompt(currentRoute ?? "/", config),
|
|
635
673
|
prepareStep: ({ stepNumber }) => {
|
|
636
674
|
if (stepNumber === 0) {
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { type GeistdocsConfig } from "../config";
|
|
2
|
+
import type { GeistdocsSourceBundle } from "../source";
|
|
3
|
+
interface CreateSearchExportRouteOptions {
|
|
4
|
+
config: Pick<GeistdocsConfig, "ai" | "basePath" | "defaultLanguage" | "siteId" | "translations">;
|
|
5
|
+
source?: GeistdocsSourceBundle;
|
|
6
|
+
sources?: GeistdocsSourceBundle[];
|
|
7
|
+
}
|
|
8
|
+
export declare const createSearchExportRoute: ({ config, source, sources, }: CreateSearchExportRouteOptions) => {
|
|
9
|
+
GET: () => Promise<Response>;
|
|
10
|
+
};
|
|
11
|
+
export {};
|
|
12
|
+
//# sourceMappingURL=search-export.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search-export.d.ts","sourceRoot":"","sources":["../../src/routes/search-export.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,eAAe,EAAiB,MAAM,WAAW,CAAC;AAOhE,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAEvD,UAAU,8BAA8B;IACtC,MAAM,EAAE,IAAI,CACV,eAAe,EACf,IAAI,GAAG,UAAU,GAAG,iBAAiB,GAAG,QAAQ,GAAG,cAAc,CAClE,CAAC;IACF,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B,OAAO,CAAC,EAAE,qBAAqB,EAAE,CAAC;CACnC;AAwBD,eAAO,MAAM,uBAAuB,GAAI,8BAIrC,8BAA8B;;CAyChC,CAAC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { getPublicPath } from "../config";
|
|
2
|
+
import { GEISTDOCS_SEARCH_EXPORT_VERSION, } from "../internal/search-export";
|
|
3
|
+
import { isPageVisibleForSurface } from "../page-visibility";
|
|
4
|
+
const resolveSources = (source, sources) => {
|
|
5
|
+
const resolvedSources = sources ?? (source ? [source] : []);
|
|
6
|
+
if (resolvedSources.length === 0) {
|
|
7
|
+
throw new Error("createSearchExportRoute requires source or sources.");
|
|
8
|
+
}
|
|
9
|
+
return resolvedSources;
|
|
10
|
+
};
|
|
11
|
+
const getLanguages = (config) => {
|
|
12
|
+
const defaultLanguage = config.defaultLanguage ?? "en";
|
|
13
|
+
return Array.from(new Set([defaultLanguage, ...Object.keys(config.translations ?? {})]));
|
|
14
|
+
};
|
|
15
|
+
export const createSearchExportRoute = ({ config, source, sources, }) => {
|
|
16
|
+
const routeSources = resolveSources(source, sources);
|
|
17
|
+
const GET = async () => {
|
|
18
|
+
const documents = await Promise.all(getLanguages(config).flatMap((locale) => routeSources.flatMap((entry, sourceIndex) => {
|
|
19
|
+
const sourceId = entry.id ?? `source-${sourceIndex + 1}`;
|
|
20
|
+
return entry.source
|
|
21
|
+
.getPages(locale)
|
|
22
|
+
.filter((page) => isPageVisibleForSurface(page, "chat"))
|
|
23
|
+
.map(async (page) => ({
|
|
24
|
+
content: await entry.getPageMarkdown(page),
|
|
25
|
+
description: page.data.description ?? "",
|
|
26
|
+
id: `${sourceId}:${locale}:${page.url}`,
|
|
27
|
+
locale,
|
|
28
|
+
sourceId,
|
|
29
|
+
sourceUrl: page.url,
|
|
30
|
+
title: page.data.title ?? page.url,
|
|
31
|
+
url: getPublicPath(page.url, config.basePath),
|
|
32
|
+
}));
|
|
33
|
+
})));
|
|
34
|
+
documents.sort((left, right) => left.id.localeCompare(right.id));
|
|
35
|
+
const exported = {
|
|
36
|
+
documents,
|
|
37
|
+
provider: config.ai?.retrieval ?? "local",
|
|
38
|
+
siteId: config.siteId,
|
|
39
|
+
version: GEISTDOCS_SEARCH_EXPORT_VERSION,
|
|
40
|
+
};
|
|
41
|
+
return Response.json(exported);
|
|
42
|
+
};
|
|
43
|
+
return { GET };
|
|
44
|
+
};
|
package/docs/agents.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Geistdocs Package Docs
|
|
2
2
|
|
|
3
|
-
These docs are bundled with `@vercel/geistdocs` 1.
|
|
3
|
+
These docs are bundled with `@vercel/geistdocs` 1.26.0 so agents can understand the installed package without web access.
|
|
4
4
|
They are generated from `apps/template/content/docs` during the package build.
|
|
5
5
|
|
|
6
6
|
## How to use these files
|
|
@@ -24,8 +24,8 @@ They are generated from `apps/template/content/docs` during the package build.
|
|
|
24
24
|
|
|
25
25
|
- [Overview](pages/index.md): Geistdocs is a packaged documentation system for creating Next.js and Fumadocs sites with shared runtime features.
|
|
26
26
|
- [Getting Started](pages/getting-started.md): Create a Geistdocs project, run it locally, and edit your first documentation page.
|
|
27
|
-
- [Environment Variables](pages/env.md): Environment variables required by Geistdocs for AI chat, proxy mode, and production URL configuration.
|
|
28
|
-
- [Deploy to Vercel](pages/deployment.md): Deploy
|
|
27
|
+
- [Environment Variables](pages/env.md): Environment variables required by Geistdocs for AI chat, Mixedbread retrieval, proxy mode, and production URL configuration.
|
|
28
|
+
- [Deploy to Vercel](pages/deployment.md): Deploy a Geistdocs site to Vercel with the required project settings, environment variables, and private-package access.
|
|
29
29
|
- [Migration guide](pages/migration.md): Move an existing documentation site to @vercel/geistdocs while keeping local content, routing, middleware behavior, and AI-readable surfaces intact.
|
|
30
30
|
- [Configuration](pages/configuration.md): Configure site metadata, navigation, AI features, page actions, translations, and local adapters.
|
|
31
31
|
- [Syntax](pages/syntax.md): Supported MDX syntax including text formatting, code blocks, line highlighting, and Mermaid diagrams.
|
|
@@ -38,6 +38,7 @@ They are generated from `apps/template/content/docs` during the package build.
|
|
|
38
38
|
- [Configure sidebar navigation](pages/guides/nested-navigation.md): Configure sidebar order, section labels, folder landing pages, and deeply nested page trees with meta.json files.
|
|
39
39
|
- [Edit on GitHub](pages/edit-on-github.md): A direct link on every documentation page that lets readers propose changes through GitHub pull requests.
|
|
40
40
|
- [Feedback](pages/feedback.md): An interactive feedback widget that collects user sentiment and creates structured GitHub Issues automatically.
|
|
41
|
+
- [Improve Ask AI answers with Mixedbread](pages/mixedbread-retrieval.md): Provision a Mixedbread Store for one Geistdocs consumer site, enable semantic retrieval, and sync documentation during production builds.
|
|
41
42
|
- [Internationalization](pages/internationalization.md): Serve documentation in multiple languages with automatic translation via the Geistdocs CLI and Fumadocs routing.
|
|
42
43
|
- [llms.txt](pages/llms-txt.md): A single endpoint that returns all documentation as plain Markdown text following the llms.txt standard.
|
|
43
44
|
- [Open in Chat](pages/open-in-chat.md): Transfer documentation context to AI chat platforms like ChatGPT, Claude, Cursor, and v0 with a single click.
|