@vercel/geistdocs 1.25.2 → 1.26.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/cli.js +8 -0
  2. package/dist/cli/index.js +6 -2
  3. package/dist/config.d.ts +5 -0
  4. package/dist/config.d.ts.map +1 -1
  5. package/dist/internal/chat-retrieval.d.ts +32 -0
  6. package/dist/internal/chat-retrieval.d.ts.map +1 -0
  7. package/dist/internal/chat-retrieval.js +91 -0
  8. package/dist/internal/mixedbread.d.ts +93 -0
  9. package/dist/internal/mixedbread.d.ts.map +1 -0
  10. package/dist/internal/mixedbread.js +286 -0
  11. package/dist/internal/page-actions-client.d.ts.map +1 -1
  12. package/dist/internal/page-actions-client.js +1 -19
  13. package/dist/internal/search-export.d.ts +18 -0
  14. package/dist/internal/search-export.d.ts.map +1 -0
  15. package/dist/internal/search-export.js +1 -0
  16. package/dist/internal/search-sync.d.ts +25 -0
  17. package/dist/internal/search-sync.d.ts.map +1 -0
  18. package/dist/internal/search-sync.js +312 -0
  19. package/dist/routes/chat.d.ts +8 -2
  20. package/dist/routes/chat.d.ts.map +1 -1
  21. package/dist/routes/chat.js +53 -15
  22. package/dist/routes/search-export.d.ts +12 -0
  23. package/dist/routes/search-export.d.ts.map +1 -0
  24. package/dist/routes/search-export.js +44 -0
  25. package/docs/agents.md +4 -3
  26. package/docs/llms.txt +384 -33
  27. package/docs/manifest.json +12 -3
  28. package/docs/pages/ask-ai.md +3 -0
  29. package/docs/pages/configuration.md +5 -2
  30. package/docs/pages/deployment.md +30 -5
  31. package/docs/pages/env.md +13 -3
  32. package/docs/pages/getting-started.md +29 -9
  33. package/docs/pages/index.md +3 -2
  34. package/docs/pages/md.md +1 -1
  35. package/docs/pages/migration.md +148 -8
  36. package/docs/pages/mixedbread-retrieval.md +144 -0
  37. package/docs/sitemap.md +3 -2
  38. package/package.json +11 -5
  39. package/template/.env.example +19 -0
  40. package/template/AGENTS.md +17 -3
  41. package/template/app/api/search/export/route.ts +8 -0
  42. package/template/content/docs/ask-ai.mdx +3 -0
  43. package/template/content/docs/configuration.mdx +5 -2
  44. package/template/content/docs/deployment.mdx +31 -6
  45. package/template/content/docs/env.mdx +14 -4
  46. package/template/content/docs/getting-started.mdx +30 -10
  47. package/template/content/docs/index.mdx +3 -2
  48. package/template/content/docs/md.mdx +1 -1
  49. package/template/content/docs/meta.json +1 -0
  50. package/template/content/docs/migration.mdx +150 -9
  51. package/template/content/docs/mixedbread-retrieval.mdx +142 -0
  52. package/template/geistdocs.tsx +12 -1
  53. package/template/lib/geistdocs/config.tsx +2 -5
  54. package/template/package.json +3 -3
@@ -0,0 +1,312 @@
1
+ import { createHash } from "node:crypto";
2
+ import { readFile } from "node:fs/promises";
3
+ import { join, resolve, sep } from "node:path";
4
+ import { GEISTDOCS_MIXEDBREAD_SCHEMA_VERSION, mixedbreadMetadataKeys, readGeistdocsMixedbreadMetadata, } from "./mixedbread.js";
5
+ import { GEISTDOCS_SEARCH_EXPORT_VERSION, } from "./search-export.js";
6
+ const SEARCH_EXPORT_ROUTE = "/api/search/export";
7
+ const SEARCH_EXPORT_BODY_PATH = "server/app/api/search/export.body";
8
+ const DEFAULT_CONCURRENCY = 4;
9
+ const DEFAULT_POLL_INTERVAL_MS = 1000;
10
+ const DEFAULT_POLL_TIMEOUT_MS = 180_000;
11
+ const isRecord = (value) => typeof value === "object" && value !== null;
12
+ const isExportDocument = (value) => {
13
+ if (!isRecord(value)) {
14
+ return false;
15
+ }
16
+ return [
17
+ "content",
18
+ "description",
19
+ "id",
20
+ "locale",
21
+ "sourceId",
22
+ "sourceUrl",
23
+ "title",
24
+ "url",
25
+ ].every((key) => typeof value[key] === "string");
26
+ };
27
+ export const parseSearchExport = (value) => {
28
+ if (!isRecord(value) ||
29
+ value.version !== GEISTDOCS_SEARCH_EXPORT_VERSION ||
30
+ (value.provider !== "local" && value.provider !== "mixedbread") ||
31
+ !Array.isArray(value.documents) ||
32
+ !value.documents.every(isExportDocument) ||
33
+ (value.siteId !== undefined && typeof value.siteId !== "string")) {
34
+ throw new Error("The Geistdocs search export is invalid or unsupported.");
35
+ }
36
+ const documentIds = new Set();
37
+ for (const document of value.documents) {
38
+ if (documentIds.has(document.id)) {
39
+ throw new Error(`The Geistdocs search export contains duplicate document ID "${document.id}".`);
40
+ }
41
+ documentIds.add(document.id);
42
+ }
43
+ return value;
44
+ };
45
+ export const loadSearchExport = async ({ cwd = process.cwd(), outputDirectory = ".next", } = {}) => {
46
+ const outputRoot = resolve(cwd, outputDirectory);
47
+ const prerenderManifestPath = join(outputRoot, "prerender-manifest.json");
48
+ const prerenderManifest = JSON.parse(await readFile(prerenderManifestPath, "utf8"));
49
+ const route = prerenderManifest.routes?.[SEARCH_EXPORT_ROUTE];
50
+ if (route?.routeType !== "route" ||
51
+ route.compute !== "static" ||
52
+ route.response !== "complete") {
53
+ throw new Error(`${SEARCH_EXPORT_ROUTE} was not statically generated by the latest Next.js build.`);
54
+ }
55
+ const appOutputRoot = resolve(outputRoot, "server/app");
56
+ const bodyPath = resolve(outputRoot, SEARCH_EXPORT_BODY_PATH);
57
+ if (!bodyPath.startsWith(`${appOutputRoot}${sep}`)) {
58
+ throw new Error("The Geistdocs search export path escaped the Next.js output.");
59
+ }
60
+ return parseSearchExport(JSON.parse(await readFile(bodyPath, "utf8")));
61
+ };
62
+ const hashValue = (value) => createHash("sha256").update(value).digest("hex");
63
+ const getDocumentHash = (document) => hashValue(JSON.stringify({
64
+ content: document.content,
65
+ description: document.description,
66
+ locale: document.locale,
67
+ sourceId: document.sourceId,
68
+ sourceUrl: document.sourceUrl,
69
+ title: document.title,
70
+ url: document.url,
71
+ }));
72
+ const createMetadata = ({ document, hash, revision, revisionAncestors, revisionTimestamp, siteId, }) => ({
73
+ [mixedbreadMetadataKeys.contentHash]: hash,
74
+ [mixedbreadMetadataKeys.description]: document.description,
75
+ [mixedbreadMetadataKeys.documentId]: document.id,
76
+ [mixedbreadMetadataKeys.locale]: document.locale,
77
+ ...(revision ? { [mixedbreadMetadataKeys.revision]: revision } : {}),
78
+ ...(revisionAncestors.length > 0
79
+ ? {
80
+ [mixedbreadMetadataKeys.revisionAncestors]: revisionAncestors,
81
+ }
82
+ : {}),
83
+ ...(revisionTimestamp === undefined
84
+ ? {}
85
+ : { [mixedbreadMetadataKeys.revisionTimestamp]: revisionTimestamp }),
86
+ [mixedbreadMetadataKeys.schemaVersion]: GEISTDOCS_MIXEDBREAD_SCHEMA_VERSION,
87
+ [mixedbreadMetadataKeys.siteId]: siteId,
88
+ [mixedbreadMetadataKeys.sourceId]: document.sourceId,
89
+ [mixedbreadMetadataKeys.sourceUrl]: document.sourceUrl,
90
+ [mixedbreadMetadataKeys.title]: document.title,
91
+ [mixedbreadMetadataKeys.url]: document.url,
92
+ });
93
+ const createDesiredDocument = (document, siteId, revision, revisionAncestors, revisionTimestamp) => {
94
+ const hash = getDocumentHash(document);
95
+ const siteHash = hashValue(siteId).slice(0, 16);
96
+ const documentHash = hashValue(document.id).slice(0, 32);
97
+ return {
98
+ document,
99
+ externalId: `geistdocs/${siteHash}/${documentHash}/${hash.slice(0, 16)}`,
100
+ hash,
101
+ metadata: createMetadata({
102
+ document,
103
+ hash,
104
+ revision,
105
+ revisionAncestors,
106
+ revisionTimestamp,
107
+ siteId,
108
+ }),
109
+ };
110
+ };
111
+ const isFromNewerRevision = (metadata, revision, revisionTimestamp) => Boolean(revision && metadata.revisionAncestors.includes(revision)) ||
112
+ (revisionTimestamp !== undefined &&
113
+ metadata.revisionTimestamp !== undefined &&
114
+ metadata.revisionTimestamp > revisionTimestamp);
115
+ const canDeleteForRevision = ({ currentRevision, currentRevisionAncestors, currentRevisionTimestamp, desired, metadata, }) => {
116
+ if (desired?.hash === metadata.contentHash) {
117
+ return true;
118
+ }
119
+ if (!(currentRevision && metadata.revision)) {
120
+ return false;
121
+ }
122
+ return (metadata.revision === currentRevision ||
123
+ currentRevisionAncestors.includes(metadata.revision) ||
124
+ (currentRevisionTimestamp !== undefined &&
125
+ metadata.revisionTimestamp !== undefined &&
126
+ metadata.revisionTimestamp < currentRevisionTimestamp));
127
+ };
128
+ const defaultSleep = (milliseconds) => new Promise((resolveSleep) => {
129
+ setTimeout(resolveSleep, milliseconds);
130
+ });
131
+ const listAllStoreFiles = async (client) => {
132
+ const files = [];
133
+ const seenCursors = new Set();
134
+ let cursor;
135
+ do {
136
+ const page = await client.listStoreFiles(cursor);
137
+ files.push(...page.files);
138
+ cursor = page.nextCursor;
139
+ if (cursor) {
140
+ if (seenCursors.has(cursor)) {
141
+ throw new Error("Mixedbread file listing repeated a pagination cursor.");
142
+ }
143
+ seenCursors.add(cursor);
144
+ }
145
+ } while (cursor);
146
+ return files;
147
+ };
148
+ const runWithConcurrency = async (inputs, concurrency, processInput) => {
149
+ const results = [];
150
+ const errors = [];
151
+ let nextIndex = 0;
152
+ const worker = async () => {
153
+ while (nextIndex < inputs.length) {
154
+ const index = nextIndex;
155
+ nextIndex += 1;
156
+ try {
157
+ results[index] = await processInput(inputs[index]);
158
+ }
159
+ catch (error) {
160
+ errors.push(error);
161
+ }
162
+ }
163
+ };
164
+ await Promise.all(Array.from({ length: Math.min(Math.max(1, concurrency), inputs.length) }, worker));
165
+ if (errors.length > 0) {
166
+ const details = errors
167
+ .map((error) => (error instanceof Error ? error.message : String(error)))
168
+ .join("\n");
169
+ throw new AggregateError(errors, `Mixedbread search sync did not complete:\n${details}`);
170
+ }
171
+ return results;
172
+ };
173
+ const waitForStoreFile = async ({ client, file, pollIntervalMs, pollTimeoutMs, sleep, }) => {
174
+ let current = file;
175
+ const deadline = Date.now() + pollTimeoutMs;
176
+ while (current.status === "pending" || current.status === "in_progress") {
177
+ if (Date.now() >= deadline) {
178
+ throw new Error(`Mixedbread timed out while indexing file ${file.id}.`);
179
+ }
180
+ await sleep(pollIntervalMs);
181
+ current = await client.retrieveStoreFile(file.id);
182
+ }
183
+ if (current.status !== "completed") {
184
+ throw new Error(`Mixedbread failed to index file ${file.id} (${current.status}).`);
185
+ }
186
+ return current;
187
+ };
188
+ const uploadDocument = async ({ client, desired, pollIntervalMs, pollTimeoutMs, sleep, }) => {
189
+ const fileId = await client.uploadFile({
190
+ content: desired.document.content,
191
+ filename: `${hashValue(desired.document.id).slice(0, 32)}.md`,
192
+ });
193
+ let storeFile;
194
+ try {
195
+ storeFile = await client.createStoreFile({
196
+ externalId: desired.externalId,
197
+ fileId,
198
+ metadata: {
199
+ ...desired.metadata,
200
+ [mixedbreadMetadataKeys.rawFileId]: fileId,
201
+ },
202
+ });
203
+ }
204
+ catch (error) {
205
+ try {
206
+ await client.deleteFile(fileId);
207
+ }
208
+ catch (cleanupError) {
209
+ throw new AggregateError([error, cleanupError], `Mixedbread failed to attach and clean up raw file ${fileId}.`);
210
+ }
211
+ throw error;
212
+ }
213
+ return waitForStoreFile({
214
+ client,
215
+ file: storeFile,
216
+ pollIntervalMs,
217
+ pollTimeoutMs,
218
+ sleep,
219
+ });
220
+ };
221
+ export const syncSearchExport = async ({ client, concurrency = DEFAULT_CONCURRENCY, manifest, pollIntervalMs = DEFAULT_POLL_INTERVAL_MS, pollTimeoutMs = DEFAULT_POLL_TIMEOUT_MS, revision, revisionAncestors = [], revisionTimestamp, sleep = defaultSleep, }) => {
222
+ const validatedManifest = parseSearchExport(manifest);
223
+ if (validatedManifest.provider !== "mixedbread") {
224
+ return {
225
+ deleted: 0,
226
+ skipped: validatedManifest.documents.length,
227
+ uploaded: 0,
228
+ };
229
+ }
230
+ const siteId = validatedManifest.siteId?.trim();
231
+ if (!siteId) {
232
+ throw new Error("Mixedbread retrieval requires a non-empty siteId in geistdocs.tsx.");
233
+ }
234
+ if (validatedManifest.documents.length === 0) {
235
+ throw new Error("The Geistdocs search export is empty. Refusing to remove indexed documents.");
236
+ }
237
+ const desiredDocuments = validatedManifest.documents.map((document) => createDesiredDocument(document, siteId, revision, revisionAncestors, revisionTimestamp));
238
+ const desiredById = new Map(desiredDocuments.map((desired) => [desired.document.id, desired]));
239
+ const remoteFiles = await listAllStoreFiles(client);
240
+ const ownedFiles = remoteFiles.flatMap((file) => {
241
+ const metadata = readGeistdocsMixedbreadMetadata(file.metadata);
242
+ return metadata?.siteId === siteId ? [{ file, metadata }] : [];
243
+ });
244
+ const existingCurrentFiles = new Map();
245
+ const newerDocumentIds = new Set();
246
+ for (const owned of ownedFiles) {
247
+ if (isFromNewerRevision(owned.metadata, revision, revisionTimestamp)) {
248
+ newerDocumentIds.add(owned.metadata.documentId);
249
+ }
250
+ const desired = desiredById.get(owned.metadata.documentId);
251
+ if (desired &&
252
+ owned.metadata.contentHash === desired.hash &&
253
+ owned.file.status === "completed" &&
254
+ !existingCurrentFiles.has(owned.metadata.documentId)) {
255
+ existingCurrentFiles.set(owned.metadata.documentId, owned);
256
+ }
257
+ }
258
+ const documentsToUpload = desiredDocuments.filter((desired) => !(existingCurrentFiles.has(desired.document.id) ||
259
+ newerDocumentIds.has(desired.document.id)));
260
+ await runWithConcurrency(documentsToUpload, concurrency, (desired) => uploadDocument({
261
+ client,
262
+ desired,
263
+ pollIntervalMs,
264
+ pollTimeoutMs,
265
+ sleep,
266
+ }));
267
+ const latestRemoteFiles = await listAllStoreFiles(client);
268
+ const latestOwnedFiles = latestRemoteFiles.flatMap((file) => {
269
+ const metadata = readGeistdocsMixedbreadMetadata(file.metadata);
270
+ return metadata?.siteId === siteId ? [{ file, metadata }] : [];
271
+ });
272
+ const supersededDocumentIds = new Set(latestOwnedFiles
273
+ .filter(({ metadata }) => isFromNewerRevision(metadata, revision, revisionTimestamp))
274
+ .map(({ metadata }) => metadata.documentId));
275
+ const keepFileIds = new Set();
276
+ for (const desired of desiredDocuments) {
277
+ if (supersededDocumentIds.has(desired.document.id)) {
278
+ continue;
279
+ }
280
+ const currentFiles = latestOwnedFiles
281
+ .filter(({ file, metadata }) => file.status === "completed" &&
282
+ metadata.documentId === desired.document.id &&
283
+ metadata.contentHash === desired.hash)
284
+ .sort((left, right) => left.file.id.localeCompare(right.file.id));
285
+ const survivor = currentFiles[0];
286
+ if (survivor) {
287
+ keepFileIds.add(survivor.file.id);
288
+ }
289
+ }
290
+ const filesToDelete = latestOwnedFiles.filter(({ file, metadata }) => file.status !== "pending" &&
291
+ file.status !== "in_progress" &&
292
+ !isFromNewerRevision(metadata, revision, revisionTimestamp) &&
293
+ canDeleteForRevision({
294
+ currentRevision: revision,
295
+ currentRevisionAncestors: revisionAncestors,
296
+ currentRevisionTimestamp: revisionTimestamp,
297
+ desired: desiredById.get(metadata.documentId),
298
+ metadata,
299
+ }) &&
300
+ !keepFileIds.has(file.id));
301
+ await runWithConcurrency(filesToDelete, concurrency, async ({ file, metadata }) => {
302
+ // Store files keep the raw file uploaded before attachment. Delete both
303
+ // owned resources so replacements do not accumulate storage.
304
+ await client.deleteFile(metadata.rawFileId ?? file.id);
305
+ await client.deleteStoreFile(file.id);
306
+ });
307
+ return {
308
+ deleted: filesToDelete.length,
309
+ skipped: desiredDocuments.length - documentsToUpload.length,
310
+ uploaded: documentsToUpload.length,
311
+ };
312
+ };
@@ -1,5 +1,6 @@
1
1
  import { type LanguageModel } from "ai";
2
2
  import { type GeistdocsConfig } from "../config";
3
+ import { type ChatMixedbreadOptions } from "../internal/chat-retrieval";
3
4
  import type { GeistdocsSourceBundle } from "../source";
4
5
  type SourceBundle = GeistdocsSourceBundle;
5
6
  interface ChatProxyConfig {
@@ -18,12 +19,17 @@ interface ChatEveAgentOptions {
18
19
  url?: string;
19
20
  }
20
21
  interface CreateChatRouteOptions {
21
- config: Pick<GeistdocsConfig, "ai" | "basePath" | "defaultLanguage" | "translations">;
22
+ config: Pick<GeistdocsConfig, "ai" | "basePath" | "defaultLanguage" | "siteId" | "translations">;
22
23
  /**
23
24
  * Server-side options for the eve agent responder configured through
24
25
  * `config.ai.eveAgent`. Mutually exclusive with `proxy`.
25
26
  */
26
27
  eveAgent?: ChatEveAgentOptions;
28
+ /**
29
+ * Server-only Mixedbread overrides. Environment variables are used when
30
+ * these values are omitted.
31
+ */
32
+ mixedbread?: ChatMixedbreadOptions;
27
33
  /**
28
34
  * Model used to answer Ask AI requests in the default AI Gateway mode.
29
35
  * Accepts any AI SDK `LanguageModel`, including AI Gateway model strings
@@ -36,7 +42,7 @@ interface CreateChatRouteOptions {
36
42
  source?: SourceBundle;
37
43
  sources?: SourceBundle[];
38
44
  }
39
- export declare const createChatRoute: ({ config, eveAgent, model, proxy, source, sources, }: CreateChatRouteOptions) => {
45
+ export declare const createChatRoute: ({ config, eveAgent, mixedbread, model, proxy, source, sources, }: CreateChatRouteOptions) => {
40
46
  POST: (req: Request) => Promise<Response>;
41
47
  maxDuration: 800;
42
48
  };
@@ -1 +1 @@
1
- {"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/routes/chat.ts"],"names":[],"mappings":"AACA,OAAO,EAIL,KAAK,aAAa,EAQnB,MAAM,IAAI,CAAC;AAGZ,OAAO,EAAE,KAAK,eAAe,EAAiB,MAAM,WAAW,CAAC;AAShE,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAEvD,KAAK,YAAY,GAAG,qBAAqB,CAAC;AAyB1C,UAAU,eAAe;IACvB,OAAO,CAAC,EAAE,WAAW,GAAG,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;IACnE,GAAG,EAAE,MAAM,CAAC;CACb;AAED,UAAU,mBAAmB;IAC3B;;;;;OAKG;IACH,OAAO,CAAC,EAAE,WAAW,GAAG,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;IACnE,qEAAqE;IACrE,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,UAAU,sBAAsB;IAC9B,MAAM,EAAE,IAAI,CACV,eAAe,EACf,IAAI,GAAG,UAAU,GAAG,iBAAiB,GAAG,cAAc,CACvD,CAAC;IACF;;;OAGG;IACH,QAAQ,CAAC,EAAE,mBAAmB,CAAC;IAC/B;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;CAC1B;AAm0BD,eAAO,MAAM,eAAe,GAAI,sDAO7B,sBAAsB;gBASE,OAAO;;CA0JjC,CAAC"}
1
+ {"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../src/routes/chat.ts"],"names":[],"mappings":"AACA,OAAO,EAIL,KAAK,aAAa,EAQnB,MAAM,IAAI,CAAC;AAGZ,OAAO,EAAE,KAAK,eAAe,EAAiB,MAAM,WAAW,CAAC;AAChE,OAAO,EAEL,KAAK,qBAAqB,EAG3B,MAAM,4BAA4B,CAAC;AASpC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAEvD,KAAK,YAAY,GAAG,qBAAqB,CAAC;AAyB1C,UAAU,eAAe;IACvB,OAAO,CAAC,EAAE,WAAW,GAAG,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;IACnE,GAAG,EAAE,MAAM,CAAC;CACb;AAED,UAAU,mBAAmB;IAC3B;;;;;OAKG;IACH,OAAO,CAAC,EAAE,WAAW,GAAG,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;IACnE,qEAAqE;IACrE,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,UAAU,sBAAsB;IAC9B,MAAM,EAAE,IAAI,CACV,eAAe,EACf,IAAI,GAAG,UAAU,GAAG,iBAAiB,GAAG,QAAQ,GAAG,cAAc,CAClE,CAAC;IACF;;;OAGG;IACH,QAAQ,CAAC,EAAE,mBAAmB,CAAC;IAC/B;;;OAGG;IACH,UAAU,CAAC,EAAE,qBAAqB,CAAC;IACnC;;;;;;OAMG;IACH,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,KAAK,CAAC,EAAE,eAAe,CAAC;IACxB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,OAAO,CAAC,EAAE,YAAY,EAAE,CAAC;CAC1B;AA82BD,eAAO,MAAM,eAAe,GAAI,kEAQ7B,sBAAsB;gBAcE,OAAO;;CA6JjC,CAAC"}
@@ -3,6 +3,7 @@ import { convertToModelMessages, createUIMessageStream, createUIMessageStreamRes
3
3
  import { initAdvancedSearch } from "fumadocs-core/search/server";
4
4
  import { z } from "zod";
5
5
  import { getPublicPath } from "../config";
6
+ import { createChatDocumentSearch, } from "../internal/chat-retrieval";
6
7
  import { buildEveAgentTranscript, capLastUserMessageText, getEveAgentSessionHandle, resolveEveAgentUrl, runEveAgentTurn, } from "../internal/eve-agent";
7
8
  import { isPageVisibleForSurface } from "../page-visibility";
8
9
  const DEFAULT_CHAT_MODEL = "openai/gpt-4.1-mini";
@@ -95,7 +96,7 @@ const createSearchServer = (sources, lang) => {
95
96
  }));
96
97
  return initAdvancedSearch({ indexes });
97
98
  };
98
- const resolveSearchResult = ({ basePath, lang, sources, url, }) => {
99
+ const resolveSearchResult = ({ basePath, content, lang, sources, url, }) => {
99
100
  const result = sources
100
101
  .map((source) => source.source.getPageByHref(url, { language: lang }))
101
102
  .find((entry) => entry?.page);
@@ -107,17 +108,23 @@ const resolveSearchResult = ({ basePath, lang, sources, url, }) => {
107
108
  return {
108
109
  title: data.title ?? page.url,
109
110
  description: data.description ?? "",
110
- content: getStructuredContent(page),
111
+ content: content?.trim() || getStructuredContent(page),
111
112
  slug: page.url,
112
113
  url: getPublicPath(page.url, basePath),
113
114
  };
114
115
  };
115
- const searchDocs = async ({ basePath, lang, limit, query, sources, }) => {
116
+ const searchDocsWithOrama = async ({ basePath, lang, limit, query, sources, }) => {
116
117
  const searchServer = createSearchServer(sources, lang);
117
118
  const results = await searchServer.search(query);
118
119
  const seenSlugs = new Set();
119
120
  return results
120
- .map(({ url }) => resolveSearchResult({ basePath, lang, sources, url }))
121
+ .map(({ content, type, url }) => resolveSearchResult({
122
+ basePath,
123
+ content: type === "text" ? content : undefined,
124
+ lang,
125
+ sources,
126
+ url,
127
+ }))
121
128
  .filter((doc) => Boolean(doc))
122
129
  .filter((doc) => {
123
130
  if (seenSlugs.has(doc.slug)) {
@@ -128,6 +135,33 @@ const searchDocs = async ({ basePath, lang, limit, query, sources, }) => {
128
135
  })
129
136
  .slice(0, limit);
130
137
  };
138
+ const createDocumentSearch = ({ config, mixedbread, sources, }) => {
139
+ const fallback = ({ lang, limit, query }) => searchDocsWithOrama({
140
+ basePath: config.basePath,
141
+ lang,
142
+ limit,
143
+ query,
144
+ sources,
145
+ });
146
+ return createChatDocumentSearch({
147
+ fallback,
148
+ mixedbread: {
149
+ ...mixedbread,
150
+ apiKey: mixedbread?.apiKey ?? process.env.MXBAI_API_KEY,
151
+ storeId: mixedbread?.storeId ?? process.env.MXBAI_STORE_ID,
152
+ },
153
+ onWarning: (message, error) => {
154
+ if (error === undefined) {
155
+ console.warn(message);
156
+ }
157
+ else {
158
+ console.warn(message, error);
159
+ }
160
+ },
161
+ provider: config.ai?.retrieval,
162
+ siteId: config.siteId,
163
+ });
164
+ };
131
165
  const formatDocForContext = (doc) => `**${doc.title}**\nURL: ${doc.url}\n${doc.description || ""}\n\n${doc.content.slice(0, MAX_CONTEXT_CHARS)}${doc.content.length > MAX_CONTEXT_CHARS ? "..." : ""}`;
132
166
  const getRouteCandidates = ({ basePath, currentRoute, lang, }) => {
133
167
  const pathname = normalizePathname(currentRoute);
@@ -184,7 +218,7 @@ const createSourceChunks = (docs) => docs.map((doc, index) => ({
184
218
  url: doc.url,
185
219
  title: doc.title,
186
220
  }));
187
- const buildProxyMessages = async ({ actualMessages, config, currentRoute, defaultLanguage, languages, pageContext, routeSources, }) => {
221
+ const buildProxyMessages = async ({ actualMessages, config, currentRoute, defaultLanguage, languages, pageContext, routeSources, searchDocuments, }) => {
188
222
  if (!isFirstUserMessage(actualMessages)) {
189
223
  return { messages: actualMessages, sourceDocs: [] };
190
224
  }
@@ -207,12 +241,10 @@ const buildProxyMessages = async ({ actualMessages, config, currentRoute, defaul
207
241
  sources: routeSources,
208
242
  });
209
243
  const relatedDocs = userQuestion.trim()
210
- ? await searchDocs({
211
- basePath: config.basePath,
244
+ ? await searchDocuments({
212
245
  lang,
213
246
  limit: MAX_PROXY_RELATED_PAGES,
214
247
  query: userQuestion,
215
- sources: routeSources,
216
248
  })
217
249
  : [];
218
250
  const filteredRelatedDocs = relatedDocs.filter((doc) => doc.slug !== currentPageDoc?.slug);
@@ -378,7 +410,7 @@ You are a helpful assistant specializing in answering questions strictly. If inf
378
410
  - Be friendly, clear, and specific.`;
379
411
  return [basePrompt, config.ai?.prompt].filter(Boolean).join("\n\n");
380
412
  };
381
- const createTools = (writer, sources, defaultLanguage, basePath) => {
413
+ const createTools = (writer, sources, searchDocuments, defaultLanguage, basePath) => {
382
414
  const log = (message) => {
383
415
  console.log(`[Geistdocs chat] ${message}`);
384
416
  };
@@ -395,12 +427,10 @@ const createTools = (writer, sources, defaultLanguage, basePath) => {
395
427
  execute: async ({ query, lang }) => {
396
428
  try {
397
429
  log(`Creating search server for language: ${lang}`);
398
- const trimmedResults = await searchDocs({
399
- basePath,
430
+ const trimmedResults = await searchDocuments({
400
431
  lang,
401
432
  limit: MAX_TOOL_RESULTS,
402
433
  query,
403
- sources,
404
434
  });
405
435
  if (trimmedResults.length === 0) {
406
436
  return `No documentation found for query: "${query}"`;
@@ -479,7 +509,7 @@ const createChatErrorResponse = (error, status) => new Response(JSON.stringify({
479
509
  status,
480
510
  headers: { "Content-Type": "application/json" },
481
511
  });
482
- const createEveAgentResponse = async ({ actualMessages, agentUrl, config, currentRoute, defaultLanguage, eveAgent, languages, messages, pageContext, req, routeSources, }) => {
512
+ const createEveAgentResponse = async ({ actualMessages, agentUrl, config, currentRoute, defaultLanguage, eveAgent, languages, messages, pageContext, req, routeSources, searchDocuments, }) => {
483
513
  const cappedMessages = capLastUserMessageText(actualMessages, MAX_EVE_AGENT_MESSAGE_CHARS);
484
514
  const { messages: contextMessages, sourceDocs } = await buildProxyMessages({
485
515
  actualMessages: cappedMessages,
@@ -489,6 +519,7 @@ const createEveAgentResponse = async ({ actualMessages, agentUrl, config, curren
489
519
  languages,
490
520
  pageContext,
491
521
  routeSources,
522
+ searchDocuments,
492
523
  });
493
524
  const lastMessage = contextMessages.at(-1);
494
525
  if (lastMessage?.role !== "user") {
@@ -530,9 +561,14 @@ const resolveSources = (source, sources) => {
530
561
  }
531
562
  return resolvedSources;
532
563
  };
533
- export const createChatRoute = ({ config, eveAgent, model = DEFAULT_CHAT_MODEL, proxy, source, sources, }) => {
564
+ export const createChatRoute = ({ config, eveAgent, mixedbread, model = DEFAULT_CHAT_MODEL, proxy, source, sources, }) => {
534
565
  const { defaultLanguage, languages } = getLanguages(config);
535
566
  const routeSources = resolveSources(source, sources);
567
+ const searchDocuments = createDocumentSearch({
568
+ config,
569
+ mixedbread,
570
+ sources: routeSources,
571
+ });
536
572
  const eveAgentUrl = resolveEveAgentUrl({
537
573
  configUrl: config.ai?.eveAgent?.url,
538
574
  hasProxy: Boolean(proxy),
@@ -564,6 +600,7 @@ export const createChatRoute = ({ config, eveAgent, model = DEFAULT_CHAT_MODEL,
564
600
  pageContext,
565
601
  req,
566
602
  routeSources,
603
+ searchDocuments,
567
604
  });
568
605
  }
569
606
  if (proxy) {
@@ -575,6 +612,7 @@ export const createChatRoute = ({ config, eveAgent, model = DEFAULT_CHAT_MODEL,
575
612
  languages,
576
613
  pageContext,
577
614
  routeSources,
615
+ searchDocuments,
578
616
  });
579
617
  const stream = createUIMessageStream({
580
618
  originalMessages: messages,
@@ -630,7 +668,7 @@ export const createChatRoute = ({ config, eveAgent, model = DEFAULT_CHAT_MODEL,
630
668
  model,
631
669
  messages: modelMessages,
632
670
  stopWhen: stepCountIs(10),
633
- tools: createTools(writer, routeSources, defaultLanguage, config.basePath),
671
+ tools: createTools(writer, routeSources, searchDocuments, defaultLanguage, config.basePath),
634
672
  system: createSystemPrompt(currentRoute ?? "/", config),
635
673
  prepareStep: ({ stepNumber }) => {
636
674
  if (stepNumber === 0) {
@@ -0,0 +1,12 @@
1
+ import { type GeistdocsConfig } from "../config";
2
+ import type { GeistdocsSourceBundle } from "../source";
3
+ interface CreateSearchExportRouteOptions {
4
+ config: Pick<GeistdocsConfig, "ai" | "basePath" | "defaultLanguage" | "siteId" | "translations">;
5
+ source?: GeistdocsSourceBundle;
6
+ sources?: GeistdocsSourceBundle[];
7
+ }
8
+ export declare const createSearchExportRoute: ({ config, source, sources, }: CreateSearchExportRouteOptions) => {
9
+ GET: () => Promise<Response>;
10
+ };
11
+ export {};
12
+ //# sourceMappingURL=search-export.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"search-export.d.ts","sourceRoot":"","sources":["../../src/routes/search-export.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,eAAe,EAAiB,MAAM,WAAW,CAAC;AAOhE,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAEvD,UAAU,8BAA8B;IACtC,MAAM,EAAE,IAAI,CACV,eAAe,EACf,IAAI,GAAG,UAAU,GAAG,iBAAiB,GAAG,QAAQ,GAAG,cAAc,CAClE,CAAC;IACF,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B,OAAO,CAAC,EAAE,qBAAqB,EAAE,CAAC;CACnC;AAwBD,eAAO,MAAM,uBAAuB,GAAI,8BAIrC,8BAA8B;;CAyChC,CAAC"}
@@ -0,0 +1,44 @@
1
+ import { getPublicPath } from "../config";
2
+ import { GEISTDOCS_SEARCH_EXPORT_VERSION, } from "../internal/search-export";
3
+ import { isPageVisibleForSurface } from "../page-visibility";
4
+ const resolveSources = (source, sources) => {
5
+ const resolvedSources = sources ?? (source ? [source] : []);
6
+ if (resolvedSources.length === 0) {
7
+ throw new Error("createSearchExportRoute requires source or sources.");
8
+ }
9
+ return resolvedSources;
10
+ };
11
+ const getLanguages = (config) => {
12
+ const defaultLanguage = config.defaultLanguage ?? "en";
13
+ return Array.from(new Set([defaultLanguage, ...Object.keys(config.translations ?? {})]));
14
+ };
15
+ export const createSearchExportRoute = ({ config, source, sources, }) => {
16
+ const routeSources = resolveSources(source, sources);
17
+ const GET = async () => {
18
+ const documents = await Promise.all(getLanguages(config).flatMap((locale) => routeSources.flatMap((entry, sourceIndex) => {
19
+ const sourceId = entry.id ?? `source-${sourceIndex + 1}`;
20
+ return entry.source
21
+ .getPages(locale)
22
+ .filter((page) => isPageVisibleForSurface(page, "chat"))
23
+ .map(async (page) => ({
24
+ content: await entry.getPageMarkdown(page),
25
+ description: page.data.description ?? "",
26
+ id: `${sourceId}:${locale}:${page.url}`,
27
+ locale,
28
+ sourceId,
29
+ sourceUrl: page.url,
30
+ title: page.data.title ?? page.url,
31
+ url: getPublicPath(page.url, config.basePath),
32
+ }));
33
+ })));
34
+ documents.sort((left, right) => left.id.localeCompare(right.id));
35
+ const exported = {
36
+ documents,
37
+ provider: config.ai?.retrieval ?? "local",
38
+ siteId: config.siteId,
39
+ version: GEISTDOCS_SEARCH_EXPORT_VERSION,
40
+ };
41
+ return Response.json(exported);
42
+ };
43
+ return { GET };
44
+ };
package/docs/agents.md CHANGED
@@ -1,6 +1,6 @@
1
1
  # Geistdocs Package Docs
2
2
 
3
- These docs are bundled with `@vercel/geistdocs` 1.25.2 so agents can understand the installed package without web access.
3
+ These docs are bundled with `@vercel/geistdocs` 1.26.1 so agents can understand the installed package without web access.
4
4
  They are generated from `apps/template/content/docs` during the package build.
5
5
 
6
6
  ## How to use these files
@@ -24,8 +24,8 @@ They are generated from `apps/template/content/docs` during the package build.
24
24
 
25
25
  - [Overview](pages/index.md): Geistdocs is a packaged documentation system for creating Next.js and Fumadocs sites with shared runtime features.
26
26
  - [Getting Started](pages/getting-started.md): Create a Geistdocs project, run it locally, and edit your first documentation page.
27
- - [Environment Variables](pages/env.md): Environment variables required by Geistdocs for AI chat, proxy mode, and production URL configuration.
28
- - [Deploy to Vercel](pages/deployment.md): Deploy your Geistdocs site to Vercel from the dashboard with the correct project settings.
27
+ - [Environment Variables](pages/env.md): Environment variables required by Geistdocs for AI chat, Mixedbread retrieval, proxy mode, and production URL configuration.
28
+ - [Deploy to Vercel](pages/deployment.md): Deploy a Geistdocs site to Vercel with the required project settings, environment variables, and private-package access.
29
29
  - [Migration guide](pages/migration.md): Move an existing documentation site to @vercel/geistdocs while keeping local content, routing, middleware behavior, and AI-readable surfaces intact.
30
30
  - [Configuration](pages/configuration.md): Configure site metadata, navigation, AI features, page actions, translations, and local adapters.
31
31
  - [Syntax](pages/syntax.md): Supported MDX syntax including text formatting, code blocks, line highlighting, and Mermaid diagrams.
@@ -38,6 +38,7 @@ They are generated from `apps/template/content/docs` during the package build.
38
38
  - [Configure sidebar navigation](pages/guides/nested-navigation.md): Configure sidebar order, section labels, folder landing pages, and deeply nested page trees with meta.json files.
39
39
  - [Edit on GitHub](pages/edit-on-github.md): A direct link on every documentation page that lets readers propose changes through GitHub pull requests.
40
40
  - [Feedback](pages/feedback.md): An interactive feedback widget that collects user sentiment and creates structured GitHub Issues automatically.
41
+ - [Improve Ask AI answers with Mixedbread](pages/mixedbread-retrieval.md): Provision a Mixedbread Store for one Geistdocs consumer site, enable semantic retrieval, and sync documentation during production builds.
41
42
  - [Internationalization](pages/internationalization.md): Serve documentation in multiple languages with automatic translation via the Geistdocs CLI and Fumadocs routing.
42
43
  - [llms.txt](pages/llms-txt.md): A single endpoint that returns all documentation as plain Markdown text following the llms.txt standard.
43
44
  - [Open in Chat](pages/open-in-chat.md): Transfer documentation context to AI chat platforms like ChatGPT, Claude, Cursor, and v0 with a single click.