@farming-labs/docs 0.2.77 → 0.2.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{agent-SRIJ-oF8.mjs → agent-Ba94vwM2.mjs} +4282 -334
- package/dist/{agent-DRKj42ht.d.mts → agent-CHgWJcK_.d.mts} +45 -3
- package/dist/{agent-D4hNGatV.mjs → agent-CM-hzhfh.mjs} +2 -2
- package/dist/{agent-evals-DmFoKy_y.mjs → agent-evals-C8dkgJMN.mjs} +1 -1
- package/dist/{agent-export-CM-p9_YD.mjs → agent-export-DdF-IVfg.mjs} +6 -5
- package/dist/agent-provenance-D8UBpnMc.mjs +463 -0
- package/dist/agent-skills-bundle.d.mts +1 -1
- package/dist/{agent-skills-server-CJ4D0G9P.d.mts → agent-skills-server-zvs0Yknt.d.mts} +2 -2
- package/dist/agent-skills-vite.d.mts +3 -3
- package/dist/{agents-BeLDNCnE.mjs → agents-v0Em2KZo.mjs} +4 -4
- package/dist/cli/index.mjs +14 -14
- package/dist/client/react.d.mts +1 -1
- package/dist/{cloud-ask-ai-BCgHPxew.d.mts → cloud-ask-ai-BwsdF7AQ.d.mts} +1 -1
- package/dist/docs-cloud-server.d.mts +2 -2
- package/dist/{doctor-DhMVKHjp.mjs → doctor-BbrN0rdC.mjs} +8 -8
- package/dist/{golden-evaluations-VzE6juQl.mjs → golden-evaluations-Bk9t-HNq.mjs} +7 -2
- package/dist/index.d.mts +5 -5
- package/dist/index.mjs +5 -5
- package/dist/{mcp-CZkQJXtA.mjs → mcp-DT-WNsA-.mjs} +4 -4
- package/dist/mcp.d.mts +5 -4
- package/dist/mcp.mjs +193 -3
- package/dist/{prompt-references-B6L3AD98.mjs → prompt-references-CoDorsvS.mjs} +2 -2
- package/dist/{retrieval-digest-rponR4jM.d.mts → retrieval-digest-CO3wqeMk.d.mts} +85 -4
- package/dist/{review-nxoV8jIm.mjs → review-DrCaaT7G.mjs} +4 -4
- package/dist/{robots-CgH6Stqn.mjs → robots-C_v2HK43.mjs} +1 -1
- package/dist/{robots-CiQT8mX2.mjs → robots-izJbM9cv.mjs} +2 -2
- package/dist/{search-BTUIkhnT.mjs → search-YiZMZDfb.mjs} +4 -4
- package/dist/server.d.mts +6 -6
- package/dist/server.mjs +5 -5
- package/dist/{sitemap-e0_GeCWJ.mjs → sitemap-NO8UMO6Y.mjs} +4 -4
- package/dist/{sitemap-server-DHaxyPAd.mjs → sitemap-server-Dgxs9V6S.mjs} +1 -1
- package/dist/{standards-discovery-BbslY4id.d.mts → standards-discovery-BWyNgZSs.d.mts} +1 -1
- package/dist/{types-B-5Kzmkn.d.mts → types-CpLnwHak.d.mts} +125 -4
- package/package.json +1 -1
- package/dist/search-Da2AKaFJ.mjs +0 -3992
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { S as upsertPageAgentContractMarkdown, b as renderPageAgentFrontmatterYamlLines, o as findDocsMarkdownSection, p as PAGE_AGENT_CONTRACT_FIELD_SCHEMA, s as parseDocsMarkdownSections } from "./markdown-sections-7OoA7ylx.mjs";
|
|
1
|
+
import { S as upsertPageAgentContractMarkdown, a as findDocsGeneratedAgentContractRanges, b as renderPageAgentFrontmatterYamlLines, d as PAGE_AGENT_CONTRACT_END_MARKER, l as stripDocsGeneratedAgentContractMarkers, m as PAGE_AGENT_CONTRACT_START_MARKER, o as findDocsMarkdownSection, p as PAGE_AGENT_CONTRACT_FIELD_SCHEMA, s as parseDocsMarkdownSections, y as renderPageAgentContractMarkdown } from "./markdown-sections-7OoA7ylx.mjs";
|
|
2
2
|
import { F as isDocsStandardsDiscoveryRequest, I as resolveDocsDiscoveryApiRoute, M as createDocsStandardsResponse, S as DOCS_AGENT_MANIFEST_FORMAT, T as DOCS_AGENT_MANIFEST_VERSION, _ as DEFAULT_API_CATALOG_FORMAT, a as DEFAULT_A2A_AGENT_CARD_ROUTE, b as DEFAULT_LEGACY_SKILLS_INDEX_ROUTE, c as DEFAULT_AGENT_SKILLS_ARCHIVE_ROUTE_PATTERN, d as DEFAULT_AGENT_SKILLS_ROUTE_PATTERN, k as buildDocsApiCatalog, l as DEFAULT_AGENT_SKILLS_INDEX_FORMAT, n as API_CATALOG_MEDIA_TYPE, r as API_CATALOG_PROFILE_URI, t as AGENT_SKILLS_DISCOVERY_SCHEMA_URI, u as DEFAULT_AGENT_SKILLS_INDEX_ROUTE, v as DEFAULT_API_CATALOG_ROUTE, w as DOCS_AGENT_MANIFEST_SCHEMA_URI } from "./standards-discovery-BKlEnK_H.mjs";
|
|
3
3
|
import { createProcessor } from "@mdx-js/mdx";
|
|
4
4
|
|
|
@@ -149,254 +149,586 @@ function normalizeInlineText(value) {
|
|
|
149
149
|
}
|
|
150
150
|
|
|
151
151
|
//#endregion
|
|
152
|
-
//#region src/
|
|
153
|
-
const
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
152
|
+
//#region src/retrieval-digest.ts
|
|
153
|
+
const SHA256_INITIAL_STATE = [
|
|
154
|
+
1779033703,
|
|
155
|
+
3144134277,
|
|
156
|
+
1013904242,
|
|
157
|
+
2773480762,
|
|
158
|
+
1359893119,
|
|
159
|
+
2600822924,
|
|
160
|
+
528734635,
|
|
161
|
+
1541459225
|
|
162
|
+
];
|
|
163
|
+
const SHA256_ROUND_CONSTANTS = [
|
|
164
|
+
1116352408,
|
|
165
|
+
1899447441,
|
|
166
|
+
3049323471,
|
|
167
|
+
3921009573,
|
|
168
|
+
961987163,
|
|
169
|
+
1508970993,
|
|
170
|
+
2453635748,
|
|
171
|
+
2870763221,
|
|
172
|
+
3624381080,
|
|
173
|
+
310598401,
|
|
174
|
+
607225278,
|
|
175
|
+
1426881987,
|
|
176
|
+
1925078388,
|
|
177
|
+
2162078206,
|
|
178
|
+
2614888103,
|
|
179
|
+
3248222580,
|
|
180
|
+
3835390401,
|
|
181
|
+
4022224774,
|
|
182
|
+
264347078,
|
|
183
|
+
604807628,
|
|
184
|
+
770255983,
|
|
185
|
+
1249150122,
|
|
186
|
+
1555081692,
|
|
187
|
+
1996064986,
|
|
188
|
+
2554220882,
|
|
189
|
+
2821834349,
|
|
190
|
+
2952996808,
|
|
191
|
+
3210313671,
|
|
192
|
+
3336571891,
|
|
193
|
+
3584528711,
|
|
194
|
+
113926993,
|
|
195
|
+
338241895,
|
|
196
|
+
666307205,
|
|
197
|
+
773529912,
|
|
198
|
+
1294757372,
|
|
199
|
+
1396182291,
|
|
200
|
+
1695183700,
|
|
201
|
+
1986661051,
|
|
202
|
+
2177026350,
|
|
203
|
+
2456956037,
|
|
204
|
+
2730485921,
|
|
205
|
+
2820302411,
|
|
206
|
+
3259730800,
|
|
207
|
+
3345764771,
|
|
208
|
+
3516065817,
|
|
209
|
+
3600352804,
|
|
210
|
+
4094571909,
|
|
211
|
+
275423344,
|
|
212
|
+
430227734,
|
|
213
|
+
506948616,
|
|
214
|
+
659060556,
|
|
215
|
+
883997877,
|
|
216
|
+
958139571,
|
|
217
|
+
1322822218,
|
|
218
|
+
1537002063,
|
|
219
|
+
1747873779,
|
|
220
|
+
1955562222,
|
|
221
|
+
2024104815,
|
|
222
|
+
2227730452,
|
|
223
|
+
2361852424,
|
|
224
|
+
2428436474,
|
|
225
|
+
2756734187,
|
|
226
|
+
3204031479,
|
|
227
|
+
3329325298
|
|
228
|
+
];
|
|
229
|
+
function rotateRight(value, bits) {
|
|
230
|
+
return value >>> bits | value << 32 - bits;
|
|
162
231
|
}
|
|
163
|
-
function
|
|
164
|
-
|
|
165
|
-
const normalized = normalizeUrlPath(`/${value.replace(/^\/+|\/+$/g, "")}`);
|
|
166
|
-
return normalized === "/" ? "" : normalized;
|
|
232
|
+
function normalizeDigestContent(value) {
|
|
233
|
+
return value.replace(/\r\n?/gu, "\n").replace(/^\uFEFF/u, "").trimEnd();
|
|
167
234
|
}
|
|
168
|
-
|
|
169
|
-
|
|
235
|
+
/** Validate the bounded HTTP(S) URL or root-relative URI reference used by provenance. */
|
|
236
|
+
function isDocsRetrievalCanonicalUrl(value) {
|
|
237
|
+
const hasUnsafeCharacter = value.includes("\\") || Array.from(value).some((character) => {
|
|
238
|
+
const codePoint = character.codePointAt(0) ?? 0;
|
|
239
|
+
return codePoint <= 31 || codePoint === 127;
|
|
240
|
+
});
|
|
241
|
+
if (!value || value !== value.trim() || value.length > 4096 || hasUnsafeCharacter) return false;
|
|
242
|
+
if (/^\/(?!\/)/u.test(value)) try {
|
|
243
|
+
new URL(value, "https://docs.local");
|
|
244
|
+
return true;
|
|
245
|
+
} catch {
|
|
246
|
+
return false;
|
|
247
|
+
}
|
|
248
|
+
try {
|
|
249
|
+
const url = new URL(value);
|
|
250
|
+
return (url.protocol === "http:" || url.protocol === "https:") && !url.username && !url.password;
|
|
251
|
+
} catch {
|
|
252
|
+
return false;
|
|
253
|
+
}
|
|
170
254
|
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
255
|
+
/**
|
|
256
|
+
* Hash a retrieval source projection with portable SHA-256.
|
|
257
|
+
*
|
|
258
|
+
* The projection is normalized by removing one leading BOM, converting CRLF/CR
|
|
259
|
+
* line endings to LF, and trimming trailing whitespace. The helper is exported so
|
|
260
|
+
* agents can independently verify provenance in Node, edge, and browser runtimes.
|
|
261
|
+
*/
|
|
262
|
+
function digestDocsRetrievalContent(value) {
|
|
263
|
+
const input = new TextEncoder().encode(normalizeDigestContent(value));
|
|
264
|
+
const paddedLength = Math.ceil((input.length + 9) / 64) * 64;
|
|
265
|
+
const padded = new Uint8Array(paddedLength);
|
|
266
|
+
padded.set(input);
|
|
267
|
+
padded[input.length] = 128;
|
|
268
|
+
const bitLength = input.length * 8;
|
|
269
|
+
const dataView = new DataView(padded.buffer);
|
|
270
|
+
dataView.setUint32(paddedLength - 8, Math.floor(bitLength / 4294967296), false);
|
|
271
|
+
dataView.setUint32(paddedLength - 4, bitLength >>> 0, false);
|
|
272
|
+
const state = Uint32Array.from(SHA256_INITIAL_STATE);
|
|
273
|
+
const words = new Uint32Array(64);
|
|
274
|
+
for (let offset = 0; offset < padded.length; offset += 64) {
|
|
275
|
+
for (let index = 0; index < 16; index += 1) words[index] = dataView.getUint32(offset + index * 4, false);
|
|
276
|
+
for (let index = 16; index < 64; index += 1) {
|
|
277
|
+
const previous15 = words[index - 15];
|
|
278
|
+
const previous2 = words[index - 2];
|
|
279
|
+
const sigma0 = rotateRight(previous15, 7) ^ rotateRight(previous15, 18) ^ previous15 >>> 3;
|
|
280
|
+
const sigma1 = rotateRight(previous2, 17) ^ rotateRight(previous2, 19) ^ previous2 >>> 10;
|
|
281
|
+
words[index] = words[index - 16] + sigma0 + words[index - 7] + sigma1 >>> 0;
|
|
282
|
+
}
|
|
283
|
+
let a = state[0];
|
|
284
|
+
let b = state[1];
|
|
285
|
+
let c = state[2];
|
|
286
|
+
let d = state[3];
|
|
287
|
+
let e = state[4];
|
|
288
|
+
let f = state[5];
|
|
289
|
+
let g = state[6];
|
|
290
|
+
let h = state[7];
|
|
291
|
+
for (let index = 0; index < 64; index += 1) {
|
|
292
|
+
const sum1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
|
|
293
|
+
const choice = e & f ^ ~e & g;
|
|
294
|
+
const temporary1 = h + sum1 + choice + SHA256_ROUND_CONSTANTS[index] + words[index] >>> 0;
|
|
295
|
+
const temporary2 = (rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22)) + (a & b ^ a & c ^ b & c) >>> 0;
|
|
296
|
+
h = g;
|
|
297
|
+
g = f;
|
|
298
|
+
f = e;
|
|
299
|
+
e = d + temporary1 >>> 0;
|
|
300
|
+
d = c;
|
|
301
|
+
c = b;
|
|
302
|
+
b = a;
|
|
303
|
+
a = temporary1 + temporary2 >>> 0;
|
|
304
|
+
}
|
|
305
|
+
state[0] = state[0] + a >>> 0;
|
|
306
|
+
state[1] = state[1] + b >>> 0;
|
|
307
|
+
state[2] = state[2] + c >>> 0;
|
|
308
|
+
state[3] = state[3] + d >>> 0;
|
|
309
|
+
state[4] = state[4] + e >>> 0;
|
|
310
|
+
state[5] = state[5] + f >>> 0;
|
|
311
|
+
state[6] = state[6] + g >>> 0;
|
|
312
|
+
state[7] = state[7] + h >>> 0;
|
|
313
|
+
}
|
|
314
|
+
return `sha256:${Array.from(state, (word) => word.toString(16).padStart(8, "0")).join("")}`;
|
|
180
315
|
}
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
316
|
+
|
|
317
|
+
//#endregion
|
|
318
|
+
//#region src/pagination.ts
|
|
319
|
+
const DOCS_PAGINATION_CURSOR_VERSION = 1;
|
|
320
|
+
const MAX_DOCS_PAGINATION_CURSOR_CHARS = 4096;
|
|
321
|
+
const MAX_DOCS_PAGINATION_STATE_CHARS = 1024;
|
|
322
|
+
const MAX_DOCS_PAGINATION_OFFSET = 1e7;
|
|
323
|
+
const DOCS_PAGINATION_DIGEST_PATTERN = /^sha256:[a-f\d]{64}$/u;
|
|
324
|
+
var DocsPaginationCursorError = class extends Error {
|
|
325
|
+
code = "invalid_cursor";
|
|
326
|
+
constructor() {
|
|
327
|
+
super("Invalid or stale pagination cursor.");
|
|
328
|
+
this.name = "DocsPaginationCursorError";
|
|
329
|
+
}
|
|
330
|
+
};
|
|
331
|
+
function paginationDigest(label, value) {
|
|
332
|
+
return digestDocsRetrievalContent(`${label}\0${value}`);
|
|
184
333
|
}
|
|
185
|
-
function
|
|
186
|
-
|
|
187
|
-
if (config === void 0) return true;
|
|
188
|
-
if (config === true) return true;
|
|
189
|
-
return config.enabled !== false;
|
|
334
|
+
function cursorChecksum(payload) {
|
|
335
|
+
return paginationDigest("docs-pagination-checksum.v1", JSON.stringify(payload));
|
|
190
336
|
}
|
|
191
|
-
function
|
|
192
|
-
const
|
|
193
|
-
|
|
194
|
-
const
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
337
|
+
function encodeBase64Url(value) {
|
|
338
|
+
const bytes = new TextEncoder().encode(value);
|
|
339
|
+
let binary = "";
|
|
340
|
+
for (const byte of bytes) binary += String.fromCharCode(byte);
|
|
341
|
+
return btoa(binary).replace(/\+/gu, "-").replace(/\//gu, "_").replace(/=+$/u, "");
|
|
342
|
+
}
|
|
343
|
+
function decodeBase64Url(value) {
|
|
344
|
+
const padded = `${value.replace(/-/gu, "+").replace(/_/gu, "/")}${"=".repeat((4 - value.length % 4) % 4)}`;
|
|
345
|
+
const binary = atob(padded);
|
|
346
|
+
const bytes = Uint8Array.from(binary, (character) => character.charCodeAt(0));
|
|
347
|
+
return new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
348
|
+
}
|
|
349
|
+
function isDigest(value) {
|
|
350
|
+
return typeof value === "string" && DOCS_PAGINATION_DIGEST_PATTERN.test(value);
|
|
351
|
+
}
|
|
352
|
+
function assertCursorOptions(options) {
|
|
353
|
+
if (!/^[a-z][a-z\d._/-]{0,127}$/u.test(options.kind) || typeof options.scope !== "string" || typeof options.snapshot !== "string") throw new TypeError("Pagination cursor options must contain a valid kind, scope, and snapshot.");
|
|
354
|
+
}
|
|
355
|
+
function createDocsPaginationCursor(offset, options) {
|
|
356
|
+
assertCursorOptions(options);
|
|
357
|
+
if (!Number.isSafeInteger(offset) || offset <= 0 || offset > MAX_DOCS_PAGINATION_OFFSET) throw new TypeError("Pagination cursor offsets must be positive safe integers.");
|
|
358
|
+
if (options.state !== void 0 && (options.state.length === 0 || options.state.length > MAX_DOCS_PAGINATION_STATE_CHARS)) throw new TypeError("Pagination provider state must be a bounded, non-empty string.");
|
|
359
|
+
const unsigned = [
|
|
360
|
+
DOCS_PAGINATION_CURSOR_VERSION,
|
|
361
|
+
options.kind,
|
|
362
|
+
offset,
|
|
363
|
+
paginationDigest("docs-pagination-scope.v1", options.scope),
|
|
364
|
+
paginationDigest("docs-pagination-snapshot.v1", options.snapshot),
|
|
365
|
+
options.state ?? null
|
|
366
|
+
];
|
|
367
|
+
return encodeBase64Url(JSON.stringify([...unsigned, cursorChecksum(unsigned)]));
|
|
368
|
+
}
|
|
369
|
+
function resolveDocsPaginationCursor(cursor, options) {
|
|
370
|
+
assertCursorOptions(options);
|
|
371
|
+
if (cursor === void 0) return { offset: 0 };
|
|
372
|
+
if (cursor.length === 0 || cursor.length > MAX_DOCS_PAGINATION_CURSOR_CHARS || !/^[A-Za-z\d_-]+$/u.test(cursor)) throw new DocsPaginationCursorError();
|
|
373
|
+
let parsed;
|
|
374
|
+
try {
|
|
375
|
+
parsed = JSON.parse(decodeBase64Url(cursor));
|
|
376
|
+
} catch {
|
|
377
|
+
throw new DocsPaginationCursorError();
|
|
378
|
+
}
|
|
379
|
+
if (!Array.isArray(parsed) || parsed.length !== 7 || encodeBase64Url(JSON.stringify(parsed)) !== cursor) throw new DocsPaginationCursorError();
|
|
380
|
+
const [version, kind, offset, scopeDigest, snapshotDigest, state, checksum] = parsed;
|
|
381
|
+
const unsigned = [
|
|
382
|
+
version,
|
|
383
|
+
kind,
|
|
384
|
+
offset,
|
|
385
|
+
scopeDigest,
|
|
386
|
+
snapshotDigest,
|
|
387
|
+
state
|
|
388
|
+
];
|
|
389
|
+
if (version !== DOCS_PAGINATION_CURSOR_VERSION || kind !== options.kind || !Number.isSafeInteger(offset) || offset <= 0 || offset > MAX_DOCS_PAGINATION_OFFSET || !isDigest(scopeDigest) || !isDigest(snapshotDigest) || !(state === null || typeof state === "string" && state.length > 0 && state.length <= MAX_DOCS_PAGINATION_STATE_CHARS) || !isDigest(checksum) || scopeDigest !== paginationDigest("docs-pagination-scope.v1", options.scope) || snapshotDigest !== paginationDigest("docs-pagination-snapshot.v1", options.snapshot) || checksum !== cursorChecksum(unsigned)) throw new DocsPaginationCursorError();
|
|
198
390
|
return {
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
baseUrl: normalizeBaseUrl(objectConfig.baseUrl ?? defaults.baseUrl),
|
|
202
|
-
manifestPath: objectConfig.manifestPath ?? DEFAULT_SITEMAP_MANIFEST_PATH,
|
|
203
|
-
xml: {
|
|
204
|
-
enabled: xmlEnabled,
|
|
205
|
-
includeLastmod: typeof xmlConfig === "object" ? xmlConfig.includeLastmod ?? true : true,
|
|
206
|
-
route: joinRoute(routePrefix, DEFAULT_SITEMAP_XML_ROUTE)
|
|
207
|
-
},
|
|
208
|
-
markdown: {
|
|
209
|
-
enabled: markdownEnabled,
|
|
210
|
-
includeDescriptions: typeof markdownConfig === "object" ? markdownConfig.includeDescriptions ?? true : true,
|
|
211
|
-
includeLastmod: typeof markdownConfig === "object" ? markdownConfig.includeLastmod ?? true : true,
|
|
212
|
-
linkTarget: typeof markdownConfig === "object" ? markdownConfig.linkTarget ?? "both" : "both",
|
|
213
|
-
route: joinRoute(routePrefix, DEFAULT_SITEMAP_MD_ROUTE),
|
|
214
|
-
docsRoute: routePrefix ? void 0 : DEFAULT_SITEMAP_MD_DOCS_ROUTE,
|
|
215
|
-
wellKnownRoute: joinRoute(routePrefix, DEFAULT_SITEMAP_MD_WELL_KNOWN_ROUTE)
|
|
216
|
-
}
|
|
391
|
+
offset,
|
|
392
|
+
...typeof state === "string" ? { state } : {}
|
|
217
393
|
};
|
|
218
394
|
}
|
|
219
|
-
function
|
|
220
|
-
|
|
221
|
-
const format = url.searchParams.get("format")?.trim();
|
|
222
|
-
if (pathname === resolveDocsDiscoveryApiRoute(options.apiRoute)) {
|
|
223
|
-
if (format === "sitemap-xml") return "xml";
|
|
224
|
-
if (format === "sitemap-md" || format === "sitemap-markdown") return "markdown";
|
|
225
|
-
}
|
|
226
|
-
const resolved = resolveDocsSitemapConfig(sitemap);
|
|
227
|
-
if (!resolved.enabled) return null;
|
|
228
|
-
if (resolved.xml.enabled && pathname === resolved.xml.route) return "xml";
|
|
229
|
-
if (resolved.markdown.enabled && (pathname === resolved.markdown.route || pathname === resolved.markdown.docsRoute || pathname === resolved.markdown.wellKnownRoute)) return "markdown";
|
|
230
|
-
return null;
|
|
395
|
+
function resolveDocsPaginationOffset(cursor, options) {
|
|
396
|
+
return resolveDocsPaginationCursor(cursor, options).offset;
|
|
231
397
|
}
|
|
232
|
-
function
|
|
233
|
-
|
|
234
|
-
|
|
398
|
+
function paginateDocsItems(items, options) {
|
|
399
|
+
if (!Number.isSafeInteger(options.pageSize) || options.pageSize <= 0) throw new TypeError("Pagination page sizes must be positive safe integers.");
|
|
400
|
+
const offset = resolveDocsPaginationOffset(options.cursor, options);
|
|
401
|
+
if (offset >= items.length && options.cursor !== void 0) throw new DocsPaginationCursorError();
|
|
402
|
+
const pageItems = items.slice(offset, offset + options.pageSize);
|
|
403
|
+
const nextOffset = offset + pageItems.length;
|
|
404
|
+
const hasMore = nextOffset < items.length;
|
|
405
|
+
return {
|
|
406
|
+
items: pageItems,
|
|
407
|
+
resultCount: pageItems.length,
|
|
408
|
+
total: items.length,
|
|
409
|
+
hasMore,
|
|
410
|
+
...hasMore ? { nextCursor: createDocsPaginationCursor(nextOffset, options) } : {}
|
|
411
|
+
};
|
|
235
412
|
}
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
413
|
+
|
|
414
|
+
//#endregion
|
|
415
|
+
//#region src/agent-scope.ts
|
|
416
|
+
/** Normalize framework aliases used by page metadata, agent contracts, MCP, and evaluations. */
|
|
417
|
+
function normalizeAgentFramework(value) {
|
|
418
|
+
const normalized = value.toLowerCase().replace(/[^a-z0-9]/gu, "");
|
|
419
|
+
if ([
|
|
420
|
+
"next",
|
|
421
|
+
"nextjs",
|
|
422
|
+
"nextjsapp",
|
|
423
|
+
"reactnext"
|
|
424
|
+
].includes(normalized)) return "nextjs";
|
|
425
|
+
if ([
|
|
426
|
+
"tanstack",
|
|
427
|
+
"tanstackstart",
|
|
428
|
+
"start"
|
|
429
|
+
].includes(normalized)) return "tanstackstart";
|
|
430
|
+
if (["svelte", "sveltekit"].includes(normalized)) return "sveltekit";
|
|
431
|
+
if (["nuxt", "nuxtjs"].includes(normalized)) return "nuxt";
|
|
432
|
+
return normalized;
|
|
433
|
+
}
|
|
434
|
+
function normalizeAgentLocale(value) {
|
|
435
|
+
return value.trim().toLowerCase().replace(/_/gu, "-");
|
|
436
|
+
}
|
|
437
|
+
function normalizeAgentVersion(value) {
|
|
438
|
+
return value.trim().toLowerCase().replace(/^v(?=\d)/u, "");
|
|
439
|
+
}
|
|
440
|
+
function normalizeAgentScopeValues(value) {
|
|
441
|
+
const values = Array.isArray(value) ? value : typeof value === "string" ? [value] : [];
|
|
442
|
+
return Array.from(new Set(values.map((item) => item.trim()).filter(Boolean)));
|
|
443
|
+
}
|
|
444
|
+
function compareVersions(left, right) {
|
|
445
|
+
for (let index = 0; index < 3; index += 1) if (left[index] !== right[index]) return left[index] > right[index] ? 1 : -1;
|
|
446
|
+
return 0;
|
|
447
|
+
}
|
|
448
|
+
function parseExactVersion(value) {
|
|
449
|
+
const match = normalizeAgentVersion(value).match(/^(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:-[0-9a-z.-]+)?(?:\+[0-9a-z.-]+)?$/iu);
|
|
450
|
+
if (!match) return void 0;
|
|
451
|
+
return [
|
|
452
|
+
Number(match[1]),
|
|
453
|
+
Number(match[2] ?? 0),
|
|
454
|
+
Number(match[3] ?? 0)
|
|
455
|
+
];
|
|
240
456
|
}
|
|
241
|
-
function
|
|
242
|
-
|
|
457
|
+
function parseVersionOperand(value) {
|
|
458
|
+
const normalized = normalizeAgentVersion(value);
|
|
459
|
+
if (normalized === "*" || normalized === "x") return {
|
|
460
|
+
version: [
|
|
461
|
+
0,
|
|
462
|
+
0,
|
|
463
|
+
0
|
|
464
|
+
],
|
|
465
|
+
components: 0,
|
|
466
|
+
wildcard: true
|
|
467
|
+
};
|
|
468
|
+
const match = normalized.match(/^(\d+)(?:\.(\d+|x|\*))?(?:\.(\d+|x|\*))?$/iu);
|
|
469
|
+
if (!match) return void 0;
|
|
470
|
+
const raw = [
|
|
471
|
+
match[1],
|
|
472
|
+
match[2],
|
|
473
|
+
match[3]
|
|
474
|
+
];
|
|
475
|
+
const wildcardIndex = raw.findIndex((part, index) => index > 0 && (part?.toLowerCase() === "x" || part === "*"));
|
|
476
|
+
if (wildcardIndex >= 0 && raw.slice(wildcardIndex + 1).some((part) => part && part.toLowerCase() !== "x" && part !== "*")) return;
|
|
477
|
+
const components = wildcardIndex >= 0 ? wildcardIndex : raw.filter(Boolean).length;
|
|
478
|
+
return {
|
|
479
|
+
version: [
|
|
480
|
+
Number(raw[0]),
|
|
481
|
+
Number(raw[1] ?? 0) || 0,
|
|
482
|
+
Number(raw[2] ?? 0) || 0
|
|
483
|
+
],
|
|
484
|
+
components,
|
|
485
|
+
wildcard: wildcardIndex >= 0
|
|
486
|
+
};
|
|
243
487
|
}
|
|
244
|
-
function
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
488
|
+
function nextVersionBoundary(operand) {
|
|
489
|
+
const [major, minor] = operand.version;
|
|
490
|
+
if (operand.components <= 0) return void 0;
|
|
491
|
+
if (operand.components === 1) return [
|
|
492
|
+
major + 1,
|
|
493
|
+
0,
|
|
494
|
+
0
|
|
495
|
+
];
|
|
496
|
+
return [
|
|
497
|
+
major,
|
|
498
|
+
minor + 1,
|
|
499
|
+
0
|
|
500
|
+
];
|
|
252
501
|
}
|
|
253
|
-
function
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
502
|
+
function operandRange(operand) {
|
|
503
|
+
if (operand.components === 0) return {
|
|
504
|
+
minimumInclusive: true,
|
|
505
|
+
maximumInclusive: false
|
|
506
|
+
};
|
|
507
|
+
if (operand.components < 3 || operand.wildcard) return {
|
|
508
|
+
minimum: operand.version,
|
|
509
|
+
minimumInclusive: true,
|
|
510
|
+
maximum: nextVersionBoundary(operand),
|
|
511
|
+
maximumInclusive: false
|
|
512
|
+
};
|
|
513
|
+
return {
|
|
514
|
+
minimum: operand.version,
|
|
515
|
+
minimumInclusive: true,
|
|
516
|
+
maximum: operand.version,
|
|
517
|
+
maximumInclusive: true
|
|
518
|
+
};
|
|
261
519
|
}
|
|
262
|
-
function
|
|
263
|
-
|
|
520
|
+
function caretRange(operand) {
|
|
521
|
+
const [major, minor, patch] = operand.version;
|
|
522
|
+
let maximum;
|
|
523
|
+
if (major > 0) maximum = [
|
|
524
|
+
major + 1,
|
|
525
|
+
0,
|
|
526
|
+
0
|
|
527
|
+
];
|
|
528
|
+
else if (operand.components <= 1) maximum = [
|
|
529
|
+
1,
|
|
530
|
+
0,
|
|
531
|
+
0
|
|
532
|
+
];
|
|
533
|
+
else if (minor > 0 || operand.components === 2) maximum = [
|
|
534
|
+
0,
|
|
535
|
+
minor + 1,
|
|
536
|
+
0
|
|
537
|
+
];
|
|
538
|
+
else maximum = [
|
|
539
|
+
0,
|
|
540
|
+
0,
|
|
541
|
+
patch + 1
|
|
542
|
+
];
|
|
543
|
+
return {
|
|
544
|
+
minimum: operand.version,
|
|
545
|
+
minimumInclusive: true,
|
|
546
|
+
maximum,
|
|
547
|
+
maximumInclusive: false
|
|
548
|
+
};
|
|
264
549
|
}
|
|
265
|
-
function
|
|
266
|
-
const
|
|
267
|
-
const
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
550
|
+
function tildeRange(operand) {
|
|
551
|
+
const [major, minor] = operand.version;
|
|
552
|
+
const maximum = operand.components <= 1 ? [
|
|
553
|
+
major + 1,
|
|
554
|
+
0,
|
|
555
|
+
0
|
|
556
|
+
] : [
|
|
557
|
+
major,
|
|
558
|
+
minor + 1,
|
|
559
|
+
0
|
|
560
|
+
];
|
|
561
|
+
return {
|
|
562
|
+
minimum: operand.version,
|
|
563
|
+
minimumInclusive: true,
|
|
564
|
+
maximum,
|
|
565
|
+
maximumInclusive: false
|
|
566
|
+
};
|
|
271
567
|
}
|
|
272
|
-
function
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
description: page.description,
|
|
289
|
-
sourcePath: page.sourcePath,
|
|
290
|
-
lastmod,
|
|
291
|
-
lastmodSource: resolvedLastmod?.lastmodSource ?? (lastmod ? "unknown" : void 0),
|
|
292
|
-
related: page.related?.map((link) => link.href)
|
|
293
|
-
});
|
|
568
|
+
function intersectRanges(left, right) {
|
|
569
|
+
let minimum = left.minimum;
|
|
570
|
+
let minimumInclusive = left.minimumInclusive;
|
|
571
|
+
if (!minimum || right.minimum && compareVersions(right.minimum, minimum) > 0) {
|
|
572
|
+
minimum = right.minimum;
|
|
573
|
+
minimumInclusive = right.minimumInclusive;
|
|
574
|
+
} else if (right.minimum && compareVersions(right.minimum, minimum) === 0) minimumInclusive = minimumInclusive && right.minimumInclusive;
|
|
575
|
+
let maximum = left.maximum;
|
|
576
|
+
let maximumInclusive = left.maximumInclusive;
|
|
577
|
+
if (!maximum || right.maximum && compareVersions(right.maximum, maximum) < 0) {
|
|
578
|
+
maximum = right.maximum;
|
|
579
|
+
maximumInclusive = right.maximumInclusive;
|
|
580
|
+
} else if (right.maximum && compareVersions(right.maximum, maximum) === 0) maximumInclusive = maximumInclusive && right.maximumInclusive;
|
|
581
|
+
if (minimum && maximum) {
|
|
582
|
+
const compared = compareVersions(minimum, maximum);
|
|
583
|
+
if (compared > 0 || compared === 0 && !(minimumInclusive && maximumInclusive)) return void 0;
|
|
294
584
|
}
|
|
295
|
-
pages.sort((left, right) => left.url.localeCompare(right.url));
|
|
296
585
|
return {
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
siteTitle: options.siteTitle,
|
|
302
|
-
pages
|
|
586
|
+
minimum,
|
|
587
|
+
minimumInclusive,
|
|
588
|
+
maximum,
|
|
589
|
+
maximumInclusive
|
|
303
590
|
};
|
|
304
591
|
}
|
|
305
|
-
function
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
592
|
+
function comparatorRange(operator, operand) {
|
|
593
|
+
if (!operator || operator === "=") return operandRange(operand);
|
|
594
|
+
if (operator === ">=") return {
|
|
595
|
+
minimum: operand.version,
|
|
596
|
+
minimumInclusive: true,
|
|
597
|
+
maximumInclusive: false
|
|
598
|
+
};
|
|
599
|
+
if (operator === ">") {
|
|
600
|
+
const partialBoundary = operand.components < 3 ? nextVersionBoundary(operand) : void 0;
|
|
601
|
+
return {
|
|
602
|
+
minimum: partialBoundary ?? operand.version,
|
|
603
|
+
minimumInclusive: Boolean(partialBoundary),
|
|
604
|
+
maximumInclusive: false
|
|
605
|
+
};
|
|
315
606
|
}
|
|
316
|
-
|
|
317
|
-
|
|
607
|
+
if (operator === "<") return {
|
|
608
|
+
minimumInclusive: true,
|
|
609
|
+
maximum: operand.version,
|
|
610
|
+
maximumInclusive: false
|
|
611
|
+
};
|
|
612
|
+
const partialBoundary = operand.components < 3 ? nextVersionBoundary(operand) : void 0;
|
|
613
|
+
return {
|
|
614
|
+
minimumInclusive: true,
|
|
615
|
+
maximum: partialBoundary ?? operand.version,
|
|
616
|
+
maximumInclusive: !partialBoundary
|
|
617
|
+
};
|
|
318
618
|
}
|
|
319
|
-
function
|
|
320
|
-
const
|
|
321
|
-
|
|
322
|
-
const
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
619
|
+
function parseVersionRangeBranch(value) {
|
|
620
|
+
const branch = normalizeAgentVersion(value).trim();
|
|
621
|
+
if (!branch) return void 0;
|
|
622
|
+
const hyphen = branch.match(/^(.+?)\s+-\s+(.+)$/u);
|
|
623
|
+
if (hyphen) {
|
|
624
|
+
const minimumOperand = parseVersionOperand(hyphen[1]);
|
|
625
|
+
const maximumOperand = parseVersionOperand(hyphen[2]);
|
|
626
|
+
if (!minimumOperand || !maximumOperand) return void 0;
|
|
627
|
+
const maximumBoundary = maximumOperand.components < 3 ? nextVersionBoundary(maximumOperand) : maximumOperand.version;
|
|
628
|
+
return intersectRanges({
|
|
629
|
+
minimum: minimumOperand.version,
|
|
630
|
+
minimumInclusive: true,
|
|
631
|
+
maximumInclusive: false
|
|
632
|
+
}, {
|
|
633
|
+
minimumInclusive: true,
|
|
634
|
+
maximum: maximumBoundary,
|
|
635
|
+
maximumInclusive: maximumOperand.components === 3
|
|
636
|
+
});
|
|
333
637
|
}
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
638
|
+
const special = branch.match(/^(\^|~)\s*(.+)$/u);
|
|
639
|
+
if (special) {
|
|
640
|
+
const operand = parseVersionOperand(special[2]);
|
|
641
|
+
if (!operand || operand.components === 0) return void 0;
|
|
642
|
+
return special[1] === "^" ? caretRange(operand) : tildeRange(operand);
|
|
643
|
+
}
|
|
644
|
+
const comparators = Array.from(branch.matchAll(/(>=|<=|>|<|=)\s*([^\s]+)/gu));
|
|
645
|
+
if (comparators.length > 0) {
|
|
646
|
+
if (comparators.map((match) => match[0]).join(" ").replace(/\s+/gu, " ") !== branch.replace(/\s+/gu, " ")) return void 0;
|
|
647
|
+
let range = {
|
|
648
|
+
minimumInclusive: true,
|
|
649
|
+
maximumInclusive: false
|
|
650
|
+
};
|
|
651
|
+
for (const comparator of comparators) {
|
|
652
|
+
const operand = parseVersionOperand(comparator[2]);
|
|
653
|
+
if (!operand) return void 0;
|
|
654
|
+
const next = intersectRanges(range, comparatorRange(comparator[1], operand));
|
|
655
|
+
if (!next) return void 0;
|
|
656
|
+
range = next;
|
|
345
657
|
}
|
|
658
|
+
return range;
|
|
346
659
|
}
|
|
347
|
-
|
|
660
|
+
const operand = parseVersionOperand(branch);
|
|
661
|
+
return operand ? operandRange(operand) : void 0;
|
|
348
662
|
}
|
|
349
|
-
function
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
663
|
+
function parseVersionRanges(value) {
|
|
664
|
+
return normalizeAgentVersion(value).split("||").map((branch) => parseVersionRangeBranch(branch)).filter((range) => Boolean(range));
|
|
665
|
+
}
|
|
666
|
+
function parseVersionConstraintBranches(value) {
|
|
667
|
+
const ranges = parseVersionRanges(value);
|
|
668
|
+
if (ranges.length > 0) return ranges.map((range) => ({
|
|
669
|
+
kind: "range",
|
|
670
|
+
range
|
|
671
|
+
}));
|
|
672
|
+
const normalized = normalizeAgentVersion(value);
|
|
673
|
+
return normalized ? [{
|
|
674
|
+
kind: "opaque",
|
|
675
|
+
value: normalized
|
|
676
|
+
}] : [];
|
|
677
|
+
}
|
|
678
|
+
function intersectVersionConstraintBranches(left, right) {
|
|
679
|
+
if (left.kind === "opaque" || right.kind === "opaque") return left.kind === "opaque" && right.kind === "opaque" && left.value === right.value ? left : void 0;
|
|
680
|
+
const range = intersectRanges(left.range, right.range);
|
|
681
|
+
return range ? {
|
|
682
|
+
kind: "range",
|
|
683
|
+
range
|
|
684
|
+
} : void 0;
|
|
685
|
+
}
|
|
686
|
+
/**
|
|
687
|
+
* Return true when choosing one alternative from every group can select one shared version.
|
|
688
|
+
*
|
|
689
|
+
* Each inner group is an OR list while the groups themselves are intersected. Computing the
|
|
690
|
+
* accumulated range prevents pairwise matches from accepting three mutually incompatible scopes.
|
|
691
|
+
*/
|
|
692
|
+
function agentVersionConstraintGroupsOverlap(groups) {
|
|
693
|
+
if (groups.length === 0) return false;
|
|
694
|
+
let intersections = groups[0].flatMap(parseVersionConstraintBranches);
|
|
695
|
+
if (intersections.length === 0) return false;
|
|
696
|
+
for (const group of groups.slice(1)) {
|
|
697
|
+
const branches = group.flatMap(parseVersionConstraintBranches);
|
|
698
|
+
if (branches.length === 0) return false;
|
|
699
|
+
intersections = intersections.flatMap((intersection) => branches.map((branch) => intersectVersionConstraintBranches(intersection, branch)).filter((value) => Boolean(value)));
|
|
700
|
+
if (intersections.length === 0) return false;
|
|
357
701
|
}
|
|
702
|
+
return true;
|
|
358
703
|
}
|
|
359
|
-
function
|
|
360
|
-
if (
|
|
361
|
-
|
|
362
|
-
|
|
704
|
+
function rangeContains(range, version) {
|
|
705
|
+
if (range.minimum) {
|
|
706
|
+
const compared = compareVersions(version, range.minimum);
|
|
707
|
+
if (compared < 0 || compared === 0 && !range.minimumInclusive) return false;
|
|
708
|
+
}
|
|
709
|
+
if (range.maximum) {
|
|
710
|
+
const compared = compareVersions(version, range.maximum);
|
|
711
|
+
if (compared > 0 || compared === 0 && !range.maximumInclusive) return false;
|
|
712
|
+
}
|
|
713
|
+
return true;
|
|
363
714
|
}
|
|
364
|
-
function
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
if (
|
|
370
|
-
|
|
371
|
-
if (
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
const
|
|
379
|
-
const
|
|
380
|
-
|
|
381
|
-
includeLastmod: resolved.xml.includeLastmod
|
|
382
|
-
}) : renderDocsSitemapMarkdown(nextManifest, {
|
|
383
|
-
baseUrl: responseBaseUrl,
|
|
384
|
-
includeDescriptions: resolved.markdown.includeDescriptions,
|
|
385
|
-
includeLastmod: resolved.markdown.includeLastmod,
|
|
386
|
-
linkTarget: resolved.markdown.linkTarget
|
|
387
|
-
});
|
|
388
|
-
const headers = new Headers({
|
|
389
|
-
"Content-Type": format === "xml" ? "application/xml; charset=utf-8" : "text/markdown; charset=utf-8",
|
|
390
|
-
"Cache-Control": "public, max-age=0, must-revalidate",
|
|
391
|
-
ETag: hashString(body)
|
|
392
|
-
});
|
|
393
|
-
const lastModified = newestLastmod(nextManifest.pages);
|
|
394
|
-
if (lastModified) headers.set("Last-Modified", (/* @__PURE__ */ new Date(`${lastModified}T00:00:00.000Z`)).toUTCString());
|
|
395
|
-
if (request.headers.get("if-none-match") === headers.get("ETag")) return new Response(null, {
|
|
396
|
-
status: 304,
|
|
397
|
-
headers
|
|
398
|
-
});
|
|
399
|
-
return new Response(body, { headers });
|
|
715
|
+
function rangesOverlap(left, right) {
|
|
716
|
+
return Boolean(intersectRanges(left, right));
|
|
717
|
+
}
|
|
718
|
+
/** Match an exact requested version against an exact or range-like documented constraint. */
|
|
719
|
+
function agentVersionConstraintMatches(requested, constraint) {
|
|
720
|
+
if (normalizeAgentVersion(requested) === normalizeAgentVersion(constraint)) return Boolean(normalizeAgentVersion(requested));
|
|
721
|
+
const wanted = parseExactVersion(requested);
|
|
722
|
+
if (!wanted) return false;
|
|
723
|
+
return parseVersionRanges(constraint).some((range) => rangeContains(range, wanted));
|
|
724
|
+
}
|
|
725
|
+
/** Return true when two exact or range-like documented version constraints can select one version. */
|
|
726
|
+
function agentVersionConstraintsOverlap(left, right) {
|
|
727
|
+
const normalizedLeft = normalizeAgentVersion(left);
|
|
728
|
+
if (normalizedLeft === normalizeAgentVersion(right)) return Boolean(normalizedLeft);
|
|
729
|
+
const leftRanges = parseVersionRanges(left);
|
|
730
|
+
const rightRanges = parseVersionRanges(right);
|
|
731
|
+
return leftRanges.some((leftRange) => rightRanges.some((rightRange) => rangesOverlap(leftRange, rightRange)));
|
|
400
732
|
}
|
|
401
733
|
|
|
402
734
|
//#endregion
|
|
@@ -1927,124 +2259,3682 @@ function extractDocsMarkdownPromptBlocks(content) {
|
|
|
1927
2259
|
if (parent) parent.completedChildren.push(pair);
|
|
1928
2260
|
else pairs.push(pair);
|
|
1929
2261
|
}
|
|
1930
|
-
for (const frame of openings) pairs.push(...frame.completedChildren);
|
|
1931
|
-
pairs.sort((left, right) => left.start - right.start);
|
|
1932
|
-
if (pairs.length === 0) return {
|
|
1933
|
-
blocks: [],
|
|
1934
|
-
markdown: content
|
|
1935
|
-
};
|
|
1936
|
-
const usedTokens = /* @__PURE__ */ new Set();
|
|
1937
|
-
const blocks = pairs.map((pair, index) => {
|
|
1938
|
-
const token = createDocsMarkdownPromptToken(content, usedTokens, index);
|
|
1939
|
-
usedTokens.add(token);
|
|
1940
|
-
return {
|
|
1941
|
-
attributes: pair.attributes,
|
|
1942
|
-
children: pair.children,
|
|
1943
|
-
token
|
|
1944
|
-
};
|
|
1945
|
-
});
|
|
1946
|
-
let markdown = content;
|
|
1947
|
-
for (let index = pairs.length - 1; index >= 0; index -= 1) {
|
|
1948
|
-
const replacement = createDocsMarkdownPromptReplacement(content, pairs[index].start, pairs[index].end, blocks[index].token, pairs[index].flow, pairs[index].container, index > 0 && pairs[index - 1].flow && /^[\t \r\n]*$/.test(content.slice(pairs[index - 1].end, pairs[index].start)), index + 1 < pairs.length && pairs[index + 1].flow && /^[\t \r\n]*$/.test(content.slice(pairs[index].end, pairs[index + 1].start)));
|
|
1949
|
-
markdown = markdown.slice(0, pairs[index].start) + replacement + markdown.slice(pairs[index].end);
|
|
2262
|
+
for (const frame of openings) pairs.push(...frame.completedChildren);
|
|
2263
|
+
pairs.sort((left, right) => left.start - right.start);
|
|
2264
|
+
if (pairs.length === 0) return {
|
|
2265
|
+
blocks: [],
|
|
2266
|
+
markdown: content
|
|
2267
|
+
};
|
|
2268
|
+
const usedTokens = /* @__PURE__ */ new Set();
|
|
2269
|
+
const blocks = pairs.map((pair, index) => {
|
|
2270
|
+
const token = createDocsMarkdownPromptToken(content, usedTokens, index);
|
|
2271
|
+
usedTokens.add(token);
|
|
2272
|
+
return {
|
|
2273
|
+
attributes: pair.attributes,
|
|
2274
|
+
children: pair.children,
|
|
2275
|
+
token
|
|
2276
|
+
};
|
|
2277
|
+
});
|
|
2278
|
+
let markdown = content;
|
|
2279
|
+
for (let index = pairs.length - 1; index >= 0; index -= 1) {
|
|
2280
|
+
const replacement = createDocsMarkdownPromptReplacement(content, pairs[index].start, pairs[index].end, blocks[index].token, pairs[index].flow, pairs[index].container, index > 0 && pairs[index - 1].flow && /^[\t \r\n]*$/.test(content.slice(pairs[index - 1].end, pairs[index].start)), index + 1 < pairs.length && pairs[index + 1].flow && /^[\t \r\n]*$/.test(content.slice(pairs[index].end, pairs[index + 1].start)));
|
|
2281
|
+
markdown = markdown.slice(0, pairs[index].start) + replacement + markdown.slice(pairs[index].end);
|
|
2282
|
+
}
|
|
2283
|
+
return {
|
|
2284
|
+
blocks,
|
|
2285
|
+
markdown
|
|
2286
|
+
};
|
|
2287
|
+
}
|
|
2288
|
+
function normalizeProjectedWhitespace(content) {
|
|
2289
|
+
const protectedRanges = findProtectedRanges(content);
|
|
2290
|
+
return content.replace(/\n{3,}/g, (newlines, offset) => {
|
|
2291
|
+
const end = offset + newlines.length;
|
|
2292
|
+
return protectedRanges.some((range) => range.start < end && range.end > offset) ? newlines : "\n\n";
|
|
2293
|
+
}).trim();
|
|
2294
|
+
}
|
|
2295
|
+
/** Find audience declarations that cannot be projected consistently at build time. */
|
|
2296
|
+
function findDocsAudienceMdxIssues(content) {
|
|
2297
|
+
const issues = [];
|
|
2298
|
+
for (const tag of findDocsAudienceMdxTags(content)) {
|
|
2299
|
+
if (tag.closing) continue;
|
|
2300
|
+
const { attributes } = tag;
|
|
2301
|
+
const hasOnlyAttribute = tag.hasOnlyAttribute ?? /(?:^|\s)only\s*=/.test(attributes);
|
|
2302
|
+
const hasSpreadAttribute = tag.hasSpreadAttribute ?? /\{\s*\.\.\./.test(attributes);
|
|
2303
|
+
if ((tag.name === "Agent" || tag.name === "Human") && hasOnlyAttribute) {
|
|
2304
|
+
const shorthandAudience = tag.name === "Agent" ? "agent" : "human";
|
|
2305
|
+
issues.push({
|
|
2306
|
+
code: tag.name === "Agent" ? "ignored-agent-only" : "ignored-human-only",
|
|
2307
|
+
index: tag.index,
|
|
2308
|
+
message: `<${tag.name}> is always ${shorthandAudience}-only, so its \`only\` prop is ignored. Use <Audience only="${shorthandAudience}"> when you need the explicit form.`
|
|
2309
|
+
});
|
|
2310
|
+
continue;
|
|
2311
|
+
}
|
|
2312
|
+
if (tag.name !== "Audience") continue;
|
|
2313
|
+
if (hasSpreadAttribute) {
|
|
2314
|
+
issues.push({
|
|
2315
|
+
code: "dynamic-only",
|
|
2316
|
+
index: tag.index,
|
|
2317
|
+
message: "Audience spread props cannot be projected consistently. Remove the spread and use the static literal `only=\"human\"` or `only=\"agent\"`."
|
|
2318
|
+
});
|
|
2319
|
+
continue;
|
|
2320
|
+
}
|
|
2321
|
+
if (tag.only) continue;
|
|
2322
|
+
if (!hasOnlyAttribute) {
|
|
2323
|
+
issues.push({
|
|
2324
|
+
code: "missing-only",
|
|
2325
|
+
index: tag.index,
|
|
2326
|
+
message: "<Audience> requires the static literal `only=\"human\"` or `only=\"agent\"`."
|
|
2327
|
+
});
|
|
2328
|
+
continue;
|
|
2329
|
+
}
|
|
2330
|
+
const dynamicOnly = tag.onlyKind === "dynamic" || /(?:^|\s)only\s*=\s*\{/.test(attributes);
|
|
2331
|
+
issues.push({
|
|
2332
|
+
code: dynamicOnly ? "dynamic-only" : "invalid-only",
|
|
2333
|
+
index: tag.index,
|
|
2334
|
+
message: dynamicOnly ? "Dynamic Audience.only expressions cannot be projected consistently. Use the static literal `only=\"human\"` or `only=\"agent\"`." : "Audience.only must be the static literal `\"human\"` or `\"agent\"`."
|
|
2335
|
+
});
|
|
2336
|
+
}
|
|
2337
|
+
return issues;
|
|
2338
|
+
}
|
|
2339
|
+
/**
|
|
2340
|
+
* Resolve MDX into its human or agent projection.
|
|
2341
|
+
*
|
|
2342
|
+
* `<Agent>` is shorthand for `<Audience only="agent">`, while `<Human>` is
|
|
2343
|
+
* shorthand for `<Audience only="human">`. Unknown `Audience.only` values are
|
|
2344
|
+
* treated as shared content so an authoring typo never silently deletes content.
|
|
2345
|
+
* These primitives shape content for each surface; they are not an access-control boundary.
|
|
2346
|
+
*/
|
|
2347
|
+
function resolveDocsAudienceMdxContent(content, audience) {
|
|
2348
|
+
const scopes = [];
|
|
2349
|
+
let output = "";
|
|
2350
|
+
let cursor = 0;
|
|
2351
|
+
let resolvedTag = false;
|
|
2352
|
+
for (const tag of findDocsAudienceMdxTags(content)) {
|
|
2353
|
+
const activeScope = scopes.at(-1);
|
|
2354
|
+
if (tag.closing && activeScope?.name !== tag.name) continue;
|
|
2355
|
+
resolvedTag = true;
|
|
2356
|
+
const visibleBeforeTag = isAudienceVisible(scopes, audience);
|
|
2357
|
+
if (visibleBeforeTag) output += content.slice(cursor, tag.index);
|
|
2358
|
+
cursor = tag.end;
|
|
2359
|
+
if (tag.closing) {
|
|
2360
|
+
if (tag.expression && visibleBeforeTag) output += "</>";
|
|
2361
|
+
scopes.pop();
|
|
2362
|
+
continue;
|
|
2363
|
+
}
|
|
2364
|
+
if (tag.selfClosing) {
|
|
2365
|
+
if (tag.expression && visibleBeforeTag) output += tag.jsxChild ? "{null}" : "null";
|
|
2366
|
+
continue;
|
|
2367
|
+
}
|
|
2368
|
+
scopes.push({
|
|
2369
|
+
name: tag.name,
|
|
2370
|
+
only: tag.only
|
|
2371
|
+
});
|
|
2372
|
+
if (tag.expression && visibleBeforeTag) output += isAudienceVisible(scopes, audience) ? "<>" : tag.jsxChild ? "{null}" : "null";
|
|
2373
|
+
}
|
|
2374
|
+
if (isAudienceVisible(scopes, audience)) output += content.slice(cursor);
|
|
2375
|
+
return resolvedTag ? normalizeProjectedWhitespace(output) : content;
|
|
2376
|
+
}
|
|
2377
|
+
/** Backwards-compatible name retained for existing integrations. */
|
|
2378
|
+
function resolveDocsAgentMdxContent(content, audience) {
|
|
2379
|
+
return resolveDocsAudienceMdxContent(content, audience);
|
|
2380
|
+
}
|
|
2381
|
+
|
|
2382
|
+
//#endregion
|
|
2383
|
+
//#region src/search.ts
|
|
2384
|
+
const DEFAULT_SEARCH_LIMIT = 10;
|
|
2385
|
+
const MAX_STRUCTURED_SEARCH_LIMIT = 25;
|
|
2386
|
+
const MAX_SEARCH_SNIPPET_CHARS = 160;
|
|
2387
|
+
const DEFAULT_MCP_PROTOCOL_VERSION = "2025-11-25";
|
|
2388
|
+
const MCP_SESSION_CLEANUP_TIMEOUT_MS = 1e3;
|
|
2389
|
+
const syncedIndexes = /* @__PURE__ */ new Map();
|
|
2390
|
+
const syncingIndexes = /* @__PURE__ */ new Map();
|
|
2391
|
+
const ALGOLIA_MAX_RECORD_BYTES = 9500;
|
|
2392
|
+
const DEFAULT_ASK_AI_CONTEXT_CHARS = 24e3;
|
|
2393
|
+
const DEFAULT_ASK_AI_RESULT_CHARS = 6e3;
|
|
2394
|
+
const MAX_SEARCH_FILTER_VALUES = 16;
|
|
2395
|
+
const MAX_SEARCH_FILTER_VALUE_CHARS = 128;
|
|
2396
|
+
const MAX_SEARCH_FILTER_RAW_CHARS = 4096;
|
|
2397
|
+
const MAX_SEARCH_FILTER_SEGMENTS = 64;
|
|
2398
|
+
const MAX_SEARCH_PROVIDER_CURSOR_STATE_CHARS = 1024;
|
|
2399
|
+
const MAX_PROVIDER_SCOPE_FILTER_IDS = 1e3;
|
|
2400
|
+
const MAX_PROVIDER_SCOPE_FILTER_CHARS = 16e3;
|
|
2401
|
+
const MAX_HOSTED_PROVIDER_CURSOR_RESULTS = 1e3;
|
|
2402
|
+
const MAX_MCP_SEARCH_CURSOR_REPLAY_PAGES = 64;
|
|
2403
|
+
const MAX_MCP_SEARCH_CURSOR_RESULTS = MAX_MCP_SEARCH_CURSOR_REPLAY_PAGES;
|
|
2404
|
+
const DOCS_UNPAGINATED_PROVIDER_QUERY = Symbol("docs-unpaginated-provider-query");
|
|
2405
|
+
const DOCS_PROVIDER_SCOPE_FILTER_OVERFLOW = Symbol("docs-provider-scope-filter-overflow");
|
|
2406
|
+
const ALGOLIA_BATCH_OPERATIONS = 1e3;
|
|
2407
|
+
const MAX_SEARCH_WARNINGS = 16;
|
|
2408
|
+
const MAX_SEARCH_WARNING_VALUES = 16;
|
|
2409
|
+
const MAX_SEARCH_WARNING_PAGE_URLS = 8;
|
|
2410
|
+
const MAX_SEARCH_FACET_VALUES = 100;
|
|
2411
|
+
const RETRIEVAL_INDEX_FORMAT = "docs-retrieval-index.v1";
|
|
2412
|
+
const MAX_RETRIEVAL_SOURCE_URL_CHARS = 4096;
|
|
2413
|
+
const MAX_RETRIEVAL_SOURCE_VALUE_CHARS = 256;
|
|
2414
|
+
const SEARCH_FILTER_FIELDS = [
|
|
2415
|
+
"framework",
|
|
2416
|
+
"version",
|
|
2417
|
+
"package",
|
|
2418
|
+
"tags"
|
|
2419
|
+
];
|
|
2420
|
+
const SEARCH_AMBIGUITY_FIELDS = [
|
|
2421
|
+
"framework",
|
|
2422
|
+
"version",
|
|
2423
|
+
"package"
|
|
2424
|
+
];
|
|
2425
|
+
const SEARCH_STOP_WORDS = new Set([
|
|
2426
|
+
"a",
|
|
2427
|
+
"an",
|
|
2428
|
+
"and",
|
|
2429
|
+
"are",
|
|
2430
|
+
"as",
|
|
2431
|
+
"at",
|
|
2432
|
+
"be",
|
|
2433
|
+
"can",
|
|
2434
|
+
"do",
|
|
2435
|
+
"does",
|
|
2436
|
+
"for",
|
|
2437
|
+
"from",
|
|
2438
|
+
"how",
|
|
2439
|
+
"i",
|
|
2440
|
+
"in",
|
|
2441
|
+
"is",
|
|
2442
|
+
"it",
|
|
2443
|
+
"of",
|
|
2444
|
+
"on",
|
|
2445
|
+
"or",
|
|
2446
|
+
"the",
|
|
2447
|
+
"this",
|
|
2448
|
+
"to",
|
|
2449
|
+
"use",
|
|
2450
|
+
"what",
|
|
2451
|
+
"when",
|
|
2452
|
+
"where",
|
|
2453
|
+
"which",
|
|
2454
|
+
"with"
|
|
2455
|
+
]);
|
|
2456
|
+
function normalizeDocsSearchFilterValue(field, value) {
|
|
2457
|
+
const bounded = value.trim().slice(0, MAX_SEARCH_FILTER_VALUE_CHARS).trim();
|
|
2458
|
+
if (!bounded) return "";
|
|
2459
|
+
if (field === "framework") return normalizeAgentFramework(bounded);
|
|
2460
|
+
if (field === "version") return normalizeAgentVersion(bounded);
|
|
2461
|
+
return bounded.toLowerCase();
|
|
2462
|
+
}
|
|
2463
|
+
function normalizeDocsSearchFilterValues(field, input) {
|
|
2464
|
+
const values = typeof input === "string" ? [input] : input ?? [];
|
|
2465
|
+
const normalized = [];
|
|
2466
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2467
|
+
let remainingChars = MAX_SEARCH_FILTER_RAW_CHARS;
|
|
2468
|
+
let remainingSegments = MAX_SEARCH_FILTER_SEGMENTS;
|
|
2469
|
+
for (const item of values) {
|
|
2470
|
+
if (remainingChars <= 0 || remainingSegments <= 0) break;
|
|
2471
|
+
const boundedItem = item.slice(0, remainingChars);
|
|
2472
|
+
remainingChars -= boundedItem.length;
|
|
2473
|
+
for (const part of boundedItem.split(",")) {
|
|
2474
|
+
if (remainingSegments <= 0) break;
|
|
2475
|
+
remainingSegments -= 1;
|
|
2476
|
+
const value = normalizeDocsSearchFilterValue(field, part);
|
|
2477
|
+
if (!value || seen.has(value)) continue;
|
|
2478
|
+
seen.add(value);
|
|
2479
|
+
normalized.push(value);
|
|
2480
|
+
if (normalized.length >= MAX_SEARCH_FILTER_VALUES) return normalized;
|
|
2481
|
+
}
|
|
2482
|
+
}
|
|
2483
|
+
return normalized;
|
|
2484
|
+
}
|
|
2485
|
+
/** Normalize scalar or array-valued scope filters for programmatic, HTTP, and MCP callers. */
|
|
2486
|
+
function normalizeDocsSearchFilters(input = {}) {
|
|
2487
|
+
const filters = {};
|
|
2488
|
+
for (const field of SEARCH_FILTER_FIELDS) {
|
|
2489
|
+
const values = normalizeDocsSearchFilterValues(field, input[field]);
|
|
2490
|
+
if (values.length > 0) filters[field] = values;
|
|
2491
|
+
}
|
|
2492
|
+
return filters;
|
|
2493
|
+
}
|
|
2494
|
+
/** Parse the public search scope parameters from a URL query string. */
|
|
2495
|
+
function resolveDocsSearchFilters(searchParams) {
|
|
2496
|
+
return normalizeDocsSearchFilters({
|
|
2497
|
+
framework: searchParams.getAll("framework"),
|
|
2498
|
+
version: searchParams.getAll("version"),
|
|
2499
|
+
package: searchParams.getAll("package"),
|
|
2500
|
+
tags: searchParams.getAll("tags")
|
|
2501
|
+
});
|
|
2502
|
+
}
|
|
2503
|
+
var DocsSearchRequestError = class extends Error {
|
|
2504
|
+
code = "invalid_search_request";
|
|
2505
|
+
constructor(message) {
|
|
2506
|
+
super(message);
|
|
2507
|
+
this.name = "DocsSearchRequestError";
|
|
2508
|
+
}
|
|
2509
|
+
};
|
|
2510
|
+
function resolveDocsSearchError(error) {
|
|
2511
|
+
if (error instanceof DocsPaginationCursorError) return {
|
|
2512
|
+
code: error.code,
|
|
2513
|
+
message: error.message
|
|
2514
|
+
};
|
|
2515
|
+
if (error instanceof DocsSearchRequestError) return {
|
|
2516
|
+
code: error.code,
|
|
2517
|
+
message: error.message
|
|
2518
|
+
};
|
|
2519
|
+
}
|
|
2520
|
+
/** Resolve search filters and the backwards-compatible structured response opt-in. */
|
|
2521
|
+
function resolveDocsSearchRequest(searchParams) {
|
|
2522
|
+
const response = searchParams.get("response");
|
|
2523
|
+
const structured = response === "structured";
|
|
2524
|
+
const facets = response === "facets";
|
|
2525
|
+
const rawFacet = searchParams.get("facet");
|
|
2526
|
+
let facet;
|
|
2527
|
+
if (rawFacet !== null) {
|
|
2528
|
+
if (!facets || !SEARCH_FILTER_FIELDS.includes(rawFacet)) throw new DocsSearchRequestError("Search facet must be one of framework, version, package, or tags and requires `response=facets`.");
|
|
2529
|
+
facet = rawFacet;
|
|
2530
|
+
}
|
|
2531
|
+
const cursor = searchParams.get("cursor") ?? void 0;
|
|
2532
|
+
if (cursor !== void 0 && !structured && !facets) throw new DocsSearchRequestError("Search cursors require `response=structured` or `response=facets`.");
|
|
2533
|
+
if (cursor !== void 0 && facets && facet === void 0) throw new DocsSearchRequestError("Facet continuation cursors require a `facet` field.");
|
|
2534
|
+
if (cursor !== void 0 && (cursor.length === 0 || cursor.length > 4096 || !/^[A-Za-z\d_-]+$/u.test(cursor))) throw new DocsPaginationCursorError();
|
|
2535
|
+
const rawLimit = searchParams.get("limit");
|
|
2536
|
+
let limit;
|
|
2537
|
+
if (rawLimit !== null && (structured || facets)) {
|
|
2538
|
+
const maximum = facets ? MAX_SEARCH_FACET_VALUES : MAX_STRUCTURED_SEARCH_LIMIT;
|
|
2539
|
+
if (!/^\d+$/u.test(rawLimit)) throw new DocsSearchRequestError(`Search limit must be an integer between 1 and ${maximum}.`);
|
|
2540
|
+
limit = Number(rawLimit);
|
|
2541
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > maximum) throw new DocsSearchRequestError(`Search limit must be an integer between 1 and ${maximum}.`);
|
|
2542
|
+
}
|
|
2543
|
+
return {
|
|
2544
|
+
filters: resolveDocsSearchFilters(searchParams),
|
|
2545
|
+
structured,
|
|
2546
|
+
...facets ? { facets: true } : {},
|
|
2547
|
+
...facet !== void 0 ? { facet } : {},
|
|
2548
|
+
...cursor !== void 0 ? { cursor } : {},
|
|
2549
|
+
...limit !== void 0 ? { limit } : {}
|
|
2550
|
+
};
|
|
2551
|
+
}
|
|
2552
|
+
function hasDocsSearchFilters(filters) {
|
|
2553
|
+
return SEARCH_FILTER_FIELDS.some((field) => (filters[field]?.length ?? 0) > 0);
|
|
2554
|
+
}
|
|
2555
|
+
function resolveProviderScopeDocumentIds(query, context, corpusId) {
|
|
2556
|
+
if (!hasDocsSearchFilters(query.filters ?? {})) return void 0;
|
|
2557
|
+
const documents = corpusId ? buildDocsSearchDocuments(context.pages, context.chunking ?? { strategy: "section" }, "human") : context.documents;
|
|
2558
|
+
const ids = Array.from(new Set(documents.map((document) => corpusId ? makeHostedProviderDocumentId(corpusId, document.id) : document.id)));
|
|
2559
|
+
return ids.length <= MAX_PROVIDER_SCOPE_FILTER_IDS ? ids : DOCS_PROVIDER_SCOPE_FILTER_OVERFLOW;
|
|
2560
|
+
}
|
|
2561
|
+
function docsSearchFiltersMatch(expected, actual) {
|
|
2562
|
+
return SEARCH_FILTER_FIELDS.every((field) => {
|
|
2563
|
+
const expectedValues = expected[field] ?? [];
|
|
2564
|
+
const actualValues = actual[field] ?? [];
|
|
2565
|
+
return expectedValues.length === actualValues.length && expectedValues.every((value) => actualValues.includes(value));
|
|
2566
|
+
});
|
|
2567
|
+
}
|
|
2568
|
+
function parseVerifiedMcpSearchFilters(value) {
|
|
2569
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
|
|
2570
|
+
const record = value;
|
|
2571
|
+
const filterFields = SEARCH_FILTER_FIELDS;
|
|
2572
|
+
if (Object.keys(record).some((field) => !filterFields.includes(field))) return;
|
|
2573
|
+
const input = {};
|
|
2574
|
+
for (const field of SEARCH_FILTER_FIELDS) {
|
|
2575
|
+
const values = record[field];
|
|
2576
|
+
if (values === void 0) continue;
|
|
2577
|
+
if (!Array.isArray(values) || values.length > MAX_SEARCH_FILTER_VALUES || values.some((item) => typeof item !== "string" || item.length > MAX_SEARCH_FILTER_VALUE_CHARS)) return;
|
|
2578
|
+
input[field] = values;
|
|
2579
|
+
}
|
|
2580
|
+
const normalized = normalizeDocsSearchFilters(input);
|
|
2581
|
+
for (const field of SEARCH_FILTER_FIELDS) {
|
|
2582
|
+
const values = record[field];
|
|
2583
|
+
if (values === void 0) continue;
|
|
2584
|
+
if (!Array.isArray(values) || values.length !== (normalized[field]?.length ?? 0) || values.some((value, index) => value !== normalized[field]?.[index])) return;
|
|
2585
|
+
}
|
|
2586
|
+
return normalized;
|
|
2587
|
+
}
|
|
2588
|
+
function normalizeDocsSearchMetadataValues(field, value) {
|
|
2589
|
+
const values = normalizeAgentScopeValues(value);
|
|
2590
|
+
return Array.from(new Set(values.map((item) => normalizeDocsSearchFilterValue(field, item)).filter((item) => Boolean(item))));
|
|
2591
|
+
}
|
|
2592
|
+
function valuesOverlap(left, right, matches) {
|
|
2593
|
+
return left.some((leftValue) => right.some((rightValue) => matches(leftValue, rightValue)));
|
|
2594
|
+
}
|
|
2595
|
+
function resolveDocsSearchPageScope(page) {
|
|
2596
|
+
const topFramework = normalizeDocsSearchMetadataValues("framework", page.framework);
|
|
2597
|
+
const contractFramework = normalizeDocsSearchMetadataValues("framework", page.agent?.appliesTo?.framework);
|
|
2598
|
+
const topVersion = normalizeDocsSearchMetadataValues("version", page.version);
|
|
2599
|
+
const contractVersion = normalizeDocsSearchMetadataValues("version", page.agent?.appliesTo?.version);
|
|
2600
|
+
const packageValues = normalizeDocsSearchMetadataValues("package", page.agent?.appliesTo?.package);
|
|
2601
|
+
const tags = normalizeDocsSearchMetadataValues("tags", page.tags);
|
|
2602
|
+
const conflicts = [];
|
|
2603
|
+
if (topFramework.length > 0 && contractFramework.length > 0 && !valuesOverlap(topFramework, contractFramework, (left, right) => left === right)) conflicts.push("framework");
|
|
2604
|
+
if (topVersion.length > 0 && contractVersion.length > 0 && !valuesOverlap(topVersion, contractVersion, agentVersionConstraintsOverlap)) conflicts.push("version");
|
|
2605
|
+
return {
|
|
2606
|
+
framework: topFramework.length > 0 && contractFramework.length > 0 ? topFramework.filter((value) => contractFramework.includes(value)) : Array.from(new Set([...topFramework, ...contractFramework])),
|
|
2607
|
+
version: Array.from(new Set([...topVersion, ...contractVersion])),
|
|
2608
|
+
package: packageValues,
|
|
2609
|
+
tags,
|
|
2610
|
+
declarations: {
|
|
2611
|
+
framework: [topFramework, contractFramework].filter((values) => values.length > 0),
|
|
2612
|
+
version: [topVersion, contractVersion].filter((values) => values.length > 0),
|
|
2613
|
+
package: packageValues.length > 0 ? [packageValues] : [],
|
|
2614
|
+
tags: tags.length > 0 ? [tags] : []
|
|
2615
|
+
},
|
|
2616
|
+
conflicts
|
|
2617
|
+
};
|
|
2618
|
+
}
|
|
2619
|
+
function docsSearchScopeValueMatches(field, requested, candidate) {
|
|
2620
|
+
return field === "version" ? agentVersionConstraintsOverlap(requested, candidate) : requested === candidate;
|
|
2621
|
+
}
|
|
2622
|
+
function docsSearchPageMatchesFilters(scope, filters) {
|
|
2623
|
+
if (scope.conflicts.length > 0) return false;
|
|
2624
|
+
return SEARCH_FILTER_FIELDS.every((field) => {
|
|
2625
|
+
const requested = filters[field];
|
|
2626
|
+
if (!requested || requested.length === 0) return true;
|
|
2627
|
+
return docsSearchScopeFieldMatches(scope, field, requested);
|
|
2628
|
+
});
|
|
2629
|
+
}
|
|
2630
|
+
function docsSearchScopeFieldMatches(scope, field, requested) {
|
|
2631
|
+
const declarations = scope.declarations[field];
|
|
2632
|
+
if (declarations.length === 0) return false;
|
|
2633
|
+
if (field === "version") return agentVersionConstraintGroupsOverlap([requested, ...declarations]);
|
|
2634
|
+
return requested.some((value) => declarations.every((candidates) => candidates.some((candidate) => docsSearchScopeValueMatches(field, value, candidate))));
|
|
2635
|
+
}
|
|
2636
|
+
function stripMarkdownText(content) {
|
|
2637
|
+
return removeMdxModuleLinesOutsideFences(content).replace(/```[^\n]*\n([\s\S]*?)```/g, "$1").replace(/```([\s\S]*?)```/g, "$1").replace(/~~~[^\n]*\n([\s\S]*?)~~~/g, "$1").replace(/~~~([\s\S]*?)~~~/g, "$1").replace(/<[^>]+\/>/g, "").replace(/<\/?[A-Z][^>]*>/g, "").replace(/<\/?[a-z][^>]*>/g, "").replace(/!\[([^\]]*)\]\([^)]+\)/g, "$1").replace(/\[([^\]]+)\]\([^)]+\)/g, "$1").replace(/^#{1,6}\s+/gm, "").replace(/^\|?[\s:-]+(\|[\s:-]+)+\|?\s*$/gm, "").replace(/\|/g, " ").replace(/^[-*+]\s+/gm, "").replace(/(\*{1,3}|_{1,3})(.*?)\1/g, "$2").replace(/`{3,}[^\n]*$/gm, "").replace(/`([^`]+)`/g, "$1").replace(/`+/g, "").replace(/^>\s+/gm, "").replace(/^[-*_]{3,}\s*$/gm, "").replace(/\n{3,}/g, "\n\n").replace(/\s{2,}/g, " ").trim();
|
|
2638
|
+
}
|
|
2639
|
+
function stripHtml(text) {
|
|
2640
|
+
return text.replace(/<[^>]+>/g, "");
|
|
2641
|
+
}
|
|
2642
|
+
function normalizeMcpSsePayload(body) {
|
|
2643
|
+
const payload = body.split("\n").filter((line) => line.startsWith("data: ")).map((line) => line.slice(6).trim()).filter(Boolean).at(-1);
|
|
2644
|
+
return payload ? JSON.parse(payload) : null;
|
|
2645
|
+
}
|
|
2646
|
+
function normalizeWhitespace(value) {
|
|
2647
|
+
return value.replace(/\s+/g, " ").trim();
|
|
2648
|
+
}
|
|
2649
|
+
function normalizeSearchPhrase(value) {
|
|
2650
|
+
return normalizeWhitespace(value.toLowerCase().replace(/[?!.,;:]+$/g, ""));
|
|
2651
|
+
}
|
|
2652
|
+
function escapeRegExp(value) {
|
|
2653
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
2654
|
+
}
|
|
2655
|
+
function literalMatchPriority(query, value) {
|
|
2656
|
+
const q = normalizeSearchPhrase(query);
|
|
2657
|
+
const text = normalizeSearchPhrase(value ?? "");
|
|
2658
|
+
if (!q || !text) return 0;
|
|
2659
|
+
if (text === q) return 2;
|
|
2660
|
+
const boundary = "[^\\p{L}\\p{N}]";
|
|
2661
|
+
return new RegExp(`(^|${boundary})${escapeRegExp(q)}(?=$|${boundary})`, "u").test(text) ? 1 : 0;
|
|
2662
|
+
}
|
|
2663
|
+
function isLiteralLookupQuery(query) {
|
|
2664
|
+
const q = normalizeSearchPhrase(query);
|
|
2665
|
+
const words = tokenizeSearchQuery(q);
|
|
2666
|
+
return words.length > 0 && words.length <= 3 && words.join(" ") === q;
|
|
2667
|
+
}
|
|
2668
|
+
function tokenizeSearchQuery(query) {
|
|
2669
|
+
return Array.from(new Set(query.toLowerCase().replace(/[^\p{L}\p{N}@/_:.-]+/gu, " ").split(/\s+/).map((word) => word.replace(/^[^\p{L}\p{N}@]+|[^\p{L}\p{N}]+$/gu, "")).filter((word) => word.length > 1 && !SEARCH_STOP_WORDS.has(word))));
|
|
2670
|
+
}
|
|
2671
|
+
function normalizeUrlRouteKey(value) {
|
|
2672
|
+
try {
|
|
2673
|
+
const url = new URL(value, "https://docs.local");
|
|
2674
|
+
return `${url.pathname || "/"}${url.search}`;
|
|
2675
|
+
} catch {
|
|
2676
|
+
return value.split("#", 1)[0] || "/";
|
|
2677
|
+
}
|
|
2678
|
+
}
|
|
2679
|
+
function normalizeAuthoredUrlIdentity(value) {
|
|
2680
|
+
const trimmed = value.trim();
|
|
2681
|
+
const explicitScheme = trimmed.match(/^([a-z][a-z\d+.-]*):/iu)?.[1]?.toLowerCase();
|
|
2682
|
+
if (explicitScheme && explicitScheme !== "http" && explicitScheme !== "https") return normalizeUrlRouteKey("/");
|
|
2683
|
+
try {
|
|
2684
|
+
const url = new URL(trimmed, "https://docs.local");
|
|
2685
|
+
const route = `${url.pathname || "/"}${url.search}`;
|
|
2686
|
+
if (explicitScheme) return `${url.origin}${route}`;
|
|
2687
|
+
if (trimmed.startsWith("//")) return `//${url.host}${route}`;
|
|
2688
|
+
return route;
|
|
2689
|
+
} catch {
|
|
2690
|
+
return normalizeUrlRouteKey(trimmed);
|
|
2691
|
+
}
|
|
2692
|
+
}
|
|
2693
|
+
function appendDocsLocaleQuery(value, locale) {
|
|
2694
|
+
const hashIndex = value.indexOf("#");
|
|
2695
|
+
const withoutHash = hashIndex >= 0 ? value.slice(0, hashIndex) : value;
|
|
2696
|
+
const hash = hashIndex >= 0 ? value.slice(hashIndex) : "";
|
|
2697
|
+
const queryIndex = withoutHash.indexOf("?");
|
|
2698
|
+
const rawQuery = queryIndex >= 0 ? withoutHash.slice(queryIndex + 1) : "";
|
|
2699
|
+
if (new URLSearchParams(rawQuery).has("lang")) return value;
|
|
2700
|
+
return `${withoutHash}${queryIndex >= 0 ? "&" : "?"}lang=${encodeURIComponent(locale)}${hash}`;
|
|
2701
|
+
}
|
|
2702
|
+
function localizeDocsSearchPage(page, localeFallback) {
|
|
2703
|
+
const rawLocale = (page.locale ?? localeFallback)?.trim();
|
|
2704
|
+
if (!rawLocale) return page;
|
|
2705
|
+
const locale = rawLocale;
|
|
2706
|
+
const url = appendDocsLocaleQuery(page.url, locale);
|
|
2707
|
+
if (url === page.url && page.locale === locale) return page;
|
|
2708
|
+
return {
|
|
2709
|
+
...page,
|
|
2710
|
+
url,
|
|
2711
|
+
locale
|
|
2712
|
+
};
|
|
2713
|
+
}
|
|
2714
|
+
function safeDecodeUrlSegment(value) {
|
|
2715
|
+
try {
|
|
2716
|
+
return decodeURIComponent(value);
|
|
2717
|
+
} catch {
|
|
2718
|
+
return value;
|
|
2719
|
+
}
|
|
2720
|
+
}
|
|
2721
|
+
function getUrlSearchSegments(value) {
|
|
2722
|
+
let pathname = "";
|
|
2723
|
+
try {
|
|
2724
|
+
pathname = new URL(value, "https://docs.local").pathname;
|
|
2725
|
+
} catch {
|
|
2726
|
+
pathname = value.split(/[?#]/)[0] ?? "";
|
|
2727
|
+
}
|
|
2728
|
+
return Array.from(new Set(pathname.split("/").flatMap((segment) => {
|
|
2729
|
+
const decoded = safeDecodeUrlSegment(segment);
|
|
2730
|
+
return [decoded, decoded.replace(/[-_]+/g, " ")];
|
|
2731
|
+
}).map(normalizeSearchPhrase).filter(Boolean)));
|
|
2732
|
+
}
|
|
2733
|
+
function resolveAskAIContextUrl(value, baseUrl) {
|
|
2734
|
+
if (!baseUrl) return value;
|
|
2735
|
+
try {
|
|
2736
|
+
return new URL(value, baseUrl).toString();
|
|
2737
|
+
} catch {
|
|
2738
|
+
return value;
|
|
2739
|
+
}
|
|
2740
|
+
}
|
|
2741
|
+
function cleanGeneratedAgentContractMarkers(content) {
|
|
2742
|
+
return stripDocsGeneratedAgentContractMarkers(content);
|
|
2743
|
+
}
|
|
2744
|
+
function getAskAIPageSectionContent(page) {
|
|
2745
|
+
return getPageAudienceSectionContent(page, "agent");
|
|
2746
|
+
}
|
|
2747
|
+
function getAskAIPageContent(page) {
|
|
2748
|
+
return cleanGeneratedAgentContractMarkers(getAskAIPageSectionContent(page));
|
|
2749
|
+
}
|
|
2750
|
+
function getPageAgentContractSearchText(page) {
|
|
2751
|
+
return stripMarkdownText(renderPageAgentContractMarkdown(page.agent).replace(PAGE_AGENT_CONTRACT_START_MARKER, "").replace(PAGE_AGENT_CONTRACT_END_MARKER, ""));
|
|
2752
|
+
}
|
|
2753
|
+
function removeMdxModuleLinesOutsideFences(content) {
|
|
2754
|
+
let inFence = false;
|
|
2755
|
+
return content.split("\n").filter((line) => {
|
|
2756
|
+
const trimmed = line.trimStart();
|
|
2757
|
+
if (trimmed.startsWith("```") || trimmed.startsWith("~~~")) {
|
|
2758
|
+
inFence = !inFence;
|
|
2759
|
+
return true;
|
|
2760
|
+
}
|
|
2761
|
+
return inFence || !/^(import|export)\s/.test(trimmed);
|
|
2762
|
+
}).join("\n");
|
|
2763
|
+
}
|
|
2764
|
+
function cleanAskAIContextMarkdown(content) {
|
|
2765
|
+
return removeMdxModuleLinesOutsideFences(content).replace(/<[^>]+\/>/g, "").replace(/<\/?[A-Z][^>]*>/g, "").replace(/<\/?[a-z][^>]*>/g, "").replace(/\n{3,}/g, "\n\n").trim();
|
|
2766
|
+
}
|
|
2767
|
+
function packageRootFromSpecifier(specifier) {
|
|
2768
|
+
if (!specifier || specifier.startsWith(".") || specifier.startsWith("/") || specifier.startsWith("@/") || specifier.startsWith("~/") || specifier.startsWith("#")) return null;
|
|
2769
|
+
const parts = specifier.split("/").filter(Boolean);
|
|
2770
|
+
if (parts.length === 0) return null;
|
|
2771
|
+
if (parts[0]?.startsWith("@")) return parts.length > 1 ? `${parts[0]}/${parts[1]}` : null;
|
|
2772
|
+
return parts[0];
|
|
2773
|
+
}
|
|
2774
|
+
function cleanPackageToken(token) {
|
|
2775
|
+
const trimmed = token.trim().replace(/^["'`]+|["'`,;]+$/g, "").replace(/\\$/g, "");
|
|
2776
|
+
if (!trimmed || trimmed.startsWith("-") || /^[A-Z_][A-Z0-9_]*=/.test(trimmed)) return null;
|
|
2777
|
+
if (/^(npm|pnpm|yarn|bun|install|add|i|x|dlx|run|exec)$/.test(trimmed)) return null;
|
|
2778
|
+
return packageRootFromSpecifier(trimmed.startsWith("@") ? trimmed.replace(/^(@[^/]+\/[^@]+)@.+$/, "$1") : trimmed.replace(/^([^@]+)@.+$/, "$1"));
|
|
2779
|
+
}
|
|
2780
|
+
function inferDocsAskAIPackageHints(content) {
|
|
2781
|
+
const packages = /* @__PURE__ */ new Set();
|
|
2782
|
+
const imports = /* @__PURE__ */ new Set();
|
|
2783
|
+
const installCommands = /* @__PURE__ */ new Set();
|
|
2784
|
+
for (const line of content.split("\n")) {
|
|
2785
|
+
const trimmed = line.trim();
|
|
2786
|
+
if (!trimmed) continue;
|
|
2787
|
+
const importSpecifier = trimmed.match(/^(?:import|export)\s+(?:type\s+)?[\s\S]*?\s+from\s+["']([^"']+)["']/)?.[1];
|
|
2788
|
+
const bareImportSpecifier = trimmed.match(/^import\s+["']([^"']+)["']/)?.[1];
|
|
2789
|
+
const requireSpecifier = trimmed.match(/require\(["']([^"']+)["']\)/)?.[1];
|
|
2790
|
+
const specifier = importSpecifier ?? bareImportSpecifier ?? requireSpecifier;
|
|
2791
|
+
const packageName = specifier ? packageRootFromSpecifier(specifier) : null;
|
|
2792
|
+
if (packageName) {
|
|
2793
|
+
packages.add(packageName);
|
|
2794
|
+
if (/^(?:import|export)\s/.test(trimmed)) imports.add(trimmed);
|
|
2795
|
+
}
|
|
2796
|
+
const installMatch = trimmed.match(/^(?:npm\s+(?:install|i)|pnpm\s+add|yarn\s+add|bun\s+add)\s+(.+)$/);
|
|
2797
|
+
if (!installMatch) continue;
|
|
2798
|
+
const commandPackages = installMatch[1].split(/\s+/).map(cleanPackageToken).filter((value) => Boolean(value));
|
|
2799
|
+
if (commandPackages.length > 0) {
|
|
2800
|
+
installCommands.add(trimmed);
|
|
2801
|
+
for (const name of commandPackages) packages.add(name);
|
|
2802
|
+
}
|
|
2803
|
+
}
|
|
2804
|
+
return {
|
|
2805
|
+
packages: Array.from(packages).slice(0, 8),
|
|
2806
|
+
imports: Array.from(imports).slice(0, 12),
|
|
2807
|
+
installCommands: Array.from(installCommands).slice(0, 8)
|
|
2808
|
+
};
|
|
2809
|
+
}
|
|
2810
|
+
function formatDocsAskAIPackageHints(hints, packageName) {
|
|
2811
|
+
const packages = packageName ? Array.from(new Set([packageName, ...hints.packages])) : hints.packages;
|
|
2812
|
+
if (packages.length === 0 && hints.imports.length === 0 && hints.installCommands.length === 0) return;
|
|
2813
|
+
const lines = ["Package and import hints inferred from the retrieved documentation context:"];
|
|
2814
|
+
if (packages.length > 0) lines.push(`- Package names found in install/import examples: ${packages.join(", ")}`);
|
|
2815
|
+
if (hints.imports.length > 0) lines.push(`- Exact import lines found in context: ${hints.imports.map((line) => `\`${line}\``).join("; ")}`);
|
|
2816
|
+
if (hints.installCommands.length > 0) lines.push(`- Exact install commands found in context: ${hints.installCommands.map((line) => `\`${line}\``).join("; ")}`);
|
|
2817
|
+
lines.push("Use these exact package names, install commands, and import lines when relevant. Do not replace them with placeholders.");
|
|
2818
|
+
return lines.join("\n");
|
|
2819
|
+
}
|
|
2820
|
+
function clampText(value, maxChars) {
|
|
2821
|
+
if (maxChars <= 0) return "";
|
|
2822
|
+
if (value.length <= maxChars) return value;
|
|
2823
|
+
return `${value.slice(0, maxChars).trimEnd()}...`;
|
|
2824
|
+
}
|
|
2825
|
+
function findPageForSearchResult(pages, result, baseUrl, pagesByRoute) {
|
|
2826
|
+
const rawUrl = result.url.trim();
|
|
2827
|
+
const explicitlySchemed = /^[a-z][a-z\d+.-]*:/iu.test(rawUrl);
|
|
2828
|
+
const explicitlyHosted = explicitlySchemed || /^[\\/]{2}/u.test(rawUrl);
|
|
2829
|
+
if (explicitlySchemed && !/^https?:/iu.test(rawUrl)) return void 0;
|
|
2830
|
+
if (explicitlyHosted && !baseUrl) return void 0;
|
|
2831
|
+
if (explicitlyHosted && baseUrl) try {
|
|
2832
|
+
if (new URL(result.url, baseUrl).origin !== new URL(baseUrl).origin) return void 0;
|
|
2833
|
+
} catch {
|
|
2834
|
+
return;
|
|
2835
|
+
}
|
|
2836
|
+
const resultPath = normalizeUrlRouteKey(result.url);
|
|
2837
|
+
return pagesByRoute?.get(resultPath) ?? pages.find((page) => normalizeUrlRouteKey(page.url) === resultPath);
|
|
2838
|
+
}
|
|
2839
|
+
function inferResultTitle(result, page) {
|
|
2840
|
+
if (page) return page.title;
|
|
2841
|
+
return stripHtml(result.content).trim().split("—")[0]?.trim() || result.url;
|
|
2842
|
+
}
|
|
2843
|
+
function formatAskAIContextResult(options) {
|
|
2844
|
+
const { result, page, maxChars, baseUrl } = options;
|
|
2845
|
+
const title = inferResultTitle(result, page);
|
|
2846
|
+
const section = result.section;
|
|
2847
|
+
const sectionSelector = getSearchResultAnchor(result.url) ?? section;
|
|
2848
|
+
const contextContent = clampText(cleanAskAIContextMarkdown(page ? sectionSelector ? cleanGeneratedAgentContractMarkers(findDocsMarkdownSection(getAskAIPageSectionContent(page), sectionSelector)?.content ?? "") : getAskAIPageContent(page) : [result.content, result.description].filter(Boolean).join("\n\n")), maxChars);
|
|
2849
|
+
return {
|
|
2850
|
+
...result,
|
|
2851
|
+
url: resolveAskAIContextUrl(result.url, baseUrl),
|
|
2852
|
+
title,
|
|
2853
|
+
contextContent
|
|
2854
|
+
};
|
|
2855
|
+
}
|
|
2856
|
+
function getSearchResultKey(result) {
|
|
2857
|
+
const anchor = getSearchResultAnchor(result.url);
|
|
2858
|
+
const sectionFallback = normalizeWhitespace(result.section ?? "").toLowerCase();
|
|
2859
|
+
return `${normalizeUrlRouteKey(result.url)}#${anchor ?? sectionFallback}`;
|
|
2860
|
+
}
|
|
2861
|
+
function getSearchResultAnchor(value) {
|
|
2862
|
+
let hash = "";
|
|
2863
|
+
try {
|
|
2864
|
+
hash = new URL(value, "https://docs.local").hash.replace(/^#/, "");
|
|
2865
|
+
} catch {
|
|
2866
|
+
const hashIndex = value.indexOf("#");
|
|
2867
|
+
hash = hashIndex >= 0 ? value.slice(hashIndex + 1) : "";
|
|
2868
|
+
}
|
|
2869
|
+
if (!hash) return void 0;
|
|
2870
|
+
try {
|
|
2871
|
+
return decodeURIComponent(hash);
|
|
2872
|
+
} catch {
|
|
2873
|
+
return hash;
|
|
2874
|
+
}
|
|
2875
|
+
}
|
|
2876
|
+
function getAskAIResultPageKey(value, baseUrl, strictExternalOrigins = false) {
|
|
2877
|
+
const path = normalizeUrlRouteKey(value);
|
|
2878
|
+
const rawValue = value.trim();
|
|
2879
|
+
if (!(/^[a-z][a-z\d+.-]*:/iu.test(rawValue) || /^[\\/]{2}/u.test(rawValue)) || !baseUrl && !strictExternalOrigins) return path;
|
|
2880
|
+
try {
|
|
2881
|
+
const fallbackBase = baseUrl ?? "https://docs.local";
|
|
2882
|
+
const parsed = new URL(value, fallbackBase);
|
|
2883
|
+
const configuredBase = baseUrl ? new URL(baseUrl) : void 0;
|
|
2884
|
+
if (configuredBase && parsed.origin === configuredBase.origin) return path;
|
|
2885
|
+
return `${parsed.protocol}//${parsed.host}${path}`;
|
|
2886
|
+
} catch {
|
|
2887
|
+
return `${value.split("#", 1)[0]}::${path}`;
|
|
2888
|
+
}
|
|
2889
|
+
}
|
|
2890
|
+
function getAskAIResultKey(result, baseUrl, strictExternalOrigins = false) {
|
|
2891
|
+
const anchor = getSearchResultAnchor(result.url);
|
|
2892
|
+
const sectionFallback = normalizeWhitespace(result.section ?? "").toLowerCase();
|
|
2893
|
+
return `${getAskAIResultPageKey(result.url, baseUrl, strictExternalOrigins)}#${anchor ?? sectionFallback}`;
|
|
2894
|
+
}
|
|
2895
|
+
function mergeSearchResults(groups, getResultKey = getSearchResultKey) {
|
|
2896
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2897
|
+
const results = [];
|
|
2898
|
+
for (const group of groups) for (const result of group) {
|
|
2899
|
+
const key = getResultKey(result);
|
|
2900
|
+
if (seen.has(key)) continue;
|
|
2901
|
+
seen.add(key);
|
|
2902
|
+
results.push(result);
|
|
2903
|
+
}
|
|
2904
|
+
return results;
|
|
2905
|
+
}
|
|
2906
|
+
function buildAudienceProjectionSearchResults(documents, query) {
|
|
2907
|
+
return documents.map((document) => {
|
|
2908
|
+
const score = scoreDocument(query, document);
|
|
2909
|
+
return {
|
|
2910
|
+
id: document.id,
|
|
2911
|
+
url: document.url,
|
|
2912
|
+
content: document.section ? `${document.title} — ${document.section}` : document.title,
|
|
2913
|
+
description: cleanSearchResultText(buildSnippet(document, query) ?? document.description),
|
|
2914
|
+
type: document.type,
|
|
2915
|
+
score,
|
|
2916
|
+
section: document.section
|
|
2917
|
+
};
|
|
2918
|
+
});
|
|
2919
|
+
}
|
|
2920
|
+
function sanitizeExternalAudienceSearchResults(results, localAudienceResults, baseUrl, preserveUnmatched, fallbackSectionToLocalPage = false) {
|
|
2921
|
+
const localByKey = new Map(localAudienceResults.map((result) => [getSearchResultKey(result), result]));
|
|
2922
|
+
const localPageResults = /* @__PURE__ */ new Map();
|
|
2923
|
+
for (const result of localAudienceResults) {
|
|
2924
|
+
const pageUrl = normalizeUrlRouteKey(result.url);
|
|
2925
|
+
if (!localPageResults.get(pageUrl) || result.type === "page") localPageResults.set(pageUrl, result);
|
|
2926
|
+
}
|
|
2927
|
+
return results.flatMap((result) => {
|
|
2928
|
+
const rawUrl = result.url.trim();
|
|
2929
|
+
const explicitlySchemed = /^[a-z][a-z\d+.-]*:/iu.test(rawUrl);
|
|
2930
|
+
const externallyHosted = explicitlySchemed && /^https?:/iu.test(rawUrl) || /^[\\/]{2}/u.test(rawUrl);
|
|
2931
|
+
const unsupportedScheme = explicitlySchemed && !/^https?:/iu.test(rawUrl);
|
|
2932
|
+
let sameOriginOrUnknown = !externallyHosted;
|
|
2933
|
+
if (unsupportedScheme) sameOriginOrUnknown = false;
|
|
2934
|
+
if (externallyHosted && baseUrl) try {
|
|
2935
|
+
sameOriginOrUnknown = new URL(result.url, baseUrl).origin === new URL(baseUrl).origin;
|
|
2936
|
+
} catch {
|
|
2937
|
+
sameOriginOrUnknown = false;
|
|
2938
|
+
}
|
|
2939
|
+
const hasSection = Boolean(getSearchResultAnchor(result.url) || result.section);
|
|
2940
|
+
const local = sameOriginOrUnknown ? localByKey.get(getSearchResultKey(result)) ?? (hasSection && !fallbackSectionToLocalPage ? void 0 : localPageResults.get(normalizeUrlRouteKey(result.url))) : void 0;
|
|
2941
|
+
if (!local) return preserveUnmatched?.(result) ? [result] : [];
|
|
2942
|
+
return [{
|
|
2943
|
+
...local,
|
|
2944
|
+
id: result.id,
|
|
2945
|
+
score: result.score ?? local.score
|
|
2946
|
+
}];
|
|
2947
|
+
});
|
|
2948
|
+
}
|
|
2949
|
+
function shouldPreserveUnmatchedExternalResult(options) {
|
|
2950
|
+
const { result, localPagePaths, baseUrl } = options;
|
|
2951
|
+
const path = normalizeUrlRouteKey(result.url);
|
|
2952
|
+
const rawUrl = result.url.trim();
|
|
2953
|
+
const explicitScheme = rawUrl.match(/^([a-z][a-z\d+.-]*):/iu)?.[1]?.toLowerCase();
|
|
2954
|
+
if (explicitScheme && explicitScheme !== "http" && explicitScheme !== "https") return false;
|
|
2955
|
+
const explicitlyHosted = /^[a-z][a-z\d+.-]*:/iu.test(rawUrl) || /^[\\/]{2}/u.test(rawUrl);
|
|
2956
|
+
const knownLocalPath = localPagePaths.has(path);
|
|
2957
|
+
if (!explicitlyHosted) return !knownLocalPath;
|
|
2958
|
+
if (!baseUrl) return true;
|
|
2959
|
+
try {
|
|
2960
|
+
if (new URL(result.url, baseUrl).origin !== new URL(baseUrl).origin) return true;
|
|
2961
|
+
} catch {
|
|
2962
|
+
return false;
|
|
2963
|
+
}
|
|
2964
|
+
return !knownLocalPath;
|
|
2965
|
+
}
|
|
2966
|
+
function getPageAudienceSource(page, audience) {
|
|
2967
|
+
return audience === "agent" ? page.agentRawContent ?? page.agentFallbackRawContent ?? page.agentContent ?? page.agentFallbackContent ?? page.rawContent ?? page.content : page.rawContent ?? page.content;
|
|
2968
|
+
}
|
|
2969
|
+
function getPageAudienceRawContent(page, audience) {
|
|
2970
|
+
return resolveDocsAudienceMdxContent(getPageAudienceSource(page, audience), audience);
|
|
2971
|
+
}
|
|
2972
|
+
function getPageAudienceSectionContent(page, audience) {
|
|
2973
|
+
const content = getPageAudienceRawContent(page, audience);
|
|
2974
|
+
return audience === "agent" ? upsertPageAgentContractMarkdown(content, page.agent) : content;
|
|
2975
|
+
}
|
|
2976
|
+
/**
|
|
2977
|
+
* Build the exact normalized-input projection represented by a retrieval source digest.
|
|
2978
|
+
* This is public so consumers can independently reproduce `source.digest`.
|
|
2979
|
+
*/
|
|
2980
|
+
function buildDocsRetrievalDigestProjection(page, audience = "human") {
|
|
2981
|
+
const audienceContent = getPageAudienceRawContent(page, audience);
|
|
2982
|
+
return audience === "agent" ? upsertPageAgentContractMarkdown(audienceContent, page.agent) : [audienceContent, renderPageAgentContractMarkdown(page.agent)].filter(Boolean).join("\n\n");
|
|
2983
|
+
}
|
|
2984
|
+
function getPageAudienceSearchText(page, audience) {
|
|
2985
|
+
return stripMarkdownText(getPageAudienceRawContent(page, audience));
|
|
2986
|
+
}
|
|
2987
|
+
function getPageAudienceIndexContent(page, audience) {
|
|
2988
|
+
return normalizeWhitespace([getPageAudienceSearchText(page, audience), getPageAgentContractSearchText(page)].join(" "));
|
|
2989
|
+
}
|
|
2990
|
+
function sortRetrievalSourceValues(values) {
|
|
2991
|
+
const sorted = Array.from(new Set(values)).sort(compareSearchMetadataValues).slice(0, MAX_SEARCH_FILTER_VALUES);
|
|
2992
|
+
return sorted.length > 0 ? sorted : void 0;
|
|
2993
|
+
}
|
|
2994
|
+
function buildDocsRetrievalSourceScope(page, audience, localeFallback) {
|
|
2995
|
+
const resolved = resolveDocsSearchPageScope(page);
|
|
2996
|
+
const rawLocale = (page.locale ?? localeFallback)?.trim();
|
|
2997
|
+
const locale = rawLocale ? normalizeAgentLocale(rawLocale).slice(0, MAX_SEARCH_FILTER_VALUE_CHARS) : void 0;
|
|
2998
|
+
const framework = sortRetrievalSourceValues(resolved.framework);
|
|
2999
|
+
const version = sortRetrievalSourceValues(resolved.version);
|
|
3000
|
+
const versionGroups = resolved.declarations.version.map((group) => sortRetrievalSourceValues(group)).filter((group) => Boolean(group)).slice(0, MAX_SEARCH_FILTER_VALUES);
|
|
3001
|
+
const packageNames = sortRetrievalSourceValues(resolved.package);
|
|
3002
|
+
const tags = sortRetrievalSourceValues(resolved.tags);
|
|
3003
|
+
const truncated = SEARCH_FILTER_FIELDS.filter((field) => {
|
|
3004
|
+
if (field === "version") return resolved.version.length > MAX_SEARCH_FILTER_VALUES || resolved.declarations.version.length > MAX_SEARCH_FILTER_VALUES || resolved.declarations.version.some((group) => group.length > MAX_SEARCH_FILTER_VALUES);
|
|
3005
|
+
return resolved[field].length > MAX_SEARCH_FILTER_VALUES;
|
|
3006
|
+
});
|
|
3007
|
+
return {
|
|
3008
|
+
audience,
|
|
3009
|
+
...locale ? { locale: [locale] } : {},
|
|
3010
|
+
...framework ? { framework } : {},
|
|
3011
|
+
...version ? { version } : {},
|
|
3012
|
+
...versionGroups.length > 1 ? { versionGroups } : {},
|
|
3013
|
+
...packageNames ? { package: packageNames } : {},
|
|
3014
|
+
...tags ? { tags } : {},
|
|
3015
|
+
...truncated.length > 0 ? { truncated } : {},
|
|
3016
|
+
...resolved.conflicts.length > 0 ? { conflicts: [...resolved.conflicts].sort((left, right) => SEARCH_FILTER_FIELDS.indexOf(left) - SEARCH_FILTER_FIELDS.indexOf(right)) } : {}
|
|
3017
|
+
};
|
|
3018
|
+
}
|
|
3019
|
+
function buildDocsRetrievalScopeIdentity(page, audience, localeFallback) {
|
|
3020
|
+
const resolved = resolveDocsSearchPageScope(page);
|
|
3021
|
+
const sortValues = (values) => Array.from(new Set(values)).sort(compareSearchMetadataValues);
|
|
3022
|
+
const declarations = Object.fromEntries(SEARCH_FILTER_FIELDS.map((field) => [field, resolved.declarations[field].map(sortValues).sort((left, right) => compareSearchMetadataValues(JSON.stringify(left), JSON.stringify(right)))]));
|
|
3023
|
+
return hashDocsRetrievalValue(JSON.stringify({
|
|
3024
|
+
audience,
|
|
3025
|
+
locale: (page.locale ?? localeFallback)?.trim(),
|
|
3026
|
+
declarations,
|
|
3027
|
+
conflicts: [...resolved.conflicts].sort((left, right) => SEARCH_FILTER_FIELDS.indexOf(left) - SEARCH_FILTER_FIELDS.indexOf(right))
|
|
3028
|
+
}));
|
|
3029
|
+
}
|
|
3030
|
+
function getUrlHash(value) {
|
|
3031
|
+
const hashIndex = value.indexOf("#");
|
|
3032
|
+
return hashIndex >= 0 ? value.slice(hashIndex) : "";
|
|
3033
|
+
}
|
|
3034
|
+
function resolveDocsRetrievalCanonicalUrl(page, resultUrl, baseUrl) {
|
|
3035
|
+
const requestedCanonical = page.canonicalUrl?.trim();
|
|
3036
|
+
const requested = requestedCanonical && requestedCanonical.length <= MAX_RETRIEVAL_SOURCE_URL_CHARS ? requestedCanonical : page.url;
|
|
3037
|
+
const explicitScheme = requested.match(/^([a-z][a-z\d+.-]*):/iu)?.[1]?.toLowerCase();
|
|
3038
|
+
const pageScheme = page.url.match(/^([a-z][a-z\d+.-]*):/iu)?.[1]?.toLowerCase();
|
|
3039
|
+
const safePageUrl = pageScheme && pageScheme !== "http" && pageScheme !== "https" ? "/" : page.url;
|
|
3040
|
+
const configured = explicitScheme && explicitScheme !== "http" && explicitScheme !== "https" ? safePageUrl : requested;
|
|
3041
|
+
let canonical = configured;
|
|
3042
|
+
if (baseUrl) try {
|
|
3043
|
+
canonical = new URL(configured, baseUrl).toString();
|
|
3044
|
+
} catch {
|
|
3045
|
+
canonical = configured;
|
|
3046
|
+
}
|
|
3047
|
+
else if (configured.startsWith("//")) canonical = `https:${configured}`;
|
|
3048
|
+
const withSection = `${canonical.split("#", 1)[0]}${getUrlHash(resultUrl)}`;
|
|
3049
|
+
if (withSection.length <= MAX_RETRIEVAL_SOURCE_URL_CHARS && isDocsRetrievalCanonicalUrl(withSection)) return withSection;
|
|
3050
|
+
const fallback = (baseUrl ? (() => {
|
|
3051
|
+
try {
|
|
3052
|
+
return new URL(safePageUrl, baseUrl).toString();
|
|
3053
|
+
} catch {
|
|
3054
|
+
return safePageUrl;
|
|
3055
|
+
}
|
|
3056
|
+
})() : safePageUrl).split("#", 1)[0];
|
|
3057
|
+
if (fallback.length <= MAX_RETRIEVAL_SOURCE_URL_CHARS && isDocsRetrievalCanonicalUrl(fallback)) return fallback;
|
|
3058
|
+
if (baseUrl) try {
|
|
3059
|
+
const root = new URL("/", baseUrl).toString();
|
|
3060
|
+
if (root.length <= MAX_RETRIEVAL_SOURCE_URL_CHARS && isDocsRetrievalCanonicalUrl(root)) return root;
|
|
3061
|
+
} catch {}
|
|
3062
|
+
return "/";
|
|
3063
|
+
}
|
|
3064
|
+
function hashDocsRetrievalValue(value) {
|
|
3065
|
+
return digestDocsRetrievalContent(value);
|
|
3066
|
+
}
|
|
3067
|
+
const docsRetrievalDigestMemo = /* @__PURE__ */ new WeakMap();
|
|
3068
|
+
function getDocsRetrievalSourceDigest(page, audience, cache) {
|
|
3069
|
+
const cached = cache?.get(page);
|
|
3070
|
+
if (cached) return cached;
|
|
3071
|
+
const source = getPageAudienceSource(page, audience);
|
|
3072
|
+
let agentContractKey = "";
|
|
3073
|
+
let cacheable = true;
|
|
3074
|
+
try {
|
|
3075
|
+
agentContractKey = JSON.stringify(page.agent ?? null);
|
|
3076
|
+
} catch {
|
|
3077
|
+
cacheable = false;
|
|
3078
|
+
}
|
|
3079
|
+
const memo = docsRetrievalDigestMemo.get(page)?.[audience];
|
|
3080
|
+
if (cacheable && memo && memo.source === source && memo.agentContractKey === agentContractKey) {
|
|
3081
|
+
cache?.set(page, memo.digest);
|
|
3082
|
+
return memo.digest;
|
|
3083
|
+
}
|
|
3084
|
+
const digest = hashDocsRetrievalValue(buildDocsRetrievalDigestProjection(page, audience));
|
|
3085
|
+
if (cacheable) {
|
|
3086
|
+
const pageMemo = docsRetrievalDigestMemo.get(page) ?? {};
|
|
3087
|
+
pageMemo[audience] = {
|
|
3088
|
+
source,
|
|
3089
|
+
agentContractKey,
|
|
3090
|
+
digest
|
|
3091
|
+
};
|
|
3092
|
+
docsRetrievalDigestMemo.set(page, pageMemo);
|
|
3093
|
+
}
|
|
3094
|
+
cache?.set(page, digest);
|
|
3095
|
+
return digest;
|
|
3096
|
+
}
|
|
3097
|
+
function resolveDocsRetrievalLastModified(page, audience) {
|
|
3098
|
+
const parseCandidate = (value) => {
|
|
3099
|
+
const normalized = value?.trim();
|
|
3100
|
+
if (!normalized) return void 0;
|
|
3101
|
+
const timestamp = Date.parse(normalized);
|
|
3102
|
+
return Number.isFinite(timestamp) ? {
|
|
3103
|
+
value: normalized,
|
|
3104
|
+
timestamp
|
|
3105
|
+
} : void 0;
|
|
3106
|
+
};
|
|
3107
|
+
const pageModified = parseCandidate(page.lastmod) ?? parseCandidate(page.lastModified);
|
|
3108
|
+
if (!(audience === "agent" && (page.agentRawContent !== void 0 || page.agentContent !== void 0))) return pageModified?.value;
|
|
3109
|
+
const agentModified = parseCandidate(page.agentLastModified);
|
|
3110
|
+
if (!renderPageAgentContractMarkdown(page.agent)) return agentModified?.value ?? pageModified?.value;
|
|
3111
|
+
if (!agentModified) return pageModified?.value;
|
|
3112
|
+
if (!pageModified) return agentModified.value;
|
|
3113
|
+
return agentModified.timestamp >= pageModified.timestamp ? agentModified.value : pageModified.value;
|
|
3114
|
+
}
|
|
3115
|
+
async function buildDocsSearchIndexGeneration(pages, options) {
|
|
3116
|
+
const sources = (await Promise.all(pages.map(async (rawPage) => {
|
|
3117
|
+
const page = localizeDocsSearchPage(rawPage, options.locale);
|
|
3118
|
+
const authoredLastModified = page.lastmod?.trim();
|
|
3119
|
+
return {
|
|
3120
|
+
canonicalIdentity: normalizeAuthoredUrlIdentity(page.canonicalUrl?.trim() || page.url),
|
|
3121
|
+
url: normalizeAuthoredUrlIdentity(page.url),
|
|
3122
|
+
indexedUrl: page.url,
|
|
3123
|
+
title: page.title,
|
|
3124
|
+
description: page.description,
|
|
3125
|
+
type: page.type,
|
|
3126
|
+
scope: buildDocsRetrievalSourceScope(page, options.audience, options.locale),
|
|
3127
|
+
scopeIdentity: buildDocsRetrievalScopeIdentity(page, options.audience, options.locale),
|
|
3128
|
+
lastModified: authoredLastModified && Number.isFinite(Date.parse(authoredLastModified)) ? authoredLastModified : void 0,
|
|
3129
|
+
digest: getDocsRetrievalSourceDigest(rawPage, options.audience, options.digestCache),
|
|
3130
|
+
agentContract: getPageAgentContractSearchText(rawPage)
|
|
3131
|
+
};
|
|
3132
|
+
}))).sort((left, right) => {
|
|
3133
|
+
const canonical = compareSearchMetadataValues(left.canonicalIdentity, right.canonicalIdentity);
|
|
3134
|
+
if (canonical !== 0) return canonical;
|
|
3135
|
+
const url = compareSearchMetadataValues(left.url, right.url);
|
|
3136
|
+
if (url !== 0) return url;
|
|
3137
|
+
const digest = compareSearchMetadataValues(left.digest, right.digest);
|
|
3138
|
+
if (digest !== 0) return digest;
|
|
3139
|
+
const title = compareSearchMetadataValues(left.title, right.title);
|
|
3140
|
+
return title !== 0 ? title : compareSearchMetadataValues(JSON.stringify(left), JSON.stringify(right));
|
|
3141
|
+
});
|
|
3142
|
+
return hashDocsRetrievalValue(JSON.stringify({
|
|
3143
|
+
format: RETRIEVAL_INDEX_FORMAT,
|
|
3144
|
+
audience: options.audience,
|
|
3145
|
+
chunking: options.chunking.strategy ?? "section",
|
|
3146
|
+
sources
|
|
3147
|
+
}));
|
|
3148
|
+
}
|
|
3149
|
+
/**
|
|
3150
|
+
* Build a deterministic, body-free inventory for content-change synchronization.
|
|
3151
|
+
*
|
|
3152
|
+
* Its generation is identical to structured search for the same pages, audience,
|
|
3153
|
+
* locale, base URL, and chunking policy.
|
|
3154
|
+
*/
|
|
3155
|
+
async function buildDocsContentSnapshot(options) {
|
|
3156
|
+
const audience = resolveDocsSearchAudience(options.audience);
|
|
3157
|
+
const search = normalizeDocsSearchConfig(options.search);
|
|
3158
|
+
const digestCache = /* @__PURE__ */ new Map();
|
|
3159
|
+
const indexGeneration = options.indexGeneration ?? await buildDocsSearchIndexGeneration(options.pages, {
|
|
3160
|
+
audience,
|
|
3161
|
+
chunking: search.chunking,
|
|
3162
|
+
locale: options.locale,
|
|
3163
|
+
baseUrl: options.baseUrl,
|
|
3164
|
+
digestCache
|
|
3165
|
+
});
|
|
3166
|
+
const documents = await Promise.all(options.pages.map(async (page) => {
|
|
3167
|
+
const localizedPage = localizeDocsSearchPage(page, options.locale);
|
|
3168
|
+
const source = await buildDocsRetrievalSource(page, localizedPage.url, {
|
|
3169
|
+
audience,
|
|
3170
|
+
chunking: search.chunking,
|
|
3171
|
+
locale: options.locale,
|
|
3172
|
+
baseUrl: options.baseUrl,
|
|
3173
|
+
indexGeneration,
|
|
3174
|
+
digestCache
|
|
3175
|
+
});
|
|
3176
|
+
const canonicalUrl = source.canonicalUrl.split("#", 1)[0];
|
|
3177
|
+
return {
|
|
3178
|
+
url: localizedPage.url,
|
|
3179
|
+
canonicalUrl,
|
|
3180
|
+
digest: hashDocsRetrievalValue(JSON.stringify({
|
|
3181
|
+
format: "docs-content-document.v1",
|
|
3182
|
+
url: localizedPage.url,
|
|
3183
|
+
canonicalUrl,
|
|
3184
|
+
title: localizedPage.title,
|
|
3185
|
+
description: localizedPage.description,
|
|
3186
|
+
type: localizedPage.type,
|
|
3187
|
+
scope: source.scope,
|
|
3188
|
+
lastModified: source.lastModified,
|
|
3189
|
+
sourceDigest: source.digest
|
|
3190
|
+
})),
|
|
3191
|
+
...source.lastModified ? { lastModified: source.lastModified } : {}
|
|
3192
|
+
};
|
|
3193
|
+
}));
|
|
3194
|
+
documents.sort((left, right) => {
|
|
3195
|
+
const canonical = compareSearchMetadataValues(left.canonicalUrl, right.canonicalUrl);
|
|
3196
|
+
return canonical !== 0 ? canonical : compareSearchMetadataValues(left.url, right.url);
|
|
3197
|
+
});
|
|
3198
|
+
for (let index = 1; index < documents.length; index += 1) if (documents[index - 1]?.canonicalUrl === documents[index]?.canonicalUrl) throw new DocsSearchRequestError(`Content-change snapshots require unique canonical URLs; duplicate: ${documents[index].canonicalUrl}`);
|
|
3199
|
+
return {
|
|
3200
|
+
format: "docs-content-snapshot.v1",
|
|
3201
|
+
audience,
|
|
3202
|
+
...options.locale ? { locale: normalizeAgentLocale(options.locale) } : {},
|
|
3203
|
+
...options.baseUrl ? { baseUrl: options.baseUrl } : {},
|
|
3204
|
+
indexGeneration,
|
|
3205
|
+
documents
|
|
3206
|
+
};
|
|
3207
|
+
}
|
|
3208
|
+
async function buildDocsRetrievalSource(rawPage, resultUrl, options) {
|
|
3209
|
+
const page = localizeDocsSearchPage(rawPage, options.locale);
|
|
3210
|
+
const lastModified = resolveDocsRetrievalLastModified(rawPage, options.audience);
|
|
3211
|
+
return {
|
|
3212
|
+
canonicalUrl: resolveDocsRetrievalCanonicalUrl(page, resultUrl, options.baseUrl),
|
|
3213
|
+
scope: buildDocsRetrievalSourceScope(page, options.audience, options.locale),
|
|
3214
|
+
...lastModified ? { lastModified } : {},
|
|
3215
|
+
digest: getDocsRetrievalSourceDigest(rawPage, options.audience, options.digestCache),
|
|
3216
|
+
indexGeneration: options.indexGeneration
|
|
3217
|
+
};
|
|
3218
|
+
}
|
|
3219
|
+
function parseRetrievalSourceString(value, maxChars = MAX_RETRIEVAL_SOURCE_VALUE_CHARS) {
|
|
3220
|
+
if (typeof value !== "string") return void 0;
|
|
3221
|
+
const normalized = value.trim();
|
|
3222
|
+
if (!normalized || normalized.length > maxChars) return void 0;
|
|
3223
|
+
if (Array.from(normalized).some((character) => {
|
|
3224
|
+
const codePoint = character.codePointAt(0) ?? 0;
|
|
3225
|
+
return codePoint <= 31 || codePoint === 127;
|
|
3226
|
+
})) return void 0;
|
|
3227
|
+
return normalized;
|
|
3228
|
+
}
|
|
3229
|
+
function parseRetrievalSourceValues(field, value) {
|
|
3230
|
+
if (!Array.isArray(value) || value.length > MAX_SEARCH_FILTER_VALUES) return { valid: false };
|
|
3231
|
+
const parsed = [];
|
|
3232
|
+
for (const item of value) {
|
|
3233
|
+
const stringValue = parseRetrievalSourceString(item, MAX_SEARCH_FILTER_VALUE_CHARS);
|
|
3234
|
+
if (!stringValue) return { valid: false };
|
|
3235
|
+
if (field === "locale") {
|
|
3236
|
+
const normalizedLocale = normalizeAgentLocale(stringValue);
|
|
3237
|
+
if (!normalizedLocale) return { valid: false };
|
|
3238
|
+
parsed.push(normalizedLocale);
|
|
3239
|
+
continue;
|
|
3240
|
+
}
|
|
3241
|
+
const normalized = normalizeDocsSearchFilterValue(field, stringValue);
|
|
3242
|
+
if (!normalized) return { valid: false };
|
|
3243
|
+
parsed.push(normalized);
|
|
3244
|
+
}
|
|
3245
|
+
return {
|
|
3246
|
+
valid: true,
|
|
3247
|
+
values: sortRetrievalSourceValues(parsed)
|
|
3248
|
+
};
|
|
3249
|
+
}
|
|
3250
|
+
function parseRetrievalSourceVersionGroups(value) {
|
|
3251
|
+
if (!Array.isArray(value) || value.length === 0 || value.length > MAX_SEARCH_FILTER_VALUES) return { valid: false };
|
|
3252
|
+
const groups = value.map((group) => parseRetrievalSourceValues("version", group));
|
|
3253
|
+
if (groups.some((group) => !group.valid || !group.values || group.values.length === 0)) return { valid: false };
|
|
3254
|
+
return {
|
|
3255
|
+
valid: true,
|
|
3256
|
+
groups: groups.map((group) => group.values)
|
|
3257
|
+
};
|
|
3258
|
+
}
|
|
3259
|
+
function parseDocsRetrievalSource(value, options = {}) {
|
|
3260
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
|
|
3261
|
+
const record = value;
|
|
3262
|
+
const canonicalUrl = parseRetrievalSourceString(record.canonicalUrl, MAX_RETRIEVAL_SOURCE_URL_CHARS);
|
|
3263
|
+
const digest = parseRetrievalSourceString(record.digest);
|
|
3264
|
+
const indexGeneration = parseRetrievalSourceString(record.indexGeneration);
|
|
3265
|
+
if (!canonicalUrl || !digest || !indexGeneration) return void 0;
|
|
3266
|
+
const digestPattern = /^sha256:[a-f\d]{64}$/iu;
|
|
3267
|
+
if (!digestPattern.test(digest) || !digestPattern.test(indexGeneration)) return void 0;
|
|
3268
|
+
if (!isDocsRetrievalCanonicalUrl(canonicalUrl) || /^\/(?!\/)/u.test(canonicalUrl) && !options.allowRootRelativeCanonical) return;
|
|
3269
|
+
if (!record.scope || typeof record.scope !== "object" || Array.isArray(record.scope)) return;
|
|
3270
|
+
const rawScope = record.scope;
|
|
3271
|
+
const audience = rawScope.audience === "human" || rawScope.audience === "agent" ? rawScope.audience : void 0;
|
|
3272
|
+
if (!audience) return void 0;
|
|
3273
|
+
const parseScopeFields = (value) => {
|
|
3274
|
+
if (!Array.isArray(value) || value.length > SEARCH_FILTER_FIELDS.length || value.some((field) => typeof field !== "string" || !SEARCH_FILTER_FIELDS.includes(field))) return { valid: false };
|
|
3275
|
+
return {
|
|
3276
|
+
valid: true,
|
|
3277
|
+
values: Array.from(new Set(value)).sort((left, right) => SEARCH_FILTER_FIELDS.indexOf(left) - SEARCH_FILTER_FIELDS.indexOf(right))
|
|
3278
|
+
};
|
|
3279
|
+
};
|
|
3280
|
+
const parseOptionalValues = (field, value) => value === void 0 ? { valid: true } : parseRetrievalSourceValues(field, value);
|
|
3281
|
+
const parseOptionalScopeFields = (value) => value === void 0 ? { valid: true } : parseScopeFields(value);
|
|
3282
|
+
const conflicts = parseOptionalScopeFields(rawScope.conflicts);
|
|
3283
|
+
const truncated = parseOptionalScopeFields(rawScope.truncated);
|
|
3284
|
+
const locale = parseOptionalValues("locale", rawScope.locale);
|
|
3285
|
+
const framework = parseOptionalValues("framework", rawScope.framework);
|
|
3286
|
+
const version = parseOptionalValues("version", rawScope.version);
|
|
3287
|
+
const versionGroups = rawScope.versionGroups === void 0 ? {
|
|
3288
|
+
valid: true,
|
|
3289
|
+
groups: void 0
|
|
3290
|
+
} : parseRetrievalSourceVersionGroups(rawScope.versionGroups);
|
|
3291
|
+
const packageNames = parseOptionalValues("package", rawScope.package);
|
|
3292
|
+
const tags = parseOptionalValues("tags", rawScope.tags);
|
|
3293
|
+
if (!conflicts.valid || !truncated.valid || !locale.valid || !framework.valid || !version.valid || !versionGroups.valid || !packageNames.valid || !tags.valid) return;
|
|
3294
|
+
const scope = {
|
|
3295
|
+
audience,
|
|
3296
|
+
...locale.values ? { locale: locale.values } : {},
|
|
3297
|
+
...framework.values ? { framework: framework.values } : {},
|
|
3298
|
+
...version.values ? { version: version.values } : {},
|
|
3299
|
+
...versionGroups.groups ? { versionGroups: versionGroups.groups } : {},
|
|
3300
|
+
...packageNames.values ? { package: packageNames.values } : {},
|
|
3301
|
+
...tags.values ? { tags: tags.values } : {},
|
|
3302
|
+
...truncated.values && truncated.values.length > 0 ? { truncated: truncated.values } : {},
|
|
3303
|
+
...conflicts.values && conflicts.values.length > 0 ? { conflicts: conflicts.values } : {}
|
|
3304
|
+
};
|
|
3305
|
+
const rawLastModified = parseRetrievalSourceString(record.lastModified);
|
|
3306
|
+
if (record.lastModified !== void 0 && (!rawLastModified || !Number.isFinite(Date.parse(rawLastModified)))) return;
|
|
3307
|
+
const lastModified = rawLastModified && Number.isFinite(Date.parse(rawLastModified)) ? rawLastModified : void 0;
|
|
3308
|
+
return {
|
|
3309
|
+
canonicalUrl,
|
|
3310
|
+
scope,
|
|
3311
|
+
...lastModified ? { lastModified } : {},
|
|
3312
|
+
digest,
|
|
3313
|
+
indexGeneration
|
|
3314
|
+
};
|
|
3315
|
+
}
|
|
3316
|
+
function docsRetrievalSourceMatchesRequest(source, audience, filters, locale) {
|
|
3317
|
+
if (source.scope.audience !== audience) return false;
|
|
3318
|
+
for (const field of SEARCH_FILTER_FIELDS) {
|
|
3319
|
+
const requested = filters?.[field];
|
|
3320
|
+
if (!requested?.length) continue;
|
|
3321
|
+
if (source.scope.conflicts?.includes(field)) return false;
|
|
3322
|
+
if (source.scope.truncated?.includes(field)) return false;
|
|
3323
|
+
if (field === "version" && source.scope.versionGroups?.length) {
|
|
3324
|
+
if (!agentVersionConstraintGroupsOverlap([requested, ...source.scope.versionGroups])) return false;
|
|
3325
|
+
continue;
|
|
3326
|
+
}
|
|
3327
|
+
const candidates = source.scope[field];
|
|
3328
|
+
if (!candidates?.length) return false;
|
|
3329
|
+
if (!requested.some((requestedValue) => candidates.some((candidate) => docsSearchScopeValueMatches(field, requestedValue, candidate)))) return false;
|
|
3330
|
+
}
|
|
3331
|
+
const requestedLocale = locale ? normalizeAgentLocale(locale) : void 0;
|
|
3332
|
+
if (requestedLocale && (!source.scope.locale?.length || !source.scope.locale.some((candidate) => normalizeAgentLocale(candidate) === requestedLocale))) return false;
|
|
3333
|
+
return true;
|
|
3334
|
+
}
|
|
3335
|
+
const HOSTED_RETRIEVAL_SOURCE_FIELDS = [
|
|
3336
|
+
"source_canonical_url",
|
|
3337
|
+
"source_scope_audience",
|
|
3338
|
+
"source_scope_locale",
|
|
3339
|
+
"source_scope_framework",
|
|
3340
|
+
"source_scope_version",
|
|
3341
|
+
"source_scope_version_groups",
|
|
3342
|
+
"source_scope_package",
|
|
3343
|
+
"source_scope_tags",
|
|
3344
|
+
"source_scope_truncated",
|
|
3345
|
+
"source_scope_conflicts",
|
|
3346
|
+
"source_last_modified",
|
|
3347
|
+
"source_digest",
|
|
3348
|
+
"source_index_generation"
|
|
3349
|
+
];
|
|
3350
|
+
function readFlattenedHostedRetrievalSource(record) {
|
|
3351
|
+
return parseDocsRetrievalSource({
|
|
3352
|
+
canonicalUrl: record.source_canonical_url,
|
|
3353
|
+
scope: {
|
|
3354
|
+
audience: record.source_scope_audience,
|
|
3355
|
+
locale: record.source_scope_locale,
|
|
3356
|
+
framework: record.source_scope_framework,
|
|
3357
|
+
version: record.source_scope_version,
|
|
3358
|
+
versionGroups: (() => {
|
|
3359
|
+
const value = record.source_scope_version_groups;
|
|
3360
|
+
if (typeof value !== "string") return value;
|
|
3361
|
+
try {
|
|
3362
|
+
return JSON.parse(value);
|
|
3363
|
+
} catch {
|
|
3364
|
+
return null;
|
|
3365
|
+
}
|
|
3366
|
+
})(),
|
|
3367
|
+
package: record.source_scope_package,
|
|
3368
|
+
tags: record.source_scope_tags,
|
|
3369
|
+
truncated: record.source_scope_truncated,
|
|
3370
|
+
conflicts: record.source_scope_conflicts
|
|
3371
|
+
},
|
|
3372
|
+
lastModified: record.source_last_modified,
|
|
3373
|
+
digest: record.source_digest,
|
|
3374
|
+
indexGeneration: record.source_index_generation
|
|
3375
|
+
}, { allowRootRelativeCanonical: true });
|
|
3376
|
+
}
|
|
3377
|
+
function readHostedRetrievalSource(record) {
|
|
3378
|
+
const hasNested = record.source !== void 0;
|
|
3379
|
+
const hasFlattened = HOSTED_RETRIEVAL_SOURCE_FIELDS.some((key) => key in record);
|
|
3380
|
+
if (hasNested) {
|
|
3381
|
+
const nested = parseDocsRetrievalSource(record.source, { allowRootRelativeCanonical: true });
|
|
3382
|
+
if (!nested) return void 0;
|
|
3383
|
+
if (!hasFlattened) return nested;
|
|
3384
|
+
const flattened = readFlattenedHostedRetrievalSource(record);
|
|
3385
|
+
return flattened && JSON.stringify(flattened) === JSON.stringify(nested) ? nested : void 0;
|
|
3386
|
+
}
|
|
3387
|
+
return readFlattenedHostedRetrievalSource(record);
|
|
3388
|
+
}
|
|
3389
|
+
function hasHostedRetrievalSource(record) {
|
|
3390
|
+
return record.source !== void 0 || [
|
|
3391
|
+
"source_corpus_id",
|
|
3392
|
+
"source_document_id",
|
|
3393
|
+
...HOSTED_RETRIEVAL_SOURCE_FIELDS
|
|
3394
|
+
].some((key) => key in record);
|
|
3395
|
+
}
|
|
3396
|
+
function serializeHostedRetrievalSource(source, corpusId, sourceDocumentId) {
|
|
3397
|
+
if (!source) return {};
|
|
3398
|
+
return {
|
|
3399
|
+
source_corpus_id: corpusId,
|
|
3400
|
+
source_document_id: sourceDocumentId,
|
|
3401
|
+
source_canonical_url: source.canonicalUrl,
|
|
3402
|
+
source_scope_audience: source.scope.audience,
|
|
3403
|
+
source_scope_locale: source.scope.locale,
|
|
3404
|
+
source_scope_framework: source.scope.framework,
|
|
3405
|
+
source_scope_version: source.scope.version,
|
|
3406
|
+
source_scope_version_groups: source.scope.versionGroups ? JSON.stringify(source.scope.versionGroups) : void 0,
|
|
3407
|
+
source_scope_package: source.scope.package,
|
|
3408
|
+
source_scope_tags: source.scope.tags,
|
|
3409
|
+
source_scope_truncated: source.scope.truncated,
|
|
3410
|
+
source_scope_conflicts: source.scope.conflicts,
|
|
3411
|
+
source_last_modified: source.lastModified,
|
|
3412
|
+
source_digest: source.digest,
|
|
3413
|
+
source_index_generation: source.indexGeneration
|
|
3414
|
+
};
|
|
3415
|
+
}
|
|
3416
|
+
function resolveHostedCorpusId(syncNamespace, context) {
|
|
3417
|
+
const explicitNamespace = syncNamespace?.trim();
|
|
3418
|
+
if (explicitNamespace && explicitNamespace.length > 1024) throw new Error("Search syncNamespace must be 1024 characters or fewer.");
|
|
3419
|
+
let canonicalIdentity;
|
|
3420
|
+
const indexBaseUrl = context.indexBaseUrl;
|
|
3421
|
+
if (!explicitNamespace && indexBaseUrl) try {
|
|
3422
|
+
const url = new URL(indexBaseUrl);
|
|
3423
|
+
if ((url.protocol === "http:" || url.protocol === "https:") && !url.username && !url.password) canonicalIdentity = `${url.origin}${url.pathname.replace(/\/+$/u, "") || "/"}`;
|
|
3424
|
+
} catch {}
|
|
3425
|
+
const identity = explicitNamespace ? `namespace:${explicitNamespace}` : canonicalIdentity ? `canonical:${canonicalIdentity}` : void 0;
|
|
3426
|
+
if (!identity) return void 0;
|
|
3427
|
+
return hashDocsRetrievalValue(JSON.stringify({
|
|
3428
|
+
format: "docs-hosted-corpus.v1",
|
|
3429
|
+
identity,
|
|
3430
|
+
audience: resolveDocsSearchAudience(context.audience),
|
|
3431
|
+
locale: context.locale ? normalizeAgentLocale(context.locale) : "__all__"
|
|
3432
|
+
}));
|
|
3433
|
+
}
|
|
3434
|
+
function resolveHostedHumanCorpusId(syncNamespace, context) {
|
|
3435
|
+
return resolveHostedCorpusId(syncNamespace, {
|
|
3436
|
+
audience: "human",
|
|
3437
|
+
locale: context.locale,
|
|
3438
|
+
baseUrl: context.baseUrl,
|
|
3439
|
+
indexBaseUrl: context.indexBaseUrl
|
|
3440
|
+
});
|
|
3441
|
+
}
|
|
3442
|
+
function makeHostedProviderDocumentId(corpusId, sourceDocumentId) {
|
|
3443
|
+
return `docs_${hashDocsRetrievalValue(JSON.stringify({
|
|
3444
|
+
format: "docs-hosted-document.v1",
|
|
3445
|
+
corpusId,
|
|
3446
|
+
sourceDocumentId
|
|
3447
|
+
})).slice(7)}`;
|
|
3448
|
+
}
|
|
3449
|
+
function readHostedSourceDocumentId(record) {
|
|
3450
|
+
return parseRetrievalSourceString(record.source_document_id, MAX_RETRIEVAL_SOURCE_URL_CHARS);
|
|
3451
|
+
}
|
|
3452
|
+
function hostedRecordMatchesCorpus(record, corpusId) {
|
|
3453
|
+
if (!corpusId) return true;
|
|
3454
|
+
if (record.source_corpus_id === corpusId) return true;
|
|
3455
|
+
return record.source_corpus_id === void 0 && !hasHostedRetrievalSource(record);
|
|
3456
|
+
}
|
|
3457
|
+
async function enrichDocsSearchResultsWithSources(options) {
|
|
3458
|
+
const digestCache = /* @__PURE__ */ new Map();
|
|
3459
|
+
const localizedPages = options.pages.map((page) => localizeDocsSearchPage(page, options.locale));
|
|
3460
|
+
const pagesByRoute = /* @__PURE__ */ new Map();
|
|
3461
|
+
for (const page of localizedPages) {
|
|
3462
|
+
const route = normalizeUrlRouteKey(page.url);
|
|
3463
|
+
if (!pagesByRoute.has(route)) pagesByRoute.set(route, page);
|
|
3464
|
+
}
|
|
3465
|
+
let resolvedIndexGeneration;
|
|
3466
|
+
const getIndexGeneration = () => {
|
|
3467
|
+
resolvedIndexGeneration ??= options.indexGeneration ? Promise.resolve(options.indexGeneration) : buildDocsSearchIndexGeneration(options.generationPages ?? options.pages, {
|
|
3468
|
+
audience: options.audience,
|
|
3469
|
+
chunking: options.chunking,
|
|
3470
|
+
locale: options.locale,
|
|
3471
|
+
baseUrl: options.baseUrl,
|
|
3472
|
+
digestCache
|
|
3473
|
+
});
|
|
3474
|
+
return resolvedIndexGeneration;
|
|
3475
|
+
};
|
|
3476
|
+
return (await Promise.all(options.results.map(async (result) => {
|
|
3477
|
+
const { source: rawSource, ...resultWithoutSource } = result;
|
|
3478
|
+
const page = findPageForSearchResult(localizedPages, result, options.baseUrl, pagesByRoute);
|
|
3479
|
+
if (page) return {
|
|
3480
|
+
...resultWithoutSource,
|
|
3481
|
+
source: await buildDocsRetrievalSource(page, result.url, {
|
|
3482
|
+
audience: options.audience,
|
|
3483
|
+
chunking: options.chunking,
|
|
3484
|
+
locale: options.locale,
|
|
3485
|
+
baseUrl: options.baseUrl,
|
|
3486
|
+
indexGeneration: await getIndexGeneration(),
|
|
3487
|
+
digestCache
|
|
3488
|
+
})
|
|
3489
|
+
};
|
|
3490
|
+
if (rawSource === void 0) return resultWithoutSource;
|
|
3491
|
+
const source = parseDocsRetrievalSource(rawSource, { allowRootRelativeCanonical: options.requireCurrentIndexGeneration });
|
|
3492
|
+
if (!source) return null;
|
|
3493
|
+
if (!docsRetrievalSourceMatchesRequest(source, options.audience, options.filters, options.locale)) return null;
|
|
3494
|
+
if (options.requireCurrentIndexGeneration && source.indexGeneration !== await getIndexGeneration()) return null;
|
|
3495
|
+
return {
|
|
3496
|
+
...resultWithoutSource,
|
|
3497
|
+
source
|
|
3498
|
+
};
|
|
3499
|
+
}))).filter((result) => Boolean(result));
|
|
3500
|
+
}
|
|
3501
|
+
async function enrichDocsSearchDocumentsWithSources(options) {
|
|
3502
|
+
const results = await enrichDocsSearchResultsWithSources({
|
|
3503
|
+
results: options.documents.map((document) => ({
|
|
3504
|
+
id: document.id,
|
|
3505
|
+
url: document.url,
|
|
3506
|
+
content: document.title,
|
|
3507
|
+
description: document.description,
|
|
3508
|
+
type: document.type,
|
|
3509
|
+
section: document.section,
|
|
3510
|
+
source: document.source
|
|
3511
|
+
})),
|
|
3512
|
+
pages: options.pages,
|
|
3513
|
+
audience: options.audience,
|
|
3514
|
+
chunking: options.chunking,
|
|
3515
|
+
locale: options.locale,
|
|
3516
|
+
baseUrl: options.baseUrl,
|
|
3517
|
+
indexGeneration: options.indexGeneration
|
|
3518
|
+
});
|
|
3519
|
+
const sources = new Map(results.map((result) => [result.id, result.source]));
|
|
3520
|
+
return options.documents.map((document) => {
|
|
3521
|
+
const source = sources.get(document.id);
|
|
3522
|
+
return source ? {
|
|
3523
|
+
...document,
|
|
3524
|
+
source
|
|
3525
|
+
} : document;
|
|
3526
|
+
});
|
|
3527
|
+
}
|
|
3528
|
+
/**
|
|
3529
|
+
* Attach the same canonical provenance used by built-in search providers.
|
|
3530
|
+
* Custom adapters can call this before persisting their own hosted records.
|
|
3531
|
+
*/
|
|
3532
|
+
function enrichDocsSearchDocumentsWithProvenance(options) {
|
|
3533
|
+
return enrichDocsSearchDocumentsWithSources({
|
|
3534
|
+
...options,
|
|
3535
|
+
audience: resolveDocsSearchAudience(options.audience),
|
|
3536
|
+
chunking: options.chunking ?? { strategy: "section" }
|
|
3537
|
+
});
|
|
3538
|
+
}
|
|
3539
|
+
function isLocalProviderResult(result, baseUrl) {
|
|
3540
|
+
const rawUrl = result.url.trim();
|
|
3541
|
+
const explicitScheme = rawUrl.match(/^([a-z][a-z\d+.-]*):/iu)?.[1]?.toLowerCase();
|
|
3542
|
+
if (explicitScheme && explicitScheme !== "http" && explicitScheme !== "https") return false;
|
|
3543
|
+
if (!(/^[a-z][a-z\d+.-]*:/iu.test(rawUrl) || /^[\\/]{2}/u.test(rawUrl))) return true;
|
|
3544
|
+
if (!baseUrl) return false;
|
|
3545
|
+
try {
|
|
3546
|
+
return new URL(result.url, baseUrl).origin === new URL(baseUrl).origin;
|
|
3547
|
+
} catch {
|
|
3548
|
+
return false;
|
|
3549
|
+
}
|
|
3550
|
+
}
|
|
3551
|
+
function hasOppositeAudienceEvidence(options) {
|
|
3552
|
+
const { result, pages, query, audience, baseUrl } = options;
|
|
3553
|
+
if (!isLocalProviderResult(result, baseUrl)) return false;
|
|
3554
|
+
const pagePath = normalizeUrlRouteKey(result.url);
|
|
3555
|
+
const page = pages.find((candidate) => normalizeUrlRouteKey(candidate.url) === pagePath);
|
|
3556
|
+
if (!page) return false;
|
|
3557
|
+
if (scoreDocument(query, pageToSearchDocument(page, audience)) > 0) return false;
|
|
3558
|
+
const oppositeAudience = audience === "agent" ? "human" : "agent";
|
|
3559
|
+
if (scoreDocument(query, pageToSearchDocument(page, oppositeAudience)) > 0) return true;
|
|
3560
|
+
const evidence = cleanSearchResultText(result.description);
|
|
3561
|
+
if (!evidence) return false;
|
|
3562
|
+
const selectedTokens = new Set(tokenizeSearchQuery(getPageAudienceSearchText(page, audience)));
|
|
3563
|
+
const oppositeTokens = new Set(tokenizeSearchQuery(getPageAudienceSearchText(page, oppositeAudience)));
|
|
3564
|
+
const evidenceTokens = [...new Set(tokenizeSearchQuery(evidence))];
|
|
3565
|
+
let selectedOnlyMatches = 0;
|
|
3566
|
+
let oppositeOnlyMatches = 0;
|
|
3567
|
+
for (const token of evidenceTokens) {
|
|
3568
|
+
const inSelected = selectedTokens.has(token);
|
|
3569
|
+
const inOpposite = oppositeTokens.has(token);
|
|
3570
|
+
if (inSelected && !inOpposite) selectedOnlyMatches += 1;
|
|
3571
|
+
if (inOpposite && !inSelected) oppositeOnlyMatches += 1;
|
|
3572
|
+
}
|
|
3573
|
+
return oppositeOnlyMatches > selectedOnlyMatches;
|
|
3574
|
+
}
|
|
3575
|
+
function pageToSearchDocument(rawPage, audience = "human") {
|
|
3576
|
+
const page = localizeDocsSearchPage(rawPage, rawPage.locale);
|
|
3577
|
+
const scope = resolveDocsSearchPageScope(page);
|
|
3578
|
+
return {
|
|
3579
|
+
id: makeDocumentId(page.url, "page"),
|
|
3580
|
+
url: page.url,
|
|
3581
|
+
title: page.title,
|
|
3582
|
+
content: getPageAudienceIndexContent(page, audience),
|
|
3583
|
+
description: page.description,
|
|
3584
|
+
type: "page",
|
|
3585
|
+
locale: page.locale,
|
|
3586
|
+
framework: scope.framework[0],
|
|
3587
|
+
version: scope.version[0],
|
|
3588
|
+
package: scope.package.length > 0 ? scope.package : void 0,
|
|
3589
|
+
tags: scope.tags.length > 0 ? scope.tags : void 0
|
|
3590
|
+
};
|
|
3591
|
+
}
|
|
3592
|
+
function buildExactPageSearchResults(query, pages, audience = "human") {
|
|
3593
|
+
const normalizedQuery = normalizeSearchPhrase(query);
|
|
3594
|
+
if (!normalizedQuery) return [];
|
|
3595
|
+
const results = [];
|
|
3596
|
+
for (const page of pages) {
|
|
3597
|
+
const document = pageToSearchDocument(page, audience);
|
|
3598
|
+
const title = normalizeSearchPhrase(page.title);
|
|
3599
|
+
const urlSegments = getUrlSearchSegments(page.url);
|
|
3600
|
+
if (!(title === normalizedQuery || urlSegments.includes(normalizedQuery))) continue;
|
|
3601
|
+
results.push({
|
|
3602
|
+
id: document.id,
|
|
3603
|
+
url: document.url,
|
|
3604
|
+
content: cleanSearchResultText(document.title) ?? document.title,
|
|
3605
|
+
description: cleanSearchResultText(buildSnippet(document, query) ?? document.description),
|
|
3606
|
+
type: "page",
|
|
3607
|
+
score: scoreDocument(query, document) + 2e3
|
|
3608
|
+
});
|
|
3609
|
+
}
|
|
3610
|
+
return results.sort((a, b) => (b.score ?? 0) - (a.score ?? 0) || a.url.localeCompare(b.url));
|
|
3611
|
+
}
|
|
3612
|
+
function hasDistinctResultSection(result) {
|
|
3613
|
+
if (result.type === "page") return false;
|
|
3614
|
+
const section = normalizeSearchPhrase(stripHtml(result.section ?? ""));
|
|
3615
|
+
if (!section) return true;
|
|
3616
|
+
return section !== normalizeSearchPhrase(normalizeSearchPhrase(stripHtml(result.content)).split(/\s+[—–]\s+/)[0] ?? "");
|
|
3617
|
+
}
|
|
3618
|
+
function insideLiteralResultPriority(query, result) {
|
|
3619
|
+
if (!hasDistinctResultSection(result) || !isLiteralLookupQuery(query)) return 0;
|
|
3620
|
+
return Math.max(literalMatchPriority(query, stripHtml(result.section ?? "")), literalMatchPriority(query, stripHtml(result.description ?? "")));
|
|
3621
|
+
}
|
|
3622
|
+
function prioritizeLiteralInsideResults(query, results) {
|
|
3623
|
+
if (!isLiteralLookupQuery(query)) return results;
|
|
3624
|
+
return [...results].sort((a, b) => {
|
|
3625
|
+
const literalDelta = insideLiteralResultPriority(query, b) - insideLiteralResultPriority(query, a);
|
|
3626
|
+
if (literalDelta) return literalDelta;
|
|
3627
|
+
return 0;
|
|
3628
|
+
});
|
|
3629
|
+
}
|
|
3630
|
+
function rankAskAIContextResult(query, result) {
|
|
3631
|
+
return scoreDocument(query, {
|
|
3632
|
+
id: result.id,
|
|
3633
|
+
url: result.url,
|
|
3634
|
+
title: result.title,
|
|
3635
|
+
content: result.contextContent,
|
|
3636
|
+
description: result.description,
|
|
3637
|
+
type: result.type,
|
|
3638
|
+
section: result.section
|
|
3639
|
+
});
|
|
3640
|
+
}
|
|
3641
|
+
function buildAskAIContextBlock(result) {
|
|
3642
|
+
const lines = [`## ${result.title}`, `URL: ${result.url}`];
|
|
3643
|
+
if (result.section) lines.push(`Section: ${result.section}`);
|
|
3644
|
+
if (result.description) lines.push(`Search snippet: ${result.description}`);
|
|
3645
|
+
if (result.source) {
|
|
3646
|
+
const scope = result.source.scope;
|
|
3647
|
+
const scopeEntries = [
|
|
3648
|
+
`audience=${scope.audience}`,
|
|
3649
|
+
scope.locale?.length ? `locale=${scope.locale.join(",")}` : void 0,
|
|
3650
|
+
scope.framework?.length ? `framework=${scope.framework.join(",")}` : void 0,
|
|
3651
|
+
scope.version?.length ? `version=${scope.version.join(",")}` : void 0,
|
|
3652
|
+
scope.package?.length ? `package=${scope.package.join(",")}` : void 0,
|
|
3653
|
+
scope.tags?.length ? `tags=${scope.tags.join(",")}` : void 0,
|
|
3654
|
+
scope.conflicts?.length ? `conflicts=${scope.conflicts.join(",")}` : void 0
|
|
3655
|
+
].filter((value) => Boolean(value));
|
|
3656
|
+
lines.push(`Canonical URL: ${result.source.canonicalUrl}`);
|
|
3657
|
+
lines.push(`Source scope: ${scopeEntries.join("; ")}`);
|
|
3658
|
+
if (result.source.lastModified) lines.push(`Source modified: ${result.source.lastModified}`);
|
|
3659
|
+
lines.push(`Source digest: ${result.source.digest}`);
|
|
3660
|
+
lines.push(`Index generation: ${result.source.indexGeneration}`);
|
|
3661
|
+
}
|
|
3662
|
+
lines.push("", result.contextContent);
|
|
3663
|
+
return lines.join("\n").trim();
|
|
3664
|
+
}
|
|
3665
|
+
function makeDocumentId(url, suffix) {
|
|
3666
|
+
return `${url}#${suffix}`;
|
|
3667
|
+
}
|
|
3668
|
+
function splitPageIntoSections(page, audience = "human") {
|
|
3669
|
+
const raw = getPageAudienceSectionContent(page, audience);
|
|
3670
|
+
const generatedContract = audience === "agent" ? findDocsGeneratedAgentContractRanges(raw)[0] : void 0;
|
|
3671
|
+
const scope = resolveDocsSearchPageScope(page);
|
|
3672
|
+
let documentIndex = 0;
|
|
3673
|
+
return parseDocsMarkdownSections(raw).flatMap((section) => {
|
|
3674
|
+
if (generatedContract && section.startLine > generatedContract.startLine && section.startLine < generatedContract.endLine) return [];
|
|
3675
|
+
const content = normalizeWhitespace(stripMarkdownText(section.content));
|
|
3676
|
+
if (!content) return [];
|
|
3677
|
+
const index = documentIndex;
|
|
3678
|
+
documentIndex += 1;
|
|
3679
|
+
return [{
|
|
3680
|
+
id: makeDocumentId(page.url, `section-${index}`),
|
|
3681
|
+
url: `${page.url.split("#", 1)[0]}#${encodeURIComponent(section.anchor)}`,
|
|
3682
|
+
title: page.title,
|
|
3683
|
+
section: section.title,
|
|
3684
|
+
content,
|
|
3685
|
+
description: page.description,
|
|
3686
|
+
type: "heading",
|
|
3687
|
+
locale: page.locale,
|
|
3688
|
+
framework: scope.framework[0],
|
|
3689
|
+
version: scope.version[0],
|
|
3690
|
+
package: scope.package.length > 0 ? scope.package : void 0,
|
|
3691
|
+
tags: scope.tags.length > 0 ? scope.tags : void 0
|
|
3692
|
+
}];
|
|
3693
|
+
});
|
|
3694
|
+
}
|
|
3695
|
+
function buildDocsSearchDocuments(pages, chunking = {}, audience = "human") {
|
|
3696
|
+
const strategy = chunking.strategy ?? "section";
|
|
3697
|
+
return pages.flatMap((rawPage) => {
|
|
3698
|
+
const page = localizeDocsSearchPage(rawPage, rawPage.locale);
|
|
3699
|
+
const base = pageToSearchDocument(page, audience);
|
|
3700
|
+
if (strategy === "page") return [base];
|
|
3701
|
+
const sections = splitPageIntoSections(page, audience);
|
|
3702
|
+
if (sections.length === 0) return [base];
|
|
3703
|
+
return [...base.content ? [base] : [], ...sections];
|
|
3704
|
+
});
|
|
3705
|
+
}
|
|
3706
|
+
function scoreDocument(query, document) {
|
|
3707
|
+
const q = normalizeSearchPhrase(query);
|
|
3708
|
+
if (!q) return 0;
|
|
3709
|
+
const words = tokenizeSearchQuery(q);
|
|
3710
|
+
const title = normalizeSearchPhrase(document.title);
|
|
3711
|
+
const section = document.section ? normalizeSearchPhrase(document.section) : "";
|
|
3712
|
+
const hasDistinctSection = Boolean(section && section !== title);
|
|
3713
|
+
const titleSection = section ? normalizeSearchPhrase(`${document.title} ${document.section}`) : "";
|
|
3714
|
+
const description = document.description ? normalizeSearchPhrase(document.description) : "";
|
|
3715
|
+
const content = normalizeSearchPhrase(document.content);
|
|
3716
|
+
const url = normalizeSearchPhrase(document.url);
|
|
3717
|
+
const urlSegments = getUrlSearchSegments(document.url);
|
|
3718
|
+
const titleTokens = tokenizeSearchQuery(title);
|
|
3719
|
+
const sectionTokens = tokenizeSearchQuery(section);
|
|
3720
|
+
let score = 0;
|
|
3721
|
+
const insideLiteralPriority = document.type !== "page" && hasDistinctSection && isLiteralLookupQuery(q) ? Math.max(literalMatchPriority(q, section), literalMatchPriority(q, description), literalMatchPriority(q, content)) : 0;
|
|
3722
|
+
if (insideLiteralPriority > 0) score += insideLiteralPriority * 2250;
|
|
3723
|
+
if (title === q) score += 1120;
|
|
3724
|
+
else if (title.startsWith(q)) score += 70;
|
|
3725
|
+
else if (title.includes(q)) score += 45;
|
|
3726
|
+
if (hasDistinctSection) {
|
|
3727
|
+
if (section === q) score += 1080;
|
|
3728
|
+
else if (section.startsWith(q)) score += 55;
|
|
3729
|
+
else if (section.includes(q)) score += 30;
|
|
3730
|
+
if (titleSection === q) score += 1e3;
|
|
3731
|
+
else if (titleSection.startsWith(q)) score += 50;
|
|
3732
|
+
else if (titleSection.includes(q)) score += 28;
|
|
3733
|
+
}
|
|
3734
|
+
if (urlSegments.includes(q)) score += 950;
|
|
3735
|
+
if (url.includes(q)) score += 12;
|
|
3736
|
+
if (description.includes(q)) score += 18;
|
|
3737
|
+
if (content.includes(q)) score += 12;
|
|
3738
|
+
let matchedWords = 0;
|
|
3739
|
+
for (const word of words) {
|
|
3740
|
+
let matched = false;
|
|
3741
|
+
if (title === word) {
|
|
3742
|
+
score += 28;
|
|
3743
|
+
matched = true;
|
|
3744
|
+
} else if (title.startsWith(word)) {
|
|
3745
|
+
score += 20;
|
|
3746
|
+
matched = true;
|
|
3747
|
+
} else if (title.includes(word)) {
|
|
3748
|
+
score += 12;
|
|
3749
|
+
matched = true;
|
|
3750
|
+
}
|
|
3751
|
+
if (hasDistinctSection) {
|
|
3752
|
+
if (section === word) {
|
|
3753
|
+
score += 22;
|
|
3754
|
+
matched = true;
|
|
3755
|
+
} else if (section.startsWith(word)) {
|
|
3756
|
+
score += 16;
|
|
3757
|
+
matched = true;
|
|
3758
|
+
} else if (section.includes(word)) {
|
|
3759
|
+
score += 10;
|
|
3760
|
+
matched = true;
|
|
3761
|
+
}
|
|
3762
|
+
}
|
|
3763
|
+
if (description.includes(word)) {
|
|
3764
|
+
score += 6;
|
|
3765
|
+
matched = true;
|
|
3766
|
+
}
|
|
3767
|
+
if (content.includes(word)) {
|
|
3768
|
+
score += 4;
|
|
3769
|
+
matched = true;
|
|
3770
|
+
}
|
|
3771
|
+
if (matched) matchedWords += 1;
|
|
3772
|
+
}
|
|
3773
|
+
if (words.length > 1) {
|
|
3774
|
+
if (hasDistinctSection && sectionTokens.length > 0 && words.every((word) => sectionTokens.includes(word))) score += 30;
|
|
3775
|
+
if (document.type === "page" && titleTokens.length > 0 && words.every((word) => titleTokens.includes(word))) score += 24;
|
|
3776
|
+
}
|
|
3777
|
+
if (matchedWords === words.length && words.length > 1) score += 20;
|
|
3778
|
+
if (score > 0 && document.type === "heading" && hasDistinctSection) score += 6;
|
|
3779
|
+
return score;
|
|
3780
|
+
}
|
|
3781
|
+
function buildSnippet(document, query) {
|
|
3782
|
+
const q = query.trim().toLowerCase();
|
|
3783
|
+
const sources = [normalizeWhitespace(stripMarkdownText(document.content)), normalizeWhitespace(stripMarkdownText(document.description ?? ""))].filter(Boolean);
|
|
3784
|
+
for (const source of sources) {
|
|
3785
|
+
if (!q) return clampSearchSnippet(source);
|
|
3786
|
+
const idx = source.toLowerCase().indexOf(q);
|
|
3787
|
+
if (idx === -1) continue;
|
|
3788
|
+
const start = Math.max(0, idx - 48);
|
|
3789
|
+
const end = Math.min(source.length, idx + q.length + 96);
|
|
3790
|
+
const prefix = start > 0 ? "..." : "";
|
|
3791
|
+
const suffix = end < source.length ? "..." : "";
|
|
3792
|
+
return clampSearchSnippet(`${prefix}${source.slice(start, end).trim()}${suffix}`);
|
|
3793
|
+
}
|
|
3794
|
+
return sources[0] ? clampSearchSnippet(sources[0]) : void 0;
|
|
3795
|
+
}
|
|
3796
|
+
function clampSearchSnippet(value) {
|
|
3797
|
+
if (value.length <= MAX_SEARCH_SNIPPET_CHARS) return value;
|
|
3798
|
+
return `${value.slice(0, MAX_SEARCH_SNIPPET_CHARS - 3).trimEnd()}...`;
|
|
3799
|
+
}
|
|
3800
|
+
function cleanSearchResultText(value) {
|
|
3801
|
+
if (!value) return void 0;
|
|
3802
|
+
return normalizeWhitespace(stripHtml(stripMarkdownText(value))) || void 0;
|
|
3803
|
+
}
|
|
3804
|
+
function getHostedIndexGenerationTag(indexGeneration) {
|
|
3805
|
+
return `docs-generation:${indexGeneration}`;
|
|
3806
|
+
}
|
|
3807
|
+
function buildAlgoliaRecord(document, corpusId) {
|
|
3808
|
+
const providerDocumentId = corpusId ? makeHostedProviderDocumentId(corpusId, document.id) : document.id;
|
|
3809
|
+
const generationTag = document.source?.indexGeneration ? getHostedIndexGenerationTag(document.source.indexGeneration) : void 0;
|
|
3810
|
+
const record = {
|
|
3811
|
+
objectID: providerDocumentId,
|
|
3812
|
+
id: providerDocumentId,
|
|
3813
|
+
url: document.url,
|
|
3814
|
+
title: document.title,
|
|
3815
|
+
section: document.section,
|
|
3816
|
+
content: document.content,
|
|
3817
|
+
description: document.description,
|
|
3818
|
+
type: document.type,
|
|
3819
|
+
locale: document.locale ?? document.source?.scope.locale?.[0],
|
|
3820
|
+
framework: document.framework,
|
|
3821
|
+
version: document.version,
|
|
3822
|
+
package: document.package,
|
|
3823
|
+
tags: document.tags,
|
|
3824
|
+
_tags: corpusId || generationTag ? [corpusId, generationTag].filter((value) => Boolean(value)) : void 0,
|
|
3825
|
+
...serializeHostedRetrievalSource(document.source, corpusId, document.id)
|
|
3826
|
+
};
|
|
3827
|
+
const encoder = new TextEncoder();
|
|
3828
|
+
const sizeOf = (value) => encoder.encode(JSON.stringify(value)).length;
|
|
3829
|
+
if (sizeOf(record) <= ALGOLIA_MAX_RECORD_BYTES) return record;
|
|
3830
|
+
delete record.description;
|
|
3831
|
+
if (sizeOf(record) <= ALGOLIA_MAX_RECORD_BYTES) return record;
|
|
3832
|
+
record.content = "";
|
|
3833
|
+
for (const field of [
|
|
3834
|
+
"package",
|
|
3835
|
+
"tags",
|
|
3836
|
+
"framework",
|
|
3837
|
+
"version"
|
|
3838
|
+
]) {
|
|
3839
|
+
if (sizeOf(record) <= ALGOLIA_MAX_RECORD_BYTES) break;
|
|
3840
|
+
delete record[field];
|
|
3841
|
+
}
|
|
3842
|
+
const trimFieldToFit = (field, value) => {
|
|
3843
|
+
let low = 0;
|
|
3844
|
+
let high = value.length;
|
|
3845
|
+
let best = "";
|
|
3846
|
+
while (low <= high) {
|
|
3847
|
+
const middle = Math.floor((low + high) / 2);
|
|
3848
|
+
const candidate = middle < value.length ? `${value.slice(0, middle).trimEnd()}...` : value;
|
|
3849
|
+
record[field] = candidate;
|
|
3850
|
+
if (sizeOf(record) <= ALGOLIA_MAX_RECORD_BYTES) {
|
|
3851
|
+
best = candidate;
|
|
3852
|
+
low = middle + 1;
|
|
3853
|
+
} else high = middle - 1;
|
|
3854
|
+
}
|
|
3855
|
+
record[field] = best;
|
|
3856
|
+
};
|
|
3857
|
+
if (sizeOf(record) > ALGOLIA_MAX_RECORD_BYTES && typeof record.section === "string") trimFieldToFit("section", record.section);
|
|
3858
|
+
if (sizeOf(record) > ALGOLIA_MAX_RECORD_BYTES && typeof record.title === "string") trimFieldToFit("title", record.title);
|
|
3859
|
+
if (sizeOf(record) > ALGOLIA_MAX_RECORD_BYTES) throw new Error(`Algolia record ${document.id} exceeds the record limit with required provenance.`);
|
|
3860
|
+
trimFieldToFit("content", document.content);
|
|
3861
|
+
if (sizeOf(record) > ALGOLIA_MAX_RECORD_BYTES) throw new Error(`Algolia record ${document.id} still exceeds the record limit after trimming.`);
|
|
3862
|
+
return record;
|
|
3863
|
+
}
|
|
3864
|
+
async function searchSimpleDocsPage(query, context) {
|
|
3865
|
+
const limit = query.limit ?? DEFAULT_SEARCH_LIMIT;
|
|
3866
|
+
const offset = query.offset ?? 0;
|
|
3867
|
+
const matches = context.documents.map((document) => ({
|
|
3868
|
+
document,
|
|
3869
|
+
score: scoreDocument(query.query, document)
|
|
3870
|
+
})).filter((item) => item.score > 0).sort((a, b) => {
|
|
3871
|
+
if (b.score !== a.score) return b.score - a.score;
|
|
3872
|
+
return a.document.url.localeCompare(b.document.url);
|
|
3873
|
+
});
|
|
3874
|
+
const results = matches.slice(offset, offset + limit).map(({ document, score }) => ({
|
|
3875
|
+
id: document.id,
|
|
3876
|
+
url: document.url,
|
|
3877
|
+
content: cleanSearchResultText(document.section ? `${document.title} — ${document.section}` : document.title) ?? (document.section ? `${document.title} — ${document.section}` : document.title),
|
|
3878
|
+
description: cleanSearchResultText(buildSnippet(document, query.query) ?? document.description),
|
|
3879
|
+
type: document.type,
|
|
3880
|
+
score,
|
|
3881
|
+
section: document.section,
|
|
3882
|
+
source: document.source
|
|
3883
|
+
}));
|
|
3884
|
+
return {
|
|
3885
|
+
results: context.deferSourceProvenance ? results : await enrichDocsSearchResultsWithSources({
|
|
3886
|
+
results,
|
|
3887
|
+
pages: context.pages,
|
|
3888
|
+
audience: resolveDocsSearchAudience(context.audience),
|
|
3889
|
+
chunking: context.chunking ?? { strategy: "section" },
|
|
3890
|
+
locale: query.locale ?? context.locale,
|
|
3891
|
+
baseUrl: context.baseUrl,
|
|
3892
|
+
indexGeneration: context.indexGeneration,
|
|
3893
|
+
filters: query.filters
|
|
3894
|
+
}),
|
|
3895
|
+
total: matches.length
|
|
3896
|
+
};
|
|
3897
|
+
}
|
|
3898
|
+
function createSimpleSearchAdapter() {
|
|
3899
|
+
return {
|
|
3900
|
+
name: "simple",
|
|
3901
|
+
async search(query, context) {
|
|
3902
|
+
return (await searchSimpleDocsPage({
|
|
3903
|
+
...query,
|
|
3904
|
+
offset: 0
|
|
3905
|
+
}, context)).results;
|
|
3906
|
+
},
|
|
3907
|
+
searchPage: searchSimpleDocsPage
|
|
3908
|
+
};
|
|
3909
|
+
}
|
|
3910
|
+
function normalizeDocsSearchConfig(search) {
|
|
3911
|
+
if (search === false) return {
|
|
3912
|
+
enabled: false,
|
|
3913
|
+
provider: "simple",
|
|
3914
|
+
maxResults: DEFAULT_SEARCH_LIMIT,
|
|
3915
|
+
chunking: { strategy: "section" }
|
|
3916
|
+
};
|
|
3917
|
+
if (!search || search === true) return {
|
|
3918
|
+
enabled: true,
|
|
3919
|
+
provider: "simple",
|
|
3920
|
+
maxResults: DEFAULT_SEARCH_LIMIT,
|
|
3921
|
+
chunking: { strategy: "section" },
|
|
3922
|
+
raw: typeof search === "object" ? search : void 0
|
|
3923
|
+
};
|
|
3924
|
+
const provider = search.provider ?? "simple";
|
|
3925
|
+
const maxResults = search.maxResults ?? DEFAULT_SEARCH_LIMIT;
|
|
3926
|
+
const chunking = search.chunking ?? { strategy: "section" };
|
|
3927
|
+
return {
|
|
3928
|
+
enabled: search.enabled ?? true,
|
|
3929
|
+
provider,
|
|
3930
|
+
maxResults,
|
|
3931
|
+
chunking,
|
|
3932
|
+
raw: search
|
|
3933
|
+
};
|
|
3934
|
+
}
|
|
3935
|
+
function getDocsSearchPaginationProviderIdentity(search) {
|
|
3936
|
+
const raw = search.raw;
|
|
3937
|
+
if (raw?.provider === "algolia") return {
|
|
3938
|
+
provider: "algolia",
|
|
3939
|
+
appId: raw.appId,
|
|
3940
|
+
indexName: raw.indexName,
|
|
3941
|
+
syncNamespace: raw.syncNamespace?.trim() || void 0
|
|
3942
|
+
};
|
|
3943
|
+
if (raw?.provider === "typesense") return {
|
|
3944
|
+
provider: "typesense",
|
|
3945
|
+
baseUrl: raw.baseUrl,
|
|
3946
|
+
collection: raw.collection,
|
|
3947
|
+
mode: raw.mode ?? "keyword",
|
|
3948
|
+
queryBy: raw.queryBy ?? [
|
|
3949
|
+
"title",
|
|
3950
|
+
"section",
|
|
3951
|
+
"content",
|
|
3952
|
+
"description"
|
|
3953
|
+
],
|
|
3954
|
+
embeddings: raw.embeddings ? {
|
|
3955
|
+
provider: raw.embeddings.provider,
|
|
3956
|
+
model: raw.embeddings.model,
|
|
3957
|
+
baseUrl: raw.embeddings.baseUrl ?? "http://127.0.0.1:11434"
|
|
3958
|
+
} : void 0,
|
|
3959
|
+
syncNamespace: raw.syncNamespace?.trim() || void 0
|
|
3960
|
+
};
|
|
3961
|
+
if (raw?.provider === "mcp") return {
|
|
3962
|
+
provider: "mcp",
|
|
3963
|
+
endpoint: raw.endpoint,
|
|
3964
|
+
toolName: raw.toolName ?? "search_docs",
|
|
3965
|
+
protocolVersion: raw.protocolVersion ?? DEFAULT_MCP_PROTOCOL_VERSION,
|
|
3966
|
+
forwardAudience: raw.forwardAudience === true
|
|
3967
|
+
};
|
|
3968
|
+
if (raw?.provider === "custom") return {
|
|
3969
|
+
provider: "custom",
|
|
3970
|
+
adapterName: typeof raw.adapter === "object" && raw.adapter ? raw.adapter.name : void 0,
|
|
3971
|
+
paginationRevision: raw.paginationRevision?.trim() || void 0
|
|
3972
|
+
};
|
|
3973
|
+
return { provider: search.provider };
|
|
3974
|
+
}
|
|
3975
|
+
async function readResponseJson(response) {
|
|
3976
|
+
const text = await response.text();
|
|
3977
|
+
return text ? JSON.parse(text) : null;
|
|
3978
|
+
}
|
|
3979
|
+
async function readMcpResponsePayload(response) {
|
|
3980
|
+
const text = await response.text();
|
|
3981
|
+
if (!text) return null;
|
|
3982
|
+
if ((response.headers.get("content-type") ?? "").includes("application/json")) return JSON.parse(text);
|
|
3983
|
+
return normalizeMcpSsePayload(text);
|
|
3984
|
+
}
|
|
3985
|
+
function ensureOk(response, message) {
|
|
3986
|
+
if (response.ok) return;
|
|
3987
|
+
throw new Error(`${message} (${response.status} ${response.statusText})`);
|
|
3988
|
+
}
|
|
3989
|
+
function ensureJsonRpcOk(payload, message) {
|
|
3990
|
+
if (payload && typeof payload === "object" && "error" in payload && payload.error && typeof payload.error === "object" && "message" in payload.error) throw new Error(`${message}: ${String(payload.error.message)}`);
|
|
3991
|
+
}
|
|
3992
|
+
function resolveMcpEndpoint(endpoint) {
|
|
3993
|
+
if (/^https?:\/\//i.test(endpoint)) return endpoint;
|
|
3994
|
+
throw new Error("Relative MCP search endpoints must be resolved before creating the MCP adapter.");
|
|
3995
|
+
}
|
|
3996
|
+
function isDocsSearchResultType(value) {
|
|
3997
|
+
return value === "page" || value === "heading" || value === "text";
|
|
3998
|
+
}
|
|
3999
|
+
function mapMcpSearchResult(value, sourceBaseUrl) {
|
|
4000
|
+
if (!value || typeof value !== "object") return null;
|
|
4001
|
+
const item = value;
|
|
4002
|
+
const section = typeof item.section === "string" ? item.section : void 0;
|
|
4003
|
+
const title = typeof item.title === "string" ? item.title : void 0;
|
|
4004
|
+
const content = typeof item.content === "string" ? item.content : title ? section ? `${title} — ${section}` : title : void 0;
|
|
4005
|
+
const url = typeof item.url === "string" ? item.url : void 0;
|
|
4006
|
+
if (!content || !url) return null;
|
|
4007
|
+
const hasSource = item.source !== void 0;
|
|
4008
|
+
let sourceValue = item.source;
|
|
4009
|
+
if (hasSource && sourceBaseUrl && sourceValue && typeof sourceValue === "object" && !Array.isArray(sourceValue)) {
|
|
4010
|
+
const sourceRecord = sourceValue;
|
|
4011
|
+
if (typeof sourceRecord.canonicalUrl === "string" && /^\/(?!\/)/u.test(sourceRecord.canonicalUrl)) try {
|
|
4012
|
+
sourceValue = {
|
|
4013
|
+
...sourceRecord,
|
|
4014
|
+
canonicalUrl: new URL(sourceRecord.canonicalUrl, sourceBaseUrl).toString()
|
|
4015
|
+
};
|
|
4016
|
+
} catch {}
|
|
4017
|
+
}
|
|
4018
|
+
const source = hasSource ? parseDocsRetrievalSource(sourceValue) : void 0;
|
|
4019
|
+
if (hasSource && !source) return null;
|
|
4020
|
+
return {
|
|
4021
|
+
id: typeof item.id === "string" ? item.id : typeof item.slug === "string" ? item.slug : url,
|
|
4022
|
+
url,
|
|
4023
|
+
content: cleanSearchResultText(content) ?? content,
|
|
4024
|
+
description: cleanSearchResultText(typeof item.description === "string" ? item.description : typeof item.excerpt === "string" ? item.excerpt : void 0) ?? void 0,
|
|
4025
|
+
type: isDocsSearchResultType(item.type) ? item.type : section ? "heading" : "page",
|
|
4026
|
+
score: typeof item.score === "number" ? item.score : void 0,
|
|
4027
|
+
section,
|
|
4028
|
+
...source ? { source } : {}
|
|
4029
|
+
};
|
|
4030
|
+
}
|
|
4031
|
+
function readMcpSearchToolPayload(payload) {
|
|
4032
|
+
if (!payload || typeof payload !== "object" || !("result" in payload) || !payload.result || typeof payload.result !== "object") return null;
|
|
4033
|
+
const result = payload.result;
|
|
4034
|
+
if (result.structuredContent && typeof result.structuredContent === "object" && !Array.isArray(result.structuredContent)) return result.structuredContent;
|
|
4035
|
+
const content = Array.isArray(result.content) ? result.content : [];
|
|
4036
|
+
const resultText = content.length > 0 && content[0] && typeof content[0] === "object" && "text" in content[0] && typeof content[0].text === "string" ? content[0].text : null;
|
|
4037
|
+
return resultText ? JSON.parse(resultText) : null;
|
|
4038
|
+
}
|
|
4039
|
+
async function createOllamaEmbedding(text, config, signal) {
|
|
4040
|
+
const response = await fetch(`${(config.baseUrl ?? "http://127.0.0.1:11434").replace(/\/$/, "")}/api/embed`, {
|
|
4041
|
+
method: "POST",
|
|
4042
|
+
headers: { "Content-Type": "application/json" },
|
|
4043
|
+
body: JSON.stringify({
|
|
4044
|
+
model: config.model,
|
|
4045
|
+
input: text
|
|
4046
|
+
}),
|
|
4047
|
+
signal
|
|
4048
|
+
});
|
|
4049
|
+
ensureOk(response, "Failed to create Ollama embedding");
|
|
4050
|
+
const payload = await readResponseJson(response);
|
|
4051
|
+
if (Array.isArray(payload.embeddings?.[0])) return payload.embeddings[0];
|
|
4052
|
+
if (Array.isArray(payload.embedding)) return payload.embedding;
|
|
4053
|
+
throw new Error("Ollama embedding response did not include an embedding vector.");
|
|
4054
|
+
}
|
|
4055
|
+
function getTypesenseSearchBase(config) {
|
|
4056
|
+
return config.baseUrl.replace(/\/$/, "");
|
|
4057
|
+
}
|
|
4058
|
+
function quoteTypesenseFilterValue(value) {
|
|
4059
|
+
return `\`${value.replace(/\\/gu, "\\\\").replace(/`/gu, "\\`")}\``;
|
|
4060
|
+
}
|
|
4061
|
+
const TYPESENSE_RETRIEVAL_SOURCE_FIELDS = [
|
|
4062
|
+
{
|
|
4063
|
+
name: "source_corpus_id",
|
|
4064
|
+
type: "string",
|
|
4065
|
+
optional: true
|
|
4066
|
+
},
|
|
4067
|
+
{
|
|
4068
|
+
name: "source_document_id",
|
|
4069
|
+
type: "string",
|
|
4070
|
+
optional: true
|
|
4071
|
+
},
|
|
4072
|
+
{
|
|
4073
|
+
name: "source_canonical_url",
|
|
4074
|
+
type: "string",
|
|
4075
|
+
optional: true
|
|
4076
|
+
},
|
|
4077
|
+
{
|
|
4078
|
+
name: "source_scope_audience",
|
|
4079
|
+
type: "string",
|
|
4080
|
+
optional: true
|
|
4081
|
+
},
|
|
4082
|
+
{
|
|
4083
|
+
name: "source_scope_locale",
|
|
4084
|
+
type: "string[]",
|
|
4085
|
+
optional: true
|
|
4086
|
+
},
|
|
4087
|
+
{
|
|
4088
|
+
name: "source_scope_framework",
|
|
4089
|
+
type: "string[]",
|
|
4090
|
+
optional: true
|
|
4091
|
+
},
|
|
4092
|
+
{
|
|
4093
|
+
name: "source_scope_version",
|
|
4094
|
+
type: "string[]",
|
|
4095
|
+
optional: true
|
|
4096
|
+
},
|
|
4097
|
+
{
|
|
4098
|
+
name: "source_scope_version_groups",
|
|
4099
|
+
type: "string",
|
|
4100
|
+
optional: true
|
|
4101
|
+
},
|
|
4102
|
+
{
|
|
4103
|
+
name: "source_scope_package",
|
|
4104
|
+
type: "string[]",
|
|
4105
|
+
optional: true
|
|
4106
|
+
},
|
|
4107
|
+
{
|
|
4108
|
+
name: "source_scope_tags",
|
|
4109
|
+
type: "string[]",
|
|
4110
|
+
optional: true
|
|
4111
|
+
},
|
|
4112
|
+
{
|
|
4113
|
+
name: "source_scope_truncated",
|
|
4114
|
+
type: "string[]",
|
|
4115
|
+
optional: true
|
|
4116
|
+
},
|
|
4117
|
+
{
|
|
4118
|
+
name: "source_scope_conflicts",
|
|
4119
|
+
type: "string[]",
|
|
4120
|
+
optional: true
|
|
4121
|
+
},
|
|
4122
|
+
{
|
|
4123
|
+
name: "source_last_modified",
|
|
4124
|
+
type: "string",
|
|
4125
|
+
optional: true
|
|
4126
|
+
},
|
|
4127
|
+
{
|
|
4128
|
+
name: "source_digest",
|
|
4129
|
+
type: "string",
|
|
4130
|
+
optional: true
|
|
4131
|
+
},
|
|
4132
|
+
{
|
|
4133
|
+
name: "source_index_generation",
|
|
4134
|
+
type: "string",
|
|
4135
|
+
optional: true
|
|
4136
|
+
}
|
|
4137
|
+
];
|
|
4138
|
+
async function ensureTypesenseCollection(config, dimensions, signal, retryAfterCreateConflict = true, retryAfterAlterFailure = true) {
|
|
4139
|
+
const baseUrl = getTypesenseSearchBase(config);
|
|
4140
|
+
const headers = {
|
|
4141
|
+
"X-TYPESENSE-API-KEY": config.adminApiKey ?? config.apiKey,
|
|
4142
|
+
"Content-Type": "application/json"
|
|
4143
|
+
};
|
|
4144
|
+
const existing = await fetch(`${baseUrl}/collections/${encodeURIComponent(config.collection)}`, {
|
|
4145
|
+
headers,
|
|
4146
|
+
signal
|
|
4147
|
+
});
|
|
4148
|
+
if (existing.ok) {
|
|
4149
|
+
const existingPayload = await readResponseJson(existing);
|
|
4150
|
+
if (!Array.isArray(existingPayload?.fields)) throw new Error("Typesense collection response did not include a fields array.");
|
|
4151
|
+
const existingFields = new Map(existingPayload.fields.flatMap((field) => typeof field.name === "string" ? [[field.name, field]] : []));
|
|
4152
|
+
const incompatibleFields = TYPESENSE_RETRIEVAL_SOURCE_FIELDS.flatMap((expected) => {
|
|
4153
|
+
const name = typeof expected.name === "string" ? expected.name : void 0;
|
|
4154
|
+
const existingField = name ? existingFields.get(name) : void 0;
|
|
4155
|
+
if (!name || !existingField) return [];
|
|
4156
|
+
return existingField.type === expected.type && existingField.optional === true ? [] : [name];
|
|
4157
|
+
});
|
|
4158
|
+
if (incompatibleFields.length > 0) throw new Error(`Typesense collection has incompatible provenance fields: ${incompatibleFields.join(", ")}.`);
|
|
4159
|
+
const missingFields = TYPESENSE_RETRIEVAL_SOURCE_FIELDS.filter((field) => typeof field.name === "string" && !existingFields.has(field.name));
|
|
4160
|
+
const embeddingField = existingPayload.fields.find((field) => field.name === "embedding");
|
|
4161
|
+
const expectedEmbeddingField = config.embeddings && dimensions ? {
|
|
4162
|
+
name: "embedding",
|
|
4163
|
+
type: "float[]",
|
|
4164
|
+
num_dim: dimensions,
|
|
4165
|
+
optional: true
|
|
4166
|
+
} : void 0;
|
|
4167
|
+
const replaceEmbedding = Boolean(expectedEmbeddingField && embeddingField && (embeddingField.type !== "float[]" || embeddingField.num_dim !== dimensions));
|
|
4168
|
+
if (replaceEmbedding) ensureOk(await fetch(`${baseUrl}/collections/${encodeURIComponent(config.collection)}`, {
|
|
4169
|
+
method: "PATCH",
|
|
4170
|
+
headers,
|
|
4171
|
+
body: JSON.stringify({ fields: [{
|
|
4172
|
+
name: "embedding",
|
|
4173
|
+
drop: true
|
|
4174
|
+
}] }),
|
|
4175
|
+
signal
|
|
4176
|
+
}), "Failed to replace an incompatible Typesense embedding field");
|
|
4177
|
+
const fieldsToAdd = [...missingFields, ...expectedEmbeddingField && (!embeddingField || replaceEmbedding) ? [expectedEmbeddingField] : []];
|
|
4178
|
+
if (fieldsToAdd.length === 0) return;
|
|
4179
|
+
const altered = await fetch(`${baseUrl}/collections/${encodeURIComponent(config.collection)}`, {
|
|
4180
|
+
method: "PATCH",
|
|
4181
|
+
headers,
|
|
4182
|
+
body: JSON.stringify({ fields: fieldsToAdd }),
|
|
4183
|
+
signal
|
|
4184
|
+
});
|
|
4185
|
+
if (!altered.ok && retryAfterAlterFailure) {
|
|
4186
|
+
await ensureTypesenseCollection(config, dimensions, signal, retryAfterCreateConflict, false);
|
|
4187
|
+
return;
|
|
4188
|
+
}
|
|
4189
|
+
ensureOk(altered, "Failed to update the Typesense search schema");
|
|
4190
|
+
return;
|
|
4191
|
+
}
|
|
4192
|
+
if (existing.status !== 404) ensureOk(existing, "Failed to inspect Typesense collection");
|
|
4193
|
+
const fields = [
|
|
4194
|
+
{
|
|
4195
|
+
name: "id",
|
|
4196
|
+
type: "string"
|
|
4197
|
+
},
|
|
4198
|
+
{
|
|
4199
|
+
name: "url",
|
|
4200
|
+
type: "string"
|
|
4201
|
+
},
|
|
4202
|
+
{
|
|
4203
|
+
name: "title",
|
|
4204
|
+
type: "string"
|
|
4205
|
+
},
|
|
4206
|
+
{
|
|
4207
|
+
name: "section",
|
|
4208
|
+
type: "string",
|
|
4209
|
+
optional: true
|
|
4210
|
+
},
|
|
4211
|
+
{
|
|
4212
|
+
name: "content",
|
|
4213
|
+
type: "string"
|
|
4214
|
+
},
|
|
4215
|
+
{
|
|
4216
|
+
name: "description",
|
|
4217
|
+
type: "string",
|
|
4218
|
+
optional: true
|
|
4219
|
+
},
|
|
4220
|
+
{
|
|
4221
|
+
name: "type",
|
|
4222
|
+
type: "string"
|
|
4223
|
+
},
|
|
4224
|
+
{
|
|
4225
|
+
name: "locale",
|
|
4226
|
+
type: "string",
|
|
4227
|
+
optional: true
|
|
4228
|
+
},
|
|
4229
|
+
{
|
|
4230
|
+
name: "framework",
|
|
4231
|
+
type: "string",
|
|
4232
|
+
optional: true
|
|
4233
|
+
},
|
|
4234
|
+
{
|
|
4235
|
+
name: "version",
|
|
4236
|
+
type: "string",
|
|
4237
|
+
optional: true
|
|
4238
|
+
},
|
|
4239
|
+
{
|
|
4240
|
+
name: "package",
|
|
4241
|
+
type: "string[]",
|
|
4242
|
+
optional: true
|
|
4243
|
+
},
|
|
4244
|
+
{
|
|
4245
|
+
name: "tags",
|
|
4246
|
+
type: "string[]",
|
|
4247
|
+
optional: true
|
|
4248
|
+
},
|
|
4249
|
+
...TYPESENSE_RETRIEVAL_SOURCE_FIELDS
|
|
4250
|
+
];
|
|
4251
|
+
if (config.embeddings && dimensions) fields.push({
|
|
4252
|
+
name: "embedding",
|
|
4253
|
+
type: "float[]",
|
|
4254
|
+
num_dim: dimensions,
|
|
4255
|
+
optional: true
|
|
4256
|
+
});
|
|
4257
|
+
const response = await fetch(`${baseUrl}/collections`, {
|
|
4258
|
+
method: "POST",
|
|
4259
|
+
headers,
|
|
4260
|
+
body: JSON.stringify({
|
|
4261
|
+
name: config.collection,
|
|
4262
|
+
fields
|
|
4263
|
+
}),
|
|
4264
|
+
signal
|
|
4265
|
+
});
|
|
4266
|
+
if (response.status === 409 && retryAfterCreateConflict) {
|
|
4267
|
+
await ensureTypesenseCollection(config, dimensions, signal, false, retryAfterAlterFailure);
|
|
4268
|
+
return;
|
|
4269
|
+
}
|
|
4270
|
+
ensureOk(response, "Failed to create Typesense collection");
|
|
4271
|
+
}
|
|
4272
|
+
async function ensureTypesenseImportSucceeded(response, expectedRecords) {
|
|
4273
|
+
ensureOk(response, "Failed to sync documents to Typesense");
|
|
4274
|
+
const lines = (await response.text()).split(/\r?\n/u).filter((line) => line.trim());
|
|
4275
|
+
if (lines.length !== expectedRecords) throw new Error(`Typesense acknowledged ${lines.length} of ${expectedRecords} imported records.`);
|
|
4276
|
+
for (const [index, line] of lines.entries()) {
|
|
4277
|
+
let result;
|
|
4278
|
+
try {
|
|
4279
|
+
result = JSON.parse(line);
|
|
4280
|
+
} catch {
|
|
4281
|
+
throw new Error(`Typesense import returned invalid JSON on record ${index + 1}.`);
|
|
4282
|
+
}
|
|
4283
|
+
if (result.success !== true) {
|
|
4284
|
+
const detail = typeof result.error === "string" ? `: ${result.error}` : "";
|
|
4285
|
+
throw new Error(`Typesense failed to import record ${index + 1}${detail}`);
|
|
4286
|
+
}
|
|
4287
|
+
}
|
|
4288
|
+
}
|
|
4289
|
+
function createTypesenseSearchAdapter(config) {
|
|
4290
|
+
const adapter = {
|
|
4291
|
+
name: "typesense",
|
|
4292
|
+
async index(context) {
|
|
4293
|
+
const adminApiKey = config.adminApiKey ?? config.apiKey;
|
|
4294
|
+
const corpusId = resolveHostedHumanCorpusId(config.syncNamespace, context);
|
|
4295
|
+
const documents = await enrichDocsSearchDocumentsWithSources({
|
|
4296
|
+
documents: context.documents,
|
|
4297
|
+
pages: context.pages,
|
|
4298
|
+
audience: resolveDocsSearchAudience(context.audience),
|
|
4299
|
+
chunking: context.chunking ?? { strategy: "section" },
|
|
4300
|
+
locale: context.locale,
|
|
4301
|
+
baseUrl: context.baseUrl,
|
|
4302
|
+
indexGeneration: context.indexGeneration
|
|
4303
|
+
});
|
|
4304
|
+
const docsForImport = await Promise.all(documents.map(async (document) => {
|
|
4305
|
+
const next = {
|
|
4306
|
+
id: corpusId ? makeHostedProviderDocumentId(corpusId, document.id) : document.id,
|
|
4307
|
+
url: document.url,
|
|
4308
|
+
title: document.title,
|
|
4309
|
+
section: document.section,
|
|
4310
|
+
content: document.content,
|
|
4311
|
+
description: document.description,
|
|
4312
|
+
type: document.type,
|
|
4313
|
+
locale: document.locale ?? document.source?.scope.locale?.[0],
|
|
4314
|
+
framework: document.framework,
|
|
4315
|
+
version: document.version,
|
|
4316
|
+
package: document.package,
|
|
4317
|
+
tags: document.tags,
|
|
4318
|
+
...serializeHostedRetrievalSource(document.source, corpusId, document.id)
|
|
4319
|
+
};
|
|
4320
|
+
if (config.mode === "hybrid" && config.embeddings) next.embedding = await createOllamaEmbedding(`${document.title}\n${document.section ?? ""}\n${document.content}`.trim(), config.embeddings, context.signal);
|
|
4321
|
+
return next;
|
|
4322
|
+
}));
|
|
4323
|
+
await ensureTypesenseCollection(config, Array.isArray(docsForImport[0]?.embedding) ? docsForImport[0].embedding.length : void 0, context.signal);
|
|
4324
|
+
if (docsForImport.length > 0) await ensureTypesenseImportSucceeded(await fetch(`${getTypesenseSearchBase(config)}/collections/${encodeURIComponent(config.collection)}/documents/import?action=upsert`, {
|
|
4325
|
+
method: "POST",
|
|
4326
|
+
headers: {
|
|
4327
|
+
"X-TYPESENSE-API-KEY": adminApiKey,
|
|
4328
|
+
"Content-Type": "text/plain"
|
|
4329
|
+
},
|
|
4330
|
+
body: docsForImport.map((document) => JSON.stringify(document)).join("\n"),
|
|
4331
|
+
signal: context.signal
|
|
4332
|
+
}), docsForImport.length);
|
|
4333
|
+
},
|
|
4334
|
+
async search(query, context) {
|
|
4335
|
+
return (await adapter.searchPage({
|
|
4336
|
+
...query,
|
|
4337
|
+
offset: 0,
|
|
4338
|
+
[DOCS_UNPAGINATED_PROVIDER_QUERY]: true
|
|
4339
|
+
}, context)).results;
|
|
4340
|
+
},
|
|
4341
|
+
async searchPage(query, context) {
|
|
4342
|
+
const exactPagination = !query[DOCS_UNPAGINATED_PROVIDER_QUERY];
|
|
4343
|
+
const corpusId = resolveHostedHumanCorpusId(config.syncNamespace, context);
|
|
4344
|
+
const scopedDocumentIds = resolveProviderScopeDocumentIds(query, context, corpusId);
|
|
4345
|
+
if (scopedDocumentIds === DOCS_PROVIDER_SCOPE_FILTER_OVERFLOW) throw new Error("Typesense scope filters exceed the provider pagination limit.");
|
|
4346
|
+
if (scopedDocumentIds?.length === 0) return {
|
|
4347
|
+
results: [],
|
|
4348
|
+
total: 0
|
|
4349
|
+
};
|
|
4350
|
+
const pageSize = query.limit ?? config.maxResults ?? DEFAULT_SEARCH_LIMIT;
|
|
4351
|
+
if ((query.offset ?? 0) % pageSize !== 0) throw new Error("Typesense search offset does not match its page boundary.");
|
|
4352
|
+
const params = new URLSearchParams({
|
|
4353
|
+
q: query.query,
|
|
4354
|
+
query_by: (config.queryBy ?? [
|
|
4355
|
+
"title",
|
|
4356
|
+
"section",
|
|
4357
|
+
"content",
|
|
4358
|
+
"description"
|
|
4359
|
+
]).join(","),
|
|
4360
|
+
per_page: String(pageSize),
|
|
4361
|
+
prioritize_exact_match: "true",
|
|
4362
|
+
num_typos: "2",
|
|
4363
|
+
highlight_fields: "content,title,section,description",
|
|
4364
|
+
page: String(Math.floor((query.offset ?? 0) / pageSize) + 1)
|
|
4365
|
+
});
|
|
4366
|
+
if (config.mode === "hybrid" && config.embeddings) {
|
|
4367
|
+
const vector = await createOllamaEmbedding(query.query, config.embeddings, context.signal);
|
|
4368
|
+
params.set("vector_query", `embedding:([${vector.join(",")}],k:${Math.max((query.limit ?? 10) * 4, 20)})`);
|
|
4369
|
+
}
|
|
4370
|
+
const filterClauses = [
|
|
4371
|
+
corpusId ? `source_corpus_id:=${quoteTypesenseFilterValue(corpusId)}` : void 0,
|
|
4372
|
+
context.indexGeneration ? `source_index_generation:=${quoteTypesenseFilterValue(context.indexGeneration)}` : void 0,
|
|
4373
|
+
scopedDocumentIds ? `id:=[${scopedDocumentIds.map(quoteTypesenseFilterValue).join(",")}]` : void 0
|
|
4374
|
+
].filter((value) => Boolean(value));
|
|
4375
|
+
const filterBy = filterClauses.length > 0 ? filterClauses.join(" && ") : void 0;
|
|
4376
|
+
if (filterBy && filterBy.length > MAX_PROVIDER_SCOPE_FILTER_CHARS) throw new Error("Typesense scope filters exceed the provider pagination limit.");
|
|
4377
|
+
const response = filterBy ? await fetch(`${getTypesenseSearchBase(config)}/multi_search`, {
|
|
4378
|
+
method: "POST",
|
|
4379
|
+
headers: {
|
|
4380
|
+
"Content-Type": "application/json",
|
|
4381
|
+
"X-TYPESENSE-API-KEY": config.apiKey
|
|
4382
|
+
},
|
|
4383
|
+
body: JSON.stringify({ searches: [{
|
|
4384
|
+
collection: config.collection,
|
|
4385
|
+
...Object.fromEntries(params),
|
|
4386
|
+
filter_by: filterBy
|
|
4387
|
+
}] }),
|
|
4388
|
+
signal: context.signal
|
|
4389
|
+
}) : await fetch(`${getTypesenseSearchBase(config)}/collections/${encodeURIComponent(config.collection)}/documents/search?${params.toString()}`, {
|
|
4390
|
+
headers: { "X-TYPESENSE-API-KEY": config.apiKey },
|
|
4391
|
+
signal: context.signal
|
|
4392
|
+
});
|
|
4393
|
+
ensureOk(response, "Typesense search failed");
|
|
4394
|
+
const rawPayload = await readResponseJson(response);
|
|
4395
|
+
const payload = filterBy ? rawPayload.results?.[0] ?? {} : rawPayload;
|
|
4396
|
+
if (exactPagination && payload.search_cutoff !== void 0 && payload.search_cutoff !== false) throw new Error("Typesense search was cut off before exact pagination completed.");
|
|
4397
|
+
const results = (payload.hits ?? []).flatMap((hit) => {
|
|
4398
|
+
const document = hit.document ?? {};
|
|
4399
|
+
if (!hostedRecordMatchesCorpus(document, corpusId)) return [];
|
|
4400
|
+
const section = typeof document.section === "string" ? document.section : void 0;
|
|
4401
|
+
const content = typeof document.title === "string" ? section ? `${document.title} — ${section}` : document.title : typeof document.content === "string" ? document.content : "Untitled result";
|
|
4402
|
+
const description = hit.highlights?.find((item) => item.field === "content")?.snippet ?? hit.highlights?.find((item) => item.field === "description")?.snippet ?? (typeof document.description === "string" ? document.description : void 0);
|
|
4403
|
+
const source = readHostedRetrievalSource(document);
|
|
4404
|
+
if (hasHostedRetrievalSource(document) && !source) return [];
|
|
4405
|
+
return [{
|
|
4406
|
+
id: readHostedSourceDocumentId(document) ?? (typeof document.id === "string" ? document.id : String(document.url ?? content)),
|
|
4407
|
+
url: typeof document.url === "string" ? document.url : "/docs",
|
|
4408
|
+
content: cleanSearchResultText(content) ?? content,
|
|
4409
|
+
description: cleanSearchResultText(description),
|
|
4410
|
+
type: typeof document.type === "string" && [
|
|
4411
|
+
"page",
|
|
4412
|
+
"heading",
|
|
4413
|
+
"text"
|
|
4414
|
+
].includes(document.type) ? document.type : section ? "heading" : "page",
|
|
4415
|
+
score: hit.text_match,
|
|
4416
|
+
section,
|
|
4417
|
+
...source ? { source } : {}
|
|
4418
|
+
}];
|
|
4419
|
+
});
|
|
4420
|
+
const exactTotal = typeof payload.found === "number" && Number.isSafeInteger(payload.found) && payload.found >= 0 ? payload.found : void 0;
|
|
4421
|
+
if (exactPagination && exactTotal === void 0) throw new Error("Typesense search did not return an exact result total.");
|
|
4422
|
+
const total = exactTotal ?? results.length;
|
|
4423
|
+
if (exactPagination && total > MAX_HOSTED_PROVIDER_CURSOR_RESULTS) throw new Error("Typesense search exceeds the exact cursor pagination window.");
|
|
4424
|
+
if (exactPagination && (query.offset ?? 0) + results.length < total && results.length !== pageSize) throw new Error("Typesense search returned an incomplete non-final page.");
|
|
4425
|
+
return {
|
|
4426
|
+
results,
|
|
4427
|
+
total
|
|
4428
|
+
};
|
|
4429
|
+
}
|
|
4430
|
+
};
|
|
4431
|
+
return adapter;
|
|
4432
|
+
}
|
|
4433
|
+
function resolveSearchRequestConfig(search, requestUrl) {
|
|
4434
|
+
if (!search || search === true || typeof search !== "object" || search.provider !== "mcp") return search;
|
|
4435
|
+
if (!requestUrl) return search;
|
|
4436
|
+
const resolvedEndpoint = new URL(search.endpoint, requestUrl);
|
|
4437
|
+
const usesDefaultSearchTool = (search.toolName ?? "search_docs") === "search_docs";
|
|
4438
|
+
const isSameOrigin = resolvedEndpoint.origin === new URL(requestUrl).origin;
|
|
4439
|
+
return {
|
|
4440
|
+
...search,
|
|
4441
|
+
endpoint: resolvedEndpoint.toString(),
|
|
4442
|
+
forwardAudience: search.forwardAudience ?? (usesDefaultSearchTool && isSameOrigin)
|
|
4443
|
+
};
|
|
4444
|
+
}
|
|
4445
|
+
/**
|
|
4446
|
+
* Resolve the public search audience without allowing malformed values to opt into agent content.
|
|
4447
|
+
* Human search remains the default for omitted, legacy, and unknown query values.
|
|
4448
|
+
*/
|
|
4449
|
+
function resolveDocsSearchAudience(value) {
|
|
4450
|
+
return value === "agent" ? "agent" : "human";
|
|
4451
|
+
}
|
|
4452
|
+
function resolveAskAISearchRequestConfig(options) {
|
|
4453
|
+
if (!options.useMcp) return resolveSearchRequestConfig(options.search, options.requestUrl);
|
|
4454
|
+
if (typeof options.useMcp === "object") return resolveSearchRequestConfig({
|
|
4455
|
+
...options.useMcp,
|
|
4456
|
+
provider: "mcp"
|
|
4457
|
+
}, options.requestUrl);
|
|
4458
|
+
if (options.mcpEnabled === false || options.mcpSearchEnabled === false || !options.mcpEndpoint) return resolveSearchRequestConfig(options.search, options.requestUrl);
|
|
4459
|
+
return resolveSearchRequestConfig({
|
|
4460
|
+
provider: "mcp",
|
|
4461
|
+
endpoint: options.mcpEndpoint
|
|
4462
|
+
}, options.requestUrl);
|
|
4463
|
+
}
|
|
4464
|
+
function createMcpSearchAdapter(config) {
|
|
4465
|
+
async function runMcpSearch(query, context, paginated) {
|
|
4466
|
+
const endpoint = resolveMcpEndpoint(config.endpoint);
|
|
4467
|
+
const protocolVersion = config.protocolVersion ?? DEFAULT_MCP_PROTOCOL_VERSION;
|
|
4468
|
+
const toolName = config.toolName ?? "search_docs";
|
|
4469
|
+
const forwardAudience = config.forwardAudience === true;
|
|
4470
|
+
const audience = resolveDocsSearchAudience(query.audience);
|
|
4471
|
+
const baseHeaders = config.headers ?? {};
|
|
4472
|
+
if (audience === "human" && !forwardAudience) throw new Error("MCP human-projection search requires forwardAudience: true on an audience-aware tool.");
|
|
4473
|
+
const initializeResponse = await fetch(endpoint, {
|
|
4474
|
+
method: "POST",
|
|
4475
|
+
headers: {
|
|
4476
|
+
...baseHeaders,
|
|
4477
|
+
"Content-Type": "application/json",
|
|
4478
|
+
accept: "application/json, text/event-stream",
|
|
4479
|
+
"mcp-protocol-version": protocolVersion
|
|
4480
|
+
},
|
|
4481
|
+
body: JSON.stringify({
|
|
4482
|
+
jsonrpc: "2.0",
|
|
4483
|
+
id: 1,
|
|
4484
|
+
method: "initialize",
|
|
4485
|
+
params: {
|
|
4486
|
+
protocolVersion,
|
|
4487
|
+
capabilities: {},
|
|
4488
|
+
clientInfo: {
|
|
4489
|
+
name: "@farming-labs/docs-search",
|
|
4490
|
+
version: "0.1.2"
|
|
4491
|
+
}
|
|
4492
|
+
}
|
|
4493
|
+
}),
|
|
4494
|
+
signal: context.signal
|
|
4495
|
+
});
|
|
4496
|
+
const initializePayload = await readMcpResponsePayload(initializeResponse);
|
|
4497
|
+
ensureOk(initializeResponse, "MCP search initialization failed");
|
|
4498
|
+
ensureJsonRpcOk(initializePayload, "MCP search initialization failed");
|
|
4499
|
+
const sessionId = initializeResponse.headers.get("mcp-session-id") ?? void 0;
|
|
4500
|
+
try {
|
|
4501
|
+
ensureOk(await fetch(endpoint, {
|
|
4502
|
+
method: "POST",
|
|
4503
|
+
headers: {
|
|
4504
|
+
...baseHeaders,
|
|
4505
|
+
"Content-Type": "application/json",
|
|
4506
|
+
accept: "application/json, text/event-stream",
|
|
4507
|
+
"mcp-protocol-version": protocolVersion,
|
|
4508
|
+
...sessionId ? { "mcp-session-id": sessionId } : {}
|
|
4509
|
+
},
|
|
4510
|
+
body: JSON.stringify({
|
|
4511
|
+
jsonrpc: "2.0",
|
|
4512
|
+
method: "notifications/initialized",
|
|
4513
|
+
params: {}
|
|
4514
|
+
}),
|
|
4515
|
+
signal: context.signal
|
|
4516
|
+
}), "MCP initialized notification failed");
|
|
4517
|
+
const pageSize = query.limit ?? config.maxResults ?? DEFAULT_SEARCH_LIMIT;
|
|
4518
|
+
const callRemoteSearchPage = async (providerCursor, id) => {
|
|
4519
|
+
const searchResponse = await fetch(endpoint, {
|
|
4520
|
+
method: "POST",
|
|
4521
|
+
headers: {
|
|
4522
|
+
...baseHeaders,
|
|
4523
|
+
"Content-Type": "application/json",
|
|
4524
|
+
accept: "application/json, text/event-stream",
|
|
4525
|
+
"mcp-protocol-version": protocolVersion,
|
|
4526
|
+
...sessionId ? { "mcp-session-id": sessionId } : {}
|
|
4527
|
+
},
|
|
4528
|
+
body: JSON.stringify({
|
|
4529
|
+
jsonrpc: "2.0",
|
|
4530
|
+
id,
|
|
4531
|
+
method: "tools/call",
|
|
4532
|
+
params: {
|
|
4533
|
+
name: toolName,
|
|
4534
|
+
arguments: {
|
|
4535
|
+
query: query.query,
|
|
4536
|
+
limit: pageSize,
|
|
4537
|
+
locale: query.locale,
|
|
4538
|
+
...providerCursor !== void 0 ? { cursor: providerCursor } : {},
|
|
4539
|
+
...forwardAudience ? { audience } : {},
|
|
4540
|
+
...query.filters?.framework ? { framework: query.filters.framework } : {},
|
|
4541
|
+
...query.filters?.version ? { version: query.filters.version } : {},
|
|
4542
|
+
...query.filters?.package ? { package: query.filters.package } : {},
|
|
4543
|
+
...query.filters?.tags ? { tags: query.filters.tags } : {}
|
|
4544
|
+
}
|
|
4545
|
+
}
|
|
4546
|
+
}),
|
|
4547
|
+
signal: context.signal
|
|
4548
|
+
});
|
|
4549
|
+
const payload = await readMcpResponsePayload(searchResponse);
|
|
4550
|
+
ensureOk(searchResponse, "MCP search request failed");
|
|
4551
|
+
ensureJsonRpcOk(payload, "MCP search request failed");
|
|
4552
|
+
const parsed = readMcpSearchToolPayload(payload);
|
|
4553
|
+
if (!parsed) return {
|
|
4554
|
+
parsed,
|
|
4555
|
+
rawResults: [],
|
|
4556
|
+
results: []
|
|
4557
|
+
};
|
|
4558
|
+
if (hasDocsSearchFilters(query.filters ?? {})) {
|
|
4559
|
+
if (Array.isArray(parsed) || parsed.format !== "docs-search.v1") {
|
|
4560
|
+
if (paginated) throw new Error("MCP search returned unverifiable pagination data.");
|
|
4561
|
+
return {
|
|
4562
|
+
parsed,
|
|
4563
|
+
rawResults: [],
|
|
4564
|
+
results: []
|
|
4565
|
+
};
|
|
4566
|
+
}
|
|
4567
|
+
const echoedFilters = parseVerifiedMcpSearchFilters(parsed.filters);
|
|
4568
|
+
if (!echoedFilters || !docsSearchFiltersMatch(query.filters ?? {}, echoedFilters)) {
|
|
4569
|
+
if (paginated) throw new Error("MCP search returned mismatched scope filters.");
|
|
4570
|
+
return {
|
|
4571
|
+
parsed,
|
|
4572
|
+
rawResults: [],
|
|
4573
|
+
results: []
|
|
4574
|
+
};
|
|
4575
|
+
}
|
|
4576
|
+
}
|
|
4577
|
+
const rawResults = Array.isArray(parsed) ? parsed : Array.isArray(parsed.results) ? parsed.results : Array.isArray(parsed.pages) ? parsed.pages : [];
|
|
4578
|
+
return {
|
|
4579
|
+
parsed,
|
|
4580
|
+
rawResults,
|
|
4581
|
+
results: rawResults.map((result) => mapMcpSearchResult(result, endpoint)).filter((result) => Boolean(result))
|
|
4582
|
+
};
|
|
4583
|
+
};
|
|
4584
|
+
if (!paginated) return (await callRemoteSearchPage(void 0, 2)).results;
|
|
4585
|
+
const targetOffset = query.offset ?? 0;
|
|
4586
|
+
if (targetOffset === 0 && query.cursor !== void 0 || targetOffset > 0 && query.cursor === void 0) throw new Error("MCP search continuation is missing or misplaced its provider cursor.");
|
|
4587
|
+
let consumed = 0;
|
|
4588
|
+
let remoteCursor;
|
|
4589
|
+
let expectedTotal;
|
|
4590
|
+
const replayResultDigests = [];
|
|
4591
|
+
for (let pageIndex = 0; pageIndex < MAX_MCP_SEARCH_CURSOR_REPLAY_PAGES; pageIndex += 1) {
|
|
4592
|
+
const { parsed, rawResults, results } = await callRemoteSearchPage(remoteCursor, pageIndex + 2);
|
|
4593
|
+
if (!parsed) {
|
|
4594
|
+
if (targetOffset === 0) return {
|
|
4595
|
+
results: [],
|
|
4596
|
+
total: 0
|
|
4597
|
+
};
|
|
4598
|
+
throw new Error("MCP search continuation ended before the requested page.");
|
|
4599
|
+
}
|
|
4600
|
+
if (Array.isArray(parsed) || parsed.format !== "docs-search.v1" || results.length !== rawResults.length || parsed.resultCount !== void 0 && parsed.resultCount !== results.length || typeof parsed.total !== "number" || !Number.isSafeInteger(parsed.total) || parsed.total < consumed + results.length || typeof parsed.hasMore !== "boolean" || parsed.nextCursor !== void 0 && (typeof parsed.nextCursor !== "string" || parsed.nextCursor.length > 768)) throw new Error("MCP search returned malformed paginated results.");
|
|
4601
|
+
const total = parsed.total;
|
|
4602
|
+
const nextCursor = typeof parsed.nextCursor === "string" ? parsed.nextCursor : void 0;
|
|
4603
|
+
const hasMore = consumed + results.length < total;
|
|
4604
|
+
if (expectedTotal !== void 0 && total !== expectedTotal || parsed.hasMore !== hasMore || nextCursor !== void 0 !== hasMore || hasMore && results.length === 0) throw new Error("MCP search returned inconsistent pagination metadata.");
|
|
4605
|
+
expectedTotal ??= total;
|
|
4606
|
+
if (total > MAX_MCP_SEARCH_CURSOR_RESULTS) throw new Error("MCP search exceeds its bounded exact pagination window.");
|
|
4607
|
+
replayResultDigests.push(...results.map((result) => hashDocsRetrievalValue(JSON.stringify(result))));
|
|
4608
|
+
const nextOffset = consumed + results.length;
|
|
4609
|
+
const nextCheckpoint = hasMore ? hashDocsRetrievalValue(JSON.stringify({
|
|
4610
|
+
format: "docs-mcp-search-replay.v1",
|
|
4611
|
+
total,
|
|
4612
|
+
nextOffset,
|
|
4613
|
+
resultDigests: replayResultDigests
|
|
4614
|
+
})) : void 0;
|
|
4615
|
+
if (nextOffset === targetOffset && nextCheckpoint !== query.cursor) throw new Error("MCP search continuation cursor is stale.");
|
|
4616
|
+
if (consumed === targetOffset) return {
|
|
4617
|
+
results,
|
|
4618
|
+
total,
|
|
4619
|
+
...nextCheckpoint !== void 0 ? { nextCursor: nextCheckpoint } : {}
|
|
4620
|
+
};
|
|
4621
|
+
if (!hasMore || nextCursor === void 0) throw new Error("MCP search continuation ended before the requested page.");
|
|
4622
|
+
consumed = nextOffset;
|
|
4623
|
+
if (consumed > targetOffset) throw new Error("MCP search continuation offset does not match a page boundary.");
|
|
4624
|
+
remoteCursor = nextCursor;
|
|
4625
|
+
}
|
|
4626
|
+
throw new Error("MCP search exceeds its bounded cursor replay limit.");
|
|
4627
|
+
} finally {
|
|
4628
|
+
if (sessionId) {
|
|
4629
|
+
const cleanupController = new AbortController();
|
|
4630
|
+
const cleanupTimeout = setTimeout(() => cleanupController.abort(), MCP_SESSION_CLEANUP_TIMEOUT_MS);
|
|
4631
|
+
try {
|
|
4632
|
+
await fetch(endpoint, {
|
|
4633
|
+
method: "DELETE",
|
|
4634
|
+
headers: {
|
|
4635
|
+
...baseHeaders,
|
|
4636
|
+
"mcp-protocol-version": protocolVersion,
|
|
4637
|
+
"mcp-session-id": sessionId
|
|
4638
|
+
},
|
|
4639
|
+
signal: cleanupController.signal
|
|
4640
|
+
});
|
|
4641
|
+
} catch {} finally {
|
|
4642
|
+
clearTimeout(cleanupTimeout);
|
|
4643
|
+
}
|
|
4644
|
+
}
|
|
4645
|
+
}
|
|
4646
|
+
}
|
|
4647
|
+
return {
|
|
4648
|
+
name: "mcp",
|
|
4649
|
+
async search(query, context) {
|
|
4650
|
+
return await runMcpSearch(query, context, false);
|
|
4651
|
+
},
|
|
4652
|
+
async searchPage(query, context) {
|
|
4653
|
+
return await runMcpSearch(query, context, true);
|
|
4654
|
+
}
|
|
4655
|
+
};
|
|
4656
|
+
}
|
|
4657
|
+
function getAlgoliaSearchBase(config) {
|
|
4658
|
+
return `https://${config.appId}-dsn.algolia.net`;
|
|
4659
|
+
}
|
|
4660
|
+
function getAlgoliaAdminBase(config) {
|
|
4661
|
+
return `https://${config.appId}.algolia.net`;
|
|
4662
|
+
}
|
|
4663
|
+
function getAlgoliaAdminHeaders(config) {
|
|
4664
|
+
return {
|
|
4665
|
+
"Content-Type": "application/json",
|
|
4666
|
+
"X-Algolia-Application-Id": config.appId,
|
|
4667
|
+
"X-Algolia-API-Key": config.adminApiKey ?? config.searchApiKey
|
|
4668
|
+
};
|
|
4669
|
+
}
|
|
4670
|
+
async function waitForAlgoliaTask(config, taskId, signal) {
|
|
4671
|
+
const deadline = Date.now() + 6e4;
|
|
4672
|
+
for (let attempt = 0; Date.now() < deadline; attempt += 1) {
|
|
4673
|
+
const response = await fetch(`${getAlgoliaAdminBase(config)}/1/indexes/${encodeURIComponent(config.indexName)}/task/${encodeURIComponent(String(taskId))}`, {
|
|
4674
|
+
headers: getAlgoliaAdminHeaders(config),
|
|
4675
|
+
signal
|
|
4676
|
+
});
|
|
4677
|
+
ensureOk(response, "Failed to inspect Algolia sync task");
|
|
4678
|
+
const payload = await readResponseJson(response);
|
|
4679
|
+
if (payload?.status === "published") return;
|
|
4680
|
+
if (payload?.status !== "notPublished") throw new Error("Algolia sync task returned an unknown status.");
|
|
4681
|
+
const delay = Math.min(100 * 2 ** Math.min(attempt, 4), 2e3);
|
|
4682
|
+
await new Promise((resolve, reject) => {
|
|
4683
|
+
let timer;
|
|
4684
|
+
const onAbort = () => {
|
|
4685
|
+
clearTimeout(timer);
|
|
4686
|
+
reject(signal?.reason ?? /* @__PURE__ */ new Error("Algolia sync was aborted."));
|
|
4687
|
+
};
|
|
4688
|
+
timer = setTimeout(() => {
|
|
4689
|
+
signal?.removeEventListener("abort", onAbort);
|
|
4690
|
+
resolve();
|
|
4691
|
+
}, Math.min(delay, Math.max(1, deadline - Date.now())));
|
|
4692
|
+
if (signal?.aborted) onAbort();
|
|
4693
|
+
else signal?.addEventListener("abort", onAbort, { once: true });
|
|
4694
|
+
});
|
|
4695
|
+
}
|
|
4696
|
+
throw new Error("Timed out waiting for Algolia to publish the synced index.");
|
|
4697
|
+
}
|
|
4698
|
+
async function executeAlgoliaBatch(config, requests, message, signal) {
|
|
4699
|
+
for (let offset = 0; offset < requests.length; offset += ALGOLIA_BATCH_OPERATIONS) {
|
|
4700
|
+
const batch = requests.slice(offset, offset + ALGOLIA_BATCH_OPERATIONS);
|
|
4701
|
+
const response = await fetch(`${getAlgoliaAdminBase(config)}/1/indexes/${encodeURIComponent(config.indexName)}/batch`, {
|
|
4702
|
+
method: "POST",
|
|
4703
|
+
headers: getAlgoliaAdminHeaders(config),
|
|
4704
|
+
body: JSON.stringify({ requests: batch }),
|
|
4705
|
+
signal
|
|
4706
|
+
});
|
|
4707
|
+
ensureOk(response, message);
|
|
4708
|
+
const payload = await readResponseJson(response);
|
|
4709
|
+
if (typeof payload?.taskID !== "string" && typeof payload?.taskID !== "number") throw new Error("Algolia sync response did not include a task ID.");
|
|
4710
|
+
await waitForAlgoliaTask(config, payload.taskID, signal);
|
|
4711
|
+
}
|
|
4712
|
+
}
|
|
4713
|
+
function quoteAlgoliaFilterValue(value) {
|
|
4714
|
+
return `"${value.replace(/\\/gu, "\\\\").replace(/"/gu, "\\\"")}"`;
|
|
4715
|
+
}
|
|
4716
|
+
function createAlgoliaSearchAdapter(config) {
|
|
4717
|
+
const adapter = {
|
|
4718
|
+
name: "algolia",
|
|
4719
|
+
async index(context) {
|
|
4720
|
+
if (!config.adminApiKey) return;
|
|
4721
|
+
const corpusId = resolveHostedHumanCorpusId(config.syncNamespace, context);
|
|
4722
|
+
await executeAlgoliaBatch(config, (await enrichDocsSearchDocumentsWithSources({
|
|
4723
|
+
documents: context.documents,
|
|
4724
|
+
pages: context.pages,
|
|
4725
|
+
audience: resolveDocsSearchAudience(context.audience),
|
|
4726
|
+
chunking: context.chunking ?? { strategy: "section" },
|
|
4727
|
+
locale: context.locale,
|
|
4728
|
+
baseUrl: context.baseUrl,
|
|
4729
|
+
indexGeneration: context.indexGeneration
|
|
4730
|
+
})).map((document) => ({
|
|
4731
|
+
action: "addObject",
|
|
4732
|
+
body: buildAlgoliaRecord(document, corpusId)
|
|
4733
|
+
})), "Failed to sync documents to Algolia", context.signal);
|
|
4734
|
+
},
|
|
4735
|
+
async search(query, context) {
|
|
4736
|
+
return (await adapter.searchPage({
|
|
4737
|
+
...query,
|
|
4738
|
+
offset: 0,
|
|
4739
|
+
[DOCS_UNPAGINATED_PROVIDER_QUERY]: true
|
|
4740
|
+
}, context)).results;
|
|
4741
|
+
},
|
|
4742
|
+
async searchPage(query, context) {
|
|
4743
|
+
const exactPagination = !query[DOCS_UNPAGINATED_PROVIDER_QUERY];
|
|
4744
|
+
const corpusId = resolveHostedHumanCorpusId(config.syncNamespace, context);
|
|
4745
|
+
const scopedDocumentIds = resolveProviderScopeDocumentIds(query, context, corpusId);
|
|
4746
|
+
if (scopedDocumentIds === DOCS_PROVIDER_SCOPE_FILTER_OVERFLOW) throw new Error("Algolia scope filters exceed the provider pagination limit.");
|
|
4747
|
+
if (scopedDocumentIds?.length === 0) return {
|
|
4748
|
+
results: [],
|
|
4749
|
+
total: 0
|
|
4750
|
+
};
|
|
4751
|
+
const scopedIdsFilter = scopedDocumentIds?.map((id) => `objectID:${quoteAlgoliaFilterValue(id)}`).join(" OR ");
|
|
4752
|
+
const filterClauses = [
|
|
4753
|
+
corpusId ? `_tags:${quoteAlgoliaFilterValue(corpusId)}` : void 0,
|
|
4754
|
+
context.indexGeneration ? `_tags:${quoteAlgoliaFilterValue(getHostedIndexGenerationTag(context.indexGeneration))}` : void 0,
|
|
4755
|
+
scopedIdsFilter ? corpusId || context.indexGeneration ? `(${scopedIdsFilter})` : scopedIdsFilter : void 0
|
|
4756
|
+
].filter((value) => Boolean(value));
|
|
4757
|
+
const filters = filterClauses.length > 0 ? filterClauses.join(" AND ") : void 0;
|
|
4758
|
+
if (filters && filters.length > MAX_PROVIDER_SCOPE_FILTER_CHARS) throw new Error("Algolia scope filters exceed the provider pagination limit.");
|
|
4759
|
+
const pageSize = query.limit ?? config.maxResults ?? DEFAULT_SEARCH_LIMIT;
|
|
4760
|
+
if ((query.offset ?? 0) % pageSize !== 0) throw new Error("Algolia search offset does not match its page boundary.");
|
|
4761
|
+
const response = await fetch(`${getAlgoliaSearchBase(config)}/1/indexes/${encodeURIComponent(config.indexName)}/query`, {
|
|
4762
|
+
method: "POST",
|
|
4763
|
+
headers: {
|
|
4764
|
+
"Content-Type": "application/json",
|
|
4765
|
+
"X-Algolia-Application-Id": config.appId,
|
|
4766
|
+
"X-Algolia-API-Key": config.searchApiKey
|
|
4767
|
+
},
|
|
4768
|
+
body: JSON.stringify({
|
|
4769
|
+
query: query.query,
|
|
4770
|
+
hitsPerPage: pageSize,
|
|
4771
|
+
page: Math.floor((query.offset ?? 0) / pageSize),
|
|
4772
|
+
restrictSearchableAttributes: [
|
|
4773
|
+
"title",
|
|
4774
|
+
"section",
|
|
4775
|
+
"content",
|
|
4776
|
+
"description"
|
|
4777
|
+
],
|
|
4778
|
+
attributesToSnippet: ["content:20"],
|
|
4779
|
+
...filters ? { filters } : {}
|
|
4780
|
+
}),
|
|
4781
|
+
signal: context.signal
|
|
4782
|
+
});
|
|
4783
|
+
ensureOk(response, "Algolia search failed");
|
|
4784
|
+
const payload = await readResponseJson(response);
|
|
4785
|
+
const results = (payload.hits ?? []).flatMap((hit) => {
|
|
4786
|
+
if (!hostedRecordMatchesCorpus(hit, corpusId)) return [];
|
|
4787
|
+
const title = typeof hit.title === "string" ? hit.title : "Untitled result";
|
|
4788
|
+
const section = typeof hit.section === "string" ? hit.section : void 0;
|
|
4789
|
+
const source = readHostedRetrievalSource(hit);
|
|
4790
|
+
if (hasHostedRetrievalSource(hit) && !source) return [];
|
|
4791
|
+
return [{
|
|
4792
|
+
id: readHostedSourceDocumentId(hit) ?? hit.objectID ?? String(hit.url ?? title),
|
|
4793
|
+
url: typeof hit.url === "string" ? hit.url : "/docs",
|
|
4794
|
+
content: cleanSearchResultText(section ? `${title} — ${section}` : title) ?? title,
|
|
4795
|
+
description: cleanSearchResultText(hit._snippetResult?.content?.value ?? hit._snippetResult?.description?.value ?? (typeof hit.description === "string" ? hit.description : void 0)),
|
|
4796
|
+
type: typeof hit.type === "string" && [
|
|
4797
|
+
"page",
|
|
4798
|
+
"heading",
|
|
4799
|
+
"text"
|
|
4800
|
+
].includes(hit.type) ? hit.type : section ? "heading" : "page",
|
|
4801
|
+
score: hit._rankingInfo?.nbTypos != null ? 100 - hit._rankingInfo.nbTypos : void 0,
|
|
4802
|
+
section,
|
|
4803
|
+
...source ? { source } : {}
|
|
4804
|
+
}];
|
|
4805
|
+
});
|
|
4806
|
+
const exactTotal = typeof payload.nbHits === "number" && Number.isSafeInteger(payload.nbHits) && payload.nbHits >= 0 ? payload.nbHits : void 0;
|
|
4807
|
+
const exactPageCount = typeof payload.nbPages === "number" && Number.isSafeInteger(payload.nbPages) && payload.nbPages >= 0 ? payload.nbPages : void 0;
|
|
4808
|
+
if (exactPagination && (exactTotal === void 0 || exactPageCount === void 0)) throw new Error("Algolia search did not return exact reachable pagination metadata.");
|
|
4809
|
+
const total = exactTotal ?? results.length;
|
|
4810
|
+
if (exactPagination && total > MAX_HOSTED_PROVIDER_CURSOR_RESULTS) throw new Error("Algolia search exceeds the exact cursor pagination window.");
|
|
4811
|
+
if (exactPagination && payload.exhaustiveNbHits === false) throw new Error("Algolia search returned an approximate result total.");
|
|
4812
|
+
if (exactPagination && exactPageCount !== void 0 && total > exactPageCount * pageSize) throw new Error("Algolia search total exceeds its reachable pagination window.");
|
|
4813
|
+
if (exactPagination && (query.offset ?? 0) + results.length < total && results.length !== pageSize) throw new Error("Algolia search returned an incomplete non-final page.");
|
|
4814
|
+
return {
|
|
4815
|
+
results,
|
|
4816
|
+
total
|
|
4817
|
+
};
|
|
4818
|
+
}
|
|
4819
|
+
};
|
|
4820
|
+
return adapter;
|
|
4821
|
+
}
|
|
4822
|
+
async function resolveSearchAdapter(search, context) {
|
|
4823
|
+
const raw = search.raw;
|
|
4824
|
+
if (search.provider === "custom" && raw?.provider === "custom") return typeof raw.adapter === "function" ? await raw.adapter(context) : raw.adapter;
|
|
4825
|
+
if (search.provider === "typesense" && raw?.provider === "typesense") return createTypesenseSearchAdapter(raw);
|
|
4826
|
+
if (search.provider === "mcp" && raw?.provider === "mcp") return createMcpSearchAdapter(raw);
|
|
4827
|
+
if (search.provider === "algolia" && raw?.provider === "algolia") return createAlgoliaSearchAdapter(raw);
|
|
4828
|
+
return createSimpleSearchAdapter();
|
|
4829
|
+
}
|
|
4830
|
+
function shouldSyncOnSearch(search) {
|
|
4831
|
+
const raw = search.raw;
|
|
4832
|
+
if (search.provider === "algolia" && raw?.provider === "algolia") return (raw.syncOnSearch ?? Boolean(raw.adminApiKey)) && Boolean(raw.adminApiKey);
|
|
4833
|
+
if (search.provider === "typesense" && raw?.provider === "typesense") return (raw.syncOnSearch ?? Boolean(raw.adminApiKey)) && Boolean(raw.adminApiKey);
|
|
4834
|
+
return false;
|
|
4835
|
+
}
|
|
4836
|
+
function getSyncKey(search, context) {
|
|
4837
|
+
const raw = search.raw;
|
|
4838
|
+
if (search.provider === "algolia" && raw?.provider === "algolia") {
|
|
4839
|
+
const corpusId = resolveHostedHumanCorpusId(raw.syncNamespace, context) ?? "__unowned__";
|
|
4840
|
+
return `algolia:${raw.appId}:${raw.indexName}:${corpusId}:${context.locale ?? "__default__"}`;
|
|
4841
|
+
}
|
|
4842
|
+
if (search.provider === "typesense" && raw?.provider === "typesense") {
|
|
4843
|
+
const corpusId = resolveHostedHumanCorpusId(raw.syncNamespace, context) ?? "__unowned__";
|
|
4844
|
+
return `typesense:${raw.baseUrl}:${raw.collection}:${corpusId}:${context.locale ?? "__default__"}`;
|
|
4845
|
+
}
|
|
4846
|
+
if (search.provider === "mcp" && raw?.provider === "mcp") return `mcp:${raw.endpoint}:${context.locale ?? "__default__"}`;
|
|
4847
|
+
return `${search.provider}:${context.locale ?? "__default__"}`;
|
|
4848
|
+
}
|
|
4849
|
+
function getSearchSyncFingerprint(search, context, indexGeneration) {
|
|
4850
|
+
const raw = search.raw;
|
|
4851
|
+
const providerConfig = search.provider === "typesense" && raw?.provider === "typesense" ? {
|
|
4852
|
+
syncNamespace: raw.syncNamespace?.trim() || void 0,
|
|
4853
|
+
mode: raw.mode ?? "keyword",
|
|
4854
|
+
embeddings: raw.embeddings ? {
|
|
4855
|
+
provider: raw.embeddings.provider,
|
|
4856
|
+
model: raw.embeddings.model,
|
|
4857
|
+
baseUrl: raw.embeddings.baseUrl
|
|
4858
|
+
} : void 0
|
|
4859
|
+
} : search.provider === "algolia" && raw?.provider === "algolia" ? { syncNamespace: raw.syncNamespace?.trim() || void 0 } : void 0;
|
|
4860
|
+
return hashDocsRetrievalValue(JSON.stringify({
|
|
4861
|
+
format: "docs-search-sync.v1",
|
|
4862
|
+
indexGeneration,
|
|
4863
|
+
canonicalBaseUrl: context.baseUrl,
|
|
4864
|
+
provider: search.provider,
|
|
4865
|
+
providerConfig
|
|
4866
|
+
}));
|
|
4867
|
+
}
|
|
4868
|
+
async function maybeSyncSearchIndex(adapter, search, context) {
|
|
4869
|
+
if (!shouldSyncOnSearch(search) || typeof adapter.index !== "function") return;
|
|
4870
|
+
const syncKey = getSyncKey(search, context);
|
|
4871
|
+
const indexGeneration = context.indexGeneration ?? await buildDocsSearchIndexGeneration(context.pages, {
|
|
4872
|
+
audience: resolveDocsSearchAudience(context.audience),
|
|
4873
|
+
chunking: context.chunking ?? search.chunking,
|
|
4874
|
+
locale: context.locale,
|
|
4875
|
+
baseUrl: context.baseUrl
|
|
4876
|
+
});
|
|
4877
|
+
context.indexGeneration = indexGeneration;
|
|
4878
|
+
const syncFingerprint = getSearchSyncFingerprint(search, context, indexGeneration);
|
|
4879
|
+
while (true) {
|
|
4880
|
+
const inFlight = syncingIndexes.get(syncKey);
|
|
4881
|
+
if (inFlight) {
|
|
4882
|
+
try {
|
|
4883
|
+
await inFlight.promise;
|
|
4884
|
+
} catch (error) {
|
|
4885
|
+
if (inFlight.fingerprint === syncFingerprint) throw error;
|
|
4886
|
+
}
|
|
4887
|
+
continue;
|
|
4888
|
+
}
|
|
4889
|
+
if (syncedIndexes.get(syncKey) === syncFingerprint) return;
|
|
4890
|
+
const sync = adapter.index(context).then(() => {
|
|
4891
|
+
syncedIndexes.set(syncKey, syncFingerprint);
|
|
4892
|
+
});
|
|
4893
|
+
syncingIndexes.set(syncKey, {
|
|
4894
|
+
fingerprint: syncFingerprint,
|
|
4895
|
+
promise: sync
|
|
4896
|
+
});
|
|
4897
|
+
try {
|
|
4898
|
+
await sync;
|
|
4899
|
+
} finally {
|
|
4900
|
+
if (syncingIndexes.get(syncKey)?.promise === sync) syncingIndexes.delete(syncKey);
|
|
4901
|
+
}
|
|
4902
|
+
}
|
|
4903
|
+
}
|
|
4904
|
+
function normalizeDocsSearchAdapterPage(value) {
|
|
4905
|
+
if (Array.isArray(value)) return {
|
|
4906
|
+
results: value,
|
|
4907
|
+
total: value.length
|
|
4908
|
+
};
|
|
4909
|
+
if (!value || !Array.isArray(value.results) || !Number.isSafeInteger(value.total) || value.total < 0 || value.results.length > value.total || value.nextCursor !== void 0 && (typeof value.nextCursor !== "string" || value.nextCursor.length === 0 || JSON.stringify([
|
|
4910
|
+
1,
|
|
4911
|
+
"provider",
|
|
4912
|
+
value.total,
|
|
4913
|
+
value.nextCursor
|
|
4914
|
+
]).length > MAX_SEARCH_PROVIDER_CURSOR_STATE_CHARS)) throw new Error("Search adapter returned invalid pagination metadata.");
|
|
4915
|
+
return value;
|
|
4916
|
+
}
|
|
4917
|
+
async function performDocsSearchInternal(options) {
|
|
4918
|
+
const search = normalizeDocsSearchConfig(options.search);
|
|
4919
|
+
if (!search.enabled) {
|
|
4920
|
+
if (options.paginationMode === "provider" || options.expectedTotal !== void 0 && options.expectedTotal !== 0) throw new DocsPaginationCursorError();
|
|
4921
|
+
return {
|
|
4922
|
+
results: [],
|
|
4923
|
+
total: 0,
|
|
4924
|
+
paginationMode: "local"
|
|
4925
|
+
};
|
|
4926
|
+
}
|
|
4927
|
+
const audience = resolveDocsSearchAudience(options.audience);
|
|
4928
|
+
const filters = normalizeDocsSearchFilters(options.filters);
|
|
4929
|
+
const hasFilters = hasDocsSearchFilters(filters);
|
|
4930
|
+
const corpusPages = options.pages.map((page) => localizeDocsSearchPage(page, options.locale));
|
|
4931
|
+
const indexBaseUrl = options.syncBaseUrl === null ? void 0 : options.syncBaseUrl ?? options.baseUrl;
|
|
4932
|
+
const scopedPages = hasFilters ? corpusPages.filter((page) => docsSearchPageMatchesFilters(resolveDocsSearchPageScope(page), filters)) : corpusPages;
|
|
4933
|
+
let resolvedDocuments;
|
|
4934
|
+
const getDocuments = () => {
|
|
4935
|
+
if (!resolvedDocuments) resolvedDocuments = buildDocsSearchDocuments(scopedPages, search.chunking, audience);
|
|
4936
|
+
return resolvedDocuments;
|
|
4937
|
+
};
|
|
4938
|
+
const context = {
|
|
4939
|
+
pages: scopedPages,
|
|
4940
|
+
get documents() {
|
|
4941
|
+
return getDocuments();
|
|
4942
|
+
},
|
|
4943
|
+
set documents(documents) {
|
|
4944
|
+
resolvedDocuments = documents;
|
|
4945
|
+
},
|
|
4946
|
+
audience,
|
|
4947
|
+
locale: options.locale,
|
|
4948
|
+
pathname: options.pathname,
|
|
4949
|
+
siteTitle: options.siteTitle,
|
|
4950
|
+
baseUrl: options.baseUrl,
|
|
4951
|
+
indexBaseUrl,
|
|
4952
|
+
chunking: search.chunking,
|
|
4953
|
+
deferSourceProvenance: true,
|
|
4954
|
+
signal: options.signal
|
|
4955
|
+
};
|
|
4956
|
+
const query = {
|
|
4957
|
+
query: options.query,
|
|
4958
|
+
limit: options.limit ?? search.maxResults,
|
|
4959
|
+
...options.paginated ? { offset: options.offset ?? 0 } : {},
|
|
4960
|
+
...options.providerCursor !== void 0 ? { cursor: options.providerCursor } : {},
|
|
4961
|
+
locale: options.locale,
|
|
4962
|
+
pathname: options.pathname,
|
|
4963
|
+
audience,
|
|
4964
|
+
...hasFilters ? { filters } : {}
|
|
4965
|
+
};
|
|
4966
|
+
let currentIndexGeneration;
|
|
4967
|
+
const getCurrentIndexGeneration = () => {
|
|
4968
|
+
currentIndexGeneration ??= options.indexGeneration ? Promise.resolve(options.indexGeneration) : buildDocsSearchIndexGeneration(options.generationPages ?? options.pages, {
|
|
4969
|
+
audience,
|
|
4970
|
+
chunking: search.chunking,
|
|
4971
|
+
locale: options.locale,
|
|
4972
|
+
baseUrl: options.baseUrl
|
|
4973
|
+
});
|
|
4974
|
+
return currentIndexGeneration;
|
|
4975
|
+
};
|
|
4976
|
+
const requireCurrentIndexGeneration = search.provider === "algolia" || search.provider === "typesense";
|
|
4977
|
+
let hostedIndexGeneration;
|
|
4978
|
+
const getHostedIndexGeneration = () => {
|
|
4979
|
+
hostedIndexGeneration ??= audience === "human" ? getCurrentIndexGeneration() : buildDocsSearchIndexGeneration(options.generationPages ?? options.pages, {
|
|
4980
|
+
audience: "human",
|
|
4981
|
+
chunking: search.chunking,
|
|
4982
|
+
locale: options.locale,
|
|
4983
|
+
baseUrl: indexBaseUrl
|
|
4984
|
+
});
|
|
4985
|
+
return hostedIndexGeneration;
|
|
4986
|
+
};
|
|
4987
|
+
const finalizeResults = async (results) => enrichDocsSearchResultsWithSources({
|
|
4988
|
+
results,
|
|
4989
|
+
pages: scopedPages,
|
|
4990
|
+
generationPages: options.generationPages ?? options.pages,
|
|
4991
|
+
audience,
|
|
4992
|
+
chunking: search.chunking,
|
|
4993
|
+
locale: options.locale,
|
|
4994
|
+
baseUrl: options.baseUrl,
|
|
4995
|
+
indexGeneration: options.indexGeneration,
|
|
4996
|
+
strictExternalOrigins: options.strictExternalOrigins,
|
|
4997
|
+
filters,
|
|
4998
|
+
requireCurrentIndexGeneration
|
|
4999
|
+
});
|
|
5000
|
+
const createLocalPage = async () => {
|
|
5001
|
+
const fallback = options.paginated ? await createSimpleSearchAdapter().searchPage(query, context) : normalizeDocsSearchAdapterPage(await createSimpleSearchAdapter().search(query, context));
|
|
5002
|
+
if (options.expectedTotal !== void 0 && fallback.total !== options.expectedTotal) throw new DocsPaginationCursorError();
|
|
5003
|
+
return {
|
|
5004
|
+
results: await finalizeResults(fallback.results),
|
|
5005
|
+
total: options.paginated ? fallback.total : fallback.results.length,
|
|
5006
|
+
paginationMode: "local"
|
|
5007
|
+
};
|
|
5008
|
+
};
|
|
5009
|
+
const fallBackFromProvider = () => {
|
|
5010
|
+
if (options.paginationMode === "provider") throw new DocsPaginationCursorError();
|
|
5011
|
+
return createLocalPage();
|
|
5012
|
+
};
|
|
5013
|
+
if (options.paginated && options.paginationMode === "local") return createLocalPage();
|
|
5014
|
+
try {
|
|
5015
|
+
const adapter = await resolveSearchAdapter(search, context);
|
|
5016
|
+
if (shouldSyncOnSearch(search) && typeof adapter.index === "function") {
|
|
5017
|
+
const syncPages = audience === "agent" || hasFilters ? corpusPages : scopedPages;
|
|
5018
|
+
await maybeSyncSearchIndex(adapter, search, {
|
|
5019
|
+
pages: syncPages,
|
|
5020
|
+
documents: buildDocsSearchDocuments(syncPages, search.chunking, "human"),
|
|
5021
|
+
audience: "human",
|
|
5022
|
+
locale: options.locale,
|
|
5023
|
+
pathname: options.pathname,
|
|
5024
|
+
siteTitle: options.siteTitle,
|
|
5025
|
+
baseUrl: indexBaseUrl,
|
|
5026
|
+
indexBaseUrl,
|
|
5027
|
+
...options.paginated && requireCurrentIndexGeneration ? { indexGeneration: await getHostedIndexGeneration() } : {},
|
|
5028
|
+
chunking: search.chunking,
|
|
5029
|
+
signal: options.signal
|
|
5030
|
+
});
|
|
5031
|
+
}
|
|
5032
|
+
if (options.paginated && requireCurrentIndexGeneration) context.indexGeneration = await getHostedIndexGeneration();
|
|
5033
|
+
if (options.paginated && search.provider === "custom" && (!adapter.searchPage || search.raw?.provider !== "custom" || !search.raw.paginationRevision?.trim())) {
|
|
5034
|
+
if (options.paginationMode === "provider") throw new DocsPaginationCursorError();
|
|
5035
|
+
return createLocalPage();
|
|
5036
|
+
}
|
|
5037
|
+
const adapterSearch = options.paginated && adapter.searchPage ? adapter.searchPage(query, context) : adapter.search(query, context);
|
|
5038
|
+
let documents;
|
|
5039
|
+
try {
|
|
5040
|
+
documents = getDocuments();
|
|
5041
|
+
} catch (error) {
|
|
5042
|
+
adapterSearch.catch(() => void 0);
|
|
5043
|
+
throw error;
|
|
5044
|
+
}
|
|
5045
|
+
const rawAdapterResult = await adapterSearch;
|
|
5046
|
+
const adapterPage = options.paginated && !adapter.searchPage && Array.isArray(rawAdapterResult) ? {
|
|
5047
|
+
results: rawAdapterResult.slice(options.offset ?? 0, (options.offset ?? 0) + (query.limit ?? search.maxResults)),
|
|
5048
|
+
total: rawAdapterResult.length
|
|
5049
|
+
} : normalizeDocsSearchAdapterPage(rawAdapterResult);
|
|
5050
|
+
const results = adapterPage.results;
|
|
5051
|
+
if (search.provider === "simple") {
|
|
5052
|
+
if (options.expectedTotal !== void 0 && adapterPage.total !== options.expectedTotal) throw new DocsPaginationCursorError();
|
|
5053
|
+
return {
|
|
5054
|
+
results: await finalizeResults(results),
|
|
5055
|
+
total: options.paginated ? adapterPage.total : results.length,
|
|
5056
|
+
paginationMode: "local"
|
|
5057
|
+
};
|
|
5058
|
+
}
|
|
5059
|
+
const localAudienceProjectionResults = buildAudienceProjectionSearchResults(documents, options.query);
|
|
5060
|
+
const localPagePaths = new Set(scopedPages.map((page) => normalizeUrlRouteKey(page.url)));
|
|
5061
|
+
const allLocalPagePaths = hasFilters && search.provider === "mcp" ? new Set(options.pages.map((page) => normalizeUrlRouteKey(page.url))) : localPagePaths;
|
|
5062
|
+
const preserveUnmatched = !hasFilters && audience === "human" || search.provider === "mcp" ? (result) => shouldPreserveUnmatchedExternalResult({
|
|
5063
|
+
result,
|
|
5064
|
+
localPagePaths: allLocalPagePaths,
|
|
5065
|
+
baseUrl: options.baseUrl
|
|
5066
|
+
}) : void 0;
|
|
5067
|
+
const isKnownLocalProviderResult = (result) => isLocalProviderResult(result, options.baseUrl) && localPagePaths.has(normalizeUrlRouteKey(result.url));
|
|
5068
|
+
const expectedProviderGeneration = requireCurrentIndexGeneration ? await getHostedIndexGeneration() : void 0;
|
|
5069
|
+
const providerAudience = requireCurrentIndexGeneration ? "human" : audience;
|
|
5070
|
+
const safeAdapterResults = sanitizeExternalAudienceSearchResults(results.filter((result) => {
|
|
5071
|
+
if (result.source === void 0) return !requireCurrentIndexGeneration || isKnownLocalProviderResult(result);
|
|
5072
|
+
const source = parseDocsRetrievalSource(result.source, { allowRootRelativeCanonical: requireCurrentIndexGeneration || isKnownLocalProviderResult(result) });
|
|
5073
|
+
const knownLocal = isKnownLocalProviderResult(result);
|
|
5074
|
+
return Boolean(source && docsRetrievalSourceMatchesRequest(source, providerAudience, knownLocal ? void 0 : filters, options.locale) && (!expectedProviderGeneration || source.indexGeneration === expectedProviderGeneration));
|
|
5075
|
+
}).filter((result) => requireCurrentIndexGeneration && isKnownLocalProviderResult(result) || !hasOppositeAudienceEvidence({
|
|
5076
|
+
result,
|
|
5077
|
+
pages: scopedPages,
|
|
5078
|
+
query: options.query,
|
|
5079
|
+
audience,
|
|
5080
|
+
baseUrl: options.baseUrl
|
|
5081
|
+
})), localAudienceProjectionResults, options.baseUrl, preserveUnmatched, requireCurrentIndexGeneration && audience === "agent");
|
|
5082
|
+
if (options.paginated) {
|
|
5083
|
+
const endOffset = (query.offset ?? 0) + safeAdapterResults.length;
|
|
5084
|
+
if (safeAdapterResults.length !== results.length || safeAdapterResults.length > (query.limit ?? search.maxResults) || endOffset > adapterPage.total || requireCurrentIndexGeneration && adapterPage.total === 0 && localAudienceProjectionResults.length > 0 || endOffset < adapterPage.total && safeAdapterResults.length === 0 || adapterPage.nextCursor !== void 0 && options.providerCursor !== void 0 && adapterPage.nextCursor === options.providerCursor || adapterPage.nextCursor !== void 0 && endOffset >= adapterPage.total || options.expectedTotal !== void 0 && adapterPage.total !== options.expectedTotal) return fallBackFromProvider();
|
|
5085
|
+
return {
|
|
5086
|
+
results: await finalizeResults(safeAdapterResults),
|
|
5087
|
+
total: adapterPage.total,
|
|
5088
|
+
...adapterPage.nextCursor !== void 0 ? { nextCursor: adapterPage.nextCursor } : {},
|
|
5089
|
+
paginationMode: "provider"
|
|
5090
|
+
};
|
|
5091
|
+
}
|
|
5092
|
+
if (options.supplementExternalResults === false) {
|
|
5093
|
+
const bounded = safeAdapterResults.slice(0, query.limit ?? search.maxResults ?? DEFAULT_SEARCH_LIMIT);
|
|
5094
|
+
return {
|
|
5095
|
+
results: await finalizeResults(bounded),
|
|
5096
|
+
total: bounded.length,
|
|
5097
|
+
paginationMode: "provider"
|
|
5098
|
+
};
|
|
5099
|
+
}
|
|
5100
|
+
const simpleAudienceResults = audience === "agent" || results.length === 0 || safeAdapterResults.length < results.length ? await createSimpleSearchAdapter().search(query, context) : [];
|
|
5101
|
+
const combinedResults = mergeSearchResults([
|
|
5102
|
+
buildExactPageSearchResults(options.query, scopedPages, audience),
|
|
5103
|
+
safeAdapterResults,
|
|
5104
|
+
simpleAudienceResults
|
|
5105
|
+
], audience === "agent" ? (result) => getAskAIResultKey(result, options.baseUrl, options.strictExternalOrigins) : void 0);
|
|
5106
|
+
const bounded = prioritizeLiteralInsideResults(options.query, combinedResults).slice(0, query.limit ?? search.maxResults ?? DEFAULT_SEARCH_LIMIT);
|
|
5107
|
+
return {
|
|
5108
|
+
results: await finalizeResults(bounded),
|
|
5109
|
+
total: bounded.length,
|
|
5110
|
+
paginationMode: "provider"
|
|
5111
|
+
};
|
|
5112
|
+
} catch (error) {
|
|
5113
|
+
if (error instanceof DocsPaginationCursorError) throw error;
|
|
5114
|
+
if (options.failureMode === "throw") throw error;
|
|
5115
|
+
return fallBackFromProvider();
|
|
5116
|
+
}
|
|
5117
|
+
}
|
|
5118
|
+
async function performDocsSearch(options) {
|
|
5119
|
+
return (await performDocsSearchInternal(options)).results;
|
|
5120
|
+
}
|
|
5121
|
+
function compareSearchMetadataValues(left, right) {
|
|
5122
|
+
if (left === right) return 0;
|
|
5123
|
+
return left < right ? -1 : 1;
|
|
5124
|
+
}
|
|
5125
|
+
function docsSearchPageMatchesFacetFilters(scope, filters, omittedField) {
|
|
5126
|
+
if (scope.conflicts.length > 0) return false;
|
|
5127
|
+
return SEARCH_FILTER_FIELDS.every((field) => {
|
|
5128
|
+
if (field === omittedField) return true;
|
|
5129
|
+
const requested = filters[field];
|
|
5130
|
+
return !requested?.length || docsSearchScopeFieldMatches(scope, field, requested);
|
|
5131
|
+
});
|
|
5132
|
+
}
|
|
5133
|
+
function buildDocsSearchFacet(pages, field, filters, options) {
|
|
5134
|
+
const counts = /* @__PURE__ */ new Map();
|
|
5135
|
+
for (const { scope } of pages) {
|
|
5136
|
+
if (!docsSearchPageMatchesFacetFilters(scope, filters, field)) continue;
|
|
5137
|
+
for (const value of new Set(scope[field])) counts.set(value, (counts.get(value) ?? 0) + 1);
|
|
5138
|
+
}
|
|
5139
|
+
const values = [...counts].sort(([left], [right]) => compareSearchMetadataValues(left, right)).map(([value, count]) => ({
|
|
5140
|
+
value,
|
|
5141
|
+
count
|
|
5142
|
+
}));
|
|
5143
|
+
const normalizedFilters = Object.fromEntries(SEARCH_FILTER_FIELDS.flatMap((filterField) => {
|
|
5144
|
+
const filterValues = filters[filterField];
|
|
5145
|
+
return filterValues?.length ? [[filterField, [...filterValues].sort(compareSearchMetadataValues)]] : [];
|
|
5146
|
+
}));
|
|
5147
|
+
const page = paginateDocsItems(values, {
|
|
5148
|
+
kind: "docs-search-facet",
|
|
5149
|
+
scope: JSON.stringify({
|
|
5150
|
+
format: "docs-search-facet-pagination.v1",
|
|
5151
|
+
field,
|
|
5152
|
+
audience: options.audience,
|
|
5153
|
+
filters: normalizedFilters,
|
|
5154
|
+
locale: options.locale ? normalizeAgentLocale(options.locale) : void 0
|
|
5155
|
+
}),
|
|
5156
|
+
snapshot: options.indexGeneration,
|
|
5157
|
+
cursor: options.cursor,
|
|
5158
|
+
pageSize: options.limit
|
|
5159
|
+
});
|
|
5160
|
+
return {
|
|
5161
|
+
valueCount: page.total,
|
|
5162
|
+
total: page.total,
|
|
5163
|
+
hasMore: page.hasMore,
|
|
5164
|
+
...page.nextCursor ? { nextCursor: page.nextCursor } : {},
|
|
5165
|
+
truncated: page.items.length < page.total,
|
|
5166
|
+
values: page.items
|
|
5167
|
+
};
|
|
5168
|
+
}
|
|
5169
|
+
/**
|
|
5170
|
+
* Build a body-free index of valid search filter values and page counts.
|
|
5171
|
+
*
|
|
5172
|
+
* Each facet applies filters from the other fields while ignoring its own selected
|
|
5173
|
+
* values. This is the conventional faceted-search behavior and lets an agent discover
|
|
5174
|
+
* safe alternatives without guessing or fetching document bodies.
|
|
5175
|
+
*/
|
|
5176
|
+
async function buildDocsSearchFacets(options) {
|
|
5177
|
+
const audience = resolveDocsSearchAudience(options.audience);
|
|
5178
|
+
const filters = normalizeDocsSearchFilters(options.filters);
|
|
5179
|
+
const search = normalizeDocsSearchConfig(options.search);
|
|
5180
|
+
const requestedLimit = options.limit ?? MAX_SEARCH_FACET_VALUES;
|
|
5181
|
+
if (!Number.isSafeInteger(requestedLimit) || requestedLimit < 1 || requestedLimit > MAX_SEARCH_FACET_VALUES) throw new DocsSearchRequestError("Search limit must be an integer between 1 and 100.");
|
|
5182
|
+
if (options.cursor !== void 0 && options.facet === void 0) throw new DocsSearchRequestError("Facet continuation cursors require a `facet` field.");
|
|
5183
|
+
const pages = options.pages.map((page) => localizeDocsSearchPage(page, options.locale)).map((page) => ({ scope: resolveDocsSearchPageScope(page) }));
|
|
5184
|
+
const indexGeneration = options.indexGeneration ?? await buildDocsSearchIndexGeneration(options.pages, {
|
|
5185
|
+
audience,
|
|
5186
|
+
chunking: search.chunking,
|
|
5187
|
+
locale: options.locale,
|
|
5188
|
+
baseUrl: options.baseUrl
|
|
5189
|
+
});
|
|
5190
|
+
return {
|
|
5191
|
+
format: "docs-search-facets.v1",
|
|
5192
|
+
audience,
|
|
5193
|
+
filters,
|
|
5194
|
+
indexGeneration,
|
|
5195
|
+
matchedPageCount: pages.filter(({ scope }) => docsSearchPageMatchesFacetFilters(scope, filters)).length,
|
|
5196
|
+
facets: Object.fromEntries(SEARCH_FILTER_FIELDS.map((field) => [field, buildDocsSearchFacet(pages, field, filters, {
|
|
5197
|
+
audience,
|
|
5198
|
+
cursor: field === options.facet ? options.cursor : void 0,
|
|
5199
|
+
indexGeneration,
|
|
5200
|
+
limit: options.facet === void 0 || field === options.facet ? requestedLimit : MAX_SEARCH_FACET_VALUES,
|
|
5201
|
+
locale: options.locale
|
|
5202
|
+
})]))
|
|
5203
|
+
};
|
|
5204
|
+
}
|
|
5205
|
+
function boundedSearchWarningValues(values) {
|
|
5206
|
+
const bounded = Array.from(new Set(values)).sort(compareSearchMetadataValues).slice(0, MAX_SEARCH_WARNING_VALUES);
|
|
5207
|
+
return bounded.length > 0 ? bounded : void 0;
|
|
5208
|
+
}
|
|
5209
|
+
function boundedSearchWarningPageUrls(pages) {
|
|
5210
|
+
const bounded = Array.from(new Set(pages.map((page) => page.url))).sort(compareSearchMetadataValues).slice(0, MAX_SEARCH_WARNING_PAGE_URLS);
|
|
5211
|
+
return bounded.length > 0 ? bounded : void 0;
|
|
5212
|
+
}
|
|
5213
|
+
function findSearchResultPages(pages, results, baseUrl) {
|
|
5214
|
+
const pagesByPath = new Map(pages.map((page) => [normalizeUrlRouteKey(page.url), page]));
|
|
5215
|
+
const found = /* @__PURE__ */ new Map();
|
|
5216
|
+
for (const result of results) {
|
|
5217
|
+
if (!isLocalProviderResult(result, baseUrl)) continue;
|
|
5218
|
+
const page = pagesByPath.get(normalizeUrlRouteKey(result.url));
|
|
5219
|
+
if (page) found.set(page.url, page);
|
|
5220
|
+
}
|
|
5221
|
+
return [...found.values()].sort((left, right) => compareSearchMetadataValues(left.url, right.url));
|
|
5222
|
+
}
|
|
5223
|
+
function searchScopeIsAmbiguous(field, values) {
|
|
5224
|
+
if (values.length < 2) return false;
|
|
5225
|
+
if (field !== "version") return true;
|
|
5226
|
+
return values.some((left, index) => values.slice(index + 1).some((right) => !agentVersionConstraintsOverlap(left, right)));
|
|
5227
|
+
}
|
|
5228
|
+
function buildDocsSearchWarnings(options) {
|
|
5229
|
+
const pageScopes = options.pages.map((page) => ({
|
|
5230
|
+
page,
|
|
5231
|
+
scope: resolveDocsSearchPageScope(page)
|
|
5232
|
+
}));
|
|
5233
|
+
const relevantPageScopes = pageScopes.filter(({ page }) => scoreDocument(options.query, pageToSearchDocument(page, options.audience)) > 0);
|
|
5234
|
+
const warnings = [];
|
|
5235
|
+
for (const field of SEARCH_FILTER_FIELDS) {
|
|
5236
|
+
const conflicting = relevantPageScopes.filter(({ scope }) => scope.conflicts.includes(field));
|
|
5237
|
+
if (conflicting.length > 0) warnings.push({
|
|
5238
|
+
code: "conflicting_scope_metadata",
|
|
5239
|
+
field,
|
|
5240
|
+
message: `${conflicting.length} page${conflicting.length === 1 ? "" : "s"} declare conflicting ${field} metadata and cannot be selected safely by scoped search.`,
|
|
5241
|
+
pageUrls: boundedSearchWarningPageUrls(conflicting.map(({ page }) => page)),
|
|
5242
|
+
count: conflicting.length
|
|
5243
|
+
});
|
|
5244
|
+
const requested = options.filters[field];
|
|
5245
|
+
if (!requested || requested.length === 0) continue;
|
|
5246
|
+
const missing = relevantPageScopes.filter(({ scope }) => scope.declarations[field].length === 0);
|
|
5247
|
+
if (missing.length > 0) warnings.push({
|
|
5248
|
+
code: "missing_scope_metadata",
|
|
5249
|
+
field,
|
|
5250
|
+
message: `${missing.length} page${missing.length === 1 ? "" : "s"} lack ${field} metadata and were excluded by the strict scope filter.`,
|
|
5251
|
+
pageUrls: boundedSearchWarningPageUrls(missing.map(({ page }) => page)),
|
|
5252
|
+
count: missing.length
|
|
5253
|
+
});
|
|
5254
|
+
const unknown = requested.filter((value) => !pageScopes.some(({ scope }) => !scope.conflicts.includes(field) && docsSearchScopeFieldMatches(scope, field, [value])));
|
|
5255
|
+
if (unknown.length > 0) warnings.push({
|
|
5256
|
+
code: "unknown_filter_value",
|
|
5257
|
+
field,
|
|
5258
|
+
message: `No page metadata matches ${field} filter value${unknown.length === 1 ? "" : "s"}: ${unknown.join(", ")}.`,
|
|
5259
|
+
values: boundedSearchWarningValues(unknown),
|
|
5260
|
+
count: unknown.length
|
|
5261
|
+
});
|
|
5262
|
+
}
|
|
5263
|
+
const resultPages = options.completeResultSet ? relevantPageScopes.filter(({ scope }) => docsSearchPageMatchesFilters(scope, options.filters)).map(({ page }) => page).sort((left, right) => compareSearchMetadataValues(left.url, right.url)) : findSearchResultPages(options.pages, options.results, options.baseUrl);
|
|
5264
|
+
for (const field of SEARCH_AMBIGUITY_FIELDS) {
|
|
5265
|
+
if ((options.filters[field]?.length ?? 0) > 0) continue;
|
|
5266
|
+
const values = resultPages.flatMap((page) => {
|
|
5267
|
+
const scope = resolveDocsSearchPageScope(page);
|
|
5268
|
+
return scope.conflicts.includes(field) ? [] : scope[field];
|
|
5269
|
+
});
|
|
5270
|
+
const unique = Array.from(new Set(values)).sort(compareSearchMetadataValues);
|
|
5271
|
+
if (!searchScopeIsAmbiguous(field, unique)) continue;
|
|
5272
|
+
const contributingPages = resultPages.filter((page) => {
|
|
5273
|
+
const scope = resolveDocsSearchPageScope(page);
|
|
5274
|
+
return !scope.conflicts.includes(field) && scope[field].length > 0;
|
|
5275
|
+
});
|
|
5276
|
+
warnings.push({
|
|
5277
|
+
code: "ambiguous_scope",
|
|
5278
|
+
field,
|
|
5279
|
+
message: `Search results span multiple ${field} scopes; add a ${field} filter before acting on scope-specific guidance.`,
|
|
5280
|
+
values: boundedSearchWarningValues(unique),
|
|
5281
|
+
pageUrls: boundedSearchWarningPageUrls(contributingPages),
|
|
5282
|
+
count: unique.length
|
|
5283
|
+
});
|
|
5284
|
+
}
|
|
5285
|
+
const codeOrder = {
|
|
5286
|
+
ambiguous_scope: 0,
|
|
5287
|
+
unknown_filter_value: 1,
|
|
5288
|
+
missing_scope_metadata: 2,
|
|
5289
|
+
conflicting_scope_metadata: 3
|
|
5290
|
+
};
|
|
5291
|
+
return warnings.sort((left, right) => {
|
|
5292
|
+
const codeDelta = codeOrder[left.code] - codeOrder[right.code];
|
|
5293
|
+
if (codeDelta !== 0) return codeDelta;
|
|
5294
|
+
return SEARCH_FILTER_FIELDS.indexOf(left.field) - SEARCH_FILTER_FIELDS.indexOf(right.field);
|
|
5295
|
+
}).slice(0, MAX_SEARCH_WARNINGS);
|
|
5296
|
+
}
|
|
5297
|
+
function parseDocsSearchCursorState(state, offset) {
|
|
5298
|
+
if (offset === 0 && state === void 0) return void 0;
|
|
5299
|
+
if (state === void 0) throw new DocsPaginationCursorError();
|
|
5300
|
+
let parsed;
|
|
5301
|
+
try {
|
|
5302
|
+
parsed = JSON.parse(state);
|
|
5303
|
+
} catch {
|
|
5304
|
+
throw new DocsPaginationCursorError();
|
|
5305
|
+
}
|
|
5306
|
+
if (!Array.isArray(parsed) || parsed.length !== 4 || JSON.stringify(parsed) !== state) throw new DocsPaginationCursorError();
|
|
5307
|
+
const [version, mode, total, providerCursor] = parsed;
|
|
5308
|
+
if (version !== 1 || mode !== "local" && mode !== "provider" || !Number.isSafeInteger(total) || total < offset || !(providerCursor === null || typeof providerCursor === "string" && providerCursor.length <= 768) || mode === "local" && providerCursor !== null) throw new DocsPaginationCursorError();
|
|
5309
|
+
return {
|
|
5310
|
+
mode,
|
|
5311
|
+
total,
|
|
5312
|
+
...typeof providerCursor === "string" ? { providerCursor } : {}
|
|
5313
|
+
};
|
|
5314
|
+
}
|
|
5315
|
+
function createDocsSearchCursorState(page) {
|
|
5316
|
+
return JSON.stringify([
|
|
5317
|
+
1,
|
|
5318
|
+
page.paginationMode,
|
|
5319
|
+
page.total,
|
|
5320
|
+
page.paginationMode === "provider" ? page.nextCursor ?? null : null
|
|
5321
|
+
]);
|
|
5322
|
+
}
|
|
5323
|
+
async function performDocsSearchWithMetadata(options) {
|
|
5324
|
+
const audience = resolveDocsSearchAudience(options.audience);
|
|
5325
|
+
const filters = normalizeDocsSearchFilters(options.filters);
|
|
5326
|
+
const search = normalizeDocsSearchConfig(options.search);
|
|
5327
|
+
const requestedLimit = options.limit ?? search.maxResults ?? DEFAULT_SEARCH_LIMIT;
|
|
5328
|
+
const limit = Number.isFinite(requestedLimit) && requestedLimit > 0 ? Math.max(1, Math.floor(requestedLimit)) : DEFAULT_SEARCH_LIMIT;
|
|
5329
|
+
const indexGeneration = options.indexGeneration ?? await buildDocsSearchIndexGeneration(options.generationPages ?? options.pages, {
|
|
5330
|
+
audience,
|
|
5331
|
+
chunking: search.chunking,
|
|
5332
|
+
locale: options.locale,
|
|
5333
|
+
baseUrl: options.baseUrl
|
|
5334
|
+
});
|
|
5335
|
+
const normalizedFilters = Object.fromEntries(SEARCH_FILTER_FIELDS.flatMap((field) => {
|
|
5336
|
+
const values = filters[field];
|
|
5337
|
+
return values?.length ? [[field, [...values].sort(compareSearchMetadataValues)]] : [];
|
|
5338
|
+
}));
|
|
5339
|
+
const provider = getDocsSearchPaginationProviderIdentity(search);
|
|
5340
|
+
const cursorOptions = {
|
|
5341
|
+
kind: "docs-search",
|
|
5342
|
+
scope: JSON.stringify({
|
|
5343
|
+
format: "docs-search-pagination.v2",
|
|
5344
|
+
query: options.query,
|
|
5345
|
+
audience,
|
|
5346
|
+
filters: normalizedFilters,
|
|
5347
|
+
locale: options.locale ? normalizeAgentLocale(options.locale) : void 0,
|
|
5348
|
+
pathname: options.pathname?.trim() || void 0,
|
|
5349
|
+
provider,
|
|
5350
|
+
limit
|
|
5351
|
+
}),
|
|
5352
|
+
snapshot: indexGeneration
|
|
5353
|
+
};
|
|
5354
|
+
const cursorPosition = resolveDocsPaginationCursor(options.cursor, cursorOptions);
|
|
5355
|
+
const offset = cursorPosition.offset;
|
|
5356
|
+
const cursorState = parseDocsSearchCursorState(cursorPosition.state, offset);
|
|
5357
|
+
const page = options.query.trim() ? await performDocsSearchInternal({
|
|
5358
|
+
...options,
|
|
5359
|
+
audience,
|
|
5360
|
+
filters,
|
|
5361
|
+
indexGeneration,
|
|
5362
|
+
limit,
|
|
5363
|
+
offset,
|
|
5364
|
+
paginated: true,
|
|
5365
|
+
providerCursor: cursorState?.providerCursor,
|
|
5366
|
+
paginationMode: cursorState?.mode,
|
|
5367
|
+
expectedTotal: cursorState?.total
|
|
5368
|
+
}) : {
|
|
5369
|
+
results: [],
|
|
5370
|
+
total: 0,
|
|
5371
|
+
paginationMode: "local"
|
|
5372
|
+
};
|
|
5373
|
+
if (options.cursor !== void 0 && offset >= page.total || page.results.length > Math.min(limit, Math.max(0, page.total - offset)) || offset < page.total && page.results.length === 0) throw new DocsPaginationCursorError();
|
|
5374
|
+
const nextOffset = Math.min(page.total, offset + page.results.length);
|
|
5375
|
+
const hasMore = nextOffset < page.total;
|
|
5376
|
+
const nextCursor = hasMore ? createDocsPaginationCursor(nextOffset, {
|
|
5377
|
+
...cursorOptions,
|
|
5378
|
+
state: createDocsSearchCursorState(page)
|
|
5379
|
+
}) : void 0;
|
|
5380
|
+
return {
|
|
5381
|
+
format: "docs-search.v1",
|
|
5382
|
+
query: options.query,
|
|
5383
|
+
audience,
|
|
5384
|
+
filters,
|
|
5385
|
+
indexGeneration,
|
|
5386
|
+
resultCount: page.results.length,
|
|
5387
|
+
total: page.total,
|
|
5388
|
+
hasMore,
|
|
5389
|
+
...nextCursor ? { nextCursor } : {},
|
|
5390
|
+
results: page.results,
|
|
5391
|
+
warnings: buildDocsSearchWarnings({
|
|
5392
|
+
pages: options.pages,
|
|
5393
|
+
results: page.results,
|
|
5394
|
+
filters,
|
|
5395
|
+
query: options.query,
|
|
5396
|
+
audience,
|
|
5397
|
+
baseUrl: options.baseUrl,
|
|
5398
|
+
completeResultSet: true
|
|
5399
|
+
})
|
|
5400
|
+
};
|
|
5401
|
+
}
|
|
5402
|
+
async function buildDocsAskAIContext(options) {
|
|
5403
|
+
const limit = options.limit ?? 5;
|
|
5404
|
+
const searchLimit = Math.max(limit * 2, limit);
|
|
5405
|
+
const initialSearch = options.search === false ? true : options.search;
|
|
5406
|
+
const primarySearch = normalizeDocsSearchConfig(initialSearch).enabled ? initialSearch : true;
|
|
5407
|
+
const searchResults = await performDocsSearch({
|
|
5408
|
+
pages: options.pages,
|
|
5409
|
+
query: options.query,
|
|
5410
|
+
search: primarySearch,
|
|
5411
|
+
audience: "agent",
|
|
5412
|
+
locale: options.locale,
|
|
5413
|
+
pathname: options.pathname,
|
|
5414
|
+
siteTitle: options.siteTitle,
|
|
5415
|
+
baseUrl: options.baseUrl,
|
|
5416
|
+
syncBaseUrl: options.syncBaseUrl,
|
|
5417
|
+
limit: searchLimit,
|
|
5418
|
+
filters: options.filters,
|
|
5419
|
+
failureMode: options.searchFailureMode,
|
|
5420
|
+
strictExternalOrigins: options.strictExternalOrigins,
|
|
5421
|
+
signal: options.signal
|
|
5422
|
+
});
|
|
5423
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5424
|
+
const maxResultChars = options.maxResultChars ?? DEFAULT_ASK_AI_RESULT_CHARS;
|
|
5425
|
+
const rankedResults = searchResults.map((result, index) => {
|
|
5426
|
+
const formatted = formatAskAIContextResult({
|
|
5427
|
+
result,
|
|
5428
|
+
page: findPageForSearchResult(options.pages, result, options.baseUrl),
|
|
5429
|
+
maxChars: maxResultChars,
|
|
5430
|
+
baseUrl: options.baseUrl
|
|
5431
|
+
});
|
|
5432
|
+
return {
|
|
5433
|
+
result,
|
|
5434
|
+
formatted,
|
|
5435
|
+
index,
|
|
5436
|
+
rank: rankAskAIContextResult(options.query, formatted)
|
|
5437
|
+
};
|
|
5438
|
+
}).sort((a, b) => b.rank - a.rank || a.index - b.index);
|
|
5439
|
+
const formattedResults = rankedResults.map((item) => item.formatted);
|
|
5440
|
+
const sectionResultPaths = new Set(formattedResults.filter((result) => result.section).map((result) => getAskAIResultPageKey(result.url, options.baseUrl, options.strictExternalOrigins)));
|
|
5441
|
+
const results = formattedResults.filter((result) => result.section || !sectionResultPaths.has(getAskAIResultPageKey(result.url, options.baseUrl, options.strictExternalOrigins))).filter((result) => {
|
|
5442
|
+
const key = getAskAIResultKey(result, options.baseUrl, options.strictExternalOrigins);
|
|
5443
|
+
if (seen.has(key)) return false;
|
|
5444
|
+
seen.add(key);
|
|
5445
|
+
return result.contextContent.length > 0;
|
|
5446
|
+
}).slice(0, limit);
|
|
5447
|
+
const maxContextChars = options.maxContextChars ?? DEFAULT_ASK_AI_CONTEXT_CHARS;
|
|
5448
|
+
const blocks = [];
|
|
5449
|
+
let usedChars = 0;
|
|
5450
|
+
for (const result of results) {
|
|
5451
|
+
const block = buildAskAIContextBlock(result);
|
|
5452
|
+
const separatorChars = blocks.length === 0 ? 0 : 7;
|
|
5453
|
+
if (usedChars + separatorChars + block.length > maxContextChars) {
|
|
5454
|
+
const remaining = maxContextChars - usedChars - separatorChars;
|
|
5455
|
+
if (remaining > 400) blocks.push(clampText(block, remaining));
|
|
5456
|
+
break;
|
|
5457
|
+
}
|
|
5458
|
+
blocks.push(block);
|
|
5459
|
+
usedChars += separatorChars + block.length;
|
|
5460
|
+
}
|
|
5461
|
+
const context = blocks.join("\n\n---\n\n");
|
|
5462
|
+
return {
|
|
5463
|
+
context,
|
|
5464
|
+
blocks: blocks.map((text, index) => ({
|
|
5465
|
+
text,
|
|
5466
|
+
result: results[index]
|
|
5467
|
+
})),
|
|
5468
|
+
results: results.slice(0, blocks.length),
|
|
5469
|
+
searchResults: rankedResults.map((item) => item.result),
|
|
5470
|
+
packageHints: inferDocsAskAIPackageHints(context)
|
|
5471
|
+
};
|
|
5472
|
+
}
|
|
5473
|
+
function createCustomSearchAdapter(adapter, options = {}) {
|
|
5474
|
+
return {
|
|
5475
|
+
provider: "custom",
|
|
5476
|
+
adapter,
|
|
5477
|
+
...options.paginationRevision ? { paginationRevision: options.paginationRevision } : {}
|
|
5478
|
+
};
|
|
5479
|
+
}
|
|
5480
|
+
|
|
5481
|
+
//#endregion
|
|
5482
|
+
//#region src/content-changes.ts
|
|
5483
|
+
const DOCS_CONTENT_CHANGES_FORMAT = "docs-content-changes.v1";
|
|
5484
|
+
const DOCS_CONTENT_SNAPSHOT_FORMAT = "docs-content-snapshot.v1";
|
|
5485
|
+
const DOCS_CONTENT_CHANGES_RESPONSE_VALUE = "changes";
|
|
5486
|
+
const DEFAULT_DOCS_CONTENT_CHANGE_SNAPSHOTS = 8;
|
|
5487
|
+
const MAX_DOCS_CONTENT_CHANGE_SNAPSHOTS = 64;
|
|
5488
|
+
const DOCS_RETRIEVAL_DIGEST_PATTERN = /^sha256:[a-f\d]{64}$/u;
|
|
5489
|
+
var DocsContentChangesRequestError = class extends Error {
|
|
5490
|
+
code = "invalid_content_changes_request";
|
|
5491
|
+
constructor(message) {
|
|
5492
|
+
super(message);
|
|
5493
|
+
this.name = "DocsContentChangesRequestError";
|
|
5494
|
+
}
|
|
5495
|
+
};
|
|
5496
|
+
function resolveDocsContentChangesConfig(input, options = {}) {
|
|
5497
|
+
const configured = input && typeof input === "object" ? input : void 0;
|
|
5498
|
+
const enabled = !options.staticExport && input !== false && configured?.enabled !== false;
|
|
5499
|
+
const requestedMax = configured?.maxSnapshots ?? DEFAULT_DOCS_CONTENT_CHANGE_SNAPSHOTS;
|
|
5500
|
+
return {
|
|
5501
|
+
enabled,
|
|
5502
|
+
maxSnapshots: Number.isSafeInteger(requestedMax) && requestedMax >= 1 && requestedMax <= MAX_DOCS_CONTENT_CHANGE_SNAPSHOTS ? requestedMax : DEFAULT_DOCS_CONTENT_CHANGE_SNAPSHOTS,
|
|
5503
|
+
...configured?.loadSnapshot ? { loadSnapshot: configured.loadSnapshot } : {},
|
|
5504
|
+
...configured?.saveSnapshot ? { saveSnapshot: configured.saveSnapshot } : {}
|
|
5505
|
+
};
|
|
5506
|
+
}
|
|
5507
|
+
function isDocsContentChangesRequest(url) {
|
|
5508
|
+
return url.searchParams.get("response") === DOCS_CONTENT_CHANGES_RESPONSE_VALUE;
|
|
5509
|
+
}
|
|
5510
|
+
function resolveDocsContentChangesRequest(url) {
|
|
5511
|
+
if (!isDocsContentChangesRequest(url)) throw new DocsContentChangesRequestError("Content changes require `response=changes`.");
|
|
5512
|
+
const rawAudience = url.searchParams.get("audience");
|
|
5513
|
+
if (rawAudience !== null && rawAudience !== "human" && rawAudience !== "agent") throw new DocsContentChangesRequestError("Content-change audience must be `human` or `agent`.");
|
|
5514
|
+
const since = url.searchParams.get("since") ?? void 0;
|
|
5515
|
+
if (since !== void 0 && !DOCS_RETRIEVAL_DIGEST_PATTERN.test(since)) throw new DocsContentChangesRequestError("Content-change `since` must be a SHA-256 index generation.");
|
|
5516
|
+
return {
|
|
5517
|
+
audience: rawAudience ?? "agent",
|
|
5518
|
+
...since ? { since } : {}
|
|
5519
|
+
};
|
|
5520
|
+
}
|
|
5521
|
+
function snapshotScope(snapshot) {
|
|
5522
|
+
return JSON.stringify({
|
|
5523
|
+
audience: snapshot.audience,
|
|
5524
|
+
locale: snapshot.locale,
|
|
5525
|
+
baseUrl: snapshot.baseUrl
|
|
5526
|
+
});
|
|
5527
|
+
}
|
|
5528
|
+
function snapshotKey(snapshot) {
|
|
5529
|
+
return `${snapshotScope(snapshot)}\0${snapshot.indexGeneration}`;
|
|
5530
|
+
}
|
|
5531
|
+
function snapshotContext(snapshot, request) {
|
|
5532
|
+
return {
|
|
5533
|
+
audience: snapshot.audience,
|
|
5534
|
+
...snapshot.locale ? { locale: snapshot.locale } : {},
|
|
5535
|
+
...snapshot.baseUrl ? { baseUrl: snapshot.baseUrl } : {},
|
|
5536
|
+
...request ? { request } : {}
|
|
5537
|
+
};
|
|
5538
|
+
}
|
|
5539
|
+
function isSnapshotDocument(value) {
|
|
5540
|
+
if (!value || typeof value !== "object") return false;
|
|
5541
|
+
const document = value;
|
|
5542
|
+
return typeof document.url === "string" && document.url.length > 0 && typeof document.canonicalUrl === "string" && isDocsRetrievalCanonicalUrl(document.canonicalUrl) && typeof document.digest === "string" && DOCS_RETRIEVAL_DIGEST_PATTERN.test(document.digest) && (document.lastModified === void 0 || typeof document.lastModified === "string" && Number.isFinite(Date.parse(document.lastModified)));
|
|
5543
|
+
}
|
|
5544
|
+
function validateLoadedSnapshot(value, generation, current) {
|
|
5545
|
+
if (value.format !== DOCS_CONTENT_SNAPSHOT_FORMAT || value.indexGeneration !== generation || !DOCS_RETRIEVAL_DIGEST_PATTERN.test(value.indexGeneration) || snapshotScope(value) !== snapshotScope(current) || !Array.isArray(value.documents) || !value.documents.every(isSnapshotDocument)) throw new TypeError("Content-change snapshot storage returned an invalid snapshot.");
|
|
5546
|
+
const documents = value.documents.map((document) => ({ ...document }));
|
|
5547
|
+
documents.sort((left, right) => left.canonicalUrl === right.canonicalUrl ? left.url.localeCompare(right.url) : left.canonicalUrl.localeCompare(right.canonicalUrl));
|
|
5548
|
+
if (documents.some((document, index) => index > 0 && documents[index - 1]?.canonicalUrl === document.canonicalUrl)) throw new TypeError("Content-change snapshot storage returned duplicate canonical URLs.");
|
|
5549
|
+
return {
|
|
5550
|
+
...value,
|
|
5551
|
+
documents
|
|
5552
|
+
};
|
|
5553
|
+
}
|
|
5554
|
+
function asChangeDocument(document) {
|
|
5555
|
+
return { ...document };
|
|
5556
|
+
}
|
|
5557
|
+
function buildDelta(current, previous, since) {
|
|
5558
|
+
const currentByUrl = new Map(current.documents.map((document) => [document.canonicalUrl, document]));
|
|
5559
|
+
const previousByUrl = new Map(previous?.documents.map((document) => [document.canonicalUrl, document]) ?? []);
|
|
5560
|
+
const added = [];
|
|
5561
|
+
const changed = [];
|
|
5562
|
+
const deleted = [];
|
|
5563
|
+
if (!since || !previous) added.push(...current.documents.map(asChangeDocument));
|
|
5564
|
+
else {
|
|
5565
|
+
for (const document of current.documents) {
|
|
5566
|
+
const old = previousByUrl.get(document.canonicalUrl);
|
|
5567
|
+
if (!old) added.push(asChangeDocument(document));
|
|
5568
|
+
else if (old.digest !== document.digest || old.lastModified !== document.lastModified || old.url !== document.url) changed.push({
|
|
5569
|
+
...asChangeDocument(document),
|
|
5570
|
+
previousDigest: old.digest,
|
|
5571
|
+
...old.lastModified ? { previousLastModified: old.lastModified } : {}
|
|
5572
|
+
});
|
|
5573
|
+
}
|
|
5574
|
+
for (const document of previous.documents) if (!currentByUrl.has(document.canonicalUrl)) deleted.push(asChangeDocument(document));
|
|
5575
|
+
}
|
|
5576
|
+
const mode = !since ? "snapshot" : previous ? "delta" : "reset";
|
|
5577
|
+
return {
|
|
5578
|
+
format: DOCS_CONTENT_CHANGES_FORMAT,
|
|
5579
|
+
audience: current.audience,
|
|
5580
|
+
...current.locale ? { locale: current.locale } : {},
|
|
5581
|
+
since: since ?? null,
|
|
5582
|
+
indexGeneration: current.indexGeneration,
|
|
5583
|
+
mode,
|
|
5584
|
+
resetRequired: mode === "reset",
|
|
5585
|
+
documentCount: current.documents.length,
|
|
5586
|
+
counts: {
|
|
5587
|
+
added: added.length,
|
|
5588
|
+
changed: changed.length,
|
|
5589
|
+
deleted: deleted.length
|
|
5590
|
+
},
|
|
5591
|
+
added,
|
|
5592
|
+
changed,
|
|
5593
|
+
deleted
|
|
5594
|
+
};
|
|
5595
|
+
}
|
|
5596
|
+
function createDocsContentChangeFeed(input) {
|
|
5597
|
+
const config = resolveDocsContentChangesConfig(input);
|
|
5598
|
+
const snapshots = /* @__PURE__ */ new Map();
|
|
5599
|
+
const saved = /* @__PURE__ */ new Set();
|
|
5600
|
+
const remember = (snapshot) => {
|
|
5601
|
+
const key = snapshotKey(snapshot);
|
|
5602
|
+
snapshots.delete(key);
|
|
5603
|
+
snapshots.set(key, snapshot);
|
|
5604
|
+
while (snapshots.size > config.maxSnapshots) {
|
|
5605
|
+
const oldest = snapshots.keys().next().value;
|
|
5606
|
+
if (!oldest) break;
|
|
5607
|
+
snapshots.delete(oldest);
|
|
5608
|
+
saved.delete(oldest);
|
|
5609
|
+
}
|
|
5610
|
+
};
|
|
5611
|
+
return { async resolve(options) {
|
|
5612
|
+
if (!config.enabled) throw new DocsContentChangesRequestError("Content changes are disabled.");
|
|
5613
|
+
const current = await buildDocsContentSnapshot(options);
|
|
5614
|
+
const currentKey = snapshotKey(current);
|
|
5615
|
+
let previous;
|
|
5616
|
+
if (options.since === current.indexGeneration) previous = current;
|
|
5617
|
+
else if (options.since) {
|
|
5618
|
+
const scopedKey = `${snapshotScope(current)}\0${options.since}`;
|
|
5619
|
+
previous = snapshots.get(scopedKey);
|
|
5620
|
+
if (!previous && config.loadSnapshot) {
|
|
5621
|
+
const loaded = await config.loadSnapshot(options.since, snapshotContext(current, options.request));
|
|
5622
|
+
if (loaded) {
|
|
5623
|
+
previous = validateLoadedSnapshot(loaded, options.since, current);
|
|
5624
|
+
remember(previous);
|
|
5625
|
+
}
|
|
5626
|
+
}
|
|
5627
|
+
}
|
|
5628
|
+
remember(current);
|
|
5629
|
+
if (config.saveSnapshot && !saved.has(currentKey)) {
|
|
5630
|
+
await config.saveSnapshot({
|
|
5631
|
+
...current,
|
|
5632
|
+
documents: current.documents.map((document) => ({ ...document }))
|
|
5633
|
+
}, snapshotContext(current, options.request));
|
|
5634
|
+
saved.add(currentKey);
|
|
5635
|
+
}
|
|
5636
|
+
return buildDelta(current, previous, options.since);
|
|
5637
|
+
} };
|
|
5638
|
+
}
|
|
5639
|
+
function contentChangesEtag(response) {
|
|
5640
|
+
return `"${digestDocsRetrievalContent(JSON.stringify([
|
|
5641
|
+
response.format,
|
|
5642
|
+
response.audience,
|
|
5643
|
+
response.locale ?? null,
|
|
5644
|
+
response.since,
|
|
5645
|
+
response.indexGeneration,
|
|
5646
|
+
response.mode
|
|
5647
|
+
]))}"`;
|
|
5648
|
+
}
|
|
5649
|
+
function requestMatchesEtag(request, etag) {
|
|
5650
|
+
const header = request.headers.get("if-none-match");
|
|
5651
|
+
if (!header) return false;
|
|
5652
|
+
return header.split(",").map((value) => value.trim().replace(/^W\//u, "")).some((value) => value === "*" || value === etag);
|
|
5653
|
+
}
|
|
5654
|
+
async function createDocsContentChangesHttpResponse(options) {
|
|
5655
|
+
let resolved;
|
|
5656
|
+
try {
|
|
5657
|
+
resolved = resolveDocsContentChangesRequest(new URL(options.request.url));
|
|
5658
|
+
} catch (error) {
|
|
5659
|
+
if (!(error instanceof DocsContentChangesRequestError)) throw error;
|
|
5660
|
+
return Response.json({ error: {
|
|
5661
|
+
code: error.code,
|
|
5662
|
+
message: error.message
|
|
5663
|
+
} }, { status: 400 });
|
|
5664
|
+
}
|
|
5665
|
+
const response = await options.feed.resolve({
|
|
5666
|
+
pages: options.pages,
|
|
5667
|
+
search: options.search,
|
|
5668
|
+
audience: resolved.audience,
|
|
5669
|
+
locale: options.locale,
|
|
5670
|
+
baseUrl: options.baseUrl,
|
|
5671
|
+
since: resolved.since,
|
|
5672
|
+
request: options.request
|
|
5673
|
+
});
|
|
5674
|
+
const etag = contentChangesEtag(response);
|
|
5675
|
+
const headers = {
|
|
5676
|
+
"Cache-Control": "public, max-age=0, s-maxage=60, stale-while-revalidate=300",
|
|
5677
|
+
ETag: etag,
|
|
5678
|
+
"X-Docs-Index-Generation": response.indexGeneration,
|
|
5679
|
+
"X-Robots-Tag": "noindex"
|
|
5680
|
+
};
|
|
5681
|
+
if (requestMatchesEtag(options.request, etag)) return new Response(null, {
|
|
5682
|
+
status: 304,
|
|
5683
|
+
headers
|
|
5684
|
+
});
|
|
5685
|
+
if (options.request.method.toUpperCase() === "HEAD") return new Response(null, { headers });
|
|
5686
|
+
return Response.json(response, { headers });
|
|
5687
|
+
}
|
|
5688
|
+
|
|
5689
|
+
//#endregion
|
|
5690
|
+
//#region src/sitemap.ts
|
|
5691
|
+
const DEFAULT_SITEMAP_XML_ROUTE = "/sitemap.xml";
|
|
5692
|
+
const DEFAULT_SITEMAP_MD_ROUTE = "/sitemap.md";
|
|
5693
|
+
const DEFAULT_SITEMAP_MD_DOCS_ROUTE = "/docs/sitemap.md";
|
|
5694
|
+
const DEFAULT_SITEMAP_MD_WELL_KNOWN_ROUTE = "/.well-known/sitemap.md";
|
|
5695
|
+
const DEFAULT_SITEMAP_MANIFEST_PATH = ".farming-labs/sitemap-manifest.json";
|
|
5696
|
+
function normalizeUrlPath(value) {
|
|
5697
|
+
const normalized = value.replace(/\/+/g, "/");
|
|
5698
|
+
if (normalized === "/") return normalized;
|
|
5699
|
+
return normalized.replace(/\/+$/, "");
|
|
5700
|
+
}
|
|
5701
|
+
function normalizeRoutePrefix(value) {
|
|
5702
|
+
if (!value) return "";
|
|
5703
|
+
const normalized = normalizeUrlPath(`/${value.replace(/^\/+|\/+$/g, "")}`);
|
|
5704
|
+
return normalized === "/" ? "" : normalized;
|
|
5705
|
+
}
|
|
5706
|
+
function joinRoute(prefix, route) {
|
|
5707
|
+
return normalizeUrlPath(`${prefix}/${route.replace(/^\/+/, "")}`);
|
|
5708
|
+
}
|
|
5709
|
+
function normalizeDateOnly(value) {
|
|
5710
|
+
if (!value) return void 0;
|
|
5711
|
+
const trimmed = value.trim();
|
|
5712
|
+
if (!trimmed) return void 0;
|
|
5713
|
+
const dateOnly = trimmed.match(/^(\d{4}-\d{2}-\d{2})/);
|
|
5714
|
+
if (dateOnly) return dateOnly[1];
|
|
5715
|
+
const parsed = new Date(trimmed);
|
|
5716
|
+
if (Number.isNaN(parsed.getTime())) return void 0;
|
|
5717
|
+
return parsed.toISOString().slice(0, 10);
|
|
5718
|
+
}
|
|
5719
|
+
function normalizeBaseUrl(value) {
|
|
5720
|
+
if (!value) return void 0;
|
|
5721
|
+
return value.trim().replace(/\/+$/, "") || void 0;
|
|
5722
|
+
}
|
|
5723
|
+
function isEnabledObject(config) {
|
|
5724
|
+
if (config === false) return false;
|
|
5725
|
+
if (config === void 0) return true;
|
|
5726
|
+
if (config === true) return true;
|
|
5727
|
+
return config.enabled !== false;
|
|
5728
|
+
}
|
|
5729
|
+
function resolveDocsSitemapConfig(sitemap, defaults = {}) {
|
|
5730
|
+
const objectConfig = sitemap && typeof sitemap === "object" ? sitemap : {};
|
|
5731
|
+
const routePrefix = normalizeRoutePrefix(objectConfig.routePrefix);
|
|
5732
|
+
const xmlConfig = objectConfig.xml;
|
|
5733
|
+
const markdownConfig = objectConfig.markdown;
|
|
5734
|
+
const xmlEnabled = xmlConfig === false ? false : typeof xmlConfig === "object" ? xmlConfig.enabled ?? true : true;
|
|
5735
|
+
const markdownEnabled = markdownConfig === false ? false : typeof markdownConfig === "object" ? markdownConfig.enabled ?? true : true;
|
|
5736
|
+
return {
|
|
5737
|
+
enabled: isEnabledObject(sitemap),
|
|
5738
|
+
routePrefix,
|
|
5739
|
+
baseUrl: normalizeBaseUrl(objectConfig.baseUrl ?? defaults.baseUrl),
|
|
5740
|
+
manifestPath: objectConfig.manifestPath ?? DEFAULT_SITEMAP_MANIFEST_PATH,
|
|
5741
|
+
xml: {
|
|
5742
|
+
enabled: xmlEnabled,
|
|
5743
|
+
includeLastmod: typeof xmlConfig === "object" ? xmlConfig.includeLastmod ?? true : true,
|
|
5744
|
+
route: joinRoute(routePrefix, DEFAULT_SITEMAP_XML_ROUTE)
|
|
5745
|
+
},
|
|
5746
|
+
markdown: {
|
|
5747
|
+
enabled: markdownEnabled,
|
|
5748
|
+
includeDescriptions: typeof markdownConfig === "object" ? markdownConfig.includeDescriptions ?? true : true,
|
|
5749
|
+
includeLastmod: typeof markdownConfig === "object" ? markdownConfig.includeLastmod ?? true : true,
|
|
5750
|
+
linkTarget: typeof markdownConfig === "object" ? markdownConfig.linkTarget ?? "both" : "both",
|
|
5751
|
+
route: joinRoute(routePrefix, DEFAULT_SITEMAP_MD_ROUTE),
|
|
5752
|
+
docsRoute: routePrefix ? void 0 : DEFAULT_SITEMAP_MD_DOCS_ROUTE,
|
|
5753
|
+
wellKnownRoute: joinRoute(routePrefix, DEFAULT_SITEMAP_MD_WELL_KNOWN_ROUTE)
|
|
5754
|
+
}
|
|
5755
|
+
};
|
|
5756
|
+
}
|
|
5757
|
+
function resolveDocsSitemapRequest(url, sitemap, options = {}) {
|
|
5758
|
+
const pathname = normalizeUrlPath(url.pathname);
|
|
5759
|
+
const format = url.searchParams.get("format")?.trim();
|
|
5760
|
+
if (pathname === resolveDocsDiscoveryApiRoute(options.apiRoute)) {
|
|
5761
|
+
if (format === "sitemap-xml") return "xml";
|
|
5762
|
+
if (format === "sitemap-md" || format === "sitemap-markdown") return "markdown";
|
|
5763
|
+
}
|
|
5764
|
+
const resolved = resolveDocsSitemapConfig(sitemap);
|
|
5765
|
+
if (!resolved.enabled) return null;
|
|
5766
|
+
if (resolved.xml.enabled && pathname === resolved.xml.route) return "xml";
|
|
5767
|
+
if (resolved.markdown.enabled && (pathname === resolved.markdown.route || pathname === resolved.markdown.docsRoute || pathname === resolved.markdown.wellKnownRoute)) return "markdown";
|
|
5768
|
+
return null;
|
|
5769
|
+
}
|
|
5770
|
+
function toDocsSitemapMarkdownUrl(url) {
|
|
5771
|
+
const normalized = normalizeUrlPath(url);
|
|
5772
|
+
return normalized.endsWith(".md") ? normalized : `${normalized}.md`;
|
|
5773
|
+
}
|
|
5774
|
+
function absolutizeUrl(baseUrl, url) {
|
|
5775
|
+
if (/^https?:\/\//i.test(url)) return url;
|
|
5776
|
+
const base = normalizeBaseUrl(baseUrl);
|
|
5777
|
+
return base ? `${base}${url.startsWith("/") ? url : `/${url}`}` : url;
|
|
5778
|
+
}
|
|
5779
|
+
function escapeXml(value) {
|
|
5780
|
+
return value.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """).replace(/'/g, "'");
|
|
5781
|
+
}
|
|
5782
|
+
function hashString(value) {
|
|
5783
|
+
let hash = 14695981039346656037n;
|
|
5784
|
+
const bytes = new TextEncoder().encode(value);
|
|
5785
|
+
for (const byte of bytes) {
|
|
5786
|
+
hash ^= BigInt(byte);
|
|
5787
|
+
hash = BigInt.asUintN(64, hash * 1099511628211n);
|
|
5788
|
+
}
|
|
5789
|
+
return `"fnv1a64-${hash.toString(16).padStart(16, "0")}"`;
|
|
5790
|
+
}
|
|
5791
|
+
function newestLastmod(pages) {
|
|
5792
|
+
let newest;
|
|
5793
|
+
for (const page of pages) {
|
|
5794
|
+
const date = normalizeDateOnly(page.lastmod);
|
|
5795
|
+
if (!date) continue;
|
|
5796
|
+
if (!newest || date > newest) newest = date;
|
|
5797
|
+
}
|
|
5798
|
+
return newest;
|
|
5799
|
+
}
|
|
5800
|
+
function titleize(value) {
|
|
5801
|
+
return value.replace(/-/g, " ").replace(/\b\w/g, (char) => char.toUpperCase());
|
|
5802
|
+
}
|
|
5803
|
+
function sitemapSectionName(entry, pageUrl) {
|
|
5804
|
+
const normalizedEntry = `/${entry.replace(/^\/+|\/+$/g, "") || "docs"}`;
|
|
5805
|
+
const normalizedUrl = normalizeUrlPath(pageUrl);
|
|
5806
|
+
if (normalizedUrl === normalizedEntry) return "Overview";
|
|
5807
|
+
const firstSegment = (normalizedUrl.startsWith(`${normalizedEntry}/`) ? normalizedUrl.slice(normalizedEntry.length + 1) : normalizedUrl.replace(/^\/+/, "")).split("/").filter(Boolean)[0];
|
|
5808
|
+
return firstSegment ? titleize(firstSegment) : "Pages";
|
|
5809
|
+
}
|
|
5810
|
+
function buildDocsSitemapManifest(options) {
|
|
5811
|
+
const baseUrl = normalizeBaseUrl(options.baseUrl);
|
|
5812
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5813
|
+
const pages = [];
|
|
5814
|
+
for (const page of options.pages) {
|
|
5815
|
+
const url = normalizeUrlPath(page.url);
|
|
5816
|
+
if (seen.has(url)) continue;
|
|
5817
|
+
seen.add(url);
|
|
5818
|
+
const resolvedLastmod = options.resolveLastmod?.(page);
|
|
5819
|
+
const fallbackLastmod = normalizeDateOnly(page.lastmod ?? page.lastModified);
|
|
5820
|
+
const lastmod = normalizeDateOnly(resolvedLastmod?.lastmod) ?? fallbackLastmod;
|
|
5821
|
+
pages.push({
|
|
5822
|
+
url,
|
|
5823
|
+
absoluteUrl: absolutizeUrl(baseUrl, url),
|
|
5824
|
+
markdownUrl: page.markdownUrl ?? toDocsSitemapMarkdownUrl(url),
|
|
5825
|
+
title: page.title,
|
|
5826
|
+
description: page.description,
|
|
5827
|
+
sourcePath: page.sourcePath,
|
|
5828
|
+
lastmod,
|
|
5829
|
+
lastmodSource: resolvedLastmod?.lastmodSource ?? (lastmod ? "unknown" : void 0),
|
|
5830
|
+
related: page.related?.map((link) => link.href)
|
|
5831
|
+
});
|
|
1950
5832
|
}
|
|
5833
|
+
pages.sort((left, right) => left.url.localeCompare(right.url));
|
|
1951
5834
|
return {
|
|
1952
|
-
|
|
1953
|
-
|
|
5835
|
+
version: 1,
|
|
5836
|
+
generatedAt: options.generatedAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
5837
|
+
baseUrl,
|
|
5838
|
+
entry: options.entry ?? "docs",
|
|
5839
|
+
siteTitle: options.siteTitle,
|
|
5840
|
+
pages
|
|
1954
5841
|
};
|
|
1955
5842
|
}
|
|
1956
|
-
function
|
|
1957
|
-
const
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
5843
|
+
function renderDocsSitemapXml(manifest, options = {}) {
|
|
5844
|
+
const baseUrl = normalizeBaseUrl(options.baseUrl ?? manifest.baseUrl);
|
|
5845
|
+
const includeLastmod = options.includeLastmod ?? true;
|
|
5846
|
+
const lines = ["<?xml version=\"1.0\" encoding=\"UTF-8\"?>", "<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">"];
|
|
5847
|
+
for (const page of manifest.pages) {
|
|
5848
|
+
lines.push(" <url>");
|
|
5849
|
+
lines.push(` <loc>${escapeXml(page.absoluteUrl ?? absolutizeUrl(baseUrl, page.url))}</loc>`);
|
|
5850
|
+
const lastmod = normalizeDateOnly(page.lastmod);
|
|
5851
|
+
if (includeLastmod && lastmod) lines.push(` <lastmod>${lastmod}</lastmod>`);
|
|
5852
|
+
lines.push(" </url>");
|
|
5853
|
+
}
|
|
5854
|
+
lines.push("</urlset>");
|
|
5855
|
+
return `${lines.join("\n")}\n`;
|
|
1962
5856
|
}
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
const
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
}
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
});
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
if (tag.only) continue;
|
|
1990
|
-
if (!hasOnlyAttribute) {
|
|
1991
|
-
issues.push({
|
|
1992
|
-
code: "missing-only",
|
|
1993
|
-
index: tag.index,
|
|
1994
|
-
message: "<Audience> requires the static literal `only=\"human\"` or `only=\"agent\"`."
|
|
1995
|
-
});
|
|
1996
|
-
continue;
|
|
5857
|
+
function renderDocsSitemapMarkdown(manifest, options = {}) {
|
|
5858
|
+
const siteTitle = manifest.siteTitle ?? "Documentation";
|
|
5859
|
+
const includeDescriptions = options.includeDescriptions ?? true;
|
|
5860
|
+
const includeLastmod = options.includeLastmod ?? true;
|
|
5861
|
+
const linkTarget = options.linkTarget ?? "both";
|
|
5862
|
+
const lines = [`# ${siteTitle} Sitemap`, ""];
|
|
5863
|
+
const baseUrl = normalizeBaseUrl(options.baseUrl ?? manifest.baseUrl);
|
|
5864
|
+
if (baseUrl) lines.push(`Base URL: ${baseUrl}`);
|
|
5865
|
+
lines.push(`Docs entry: /${manifest.entry.replace(/^\/+|\/+$/g, "") || "docs"}`);
|
|
5866
|
+
lines.push(`Generated: ${normalizeDateOnly(manifest.generatedAt) ?? manifest.generatedAt}`, "");
|
|
5867
|
+
const groups = /* @__PURE__ */ new Map();
|
|
5868
|
+
for (const page of manifest.pages) {
|
|
5869
|
+
const sectionName = sitemapSectionName(manifest.entry, page.url);
|
|
5870
|
+
groups.set(sectionName, [...groups.get(sectionName) ?? [], page]);
|
|
5871
|
+
}
|
|
5872
|
+
for (const [sectionName, pages] of groups) {
|
|
5873
|
+
lines.push(`## ${sectionName}`, "");
|
|
5874
|
+
for (const page of pages) {
|
|
5875
|
+
const primaryHref = linkTarget === "markdown" ? page.markdownUrl : page.url;
|
|
5876
|
+
lines.push(`- [${page.title}](${primaryHref})`);
|
|
5877
|
+
if (linkTarget === "both" || linkTarget === "markdown") lines.push(` Markdown: ${page.markdownUrl}`);
|
|
5878
|
+
if (includeDescriptions && page.description) lines.push(` Description: ${page.description}`);
|
|
5879
|
+
const lastmod = normalizeDateOnly(page.lastmod);
|
|
5880
|
+
if (includeLastmod && lastmod) lines.push(` Last updated: ${lastmod}`);
|
|
5881
|
+
if (page.related && page.related.length > 0) lines.push(` Related: ${page.related.join(", ")}`);
|
|
5882
|
+
lines.push("");
|
|
1997
5883
|
}
|
|
1998
|
-
const dynamicOnly = tag.onlyKind === "dynamic" || /(?:^|\s)only\s*=\s*\{/.test(attributes);
|
|
1999
|
-
issues.push({
|
|
2000
|
-
code: dynamicOnly ? "dynamic-only" : "invalid-only",
|
|
2001
|
-
index: tag.index,
|
|
2002
|
-
message: dynamicOnly ? "Dynamic Audience.only expressions cannot be projected consistently. Use the static literal `only=\"human\"` or `only=\"agent\"`." : "Audience.only must be the static literal `\"human\"` or `\"agent\"`."
|
|
2003
|
-
});
|
|
2004
5884
|
}
|
|
2005
|
-
return
|
|
5885
|
+
return lines.join("\n").replace(/\n{3,}/g, "\n\n");
|
|
2006
5886
|
}
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
function resolveDocsAudienceMdxContent(content, audience) {
|
|
2016
|
-
const scopes = [];
|
|
2017
|
-
let output = "";
|
|
2018
|
-
let cursor = 0;
|
|
2019
|
-
let resolvedTag = false;
|
|
2020
|
-
for (const tag of findDocsAudienceMdxTags(content)) {
|
|
2021
|
-
const activeScope = scopes.at(-1);
|
|
2022
|
-
if (tag.closing && activeScope?.name !== tag.name) continue;
|
|
2023
|
-
resolvedTag = true;
|
|
2024
|
-
const visibleBeforeTag = isAudienceVisible(scopes, audience);
|
|
2025
|
-
if (visibleBeforeTag) output += content.slice(cursor, tag.index);
|
|
2026
|
-
cursor = tag.end;
|
|
2027
|
-
if (tag.closing) {
|
|
2028
|
-
if (tag.expression && visibleBeforeTag) output += "</>";
|
|
2029
|
-
scopes.pop();
|
|
2030
|
-
continue;
|
|
2031
|
-
}
|
|
2032
|
-
if (tag.selfClosing) {
|
|
2033
|
-
if (tag.expression && visibleBeforeTag) output += tag.jsxChild ? "{null}" : "null";
|
|
2034
|
-
continue;
|
|
2035
|
-
}
|
|
2036
|
-
scopes.push({
|
|
2037
|
-
name: tag.name,
|
|
2038
|
-
only: tag.only
|
|
2039
|
-
});
|
|
2040
|
-
if (tag.expression && visibleBeforeTag) output += isAudienceVisible(scopes, audience) ? "<>" : tag.jsxChild ? "{null}" : "null";
|
|
5887
|
+
function readDocsSitemapManifestFromContentMap(contentMap) {
|
|
5888
|
+
if (!contentMap) return null;
|
|
5889
|
+
const raw = contentMap[`/${DEFAULT_SITEMAP_MANIFEST_PATH}`] ?? contentMap[DEFAULT_SITEMAP_MANIFEST_PATH];
|
|
5890
|
+
if (!raw) return null;
|
|
5891
|
+
try {
|
|
5892
|
+
return JSON.parse(raw);
|
|
5893
|
+
} catch {
|
|
5894
|
+
return null;
|
|
2041
5895
|
}
|
|
2042
|
-
if (isAudienceVisible(scopes, audience)) output += content.slice(cursor);
|
|
2043
|
-
return resolvedTag ? normalizeProjectedWhitespace(output) : content;
|
|
2044
5896
|
}
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
5897
|
+
function resolveDocsSitemapPageLastmod(manifest, url) {
|
|
5898
|
+
if (!manifest) return void 0;
|
|
5899
|
+
const normalizedUrl = normalizeUrlPath(url);
|
|
5900
|
+
return normalizeDateOnly(manifest.pages.find((entry) => normalizeUrlPath(entry.url) === normalizedUrl)?.lastmod);
|
|
5901
|
+
}
|
|
5902
|
+
function createDocsSitemapResponse({ request, apiRoute, sitemap, entry = "docs", siteTitle, baseUrl, pages, manifest }) {
|
|
5903
|
+
const url = new URL(request.url);
|
|
5904
|
+
const format = resolveDocsSitemapRequest(url, sitemap, { apiRoute });
|
|
5905
|
+
if (!format) return null;
|
|
5906
|
+
const resolved = resolveDocsSitemapConfig(sitemap, { baseUrl });
|
|
5907
|
+
if (!resolved.enabled) return null;
|
|
5908
|
+
if (format === "xml" && !resolved.xml.enabled) return null;
|
|
5909
|
+
if (format === "markdown" && !resolved.markdown.enabled) return null;
|
|
5910
|
+
const nextManifest = manifest ?? buildDocsSitemapManifest({
|
|
5911
|
+
pages,
|
|
5912
|
+
entry,
|
|
5913
|
+
siteTitle,
|
|
5914
|
+
baseUrl: resolved.baseUrl ?? url.origin
|
|
5915
|
+
});
|
|
5916
|
+
const responseBaseUrl = resolved.baseUrl ?? nextManifest.baseUrl ?? url.origin;
|
|
5917
|
+
const body = format === "xml" ? renderDocsSitemapXml(nextManifest, {
|
|
5918
|
+
baseUrl: responseBaseUrl,
|
|
5919
|
+
includeLastmod: resolved.xml.includeLastmod
|
|
5920
|
+
}) : renderDocsSitemapMarkdown(nextManifest, {
|
|
5921
|
+
baseUrl: responseBaseUrl,
|
|
5922
|
+
includeDescriptions: resolved.markdown.includeDescriptions,
|
|
5923
|
+
includeLastmod: resolved.markdown.includeLastmod,
|
|
5924
|
+
linkTarget: resolved.markdown.linkTarget
|
|
5925
|
+
});
|
|
5926
|
+
const headers = new Headers({
|
|
5927
|
+
"Content-Type": format === "xml" ? "application/xml; charset=utf-8" : "text/markdown; charset=utf-8",
|
|
5928
|
+
"Cache-Control": "public, max-age=0, must-revalidate",
|
|
5929
|
+
ETag: hashString(body)
|
|
5930
|
+
});
|
|
5931
|
+
const lastModified = newestLastmod(nextManifest.pages);
|
|
5932
|
+
if (lastModified) headers.set("Last-Modified", (/* @__PURE__ */ new Date(`${lastModified}T00:00:00.000Z`)).toUTCString());
|
|
5933
|
+
if (request.headers.get("if-none-match") === headers.get("ETag")) return new Response(null, {
|
|
5934
|
+
status: 304,
|
|
5935
|
+
headers
|
|
5936
|
+
});
|
|
5937
|
+
return new Response(body, { headers });
|
|
2048
5938
|
}
|
|
2049
5939
|
|
|
2050
5940
|
//#endregion
|
|
@@ -2319,6 +6209,7 @@ function buildDocsDiagnostics(config, options = {}) {
|
|
|
2319
6209
|
const i18n = options.i18n ?? null;
|
|
2320
6210
|
const localesEnabled = Boolean(i18n?.locales.length);
|
|
2321
6211
|
const search = resolveDocsDiagnosticsSearch(input.search, staticExport);
|
|
6212
|
+
const contentChanges = resolveDocsContentChangesConfig((isPlainObject(input.agent) ? input.agent : void 0)?.contentChanges, { staticExport });
|
|
2322
6213
|
const ai = resolveDocsDiagnosticsAi(input.ai, staticExport);
|
|
2323
6214
|
const llms = resolveDocsDiagnosticsLlms(input.llmsTxt);
|
|
2324
6215
|
const apiCatalog = resolveDocsDiagnosticsApiCatalog(input.llmsTxt, staticExport, options.apiCatalog);
|
|
@@ -2387,6 +6278,7 @@ function buildDocsDiagnostics(config, options = {}) {
|
|
|
2387
6278
|
skill: apiQueryRoute("format=skill"),
|
|
2388
6279
|
search: search.enabled ? apiQueryRoute("query={query}") : null,
|
|
2389
6280
|
agentSearch: search.enabled ? apiQueryRoute("query={query}&audience=agent") : null,
|
|
6281
|
+
contentChanges: contentChanges.enabled ? apiQueryRoute(`audience=agent&response=${DOCS_CONTENT_CHANGES_RESPONSE_VALUE}`) : null,
|
|
2390
6282
|
askAi: ai.enabled ? apiRoute : null,
|
|
2391
6283
|
mcp: mcp.enabled ? mcp.route : null,
|
|
2392
6284
|
llmsTxt: llms.enabled ? DEFAULT_LLMS_TXT_ROUTE : null,
|
|
@@ -2420,10 +6312,12 @@ function buildDocsDiagnostics(config, options = {}) {
|
|
|
2420
6312
|
routes: {
|
|
2421
6313
|
human: search.enabled ? apiQueryRoute("query={query}") : null,
|
|
2422
6314
|
agent: search.enabled ? apiQueryRoute("query={query}&audience=agent") : null,
|
|
2423
|
-
structuredAgent: search.enabled ? apiQueryRoute("query={query}&audience=agent&response=structured") : null
|
|
6315
|
+
structuredAgent: search.enabled ? apiQueryRoute("query={query}&audience=agent&response=structured") : null,
|
|
6316
|
+
facets: search.enabled ? apiQueryRoute("audience=agent&response=facets") : null
|
|
2424
6317
|
},
|
|
2425
6318
|
agentEndpoint: search.enabled ? apiQueryRoute("query={query}&audience=agent") : null,
|
|
2426
6319
|
structuredAgentEndpoint: search.enabled ? apiQueryRoute("query={query}&audience=agent&response=structured") : null,
|
|
6320
|
+
facetsEndpoint: search.enabled ? apiQueryRoute("audience=agent&response=facets") : null,
|
|
2427
6321
|
audienceParam: "audience",
|
|
2428
6322
|
defaultAudience: "human",
|
|
2429
6323
|
supportedAudiences: ["human", "agent"],
|
|
@@ -2434,7 +6328,10 @@ function buildDocsDiagnostics(config, options = {}) {
|
|
|
2434
6328
|
tags: "tags"
|
|
2435
6329
|
},
|
|
2436
6330
|
responseFormat: "docs-search.v1",
|
|
6331
|
+
facetsResponseFormat: "docs-search-facets.v1",
|
|
2437
6332
|
warningsField: "warnings",
|
|
6333
|
+
facetParam: "facet",
|
|
6334
|
+
limitParam: "limit",
|
|
2438
6335
|
cursorParam: "cursor",
|
|
2439
6336
|
nextCursorField: "nextCursor",
|
|
2440
6337
|
hasMoreField: "hasMore",
|
|
@@ -2442,6 +6339,19 @@ function buildDocsDiagnostics(config, options = {}) {
|
|
|
2442
6339
|
provider: search.provider,
|
|
2443
6340
|
transport: "GET"
|
|
2444
6341
|
},
|
|
6342
|
+
contentChanges: {
|
|
6343
|
+
status: contentChanges.enabled ? "enabled" : "disabled",
|
|
6344
|
+
reason: staticExport ? "Runtime content-change feeds are unavailable in static exports." : contentChanges.enabled ? void 0 : "configured-disabled",
|
|
6345
|
+
route: contentChanges.enabled ? apiQueryRoute(`audience=agent&response=${DOCS_CONTENT_CHANGES_RESPONSE_VALUE}`) : null,
|
|
6346
|
+
transport: "GET",
|
|
6347
|
+
format: DOCS_CONTENT_CHANGES_FORMAT,
|
|
6348
|
+
audienceParam: "audience",
|
|
6349
|
+
defaultAudience: "agent",
|
|
6350
|
+
supportedAudiences: ["human", "agent"],
|
|
6351
|
+
sinceParam: "since",
|
|
6352
|
+
generationField: "indexGeneration",
|
|
6353
|
+
resetRequiredField: "resetRequired"
|
|
6354
|
+
},
|
|
2445
6355
|
ai: {
|
|
2446
6356
|
status: ai.enabled ? "enabled" : "disabled",
|
|
2447
6357
|
reason: ai.reason,
|
|
@@ -3915,7 +7825,7 @@ function renderDocsMarkdownNotFound({ entry = "docs", apiRoute, requestedPath, o
|
|
|
3915
7825
|
lines.push(`- [${match.title}](${match.markdownUrl}) (${confidence} confidence)${match.description ? ` - ${match.description}` : ""}`);
|
|
3916
7826
|
}
|
|
3917
7827
|
}
|
|
3918
|
-
lines.push("", "## Recovery", "", "- If a closest match looks right, fetch that markdown URL directly.", "- If the match is uncertain, search the docs first and then fetch the smallest page that answers the task.", "- Use the sitemap routes below to browse the full docs index before guessing another slug.", "", "## Discovery Routes", "", "Use these discovery routes to find the right page:", "", `- Agent discovery spec: \`${DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE}\``, `- Agent discovery fallback: \`${DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE}\``, `- Agent discovery API: \`${agentSpecApiRoute}\``, `- API catalog: \`${DEFAULT_API_CATALOG_ROUTE}\``, `- Agent Skills index: \`${DEFAULT_AGENT_SKILLS_INDEX_ROUTE}\``, `- Agent instructions: \`${DEFAULT_AGENTS_MD_ROUTE}\``, `- Structured agent search: \`${resolvedApiRoute}?query={query}&audience=agent&response=structured\` (optionally repeat \`framework\`, \`version\`, \`package\`, or \`tags\`).`, `- Docs index markdown: \`/${normalizedEntry}.md\``, `- Requested markdown API route: \`${requestedApiRoute}\``);
|
|
7828
|
+
lines.push("", "## Recovery", "", "- If a closest match looks right, fetch that markdown URL directly.", "- If the match is uncertain, search the docs first and then fetch the smallest page that answers the task.", "- Use the sitemap routes below to browse the full docs index before guessing another slug.", "", "## Discovery Routes", "", "Use these discovery routes to find the right page:", "", `- Agent discovery spec: \`${DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE}\``, `- Agent discovery fallback: \`${DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE}\``, `- Agent discovery API: \`${agentSpecApiRoute}\``, `- API catalog: \`${DEFAULT_API_CATALOG_ROUTE}\``, `- Agent Skills index: \`${DEFAULT_AGENT_SKILLS_INDEX_ROUTE}\``, `- Agent instructions: \`${DEFAULT_AGENTS_MD_ROUTE}\``, `- Search scope facets: \`${resolvedApiRoute}?audience=agent&response=facets\` (discover valid \`framework\`, \`version\`, \`package\`, and \`tags\` values without page bodies; continue large fields with \`facet={field}&limit={limit}&cursor={nextCursor}\`).`, `- Structured agent search: \`${resolvedApiRoute}?query={query}&audience=agent&response=structured\` (optionally repeat \`framework\`, \`version\`, \`package\`, or \`tags\`).`, `- Docs index markdown: \`/${normalizedEntry}.md\``, `- Requested markdown API route: \`${requestedApiRoute}\``);
|
|
3919
7829
|
if (sitemapConfig.enabled) {
|
|
3920
7830
|
if (sitemapConfig.markdown.enabled) {
|
|
3921
7831
|
lines.push(`- Semantic sitemap: \`${sitemapConfig.markdown.route}\``);
|
|
@@ -4136,7 +8046,7 @@ function shouldRenderLlmsDirective(options) {
|
|
|
4136
8046
|
if (options?.llms && typeof options.llms === "object" && options.llms.enabled === false) return false;
|
|
4137
8047
|
return true;
|
|
4138
8048
|
}
|
|
4139
|
-
function resolveDocsAgentDocumentContext({ entry = "docs", apiRoute, apiCatalog: explicitApiCatalog, search, mcp, feedback, llms, sitemap, robots, openapi, markdown }) {
|
|
8049
|
+
function resolveDocsAgentDocumentContext({ entry = "docs", apiRoute, apiCatalog: explicitApiCatalog, search, contentChanges, mcp, feedback, llms, sitemap, robots, openapi, markdown }) {
|
|
4140
8050
|
const feedbackRoute = feedback?.route ?? DEFAULT_AGENT_FEEDBACK_ROUTE;
|
|
4141
8051
|
const resolvedApiRoute = resolveDocsDiscoveryApiRoute(apiRoute);
|
|
4142
8052
|
const openapiConfig = resolveDocsOpenApiDiscoveryConfig(openapi);
|
|
@@ -4147,6 +8057,7 @@ function resolveDocsAgentDocumentContext({ entry = "docs", apiRoute, apiCatalog:
|
|
|
4147
8057
|
siteDescription: llms?.siteDescription ? compactSkillText(llms.siteDescription) : void 0,
|
|
4148
8058
|
llmsEnabled: llms?.enabled ?? true,
|
|
4149
8059
|
searchEnabled: isSearchEnabled(search),
|
|
8060
|
+
contentChangesEnabled: resolveDocsContentChangesConfig(contentChanges).enabled,
|
|
4150
8061
|
mcpEnabled: mcp.enabled,
|
|
4151
8062
|
feedbackEnabled: feedback?.enabled ?? false,
|
|
4152
8063
|
sitemapConfig: resolveDocsSitemapConfig(sitemap),
|
|
@@ -4170,7 +8081,11 @@ function appendDocsMarkdownNegotiationStartLines(lines, context, variant) {
|
|
|
4170
8081
|
}
|
|
4171
8082
|
function appendDocsSearchStartLine(lines, context, variant) {
|
|
4172
8083
|
if (!context.searchEnabled) return;
|
|
4173
|
-
lines.push(variant === "skill" ? `- Search with ${context.apiRoute}?query={query}&audience=agent&response=structured when you do not know the page; optionally repeat framework, version, package, or tags filters.` : `- Search with ${context.apiRoute}?query={query}&audience=agent&response=structured when the route is unknown; optionally repeat framework, version, package, or tags filters.`);
|
|
8084
|
+
lines.push(`- Discover valid search filters with ${context.apiRoute}?audience=agent&response=facets before choosing framework, version, package, or tags values; continue a large field with facet={field}, limit={limit}, and its nextCursor.`, variant === "skill" ? `- Search with ${context.apiRoute}?query={query}&audience=agent&response=structured when you do not know the page; optionally repeat framework, version, package, or tags filters.` : `- Search with ${context.apiRoute}?query={query}&audience=agent&response=structured when the route is unknown; optionally repeat framework, version, package, or tags filters.`);
|
|
8085
|
+
}
|
|
8086
|
+
function appendDocsContentChangesStartLine(lines, context) {
|
|
8087
|
+
if (!context.contentChangesEnabled) return;
|
|
8088
|
+
lines.push(`- Synchronize changed documents with ${context.apiRoute}?audience=agent&response=${DOCS_CONTENT_CHANGES_RESPONSE_VALUE}; pass the returned indexGeneration as since on the next request and clear stale state when resetRequired is true.`);
|
|
4174
8089
|
}
|
|
4175
8090
|
function appendDocsOpenApiStartLine(lines, context, variant) {
|
|
4176
8091
|
if (!context.openapiConfig.enabled || !context.openapiConfig.url) return;
|
|
@@ -4212,6 +8127,7 @@ function appendDocsAgentStartHereLines(lines, context, variant) {
|
|
|
4212
8127
|
if (variant === "skill") {
|
|
4213
8128
|
appendDocsMarkdownNegotiationStartLines(lines, context, variant);
|
|
4214
8129
|
appendDocsSearchStartLine(lines, context, variant);
|
|
8130
|
+
appendDocsContentChangesStartLine(lines, context);
|
|
4215
8131
|
appendDocsOpenApiStartLine(lines, context, variant);
|
|
4216
8132
|
appendDocsLlmsStartLines(lines, context, variant);
|
|
4217
8133
|
appendDocsSitemapStartLines(lines, context, variant);
|
|
@@ -4224,6 +8140,7 @@ function appendDocsAgentStartHereLines(lines, context, variant) {
|
|
|
4224
8140
|
appendDocsSitemapStartLines(lines, context, variant);
|
|
4225
8141
|
appendDocsRobotsStartLine(lines, context, variant);
|
|
4226
8142
|
appendDocsSearchStartLine(lines, context, variant);
|
|
8143
|
+
appendDocsContentChangesStartLine(lines, context);
|
|
4227
8144
|
appendDocsOpenApiStartLine(lines, context, variant);
|
|
4228
8145
|
appendDocsMcpStartLine(lines, context, variant);
|
|
4229
8146
|
appendDocsFeedbackStartLine(lines, context, variant);
|
|
@@ -4262,6 +8179,7 @@ function appendDocsAgentPublicRouteLines(lines, context, variant) {
|
|
|
4262
8179
|
appendDocsOpenApiRouteLines(lines, context);
|
|
4263
8180
|
appendDocsSitemapRouteLines(lines, context);
|
|
4264
8181
|
appendDocsMcpRouteLines(lines, context);
|
|
8182
|
+
if (context.contentChangesEnabled) lines.push(`- Content changes: ${context.apiRoute}?audience=agent&response=${DOCS_CONTENT_CHANGES_RESPONSE_VALUE}`);
|
|
4265
8183
|
return;
|
|
4266
8184
|
}
|
|
4267
8185
|
lines.push(`- Agent instructions: ${DEFAULT_AGENTS_MD_ROUTE}`, `- Agent instructions well-known alias: ${DEFAULT_AGENTS_MD_WELL_KNOWN_ROUTE}`, `- Agent instructions API format: ${context.apiRoute}?format=agents`, `- Agent instructions aliases: ${DEFAULT_AGENT_MD_ROUTE}, ${DEFAULT_AGENT_MD_WELL_KNOWN_ROUTE}`, `- Site skill: ${DEFAULT_SKILL_MD_ROUTE}`, `- Site skill well-known alias: ${DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE}`, `- Site skill API format: ${context.apiRoute}?format=skill`, `- Markdown root: /${context.normalizedEntry}.md`, `- Markdown pages: /${context.normalizedEntry}/{slug}.md`, `- Agent discovery: ${DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE}`, `- Agent discovery fallback: ${DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE}`);
|
|
@@ -4273,6 +8191,7 @@ function appendDocsAgentPublicRouteLines(lines, context, variant) {
|
|
|
4273
8191
|
appendDocsSitemapRouteLines(lines, context);
|
|
4274
8192
|
appendDocsOpenApiRouteLines(lines, context);
|
|
4275
8193
|
appendDocsMcpRouteLines(lines, context);
|
|
8194
|
+
if (context.contentChangesEnabled) lines.push(`- Content changes: ${context.apiRoute}?audience=agent&response=${DOCS_CONTENT_CHANGES_RESPONSE_VALUE}`);
|
|
4276
8195
|
}
|
|
4277
8196
|
function renderDocsMarkdownDocument(page, options) {
|
|
4278
8197
|
const explicitAgentContent = page.agentRawContent ?? page.agentContent;
|
|
@@ -4334,12 +8253,13 @@ function resolveDocsAgentContractMcpTools(mcp) {
|
|
|
4334
8253
|
if (mcp.tools.readTask !== false) tools.read = "read_task";
|
|
4335
8254
|
return tools.list || tools.read ? tools : void 0;
|
|
4336
8255
|
}
|
|
4337
|
-
function buildDocsAgentDiscoverySpec({ origin, entry = "docs", apiRoute, apiCatalog: explicitApiCatalog, i18n = null, search, mcp, feedback, llms, sitemap, robots, openapi, markdown, publishedSkills = [], agentCard }) {
|
|
8256
|
+
function buildDocsAgentDiscoverySpec({ origin, entry = "docs", apiRoute, apiCatalog: explicitApiCatalog, i18n = null, search, contentChanges, mcp, feedback, llms, sitemap, robots, openapi, markdown, publishedSkills = [], agentCard }) {
|
|
4338
8257
|
const normalizedEntry = normalizeDocsPathSegment(entry) || "docs";
|
|
4339
8258
|
const resolvedApiRoute = resolveDocsDiscoveryApiRoute(apiRoute);
|
|
4340
8259
|
const apiQueryRoute = (query) => `${resolvedApiRoute}?${query}`;
|
|
4341
8260
|
const localesEnabled = i18n !== null;
|
|
4342
8261
|
const searchEnabled = isSearchEnabled(search);
|
|
8262
|
+
const contentChangesEnabled = resolveDocsContentChangesConfig(contentChanges).enabled;
|
|
4343
8263
|
const feedbackRoute = feedback?.route ?? DEFAULT_AGENT_FEEDBACK_ROUTE;
|
|
4344
8264
|
const feedbackSchemaRoute = feedback?.schemaRoute ?? `${feedbackRoute}/schema`;
|
|
4345
8265
|
const llmsEnabled = llms?.enabled ?? true;
|
|
@@ -4386,6 +8306,7 @@ function buildDocsAgentDiscoverySpec({ origin, entry = "docs", apiRoute, apiCata
|
|
|
4386
8306
|
agentSkillsDiscovery: true,
|
|
4387
8307
|
mcp: mcp.enabled,
|
|
4388
8308
|
search: searchEnabled,
|
|
8309
|
+
contentChanges: contentChangesEnabled,
|
|
4389
8310
|
sitemap: sitemapConfig.enabled,
|
|
4390
8311
|
robots: robotsEnabled,
|
|
4391
8312
|
structuredData: true,
|
|
@@ -4412,7 +8333,8 @@ function buildDocsAgentDiscoverySpec({ origin, entry = "docs", apiRoute, apiCata
|
|
|
4412
8333
|
agentSkillsIndex: DEFAULT_AGENT_SKILLS_INDEX_ROUTE,
|
|
4413
8334
|
legacySkillsIndex: DEFAULT_LEGACY_SKILLS_INDEX_ROUTE,
|
|
4414
8335
|
...agentCard ? { agentCard: DEFAULT_A2A_AGENT_CARD_ROUTE } : {},
|
|
4415
|
-
openapi: defaultOpenapiRoute
|
|
8336
|
+
openapi: defaultOpenapiRoute,
|
|
8337
|
+
...contentChangesEnabled ? { contentChanges: apiQueryRoute(`audience=agent&response=${DOCS_CONTENT_CHANGES_RESPONSE_VALUE}`) } : {}
|
|
4416
8338
|
},
|
|
4417
8339
|
apiCatalog: {
|
|
4418
8340
|
enabled: apiCatalogEnabled,
|
|
@@ -4530,6 +8452,7 @@ function buildDocsAgentDiscoverySpec({ origin, entry = "docs", apiRoute, apiCata
|
|
|
4530
8452
|
endpoint: apiQueryRoute("query={query}"),
|
|
4531
8453
|
agentEndpoint: apiQueryRoute("query={query}&audience=agent"),
|
|
4532
8454
|
structuredAgentEndpoint: apiQueryRoute("query={query}&audience=agent&response=structured"),
|
|
8455
|
+
facetsEndpoint: apiQueryRoute("audience=agent&response=facets"),
|
|
4533
8456
|
method: "GET",
|
|
4534
8457
|
queryParam: "query",
|
|
4535
8458
|
localeParam: "lang",
|
|
@@ -4538,7 +8461,9 @@ function buildDocsAgentDiscoverySpec({ origin, entry = "docs", apiRoute, apiCata
|
|
|
4538
8461
|
supportedAudiences: ["human", "agent"],
|
|
4539
8462
|
responseParam: "response",
|
|
4540
8463
|
structuredResponseValue: "structured",
|
|
8464
|
+
facetsResponseValue: "facets",
|
|
4541
8465
|
responseFormat: "docs-search.v1",
|
|
8466
|
+
facetsResponseFormat: "docs-search-facets.v1",
|
|
4542
8467
|
filterParams: {
|
|
4543
8468
|
framework: "framework",
|
|
4544
8469
|
version: "version",
|
|
@@ -4552,11 +8477,34 @@ function buildDocsAgentDiscoverySpec({ origin, entry = "docs", apiRoute, apiCata
|
|
|
4552
8477
|
"tags"
|
|
4553
8478
|
],
|
|
4554
8479
|
warningsField: "warnings",
|
|
8480
|
+
facetParam: "facet",
|
|
8481
|
+
limitParam: "limit",
|
|
4555
8482
|
cursorParam: "cursor",
|
|
4556
8483
|
nextCursorField: "nextCursor",
|
|
4557
8484
|
hasMoreField: "hasMore",
|
|
4558
8485
|
totalField: "total"
|
|
4559
8486
|
},
|
|
8487
|
+
contentChanges: {
|
|
8488
|
+
enabled: contentChangesEnabled,
|
|
8489
|
+
endpoint: contentChangesEnabled ? apiQueryRoute(`audience=agent&response=${DOCS_CONTENT_CHANGES_RESPONSE_VALUE}`) : null,
|
|
8490
|
+
method: "GET",
|
|
8491
|
+
audienceParam: "audience",
|
|
8492
|
+
defaultAudience: "agent",
|
|
8493
|
+
supportedAudiences: ["human", "agent"],
|
|
8494
|
+
responseParam: "response",
|
|
8495
|
+
responseValue: DOCS_CONTENT_CHANGES_RESPONSE_VALUE,
|
|
8496
|
+
sinceParam: "since",
|
|
8497
|
+
format: DOCS_CONTENT_CHANGES_FORMAT,
|
|
8498
|
+
generationField: "indexGeneration",
|
|
8499
|
+
resetRequiredField: "resetRequired",
|
|
8500
|
+
modes: [
|
|
8501
|
+
"snapshot",
|
|
8502
|
+
"delta",
|
|
8503
|
+
"reset"
|
|
8504
|
+
],
|
|
8505
|
+
bodyFree: true,
|
|
8506
|
+
etag: true
|
|
8507
|
+
},
|
|
4560
8508
|
agents: {
|
|
4561
8509
|
enabled: true,
|
|
4562
8510
|
file: "AGENTS.md",
|
|
@@ -4730,4 +8678,4 @@ function toYamlString(value) {
|
|
|
4730
8678
|
}
|
|
4731
8679
|
|
|
4732
8680
|
//#endregion
|
|
4733
|
-
export { matchesDocsLlmsTxtSection as $,
|
|
8681
|
+
export { matchesDocsLlmsTxtSection as $, resolveDocsMcpResourceLocation as $n, buildDocsSearchDocuments as $t, buildDocsAgentDiscoverySpec as A, normalizeAgentScopeValues as An, DEFAULT_SITEMAP_XML_ROUTE as At, findDocsMarkdownPage as B, DEFAULT_MCP_ROUTE as Bn, DOCS_CONTENT_CHANGES_FORMAT as Bt, DOCS_AI_AGENT_USER_AGENT_HEADER_PATTERN as C, resolveDocsAudienceExposure as Cn, selectDocsMarkdownSection as Ct, DOCS_MARKDOWN_SIGNATURE_AGENT_HEADER as D, agentVersionConstraintsOverlap as Dn, DEFAULT_SITEMAP_MD_DOCS_ROUTE as Dt, DOCS_MARKDOWN_SECTION_INDEX_FORMAT as E, agentVersionConstraintMatches as En, DEFAULT_SITEMAP_MANIFEST_PATH as Et, buildDocsMcpEndpointCandidates as F, isDocsRetrievalCanonicalUrl as Fn, renderDocsSitemapXml as Ft, isDocsAgentDiscoveryRequest as G, hasDocsMcpProtectedResourceConfig as Gn, createDocsContentChangesHttpResponse as Gt, getDocsMarkdownCanonicalLinkHeader as H, buildDocsMcpProtectedResourceMetadataRoute as Hn, DOCS_CONTENT_SNAPSHOT_FORMAT as Ht, collectDocsMarkdownSections as I, normalizeDocsRelated as In, resolveDocsSitemapConfig as It, isDocsDiagnosticsRequest as J, isDocsMcpRequest as Jn, resolveDocsContentChangesRequest as Jt, isDocsAgentsRequest as K, isDocsMcpOAuthScopeToken as Kn, isDocsContentChangesRequest as Kt, createDocsMarkdownResponse as L, renderDocsRelatedMarkdownLines as Ln, resolveDocsSitemapPageLastmod as Lt, buildDocsConfigMap as M, DocsPaginationCursorError as Mn, createDocsSitemapResponse as Mt, buildDocsDiagnostics as N, paginateDocsItems as Nn, readDocsSitemapManifestFromContentMap as Nt, DOCS_TRADITIONAL_BOT_USER_AGENT_HEADER_PATTERN as O, normalizeAgentFramework as On, DEFAULT_SITEMAP_MD_ROUTE as Ot, buildDocsMarkdownSectionIndex as P, digestDocsRetrievalContent as Pn, renderDocsSitemapMarkdown as Pt, isDocsSkillRequest as Q, resolveDocsMcpProtectedResourceMetadataLocation as Qn, buildDocsRetrievalDigestProjection as Qt, createDocsStandardsDiscoveryResponse as R, DEFAULT_MCP_PROTECTED_RESOURCE_METADATA_ROUTE as Rn, resolveDocsSitemapRequest as Rt, DEFAULT_SKILL_MD_WELL_KNOWN_ROUTE as S, resolveDocsAgentMdxContent as Sn, selectDocsLlmsTxtContent as St, DOCS_CONFIG_MAP_TOP_LEVEL_KEYS as T, agentVersionConstraintGroupsOverlap as Tn, validateDocsAgentFeedbackPayload as Tt, getDocsMarkdownVaryHeader as U, getDocsMcpProtectedResourceMetadataRoutes as Un, DocsContentChangesRequestError as Ut, getDocsLlmsTxtMaxCharsIssue as V, DEFAULT_MCP_WELL_KNOWN_ROUTE as Vn, DOCS_CONTENT_CHANGES_RESPONSE_VALUE as Vt, hasDocsMarkdownSignatureAgent as W, getDocsMcpResourcePaths as Wn, createDocsContentChangeFeed as Wt, isDocsMarkdownSectionIndexRequest as X, normalizeDocsMcpAuthorizationServerUrls as Xn, buildDocsAskAIContext as Xt, isDocsLlmsTxtPublicRequest as Y, isDocsMcpResourcePath as Yn, DocsSearchRequestError as Yt, isDocsPublicGetRequest as Z, normalizeDocsMcpEndpointPath as Zn, buildDocsContentSnapshot as Zt, DEFAULT_LLMS_TXT_MAX_CHARS as _, resolveDocsSearchRequest as _n, resolveDocsMarkdownRequest as _t, DEFAULT_AGENT_MD_ROUTE as a, createTypesenseSearchAdapter as an, renderDocsMarkdownDocument as at, DEFAULT_OPENAPI_SCHEMA_ROUTE as b, findDocsAudienceMdxIssues as bn, resolveDocsRequestApiRoute as bt, DEFAULT_AGENT_SPEC_WELL_KNOWN_JSON_ROUTE as c, inferDocsAskAIPackageHints as cn, resolveDocsAgentContractMcpTools as ct, DEFAULT_DOCS_CONFIG_FORMAT as d, performDocsSearchWithMetadata as dn, resolveDocsAgentsFormat as dt, buildDocsSearchFacets as en, normalizeDocsPathSegment as et, DEFAULT_DOCS_CONFIG_ROUTE as f, resolveAskAISearchRequestConfig as fn, resolveDocsLlmsTxtFormat as ft, DEFAULT_LLMS_FULL_TXT_WELL_KNOWN_ROUTE as g, resolveDocsSearchFilters as gn, resolveDocsMarkdownRecovery as gt, DEFAULT_LLMS_FULL_TXT_ROUTE as h, resolveDocsSearchError as hn, resolveDocsMarkdownCanonicalUrl as ht, DEFAULT_AGENT_FEEDBACK_ROUTE as i, createSimpleSearchAdapter as in, renderDocsLlmsTxt as it, buildDocsAgentFeedbackSchema as j, normalizeAgentVersion as jn, buildDocsSitemapManifest as jt, acceptsDocsMarkdown as k, normalizeAgentLocale as kn, DEFAULT_SITEMAP_MD_WELL_KNOWN_ROUTE as kt, DEFAULT_AGENT_SPEC_WELL_KNOWN_ROUTE as l, normalizeDocsSearchFilters as ln, resolveDocsAgentFeedbackConfig as lt, DEFAULT_DOCS_DIAGNOSTICS_ROUTE as m, resolveDocsSearchAudience as mn, resolveDocsLlmsTxtSections as mt, DEFAULT_AGENTS_MD_WELL_KNOWN_ROUTE as n, createCustomSearchAdapter as nn, parseDocsAgentFeedbackData as nt, DEFAULT_AGENT_MD_WELL_KNOWN_ROUTE as o, enrichDocsSearchDocumentsWithProvenance as on, renderDocsMarkdownNotFound as ot, DEFAULT_DOCS_DIAGNOSTICS_FORMAT as p, resolveDocsRetrievalLastModified as pn, resolveDocsLlmsTxtRequest as pt, isDocsConfigRequest as q, isDocsMcpProtectedResourceMetadataPath as qn, resolveDocsContentChangesConfig as qt, DEFAULT_AGENT_FEEDBACK_PAYLOAD_SCHEMA as r, createMcpSearchAdapter as rn, renderDocsAgentsDocument as rt, DEFAULT_AGENT_SPEC_ROUTE as s, formatDocsAskAIPackageHints as sn, renderDocsSkillDocument as st, DEFAULT_AGENTS_MD_ROUTE as t, createAlgoliaSearchAdapter as tn, normalizeDocsUrlPath as tt, DEFAULT_DOCS_API_ROUTE as u, performDocsSearch as un, resolveDocsAgentFeedbackRequest as ut, DEFAULT_LLMS_TXT_ROUTE as v, resolveSearchRequestConfig as vn, resolveDocsMarkdownSectionRequest as vt, DOCS_BOT_LIKE_USER_AGENT_HEADER_PATTERN as w, resolveDocsAudienceMdxContent as wn, toDocsMarkdownUrl as wt, DEFAULT_SKILL_MD_ROUTE as x, findDocsAudienceMdxTags as xn, resolveDocsSkillFormat as xt, DEFAULT_LLMS_TXT_WELL_KNOWN_ROUTE as y, extractDocsMarkdownPromptBlocks as yn, resolveDocsOpenApiDiscoveryConfig as yt, detectDocsMarkdownAgentRequest as z, DEFAULT_MCP_PUBLIC_ROUTE as zn, toDocsSitemapMarkdownUrl as zt };
|