@farming-labs/docs 0.2.72 → 0.2.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{agent-pS2nNQS3.mjs → agent-BJ0KXN-A.mjs} +583 -765
- package/dist/{agent-DbDfIqhe.mjs → agent-D0LEY2Vy.mjs} +4 -3
- package/dist/{agent-evals-CjYzCQMZ.mjs → agent-evals-LIScgQVm.mjs} +86 -43
- package/dist/{agent-export-CpQhD8eA.mjs → agent-export-B92pNe6g.mjs} +12 -11
- package/dist/agent-skills-bundle.d.mts +1 -1
- package/dist/{agent-skills-server-DZnNB4_S.d.mts → agent-skills-server-BqvIGQb1.d.mts} +2 -2
- package/dist/{agent-skills-server-BPVv4gMg.mjs → agent-skills-server-noFn1l_p.mjs} +1 -1
- package/dist/agent-skills-vite.d.mts +3 -3
- package/dist/agent-skills-vite.mjs +2 -2
- package/dist/{agents-B-1h9Jlm.mjs → agents-C228QGbp.mjs} +10 -9
- package/dist/cli/index.mjs +34 -30
- package/dist/client/react.d.mts +1 -1
- package/dist/{cloud-ask-ai-D9gIN4Vr.d.mts → cloud-ask-ai-6X9pVjIY.d.mts} +1 -1
- package/dist/{cloud-CyysfDny.mjs → cloud-pdNC-tyj.mjs} +3 -3
- package/dist/{codeblocks-DAj7EAwt.mjs → codeblocks-CFuurVIH.mjs} +2 -2
- package/dist/{dev-qE7MYEh9.mjs → dev-Cc1cFFj-.mjs} +2 -2
- package/dist/docs-cloud-server.d.mts +2 -2
- package/dist/docs-cloud-server.mjs +1 -1
- package/dist/{doctor-CBy8-tPu.mjs → doctor-CgrUZ3tY.mjs} +15 -14
- package/dist/{downgrade-DCxezPZI.mjs → downgrade-w7e6Se0L.mjs} +2 -2
- package/dist/{golden-evaluations-DGg_eHIY.mjs → golden-evaluations-DTEPZeb0.mjs} +4 -3
- package/dist/index.d.mts +72 -6
- package/dist/index.mjs +9 -8
- package/dist/{init-p1MHf06G.mjs → init-DAkI0Lkf.mjs} +3 -3
- package/dist/markdown-rendering.d.mts +75 -0
- package/dist/markdown-rendering.mjs +618 -0
- package/dist/markdown-sections-7OoA7ylx.mjs +1492 -0
- package/dist/{mcp-BNK45dzD.mjs → mcp-YD37HlPY.mjs} +12 -9
- package/dist/mcp.d.mts +13 -2
- package/dist/mcp.mjs +121 -60
- package/dist/{metadata-D6qlWDu-.mjs → metadata-N252j5_a.mjs} +1 -1
- package/dist/{package-version-BXHcIm1T.mjs → package-version-n5AFur8a.mjs} +1 -1
- package/dist/{reading-time-DvTuW2Iy.mjs → prompt-references-BIS1eXt4.mjs} +158 -4
- package/dist/{search-1aftosa5.d.mts → retrieval-digest-9Kp3uSaZ.d.mts} +69 -5
- package/dist/{review-Cace7A2c.mjs → review-Dbzkdl6K.mjs} +10 -9
- package/dist/{robots-DmE_wXvG.mjs → robots-DUO0r4UI.mjs} +6 -5
- package/dist/{robots-C2SindI5.mjs → robots-RbgmSvPH.mjs} +2 -2
- package/dist/{search-BbArHODP.mjs → search-DIyIl-tE.mjs} +1288 -149
- package/dist/{search-DB3svkNQ.mjs → search-hcqaEZVN.mjs} +66 -27
- package/dist/server.d.mts +5 -5
- package/dist/server.mjs +10 -9
- package/dist/{sitemap-DPTLh_26.mjs → sitemap-jhkS_jYx.mjs} +10 -9
- package/dist/{sitemap-server-PUoOhplO.mjs → sitemap-server-BfyBM7_V.mjs} +1 -1
- package/dist/{standards-discovery-CYfiQE13.mjs → standards-discovery-BwL5ffGx.mjs} +4 -3
- package/dist/{standards-discovery-DRJ10g7B.d.mts → standards-discovery-CdrOE-05.d.mts} +5 -4
- package/dist/{types-B9DNNAdT.d.mts → types-clg5J1R4.d.mts} +60 -1
- package/dist/{upgrade-DDJQ5l6n.mjs → upgrade-oz-GChgt.mjs} +2 -2
- package/package.json +9 -1
- /package/dist/{cloud-ask-ai-hnJfj8-X.mjs → cloud-ask-ai-sbpjOR2K.mjs} +0 -0
- /package/dist/{code-blocks-qe0T8-xe.mjs → code-blocks-DnNVNK2M.mjs} +0 -0
- /package/dist/{config-z7_R0f86.mjs → config-Wcdj-D0a.mjs} +0 -0
- /package/dist/{errors-Cc_qplZU.mjs → errors-DbOhkE1h.mjs} +0 -0
- /package/dist/{i18n-CCaFUnAN.mjs → i18n-hHVWcflJ.mjs} +0 -0
- /package/dist/{templates-BJnf6B8l.mjs → templates-C6avM0U3.mjs} +0 -0
- /package/dist/{utils-C4RC8Zq_.mjs → utils-DpiIioYb.mjs} +0 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { G as isDocsAgentDiscoveryRequest, K as isDocsAgentsRequest, Q as isDocsSkillRequest, _t as resolveDocsMarkdownRequest, dt as resolveDocsAgentsFormat, jt as resolveDocsAudienceMdxContent, lt as resolveDocsAgentFeedbackConfig, pt as resolveDocsLlmsTxtRequest, ut as resolveDocsAgentFeedbackRequest, xt as resolveDocsSkillFormat } from "./agent-BJ0KXN-A.mjs";
|
|
2
|
+
import { S as upsertPageAgentContractMarkdown, a as findDocsGeneratedAgentContractRanges, d as PAGE_AGENT_CONTRACT_END_MARKER, l as stripDocsGeneratedAgentContractMarkers, m as PAGE_AGENT_CONTRACT_START_MARKER, o as findDocsMarkdownSection, s as parseDocsMarkdownSections, y as renderPageAgentContractMarkdown } from "./markdown-sections-7OoA7ylx.mjs";
|
|
2
3
|
|
|
3
4
|
//#region src/telemetry.ts
|
|
4
5
|
const DOCS_PACKAGE_NAME = "@farming-labs/docs";
|
|
@@ -457,6 +458,172 @@ function serializeGeneratedAgentDocument(content, provenance) {
|
|
|
457
458
|
].join("\n")}\n`;
|
|
458
459
|
}
|
|
459
460
|
|
|
461
|
+
//#endregion
|
|
462
|
+
//#region src/retrieval-digest.ts
|
|
463
|
+
const SHA256_INITIAL_STATE = [
|
|
464
|
+
1779033703,
|
|
465
|
+
3144134277,
|
|
466
|
+
1013904242,
|
|
467
|
+
2773480762,
|
|
468
|
+
1359893119,
|
|
469
|
+
2600822924,
|
|
470
|
+
528734635,
|
|
471
|
+
1541459225
|
|
472
|
+
];
|
|
473
|
+
const SHA256_ROUND_CONSTANTS = [
|
|
474
|
+
1116352408,
|
|
475
|
+
1899447441,
|
|
476
|
+
3049323471,
|
|
477
|
+
3921009573,
|
|
478
|
+
961987163,
|
|
479
|
+
1508970993,
|
|
480
|
+
2453635748,
|
|
481
|
+
2870763221,
|
|
482
|
+
3624381080,
|
|
483
|
+
310598401,
|
|
484
|
+
607225278,
|
|
485
|
+
1426881987,
|
|
486
|
+
1925078388,
|
|
487
|
+
2162078206,
|
|
488
|
+
2614888103,
|
|
489
|
+
3248222580,
|
|
490
|
+
3835390401,
|
|
491
|
+
4022224774,
|
|
492
|
+
264347078,
|
|
493
|
+
604807628,
|
|
494
|
+
770255983,
|
|
495
|
+
1249150122,
|
|
496
|
+
1555081692,
|
|
497
|
+
1996064986,
|
|
498
|
+
2554220882,
|
|
499
|
+
2821834349,
|
|
500
|
+
2952996808,
|
|
501
|
+
3210313671,
|
|
502
|
+
3336571891,
|
|
503
|
+
3584528711,
|
|
504
|
+
113926993,
|
|
505
|
+
338241895,
|
|
506
|
+
666307205,
|
|
507
|
+
773529912,
|
|
508
|
+
1294757372,
|
|
509
|
+
1396182291,
|
|
510
|
+
1695183700,
|
|
511
|
+
1986661051,
|
|
512
|
+
2177026350,
|
|
513
|
+
2456956037,
|
|
514
|
+
2730485921,
|
|
515
|
+
2820302411,
|
|
516
|
+
3259730800,
|
|
517
|
+
3345764771,
|
|
518
|
+
3516065817,
|
|
519
|
+
3600352804,
|
|
520
|
+
4094571909,
|
|
521
|
+
275423344,
|
|
522
|
+
430227734,
|
|
523
|
+
506948616,
|
|
524
|
+
659060556,
|
|
525
|
+
883997877,
|
|
526
|
+
958139571,
|
|
527
|
+
1322822218,
|
|
528
|
+
1537002063,
|
|
529
|
+
1747873779,
|
|
530
|
+
1955562222,
|
|
531
|
+
2024104815,
|
|
532
|
+
2227730452,
|
|
533
|
+
2361852424,
|
|
534
|
+
2428436474,
|
|
535
|
+
2756734187,
|
|
536
|
+
3204031479,
|
|
537
|
+
3329325298
|
|
538
|
+
];
|
|
539
|
+
function rotateRight(value, bits) {
|
|
540
|
+
return value >>> bits | value << 32 - bits;
|
|
541
|
+
}
|
|
542
|
+
function normalizeDigestContent(value) {
|
|
543
|
+
return value.replace(/\r\n?/gu, "\n").replace(/^\uFEFF/u, "").trimEnd();
|
|
544
|
+
}
|
|
545
|
+
/** Validate the bounded HTTP(S) URL or root-relative URI reference used by provenance. */
|
|
546
|
+
function isDocsRetrievalCanonicalUrl(value) {
|
|
547
|
+
const hasUnsafeCharacter = value.includes("\\") || Array.from(value).some((character) => {
|
|
548
|
+
const codePoint = character.codePointAt(0) ?? 0;
|
|
549
|
+
return codePoint <= 31 || codePoint === 127;
|
|
550
|
+
});
|
|
551
|
+
if (!value || value !== value.trim() || value.length > 4096 || hasUnsafeCharacter) return false;
|
|
552
|
+
if (/^\/(?!\/)/u.test(value)) try {
|
|
553
|
+
new URL(value, "https://docs.local");
|
|
554
|
+
return true;
|
|
555
|
+
} catch {
|
|
556
|
+
return false;
|
|
557
|
+
}
|
|
558
|
+
try {
|
|
559
|
+
const url = new URL(value);
|
|
560
|
+
return (url.protocol === "http:" || url.protocol === "https:") && !url.username && !url.password;
|
|
561
|
+
} catch {
|
|
562
|
+
return false;
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
/**
|
|
566
|
+
* Hash a retrieval source projection with portable SHA-256.
|
|
567
|
+
*
|
|
568
|
+
* The projection is normalized by removing one leading BOM, converting CRLF/CR
|
|
569
|
+
* line endings to LF, and trimming trailing whitespace. The helper is exported so
|
|
570
|
+
* agents can independently verify provenance in Node, edge, and browser runtimes.
|
|
571
|
+
*/
|
|
572
|
+
function digestDocsRetrievalContent(value) {
|
|
573
|
+
const input = new TextEncoder().encode(normalizeDigestContent(value));
|
|
574
|
+
const paddedLength = Math.ceil((input.length + 9) / 64) * 64;
|
|
575
|
+
const padded = new Uint8Array(paddedLength);
|
|
576
|
+
padded.set(input);
|
|
577
|
+
padded[input.length] = 128;
|
|
578
|
+
const bitLength = input.length * 8;
|
|
579
|
+
const dataView = new DataView(padded.buffer);
|
|
580
|
+
dataView.setUint32(paddedLength - 8, Math.floor(bitLength / 4294967296), false);
|
|
581
|
+
dataView.setUint32(paddedLength - 4, bitLength >>> 0, false);
|
|
582
|
+
const state = Uint32Array.from(SHA256_INITIAL_STATE);
|
|
583
|
+
const words = new Uint32Array(64);
|
|
584
|
+
for (let offset = 0; offset < padded.length; offset += 64) {
|
|
585
|
+
for (let index = 0; index < 16; index += 1) words[index] = dataView.getUint32(offset + index * 4, false);
|
|
586
|
+
for (let index = 16; index < 64; index += 1) {
|
|
587
|
+
const previous15 = words[index - 15];
|
|
588
|
+
const previous2 = words[index - 2];
|
|
589
|
+
const sigma0 = rotateRight(previous15, 7) ^ rotateRight(previous15, 18) ^ previous15 >>> 3;
|
|
590
|
+
const sigma1 = rotateRight(previous2, 17) ^ rotateRight(previous2, 19) ^ previous2 >>> 10;
|
|
591
|
+
words[index] = words[index - 16] + sigma0 + words[index - 7] + sigma1 >>> 0;
|
|
592
|
+
}
|
|
593
|
+
let a = state[0];
|
|
594
|
+
let b = state[1];
|
|
595
|
+
let c = state[2];
|
|
596
|
+
let d = state[3];
|
|
597
|
+
let e = state[4];
|
|
598
|
+
let f = state[5];
|
|
599
|
+
let g = state[6];
|
|
600
|
+
let h = state[7];
|
|
601
|
+
for (let index = 0; index < 64; index += 1) {
|
|
602
|
+
const sum1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
|
|
603
|
+
const choice = e & f ^ ~e & g;
|
|
604
|
+
const temporary1 = h + sum1 + choice + SHA256_ROUND_CONSTANTS[index] + words[index] >>> 0;
|
|
605
|
+
const temporary2 = (rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22)) + (a & b ^ a & c ^ b & c) >>> 0;
|
|
606
|
+
h = g;
|
|
607
|
+
g = f;
|
|
608
|
+
f = e;
|
|
609
|
+
e = d + temporary1 >>> 0;
|
|
610
|
+
d = c;
|
|
611
|
+
c = b;
|
|
612
|
+
b = a;
|
|
613
|
+
a = temporary1 + temporary2 >>> 0;
|
|
614
|
+
}
|
|
615
|
+
state[0] = state[0] + a >>> 0;
|
|
616
|
+
state[1] = state[1] + b >>> 0;
|
|
617
|
+
state[2] = state[2] + c >>> 0;
|
|
618
|
+
state[3] = state[3] + d >>> 0;
|
|
619
|
+
state[4] = state[4] + e >>> 0;
|
|
620
|
+
state[5] = state[5] + f >>> 0;
|
|
621
|
+
state[6] = state[6] + g >>> 0;
|
|
622
|
+
state[7] = state[7] + h >>> 0;
|
|
623
|
+
}
|
|
624
|
+
return `sha256:${Array.from(state, (word) => word.toString(16).padStart(8, "0")).join("")}`;
|
|
625
|
+
}
|
|
626
|
+
|
|
460
627
|
//#endregion
|
|
461
628
|
//#region src/agent-scope.ts
|
|
462
629
|
/** Normalize framework aliases used by page metadata, agent contracts, MCP, and evaluations. */
|
|
@@ -783,7 +950,8 @@ const DEFAULT_SEARCH_LIMIT = 10;
|
|
|
783
950
|
const MAX_SEARCH_SNIPPET_CHARS = 160;
|
|
784
951
|
const DEFAULT_MCP_PROTOCOL_VERSION = "2025-11-25";
|
|
785
952
|
const MCP_SESSION_CLEANUP_TIMEOUT_MS = 1e3;
|
|
786
|
-
const syncedIndexes = /* @__PURE__ */ new
|
|
953
|
+
const syncedIndexes = /* @__PURE__ */ new Map();
|
|
954
|
+
const syncingIndexes = /* @__PURE__ */ new Map();
|
|
787
955
|
const ALGOLIA_MAX_RECORD_BYTES = 9500;
|
|
788
956
|
const DEFAULT_ASK_AI_CONTEXT_CHARS = 24e3;
|
|
789
957
|
const DEFAULT_ASK_AI_RESULT_CHARS = 6e3;
|
|
@@ -793,9 +961,13 @@ const MAX_SEARCH_FILTER_RAW_CHARS = 4096;
|
|
|
793
961
|
const MAX_SEARCH_FILTER_SEGMENTS = 64;
|
|
794
962
|
const MAX_PROVIDER_SCOPE_FILTER_IDS = 1e3;
|
|
795
963
|
const MAX_PROVIDER_SCOPE_FILTER_CHARS = 16e3;
|
|
964
|
+
const ALGOLIA_BATCH_OPERATIONS = 1e3;
|
|
796
965
|
const MAX_SEARCH_WARNINGS = 16;
|
|
797
966
|
const MAX_SEARCH_WARNING_VALUES = 16;
|
|
798
967
|
const MAX_SEARCH_WARNING_PAGE_URLS = 8;
|
|
968
|
+
const RETRIEVAL_INDEX_FORMAT = "docs-retrieval-index.v1";
|
|
969
|
+
const MAX_RETRIEVAL_SOURCE_URL_CHARS = 4096;
|
|
970
|
+
const MAX_RETRIEVAL_SOURCE_VALUE_CHARS = 256;
|
|
799
971
|
const SEARCH_FILTER_FIELDS = [
|
|
800
972
|
"framework",
|
|
801
973
|
"version",
|
|
@@ -895,9 +1067,10 @@ function resolveDocsSearchRequest(searchParams) {
|
|
|
895
1067
|
function hasDocsSearchFilters(filters) {
|
|
896
1068
|
return SEARCH_FILTER_FIELDS.some((field) => (filters[field]?.length ?? 0) > 0);
|
|
897
1069
|
}
|
|
898
|
-
function resolveProviderScopeDocumentIds(query, context) {
|
|
1070
|
+
function resolveProviderScopeDocumentIds(query, context, corpusId) {
|
|
899
1071
|
if (!hasDocsSearchFilters(query.filters ?? {})) return void 0;
|
|
900
|
-
const
|
|
1072
|
+
const documents = corpusId ? buildDocsSearchDocuments(context.pages, context.chunking ?? { strategy: "section" }, "human") : context.documents;
|
|
1073
|
+
const ids = Array.from(new Set(documents.map((document) => corpusId ? makeHostedProviderDocumentId(corpusId, document.id) : document.id)));
|
|
901
1074
|
return ids.length <= MAX_PROVIDER_SCOPE_FILTER_IDS ? ids : [];
|
|
902
1075
|
}
|
|
903
1076
|
function docsSearchFiltersMatch(expected, actual) {
|
|
@@ -1010,13 +1183,49 @@ function isLiteralLookupQuery(query) {
|
|
|
1010
1183
|
function tokenizeSearchQuery(query) {
|
|
1011
1184
|
return Array.from(new Set(query.toLowerCase().replace(/[^\p{L}\p{N}@/_:.-]+/gu, " ").split(/\s+/).map((word) => word.replace(/^[^\p{L}\p{N}@]+|[^\p{L}\p{N}]+$/gu, "")).filter((word) => word.length > 1 && !SEARCH_STOP_WORDS.has(word))));
|
|
1012
1185
|
}
|
|
1013
|
-
function
|
|
1186
|
+
function normalizeUrlRouteKey(value) {
|
|
1014
1187
|
try {
|
|
1015
|
-
|
|
1188
|
+
const url = new URL(value, "https://docs.local");
|
|
1189
|
+
return `${url.pathname || "/"}${url.search}`;
|
|
1016
1190
|
} catch {
|
|
1017
|
-
return value.split(
|
|
1191
|
+
return value.split("#", 1)[0] || "/";
|
|
1018
1192
|
}
|
|
1019
1193
|
}
|
|
1194
|
+
function normalizeAuthoredUrlIdentity(value) {
|
|
1195
|
+
const trimmed = value.trim();
|
|
1196
|
+
const explicitScheme = trimmed.match(/^([a-z][a-z\d+.-]*):/iu)?.[1]?.toLowerCase();
|
|
1197
|
+
if (explicitScheme && explicitScheme !== "http" && explicitScheme !== "https") return normalizeUrlRouteKey("/");
|
|
1198
|
+
try {
|
|
1199
|
+
const url = new URL(trimmed, "https://docs.local");
|
|
1200
|
+
const route = `${url.pathname || "/"}${url.search}`;
|
|
1201
|
+
if (explicitScheme) return `${url.origin}${route}`;
|
|
1202
|
+
if (trimmed.startsWith("//")) return `//${url.host}${route}`;
|
|
1203
|
+
return route;
|
|
1204
|
+
} catch {
|
|
1205
|
+
return normalizeUrlRouteKey(trimmed);
|
|
1206
|
+
}
|
|
1207
|
+
}
|
|
1208
|
+
function appendDocsLocaleQuery(value, locale) {
|
|
1209
|
+
const hashIndex = value.indexOf("#");
|
|
1210
|
+
const withoutHash = hashIndex >= 0 ? value.slice(0, hashIndex) : value;
|
|
1211
|
+
const hash = hashIndex >= 0 ? value.slice(hashIndex) : "";
|
|
1212
|
+
const queryIndex = withoutHash.indexOf("?");
|
|
1213
|
+
const rawQuery = queryIndex >= 0 ? withoutHash.slice(queryIndex + 1) : "";
|
|
1214
|
+
if (new URLSearchParams(rawQuery).has("lang")) return value;
|
|
1215
|
+
return `${withoutHash}${queryIndex >= 0 ? "&" : "?"}lang=${encodeURIComponent(locale)}${hash}`;
|
|
1216
|
+
}
|
|
1217
|
+
function localizeDocsSearchPage(page, localeFallback) {
|
|
1218
|
+
const rawLocale = (page.locale ?? localeFallback)?.trim();
|
|
1219
|
+
if (!rawLocale) return page;
|
|
1220
|
+
const locale = rawLocale;
|
|
1221
|
+
const url = appendDocsLocaleQuery(page.url, locale);
|
|
1222
|
+
if (url === page.url && page.locale === locale) return page;
|
|
1223
|
+
return {
|
|
1224
|
+
...page,
|
|
1225
|
+
url,
|
|
1226
|
+
locale
|
|
1227
|
+
};
|
|
1228
|
+
}
|
|
1020
1229
|
function safeDecodeUrlSegment(value) {
|
|
1021
1230
|
try {
|
|
1022
1231
|
return decodeURIComponent(value);
|
|
@@ -1044,8 +1253,14 @@ function resolveAskAIContextUrl(value, baseUrl) {
|
|
|
1044
1253
|
return value;
|
|
1045
1254
|
}
|
|
1046
1255
|
}
|
|
1256
|
+
function cleanGeneratedAgentContractMarkers(content) {
|
|
1257
|
+
return stripDocsGeneratedAgentContractMarkers(content);
|
|
1258
|
+
}
|
|
1259
|
+
function getAskAIPageSectionContent(page) {
|
|
1260
|
+
return getPageAudienceSectionContent(page, "agent");
|
|
1261
|
+
}
|
|
1047
1262
|
function getAskAIPageContent(page) {
|
|
1048
|
-
return
|
|
1263
|
+
return cleanGeneratedAgentContractMarkers(getAskAIPageSectionContent(page));
|
|
1049
1264
|
}
|
|
1050
1265
|
function getPageAgentContractSearchText(page) {
|
|
1051
1266
|
return stripMarkdownText(renderPageAgentContractMarkdown(page.agent).replace(PAGE_AGENT_CONTRACT_START_MARKER, "").replace(PAGE_AGENT_CONTRACT_END_MARKER, ""));
|
|
@@ -1122,19 +1337,19 @@ function clampText(value, maxChars) {
|
|
|
1122
1337
|
if (value.length <= maxChars) return value;
|
|
1123
1338
|
return `${value.slice(0, maxChars).trimEnd()}...`;
|
|
1124
1339
|
}
|
|
1125
|
-
function findPageForSearchResult(pages, result, baseUrl,
|
|
1340
|
+
function findPageForSearchResult(pages, result, baseUrl, pagesByRoute) {
|
|
1126
1341
|
const rawUrl = result.url.trim();
|
|
1127
1342
|
const explicitlySchemed = /^[a-z][a-z\d+.-]*:/iu.test(rawUrl);
|
|
1128
1343
|
const explicitlyHosted = explicitlySchemed || /^[\\/]{2}/u.test(rawUrl);
|
|
1129
1344
|
if (explicitlySchemed && !/^https?:/iu.test(rawUrl)) return void 0;
|
|
1130
|
-
if (explicitlyHosted && !baseUrl
|
|
1345
|
+
if (explicitlyHosted && !baseUrl) return void 0;
|
|
1131
1346
|
if (explicitlyHosted && baseUrl) try {
|
|
1132
1347
|
if (new URL(result.url, baseUrl).origin !== new URL(baseUrl).origin) return void 0;
|
|
1133
1348
|
} catch {
|
|
1134
1349
|
return;
|
|
1135
1350
|
}
|
|
1136
|
-
const resultPath =
|
|
1137
|
-
return pages.find((page) =>
|
|
1351
|
+
const resultPath = normalizeUrlRouteKey(result.url);
|
|
1352
|
+
return pagesByRoute?.get(resultPath) ?? pages.find((page) => normalizeUrlRouteKey(page.url) === resultPath);
|
|
1138
1353
|
}
|
|
1139
1354
|
function inferResultTitle(result, page) {
|
|
1140
1355
|
if (page) return page.title;
|
|
@@ -1145,7 +1360,7 @@ function formatAskAIContextResult(options) {
|
|
|
1145
1360
|
const title = inferResultTitle(result, page);
|
|
1146
1361
|
const section = result.section;
|
|
1147
1362
|
const sectionSelector = getSearchResultAnchor(result.url) ?? section;
|
|
1148
|
-
const contextContent = clampText(cleanAskAIContextMarkdown(page ? sectionSelector ? findDocsMarkdownSection(
|
|
1363
|
+
const contextContent = clampText(cleanAskAIContextMarkdown(page ? sectionSelector ? cleanGeneratedAgentContractMarkers(findDocsMarkdownSection(getAskAIPageSectionContent(page), sectionSelector)?.content ?? "") : getAskAIPageContent(page) : [result.content, result.description].filter(Boolean).join("\n\n")), maxChars);
|
|
1149
1364
|
return {
|
|
1150
1365
|
...result,
|
|
1151
1366
|
url: resolveAskAIContextUrl(result.url, baseUrl),
|
|
@@ -1154,20 +1369,17 @@ function formatAskAIContextResult(options) {
|
|
|
1154
1369
|
};
|
|
1155
1370
|
}
|
|
1156
1371
|
function getSearchResultKey(result) {
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
} catch {
|
|
1161
|
-
hash = result.url.split("#")[1]?.split(/[?&]/)[0] ?? "";
|
|
1162
|
-
}
|
|
1163
|
-
return `${normalizeUrlPathname(result.url)}#${normalizeWhitespace(hash || result.section || "").toLowerCase()}`;
|
|
1372
|
+
const anchor = getSearchResultAnchor(result.url);
|
|
1373
|
+
const sectionFallback = normalizeWhitespace(result.section ?? "").toLowerCase();
|
|
1374
|
+
return `${normalizeUrlRouteKey(result.url)}#${anchor ?? sectionFallback}`;
|
|
1164
1375
|
}
|
|
1165
1376
|
function getSearchResultAnchor(value) {
|
|
1166
1377
|
let hash = "";
|
|
1167
1378
|
try {
|
|
1168
1379
|
hash = new URL(value, "https://docs.local").hash.replace(/^#/, "");
|
|
1169
1380
|
} catch {
|
|
1170
|
-
|
|
1381
|
+
const hashIndex = value.indexOf("#");
|
|
1382
|
+
hash = hashIndex >= 0 ? value.slice(hashIndex + 1) : "";
|
|
1171
1383
|
}
|
|
1172
1384
|
if (!hash) return void 0;
|
|
1173
1385
|
try {
|
|
@@ -1177,7 +1389,7 @@ function getSearchResultAnchor(value) {
|
|
|
1177
1389
|
}
|
|
1178
1390
|
}
|
|
1179
1391
|
function getAskAIResultPageKey(value, baseUrl, strictExternalOrigins = false) {
|
|
1180
|
-
const path =
|
|
1392
|
+
const path = normalizeUrlRouteKey(value);
|
|
1181
1393
|
const rawValue = value.trim();
|
|
1182
1394
|
if (!(/^[a-z][a-z\d+.-]*:/iu.test(rawValue) || /^[\\/]{2}/u.test(rawValue)) || !baseUrl && !strictExternalOrigins) return path;
|
|
1183
1395
|
try {
|
|
@@ -1191,7 +1403,9 @@ function getAskAIResultPageKey(value, baseUrl, strictExternalOrigins = false) {
|
|
|
1191
1403
|
}
|
|
1192
1404
|
}
|
|
1193
1405
|
function getAskAIResultKey(result, baseUrl, strictExternalOrigins = false) {
|
|
1194
|
-
|
|
1406
|
+
const anchor = getSearchResultAnchor(result.url);
|
|
1407
|
+
const sectionFallback = normalizeWhitespace(result.section ?? "").toLowerCase();
|
|
1408
|
+
return `${getAskAIResultPageKey(result.url, baseUrl, strictExternalOrigins)}#${anchor ?? sectionFallback}`;
|
|
1195
1409
|
}
|
|
1196
1410
|
function mergeSearchResults(groups, getResultKey = getSearchResultKey) {
|
|
1197
1411
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -1218,11 +1432,11 @@ function buildAudienceProjectionSearchResults(documents, query) {
|
|
|
1218
1432
|
};
|
|
1219
1433
|
});
|
|
1220
1434
|
}
|
|
1221
|
-
function sanitizeExternalAudienceSearchResults(results, localAudienceResults, baseUrl,
|
|
1435
|
+
function sanitizeExternalAudienceSearchResults(results, localAudienceResults, baseUrl, preserveUnmatched, fallbackSectionToLocalPage = false) {
|
|
1222
1436
|
const localByKey = new Map(localAudienceResults.map((result) => [getSearchResultKey(result), result]));
|
|
1223
1437
|
const localPageResults = /* @__PURE__ */ new Map();
|
|
1224
1438
|
for (const result of localAudienceResults) {
|
|
1225
|
-
const pageUrl =
|
|
1439
|
+
const pageUrl = normalizeUrlRouteKey(result.url);
|
|
1226
1440
|
if (!localPageResults.get(pageUrl) || result.type === "page") localPageResults.set(pageUrl, result);
|
|
1227
1441
|
}
|
|
1228
1442
|
return results.flatMap((result) => {
|
|
@@ -1230,7 +1444,7 @@ function sanitizeExternalAudienceSearchResults(results, localAudienceResults, ba
|
|
|
1230
1444
|
const explicitlySchemed = /^[a-z][a-z\d+.-]*:/iu.test(rawUrl);
|
|
1231
1445
|
const externallyHosted = explicitlySchemed && /^https?:/iu.test(rawUrl) || /^[\\/]{2}/u.test(rawUrl);
|
|
1232
1446
|
const unsupportedScheme = explicitlySchemed && !/^https?:/iu.test(rawUrl);
|
|
1233
|
-
let sameOriginOrUnknown = !externallyHosted
|
|
1447
|
+
let sameOriginOrUnknown = !externallyHosted;
|
|
1234
1448
|
if (unsupportedScheme) sameOriginOrUnknown = false;
|
|
1235
1449
|
if (externallyHosted && baseUrl) try {
|
|
1236
1450
|
sameOriginOrUnknown = new URL(result.url, baseUrl).origin === new URL(baseUrl).origin;
|
|
@@ -1238,7 +1452,7 @@ function sanitizeExternalAudienceSearchResults(results, localAudienceResults, ba
|
|
|
1238
1452
|
sameOriginOrUnknown = false;
|
|
1239
1453
|
}
|
|
1240
1454
|
const hasSection = Boolean(getSearchResultAnchor(result.url) || result.section);
|
|
1241
|
-
const local = sameOriginOrUnknown ? localByKey.get(getSearchResultKey(result)) ?? (hasSection ? void 0 : localPageResults.get(
|
|
1455
|
+
const local = sameOriginOrUnknown ? localByKey.get(getSearchResultKey(result)) ?? (hasSection && !fallbackSectionToLocalPage ? void 0 : localPageResults.get(normalizeUrlRouteKey(result.url))) : void 0;
|
|
1242
1456
|
if (!local) return preserveUnmatched?.(result) ? [result] : [];
|
|
1243
1457
|
return [{
|
|
1244
1458
|
...local,
|
|
@@ -1249,14 +1463,14 @@ function sanitizeExternalAudienceSearchResults(results, localAudienceResults, ba
|
|
|
1249
1463
|
}
|
|
1250
1464
|
function shouldPreserveUnmatchedExternalResult(options) {
|
|
1251
1465
|
const { result, localPagePaths, baseUrl } = options;
|
|
1252
|
-
const path =
|
|
1466
|
+
const path = normalizeUrlRouteKey(result.url);
|
|
1253
1467
|
const rawUrl = result.url.trim();
|
|
1254
1468
|
const explicitScheme = rawUrl.match(/^([a-z][a-z\d+.-]*):/iu)?.[1]?.toLowerCase();
|
|
1255
1469
|
if (explicitScheme && explicitScheme !== "http" && explicitScheme !== "https") return false;
|
|
1256
1470
|
const explicitlyHosted = /^[a-z][a-z\d+.-]*:/iu.test(rawUrl) || /^[\\/]{2}/u.test(rawUrl);
|
|
1257
1471
|
const knownLocalPath = localPagePaths.has(path);
|
|
1258
1472
|
if (!explicitlyHosted) return !knownLocalPath;
|
|
1259
|
-
if (!baseUrl) return
|
|
1473
|
+
if (!baseUrl) return true;
|
|
1260
1474
|
try {
|
|
1261
1475
|
if (new URL(result.url, baseUrl).origin !== new URL(baseUrl).origin) return true;
|
|
1262
1476
|
} catch {
|
|
@@ -1264,18 +1478,526 @@ function shouldPreserveUnmatchedExternalResult(options) {
|
|
|
1264
1478
|
}
|
|
1265
1479
|
return !knownLocalPath;
|
|
1266
1480
|
}
|
|
1481
|
+
function getPageAudienceSource(page, audience) {
|
|
1482
|
+
return audience === "agent" ? page.agentRawContent ?? page.agentFallbackRawContent ?? page.agentContent ?? page.agentFallbackContent ?? page.rawContent ?? page.content : page.rawContent ?? page.content;
|
|
1483
|
+
}
|
|
1267
1484
|
function getPageAudienceRawContent(page, audience) {
|
|
1268
|
-
return resolveDocsAudienceMdxContent(
|
|
1485
|
+
return resolveDocsAudienceMdxContent(getPageAudienceSource(page, audience), audience);
|
|
1486
|
+
}
|
|
1487
|
+
function getPageAudienceSectionContent(page, audience) {
|
|
1488
|
+
const content = getPageAudienceRawContent(page, audience);
|
|
1489
|
+
return audience === "agent" ? upsertPageAgentContractMarkdown(content, page.agent) : content;
|
|
1490
|
+
}
|
|
1491
|
+
/**
|
|
1492
|
+
* Build the exact normalized-input projection represented by a retrieval source digest.
|
|
1493
|
+
* This is public so consumers can independently reproduce `source.digest`.
|
|
1494
|
+
*/
|
|
1495
|
+
function buildDocsRetrievalDigestProjection(page, audience = "human") {
|
|
1496
|
+
const audienceContent = getPageAudienceRawContent(page, audience);
|
|
1497
|
+
return audience === "agent" ? upsertPageAgentContractMarkdown(audienceContent, page.agent) : [audienceContent, renderPageAgentContractMarkdown(page.agent)].filter(Boolean).join("\n\n");
|
|
1269
1498
|
}
|
|
1270
1499
|
function getPageAudienceSearchText(page, audience) {
|
|
1271
1500
|
return stripMarkdownText(getPageAudienceRawContent(page, audience));
|
|
1272
1501
|
}
|
|
1273
|
-
function
|
|
1502
|
+
function getPageAudienceIndexContent(page, audience) {
|
|
1503
|
+
return normalizeWhitespace([getPageAudienceSearchText(page, audience), getPageAgentContractSearchText(page)].join(" "));
|
|
1504
|
+
}
|
|
1505
|
+
function sortRetrievalSourceValues(values) {
|
|
1506
|
+
const sorted = Array.from(new Set(values)).sort(compareSearchMetadataValues).slice(0, MAX_SEARCH_FILTER_VALUES);
|
|
1507
|
+
return sorted.length > 0 ? sorted : void 0;
|
|
1508
|
+
}
|
|
1509
|
+
function buildDocsRetrievalSourceScope(page, audience, localeFallback) {
|
|
1510
|
+
const resolved = resolveDocsSearchPageScope(page);
|
|
1511
|
+
const rawLocale = (page.locale ?? localeFallback)?.trim();
|
|
1512
|
+
const locale = rawLocale ? normalizeAgentLocale(rawLocale).slice(0, MAX_SEARCH_FILTER_VALUE_CHARS) : void 0;
|
|
1513
|
+
const framework = sortRetrievalSourceValues(resolved.framework);
|
|
1514
|
+
const version = sortRetrievalSourceValues(resolved.version);
|
|
1515
|
+
const versionGroups = resolved.declarations.version.map((group) => sortRetrievalSourceValues(group)).filter((group) => Boolean(group)).slice(0, MAX_SEARCH_FILTER_VALUES);
|
|
1516
|
+
const packageNames = sortRetrievalSourceValues(resolved.package);
|
|
1517
|
+
const tags = sortRetrievalSourceValues(resolved.tags);
|
|
1518
|
+
const truncated = SEARCH_FILTER_FIELDS.filter((field) => {
|
|
1519
|
+
if (field === "version") return resolved.version.length > MAX_SEARCH_FILTER_VALUES || resolved.declarations.version.length > MAX_SEARCH_FILTER_VALUES || resolved.declarations.version.some((group) => group.length > MAX_SEARCH_FILTER_VALUES);
|
|
1520
|
+
return resolved[field].length > MAX_SEARCH_FILTER_VALUES;
|
|
1521
|
+
});
|
|
1522
|
+
return {
|
|
1523
|
+
audience,
|
|
1524
|
+
...locale ? { locale: [locale] } : {},
|
|
1525
|
+
...framework ? { framework } : {},
|
|
1526
|
+
...version ? { version } : {},
|
|
1527
|
+
...versionGroups.length > 1 ? { versionGroups } : {},
|
|
1528
|
+
...packageNames ? { package: packageNames } : {},
|
|
1529
|
+
...tags ? { tags } : {},
|
|
1530
|
+
...truncated.length > 0 ? { truncated } : {},
|
|
1531
|
+
...resolved.conflicts.length > 0 ? { conflicts: [...resolved.conflicts].sort((left, right) => SEARCH_FILTER_FIELDS.indexOf(left) - SEARCH_FILTER_FIELDS.indexOf(right)) } : {}
|
|
1532
|
+
};
|
|
1533
|
+
}
|
|
1534
|
+
function buildDocsRetrievalScopeIdentity(page, audience, localeFallback) {
|
|
1535
|
+
const resolved = resolveDocsSearchPageScope(page);
|
|
1536
|
+
const sortValues = (values) => Array.from(new Set(values)).sort(compareSearchMetadataValues);
|
|
1537
|
+
const declarations = Object.fromEntries(SEARCH_FILTER_FIELDS.map((field) => [field, resolved.declarations[field].map(sortValues).sort((left, right) => compareSearchMetadataValues(JSON.stringify(left), JSON.stringify(right)))]));
|
|
1538
|
+
return hashDocsRetrievalValue(JSON.stringify({
|
|
1539
|
+
audience,
|
|
1540
|
+
locale: (page.locale ?? localeFallback)?.trim(),
|
|
1541
|
+
declarations,
|
|
1542
|
+
conflicts: [...resolved.conflicts].sort((left, right) => SEARCH_FILTER_FIELDS.indexOf(left) - SEARCH_FILTER_FIELDS.indexOf(right))
|
|
1543
|
+
}));
|
|
1544
|
+
}
|
|
1545
|
+
function getUrlHash(value) {
|
|
1546
|
+
const hashIndex = value.indexOf("#");
|
|
1547
|
+
return hashIndex >= 0 ? value.slice(hashIndex) : "";
|
|
1548
|
+
}
|
|
1549
|
+
function resolveDocsRetrievalCanonicalUrl(page, resultUrl, baseUrl) {
|
|
1550
|
+
const requestedCanonical = page.canonicalUrl?.trim();
|
|
1551
|
+
const requested = requestedCanonical && requestedCanonical.length <= MAX_RETRIEVAL_SOURCE_URL_CHARS ? requestedCanonical : page.url;
|
|
1552
|
+
const explicitScheme = requested.match(/^([a-z][a-z\d+.-]*):/iu)?.[1]?.toLowerCase();
|
|
1553
|
+
const pageScheme = page.url.match(/^([a-z][a-z\d+.-]*):/iu)?.[1]?.toLowerCase();
|
|
1554
|
+
const safePageUrl = pageScheme && pageScheme !== "http" && pageScheme !== "https" ? "/" : page.url;
|
|
1555
|
+
const configured = explicitScheme && explicitScheme !== "http" && explicitScheme !== "https" ? safePageUrl : requested;
|
|
1556
|
+
let canonical = configured;
|
|
1557
|
+
if (baseUrl) try {
|
|
1558
|
+
canonical = new URL(configured, baseUrl).toString();
|
|
1559
|
+
} catch {
|
|
1560
|
+
canonical = configured;
|
|
1561
|
+
}
|
|
1562
|
+
else if (configured.startsWith("//")) canonical = `https:${configured}`;
|
|
1563
|
+
const withSection = `${canonical.split("#", 1)[0]}${getUrlHash(resultUrl)}`;
|
|
1564
|
+
if (withSection.length <= MAX_RETRIEVAL_SOURCE_URL_CHARS && isDocsRetrievalCanonicalUrl(withSection)) return withSection;
|
|
1565
|
+
const fallback = (baseUrl ? (() => {
|
|
1566
|
+
try {
|
|
1567
|
+
return new URL(safePageUrl, baseUrl).toString();
|
|
1568
|
+
} catch {
|
|
1569
|
+
return safePageUrl;
|
|
1570
|
+
}
|
|
1571
|
+
})() : safePageUrl).split("#", 1)[0];
|
|
1572
|
+
if (fallback.length <= MAX_RETRIEVAL_SOURCE_URL_CHARS && isDocsRetrievalCanonicalUrl(fallback)) return fallback;
|
|
1573
|
+
if (baseUrl) try {
|
|
1574
|
+
const root = new URL("/", baseUrl).toString();
|
|
1575
|
+
if (root.length <= MAX_RETRIEVAL_SOURCE_URL_CHARS && isDocsRetrievalCanonicalUrl(root)) return root;
|
|
1576
|
+
} catch {}
|
|
1577
|
+
return "/";
|
|
1578
|
+
}
|
|
1579
|
+
function hashDocsRetrievalValue(value) {
|
|
1580
|
+
return digestDocsRetrievalContent(value);
|
|
1581
|
+
}
|
|
1582
|
+
const docsRetrievalDigestMemo = /* @__PURE__ */ new WeakMap();
|
|
1583
|
+
function getDocsRetrievalSourceDigest(page, audience, cache) {
|
|
1584
|
+
const cached = cache?.get(page);
|
|
1585
|
+
if (cached) return cached;
|
|
1586
|
+
const source = getPageAudienceSource(page, audience);
|
|
1587
|
+
let agentContractKey = "";
|
|
1588
|
+
let cacheable = true;
|
|
1589
|
+
try {
|
|
1590
|
+
agentContractKey = JSON.stringify(page.agent ?? null);
|
|
1591
|
+
} catch {
|
|
1592
|
+
cacheable = false;
|
|
1593
|
+
}
|
|
1594
|
+
const memo = docsRetrievalDigestMemo.get(page)?.[audience];
|
|
1595
|
+
if (cacheable && memo && memo.source === source && memo.agentContractKey === agentContractKey) {
|
|
1596
|
+
cache?.set(page, memo.digest);
|
|
1597
|
+
return memo.digest;
|
|
1598
|
+
}
|
|
1599
|
+
const digest = hashDocsRetrievalValue(buildDocsRetrievalDigestProjection(page, audience));
|
|
1600
|
+
if (cacheable) {
|
|
1601
|
+
const pageMemo = docsRetrievalDigestMemo.get(page) ?? {};
|
|
1602
|
+
pageMemo[audience] = {
|
|
1603
|
+
source,
|
|
1604
|
+
agentContractKey,
|
|
1605
|
+
digest
|
|
1606
|
+
};
|
|
1607
|
+
docsRetrievalDigestMemo.set(page, pageMemo);
|
|
1608
|
+
}
|
|
1609
|
+
cache?.set(page, digest);
|
|
1610
|
+
return digest;
|
|
1611
|
+
}
|
|
1612
|
+
function resolveDocsRetrievalLastModified(page, audience) {
|
|
1613
|
+
const parseCandidate = (value) => {
|
|
1614
|
+
const normalized = value?.trim();
|
|
1615
|
+
if (!normalized) return void 0;
|
|
1616
|
+
const timestamp = Date.parse(normalized);
|
|
1617
|
+
return Number.isFinite(timestamp) ? {
|
|
1618
|
+
value: normalized,
|
|
1619
|
+
timestamp
|
|
1620
|
+
} : void 0;
|
|
1621
|
+
};
|
|
1622
|
+
const pageModified = parseCandidate(page.lastmod) ?? parseCandidate(page.lastModified);
|
|
1623
|
+
if (!(audience === "agent" && (page.agentRawContent !== void 0 || page.agentContent !== void 0))) return pageModified?.value;
|
|
1624
|
+
const agentModified = parseCandidate(page.agentLastModified);
|
|
1625
|
+
if (!renderPageAgentContractMarkdown(page.agent)) return agentModified?.value ?? pageModified?.value;
|
|
1626
|
+
if (!agentModified) return pageModified?.value;
|
|
1627
|
+
if (!pageModified) return agentModified.value;
|
|
1628
|
+
return agentModified.timestamp >= pageModified.timestamp ? agentModified.value : pageModified.value;
|
|
1629
|
+
}
|
|
1630
|
+
async function buildDocsSearchIndexGeneration(pages, options) {
|
|
1631
|
+
const sources = (await Promise.all(pages.map(async (rawPage) => {
|
|
1632
|
+
const page = localizeDocsSearchPage(rawPage, options.locale);
|
|
1633
|
+
const authoredLastModified = page.lastmod?.trim();
|
|
1634
|
+
return {
|
|
1635
|
+
canonicalIdentity: normalizeAuthoredUrlIdentity(page.canonicalUrl?.trim() || page.url),
|
|
1636
|
+
url: normalizeAuthoredUrlIdentity(page.url),
|
|
1637
|
+
indexedUrl: page.url,
|
|
1638
|
+
title: page.title,
|
|
1639
|
+
description: page.description,
|
|
1640
|
+
type: page.type,
|
|
1641
|
+
scope: buildDocsRetrievalSourceScope(page, options.audience, options.locale),
|
|
1642
|
+
scopeIdentity: buildDocsRetrievalScopeIdentity(page, options.audience, options.locale),
|
|
1643
|
+
lastModified: authoredLastModified && Number.isFinite(Date.parse(authoredLastModified)) ? authoredLastModified : void 0,
|
|
1644
|
+
digest: getDocsRetrievalSourceDigest(rawPage, options.audience, options.digestCache),
|
|
1645
|
+
agentContract: getPageAgentContractSearchText(rawPage)
|
|
1646
|
+
};
|
|
1647
|
+
}))).sort((left, right) => {
|
|
1648
|
+
const canonical = compareSearchMetadataValues(left.canonicalIdentity, right.canonicalIdentity);
|
|
1649
|
+
if (canonical !== 0) return canonical;
|
|
1650
|
+
const url = compareSearchMetadataValues(left.url, right.url);
|
|
1651
|
+
if (url !== 0) return url;
|
|
1652
|
+
const digest = compareSearchMetadataValues(left.digest, right.digest);
|
|
1653
|
+
if (digest !== 0) return digest;
|
|
1654
|
+
const title = compareSearchMetadataValues(left.title, right.title);
|
|
1655
|
+
return title !== 0 ? title : compareSearchMetadataValues(JSON.stringify(left), JSON.stringify(right));
|
|
1656
|
+
});
|
|
1657
|
+
return hashDocsRetrievalValue(JSON.stringify({
|
|
1658
|
+
format: RETRIEVAL_INDEX_FORMAT,
|
|
1659
|
+
audience: options.audience,
|
|
1660
|
+
chunking: options.chunking.strategy ?? "section",
|
|
1661
|
+
sources
|
|
1662
|
+
}));
|
|
1663
|
+
}
|
|
1664
|
+
async function buildDocsRetrievalSource(rawPage, resultUrl, options) {
|
|
1665
|
+
const page = localizeDocsSearchPage(rawPage, options.locale);
|
|
1666
|
+
const lastModified = resolveDocsRetrievalLastModified(rawPage, options.audience);
|
|
1667
|
+
return {
|
|
1668
|
+
canonicalUrl: resolveDocsRetrievalCanonicalUrl(page, resultUrl, options.baseUrl),
|
|
1669
|
+
scope: buildDocsRetrievalSourceScope(page, options.audience, options.locale),
|
|
1670
|
+
...lastModified ? { lastModified } : {},
|
|
1671
|
+
digest: getDocsRetrievalSourceDigest(rawPage, options.audience, options.digestCache),
|
|
1672
|
+
indexGeneration: options.indexGeneration
|
|
1673
|
+
};
|
|
1674
|
+
}
|
|
1675
|
+
function parseRetrievalSourceString(value, maxChars = MAX_RETRIEVAL_SOURCE_VALUE_CHARS) {
|
|
1676
|
+
if (typeof value !== "string") return void 0;
|
|
1677
|
+
const normalized = value.trim();
|
|
1678
|
+
if (!normalized || normalized.length > maxChars) return void 0;
|
|
1679
|
+
if (Array.from(normalized).some((character) => {
|
|
1680
|
+
const codePoint = character.codePointAt(0) ?? 0;
|
|
1681
|
+
return codePoint <= 31 || codePoint === 127;
|
|
1682
|
+
})) return void 0;
|
|
1683
|
+
return normalized;
|
|
1684
|
+
}
|
|
1685
|
+
function parseRetrievalSourceValues(field, value) {
|
|
1686
|
+
if (!Array.isArray(value) || value.length > MAX_SEARCH_FILTER_VALUES) return { valid: false };
|
|
1687
|
+
const parsed = [];
|
|
1688
|
+
for (const item of value) {
|
|
1689
|
+
const stringValue = parseRetrievalSourceString(item, MAX_SEARCH_FILTER_VALUE_CHARS);
|
|
1690
|
+
if (!stringValue) return { valid: false };
|
|
1691
|
+
if (field === "locale") {
|
|
1692
|
+
const normalizedLocale = normalizeAgentLocale(stringValue);
|
|
1693
|
+
if (!normalizedLocale) return { valid: false };
|
|
1694
|
+
parsed.push(normalizedLocale);
|
|
1695
|
+
continue;
|
|
1696
|
+
}
|
|
1697
|
+
const normalized = normalizeDocsSearchFilterValue(field, stringValue);
|
|
1698
|
+
if (!normalized) return { valid: false };
|
|
1699
|
+
parsed.push(normalized);
|
|
1700
|
+
}
|
|
1701
|
+
return {
|
|
1702
|
+
valid: true,
|
|
1703
|
+
values: sortRetrievalSourceValues(parsed)
|
|
1704
|
+
};
|
|
1705
|
+
}
|
|
1706
|
+
function parseRetrievalSourceVersionGroups(value) {
|
|
1707
|
+
if (!Array.isArray(value) || value.length === 0 || value.length > MAX_SEARCH_FILTER_VALUES) return { valid: false };
|
|
1708
|
+
const groups = value.map((group) => parseRetrievalSourceValues("version", group));
|
|
1709
|
+
if (groups.some((group) => !group.valid || !group.values || group.values.length === 0)) return { valid: false };
|
|
1710
|
+
return {
|
|
1711
|
+
valid: true,
|
|
1712
|
+
groups: groups.map((group) => group.values)
|
|
1713
|
+
};
|
|
1714
|
+
}
|
|
1715
|
+
function parseDocsRetrievalSource(value, options = {}) {
|
|
1716
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return void 0;
|
|
1717
|
+
const record = value;
|
|
1718
|
+
const canonicalUrl = parseRetrievalSourceString(record.canonicalUrl, MAX_RETRIEVAL_SOURCE_URL_CHARS);
|
|
1719
|
+
const digest = parseRetrievalSourceString(record.digest);
|
|
1720
|
+
const indexGeneration = parseRetrievalSourceString(record.indexGeneration);
|
|
1721
|
+
if (!canonicalUrl || !digest || !indexGeneration) return void 0;
|
|
1722
|
+
const digestPattern = /^sha256:[a-f\d]{64}$/iu;
|
|
1723
|
+
if (!digestPattern.test(digest) || !digestPattern.test(indexGeneration)) return void 0;
|
|
1724
|
+
if (!isDocsRetrievalCanonicalUrl(canonicalUrl) || /^\/(?!\/)/u.test(canonicalUrl) && !options.allowRootRelativeCanonical) return;
|
|
1725
|
+
if (!record.scope || typeof record.scope !== "object" || Array.isArray(record.scope)) return;
|
|
1726
|
+
const rawScope = record.scope;
|
|
1727
|
+
const audience = rawScope.audience === "human" || rawScope.audience === "agent" ? rawScope.audience : void 0;
|
|
1728
|
+
if (!audience) return void 0;
|
|
1729
|
+
const parseScopeFields = (value) => {
|
|
1730
|
+
if (!Array.isArray(value) || value.length > SEARCH_FILTER_FIELDS.length || value.some((field) => typeof field !== "string" || !SEARCH_FILTER_FIELDS.includes(field))) return { valid: false };
|
|
1731
|
+
return {
|
|
1732
|
+
valid: true,
|
|
1733
|
+
values: Array.from(new Set(value)).sort((left, right) => SEARCH_FILTER_FIELDS.indexOf(left) - SEARCH_FILTER_FIELDS.indexOf(right))
|
|
1734
|
+
};
|
|
1735
|
+
};
|
|
1736
|
+
const parseOptionalValues = (field, value) => value === void 0 ? { valid: true } : parseRetrievalSourceValues(field, value);
|
|
1737
|
+
const parseOptionalScopeFields = (value) => value === void 0 ? { valid: true } : parseScopeFields(value);
|
|
1738
|
+
const conflicts = parseOptionalScopeFields(rawScope.conflicts);
|
|
1739
|
+
const truncated = parseOptionalScopeFields(rawScope.truncated);
|
|
1740
|
+
const locale = parseOptionalValues("locale", rawScope.locale);
|
|
1741
|
+
const framework = parseOptionalValues("framework", rawScope.framework);
|
|
1742
|
+
const version = parseOptionalValues("version", rawScope.version);
|
|
1743
|
+
const versionGroups = rawScope.versionGroups === void 0 ? {
|
|
1744
|
+
valid: true,
|
|
1745
|
+
groups: void 0
|
|
1746
|
+
} : parseRetrievalSourceVersionGroups(rawScope.versionGroups);
|
|
1747
|
+
const packageNames = parseOptionalValues("package", rawScope.package);
|
|
1748
|
+
const tags = parseOptionalValues("tags", rawScope.tags);
|
|
1749
|
+
if (!conflicts.valid || !truncated.valid || !locale.valid || !framework.valid || !version.valid || !versionGroups.valid || !packageNames.valid || !tags.valid) return;
|
|
1750
|
+
const scope = {
|
|
1751
|
+
audience,
|
|
1752
|
+
...locale.values ? { locale: locale.values } : {},
|
|
1753
|
+
...framework.values ? { framework: framework.values } : {},
|
|
1754
|
+
...version.values ? { version: version.values } : {},
|
|
1755
|
+
...versionGroups.groups ? { versionGroups: versionGroups.groups } : {},
|
|
1756
|
+
...packageNames.values ? { package: packageNames.values } : {},
|
|
1757
|
+
...tags.values ? { tags: tags.values } : {},
|
|
1758
|
+
...truncated.values && truncated.values.length > 0 ? { truncated: truncated.values } : {},
|
|
1759
|
+
...conflicts.values && conflicts.values.length > 0 ? { conflicts: conflicts.values } : {}
|
|
1760
|
+
};
|
|
1761
|
+
const rawLastModified = parseRetrievalSourceString(record.lastModified);
|
|
1762
|
+
if (record.lastModified !== void 0 && (!rawLastModified || !Number.isFinite(Date.parse(rawLastModified)))) return;
|
|
1763
|
+
const lastModified = rawLastModified && Number.isFinite(Date.parse(rawLastModified)) ? rawLastModified : void 0;
|
|
1764
|
+
return {
|
|
1765
|
+
canonicalUrl,
|
|
1766
|
+
scope,
|
|
1767
|
+
...lastModified ? { lastModified } : {},
|
|
1768
|
+
digest,
|
|
1769
|
+
indexGeneration
|
|
1770
|
+
};
|
|
1771
|
+
}
|
|
1772
|
+
function docsRetrievalSourceMatchesRequest(source, audience, filters, locale) {
|
|
1773
|
+
if (source.scope.audience !== audience) return false;
|
|
1774
|
+
for (const field of SEARCH_FILTER_FIELDS) {
|
|
1775
|
+
const requested = filters?.[field];
|
|
1776
|
+
if (!requested?.length) continue;
|
|
1777
|
+
if (source.scope.conflicts?.includes(field)) return false;
|
|
1778
|
+
if (source.scope.truncated?.includes(field)) return false;
|
|
1779
|
+
if (field === "version" && source.scope.versionGroups?.length) {
|
|
1780
|
+
if (!agentVersionConstraintGroupsOverlap([requested, ...source.scope.versionGroups])) return false;
|
|
1781
|
+
continue;
|
|
1782
|
+
}
|
|
1783
|
+
const candidates = source.scope[field];
|
|
1784
|
+
if (!candidates?.length) return false;
|
|
1785
|
+
if (!requested.some((requestedValue) => candidates.some((candidate) => docsSearchScopeValueMatches(field, requestedValue, candidate)))) return false;
|
|
1786
|
+
}
|
|
1787
|
+
const requestedLocale = locale ? normalizeAgentLocale(locale) : void 0;
|
|
1788
|
+
if (requestedLocale && (!source.scope.locale?.length || !source.scope.locale.some((candidate) => normalizeAgentLocale(candidate) === requestedLocale))) return false;
|
|
1789
|
+
return true;
|
|
1790
|
+
}
|
|
1791
|
+
const HOSTED_RETRIEVAL_SOURCE_FIELDS = [
|
|
1792
|
+
"source_canonical_url",
|
|
1793
|
+
"source_scope_audience",
|
|
1794
|
+
"source_scope_locale",
|
|
1795
|
+
"source_scope_framework",
|
|
1796
|
+
"source_scope_version",
|
|
1797
|
+
"source_scope_version_groups",
|
|
1798
|
+
"source_scope_package",
|
|
1799
|
+
"source_scope_tags",
|
|
1800
|
+
"source_scope_truncated",
|
|
1801
|
+
"source_scope_conflicts",
|
|
1802
|
+
"source_last_modified",
|
|
1803
|
+
"source_digest",
|
|
1804
|
+
"source_index_generation"
|
|
1805
|
+
];
|
|
1806
|
+
function readFlattenedHostedRetrievalSource(record) {
|
|
1807
|
+
return parseDocsRetrievalSource({
|
|
1808
|
+
canonicalUrl: record.source_canonical_url,
|
|
1809
|
+
scope: {
|
|
1810
|
+
audience: record.source_scope_audience,
|
|
1811
|
+
locale: record.source_scope_locale,
|
|
1812
|
+
framework: record.source_scope_framework,
|
|
1813
|
+
version: record.source_scope_version,
|
|
1814
|
+
versionGroups: (() => {
|
|
1815
|
+
const value = record.source_scope_version_groups;
|
|
1816
|
+
if (typeof value !== "string") return value;
|
|
1817
|
+
try {
|
|
1818
|
+
return JSON.parse(value);
|
|
1819
|
+
} catch {
|
|
1820
|
+
return null;
|
|
1821
|
+
}
|
|
1822
|
+
})(),
|
|
1823
|
+
package: record.source_scope_package,
|
|
1824
|
+
tags: record.source_scope_tags,
|
|
1825
|
+
truncated: record.source_scope_truncated,
|
|
1826
|
+
conflicts: record.source_scope_conflicts
|
|
1827
|
+
},
|
|
1828
|
+
lastModified: record.source_last_modified,
|
|
1829
|
+
digest: record.source_digest,
|
|
1830
|
+
indexGeneration: record.source_index_generation
|
|
1831
|
+
}, { allowRootRelativeCanonical: true });
|
|
1832
|
+
}
|
|
1833
|
+
function readHostedRetrievalSource(record) {
|
|
1834
|
+
const hasNested = record.source !== void 0;
|
|
1835
|
+
const hasFlattened = HOSTED_RETRIEVAL_SOURCE_FIELDS.some((key) => key in record);
|
|
1836
|
+
if (hasNested) {
|
|
1837
|
+
const nested = parseDocsRetrievalSource(record.source, { allowRootRelativeCanonical: true });
|
|
1838
|
+
if (!nested) return void 0;
|
|
1839
|
+
if (!hasFlattened) return nested;
|
|
1840
|
+
const flattened = readFlattenedHostedRetrievalSource(record);
|
|
1841
|
+
return flattened && JSON.stringify(flattened) === JSON.stringify(nested) ? nested : void 0;
|
|
1842
|
+
}
|
|
1843
|
+
return readFlattenedHostedRetrievalSource(record);
|
|
1844
|
+
}
|
|
1845
|
+
function hasHostedRetrievalSource(record) {
|
|
1846
|
+
return record.source !== void 0 || [
|
|
1847
|
+
"source_corpus_id",
|
|
1848
|
+
"source_document_id",
|
|
1849
|
+
...HOSTED_RETRIEVAL_SOURCE_FIELDS
|
|
1850
|
+
].some((key) => key in record);
|
|
1851
|
+
}
|
|
1852
|
+
function serializeHostedRetrievalSource(source, corpusId, sourceDocumentId) {
|
|
1853
|
+
if (!source) return {};
|
|
1854
|
+
return {
|
|
1855
|
+
source_corpus_id: corpusId,
|
|
1856
|
+
source_document_id: sourceDocumentId,
|
|
1857
|
+
source_canonical_url: source.canonicalUrl,
|
|
1858
|
+
source_scope_audience: source.scope.audience,
|
|
1859
|
+
source_scope_locale: source.scope.locale,
|
|
1860
|
+
source_scope_framework: source.scope.framework,
|
|
1861
|
+
source_scope_version: source.scope.version,
|
|
1862
|
+
source_scope_version_groups: source.scope.versionGroups ? JSON.stringify(source.scope.versionGroups) : void 0,
|
|
1863
|
+
source_scope_package: source.scope.package,
|
|
1864
|
+
source_scope_tags: source.scope.tags,
|
|
1865
|
+
source_scope_truncated: source.scope.truncated,
|
|
1866
|
+
source_scope_conflicts: source.scope.conflicts,
|
|
1867
|
+
source_last_modified: source.lastModified,
|
|
1868
|
+
source_digest: source.digest,
|
|
1869
|
+
source_index_generation: source.indexGeneration
|
|
1870
|
+
};
|
|
1871
|
+
}
|
|
1872
|
+
function resolveHostedCorpusId(syncNamespace, context) {
|
|
1873
|
+
const explicitNamespace = syncNamespace?.trim();
|
|
1874
|
+
if (explicitNamespace && explicitNamespace.length > 1024) throw new Error("Search syncNamespace must be 1024 characters or fewer.");
|
|
1875
|
+
let canonicalIdentity;
|
|
1876
|
+
const indexBaseUrl = context.indexBaseUrl;
|
|
1877
|
+
if (!explicitNamespace && indexBaseUrl) try {
|
|
1878
|
+
const url = new URL(indexBaseUrl);
|
|
1879
|
+
if ((url.protocol === "http:" || url.protocol === "https:") && !url.username && !url.password) canonicalIdentity = `${url.origin}${url.pathname.replace(/\/+$/u, "") || "/"}`;
|
|
1880
|
+
} catch {}
|
|
1881
|
+
const identity = explicitNamespace ? `namespace:${explicitNamespace}` : canonicalIdentity ? `canonical:${canonicalIdentity}` : void 0;
|
|
1882
|
+
if (!identity) return void 0;
|
|
1883
|
+
return hashDocsRetrievalValue(JSON.stringify({
|
|
1884
|
+
format: "docs-hosted-corpus.v1",
|
|
1885
|
+
identity,
|
|
1886
|
+
audience: resolveDocsSearchAudience(context.audience),
|
|
1887
|
+
locale: context.locale ? normalizeAgentLocale(context.locale) : "__all__"
|
|
1888
|
+
}));
|
|
1889
|
+
}
|
|
1890
|
+
function resolveHostedHumanCorpusId(syncNamespace, context) {
|
|
1891
|
+
return resolveHostedCorpusId(syncNamespace, {
|
|
1892
|
+
audience: "human",
|
|
1893
|
+
locale: context.locale,
|
|
1894
|
+
baseUrl: context.baseUrl,
|
|
1895
|
+
indexBaseUrl: context.indexBaseUrl
|
|
1896
|
+
});
|
|
1897
|
+
}
|
|
1898
|
+
function makeHostedProviderDocumentId(corpusId, sourceDocumentId) {
|
|
1899
|
+
return `docs_${hashDocsRetrievalValue(JSON.stringify({
|
|
1900
|
+
format: "docs-hosted-document.v1",
|
|
1901
|
+
corpusId,
|
|
1902
|
+
sourceDocumentId
|
|
1903
|
+
})).slice(7)}`;
|
|
1904
|
+
}
|
|
1905
|
+
function readHostedSourceDocumentId(record) {
|
|
1906
|
+
return parseRetrievalSourceString(record.source_document_id, MAX_RETRIEVAL_SOURCE_URL_CHARS);
|
|
1907
|
+
}
|
|
1908
|
+
function hostedRecordMatchesCorpus(record, corpusId) {
|
|
1909
|
+
if (!corpusId) return true;
|
|
1910
|
+
if (record.source_corpus_id === corpusId) return true;
|
|
1911
|
+
return record.source_corpus_id === void 0 && !hasHostedRetrievalSource(record);
|
|
1912
|
+
}
|
|
1913
|
+
async function enrichDocsSearchResultsWithSources(options) {
|
|
1914
|
+
const digestCache = /* @__PURE__ */ new Map();
|
|
1915
|
+
const localizedPages = options.pages.map((page) => localizeDocsSearchPage(page, options.locale));
|
|
1916
|
+
const pagesByRoute = /* @__PURE__ */ new Map();
|
|
1917
|
+
for (const page of localizedPages) {
|
|
1918
|
+
const route = normalizeUrlRouteKey(page.url);
|
|
1919
|
+
if (!pagesByRoute.has(route)) pagesByRoute.set(route, page);
|
|
1920
|
+
}
|
|
1921
|
+
let resolvedIndexGeneration;
|
|
1922
|
+
const getIndexGeneration = () => {
|
|
1923
|
+
resolvedIndexGeneration ??= options.indexGeneration ? Promise.resolve(options.indexGeneration) : buildDocsSearchIndexGeneration(options.generationPages ?? options.pages, {
|
|
1924
|
+
audience: options.audience,
|
|
1925
|
+
chunking: options.chunking,
|
|
1926
|
+
locale: options.locale,
|
|
1927
|
+
baseUrl: options.baseUrl,
|
|
1928
|
+
digestCache
|
|
1929
|
+
});
|
|
1930
|
+
return resolvedIndexGeneration;
|
|
1931
|
+
};
|
|
1932
|
+
return (await Promise.all(options.results.map(async (result) => {
|
|
1933
|
+
const { source: rawSource, ...resultWithoutSource } = result;
|
|
1934
|
+
const page = findPageForSearchResult(localizedPages, result, options.baseUrl, pagesByRoute);
|
|
1935
|
+
if (page) return {
|
|
1936
|
+
...resultWithoutSource,
|
|
1937
|
+
source: await buildDocsRetrievalSource(page, result.url, {
|
|
1938
|
+
audience: options.audience,
|
|
1939
|
+
chunking: options.chunking,
|
|
1940
|
+
locale: options.locale,
|
|
1941
|
+
baseUrl: options.baseUrl,
|
|
1942
|
+
indexGeneration: await getIndexGeneration(),
|
|
1943
|
+
digestCache
|
|
1944
|
+
})
|
|
1945
|
+
};
|
|
1946
|
+
if (rawSource === void 0) return resultWithoutSource;
|
|
1947
|
+
const source = parseDocsRetrievalSource(rawSource, { allowRootRelativeCanonical: options.requireCurrentIndexGeneration });
|
|
1948
|
+
if (!source) return null;
|
|
1949
|
+
if (!docsRetrievalSourceMatchesRequest(source, options.audience, options.filters, options.locale)) return null;
|
|
1950
|
+
if (options.requireCurrentIndexGeneration && source.indexGeneration !== await getIndexGeneration()) return null;
|
|
1951
|
+
return {
|
|
1952
|
+
...resultWithoutSource,
|
|
1953
|
+
source
|
|
1954
|
+
};
|
|
1955
|
+
}))).filter((result) => Boolean(result));
|
|
1956
|
+
}
|
|
1957
|
+
async function enrichDocsSearchDocumentsWithSources(options) {
|
|
1958
|
+
const results = await enrichDocsSearchResultsWithSources({
|
|
1959
|
+
results: options.documents.map((document) => ({
|
|
1960
|
+
id: document.id,
|
|
1961
|
+
url: document.url,
|
|
1962
|
+
content: document.title,
|
|
1963
|
+
description: document.description,
|
|
1964
|
+
type: document.type,
|
|
1965
|
+
section: document.section,
|
|
1966
|
+
source: document.source
|
|
1967
|
+
})),
|
|
1968
|
+
pages: options.pages,
|
|
1969
|
+
audience: options.audience,
|
|
1970
|
+
chunking: options.chunking,
|
|
1971
|
+
locale: options.locale,
|
|
1972
|
+
baseUrl: options.baseUrl,
|
|
1973
|
+
indexGeneration: options.indexGeneration
|
|
1974
|
+
});
|
|
1975
|
+
const sources = new Map(results.map((result) => [result.id, result.source]));
|
|
1976
|
+
return options.documents.map((document) => {
|
|
1977
|
+
const source = sources.get(document.id);
|
|
1978
|
+
return source ? {
|
|
1979
|
+
...document,
|
|
1980
|
+
source
|
|
1981
|
+
} : document;
|
|
1982
|
+
});
|
|
1983
|
+
}
|
|
1984
|
+
/**
|
|
1985
|
+
* Attach the same canonical provenance used by built-in search providers.
|
|
1986
|
+
* Custom adapters can call this before persisting their own hosted records.
|
|
1987
|
+
*/
|
|
1988
|
+
function enrichDocsSearchDocumentsWithProvenance(options) {
|
|
1989
|
+
return enrichDocsSearchDocumentsWithSources({
|
|
1990
|
+
...options,
|
|
1991
|
+
audience: resolveDocsSearchAudience(options.audience),
|
|
1992
|
+
chunking: options.chunking ?? { strategy: "section" }
|
|
1993
|
+
});
|
|
1994
|
+
}
|
|
1995
|
+
function isLocalProviderResult(result, baseUrl) {
|
|
1274
1996
|
const rawUrl = result.url.trim();
|
|
1275
1997
|
const explicitScheme = rawUrl.match(/^([a-z][a-z\d+.-]*):/iu)?.[1]?.toLowerCase();
|
|
1276
1998
|
if (explicitScheme && explicitScheme !== "http" && explicitScheme !== "https") return false;
|
|
1277
1999
|
if (!(/^[a-z][a-z\d+.-]*:/iu.test(rawUrl) || /^[\\/]{2}/u.test(rawUrl))) return true;
|
|
1278
|
-
if (!baseUrl) return
|
|
2000
|
+
if (!baseUrl) return false;
|
|
1279
2001
|
try {
|
|
1280
2002
|
return new URL(result.url, baseUrl).origin === new URL(baseUrl).origin;
|
|
1281
2003
|
} catch {
|
|
@@ -1283,10 +2005,10 @@ function isLocalProviderResult(result, baseUrl, strictExternalOrigins = false) {
|
|
|
1283
2005
|
}
|
|
1284
2006
|
}
|
|
1285
2007
|
function hasOppositeAudienceEvidence(options) {
|
|
1286
|
-
const { result, pages, query, audience, baseUrl
|
|
1287
|
-
if (!isLocalProviderResult(result, baseUrl
|
|
1288
|
-
const pagePath =
|
|
1289
|
-
const page = pages.find((candidate) =>
|
|
2008
|
+
const { result, pages, query, audience, baseUrl } = options;
|
|
2009
|
+
if (!isLocalProviderResult(result, baseUrl)) return false;
|
|
2010
|
+
const pagePath = normalizeUrlRouteKey(result.url);
|
|
2011
|
+
const page = pages.find((candidate) => normalizeUrlRouteKey(candidate.url) === pagePath);
|
|
1290
2012
|
if (!page) return false;
|
|
1291
2013
|
if (scoreDocument(query, pageToSearchDocument(page, audience)) > 0) return false;
|
|
1292
2014
|
const oppositeAudience = audience === "agent" ? "human" : "agent";
|
|
@@ -1306,13 +2028,14 @@ function hasOppositeAudienceEvidence(options) {
|
|
|
1306
2028
|
}
|
|
1307
2029
|
return oppositeOnlyMatches > selectedOnlyMatches;
|
|
1308
2030
|
}
|
|
1309
|
-
function pageToSearchDocument(
|
|
2031
|
+
function pageToSearchDocument(rawPage, audience = "human") {
|
|
2032
|
+
const page = localizeDocsSearchPage(rawPage, rawPage.locale);
|
|
1310
2033
|
const scope = resolveDocsSearchPageScope(page);
|
|
1311
2034
|
return {
|
|
1312
2035
|
id: makeDocumentId(page.url, "page"),
|
|
1313
2036
|
url: page.url,
|
|
1314
2037
|
title: page.title,
|
|
1315
|
-
content:
|
|
2038
|
+
content: getPageAudienceIndexContent(page, audience),
|
|
1316
2039
|
description: page.description,
|
|
1317
2040
|
type: "page",
|
|
1318
2041
|
locale: page.locale,
|
|
@@ -1375,6 +2098,23 @@ function buildAskAIContextBlock(result) {
|
|
|
1375
2098
|
const lines = [`## ${result.title}`, `URL: ${result.url}`];
|
|
1376
2099
|
if (result.section) lines.push(`Section: ${result.section}`);
|
|
1377
2100
|
if (result.description) lines.push(`Search snippet: ${result.description}`);
|
|
2101
|
+
if (result.source) {
|
|
2102
|
+
const scope = result.source.scope;
|
|
2103
|
+
const scopeEntries = [
|
|
2104
|
+
`audience=${scope.audience}`,
|
|
2105
|
+
scope.locale?.length ? `locale=${scope.locale.join(",")}` : void 0,
|
|
2106
|
+
scope.framework?.length ? `framework=${scope.framework.join(",")}` : void 0,
|
|
2107
|
+
scope.version?.length ? `version=${scope.version.join(",")}` : void 0,
|
|
2108
|
+
scope.package?.length ? `package=${scope.package.join(",")}` : void 0,
|
|
2109
|
+
scope.tags?.length ? `tags=${scope.tags.join(",")}` : void 0,
|
|
2110
|
+
scope.conflicts?.length ? `conflicts=${scope.conflicts.join(",")}` : void 0
|
|
2111
|
+
].filter((value) => Boolean(value));
|
|
2112
|
+
lines.push(`Canonical URL: ${result.source.canonicalUrl}`);
|
|
2113
|
+
lines.push(`Source scope: ${scopeEntries.join("; ")}`);
|
|
2114
|
+
if (result.source.lastModified) lines.push(`Source modified: ${result.source.lastModified}`);
|
|
2115
|
+
lines.push(`Source digest: ${result.source.digest}`);
|
|
2116
|
+
lines.push(`Index generation: ${result.source.indexGeneration}`);
|
|
2117
|
+
}
|
|
1378
2118
|
lines.push("", result.contextContent);
|
|
1379
2119
|
return lines.join("\n").trim();
|
|
1380
2120
|
}
|
|
@@ -1382,14 +2122,19 @@ function makeDocumentId(url, suffix) {
|
|
|
1382
2122
|
return `${url}#${suffix}`;
|
|
1383
2123
|
}
|
|
1384
2124
|
function splitPageIntoSections(page, audience = "human") {
|
|
1385
|
-
const raw =
|
|
2125
|
+
const raw = getPageAudienceSectionContent(page, audience);
|
|
2126
|
+
const generatedContract = audience === "agent" ? findDocsGeneratedAgentContractRanges(raw)[0] : void 0;
|
|
1386
2127
|
const scope = resolveDocsSearchPageScope(page);
|
|
1387
|
-
|
|
2128
|
+
let documentIndex = 0;
|
|
2129
|
+
return parseDocsMarkdownSections(raw).flatMap((section) => {
|
|
2130
|
+
if (generatedContract && section.startLine > generatedContract.startLine && section.startLine < generatedContract.endLine) return [];
|
|
1388
2131
|
const content = normalizeWhitespace(stripMarkdownText(section.content));
|
|
1389
2132
|
if (!content) return [];
|
|
2133
|
+
const index = documentIndex;
|
|
2134
|
+
documentIndex += 1;
|
|
1390
2135
|
return [{
|
|
1391
2136
|
id: makeDocumentId(page.url, `section-${index}`),
|
|
1392
|
-
url: `${page.url}#${section.anchor}`,
|
|
2137
|
+
url: `${page.url.split("#", 1)[0]}#${encodeURIComponent(section.anchor)}`,
|
|
1393
2138
|
title: page.title,
|
|
1394
2139
|
section: section.title,
|
|
1395
2140
|
content,
|
|
@@ -1405,7 +2150,8 @@ function splitPageIntoSections(page, audience = "human") {
|
|
|
1405
2150
|
}
|
|
1406
2151
|
function buildDocsSearchDocuments(pages, chunking = {}, audience = "human") {
|
|
1407
2152
|
const strategy = chunking.strategy ?? "section";
|
|
1408
|
-
return pages.flatMap((
|
|
2153
|
+
return pages.flatMap((rawPage) => {
|
|
2154
|
+
const page = localizeDocsSearchPage(rawPage, rawPage.locale);
|
|
1409
2155
|
const base = pageToSearchDocument(page, audience);
|
|
1410
2156
|
if (strategy === "page") return [base];
|
|
1411
2157
|
const sections = splitPageIntoSections(page, audience);
|
|
@@ -1511,50 +2257,60 @@ function cleanSearchResultText(value) {
|
|
|
1511
2257
|
if (!value) return void 0;
|
|
1512
2258
|
return normalizeWhitespace(stripHtml(stripMarkdownText(value))) || void 0;
|
|
1513
2259
|
}
|
|
1514
|
-
function
|
|
1515
|
-
|
|
1516
|
-
const encoder = new TextEncoder();
|
|
1517
|
-
if (encoder.encode(value).length <= maxBytes) return value;
|
|
1518
|
-
let low = 0;
|
|
1519
|
-
let high = value.length;
|
|
1520
|
-
let best = "";
|
|
1521
|
-
while (low <= high) {
|
|
1522
|
-
const mid = Math.floor((low + high) / 2);
|
|
1523
|
-
const next = `${value.slice(0, mid).trimEnd()}...`;
|
|
1524
|
-
if (encoder.encode(next).length <= maxBytes) {
|
|
1525
|
-
best = next;
|
|
1526
|
-
low = mid + 1;
|
|
1527
|
-
} else high = mid - 1;
|
|
1528
|
-
}
|
|
1529
|
-
return best;
|
|
1530
|
-
}
|
|
1531
|
-
function buildAlgoliaRecord(document) {
|
|
2260
|
+
function buildAlgoliaRecord(document, corpusId) {
|
|
2261
|
+
const providerDocumentId = corpusId ? makeHostedProviderDocumentId(corpusId, document.id) : document.id;
|
|
1532
2262
|
const record = {
|
|
1533
|
-
objectID:
|
|
1534
|
-
id:
|
|
2263
|
+
objectID: providerDocumentId,
|
|
2264
|
+
id: providerDocumentId,
|
|
1535
2265
|
url: document.url,
|
|
1536
2266
|
title: document.title,
|
|
1537
2267
|
section: document.section,
|
|
1538
2268
|
content: document.content,
|
|
1539
2269
|
description: document.description,
|
|
1540
2270
|
type: document.type,
|
|
1541
|
-
locale: document.locale,
|
|
2271
|
+
locale: document.locale ?? document.source?.scope.locale?.[0],
|
|
1542
2272
|
framework: document.framework,
|
|
1543
2273
|
version: document.version,
|
|
1544
2274
|
package: document.package,
|
|
1545
|
-
tags: document.tags
|
|
2275
|
+
tags: document.tags,
|
|
2276
|
+
_tags: corpusId ? [corpusId] : void 0,
|
|
2277
|
+
...serializeHostedRetrievalSource(document.source, corpusId, document.id)
|
|
1546
2278
|
};
|
|
1547
2279
|
const encoder = new TextEncoder();
|
|
1548
2280
|
const sizeOf = (value) => encoder.encode(JSON.stringify(value)).length;
|
|
1549
2281
|
if (sizeOf(record) <= ALGOLIA_MAX_RECORD_BYTES) return record;
|
|
1550
2282
|
delete record.description;
|
|
1551
2283
|
if (sizeOf(record) <= ALGOLIA_MAX_RECORD_BYTES) return record;
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
2284
|
+
record.content = "";
|
|
2285
|
+
for (const field of [
|
|
2286
|
+
"package",
|
|
2287
|
+
"tags",
|
|
2288
|
+
"framework",
|
|
2289
|
+
"version"
|
|
2290
|
+
]) {
|
|
2291
|
+
if (sizeOf(record) <= ALGOLIA_MAX_RECORD_BYTES) break;
|
|
2292
|
+
delete record[field];
|
|
2293
|
+
}
|
|
2294
|
+
const trimFieldToFit = (field, value) => {
|
|
2295
|
+
let low = 0;
|
|
2296
|
+
let high = value.length;
|
|
2297
|
+
let best = "";
|
|
2298
|
+
while (low <= high) {
|
|
2299
|
+
const middle = Math.floor((low + high) / 2);
|
|
2300
|
+
const candidate = middle < value.length ? `${value.slice(0, middle).trimEnd()}...` : value;
|
|
2301
|
+
record[field] = candidate;
|
|
2302
|
+
if (sizeOf(record) <= ALGOLIA_MAX_RECORD_BYTES) {
|
|
2303
|
+
best = candidate;
|
|
2304
|
+
low = middle + 1;
|
|
2305
|
+
} else high = middle - 1;
|
|
2306
|
+
}
|
|
2307
|
+
record[field] = best;
|
|
2308
|
+
};
|
|
2309
|
+
if (sizeOf(record) > ALGOLIA_MAX_RECORD_BYTES && typeof record.section === "string") trimFieldToFit("section", record.section);
|
|
2310
|
+
if (sizeOf(record) > ALGOLIA_MAX_RECORD_BYTES && typeof record.title === "string") trimFieldToFit("title", record.title);
|
|
2311
|
+
if (sizeOf(record) > ALGOLIA_MAX_RECORD_BYTES) throw new Error(`Algolia record ${document.id} exceeds the record limit with required provenance.`);
|
|
2312
|
+
trimFieldToFit("content", document.content);
|
|
2313
|
+
if (sizeOf(record) > ALGOLIA_MAX_RECORD_BYTES) throw new Error(`Algolia record ${document.id} still exceeds the record limit after trimming.`);
|
|
1558
2314
|
return record;
|
|
1559
2315
|
}
|
|
1560
2316
|
function createSimpleSearchAdapter() {
|
|
@@ -1562,7 +2318,7 @@ function createSimpleSearchAdapter() {
|
|
|
1562
2318
|
name: "simple",
|
|
1563
2319
|
async search(query, context) {
|
|
1564
2320
|
const limit = query.limit ?? DEFAULT_SEARCH_LIMIT;
|
|
1565
|
-
|
|
2321
|
+
const results = context.documents.map((document) => ({
|
|
1566
2322
|
document,
|
|
1567
2323
|
score: scoreDocument(query.query, document)
|
|
1568
2324
|
})).filter((item) => item.score > 0).sort((a, b) => {
|
|
@@ -1575,8 +2331,20 @@ function createSimpleSearchAdapter() {
|
|
|
1575
2331
|
description: cleanSearchResultText(buildSnippet(document, query.query) ?? document.description),
|
|
1576
2332
|
type: document.type,
|
|
1577
2333
|
score,
|
|
1578
|
-
section: document.section
|
|
2334
|
+
section: document.section,
|
|
2335
|
+
source: document.source
|
|
1579
2336
|
}));
|
|
2337
|
+
if (context.deferSourceProvenance) return results;
|
|
2338
|
+
return enrichDocsSearchResultsWithSources({
|
|
2339
|
+
results,
|
|
2340
|
+
pages: context.pages,
|
|
2341
|
+
audience: resolveDocsSearchAudience(context.audience),
|
|
2342
|
+
chunking: context.chunking ?? { strategy: "section" },
|
|
2343
|
+
locale: query.locale ?? context.locale,
|
|
2344
|
+
baseUrl: context.baseUrl,
|
|
2345
|
+
indexGeneration: context.indexGeneration,
|
|
2346
|
+
filters: query.filters
|
|
2347
|
+
});
|
|
1580
2348
|
}
|
|
1581
2349
|
};
|
|
1582
2350
|
}
|
|
@@ -1629,7 +2397,7 @@ function resolveMcpEndpoint(endpoint) {
|
|
|
1629
2397
|
function isDocsSearchResultType(value) {
|
|
1630
2398
|
return value === "page" || value === "heading" || value === "text";
|
|
1631
2399
|
}
|
|
1632
|
-
function mapMcpSearchResult(value) {
|
|
2400
|
+
function mapMcpSearchResult(value, sourceBaseUrl) {
|
|
1633
2401
|
if (!value || typeof value !== "object") return null;
|
|
1634
2402
|
const item = value;
|
|
1635
2403
|
const section = typeof item.section === "string" ? item.section : void 0;
|
|
@@ -1637,6 +2405,19 @@ function mapMcpSearchResult(value) {
|
|
|
1637
2405
|
const content = typeof item.content === "string" ? item.content : title ? section ? `${title} — ${section}` : title : void 0;
|
|
1638
2406
|
const url = typeof item.url === "string" ? item.url : void 0;
|
|
1639
2407
|
if (!content || !url) return null;
|
|
2408
|
+
const hasSource = item.source !== void 0;
|
|
2409
|
+
let sourceValue = item.source;
|
|
2410
|
+
if (hasSource && sourceBaseUrl && sourceValue && typeof sourceValue === "object" && !Array.isArray(sourceValue)) {
|
|
2411
|
+
const sourceRecord = sourceValue;
|
|
2412
|
+
if (typeof sourceRecord.canonicalUrl === "string" && /^\/(?!\/)/u.test(sourceRecord.canonicalUrl)) try {
|
|
2413
|
+
sourceValue = {
|
|
2414
|
+
...sourceRecord,
|
|
2415
|
+
canonicalUrl: new URL(sourceRecord.canonicalUrl, sourceBaseUrl).toString()
|
|
2416
|
+
};
|
|
2417
|
+
} catch {}
|
|
2418
|
+
}
|
|
2419
|
+
const source = hasSource ? parseDocsRetrievalSource(sourceValue) : void 0;
|
|
2420
|
+
if (hasSource && !source) return null;
|
|
1640
2421
|
return {
|
|
1641
2422
|
id: typeof item.id === "string" ? item.id : typeof item.slug === "string" ? item.slug : url,
|
|
1642
2423
|
url,
|
|
@@ -1644,7 +2425,8 @@ function mapMcpSearchResult(value) {
|
|
|
1644
2425
|
description: cleanSearchResultText(typeof item.description === "string" ? item.description : typeof item.excerpt === "string" ? item.excerpt : void 0) ?? void 0,
|
|
1645
2426
|
type: isDocsSearchResultType(item.type) ? item.type : section ? "heading" : "page",
|
|
1646
2427
|
score: typeof item.score === "number" ? item.score : void 0,
|
|
1647
|
-
section
|
|
2428
|
+
section,
|
|
2429
|
+
...source ? { source } : {}
|
|
1648
2430
|
};
|
|
1649
2431
|
}
|
|
1650
2432
|
function readMcpSearchToolPayload(payload) {
|
|
@@ -1677,7 +2459,84 @@ function getTypesenseSearchBase(config) {
|
|
|
1677
2459
|
function quoteTypesenseFilterValue(value) {
|
|
1678
2460
|
return `\`${value.replace(/\\/gu, "\\\\").replace(/`/gu, "\\`")}\``;
|
|
1679
2461
|
}
|
|
1680
|
-
|
|
2462
|
+
const TYPESENSE_RETRIEVAL_SOURCE_FIELDS = [
|
|
2463
|
+
{
|
|
2464
|
+
name: "source_corpus_id",
|
|
2465
|
+
type: "string",
|
|
2466
|
+
optional: true
|
|
2467
|
+
},
|
|
2468
|
+
{
|
|
2469
|
+
name: "source_document_id",
|
|
2470
|
+
type: "string",
|
|
2471
|
+
optional: true
|
|
2472
|
+
},
|
|
2473
|
+
{
|
|
2474
|
+
name: "source_canonical_url",
|
|
2475
|
+
type: "string",
|
|
2476
|
+
optional: true
|
|
2477
|
+
},
|
|
2478
|
+
{
|
|
2479
|
+
name: "source_scope_audience",
|
|
2480
|
+
type: "string",
|
|
2481
|
+
optional: true
|
|
2482
|
+
},
|
|
2483
|
+
{
|
|
2484
|
+
name: "source_scope_locale",
|
|
2485
|
+
type: "string[]",
|
|
2486
|
+
optional: true
|
|
2487
|
+
},
|
|
2488
|
+
{
|
|
2489
|
+
name: "source_scope_framework",
|
|
2490
|
+
type: "string[]",
|
|
2491
|
+
optional: true
|
|
2492
|
+
},
|
|
2493
|
+
{
|
|
2494
|
+
name: "source_scope_version",
|
|
2495
|
+
type: "string[]",
|
|
2496
|
+
optional: true
|
|
2497
|
+
},
|
|
2498
|
+
{
|
|
2499
|
+
name: "source_scope_version_groups",
|
|
2500
|
+
type: "string",
|
|
2501
|
+
optional: true
|
|
2502
|
+
},
|
|
2503
|
+
{
|
|
2504
|
+
name: "source_scope_package",
|
|
2505
|
+
type: "string[]",
|
|
2506
|
+
optional: true
|
|
2507
|
+
},
|
|
2508
|
+
{
|
|
2509
|
+
name: "source_scope_tags",
|
|
2510
|
+
type: "string[]",
|
|
2511
|
+
optional: true
|
|
2512
|
+
},
|
|
2513
|
+
{
|
|
2514
|
+
name: "source_scope_truncated",
|
|
2515
|
+
type: "string[]",
|
|
2516
|
+
optional: true
|
|
2517
|
+
},
|
|
2518
|
+
{
|
|
2519
|
+
name: "source_scope_conflicts",
|
|
2520
|
+
type: "string[]",
|
|
2521
|
+
optional: true
|
|
2522
|
+
},
|
|
2523
|
+
{
|
|
2524
|
+
name: "source_last_modified",
|
|
2525
|
+
type: "string",
|
|
2526
|
+
optional: true
|
|
2527
|
+
},
|
|
2528
|
+
{
|
|
2529
|
+
name: "source_digest",
|
|
2530
|
+
type: "string",
|
|
2531
|
+
optional: true
|
|
2532
|
+
},
|
|
2533
|
+
{
|
|
2534
|
+
name: "source_index_generation",
|
|
2535
|
+
type: "string",
|
|
2536
|
+
optional: true
|
|
2537
|
+
}
|
|
2538
|
+
];
|
|
2539
|
+
async function ensureTypesenseCollection(config, dimensions, signal, retryAfterCreateConflict = true, retryAfterAlterFailure = true) {
|
|
1681
2540
|
const baseUrl = getTypesenseSearchBase(config);
|
|
1682
2541
|
const headers = {
|
|
1683
2542
|
"X-TYPESENSE-API-KEY": config.adminApiKey ?? config.apiKey,
|
|
@@ -1687,7 +2546,50 @@ async function ensureTypesenseCollection(config, dimensions, signal) {
|
|
|
1687
2546
|
headers,
|
|
1688
2547
|
signal
|
|
1689
2548
|
});
|
|
1690
|
-
if (existing.ok)
|
|
2549
|
+
if (existing.ok) {
|
|
2550
|
+
const existingPayload = await readResponseJson(existing);
|
|
2551
|
+
if (!Array.isArray(existingPayload?.fields)) throw new Error("Typesense collection response did not include a fields array.");
|
|
2552
|
+
const existingFields = new Map(existingPayload.fields.flatMap((field) => typeof field.name === "string" ? [[field.name, field]] : []));
|
|
2553
|
+
const incompatibleFields = TYPESENSE_RETRIEVAL_SOURCE_FIELDS.flatMap((expected) => {
|
|
2554
|
+
const name = typeof expected.name === "string" ? expected.name : void 0;
|
|
2555
|
+
const existingField = name ? existingFields.get(name) : void 0;
|
|
2556
|
+
if (!name || !existingField) return [];
|
|
2557
|
+
return existingField.type === expected.type && existingField.optional === true ? [] : [name];
|
|
2558
|
+
});
|
|
2559
|
+
if (incompatibleFields.length > 0) throw new Error(`Typesense collection has incompatible provenance fields: ${incompatibleFields.join(", ")}.`);
|
|
2560
|
+
const missingFields = TYPESENSE_RETRIEVAL_SOURCE_FIELDS.filter((field) => typeof field.name === "string" && !existingFields.has(field.name));
|
|
2561
|
+
const embeddingField = existingPayload.fields.find((field) => field.name === "embedding");
|
|
2562
|
+
const expectedEmbeddingField = config.embeddings && dimensions ? {
|
|
2563
|
+
name: "embedding",
|
|
2564
|
+
type: "float[]",
|
|
2565
|
+
num_dim: dimensions,
|
|
2566
|
+
optional: true
|
|
2567
|
+
} : void 0;
|
|
2568
|
+
const replaceEmbedding = Boolean(expectedEmbeddingField && embeddingField && (embeddingField.type !== "float[]" || embeddingField.num_dim !== dimensions));
|
|
2569
|
+
if (replaceEmbedding) ensureOk(await fetch(`${baseUrl}/collections/${encodeURIComponent(config.collection)}`, {
|
|
2570
|
+
method: "PATCH",
|
|
2571
|
+
headers,
|
|
2572
|
+
body: JSON.stringify({ fields: [{
|
|
2573
|
+
name: "embedding",
|
|
2574
|
+
drop: true
|
|
2575
|
+
}] }),
|
|
2576
|
+
signal
|
|
2577
|
+
}), "Failed to replace an incompatible Typesense embedding field");
|
|
2578
|
+
const fieldsToAdd = [...missingFields, ...expectedEmbeddingField && (!embeddingField || replaceEmbedding) ? [expectedEmbeddingField] : []];
|
|
2579
|
+
if (fieldsToAdd.length === 0) return;
|
|
2580
|
+
const altered = await fetch(`${baseUrl}/collections/${encodeURIComponent(config.collection)}`, {
|
|
2581
|
+
method: "PATCH",
|
|
2582
|
+
headers,
|
|
2583
|
+
body: JSON.stringify({ fields: fieldsToAdd }),
|
|
2584
|
+
signal
|
|
2585
|
+
});
|
|
2586
|
+
if (!altered.ok && retryAfterAlterFailure) {
|
|
2587
|
+
await ensureTypesenseCollection(config, dimensions, signal, retryAfterCreateConflict, false);
|
|
2588
|
+
return;
|
|
2589
|
+
}
|
|
2590
|
+
ensureOk(altered, "Failed to update the Typesense search schema");
|
|
2591
|
+
return;
|
|
2592
|
+
}
|
|
1691
2593
|
if (existing.status !== 404) ensureOk(existing, "Failed to inspect Typesense collection");
|
|
1692
2594
|
const fields = [
|
|
1693
2595
|
{
|
|
@@ -1744,7 +2646,8 @@ async function ensureTypesenseCollection(config, dimensions, signal) {
|
|
|
1744
2646
|
name: "tags",
|
|
1745
2647
|
type: "string[]",
|
|
1746
2648
|
optional: true
|
|
1747
|
-
}
|
|
2649
|
+
},
|
|
2650
|
+
...TYPESENSE_RETRIEVAL_SOURCE_FIELDS
|
|
1748
2651
|
];
|
|
1749
2652
|
if (config.embeddings && dimensions) fields.push({
|
|
1750
2653
|
name: "embedding",
|
|
@@ -1752,7 +2655,7 @@ async function ensureTypesenseCollection(config, dimensions, signal) {
|
|
|
1752
2655
|
num_dim: dimensions,
|
|
1753
2656
|
optional: true
|
|
1754
2657
|
});
|
|
1755
|
-
|
|
2658
|
+
const response = await fetch(`${baseUrl}/collections`, {
|
|
1756
2659
|
method: "POST",
|
|
1757
2660
|
headers,
|
|
1758
2661
|
body: JSON.stringify({
|
|
@@ -1760,34 +2663,66 @@ async function ensureTypesenseCollection(config, dimensions, signal) {
|
|
|
1760
2663
|
fields
|
|
1761
2664
|
}),
|
|
1762
2665
|
signal
|
|
1763
|
-
})
|
|
2666
|
+
});
|
|
2667
|
+
if (response.status === 409 && retryAfterCreateConflict) {
|
|
2668
|
+
await ensureTypesenseCollection(config, dimensions, signal, false, retryAfterAlterFailure);
|
|
2669
|
+
return;
|
|
2670
|
+
}
|
|
2671
|
+
ensureOk(response, "Failed to create Typesense collection");
|
|
2672
|
+
}
|
|
2673
|
+
async function ensureTypesenseImportSucceeded(response, expectedRecords) {
|
|
2674
|
+
ensureOk(response, "Failed to sync documents to Typesense");
|
|
2675
|
+
const lines = (await response.text()).split(/\r?\n/u).filter((line) => line.trim());
|
|
2676
|
+
if (lines.length !== expectedRecords) throw new Error(`Typesense acknowledged ${lines.length} of ${expectedRecords} imported records.`);
|
|
2677
|
+
for (const [index, line] of lines.entries()) {
|
|
2678
|
+
let result;
|
|
2679
|
+
try {
|
|
2680
|
+
result = JSON.parse(line);
|
|
2681
|
+
} catch {
|
|
2682
|
+
throw new Error(`Typesense import returned invalid JSON on record ${index + 1}.`);
|
|
2683
|
+
}
|
|
2684
|
+
if (result.success !== true) {
|
|
2685
|
+
const detail = typeof result.error === "string" ? `: ${result.error}` : "";
|
|
2686
|
+
throw new Error(`Typesense failed to import record ${index + 1}${detail}`);
|
|
2687
|
+
}
|
|
2688
|
+
}
|
|
1764
2689
|
}
|
|
1765
2690
|
function createTypesenseSearchAdapter(config) {
|
|
1766
2691
|
return {
|
|
1767
2692
|
name: "typesense",
|
|
1768
2693
|
async index(context) {
|
|
1769
2694
|
const adminApiKey = config.adminApiKey ?? config.apiKey;
|
|
1770
|
-
const
|
|
2695
|
+
const corpusId = resolveHostedHumanCorpusId(config.syncNamespace, context);
|
|
2696
|
+
const documents = await enrichDocsSearchDocumentsWithSources({
|
|
2697
|
+
documents: context.documents,
|
|
2698
|
+
pages: context.pages,
|
|
2699
|
+
audience: resolveDocsSearchAudience(context.audience),
|
|
2700
|
+
chunking: context.chunking ?? { strategy: "section" },
|
|
2701
|
+
locale: context.locale,
|
|
2702
|
+
baseUrl: context.baseUrl,
|
|
2703
|
+
indexGeneration: context.indexGeneration
|
|
2704
|
+
});
|
|
2705
|
+
const docsForImport = await Promise.all(documents.map(async (document) => {
|
|
1771
2706
|
const next = {
|
|
1772
|
-
id: document.id,
|
|
2707
|
+
id: corpusId ? makeHostedProviderDocumentId(corpusId, document.id) : document.id,
|
|
1773
2708
|
url: document.url,
|
|
1774
2709
|
title: document.title,
|
|
1775
2710
|
section: document.section,
|
|
1776
2711
|
content: document.content,
|
|
1777
2712
|
description: document.description,
|
|
1778
2713
|
type: document.type,
|
|
1779
|
-
locale: document.locale,
|
|
2714
|
+
locale: document.locale ?? document.source?.scope.locale?.[0],
|
|
1780
2715
|
framework: document.framework,
|
|
1781
2716
|
version: document.version,
|
|
1782
2717
|
package: document.package,
|
|
1783
|
-
tags: document.tags
|
|
2718
|
+
tags: document.tags,
|
|
2719
|
+
...serializeHostedRetrievalSource(document.source, corpusId, document.id)
|
|
1784
2720
|
};
|
|
1785
2721
|
if (config.mode === "hybrid" && config.embeddings) next.embedding = await createOllamaEmbedding(`${document.title}\n${document.section ?? ""}\n${document.content}`.trim(), config.embeddings, context.signal);
|
|
1786
2722
|
return next;
|
|
1787
2723
|
}));
|
|
1788
|
-
if (docsForImport.length === 0) return;
|
|
1789
2724
|
await ensureTypesenseCollection(config, Array.isArray(docsForImport[0]?.embedding) ? docsForImport[0].embedding.length : void 0, context.signal);
|
|
1790
|
-
|
|
2725
|
+
if (docsForImport.length > 0) await ensureTypesenseImportSucceeded(await fetch(`${getTypesenseSearchBase(config)}/collections/${encodeURIComponent(config.collection)}/documents/import?action=upsert`, {
|
|
1791
2726
|
method: "POST",
|
|
1792
2727
|
headers: {
|
|
1793
2728
|
"X-TYPESENSE-API-KEY": adminApiKey,
|
|
@@ -1795,10 +2730,11 @@ function createTypesenseSearchAdapter(config) {
|
|
|
1795
2730
|
},
|
|
1796
2731
|
body: docsForImport.map((document) => JSON.stringify(document)).join("\n"),
|
|
1797
2732
|
signal: context.signal
|
|
1798
|
-
}),
|
|
2733
|
+
}), docsForImport.length);
|
|
1799
2734
|
},
|
|
1800
2735
|
async search(query, context) {
|
|
1801
|
-
const
|
|
2736
|
+
const corpusId = resolveHostedHumanCorpusId(config.syncNamespace, context);
|
|
2737
|
+
const scopedDocumentIds = resolveProviderScopeDocumentIds(query, context, corpusId);
|
|
1802
2738
|
if (scopedDocumentIds?.length === 0) return [];
|
|
1803
2739
|
const params = new URLSearchParams({
|
|
1804
2740
|
q: query.query,
|
|
@@ -1817,7 +2753,8 @@ function createTypesenseSearchAdapter(config) {
|
|
|
1817
2753
|
const vector = await createOllamaEmbedding(query.query, config.embeddings, context.signal);
|
|
1818
2754
|
params.set("vector_query", `embedding:([${vector.join(",")}],k:${Math.max((query.limit ?? 10) * 4, 20)})`);
|
|
1819
2755
|
}
|
|
1820
|
-
const
|
|
2756
|
+
const filterClauses = [corpusId ? `source_corpus_id:=${quoteTypesenseFilterValue(corpusId)}` : void 0, scopedDocumentIds ? `id:=[${scopedDocumentIds.map(quoteTypesenseFilterValue).join(",")}]` : void 0].filter((value) => Boolean(value));
|
|
2757
|
+
const filterBy = filterClauses.length > 0 ? filterClauses.join(" && ") : void 0;
|
|
1821
2758
|
if (filterBy && filterBy.length > MAX_PROVIDER_SCOPE_FILTER_CHARS) return [];
|
|
1822
2759
|
const response = filterBy ? await fetch(`${getTypesenseSearchBase(config)}/multi_search`, {
|
|
1823
2760
|
method: "POST",
|
|
@@ -1837,13 +2774,16 @@ function createTypesenseSearchAdapter(config) {
|
|
|
1837
2774
|
});
|
|
1838
2775
|
ensureOk(response, "Typesense search failed");
|
|
1839
2776
|
const rawPayload = await readResponseJson(response);
|
|
1840
|
-
return ((filterBy ? rawPayload.results?.[0] ?? {} : rawPayload).hits ?? []).
|
|
2777
|
+
return ((filterBy ? rawPayload.results?.[0] ?? {} : rawPayload).hits ?? []).flatMap((hit) => {
|
|
1841
2778
|
const document = hit.document ?? {};
|
|
2779
|
+
if (!hostedRecordMatchesCorpus(document, corpusId)) return [];
|
|
1842
2780
|
const section = typeof document.section === "string" ? document.section : void 0;
|
|
1843
2781
|
const content = typeof document.title === "string" ? section ? `${document.title} — ${section}` : document.title : typeof document.content === "string" ? document.content : "Untitled result";
|
|
1844
2782
|
const description = hit.highlights?.find((item) => item.field === "content")?.snippet ?? hit.highlights?.find((item) => item.field === "description")?.snippet ?? (typeof document.description === "string" ? document.description : void 0);
|
|
1845
|
-
|
|
1846
|
-
|
|
2783
|
+
const source = readHostedRetrievalSource(document);
|
|
2784
|
+
if (hasHostedRetrievalSource(document) && !source) return [];
|
|
2785
|
+
return [{
|
|
2786
|
+
id: readHostedSourceDocumentId(document) ?? (typeof document.id === "string" ? document.id : String(document.url ?? content)),
|
|
1847
2787
|
url: typeof document.url === "string" ? document.url : "/docs",
|
|
1848
2788
|
content: cleanSearchResultText(content) ?? content,
|
|
1849
2789
|
description: cleanSearchResultText(description),
|
|
@@ -1853,8 +2793,9 @@ function createTypesenseSearchAdapter(config) {
|
|
|
1853
2793
|
"text"
|
|
1854
2794
|
].includes(document.type) ? document.type : section ? "heading" : "page",
|
|
1855
2795
|
score: hit.text_match,
|
|
1856
|
-
section
|
|
1857
|
-
|
|
2796
|
+
section,
|
|
2797
|
+
...source ? { source } : {}
|
|
2798
|
+
}];
|
|
1858
2799
|
});
|
|
1859
2800
|
}
|
|
1860
2801
|
};
|
|
@@ -1968,7 +2909,7 @@ function createMcpSearchAdapter(config) {
|
|
|
1968
2909
|
const echoedFilters = parseVerifiedMcpSearchFilters(parsed.filters);
|
|
1969
2910
|
if (!echoedFilters || !docsSearchFiltersMatch(query.filters ?? {}, echoedFilters)) return [];
|
|
1970
2911
|
}
|
|
1971
|
-
return (Array.isArray(parsed) ? parsed : Array.isArray(parsed.results) ? parsed.results : Array.isArray(parsed.pages) ? parsed.pages : []).map(mapMcpSearchResult).filter((result) => Boolean(result));
|
|
2912
|
+
return (Array.isArray(parsed) ? parsed : Array.isArray(parsed.results) ? parsed.results : Array.isArray(parsed.pages) ? parsed.pages : []).map((result) => mapMcpSearchResult(result, endpoint)).filter((result) => Boolean(result));
|
|
1972
2913
|
} finally {
|
|
1973
2914
|
if (sessionId) {
|
|
1974
2915
|
const cleanupController = new AbortController();
|
|
@@ -1991,9 +2932,62 @@ function createMcpSearchAdapter(config) {
|
|
|
1991
2932
|
}
|
|
1992
2933
|
};
|
|
1993
2934
|
}
|
|
1994
|
-
function
|
|
2935
|
+
function getAlgoliaSearchBase(config) {
|
|
1995
2936
|
return `https://${config.appId}-dsn.algolia.net`;
|
|
1996
2937
|
}
|
|
2938
|
+
function getAlgoliaAdminBase(config) {
|
|
2939
|
+
return `https://${config.appId}.algolia.net`;
|
|
2940
|
+
}
|
|
2941
|
+
function getAlgoliaAdminHeaders(config) {
|
|
2942
|
+
return {
|
|
2943
|
+
"Content-Type": "application/json",
|
|
2944
|
+
"X-Algolia-Application-Id": config.appId,
|
|
2945
|
+
"X-Algolia-API-Key": config.adminApiKey ?? config.searchApiKey
|
|
2946
|
+
};
|
|
2947
|
+
}
|
|
2948
|
+
async function waitForAlgoliaTask(config, taskId, signal) {
|
|
2949
|
+
const deadline = Date.now() + 6e4;
|
|
2950
|
+
for (let attempt = 0; Date.now() < deadline; attempt += 1) {
|
|
2951
|
+
const response = await fetch(`${getAlgoliaAdminBase(config)}/1/indexes/${encodeURIComponent(config.indexName)}/task/${encodeURIComponent(String(taskId))}`, {
|
|
2952
|
+
headers: getAlgoliaAdminHeaders(config),
|
|
2953
|
+
signal
|
|
2954
|
+
});
|
|
2955
|
+
ensureOk(response, "Failed to inspect Algolia sync task");
|
|
2956
|
+
const payload = await readResponseJson(response);
|
|
2957
|
+
if (payload?.status === "published") return;
|
|
2958
|
+
if (payload?.status !== "notPublished") throw new Error("Algolia sync task returned an unknown status.");
|
|
2959
|
+
const delay = Math.min(100 * 2 ** Math.min(attempt, 4), 2e3);
|
|
2960
|
+
await new Promise((resolve, reject) => {
|
|
2961
|
+
let timer;
|
|
2962
|
+
const onAbort = () => {
|
|
2963
|
+
clearTimeout(timer);
|
|
2964
|
+
reject(signal?.reason ?? /* @__PURE__ */ new Error("Algolia sync was aborted."));
|
|
2965
|
+
};
|
|
2966
|
+
timer = setTimeout(() => {
|
|
2967
|
+
signal?.removeEventListener("abort", onAbort);
|
|
2968
|
+
resolve();
|
|
2969
|
+
}, Math.min(delay, Math.max(1, deadline - Date.now())));
|
|
2970
|
+
if (signal?.aborted) onAbort();
|
|
2971
|
+
else signal?.addEventListener("abort", onAbort, { once: true });
|
|
2972
|
+
});
|
|
2973
|
+
}
|
|
2974
|
+
throw new Error("Timed out waiting for Algolia to publish the synced index.");
|
|
2975
|
+
}
|
|
2976
|
+
async function executeAlgoliaBatch(config, requests, message, signal) {
|
|
2977
|
+
for (let offset = 0; offset < requests.length; offset += ALGOLIA_BATCH_OPERATIONS) {
|
|
2978
|
+
const batch = requests.slice(offset, offset + ALGOLIA_BATCH_OPERATIONS);
|
|
2979
|
+
const response = await fetch(`${getAlgoliaAdminBase(config)}/1/indexes/${encodeURIComponent(config.indexName)}/batch`, {
|
|
2980
|
+
method: "POST",
|
|
2981
|
+
headers: getAlgoliaAdminHeaders(config),
|
|
2982
|
+
body: JSON.stringify({ requests: batch }),
|
|
2983
|
+
signal
|
|
2984
|
+
});
|
|
2985
|
+
ensureOk(response, message);
|
|
2986
|
+
const payload = await readResponseJson(response);
|
|
2987
|
+
if (typeof payload?.taskID !== "string" && typeof payload?.taskID !== "number") throw new Error("Algolia sync response did not include a task ID.");
|
|
2988
|
+
await waitForAlgoliaTask(config, payload.taskID, signal);
|
|
2989
|
+
}
|
|
2990
|
+
}
|
|
1997
2991
|
function quoteAlgoliaFilterValue(value) {
|
|
1998
2992
|
return `"${value.replace(/\\/gu, "\\\\").replace(/"/gu, "\\\"")}"`;
|
|
1999
2993
|
}
|
|
@@ -2002,26 +2996,29 @@ function createAlgoliaSearchAdapter(config) {
|
|
|
2002
2996
|
name: "algolia",
|
|
2003
2997
|
async index(context) {
|
|
2004
2998
|
if (!config.adminApiKey) return;
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
}), "Failed to sync documents to Algolia");
|
|
2999
|
+
const corpusId = resolveHostedHumanCorpusId(config.syncNamespace, context);
|
|
3000
|
+
await executeAlgoliaBatch(config, (await enrichDocsSearchDocumentsWithSources({
|
|
3001
|
+
documents: context.documents,
|
|
3002
|
+
pages: context.pages,
|
|
3003
|
+
audience: resolveDocsSearchAudience(context.audience),
|
|
3004
|
+
chunking: context.chunking ?? { strategy: "section" },
|
|
3005
|
+
locale: context.locale,
|
|
3006
|
+
baseUrl: context.baseUrl,
|
|
3007
|
+
indexGeneration: context.indexGeneration
|
|
3008
|
+
})).map((document) => ({
|
|
3009
|
+
action: "addObject",
|
|
3010
|
+
body: buildAlgoliaRecord(document, corpusId)
|
|
3011
|
+
})), "Failed to sync documents to Algolia", context.signal);
|
|
2018
3012
|
},
|
|
2019
3013
|
async search(query, context) {
|
|
2020
|
-
const
|
|
3014
|
+
const corpusId = resolveHostedHumanCorpusId(config.syncNamespace, context);
|
|
3015
|
+
const scopedDocumentIds = resolveProviderScopeDocumentIds(query, context, corpusId);
|
|
2021
3016
|
if (scopedDocumentIds?.length === 0) return [];
|
|
2022
|
-
const
|
|
3017
|
+
const scopedIdsFilter = scopedDocumentIds?.map((id) => `objectID:${quoteAlgoliaFilterValue(id)}`).join(" OR ");
|
|
3018
|
+
const filterClauses = [corpusId ? `_tags:${quoteAlgoliaFilterValue(corpusId)}` : void 0, scopedIdsFilter ? corpusId ? `(${scopedIdsFilter})` : scopedIdsFilter : void 0].filter((value) => Boolean(value));
|
|
3019
|
+
const filters = filterClauses.length > 0 ? filterClauses.join(" AND ") : void 0;
|
|
2023
3020
|
if (filters && filters.length > MAX_PROVIDER_SCOPE_FILTER_CHARS) return [];
|
|
2024
|
-
const response = await fetch(`${
|
|
3021
|
+
const response = await fetch(`${getAlgoliaSearchBase(config)}/1/indexes/${encodeURIComponent(config.indexName)}/query`, {
|
|
2025
3022
|
method: "POST",
|
|
2026
3023
|
headers: {
|
|
2027
3024
|
"Content-Type": "application/json",
|
|
@@ -2031,17 +3028,26 @@ function createAlgoliaSearchAdapter(config) {
|
|
|
2031
3028
|
body: JSON.stringify({
|
|
2032
3029
|
query: query.query,
|
|
2033
3030
|
hitsPerPage: query.limit ?? config.maxResults ?? DEFAULT_SEARCH_LIMIT,
|
|
3031
|
+
restrictSearchableAttributes: [
|
|
3032
|
+
"title",
|
|
3033
|
+
"section",
|
|
3034
|
+
"content",
|
|
3035
|
+
"description"
|
|
3036
|
+
],
|
|
2034
3037
|
attributesToSnippet: ["content:20"],
|
|
2035
3038
|
...filters ? { filters } : {}
|
|
2036
3039
|
}),
|
|
2037
3040
|
signal: context.signal
|
|
2038
3041
|
});
|
|
2039
3042
|
ensureOk(response, "Algolia search failed");
|
|
2040
|
-
return ((await readResponseJson(response)).hits ?? []).
|
|
3043
|
+
return ((await readResponseJson(response)).hits ?? []).flatMap((hit) => {
|
|
3044
|
+
if (!hostedRecordMatchesCorpus(hit, corpusId)) return [];
|
|
2041
3045
|
const title = typeof hit.title === "string" ? hit.title : "Untitled result";
|
|
2042
3046
|
const section = typeof hit.section === "string" ? hit.section : void 0;
|
|
2043
|
-
|
|
2044
|
-
|
|
3047
|
+
const source = readHostedRetrievalSource(hit);
|
|
3048
|
+
if (hasHostedRetrievalSource(hit) && !source) return [];
|
|
3049
|
+
return [{
|
|
3050
|
+
id: readHostedSourceDocumentId(hit) ?? hit.objectID ?? String(hit.url ?? title),
|
|
2045
3051
|
url: typeof hit.url === "string" ? hit.url : "/docs",
|
|
2046
3052
|
content: cleanSearchResultText(section ? `${title} — ${section}` : title) ?? title,
|
|
2047
3053
|
description: cleanSearchResultText(hit._snippetResult?.content?.value ?? hit._snippetResult?.description?.value ?? (typeof hit.description === "string" ? hit.description : void 0)),
|
|
@@ -2051,8 +3057,9 @@ function createAlgoliaSearchAdapter(config) {
|
|
|
2051
3057
|
"text"
|
|
2052
3058
|
].includes(hit.type) ? hit.type : section ? "heading" : "page",
|
|
2053
3059
|
score: hit._rankingInfo?.nbTypos != null ? 100 - hit._rankingInfo.nbTypos : void 0,
|
|
2054
|
-
section
|
|
2055
|
-
|
|
3060
|
+
section,
|
|
3061
|
+
...source ? { source } : {}
|
|
3062
|
+
}];
|
|
2056
3063
|
});
|
|
2057
3064
|
}
|
|
2058
3065
|
};
|
|
@@ -2073,17 +3080,71 @@ function shouldSyncOnSearch(search) {
|
|
|
2073
3080
|
}
|
|
2074
3081
|
function getSyncKey(search, context) {
|
|
2075
3082
|
const raw = search.raw;
|
|
2076
|
-
if (search.provider === "algolia" && raw?.provider === "algolia")
|
|
2077
|
-
|
|
3083
|
+
if (search.provider === "algolia" && raw?.provider === "algolia") {
|
|
3084
|
+
const corpusId = resolveHostedHumanCorpusId(raw.syncNamespace, context) ?? "__unowned__";
|
|
3085
|
+
return `algolia:${raw.appId}:${raw.indexName}:${corpusId}:${context.locale ?? "__default__"}`;
|
|
3086
|
+
}
|
|
3087
|
+
if (search.provider === "typesense" && raw?.provider === "typesense") {
|
|
3088
|
+
const corpusId = resolveHostedHumanCorpusId(raw.syncNamespace, context) ?? "__unowned__";
|
|
3089
|
+
return `typesense:${raw.baseUrl}:${raw.collection}:${corpusId}:${context.locale ?? "__default__"}`;
|
|
3090
|
+
}
|
|
2078
3091
|
if (search.provider === "mcp" && raw?.provider === "mcp") return `mcp:${raw.endpoint}:${context.locale ?? "__default__"}`;
|
|
2079
3092
|
return `${search.provider}:${context.locale ?? "__default__"}`;
|
|
2080
3093
|
}
|
|
3094
|
+
function getSearchSyncFingerprint(search, context, indexGeneration) {
|
|
3095
|
+
const raw = search.raw;
|
|
3096
|
+
const providerConfig = search.provider === "typesense" && raw?.provider === "typesense" ? {
|
|
3097
|
+
syncNamespace: raw.syncNamespace?.trim() || void 0,
|
|
3098
|
+
mode: raw.mode ?? "keyword",
|
|
3099
|
+
embeddings: raw.embeddings ? {
|
|
3100
|
+
provider: raw.embeddings.provider,
|
|
3101
|
+
model: raw.embeddings.model,
|
|
3102
|
+
baseUrl: raw.embeddings.baseUrl
|
|
3103
|
+
} : void 0
|
|
3104
|
+
} : search.provider === "algolia" && raw?.provider === "algolia" ? { syncNamespace: raw.syncNamespace?.trim() || void 0 } : void 0;
|
|
3105
|
+
return hashDocsRetrievalValue(JSON.stringify({
|
|
3106
|
+
format: "docs-search-sync.v1",
|
|
3107
|
+
indexGeneration,
|
|
3108
|
+
canonicalBaseUrl: context.baseUrl,
|
|
3109
|
+
provider: search.provider,
|
|
3110
|
+
providerConfig
|
|
3111
|
+
}));
|
|
3112
|
+
}
|
|
2081
3113
|
async function maybeSyncSearchIndex(adapter, search, context) {
|
|
2082
3114
|
if (!shouldSyncOnSearch(search) || typeof adapter.index !== "function") return;
|
|
2083
3115
|
const syncKey = getSyncKey(search, context);
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
3116
|
+
const indexGeneration = context.indexGeneration ?? await buildDocsSearchIndexGeneration(context.pages, {
|
|
3117
|
+
audience: resolveDocsSearchAudience(context.audience),
|
|
3118
|
+
chunking: context.chunking ?? search.chunking,
|
|
3119
|
+
locale: context.locale,
|
|
3120
|
+
baseUrl: context.baseUrl
|
|
3121
|
+
});
|
|
3122
|
+
context.indexGeneration = indexGeneration;
|
|
3123
|
+
const syncFingerprint = getSearchSyncFingerprint(search, context, indexGeneration);
|
|
3124
|
+
while (true) {
|
|
3125
|
+
const inFlight = syncingIndexes.get(syncKey);
|
|
3126
|
+
if (inFlight) {
|
|
3127
|
+
try {
|
|
3128
|
+
await inFlight.promise;
|
|
3129
|
+
} catch (error) {
|
|
3130
|
+
if (inFlight.fingerprint === syncFingerprint) throw error;
|
|
3131
|
+
}
|
|
3132
|
+
continue;
|
|
3133
|
+
}
|
|
3134
|
+
if (syncedIndexes.get(syncKey) === syncFingerprint) return;
|
|
3135
|
+
const sync = adapter.index(context).then(() => {
|
|
3136
|
+
syncedIndexes.set(syncKey, syncFingerprint);
|
|
3137
|
+
});
|
|
3138
|
+
syncingIndexes.set(syncKey, {
|
|
3139
|
+
fingerprint: syncFingerprint,
|
|
3140
|
+
promise: sync
|
|
3141
|
+
});
|
|
3142
|
+
try {
|
|
3143
|
+
await sync;
|
|
3144
|
+
} finally {
|
|
3145
|
+
if (syncingIndexes.get(syncKey)?.promise === sync) syncingIndexes.delete(syncKey);
|
|
3146
|
+
}
|
|
3147
|
+
}
|
|
2087
3148
|
}
|
|
2088
3149
|
async function performDocsSearch(options) {
|
|
2089
3150
|
const search = normalizeDocsSearchConfig(options.search);
|
|
@@ -2091,15 +3152,30 @@ async function performDocsSearch(options) {
|
|
|
2091
3152
|
const audience = resolveDocsSearchAudience(options.audience);
|
|
2092
3153
|
const filters = normalizeDocsSearchFilters(options.filters);
|
|
2093
3154
|
const hasFilters = hasDocsSearchFilters(filters);
|
|
2094
|
-
const
|
|
2095
|
-
const
|
|
3155
|
+
const corpusPages = options.pages.map((page) => localizeDocsSearchPage(page, options.locale));
|
|
3156
|
+
const indexBaseUrl = options.syncBaseUrl === null ? void 0 : options.syncBaseUrl ?? options.baseUrl;
|
|
3157
|
+
const scopedPages = hasFilters ? corpusPages.filter((page) => docsSearchPageMatchesFilters(resolveDocsSearchPageScope(page), filters)) : corpusPages;
|
|
3158
|
+
let resolvedDocuments;
|
|
3159
|
+
const getDocuments = () => {
|
|
3160
|
+
if (!resolvedDocuments) resolvedDocuments = buildDocsSearchDocuments(scopedPages, search.chunking, audience);
|
|
3161
|
+
return resolvedDocuments;
|
|
3162
|
+
};
|
|
2096
3163
|
const context = {
|
|
2097
3164
|
pages: scopedPages,
|
|
2098
|
-
documents
|
|
3165
|
+
get documents() {
|
|
3166
|
+
return getDocuments();
|
|
3167
|
+
},
|
|
3168
|
+
set documents(documents) {
|
|
3169
|
+
resolvedDocuments = documents;
|
|
3170
|
+
},
|
|
2099
3171
|
audience,
|
|
2100
3172
|
locale: options.locale,
|
|
2101
3173
|
pathname: options.pathname,
|
|
2102
3174
|
siteTitle: options.siteTitle,
|
|
3175
|
+
baseUrl: options.baseUrl,
|
|
3176
|
+
indexBaseUrl,
|
|
3177
|
+
chunking: search.chunking,
|
|
3178
|
+
deferSourceProvenance: true,
|
|
2103
3179
|
signal: options.signal
|
|
2104
3180
|
};
|
|
2105
3181
|
const query = {
|
|
@@ -2110,43 +3186,96 @@ async function performDocsSearch(options) {
|
|
|
2110
3186
|
audience,
|
|
2111
3187
|
...hasFilters ? { filters } : {}
|
|
2112
3188
|
};
|
|
3189
|
+
let currentIndexGeneration;
|
|
3190
|
+
const getCurrentIndexGeneration = () => {
|
|
3191
|
+
currentIndexGeneration ??= options.indexGeneration ? Promise.resolve(options.indexGeneration) : buildDocsSearchIndexGeneration(options.generationPages ?? options.pages, {
|
|
3192
|
+
audience,
|
|
3193
|
+
chunking: search.chunking,
|
|
3194
|
+
locale: options.locale,
|
|
3195
|
+
baseUrl: options.baseUrl
|
|
3196
|
+
});
|
|
3197
|
+
return currentIndexGeneration;
|
|
3198
|
+
};
|
|
3199
|
+
const requireCurrentIndexGeneration = search.provider === "algolia" || search.provider === "typesense";
|
|
3200
|
+
const finalizeResults = async (results) => enrichDocsSearchResultsWithSources({
|
|
3201
|
+
results,
|
|
3202
|
+
pages: scopedPages,
|
|
3203
|
+
generationPages: options.generationPages ?? options.pages,
|
|
3204
|
+
audience,
|
|
3205
|
+
chunking: search.chunking,
|
|
3206
|
+
locale: options.locale,
|
|
3207
|
+
baseUrl: options.baseUrl,
|
|
3208
|
+
indexGeneration: options.indexGeneration,
|
|
3209
|
+
strictExternalOrigins: options.strictExternalOrigins,
|
|
3210
|
+
filters,
|
|
3211
|
+
requireCurrentIndexGeneration
|
|
3212
|
+
});
|
|
2113
3213
|
try {
|
|
2114
3214
|
const adapter = await resolveSearchAdapter(search, context);
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
3215
|
+
if (shouldSyncOnSearch(search) && typeof adapter.index === "function") {
|
|
3216
|
+
const syncPages = audience === "agent" || hasFilters ? corpusPages : scopedPages;
|
|
3217
|
+
await maybeSyncSearchIndex(adapter, search, {
|
|
3218
|
+
pages: syncPages,
|
|
3219
|
+
documents: buildDocsSearchDocuments(syncPages, search.chunking, "human"),
|
|
3220
|
+
audience: "human",
|
|
3221
|
+
locale: options.locale,
|
|
3222
|
+
pathname: options.pathname,
|
|
3223
|
+
siteTitle: options.siteTitle,
|
|
3224
|
+
baseUrl: indexBaseUrl,
|
|
3225
|
+
indexBaseUrl,
|
|
3226
|
+
chunking: search.chunking,
|
|
3227
|
+
signal: options.signal
|
|
3228
|
+
});
|
|
3229
|
+
}
|
|
3230
|
+
const adapterSearch = adapter.search(query, context);
|
|
3231
|
+
let documents;
|
|
3232
|
+
try {
|
|
3233
|
+
documents = getDocuments();
|
|
3234
|
+
} catch (error) {
|
|
3235
|
+
adapterSearch.catch(() => void 0);
|
|
3236
|
+
throw error;
|
|
3237
|
+
}
|
|
3238
|
+
const results = await adapterSearch;
|
|
3239
|
+
if (search.provider === "simple") return finalizeResults(results);
|
|
2123
3240
|
const localAudienceProjectionResults = buildAudienceProjectionSearchResults(documents, options.query);
|
|
2124
|
-
const localPagePaths = new Set(scopedPages.map((page) =>
|
|
2125
|
-
const allLocalPagePaths = hasFilters && search.provider === "mcp" ? new Set(options.pages.map((page) =>
|
|
3241
|
+
const localPagePaths = new Set(scopedPages.map((page) => normalizeUrlRouteKey(page.url)));
|
|
3242
|
+
const allLocalPagePaths = hasFilters && search.provider === "mcp" ? new Set(options.pages.map((page) => normalizeUrlRouteKey(page.url))) : localPagePaths;
|
|
2126
3243
|
const preserveUnmatched = !hasFilters && audience === "human" || search.provider === "mcp" ? (result) => shouldPreserveUnmatchedExternalResult({
|
|
2127
3244
|
result,
|
|
2128
3245
|
localPagePaths: allLocalPagePaths,
|
|
2129
3246
|
baseUrl: options.baseUrl
|
|
2130
3247
|
}) : void 0;
|
|
2131
|
-
const
|
|
3248
|
+
const isKnownLocalProviderResult = (result) => isLocalProviderResult(result, options.baseUrl) && localPagePaths.has(normalizeUrlRouteKey(result.url));
|
|
3249
|
+
const expectedProviderGeneration = requireCurrentIndexGeneration ? audience === "human" ? await getCurrentIndexGeneration() : await buildDocsSearchIndexGeneration(options.generationPages ?? options.pages, {
|
|
3250
|
+
audience: "human",
|
|
3251
|
+
chunking: search.chunking,
|
|
3252
|
+
locale: options.locale,
|
|
3253
|
+
baseUrl: indexBaseUrl
|
|
3254
|
+
}) : void 0;
|
|
3255
|
+
const providerAudience = requireCurrentIndexGeneration ? "human" : audience;
|
|
3256
|
+
const safeAdapterResults = sanitizeExternalAudienceSearchResults(results.filter((result) => {
|
|
3257
|
+
if (result.source === void 0) return !requireCurrentIndexGeneration || isKnownLocalProviderResult(result);
|
|
3258
|
+
const source = parseDocsRetrievalSource(result.source, { allowRootRelativeCanonical: requireCurrentIndexGeneration || isKnownLocalProviderResult(result) });
|
|
3259
|
+
const knownLocal = isKnownLocalProviderResult(result);
|
|
3260
|
+
return Boolean(source && docsRetrievalSourceMatchesRequest(source, providerAudience, knownLocal ? void 0 : filters, options.locale) && (!expectedProviderGeneration || source.indexGeneration === expectedProviderGeneration));
|
|
3261
|
+
}).filter((result) => requireCurrentIndexGeneration && isKnownLocalProviderResult(result) || !hasOppositeAudienceEvidence({
|
|
2132
3262
|
result,
|
|
2133
3263
|
pages: scopedPages,
|
|
2134
3264
|
query: options.query,
|
|
2135
3265
|
audience,
|
|
2136
|
-
baseUrl: options.baseUrl
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
if (options.supplementExternalResults === false) return safeAdapterResults.slice(0, query.limit ?? search.maxResults ?? DEFAULT_SEARCH_LIMIT);
|
|
3266
|
+
baseUrl: options.baseUrl
|
|
3267
|
+
})), localAudienceProjectionResults, options.baseUrl, preserveUnmatched, requireCurrentIndexGeneration && audience === "agent");
|
|
3268
|
+
if (options.supplementExternalResults === false) return finalizeResults(safeAdapterResults.slice(0, query.limit ?? search.maxResults ?? DEFAULT_SEARCH_LIMIT));
|
|
2140
3269
|
const simpleAudienceResults = audience === "agent" || results.length === 0 || safeAdapterResults.length < results.length ? await createSimpleSearchAdapter().search(query, context) : [];
|
|
2141
3270
|
const combinedResults = mergeSearchResults([
|
|
2142
3271
|
buildExactPageSearchResults(options.query, scopedPages, audience),
|
|
2143
3272
|
safeAdapterResults,
|
|
2144
3273
|
simpleAudienceResults
|
|
2145
3274
|
], audience === "agent" ? (result) => getAskAIResultKey(result, options.baseUrl, options.strictExternalOrigins) : void 0);
|
|
2146
|
-
return prioritizeLiteralInsideResults(options.query, combinedResults).slice(0, query.limit ?? search.maxResults ?? DEFAULT_SEARCH_LIMIT);
|
|
3275
|
+
return finalizeResults(prioritizeLiteralInsideResults(options.query, combinedResults).slice(0, query.limit ?? search.maxResults ?? DEFAULT_SEARCH_LIMIT));
|
|
2147
3276
|
} catch (error) {
|
|
2148
3277
|
if (options.failureMode === "throw") throw error;
|
|
2149
|
-
return createSimpleSearchAdapter().search(query, context);
|
|
3278
|
+
return finalizeResults(await createSimpleSearchAdapter().search(query, context));
|
|
2150
3279
|
}
|
|
2151
3280
|
}
|
|
2152
3281
|
function compareSearchMetadataValues(left, right) {
|
|
@@ -2162,11 +3291,11 @@ function boundedSearchWarningPageUrls(pages) {
|
|
|
2162
3291
|
return bounded.length > 0 ? bounded : void 0;
|
|
2163
3292
|
}
|
|
2164
3293
|
function findSearchResultPages(pages, results, baseUrl) {
|
|
2165
|
-
const pagesByPath = new Map(pages.map((page) => [
|
|
3294
|
+
const pagesByPath = new Map(pages.map((page) => [normalizeUrlRouteKey(page.url), page]));
|
|
2166
3295
|
const found = /* @__PURE__ */ new Map();
|
|
2167
3296
|
for (const result of results) {
|
|
2168
|
-
if (!isLocalProviderResult(result, baseUrl
|
|
2169
|
-
const page = pagesByPath.get(
|
|
3297
|
+
if (!isLocalProviderResult(result, baseUrl)) continue;
|
|
3298
|
+
const page = pagesByPath.get(normalizeUrlRouteKey(result.url));
|
|
2170
3299
|
if (page) found.set(page.url, page);
|
|
2171
3300
|
}
|
|
2172
3301
|
return [...found.values()].sort((left, right) => compareSearchMetadataValues(left.url, right.url));
|
|
@@ -2248,16 +3377,25 @@ function buildDocsSearchWarnings(options) {
|
|
|
2248
3377
|
async function performDocsSearchWithMetadata(options) {
|
|
2249
3378
|
const audience = resolveDocsSearchAudience(options.audience);
|
|
2250
3379
|
const filters = normalizeDocsSearchFilters(options.filters);
|
|
2251
|
-
const
|
|
2252
|
-
|
|
3380
|
+
const search = normalizeDocsSearchConfig(options.search);
|
|
3381
|
+
const indexGeneration = options.indexGeneration ?? await buildDocsSearchIndexGeneration(options.generationPages ?? options.pages, {
|
|
2253
3382
|
audience,
|
|
2254
|
-
|
|
3383
|
+
chunking: search.chunking,
|
|
3384
|
+
locale: options.locale,
|
|
3385
|
+
baseUrl: options.baseUrl
|
|
2255
3386
|
});
|
|
3387
|
+
const results = options.query.trim() ? await performDocsSearch({
|
|
3388
|
+
...options,
|
|
3389
|
+
audience,
|
|
3390
|
+
filters,
|
|
3391
|
+
indexGeneration
|
|
3392
|
+
}) : [];
|
|
2256
3393
|
return {
|
|
2257
3394
|
format: "docs-search.v1",
|
|
2258
3395
|
query: options.query,
|
|
2259
3396
|
audience,
|
|
2260
3397
|
filters,
|
|
3398
|
+
indexGeneration,
|
|
2261
3399
|
resultCount: results.length,
|
|
2262
3400
|
results,
|
|
2263
3401
|
warnings: buildDocsSearchWarnings({
|
|
@@ -2284,6 +3422,7 @@ async function buildDocsAskAIContext(options) {
|
|
|
2284
3422
|
pathname: options.pathname,
|
|
2285
3423
|
siteTitle: options.siteTitle,
|
|
2286
3424
|
baseUrl: options.baseUrl,
|
|
3425
|
+
syncBaseUrl: options.syncBaseUrl,
|
|
2287
3426
|
limit: searchLimit,
|
|
2288
3427
|
filters: options.filters,
|
|
2289
3428
|
failureMode: options.searchFailureMode,
|
|
@@ -2295,7 +3434,7 @@ async function buildDocsAskAIContext(options) {
|
|
|
2295
3434
|
const rankedResults = searchResults.map((result, index) => {
|
|
2296
3435
|
const formatted = formatAskAIContextResult({
|
|
2297
3436
|
result,
|
|
2298
|
-
page: findPageForSearchResult(options.pages, result, options.baseUrl
|
|
3437
|
+
page: findPageForSearchResult(options.pages, result, options.baseUrl),
|
|
2299
3438
|
maxChars: maxResultChars,
|
|
2300
3439
|
baseUrl: options.baseUrl
|
|
2301
3440
|
});
|
|
@@ -2348,4 +3487,4 @@ function createCustomSearchAdapter(adapter) {
|
|
|
2348
3487
|
}
|
|
2349
3488
|
|
|
2350
3489
|
//#endregion
|
|
2351
|
-
export {
|
|
3490
|
+
export { GENERATED_AGENT_PROVENANCE_MARKER as A, resolveSidebarFolderIndexBehaviorForPath as B, agentVersionConstraintsOverlap as C, normalizeAgentVersion as D, normalizeAgentScopeValues as E, serializeGeneratedAgentDocument as F, getDocsTelemetryFeatures as G, emitDocsTelemetryEvent as H, stripGeneratedAgentProvenance as I, normalizeDocsTelemetryOrigin as J, inferDocsTelemetryAgentSurface as K, applySidebarFolderIndexBehavior as L, hashGeneratedAgentContent as M, normalizeGeneratedAgentContent as N, digestDocsRetrievalContent as O, parseGeneratedAgentDocument as P, resolvePageSidebarFolderIndexBehavior as R, agentVersionConstraintMatches as S, normalizeAgentLocale as T, emitDocsTelemetryMcpToolEvent as U, emitDocsTelemetryAgentSurfaceEvent as V, emitDocsTelemetryProjectEvent as W, resolveDocsTelemetryConfig as Y, resolveDocsSearchAudience as _, createCustomSearchAdapter as a, resolveSearchRequestConfig as b, createTypesenseSearchAdapter as c, inferDocsAskAIPackageHints as d, normalizeDocsSearchFilters as f, resolveDocsRetrievalLastModified as g, resolveAskAISearchRequestConfig as h, createAlgoliaSearchAdapter as i, GENERATED_AGENT_PROVENANCE_VERSION as j, isDocsRetrievalCanonicalUrl as k, enrichDocsSearchDocumentsWithProvenance as l, performDocsSearchWithMetadata as m, buildDocsRetrievalDigestProjection as n, createMcpSearchAdapter as o, performDocsSearch as p, isLocalDocsTelemetryOrigin as q, buildDocsSearchDocuments as r, createSimpleSearchAdapter as s, buildDocsAskAIContext as t, formatDocsAskAIPackageHints as u, resolveDocsSearchFilters as v, normalizeAgentFramework as w, agentVersionConstraintGroupsOverlap as x, resolveDocsSearchRequest as y, resolveSidebarFolderIndexBehavior as z };
|