@expo/code-review-cli 0.12.0 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -43
- package/build/commands/init.js +3 -2
- package/build/config/schema.js +0 -9
- package/build/core/auth.js +1 -0
- package/build/core/claude-code.js +23 -3
- package/build/core/opencode.js +23 -4
- package/build/core/prompts.js +35 -6
- package/build/core/render.js +13 -0
- package/build/core/research.js +232 -9
- package/build/core/review.js +72 -24
- package/build/core/schema.js +14 -0
- package/build/core/tools.js +5 -0
- package/build/research-mcp/apple-docc.js +75 -36
- package/build/research-mcp/audit.js +163 -0
- package/build/research-mcp/brave-search.js +68 -0
- package/build/research-mcp/cli.js +29 -7
- package/build/research-mcp/crawler.js +2 -48
- package/build/research-mcp/direct-fetch.js +100 -0
- package/build/research-mcp/fetch-document.js +76 -0
- package/build/research-mcp/html.js +7 -1
- package/build/research-mcp/okhttp-search.js +94 -0
- package/build/research-mcp/query-sanitizer.js +146 -0
- package/build/research-mcp/remote-search.js +175 -0
- package/build/research-mcp/server.js +236 -65
- package/package.json +1 -1
- package/templates/atlantis.yml +2 -0
- package/templates/command.yml +2 -0
- package/templates/config.jsonc +5 -8
- package/templates/coordinator.md +2 -0
- package/templates/shared.md +7 -1
- package/templates/workflow.yml +2 -0
|
@@ -96,7 +96,13 @@ export function chunkDocument(document, indexedAt, targetCharacters = 1400, over
|
|
|
96
96
|
if (passage) {
|
|
97
97
|
passages.push(passage);
|
|
98
98
|
}
|
|
99
|
-
|
|
99
|
+
const overlapStart = Math.max(0, passage.length - overlapCharacters);
|
|
100
|
+
const overlapTail = passage.slice(overlapStart);
|
|
101
|
+
const sentenceBoundary = overlapTail.match(/[.!?]\s+/);
|
|
102
|
+
current =
|
|
103
|
+
sentenceBoundary?.index !== undefined
|
|
104
|
+
? overlapTail.slice(sentenceBoundary.index + sentenceBoundary[0].length)
|
|
105
|
+
: "";
|
|
100
106
|
};
|
|
101
107
|
for (const paragraph of paragraphs) {
|
|
102
108
|
if (current && current.length + paragraph.length + 2 > targetCharacters) {
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { chunkDocument, extractDocumentationPage } from "./html.js";
|
|
3
|
+
import { okHttpProvider, resolveAllowedUrl } from "./providers.js";
|
|
4
|
+
import { readBodyWithLimit } from "./response.js";
|
|
5
|
+
import { buildSearchIndex, searchDocumentation } from "./search-index.js";
|
|
6
|
+
const OKHTTP_SEARCH_INDEX = "https://lysine.dev/okhttp/search/search_index.json";
|
|
7
|
+
const OKHTTP_SEARCH_INDEX_LIMIT_BYTES = 1_000_000;
|
|
8
|
+
const OKHTTP_SEARCH_TIMEOUT_MS = 10_000;
|
|
9
|
+
const okHttpSearchIndexSchema = z.object({
|
|
10
|
+
docs: z
|
|
11
|
+
.array(z.object({
|
|
12
|
+
location: z.string().max(2_000),
|
|
13
|
+
title: z.string().min(1).max(500),
|
|
14
|
+
text: z.string().max(100_000),
|
|
15
|
+
}))
|
|
16
|
+
.max(1_000),
|
|
17
|
+
});
|
|
18
|
+
const indexCache = new WeakMap();
|
|
19
|
+
async function loadOkHttpSearchIndex(fetchImplementation) {
|
|
20
|
+
const response = await fetchImplementation(OKHTTP_SEARCH_INDEX, {
|
|
21
|
+
redirect: "manual",
|
|
22
|
+
signal: AbortSignal.timeout(OKHTTP_SEARCH_TIMEOUT_MS),
|
|
23
|
+
headers: {
|
|
24
|
+
accept: "application/json",
|
|
25
|
+
"accept-language": "en-US,en;q=0.9",
|
|
26
|
+
"user-agent": "review-research-mcp/0.2 (+official documentation search index)",
|
|
27
|
+
},
|
|
28
|
+
});
|
|
29
|
+
if (response.status >= 300 && response.status < 400) {
|
|
30
|
+
throw new Error(`OkHttp search index unexpectedly redirected with HTTP ${response.status}`);
|
|
31
|
+
}
|
|
32
|
+
if (!response.ok) {
|
|
33
|
+
throw new Error(`OkHttp search index returned HTTP ${response.status}`);
|
|
34
|
+
}
|
|
35
|
+
const contentType = response.headers.get("content-type")?.toLowerCase() ?? "";
|
|
36
|
+
if (!contentType.includes("application/json")) {
|
|
37
|
+
throw new Error(`OkHttp search index returned unsupported content type: ${contentType || "missing"}`);
|
|
38
|
+
}
|
|
39
|
+
const parsed = okHttpSearchIndexSchema.parse(JSON.parse(await readBodyWithLimit(response, OKHTTP_SEARCH_INDEX_LIMIT_BYTES)));
|
|
40
|
+
const indexedAt = new Date().toISOString();
|
|
41
|
+
const seenDocuments = new Set();
|
|
42
|
+
const documents = parsed.docs.flatMap((entry) => {
|
|
43
|
+
try {
|
|
44
|
+
const url = resolveAllowedUrl(okHttpProvider, entry.location, "https://lysine.dev/okhttp/");
|
|
45
|
+
const key = `${url.href}|${entry.title}`;
|
|
46
|
+
if (seenDocuments.has(key))
|
|
47
|
+
return [];
|
|
48
|
+
seenDocuments.add(key);
|
|
49
|
+
const extracted = extractDocumentationPage(`<!doctype html><html><body><main><h1>${entry.title}</h1>${entry.text}</main></body></html>`, url.href, "android", { provider: "okhttp", sourceKind: "official-guide" });
|
|
50
|
+
return extracted ? [extracted.document] : [];
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
return [];
|
|
54
|
+
}
|
|
55
|
+
});
|
|
56
|
+
const chunks = documents.flatMap((document) => chunkDocument(document, indexedAt));
|
|
57
|
+
return buildSearchIndex(chunks, documents.length, indexedAt);
|
|
58
|
+
}
|
|
59
|
+
function cachedOkHttpSearchIndex(fetchImplementation) {
|
|
60
|
+
const cached = indexCache.get(fetchImplementation);
|
|
61
|
+
if (cached)
|
|
62
|
+
return cached;
|
|
63
|
+
const pending = loadOkHttpSearchIndex(fetchImplementation);
|
|
64
|
+
indexCache.set(fetchImplementation, pending);
|
|
65
|
+
return pending;
|
|
66
|
+
}
|
|
67
|
+
export async function searchOkHttpDocumentation(query, limit, fetchImplementation = fetch) {
|
|
68
|
+
const index = await cachedOkHttpSearchIndex(fetchImplementation);
|
|
69
|
+
const search = (value) => searchDocumentation(index, value, {
|
|
70
|
+
platform: "android",
|
|
71
|
+
providers: ["okhttp"],
|
|
72
|
+
sourceKinds: ["official-guide"],
|
|
73
|
+
limit,
|
|
74
|
+
});
|
|
75
|
+
const conceptTokens = [...new Set(query.match(/\b[a-z][a-z0-9-]{3,}\b/g) ?? [])];
|
|
76
|
+
const concepts = conceptTokens.join(" ");
|
|
77
|
+
const candidates = concepts && concepts !== query
|
|
78
|
+
? [
|
|
79
|
+
...conceptTokens.flatMap((concept) => search(concept).slice(0, 1)),
|
|
80
|
+
...search(concepts),
|
|
81
|
+
...search(query),
|
|
82
|
+
]
|
|
83
|
+
: search(query);
|
|
84
|
+
const seen = new Set();
|
|
85
|
+
return candidates
|
|
86
|
+
.filter((result) => {
|
|
87
|
+
const key = `${result.url}|${result.title}`;
|
|
88
|
+
if (seen.has(key))
|
|
89
|
+
return false;
|
|
90
|
+
seen.add(key);
|
|
91
|
+
return true;
|
|
92
|
+
})
|
|
93
|
+
.slice(0, limit);
|
|
94
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
const MAX_QUERY_CHARACTERS = 160;
|
|
2
|
+
const MAX_QUERY_TOKENS = 8;
|
|
3
|
+
const MAX_TOKEN_CHARACTERS = 64;
|
|
4
|
+
const SECRET_SHAPE = /(?:-----BEGIN [A-Z ]*PRIVATE KEY-----|\bAKIA[0-9A-Z]{16}\b|\b(?:sk|xox[baprs]|gh[opusr])[-_][A-Za-z0-9_-]{12,}\b|\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b)/i;
|
|
5
|
+
const NAMED_SECRET = /\b(?:api[-_ ]?key|access[-_ ]?token|auth[-_ ]?token|password|passwd|secret|credential)\b\s*(?::|=|is)?\s*\S+/i;
|
|
6
|
+
const QUOTED_LITERAL = /"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`/g;
|
|
7
|
+
const URL_OR_EMAIL = /\b(?:https?:\/\/|www\.)\S+|\b[^\s@]+@[^\s@]+\.[^\s@]+\b/gi;
|
|
8
|
+
const PATH_LIKE = /(?:^|\s)(?:\.{0,2}[\\/]|[A-Za-z]:[\\/]|~[\\/])\S+/g;
|
|
9
|
+
const QUERY_TOKEN = /[A-Za-z_][A-Za-z0-9_.:$#<>()[\]-]{0,63}|\b\d{1,4}(?:\.\d{1,3}){0,2}\b/g;
|
|
10
|
+
const PROSE_STOP_WORDS = new Set([
|
|
11
|
+
"a",
|
|
12
|
+
"about",
|
|
13
|
+
"an",
|
|
14
|
+
"and",
|
|
15
|
+
"are",
|
|
16
|
+
"can",
|
|
17
|
+
"could",
|
|
18
|
+
"do",
|
|
19
|
+
"does",
|
|
20
|
+
"documentation",
|
|
21
|
+
"explain",
|
|
22
|
+
"find",
|
|
23
|
+
"for",
|
|
24
|
+
"from",
|
|
25
|
+
"how",
|
|
26
|
+
"i",
|
|
27
|
+
"in",
|
|
28
|
+
"is",
|
|
29
|
+
"it",
|
|
30
|
+
"of",
|
|
31
|
+
"on",
|
|
32
|
+
"or",
|
|
33
|
+
"please",
|
|
34
|
+
"search",
|
|
35
|
+
"show",
|
|
36
|
+
"tell",
|
|
37
|
+
"the",
|
|
38
|
+
"this",
|
|
39
|
+
"to",
|
|
40
|
+
"what",
|
|
41
|
+
"when",
|
|
42
|
+
"where",
|
|
43
|
+
"why",
|
|
44
|
+
"with",
|
|
45
|
+
"work",
|
|
46
|
+
"works",
|
|
47
|
+
]);
|
|
48
|
+
function shannonEntropy(value) {
|
|
49
|
+
const counts = new Map();
|
|
50
|
+
for (const character of value)
|
|
51
|
+
counts.set(character, (counts.get(character) ?? 0) + 1);
|
|
52
|
+
let entropy = 0;
|
|
53
|
+
for (const count of counts.values()) {
|
|
54
|
+
const probability = count / value.length;
|
|
55
|
+
entropy -= probability * Math.log2(probability);
|
|
56
|
+
}
|
|
57
|
+
return entropy;
|
|
58
|
+
}
|
|
59
|
+
function looksHighEntropy(value) {
|
|
60
|
+
if (/^[A-Fa-f0-9]{24,}$/.test(value))
|
|
61
|
+
return true;
|
|
62
|
+
if (/^[A-Za-z0-9+/]{20,}={0,2}$/.test(value) && /[+/=]/.test(value))
|
|
63
|
+
return true;
|
|
64
|
+
return (value.length >= 24 &&
|
|
65
|
+
/[a-z]/.test(value) &&
|
|
66
|
+
/[A-Z]/.test(value) &&
|
|
67
|
+
/\d/.test(value) &&
|
|
68
|
+
shannonEntropy(value) >= 4.2);
|
|
69
|
+
}
|
|
70
|
+
function isApiAnchor(value) {
|
|
71
|
+
return (/[a-z][A-Z]/.test(value) ||
|
|
72
|
+
/[A-Z][A-Za-z0-9_]{1,}/.test(value) ||
|
|
73
|
+
/[._:$#()]/.test(value) ||
|
|
74
|
+
/_[A-Z0-9]/.test(value));
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Convert an agent-authored search into a short API-symbol query. Dangerous shapes
|
|
78
|
+
* fail closed; prose, literals, URLs, paths, and unsupported punctuation are removed.
|
|
79
|
+
*/
|
|
80
|
+
export function sanitizeDocumentationQuery(rawQuery) {
|
|
81
|
+
const visible = rawQuery
|
|
82
|
+
.normalize("NFKC")
|
|
83
|
+
// oxlint-disable-next-line no-control-regex -- outbound query containment
|
|
84
|
+
.replace(/[\u0000-\u001f\u007f]/g, " ")
|
|
85
|
+
.replace(/\s+/g, " ")
|
|
86
|
+
.trim();
|
|
87
|
+
if (!visible || visible.length > 300) {
|
|
88
|
+
throw new Error("Query must contain between 1 and 300 visible characters");
|
|
89
|
+
}
|
|
90
|
+
if (SECRET_SHAPE.test(visible) || NAMED_SECRET.test(visible)) {
|
|
91
|
+
throw new Error("Query contains credential-shaped or secret-labeled material");
|
|
92
|
+
}
|
|
93
|
+
const candidates = (visible
|
|
94
|
+
.replace(QUOTED_LITERAL, " ")
|
|
95
|
+
.replace(URL_OR_EMAIL, " ")
|
|
96
|
+
.replace(PATH_LIKE, " ")
|
|
97
|
+
.match(QUERY_TOKEN) ?? []).filter((token) => {
|
|
98
|
+
if (token.length > MAX_TOKEN_CHARACTERS || looksHighEntropy(token))
|
|
99
|
+
return false;
|
|
100
|
+
return !PROSE_STOP_WORDS.has(token.toLowerCase());
|
|
101
|
+
});
|
|
102
|
+
const unique = [...new Set(candidates)].slice(0, MAX_QUERY_TOKENS);
|
|
103
|
+
if (!unique.some(isApiAnchor)) {
|
|
104
|
+
throw new Error("Query must include an API-like symbol or member name");
|
|
105
|
+
}
|
|
106
|
+
const sanitized = unique.join(" ").slice(0, MAX_QUERY_CHARACTERS).trim();
|
|
107
|
+
if (!sanitized)
|
|
108
|
+
throw new Error("Query contained no safe documentation terms");
|
|
109
|
+
return sanitized;
|
|
110
|
+
}
|
|
111
|
+
/** Reject URL decorations and path segments that could encode arbitrary outbound data. */
|
|
112
|
+
export function assertSafeDocumentationUrlShape(rawUrl) {
|
|
113
|
+
let url;
|
|
114
|
+
try {
|
|
115
|
+
url = new URL(rawUrl);
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
throw new Error("Invalid documentation URL");
|
|
119
|
+
}
|
|
120
|
+
if (url.protocol !== "https:" || url.username || url.password || url.port) {
|
|
121
|
+
throw new Error("Documentation URL must use plain HTTPS without credentials or a port");
|
|
122
|
+
}
|
|
123
|
+
if (url.search || url.hash) {
|
|
124
|
+
throw new Error("Documentation URL must not contain a query string or fragment");
|
|
125
|
+
}
|
|
126
|
+
const segments = url.pathname.split("/").filter(Boolean);
|
|
127
|
+
for (const segment of segments) {
|
|
128
|
+
let decoded;
|
|
129
|
+
try {
|
|
130
|
+
decoded = decodeURIComponent(segment);
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
throw new Error("Documentation URL contains an invalid encoded path segment");
|
|
134
|
+
}
|
|
135
|
+
// oxlint-disable-next-line no-control-regex -- URL path is an outbound data boundary
|
|
136
|
+
const containsControlCharacter = /[\\/\u0000-\u001f\u007f]/.test(decoded);
|
|
137
|
+
if (decoded.length > 120 ||
|
|
138
|
+
containsControlCharacter ||
|
|
139
|
+
SECRET_SHAPE.test(decoded) ||
|
|
140
|
+
NAMED_SECRET.test(decoded) ||
|
|
141
|
+
looksHighEntropy(decoded)) {
|
|
142
|
+
throw new Error("Documentation URL contains a suspicious path segment");
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return url;
|
|
146
|
+
}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { searchBrave } from "./brave-search.js";
|
|
3
|
+
import { fetchDocumentationDocument } from "./fetch-document.js";
|
|
4
|
+
import { chunkDocument } from "./html.js";
|
|
5
|
+
import { getProvider, resolveAllowedUrl } from "./providers.js";
|
|
6
|
+
import { buildSearchIndex, searchDocumentation } from "./search-index.js";
|
|
7
|
+
const providerSearchDefinitions = {
|
|
8
|
+
apple: {
|
|
9
|
+
scopes: ["developer.apple.com/documentation"],
|
|
10
|
+
sourceKind: "official-api",
|
|
11
|
+
},
|
|
12
|
+
"apple-releases": {
|
|
13
|
+
scopes: ["developer.apple.com/documentation/xcode-release-notes"],
|
|
14
|
+
sourceKind: "release-notes",
|
|
15
|
+
},
|
|
16
|
+
"swift-evolution": {
|
|
17
|
+
scopes: ["github.com/swiftlang/swift-evolution/blob/main/proposals"],
|
|
18
|
+
sourceKind: "official-guide",
|
|
19
|
+
},
|
|
20
|
+
android: {
|
|
21
|
+
scopes: ["developer.android.com/reference"],
|
|
22
|
+
sourceKind: "official-api",
|
|
23
|
+
},
|
|
24
|
+
"android-releases": {
|
|
25
|
+
scopes: ["developer.android.com/about/versions"],
|
|
26
|
+
sourceKind: "release-notes",
|
|
27
|
+
},
|
|
28
|
+
media3: {
|
|
29
|
+
scopes: ["developer.android.com"],
|
|
30
|
+
sourceKind: "official-guide",
|
|
31
|
+
},
|
|
32
|
+
glide: {
|
|
33
|
+
scopes: ["bumptech.github.io/glide"],
|
|
34
|
+
sourceKind: "official-guide",
|
|
35
|
+
},
|
|
36
|
+
okhttp: {
|
|
37
|
+
scopes: ["lysine.dev/okhttp"],
|
|
38
|
+
sourceKind: "official-guide",
|
|
39
|
+
},
|
|
40
|
+
"kotlin-coroutines": {
|
|
41
|
+
scopes: ["kotlinlang.org"],
|
|
42
|
+
sourceKind: "official-guide",
|
|
43
|
+
},
|
|
44
|
+
gradle: {
|
|
45
|
+
scopes: ["docs.gradle.org/current"],
|
|
46
|
+
sourceKind: "official-guide",
|
|
47
|
+
},
|
|
48
|
+
agp: {
|
|
49
|
+
scopes: ["developer.android.com"],
|
|
50
|
+
sourceKind: "release-notes",
|
|
51
|
+
},
|
|
52
|
+
"jetbrains-issues": {
|
|
53
|
+
scopes: ["youtrack.jetbrains.com/issue"],
|
|
54
|
+
sourceKind: "issue-tracker",
|
|
55
|
+
},
|
|
56
|
+
"react-native": {
|
|
57
|
+
scopes: ["reactnative.dev"],
|
|
58
|
+
sourceKind: "official-api",
|
|
59
|
+
},
|
|
60
|
+
"react-native-reanimated": {
|
|
61
|
+
scopes: ["docs.swmansion.com/react-native-reanimated/docs"],
|
|
62
|
+
sourceKind: "official-api",
|
|
63
|
+
},
|
|
64
|
+
"react-native-gesture-handler": {
|
|
65
|
+
scopes: ["docs.swmansion.com/react-native-gesture-handler/docs/gestures"],
|
|
66
|
+
sourceKind: "official-api",
|
|
67
|
+
},
|
|
68
|
+
"react-native-screens": {
|
|
69
|
+
scopes: ["docs.swmansion.com"],
|
|
70
|
+
sourceKind: "official-api",
|
|
71
|
+
},
|
|
72
|
+
"react-native-worklets": {
|
|
73
|
+
scopes: ["docs.swmansion.com/react-native-worklets/docs"],
|
|
74
|
+
sourceKind: "official-api",
|
|
75
|
+
},
|
|
76
|
+
};
|
|
77
|
+
function appleReleaseScopes(query) {
|
|
78
|
+
if (/\b(?:ios|ipados)\b/i.test(query)) {
|
|
79
|
+
return ["developer.apple.com/documentation/ios-ipados-release-notes"];
|
|
80
|
+
}
|
|
81
|
+
if (/\bmacos\b/i.test(query)) {
|
|
82
|
+
return ["developer.apple.com/documentation/macos-release-notes"];
|
|
83
|
+
}
|
|
84
|
+
if (/\btvos\b/i.test(query)) {
|
|
85
|
+
return ["developer.apple.com/documentation/tvos-release-notes"];
|
|
86
|
+
}
|
|
87
|
+
if (/\bwatchos\b/i.test(query)) {
|
|
88
|
+
return ["developer.apple.com/documentation/watchos-release-notes"];
|
|
89
|
+
}
|
|
90
|
+
if (/\bvisionos\b/i.test(query)) {
|
|
91
|
+
return ["developer.apple.com/documentation/visionos-release-notes"];
|
|
92
|
+
}
|
|
93
|
+
return providerSearchDefinitions["apple-releases"].scopes;
|
|
94
|
+
}
|
|
95
|
+
function bestPassage(document, query, indexedAt) {
|
|
96
|
+
const chunks = chunkDocument(document, indexedAt);
|
|
97
|
+
if (chunks.length === 0) {
|
|
98
|
+
return { passage: document.body.slice(0, 1_400), relevance: 0 };
|
|
99
|
+
}
|
|
100
|
+
const index = buildSearchIndex(chunks, 1, indexedAt);
|
|
101
|
+
const result = searchDocumentation(index, query, {
|
|
102
|
+
platform: document.platform,
|
|
103
|
+
providers: document.provider ? [document.provider] : undefined,
|
|
104
|
+
limit: 1,
|
|
105
|
+
})[0];
|
|
106
|
+
return {
|
|
107
|
+
passage: result?.passage ?? chunks[0].passage,
|
|
108
|
+
relevance: result?.score ?? 0,
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
export async function searchRemoteDocumentation(providerId, query, limit, options) {
|
|
112
|
+
const definition = providerSearchDefinitions[providerId];
|
|
113
|
+
if (options.sourceKinds && !options.sourceKinds.includes(definition.sourceKind)) {
|
|
114
|
+
return { results: [], warnings: [] };
|
|
115
|
+
}
|
|
116
|
+
const provider = getProvider(providerId);
|
|
117
|
+
const fetchImplementation = options.fetchImplementation ?? fetch;
|
|
118
|
+
const hits = await searchBrave(query, providerId === "apple-releases" ? appleReleaseScopes(query) : definition.scopes, Math.min(10, Math.max(limit * 2, 4)), options.apiKey, fetchImplementation);
|
|
119
|
+
const candidates = [];
|
|
120
|
+
const seen = new Set();
|
|
121
|
+
for (const [position, hit] of hits.entries()) {
|
|
122
|
+
try {
|
|
123
|
+
const url = resolveAllowedUrl(provider, hit.url);
|
|
124
|
+
if (seen.has(url.href))
|
|
125
|
+
continue;
|
|
126
|
+
seen.add(url.href);
|
|
127
|
+
candidates.push({ url, position });
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
// Search ranking is discovery only. The provider allowlist is authoritative.
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
const warnings = [];
|
|
134
|
+
const indexedAt = new Date().toISOString();
|
|
135
|
+
const fetched = await Promise.all(candidates.map(async ({ url, position }) => {
|
|
136
|
+
try {
|
|
137
|
+
const document = await fetchDocumentationDocument(provider, url.href, definition.sourceKind, fetchImplementation);
|
|
138
|
+
if (!document || (options.language && document.language !== options.language))
|
|
139
|
+
return null;
|
|
140
|
+
const id = createHash("sha256")
|
|
141
|
+
.update(`${providerId}\0${document.url}\0${document.title}`)
|
|
142
|
+
.digest("hex")
|
|
143
|
+
.slice(0, 16);
|
|
144
|
+
const selected = bestPassage(document, query, indexedAt);
|
|
145
|
+
const result = {
|
|
146
|
+
id: `remote:${providerId}:${id}`,
|
|
147
|
+
platform: document.platform,
|
|
148
|
+
provider: providerId,
|
|
149
|
+
sourceKind: definition.sourceKind,
|
|
150
|
+
title: document.title,
|
|
151
|
+
url: document.url,
|
|
152
|
+
passage: selected.passage,
|
|
153
|
+
...(document.framework ? { framework: document.framework } : {}),
|
|
154
|
+
...(document.symbol ? { symbol: document.symbol } : {}),
|
|
155
|
+
...(document.language ? { language: document.language } : {}),
|
|
156
|
+
...(document.availability ? { availability: document.availability } : {}),
|
|
157
|
+
indexedAt,
|
|
158
|
+
score: selected.relevance * 100 + hits.length - position,
|
|
159
|
+
};
|
|
160
|
+
return result;
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
164
|
+
warnings.push(`${provider.displayName} fetch failed for ${url.href}: ${message}`);
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
}));
|
|
168
|
+
return {
|
|
169
|
+
results: fetched
|
|
170
|
+
.flatMap((result) => (result ? [result] : []))
|
|
171
|
+
.sort((left, right) => right.score - left.score)
|
|
172
|
+
.slice(0, limit),
|
|
173
|
+
warnings: warnings.slice(0, 5),
|
|
174
|
+
};
|
|
175
|
+
}
|