@expo/code-review-cli 0.12.4 → 0.12.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -15
- package/build/commands/ci.js +4 -4
- package/build/config/load.js +1 -1
- package/build/config/schema.js +6 -3
- package/build/core/research.js +37 -33
- package/build/research-mcp/audit.js +2 -1
- package/build/research-mcp/child-env.js +73 -0
- package/build/research-mcp/cli.js +11 -56
- package/build/research-mcp/direct-fetch.js +1 -1
- package/build/research-mcp/fetch-document.js +0 -3
- package/build/research-mcp/network.js +75 -0
- package/build/research-mcp/okhttp-search.js +1 -1
- package/build/research-mcp/remote-search.js +4 -2
- package/build/research-mcp/search-index.js +7 -31
- package/build/research-mcp/server.js +22 -29
- package/build/research-mcp/wrapper.js +54 -0
- package/package.json +2 -4
- package/templates/config.jsonc +15 -4
- package/build/research-mcp/crawler.js +0 -148
- package/build/research-mcp/paths.js +0 -4
- package/research/sources.json +0 -295
package/README.md
CHANGED
|
@@ -190,7 +190,7 @@ Enable it only in the root config, which CI loads from the PR's trusted base:
|
|
|
190
190
|
"enabled": true,
|
|
191
191
|
"maxQueries": 8,
|
|
192
192
|
"resultsPerQuery": 2,
|
|
193
|
-
"timeoutMs":
|
|
193
|
+
"timeoutMs": 30000
|
|
194
194
|
}
|
|
195
195
|
}
|
|
196
196
|
```
|
|
@@ -205,13 +205,26 @@ the current absolute Node executable, so a PR-owned `PATH` entry cannot replace
|
|
|
205
205
|
either component. Each review gets an owner-only temporary MCP config and append-only
|
|
206
206
|
audit. Claude receives that explicit config under `--strict-mcp-config`, with project
|
|
207
207
|
settings and slash commands disabled; OpenCode receives the same fixed local command.
|
|
208
|
-
The Brave credential is passed to the MCP child, not the model process.
|
|
209
208
|
|
|
210
|
-
|
|
209
|
+
That command is a wrapper, not the server. Both engines merge a configured MCP `env`
|
|
210
|
+
onto their own environment instead of replacing it, so the declared block alone cannot
|
|
211
|
+
bound the child — the engine's model credential reaches it, and OpenCode additionally
|
|
212
|
+
passes down the runner's whole ambient environment. The wrapper therefore rebuilds the
|
|
213
|
+
environment from an explicit allowlist before starting the server, and loads no parser
|
|
214
|
+
and opens no socket of its own. The server process sees the search key, the call
|
|
215
|
+
bounds, and locale/proxy settings; it never sees a model credential. The Brave
|
|
216
|
+
credential travels the same way and is never added to the model process env.
|
|
217
|
+
|
|
218
|
+
The MCP bounds the shape, host, and volume of outbound requests. It is not a
|
|
219
|
+
confidentiality boundary: the reviewing model chooses the query terms and URLs, and a
|
|
220
|
+
low-entropy identifier can carry repository-derived data past every check below.
|
|
221
|
+
Enable research only where repository-derived terms may be shared with Brave and the
|
|
222
|
+
documentation providers. Search queries are normalized before
|
|
211
223
|
logging or networking: quoted literals, URLs, email addresses, paths, prose stop
|
|
212
224
|
words, overlong/high-entropy tokens, and unsupported punctuation are removed;
|
|
213
|
-
credential-shaped or secret-labeled input fails closed. The remaining query must be
|
|
214
|
-
|
|
225
|
+
credential-shaped or secret-labeled input fails closed. The remaining query must be at
|
|
226
|
+
most eight short tokens and either contain an API-like symbol or be a short multi-word
|
|
227
|
+
lowercase concept phrase. Direct URLs must use plain
|
|
215
228
|
HTTPS with no credentials, port, query string, or fragment; suspicious/high-entropy
|
|
216
229
|
path segments fail closed. The fixed provider host/path allowlist and redirect,
|
|
217
230
|
response-size, content-type, and timeout checks still apply after that first gate.
|
|
@@ -219,6 +232,17 @@ These deterministic checks greatly reduce accidental exfiltration; they are not
|
|
|
219
232
|
proof that every low-entropy string is harmless, so reviewer prompts also forbid
|
|
220
233
|
sending repository text and the review-wide MCP budget defaults to eight calls.
|
|
221
234
|
|
|
235
|
+
`maxQueries` bounds MCP calls, not network requests. One search selects up to four
|
|
236
|
+
providers, and each issues its own discovery request plus a page fetch per candidate,
|
|
237
|
+
so eight calls can mean roughly thirty discovery requests and over a hundred page
|
|
238
|
+
downloads. Every call therefore reports its own ledger — discovery requests, page
|
|
239
|
+
fetches, redirect hops, total HTTP requests, and elapsed time — and the review log and
|
|
240
|
+
Actions summary report the totals. `timeoutMs` is the MCP's own end-to-end deadline for
|
|
241
|
+
one call, enforced by the server across discovery, redirects, retrieval, and
|
|
242
|
+
extraction; a call that hits it returns what it already has rather than failing. It has
|
|
243
|
+
to live there because OpenCode's `timeout` bounds only tool discovery and Claude
|
|
244
|
+
provides no per-call timeout at all.
|
|
245
|
+
|
|
222
246
|
For non-Expo providers, discovery sends a fixed, provider-owned `site:` scope plus
|
|
223
247
|
the bounded query to Brave's fixed Web Search endpoint. Search snippets and titles
|
|
224
248
|
are never treated as evidence. ECR independently rejects off-allowlist result URLs,
|
|
@@ -276,8 +300,11 @@ same audit trail in the step summary, while `.runs/reviews.jsonl` keeps the quer
|
|
|
276
300
|
plus bounded returned passages for short-lived operational inspection. Reviewers
|
|
277
301
|
are instructed to attach `sources` only when documentation materially supports a
|
|
278
302
|
finding. ECR accepts only exact URLs returned during that review, restores canonical
|
|
279
|
-
titles, carries citations through coordination, and renders them below the finding
|
|
280
|
-
|
|
303
|
+
titles, carries citations through coordination, and renders them below the finding.
|
|
304
|
+
A citation to a URL this review never retrieved is dropped outright. Relatedness is a
|
|
305
|
+
separate, weaker guarantee: a cited finding is escalated to the verifier with the
|
|
306
|
+
audited passage inline, which judges whether that passage actually supports the claim
|
|
307
|
+
and strips the citation when it does not.
|
|
281
308
|
|
|
282
309
|
Reviewers also emit a bounded `researchDecisions` record only when documentation
|
|
283
310
|
materially confirms a finding candidate or proves one safe. ECR grounds those records
|
|
@@ -298,14 +325,11 @@ downloads the fixed official `lysine.dev` static search index, validates and ind
|
|
|
298
325
|
it in memory once per MCP process, and rejects any entry outside the existing OkHttp
|
|
299
326
|
allowlist. Brave remains a fallback if that official index is unavailable.
|
|
300
327
|
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
`
|
|
305
|
-
|
|
306
|
-
`research/sources.json`; `seedUrls` are deterministic starting pages for the
|
|
307
|
-
bounded crawler, and extraction/indexing use no LLM. Installations requiring a fully
|
|
308
|
-
offline review can supply a separately built, verified index and omit the Brave key.
|
|
328
|
+
There is no offline index. Every passage a review sees is fetched live from the
|
|
329
|
+
provider allowlist during that review, so evidence is never served from a local
|
|
330
|
+
artifact whose contents ECR cannot vouch for. The crawler, its seed catalog, and
|
|
331
|
+
`research.indexPath` were removed once live discovery replaced them; a config still
|
|
332
|
+
naming `indexPath` fails to parse rather than silently ignoring it.
|
|
309
333
|
Installation-specific provider configuration is intentionally
|
|
310
334
|
deferred: when added, it should follow the trusted root-config model used for agents
|
|
311
335
|
without permitting PR-controlled URLs, commands, or executable parsers. Expo skills
|
package/build/commands/ci.js
CHANGED
|
@@ -463,10 +463,10 @@ async function runLegacyCi(source, repo, prNumber, cwd, configRoot, options) {
|
|
|
463
463
|
// Dynamic stack context and model-backed reply adjudication have inputs outside
|
|
464
464
|
// the scoped diff. Keep those paths fresh until their inputs join the cache key.
|
|
465
465
|
// A maintainer's explicit /review is also always a real rerun.
|
|
466
|
-
// Research
|
|
467
|
-
//
|
|
468
|
-
//
|
|
469
|
-
const cacheAllowed = !bypassTriggerGate && !stack && !feedback &&
|
|
466
|
+
// Research no longer forces a fresh run: with the local index gone, every result
|
|
467
|
+
// is fetched live from an allowlisted host during the run, so there is no mounted
|
|
468
|
+
// artifact whose contents could drift out from under the cache key.
|
|
469
|
+
const cacheAllowed = !bypassTriggerGate && !stack && !feedback && metadata !== undefined;
|
|
470
470
|
let inputHash;
|
|
471
471
|
try {
|
|
472
472
|
if (cacheAllowed) {
|
package/build/config/load.js
CHANGED
|
@@ -31,7 +31,7 @@ const RESEARCH_CONFIG_DEFAULTS = {
|
|
|
31
31
|
enabled: false,
|
|
32
32
|
maxQueries: 8,
|
|
33
33
|
resultsPerQuery: 2,
|
|
34
|
-
timeoutMs:
|
|
34
|
+
timeoutMs: 30_000,
|
|
35
35
|
};
|
|
36
36
|
/** Default OpenCode tool toggles for a reviewer: read the repo, never mutate it. */
|
|
37
37
|
const DEFAULT_AGENT_TOOLS = toolMap(["read", "grep", "glob", "list"]);
|
package/build/config/schema.js
CHANGED
|
@@ -61,10 +61,13 @@ export const ReviewConfigSchema = z.object({
|
|
|
61
61
|
research: z
|
|
62
62
|
.object({
|
|
63
63
|
enabled: z.boolean().default(false),
|
|
64
|
+
// Removed with the offline index. An unknown key would be stripped silently,
|
|
65
|
+
// so name it explicitly: a config carrying it is stale, and quietly ignoring
|
|
66
|
+
// the setting is worse than refusing to start.
|
|
64
67
|
indexPath: z
|
|
65
|
-
.
|
|
66
|
-
.
|
|
67
|
-
|
|
68
|
+
.never({
|
|
69
|
+
error: "research.indexPath was removed — documentation is now always fetched live from the provider allowlist. Delete this key.",
|
|
70
|
+
})
|
|
68
71
|
.optional(),
|
|
69
72
|
maxQueries: z.number().int().min(1).max(20).default(8),
|
|
70
73
|
resultsPerQuery: z.number().int().min(1).max(3).default(2),
|
package/build/core/research.js
CHANGED
|
@@ -7,6 +7,8 @@ import { tmpdir } from "node:os";
|
|
|
7
7
|
import path from "node:path";
|
|
8
8
|
import { fileURLToPath } from "node:url";
|
|
9
9
|
import { readResearchAudit } from "../research-mcp/audit.js";
|
|
10
|
+
import { researchChildEnvironment } from "../research-mcp/child-env.js";
|
|
11
|
+
import { totalResearchNetwork } from "../research-mcp/network.js";
|
|
10
12
|
export { OPENCODE_RESEARCH_TOOLS } from "./tools.js";
|
|
11
13
|
export const RESEARCH_DECISION_COUNT_LIMIT = 16;
|
|
12
14
|
export const RESEARCH_DECISION_BYTES_LIMIT = 20_000;
|
|
@@ -15,33 +17,17 @@ export const CLAUDE_RESEARCH_TOOLS = [
|
|
|
15
17
|
`mcp__${RESEARCH_MCP_SERVER_NAME}__search_platform_docs`,
|
|
16
18
|
`mcp__${RESEARCH_MCP_SERVER_NAME}__fetch_platform_doc`,
|
|
17
19
|
];
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
const RESEARCH_SEARCH_API_KEY = "BRAVE_SEARCH_API_KEY";
|
|
27
|
-
export function researchChildEnvironment(source = process.env) {
|
|
28
|
-
const environment = {
|
|
29
|
-
LANG: "C.UTF-8",
|
|
30
|
-
LC_ALL: "C.UTF-8",
|
|
31
|
-
...(process.platform === "win32" && source.SystemRoot ? { SystemRoot: source.SystemRoot } : {}),
|
|
32
|
-
};
|
|
33
|
-
for (const key of RESEARCH_PROXY_ENV_KEYS) {
|
|
34
|
-
if (source[key])
|
|
35
|
-
environment[key] = source[key];
|
|
36
|
-
}
|
|
37
|
-
if (source[RESEARCH_SEARCH_API_KEY]) {
|
|
38
|
-
environment[RESEARCH_SEARCH_API_KEY] = source[RESEARCH_SEARCH_API_KEY];
|
|
39
|
-
}
|
|
40
|
-
return environment;
|
|
41
|
-
}
|
|
20
|
+
export { researchChildEnvironment };
|
|
21
|
+
/**
|
|
22
|
+
* The engine spawns the WRAPPER, not the server. Both Claude Code and OpenCode
|
|
23
|
+
* merge the config's `env` block onto their own environment instead of replacing
|
|
24
|
+
* it, so the block below cannot bound the child on its own; the wrapper rebuilds
|
|
25
|
+
* the environment from an explicit allowlist before the real server starts. See
|
|
26
|
+
* research-mcp/wrapper.ts.
|
|
27
|
+
*/
|
|
42
28
|
export function bundledResearchServer() {
|
|
43
|
-
const builtEntry = fileURLToPath(new URL("../research-mcp/
|
|
44
|
-
const sourceEntry = fileURLToPath(new URL("../research-mcp/
|
|
29
|
+
const builtEntry = fileURLToPath(new URL("../research-mcp/wrapper.js", import.meta.url));
|
|
30
|
+
const sourceEntry = fileURLToPath(new URL("../research-mcp/wrapper.ts", import.meta.url));
|
|
45
31
|
return {
|
|
46
32
|
command: process.execPath,
|
|
47
33
|
args: [existsSync(builtEntry) ? builtEntry : sourceEntry],
|
|
@@ -49,8 +35,12 @@ export function bundledResearchServer() {
|
|
|
49
35
|
}
|
|
50
36
|
/**
|
|
51
37
|
* Create one owner-only MCP configuration and append-only audit for a review run.
|
|
52
|
-
* The model process receives only the config path
|
|
53
|
-
*
|
|
38
|
+
* The model process receives only the config path.
|
|
39
|
+
*
|
|
40
|
+
* The `env` block below is a request the engine merges rather than applies — see
|
|
41
|
+
* bundledResearchServer. The wrapper it names is what actually bounds the server's
|
|
42
|
+
* environment, so the Brave credential and these limits are the only things the
|
|
43
|
+
* server can see, whatever the engine passed down.
|
|
54
44
|
*/
|
|
55
45
|
export async function createResearchMcpRuntime(config) {
|
|
56
46
|
if (!config.enabled)
|
|
@@ -59,17 +49,17 @@ export async function createResearchMcpRuntime(config) {
|
|
|
59
49
|
const auditPath = path.join(directory, "audit.jsonl");
|
|
60
50
|
const claudeConfigPath = path.join(directory, "mcp.json");
|
|
61
51
|
const server = bundledResearchServer();
|
|
62
|
-
const args = [
|
|
63
|
-
...server.args,
|
|
64
|
-
"serve",
|
|
65
|
-
...(config.indexPath ? ["--index", config.indexPath] : []),
|
|
66
|
-
];
|
|
52
|
+
const args = [...server.args, "serve"];
|
|
67
53
|
const child = researchChildEnvironment();
|
|
68
54
|
const environment = Object.fromEntries(Object.entries({
|
|
69
55
|
...child,
|
|
70
56
|
REVIEW_RESEARCH_AUDIT_PATH: auditPath,
|
|
71
57
|
REVIEW_RESEARCH_MAX_CALLS: String(config.maxQueries),
|
|
72
58
|
REVIEW_RESEARCH_MAX_RESULTS: String(config.resultsPerQuery),
|
|
59
|
+
// The MCP enforces this itself as a per-call deadline. OpenCode's `timeout`
|
|
60
|
+
// only bounds tool DISCOVERY and Claude has no equivalent, so neither engine
|
|
61
|
+
// can bound how long a call actually runs — the server has to.
|
|
62
|
+
REVIEW_RESEARCH_TIMEOUT_MS: String(config.timeoutMs),
|
|
73
63
|
}).flatMap(([key, value]) => (value === undefined ? [] : [[key, value]])));
|
|
74
64
|
await writeFile(claudeConfigPath, `${JSON.stringify({
|
|
75
65
|
mcpServers: {
|
|
@@ -131,6 +121,9 @@ export async function researchProvenanceFromAudit(auditPath) {
|
|
|
131
121
|
warnings: [...new Set(warnings)].slice(0, 10),
|
|
132
122
|
};
|
|
133
123
|
const provenance = toResearchProvenance(run);
|
|
124
|
+
const ledgers = records.flatMap((record) => (record.network ? [record.network] : []));
|
|
125
|
+
if (ledgers.length > 0)
|
|
126
|
+
provenance.network = totalResearchNetwork(ledgers);
|
|
134
127
|
if (rejections.length > 0) {
|
|
135
128
|
const counts = new Map();
|
|
136
129
|
for (const rejection of rejections) {
|
|
@@ -199,6 +192,11 @@ export function formatResearchProgress(provenance) {
|
|
|
199
192
|
lines.push(` result: ${cleanEvidenceText(result.title, 160)} (${result.provider}/${result.sourceKind}) — ${result.url}`);
|
|
200
193
|
}
|
|
201
194
|
}
|
|
195
|
+
if (provenance.network) {
|
|
196
|
+
const { searchRequests, documentRequests, redirects, totalRequests } = provenance.network;
|
|
197
|
+
lines.push(` research network: ${searchRequests} search request(s), ${documentRequests} page fetch(es), ` +
|
|
198
|
+
`${redirects} redirect(s) — ${totalRequests} HTTP request(s) total`);
|
|
199
|
+
}
|
|
202
200
|
for (const rejection of provenance.rejections ?? []) {
|
|
203
201
|
lines.push(` research: ${rejection.count} ${rejection.tool} call(s) rejected before execution (${rejection.reason})`);
|
|
204
202
|
}
|
|
@@ -242,6 +240,12 @@ export function renderResearchMarkdown(provenance) {
|
|
|
242
240
|
lines.push(` - [${escapeMarkdownLabel(result.title)}](<${result.url}>) — ${escapeMarkdownLabel(result.provider)}/${escapeMarkdownLabel(result.sourceKind)}`);
|
|
243
241
|
}
|
|
244
242
|
}
|
|
243
|
+
if (provenance.network) {
|
|
244
|
+
const { searchRequests, documentRequests, redirects, totalRequests, elapsedMs } = provenance.network;
|
|
245
|
+
lines.push("", `Outbound: **${searchRequests}** search request(s), **${documentRequests}** page fetch(es), ` +
|
|
246
|
+
`**${redirects}** redirect(s) — **${totalRequests}** HTTP request(s) across ` +
|
|
247
|
+
`${(elapsedMs / 1000).toFixed(1)}s of call time.`);
|
|
248
|
+
}
|
|
245
249
|
for (const rejection of provenance.rejections ?? []) {
|
|
246
250
|
lines.push(`- ⚠️ ${rejection.count} \`${escapeMarkdownLabel(rejection.tool)}\` call(s) rejected before execution (${escapeMarkdownLabel(rejection.reason)}).`);
|
|
247
251
|
}
|
|
@@ -129,7 +129,7 @@ export class ResearchAudit {
|
|
|
129
129
|
});
|
|
130
130
|
return requestId;
|
|
131
131
|
}
|
|
132
|
-
async complete(requestId, tool, input, results, warnings = []) {
|
|
132
|
+
async complete(requestId, tool, input, results, warnings = [], network) {
|
|
133
133
|
await this.append({
|
|
134
134
|
type: "completed",
|
|
135
135
|
requestId,
|
|
@@ -137,6 +137,7 @@ export class ResearchAudit {
|
|
|
137
137
|
input,
|
|
138
138
|
results: results.map(boundedResult),
|
|
139
139
|
warnings: warnings.slice(0, 10).map((warning) => warning.slice(0, 500)),
|
|
140
|
+
...(network ? { network } : {}),
|
|
140
141
|
timestamp: new Date().toISOString(),
|
|
141
142
|
});
|
|
142
143
|
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// @ref LLP 0013#one-package-two-binaries [implements] — the bounded MCP's environment is constructed, never inherited
|
|
2
|
+
/**
|
|
3
|
+
* The single definition of what the bounded documentation MCP is allowed to see
|
|
4
|
+
* in its environment.
|
|
5
|
+
*
|
|
6
|
+
* Two callers share it, and they need it for different reasons:
|
|
7
|
+
*
|
|
8
|
+
* - `createResearchMcpRuntime` writes `researchChildEnvironment()` into the
|
|
9
|
+
* engine's MCP configuration. That block is a REQUEST, not a guarantee: both
|
|
10
|
+
* Claude Code and OpenCode merge it onto the environment the engine already
|
|
11
|
+
* has rather than replacing it, so the declared allowlist alone never bounds
|
|
12
|
+
* the child.
|
|
13
|
+
* - `wrapper.ts` applies `researchWrapperEnvironment()` when it spawns the real
|
|
14
|
+
* server. That IS the guarantee — the server process is handed a constructed
|
|
15
|
+
* environment, so whatever the engine merged in stops at the wrapper.
|
|
16
|
+
*
|
|
17
|
+
* Nothing here imports a parser, a network client, or the config schema: the
|
|
18
|
+
* wrapper must stay loadable without pulling untrusted-content machinery into
|
|
19
|
+
* the one process that still holds the engine's credentials.
|
|
20
|
+
*/
|
|
21
|
+
const PROXY_ENV_KEYS = [
|
|
22
|
+
"HTTP_PROXY",
|
|
23
|
+
"HTTPS_PROXY",
|
|
24
|
+
"NO_PROXY",
|
|
25
|
+
"http_proxy",
|
|
26
|
+
"https_proxy",
|
|
27
|
+
"no_proxy",
|
|
28
|
+
];
|
|
29
|
+
export const RESEARCH_SEARCH_API_KEY = "BRAVE_SEARCH_API_KEY";
|
|
30
|
+
/**
|
|
31
|
+
* Bounds ECR sets on the child through the config env block. These are inputs to
|
|
32
|
+
* the server, so the wrapper must forward them; every one is a number, a path ECR
|
|
33
|
+
* itself chose, or both, and none is a credential.
|
|
34
|
+
*/
|
|
35
|
+
export const RESEARCH_RUNTIME_ENV_KEYS = [
|
|
36
|
+
"REVIEW_RESEARCH_AUDIT_PATH",
|
|
37
|
+
"REVIEW_RESEARCH_MAX_CALLS",
|
|
38
|
+
"REVIEW_RESEARCH_MAX_RESULTS",
|
|
39
|
+
"REVIEW_RESEARCH_TIMEOUT_MS",
|
|
40
|
+
];
|
|
41
|
+
/**
|
|
42
|
+
* Locale, proxy configuration, and the search-only credential — the environment
|
|
43
|
+
* the MCP actually needs to do its job.
|
|
44
|
+
*
|
|
45
|
+
* NODE_OPTIONS is deliberately absent: it is arbitrary code injection into the
|
|
46
|
+
* process that parses untrusted remote documents. So are PATH and HOME — the
|
|
47
|
+
* wrapper spawns the server by absolute path and the server writes only to the
|
|
48
|
+
* audit path it is given.
|
|
49
|
+
*/
|
|
50
|
+
export function researchChildEnvironment(source = process.env) {
|
|
51
|
+
const environment = {
|
|
52
|
+
LANG: "C.UTF-8",
|
|
53
|
+
LC_ALL: "C.UTF-8",
|
|
54
|
+
...(process.platform === "win32" && source.SystemRoot ? { SystemRoot: source.SystemRoot } : {}),
|
|
55
|
+
};
|
|
56
|
+
for (const key of PROXY_ENV_KEYS) {
|
|
57
|
+
if (source[key])
|
|
58
|
+
environment[key] = source[key];
|
|
59
|
+
}
|
|
60
|
+
if (source[RESEARCH_SEARCH_API_KEY]) {
|
|
61
|
+
environment[RESEARCH_SEARCH_API_KEY] = source[RESEARCH_SEARCH_API_KEY];
|
|
62
|
+
}
|
|
63
|
+
return environment;
|
|
64
|
+
}
|
|
65
|
+
/** The child environment plus the bounds ECR passes through the config block. */
|
|
66
|
+
export function researchWrapperEnvironment(source = process.env) {
|
|
67
|
+
const environment = researchChildEnvironment(source);
|
|
68
|
+
for (const key of RESEARCH_RUNTIME_ENV_KEYS) {
|
|
69
|
+
if (source[key] !== undefined)
|
|
70
|
+
environment[key] = source[key];
|
|
71
|
+
}
|
|
72
|
+
return environment;
|
|
73
|
+
}
|
|
@@ -1,25 +1,18 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// @ref LLP 0013#one-package-two-binaries [implements] — the package's second binary
|
|
3
|
-
// @ref LLP 0013#search-fetch-and-optional-index-boundary [implements] —
|
|
4
|
-
import { parseArgs } from "node:util";
|
|
5
|
-
import { defaultConfigPath } from "./paths.js";
|
|
2
|
+
// @ref LLP 0013#one-package-two-binaries [implements] — the package's second binary serves the bounded MCP
|
|
3
|
+
// @ref LLP 0013#search-fetch-and-optional-index-boundary [implements] — live discovery is the only evidence path
|
|
6
4
|
import { runStdioServer } from "./server.js";
|
|
7
|
-
import { PLATFORMS } from "./types.js";
|
|
8
5
|
function printHelp() {
|
|
9
6
|
process.stdout.write(`review-research-mcp
|
|
10
7
|
|
|
11
8
|
Usage:
|
|
12
9
|
review-research-mcp [serve]
|
|
13
|
-
review-research-mcp serve [--index PATH]
|
|
14
|
-
review-research-mcp update [--config PATH] [--output PATH]
|
|
15
|
-
[--platform apple|android|react-native] [--max-pages NUMBER]
|
|
16
10
|
|
|
17
|
-
The serve command uses BRAVE_SEARCH_API_KEY for scoped web discovery
|
|
18
|
-
allowlisted official pages
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
an optional offline crawler for operator-managed fallback indexes.
|
|
11
|
+
The serve command uses BRAVE_SEARCH_API_KEY for scoped web discovery and fetches only
|
|
12
|
+
allowlisted official pages. Expo-provider searches use Expo's public documentation
|
|
13
|
+
index. Its fetch_platform_doc tool can fetch one exact allowlisted documentation URL
|
|
14
|
+
without a search key and return focused, section, or bounded-document extracted
|
|
15
|
+
context.
|
|
23
16
|
`);
|
|
24
17
|
}
|
|
25
18
|
function boundedInteger(name, fallback, minimum, maximum) {
|
|
@@ -43,60 +36,22 @@ async function main() {
|
|
|
43
36
|
printHelp();
|
|
44
37
|
return;
|
|
45
38
|
}
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
index: { type: "string" },
|
|
50
|
-
},
|
|
51
|
-
strict: true,
|
|
52
|
-
});
|
|
53
|
-
const indexPath = values.index ?? process.env.REVIEW_RESEARCH_INDEX_PATH;
|
|
39
|
+
if (rest.length > 0) {
|
|
40
|
+
throw new Error(`Unexpected argument: ${rest[0]}`);
|
|
41
|
+
}
|
|
54
42
|
await runStdioServer({
|
|
55
|
-
...(indexPath ? { indexPath } : {}),
|
|
56
43
|
...(process.env.REVIEW_RESEARCH_AUDIT_PATH
|
|
57
44
|
? { auditPath: process.env.REVIEW_RESEARCH_AUDIT_PATH }
|
|
58
45
|
: {}),
|
|
59
46
|
maxCalls: boundedInteger("REVIEW_RESEARCH_MAX_CALLS", 8, 1, 20),
|
|
60
47
|
maxResultsPerCall: boundedInteger("REVIEW_RESEARCH_MAX_RESULTS", 3, 1, 3),
|
|
48
|
+
timeoutMs: boundedInteger("REVIEW_RESEARCH_TIMEOUT_MS", 30_000, 1_000, 60_000),
|
|
61
49
|
...(process.env.BRAVE_SEARCH_API_KEY
|
|
62
50
|
? { braveApiKey: process.env.BRAVE_SEARCH_API_KEY }
|
|
63
51
|
: {}),
|
|
64
52
|
});
|
|
65
53
|
return;
|
|
66
54
|
}
|
|
67
|
-
if (command === "update") {
|
|
68
|
-
if (rest.includes("--help") || rest.includes("-h")) {
|
|
69
|
-
printHelp();
|
|
70
|
-
return;
|
|
71
|
-
}
|
|
72
|
-
const { values } = parseArgs({
|
|
73
|
-
args: rest,
|
|
74
|
-
options: {
|
|
75
|
-
config: { type: "string" },
|
|
76
|
-
output: { type: "string" },
|
|
77
|
-
platform: { type: "string", multiple: true },
|
|
78
|
-
"max-pages": { type: "string" },
|
|
79
|
-
},
|
|
80
|
-
strict: true,
|
|
81
|
-
});
|
|
82
|
-
const invalidPlatform = values.platform?.find((platform) => !PLATFORMS.includes(platform));
|
|
83
|
-
if (invalidPlatform) {
|
|
84
|
-
throw new Error(`Unknown platform: ${invalidPlatform}`);
|
|
85
|
-
}
|
|
86
|
-
const maxPages = values["max-pages"] ? Number(values["max-pages"]) : undefined;
|
|
87
|
-
if (maxPages !== undefined && (!Number.isInteger(maxPages) || maxPages < 1)) {
|
|
88
|
-
throw new Error("--max-pages must be a positive integer");
|
|
89
|
-
}
|
|
90
|
-
const { updateDocumentationIndex } = await import("./crawler.js");
|
|
91
|
-
const result = await updateDocumentationIndex({
|
|
92
|
-
configPath: values.config ?? defaultConfigPath,
|
|
93
|
-
...(values.output ? { outputPath: values.output } : {}),
|
|
94
|
-
...(values.platform ? { platforms: values.platform } : {}),
|
|
95
|
-
...(maxPages ? { maxPagesPerProvider: maxPages } : {}),
|
|
96
|
-
});
|
|
97
|
-
process.stderr.write(`${JSON.stringify(result, null, 2)}\n`);
|
|
98
|
-
return;
|
|
99
|
-
}
|
|
100
55
|
throw new Error(`Unknown command: ${command}`);
|
|
101
56
|
}
|
|
102
57
|
main().catch((error) => {
|
|
@@ -55,7 +55,7 @@ function withScore(chunk, score = 0) {
|
|
|
55
55
|
function rankedAnchor(chunks, document, provider, query) {
|
|
56
56
|
if (!query?.trim())
|
|
57
57
|
return chunks[0] ? withScore(chunks[0]) : undefined;
|
|
58
|
-
const index = buildSearchIndex(chunks
|
|
58
|
+
const index = buildSearchIndex(chunks);
|
|
59
59
|
return (searchDocumentation(index, query, {
|
|
60
60
|
platform: document.platform,
|
|
61
61
|
providers: [provider],
|
|
@@ -5,9 +5,6 @@ import { resolveAllowedRequestUrl, resolveAllowedUrl, } from "./providers.js";
|
|
|
5
5
|
import { readBodyWithLimit } from "./response.js";
|
|
6
6
|
import { extractYouTrackIssue } from "./youtrack.js";
|
|
7
7
|
export const onDemandFetchLimits = {
|
|
8
|
-
maxPagesPerProvider: 10,
|
|
9
|
-
maxDepth: 0,
|
|
10
|
-
delayMs: 0,
|
|
11
8
|
timeoutMs: 10_000,
|
|
12
9
|
maxResponseBytes: 5_000_000,
|
|
13
10
|
};
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/** Discovery endpoints. Everything else is a documentation page or asset. */
|
|
2
|
+
const SEARCH_ENDPOINTS = [
|
|
3
|
+
"https://api.search.brave.com/",
|
|
4
|
+
"https://qex7pb7d46-dsn.algolia.net/",
|
|
5
|
+
"https://lysine.dev/okhttp/search/search_index.json",
|
|
6
|
+
];
|
|
7
|
+
export class ResearchDeadlineError extends Error {
|
|
8
|
+
constructor(timeoutMs) {
|
|
9
|
+
super(`Documentation research call exceeded its ${timeoutMs}ms deadline`);
|
|
10
|
+
this.name = "ResearchDeadlineError";
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
export function createResearchNetwork(base, timeoutMs) {
|
|
14
|
+
const startedAt = Date.now();
|
|
15
|
+
const controller = new AbortController();
|
|
16
|
+
const timer = setTimeout(() => controller.abort(new ResearchDeadlineError(timeoutMs)), timeoutMs);
|
|
17
|
+
// The MCP is a short-lived stdio process; never hold the loop open for this.
|
|
18
|
+
timer.unref?.();
|
|
19
|
+
let searchRequests = 0;
|
|
20
|
+
let documentRequests = 0;
|
|
21
|
+
let redirects = 0;
|
|
22
|
+
let totalRequests = 0;
|
|
23
|
+
const wrapped = async (input, init) => {
|
|
24
|
+
if (controller.signal.aborted)
|
|
25
|
+
throw new ResearchDeadlineError(timeoutMs);
|
|
26
|
+
const href = typeof input === "string" ? input : input instanceof URL ? input.href : input.url;
|
|
27
|
+
const isSearch = SEARCH_ENDPOINTS.some((endpoint) => href.startsWith(endpoint));
|
|
28
|
+
totalRequests++;
|
|
29
|
+
if (isSearch)
|
|
30
|
+
searchRequests++;
|
|
31
|
+
else
|
|
32
|
+
documentRequests++;
|
|
33
|
+
// Keep the caller's own per-attempt timeout AND add the call deadline, so a
|
|
34
|
+
// single slow hop still fails fast while the whole call stays bounded.
|
|
35
|
+
const signal = init?.signal
|
|
36
|
+
? AbortSignal.any([init.signal, controller.signal])
|
|
37
|
+
: controller.signal;
|
|
38
|
+
const response = await base(input, { ...init, signal });
|
|
39
|
+
if (response.status >= 300 && response.status < 400) {
|
|
40
|
+
redirects++;
|
|
41
|
+
// A redirect hop is a round trip, not a distinct resource, so undo the
|
|
42
|
+
// classification count. It has to be the SAME counter that was
|
|
43
|
+
// incremented: the search backends use `redirect: "manual"` too and treat
|
|
44
|
+
// a 3xx as an error, so unconditionally decrementing documentRequests
|
|
45
|
+
// drives it negative the first time a discovery endpoint redirects.
|
|
46
|
+
if (isSearch)
|
|
47
|
+
searchRequests--;
|
|
48
|
+
else
|
|
49
|
+
documentRequests--;
|
|
50
|
+
}
|
|
51
|
+
return response;
|
|
52
|
+
};
|
|
53
|
+
return {
|
|
54
|
+
fetch: wrapped,
|
|
55
|
+
signal: controller.signal,
|
|
56
|
+
expired: () => controller.signal.aborted,
|
|
57
|
+
counts: () => ({
|
|
58
|
+
searchRequests,
|
|
59
|
+
documentRequests,
|
|
60
|
+
redirects,
|
|
61
|
+
totalRequests,
|
|
62
|
+
elapsedMs: Date.now() - startedAt,
|
|
63
|
+
}),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
/** Sum per-call ledgers into one review-wide total. */
|
|
67
|
+
export function totalResearchNetwork(counts) {
|
|
68
|
+
return counts.reduce((total, entry) => ({
|
|
69
|
+
searchRequests: total.searchRequests + entry.searchRequests,
|
|
70
|
+
documentRequests: total.documentRequests + entry.documentRequests,
|
|
71
|
+
redirects: total.redirects + entry.redirects,
|
|
72
|
+
totalRequests: total.totalRequests + entry.totalRequests,
|
|
73
|
+
elapsedMs: total.elapsedMs + entry.elapsedMs,
|
|
74
|
+
}), { searchRequests: 0, documentRequests: 0, redirects: 0, totalRequests: 0, elapsedMs: 0 });
|
|
75
|
+
}
|
|
@@ -54,7 +54,7 @@ async function loadOkHttpSearchIndex(fetchImplementation) {
|
|
|
54
54
|
}
|
|
55
55
|
});
|
|
56
56
|
const chunks = documents.flatMap((document) => chunkDocument(document, indexedAt));
|
|
57
|
-
return buildSearchIndex(chunks
|
|
57
|
+
return buildSearchIndex(chunks);
|
|
58
58
|
}
|
|
59
59
|
function cachedOkHttpSearchIndex(fetchImplementation) {
|
|
60
60
|
const cached = indexCache.get(fetchImplementation);
|
|
@@ -101,7 +101,7 @@ function bestPassage(document, query, indexedAt) {
|
|
|
101
101
|
if (chunks.length === 0) {
|
|
102
102
|
return { passage: document.body.slice(0, 1_400), relevance: 0 };
|
|
103
103
|
}
|
|
104
|
-
const index = buildSearchIndex(chunks
|
|
104
|
+
const index = buildSearchIndex(chunks);
|
|
105
105
|
const result = searchDocumentation(index, query, {
|
|
106
106
|
platform: document.platform,
|
|
107
107
|
providers: document.provider ? [document.provider] : undefined,
|
|
@@ -175,7 +175,9 @@ export async function searchRemoteDocumentation(providerId, query, limit, option
|
|
|
175
175
|
// downloading every discovery candidate.
|
|
176
176
|
const fetched = [];
|
|
177
177
|
let candidateIndex = 0;
|
|
178
|
-
while (fetched.length < limit &&
|
|
178
|
+
while (fetched.length < limit &&
|
|
179
|
+
candidateIndex < candidates.length &&
|
|
180
|
+
!options.deadline?.aborted) {
|
|
179
181
|
const outstanding = limit - fetched.length;
|
|
180
182
|
const batch = candidates.slice(candidateIndex, candidateIndex + outstanding);
|
|
181
183
|
candidateIndex += batch.length;
|
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
|
|
2
|
-
import path from "node:path";
|
|
3
1
|
import MiniSearch from "minisearch";
|
|
4
2
|
const miniSearchOptions = {
|
|
5
3
|
fields: ["title", "passage", "framework", "symbol", "provider"],
|
|
@@ -20,37 +18,15 @@ const miniSearchOptions = {
|
|
|
20
18
|
"indexedAt",
|
|
21
19
|
],
|
|
22
20
|
};
|
|
23
|
-
|
|
21
|
+
/**
|
|
22
|
+
* Build a throwaway in-memory index over one fetch's chunks, purely to rank them.
|
|
23
|
+
* Nothing is serialized or persisted: since the offline index was removed, every
|
|
24
|
+
* caller builds this per request and discards it.
|
|
25
|
+
*/
|
|
26
|
+
export function buildSearchIndex(chunks) {
|
|
24
27
|
const miniSearch = new MiniSearch(miniSearchOptions);
|
|
25
28
|
miniSearch.addAll(chunks);
|
|
26
|
-
|
|
27
|
-
...new Set(chunks.map((chunk) => chunk.provider ?? chunk.platform)),
|
|
28
|
-
].sort();
|
|
29
|
-
return {
|
|
30
|
-
miniSearch,
|
|
31
|
-
serialized: {
|
|
32
|
-
schemaVersion: 1,
|
|
33
|
-
generatedAt,
|
|
34
|
-
documentCount,
|
|
35
|
-
chunkCount: chunks.length,
|
|
36
|
-
providers,
|
|
37
|
-
searchIndex: miniSearch.toJSON(),
|
|
38
|
-
},
|
|
39
|
-
};
|
|
40
|
-
}
|
|
41
|
-
export async function writeSearchIndex(filePath, index) {
|
|
42
|
-
await mkdir(path.dirname(filePath), { recursive: true });
|
|
43
|
-
const temporaryPath = `${filePath}.tmp-${process.pid}`;
|
|
44
|
-
await writeFile(temporaryPath, `${JSON.stringify(index)}\n`, { mode: 0o644 });
|
|
45
|
-
await rename(temporaryPath, filePath);
|
|
46
|
-
}
|
|
47
|
-
export async function loadSearchIndex(filePath) {
|
|
48
|
-
const serialized = JSON.parse(await readFile(filePath, "utf8"));
|
|
49
|
-
if (serialized.schemaVersion !== 1 || typeof serialized.searchIndex !== "object") {
|
|
50
|
-
throw new Error(`Unsupported or invalid search index at ${filePath}`);
|
|
51
|
-
}
|
|
52
|
-
const miniSearch = MiniSearch.loadJSON(JSON.stringify(serialized.searchIndex), miniSearchOptions);
|
|
53
|
-
return { serialized, miniSearch };
|
|
29
|
+
return { miniSearch };
|
|
54
30
|
}
|
|
55
31
|
function searchOptions(combineWith, exact = false) {
|
|
56
32
|
return {
|