@expo/code-review-cli 0.12.4 → 0.12.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -190,7 +190,7 @@ Enable it only in the root config, which CI loads from the PR's trusted base:
190
190
  "enabled": true,
191
191
  "maxQueries": 8,
192
192
  "resultsPerQuery": 2,
193
- "timeoutMs": 15000
193
+ "timeoutMs": 30000
194
194
  }
195
195
  }
196
196
  ```
@@ -205,13 +205,26 @@ the current absolute Node executable, so a PR-owned `PATH` entry cannot replace
205
205
  either component. Each review gets an owner-only temporary MCP config and append-only
206
206
  audit. Claude receives that explicit config under `--strict-mcp-config`, with project
207
207
  settings and slash commands disabled; OpenCode receives the same fixed local command.
208
- The Brave credential is passed to the MCP child, not the model process.
209
208
 
210
- The MCP is the outbound security boundary. Search queries are normalized before
209
+ That command is a wrapper, not the server. Both engines merge a configured MCP `env`
210
+ onto their own environment instead of replacing it, so the declared block alone cannot
211
+ bound the child — the engine's model credential reaches it, and OpenCode additionally
212
+ passes down the runner's whole ambient environment. The wrapper therefore rebuilds the
213
+ environment from an explicit allowlist before starting the server, and loads no parser
214
+ and opens no socket of its own. The server process sees the search key, the call
215
+ bounds, and locale/proxy settings; it never sees a model credential. The Brave
216
+ credential travels the same way and is never added to the model process env.
217
+
218
+ The MCP bounds the shape, host, and volume of outbound requests. It is not a
219
+ confidentiality boundary: the reviewing model chooses the query terms and URLs, and a
220
+ low-entropy identifier can carry repository-derived data past every check below.
221
+ Enable research only where repository-derived terms may be shared with Brave and the
222
+ documentation providers. Search queries are normalized before
211
223
  logging or networking: quoted literals, URLs, email addresses, paths, prose stop
212
224
  words, overlong/high-entropy tokens, and unsupported punctuation are removed;
213
- credential-shaped or secret-labeled input fails closed. The remaining query must be
214
- at most eight short tokens and contain an API-like symbol. Direct URLs must use plain
225
+ credential-shaped or secret-labeled input fails closed. The remaining query must be at
226
+ most eight short tokens and either contain an API-like symbol or be a short multi-word
227
+ lowercase concept phrase. Direct URLs must use plain
215
228
  HTTPS with no credentials, port, query string, or fragment; suspicious/high-entropy
216
229
  path segments fail closed. The fixed provider host/path allowlist and redirect,
217
230
  response-size, content-type, and timeout checks still apply after that first gate.
@@ -219,6 +232,17 @@ These deterministic checks greatly reduce accidental exfiltration; they are not
219
232
  proof that every low-entropy string is harmless, so reviewer prompts also forbid
220
233
  sending repository text and the review-wide MCP budget defaults to eight calls.
221
234
 
235
+ `maxQueries` bounds MCP calls, not network requests. One search selects up to four
236
+ providers, and each issues its own discovery request plus a page fetch per candidate,
237
+ so eight calls can mean roughly thirty discovery requests and over a hundred page
238
+ downloads. Every call therefore reports its own ledger — discovery requests, page
239
+ fetches, redirect hops, total HTTP requests, and elapsed time — and the review log and
240
+ Actions summary report the totals. `timeoutMs` is the MCP's own end-to-end deadline for
241
+ one call, enforced by the server across discovery, redirects, retrieval, and
242
+ extraction; a call that hits it returns what it already has rather than failing. It has
243
+ to live there because OpenCode's `timeout` bounds only tool discovery and Claude
244
+ provides no per-call timeout at all.
245
+
222
246
  For non-Expo providers, discovery sends a fixed, provider-owned `site:` scope plus
223
247
  the bounded query to Brave's fixed Web Search endpoint. Search snippets and titles
224
248
  are never treated as evidence. ECR independently rejects off-allowlist result URLs,
@@ -276,8 +300,11 @@ same audit trail in the step summary, while `.runs/reviews.jsonl` keeps the quer
276
300
  plus bounded returned passages for short-lived operational inspection. Reviewers
277
301
  are instructed to attach `sources` only when documentation materially supports a
278
302
  finding. ECR accepts only exact URLs returned during that review, restores canonical
279
- titles, carries citations through coordination, and renders them below the finding;
280
- invented or unrelated citations are dropped.
303
+ titles, carries citations through coordination, and renders them below the finding.
304
+ A citation to a URL this review never retrieved is dropped outright. Relatedness is a
305
+ separate, weaker guarantee: a cited finding is escalated to the verifier with the
306
+ audited passage inline, which judges whether that passage actually supports the claim
307
+ and strips the citation when it does not.
281
308
 
282
309
  Reviewers also emit a bounded `researchDecisions` record only when documentation
283
310
  materially confirms a finding candidate or proves one safe. ECR grounds those records
@@ -298,14 +325,11 @@ downloads the fixed official `lysine.dev` static search index, validates and ind
298
325
  it in memory once per MCP process, and rejects any entry outside the existing OkHttp
299
326
  allowlist. Brave remains a fallback if that official index is unavailable.
300
327
 
301
- An absolute `research.indexPath` remains available as an optional local fallback.
302
- The MCP and its trusted updater ship with ECR: build that fallback from this
303
- repository with `bun run research:update`, or from an installed package with
304
- `review-research-mcp update`. This is operator/scheduled offline tooling, not a
305
- step to run before each review. The built-in seed catalog lives in
306
- `research/sources.json`; `seedUrls` are deterministic starting pages for the
307
- bounded crawler, and extraction/indexing use no LLM. Installations requiring a fully
308
- offline review can supply a separately built, verified index and omit the Brave key.
328
+ There is no offline index. Every passage a review sees is fetched live from the
329
+ provider allowlist during that review, so evidence is never served from a local
330
+ artifact whose contents ECR cannot vouch for. The crawler, its seed catalog, and
331
+ `research.indexPath` were removed once live discovery replaced them; a config still
332
+ naming `indexPath` fails to parse rather than silently ignoring it.
309
333
  Installation-specific provider configuration is intentionally
310
334
  deferred: when added, it should follow the trusted root-config model used for agents
311
335
  without permitting PR-controlled URLs, commands, or executable parsers. Expo skills
@@ -463,10 +463,10 @@ async function runLegacyCi(source, repo, prNumber, cwd, configRoot, options) {
463
463
  // Dynamic stack context and model-backed reply adjudication have inputs outside
464
464
  // the scoped diff. Keep those paths fresh until their inputs join the cache key.
465
465
  // A maintainer's explicit /review is also always a real rerun.
466
- // Research output depends on the mounted index contents, not merely its configured
467
- // path. Until a signed index digest joins the cache key, a researched review must
468
- // run fresh rather than reuse evidence from an older artifact at the same path.
469
- const cacheAllowed = !bypassTriggerGate && !stack && !feedback && !config.research.enabled && metadata !== undefined;
466
+ // Research no longer forces a fresh run: with the local index gone, every result
467
+ // is fetched live from an allowlisted host during the run, so there is no mounted
468
+ // artifact whose contents could drift out from under the cache key.
469
+ const cacheAllowed = !bypassTriggerGate && !stack && !feedback && metadata !== undefined;
470
470
  let inputHash;
471
471
  try {
472
472
  if (cacheAllowed) {
@@ -31,7 +31,7 @@ const RESEARCH_CONFIG_DEFAULTS = {
31
31
  enabled: false,
32
32
  maxQueries: 8,
33
33
  resultsPerQuery: 2,
34
- timeoutMs: 15_000,
34
+ timeoutMs: 30_000,
35
35
  };
36
36
  /** Default OpenCode tool toggles for a reviewer: read the repo, never mutate it. */
37
37
  const DEFAULT_AGENT_TOOLS = toolMap(["read", "grep", "glob", "list"]);
@@ -61,10 +61,13 @@ export const ReviewConfigSchema = z.object({
61
61
  research: z
62
62
  .object({
63
63
  enabled: z.boolean().default(false),
64
+ // Removed with the offline index. An unknown key would be stripped silently,
65
+ // so name it explicitly: a config carrying it is stale, and quietly ignoring
66
+ // the setting is worse than refusing to start.
64
67
  indexPath: z
65
- .string()
66
- .min(1)
67
- .refine((value) => path.isAbsolute(value), "research.indexPath must be an absolute path")
68
+ .never({
69
+ error: "research.indexPath was removed — documentation is now always fetched live from the provider allowlist. Delete this key.",
70
+ })
68
71
  .optional(),
69
72
  maxQueries: z.number().int().min(1).max(20).default(8),
70
73
  resultsPerQuery: z.number().int().min(1).max(3).default(2),
@@ -7,6 +7,8 @@ import { tmpdir } from "node:os";
7
7
  import path from "node:path";
8
8
  import { fileURLToPath } from "node:url";
9
9
  import { readResearchAudit } from "../research-mcp/audit.js";
10
+ import { researchChildEnvironment } from "../research-mcp/child-env.js";
11
+ import { totalResearchNetwork } from "../research-mcp/network.js";
10
12
  export { OPENCODE_RESEARCH_TOOLS } from "./tools.js";
11
13
  export const RESEARCH_DECISION_COUNT_LIMIT = 16;
12
14
  export const RESEARCH_DECISION_BYTES_LIMIT = 20_000;
@@ -15,33 +17,17 @@ export const CLAUDE_RESEARCH_TOOLS = [
15
17
  `mcp__${RESEARCH_MCP_SERVER_NAME}__search_platform_docs`,
16
18
  `mcp__${RESEARCH_MCP_SERVER_NAME}__fetch_platform_doc`,
17
19
  ];
18
- const RESEARCH_PROXY_ENV_KEYS = [
19
- "HTTP_PROXY",
20
- "HTTPS_PROXY",
21
- "NO_PROXY",
22
- "http_proxy",
23
- "https_proxy",
24
- "no_proxy",
25
- ];
26
- const RESEARCH_SEARCH_API_KEY = "BRAVE_SEARCH_API_KEY";
27
- export function researchChildEnvironment(source = process.env) {
28
- const environment = {
29
- LANG: "C.UTF-8",
30
- LC_ALL: "C.UTF-8",
31
- ...(process.platform === "win32" && source.SystemRoot ? { SystemRoot: source.SystemRoot } : {}),
32
- };
33
- for (const key of RESEARCH_PROXY_ENV_KEYS) {
34
- if (source[key])
35
- environment[key] = source[key];
36
- }
37
- if (source[RESEARCH_SEARCH_API_KEY]) {
38
- environment[RESEARCH_SEARCH_API_KEY] = source[RESEARCH_SEARCH_API_KEY];
39
- }
40
- return environment;
41
- }
20
+ export { researchChildEnvironment };
21
+ /**
22
+ * The engine spawns the WRAPPER, not the server. Both Claude Code and OpenCode
23
+ * merge the config's `env` block onto their own environment instead of replacing
24
+ * it, so the block below cannot bound the child on its own; the wrapper rebuilds
25
+ * the environment from an explicit allowlist before the real server starts. See
26
+ * research-mcp/wrapper.ts.
27
+ */
42
28
  export function bundledResearchServer() {
43
- const builtEntry = fileURLToPath(new URL("../research-mcp/cli.js", import.meta.url));
44
- const sourceEntry = fileURLToPath(new URL("../research-mcp/cli.ts", import.meta.url));
29
+ const builtEntry = fileURLToPath(new URL("../research-mcp/wrapper.js", import.meta.url));
30
+ const sourceEntry = fileURLToPath(new URL("../research-mcp/wrapper.ts", import.meta.url));
45
31
  return {
46
32
  command: process.execPath,
47
33
  args: [existsSync(builtEntry) ? builtEntry : sourceEntry],
@@ -49,8 +35,12 @@ export function bundledResearchServer() {
49
35
  }
50
36
  /**
51
37
  * Create one owner-only MCP configuration and append-only audit for a review run.
52
- * The model process receives only the config path; the Brave credential is passed
53
- * directly to the bounded MCP child and never added to the model process env.
38
+ * The model process receives only the config path.
39
+ *
40
+ * The `env` block below is a request the engine merges rather than applies — see
41
+ * bundledResearchServer. The wrapper it names is what actually bounds the server's
42
+ * environment, so the Brave credential and these limits are the only things the
43
+ * server can see, whatever the engine passed down.
54
44
  */
55
45
  export async function createResearchMcpRuntime(config) {
56
46
  if (!config.enabled)
@@ -59,17 +49,17 @@ export async function createResearchMcpRuntime(config) {
59
49
  const auditPath = path.join(directory, "audit.jsonl");
60
50
  const claudeConfigPath = path.join(directory, "mcp.json");
61
51
  const server = bundledResearchServer();
62
- const args = [
63
- ...server.args,
64
- "serve",
65
- ...(config.indexPath ? ["--index", config.indexPath] : []),
66
- ];
52
+ const args = [...server.args, "serve"];
67
53
  const child = researchChildEnvironment();
68
54
  const environment = Object.fromEntries(Object.entries({
69
55
  ...child,
70
56
  REVIEW_RESEARCH_AUDIT_PATH: auditPath,
71
57
  REVIEW_RESEARCH_MAX_CALLS: String(config.maxQueries),
72
58
  REVIEW_RESEARCH_MAX_RESULTS: String(config.resultsPerQuery),
59
+ // The MCP enforces this itself as a per-call deadline. OpenCode's `timeout`
60
+ // only bounds tool DISCOVERY and Claude has no equivalent, so neither engine
61
+ // can bound how long a call actually runs — the server has to.
62
+ REVIEW_RESEARCH_TIMEOUT_MS: String(config.timeoutMs),
73
63
  }).flatMap(([key, value]) => (value === undefined ? [] : [[key, value]])));
74
64
  await writeFile(claudeConfigPath, `${JSON.stringify({
75
65
  mcpServers: {
@@ -131,6 +121,9 @@ export async function researchProvenanceFromAudit(auditPath) {
131
121
  warnings: [...new Set(warnings)].slice(0, 10),
132
122
  };
133
123
  const provenance = toResearchProvenance(run);
124
+ const ledgers = records.flatMap((record) => (record.network ? [record.network] : []));
125
+ if (ledgers.length > 0)
126
+ provenance.network = totalResearchNetwork(ledgers);
134
127
  if (rejections.length > 0) {
135
128
  const counts = new Map();
136
129
  for (const rejection of rejections) {
@@ -199,6 +192,11 @@ export function formatResearchProgress(provenance) {
199
192
  lines.push(` result: ${cleanEvidenceText(result.title, 160)} (${result.provider}/${result.sourceKind}) — ${result.url}`);
200
193
  }
201
194
  }
195
+ if (provenance.network) {
196
+ const { searchRequests, documentRequests, redirects, totalRequests } = provenance.network;
197
+ lines.push(` research network: ${searchRequests} search request(s), ${documentRequests} page fetch(es), ` +
198
+ `${redirects} redirect(s) — ${totalRequests} HTTP request(s) total`);
199
+ }
202
200
  for (const rejection of provenance.rejections ?? []) {
203
201
  lines.push(` research: ${rejection.count} ${rejection.tool} call(s) rejected before execution (${rejection.reason})`);
204
202
  }
@@ -242,6 +240,12 @@ export function renderResearchMarkdown(provenance) {
242
240
  lines.push(` - [${escapeMarkdownLabel(result.title)}](<${result.url}>) — ${escapeMarkdownLabel(result.provider)}/${escapeMarkdownLabel(result.sourceKind)}`);
243
241
  }
244
242
  }
243
+ if (provenance.network) {
244
+ const { searchRequests, documentRequests, redirects, totalRequests, elapsedMs } = provenance.network;
245
+ lines.push("", `Outbound: **${searchRequests}** search request(s), **${documentRequests}** page fetch(es), ` +
246
+ `**${redirects}** redirect(s) — **${totalRequests}** HTTP request(s) across ` +
247
+ `${(elapsedMs / 1000).toFixed(1)}s of call time.`);
248
+ }
245
249
  for (const rejection of provenance.rejections ?? []) {
246
250
  lines.push(`- ⚠️ ${rejection.count} \`${escapeMarkdownLabel(rejection.tool)}\` call(s) rejected before execution (${escapeMarkdownLabel(rejection.reason)}).`);
247
251
  }
@@ -129,7 +129,7 @@ export class ResearchAudit {
129
129
  });
130
130
  return requestId;
131
131
  }
132
- async complete(requestId, tool, input, results, warnings = []) {
132
+ async complete(requestId, tool, input, results, warnings = [], network) {
133
133
  await this.append({
134
134
  type: "completed",
135
135
  requestId,
@@ -137,6 +137,7 @@ export class ResearchAudit {
137
137
  input,
138
138
  results: results.map(boundedResult),
139
139
  warnings: warnings.slice(0, 10).map((warning) => warning.slice(0, 500)),
140
+ ...(network ? { network } : {}),
140
141
  timestamp: new Date().toISOString(),
141
142
  });
142
143
  }
@@ -0,0 +1,73 @@
1
+ // @ref LLP 0013#one-package-two-binaries [implements] — the bounded MCP's environment is constructed, never inherited
2
+ /**
3
+ * The single definition of what the bounded documentation MCP is allowed to see
4
+ * in its environment.
5
+ *
6
+ * Two callers share it, and they need it for different reasons:
7
+ *
8
+ * - `createResearchMcpRuntime` writes `researchChildEnvironment()` into the
9
+ * engine's MCP configuration. That block is a REQUEST, not a guarantee: both
10
+ * Claude Code and OpenCode merge it onto the environment the engine already
11
+ * has rather than replacing it, so the declared allowlist alone never bounds
12
+ * the child.
13
+ * - `wrapper.ts` applies `researchWrapperEnvironment()` when it spawns the real
14
+ * server. That IS the guarantee — the server process is handed a constructed
15
+ * environment, so whatever the engine merged in stops at the wrapper.
16
+ *
17
+ * Nothing here imports a parser, a network client, or the config schema: the
18
+ * wrapper must stay loadable without pulling untrusted-content machinery into
19
+ * the one process that still holds the engine's credentials.
20
+ */
21
+ const PROXY_ENV_KEYS = [
22
+ "HTTP_PROXY",
23
+ "HTTPS_PROXY",
24
+ "NO_PROXY",
25
+ "http_proxy",
26
+ "https_proxy",
27
+ "no_proxy",
28
+ ];
29
+ export const RESEARCH_SEARCH_API_KEY = "BRAVE_SEARCH_API_KEY";
30
+ /**
31
+ * Bounds ECR sets on the child through the config env block. These are inputs to
32
+ * the server, so the wrapper must forward them; every one is a number, a path ECR
33
+ * itself chose, or both, and none is a credential.
34
+ */
35
+ export const RESEARCH_RUNTIME_ENV_KEYS = [
36
+ "REVIEW_RESEARCH_AUDIT_PATH",
37
+ "REVIEW_RESEARCH_MAX_CALLS",
38
+ "REVIEW_RESEARCH_MAX_RESULTS",
39
+ "REVIEW_RESEARCH_TIMEOUT_MS",
40
+ ];
41
+ /**
42
+ * Locale, proxy configuration, and the search-only credential — the environment
43
+ * the MCP actually needs to do its job.
44
+ *
45
+ * NODE_OPTIONS is deliberately absent: it is arbitrary code injection into the
46
+ * process that parses untrusted remote documents. So are PATH and HOME — the
47
+ * wrapper spawns the server by absolute path and the server writes only to the
48
+ * audit path it is given.
49
+ */
50
+ export function researchChildEnvironment(source = process.env) {
51
+ const environment = {
52
+ LANG: "C.UTF-8",
53
+ LC_ALL: "C.UTF-8",
54
+ ...(process.platform === "win32" && source.SystemRoot ? { SystemRoot: source.SystemRoot } : {}),
55
+ };
56
+ for (const key of PROXY_ENV_KEYS) {
57
+ if (source[key])
58
+ environment[key] = source[key];
59
+ }
60
+ if (source[RESEARCH_SEARCH_API_KEY]) {
61
+ environment[RESEARCH_SEARCH_API_KEY] = source[RESEARCH_SEARCH_API_KEY];
62
+ }
63
+ return environment;
64
+ }
65
+ /** The child environment plus the bounds ECR passes through the config block. */
66
+ export function researchWrapperEnvironment(source = process.env) {
67
+ const environment = researchChildEnvironment(source);
68
+ for (const key of RESEARCH_RUNTIME_ENV_KEYS) {
69
+ if (source[key] !== undefined)
70
+ environment[key] = source[key];
71
+ }
72
+ return environment;
73
+ }
@@ -1,25 +1,18 @@
1
1
  #!/usr/bin/env node
2
- // @ref LLP 0013#one-package-two-binaries [implements] — the package's second binary owns serve/update dispatch
3
- // @ref LLP 0013#search-fetch-and-optional-index-boundary [implements] — review-facing serve and operator-only update stay separate
4
- import { parseArgs } from "node:util";
5
- import { defaultConfigPath } from "./paths.js";
2
+ // @ref LLP 0013#one-package-two-binaries [implements] — the package's second binary serves the bounded MCP
3
+ // @ref LLP 0013#search-fetch-and-optional-index-boundary [implements] — live discovery is the only evidence path
6
4
  import { runStdioServer } from "./server.js";
7
- import { PLATFORMS } from "./types.js";
8
5
  function printHelp() {
9
6
  process.stdout.write(`review-research-mcp
10
7
 
11
8
  Usage:
12
9
  review-research-mcp [serve]
13
- review-research-mcp serve [--index PATH]
14
- review-research-mcp update [--config PATH] [--output PATH]
15
- [--platform apple|android|react-native] [--max-pages NUMBER]
16
10
 
17
- The serve command uses BRAVE_SEARCH_API_KEY for scoped web discovery, fetches only
18
- allowlisted official pages, and optionally falls back to a local index. Expo-provider
19
- searches use Expo's public documentation index. Its fetch_platform_doc tool can fetch
20
- one exact allowlisted documentation URL without a search key and return focused,
21
- section, or bounded-document extracted context. The update command is
22
- an optional offline crawler for operator-managed fallback indexes.
11
+ The serve command uses BRAVE_SEARCH_API_KEY for scoped web discovery and fetches only
12
+ allowlisted official pages. Expo-provider searches use Expo's public documentation
13
+ index. Its fetch_platform_doc tool can fetch one exact allowlisted documentation URL
14
+ without a search key and return focused, section, or bounded-document extracted
15
+ context.
23
16
  `);
24
17
  }
25
18
  function boundedInteger(name, fallback, minimum, maximum) {
@@ -43,60 +36,22 @@ async function main() {
43
36
  printHelp();
44
37
  return;
45
38
  }
46
- const { values } = parseArgs({
47
- args: rest,
48
- options: {
49
- index: { type: "string" },
50
- },
51
- strict: true,
52
- });
53
- const indexPath = values.index ?? process.env.REVIEW_RESEARCH_INDEX_PATH;
39
+ if (rest.length > 0) {
40
+ throw new Error(`Unexpected argument: ${rest[0]}`);
41
+ }
54
42
  await runStdioServer({
55
- ...(indexPath ? { indexPath } : {}),
56
43
  ...(process.env.REVIEW_RESEARCH_AUDIT_PATH
57
44
  ? { auditPath: process.env.REVIEW_RESEARCH_AUDIT_PATH }
58
45
  : {}),
59
46
  maxCalls: boundedInteger("REVIEW_RESEARCH_MAX_CALLS", 8, 1, 20),
60
47
  maxResultsPerCall: boundedInteger("REVIEW_RESEARCH_MAX_RESULTS", 3, 1, 3),
48
+ timeoutMs: boundedInteger("REVIEW_RESEARCH_TIMEOUT_MS", 30_000, 1_000, 60_000),
61
49
  ...(process.env.BRAVE_SEARCH_API_KEY
62
50
  ? { braveApiKey: process.env.BRAVE_SEARCH_API_KEY }
63
51
  : {}),
64
52
  });
65
53
  return;
66
54
  }
67
- if (command === "update") {
68
- if (rest.includes("--help") || rest.includes("-h")) {
69
- printHelp();
70
- return;
71
- }
72
- const { values } = parseArgs({
73
- args: rest,
74
- options: {
75
- config: { type: "string" },
76
- output: { type: "string" },
77
- platform: { type: "string", multiple: true },
78
- "max-pages": { type: "string" },
79
- },
80
- strict: true,
81
- });
82
- const invalidPlatform = values.platform?.find((platform) => !PLATFORMS.includes(platform));
83
- if (invalidPlatform) {
84
- throw new Error(`Unknown platform: ${invalidPlatform}`);
85
- }
86
- const maxPages = values["max-pages"] ? Number(values["max-pages"]) : undefined;
87
- if (maxPages !== undefined && (!Number.isInteger(maxPages) || maxPages < 1)) {
88
- throw new Error("--max-pages must be a positive integer");
89
- }
90
- const { updateDocumentationIndex } = await import("./crawler.js");
91
- const result = await updateDocumentationIndex({
92
- configPath: values.config ?? defaultConfigPath,
93
- ...(values.output ? { outputPath: values.output } : {}),
94
- ...(values.platform ? { platforms: values.platform } : {}),
95
- ...(maxPages ? { maxPagesPerProvider: maxPages } : {}),
96
- });
97
- process.stderr.write(`${JSON.stringify(result, null, 2)}\n`);
98
- return;
99
- }
100
55
  throw new Error(`Unknown command: ${command}`);
101
56
  }
102
57
  main().catch((error) => {
@@ -55,7 +55,7 @@ function withScore(chunk, score = 0) {
55
55
  function rankedAnchor(chunks, document, provider, query) {
56
56
  if (!query?.trim())
57
57
  return chunks[0] ? withScore(chunks[0]) : undefined;
58
- const index = buildSearchIndex(chunks, 1);
58
+ const index = buildSearchIndex(chunks);
59
59
  return (searchDocumentation(index, query, {
60
60
  platform: document.platform,
61
61
  providers: [provider],
@@ -5,9 +5,6 @@ import { resolveAllowedRequestUrl, resolveAllowedUrl, } from "./providers.js";
5
5
  import { readBodyWithLimit } from "./response.js";
6
6
  import { extractYouTrackIssue } from "./youtrack.js";
7
7
  export const onDemandFetchLimits = {
8
- maxPagesPerProvider: 10,
9
- maxDepth: 0,
10
- delayMs: 0,
11
8
  timeoutMs: 10_000,
12
9
  maxResponseBytes: 5_000_000,
13
10
  };
@@ -0,0 +1,75 @@
1
+ /** Discovery endpoints. Everything else is a documentation page or asset. */
2
+ const SEARCH_ENDPOINTS = [
3
+ "https://api.search.brave.com/",
4
+ "https://qex7pb7d46-dsn.algolia.net/",
5
+ "https://lysine.dev/okhttp/search/search_index.json",
6
+ ];
7
+ export class ResearchDeadlineError extends Error {
8
+ constructor(timeoutMs) {
9
+ super(`Documentation research call exceeded its ${timeoutMs}ms deadline`);
10
+ this.name = "ResearchDeadlineError";
11
+ }
12
+ }
13
+ export function createResearchNetwork(base, timeoutMs) {
14
+ const startedAt = Date.now();
15
+ const controller = new AbortController();
16
+ const timer = setTimeout(() => controller.abort(new ResearchDeadlineError(timeoutMs)), timeoutMs);
17
+ // The MCP is a short-lived stdio process; never hold the loop open for this.
18
+ timer.unref?.();
19
+ let searchRequests = 0;
20
+ let documentRequests = 0;
21
+ let redirects = 0;
22
+ let totalRequests = 0;
23
+ const wrapped = async (input, init) => {
24
+ if (controller.signal.aborted)
25
+ throw new ResearchDeadlineError(timeoutMs);
26
+ const href = typeof input === "string" ? input : input instanceof URL ? input.href : input.url;
27
+ const isSearch = SEARCH_ENDPOINTS.some((endpoint) => href.startsWith(endpoint));
28
+ totalRequests++;
29
+ if (isSearch)
30
+ searchRequests++;
31
+ else
32
+ documentRequests++;
33
+ // Keep the caller's own per-attempt timeout AND add the call deadline, so a
34
+ // single slow hop still fails fast while the whole call stays bounded.
35
+ const signal = init?.signal
36
+ ? AbortSignal.any([init.signal, controller.signal])
37
+ : controller.signal;
38
+ const response = await base(input, { ...init, signal });
39
+ if (response.status >= 300 && response.status < 400) {
40
+ redirects++;
41
+ // A redirect hop is a round trip, not a distinct resource, so undo the
42
+ // classification count. It has to be the SAME counter that was
43
+ // incremented: the search backends use `redirect: "manual"` too and treat
44
+ // a 3xx as an error, so unconditionally decrementing documentRequests
45
+ // drives it negative the first time a discovery endpoint redirects.
46
+ if (isSearch)
47
+ searchRequests--;
48
+ else
49
+ documentRequests--;
50
+ }
51
+ return response;
52
+ };
53
+ return {
54
+ fetch: wrapped,
55
+ signal: controller.signal,
56
+ expired: () => controller.signal.aborted,
57
+ counts: () => ({
58
+ searchRequests,
59
+ documentRequests,
60
+ redirects,
61
+ totalRequests,
62
+ elapsedMs: Date.now() - startedAt,
63
+ }),
64
+ };
65
+ }
66
+ /** Sum per-call ledgers into one review-wide total. */
67
+ export function totalResearchNetwork(counts) {
68
+ return counts.reduce((total, entry) => ({
69
+ searchRequests: total.searchRequests + entry.searchRequests,
70
+ documentRequests: total.documentRequests + entry.documentRequests,
71
+ redirects: total.redirects + entry.redirects,
72
+ totalRequests: total.totalRequests + entry.totalRequests,
73
+ elapsedMs: total.elapsedMs + entry.elapsedMs,
74
+ }), { searchRequests: 0, documentRequests: 0, redirects: 0, totalRequests: 0, elapsedMs: 0 });
75
+ }
@@ -54,7 +54,7 @@ async function loadOkHttpSearchIndex(fetchImplementation) {
54
54
  }
55
55
  });
56
56
  const chunks = documents.flatMap((document) => chunkDocument(document, indexedAt));
57
- return buildSearchIndex(chunks, documents.length, indexedAt);
57
+ return buildSearchIndex(chunks);
58
58
  }
59
59
  function cachedOkHttpSearchIndex(fetchImplementation) {
60
60
  const cached = indexCache.get(fetchImplementation);
@@ -101,7 +101,7 @@ function bestPassage(document, query, indexedAt) {
101
101
  if (chunks.length === 0) {
102
102
  return { passage: document.body.slice(0, 1_400), relevance: 0 };
103
103
  }
104
- const index = buildSearchIndex(chunks, 1, indexedAt);
104
+ const index = buildSearchIndex(chunks);
105
105
  const result = searchDocumentation(index, query, {
106
106
  platform: document.platform,
107
107
  providers: document.provider ? [document.provider] : undefined,
@@ -175,7 +175,9 @@ export async function searchRemoteDocumentation(providerId, query, limit, option
175
175
  // downloading every discovery candidate.
176
176
  const fetched = [];
177
177
  let candidateIndex = 0;
178
- while (fetched.length < limit && candidateIndex < candidates.length) {
178
+ while (fetched.length < limit &&
179
+ candidateIndex < candidates.length &&
180
+ !options.deadline?.aborted) {
179
181
  const outstanding = limit - fetched.length;
180
182
  const batch = candidates.slice(candidateIndex, candidateIndex + outstanding);
181
183
  candidateIndex += batch.length;
@@ -1,5 +1,3 @@
1
- import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
2
- import path from "node:path";
3
1
  import MiniSearch from "minisearch";
4
2
  const miniSearchOptions = {
5
3
  fields: ["title", "passage", "framework", "symbol", "provider"],
@@ -20,37 +18,15 @@ const miniSearchOptions = {
20
18
  "indexedAt",
21
19
  ],
22
20
  };
23
- export function buildSearchIndex(chunks, documentCount, generatedAt = new Date().toISOString()) {
21
+ /**
22
+ * Build a throwaway in-memory index over one fetch's chunks, purely to rank them.
23
+ * Nothing is serialized or persisted: since the offline index was removed, every
24
+ * caller builds this per request and discards it.
25
+ */
26
+ export function buildSearchIndex(chunks) {
24
27
  const miniSearch = new MiniSearch(miniSearchOptions);
25
28
  miniSearch.addAll(chunks);
26
- const providers = [
27
- ...new Set(chunks.map((chunk) => chunk.provider ?? chunk.platform)),
28
- ].sort();
29
- return {
30
- miniSearch,
31
- serialized: {
32
- schemaVersion: 1,
33
- generatedAt,
34
- documentCount,
35
- chunkCount: chunks.length,
36
- providers,
37
- searchIndex: miniSearch.toJSON(),
38
- },
39
- };
40
- }
41
- export async function writeSearchIndex(filePath, index) {
42
- await mkdir(path.dirname(filePath), { recursive: true });
43
- const temporaryPath = `${filePath}.tmp-${process.pid}`;
44
- await writeFile(temporaryPath, `${JSON.stringify(index)}\n`, { mode: 0o644 });
45
- await rename(temporaryPath, filePath);
46
- }
47
- export async function loadSearchIndex(filePath) {
48
- const serialized = JSON.parse(await readFile(filePath, "utf8"));
49
- if (serialized.schemaVersion !== 1 || typeof serialized.searchIndex !== "object") {
50
- throw new Error(`Unsupported or invalid search index at ${filePath}`);
51
- }
52
- const miniSearch = MiniSearch.loadJSON(JSON.stringify(serialized.searchIndex), miniSearchOptions);
53
- return { serialized, miniSearch };
29
+ return { miniSearch };
54
30
  }
55
31
  function searchOptions(combineWith, exact = false) {
56
32
  return {