context-mode 1.0.103 → 1.0.104

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/.claude-plugin/marketplace.json +2 -2
  2. package/.claude-plugin/plugin.json +1 -1
  3. package/.openclaw-plugin/openclaw.plugin.json +1 -1
  4. package/.openclaw-plugin/package.json +1 -1
  5. package/README.md +66 -5
  6. package/bin/statusline.mjs +321 -0
  7. package/build/adapters/antigravity/index.d.ts +6 -0
  8. package/build/adapters/antigravity/index.js +10 -0
  9. package/build/adapters/base.d.ts +23 -0
  10. package/build/adapters/base.js +29 -0
  11. package/build/adapters/codex/index.d.ts +10 -0
  12. package/build/adapters/codex/index.js +22 -4
  13. package/build/adapters/cursor/index.d.ts +7 -0
  14. package/build/adapters/cursor/index.js +11 -0
  15. package/build/adapters/detect.d.ts +12 -1
  16. package/build/adapters/detect.js +69 -7
  17. package/build/adapters/gemini-cli/index.d.ts +8 -1
  18. package/build/adapters/gemini-cli/index.js +19 -7
  19. package/build/adapters/jetbrains-copilot/index.d.ts +7 -0
  20. package/build/adapters/jetbrains-copilot/index.js +12 -0
  21. package/build/adapters/kiro/index.d.ts +8 -0
  22. package/build/adapters/kiro/index.js +12 -0
  23. package/build/adapters/openclaw/index.d.ts +17 -0
  24. package/build/adapters/openclaw/index.js +29 -4
  25. package/build/adapters/opencode/index.d.ts +8 -0
  26. package/build/adapters/opencode/index.js +18 -6
  27. package/build/adapters/qwen-code/index.d.ts +1 -0
  28. package/build/adapters/qwen-code/index.js +3 -0
  29. package/build/adapters/types.d.ts +33 -0
  30. package/build/adapters/vscode-copilot/index.d.ts +6 -0
  31. package/build/adapters/vscode-copilot/index.js +10 -0
  32. package/build/adapters/zed/index.d.ts +1 -0
  33. package/build/adapters/zed/index.js +3 -0
  34. package/build/cli.d.ts +15 -0
  35. package/build/cli.js +62 -16
  36. package/build/concurrency/runPool.d.ts +36 -0
  37. package/build/concurrency/runPool.js +51 -0
  38. package/build/executor.d.ts +11 -1
  39. package/build/executor.js +59 -16
  40. package/build/fetch-cache.d.ts +13 -0
  41. package/build/fetch-cache.js +15 -0
  42. package/build/lifecycle.d.ts +6 -2
  43. package/build/lifecycle.js +29 -2
  44. package/build/opencode-plugin.d.ts +6 -0
  45. package/build/opencode-plugin.js +60 -1
  46. package/build/routing-block.d.ts +8 -0
  47. package/build/routing-block.js +86 -0
  48. package/build/runtime.d.ts +1 -0
  49. package/build/runtime.js +54 -3
  50. package/build/search/auto-memory.d.ts +23 -10
  51. package/build/search/auto-memory.js +64 -26
  52. package/build/search/unified.d.ts +3 -0
  53. package/build/search/unified.js +2 -2
  54. package/build/server.d.ts +42 -0
  55. package/build/server.js +693 -164
  56. package/build/session/analytics.d.ts +49 -1
  57. package/build/session/analytics.js +278 -16
  58. package/build/session/db.d.ts +39 -8
  59. package/build/session/db.js +170 -19
  60. package/build/session/extract.js +124 -2
  61. package/build/tool-naming.d.ts +4 -0
  62. package/build/tool-naming.js +24 -0
  63. package/cli.bundle.mjs +201 -159
  64. package/configs/antigravity/GEMINI.md +11 -0
  65. package/configs/claude-code/CLAUDE.md +11 -0
  66. package/configs/codex/AGENTS.md +11 -0
  67. package/configs/cursor/context-mode.mdc +11 -0
  68. package/configs/gemini-cli/GEMINI.md +11 -0
  69. package/configs/jetbrains-copilot/copilot-instructions.md +3 -0
  70. package/configs/kilo/AGENTS.md +11 -0
  71. package/configs/kiro/KIRO.md +11 -0
  72. package/configs/openclaw/AGENTS.md +11 -0
  73. package/configs/opencode/AGENTS.md +11 -0
  74. package/configs/pi/AGENTS.md +11 -0
  75. package/configs/qwen-code/QWEN.md +11 -0
  76. package/configs/vscode-copilot/copilot-instructions.md +3 -0
  77. package/configs/zed/AGENTS.md +11 -0
  78. package/hooks/auto-injection.mjs +36 -10
  79. package/hooks/cache-heal-utils.mjs +231 -0
  80. package/hooks/codex/sessionstart.mjs +7 -4
  81. package/hooks/core/routing.mjs +5 -0
  82. package/hooks/cursor/sessionstart.mjs +7 -4
  83. package/hooks/formatters/claude-code.mjs +20 -0
  84. package/hooks/gemini-cli/sessionstart.mjs +7 -2
  85. package/hooks/jetbrains-copilot/sessionstart.mjs +7 -2
  86. package/hooks/normalize-hooks.mjs +184 -0
  87. package/hooks/session-db.bundle.mjs +33 -14
  88. package/hooks/session-extract.bundle.mjs +2 -2
  89. package/hooks/session-helpers.mjs +68 -20
  90. package/hooks/session-loaders.mjs +8 -2
  91. package/hooks/sessionstart.mjs +8 -2
  92. package/hooks/vscode-copilot/sessionstart.mjs +7 -2
  93. package/openclaw.plugin.json +1 -1
  94. package/package.json +2 -1
  95. package/server.bundle.mjs +164 -125
  96. package/skills/ctx-insight/SKILL.md +1 -1
  97. package/start.mjs +63 -3
package/build/server.js CHANGED
@@ -3,15 +3,17 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
3
3
  import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
4
4
  import { createRequire } from "node:module";
5
5
  import { createHash } from "node:crypto";
6
- import { existsSync, unlinkSync, readdirSync, readFileSync, writeFileSync, rmSync, mkdirSync, cpSync, statSync, symlinkSync, lstatSync } from "node:fs";
6
+ import { existsSync, unlinkSync, readdirSync, readFileSync, writeFileSync, renameSync, rmSync, mkdirSync, cpSync, statSync, symlinkSync, lstatSync } from "node:fs";
7
7
  import { execSync } from "node:child_process";
8
- import { join, dirname, resolve, sep } from "node:path";
8
+ import { join, dirname, resolve, sep, isAbsolute } from "node:path";
9
9
  import { fileURLToPath } from "node:url";
10
- import { homedir, tmpdir } from "node:os";
10
+ import { homedir, tmpdir, cpus } from "node:os";
11
11
  import { request as httpsRequest } from "node:https";
12
12
  import { z } from "zod";
13
13
  import { PolyglotExecutor } from "./executor.js";
14
+ import { runPool } from "./concurrency/runPool.js";
14
15
  import { ContentStore, cleanupStaleDBs, cleanupStaleContentDBs } from "./store.js";
16
+ import { composeFetchCacheKey } from "./fetch-cache.js";
15
17
  import { readBashPolicies, evaluateCommandDenyOnly, extractShellCommands, readToolDenyPatterns, evaluateFilePath, } from "./security.js";
16
18
  import { detectRuntimes, getRuntimeSummary, getAvailableLanguages, hasBunRuntime, } from "./runtime.js";
17
19
  import { classifyNonZeroExit } from "./exit-classify.js";
@@ -19,8 +21,9 @@ import { startLifecycleGuard } from "./lifecycle.js";
19
21
  import { getWorktreeSuffix, SessionDB } from "./session/db.js";
20
22
  import { searchAllSources } from "./search/unified.js";
21
23
  import { buildNodeCommand } from "./adapters/types.js";
24
+ import { detectPlatform, getSessionDirSegments } from "./adapters/detect.js";
22
25
  import { loadDatabase } from "./db-base.js";
23
- import { AnalyticsEngine, formatReport } from "./session/analytics.js";
26
+ import { AnalyticsEngine, formatReport, getLifetimeStats, OPUS_INPUT_PRICE_PER_TOKEN } from "./session/analytics.js";
24
27
  const __pkg_dir = dirname(fileURLToPath(import.meta.url));
25
28
  const VERSION = (() => {
26
29
  for (const rel of ["../package.json", "./package.json"]) {
@@ -57,7 +60,7 @@ server.server.setRequestHandler(ListResourcesRequestSchema, async () => ({ resou
57
60
  server.server.setRequestHandler(ListResourceTemplatesRequestSchema, async () => ({ resourceTemplates: [] }));
58
61
  const executor = new PolyglotExecutor({
59
62
  runtimes,
60
- projectRoot: process.env.CLAUDE_PROJECT_DIR,
63
+ projectRoot: () => getProjectDir(),
61
64
  });
62
65
  // ─────────────────────────────────────────────────────────
63
66
  // FS read tracking preload for ctx_batch_execute
@@ -109,6 +112,20 @@ let _insightChild = null;
109
112
  function getSessionDir() {
110
113
  if (_detectedAdapter)
111
114
  return _detectedAdapter.getSessionDir();
115
+ // Pre-detection path (race window before MCP `initialize` completes):
116
+ // call detectPlatform() (sync, env-var-based) and look up segments via
117
+ // getSessionDirSegments() (sync map, no adapter instantiation). This keeps
118
+ // non-Claude platforms from spilling sessions into ~/.claude/.
119
+ try {
120
+ const signal = detectPlatform();
121
+ const segments = getSessionDirSegments(signal.platform);
122
+ if (segments) {
123
+ const dir = join(homedir(), ...segments, "context-mode", "sessions");
124
+ mkdirSync(dir, { recursive: true });
125
+ return dir;
126
+ }
127
+ }
128
+ catch { /* fall through to .claude fallback */ }
112
129
  const dir = join(homedir(), ".claude", "context-mode", "sessions");
113
130
  mkdirSync(dir, { recursive: true });
114
131
  return dir;
@@ -130,9 +147,18 @@ function getProjectDir() {
130
147
  || process.env.VSCODE_CWD
131
148
  || process.env.OPENCODE_PROJECT_DIR
132
149
  || process.env.PI_PROJECT_DIR
150
+ || process.env.IDEA_INITIAL_DIRECTORY
133
151
  || process.env.CONTEXT_MODE_PROJECT_DIR
134
152
  || process.cwd();
135
153
  }
154
+ /**
155
+ * Resolve a possibly-relative path against the project directory (full env cascade),
156
+ * not the MCP server's process.cwd(). MCP server is spawned by the host and its cwd
157
+ * is unrelated to where the user is working.
158
+ */
159
+ function resolveProjectPath(filePath) {
160
+ return isAbsolute(filePath) ? filePath : resolve(getProjectDir(), filePath);
161
+ }
136
162
  /**
137
163
  * Consistent project dir hashing across all DB paths.
138
164
  * Normalizes Windows backslashes before hashing so the same project
@@ -322,10 +348,120 @@ function trackResponse(toolName, response) {
322
348
  sessionStats.calls[toolName] = (sessionStats.calls[toolName] || 0) + 1;
323
349
  sessionStats.bytesReturned[toolName] =
324
350
  (sessionStats.bytesReturned[toolName] || 0) + bytes;
351
+ // Persist a sidecar JSON snapshot for the statusline — read at ~3-5 Hz by
352
+ // bin/statusline.mjs (and any external dashboard) so they don't have to
353
+ // open the SQLite database. Throttled inside persistStats() (500ms) so
354
+ // it's safe to call on every response. The b392c2f concurrency refactor
355
+ // dropped the SessionDB tool-call counter (`persistToolCallCounter`); we
356
+ // keep persistStats here because the statusline depends on it.
357
+ persistStats();
325
358
  return response;
326
359
  }
327
360
  function trackIndexed(bytes) {
328
361
  sessionStats.bytesIndexed += bytes;
362
+ persistStats();
363
+ }
364
+ // ─────────────────────────────────────────────────────────
365
+ // Stats persistence — written after every tool call so
366
+ // external readers (status line scripts, dashboards, hooks)
367
+ // can see real-time savings without spawning an MCP client.
368
+ // ─────────────────────────────────────────────────────────
369
+ const STATS_PERSIST_THROTTLE_MS = 500;
370
+ // Schema version for the persisted stats payload (~/.claude/context-mode/sessions/stats-*.json).
371
+ // Bump when a field is added/renamed/removed. Statusline reads `schemaVersion ?? 0` and warns when
372
+ // it sees a future schema, so legacy bundles degrade gracefully on upgrade rather than silently
373
+ // rendering missing fields (PR #401 architect review P1.3).
374
+ // v2: added tokens_saved_lifetime + dollars_saved_lifetime.
375
+ const STATS_SCHEMA_VERSION = 2;
376
+ // OPUS_INPUT_PRICE_PER_TOKEN intentionally NOT defined here — single source in
377
+ // src/session/analytics.ts re-exported above. (P1.1 — pricing constant dedup,
378
+ // PR #401 architect + ops 2-vote convergence.)
379
+ const LIFETIME_REFRESH_MS = 30_000;
380
+ // Matches the conversion factor in src/session/analytics.ts renderBottomLine:
381
+ // ~1KB per session event ÷ 4 bytes/token = 256 tokens/event.
382
+ const TOKENS_PER_EVENT = 256;
383
+ let _lastStatsPersist = 0;
384
+ let _lifetimeCache;
385
+ /**
386
+ * Resolve the per-session stats file path.
387
+ *
388
+ * The session id mirrors the Claude Code adapter contract
389
+ * (`pid-<parent pid>`), so a status line script can derive
390
+ * the same id from `$PPID` without coupling to MCP.
391
+ */
392
+ function getStatsFilePath() {
393
+ const sessionId = process.env.CLAUDE_SESSION_ID || `pid-${process.ppid}`;
394
+ return join(getSessionDir(), `stats-${sessionId}.json`);
395
+ }
396
+ function persistStats() {
397
+ const now = Date.now();
398
+ if (now - _lastStatsPersist < STATS_PERSIST_THROTTLE_MS)
399
+ return;
400
+ _lastStatsPersist = now;
401
+ try {
402
+ const totalReturned = Object.values(sessionStats.bytesReturned).reduce((a, b) => a + b, 0);
403
+ const totalCalls = Object.values(sessionStats.calls).reduce((a, b) => a + b, 0);
404
+ const keptOut = sessionStats.bytesIndexed +
405
+ sessionStats.bytesSandboxed +
406
+ sessionStats.cacheBytesSaved;
407
+ const totalProcessed = keptOut + totalReturned;
408
+ const reductionPct = totalProcessed > 0
409
+ ? Math.round((1 - totalReturned / totalProcessed) * 100)
410
+ : 0;
411
+ const tokensSaved = Math.round(keptOut / 4);
412
+ // Lifetime savings — cached separately because getLifetimeStats() scans
413
+ // disk (per-project SessionDBs + auto-memory dirs) and is too expensive
414
+ // for the 500ms persist throttle. Refresh every 30s; the statusline
415
+ // doesn't need second-by-second lifetime accuracy.
416
+ let lifetimeTokens = _lifetimeCache?.tokens ?? 0;
417
+ if (!_lifetimeCache || now - _lifetimeCache.computedAt > LIFETIME_REFRESH_MS) {
418
+ try {
419
+ const life = getLifetimeStats({ sessionsDir: getSessionDir() });
420
+ lifetimeTokens = (life?.totalEvents ?? 0) * TOKENS_PER_EVENT;
421
+ _lifetimeCache = { tokens: lifetimeTokens, computedAt: now };
422
+ }
423
+ catch {
424
+ // best-effort — keep stale cache or 0
425
+ }
426
+ }
427
+ const payload = {
428
+ schemaVersion: STATS_SCHEMA_VERSION,
429
+ version: VERSION,
430
+ updated_at: now,
431
+ session_start: sessionStats.sessionStart,
432
+ uptime_ms: now - sessionStats.sessionStart,
433
+ total_calls: totalCalls,
434
+ bytes_returned: totalReturned,
435
+ bytes_indexed: sessionStats.bytesIndexed,
436
+ bytes_sandboxed: sessionStats.bytesSandboxed,
437
+ cache_hits: sessionStats.cacheHits,
438
+ cache_bytes_saved: sessionStats.cacheBytesSaved,
439
+ kept_out: keptOut,
440
+ total_processed: totalProcessed,
441
+ reduction_pct: reductionPct,
442
+ tokens_saved: tokensSaved,
443
+ // statusline-facing $ values — pre-computed at Opus input rate so the
444
+ // statusline doesn't have to know pricing. Lets us evolve pricing in
445
+ // one place without touching consumers.
446
+ dollars_saved_session: +(tokensSaved * OPUS_INPUT_PRICE_PER_TOKEN).toFixed(2),
447
+ tokens_saved_lifetime: lifetimeTokens,
448
+ dollars_saved_lifetime: +(lifetimeTokens * OPUS_INPUT_PRICE_PER_TOKEN).toFixed(2),
449
+ by_tool: Object.fromEntries(Object.keys({ ...sessionStats.calls, ...sessionStats.bytesReturned }).map((t) => [
450
+ t,
451
+ {
452
+ calls: sessionStats.calls[t] || 0,
453
+ bytes: sessionStats.bytesReturned[t] || 0,
454
+ },
455
+ ])),
456
+ };
457
+ const filePath = getStatsFilePath();
458
+ const tmpPath = `${filePath}.tmp`;
459
+ writeFileSync(tmpPath, JSON.stringify(payload));
460
+ renameSync(tmpPath, filePath);
461
+ }
462
+ catch {
463
+ // best-effort — never break tool calls because of stats persistence
464
+ }
329
465
  }
330
466
  // ==============================================================================
331
467
  // Security: server-side deny firewall
@@ -387,7 +523,7 @@ function checkNonShellDenyPolicy(code, language, toolName) {
387
523
  */
388
524
  function checkFilePathDenyPolicy(filePath, toolName) {
389
525
  try {
390
- const projectDir = process.env.CLAUDE_PROJECT_DIR ?? process.cwd();
526
+ const projectDir = getProjectDir();
391
527
  const denyGlobs = readToolDenyPatterns("Read", projectDir);
392
528
  const result = evaluateFilePath(filePath, denyGlobs, process.platform === "win32", projectDir);
393
529
  if (result.denied) {
@@ -539,6 +675,94 @@ export function formatBatchQueryResults(store, queries, source, maxOutput = 80 *
539
675
  sections.push(`\n> **Tip:** Results are scoped to this batch only. To search across all indexed sources, use \`ctx_search(queries: [...])\`.`);
540
676
  return sections;
541
677
  }
678
+ function formatCommandOutput(label, raw, onFsBytes) {
679
+ let output = raw || "(no output)";
680
+ const fsMatches = output.matchAll(/__CM_FS__:(\d+)/g);
681
+ let cmdFsBytes = 0;
682
+ for (const m of fsMatches)
683
+ cmdFsBytes += parseInt(m[1]);
684
+ if (cmdFsBytes > 0) {
685
+ onFsBytes?.(cmdFsBytes);
686
+ output = output.replace(/__CM_FS__:\d+\n?/g, "");
687
+ }
688
+ return `# ${label}\n\n${output}\n`;
689
+ }
690
+ /**
691
+ * Execute batch commands. concurrency=1 preserves the legacy serial path
692
+ * (shared timeout budget + cascading skip-on-timeout). concurrency>1 runs
693
+ * commands concurrently with at most N in flight; each command receives the
694
+ * full timeout, output is collated by input index, and per-command timeouts
695
+ * record `(timed out)` blocks without skipping siblings.
696
+ */
697
+ export async function runBatchCommands(commands, opts, executor) {
698
+ const { timeout, concurrency, nodeOptsPrefix, onFsBytes } = opts;
699
+ if (concurrency <= 1) {
700
+ // Serial path — shared timeout budget, cascading skip on timeout.
701
+ const outputs = [];
702
+ const startTime = Date.now();
703
+ let timedOut = false;
704
+ for (let i = 0; i < commands.length; i++) {
705
+ const cmd = commands[i];
706
+ const elapsed = Date.now() - startTime;
707
+ const remaining = timeout - elapsed;
708
+ if (remaining <= 0) {
709
+ outputs.push(`# ${cmd.label}\n\n(skipped — batch timeout exceeded)\n`);
710
+ timedOut = true;
711
+ continue;
712
+ }
713
+ const result = await executor.execute({
714
+ language: "shell",
715
+ code: `${nodeOptsPrefix}${cmd.command} 2>&1`,
716
+ timeout: remaining,
717
+ });
718
+ outputs.push(formatCommandOutput(cmd.label, result.stdout, onFsBytes));
719
+ if (result.timedOut) {
720
+ timedOut = true;
721
+ for (let j = i + 1; j < commands.length; j++) {
722
+ outputs.push(`# ${commands[j].label}\n\n(skipped — batch timeout exceeded)\n`);
723
+ }
724
+ break;
725
+ }
726
+ }
727
+ return { outputs, timedOut };
728
+ }
729
+ // Parallel path — delegated to the shared runPool primitive.
730
+ // Each job returns { output, timedOut }; runPool handles in-flight cap,
731
+ // throw isolation (Promise.allSettled semantics), and order preservation.
732
+ const jobs = commands.map((cmd) => ({
733
+ run: async () => {
734
+ const result = await executor.execute({
735
+ language: "shell",
736
+ code: `${nodeOptsPrefix}${cmd.command} 2>&1`,
737
+ timeout,
738
+ });
739
+ // Always route partial stdout through formatCommandOutput so __CM_FS__
740
+ // markers are stripped + counted, even when the command timed out.
741
+ const formatted = formatCommandOutput(cmd.label, result.stdout, onFsBytes);
742
+ const output = result.timedOut
743
+ ? formatted.replace(/\n$/, "") + `\n(timed out after ${timeout}ms)\n`
744
+ : formatted;
745
+ return { output, timedOut: !!result.timedOut };
746
+ },
747
+ }));
748
+ const { settled } = await runPool(jobs, { concurrency });
749
+ const outputs = new Array(commands.length);
750
+ let timedOut = false;
751
+ for (let i = 0; i < settled.length; i++) {
752
+ const r = settled[i];
753
+ if (r.status === "fulfilled") {
754
+ outputs[i] = r.value.output;
755
+ if (r.value.timedOut)
756
+ timedOut = true;
757
+ }
758
+ else {
759
+ // Isolated executor throw (spawn EAGAIN, ENOMEM, EMFILE, …) — siblings keep running.
760
+ const message = r.reason instanceof Error ? r.reason.message : String(r.reason);
761
+ outputs[i] = `# ${commands[i].label}\n\n(executor error: ${message})\n`;
762
+ }
763
+ }
764
+ return { outputs, timedOut };
765
+ }
542
766
  // ─────────────────────────────────────────────────────────
543
767
  // Tool: execute
544
768
  // ─────────────────────────────────────────────────────────
@@ -1009,18 +1233,19 @@ server.registerTool("ctx_index", {
1009
1233
  });
1010
1234
  }
1011
1235
  try {
1236
+ const resolvedPath = path ? resolveProjectPath(path) : undefined;
1012
1237
  // Track the raw bytes being indexed (content or file)
1013
1238
  if (content)
1014
1239
  trackIndexed(Buffer.byteLength(content));
1015
- else if (path) {
1240
+ else if (resolvedPath) {
1016
1241
  try {
1017
1242
  const fs = await import("fs");
1018
- trackIndexed(fs.readFileSync(path).byteLength);
1243
+ trackIndexed(fs.readFileSync(resolvedPath).byteLength);
1019
1244
  }
1020
1245
  catch { /* ignore — file read errors handled by store */ }
1021
1246
  }
1022
1247
  const store = getStore();
1023
- const result = store.index({ content, path, source });
1248
+ const result = store.index({ content, path: resolvedPath, source: source ?? resolvedPath });
1024
1249
  return trackResponse("ctx_index", {
1025
1250
  content: [
1026
1251
  {
@@ -1178,14 +1403,14 @@ server.registerTool("ctx_search", {
1178
1403
  if (sort === "timeline") {
1179
1404
  try {
1180
1405
  const sessionsDir = getSessionDir();
1181
- const dbFile = join(sessionsDir, `${hashProjectDir()}.db`);
1406
+ const dbFile = join(sessionsDir, `${hashProjectDir()}${getWorktreeSuffix()}.db`);
1182
1407
  if (existsSync(dbFile)) {
1183
1408
  timelineDB = new SessionDB({ dbPath: dbFile });
1184
1409
  }
1185
1410
  }
1186
1411
  catch { /* SessionDB unavailable — search ContentStore + auto-memory only */ }
1187
1412
  }
1188
- const configDir = process.env.CLAUDE_CONFIG_DIR || join(homedir(), ".claude");
1413
+ const configDir = _detectedAdapter?.getConfigDir() ?? (process.env.CLAUDE_CONFIG_DIR || join(homedir(), ".claude"));
1189
1414
  try {
1190
1415
  for (const q of queryList) {
1191
1416
  if (totalSize > MAX_TOTAL) {
@@ -1204,6 +1429,7 @@ server.registerTool("ctx_search", {
1204
1429
  sessionDB: timelineDB,
1205
1430
  projectDir: getProjectDir(),
1206
1431
  configDir,
1432
+ adapter: _detectedAdapter ?? undefined,
1207
1433
  });
1208
1434
  }
1209
1435
  else {
@@ -1342,54 +1568,178 @@ async function main() {
1342
1568
  main();
1343
1569
  `;
1344
1570
  }
1345
- server.registerTool("ctx_fetch_and_index", {
1346
- title: "Fetch & Index URL",
1347
- description: "Fetches URL content, converts HTML to markdown, indexes into searchable knowledge base, " +
1348
- "and returns a ~3KB preview. Full content stays in sandbox use ctx_search() for deeper lookups.\n\n" +
1349
- "Better than WebFetch: preview is immediate, full content is searchable, raw HTML never enters context.\n\n" +
1350
- "Content-type aware: HTML is converted to markdown, JSON is chunked by key paths, plain text is indexed directly.\n\n" +
1351
- "When reporting results terse like caveman. Technical substance exact. Only fluff die. Pattern: [thing] [action] [reason]. [next step].",
1352
- inputSchema: z.object({
1353
- url: z.string().describe("The URL to fetch and index"),
1354
- source: z
1355
- .string()
1356
- .optional()
1357
- .describe("Label for the indexed content (e.g., 'React useEffect docs', 'Supabase Auth API')"),
1358
- force: z
1359
- .boolean()
1360
- .optional()
1361
- .describe("Skip cache and re-fetch even if content was recently indexed"),
1362
- }),
1363
- }, async ({ url, source, force }) => {
1364
- // TTL cache: if source was indexed within 24h, return cached hint
1571
+ // ─────────────────────────────────────────────────────────
1572
+ // fetch_and_index helpers split into parallel-safe fetch and serial-only index
1573
+ // ─────────────────────────────────────────────────────────
1574
+ const FETCH_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
1575
+ const FETCH_PREVIEW_LIMIT = 3072;
1576
+ /**
1577
+ * Pure fetch step TTL cache check + subprocess fetch. SAFE TO RUN IN PARALLEL.
1578
+ * Performs zero SQLite writes (only reads source meta). Caller must funnel
1579
+ * fetched results through `indexFetched` serially to avoid FTS5 WAL contention.
1580
+ */
1581
+ /**
1582
+ * SSRF guard for ctx_fetch_and_index: validate URL scheme + resolve target IP +
1583
+ * block link-local / IMDS / multicast / reserved IP ranges. Returns null if
1584
+ * safe; returns a FetchOneResult fetch_error if blocked.
1585
+ *
1586
+ * Policy (PR #401 ops review, developer-friendly default):
1587
+ *
1588
+ * **HARD BLOCK** (no legitimate dev workflow):
1589
+ * - file://, gopher://, javascript:, data: schemes (only http: and https:)
1590
+ * - 169.254.0.0/16 link-local (INCLUDES 169.254.169.254 = AWS/GCP/Azure IMDS
1591
+ * cloud credential endpoint — high-value target for indirect prompt injection)
1592
+ * - IPv6 link-local fe80::/10
1593
+ * - Multicast (224+ IPv4, ff00::/8 IPv6) and reserved (0.0.0.0/8) ranges
1594
+ *
1595
+ * **ALLOW by default** (legitimate developer use cases dominate):
1596
+ * - localhost, 127.x.x.x, ::1 (local dev servers — Next.js, Vite, Postgres, …)
1597
+ * - 10.x, 172.16-31.x, 192.168.x RFC1918 private (developer's internal network)
1598
+ *
1599
+ * **STRICT MODE** opt-in via env var: `CTX_FETCH_STRICT=1`
1600
+ * - Blocks loopback + RFC1918 too
1601
+ * - For hosted/CI environments where the runtime isn't the user's own machine
1602
+ *
1603
+ * DNS resolution is performed against the resolved IP (not just URL parse) so a
1604
+ * hostname like `evil.com` pointing to 169.254.169.254 is rejected — defends
1605
+ * against attacker-controlled DNS records and DNS rebinding.
1606
+ */
1607
+ async function ssrfGuard(rawUrl) {
1608
+ let parsed;
1609
+ try {
1610
+ parsed = new URL(rawUrl);
1611
+ }
1612
+ catch {
1613
+ return { kind: "fetch_error", url: rawUrl, error: "invalid URL", reason: "exit" };
1614
+ }
1615
+ // 1. Scheme allowlist — http and https only
1616
+ if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
1617
+ return {
1618
+ kind: "fetch_error",
1619
+ url: rawUrl,
1620
+ error: `URL scheme "${parsed.protocol}" not allowed (only http: and https:)`,
1621
+ reason: "exit",
1622
+ };
1623
+ }
1624
+ const strict = process.env.CTX_FETCH_STRICT === "1";
1625
+ // 2. DNS resolve + check IP ranges (hard-block + optional strict-mode block)
1626
+ try {
1627
+ const { lookup } = await import("node:dns/promises");
1628
+ const records = await lookup(parsed.hostname, { all: true, verbatim: true });
1629
+ for (const rec of records) {
1630
+ const verdict = classifyIp(rec.address);
1631
+ if (verdict === "block") {
1632
+ return {
1633
+ kind: "fetch_error",
1634
+ url: rawUrl,
1635
+ error: `URL "${parsed.hostname}" resolves to ${rec.address} — blocked (link-local / IMDS / multicast / reserved)`,
1636
+ reason: "exit",
1637
+ };
1638
+ }
1639
+ if (verdict === "private" && strict) {
1640
+ return {
1641
+ kind: "fetch_error",
1642
+ url: rawUrl,
1643
+ error: `URL "${parsed.hostname}" resolves to private IP ${rec.address} — blocked under CTX_FETCH_STRICT=1`,
1644
+ reason: "exit",
1645
+ };
1646
+ }
1647
+ }
1648
+ }
1649
+ catch (err) {
1650
+ return {
1651
+ kind: "fetch_error",
1652
+ url: rawUrl,
1653
+ error: `DNS lookup failed for "${parsed.hostname}": ${err instanceof Error ? err.message : String(err)}`,
1654
+ reason: "exit",
1655
+ };
1656
+ }
1657
+ return null; // safe to fetch
1658
+ }
1659
+ /**
1660
+ * Classify an IP address.
1661
+ * - "block": always blocked (link-local/IMDS/multicast/reserved/malformed)
1662
+ * - "private": loopback or RFC1918 — allowed by default, blocked in strict mode
1663
+ * - "public": safe to fetch
1664
+ *
1665
+ * Exported (via the function name) so SSRF tests can exercise the matcher directly.
1666
+ */
1667
+ export function classifyIp(ip) {
1668
+ const lower = ip.toLowerCase();
1669
+ // IPv6 takes priority — check for `:` first so IPv4-mapped addresses
1670
+ // (`::ffff:127.0.0.1`) don't get incorrectly routed through the IPv4 parser.
1671
+ if (lower.includes(":")) {
1672
+ // IPv4-mapped IPv6 (`::ffff:127.0.0.1`) — recurse through IPv4 classifier
1673
+ const v4MappedMatch = lower.match(/^::ffff:([\d.]+)$/);
1674
+ if (v4MappedMatch)
1675
+ return classifyIp(v4MappedMatch[1]);
1676
+ // Hard-block
1677
+ if (lower === "::")
1678
+ return "block"; // unspecified
1679
+ if (lower.startsWith("fe8") || lower.startsWith("fe9") ||
1680
+ lower.startsWith("fea") || lower.startsWith("feb"))
1681
+ return "block"; // fe80::/10 link-local
1682
+ if (lower.startsWith("ff"))
1683
+ return "block"; // ff00::/8 multicast
1684
+ // Private (loopback + ULA)
1685
+ if (lower === "::1")
1686
+ return "private";
1687
+ if (lower.startsWith("fc") || lower.startsWith("fd"))
1688
+ return "private"; // fc00::/7 ULA
1689
+ return "public";
1690
+ }
1691
+ // IPv4 (or non-IP string — malformed = block)
1692
+ if (!ip.includes("."))
1693
+ return "block"; // not an IP at all
1694
+ const parts = ip.split(".").map((p) => parseInt(p, 10));
1695
+ if (parts.length !== 4 || parts.some((p) => isNaN(p) || p < 0 || p > 255))
1696
+ return "block";
1697
+ const [a, b] = parts;
1698
+ // Hard-block (no legitimate use)
1699
+ if (a === 169 && b === 254)
1700
+ return "block"; // link-local incl. 169.254.169.254 (IMDS)
1701
+ if (a === 0)
1702
+ return "block"; // 0.0.0.0/8 (current network)
1703
+ if (a >= 224)
1704
+ return "block"; // 224.0.0.0+ multicast/reserved
1705
+ // Private (loopback + RFC1918) — allow by default
1706
+ if (a === 127)
1707
+ return "private"; // 127.0.0.0/8 loopback
1708
+ if (a === 10)
1709
+ return "private"; // 10.0.0.0/8
1710
+ if (a === 172 && b >= 16 && b <= 31)
1711
+ return "private"; // 172.16.0.0/12
1712
+ if (a === 192 && b === 168)
1713
+ return "private"; // 192.168.0.0/16
1714
+ return "public";
1715
+ }
1716
+ async function fetchOneUrl(url, source, force) {
1717
+ // SSRF guard — reject file://, javascript:, loopback, RFC1918, IMDS, link-local
1718
+ // BEFORE any cache lookup or subprocess spawn. Even cached entries shouldn't
1719
+ // serve a previously-poisoned source label.
1720
+ const ssrfBlock = await ssrfGuard(url);
1721
+ if (ssrfBlock)
1722
+ return ssrfBlock;
1365
1723
  if (!force) {
1366
1724
  const store = getStore();
1367
- const label = source ?? url;
1368
- const meta = store.getSourceMeta(label);
1725
+ // Cache key composes (source, url) so two distinct URLs sharing the same
1726
+ // `source` label do not collide — they each get their own cache slot
1727
+ // (commit 1f1243e regression test enforced).
1728
+ const cacheKey = composeFetchCacheKey(source, url);
1729
+ const meta = store.getSourceMeta(cacheKey);
1369
1730
  if (meta) {
1370
1731
  const indexedAt = new Date(meta.indexedAt + "Z"); // SQLite datetime is UTC without Z
1371
1732
  const ageMs = Date.now() - indexedAt.getTime();
1372
- const TTL_MS = 24 * 60 * 60 * 1000; // 24 hours
1373
- if (ageMs < TTL_MS) {
1733
+ if (ageMs < FETCH_TTL_MS) {
1374
1734
  const ageHours = Math.floor(ageMs / (60 * 60 * 1000));
1375
1735
  const ageMin = Math.floor(ageMs / (60 * 1000));
1376
1736
  const ageStr = ageHours > 0 ? `${ageHours}h ago` : ageMin > 0 ? `${ageMin}m ago` : "just now";
1377
- // Track cache savings estimate ~1.6KB per chunk (average indexed content size)
1378
- const estimatedBytes = meta.chunkCount * 1600;
1379
- sessionStats.cacheHits++;
1380
- sessionStats.cacheBytesSaved += estimatedBytes;
1381
- return trackResponse("ctx_fetch_and_index", {
1382
- content: [{
1383
- type: "text",
1384
- text: `Cached: **${meta.label}** — ${meta.chunkCount} sections, indexed ${ageStr} (fresh, TTL: 24h).\nTo refresh: call ctx_fetch_and_index again with \`force: true\`.\n\nYou MUST call ctx_search() to answer questions about this content — this cached response contains no content.\nUse: ctx_search(queries: [...], source: "${meta.label}")`,
1385
- }],
1386
- });
1737
+ const estimatedBytes = meta.chunkCount * 1600; // ~1.6KB/chunk avg
1738
+ return { kind: "cached", label: meta.label, chunkCount: meta.chunkCount, estimatedBytes, ageStr };
1387
1739
  }
1388
- // Stale (>24h) — fall through to re-fetch silently
1740
+ // Stale — fall through to re-fetch silently
1389
1741
  }
1390
1742
  }
1391
- // Generate a unique temp file path for the subprocess to write fetched content.
1392
- // This bypasses the executor's 100KB stdout truncation — content goes file→handler directly.
1393
1743
  const outputPath = join(tmpdir(), `ctx-fetch-${Date.now()}-${Math.random().toString(36).slice(2)}.dat`);
1394
1744
  try {
1395
1745
  const fetchCode = buildFetchCode(url, outputPath);
@@ -1399,93 +1749,258 @@ server.registerTool("ctx_fetch_and_index", {
1399
1749
  timeout: 30_000,
1400
1750
  });
1401
1751
  if (result.exitCode !== 0) {
1402
- return trackResponse("ctx_fetch_and_index", {
1403
- content: [
1404
- {
1405
- type: "text",
1406
- text: `Failed to fetch ${url}: ${result.stderr || result.stdout}`,
1407
- },
1408
- ],
1409
- isError: true,
1410
- });
1752
+ return { kind: "fetch_error", url, error: result.stderr || result.stdout || "unknown error", reason: "exit" };
1411
1753
  }
1412
- // Parse content-type marker from stdout (content is in the temp file)
1413
- const store = getStore();
1414
1754
  const header = (result.stdout || "").trim();
1415
- // Read full content from temp file
1416
1755
  let markdown;
1417
1756
  try {
1418
1757
  markdown = readFileSync(outputPath, "utf-8").trim();
1419
1758
  }
1420
1759
  catch {
1421
- return trackResponse("ctx_fetch_and_index", {
1422
- content: [
1423
- {
1424
- type: "text",
1425
- text: `Fetched ${url} but could not read subprocess output`,
1426
- },
1427
- ],
1428
- isError: true,
1429
- });
1760
+ return { kind: "fetch_error", url, error: "could not read subprocess output", reason: "read" };
1430
1761
  }
1431
1762
  if (markdown.length === 0) {
1763
+ return { kind: "fetch_error", url, error: "empty content", reason: "empty" };
1764
+ }
1765
+ return { kind: "fetched", url, source, markdown, header };
1766
+ }
1767
+ catch (err) {
1768
+ return {
1769
+ kind: "fetch_error",
1770
+ url,
1771
+ error: err instanceof Error ? err.message : String(err),
1772
+ reason: "throw",
1773
+ };
1774
+ }
1775
+ finally {
1776
+ try {
1777
+ rmSync(outputPath);
1778
+ }
1779
+ catch { /* already gone */ }
1780
+ }
1781
+ }
1782
+ /**
1783
+ * Serial-only indexing step — single FTS5 write per call. Caller loops over
1784
+ * fetched results and calls this one-at-a-time to avoid SQLite WAL contention
1785
+ * (PRD finding E).
1786
+ */
1787
+ function indexFetched(f) {
1788
+ const store = getStore();
1789
+ // Storage label composed via composeFetchCacheKey so two URLs sharing a
1790
+ // `source` label do not overwrite each other (commit 1f1243e). ctx_search()
1791
+ // still finds both via LIKE-mode source filter on the `source` substring.
1792
+ const storageLabel = composeFetchCacheKey(f.source, f.url);
1793
+ let indexed;
1794
+ if (f.header === "__CM_CT__:json") {
1795
+ indexed = store.indexJSON(f.markdown, storageLabel);
1796
+ }
1797
+ else if (f.header === "__CM_CT__:text") {
1798
+ indexed = store.indexPlainText(f.markdown, storageLabel);
1799
+ }
1800
+ else {
1801
+ indexed = store.index({ content: f.markdown, source: storageLabel });
1802
+ }
1803
+ // Track AFTER the FTS5 write succeeds — failed indexes shouldn't inflate the counter.
1804
+ trackIndexed(Buffer.byteLength(f.markdown));
1805
+ const preview = f.markdown.length > FETCH_PREVIEW_LIMIT
1806
+ ? f.markdown.slice(0, FETCH_PREVIEW_LIMIT) + "\n\n…[truncated — use ctx_search() for full content]"
1807
+ : f.markdown;
1808
+ return {
1809
+ label: indexed.label,
1810
+ totalChunks: indexed.totalChunks,
1811
+ totalBytes: Buffer.byteLength(f.markdown),
1812
+ preview,
1813
+ };
1814
+ }
1815
+ server.registerTool("ctx_fetch_and_index", {
1816
+ title: "Fetch & Index URL(s)",
1817
+ description: "Fetches URL content, converts HTML to markdown, indexes into searchable knowledge base, " +
1818
+ "and returns a ~3KB preview. Full content stays in sandbox — use ctx_search() for deeper lookups.\n\n" +
1819
+ "Better than WebFetch: preview is immediate, full content is searchable, raw HTML never enters context.\n\n" +
1820
+ "Content-type aware: HTML is converted to markdown, JSON is chunked by key paths, plain text is indexed directly.\n\n" +
1821
+ "PARALLELIZE I/O: For multi-URL research (library evaluation, migration scans, doc comparisons), pass `requests: [{url, source}, ...]` with `concurrency: 4-8` — speeds up by 3-5x on real workloads.\n" +
1822
+ " ✅ Use concurrency: 4-8 for: library docs sweep, multi-changelog scan, competitive pricing pages, multi-region docs, GitHub raw file pulls.\n" +
1823
+ " ❌ Single URL → use the legacy {url, source} shape (concurrency irrelevant).\n" +
1824
+ " Example: requests: [{url: 'https://react.dev/...', source: 'react'}, {url: 'https://vuejs.org/...', source: 'vue'}], concurrency: 5.\n" +
1825
+ " Indexing is serial regardless of concurrency — fetches race, FTS5 writes don't (avoids SQLite WAL contention).\n\n" +
1826
+ "When reporting results — terse like caveman. Technical substance exact. Only fluff die. Pattern: [thing] [action] [reason]. [next step].",
1827
+ inputSchema: z.object({
1828
+ url: z.string().optional().describe("Single URL to fetch and index (legacy single-shape)"),
1829
+ source: z
1830
+ .string()
1831
+ .optional()
1832
+ .describe("Label for the indexed content when using single `url` (e.g., 'React useEffect docs', 'Supabase Auth API'). For batch, put source in each requests entry."),
1833
+ requests: z
1834
+ .array(z.object({
1835
+ url: z.string().describe("URL to fetch"),
1836
+ source: z.string().optional().describe("Label for this URL's indexed content"),
1837
+ }))
1838
+ .min(1)
1839
+ .optional()
1840
+ .describe("Batch shape: array of {url, source?} entries. Use with concurrency>1 for parallel fetch. " +
1841
+ "Each request indexed under its own source label. Output preserves input order."),
1842
+ concurrency: z
1843
+ .coerce.number()
1844
+ .int()
1845
+ .min(1)
1846
+ .max(8)
1847
+ .optional()
1848
+ .default(1)
1849
+ .describe("Max URLs to fetch in parallel (1-8, default: 1). " +
1850
+ "Use 4-8 for I/O-bound multi-URL batches (library docs, changelogs, pricing pages). " +
1851
+ "Capped by os.cpus().length on small machines (response notes when capped). " +
1852
+ "Indexing is always serial regardless — only fetches race."),
1853
+ force: z
1854
+ .boolean()
1855
+ .optional()
1856
+ .describe("Skip cache and re-fetch even if content was recently indexed"),
1857
+ }),
1858
+ }, async ({ url, source, requests, concurrency, force }) => {
1859
+ // Normalize input: legacy {url} or new {requests: [...]}.
1860
+ // requests wins when both are provided (explicit batch intent).
1861
+ const batch = requests
1862
+ ? requests
1863
+ : url
1864
+ ? [{ url, source }]
1865
+ : [];
1866
+ if (batch.length === 0) {
1867
+ return trackResponse("ctx_fetch_and_index", {
1868
+ content: [{
1869
+ type: "text",
1870
+ text: "ctx_fetch_and_index requires either `url` (single) or `requests: [{url, source?}, ...]` (batch).",
1871
+ }],
1872
+ isError: true,
1873
+ });
1874
+ }
1875
+ const isLegacySingle = !requests && batch.length === 1;
1876
+ const requestedConcurrency = concurrency ?? 1;
1877
+ // Parallel fetch via shared runPool primitive. capByCpuCount only for batch
1878
+ // — single-URL doesn't need the cap (only one job, executor is one subprocess).
1879
+ const jobs = batch.map((req) => ({
1880
+ run: () => fetchOneUrl(req.url, req.source, force),
1881
+ }));
1882
+ const { settled, effectiveConcurrency, capped } = await runPool(jobs, {
1883
+ concurrency: requestedConcurrency,
1884
+ capByCpuCount: !isLegacySingle && requestedConcurrency > 1,
1885
+ });
1886
+ const finalized = [];
1887
+ for (let i = 0; i < settled.length; i++) {
1888
+ const r = settled[i];
1889
+ if (r.status === "rejected") {
1890
+ const message = r.reason instanceof Error ? r.reason.message : String(r.reason);
1891
+ finalized.push({ kind: "job_error", url: batch[i].url, error: message });
1892
+ continue;
1893
+ }
1894
+ const v = r.value;
1895
+ if (v.kind === "cached") {
1896
+ sessionStats.cacheHits++;
1897
+ sessionStats.cacheBytesSaved += v.estimatedBytes;
1898
+ finalized.push({ kind: "cached", label: v.label, chunkCount: v.chunkCount, ageStr: v.ageStr });
1899
+ }
1900
+ else if (v.kind === "fetch_error") {
1901
+ finalized.push({ kind: "fetch_error", url: v.url, error: v.error, reason: v.reason });
1902
+ }
1903
+ else {
1904
+ // Serial FTS5 write here — no parallel store.index calls.
1905
+ finalized.push({ kind: "fetched", indexed: indexFetched(v) });
1906
+ }
1907
+ }
1908
+ // Backward-compat single-URL response shape — preserve the EXACT original wording.
1909
+ if (isLegacySingle) {
1910
+ const r = finalized[0];
1911
+ if (r.kind === "cached") {
1432
1912
  return trackResponse("ctx_fetch_and_index", {
1433
- content: [
1434
- {
1913
+ content: [{
1435
1914
  type: "text",
1436
- text: `Fetched ${url} but got empty content`,
1437
- },
1438
- ],
1439
- isError: true,
1915
+ text: `Cached: **${r.label}** — ${r.chunkCount} sections, indexed ${r.ageStr} (fresh, TTL: 24h).\nTo refresh: call ctx_fetch_and_index again with \`force: true\`.\n\nYou MUST call ctx_search() to answer questions about this content — this cached response contains no content.\nUse: ctx_search(queries: [...], source: "${r.label}")`,
1916
+ }],
1440
1917
  });
1441
1918
  }
1442
- trackIndexed(Buffer.byteLength(markdown));
1443
- // Route to the appropriate indexing strategy based on Content-Type
1444
- let indexed;
1445
- if (header === "__CM_CT__:json") {
1446
- indexed = store.indexJSON(markdown, source ?? url);
1919
+ if (r.kind === "fetched") {
1920
+ const totalKB = (r.indexed.totalBytes / 1024).toFixed(1);
1921
+ const text = [
1922
+ `Fetched and indexed **${r.indexed.totalChunks} sections** (${totalKB}KB) from: ${r.indexed.label}`,
1923
+ `Full content indexed in sandbox — use ctx_search(queries: [...], source: "${r.indexed.label}") for specific lookups.`,
1924
+ "",
1925
+ "---",
1926
+ "",
1927
+ r.indexed.preview,
1928
+ ].join("\n");
1929
+ return trackResponse("ctx_fetch_and_index", {
1930
+ content: [{ type: "text", text }],
1931
+ });
1447
1932
  }
1448
- else if (header === "__CM_CT__:text") {
1449
- indexed = store.indexPlainText(markdown, source ?? url);
1933
+ // fetch_error preserve original error wording per reason
1934
+ if (r.kind === "fetch_error") {
1935
+ const text = r.reason === "empty" ? `Fetched ${r.url} but got empty content`
1936
+ : r.reason === "read" ? `Fetched ${r.url} but could not read subprocess output`
1937
+ : r.reason === "exit" ? `Failed to fetch ${r.url}: ${r.error}`
1938
+ : /* throw */ `Fetch error: ${r.error}`;
1939
+ return trackResponse("ctx_fetch_and_index", {
1940
+ content: [{ type: "text", text }],
1941
+ isError: true,
1942
+ });
1450
1943
  }
1451
- else {
1452
- // HTML (default) — content is already converted to markdown
1453
- indexed = store.index({ content: markdown, source: source ?? url });
1454
- }
1455
- // Build preview — first ~3KB of markdown for immediate use
1456
- const PREVIEW_LIMIT = 3072;
1457
- const preview = markdown.length > PREVIEW_LIMIT
1458
- ? markdown.slice(0, PREVIEW_LIMIT) + "\n\n…[truncated — use ctx_search() for full content]"
1459
- : markdown;
1460
- const totalKB = (Buffer.byteLength(markdown) / 1024).toFixed(1);
1461
- const text = [
1462
- `Fetched and indexed **${indexed.totalChunks} sections** (${totalKB}KB) from: ${indexed.label}`,
1463
- `Full content indexed in sandbox — use ctx_search(queries: [...], source: "${indexed.label}") for specific lookups.`,
1464
- "",
1465
- "---",
1466
- "",
1467
- preview,
1468
- ].join("\n");
1469
- return trackResponse("ctx_fetch_and_index", {
1470
- content: [{ type: "text", text }],
1471
- });
1472
- }
1473
- catch (err) {
1474
- const message = err instanceof Error ? err.message : String(err);
1944
+ // job_error
1475
1945
  return trackResponse("ctx_fetch_and_index", {
1476
- content: [
1477
- { type: "text", text: `Fetch error: ${message}` },
1478
- ],
1946
+ content: [{ type: "text", text: `Fetch error: ${r.error}` }],
1479
1947
  isError: true,
1480
1948
  });
1481
1949
  }
1482
- finally {
1483
- // Clean up temp file
1484
- try {
1485
- rmSync(outputPath);
1950
+ // Batch response — aggregated summary; isError only when EVERY URL failed.
1951
+ // Per-URL preview capped tightly so a 8-URL batch doesn't undo the
1952
+ // context-savings the tool exists to deliver (PRD review finding G1).
1953
+ const FETCH_BATCH_PREVIEW_LIMIT = 384; // ~3KB total for 8-URL batches
1954
+ const lines = [];
1955
+ let totalSections = 0;
1956
+ let totalBytes = 0;
1957
+ let cachedCount = 0;
1958
+ let fetchedCount = 0;
1959
+ let errorCount = 0;
1960
+ const snippets = [];
1961
+ for (const r of finalized) {
1962
+ if (r.kind === "cached") {
1963
+ cachedCount++;
1964
+ lines.push(`- [cache] ${r.label} — ${r.chunkCount} sections (${r.ageStr})`);
1965
+ }
1966
+ else if (r.kind === "fetched") {
1967
+ fetchedCount++;
1968
+ totalSections += r.indexed.totalChunks;
1969
+ totalBytes += r.indexed.totalBytes;
1970
+ const kb = (r.indexed.totalBytes / 1024).toFixed(1);
1971
+ lines.push(`- [new] ${r.indexed.label} — ${r.indexed.totalChunks} sections (${kb}KB)`);
1972
+ const snippet = r.indexed.preview.length > FETCH_BATCH_PREVIEW_LIMIT
1973
+ ? r.indexed.preview.slice(0, FETCH_BATCH_PREVIEW_LIMIT).trimEnd() + "…"
1974
+ : r.indexed.preview;
1975
+ snippets.push(`### ${r.indexed.label}\n\n${snippet}`);
1486
1976
  }
1487
- catch { /* already gone */ }
1488
- }
1977
+ else {
1978
+ errorCount++;
1979
+ lines.push(`- [err] ${r.url}: ${r.error}`);
1980
+ }
1981
+ }
1982
+ const totalKB = (totalBytes / 1024).toFixed(1);
1983
+ const cappedNote = capped
1984
+ ? ` cap=${effectiveConcurrency}/${cpus().length}cpu`
1985
+ : "";
1986
+ // Caveman style — terse status line: counts + sections + size.
1987
+ // Singular forms used at count=1 to avoid grammar drift ("1 errors" → "1 error").
1988
+ const fmt = (n, sing, plur) => `${n} ${n === 1 ? sing : plur}`;
1989
+ const headerLine = `fetched ${batch.length} c=${effectiveConcurrency}${cappedNote}. ` +
1990
+ `ok=${fetchedCount} cache=${cachedCount} err=${errorCount}. ` +
1991
+ `${fmt(totalSections, "section", "sections")} ${totalKB}KB.`;
1992
+ const text = [
1993
+ headerLine,
1994
+ "",
1995
+ ...lines,
1996
+ "",
1997
+ `ctx_search(queries: [...], source: "<label>") for full content.`,
1998
+ ...(snippets.length > 0 ? ["", "---", "", ...snippets] : []),
1999
+ ].join("\n");
2000
+ return trackResponse("ctx_fetch_and_index", {
2001
+ content: [{ type: "text", text }],
2002
+ isError: errorCount === batch.length, // only mark error if every URL failed
2003
+ });
1489
2004
  });
1490
2005
  // ─────────────────────────────────────────────────────────
1491
2006
  // Tool: batch_execute
@@ -1497,7 +2012,12 @@ server.registerTool("ctx_batch_execute", {
1497
2012
  "THIS IS THE PRIMARY TOOL. Use this instead of multiple ctx_execute() calls.\n\n" +
1498
2013
  "One ctx_batch_execute call replaces 30+ ctx_execute calls + 10+ ctx_search calls.\n" +
1499
2014
  "Provide all commands to run and all queries to search — everything happens in one round trip.\n\n" +
1500
- "THINK IN CODE: When commands produce data you need to analyze, add processing commands that filter and summarize. Don't pull raw output into context — let the sandbox do the work.\n\n" +
2015
+ "PARALLELIZE I/O: For I/O-bound batches (network calls, slow API queries, multi-URL fetches), ALWAYS pass concurrency: 4-8 speeds up by 3-5x on real workloads.\n" +
2016
+ " ✅ Use concurrency: 4-8 for: gh API calls, curl/web fetches, multi-region cloud queries, multi-repo git reads, dig/DNS, docker inspect.\n" +
2017
+ " ❌ Keep concurrency: 1 for: npm test, build, lint, image processing (CPU-bound), or commands sharing state (ports, lock files, same-repo writes).\n" +
2018
+ " Example: [gh issue view 1, gh issue view 2, gh issue view 3] → concurrency: 3.\n" +
2019
+ " Speedup depends on workload — applies to I/O wait, not CPU work.\n\n" +
2020
+ "THINK IN CODE — NON-NEGOTIABLE: When commands produce data you need to analyze, count, filter, compare, or transform — add a processing command that runs JavaScript and console.log() ONLY the answer. NEVER pull raw output into context to reason over. Concurrency parallelizes the FETCH; THINK IN CODE owns the PROCESSING. One programmed analysis replaces ten read-and-reason rounds. Pure JavaScript, Node.js built-ins (fs, path, child_process), try/catch, null-safe.\n\n" +
1501
2021
  "When reporting results — terse like caveman. Technical substance exact. Only fluff die. Pattern: [thing] [action] [reason]. [next step].",
1502
2022
  inputSchema: z.object({
1503
2023
  commands: z.preprocess(coerceCommandsArray, z
@@ -1510,7 +2030,8 @@ server.registerTool("ctx_batch_execute", {
1510
2030
  .describe("Shell command to execute"),
1511
2031
  }))
1512
2032
  .min(1)
1513
- .describe("Commands to execute as a batch. Each runs sequentially, output is labeled with the section header.")),
2033
+ .describe("Commands to execute as a batch. Output is labeled with the section header. " +
2034
+ "Default order is sequential; pass concurrency>1 to run in parallel (output stays in input order).")),
1514
2035
  queries: z.preprocess(coerceJsonArray, z
1515
2036
  .array(z.string())
1516
2037
  .min(1)
@@ -1521,9 +2042,21 @@ server.registerTool("ctx_batch_execute", {
1521
2042
  .coerce.number()
1522
2043
  .optional()
1523
2044
  .default(60000)
1524
- .describe("Max execution time in ms (default: 60s)"),
2045
+ .describe("Max execution time in ms (default: 60s). With concurrency=1, shared budget across commands; with concurrency>1, applied per-command."),
2046
+ concurrency: z
2047
+ .coerce.number()
2048
+ .int()
2049
+ .min(1)
2050
+ .max(8)
2051
+ .optional()
2052
+ .default(1)
2053
+ .describe("Max commands to run in parallel (1-8, default: 1). " +
2054
+ "Use 4-8 for I/O-bound batches (network, gh, curl, multi-repo git reads). " +
2055
+ "Keep at 1 for CPU-bound (npm test, build, lint) or stateful commands (ports, locks). " +
2056
+ ">1 switches to per-command timeouts (no shared budget) and " +
2057
+ "individual `(timed out)` blocks instead of cascading skip."),
1525
2058
  }),
1526
- }, async ({ commands, queries, timeout }) => {
2059
+ }, async ({ commands, queries, timeout, concurrency }) => {
1527
2060
  // Security: check each command against deny patterns
1528
2061
  for (const cmd of commands) {
1529
2062
  const denied = checkDenyPolicy(cmd.command, "batch_execute");
@@ -1531,51 +2064,18 @@ server.registerTool("ctx_batch_execute", {
1531
2064
  return denied;
1532
2065
  }
1533
2066
  try {
1534
- // Execute each command individually so every command gets its own
1535
- // output capture. Full stdout is preserved and indexed into FTS5.
1536
- // (Issue #61, #197)
1537
- const perCommandOutputs = [];
1538
- const startTime = Date.now();
1539
- let timedOut = false;
1540
2067
  // Inject NODE_OPTIONS for FS read tracking in spawned Node processes.
1541
2068
  // The executor denies NODE_OPTIONS in its env (security), so we set it
1542
2069
  // as an inline shell prefix. This only affects child `node` invocations.
1543
2070
  const nodeOptsPrefix = `NODE_OPTIONS="--require ${CM_FS_PRELOAD}" `;
1544
- for (const cmd of commands) {
1545
- const elapsed = Date.now() - startTime;
1546
- const remaining = timeout - elapsed;
1547
- if (remaining <= 0) {
1548
- perCommandOutputs.push(`# ${cmd.label}\n\n(skipped — batch timeout exceeded)\n`);
1549
- timedOut = true;
1550
- continue;
1551
- }
1552
- const result = await executor.execute({
1553
- language: "shell",
1554
- code: `${nodeOptsPrefix}${cmd.command} 2>&1`,
1555
- timeout: remaining,
1556
- });
1557
- let output = result.stdout || "(no output)";
1558
- // Parse and strip __CM_FS__ markers emitted by the preload script.
1559
- // Because 2>&1 merges stderr into stdout, markers appear in output.
1560
- const fsMatches = output.matchAll(/__CM_FS__:(\d+)/g);
1561
- let cmdFsBytes = 0;
1562
- for (const m of fsMatches)
1563
- cmdFsBytes += parseInt(m[1]);
1564
- if (cmdFsBytes > 0) {
1565
- sessionStats.bytesSandboxed += cmdFsBytes;
1566
- output = output.replace(/__CM_FS__:\d+\n?/g, "");
1567
- }
1568
- perCommandOutputs.push(`# ${cmd.label}\n\n${output}\n`);
1569
- if (result.timedOut) {
1570
- timedOut = true;
1571
- // Mark remaining commands as skipped
1572
- const idx = commands.indexOf(cmd);
1573
- for (let i = idx + 1; i < commands.length; i++) {
1574
- perCommandOutputs.push(`# ${commands[i].label}\n\n(skipped — batch timeout exceeded)\n`);
1575
- }
1576
- break;
1577
- }
1578
- }
2071
+ // Full stdout is preserved per-command and indexed into FTS5 (Issue #61, #197).
2072
+ // Concurrency>1 switches to a worker pool with per-command timeouts.
2073
+ const { outputs: perCommandOutputs, timedOut } = await runBatchCommands(commands, {
2074
+ timeout,
2075
+ concurrency,
2076
+ nodeOptsPrefix,
2077
+ onFsBytes: (bytes) => { sessionStats.bytesSandboxed += bytes; },
2078
+ }, executor);
1579
2079
  const stdout = perCommandOutputs.join("\n");
1580
2080
  const totalBytes = Buffer.byteLength(stdout);
1581
2081
  const totalLines = stdout.split("\n").length;
@@ -1678,24 +2178,37 @@ server.registerTool("ctx_stats", {
1678
2178
  try {
1679
2179
  const engine = new AnalyticsEngine(sdb);
1680
2180
  const report = engine.queryAll(sessionStats);
1681
- text = formatReport(report, VERSION, _latestVersion);
2181
+ // MCP usage is read-only and cheap; only available when DB exists.
2182
+ const mcpUsage = engine.getMcpToolUsage();
2183
+ // Lifetime stats span every project's SessionDB + auto-memory dir
2184
+ // (Bugs #3/#4); failures are absorbed inside getLifetimeStats so a
2185
+ // corrupt sidecar can never break ctx_stats.
2186
+ const lifetime = getLifetimeStats();
2187
+ text = formatReport(report, VERSION, _latestVersion, { lifetime, mcpUsage });
1682
2188
  }
1683
2189
  finally {
1684
2190
  sdb.close();
1685
2191
  }
1686
2192
  }
1687
2193
  else {
1688
- // No session DB — build a minimal report from runtime stats only
2194
+ // No session DB — build a minimal report from runtime stats only.
2195
+ // Lifetime still meaningful (other projects, auto-memory) so include it.
1689
2196
  const engine = new AnalyticsEngine(createMinimalDb());
1690
2197
  const report = engine.queryAll(sessionStats);
1691
- text = formatReport(report, VERSION, _latestVersion);
2198
+ const lifetime = getLifetimeStats();
2199
+ text = formatReport(report, VERSION, _latestVersion, { lifetime });
1692
2200
  }
1693
2201
  }
1694
2202
  catch {
1695
2203
  // Session DB not available or incompatible — build minimal report from runtime stats
1696
2204
  const engine = new AnalyticsEngine(createMinimalDb());
1697
2205
  const report = engine.queryAll(sessionStats);
1698
- text = formatReport(report, VERSION, _latestVersion);
2206
+ let lifetime;
2207
+ try {
2208
+ lifetime = getLifetimeStats();
2209
+ }
2210
+ catch { /* never block ctx_stats */ }
2211
+ text = formatReport(report, VERSION, _latestVersion, lifetime ? { lifetime } : undefined);
1699
2212
  }
1700
2213
  return trackResponse("ctx_stats", {
1701
2214
  content: [{ type: "text", text }],
@@ -1985,6 +2498,13 @@ server.registerTool("ctx_purge", {
1985
2498
  sessionStats.cacheBytesSaved = 0;
1986
2499
  sessionStats.sessionStart = Date.now();
1987
2500
  deleted.push("session stats");
2501
+ // Also drop the persisted stats file so external readers see a fresh state
2502
+ try {
2503
+ const statsFile = getStatsFilePath();
2504
+ if (existsSync(statsFile))
2505
+ unlinkSync(statsFile);
2506
+ }
2507
+ catch { /* best effort */ }
1988
2508
  return trackResponse("ctx_purge", {
1989
2509
  content: [{
1990
2510
  type: "text",
@@ -2231,6 +2751,15 @@ async function main() {
2231
2751
  }
2232
2752
  };
2233
2753
  const gracefulShutdown = async () => {
2754
+ // Final stats flush — bypass throttle so the last 0-500ms of
2755
+ // bytes_indexed / bytes_returned aren't silently lost on SIGTERM/SIGINT
2756
+ // (PR #401 grill-me review B1: persistStats early-returns inside throttle
2757
+ // window; gracefulShutdown previously did NOT bypass).
2758
+ try {
2759
+ _lastStatsPersist = 0;
2760
+ persistStats();
2761
+ }
2762
+ catch { /* best effort — never block shutdown */ }
2234
2763
  shutdown();
2235
2764
  process.exit(0);
2236
2765
  };