crawlforge-mcp-server 6.4.0 → 6.6.0

package/src/tools/advanced/BrowserSessionTool.js

@@ -0,0 +1,476 @@
+/**
+ * BrowserSessionTool — `browser_session`: one browser page the caller keeps
+ * across several tool calls, driven by an `operation` enum.
+ *
+ * `scrape_with_actions` is one-shot and blind: the agent has to name up to 20
+ * actions up front, guessing selectors for a page it has never seen, and the
+ * browser closes when the call returns. A session inverts that loop — open,
+ * look (`snapshot`), act on the refs the snapshot handed back, look again —
+ * and one login is paid for once instead of once per call.
+ *
+ * Everything here is assembled from parts that already exist, deliberately:
+ *   - `ActionExecutor.initializePage()` is the `open` primitive, SSRF guard and
+ *     robots gate included, and `executeActionsOnPage()` runs actions against a
+ *     page it does not own (the `navigate` action re-gates every hop itself).
+ *   - Element refs live in a page-scoped WeakMap in core/browser/snapshot.js, so
+ *     a session that keeps its page keeps its refs across calls for free, and
+ *     loses them exactly when it should — on navigation.
+ *   - `BrowserSessionStore` holds the sessions, the two TTL clocks and the caps.
+ *   - `ExtractContentTool` turns the live DOM into the requested formats.
+ *
+ * Ownership is a tenant boundary, not a nicety — see ownerId() for what that
+ * means for the hosted REST path, which is served only to a request carrying a
+ * per-user owner token and refused to any other.
+ */
+
+import { z } from 'zod';
+import { createHash } from 'node:crypto';
+
+import ActionExecutor from '../../core/ActionExecutor.js';
+import ExtractContentTool from '../extract/extractContent.js';
+import BrowserSessionStore, {
+  TTL_MIN_MS,
+  TTL_MAX_MS,
+  ACTIVITY_TTL_MIN_MS,
+  ACTIVITY_TTL_MAX_MS
+} from '../../core/browser/SessionStore.js';
+import { captureSnapshot } from '../../core/browser/snapshot.js';
+import authManager from '../../core/AuthManager.js';
+import { isCreatorModeVerified } from '../../core/creatorMode.js';
+import { internalOwnerToken, isInternalRequest } from '../../server/requestContext.js';
+import { isRemoteTransport } from '../../utils/remoteMode.js';
+import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js';
+
+const SECOND = 1000;
+
+/** The prefix that marks an owner id derived from a hosted REST owner token. */
+const REST_OWNER_PREFIX = 'rest:';
+
+/**
+ * A hosted REST customer may hold ONE session at a time, where a stdio install
+ * keeps the store's default of three.
+ *
+ * Arithmetic, not caution. On Render MAX_BROWSER_CONTEXTS=6, so
+ * DEFAULT_MAX_SESSIONS_TOTAL is floor(6/2) = 3 for the WHOLE box while
+ * DEFAULT_MAX_SESSIONS_PER_OWNER is 3 — meaning one REST customer opening three
+ * sessions occupies the entire hosted session capacity, and every other paying
+ * customer is refused until those sessions age out, up to ten minutes later.
+ * Capping REST owners at one lets three distinct customers work at once.
+ *
+ * stdio and self-hosted installs keep the three: there the process IS the
+ * customer, the box is theirs, and there is nobody else to lock out.
+ */
+const REST_MAX_SESSIONS_PER_OWNER = 1;
+
+/**
+ * The action array is `scrape_with_actions`' own, passed through untouched:
+ * ActionExecutor validates each action against its own union as it runs it, and
+ * a fourth copy of that union here could only drift from the three that exist.
+ *
+ * The two fields below are not decoration. ActionExecutor parses each action but
+ * discards the parsed value, so an action that arrives without them keeps the
+ * undefined it came with — and `action.retries > 0` is the gate on error
+ * recovery, `action.continueOnError` the per-action failure policy. These are
+ * the same defaults ScrapeWithActionsTool's schema stamps.
+ */
+const SessionActionSchema = z.object({
+  type: z.string(),
+  continueOnError: z.boolean().default(false),
+  retries: z.number().min(0).max(5).default(1)
+}).passthrough();
+
+const BrowserSessionSchema = z.object({
+  operation: z.enum(['open', 'snapshot', 'act', 'read', 'screenshot', 'close', 'list']),
+  session_id: z.string().optional(),
+
+  // open. ttl/activity_ttl are seconds, as Firecrawl's are, so anyone arriving
+  // from their docs reads the same numbers; the store works in milliseconds.
+  url: z.string().url().optional(),
+  stealth: z.boolean().default(false),
+  ttl: z.number().min(TTL_MIN_MS / SECOND).max(TTL_MAX_MS / SECOND).optional(),
+  activity_ttl: z.number().min(ACTIVITY_TTL_MIN_MS / SECOND).max(ACTIVITY_TTL_MAX_MS / SECOND).optional(),
+  viewport: z.object({
+    width: z.number().min(800).max(1920),
+    height: z.number().min(600).max(1080)
+  }).optional(),
+  timeout: z.number().min(10000).max(120000).default(30000),
+
+  // Applies to the call it is sent on: on `open` to the first load, on `act` to
+  // every navigate in that call. It is never remembered by the session, so an
+  // override has to be repeated as deliberately as it was made.
+  respect_robots: z.boolean().optional(),
+
+  // snapshot
+  interactive_only: z.boolean().default(true),
+  max_nodes: z.number().min(1).max(1000).optional(),
+
+  // act
+  actions: z.array(SessionActionSchema).min(1).max(20).optional(),
+  continue_on_error: z.boolean().default(false),
+
+  // read
+  formats: z.array(z.enum(['markdown', 'html', 'text', 'json'])).default(['markdown']),
+
+  // screenshot
+  full_page: z.boolean().default(false),
+  format: z.enum(['png', 'jpeg']).default('png'),
+  quality: z.number().min(0).max(100).default(80),
+  selector: z.string().optional()
+});
+
+/** A refusal a caller (and a test) can match on by code rather than by prose. */
+function refuse(code, message) {
+  const error = new Error(message);
+  error.name = 'BrowserSessionRefusal';
+  error.code = code;
+  return error;
+}
+
+/**
+ * The session fields every operation echoes back, in the same shape a store
+ * list row carries. Read it after touch() so the idle clock is the fresh one.
+ */
+function sessionInfo(session) {
+  return {
+    sessionId: session.id,
+    url: session.url,
+    stealth: session.stealth,
+    expiresAt: session.createdAt + session.ttlMs,
+    idleExpiresAt: session.lastUsedAt + session.activityTtlMs
+  };
+}
+
+export class BrowserSessionTool {
+  constructor(options = {}) {
+    const {
+      actionExecutor = null,
+      extractContentTool = null,
+      store = null,
+      storeOptions = {},
+      enableLogging = true
+    } = options;
+
+    // An injected executor belongs to whoever built it (server.js hands us
+    // scrape_with_actions'), and destroying it would take that tool's browser
+    // down with ours. Only an executor we made ourselves is ours to destroy.
+    this._ownsExecutor = !actionExecutor;
+    this.actionExecutor = actionExecutor || new ActionExecutor({ enableLogging });
+    this.extractContentTool = extractContentTool || new ExtractContentTool();
+    this.storeOptions = storeOptions;
+    this.store = store || new BrowserSessionStore(storeOptions);
+  }
+
+  async execute(params) {
+    const validated = BrowserSessionSchema.parse(params);
+    const ownerId = this.ownerId();
+
+    switch (validated.operation) {
+      case 'open': return await this.openSession(validated, ownerId);
+      case 'snapshot': return await this.snapshotSession(validated, ownerId);
+      case 'act': return await this.actOnSession(validated, ownerId);
+      case 'read': return await this.readSession(validated, ownerId);
+      case 'screenshot': return await this.screenshotSession(validated, ownerId);
+      case 'close': return await this.closeSession(validated, ownerId);
+      case 'list': return this.listSessions(ownerId);
+    }
+  }
+
+  /**
+   * Who the caller is — the identity every session is bound to and every
+   * lookup is scoped by.
+   *
+   * On the hosted REST path the shared secret is not an identity. The website's
+   * proxy authenticates to this server with a single X-Internal-Secret
+   * (`authenticateRequest` in src/server/transports/streamableHttp.js), so
+   * every REST customer arrives as the same internal caller; binding a session
+   * to THAT would put all of them inside one tenant, where any customer could
+   * name any other customer's session id and be handed their logged-in browser.
+   * The per-user owner token is the missing half: the proxy derives it per end
+   * user and sends it on X-CrawlForge-Owner, the transport honours it only on a
+   * request that already proved the secret, and it lands here as the tenant key.
+   *
+   * NO FALLBACK WHEN IT IS ABSENT, DELIBERATELY — do not "tidy" the refusal
+   * below into a default owner. It is what makes the two repos safe to deploy
+   * in either order: an old server ignores the new header, a new server meets
+   * an old website that sends none, and in both cases sessions are refused
+   * rather than silently collapsing into one shared tenant. A missing owner is
+   * a missing tenant boundary, and the honest answer to that is "no session".
+   *
+   * Everywhere else the install is the tenant: over stdio, and over self-hosted
+   * HTTP authenticated with the install's API key or an OAuth token, the same
+   * configured key stands behind every request. Its digest is the owner id; the
+   * key itself never leaves this method.
+   */
+  ownerId() {
+    if (isInternalRequest()) {
+      const ownerToken = internalOwnerToken();
+      if (ownerToken) return `${REST_OWNER_PREFIX}${ownerToken}`;
+
+      throw refuse(
+        'SESSIONS_NOT_AVAILABLE_OVER_REST',
+        'browser_session could not be opened over the CrawlForge REST API: the API proxy ' +
+        'authenticated as a shared internal caller without saying which customer this request ' +
+        'belongs to, so a session id could not be bound to the account that opened it. That is ' +
+        'usually a version skew mid-deploy — try again shortly. Meanwhile, use ' +
+        'scrape_with_actions for a one-shot interaction chain, or run the CrawlForge MCP ' +
+        'server locally (stdio) where sessions work normally.'
+      );
+    }
+
+    const apiKey = authManager.getConfig()?.apiKey;
+    return apiKey
+      ? `key:${createHash('sha256').update(apiKey).digest('hex').slice(0, 16)}`
+      : 'local';
+  }
+
+  /**
+   * The session this operation names, or the same "session not found" an
+   * unknown id gets. The store raises one error for unknown, wrong-owner and
+   * expired alike, which is what keeps ids non-enumerable — never answer a
+   * wrong owner with "forbidden".
+   */
+  requireSession(params, ownerId) {
+    if (!params.session_id) {
+      throw new Error(
+        `operation "${params.operation}" requires session_id — the id returned by operation:"open".`
+      );
+    }
+    return this.store.get(params.session_id, ownerId);
+  }
+
+  async openSession(params, ownerId) {
+    if (!params.url) {
+      throw new Error('operation "open" requires a url to load the session on.');
+    }
+
+    const browserOptions = {
+      headless: true,
+      viewportWidth: params.viewport?.width,
+      viewportHeight: params.viewport?.height,
+      timeout: params.timeout,
+      respectRobots: params.respect_robots
+    };
+    if (params.stealth) {
+      browserOptions.stealthMode = { enabled: true };
+    }
+
+    // initializePage runs the SSRF guard, then the blocklist/robots gate, and
+    // only then creates a page and navigates — closing the page itself if any
+    // of that fails. That is the whole of 2.5's gating on `open`, which is why
+    // this is not page.goto() behind a gate written here.
+    const page = await this.actionExecutor.initializePage(params.url, browserOptions);
+    const releasePage = this.releaser(page, params.stealth);
+
+    let session;
+    try {
+      session = this.store.create({
+        ownerId,
+        page,
+        releasePage,
+        url: page.url(),
+        stealth: params.stealth,
+        ttlMs: params.ttl === undefined ? undefined : params.ttl * SECOND,
+        activityTtlMs: params.activity_ttl === undefined ? undefined : params.activity_ttl * SECOND,
+        // undefined for every other owner, which leaves the store's own cap in
+        // force — the override exists for the hosted box alone.
+        maxPerOwner: ownerId.startsWith(REST_OWNER_PREFIX) ? REST_MAX_SESSIONS_PER_OWNER : undefined
+      });
+    } catch (error) {
+      // A cap refusal arrives with a live page in hand. Give it back before
+      // rethrowing, or the refused call leaks the context it just pinned.
+      await releasePage();
+      throw error;
+    }
+
+    return { success: true, operation: 'open', ...sessionInfo(session) };
+  }
+
+  async snapshotSession(params, ownerId) {
+    const session = this.requireSession(params, ownerId);
+
+    const snapshot = await captureSnapshot(session.page, {
+      interactiveOnly: params.interactive_only,
+      maxNodes: params.max_nodes
+    });
+
+    this.store.touch(session, session.page.url());
+    return { success: true, operation: 'snapshot', ...sessionInfo(session), snapshot };
+  }
+
+  async actOnSession(params, ownerId) {
+    if (!params.actions?.length) {
+      throw new Error('operation "act" requires an actions array.');
+    }
+    const session = this.requireSession(params, ownerId);
+
+    // D6: arbitrary JavaScript in a browser on OUR infrastructure is a
+    // materially different act from the same JavaScript in a browser on the
+    // caller's own laptop. Over stdio or loopback the caller is the local user
+    // and executeJavaScriptAction's own ALLOW_JAVASCRIPT_EXECUTION flag is the
+    // control; served to a network, it is refused outright. Creator mode is the
+    // maintainer's own box, so it keeps the local answer.
+    if (params.actions.some((action) => action.type === 'executeJavaScript') &&
+        isRemoteTransport() && !isCreatorModeVerified()) {
+      throw refuse(
+        'JS_EXECUTION_REFUSED_REMOTE',
+        'executeJavaScript is refused in a browser session on a remotely-served CrawlForge ' +
+        'instance: the script would run in a browser on the server, not on your machine. ' +
+        'Use the click / type / select / press actions, or run the MCP server locally over stdio.'
+      );
+    }
+
+    // Every `navigate` in here re-runs the SSRF guard and the blocklist/robots
+    // gate inside executeNavigateAction — verified, and the reason the gate is
+    // not repeated here. A long-lived session is a repeatable navigation
+    // primitive, so that per-hop check is what stops it becoming an SSRF hop.
+    const result = await this.actionExecutor.executeActionsOnPage(session.page, params.actions, {
+      continueOnError: params.continue_on_error,
+      timeout: params.timeout,
+      browserOptions: { respectRobots: params.respect_robots }
+    });
+
+    this.store.touch(session, result.finalUrl);
+    return {
+      success: result.success,
+      operation: 'act',
+      ...sessionInfo(session),
+      error: result.error,
+      actionResults: result.results,
+      screenshots: result.screenshots,
+      ...(result.capturedStates.length > 0 ? { capturedStates: result.capturedStates } : {}),
+      stats: result.stats
+    };
+  }
+
+  async readSession(params, ownerId) {
+    const session = this.requireSession(params, ownerId);
+    const url = session.page.url();
+    const html = await session.page.content();
+
+    const options = {};
+    if (params.formats.includes('markdown')) options.outputFormat = 'markdown';
+    if (params.formats.includes('html')) options.includeRawHTML = true;
+
+    // The live post-action DOM is already in hand, so extract_content is handed
+    // that rather than the url: a re-fetch would arrive without the session's
+    // cookies and before everything the session has clicked, which is the whole
+    // point of having one.
+    const extracted = await this.extractContentTool.execute({ url, html, options });
+
+    const content = {};
+    if (params.formats.includes('text')) {
+      content.text = extracted.content?.text || '';
+    }
+    if (params.formats.includes('html')) {
+      content.html = extracted.content?.html || html;
+    }
+    if (params.formats.includes('markdown')) {
+      // Readability finds no article on most app pages, and then no markdown is
+      // produced at all (R20, 2026-09-07). Convert the DOM we hold instead of
+      // handing back a placeholder.
+      content.markdown = extracted.content?.markdown || htmlToMarkdown(html);
+    }
+    if (params.formats.includes('json')) {
+      content.json = {
+        title: extracted.title ?? null,
+        metadata: extracted.metadata || {},
+        structuredData: extracted.structuredData
+      };
+    }
+
+    this.store.touch(session, url);
+    return {
+      success: true,
+      operation: 'read',
+      ...sessionInfo(session),
+      title: extracted.title ?? null,
+      extractionMethod: extracted.extractionMethod,
+      content
+    };
+  }
+
+  async screenshotSession(params, ownerId) {
+    const session = this.requireSession(params, ownerId);
+
+    const shot = await this.actionExecutor.captureScreenshot(session.page, {
+      fullPage: params.full_page,
+      format: params.format,
+      quality: params.quality,
+      selector: params.selector
+    });
+
+    this.store.touch(session, session.page.url());
+    // The actionId is what lets the server publish the image as a
+    // crawlforge://screenshot/{actionId} resource and drop the base64 from the
+    // result — a full-page PNG inline is megabytes (R21, 2026-09-09).
+    return {
+      success: true,
+      operation: 'screenshot',
+      ...sessionInfo(session),
+      screenshot: { actionId: this.actionExecutor.generateActionId(), ...shot }
+    };
+  }
+
+  async closeSession(params, ownerId) {
+    if (!params.session_id) {
+      throw new Error('operation "close" requires session_id.');
+    }
+    await this.store.close(params.session_id, ownerId);
+    return { success: true, operation: 'close', sessionId: params.session_id, closed: true };
+  }
+
+  listSessions(ownerId) {
+    const sessions = this.store.list(ownerId).map(({ id, ...rest }) => ({ sessionId: id, ...rest }));
+    return { success: true, operation: 'list', count: sessions.length, sessions };
+  }
+
+  /**
+   * How a page goes back: the closure the store calls on close, on expiry and
+   * at shutdown.
+   *
+   * Copied from executeActionChain's `finally`, and the halves are not
+   * interchangeable. A stealth page goes through the manager so its pooled
+   * context slot is freed as well as the renderer; a standard page must also
+   * close the BrowserContext createPage() gave it, because nothing else tracks
+   * that one. Getting this wrong pins a context until the process dies, which
+   * on the 2 GB Render box is an outage rather than a leak.
+   */
+  releaser(page, stealth) {
+    return async () => {
+      if (stealth) {
+        await this.actionExecutor.browserProcessor.releaseStealthPage(page);
+        return;
+      }
+      const context = page.context();
+      try { await page.close(); } catch (_) { /* ignore close errors */ }
+      try { await context.close(); } catch (_) { /* ignore close errors */ }
+    };
+  }
+
+  /**
+   * Close every live session, leaving the tool able to open more.
+   *
+   * This is the half the stealth-cleanup lever needs: `stealth_mode`
+   * operation:"cleanup" tears down the stealth browser, so every page a stealth
+   * session is holding dies with it and those sessions have to go too — but the
+   * tool itself must still work afterwards. The store is replaced rather than
+   * reused because destroy() also stops its sweep timer, and an injected store
+   * is replaced along with the rest.
+   */
+  async cleanup() {
+    await this.store.destroy();
+    this.store = new BrowserSessionStore(this.storeOptions);
+  }
+
+  /**
+   * Process exit: close the sessions, then the browser — but only if the
+   * executor is ours. Sessions always close here; that is why this tool is in
+   * server.js's shutdown list even when it shares another tool's executor.
+   */
+  async destroy() {
+    await this.store.destroy();
+    if (this._ownsExecutor) await this.actionExecutor.destroy();
+  }
+}
+
+export default BrowserSessionTool;

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -129,6 +129,14 @@ const ExecuteJavaScriptActionSchema = BaseActionSchema.extend({
   returnResult: z.boolean().default(true)
 });
 
+// camelCase fields, matching ActionExecutor's SnapshotActionSchema - this
+// union runs first, so a mismatch here rejects the action before it gets there.
+const SnapshotActionSchema = BaseActionSchema.extend({
+  type: z.literal('snapshot'),
+  interactiveOnly: z.boolean().default(true),
+  maxNodes: z.number().min(1).max(1000).optional()
+});
+
 const ActionSchema = z.union([
   WaitActionSchema,
   ClickActionSchema,
@@ -139,7 +147,8 @@ const ActionSchema = z.union([
   HoverActionSchema,
   NavigateActionSchema,
   ScreenshotActionSchema,
-  ExecuteJavaScriptActionSchema
+  ExecuteJavaScriptActionSchema,
+  SnapshotActionSchema
 ]);
 
 // Form field schema for auto-fill

package/src/tools/crawl/mapSite.js

@@ -10,6 +10,44 @@ import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
 import { preflightFetch } from '../../utils/robotsGate.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 
+/**
+ * The path prefix a seed URL asks for: nps.gov/yell/ means the Yellowstone
+ * subtree, not the first 200 URLs of the park service's site-wide sitemap
+ * (R21, 2026-09-09: map_site returned Abraham Lincoln Birthplace pages for it).
+ * A page URL scopes to its directory; the site root scopes to nothing.
+ * @param {string} url
+ * @returns {string|null} e.g. "/yell/", or null for the root
+ */
+export function scopePathOf(url) {
+  try {
+    const { pathname } = new URL(url);
+    const dir = pathname.endsWith('/') ? pathname : pathname.slice(0, pathname.lastIndexOf('/') + 1);
+    return dir && dir !== '/' ? dir : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * How many of a search's terms a URL's own path and query contain — the
+ * relevance signal the generic ranker lacks when every candidate is a bare
+ * URL (it scored 0.19 for all 200 nps.gov URLs of a "fees" search, none of
+ * which mentioned fees).
+ * @param {string} url
+ * @param {string} search
+ * @returns {number}
+ */
+export function searchScore(url, search) {
+  const terms = [...new Set(String(search || '').toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((t) => t.length >= 2))];
+  if (terms.length === 0) return 0;
+  let haystack = url.toLowerCase();
+  try {
+    const { pathname, search: query } = new URL(url);
+    haystack = decodeURIComponent(pathname + query).toLowerCase();
+  } catch { /* keep the raw url */ }
+  return terms.filter((t) => haystack.includes(t)).length;
+}
+
 // Lazy singleton — avoids creating a CacheManager timer per request
 let _ranker = null;
 function getRanker() {
@@ -107,22 +145,47 @@ export class MapSiteTool {
         }
       }
 
+      // A seed with a path asks for that subtree, and a search needs a pool
+      // wider than max_urls to rank — otherwise the cut falls before the
+      // relevant URLs ever enter (the sitemap head is alphabetical).
+      const scopePath = scopePathOf(validated.url);
+      const widen = scopePath || validated.search;
+      const poolLimit = widen ? Math.min(10000, Math.max(validated.max_urls * 10, 2000)) : validated.max_urls;
+      const warnings = [];
+
       // Try to fetch sitemap first
       if (validated.include_sitemap) {
-        const sitemapUrls = await this.fetchSitemapUrls(baseUrl, domainFilter, validated.max_urls);
+        const sitemapUrls = await this.fetchSitemapUrls(baseUrl, domainFilter, poolLimit, scopePath);
         sitemapUrls.forEach(url => urls.add(normalizeUrl(url)));
       }
 
       // Fetch and parse the main page for additional URLs
       const pageUrls = await this.fetchPageUrls(validated.url, domainFilter, identity);
       pageUrls.forEach(url => {
-        if (urls.size < validated.max_urls) {
+        if (urls.size < poolLimit) {
           urls.add(normalizeUrl(url));
         }
       });
 
+      let pool = Array.from(urls);
+      if (scopePath) {
+        const inScope = pool.filter((u) => { try { return new URL(u).pathname.startsWith(scopePath); } catch { return false; } });
+        if (inScope.length > 0) {
+          pool = inScope;
+        } else {
+          warnings.push(`No URL under ${scopePath} was found in the sitemap or on the page; the whole site is listed instead.`);
+        }
+      }
+      if (validated.search) {
+        // Stable: relevance first, discovery order among equals.
+        pool = pool
+          .map((url, i) => ({ url, i, score: searchScore(url, validated.search) }))
+          .sort((a, b) => b.score - a.score || a.i - b.i)
+          .map((x) => x.url);
+      }
+
       // Convert to array and limit
-      const urlArray = Array.from(urls).slice(0, validated.max_urls);
+      const urlArray = pool.slice(0, validated.max_urls);
 
       // Fetch metadata if requested
       if (validated.include_metadata) {
@@ -142,7 +205,9 @@ export class MapSiteTool {
         site_map: this.generateSiteMap(urlArray),
         statistics: this.generateStatistics(urlArray),
         domain_filter_config: domainFilter ? domainFilter.exportConfig() : null,
-        filter_stats: domainFilter ? domainFilter.getStats() : null
+        filter_stats: domainFilter ? domainFilter.getStats() : null,
+        ...(scopePath ? { scope: scopePath } : {}),
+        ...(warnings.length ? { warnings } : {})
       };
 
       // Optional: rank URLs by relevance to a search string
@@ -157,7 +222,11 @@ export class MapSiteTool {
             return { link: url, title, snippet: '' };
           });
           const ranked = await getRanker().rankResults(rankerInput, validated.search);
-          result.ranked_urls = ranked.map(r => ({ url: r.link, score: r.finalScore ?? 0 }));
+          // The ranker's score is flat across bare URLs; the path-term count
+          // is what separates /yell/planyourvisit/fees.htm from the rest.
+          result.ranked_urls = ranked
+            .map(r => ({ url: r.link, score: Number(((r.finalScore ?? 0) + searchScore(r.link, validated.search)).toFixed(3)) }))
+            .sort((a, b) => b.score - a.score);
         } catch {
           // ranking is best-effort; don't fail the whole call
           result.ranked_urls = urlArray.map(u => ({ url: u, score: 0 }));
@@ -193,7 +262,11 @@ export class MapSiteTool {
     });
   }
 
-  async fetchSitemapUrls(baseUrl, domainFilter = null, maxUrls = Infinity) {
+  async fetchSitemapUrls(baseUrl, domainFilter = null, maxUrls = Infinity, scopePath = null) {
+    const inScope = (url) => {
+      if (!scopePath) return true;
+      try { return new URL(url).pathname.startsWith(scopePath); } catch { return false; }
+    };
     // Discover sitemaps via robots.txt and common paths, then parse with full
     // SitemapParser support (sitemap-index recursion, gzip, CDATA/entities).
     const discovered = await this.sitemapParser.discoverSitemaps(baseUrl, {
@@ -212,6 +285,7 @@ export class MapSiteTool {
         if (parsed.success) {
           for (const entry of parsed.urls) {
             const url = entry.loc || entry;
+            if (!inScope(url)) continue;
             if (!domainFilter || domainFilter.isAllowed(url).allowed) {
               urls.add(url);
             }
@@ -267,7 +341,7 @@ export class MapSiteTool {
       // A gate refusal is the answer to the request, not a page we failed to
       // read: surface it instead of returning an emptier map than the caller
       // would notice.
-      if (error.code === 'ROBOTS_DISALLOWED' || error.code === 'HOST_BLOCKED') throw error;
+      if (error.code === 'ROBOTS_DISALLOWED' || error.code === 'HOST_BLOCKED' || error.code === 'USE_REDDIT_SEARCH') throw error;
       return [];
     }
   }

package/src/tools/extract/extractEmbeddedState.js

@@ -41,7 +41,23 @@ export async function extractEmbeddedStateHandler({ url, path, user_agent, respe
       );
     }
 
-    const data = path ? selectJsonPath(state.data, path) : state.data;
+    // A path is naturally written against the payload ("props.pageProps"),
+    // not against this tool's envelope ("next_data.props.pageProps"). When
+    // the page carries exactly one payload and the path's root is not one of
+    // the envelope keys, read it inside that payload and say so (R21,
+    // 2026-09-09: four Next.js pages in a row failed on the bare path).
+    let effectivePath = path;
+    if (path && state.found.length === 1) {
+      const root = path.split(/[.[]/)[0];
+      if (root && !(root in state.data)) {
+        effectivePath = `${state.found[0].name}.${path}`;
+        warnings.push(
+          `path "${path}" was read as "${effectivePath}": "${state.found[0].name}" is the only payload on this page, so the path is resolved inside it.`
+        );
+      }
+    }
+
+    const data = effectivePath ? selectJsonPath(state.data, effectivePath) : state.data;
     const bytes = Buffer.byteLength(JSON.stringify(data) ?? '');
 
     if (!path && bytes > LARGE_RESULT_BYTES) {
@@ -57,7 +73,7 @@ export async function extractEmbeddedStateHandler({ url, path, user_agent, respe
         text: JSON.stringify({
           url: finalUrl,
           found: state.found,
-          path: path || null,
+          path: effectivePath || null,
           bytes,
           data,
           warnings