crawlforge-mcp-server 5.1.0 → 5.2.0

package/src/tools/search/adapters/redditOfficialApi.js

@@ -0,0 +1,196 @@
+/**
+ * Reddit Official Data API adapter (app-only OAuth)
+ *
+ * The optional, fully ToS-compliant live path for reddit_search. When a user
+ * supplies THEIR OWN Reddit app credentials (REDDIT_CLIENT_ID /
+ * REDDIT_CLIENT_SECRET), reddit_search can read Reddit's own API directly
+ * instead of a community archive — giving live scores, complete comment trees,
+ * and up-to-the-minute listings, on the user's own 100-QPM free quota.
+ *
+ * Auth: "Application Only OAuth" (client_credentials grant). This needs only a
+ * client id + secret from a Reddit "script" or "web app" registered at
+ * https://www.reddit.com/prefs/apps — no Reddit username/password, because we
+ * only read public data. The token endpoint (www.reddit.com/api/v1/access_token)
+ * is the OAuth server and is NOT behind Reddit's anti-scraper wall; all data
+ * calls then go to https://oauth.reddit.com with the bearer token.
+ *
+ * Deliberate scope: the official API can serve `posts` search/listings and
+ * `thread` reads. It has NO comment full-text search (that was Pushshift's
+ * superpower), and its listings/search cannot filter by an arbitrary date
+ * range — only coarse `t` buckets. Those two cases are surfaced as errors so
+ * the caller falls back to the archives, which do support them.
+ */
+
+import { normalizePost, normalizeTreeNodes, stripIdPrefix, stripNamePrefix } from '../redditNormalize.js';
+
+const TOKEN_URL = 'https://www.reddit.com/api/v1/access_token';
+const API_BASE = 'https://oauth.reddit.com';
+
+/** Reddit requires a descriptive, unique User-Agent; generic ones are throttled. */
+const DEFAULT_USER_AGENT = 'CrawlForge-MCP/5.2.0 (+https://www.crawlforge.dev)';
+
+/** Our sort is asc/desc by post date; Reddit listings/search only go newest-first. */
+const REDDIT_SORT = 'new';
+
+export class RedditOfficialApiAdapter {
+  constructor(clientId, clientSecret, options = {}) {
+    if (!clientId || !clientSecret) {
+      throw new Error('Reddit API credentials are required (client id + secret).');
+    }
+    this.clientId = clientId;
+    this.clientSecret = clientSecret;
+    this.userAgent = options.userAgent || process.env.REDDIT_USER_AGENT || DEFAULT_USER_AGENT;
+    this.tokenUrl = options.tokenUrl || TOKEN_URL;
+    this.apiBaseUrl = options.apiBaseUrl || API_BASE;
+    this.timeoutMs = options.timeoutMs ?? 30000;
+    this.authHeader = 'Basic ' + Buffer.from(`${clientId}:${clientSecret}`).toString('base64');
+    // Cached app-only token: { value, expiresAt(ms epoch) }.
+    this._token = null;
+  }
+
+  /** Fetch (or reuse) an app-only bearer token. Refreshed a minute before expiry. */
+  async #getToken(force = false) {
+    if (!force && this._token && Date.now() < this._token.expiresAt) {
+      return this._token.value;
+    }
+    let response;
+    try {
+      response = await fetch(this.tokenUrl, {
+        method: 'POST',
+        headers: {
+          Authorization: this.authHeader,
+          'Content-Type': 'application/x-www-form-urlencoded',
+          'User-Agent': this.userAgent,
+        },
+        body: 'grant_type=client_credentials',
+        signal: AbortSignal.timeout(this.timeoutMs),
+      });
+    } catch (error) {
+      if (error.name === 'TimeoutError' || error.name === 'AbortError') {
+        throw new Error(`Reddit token request timed out after ${this.timeoutMs}ms`);
+      }
+      throw new Error(`Reddit token network error: ${error.message}`);
+    }
+    if (!response.ok) {
+      // 401 here means the client id/secret are wrong or the app was revoked.
+      const hint = response.status === 401
+        ? ' — check REDDIT_CLIENT_ID / REDDIT_CLIENT_SECRET (and that the app is a "script" or "web app")'
+        : '';
+      throw new Error(`Reddit OAuth failed: HTTP ${response.status} ${response.statusText}${hint}`);
+    }
+    const data = await response.json();
+    if (!data.access_token) throw new Error('Reddit OAuth returned no access_token');
+    const ttlMs = (Number(data.expires_in) || 3600) * 1000;
+    this._token = { value: data.access_token, expiresAt: Date.now() + ttlMs - 60000 };
+    return this._token.value;
+  }
+
+  /** Authenticated GET against oauth.reddit.com, with one token-refresh retry on 401. */
+  async #get(path, queryParams) {
+    const url = `${this.apiBaseUrl}${path}?${new URLSearchParams({ raw_json: '1', ...queryParams })}`;
+    for (let attempt = 0; attempt < 2; attempt++) {
+      const token = await this.#getToken(attempt > 0); // force refresh on the retry
+      let response;
+      try {
+        response = await fetch(url, {
+          headers: { Authorization: `Bearer ${token}`, 'User-Agent': this.userAgent },
+          signal: AbortSignal.timeout(this.timeoutMs),
+        });
+      } catch (error) {
+        if (error.name === 'TimeoutError' || error.name === 'AbortError') {
+          throw new Error(`Reddit API request timed out after ${this.timeoutMs}ms`);
+        }
+        throw new Error(`Reddit API network error: ${error.message}`);
+      }
+      if (response.status === 401 && attempt === 0) continue; // stale token → refresh + retry once
+      if (response.status === 429) {
+        const reset = response.headers?.get?.('x-ratelimit-reset');
+        throw new Error(`Reddit API rate limited (429)${reset ? `, retry in ${reset}s` : ''} — you are over your app's 100 QPM quota`);
+      }
+      if (!response.ok) {
+        throw new Error(`Reddit API HTTP ${response.status} ${response.statusText}`);
+      }
+      return response.json();
+    }
+    // Unreachable in practice: the loop returns or throws on each path.
+    throw new Error('Reddit API authorization failed after token refresh');
+  }
+
+  /** Data is authoritative and live — say so, and drop the archive caveats. */
+  #notes() {
+    return [
+      'Data from the official Reddit Data API (oauth.reddit.com) via your configured Reddit app credentials — live and authoritative (real scores, complete comment trees).',
+      'Consumes your Reddit app\'s own 100 QPM free-tier quota, not CrawlForge credits.',
+    ];
+  }
+
+  /**
+   * Search/list posts. Mirrors the archive path's contract; validation of the
+   * required-scope rules happens in redditSearch.js before we get here.
+   */
+  async searchPosts(v, { subreddit, author }) {
+    // The official API cannot honor an arbitrary date range (only coarse `t`
+    // buckets). Reject so `auto` mode falls back to the archives, which can.
+    if (v.after || v.before) {
+      throw new Error('official Reddit API cannot filter by date range — unset after/before, or use source:"arctic_shift"/"pullpush"');
+    }
+    const sr = stripNamePrefix(subreddit);
+    const au = stripNamePrefix(author);
+    const limit = String(v.limit);
+
+    let path;
+    let query;
+    if (v.query) {
+      const q = au ? `author:${au} ${v.query}` : v.query;
+      if (sr) {
+        path = `/r/${encodeURIComponent(sr)}/search`;
+        query = { q, restrict_sr: 'true', sort: REDDIT_SORT, type: 'link', limit };
+      } else {
+        path = '/search';
+        query = { q, sort: REDDIT_SORT, type: 'link', limit };
+      }
+    } else if (au && !sr) {
+      path = `/user/${encodeURIComponent(au)}/submitted`;
+      query = { sort: REDDIT_SORT, limit };
+    } else if (sr && au) {
+      path = `/r/${encodeURIComponent(sr)}/search`;
+      query = { q: `author:${au}`, restrict_sr: 'true', sort: REDDIT_SORT, type: 'link', limit };
+    } else {
+      path = `/r/${encodeURIComponent(sr)}/new`;
+      query = { limit };
+    }
+
+    const data = await this.#get(path, query);
+    const children = Array.isArray(data?.data?.children) ? data.data.children : [];
+    let results = children
+      .filter((c) => c?.kind === 't3' && c.data)
+      .map((c) => normalizePost(c.data));
+    // Reddit only returns newest-first; approximate ascending by reversing the page.
+    if (v.sort === 'asc') results = results.reverse();
+
+    return {
+      source: 'reddit_api', mode: 'posts',
+      query: v.query ?? null, subreddit: sr ?? null, author: au ?? null,
+      count: results.length, results,
+      after_cursor: data?.data?.after ?? null,
+      notes: this.#notes(), checkedAt: new Date().toISOString(),
+    };
+  }
+
+  /** Read a post plus its nested comment tree from the official API. */
+  async getThread(v) {
+    const id = stripIdPrefix(v.link_id);
+    const data = await this.#get(`/comments/${encodeURIComponent(id)}`, { limit: String(v.limit) });
+    // Reddit returns [postListing, commentsListing].
+    const postChild = data?.[0]?.data?.children?.[0];
+    const post = postChild?.data ? normalizePost(postChild.data) : null;
+    const comments = normalizeTreeNodes(data?.[1]?.data?.children);
+    return {
+      source: 'reddit_api', mode: 'thread', link_id: id,
+      post, comments, comment_count: comments.length,
+      notes: this.#notes(), checkedAt: new Date().toISOString(),
+    };
+  }
+}
+
+export default RedditOfficialApiAdapter;

package/src/tools/search/redditNormalize.js

@@ -0,0 +1,95 @@
+/**
+ * Shared Reddit result normalizers.
+ *
+ * Both the community-archive path (redditSearch.js) and the official Reddit
+ * Data API adapter (adapters/redditOfficialApi.js) reduce raw Reddit post,
+ * comment, and comment-tree objects to one stable output shape. Keeping the
+ * normalizers here lets both reuse them without a circular import.
+ *
+ * All three sources (Arctic Shift, PullPush, and Reddit's own API) store the
+ * same raw Reddit object fields, and Arctic Shift deliberately mirrors Reddit's
+ * `/comments` tree shape — so a single set of normalizers covers all of them.
+ */
+
+/** Cap selftext/body length so a 100-result payload stays LLM-friendly. */
+export const TEXT_MAX = 2000;
+
+/** "t3_abc123" / "t1_abc123" → "abc123" (all backends accept bare IDs). */
+export function stripIdPrefix(id) {
+  return String(id).replace(/^t[13]_/, '');
+}
+
+/** "r/Foo" → "Foo", "u/bar" → "bar". */
+export function stripNamePrefix(name) {
+  return name == null ? name : String(name).replace(/^[ru]\//, '');
+}
+
+export function truncate(text) {
+  if (typeof text !== 'string' || text.length <= TEXT_MAX) {
+    return { text: text ?? null, truncated: false };
+  }
+  return { text: text.slice(0, TEXT_MAX), truncated: true };
+}
+
+export function toIso(epochSeconds) {
+  return typeof epochSeconds === 'number'
+    ? new Date(epochSeconds * 1000).toISOString()
+    : null;
+}
+
+/** Raw Reddit post object → the stable subset that matters. */
+export function normalizePost(raw) {
+  const { text: selftext, truncated } = truncate(raw.selftext);
+  return {
+    id: raw.id ?? null,
+    title: raw.title ?? null,
+    author: raw.author ?? null,
+    subreddit: raw.subreddit ?? null,
+    created_utc: raw.created_utc ?? null,
+    created_iso: toIso(raw.created_utc),
+    score: raw.score ?? null,
+    num_comments: raw.num_comments ?? null,
+    upvote_ratio: raw.upvote_ratio ?? null,
+    flair: raw.link_flair_text ?? null,
+    over_18: raw.over_18 ?? null,
+    selftext,
+    selftext_truncated: truncated,
+    url: raw.url ?? null,
+    permalink: raw.permalink ? `https://www.reddit.com${raw.permalink}` : null,
+  };
+}
+
+export function normalizeComment(raw) {
+  const { text: body, truncated } = truncate(raw.body);
+  return {
+    id: raw.id ?? null,
+    author: raw.author ?? null,
+    subreddit: raw.subreddit ?? null,
+    created_utc: raw.created_utc ?? null,
+    created_iso: toIso(raw.created_utc),
+    score: raw.score ?? null,
+    body,
+    body_truncated: truncated,
+    link_id: raw.link_id ? stripIdPrefix(raw.link_id) : null,
+    parent_id: raw.parent_id ?? null,
+    permalink: raw.permalink ? `https://www.reddit.com${raw.permalink}` : null,
+  };
+}
+
+/**
+ * Reddit-API-style comment tree → nested {..comment, replies:[...]} shape.
+ * Nodes look like {kind:"t1", data:{...comment, replies:{data:{children:[...]}}}}
+ * and {kind:"more", data:{count, children:[ids]}} for collapsed branches.
+ * Both Arctic Shift's /api/comments/tree and Reddit's /comments/{id} use this.
+ */
+export function normalizeTreeNodes(nodes) {
+  if (!Array.isArray(nodes)) return [];
+  return nodes.map((node) => {
+    if (node?.kind === 'more') {
+      return { more_count: node.data?.count ?? null, more_ids: node.data?.children ?? [] };
+    }
+    const data = node?.data ?? {};
+    const children = data.replies?.data?.children;
+    return { ...normalizeComment(data), replies: normalizeTreeNodes(children) };
+  });
+}

package/src/tools/search/redditSearch.js

@@ -18,9 +18,26 @@
  * Routing: scoped searches go to Arctic Shift (fresher) with PullPush as an
  * error-only fallback; unscoped keyword searches can only go to PullPush.
  * Both services are free and need no credentials.
+ *
+ * Optional official-API path: if the user sets REDDIT_CLIENT_ID and
+ * REDDIT_CLIENT_SECRET (their own Reddit app), posts/thread requests can read
+ * Reddit's official Data API (live scores, complete comment trees) on their own
+ * free quota, preferred in `auto` mode with the archives as fallback. Absent
+ * those vars — the default — this tool never touches reddit.com. Comment
+ * full-text search and date-range filters always use the archives (the official
+ * API supports neither). See adapters/redditOfficialApi.js and
+ * docs/reddit-access-and-oauth.md.
  */
 
 import { z } from 'zod';
+import {
+  normalizePost,
+  normalizeComment,
+  normalizeTreeNodes,
+  stripIdPrefix,
+  stripNamePrefix,
+} from './redditNormalize.js';
+import { RedditOfficialApiAdapter } from './adapters/redditOfficialApi.js';
 
 const ARCTIC_SHIFT_BASE = 'https://arctic-shift.photon-reddit.com';
 const PULLPUSH_BASE = 'https://api.pullpush.io';
@@ -30,10 +47,7 @@ const PULLPUSH_BASE = 'https://api.pullpush.io';
  * into a shared bucket (422 "Timeout. Maybe slow down a bit" while curl got
  * 200 for the same URL); with a descriptive UA it answers instantly.
  */
-const USER_AGENT = 'CrawlForge-MCP/5.1.0 (+https://www.crawlforge.dev)';
-
-/** Cap selftext/body length so a 100-result payload stays LLM-friendly. */
-const TEXT_MAX = 2000;
+const USER_AGENT = 'CrawlForge-MCP/5.2.0 (+https://www.crawlforge.dev)';
 
 const RedditSearchSchema = z.object({
   query: z.string().min(1).optional(),
@@ -45,19 +59,9 @@ const RedditSearchSchema = z.object({
   before: z.string().min(1).optional(),
   limit: z.number().int().min(1).max(100).optional().default(25),
   sort: z.enum(['asc', 'desc']).optional().default('desc'),
-  source: z.enum(['auto', 'arctic_shift', 'pullpush']).optional().default('auto'),
+  source: z.enum(['auto', 'arctic_shift', 'pullpush', 'reddit_api']).optional().default('auto'),
 });
 
-/** "t3_abc123" / "t1_abc123" → "abc123" (both archives accept bare IDs). */
-function stripIdPrefix(id) {
-  return String(id).replace(/^t[13]_/, '');
-}
-
-/** "r/Foo" → "Foo", "u/bar" → "bar" (Arctic Shift ignores prefixes; PullPush doesn't). */
-function stripNamePrefix(name) {
-  return name == null ? name : String(name).replace(/^[ru]\//, '');
-}
-
 /**
  * PullPush (Pushshift schema) wants epoch seconds for after/before. Pass
  * through epoch and offset forms ("7d"); convert ISO dates. Arctic Shift
@@ -72,76 +76,6 @@ function toEpochSeconds(value) {
   return String(Math.floor(ms / 1000));
 }
 
-function truncate(text) {
-  if (typeof text !== 'string' || text.length <= TEXT_MAX) {
-    return { text: text ?? null, truncated: false };
-  }
-  return { text: text.slice(0, TEXT_MAX), truncated: true };
-}
-
-function toIso(epochSeconds) {
-  return typeof epochSeconds === 'number'
-    ? new Date(epochSeconds * 1000).toISOString()
-    : null;
-}
-
-/** Both archives store raw Reddit post objects — reduce to the fields that matter. */
-function normalizePost(raw) {
-  const { text: selftext, truncated } = truncate(raw.selftext);
-  return {
-    id: raw.id ?? null,
-    title: raw.title ?? null,
-    author: raw.author ?? null,
-    subreddit: raw.subreddit ?? null,
-    created_utc: raw.created_utc ?? null,
-    created_iso: toIso(raw.created_utc),
-    score: raw.score ?? null,
-    num_comments: raw.num_comments ?? null,
-    upvote_ratio: raw.upvote_ratio ?? null,
-    flair: raw.link_flair_text ?? null,
-    over_18: raw.over_18 ?? null,
-    selftext,
-    selftext_truncated: truncated,
-    url: raw.url ?? null,
-    permalink: raw.permalink ? `https://www.reddit.com${raw.permalink}` : null,
-  };
-}
-
-function normalizeComment(raw) {
-  const { text: body, truncated } = truncate(raw.body);
-  return {
-    id: raw.id ?? null,
-    author: raw.author ?? null,
-    subreddit: raw.subreddit ?? null,
-    created_utc: raw.created_utc ?? null,
-    created_iso: toIso(raw.created_utc),
-    score: raw.score ?? null,
-    body,
-    body_truncated: truncated,
-    link_id: raw.link_id ? stripIdPrefix(raw.link_id) : null,
-    parent_id: raw.parent_id ?? null,
-    permalink: raw.permalink ? `https://www.reddit.com${raw.permalink}` : null,
-  };
-}
-
-/**
- * Arctic Shift's /api/comments/tree returns Reddit-API-style nodes:
- * {kind:"t1", data:{...comment, replies:{kind:"Listing", data:{children:[...]}}}}
- * and {kind:"more", data:{count, children:[ids]}} for collapsed branches.
- * Flatten to a nested {..comment, replies:[...]} shape.
- */
-function normalizeTreeNodes(nodes) {
-  if (!Array.isArray(nodes)) return [];
-  return nodes.map((node) => {
-    if (node?.kind === 'more') {
-      return { more_count: node.data?.count ?? null, more_ids: node.data?.children ?? [] };
-    }
-    const data = node?.data ?? {};
-    const children = data.replies?.data?.children;
-    return { ...normalizeComment(data), replies: normalizeTreeNodes(children) };
-  });
-}
-
 export class RedditSearchTool {
   constructor(options = {}) {
     // Overridable for tests / self-hosted mirrors.
@@ -151,6 +85,27 @@ export class RedditSearchTool {
     this.timeoutMs = options.timeoutMs ?? (Number(process.env.REDDIT_SEARCH_TIMEOUT_MS) || 30000);
     // Pause before the single retry of a transient throttle response.
     this.retryDelayMs = options.retryDelayMs ?? 3000;
+
+    // Optional official-API path. When the user supplies THEIR OWN Reddit app
+    // credentials, reddit_search can read Reddit's own API (live, authoritative)
+    // instead of the archives. Absent (the default), the tool is unchanged.
+    this.redditClientId = options.redditClientId || process.env.REDDIT_CLIENT_ID || null;
+    this.redditClientSecret = options.redditClientSecret || process.env.REDDIT_CLIENT_SECRET || null;
+    this.officialConfigured = Boolean(this.redditClientId && this.redditClientSecret);
+    // Lazily constructed on first official-path use; overridable for tests.
+    this._officialAdapter = options.officialAdapter || null;
+  }
+
+  /** The official-API adapter, built once from the configured credentials. */
+  #official() {
+    if (!this._officialAdapter) {
+      this._officialAdapter = new RedditOfficialApiAdapter(
+        this.redditClientId,
+        this.redditClientSecret,
+        { timeoutMs: this.timeoutMs },
+      );
+    }
+    return this._officialAdapter;
   }
 
   async execute(params) {
@@ -168,9 +123,20 @@ export class RedditSearchTool {
     // Arctic Shift keyword search must be scoped (its documented constraint).
     const scoped = Boolean(subreddit || author || (v.mode === 'comments' && v.link_id));
     const arcticPossible = v.mode === 'thread' || !v.query || scoped;
+    // The official API serves posts search/listings and thread reads. It has no
+    // comment full-text search, so `comments` mode always uses the archives.
+    const officialPossible = this.officialConfigured && (v.mode === 'thread' || v.mode === 'posts');
 
     let order; // backends to try, in order
-    if (v.source === 'arctic_shift') {
+    if (v.source === 'reddit_api') {
+      if (!this.officialConfigured) {
+        throw new Error('source:"reddit_api" needs Reddit app credentials — set REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET (create a "script" app at https://www.reddit.com/prefs/apps)');
+      }
+      if (v.mode === 'comments') {
+        throw new Error('the official Reddit API has no comment full-text search — use comments mode with source:"arctic_shift"/"pullpush", or read a whole thread with mode:"thread"');
+      }
+      order = ['reddit_api'];
+    } else if (v.source === 'arctic_shift') {
       if (!arcticPossible) {
         throw new Error('Arctic Shift cannot keyword-search across all of Reddit — add a subreddit or author scope, or use source:"pullpush"');
       }
@@ -179,17 +145,27 @@ export class RedditSearchTool {
       if (v.mode === 'thread') throw new Error('thread mode requires Arctic Shift (source:"pullpush" only supports posts/comments search)');
       order = ['pullpush'];
     } else {
-      order = v.mode === 'thread' ? ['arctic_shift']
+      // auto: prefer the user's own official API (live, authoritative) when it
+      // can serve this request, then fall back to the community archives.
+      const archives = v.mode === 'thread' ? ['arctic_shift']
         : arcticPossible ? ['arctic_shift', 'pullpush']
         : ['pullpush'];
+      order = officialPossible ? ['reddit_api', ...archives] : archives;
     }
 
     const errors = [];
     for (const source of order) {
       try {
-        const result = source === 'arctic_shift'
-          ? await this.#searchArcticShift(v, { subreddit, author })
-          : await this.#searchPullPush(v, { subreddit, author });
+        let result;
+        if (source === 'reddit_api') {
+          result = v.mode === 'thread'
+            ? await this.#official().getThread(v)
+            : await this.#official().searchPosts(v, { subreddit, author });
+        } else if (source === 'arctic_shift') {
+          result = await this.#searchArcticShift(v, { subreddit, author });
+        } else {
+          result = await this.#searchPullPush(v, { subreddit, author });
+        }
         if (errors.length > 0) result.fallback_used = `primary source failed (${errors[0]}), fell back to ${source}`;
         return result;
       } catch (error) {
@@ -201,7 +177,7 @@ export class RedditSearchTool {
     const hint = order.length === 1 && order[0] === 'pullpush' && v.source === 'auto'
       ? ' Tip: add a subreddit or author filter to route to the more reliable Arctic Shift archive.'
       : '';
-    throw new Error(`All Reddit archive sources failed — ${errors.join('; ')}.${hint}`);
+    throw new Error(`All Reddit sources failed — ${errors.join('; ')}.${hint}`);
   }
 
   async #searchArcticShift(v, { subreddit, author }) {

package/src/tools/templates/ScrapeTemplateTool.js

@@ -6,7 +6,7 @@
  *   const result = await tool.execute({ template: "github-repo", url: "https://github.com/user/repo" });
  */
 
-import { TemplateRegistry } from './TemplateRegistry.js';
+import { TemplateRegistry } from 'crawlforge-extractors';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 
 export class ScrapeTemplateTool {
@@ -35,12 +35,17 @@ export class ScrapeTemplateTool {
       throw new Error(`Unknown template "${template}". Available templates: ${available}`);
     }
 
+    // A template may redirect its own fetch to a machine-readable endpoint
+    // (shopify-product reads /products/<handle>.json). Same host either way,
+    // so the SSRF guard below still applies.
+    const fetchUrl = template === 'list' ? url : (tpl.resolveUrl ? tpl.resolveUrl(url) : url);
+
     // Fetch the page
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), timeout);
     let html;
     try {
-      const response = await safeFetch(url, {
+      const response = await safeFetch(fetchUrl, {
         signal: controller.signal,
         headers: {
           'User-Agent': 'Mozilla/5.0 (compatible; CrawlForge-TemplateScraper/4.0)'
@@ -61,7 +66,7 @@ export class ScrapeTemplateTool {
     }
 
     // Run the template extractor
-    const result = await this.registry.run(template, html, url);
+    const result = await this.registry.run(template, html, url, fetchUrl);
     return result;
   }
 }