crawlforge-mcp-server 5.0.5 → 5.2.0

package/src/tools/search/adapters/redditOfficialApi.js

@@ -0,0 +1,196 @@
+/**
+ * Reddit Official Data API adapter (app-only OAuth)
+ *
+ * The optional, fully ToS-compliant live path for reddit_search. When a user
+ * supplies THEIR OWN Reddit app credentials (REDDIT_CLIENT_ID /
+ * REDDIT_CLIENT_SECRET), reddit_search can read Reddit's own API directly
+ * instead of a community archive — giving live scores, complete comment trees,
+ * and up-to-the-minute listings, on the user's own 100-QPM free quota.
+ *
+ * Auth: "Application Only OAuth" (client_credentials grant). This needs only a
+ * client id + secret from a Reddit "script" or "web app" registered at
+ * https://www.reddit.com/prefs/apps — no Reddit username/password, because we
+ * only read public data. The token endpoint (www.reddit.com/api/v1/access_token)
+ * is the OAuth server and is NOT behind Reddit's anti-scraper wall; all data
+ * calls then go to https://oauth.reddit.com with the bearer token.
+ *
+ * Deliberate scope: the official API can serve `posts` search/listings and
+ * `thread` reads. It has NO comment full-text search (that was Pushshift's
+ * superpower), and its listings/search cannot filter by an arbitrary date
+ * range — only coarse `t` buckets. Those two cases are surfaced as errors so
+ * the caller falls back to the archives, which do support them.
+ */
+
+import { normalizePost, normalizeTreeNodes, stripIdPrefix, stripNamePrefix } from '../redditNormalize.js';
+
+const TOKEN_URL = 'https://www.reddit.com/api/v1/access_token';
+const API_BASE = 'https://oauth.reddit.com';
+
+/** Reddit requires a descriptive, unique User-Agent; generic ones are throttled. */
+const DEFAULT_USER_AGENT = 'CrawlForge-MCP/5.2.0 (+https://www.crawlforge.dev)';
+
+/** Our sort is asc/desc by post date; Reddit listings/search only go newest-first. */
+const REDDIT_SORT = 'new';
+
+export class RedditOfficialApiAdapter {
+  constructor(clientId, clientSecret, options = {}) {
+    if (!clientId || !clientSecret) {
+      throw new Error('Reddit API credentials are required (client id + secret).');
+    }
+    this.clientId = clientId;
+    this.clientSecret = clientSecret;
+    this.userAgent = options.userAgent || process.env.REDDIT_USER_AGENT || DEFAULT_USER_AGENT;
+    this.tokenUrl = options.tokenUrl || TOKEN_URL;
+    this.apiBaseUrl = options.apiBaseUrl || API_BASE;
+    this.timeoutMs = options.timeoutMs ?? 30000;
+    this.authHeader = 'Basic ' + Buffer.from(`${clientId}:${clientSecret}`).toString('base64');
+    // Cached app-only token: { value, expiresAt(ms epoch) }.
+    this._token = null;
+  }
+
+  /** Fetch (or reuse) an app-only bearer token. Refreshed a minute before expiry. */
+  async #getToken(force = false) {
+    if (!force && this._token && Date.now() < this._token.expiresAt) {
+      return this._token.value;
+    }
+    let response;
+    try {
+      response = await fetch(this.tokenUrl, {
+        method: 'POST',
+        headers: {
+          Authorization: this.authHeader,
+          'Content-Type': 'application/x-www-form-urlencoded',
+          'User-Agent': this.userAgent,
+        },
+        body: 'grant_type=client_credentials',
+        signal: AbortSignal.timeout(this.timeoutMs),
+      });
+    } catch (error) {
+      if (error.name === 'TimeoutError' || error.name === 'AbortError') {
+        throw new Error(`Reddit token request timed out after ${this.timeoutMs}ms`);
+      }
+      throw new Error(`Reddit token network error: ${error.message}`);
+    }
+    if (!response.ok) {
+      // 401 here means the client id/secret are wrong or the app was revoked.
+      const hint = response.status === 401
+        ? ' — check REDDIT_CLIENT_ID / REDDIT_CLIENT_SECRET (and that the app is a "script" or "web app")'
+        : '';
+      throw new Error(`Reddit OAuth failed: HTTP ${response.status} ${response.statusText}${hint}`);
+    }
+    const data = await response.json();
+    if (!data.access_token) throw new Error('Reddit OAuth returned no access_token');
+    const ttlMs = (Number(data.expires_in) || 3600) * 1000;
+    this._token = { value: data.access_token, expiresAt: Date.now() + ttlMs - 60000 };
+    return this._token.value;
+  }
+
+  /** Authenticated GET against oauth.reddit.com, with one token-refresh retry on 401. */
+  async #get(path, queryParams) {
+    const url = `${this.apiBaseUrl}${path}?${new URLSearchParams({ raw_json: '1', ...queryParams })}`;
+    for (let attempt = 0; attempt < 2; attempt++) {
+      const token = await this.#getToken(attempt > 0); // force refresh on the retry
+      let response;
+      try {
+        response = await fetch(url, {
+          headers: { Authorization: `Bearer ${token}`, 'User-Agent': this.userAgent },
+          signal: AbortSignal.timeout(this.timeoutMs),
+        });
+      } catch (error) {
+        if (error.name === 'TimeoutError' || error.name === 'AbortError') {
+          throw new Error(`Reddit API request timed out after ${this.timeoutMs}ms`);
+        }
+        throw new Error(`Reddit API network error: ${error.message}`);
+      }
+      if (response.status === 401 && attempt === 0) continue; // stale token → refresh + retry once
+      if (response.status === 429) {
+        const reset = response.headers?.get?.('x-ratelimit-reset');
+        throw new Error(`Reddit API rate limited (429)${reset ? `, retry in ${reset}s` : ''} — you are over your app's 100 QPM quota`);
+      }
+      if (!response.ok) {
+        throw new Error(`Reddit API HTTP ${response.status} ${response.statusText}`);
+      }
+      return response.json();
+    }
+    // Unreachable in practice: the loop returns or throws on each path.
+    throw new Error('Reddit API authorization failed after token refresh');
+  }
+
+  /** Data is authoritative and live — say so, and drop the archive caveats. */
+  #notes() {
+    return [
+      'Data from the official Reddit Data API (oauth.reddit.com) via your configured Reddit app credentials — live and authoritative (real scores, complete comment trees).',
+      'Consumes your Reddit app\'s own 100 QPM free-tier quota, not CrawlForge credits.',
+    ];
+  }
+
+  /**
+   * Search/list posts. Mirrors the archive path's contract; validation of the
+   * required-scope rules happens in redditSearch.js before we get here.
+   */
+  async searchPosts(v, { subreddit, author }) {
+    // The official API cannot honor an arbitrary date range (only coarse `t`
+    // buckets). Reject so `auto` mode falls back to the archives, which can.
+    if (v.after || v.before) {
+      throw new Error('official Reddit API cannot filter by date range — unset after/before, or use source:"arctic_shift"/"pullpush"');
+    }
+    const sr = stripNamePrefix(subreddit);
+    const au = stripNamePrefix(author);
+    const limit = String(v.limit);
+
+    let path;
+    let query;
+    if (v.query) {
+      const q = au ? `author:${au} ${v.query}` : v.query;
+      if (sr) {
+        path = `/r/${encodeURIComponent(sr)}/search`;
+        query = { q, restrict_sr: 'true', sort: REDDIT_SORT, type: 'link', limit };
+      } else {
+        path = '/search';
+        query = { q, sort: REDDIT_SORT, type: 'link', limit };
+      }
+    } else if (au && !sr) {
+      path = `/user/${encodeURIComponent(au)}/submitted`;
+      query = { sort: REDDIT_SORT, limit };
+    } else if (sr && au) {
+      path = `/r/${encodeURIComponent(sr)}/search`;
+      query = { q: `author:${au}`, restrict_sr: 'true', sort: REDDIT_SORT, type: 'link', limit };
+    } else {
+      path = `/r/${encodeURIComponent(sr)}/new`;
+      query = { limit };
+    }
+
+    const data = await this.#get(path, query);
+    const children = Array.isArray(data?.data?.children) ? data.data.children : [];
+    let results = children
+      .filter((c) => c?.kind === 't3' && c.data)
+      .map((c) => normalizePost(c.data));
+    // Reddit only returns newest-first; approximate ascending by reversing the page.
+    if (v.sort === 'asc') results = results.reverse();
+
+    return {
+      source: 'reddit_api', mode: 'posts',
+      query: v.query ?? null, subreddit: sr ?? null, author: au ?? null,
+      count: results.length, results,
+      after_cursor: data?.data?.after ?? null,
+      notes: this.#notes(), checkedAt: new Date().toISOString(),
+    };
+  }
+
+  /** Read a post plus its nested comment tree from the official API. */
+  async getThread(v) {
+    const id = stripIdPrefix(v.link_id);
+    const data = await this.#get(`/comments/${encodeURIComponent(id)}`, { limit: String(v.limit) });
+    // Reddit returns [postListing, commentsListing].
+    const postChild = data?.[0]?.data?.children?.[0];
+    const post = postChild?.data ? normalizePost(postChild.data) : null;
+    const comments = normalizeTreeNodes(data?.[1]?.data?.children);
+    return {
+      source: 'reddit_api', mode: 'thread', link_id: id,
+      post, comments, comment_count: comments.length,
+      notes: this.#notes(), checkedAt: new Date().toISOString(),
+    };
+  }
+}
+
+export default RedditOfficialApiAdapter;

package/src/tools/search/redditNormalize.js

@@ -0,0 +1,95 @@
+/**
+ * Shared Reddit result normalizers.
+ *
+ * Both the community-archive path (redditSearch.js) and the official Reddit
+ * Data API adapter (adapters/redditOfficialApi.js) reduce raw Reddit post,
+ * comment, and comment-tree objects to one stable output shape. Keeping the
+ * normalizers here lets both reuse them without a circular import.
+ *
+ * All three sources (Arctic Shift, PullPush, and Reddit's own API) store the
+ * same raw Reddit object fields, and Arctic Shift deliberately mirrors Reddit's
+ * `/comments` tree shape — so a single set of normalizers covers all of them.
+ */
+
+/** Cap selftext/body length so a 100-result payload stays LLM-friendly. */
+export const TEXT_MAX = 2000;
+
+/** "t3_abc123" / "t1_abc123" → "abc123" (all backends accept bare IDs). */
+export function stripIdPrefix(id) {
+  return String(id).replace(/^t[13]_/, '');
+}
+
+/** "r/Foo" → "Foo", "u/bar" → "bar". */
+export function stripNamePrefix(name) {
+  return name == null ? name : String(name).replace(/^[ru]\//, '');
+}
+
+export function truncate(text) {
+  if (typeof text !== 'string' || text.length <= TEXT_MAX) {
+    return { text: text ?? null, truncated: false };
+  }
+  return { text: text.slice(0, TEXT_MAX), truncated: true };
+}
+
+export function toIso(epochSeconds) {
+  return typeof epochSeconds === 'number'
+    ? new Date(epochSeconds * 1000).toISOString()
+    : null;
+}
+
+/** Raw Reddit post object → the stable subset that matters. */
+export function normalizePost(raw) {
+  const { text: selftext, truncated } = truncate(raw.selftext);
+  return {
+    id: raw.id ?? null,
+    title: raw.title ?? null,
+    author: raw.author ?? null,
+    subreddit: raw.subreddit ?? null,
+    created_utc: raw.created_utc ?? null,
+    created_iso: toIso(raw.created_utc),
+    score: raw.score ?? null,
+    num_comments: raw.num_comments ?? null,
+    upvote_ratio: raw.upvote_ratio ?? null,
+    flair: raw.link_flair_text ?? null,
+    over_18: raw.over_18 ?? null,
+    selftext,
+    selftext_truncated: truncated,
+    url: raw.url ?? null,
+    permalink: raw.permalink ? `https://www.reddit.com${raw.permalink}` : null,
+  };
+}
+
+export function normalizeComment(raw) {
+  const { text: body, truncated } = truncate(raw.body);
+  return {
+    id: raw.id ?? null,
+    author: raw.author ?? null,
+    subreddit: raw.subreddit ?? null,
+    created_utc: raw.created_utc ?? null,
+    created_iso: toIso(raw.created_utc),
+    score: raw.score ?? null,
+    body,
+    body_truncated: truncated,
+    link_id: raw.link_id ? stripIdPrefix(raw.link_id) : null,
+    parent_id: raw.parent_id ?? null,
+    permalink: raw.permalink ? `https://www.reddit.com${raw.permalink}` : null,
+  };
+}
+
+/**
+ * Reddit-API-style comment tree → nested {..comment, replies:[...]} shape.
+ * Nodes look like {kind:"t1", data:{...comment, replies:{data:{children:[...]}}}}
+ * and {kind:"more", data:{count, children:[ids]}} for collapsed branches.
+ * Both Arctic Shift's /api/comments/tree and Reddit's /comments/{id} use this.
+ */
+export function normalizeTreeNodes(nodes) {
+  if (!Array.isArray(nodes)) return [];
+  return nodes.map((node) => {
+    if (node?.kind === 'more') {
+      return { more_count: node.data?.count ?? null, more_ids: node.data?.children ?? [] };
+    }
+    const data = node?.data ?? {};
+    const children = data.replies?.data?.children;
+    return { ...normalizeComment(data), replies: normalizeTreeNodes(children) };
+  });
+}

package/src/tools/search/redditSearch.js

@@ -0,0 +1,326 @@
+/**
+ * reddit_search tool
+ *
+ * Searches Reddit posts and comments, or reads a full comment thread.
+ *
+ * reddit.com hard-blocks non-browser clients (403 on fetch, stealth browsers
+ * included — the block is IP/TLS-reputation based), so this tool never touches
+ * reddit.com. It queries the two community-run Reddit archives instead:
+ *
+ * - Arctic Shift (https://arctic-shift.photon-reddit.com) — near-real-time
+ *   ingestion, comment trees, richer endpoints. Constraint from its API docs:
+ *   keyword search (`query`/`body`) only works when scoped to a subreddit,
+ *   author, or post — NOT across all of Reddit.
+ * - PullPush (https://api.pullpush.io) — Pushshift-compatible, supports
+ *   cross-subreddit full-text search, but has known post-2023 archive gaps
+ *   and recurring outages.
+ *
+ * Routing: scoped searches go to Arctic Shift (fresher) with PullPush as an
+ * error-only fallback; unscoped keyword searches can only go to PullPush.
+ * Both services are free and need no credentials.
+ *
+ * Optional official-API path: if the user sets REDDIT_CLIENT_ID and
+ * REDDIT_CLIENT_SECRET (their own Reddit app), posts/thread requests can read
+ * Reddit's official Data API (live scores, complete comment trees) on their own
+ * free quota, preferred in `auto` mode with the archives as fallback. Absent
+ * those vars — the default — this tool never touches reddit.com. Comment
+ * full-text search and date-range filters always use the archives (the official
+ * API supports neither). See adapters/redditOfficialApi.js and
+ * docs/reddit-access-and-oauth.md.
+ */
+
+import { z } from 'zod';
+import {
+  normalizePost,
+  normalizeComment,
+  normalizeTreeNodes,
+  stripIdPrefix,
+  stripNamePrefix,
+} from './redditNormalize.js';
+import { RedditOfficialApiAdapter } from './adapters/redditOfficialApi.js';
+
+const ARCTIC_SHIFT_BASE = 'https://arctic-shift.photon-reddit.com';
+const PULLPUSH_BASE = 'https://api.pullpush.io';
+
+/**
+ * Identify ourselves. Verified live: Arctic Shift throttles UA-less clients
+ * into a shared bucket (422 "Timeout. Maybe slow down a bit" while curl got
+ * 200 for the same URL); with a descriptive UA it answers instantly.
+ */
+const USER_AGENT = 'CrawlForge-MCP/5.2.0 (+https://www.crawlforge.dev)';
+
+const RedditSearchSchema = z.object({
+  query: z.string().min(1).optional(),
+  subreddit: z.string().min(1).optional(),
+  author: z.string().min(1).optional(),
+  mode: z.enum(['posts', 'comments', 'thread']).optional().default('posts'),
+  link_id: z.string().min(1).optional(), // post ID — required for thread mode
+  after: z.string().min(1).optional(),
+  before: z.string().min(1).optional(),
+  limit: z.number().int().min(1).max(100).optional().default(25),
+  sort: z.enum(['asc', 'desc']).optional().default('desc'),
+  source: z.enum(['auto', 'arctic_shift', 'pullpush', 'reddit_api']).optional().default('auto'),
+});
+
+/**
+ * PullPush (Pushshift schema) wants epoch seconds for after/before. Pass
+ * through epoch and offset forms ("7d"); convert ISO dates. Arctic Shift
+ * accepts all of these natively, so this is only used on the PullPush path.
+ */
+function toEpochSeconds(value) {
+  if (/^\d+$/.test(value) || /^\d+[a-z]+$/i.test(value)) return value;
+  const ms = Date.parse(value);
+  if (Number.isNaN(ms)) {
+    throw new Error(`Unparseable date "${value}" — use ISO 8601, epoch seconds, or an offset like "7d"`);
+  }
+  return String(Math.floor(ms / 1000));
+}
+
+export class RedditSearchTool {
+  constructor(options = {}) {
+    // Overridable for tests / self-hosted mirrors.
+    this.arcticBaseUrl = options.arcticBaseUrl || ARCTIC_SHIFT_BASE;
+    this.pullpushBaseUrl = options.pullpushBaseUrl || PULLPUSH_BASE;
+    // Community services with no SLA — generous but bounded.
+    this.timeoutMs = options.timeoutMs ?? (Number(process.env.REDDIT_SEARCH_TIMEOUT_MS) || 30000);
+    // Pause before the single retry of a transient throttle response.
+    this.retryDelayMs = options.retryDelayMs ?? 3000;
+
+    // Optional official-API path. When the user supplies THEIR OWN Reddit app
+    // credentials, reddit_search can read Reddit's own API (live, authoritative)
+    // instead of the archives. Absent (the default), the tool is unchanged.
+    this.redditClientId = options.redditClientId || process.env.REDDIT_CLIENT_ID || null;
+    this.redditClientSecret = options.redditClientSecret || process.env.REDDIT_CLIENT_SECRET || null;
+    this.officialConfigured = Boolean(this.redditClientId && this.redditClientSecret);
+    // Lazily constructed on first official-path use; overridable for tests.
+    this._officialAdapter = options.officialAdapter || null;
+  }
+
+  /** The official-API adapter, built once from the configured credentials. */
+  #official() {
+    if (!this._officialAdapter) {
+      this._officialAdapter = new RedditOfficialApiAdapter(
+        this.redditClientId,
+        this.redditClientSecret,
+        { timeoutMs: this.timeoutMs },
+      );
+    }
+    return this._officialAdapter;
+  }
+
+  async execute(params) {
+    const v = RedditSearchSchema.parse(params);
+
+    // Cross-field validation zod can't express per-mode.
+    if (v.mode === 'thread') {
+      if (!v.link_id) throw new Error('thread mode requires link_id (the post ID, e.g. "1twm1zh" or "t3_1twm1zh")');
+    } else if (!v.query && !v.subreddit && !v.author && !(v.mode === 'comments' && v.link_id)) {
+      throw new Error(`${v.mode} mode requires at least one of: query, subreddit, author${v.mode === 'comments' ? ', link_id' : ''}`);
+    }
+
+    const subreddit = stripNamePrefix(v.subreddit);
+    const author = stripNamePrefix(v.author);
+    // Arctic Shift keyword search must be scoped (its documented constraint).
+    const scoped = Boolean(subreddit || author || (v.mode === 'comments' && v.link_id));
+    const arcticPossible = v.mode === 'thread' || !v.query || scoped;
+    // The official API serves posts search/listings and thread reads. It has no
+    // comment full-text search, so `comments` mode always uses the archives.
+    const officialPossible = this.officialConfigured && (v.mode === 'thread' || v.mode === 'posts');
+
+    let order; // backends to try, in order
+    if (v.source === 'reddit_api') {
+      if (!this.officialConfigured) {
+        throw new Error('source:"reddit_api" needs Reddit app credentials — set REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET (create a "script" app at https://www.reddit.com/prefs/apps)');
+      }
+      if (v.mode === 'comments') {
+        throw new Error('the official Reddit API has no comment full-text search — use comments mode with source:"arctic_shift"/"pullpush", or read a whole thread with mode:"thread"');
+      }
+      order = ['reddit_api'];
+    } else if (v.source === 'arctic_shift') {
+      if (!arcticPossible) {
+        throw new Error('Arctic Shift cannot keyword-search across all of Reddit — add a subreddit or author scope, or use source:"pullpush"');
+      }
+      order = ['arctic_shift'];
+    } else if (v.source === 'pullpush') {
+      if (v.mode === 'thread') throw new Error('thread mode requires Arctic Shift (source:"pullpush" only supports posts/comments search)');
+      order = ['pullpush'];
+    } else {
+      // auto: prefer the user's own official API (live, authoritative) when it
+      // can serve this request, then fall back to the community archives.
+      const archives = v.mode === 'thread' ? ['arctic_shift']
+        : arcticPossible ? ['arctic_shift', 'pullpush']
+        : ['pullpush'];
+      order = officialPossible ? ['reddit_api', ...archives] : archives;
+    }
+
+    const errors = [];
+    for (const source of order) {
+      try {
+        let result;
+        if (source === 'reddit_api') {
+          result = v.mode === 'thread'
+            ? await this.#official().getThread(v)
+            : await this.#official().searchPosts(v, { subreddit, author });
+        } else if (source === 'arctic_shift') {
+          result = await this.#searchArcticShift(v, { subreddit, author });
+        } else {
+          result = await this.#searchPullPush(v, { subreddit, author });
+        }
+        if (errors.length > 0) result.fallback_used = `primary source failed (${errors[0]}), fell back to ${source}`;
+        return result;
+      } catch (error) {
+        errors.push(`${source}: ${error.message}`);
+      }
+    }
+    // Unscoped keyword searches have no Arctic Shift fallback (it requires a
+    // scope — verified live: HTTP 400 without one), so point at the fix.
+    const hint = order.length === 1 && order[0] === 'pullpush' && v.source === 'auto'
+      ? ' Tip: add a subreddit or author filter to route to the more reliable Arctic Shift archive.'
+      : '';
+    throw new Error(`All Reddit sources failed — ${errors.join('; ')}.${hint}`);
+  }
+
+  async #searchArcticShift(v, { subreddit, author }) {
+    const notes = [
+      'Data from the Arctic Shift community archive (arctic-shift.photon-reddit.com), not reddit.com (which blocks scrapers).',
+      'Scores and comment counts of content less than ~36h old may read 0/1 — the archive captures content the moment it is posted.',
+    ];
+
+    if (v.mode === 'thread') {
+      const id = stripIdPrefix(v.link_id);
+      const postData = await this.#get(`${this.arcticBaseUrl}/api/posts/ids`, { ids: id });
+      const post = postData.data?.[0] ? normalizePost(postData.data[0]) : null;
+      const treeData = await this.#get(`${this.arcticBaseUrl}/api/comments/tree`, {
+        link_id: `t3_${id}`,
+        limit: String(v.limit),
+      });
+      const comments = normalizeTreeNodes(treeData.data);
+      return {
+        source: 'arctic_shift', mode: 'thread', link_id: id,
+        post, comments, comment_count: comments.length,
+        notes, checkedAt: new Date().toISOString(),
+      };
+    }
+
+    const query = {
+      limit: String(v.limit),
+      sort: v.sort,
+      ...(subreddit && { subreddit }),
+      ...(author && { author }),
+      ...(v.after && { after: v.after }),
+      ...(v.before && { before: v.before }),
+    };
+    let path;
+    if (v.mode === 'posts') {
+      path = '/api/posts/search';
+      if (v.query) query.query = v.query; // searches title + selftext
+    } else {
+      path = '/api/comments/search';
+      if (v.query) query.body = v.query;
+      if (v.link_id) query.link_id = stripIdPrefix(v.link_id);
+    }
+    const data = await this.#get(`${this.arcticBaseUrl}${path}`, query);
+    const rows = Array.isArray(data.data) ? data.data : [];
+    const results = v.mode === 'posts' ? rows.map(normalizePost) : rows.map(normalizeComment);
+    return {
+      source: 'arctic_shift', mode: v.mode,
+      query: v.query ?? null, subreddit: subreddit ?? null, author: author ?? null,
+      count: results.length, results,
+      notes, checkedAt: new Date().toISOString(),
+    };
+  }
+
+  async #searchPullPush(v, { subreddit, author }) {
+    if (v.mode === 'thread') throw new Error('thread mode is Arctic Shift only');
+    const notes = [
+      'Data from the PullPush community archive (api.pullpush.io), not reddit.com (which blocks scrapers).',
+      'PullPush has known gaps in its post-2023 archive — an empty result does not prove the content does not exist.',
+    ];
+    const query = {
+      size: String(v.limit),
+      sort: v.sort,
+      sort_type: 'created_utc',
+      ...(v.query && { q: v.query }),
+      ...(subreddit && { subreddit }),
+      ...(author && { author }),
+      ...(v.after && { after: toEpochSeconds(v.after) }),
+      ...(v.before && { before: toEpochSeconds(v.before) }),
+    };
+    if (v.mode === 'comments' && v.link_id) query.link_id = stripIdPrefix(v.link_id);
+    const path = v.mode === 'posts' ? '/reddit/search/submission/' : '/reddit/search/comment/';
+    const data = await this.#get(`${this.pullpushBaseUrl}${path}`, query);
+    if (data.error) throw new Error(`PullPush error: ${data.error}`);
+    const rows = Array.isArray(data.data) ? data.data : [];
+    const results = v.mode === 'posts' ? rows.map(normalizePost) : rows.map(normalizeComment);
+    return {
+      source: 'pullpush', mode: v.mode,
+      query: v.query ?? null, subreddit: subreddit ?? null, author: author ?? null,
+      count: results.length, results,
+      notes, checkedAt: new Date().toISOString(),
+    };
+  }
+
+  /**
+   * GET with one bounded retry: both archives shed load transiently (PullPush
+   * 429s at ~15 req/min; Arctic Shift answers 422 "Timeout. Maybe slow down a
+   * bit" under per-IP pressure — observed live) and usually recover in seconds.
+   */
+  async #get(base, queryParams) {
+    const url = `${base}?${new URLSearchParams(queryParams)}`;
+    let lastError;
+    for (let attempt = 0; attempt < 2; attempt++) {
+      if (attempt > 0) await new Promise((resolve) => setTimeout(resolve, this.retryDelayMs));
+      try {
+        return await this.#getOnce(url);
+      } catch (error) {
+        lastError = error;
+        if (!error.retryable) throw error;
+      }
+    }
+    throw lastError;
+  }
+
+  async #getOnce(url) {
+    let response;
+    try {
+      response = await fetch(url, {
+        headers: { Accept: 'application/json', 'User-Agent': USER_AGENT },
+        signal: AbortSignal.timeout(this.timeoutMs),
+      });
+    } catch (error) {
+      if (error.name === 'TimeoutError' || error.name === 'AbortError') {
+        throw new Error(`request timed out after ${this.timeoutMs}ms`);
+      }
+      throw new Error(`network error: ${error.message}`);
+    }
+    if (response.status === 429) {
+      const reset = response.headers?.get?.('x-ratelimit-reset');
+      // PullPush's 429 body states its actual policy ("does not provide free
+      // scraping resources for agents...") — pass that through verbatim.
+      let detail = '';
+      try { detail = (await response.json())?.error ?? ''; } catch { /* no body */ }
+      throw Object.assign(
+        new Error(`rate limited (429)${reset ? `, retry in ${reset}s` : ''}${detail ? ` — ${detail}` : ''}`),
+        { retryable: true },
+      );
+    }
+    if (!response.ok) {
+      // Both archives put the real reason in the body (e.g. Arctic Shift's
+      // throttle/parameter complaints) — surface it, bounded.
+      let detail = '';
+      try {
+        const body = await response.text();
+        let msg = body.slice(0, 200);
+        try { msg = JSON.parse(body)?.error || msg; } catch { /* non-JSON body — use it raw */ }
+        detail = msg ? `: ${msg}` : '';
+      } catch { /* unreadable body — status alone will have to do */ }
+      throw Object.assign(
+        new Error(`HTTP ${response.status} ${response.statusText}${detail}`),
+        { retryable: response.status === 422 && /timeout|slow down/i.test(detail) },
+      );
+    }
+    return response.json();
+  }
+}
+
+export default RedditSearchTool;

package/src/tools/templates/ScrapeTemplateTool.js

@@ -6,7 +6,7 @@
  *   const result = await tool.execute({ template: "github-repo", url: "https://github.com/user/repo" });
  */
 
-import { TemplateRegistry } from './TemplateRegistry.js';
+import { TemplateRegistry } from 'crawlforge-extractors';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 
 export class ScrapeTemplateTool {
@@ -35,12 +35,17 @@ export class ScrapeTemplateTool {
       throw new Error(`Unknown template "${template}". Available templates: ${available}`);
     }
 
+    // A template may redirect its own fetch to a machine-readable endpoint
+    // (shopify-product reads /products/<handle>.json). Same host either way,
+    // so the SSRF guard below still applies.
+    const fetchUrl = template === 'list' ? url : (tpl.resolveUrl ? tpl.resolveUrl(url) : url);
+
     // Fetch the page
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), timeout);
     let html;
     try {
-      const response = await safeFetch(url, {
+      const response = await safeFetch(fetchUrl, {
         signal: controller.signal,
         headers: {
           'User-Agent': 'Mozilla/5.0 (compatible; CrawlForge-TemplateScraper/4.0)'
@@ -61,7 +66,7 @@ export class ScrapeTemplateTool {
     }
 
     // Run the template extractor
-    const result = await this.registry.run(template, html, url);
+    const result = await this.registry.run(template, html, url, fetchUrl);
     return result;
   }
 }