mcp-unknowncheatz 0.3.0

package/src/tools/bulk-get-threads.ts

@@ -0,0 +1,288 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
+import { fetchHtml } from "../crawl.js";
+import { parseThread } from "../parsers/thread.js";
+import { parseCodeBlocks } from "../parsers/code-blocks.js";
+import { validateUrl } from "../browser.js";
+import type { ThreadPost } from "../types.js";
+import type { AuthorReputation } from "../parsers/reputation.js";
+
+const MAX_URLS = 20;
+const MAX_PAGES_PER_THREAD = 10;
+const MAX_CONTENT_CHARS_PER_POST = 4_000;
+const MAX_CODE_CHARS = 3_000;
+
+function truncate(text: string, limit: number): string {
+  if (text.length <= limit) return text;
+  return `${text.slice(0, limit)}\n... [truncated, ${text.length} chars total]`;
+}
+
+function buildPageUrl(baseUrl: string, page: number): string {
+  const url = new URL(baseUrl);
+  url.searchParams.set("page", String(page));
+  return url.toString();
+}
+
+interface AuthorAgg {
+  author: string;
+  bestReputation: AuthorReputation | null;
+  postCount: number;
+  hasNegative: boolean;
+}
+
+function aggregateAuthors(posts: ThreadPost[]): AuthorAgg[] {
+  const map = new Map<string, AuthorAgg>();
+
+  for (const post of posts) {
+    if (!post.author) continue;
+    const existing = map.get(post.author) ?? {
+      author: post.author,
+      bestReputation: null,
+      postCount: 0,
+      hasNegative: false,
+    };
+    existing.postCount += 1;
+
+    const rep = post.reputation ?? null;
+    if (rep) {
+      if (rep.negativeDots > 0 || rep.sign === "negative") existing.hasNegative = true;
+      if (!existing.bestReputation || (rep.trustScore > existing.bestReputation.trustScore)) {
+        existing.bestReputation = rep;
+      }
+    }
+
+    map.set(post.author, existing);
+  }
+
+  return [...map.values()].sort((a, b) => {
+    const at = a.bestReputation?.trustScore ?? 0;
+    const bt = b.bestReputation?.trustScore ?? 0;
+    return bt - at;
+  });
+}
+
+export function registerBulkGetThreads(server: McpServer): void {
+  server.tool(
+    "bulk_get_threads",
+    "Fetch multiple UC threads (cached + rate-limited). Includes author reputation, trust scores, and OP-rep filters so untrustworthy threads can be skipped.",
+    {
+      urls: z
+        .array(z.string().url())
+        .min(1)
+        .max(MAX_URLS)
+        .describe(`Thread URLs to fetch (max ${MAX_URLS})`),
+      include_posts: z
+        .boolean()
+        .optional()
+        .default(true)
+        .describe("Include post content (default true). Set false for a summary-only crawl."),
+      include_code: z
+        .boolean()
+        .optional()
+        .default(true)
+        .describe("Extract code blocks per thread (default true)"),
+      code_limit_per_thread: z
+        .number()
+        .int()
+        .min(1)
+        .max(20)
+        .optional()
+        .default(5)
+        .describe("Max code blocks returned per thread (default 5)"),
+      fetch_all_pages: z
+        .boolean()
+        .optional()
+        .default(false)
+        .describe(`If true, fetch every page of each thread (cap ${MAX_PAGES_PER_THREAD})`),
+      post_content_chars: z
+        .number()
+        .int()
+        .min(200)
+        .max(20_000)
+        .optional()
+        .default(MAX_CONTENT_CHARS_PER_POST)
+        .describe("Max characters of body text kept per post"),
+      min_op_rep: z
+        .number()
+        .int()
+        .optional()
+        .describe("Skip threads whose original poster has reputation below this value"),
+      exclude_negative_op: z
+        .boolean()
+        .optional()
+        .default(false)
+        .describe("Skip threads whose original poster has any negative-rep dot"),
+      min_op_trust: z
+        .number()
+        .int()
+        .min(0)
+        .max(100)
+        .optional()
+        .describe("Skip threads whose OP trust score (0-100) is below this"),
+    },
+    async ({
+      urls,
+      include_posts,
+      include_code,
+      code_limit_per_thread,
+      fetch_all_pages,
+      post_content_chars,
+      min_op_rep,
+      exclude_negative_op,
+      min_op_trust,
+    }) => {
+      const results: unknown[] = [];
+      const skipped: Array<{ url: string; reason: string; opAuthor?: string; opReputation?: AuthorReputation }> = [];
+      let successCount = 0;
+      let errorCount = 0;
+
+      for (const url of urls) {
+        try {
+          validateUrl(url);
+
+          const firstHtml = await fetchHtml(url);
+          const first = parseThread(firstHtml, url, 1);
+          const opPost = first.posts[0];
+          const opRep = opPost?.reputation ?? null;
+
+          if (opRep) {
+            if (exclude_negative_op && (opRep.negativeDots > 0 || opRep.sign === "negative")) {
+              skipped.push({
+                url,
+                reason: "OP has negative reputation",
+                opAuthor: opPost?.author,
+                opReputation: opRep,
+              });
+              continue;
+            }
+            if (min_op_rep !== undefined && (opRep.score ?? 0) < min_op_rep) {
+              skipped.push({
+                url,
+                reason: `OP rep ${opRep.score ?? "unknown"} < min_op_rep ${min_op_rep}`,
+                opAuthor: opPost?.author,
+                opReputation: opRep,
+              });
+              continue;
+            }
+            if (min_op_trust !== undefined && opRep.trustScore < min_op_trust) {
+              skipped.push({
+                url,
+                reason: `OP trustScore ${opRep.trustScore} < min_op_trust ${min_op_trust}`,
+                opAuthor: opPost?.author,
+                opReputation: opRep,
+              });
+              continue;
+            }
+          }
+
+          let allPosts: ThreadPost[] = [...first.posts];
+          const pagesFetched: number[] = [1];
+
+          if (fetch_all_pages && first.totalPages > 1) {
+            const limit = Math.min(first.totalPages, MAX_PAGES_PER_THREAD);
+            for (let pageNum = 2; pageNum <= limit; pageNum++) {
+              const pageUrl = buildPageUrl(url, pageNum);
+              try {
+                const pageHtml = await fetchHtml(pageUrl);
+                const parsed = parseThread(pageHtml, pageUrl, pageNum);
+                allPosts.push(...parsed.posts);
+                pagesFetched.push(pageNum);
+              } catch (pageErr) {
+                console.error(`[bulk] Page ${pageNum} of ${url} failed:`, pageErr);
+                break;
+              }
+            }
+          }
+
+          const authors = aggregateAuthors(allPosts);
+
+          const threadResult: Record<string, unknown> = {
+            url,
+            title: first.title,
+            currentPagesFetched: pagesFetched,
+            totalPages: first.totalPages,
+            postCount: allPosts.length,
+            op: opPost
+              ? {
+                  author: opPost.author,
+                  date: opPost.date,
+                  reputation: opRep,
+                }
+              : null,
+            authorScoring: {
+              uniqueAuthors: authors.length,
+              topAuthors: authors.slice(0, 5).map((a) => ({
+                author: a.author,
+                postCount: a.postCount,
+                trustScore: a.bestReputation?.trustScore ?? 0,
+                tier: a.bestReputation?.tier ?? "unknown",
+                sign: a.bestReputation?.sign ?? "unknown",
+                score: a.bestReputation?.score ?? null,
+                description: a.bestReputation?.description,
+              })),
+              flaggedAuthors: authors.filter((a) => a.hasNegative).map((a) => a.author),
+            },
+          };
+
+          if (include_posts) {
+            threadResult.posts = allPosts.map((post) => ({
+              author: post.author,
+              date: post.date,
+              postNumber: post.postNumber,
+              content: truncate(post.content, post_content_chars),
+              linkCount: post.links.length,
+              imageCount: post.images.length,
+              reputation: post.reputation
+                ? {
+                    score: post.reputation.score,
+                    power: post.reputation.power,
+                    sign: post.reputation.sign,
+                    tier: post.reputation.tier,
+                    trustScore: post.reputation.trustScore,
+                    description: post.reputation.description,
+                    positiveDots: post.reputation.positiveDots,
+                    highPositiveDots: post.reputation.highPositiveDots,
+                    negativeDots: post.reputation.negativeDots,
+                  }
+                : null,
+            }));
+          }
+
+          if (include_code) {
+            const codeBlocks = parseCodeBlocks(firstHtml)
+              .slice(0, code_limit_per_thread)
+              .map((block) => ({
+                ...block,
+                code: truncate(block.code, MAX_CODE_CHARS),
+              }));
+            threadResult.codeBlocks = codeBlocks;
+            threadResult.codeBlockCount = codeBlocks.length;
+          }
+
+          results.push(threadResult);
+          successCount++;
+        } catch (err) {
+          const message = err instanceof Error ? err.message : String(err);
+          results.push({ url, error: message });
+          errorCount++;
+        }
+      }
+
+      return {
+        content: [
+          {
+            type: "text",
+            text: JSON.stringify({
+              requested: urls.length,
+              succeeded: successCount,
+              failed: errorCount,
+              skipped: skipped.length,
+              skippedDetail: skipped,
+              threads: results,
+            }),
+          },
+        ],
+      };
+    }
+  );
+}

package/src/tools/cache-control.ts

@@ -0,0 +1,24 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
+import { clearCache, getCacheStats } from "../crawl.js";
+
+export function registerCacheControl(server: McpServer): void {
+  server.tool(
+    "crawl_cache",
+    "Inspect or clear the in-memory HTML cache used by crawler tools.",
+    {
+      action: z.enum(["stats", "clear"]).default("stats").describe("'stats' to inspect, 'clear' to purge"),
+    },
+    async ({ action }) => {
+      if (action === "clear") {
+        const removed = clearCache();
+        return {
+          content: [{ type: "text", text: JSON.stringify({ ok: true, removed }) }],
+        };
+      }
+      return {
+        content: [{ type: "text", text: JSON.stringify(getCacheStats()) }],
+      };
+    }
+  );
+}

package/src/tools/check-login.ts

@@ -0,0 +1,41 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { load } from "cheerio";
+import { navigateWithRetry } from "../browser.js";
+import { isLoggedIn } from "../auth.js";
+
+const UC_HOME = "https://www.unknowncheats.me/forum/";
+
+export function registerCheckLogin(server: McpServer): void {
+  server.tool("check_login", "Check if the browser session is logged into UnknownCheats", {}, async () => {
+    try {
+      const { html } = await navigateWithRetry(UC_HOME);
+      const $ = load(html);
+
+      const loggedIn = isLoggedIn(html);
+
+      let username: string | undefined;
+      if (loggedIn) {
+        // Try common vBulletin welcome selectors
+        const welcomeEl = $("#welcomelink, .welcomelink, #userlinks .bolds").first();
+        const welcomeText = welcomeEl.text().trim();
+        const match = welcomeText.match(/Welcome,?\s+(.+)/i);
+        if (match) {
+          username = match[1].replace(/[!.]+$/, "").trim();
+        } else {
+          // Fallback: grab username from user CP link text
+          username = $('a[href*="usercp.php"]').first().text().trim() || undefined;
+        }
+      }
+
+      return {
+        content: [{ type: "text", text: JSON.stringify({ loggedIn, username }) }],
+      };
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      return {
+        content: [{ type: "text", text: `Error: ${message}` }],
+        isError: true,
+      };
+    }
+  });
+}

package/src/tools/crawl-subforum.ts

@@ -0,0 +1,128 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
+import { fetchHtml } from "../crawl.js";
+import {
+  filterThreads,
+  parsePaginationInfo,
+  parseThreadList,
+  type ThreadListEntry,
+} from "../parsers/thread-list.js";
+
+const HARD_PAGE_CAP = 30;
+
+function buildPageUrl(subforum: string, page: number): string {
+  const base = `https://www.unknowncheats.me/forum/${subforum}/`;
+  if (page <= 1) return base;
+  return `${base}index${page}.html`;
+}
+
+export function registerCrawlSubforum(server: McpServer): void {
+  server.tool(
+    "crawl_subforum",
+    "Walk multiple pages of a subforum thread listing with filters (query, min_replies, min_views, author, prefix). Great for building a corpus of relevant threads.",
+    {
+      subforum: z.string().describe("Subforum slug (e.g. 'apex-legends'). Use list_subforums to discover."),
+      max_pages: z
+        .number()
+        .int()
+        .min(1)
+        .max(HARD_PAGE_CAP)
+        .optional()
+        .default(3)
+        .describe(`How many pages of the subforum to walk (max ${HARD_PAGE_CAP}, default 3)`),
+      query: z.string().optional().describe("Optional keyword filter on title/snippet (AND semantics)"),
+      min_replies: z.number().int().min(0).optional().describe("Only include threads with at least this many replies"),
+      min_views: z.number().int().min(0).optional().describe("Only include threads with at least this many views"),
+      author: z.string().optional().describe("Only include threads by this author (case-insensitive substring)"),
+      prefix: z.string().optional().describe("Only include threads with this prefix label"),
+      include_sticky: z.boolean().optional().default(false).describe("Include sticky/announcement threads (default false)"),
+      sort_by: z
+        .enum(["default", "replies", "views", "title"])
+        .optional()
+        .default("default")
+        .describe("Sort the collected results locally"),
+      limit: z.number().int().min(1).max(500).optional().default(100).describe("Max threads returned after filtering/sorting"),
+    },
+    async ({ subforum, max_pages, query, min_replies, min_views, author, prefix, include_sticky, sort_by, limit }) => {
+      try {
+        const pagesWalked: number[] = [];
+        const collected: ThreadListEntry[] = [];
+        const seen = new Set<string>();
+        let totalPagesAvailable = 1;
+
+        for (let page = 1; page <= max_pages; page++) {
+          const url = buildPageUrl(subforum, page);
+          let html: string;
+          try {
+            html = await fetchHtml(url);
+          } catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            console.error(`[crawl-subforum] Failed page ${page}: ${message}`);
+            break;
+          }
+
+          const pageInfo = parsePaginationInfo(html);
+          totalPagesAvailable = pageInfo.totalPages;
+          pagesWalked.push(page);
+
+          const threads = parseThreadList(html);
+          for (const thread of threads) {
+            if (seen.has(thread.url)) continue;
+            seen.add(thread.url);
+            collected.push(thread);
+          }
+
+          if (page >= totalPagesAvailable) break;
+        }
+
+        const filtered = filterThreads(collected, {
+          query,
+          minReplies: min_replies,
+          minViews: min_views,
+          author,
+          prefix,
+          includeSticky: include_sticky,
+        });
+
+        const sorted = [...filtered];
+        if (sort_by === "replies") sorted.sort((a, b) => b.replies - a.replies);
+        else if (sort_by === "views") sorted.sort((a, b) => b.views - a.views);
+        else if (sort_by === "title") sorted.sort((a, b) => a.title.localeCompare(b.title));
+
+        const capped = sorted.slice(0, limit);
+
+        return {
+          content: [
+            {
+              type: "text",
+              text: JSON.stringify({
+                subforum,
+                pagesWalked,
+                totalPagesAvailable,
+                collected: collected.length,
+                matched: filtered.length,
+                returned: capped.length,
+                filter: {
+                  query,
+                  min_replies,
+                  min_views,
+                  author,
+                  prefix,
+                  include_sticky,
+                  sort_by,
+                },
+                results: capped,
+              }),
+            },
+          ],
+        };
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return {
+          content: [{ type: "text", text: `Error: ${message}` }],
+          isError: true,
+        };
+      }
+    }
+  );
+}

package/src/tools/debug-page.ts

@@ -0,0 +1,102 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
+import { load } from "cheerio";
+import { navigateWithRetry } from "../browser.js";
+
+export function registerDebugPage(server: McpServer): void {
+  server.tool(
+    "debug_page",
+    "Fetch a page and return its HTML snippet + key element selectors for debugging",
+    {
+      url: z.string().url().describe("URL to inspect"),
+      selector: z.string().optional().describe("CSS selector to extract (returns matched outerHTML)"),
+    },
+    async ({ url, selector }) => {
+      try {
+        const { html } = await navigateWithRetry(url);
+        const $ = load(html);
+
+        if (selector) {
+          const matched: string[] = [];
+          $(selector)
+            .slice(0, 5)
+            .each((_, el) => {
+              matched.push($.html(el)?.slice(0, 500) ?? "");
+            });
+          return {
+            content: [
+              {
+                type: "text",
+                text: JSON.stringify({ selector, count: $(selector).length, samples: matched }),
+              },
+            ],
+          };
+        }
+
+        // Return structural overview: tag names + classes of top-level body children
+        const structure: string[] = [];
+        $("body")
+          .children()
+          .slice(0, 30)
+          .each((_, el) => {
+            const e = $(el);
+            const id = e.attr("id") ? `#${e.attr("id")}` : "";
+            const cls = e.attr("class")
+              ? "." +
+                e
+                  .attr("class")!
+                  .trim()
+                  .split(/\s+/)
+                  .slice(0, 3)
+                  .join(".")
+              : "";
+            structure.push(`<${el.tagName}${id}${cls}>`);
+          });
+
+        // Also grab title and first post candidate
+        const titleCandidates = [
+          { sel: "h1", text: $("h1").first().text().trim() },
+          { sel: ".threadtitle", text: $(".threadtitle").first().text().trim() },
+          { sel: "#pagetitle", text: $("#pagetitle").first().text().trim() },
+          { sel: "title", text: $("title").first().text().trim() },
+        ];
+
+        const postCandidates = [
+          { sel: ".postcontainer", count: $(".postcontainer").length },
+          { sel: ".postbitlegacy", count: $(".postbitlegacy").length },
+          { sel: "li[id^='post_']", count: $("li[id^='post_']").length },
+          { sel: "div[id^='post_']", count: $("div[id^='post_']").length },
+          { sel: ".message", count: $(".message").length },
+          { sel: "article", count: $("article").length },
+          { sel: ".post", count: $(".post").length },
+          { sel: "td.alt1", count: $("td.alt1").length },
+        ];
+
+        const paginationCandidates = [
+          { sel: ".pagination", html: $(".pagination").first().html()?.slice(0, 300) },
+          { sel: ".pagenav", html: $(".pagenav").first().html()?.slice(0, 300) },
+          { sel: "nav", html: $("nav").first().html()?.slice(0, 300) },
+        ];
+
+        return {
+          content: [
+            {
+              type: "text",
+              text: JSON.stringify(
+                { structure, titleCandidates, postCandidates, paginationCandidates },
+                null,
+                2
+              ),
+            },
+          ],
+        };
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return {
+          content: [{ type: "text", text: `Error: ${message}` }],
+          isError: true,
+        };
+      }
+    }
+  );
+}