mcp-unknowncheatz 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -2
- package/package.json +4 -3
- package/src/browser.ts +79 -26
- package/src/crawl.ts +55 -7
- package/src/forum-index.ts +261 -0
- package/src/index.ts +3 -1
- package/src/inspect-html.ts +31 -0
- package/src/offset-discovery.ts +22 -0
- package/src/search-fallback.ts +6 -3
- package/src/sync-index.ts +113 -0
- package/src/tools/bulk-get-threads.ts +24 -4
- package/src/tools/check-login.ts +3 -2
- package/src/tools/crawl-subforum.ts +14 -3
- package/src/tools/debug-page.ts +3 -2
- package/src/tools/download-file.ts +3 -2
- package/src/tools/extract-code.ts +3 -2
- package/src/tools/find-latest-offsets.ts +33 -17
- package/src/tools/forum-index.ts +66 -0
- package/src/tools/get-thread.ts +45 -12
- package/src/tools/get-user-reputation.ts +3 -2
- package/src/tools/list-subforums.ts +3 -2
- package/src/tools/login.ts +3 -2
- package/src/tools/search-forum.ts +21 -5
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { load } from "cheerio";
|
|
2
|
+
import { parsePaginationInfo, parseThreadList } from "./parsers/thread-list.js";
|
|
3
|
+
import { parseSubforums } from "./parsers/subforums.js";
|
|
4
|
+
import { parseThread } from "./parsers/thread.js";
|
|
5
|
+
|
|
6
|
+
export function inspectForumHtml(html: string) {
|
|
7
|
+
const $ = load(html);
|
|
8
|
+
const listing = parseThreadList(html);
|
|
9
|
+
const thread = parseThread(html, "https://www.unknowncheats.me/forum/showthread.php?t=0");
|
|
10
|
+
const subforums = parseSubforums(html);
|
|
11
|
+
const title = $("title").first().text().trim();
|
|
12
|
+
return {
|
|
13
|
+
title,
|
|
14
|
+
challengeDetected: /Just a moment|cf-browser-verification|Checking your browser/i.test(html),
|
|
15
|
+
likelyPage: thread.posts.length > 0 ? "thread" : listing.length > 0 ? "listing" : subforums.length > 0 ? "forum_index" : "unrecognized",
|
|
16
|
+
selectors: {
|
|
17
|
+
subforumLinks: $("a[href*='/forum/']").length,
|
|
18
|
+
threadTitleLinks: $("a[id^='thread_title_']").length,
|
|
19
|
+
postTables: $("table[id^='post']").length,
|
|
20
|
+
postMessages: $("div[id^='post_message_']").length,
|
|
21
|
+
pagination: $(".pagenav").length,
|
|
22
|
+
},
|
|
23
|
+
parsed: {
|
|
24
|
+
subforums: subforums.length,
|
|
25
|
+
threads: listing.length,
|
|
26
|
+
posts: thread.posts.length,
|
|
27
|
+
listingPages: parsePaginationInfo(html).totalPages,
|
|
28
|
+
threadPages: thread.totalPages,
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
}
|
package/src/offset-discovery.ts
CHANGED
|
@@ -6,9 +6,31 @@ export function normalizeName(value: string): string {
|
|
|
6
6
|
return value.toLowerCase().replace(/['’]/g, "").replace(/[^a-z0-9]+/g, " ").trim();
|
|
7
7
|
}
|
|
8
8
|
|
|
9
|
+
const GAME_FORUM_ALIASES: Record<string, string> = {
|
|
10
|
+
pubg: "playerunknown-s-battlegrounds",
|
|
11
|
+
"playerunknowns battlegrounds": "playerunknown-s-battlegrounds",
|
|
12
|
+
cs2: "counter-strike-2-a",
|
|
13
|
+
"counter strike 2": "counter-strike-2-a",
|
|
14
|
+
csgo: "counterstrike-global-offensive",
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
export function preferredForumSlug(queryText: string): string | undefined {
|
|
18
|
+
const query = normalizeName(queryText);
|
|
19
|
+
if (query.includes("pubg mobile")) return undefined;
|
|
20
|
+
return Object.entries(GAME_FORUM_ALIASES)
|
|
21
|
+
.sort(([a], [b]) => b.length - a.length)
|
|
22
|
+
.find(([name]) => query === name || query.startsWith(`${name} `) ||
|
|
23
|
+
query.endsWith(` ${name}`) || query.includes(` ${name} `))?.[1];
|
|
24
|
+
}
|
|
25
|
+
|
|
9
26
|
export function rankGameForums(game: string, forums: Subforum[]): Subforum[] {
|
|
10
27
|
const query = normalizeName(game);
|
|
11
28
|
if (!query) return [];
|
|
29
|
+
const alias = preferredForumSlug(game);
|
|
30
|
+
if (alias) {
|
|
31
|
+
const exact = forums.find((forum) => forum.slug === alias);
|
|
32
|
+
if (exact) return [exact];
|
|
33
|
+
}
|
|
12
34
|
const terms = query.split(" ");
|
|
13
35
|
return forums
|
|
14
36
|
.map((forum) => {
|
package/src/search-fallback.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { load } from "cheerio";
|
|
2
2
|
import { filterThreads, parseThreadList, type ThreadListEntry } from "./parsers/thread-list.js";
|
|
3
|
+
import { preferredForumSlug } from "./offset-discovery.js";
|
|
3
4
|
|
|
4
5
|
function discoverSubforumSlugs(html: string): Array<{ slug: string; label: string }> {
|
|
5
6
|
const $ = load(html);
|
|
@@ -26,11 +27,13 @@ function rankSubforums(
|
|
|
26
27
|
): Array<{ slug: string; label: string; score: number }> {
|
|
27
28
|
const terms = query.toLowerCase().split(/\s+/).filter(Boolean);
|
|
28
29
|
const slugGuess = query.trim().toLowerCase().replace(/\s+/g, "-");
|
|
30
|
+
const preferredSlug = preferredForumSlug(query);
|
|
29
31
|
|
|
30
32
|
const ranked = subforums.map((entry) => {
|
|
31
33
|
const haystack = `${entry.slug} ${entry.label}`.toLowerCase();
|
|
32
34
|
let score = terms.filter((term) => haystack.includes(term)).length;
|
|
33
35
|
if (entry.slug === slugGuess) score += 10;
|
|
36
|
+
if (entry.slug === preferredSlug) score += 20;
|
|
34
37
|
if (entry.slug.includes(slugGuess) || slugGuess.includes(entry.slug)) score += 3;
|
|
35
38
|
return { ...entry, score };
|
|
36
39
|
});
|
|
@@ -40,10 +43,10 @@ function rankSubforums(
|
|
|
40
43
|
|
|
41
44
|
export async function searchViaSubforums(
|
|
42
45
|
query: string,
|
|
43
|
-
fetchHtml: (url: string) => Promise<string
|
|
46
|
+
fetchHtml: (url: string) => Promise<string>,
|
|
47
|
+
knownSubforums?: Array<{ slug: string; label: string }>,
|
|
44
48
|
): Promise<{ results: ThreadListEntry[]; scannedSubforums: string[] }> {
|
|
45
|
-
const
|
|
46
|
-
const subforums = discoverSubforumSlugs(indexHtml);
|
|
49
|
+
const subforums = knownSubforums ?? discoverSubforumSlugs(await fetchHtml("https://www.unknowncheats.me/forum/index.php"));
|
|
47
50
|
const ranked = rankSubforums(subforums, query);
|
|
48
51
|
|
|
49
52
|
const candidates = ranked.length > 0
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { fetchHtml } from "./crawl.js";
|
|
2
|
+
import { validateUrl } from "./browser.js";
|
|
3
|
+
import { ForumIndex } from "./forum-index.js";
|
|
4
|
+
import { parsePaginationInfo, parseThreadList, type ThreadListEntry } from "./parsers/thread-list.js";
|
|
5
|
+
import { parseThread } from "./parsers/thread.js";
|
|
6
|
+
import type { ThreadPost } from "./types.js";
|
|
7
|
+
|
|
8
|
+
export interface SyncResult {
|
|
9
|
+
subforum: string;
|
|
10
|
+
listingPages: number[];
|
|
11
|
+
threadsSeen: number;
|
|
12
|
+
threadsUpdated: number;
|
|
13
|
+
threadsSkipped: number;
|
|
14
|
+
postPagesFetched: number;
|
|
15
|
+
timeBudgetReached: boolean;
|
|
16
|
+
coverage: "first_and_recent_pages";
|
|
17
|
+
errors: Array<{ url: string; message: string }>;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function listingUrl(slug: string, page: number): string {
|
|
21
|
+
const base = `https://www.unknowncheats.me/forum/${slug}/`;
|
|
22
|
+
return page === 1 ? base : `${base}index${page}.html`;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function threadPageUrl(url: string, page: number): string {
|
|
26
|
+
const target = new URL(url);
|
|
27
|
+
target.searchParams.set("page", String(page));
|
|
28
|
+
return target.toString();
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export async function syncSubforumIndex(
|
|
32
|
+
index: ForumIndex,
|
|
33
|
+
slug: string,
|
|
34
|
+
options: { maxListingPages: number; maxThreads: number; recentPages: number; maxPostAgeMs?: number; deadlineAt?: number },
|
|
35
|
+
fetcher: (url: string) => Promise<string> = fetchHtml,
|
|
36
|
+
): Promise<SyncResult> {
|
|
37
|
+
if (!/^[a-z0-9][a-z0-9-]{1,80}$/.test(slug)) throw new Error("Invalid subforum slug");
|
|
38
|
+
const result: SyncResult = {
|
|
39
|
+
subforum: slug, listingPages: [], threadsSeen: 0, threadsUpdated: 0,
|
|
40
|
+
threadsSkipped: 0, postPagesFetched: 0, timeBudgetReached: false,
|
|
41
|
+
coverage: "first_and_recent_pages", errors: [],
|
|
42
|
+
};
|
|
43
|
+
const seen = new Map<string, ThreadListEntry>();
|
|
44
|
+
|
|
45
|
+
for (let page = 1; page <= options.maxListingPages; page++) {
|
|
46
|
+
if (options.deadlineAt !== undefined && Date.now() >= options.deadlineAt) {
|
|
47
|
+
result.timeBudgetReached = true;
|
|
48
|
+
break;
|
|
49
|
+
}
|
|
50
|
+
const url = listingUrl(slug, page);
|
|
51
|
+
try {
|
|
52
|
+
const html = await fetcher(url);
|
|
53
|
+
const entries = parseThreadList(html);
|
|
54
|
+
if (entries.length === 0) throw new Error("No thread links parsed");
|
|
55
|
+
const validEntries = entries.filter((entry) => {
|
|
56
|
+
try {
|
|
57
|
+
validateUrl(entry.url);
|
|
58
|
+
return true;
|
|
59
|
+
} catch {
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
if (validEntries.length === 0) throw new Error("No allowed thread URLs parsed");
|
|
64
|
+
index.recordListing(slug, page, validEntries);
|
|
65
|
+
for (const entry of validEntries) seen.set(entry.threadId, entry);
|
|
66
|
+
result.listingPages.push(page);
|
|
67
|
+
if (page >= parsePaginationInfo(html).totalPages) break;
|
|
68
|
+
} catch (error) {
|
|
69
|
+
if (options.deadlineAt !== undefined && Date.now() >= options.deadlineAt) result.timeBudgetReached = true;
|
|
70
|
+
result.errors.push({ url, message: error instanceof Error ? error.message : String(error) });
|
|
71
|
+
break;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
result.threadsSeen = seen.size;
|
|
76
|
+
let attempted = 0;
|
|
77
|
+
for (const entry of seen.values()) {
|
|
78
|
+
if (options.deadlineAt !== undefined && Date.now() >= options.deadlineAt) {
|
|
79
|
+
result.timeBudgetReached = true;
|
|
80
|
+
break;
|
|
81
|
+
}
|
|
82
|
+
if (attempted >= options.maxThreads) break;
|
|
83
|
+
if (!index.needsPosts(entry.threadId, options.maxPostAgeMs)) {
|
|
84
|
+
result.threadsSkipped++;
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
attempted++;
|
|
88
|
+
try {
|
|
89
|
+
const firstHtml = await fetcher(entry.url);
|
|
90
|
+
const first = parseThread(firstHtml, entry.url, 1);
|
|
91
|
+
if (first.posts.length === 0) throw new Error("No posts parsed from first page");
|
|
92
|
+
const pages: Array<{ page: number; posts: ThreadPost[] }> = [{ page: 1, posts: first.posts }];
|
|
93
|
+
const start = Math.max(2, first.totalPages - options.recentPages + 1);
|
|
94
|
+
for (let page = start; page <= first.totalPages; page++) {
|
|
95
|
+
if (options.deadlineAt !== undefined && Date.now() >= options.deadlineAt) {
|
|
96
|
+
result.timeBudgetReached = true;
|
|
97
|
+
throw new Error("Time budget reached before all sampled post pages were fetched");
|
|
98
|
+
}
|
|
99
|
+
const url = threadPageUrl(entry.url, page);
|
|
100
|
+
const parsed = parseThread(await fetcher(url), url, page);
|
|
101
|
+
if (parsed.posts.length === 0) throw new Error(`No posts parsed from page ${page}`);
|
|
102
|
+
pages.push({ page, posts: parsed.posts });
|
|
103
|
+
}
|
|
104
|
+
index.upsertPosts(entry.threadId, pages);
|
|
105
|
+
result.threadsUpdated++;
|
|
106
|
+
result.postPagesFetched += pages.length;
|
|
107
|
+
} catch (error) {
|
|
108
|
+
if (options.deadlineAt !== undefined && Date.now() >= options.deadlineAt) result.timeBudgetReached = true;
|
|
109
|
+
result.errors.push({ url: entry.url, message: error instanceof Error ? error.message : String(error) });
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return result;
|
|
113
|
+
}
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import { withBrowserSession } from "../browser.js";
|
|
1
2
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
3
|
import { z } from "zod";
|
|
3
4
|
import { fetchHtml } from "../crawl.js";
|
|
4
5
|
import { parseThread } from "../parsers/thread.js";
|
|
5
6
|
import { parseCodeBlocks } from "../parsers/code-blocks.js";
|
|
6
7
|
import { validateUrl } from "../browser.js";
|
|
8
|
+
import { getForumIndex } from "../forum-index.js";
|
|
7
9
|
import type { ThreadPost } from "../types.js";
|
|
8
10
|
import type { AuthorReputation } from "../parsers/reputation.js";
|
|
9
11
|
|
|
@@ -130,18 +132,27 @@ export function registerBulkGetThreads(server: McpServer): void {
|
|
|
130
132
|
min_op_rep,
|
|
131
133
|
exclude_negative_op,
|
|
132
134
|
min_op_trust,
|
|
133
|
-
}) => {
|
|
135
|
+
}) => withBrowserSession(async () => {
|
|
134
136
|
const results: unknown[] = [];
|
|
135
137
|
const skipped: Array<{ url: string; reason: string; opAuthor?: string; opReputation?: AuthorReputation }> = [];
|
|
136
138
|
let successCount = 0;
|
|
137
139
|
let errorCount = 0;
|
|
140
|
+
let processed = 0;
|
|
141
|
+
let timeBudgetReached = false;
|
|
142
|
+
const deadlineAt = Date.now() + 45_000;
|
|
138
143
|
|
|
139
144
|
for (const url of urls) {
|
|
145
|
+
if (Date.now() >= deadlineAt) {
|
|
146
|
+
timeBudgetReached = true;
|
|
147
|
+
break;
|
|
148
|
+
}
|
|
149
|
+
processed++;
|
|
140
150
|
try {
|
|
141
151
|
validateUrl(url);
|
|
142
152
|
|
|
143
|
-
const firstHtml = await fetchHtml(url);
|
|
153
|
+
const firstHtml = await fetchHtml(url, { deadlineAt });
|
|
144
154
|
const first = parseThread(firstHtml, url, 1);
|
|
155
|
+
getForumIndex().recordThreadPage(first, 1);
|
|
145
156
|
const opPost = first.posts[0];
|
|
146
157
|
const opRep = opPost?.reputation ?? null;
|
|
147
158
|
|
|
@@ -181,13 +192,19 @@ export function registerBulkGetThreads(server: McpServer): void {
|
|
|
181
192
|
if (fetch_all_pages && first.totalPages > 1) {
|
|
182
193
|
const limit = Math.min(first.totalPages, MAX_PAGES_PER_THREAD);
|
|
183
194
|
for (let pageNum = 2; pageNum <= limit; pageNum++) {
|
|
195
|
+
if (Date.now() >= deadlineAt) {
|
|
196
|
+
timeBudgetReached = true;
|
|
197
|
+
break;
|
|
198
|
+
}
|
|
184
199
|
const pageUrl = buildPageUrl(url, pageNum);
|
|
185
200
|
try {
|
|
186
|
-
const pageHtml = await fetchHtml(pageUrl);
|
|
201
|
+
const pageHtml = await fetchHtml(pageUrl, { deadlineAt });
|
|
187
202
|
const parsed = parseThread(pageHtml, pageUrl, pageNum);
|
|
203
|
+
getForumIndex().recordThreadPage(parsed, pageNum);
|
|
188
204
|
allPosts.push(...parsed.posts);
|
|
189
205
|
pagesFetched.push(pageNum);
|
|
190
206
|
} catch (pageErr) {
|
|
207
|
+
if (Date.now() >= deadlineAt) timeBudgetReached = true;
|
|
191
208
|
console.error(`[bulk] Page ${pageNum} of ${url} failed:`, pageErr);
|
|
192
209
|
break;
|
|
193
210
|
}
|
|
@@ -262,6 +279,7 @@ export function registerBulkGetThreads(server: McpServer): void {
|
|
|
262
279
|
results.push(threadResult);
|
|
263
280
|
successCount++;
|
|
264
281
|
} catch (err) {
|
|
282
|
+
if (Date.now() >= deadlineAt) timeBudgetReached = true;
|
|
265
283
|
const message = err instanceof Error ? err.message : String(err);
|
|
266
284
|
results.push({ url, error: message });
|
|
267
285
|
errorCount++;
|
|
@@ -276,6 +294,8 @@ export function registerBulkGetThreads(server: McpServer): void {
|
|
|
276
294
|
requested: urls.length,
|
|
277
295
|
succeeded: successCount,
|
|
278
296
|
failed: errorCount,
|
|
297
|
+
timeBudgetReached,
|
|
298
|
+
remainingUrls: urls.slice(processed),
|
|
279
299
|
skipped: skipped.length,
|
|
280
300
|
skippedDetail: skipped,
|
|
281
301
|
threads: results,
|
|
@@ -283,6 +303,6 @@ export function registerBulkGetThreads(server: McpServer): void {
|
|
|
283
303
|
},
|
|
284
304
|
],
|
|
285
305
|
};
|
|
286
|
-
}
|
|
306
|
+
})
|
|
287
307
|
);
|
|
288
308
|
}
|
package/src/tools/check-login.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { withBrowserSession } from "../browser.js";
|
|
1
2
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
3
|
import { load } from "cheerio";
|
|
3
4
|
import { navigateWithRetry } from "../browser.js";
|
|
@@ -6,7 +7,7 @@ import { isLoggedIn } from "../auth.js";
|
|
|
6
7
|
const UC_HOME = "https://www.unknowncheats.me/forum/";
|
|
7
8
|
|
|
8
9
|
export function registerCheckLogin(server: McpServer): void {
|
|
9
|
-
server.tool("check_login", "Check if the browser session is logged into UnknownCheats", {}, async () => {
|
|
10
|
+
server.tool("check_login", "Check if the browser session is logged into UnknownCheats", {}, async () => withBrowserSession(async () => {
|
|
10
11
|
try {
|
|
11
12
|
const { html } = await navigateWithRetry(UC_HOME);
|
|
12
13
|
const $ = load(html);
|
|
@@ -37,5 +38,5 @@ export function registerCheckLogin(server: McpServer): void {
|
|
|
37
38
|
isError: true,
|
|
38
39
|
};
|
|
39
40
|
}
|
|
40
|
-
});
|
|
41
|
+
}));
|
|
41
42
|
}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
import { withBrowserSession } from "../browser.js";
|
|
1
2
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
3
|
import { z } from "zod";
|
|
3
4
|
import { fetchHtml } from "../crawl.js";
|
|
5
|
+
import { getForumIndex } from "../forum-index.js";
|
|
4
6
|
import {
|
|
5
7
|
filterThreads,
|
|
6
8
|
parsePaginationInfo,
|
|
@@ -43,19 +45,26 @@ export function registerCrawlSubforum(server: McpServer): void {
|
|
|
43
45
|
.describe("Sort the collected results locally"),
|
|
44
46
|
limit: z.number().int().min(1).max(500).optional().default(100).describe("Max threads returned after filtering/sorting"),
|
|
45
47
|
},
|
|
46
|
-
async ({ subforum, max_pages, query, min_replies, min_views, author, prefix, include_sticky, sort_by, limit }) => {
|
|
48
|
+
async ({ subforum, max_pages, query, min_replies, min_views, author, prefix, include_sticky, sort_by, limit }) => withBrowserSession(async () => {
|
|
47
49
|
try {
|
|
50
|
+
const deadlineAt = Date.now() + 45_000;
|
|
51
|
+
let timeBudgetReached = false;
|
|
48
52
|
const pagesWalked: number[] = [];
|
|
49
53
|
const collected: ThreadListEntry[] = [];
|
|
50
54
|
const seen = new Set<string>();
|
|
51
55
|
let totalPagesAvailable = 1;
|
|
52
56
|
|
|
53
57
|
for (let page = 1; page <= max_pages; page++) {
|
|
58
|
+
if (Date.now() >= deadlineAt) {
|
|
59
|
+
timeBudgetReached = true;
|
|
60
|
+
break;
|
|
61
|
+
}
|
|
54
62
|
const url = buildPageUrl(subforum, page);
|
|
55
63
|
let html: string;
|
|
56
64
|
try {
|
|
57
|
-
html = await fetchHtml(url);
|
|
65
|
+
html = await fetchHtml(url, { deadlineAt });
|
|
58
66
|
} catch (err) {
|
|
67
|
+
if (Date.now() >= deadlineAt) timeBudgetReached = true;
|
|
59
68
|
const message = err instanceof Error ? err.message : String(err);
|
|
60
69
|
console.error(`[crawl-subforum] Failed page ${page}: ${message}`);
|
|
61
70
|
break;
|
|
@@ -66,6 +75,7 @@ export function registerCrawlSubforum(server: McpServer): void {
|
|
|
66
75
|
pagesWalked.push(page);
|
|
67
76
|
|
|
68
77
|
const threads = parseThreadList(html);
|
|
78
|
+
if (threads.length > 0) getForumIndex().recordListing(subforum, page, threads);
|
|
69
79
|
for (const thread of threads) {
|
|
70
80
|
if (seen.has(thread.url)) continue;
|
|
71
81
|
seen.add(thread.url);
|
|
@@ -98,6 +108,7 @@ export function registerCrawlSubforum(server: McpServer): void {
|
|
|
98
108
|
text: JSON.stringify({
|
|
99
109
|
subforum,
|
|
100
110
|
pagesWalked,
|
|
111
|
+
timeBudgetReached,
|
|
101
112
|
totalPagesAvailable,
|
|
102
113
|
collected: collected.length,
|
|
103
114
|
matched: filtered.length,
|
|
@@ -123,6 +134,6 @@ export function registerCrawlSubforum(server: McpServer): void {
|
|
|
123
134
|
isError: true,
|
|
124
135
|
};
|
|
125
136
|
}
|
|
126
|
-
}
|
|
137
|
+
})
|
|
127
138
|
);
|
|
128
139
|
}
|
package/src/tools/debug-page.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { withBrowserSession } from "../browser.js";
|
|
1
2
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
3
|
import { z } from "zod";
|
|
3
4
|
import { load } from "cheerio";
|
|
@@ -11,7 +12,7 @@ export function registerDebugPage(server: McpServer): void {
|
|
|
11
12
|
url: z.string().url().describe("URL to inspect"),
|
|
12
13
|
selector: z.string().optional().describe("CSS selector to extract (returns matched outerHTML)"),
|
|
13
14
|
},
|
|
14
|
-
async ({ url, selector }) => {
|
|
15
|
+
async ({ url, selector }) => withBrowserSession(async () => {
|
|
15
16
|
try {
|
|
16
17
|
const { html } = await navigateWithRetry(url);
|
|
17
18
|
const $ = load(html);
|
|
@@ -97,6 +98,6 @@ export function registerDebugPage(server: McpServer): void {
|
|
|
97
98
|
isError: true,
|
|
98
99
|
};
|
|
99
100
|
}
|
|
100
|
-
}
|
|
101
|
+
})
|
|
101
102
|
);
|
|
102
103
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { withBrowserSession } from "../browser.js";
|
|
1
2
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
3
|
import { z } from "zod";
|
|
3
4
|
import { getPage, navigateWithRetry, validateUrl } from "../browser.js";
|
|
@@ -148,7 +149,7 @@ export function registerDownloadFile(server: McpServer): void {
|
|
|
148
149
|
.default(true)
|
|
149
150
|
.describe("If true, reads and returns text file contents for analysis (default true)"),
|
|
150
151
|
},
|
|
151
|
-
async ({ url, analyze }) => {
|
|
152
|
+
async ({ url, analyze }) => withBrowserSession(async () => {
|
|
152
153
|
try {
|
|
153
154
|
validateUrl(url);
|
|
154
155
|
await mkdir(DOWNLOADS_DIR, { recursive: true });
|
|
@@ -401,6 +402,6 @@ export function registerDownloadFile(server: McpServer): void {
|
|
|
401
402
|
isError: true,
|
|
402
403
|
};
|
|
403
404
|
}
|
|
404
|
-
}
|
|
405
|
+
})
|
|
405
406
|
);
|
|
406
407
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { withBrowserSession } from "../browser.js";
|
|
1
2
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
3
|
import { z } from "zod";
|
|
3
4
|
import { fetchHtml } from "../crawl.js";
|
|
@@ -29,7 +30,7 @@ export function registerExtractCode(server: McpServer): void {
|
|
|
29
30
|
.default(false)
|
|
30
31
|
.describe("If true, exports ALL code blocks to a JSON file instead of returning them inline. Recommended when a page has many code blocks."),
|
|
31
32
|
},
|
|
32
|
-
async ({ url, limit, export_to_file }) => {
|
|
33
|
+
async ({ url, limit, export_to_file }) => withBrowserSession(async () => {
|
|
33
34
|
try {
|
|
34
35
|
const html = await fetchHtml(url);
|
|
35
36
|
const all = parseCodeBlocks(html);
|
|
@@ -103,6 +104,6 @@ export function registerExtractCode(server: McpServer): void {
|
|
|
103
104
|
isError: true,
|
|
104
105
|
};
|
|
105
106
|
}
|
|
106
|
-
}
|
|
107
|
+
})
|
|
107
108
|
);
|
|
108
109
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { withBrowserSession } from "../browser.js";
|
|
1
2
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
3
|
import { z } from "zod";
|
|
3
4
|
import { navigateWithRetry, validateUrl } from "../browser.js";
|
|
@@ -6,6 +7,7 @@ import type { Subforum } from "../parsers/subforums.js";
|
|
|
6
7
|
import { parseThreadList, parsePaginationInfo } from "../parsers/thread-list.js";
|
|
7
8
|
import { parseThread } from "../parsers/thread.js";
|
|
8
9
|
import { containsOffsetUpdate, normalizeName, rankGameForums, rankOffsetThreads, type OffsetThread } from "../offset-discovery.js";
|
|
10
|
+
import { getForumIndex } from "../forum-index.js";
|
|
9
11
|
|
|
10
12
|
const MAX_LISTING_PAGES = 10;
|
|
11
13
|
const MAX_THREAD_PAGES = 50;
|
|
@@ -17,27 +19,28 @@ function withPage(url: string, page: number): string {
|
|
|
17
19
|
return target.toString();
|
|
18
20
|
}
|
|
19
21
|
|
|
20
|
-
async function readPage(page: ForumPage, url: string): Promise<{ page: ForumPage; html: string }> {
|
|
22
|
+
async function readPage(page: ForumPage, url: string, deadlineAt: number): Promise<{ page: ForumPage; html: string }> {
|
|
21
23
|
validateUrl(url);
|
|
24
|
+
if (Date.now() >= deadlineAt) throw new Error("Offsets lookup time budget exhausted");
|
|
22
25
|
try {
|
|
23
|
-
const response = await page.evaluate(async (target) => {
|
|
24
|
-
const result = await fetch(target, { credentials: "include", signal: AbortSignal.timeout(
|
|
26
|
+
const response = await page.evaluate(async ({ target, timeoutMs }) => {
|
|
27
|
+
const result = await fetch(target, { credentials: "include", signal: AbortSignal.timeout(timeoutMs) });
|
|
25
28
|
return { ok: result.ok, url: result.url, html: await result.text() };
|
|
26
|
-
}, url);
|
|
29
|
+
}, { target: url, timeoutMs: Math.max(1, Math.min(8_000, deadlineAt - Date.now())) });
|
|
27
30
|
validateUrl(response.url);
|
|
28
31
|
if (!response.ok || /Just a moment|cf-browser-verification|Checking your browser/i.test(response.html)) {
|
|
29
32
|
throw new Error("Forum returned a challenge or an error");
|
|
30
33
|
}
|
|
31
34
|
return { page, html: response.html };
|
|
32
35
|
} catch {
|
|
33
|
-
return navigateWithRetry(url);
|
|
36
|
+
return navigateWithRetry(url, deadlineAt);
|
|
34
37
|
}
|
|
35
38
|
}
|
|
36
39
|
|
|
37
40
|
export function registerFindLatestOffsets(server: McpServer): void {
|
|
38
41
|
server.tool(
|
|
39
42
|
"find_latest_offsets",
|
|
40
|
-
"Discover
|
|
43
|
+
"Use when asked for the newest game offsets on UnknownCheats. Discover the game's offsets thread from live listings, then scan from its last page backward for the newest matching post; this does not verify the offsets against a game build.",
|
|
41
44
|
{
|
|
42
45
|
game: z.string().min(1).describe("Game name, such as Apex Legends or PUBG"),
|
|
43
46
|
subforum_slug: z.string().optional().describe("Exact subforum slug from list_subforums when needed"),
|
|
@@ -45,10 +48,12 @@ export function registerFindLatestOffsets(server: McpServer): void {
|
|
|
45
48
|
max_listing_pages: z.number().int().min(1).max(MAX_LISTING_PAGES).optional().default(5).describe("Maximum game-forum listing pages to inspect"),
|
|
46
49
|
max_thread_pages: z.number().int().min(1).max(MAX_THREAD_PAGES).optional().default(20).describe("Maximum recent thread pages to inspect"),
|
|
47
50
|
},
|
|
48
|
-
|
|
51
|
+
{ readOnlyHint: true, destructiveHint: false, openWorldHint: true },
|
|
52
|
+
async ({ game, subforum_slug, thread_url, max_listing_pages, max_thread_pages }) => withBrowserSession(async () => {
|
|
53
|
+
const deadlineAt = Date.now() + 45_000;
|
|
49
54
|
try {
|
|
50
55
|
if (thread_url) validateUrl(thread_url);
|
|
51
|
-
const entry = thread_url ? await navigateWithRetry(thread_url) : null;
|
|
56
|
+
const entry = thread_url ? await navigateWithRetry(thread_url, deadlineAt) : null;
|
|
52
57
|
let browserPage: ForumPage | null = entry?.page ?? null;
|
|
53
58
|
let catalog: ForumCatalog | null = null;
|
|
54
59
|
let fromCache = false;
|
|
@@ -64,7 +69,7 @@ export function registerFindLatestOffsets(server: McpServer): void {
|
|
|
64
69
|
catalog = await readForumCatalog();
|
|
65
70
|
fromCache = catalog !== null;
|
|
66
71
|
if (!catalog) {
|
|
67
|
-
const index = await navigateWithRetry(FORUM_INDEX);
|
|
72
|
+
const index = await navigateWithRetry(FORUM_INDEX, deadlineAt);
|
|
68
73
|
browserPage = index.page;
|
|
69
74
|
catalog = await saveForumCatalog(index.html);
|
|
70
75
|
}
|
|
@@ -72,7 +77,7 @@ export function registerFindLatestOffsets(server: McpServer): void {
|
|
|
72
77
|
forumChoices = rankGameForums(game, forums);
|
|
73
78
|
forum = subforum_slug ? forums.find((item) => item.slug === subforum_slug) ?? null : forumChoices[0] ?? null;
|
|
74
79
|
if (!forum && fromCache) {
|
|
75
|
-
const index = await navigateWithRetry(FORUM_INDEX);
|
|
80
|
+
const index = await navigateWithRetry(FORUM_INDEX, deadlineAt);
|
|
76
81
|
browserPage = index.page;
|
|
77
82
|
catalog = await saveForumCatalog(index.html);
|
|
78
83
|
fromCache = false;
|
|
@@ -96,12 +101,14 @@ export function registerFindLatestOffsets(server: McpServer): void {
|
|
|
96
101
|
}
|
|
97
102
|
|
|
98
103
|
for (let number = 1; number <= max_listing_pages; number++) {
|
|
104
|
+
if (Date.now() >= deadlineAt) break;
|
|
99
105
|
const url = number === 1 ? forum.url : `${forum.url}index${number}.html`;
|
|
100
|
-
const response = browserPage ? await readPage(browserPage, url) : await navigateWithRetry(url);
|
|
106
|
+
const response = browserPage ? await readPage(browserPage, url, deadlineAt) : await navigateWithRetry(url, deadlineAt);
|
|
101
107
|
browserPage = response.page;
|
|
102
108
|
listingPagesScanned.push(url);
|
|
103
109
|
const threads = parseThreadList(response.html);
|
|
104
110
|
if (threads.length === 0) throw new Error(`No thread list parsed from ${url}`);
|
|
111
|
+
getForumIndex().recordListing(forum.slug, number, threads);
|
|
105
112
|
candidates.push(...rankOffsetThreads(threads, url));
|
|
106
113
|
if (candidates.length > 0 || number >= parsePaginationInfo(response.html).totalPages) break;
|
|
107
114
|
}
|
|
@@ -112,7 +119,7 @@ export function registerFindLatestOffsets(server: McpServer): void {
|
|
|
112
119
|
const selected = candidates[0];
|
|
113
120
|
if (!selected) {
|
|
114
121
|
return { content: [{ type: "text", text: JSON.stringify({
|
|
115
|
-
found: false, reason: "offset_thread_not_found_in_scanned_listings", game, forum,
|
|
122
|
+
found: false, reason: Date.now() >= deadlineAt ? "time_budget_reached" : "offset_thread_not_found_in_scanned_listings", game, forum,
|
|
116
123
|
forumIndex: { url: FORUM_INDEX, indexedAt: catalog.indexedAt, fromCache }, listingPagesScanned,
|
|
117
124
|
}) }] };
|
|
118
125
|
}
|
|
@@ -121,7 +128,7 @@ export function registerFindLatestOffsets(server: McpServer): void {
|
|
|
121
128
|
discoveredAt = selected.listingPage;
|
|
122
129
|
}
|
|
123
130
|
|
|
124
|
-
const first = entry ?? await readPage(browserPage!, selectedUrl);
|
|
131
|
+
const first = entry ?? await readPage(browserPage!, selectedUrl, deadlineAt);
|
|
125
132
|
browserPage = first.page;
|
|
126
133
|
const thread = parseThread(first.html, selectedUrl);
|
|
127
134
|
selectedTitle ??= thread.title;
|
|
@@ -130,11 +137,14 @@ export function registerFindLatestOffsets(server: McpServer): void {
|
|
|
130
137
|
const oldestPage = Math.max(1, thread.totalPages - max_thread_pages + 1);
|
|
131
138
|
|
|
132
139
|
for (let number = thread.totalPages; number >= oldestPage; number--) {
|
|
140
|
+
if (Date.now() >= deadlineAt) break;
|
|
133
141
|
const url = withPage(selectedUrl, number);
|
|
134
|
-
const response: { page: ForumPage; html: string } = number === 1 && thread.totalPages === 1 ? first : await readPage(browserPage!, url);
|
|
142
|
+
const response: { page: ForumPage; html: string } = number === 1 && thread.totalPages === 1 ? first : await readPage(browserPage!, url, deadlineAt);
|
|
135
143
|
browserPage = response.page;
|
|
136
|
-
const
|
|
144
|
+
const parsed = parseThread(response.html, url, number);
|
|
145
|
+
const posts = parsed.posts;
|
|
137
146
|
if (posts.length === 0) throw new Error(`No posts parsed from ${url}`);
|
|
147
|
+
getForumIndex().recordThreadPage(parsed, number);
|
|
138
148
|
pagesScanned.push(number);
|
|
139
149
|
const match = posts.reverse().find(containsOffsetUpdate);
|
|
140
150
|
if (match) {
|
|
@@ -152,7 +162,7 @@ export function registerFindLatestOffsets(server: McpServer): void {
|
|
|
152
162
|
}
|
|
153
163
|
|
|
154
164
|
return { content: [{ type: "text", text: JSON.stringify({
|
|
155
|
-
found: false, reason: "no_offset_update_in_scanned_pages", game, forum,
|
|
165
|
+
found: false, reason: Date.now() >= deadlineAt ? "time_budget_reached" : "no_offset_update_in_scanned_pages", game, forum,
|
|
156
166
|
forumIndex: catalog ? { url: FORUM_INDEX, indexedAt: catalog.indexedAt, fromCache } : undefined,
|
|
157
167
|
thread: { title: selectedTitle, url: selectedUrl, discoveredAt }, candidateThreads: candidates,
|
|
158
168
|
listingPagesScanned, totalThreadPages: thread.totalPages, pagesScanned,
|
|
@@ -160,8 +170,14 @@ export function registerFindLatestOffsets(server: McpServer): void {
|
|
|
160
170
|
}) }] };
|
|
161
171
|
} catch (err) {
|
|
162
172
|
const message = err instanceof Error ? err.message : String(err);
|
|
173
|
+
if (Date.now() >= deadlineAt) {
|
|
174
|
+
return { content: [{ type: "text", text: JSON.stringify({
|
|
175
|
+
found: false, reason: "time_budget_reached", game, thread_url,
|
|
176
|
+
checkedAt: new Date().toISOString(), error: message,
|
|
177
|
+
}) }] };
|
|
178
|
+
}
|
|
163
179
|
return { content: [{ type: "text", text: `Error: ${message}` }], isError: true };
|
|
164
180
|
}
|
|
165
|
-
}
|
|
181
|
+
})
|
|
166
182
|
);
|
|
167
183
|
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { withBrowserSession } from "../browser.js";
|
|
2
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
import { getForumIndex } from "../forum-index.js";
|
|
5
|
+
import { fetchHtml } from "../crawl.js";
|
|
6
|
+
import { syncSubforumIndex } from "../sync-index.js";
|
|
7
|
+
|
|
8
|
+
export function registerForumIndex(server: McpServer): void {
|
|
9
|
+
server.tool(
|
|
10
|
+
"index_subforum",
|
|
11
|
+
"Incrementally index a bounded subforum listing and the first/recent pages of changed threads.",
|
|
12
|
+
{
|
|
13
|
+
subforum: z.string().describe("Exact slug from list_subforums"),
|
|
14
|
+
max_listing_pages: z.number().int().min(1).max(5).optional().default(1),
|
|
15
|
+
max_threads: z.number().int().min(0).max(20).optional().default(5),
|
|
16
|
+
recent_pages: z.number().int().min(1).max(3).optional().default(1),
|
|
17
|
+
},
|
|
18
|
+
{ readOnlyHint: false, destructiveHint: false, openWorldHint: true },
|
|
19
|
+
async ({ subforum, max_listing_pages, max_threads, recent_pages }) => withBrowserSession(async () => {
|
|
20
|
+
try {
|
|
21
|
+
const deadlineAt = Date.now() + 45_000;
|
|
22
|
+
const result = await syncSubforumIndex(getForumIndex(), subforum, {
|
|
23
|
+
maxListingPages: max_listing_pages, maxThreads: max_threads, recentPages: recent_pages,
|
|
24
|
+
deadlineAt,
|
|
25
|
+
}, (url) => fetchHtml(url, { deadlineAt }));
|
|
26
|
+
return { content: [{ type: "text", text: JSON.stringify(result) }] };
|
|
27
|
+
} catch (error) {
|
|
28
|
+
return { content: [{ type: "text", text: `Error: ${error instanceof Error ? error.message : String(error)}` }], isError: true };
|
|
29
|
+
}
|
|
30
|
+
}),
|
|
31
|
+
);
|
|
32
|
+
|
|
33
|
+
server.tool(
|
|
34
|
+
"search_index",
|
|
35
|
+
"Use first for forum-related game hacking, cheat, anti-cheat, reversing, or offsets questions. Search locally indexed thread titles, snippets and sampled post pages without fetching the forum. Results include timestamps and may be partial or stale.",
|
|
36
|
+
{
|
|
37
|
+
query: z.string().min(1),
|
|
38
|
+
subforum: z.string().optional(),
|
|
39
|
+
limit: z.number().int().min(1).max(100).optional().default(20),
|
|
40
|
+
},
|
|
41
|
+
{ readOnlyHint: true, destructiveHint: false, openWorldHint: false },
|
|
42
|
+
async ({ query, subforum, limit }) => {
|
|
43
|
+
try {
|
|
44
|
+
const index = getForumIndex();
|
|
45
|
+
const hits = index.search(query, subforum, limit);
|
|
46
|
+
return { content: [{ type: "text", text: JSON.stringify({ source: "local_index", count: hits.length, coverage: index.status(subforum), hits }) }] };
|
|
47
|
+
} catch (error) {
|
|
48
|
+
return { content: [{ type: "text", text: `Error: ${error instanceof Error ? error.message : String(error)}` }], isError: true };
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
);
|
|
52
|
+
|
|
53
|
+
server.tool(
|
|
54
|
+
"index_status",
|
|
55
|
+
"Show local index coverage, counts, and last update times; does not contact the forum.",
|
|
56
|
+
{ subforum: z.string().optional().describe("Optional subforum slug to inspect indexed listing page numbers") },
|
|
57
|
+
{ readOnlyHint: true, destructiveHint: false, openWorldHint: false },
|
|
58
|
+
async ({ subforum }) => {
|
|
59
|
+
try {
|
|
60
|
+
return { content: [{ type: "text", text: JSON.stringify(getForumIndex().status(subforum)) }] };
|
|
61
|
+
} catch (error) {
|
|
62
|
+
return { content: [{ type: "text", text: `Error: ${error instanceof Error ? error.message : String(error)}` }], isError: true };
|
|
63
|
+
}
|
|
64
|
+
},
|
|
65
|
+
);
|
|
66
|
+
}
|