@mrscraper/mcp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +642 -0
- package/dist/api.d.ts +102 -0
- package/dist/api.d.ts.map +1 -0
- package/dist/api.js +203 -0
- package/dist/api.js.map +1 -0
- package/dist/app.d.ts +17 -0
- package/dist/app.d.ts.map +1 -0
- package/dist/app.js +123 -0
- package/dist/app.js.map +1 -0
- package/dist/auth.d.ts +7 -0
- package/dist/auth.d.ts.map +1 -0
- package/dist/auth.js +64 -0
- package/dist/auth.js.map +1 -0
- package/dist/bin.d.ts +3 -0
- package/dist/bin.d.ts.map +1 -0
- package/dist/bin.js +8 -0
- package/dist/bin.js.map +1 -0
- package/dist/compliance.d.ts +4 -0
- package/dist/compliance.d.ts.map +1 -0
- package/dist/compliance.js +10 -0
- package/dist/compliance.js.map +1 -0
- package/dist/config.d.ts +28 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +63 -0
- package/dist/config.js.map +1 -0
- package/dist/http.d.ts +19 -0
- package/dist/http.d.ts.map +1 -0
- package/dist/http.js +156 -0
- package/dist/http.js.map +1 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/index.js.map +1 -0
- package/dist/runtime.d.ts +8 -0
- package/dist/runtime.d.ts.map +1 -0
- package/dist/runtime.js +46 -0
- package/dist/runtime.js.map +1 -0
- package/dist/server.d.ts +9 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +24 -0
- package/dist/server.js.map +1 -0
- package/dist/status.d.ts +21 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.js +62 -0
- package/dist/status.js.map +1 -0
- package/dist/tools.d.ts +122 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/tools.js +664 -0
- package/dist/tools.js.map +1 -0
- package/package.json +77 -0
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
export declare const VERSION = "0.1.0";
|
|
2
|
+
export declare const TOOL_NAMES: readonly ["fetch", "scrape", "serp", "status", "rerun", "results", "result"];
|
|
3
|
+
export type Transport = "stdio" | "http";
|
|
4
|
+
export interface ApiEndpoints {
|
|
5
|
+
apiBaseUrl: string;
|
|
6
|
+
fetchHtmlBaseUrl: string;
|
|
7
|
+
syncScraperBaseUrl: string;
|
|
8
|
+
subscriptionAccounts: string;
|
|
9
|
+
analyticStatuses: string;
|
|
10
|
+
scrapersAi: string;
|
|
11
|
+
scrapersAiRerun: string;
|
|
12
|
+
scrapersAiRerunBulk: string;
|
|
13
|
+
scrapersManualRerun: string;
|
|
14
|
+
scrapersManualRerunBulk: string;
|
|
15
|
+
results: string;
|
|
16
|
+
googleSerpSync: string;
|
|
17
|
+
}
|
|
18
|
+
export declare function getApiEndpoints(): ApiEndpoints;
|
|
19
|
+
export declare function resolveTransport(value?: string | undefined): Transport;
|
|
20
|
+
export declare function environmentFlag(name: string, fallback: boolean): boolean;
|
|
21
|
+
export declare function httpAuthEnabled(): boolean;
|
|
22
|
+
export declare function parseAllowedOrigins(): string[];
|
|
23
|
+
export declare function httpRuntimeConfig(): {
|
|
24
|
+
host: string;
|
|
25
|
+
port: number;
|
|
26
|
+
allowedOrigins: string[];
|
|
27
|
+
};
|
|
28
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,eAAO,MAAM,UAAU,YACrB,OAAO,EACP,QAAQ,EACR,MAAM,EACN,QAAQ,EACR,OAAO,EACP,SAAS,EACT,QAAQ,CACA,CAAC;AAEX,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,MAAM,CAAC;AAEzC,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,CAAC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;CACxB;AAMD,wBAAgB,eAAe,IAAI,YAAY,CA4B9C;AAED,wBAAgB,gBAAgB,CAAC,KAAK,qBAAwB,GAAG,SAAS,CAQzE;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,GAAG,OAAO,CAIxE;AAED,wBAAgB,eAAe,IAAI,OAAO,CAEzC;AAED,wBAAgB,mBAAmB,IAAI,MAAM,EAAE,CAK9C;AAED,wBAAgB,iBAAiB,IAAI;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,EAAE,CAAC;CAC1B,CAOA"}
|
package/dist/config.js
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
export const VERSION = "0.1.0";
|
|
2
|
+
export const TOOL_NAMES = [
|
|
3
|
+
"fetch",
|
|
4
|
+
"scrape",
|
|
5
|
+
"serp",
|
|
6
|
+
"status",
|
|
7
|
+
"rerun",
|
|
8
|
+
"results",
|
|
9
|
+
"result",
|
|
10
|
+
];
|
|
11
|
+
function baseUrl(environmentName, fallback) {
|
|
12
|
+
return (process.env[environmentName] || fallback).replace(/\/+$/, "");
|
|
13
|
+
}
|
|
14
|
+
export function getApiEndpoints() {
|
|
15
|
+
const apiBaseUrl = baseUrl("MRSCRAPER_API_BASE_URL", "https://api.app.mrscraper.com/api/v1");
|
|
16
|
+
const fetchHtmlBaseUrl = baseUrl("MRSCRAPER_FETCH_BASE_URL", "https://api.mrscraper.com");
|
|
17
|
+
const syncScraperBaseUrl = baseUrl("MRSCRAPER_SYNC_BASE_URL", "https://sync.scraper.mrscraper.com");
|
|
18
|
+
return {
|
|
19
|
+
apiBaseUrl,
|
|
20
|
+
fetchHtmlBaseUrl,
|
|
21
|
+
syncScraperBaseUrl,
|
|
22
|
+
subscriptionAccounts: `${apiBaseUrl}/subscription-accounts`,
|
|
23
|
+
analyticStatuses: `${apiBaseUrl}/analytic/statuses`,
|
|
24
|
+
scrapersAi: `${apiBaseUrl}/scrapers-ai`,
|
|
25
|
+
scrapersAiRerun: `${apiBaseUrl}/scrapers-ai-rerun`,
|
|
26
|
+
scrapersAiRerunBulk: `${apiBaseUrl}/scrapers-ai-rerun/bulk`,
|
|
27
|
+
scrapersManualRerun: `${apiBaseUrl}/scrapers-manual-rerun`,
|
|
28
|
+
scrapersManualRerunBulk: `${apiBaseUrl}/scrapers-manual-rerun/bulk`,
|
|
29
|
+
results: `${apiBaseUrl}/results`,
|
|
30
|
+
googleSerpSync: `${syncScraperBaseUrl}/api/google/serp/v2/sync`,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
export function resolveTransport(value = process.env.TRANSPORT) {
|
|
34
|
+
const transport = (value ?? "stdio").trim().toLowerCase();
|
|
35
|
+
if (transport !== "stdio" && transport !== "http") {
|
|
36
|
+
throw new Error(`Unsupported TRANSPORT=${JSON.stringify(transport)}. Expected one of: http, stdio.`);
|
|
37
|
+
}
|
|
38
|
+
return transport;
|
|
39
|
+
}
|
|
40
|
+
export function environmentFlag(name, fallback) {
|
|
41
|
+
const raw = process.env[name];
|
|
42
|
+
if (raw === undefined)
|
|
43
|
+
return fallback;
|
|
44
|
+
return !["0", "false", "no", "off"].includes(raw.trim().toLowerCase());
|
|
45
|
+
}
|
|
46
|
+
export function httpAuthEnabled() {
|
|
47
|
+
return environmentFlag("MRSCRAPER_HTTP_AUTH", true);
|
|
48
|
+
}
|
|
49
|
+
export function parseAllowedOrigins() {
|
|
50
|
+
return (process.env.MRSCRAPER_ALLOWED_ORIGINS || "")
|
|
51
|
+
.split(",")
|
|
52
|
+
.map((origin) => origin.trim().replace(/\/$/, ""))
|
|
53
|
+
.filter(Boolean);
|
|
54
|
+
}
|
|
55
|
+
export function httpRuntimeConfig() {
|
|
56
|
+
const host = process.env.HOST || "127.0.0.1";
|
|
57
|
+
const port = Number(process.env.PORT || "8000");
|
|
58
|
+
if (!Number.isInteger(port) || port < 1 || port > 65_535) {
|
|
59
|
+
throw new Error("PORT must be an integer between 1 and 65535");
|
|
60
|
+
}
|
|
61
|
+
return { host, port, allowedOrigins: parseAllowedOrigins() };
|
|
62
|
+
}
|
|
63
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAE/B,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,OAAO;IACP,QAAQ;IACR,MAAM;IACN,QAAQ;IACR,OAAO;IACP,SAAS;IACT,QAAQ;CACA,CAAC;AAmBX,SAAS,OAAO,CAAC,eAAuB,EAAE,QAAgB;IACxD,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,QAAQ,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,eAAe;IAC7B,MAAM,UAAU,GAAG,OAAO,CACxB,wBAAwB,EACxB,sCAAsC,CACvC,CAAC;IACF,MAAM,gBAAgB,GAAG,OAAO,CAC9B,0BAA0B,EAC1B,2BAA2B,CAC5B,CAAC;IACF,MAAM,kBAAkB,GAAG,OAAO,CAChC,yBAAyB,EACzB,oCAAoC,CACrC,CAAC;IAEF,OAAO;QACL,UAAU;QACV,gBAAgB;QAChB,kBAAkB;QAClB,oBAAoB,EAAE,GAAG,UAAU,wBAAwB;QAC3D,gBAAgB,EAAE,GAAG,UAAU,oBAAoB;QACnD,UAAU,EAAE,GAAG,UAAU,cAAc;QACvC,eAAe,EAAE,GAAG,UAAU,oBAAoB;QAClD,mBAAmB,EAAE,GAAG,UAAU,yBAAyB;QAC3D,mBAAmB,EAAE,GAAG,UAAU,wBAAwB;QAC1D,uBAAuB,EAAE,GAAG,UAAU,6BAA6B;QACnE,OAAO,EAAE,GAAG,UAAU,UAAU;QAChC,cAAc,EAAE,GAAG,kBAAkB,0BAA0B;KAChE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS;IAC5D,MAAM,SAAS,GAAG,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC1D,IAAI,SAAS,KAAK,OAAO,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CACb,yBAAyB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,iCAAiC,CACpF,CAAC;IACJ,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,QAAiB;IAC7D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACvC,OAAO,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,eAAe;IAC7B,OAAO,eAAe,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,mBAAmB;IACjC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,EAAE,CAAC;SACjD,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;SACjD,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,iBAAiB;IAK/B,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW,CAAC;IAC7C,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,MAAM,CAAC,CAAC;IAChD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,MAAM,EAAE,CAAC;QACzD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE,mBAAmB,EAAE,EAAE,CAAC;AAC/D,CAAC"}
|
package/dist/http.d.ts
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { JSONValue } from "@modelcontextprotocol/server";
|
|
2
|
+
export declare const DEFAULT_TIMEOUT_SECONDS = 600;
|
|
3
|
+
export interface ApiResponse {
|
|
4
|
+
status_code: number | null;
|
|
5
|
+
data: JSONValue;
|
|
6
|
+
headers: Record<string, string>;
|
|
7
|
+
error?: string;
|
|
8
|
+
[key: string]: unknown;
|
|
9
|
+
}
|
|
10
|
+
export interface HttpRequestOptions {
|
|
11
|
+
headers?: Record<string, string>;
|
|
12
|
+
params?: Record<string, string | number | boolean | null | undefined>;
|
|
13
|
+
json?: unknown;
|
|
14
|
+
timeout?: number;
|
|
15
|
+
fetchFn?: typeof fetch;
|
|
16
|
+
}
|
|
17
|
+
export declare function sanitizeResponseData(value: unknown, seen?: WeakSet<object>): JSONValue;
|
|
18
|
+
export declare function request(method: string, url: string, options?: HttpRequestOptions): Promise<ApiResponse>;
|
|
19
|
+
//# sourceMappingURL=http.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AAE9D,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAgC3C,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,IAAI,EAAE,SAAS,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC,CAAC;IACtE,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACxB;AAUD,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,OAAO,EACd,IAAI,kBAAwB,GAC3B,SAAS,CAqCX;AAyBD,wBAAsB,OAAO,CAC3B,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,WAAW,CAAC,CAqEtB"}
|
package/dist/http.js
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
export const DEFAULT_TIMEOUT_SECONDS = 600;
|
|
2
|
+
const SENSITIVE_RESPONSE_HEADERS = new Set([
|
|
3
|
+
"authorization",
|
|
4
|
+
"proxy-authorization",
|
|
5
|
+
"set-cookie",
|
|
6
|
+
"set-cookie2",
|
|
7
|
+
"x-api-token",
|
|
8
|
+
]);
|
|
9
|
+
const SENSITIVE_DATA_KEYS = new Set([
|
|
10
|
+
"accesstoken",
|
|
11
|
+
"api_key",
|
|
12
|
+
"apikey",
|
|
13
|
+
"apitoken",
|
|
14
|
+
"authorization",
|
|
15
|
+
"cookie",
|
|
16
|
+
"latestapitoken",
|
|
17
|
+
"password",
|
|
18
|
+
"refreshtoken",
|
|
19
|
+
"secret",
|
|
20
|
+
"set-cookie",
|
|
21
|
+
"token",
|
|
22
|
+
"x-api-token",
|
|
23
|
+
]);
|
|
24
|
+
const API_TOKEN_PATTERN = /\batk_[A-Za-z0-9_-]{12,}\b/g;
|
|
25
|
+
const AUTHORIZATION_PATTERN = /(authorization\s*:\s*bearer\s+)[^\s'"\\]+/gi;
|
|
26
|
+
const API_HEADER_PATTERN = /(x-api-token\s*:\s*)[^\s'"\\]+/gi;
|
|
27
|
+
const SIGNED_QUERY_PATTERN = /([?&](?:token|api[_-]?key|signature|sig|x-amz-(?:credential|security-token|signature))=)[^&\s'"\\]+/gi;
|
|
28
|
+
function redactSensitiveString(value) {
|
|
29
|
+
return value
|
|
30
|
+
.replace(API_TOKEN_PATTERN, "[REDACTED_API_TOKEN]")
|
|
31
|
+
.replace(AUTHORIZATION_PATTERN, "$1[REDACTED]")
|
|
32
|
+
.replace(API_HEADER_PATTERN, "$1[REDACTED]")
|
|
33
|
+
.replace(SIGNED_QUERY_PATTERN, "$1[REDACTED]");
|
|
34
|
+
}
|
|
35
|
+
export function sanitizeResponseData(value, seen = new WeakSet()) {
|
|
36
|
+
if (value === null || typeof value === "string")
|
|
37
|
+
return value;
|
|
38
|
+
if (typeof value === "boolean" || typeof value === "number")
|
|
39
|
+
return value;
|
|
40
|
+
if (typeof value !== "object")
|
|
41
|
+
return String(value);
|
|
42
|
+
if (seen.has(value))
|
|
43
|
+
return "[CIRCULAR]";
|
|
44
|
+
seen.add(value);
|
|
45
|
+
if (Array.isArray(value)) {
|
|
46
|
+
return value.map((item) => sanitizeResponseData(item, seen));
|
|
47
|
+
}
|
|
48
|
+
const sanitized = {};
|
|
49
|
+
for (const [key, item] of Object.entries(value)) {
|
|
50
|
+
const normalizedKey = key.toLowerCase();
|
|
51
|
+
const isExtractedPayload = normalizedKey === "data" &&
|
|
52
|
+
[
|
|
53
|
+
"id",
|
|
54
|
+
"scraperId",
|
|
55
|
+
"status",
|
|
56
|
+
"type",
|
|
57
|
+
"url",
|
|
58
|
+
"runtime",
|
|
59
|
+
"tokenUsage",
|
|
60
|
+
].some((marker) => Object.hasOwn(value, marker));
|
|
61
|
+
if (isExtractedPayload) {
|
|
62
|
+
sanitized[key] = item;
|
|
63
|
+
}
|
|
64
|
+
else if (SENSITIVE_DATA_KEYS.has(normalizedKey)) {
|
|
65
|
+
sanitized[key] = "[REDACTED]";
|
|
66
|
+
}
|
|
67
|
+
else if (normalizedKey === "curl" && typeof item === "string") {
|
|
68
|
+
sanitized[key] = redactSensitiveString(item);
|
|
69
|
+
}
|
|
70
|
+
else {
|
|
71
|
+
sanitized[key] = sanitizeResponseData(item, seen);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return sanitized;
|
|
75
|
+
}
|
|
76
|
+
function responseHeaders(response) {
|
|
77
|
+
return Object.fromEntries([...response.headers.entries()].filter(([name]) => !SENSITIVE_RESPONSE_HEADERS.has(name.toLowerCase())));
|
|
78
|
+
}
|
|
79
|
+
async function responseData(response) {
|
|
80
|
+
const body = await response.text();
|
|
81
|
+
const contentType = (response.headers.get("content-type") || "").toLowerCase();
|
|
82
|
+
if (contentType.includes("application/json")) {
|
|
83
|
+
try {
|
|
84
|
+
return sanitizeResponseData(body ? JSON.parse(body) : null);
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
return sanitizeResponseData(body);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return body;
|
|
91
|
+
}
|
|
92
|
+
export async function request(method, url, options = {}) {
|
|
93
|
+
const timeout = Number(options.timeout ?? DEFAULT_TIMEOUT_SECONDS);
|
|
94
|
+
const timeoutSeconds = Number.isFinite(timeout) && timeout > 0 ? timeout : DEFAULT_TIMEOUT_SECONDS;
|
|
95
|
+
const target = new URL(url);
|
|
96
|
+
for (const [name, value] of Object.entries(options.params || {})) {
|
|
97
|
+
if (value !== undefined && value !== null) {
|
|
98
|
+
target.searchParams.set(name, String(value));
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const controller = new AbortController();
|
|
102
|
+
const timer = setTimeout(() => controller.abort(), Math.max(1, Math.ceil(timeoutSeconds * 1_000)));
|
|
103
|
+
const headers = { ...(options.headers || {}) };
|
|
104
|
+
const init = {
|
|
105
|
+
method,
|
|
106
|
+
headers,
|
|
107
|
+
signal: controller.signal,
|
|
108
|
+
};
|
|
109
|
+
if (options.json !== undefined) {
|
|
110
|
+
headers["Content-Type"] = "application/json";
|
|
111
|
+
headers.accept = "application/json";
|
|
112
|
+
init.body = JSON.stringify(options.json);
|
|
113
|
+
}
|
|
114
|
+
try {
|
|
115
|
+
const response = await (options.fetchFn || fetch)(target, init);
|
|
116
|
+
const data = await responseData(response);
|
|
117
|
+
const safeHeaders = responseHeaders(response);
|
|
118
|
+
if (response.status === 401) {
|
|
119
|
+
return {
|
|
120
|
+
error: "Unauthorized or invalid token. Run `mrscraper login` or visit https://app.mrscraper.com.",
|
|
121
|
+
status_code: response.status,
|
|
122
|
+
data,
|
|
123
|
+
headers: safeHeaders,
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
if (!response.ok) {
|
|
127
|
+
return {
|
|
128
|
+
error: `HTTP ${response.status}`,
|
|
129
|
+
status_code: response.status,
|
|
130
|
+
data,
|
|
131
|
+
headers: safeHeaders,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
return {
|
|
135
|
+
status_code: response.status,
|
|
136
|
+
data,
|
|
137
|
+
headers: safeHeaders,
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
catch (error) {
|
|
141
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
142
|
+
const aborted = controller.signal.aborted ||
|
|
143
|
+
(error instanceof Error && error.name === "AbortError") ||
|
|
144
|
+
/abort(?:ed)?/i.test(message);
|
|
145
|
+
return {
|
|
146
|
+
error: aborted ? `Request timed out after ${timeoutSeconds}s` : message,
|
|
147
|
+
status_code: null,
|
|
148
|
+
data: null,
|
|
149
|
+
headers: {},
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
finally {
|
|
153
|
+
clearTimeout(timer);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
//# sourceMappingURL=http.js.map
|
package/dist/http.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"http.js","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAE3C,MAAM,0BAA0B,GAAG,IAAI,GAAG,CAAC;IACzC,eAAe;IACf,qBAAqB;IACrB,YAAY;IACZ,aAAa;IACb,aAAa;CACd,CAAC,CAAC;AAEH,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC;IAClC,aAAa;IACb,SAAS;IACT,QAAQ;IACR,UAAU;IACV,eAAe;IACf,QAAQ;IACR,gBAAgB;IAChB,UAAU;IACV,cAAc;IACd,QAAQ;IACR,YAAY;IACZ,OAAO;IACP,aAAa;CACd,CAAC,CAAC;AAEH,MAAM,iBAAiB,GAAG,6BAA6B,CAAC;AACxD,MAAM,qBAAqB,GAAG,6CAA6C,CAAC;AAC5E,MAAM,kBAAkB,GAAG,kCAAkC,CAAC;AAC9D,MAAM,oBAAoB,GACxB,uGAAuG,CAAC;AAkB1G,SAAS,qBAAqB,CAAC,KAAa;IAC1C,OAAO,KAAK;SACT,OAAO,CAAC,iBAAiB,EAAE,sBAAsB,CAAC;SAClD,OAAO,CAAC,qBAAqB,EAAE,cAAc,CAAC;SAC9C,OAAO,CAAC,kBAAkB,EAAE,cAAc,CAAC;SAC3C,OAAO,CAAC,oBAAoB,EAAE,cAAc,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,KAAc,EACd,IAAI,GAAG,IAAI,OAAO,EAAU;IAE5B,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC9D,IAAI,OAAO,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC1E,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACpD,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;QAAE,OAAO,YAAY,CAAC;IACzC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAEhB,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,oBAAoB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,MAAM,SAAS,GAA8B,EAAE,CAAC;IAChD,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,aAAa,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QACxC,MAAM,kBAAkB,GACtB,aAAa,KAAK,MAAM;YACxB;gBACE,IAAI;gBACJ,WAAW;gBACX,QAAQ;gBACR,MAAM;gBACN,KAAK;gBACL,SAAS;gBACT,YAAY;aACb,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;QAEnD,IAAI,kBAAkB,EAAE,CAAC;YACvB,SAAS,CAAC,GAAG,CAAC,GAAG,IAAiB,CAAC;QACrC,CAAC;aAAM,IAAI,mBAAmB,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC;YAClD,SAAS,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC;QAChC,CAAC;aAAM,IAAI,aAAa,KAAK,MAAM,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAChE,SAAS,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,GAAG,CAAC,GAAG,oBAAoB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,eAAe,CAAC,QAAkB;IACzC,OAAO,MAAM,CAAC,WAAW,CACvB,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,MAAM,CACpC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,0BAA0B,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAChE,CACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,QAAkB;IAC5C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,WAAW,GAAG,CAClB,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAC3C,CAAC,WAAW,EAAE,CAAC;IAChB,IAAI,WAAW,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;QAC7C,IAAI,CAAC;YACH,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC9D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,GAAW,EACX,OAAO,GAAuB,EAAE;IAEhC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,IAAI,uBAAuB,CAAC,CAAC;IACnE,MAAM,cAAc,GAClB,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,uBAAuB,CAAC;IAC9E,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;QACjE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YAC1C,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EACxB,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC,CAAC,CAC/C,CAAC;IACF,MAAM,OAAO,GAAG,EAAE,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAC;IAC/C,MAAM,IAAI,GAAgB;QACxB,MAAM;QACN,OAAO;QACP,MAAM,EAAE,UAAU,CAAC,MAAM;KAC1B,CAAC;IACF,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/B,OAAO,CAAC,cAAc,CAAC,GAAG,kBAAkB,CAAC;QAC7C,OAAO,CAAC,MAAM,GAAG,kBAAkB,CAAC;QACpC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAChE,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC9C,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,OAAO;gBACL,KAAK,EACH,0FAA0F;gBAC5F,WAAW,EAAE,QAAQ,CAAC,MAAM;gBAC5B,IAAI;gBACJ,OAAO,EAAE,WAAW;aACrB,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,OAAO;gBACL,KAAK,EAAE,QAAQ,QAAQ,CAAC,MAAM,EAAE;gBAChC,WAAW,EAAE,QAAQ,CAAC,MAAM;gBAC5B,IAAI;gBACJ,OAAO,EAAE,WAAW;aACrB,CAAC;QACJ,CAAC;QACD,OAAO;YACL,WAAW,EAAE,QAAQ,CAAC,MAAM;YAC5B,IAAI;YACJ,OAAO,EAAE,WAAW;SACrB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,MAAM,OAAO,GACX,UAAU,CAAC,MAAM,CAAC,OAAO;YACzB,CAAC,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC;YACvD,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChC,OAAO;YACL,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,2BAA2B,cAAc,GAAG,CAAC,CAAC,CAAC,OAAO;YACvE,WAAW,EAAE,IAAI;YACjB,IAAI,EAAE,IAAI;YACV,OAAO,EAAE,EAAE;SACZ,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export * from "./api.js";
|
|
2
|
+
export * from "./app.js";
|
|
3
|
+
export * from "./auth.js";
|
|
4
|
+
export * from "./compliance.js";
|
|
5
|
+
export * from "./config.js";
|
|
6
|
+
export * from "./http.js";
|
|
7
|
+
export * from "./runtime.js";
|
|
8
|
+
export * from "./server.js";
|
|
9
|
+
export * from "./status.js";
|
|
10
|
+
export * from "./tools.js";
|
|
11
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export * from "./api.js";
|
|
2
|
+
export * from "./app.js";
|
|
3
|
+
export * from "./auth.js";
|
|
4
|
+
export * from "./compliance.js";
|
|
5
|
+
export * from "./config.js";
|
|
6
|
+
export * from "./http.js";
|
|
7
|
+
export * from "./runtime.js";
|
|
8
|
+
export * from "./server.js";
|
|
9
|
+
export * from "./status.js";
|
|
10
|
+
export * from "./tools.js";
|
|
11
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { Server as HttpServer } from "node:http";
|
|
2
|
+
export interface RunningHttpServer {
|
|
3
|
+
server: HttpServer;
|
|
4
|
+
close: () => Promise<void>;
|
|
5
|
+
}
|
|
6
|
+
export declare function startHttpServer(): Promise<RunningHttpServer>;
|
|
7
|
+
export declare function run(): Promise<void>;
|
|
8
|
+
//# sourceMappingURL=runtime.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime.d.ts","sourceRoot":"","sources":["../src/runtime.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,WAAW,CAAC;AAQtD,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,UAAU,CAAC;IACnB,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5B;AAED,wBAAsB,eAAe,IAAI,OAAO,CAAC,iBAAiB,CAAC,CAwBlE;AAED,wBAAsB,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAsBzC"}
|
package/dist/runtime.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { serveStdio } from "@modelcontextprotocol/server/stdio";
|
|
2
|
+
import { createHttpApp } from "./app.js";
|
|
3
|
+
import { httpRuntimeConfig, resolveTransport } from "./config.js";
|
|
4
|
+
import { createServerFactory } from "./server.js";
|
|
5
|
+
export async function startHttpServer() {
|
|
6
|
+
const config = httpRuntimeConfig();
|
|
7
|
+
const { app, handler } = createHttpApp({
|
|
8
|
+
host: config.host,
|
|
9
|
+
allowedOrigins: config.allowedOrigins,
|
|
10
|
+
});
|
|
11
|
+
const server = await new Promise((resolve, reject) => {
|
|
12
|
+
const listener = app.listen(config.port, config.host, () => resolve(listener));
|
|
13
|
+
listener.once("error", reject);
|
|
14
|
+
});
|
|
15
|
+
console.error(`MrScraper MCP ${config.host}:${config.port}/mcp (health: /health, ready: /ready)`);
|
|
16
|
+
return {
|
|
17
|
+
server,
|
|
18
|
+
async close() {
|
|
19
|
+
await handler.close();
|
|
20
|
+
await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())));
|
|
21
|
+
},
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
export async function run() {
|
|
25
|
+
const transport = resolveTransport();
|
|
26
|
+
if (transport === "http") {
|
|
27
|
+
const running = await startHttpServer();
|
|
28
|
+
const shutdown = async () => {
|
|
29
|
+
await running.close();
|
|
30
|
+
process.exit(0);
|
|
31
|
+
};
|
|
32
|
+
process.once("SIGINT", shutdown);
|
|
33
|
+
process.once("SIGTERM", shutdown);
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
const handle = serveStdio(createServerFactory(), {
|
|
37
|
+
onerror: (error) => console.error("MCP stdio error:", error),
|
|
38
|
+
});
|
|
39
|
+
const shutdown = async () => {
|
|
40
|
+
await handle.close();
|
|
41
|
+
process.exit(0);
|
|
42
|
+
};
|
|
43
|
+
process.once("SIGINT", shutdown);
|
|
44
|
+
process.once("SIGTERM", shutdown);
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=runtime.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime.js","sourceRoot":"","sources":["../src/runtime.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,oCAAoC,CAAC;AAEhE,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAClE,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAOlD,MAAM,CAAC,KAAK,UAAU,eAAe;IACnC,MAAM,MAAM,GAAG,iBAAiB,EAAE,CAAC;IACnC,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC;QACrC,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,cAAc,EAAE,MAAM,CAAC,cAAc;KACtC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,IAAI,OAAO,CAAa,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC/D,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE,CACzD,OAAO,CAAC,QAAQ,CAAC,CAClB,CAAC;QACF,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,KAAK,CACX,iBAAiB,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,uCAAuC,CACnF,CAAC;IACF,OAAO;QACL,MAAM;QACN,KAAK,CAAC,KAAK;YACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACtB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAC1C,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAC7D,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,GAAG;IACvB,MAAM,SAAS,GAAG,gBAAgB,EAAE,CAAC;IACrC,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,MAAM,eAAe,EAAE,CAAC;QACxC,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;YAC1B,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACjC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAClC,OAAO;IACT,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,mBAAmB,EAAE,EAAE;QAC/C,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC;KAC7D,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;QAC1B,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;IACF,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACjC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;AACpC,CAAC"}
|
package/dist/server.d.ts
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { McpServer, type McpRequestContext, type McpServerFactory } from "@modelcontextprotocol/server";
|
|
2
|
+
import { type ToolDependencies } from "./tools.js";
|
|
3
|
+
export declare const SERVER_INSTRUCTIONS: string;
|
|
4
|
+
export interface ServerFactoryOptions extends ToolDependencies {
|
|
5
|
+
resolveToken?: (context: McpRequestContext) => string;
|
|
6
|
+
}
|
|
7
|
+
export declare function createMrscraperServer(context: McpRequestContext, options?: ServerFactoryOptions): McpServer;
|
|
8
|
+
export declare function createServerFactory(options?: ServerFactoryOptions): McpServerFactory;
|
|
9
|
+
//# sourceMappingURL=server.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,SAAS,EACT,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACtB,MAAM,8BAA8B,CAAC;AAKtC,OAAO,EAAiB,KAAK,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAElE,eAAO,MAAM,mBAAmB,QAUI,CAAC;AAErC,MAAM,WAAW,oBAAqB,SAAQ,gBAAgB;IAC5D,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,iBAAiB,KAAK,MAAM,CAAC;CACvD;AAED,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,iBAAiB,EAC1B,OAAO,GAAE,oBAAyB,GACjC,SAAS,CAWX;AAED,wBAAgB,mBAAmB,CACjC,OAAO,GAAE,oBAAyB,GACjC,gBAAgB,CAElB"}
|
package/dist/server.js
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { McpServer, } from "@modelcontextprotocol/server";
|
|
2
|
+
import { resolveApiToken } from "./auth.js";
|
|
3
|
+
import { MANUAL_SCRAPER_SERVER_INSTRUCTIONS } from "./compliance.js";
|
|
4
|
+
import { VERSION } from "./config.js";
|
|
5
|
+
import { registerTools } from "./tools.js";
|
|
6
|
+
export const SERVER_INSTRUCTIONS = "MrScraper provides seven web-data tools: `fetch`, `scrape`, `serp`, " +
|
|
7
|
+
"`status`, `rerun`, `results`, and `result`. " +
|
|
8
|
+
"Use `fetch` for the Web Unblocker response from a known URL, `scrape` for requested structured fields, " +
|
|
9
|
+
"and `serp` when starting from a Google query instead of a known URL. " +
|
|
10
|
+
"After a successful `scrape`, surface its saved `scraperId` and explain that `rerun` can reproduce the saved extraction configuration on the same or another URL. " +
|
|
11
|
+
"Use `rerun` for saved scraper configurations: `type` identifies an AI scraper or dashboard-built manual workflow, while `bulk` independently selects one URL or a URL list. " +
|
|
12
|
+
"Bulk reruns are asynchronous; retain `bulkResultId` and use `result` to inspect them until completion. " +
|
|
13
|
+
"Use `results` / `result` to inspect stored work. `status` reports account usage and optional domain outcomes. " +
|
|
14
|
+
"Tools do not accept API tokens as arguments. " +
|
|
15
|
+
MANUAL_SCRAPER_SERVER_INSTRUCTIONS;
|
|
16
|
+
export function createMrscraperServer(context, options = {}) {
|
|
17
|
+
const server = new McpServer({ name: "MrScraper MCP Server", version: VERSION }, { instructions: SERVER_INSTRUCTIONS });
|
|
18
|
+
registerTools(server, () => (options.resolveToken || resolveApiToken)(context), options);
|
|
19
|
+
return server;
|
|
20
|
+
}
|
|
21
|
+
export function createServerFactory(options = {}) {
|
|
22
|
+
return (context) => createMrscraperServer(context, options);
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=server.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"server.js","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,SAAS,GAGV,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,EAAE,kCAAkC,EAAE,MAAM,iBAAiB,CAAC;AACrE,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,aAAa,EAAyB,MAAM,YAAY,CAAC;AAElE,MAAM,CAAC,MAAM,mBAAmB,GAC9B,sEAAsE;IACtE,8CAA8C;IAC9C,yGAAyG;IACzG,uEAAuE;IACvE,mKAAmK;IACnK,8KAA8K;IAC9K,yGAAyG;IACzG,gHAAgH;IAChH,+CAA+C;IAC/C,kCAAkC,CAAC;AAMrC,MAAM,UAAU,qBAAqB,CACnC,OAA0B,EAC1B,OAAO,GAAyB,EAAE;IAElC,MAAM,MAAM,GAAG,IAAI,SAAS,CAC1B,EAAE,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,OAAO,EAAE,EAClD,EAAE,YAAY,EAAE,mBAAmB,EAAE,CACtC,CAAC;IACF,aAAa,CACX,MAAM,EACN,GAAG,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,IAAI,eAAe,CAAC,CAAC,OAAO,CAAC,EACxD,OAAO,CACR,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,OAAO,GAAyB,EAAE;IAElC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;AAC9D,CAAC"}
|
package/dist/status.d.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export declare function formatApiDate(date: Date): string;
|
|
2
|
+
export declare function parseStatusDate(value: string | undefined, now?: Date, fallbackDuration?: string): Date;
|
|
3
|
+
export interface SubscriptionSummary {
|
|
4
|
+
subscription_status: unknown;
|
|
5
|
+
enterprise: boolean;
|
|
6
|
+
token_usage: number;
|
|
7
|
+
token_limit: number;
|
|
8
|
+
token_remaining: number;
|
|
9
|
+
usage_percent: number;
|
|
10
|
+
rate_limit: number;
|
|
11
|
+
rate_ttl: number;
|
|
12
|
+
auto_renew: boolean;
|
|
13
|
+
ends_at: unknown;
|
|
14
|
+
user: {
|
|
15
|
+
name: unknown;
|
|
16
|
+
email: unknown;
|
|
17
|
+
verified: boolean;
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
export declare function summarizeSubscriptionAccount(account: Record<string, unknown>): SubscriptionSummary;
|
|
21
|
+
//# sourceMappingURL=status.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"status.d.ts","sourceRoot":"","sources":["../src/status.ts"],"names":[],"mappings":"AAWA,wBAAgB,aAAa,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAQhD;AAED,wBAAgB,eAAe,CAC7B,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,GAAG,OAAa,EAChB,gBAAgB,SAAQ,GACvB,IAAI,CAuBN;AAED,MAAM,WAAW,mBAAmB;IAClC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,UAAU,EAAE,OAAO,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE;QACJ,IAAI,EAAE,OAAO,CAAC;QACd,KAAK,EAAE,OAAO,CAAC;QACf,QAAQ,EAAE,OAAO,CAAC;KACnB,CAAC;CACH;AAED,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC/B,mBAAmB,CAsBrB"}
|
package/dist/status.js
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
function numberOrZero(value) {
|
|
2
|
+
const parsed = Number(value);
|
|
3
|
+
return Number.isFinite(parsed) ? parsed : 0;
|
|
4
|
+
}
|
|
5
|
+
function recordOrEmpty(value) {
|
|
6
|
+
return value !== null && typeof value === "object" && !Array.isArray(value)
|
|
7
|
+
? value
|
|
8
|
+
: {};
|
|
9
|
+
}
|
|
10
|
+
export function formatApiDate(date) {
|
|
11
|
+
if (!(date instanceof Date) || Number.isNaN(date.getTime())) {
|
|
12
|
+
throw new Error("Invalid date");
|
|
13
|
+
}
|
|
14
|
+
return date
|
|
15
|
+
.toISOString()
|
|
16
|
+
.replace("T", " ")
|
|
17
|
+
.replace(/\.\d{3}Z$/, "");
|
|
18
|
+
}
|
|
19
|
+
export function parseStatusDate(value, now = new Date(), fallbackDuration = "24h") {
|
|
20
|
+
const input = (value || fallbackDuration).trim().toLowerCase();
|
|
21
|
+
if (input === "now")
|
|
22
|
+
return new Date(now);
|
|
23
|
+
const relative = /^(\d+)(m|h|d|w)$/.exec(input);
|
|
24
|
+
if (relative) {
|
|
25
|
+
const amount = Number(relative[1]);
|
|
26
|
+
const units = {
|
|
27
|
+
m: 60_000,
|
|
28
|
+
h: 3_600_000,
|
|
29
|
+
d: 86_400_000,
|
|
30
|
+
w: 604_800_000,
|
|
31
|
+
};
|
|
32
|
+
return new Date(now.getTime() - amount * (units[relative[2]] || 0));
|
|
33
|
+
}
|
|
34
|
+
const parsed = new Date(value || "");
|
|
35
|
+
if (Number.isNaN(parsed.getTime())) {
|
|
36
|
+
throw new Error(`Invalid date "${value}". Use ISO 8601, "now", or a duration such as 24h or 7d.`);
|
|
37
|
+
}
|
|
38
|
+
return parsed;
|
|
39
|
+
}
|
|
40
|
+
export function summarizeSubscriptionAccount(account) {
|
|
41
|
+
const tokenLimit = numberOrZero(account.tokenLimit);
|
|
42
|
+
const tokenUsage = numberOrZero(account.tokenUsage);
|
|
43
|
+
const user = recordOrEmpty(account.user);
|
|
44
|
+
return {
|
|
45
|
+
subscription_status: account.stripeStatus ?? null,
|
|
46
|
+
enterprise: Boolean(account.isEnterprise),
|
|
47
|
+
token_usage: tokenUsage,
|
|
48
|
+
token_limit: tokenLimit,
|
|
49
|
+
token_remaining: Math.max(0, tokenLimit - tokenUsage),
|
|
50
|
+
usage_percent: tokenLimit > 0 ? Number(((tokenUsage / tokenLimit) * 100).toFixed(2)) : 0,
|
|
51
|
+
rate_limit: numberOrZero(account.rateLimit),
|
|
52
|
+
rate_ttl: numberOrZero(account.rateTtl),
|
|
53
|
+
auto_renew: Boolean(account.isAutoRenew),
|
|
54
|
+
ends_at: account.endsAt ?? null,
|
|
55
|
+
user: {
|
|
56
|
+
name: user.name ?? null,
|
|
57
|
+
email: user.email ?? null,
|
|
58
|
+
verified: Boolean(user.isVerified),
|
|
59
|
+
},
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=status.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"status.js","sourceRoot":"","sources":["../src/status.ts"],"names":[],"mappings":"AAAA,SAAS,YAAY,CAAC,KAAc;IAClC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACzE,CAAC,CAAE,KAAiC;QACpC,CAAC,CAAC,EAAE,CAAC;AACT,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAU;IACtC,IAAI,CAAC,CAAC,IAAI,YAAY,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,IAAI;SACR,WAAW,EAAE;SACb,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC;SACjB,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,KAAyB,EACzB,GAAG,GAAG,IAAI,IAAI,EAAE,EAChB,gBAAgB,GAAG,KAAK;IAExB,MAAM,KAAK,GAAG,CAAC,KAAK,IAAI,gBAAgB,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC/D,IAAI,KAAK,KAAK,KAAK;QAAE,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;IAE1C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChD,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,MAAM,KAAK,GAA2B;YACpC,CAAC,EAAE,MAAM;YACT,CAAC,EAAE,SAAS;YACZ,CAAC,EAAE,UAAU;YACb,CAAC,EAAE,WAAW;SACf,CAAC;QACF,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvE,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IACrC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CACb,iBAAiB,KAAK,0DAA0D,CACjF,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAoBD,MAAM,UAAU,4BAA4B,CAC1C,OAAgC;IAEhC,MAAM,UAAU,GAAG,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,UAAU,GAAG,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACpD,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,OAAO;QACL,mBAAmB,EAAE,OAAO,CAAC,YAAY,IAAI,IAAI;QACjD,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC;QACzC,WAAW,EAAE,UAAU;QACvB,WAAW,EAAE,UAAU;QACvB,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,UAAU,CAAC;QACrD,aAAa,EACX,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,GAAG,UAAU,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC;QAC3C,QAAQ,EAAE,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC;QACvC,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC;QACxC,OAAO,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI;QAC/B,IAAI,EAAE;YACJ,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI;YACvB,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI;YACzB,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC;SACnC;KACF,CAAC;AACJ,CAAC"}
|