@mrscraper/mcp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +642 -0
- package/dist/api.d.ts +102 -0
- package/dist/api.d.ts.map +1 -0
- package/dist/api.js +203 -0
- package/dist/api.js.map +1 -0
- package/dist/app.d.ts +17 -0
- package/dist/app.d.ts.map +1 -0
- package/dist/app.js +123 -0
- package/dist/app.js.map +1 -0
- package/dist/auth.d.ts +7 -0
- package/dist/auth.d.ts.map +1 -0
- package/dist/auth.js +64 -0
- package/dist/auth.js.map +1 -0
- package/dist/bin.d.ts +3 -0
- package/dist/bin.d.ts.map +1 -0
- package/dist/bin.js +8 -0
- package/dist/bin.js.map +1 -0
- package/dist/compliance.d.ts +4 -0
- package/dist/compliance.d.ts.map +1 -0
- package/dist/compliance.js +10 -0
- package/dist/compliance.js.map +1 -0
- package/dist/config.d.ts +28 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +63 -0
- package/dist/config.js.map +1 -0
- package/dist/http.d.ts +19 -0
- package/dist/http.d.ts.map +1 -0
- package/dist/http.js +156 -0
- package/dist/http.js.map +1 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/index.js.map +1 -0
- package/dist/runtime.d.ts +8 -0
- package/dist/runtime.d.ts.map +1 -0
- package/dist/runtime.js +46 -0
- package/dist/runtime.js.map +1 -0
- package/dist/server.d.ts +9 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +24 -0
- package/dist/server.js.map +1 -0
- package/dist/status.d.ts +21 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.js +62 -0
- package/dist/status.js.map +1 -0
- package/dist/tools.d.ts +122 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/tools.js +664 -0
- package/dist/tools.js.map +1 -0
- package/package.json +77 -0
package/dist/api.d.ts
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { type ApiResponse } from "./http.js";
|
|
2
|
+
export type Agent = "general" | "listing" | "map";
|
|
3
|
+
export type SerpFormat = "json" | "html";
|
|
4
|
+
interface FetchDependency {
|
|
5
|
+
fetchFn?: typeof fetch;
|
|
6
|
+
}
|
|
7
|
+
export declare function getAuthHeaders(token: string): Record<string, string>;
|
|
8
|
+
export interface FetchContentOptions extends FetchDependency {
|
|
9
|
+
token: string;
|
|
10
|
+
url: string;
|
|
11
|
+
timeout?: number;
|
|
12
|
+
geoCode?: string | null;
|
|
13
|
+
browserRendering?: boolean;
|
|
14
|
+
waitForSelector?: string | null;
|
|
15
|
+
homePage?: boolean;
|
|
16
|
+
blockResources?: boolean;
|
|
17
|
+
maxRetries?: number;
|
|
18
|
+
tokenCap?: number | null;
|
|
19
|
+
}
|
|
20
|
+
export declare function fetchContentApi({ token, url, timeout, geoCode, browserRendering, waitForSelector, homePage, blockResources, maxRetries, tokenCap, fetchFn, }: FetchContentOptions): Promise<ApiResponse>;
|
|
21
|
+
export interface CreateAiScraperOptions extends FetchDependency {
|
|
22
|
+
token: string;
|
|
23
|
+
url: string;
|
|
24
|
+
message?: string;
|
|
25
|
+
agent?: Agent;
|
|
26
|
+
proxyCountry?: string | null;
|
|
27
|
+
maxDepth?: number;
|
|
28
|
+
maxPages?: number;
|
|
29
|
+
limit?: number;
|
|
30
|
+
includePatterns?: string;
|
|
31
|
+
excludePatterns?: string;
|
|
32
|
+
}
|
|
33
|
+
export declare function createAiScraperApi({ token, url, message, agent, proxyCountry, maxDepth, maxPages, limit, includePatterns, excludePatterns, fetchFn, }: CreateAiScraperOptions): Promise<ApiResponse>;
|
|
34
|
+
export interface RerunAiOptions extends FetchDependency {
|
|
35
|
+
token: string;
|
|
36
|
+
scraperId: string;
|
|
37
|
+
url: string;
|
|
38
|
+
maxDepth?: number;
|
|
39
|
+
maxPages?: number;
|
|
40
|
+
limit?: number;
|
|
41
|
+
includePatterns?: string;
|
|
42
|
+
excludePatterns?: string;
|
|
43
|
+
}
|
|
44
|
+
export declare function rerunAiScraperApi({ token, scraperId, url, maxDepth, maxPages, limit, includePatterns, excludePatterns, fetchFn, }: RerunAiOptions): Promise<ApiResponse>;
|
|
45
|
+
interface BulkRerunOptions extends FetchDependency {
|
|
46
|
+
token: string;
|
|
47
|
+
scraperId: string;
|
|
48
|
+
urls: string[];
|
|
49
|
+
}
|
|
50
|
+
export declare function bulkRerunAiScraperApi({ token, scraperId, urls, fetchFn, }: BulkRerunOptions): Promise<ApiResponse>;
|
|
51
|
+
interface ManualRerunOptions extends FetchDependency {
|
|
52
|
+
token: string;
|
|
53
|
+
scraperId: string;
|
|
54
|
+
url: string;
|
|
55
|
+
}
|
|
56
|
+
export declare function rerunManualScraperApi({ token, scraperId, url, fetchFn, }: ManualRerunOptions): Promise<ApiResponse>;
|
|
57
|
+
export declare function bulkRerunManualScraperApi({ token, scraperId, urls, fetchFn, }: BulkRerunOptions): Promise<ApiResponse>;
|
|
58
|
+
export interface GetResultsOptions extends FetchDependency {
|
|
59
|
+
token: string;
|
|
60
|
+
sortField?: string;
|
|
61
|
+
sortOrder?: string;
|
|
62
|
+
pageSize?: number;
|
|
63
|
+
page?: number;
|
|
64
|
+
search?: string | null;
|
|
65
|
+
dateRangeColumn?: string | null;
|
|
66
|
+
startAt?: string | null;
|
|
67
|
+
endAt?: string | null;
|
|
68
|
+
}
|
|
69
|
+
export declare function getAllResultsApi({ token, sortField, sortOrder, pageSize, page, search, dateRangeColumn, startAt, endAt, fetchFn, }: GetResultsOptions): Promise<ApiResponse>;
|
|
70
|
+
export declare function getResultByIdApi(token: string, resultId: string, options?: FetchDependency): Promise<ApiResponse>;
|
|
71
|
+
export declare function getSubscriptionAccountApi(token: string, options?: FetchDependency): Promise<ApiResponse>;
|
|
72
|
+
export interface AnalyticStatusesOptions extends FetchDependency {
|
|
73
|
+
token: string;
|
|
74
|
+
domain: string;
|
|
75
|
+
startDate: string;
|
|
76
|
+
endDate: string;
|
|
77
|
+
action?: string;
|
|
78
|
+
apiTokenName?: string;
|
|
79
|
+
}
|
|
80
|
+
export declare function getAnalyticStatusesApi({ token, domain, startDate, endDate, action, apiTokenName, fetchFn, }: AnalyticStatusesOptions): Promise<ApiResponse>;
|
|
81
|
+
export interface NormalizedSerpInput {
|
|
82
|
+
query: string;
|
|
83
|
+
region: string | null;
|
|
84
|
+
language: string | null;
|
|
85
|
+
page: number | null;
|
|
86
|
+
}
|
|
87
|
+
export declare function normalizeSerpInput(input: string): NormalizedSerpInput;
|
|
88
|
+
export interface GoogleSerpOptions extends FetchDependency {
|
|
89
|
+
token: string;
|
|
90
|
+
queryOrUrl: string;
|
|
91
|
+
region?: string | null;
|
|
92
|
+
language?: string | null;
|
|
93
|
+
page?: number | null;
|
|
94
|
+
format?: SerpFormat;
|
|
95
|
+
renderJs?: boolean;
|
|
96
|
+
raw?: boolean;
|
|
97
|
+
timeout?: number;
|
|
98
|
+
}
|
|
99
|
+
export declare function googleSerpSyncApi({ token, queryOrUrl, region, language, page, format, renderJs, raw, timeout, fetchFn, }: GoogleSerpOptions): Promise<ApiResponse>;
|
|
100
|
+
export declare function parseBulkUrls(raw: string): string[];
|
|
101
|
+
export {};
|
|
102
|
+
//# sourceMappingURL=api.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAEA,OAAO,EAAW,KAAK,WAAW,EAAE,MAAM,WAAW,CAAC;AAEtD,MAAM,MAAM,KAAK,GAAG,SAAS,GAAG,SAAS,GAAG,KAAK,CAAC;AAClD,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC;AAEzC,UAAU,eAAe;IACvB,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACxB;AAQD,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAQpE;AAaD,MAAM,WAAW,mBAAoB,SAAQ,eAAe;IAC1D,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,wBAAsB,eAAe,CAAC,EACpC,KAAK,EACL,GAAG,EACH,OAAY,EACZ,OAAc,EACd,gBAAwB,EACxB,eAAsB,EACtB,QAAgB,EAChB,cAAsB,EACtB,UAAc,EACd,QAAe,EACf,OAAO,GACR,EAAE,mBAAmB,GAAG,OAAO,CAAC,WAAW,CAAC,CAiB5C;AAED,MAAM,WAAW,sBAAuB,SAAQ,eAAe;IAC7D,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,wBAAsB,kBAAkB,CAAC,EACvC,KAAK,EACL,GAAG,EACH,OAAO,EACP,KAAiB,EACjB,YAAmB,EACnB,QAAQ,EACR,QAAQ,EACR,KAAK,EACL,eAAe,EACf,eAAe,EACf,OAAO,GACR,EAAE,sBAAsB,GAAG,OAAO,CAAC,WAAW,CAAC,CAsC/C;AAED,MAAM,WAAW,cAAe,SAAQ,eAAe;IACrD,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,wBAAsB,iBAAiB,CAAC,EACtC,KAAK,EACL,SAAS,EACT,GAAG,EACH,QAAY,EACZ,QAAa,EACb,KAAa,EACb,eAAoB,EACpB,eAAoB,EACpB,OAAO,GACR,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,CAAC,CAcvC;AAED,UAAU,gBAAiB,SAAQ,eAAe;IAChD,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,KAAK,EACL,SAAS,EACT,IAAI,EACJ,OAAO,GACR,EAAE,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,CAMzC;AAED,UAAU,kBAAmB,SAAQ,eAAe;IAClD,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;CACb;AAED,wBAAsB,qBAAqB,CAAC,EAC1C,KAAK,EACL,SAAS,EACT,GAAG,EACH,OAAO,GACR,EAAE,kBAAkB,GAAG,OAAO,CAAC,WAAW,CAAC,CAM3C;AAED,wBAAsB,yBAAyB,CAAC,EAC9C,KAAK,EACL,SAAS,EACT,IAAI,EACJ,OAAO,GACR,EAAE,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,CAMzC;AAED,MAAM,WAAW,iBAAkB,SAAQ,eAAe;IACxD,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED,wBAAsB,gBAAgB,CAAC,EACrC,KAAK,EACL,SAAuB,EACvB,SAAkB,EAClB,QAAa,EACb,IAAQ,EACR,MAAa,EACb,eAAsB,EACtB,OAAc,EACd,KAAY,EACZ,OAAO,GACR,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAgB1C;AAED,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,WAAW,CAAC,CAKtB;AAED,wBAAsB,yBAAyB,CAC7C,KAAK,EAAE,MAAM,EACb,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,WAAW,CAAC,CAKtB;AAED,MAAM,WAAW,uBAAwB,SAAQ,eAAe;IAC9D,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,KAAK,EACL,MAAM,EACN,SAAS,EACT,OAAO,EACP,MAAW,EACX,YAAiB,EACjB,OAAO,GACR,EAAE,uBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC,CAMhD;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,mBAAmB,CAoBrE;AAED,MAAM,WAAW,iBAAkB,SAAQ,eAAe;IACxD,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAsB,iBAAiB,CAAC,EACtC,KAAK,EACL,UAAU,EACV,MAAa,EACb,QAAe,EACf,IAAW,EACX,MAAe,EACf,QAAgB,EAChB,GAAW,EACX,OAAa,EACb,OAAO,GACR,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAiB1C;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAKnD"}
|
package/dist/api.js
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
import { normalizeBearerToken } from "./auth.js";
|
|
2
|
+
import { getApiEndpoints } from "./config.js";
|
|
3
|
+
import { request } from "./http.js";
|
|
4
|
+
const CLI_FETCH_HEADERS = {
|
|
5
|
+
"User-Agent": "node",
|
|
6
|
+
"Accept-Language": "*",
|
|
7
|
+
"Sec-Fetch-Mode": "cors",
|
|
8
|
+
};
|
|
9
|
+
export function getAuthHeaders(token) {
|
|
10
|
+
const apiToken = normalizeBearerToken(token);
|
|
11
|
+
if (!apiToken)
|
|
12
|
+
throw new Error("API token is required");
|
|
13
|
+
return {
|
|
14
|
+
...CLI_FETCH_HEADERS,
|
|
15
|
+
Authorization: `Bearer ${apiToken}`,
|
|
16
|
+
"x-api-token": apiToken,
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
function compact(values) {
|
|
20
|
+
return Object.fromEntries(Object.entries(values).filter((entry) => entry[1] !== null && entry[1] !== undefined));
|
|
21
|
+
}
|
|
22
|
+
export async function fetchContentApi({ token, url, timeout = 30, geoCode = null, browserRendering = false, waitForSelector = null, homePage = false, blockResources = false, maxRetries = 3, tokenCap = null, fetchFn, }) {
|
|
23
|
+
return request("GET", getApiEndpoints().fetchHtmlBaseUrl, {
|
|
24
|
+
headers: getAuthHeaders(token),
|
|
25
|
+
params: compact({
|
|
26
|
+
url,
|
|
27
|
+
timeout,
|
|
28
|
+
geoCode,
|
|
29
|
+
browserRendering,
|
|
30
|
+
waitForSelector,
|
|
31
|
+
homePage,
|
|
32
|
+
blockResources,
|
|
33
|
+
maxRetries,
|
|
34
|
+
tokenCap,
|
|
35
|
+
}),
|
|
36
|
+
timeout: timeout + 30,
|
|
37
|
+
fetchFn,
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
export async function createAiScraperApi({ token, url, message, agent = "general", proxyCountry = null, maxDepth, maxPages, limit, includePatterns, excludePatterns, fetchFn, }) {
|
|
41
|
+
if (!(agent === "general" || agent === "listing" || agent === "map")) {
|
|
42
|
+
throw new Error("agent must be general, listing, or map");
|
|
43
|
+
}
|
|
44
|
+
if ((agent === "general" || agent === "listing") && !message?.trim()) {
|
|
45
|
+
throw new Error("An extraction message is required for general and listing agents");
|
|
46
|
+
}
|
|
47
|
+
if (agent === "map" && proxyCountry !== null && proxyCountry !== undefined) {
|
|
48
|
+
throw new Error("The map agent does not accept proxyCountry");
|
|
49
|
+
}
|
|
50
|
+
const payload = { url, agent };
|
|
51
|
+
if (agent === "general" || agent === "listing") {
|
|
52
|
+
payload.message = message;
|
|
53
|
+
if (proxyCountry !== null && proxyCountry !== undefined) {
|
|
54
|
+
payload.proxyCountry = proxyCountry;
|
|
55
|
+
}
|
|
56
|
+
if (agent === "listing" && maxPages !== undefined) {
|
|
57
|
+
payload.maxPages = maxPages;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
if (message)
|
|
62
|
+
throw new Error("The map agent does not accept an extraction prompt");
|
|
63
|
+
if (maxDepth !== undefined)
|
|
64
|
+
payload.maxDepth = maxDepth;
|
|
65
|
+
if (maxPages !== undefined)
|
|
66
|
+
payload.maxPages = maxPages;
|
|
67
|
+
if (limit !== undefined)
|
|
68
|
+
payload.limit = limit;
|
|
69
|
+
if (includePatterns !== undefined)
|
|
70
|
+
payload.includePatterns = includePatterns;
|
|
71
|
+
if (excludePatterns !== undefined)
|
|
72
|
+
payload.excludePatterns = excludePatterns;
|
|
73
|
+
}
|
|
74
|
+
return request("POST", getApiEndpoints().scrapersAi, {
|
|
75
|
+
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
76
|
+
json: payload,
|
|
77
|
+
fetchFn,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
export async function rerunAiScraperApi({ token, scraperId, url, maxDepth = 2, maxPages = 50, limit = 1_000, includePatterns = "", excludePatterns = "", fetchFn, }) {
|
|
81
|
+
return request("POST", getApiEndpoints().scrapersAiRerun, {
|
|
82
|
+
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
83
|
+
json: {
|
|
84
|
+
scraperId,
|
|
85
|
+
url,
|
|
86
|
+
maxDepth,
|
|
87
|
+
maxPages,
|
|
88
|
+
limit,
|
|
89
|
+
includePatterns,
|
|
90
|
+
excludePatterns,
|
|
91
|
+
},
|
|
92
|
+
fetchFn,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
export async function bulkRerunAiScraperApi({ token, scraperId, urls, fetchFn, }) {
|
|
96
|
+
return request("POST", getApiEndpoints().scrapersAiRerunBulk, {
|
|
97
|
+
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
98
|
+
json: { scraperId, urls },
|
|
99
|
+
fetchFn,
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
export async function rerunManualScraperApi({ token, scraperId, url, fetchFn, }) {
|
|
103
|
+
return request("POST", getApiEndpoints().scrapersManualRerun, {
|
|
104
|
+
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
105
|
+
json: { scraperId, url },
|
|
106
|
+
fetchFn,
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
export async function bulkRerunManualScraperApi({ token, scraperId, urls, fetchFn, }) {
|
|
110
|
+
return request("POST", getApiEndpoints().scrapersManualRerunBulk, {
|
|
111
|
+
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
112
|
+
json: { scraperId, urls },
|
|
113
|
+
fetchFn,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
export async function getAllResultsApi({ token, sortField = "updatedAt", sortOrder = "DESC", pageSize = 10, page = 1, search = null, dateRangeColumn = null, startAt = null, endAt = null, fetchFn, }) {
|
|
117
|
+
const params = {
|
|
118
|
+
sortField,
|
|
119
|
+
sortOrder,
|
|
120
|
+
pageSize,
|
|
121
|
+
page,
|
|
122
|
+
};
|
|
123
|
+
if (search)
|
|
124
|
+
params.search = search;
|
|
125
|
+
if (dateRangeColumn)
|
|
126
|
+
params.dateRangeColumn = dateRangeColumn;
|
|
127
|
+
if (startAt)
|
|
128
|
+
params.startAt = startAt;
|
|
129
|
+
if (endAt)
|
|
130
|
+
params.endAt = endAt;
|
|
131
|
+
return request("GET", getApiEndpoints().results, {
|
|
132
|
+
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
133
|
+
params,
|
|
134
|
+
fetchFn,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
export async function getResultByIdApi(token, resultId, options = {}) {
|
|
138
|
+
return request("GET", `${getApiEndpoints().results}/${resultId}`, {
|
|
139
|
+
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
140
|
+
fetchFn: options.fetchFn,
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
export async function getSubscriptionAccountApi(token, options = {}) {
|
|
144
|
+
return request("GET", getApiEndpoints().subscriptionAccounts, {
|
|
145
|
+
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
146
|
+
fetchFn: options.fetchFn,
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
export async function getAnalyticStatusesApi({ token, domain, startDate, endDate, action = "", apiTokenName = "", fetchFn, }) {
|
|
150
|
+
return request("GET", getApiEndpoints().analyticStatuses, {
|
|
151
|
+
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
152
|
+
params: { domain, startDate, endDate, action, apiTokenName },
|
|
153
|
+
fetchFn,
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
export function normalizeSerpInput(input) {
|
|
157
|
+
const value = String(input || "").trim();
|
|
158
|
+
if (!value)
|
|
159
|
+
throw new Error("A Google search query or URL is required");
|
|
160
|
+
try {
|
|
161
|
+
const parsed = new URL(value);
|
|
162
|
+
const query = parsed.searchParams.get("q")?.trim();
|
|
163
|
+
if (!query)
|
|
164
|
+
throw new Error("Google search URL must contain a q parameter");
|
|
165
|
+
const start = Number(parsed.searchParams.get("start") || 0);
|
|
166
|
+
return {
|
|
167
|
+
query,
|
|
168
|
+
region: parsed.searchParams.get("gl"),
|
|
169
|
+
language: parsed.searchParams.get("hl"),
|
|
170
|
+
page: Number.isFinite(start) && start > 0 ? Math.floor(start / 10) + 1 : null,
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
catch (error) {
|
|
174
|
+
if (/^https?:\/\//i.test(value))
|
|
175
|
+
throw error;
|
|
176
|
+
return { query: value, region: null, language: null, page: null };
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
export async function googleSerpSyncApi({ token, queryOrUrl, region = null, language = null, page = null, format = "json", renderJs = false, raw = false, timeout = 120, fetchFn, }) {
|
|
180
|
+
const normalized = normalizeSerpInput(queryOrUrl);
|
|
181
|
+
const resolvedFormat = raw ? "html" : format;
|
|
182
|
+
const payload = compact({
|
|
183
|
+
query: normalized.query,
|
|
184
|
+
region: region || normalized.region,
|
|
185
|
+
language: language || normalized.language,
|
|
186
|
+
page: page || normalized.page,
|
|
187
|
+
format: resolvedFormat,
|
|
188
|
+
renderJs: Boolean(renderJs),
|
|
189
|
+
});
|
|
190
|
+
return request("POST", getApiEndpoints().googleSerpSync, {
|
|
191
|
+
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
192
|
+
json: payload,
|
|
193
|
+
timeout,
|
|
194
|
+
fetchFn,
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
export function parseBulkUrls(raw) {
|
|
198
|
+
return raw
|
|
199
|
+
.split(/[,\n]/g)
|
|
200
|
+
.map((part) => part.trim())
|
|
201
|
+
.filter(Boolean);
|
|
202
|
+
}
|
|
203
|
+
//# sourceMappingURL=api.js.map
|
package/dist/api.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"api.js","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAoB,MAAM,WAAW,CAAC;AAStD,MAAM,iBAAiB,GAAG;IACxB,YAAY,EAAE,MAAM;IACpB,iBAAiB,EAAE,GAAG;IACtB,gBAAgB,EAAE,MAAM;CACzB,CAAC;AAEF,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC7C,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACxD,OAAO;QACL,GAAG,iBAAiB;QACpB,aAAa,EAAE,UAAU,QAAQ,EAAE;QACnC,aAAa,EAAE,QAAQ;KACxB,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CACd,MAAoE;IAEpE,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAC3B,CAAC,KAAK,EAAgD,EAAE,CACtD,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS,CAC9C,CACF,CAAC;AACJ,CAAC;AAeD,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,EACpC,KAAK,EACL,GAAG,EACH,OAAO,GAAG,EAAE,EACZ,OAAO,GAAG,IAAI,EACd,gBAAgB,GAAG,KAAK,EACxB,eAAe,GAAG,IAAI,EACtB,QAAQ,GAAG,KAAK,EAChB,cAAc,GAAG,KAAK,EACtB,UAAU,GAAG,CAAC,EACd,QAAQ,GAAG,IAAI,EACf,OAAO,GACa;IACpB,OAAO,OAAO,CAAC,KAAK,EAAE,eAAe,EAAE,CAAC,gBAAgB,EAAE;QACxD,OAAO,EAAE,cAAc,CAAC,KAAK,CAAC;QAC9B,MAAM,EAAE,OAAO,CAAC;YACd,GAAG;YACH,OAAO;YACP,OAAO;YACP,gBAAgB;YAChB,eAAe;YACf,QAAQ;YACR,cAAc;YACd,UAAU;YACV,QAAQ;SACT,CAAC;QACF,OAAO,EAAE,OAAO,GAAG,EAAE;QACrB,OAAO;KACR,CAAC,CAAC;AACL,CAAC;AAeD,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,EACvC,KAAK,EACL,GAAG,EACH,OAAO,EACP,KAAK,GAAG,SAAS,EACjB,YAAY,GAAG,IAAI,EACnB,QAAQ,EACR,QAAQ,EACR,KAAK,EACL,eAAe,EACf,eAAe,EACf,OAAO,GACgB;IACvB,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,KAAK,CAAC,EAAE,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;QACrE,MAAM,IAAI,KAAK,CACb,kEAAkE,CACnE,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,KAAK,KAAK,IAAI,YAAY,KAAK,IAAI,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QAC3E,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,OAAO,GAA4B,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;IACxD,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC/C,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC;QAC1B,IAAI,YAAY,KAAK,IAAI,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YACxD,OAAO,CAAC,YAAY,GAAG,YAAY,CAAC;QACtC,CAAC;QACD,IAAI,KAAK,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAClD,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC9B,CAAC;IACH,CAAC;SAAM,CAAC;QACN,IAAI,OAAO;YACT,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;QACxE,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACxD,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACxD,IAAI,KAAK,KAAK,SAAS;YAAE,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC;QAC/C,IAAI,eAAe,KAAK,SAAS;YAC/B,OAAO,CAAC,eAAe,GAAG,eAAe,CAAC;QAC5C,IAAI,eAAe,KAAK,SAAS;YAC/B,OAAO,CAAC,eAAe,GAAG,eAAe,CAAC;IAC9C,CAAC;IACD,OAAO,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,CAAC,UAAU,EAAE;QACnD,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,EAAE;QACjE,IAAI,EAAE,OAAO;QACb,OAAO;KACR,CAAC,CAAC;AACL,CAAC;AAaD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EACtC,KAAK,EACL,SAAS,EACT,GAAG,EACH,QAAQ,GAAG,CAAC,EACZ,QAAQ,GAAG,EAAE,EACb,KAAK,GAAG,KAAK,EACb,eAAe,GAAG,EAAE,EACpB,eAAe,GAAG,EAAE,EACpB,OAAO,GACQ;IACf,OAAO,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,CAAC,eAAe,EAAE;QACxD,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,EAAE;QACjE,IAAI,EAAE;YACJ,SAAS;YACT,GAAG;YACH,QAAQ;YACR,QAAQ;YACR,KAAK;YACL,eAAe;YACf,eAAe;SAChB;QACD,OAAO;KACR,CAAC,CAAC;AACL,CAAC;AAQD,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,EAC1C,KAAK,EACL,SAAS,EACT,IAAI,EACJ,OAAO,GACU;IACjB,OAAO,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,CAAC,mBAAmB,EAAE;QAC5D,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,EAAE;QACjE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE;QACzB,OAAO;KACR,CAAC,CAAC;AACL,CAAC;AAQD,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,EAC1C,KAAK,EACL,SAAS,EACT,GAAG,EACH,OAAO,GACY;IACnB,OAAO,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,CAAC,mBAAmB,EAAE;QAC5D,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,EAAE;QACjE,IAAI,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE;QACxB,OAAO;KACR,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAAC,EAC9C,KAAK,EACL,SAAS,EACT,IAAI,EACJ,OAAO,GACU;IACjB,OAAO,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,CAAC,uBAAuB,EAAE;QAChE,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,EAAE;QACjE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE;QACzB,OAAO;KACR,CAAC,CAAC;AACL,CAAC;AAcD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EACrC,KAAK,EACL,SAAS,GAAG,WAAW,EACvB,SAAS,GAAG,MAAM,EAClB,QAAQ,GAAG,EAAE,EACb,IAAI,GAAG,CAAC,EACR,MAAM,GAAG,IAAI,EACb,eAAe,GAAG,IAAI,EACtB,OAAO,GAAG,IAAI,EACd,KAAK,GAAG,IAAI,EACZ,OAAO,GACW;IAClB,MAAM,MAAM,GAAoC;QAC9C,SAAS;QACT,SAAS;QACT,QAAQ;QACR,IAAI;KACL,CAAC;IACF,IAAI,MAAM;QAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;IACnC,IAAI,eAAe;QAAE,MAAM,CAAC,eAAe,GAAG,eAAe,CAAC;IAC9D,IAAI,OAAO;QAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;IACtC,IAAI,KAAK;QAAE,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;IAChC,OAAO,OAAO,CAAC,KAAK,EAAE,eAAe,EAAE,CAAC,OAAO,EAAE;QAC/C,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,EAAE;QACjE,MAAM;QACN,OAAO;KACR,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,KAAa,EACb,QAAgB,EAChB,OAAO,GAAoB,EAAE;IAE7B,OAAO,OAAO,CAAC,KAAK,EAAE,GAAG,eAAe,EAAE,CAAC,OAAO,IAAI,QAAQ,EAAE,EAAE;QAChE,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,EAAE;QACjE,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,KAAa,EACb,OAAO,GAAoB,EAAE;IAE7B,OAAO,OAAO,CAAC,KAAK,EAAE,eAAe,EAAE,CAAC,oBAAoB,EAAE;QAC5D,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,EAAE;QACjE,OAAO,EAAE,OAAO,CAAC,OAAO;KACzB,CAAC,CAAC;AACL,CAAC;AAWD,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,EAC3C,KAAK,EACL,MAAM,EACN,SAAS,EACT,OAAO,EACP,MAAM,GAAG,EAAE,EACX,YAAY,GAAG,EAAE,EACjB,OAAO,GACiB;IACxB,OAAO,OAAO,CAAC,KAAK,EAAE,eAAe,EAAE,CAAC,gBAAgB,EAAE;QACxD,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,EAAE;QACjE,MAAM,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE;QAC5D,OAAO;KACR,CAAC,CAAC;AACL,CAAC;AASD,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC9C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAExE,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC;QACnD,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAC5E,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5D,OAAO;YACL,KAAK;YACL,MAAM,EAAE,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC;YACrC,QAAQ,EAAE,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC;YACvC,IAAI,EACF,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;SAC1E,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,MAAM,KAAK,CAAC;QAC7C,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACpE,CAAC;AACH,CAAC;AAcD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EACtC,KAAK,EACL,UAAU,EACV,MAAM,GAAG,IAAI,EACb,QAAQ,GAAG,IAAI,EACf,IAAI,GAAG,IAAI,EACX,MAAM,GAAG,MAAM,EACf,QAAQ,GAAG,KAAK,EAChB,GAAG,GAAG,KAAK,EACX,OAAO,GAAG,GAAG,EACb,OAAO,GACW;IAClB,MAAM,UAAU,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC;IAClD,MAAM,cAAc,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAC7C,MAAM,OAAO,GAAG,OAAO,CAAC;QACtB,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,MAAM,EAAE,MAAM,IAAI,UAAU,CAAC,MAAM;QACnC,QAAQ,EAAE,QAAQ,IAAI,UAAU,CAAC,QAAQ;QACzC,IAAI,EAAE,IAAI,IAAI,UAAU,CAAC,IAAI;QAC7B,MAAM,EAAE,cAAc;QACtB,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC;KAC5B,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,CAAC,cAAc,EAAE;QACvD,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,EAAE;QACjE,IAAI,EAAE,OAAO;QACb,OAAO;QACP,OAAO;KACR,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,GAAW;IACvC,OAAO,GAAG;SACP,KAAK,CAAC,QAAQ,CAAC;SACf,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,OAAO,CAAC,CAAC;AACrB,CAAC"}
|
package/dist/app.d.ts
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type McpHttpHandler } from "@modelcontextprotocol/server";
|
|
2
|
+
import type { Express } from "express";
|
|
3
|
+
import { type ServerFactoryOptions } from "./server.js";
|
|
4
|
+
export interface HttpAppOptions extends ServerFactoryOptions {
|
|
5
|
+
host?: string;
|
|
6
|
+
allowedOrigins?: string[];
|
|
7
|
+
authEnabled?: boolean;
|
|
8
|
+
validateToken?: (token: string) => Promise<boolean>;
|
|
9
|
+
logPayloads?: boolean;
|
|
10
|
+
payloadLogMax?: number;
|
|
11
|
+
}
|
|
12
|
+
export interface HttpApp {
|
|
13
|
+
app: Express;
|
|
14
|
+
handler: McpHttpHandler;
|
|
15
|
+
}
|
|
16
|
+
export declare function createHttpApp(options?: HttpAppOptions): HttpApp;
|
|
17
|
+
//# sourceMappingURL=app.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"app.d.ts","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AAKA,OAAO,EAEL,KAAK,cAAc,EACpB,MAAM,8BAA8B,CAAC;AACtC,OAAO,KAAK,EAAE,OAAO,EAAmC,MAAM,SAAS,CAAC;AASxE,OAAO,EAAuB,KAAK,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAE7E,MAAM,WAAW,cAAe,SAAQ,oBAAoB;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;IAC1B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACpD,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,OAAO;IACtB,GAAG,EAAE,OAAO,CAAC;IACb,OAAO,EAAE,cAAc,CAAC;CACzB;AA8ED,wBAAgB,aAAa,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAmEnE"}
|
package/dist/app.js
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { createMcpExpressApp, requireBearerAuth, } from "@modelcontextprotocol/express";
|
|
2
|
+
import { toNodeHandler } from "@modelcontextprotocol/node";
|
|
3
|
+
import { createMcpHandler, } from "@modelcontextprotocol/server";
|
|
4
|
+
import { createTokenVerifier } from "./auth.js";
|
|
5
|
+
import { TOOL_NAMES, VERSION, httpAuthEnabled, httpRuntimeConfig, } from "./config.js";
|
|
6
|
+
import { createServerFactory } from "./server.js";
|
|
7
|
+
const LOCAL_ORIGIN_HOSTNAMES = ["localhost", "127.0.0.1", "[::1]"];
|
|
8
|
+
function normalizeAllowedOrigins(origins) {
|
|
9
|
+
const normalized = origins.map((origin) => {
|
|
10
|
+
try {
|
|
11
|
+
const parsed = new URL(origin);
|
|
12
|
+
if (!["http:", "https:"].includes(parsed.protocol) ||
|
|
13
|
+
parsed.pathname !== "/" ||
|
|
14
|
+
parsed.search ||
|
|
15
|
+
parsed.hash) {
|
|
16
|
+
throw new Error("not an HTTP origin");
|
|
17
|
+
}
|
|
18
|
+
return parsed;
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
throw new Error(`Invalid MRSCRAPER_ALLOWED_ORIGINS entry ${JSON.stringify(origin)}; expected a full origin such as https://agent.example.com`);
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
return {
|
|
25
|
+
origins: new Set(normalized.map((origin) => origin.origin)),
|
|
26
|
+
hostnames: [
|
|
27
|
+
...new Set([
|
|
28
|
+
...LOCAL_ORIGIN_HOSTNAMES,
|
|
29
|
+
...normalized.map((origin) => origin.hostname),
|
|
30
|
+
]),
|
|
31
|
+
],
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
function exactOriginProtection(allowedOrigins) {
|
|
35
|
+
return (request, response, next) => {
|
|
36
|
+
const origin = request.get("origin");
|
|
37
|
+
if (!origin) {
|
|
38
|
+
next();
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
try {
|
|
42
|
+
const parsed = new URL(origin);
|
|
43
|
+
if (LOCAL_ORIGIN_HOSTNAMES.includes(parsed.hostname) ||
|
|
44
|
+
allowedOrigins.has(parsed.origin)) {
|
|
45
|
+
next();
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
// Malformed and untrusted origins are handled by the same 403 response.
|
|
51
|
+
}
|
|
52
|
+
response.status(403).send("Forbidden Origin");
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
function requestPayloadLogger(enabled, maxCharacters) {
|
|
56
|
+
return (request, _response, next) => {
|
|
57
|
+
if (enabled &&
|
|
58
|
+
["POST", "PUT", "PATCH", "DELETE"].includes(request.method) &&
|
|
59
|
+
request.body !== undefined) {
|
|
60
|
+
const serialized = JSON.stringify(request.body);
|
|
61
|
+
const rendered = serialized.length > maxCharacters
|
|
62
|
+
? `${serialized.slice(0, maxCharacters)}... (truncated)`
|
|
63
|
+
: serialized;
|
|
64
|
+
console.error(`${request.method} ${request.path} payload: ${rendered}`);
|
|
65
|
+
}
|
|
66
|
+
next();
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
export function createHttpApp(options = {}) {
|
|
70
|
+
const runtime = httpRuntimeConfig();
|
|
71
|
+
const host = options.host || runtime.host;
|
|
72
|
+
const origins = options.allowedOrigins || runtime.allowedOrigins;
|
|
73
|
+
const allowedOrigins = normalizeAllowedOrigins(origins);
|
|
74
|
+
const app = createMcpExpressApp({
|
|
75
|
+
host,
|
|
76
|
+
allowedOrigins: allowedOrigins.hostnames,
|
|
77
|
+
});
|
|
78
|
+
const handler = createMcpHandler(createServerFactory(options), {
|
|
79
|
+
onerror: (error) => console.error("MCP request error:", error),
|
|
80
|
+
});
|
|
81
|
+
const nodeHandler = toNodeHandler(handler);
|
|
82
|
+
const logPayloads = options.logPayloads ??
|
|
83
|
+
["1", "true", "yes"].includes((process.env.MRSCRAPER_LOG_HTTP_PAYLOAD || "").toLowerCase());
|
|
84
|
+
const configuredMax = Number(options.payloadLogMax ?? process.env.MRSCRAPER_LOG_HTTP_PAYLOAD_MAX ?? 8192);
|
|
85
|
+
const payloadLogMax = Number.isInteger(configuredMax) && configuredMax > 0 ? configuredMax : 8192;
|
|
86
|
+
app.use(exactOriginProtection(allowedOrigins.origins));
|
|
87
|
+
app.use(requestPayloadLogger(logPayloads, payloadLogMax));
|
|
88
|
+
app.get("/health", (_request, response) => {
|
|
89
|
+
response.json({ status: "ok", service: "mrscraper-mcp", version: VERSION });
|
|
90
|
+
});
|
|
91
|
+
app.get("/ready", (_request, response) => {
|
|
92
|
+
response.json({
|
|
93
|
+
status: "ready",
|
|
94
|
+
service: "mrscraper-mcp",
|
|
95
|
+
version: VERSION,
|
|
96
|
+
tools: TOOL_NAMES.length,
|
|
97
|
+
});
|
|
98
|
+
});
|
|
99
|
+
const authEnabled = options.authEnabled ?? httpAuthEnabled();
|
|
100
|
+
const auth = authEnabled
|
|
101
|
+
? [
|
|
102
|
+
requireBearerAuth({
|
|
103
|
+
verifier: createTokenVerifier(options.validateToken),
|
|
104
|
+
}),
|
|
105
|
+
]
|
|
106
|
+
: [];
|
|
107
|
+
app.all("/mcp", ...auth, async (request, response, next) => {
|
|
108
|
+
try {
|
|
109
|
+
await nodeHandler(request, response, request.body);
|
|
110
|
+
}
|
|
111
|
+
catch (error) {
|
|
112
|
+
next(error);
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
app.use((error, _request, response, _next) => {
|
|
116
|
+
console.error("HTTP server error:", error);
|
|
117
|
+
if (!response.headersSent) {
|
|
118
|
+
response.status(500).json({ error: "Internal server error" });
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
return { app, handler };
|
|
122
|
+
}
|
|
123
|
+
//# sourceMappingURL=app.js.map
|
package/dist/app.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"app.js","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mBAAmB,EACnB,iBAAiB,GAClB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EACL,gBAAgB,GAEjB,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EACL,UAAU,EACV,OAAO,EACP,eAAe,EACf,iBAAiB,GAClB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,mBAAmB,EAA6B,MAAM,aAAa,CAAC;AAgB7E,MAAM,sBAAsB,GAAG,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;AAEnE,SAAS,uBAAuB,CAAC,OAAiB;IAIhD,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACxC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;YAC/B,IACE,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAC9C,MAAM,CAAC,QAAQ,KAAK,GAAG;gBACvB,MAAM,CAAC,MAAM;gBACb,MAAM,CAAC,IAAI,EACX,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;YACxC,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CACb,2CAA2C,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,4DAA4D,CAC9H,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO;QACL,OAAO,EAAE,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3D,SAAS,EAAE;YACT,GAAG,IAAI,GAAG,CAAC;gBACT,GAAG,sBAAsB;gBACzB,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC;aAC/C,CAAC;SACH;KACF,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,cAA2B;IACxD,OAAO,CAAC,OAAgB,EAAE,QAAkB,EAAE,IAAkB,EAAQ,EAAE;QACxE,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,IAAI,EAAE,CAAC;YACP,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;YAC/B,IACE,sBAAsB,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAChD,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EACjC,CAAC;gBACD,IAAI,EAAE,CAAC;gBACP,OAAO;YACT,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,wEAAwE;QAC1E,CAAC;QACD,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAChD,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAgB,EAAE,aAAqB;IACnE,OAAO,CAAC,OAAgB,EAAE,SAAmB,EAAE,IAAkB,EAAQ,EAAE;QACzE,IACE,OAAO;YACP,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC;YAC3D,OAAO,CAAC,IAAI,KAAK,SAAS,EAC1B,CAAC;YACD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAChD,MAAM,QAAQ,GACZ,UAAU,CAAC,MAAM,GAAG,aAAa;gBAC/B,CAAC,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,iBAAiB;gBACxD,CAAC,CAAC,UAAU,CAAC;YACjB,OAAO,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,aAAa,QAAQ,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,IAAI,EAAE,CAAC;IACT,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAO,GAAmB,EAAE;IACxD,MAAM,OAAO,GAAG,iBAAiB,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAC1C,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,cAAc,CAAC;IACjE,MAAM,cAAc,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;IACxD,MAAM,GAAG,GAAG,mBAAmB,CAAC;QAC9B,IAAI;QACJ,cAAc,EAAE,cAAc,CAAC,SAAS;KACzC,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,gBAAgB,CAAC,mBAAmB,CAAC,OAAO,CAAC,EAAE;QAC7D,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,KAAK,CAAC;KAC/D,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,WAAW,GACf,OAAO,CAAC,WAAW;QACnB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,QAAQ,CAC3B,CAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAC7D,CAAC;IACJ,MAAM,aAAa,GAAG,MAAM,CAC1B,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,IAAI,CAC5E,CAAC;IACF,MAAM,aAAa,GACjB,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC;IAE9E,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC;IACvD,GAAG,CAAC,GAAG,CAAC,oBAAoB,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC;IAC1D,GAAG,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE;QACxC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAC9E,CAAC,CAAC,CAAC;IACH,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE;QACvC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,OAAO;YACf,OAAO,EAAE,eAAe;YACxB,OAAO,EAAE,OAAO;YAChB,KAAK,EAAE,UAAU,CAAC,MAAM;SACzB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,eAAe,EAAE,CAAC;IAC7D,MAAM,IAAI,GAAG,WAAW;QACtB,CAAC,CAAC;YACE,iBAAiB,CAAC;gBAChB,QAAQ,EAAE,mBAAmB,CAAC,OAAO,CAAC,aAAa,CAAC;aACrD,CAAC;SACH;QACH,CAAC,CAAC,EAAE,CAAC;IACP,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE;QACzD,IAAI,CAAC;YACH,MAAM,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IACH,GAAG,CAAC,GAAG,CACL,CACE,KAAc,EACd,QAAiB,EACjB,QAAkB,EAClB,KAAmB,EACnB,EAAE;QACF,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC1B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC;QAChE,CAAC;IACH,CAAC,CACF,CAAC;IACF,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AAC1B,CAAC"}
|
package/dist/auth.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { McpRequestContext, OAuthTokenVerifier } from "@modelcontextprotocol/server";
|
|
2
|
+
export declare const MISSING_TOKEN_MESSAGE = "MrScraper API token is required. For HTTP, configure the MCP client with \"headers\": {\"Authorization\": \"Bearer <your-token>\"}. For stdio, set MRSCRAPER_API_KEY or MRSCRAPER_API_TOKEN.";
|
|
3
|
+
export declare function normalizeBearerToken(token: string): string;
|
|
4
|
+
export declare function resolveApiToken(context?: Partial<McpRequestContext>): string;
|
|
5
|
+
export declare function validateApiToken(token: string, fetchFn?: typeof fetch): Promise<boolean>;
|
|
6
|
+
export declare function createTokenVerifier(validate?: (token: string) => Promise<boolean>): OAuthTokenVerifier;
|
|
7
|
+
//# sourceMappingURL=auth.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"auth.d.ts","sourceRoot":"","sources":["../src/auth.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,iBAAiB,EACjB,kBAAkB,EACnB,MAAM,8BAA8B,CAAC;AAQtC,eAAO,MAAM,qBAAqB,iMACwJ,CAAC;AAE3L,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAK1D;AAUD,wBAAgB,eAAe,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,MAAM,CAgB5E;AAED,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,OAAO,KAAK,GACrB,OAAO,CAAC,OAAO,CAAC,CAalB;AAED,wBAAgB,mBAAmB,CACjC,QAAQ,GAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAoB,GAC/D,kBAAkB,CAiBpB"}
|
package/dist/auth.js
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { OAuthError, OAuthErrorCode } from "@modelcontextprotocol/server";
|
|
2
|
+
import { getApiEndpoints } from "./config.js";
|
|
3
|
+
import { request } from "./http.js";
|
|
4
|
+
const TOKEN_VALIDATE_TIMEOUT_SECONDS = 15;
|
|
5
|
+
export const MISSING_TOKEN_MESSAGE = 'MrScraper API token is required. For HTTP, configure the MCP client with "headers": {"Authorization": "Bearer <your-token>"}. For stdio, set MRSCRAPER_API_KEY or MRSCRAPER_API_TOKEN.';
|
|
6
|
+
export function normalizeBearerToken(token) {
|
|
7
|
+
return token
|
|
8
|
+
.trim()
|
|
9
|
+
.replace(/^bearer\s+/i, "")
|
|
10
|
+
.trim();
|
|
11
|
+
}
|
|
12
|
+
function environmentApiToken() {
|
|
13
|
+
for (const name of ["MRSCRAPER_API_KEY", "MRSCRAPER_API_TOKEN"]) {
|
|
14
|
+
const value = process.env[name]?.trim();
|
|
15
|
+
if (value)
|
|
16
|
+
return value;
|
|
17
|
+
}
|
|
18
|
+
return undefined;
|
|
19
|
+
}
|
|
20
|
+
export function resolveApiToken(context) {
|
|
21
|
+
const verified = context?.authInfo?.token?.trim();
|
|
22
|
+
if (verified)
|
|
23
|
+
return normalizeBearerToken(verified);
|
|
24
|
+
const requestInfo = context?.requestInfo;
|
|
25
|
+
if (requestInfo) {
|
|
26
|
+
for (const name of ["x-api-token", "authorization"]) {
|
|
27
|
+
const value = requestInfo.headers.get(name)?.trim();
|
|
28
|
+
if (value)
|
|
29
|
+
return normalizeBearerToken(value);
|
|
30
|
+
}
|
|
31
|
+
throw new Error(MISSING_TOKEN_MESSAGE);
|
|
32
|
+
}
|
|
33
|
+
const environmentToken = environmentApiToken();
|
|
34
|
+
if (environmentToken)
|
|
35
|
+
return environmentToken;
|
|
36
|
+
throw new Error(MISSING_TOKEN_MESSAGE);
|
|
37
|
+
}
|
|
38
|
+
export async function validateApiToken(token, fetchFn) {
|
|
39
|
+
const apiToken = normalizeBearerToken(token);
|
|
40
|
+
if (!apiToken)
|
|
41
|
+
return false;
|
|
42
|
+
const response = await request("GET", getApiEndpoints().subscriptionAccounts, {
|
|
43
|
+
headers: { accept: "application/json", "x-api-token": apiToken },
|
|
44
|
+
timeout: TOKEN_VALIDATE_TIMEOUT_SECONDS,
|
|
45
|
+
fetchFn,
|
|
46
|
+
});
|
|
47
|
+
return response.status_code === 200 && !response.error;
|
|
48
|
+
}
|
|
49
|
+
export function createTokenVerifier(validate = validateApiToken) {
|
|
50
|
+
return {
|
|
51
|
+
async verifyAccessToken(token) {
|
|
52
|
+
if (!(await validate(token))) {
|
|
53
|
+
throw new OAuthError(OAuthErrorCode.InvalidToken, "Invalid MrScraper API token");
|
|
54
|
+
}
|
|
55
|
+
return {
|
|
56
|
+
token: normalizeBearerToken(token),
|
|
57
|
+
clientId: "mrscraper-mcp",
|
|
58
|
+
scopes: [],
|
|
59
|
+
expiresAt: Math.floor(Date.now() / 1_000) + 300,
|
|
60
|
+
};
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
//# sourceMappingURL=auth.js.map
|
package/dist/auth.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"auth.js","sourceRoot":"","sources":["../src/auth.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAE1E,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,MAAM,8BAA8B,GAAG,EAAE,CAAC;AAE1C,MAAM,CAAC,MAAM,qBAAqB,GAChC,wLAAwL,CAAC;AAE3L,MAAM,UAAU,oBAAoB,CAAC,KAAa;IAChD,OAAO,KAAK;SACT,IAAI,EAAE;SACN,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;SAC1B,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,mBAAmB;IAC1B,KAAK,MAAM,IAAI,IAAI,CAAC,mBAAmB,EAAE,qBAAqB,CAAC,EAAE,CAAC;QAChE,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC;QACxC,IAAI,KAAK;YAAE,OAAO,KAAK,CAAC;IAC1B,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAAoC;IAClE,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAClD,IAAI,QAAQ;QAAE,OAAO,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAEpD,MAAM,WAAW,GAAG,OAAO,EAAE,WAAW,CAAC;IACzC,IAAI,WAAW,EAAE,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC;YACpD,IAAI,KAAK;gBAAE,OAAO,oBAAoB,CAAC,KAAK,CAAC,CAAC;QAChD,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IAED,MAAM,gBAAgB,GAAG,mBAAmB,EAAE,CAAC;IAC/C,IAAI,gBAAgB;QAAE,OAAO,gBAAgB,CAAC;IAC9C,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;AACzC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,KAAa,EACb,OAAsB;IAEtB,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC7C,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5B,MAAM,QAAQ,GAAG,MAAM,OAAO,CAC5B,KAAK,EACL,eAAe,EAAE,CAAC,oBAAoB,EACtC;QACE,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,aAAa,EAAE,QAAQ,EAAE;QAChE,OAAO,EAAE,8BAA8B;QACvC,OAAO;KACR,CACF,CAAC;IACF,OAAO,QAAQ,CAAC,WAAW,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,QAAQ,GAAwC,gBAAgB;IAEhE,OAAO;QACL,KAAK,CAAC,iBAAiB,CAAC,KAAa;YACnC,IAAI,CAAC,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,UAAU,CAClB,cAAc,CAAC,YAAY,EAC3B,6BAA6B,CAC9B,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,KAAK,EAAE,oBAAoB,CAAC,KAAK,CAAC;gBAClC,QAAQ,EAAE,eAAe;gBACzB,MAAM,EAAE,EAAE;gBACV,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,GAAG,GAAG;aAChD,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/dist/bin.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bin.d.ts","sourceRoot":"","sources":["../src/bin.ts"],"names":[],"mappings":";AAEA,OAAO,eAAe,CAAC"}
|
package/dist/bin.js
ADDED
package/dist/bin.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bin.js","sourceRoot":"","sources":["../src/bin.ts"],"names":[],"mappings":";AAEA,OAAO,eAAe,CAAC;AAEvB,OAAO,EAAE,GAAG,EAAE,MAAM,cAAc,CAAC;AAEnC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IAC7B,OAAO,CAAC,KAAK,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IACtE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export declare const MANUAL_SCRAPER_COMPLIANCE_WARNING = "### Compliance & Legal Risk\nWARNING\n**Scraping login-protected pages carries serious legal and compliance risks.** Many websites explicitly prohibit automated access in their Terms of Service, and bypassing authentication to scrape content may expose you to legal action including lawsuits, account termination, and financial penalties. By proceeding on scraping login-protected pages, you confirm that you have read and understood the target website's Terms of Service, and you **fully accept all legal, financial, and ethical responsibility** for your actions.";
|
|
2
|
+
export declare const MANUAL_SCRAPER_AGENT_COMPLIANCE_NOTES = "Agent compliance (manual scrapers):\n- Before the FIRST call to any manual scraper tool in this conversation, show the user this warning exactly once and wait for acknowledgment. Do not call manual scraper tools until the user accepts the risk.\n- Do not repeat this warning on later manual scraper calls in the same conversation.\n\n### Compliance & Legal Risk\nWARNING\n**Scraping login-protected pages carries serious legal and compliance risks.** Many websites explicitly prohibit automated access in their Terms of Service, and bypassing authentication to scrape content may expose you to legal action including lawsuits, account termination, and financial penalties. By proceeding on scraping login-protected pages, you confirm that you have read and understood the target website's Terms of Service, and you **fully accept all legal, financial, and ethical responsibility** for your actions.";
|
|
3
|
+
export declare const MANUAL_SCRAPER_SERVER_INSTRUCTIONS = "Before the first manual scraper tool call in a conversation, agents must show the Compliance & Legal Risk warning once and obtain user acknowledgment; do not repeat the warning on later manual scraper calls.";
|
|
4
|
+
//# sourceMappingURL=compliance.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compliance.d.ts","sourceRoot":"","sources":["../src/compliance.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,iCAAiC,yjBAEie,CAAC;AAEhhB,eAAO,MAAM,qCAAqC,u4BAIb,CAAC;AAEtC,eAAO,MAAM,kCAAkC,oNACoK,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export const MANUAL_SCRAPER_COMPLIANCE_WARNING = `### Compliance & Legal Risk
|
|
2
|
+
WARNING
|
|
3
|
+
**Scraping login-protected pages carries serious legal and compliance risks.** Many websites explicitly prohibit automated access in their Terms of Service, and bypassing authentication to scrape content may expose you to legal action including lawsuits, account termination, and financial penalties. By proceeding on scraping login-protected pages, you confirm that you have read and understood the target website's Terms of Service, and you **fully accept all legal, financial, and ethical responsibility** for your actions.`;
|
|
4
|
+
export const MANUAL_SCRAPER_AGENT_COMPLIANCE_NOTES = `Agent compliance (manual scrapers):
|
|
5
|
+
- Before the FIRST call to any manual scraper tool in this conversation, show the user this warning exactly once and wait for acknowledgment. Do not call manual scraper tools until the user accepts the risk.
|
|
6
|
+
- Do not repeat this warning on later manual scraper calls in the same conversation.
|
|
7
|
+
|
|
8
|
+
${MANUAL_SCRAPER_COMPLIANCE_WARNING}`;
|
|
9
|
+
export const MANUAL_SCRAPER_SERVER_INSTRUCTIONS = "Before the first manual scraper tool call in a conversation, agents must show the Compliance & Legal Risk warning once and obtain user acknowledgment; do not repeat the warning on later manual scraper calls.";
|
|
10
|
+
//# sourceMappingURL=compliance.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compliance.js","sourceRoot":"","sources":["../src/compliance.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,iCAAiC,GAAG;;+gBAE8d,CAAC;AAEhhB,MAAM,CAAC,MAAM,qCAAqC,GAAG;;;;EAInD,iCAAiC,EAAE,CAAC;AAEtC,MAAM,CAAC,MAAM,kCAAkC,GAC7C,iNAAiN,CAAC"}
|