@mrscraper/mcp 0.1.3 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +235 -409
- package/dist/api.d.ts +26 -87
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +71 -188
- package/dist/api.js.map +1 -1
- package/dist/config.d.ts +5 -12
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +6 -25
- package/dist/config.js.map +1 -1
- package/dist/http.d.ts.map +1 -1
- package/dist/http.js +5 -1
- package/dist/http.js.map +1 -1
- package/dist/index.d.ts +0 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -2
- package/dist/index.js.map +1 -1
- package/dist/scopes.d.ts.map +1 -1
- package/dist/scopes.js +2 -6
- package/dist/scopes.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +3 -16
- package/dist/server.js.map +1 -1
- package/dist/status.d.ts +0 -2
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +2 -30
- package/dist/status.js.map +1 -1
- package/dist/tools.d.ts +30 -99
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +184 -637
- package/dist/tools.js.map +1 -1
- package/dist/widgets/bundles.generated.d.ts.map +1 -1
- package/dist/widgets/bundles.generated.js +2 -3
- package/dist/widgets/bundles.generated.js.map +1 -1
- package/dist/widgets/index.d.ts +1 -7
- package/dist/widgets/index.d.ts.map +1 -1
- package/dist/widgets/index.js +2 -8
- package/dist/widgets/index.js.map +1 -1
- package/package.json +4 -4
- package/dist/compliance.d.ts +0 -4
- package/dist/compliance.d.ts.map +0 -1
- package/dist/compliance.js +0 -10
- package/dist/compliance.js.map +0 -1
package/dist/api.d.ts
CHANGED
|
@@ -1,114 +1,53 @@
|
|
|
1
1
|
import { type ApiResponse } from "./http.js";
|
|
2
|
-
export type
|
|
3
|
-
export type ScrapeMode = "Cheap" | "Super";
|
|
4
|
-
export type SerpFormat = "json" | "html";
|
|
2
|
+
export type SearchFormat = "json" | "html";
|
|
5
3
|
interface FetchDependency {
|
|
6
4
|
fetchFn?: typeof fetch;
|
|
7
5
|
}
|
|
8
6
|
export declare function getAuthHeaders(token: string): Record<string, string>;
|
|
9
|
-
export interface
|
|
7
|
+
export interface ScraperApiOptions extends FetchDependency {
|
|
10
8
|
token: string;
|
|
11
9
|
url: string;
|
|
10
|
+
html?: boolean;
|
|
11
|
+
markdown?: boolean;
|
|
12
12
|
timeout?: number;
|
|
13
|
-
geoCode?: string
|
|
14
|
-
|
|
15
|
-
|
|
13
|
+
geoCode?: string;
|
|
14
|
+
proxyCountry?: string | null;
|
|
15
|
+
blockResources?: boolean | null;
|
|
16
|
+
browserRendering?: boolean | null;
|
|
17
|
+
super?: boolean;
|
|
18
|
+
useProxy?: boolean;
|
|
19
|
+
proxy?: string | null;
|
|
16
20
|
waitForSelector?: string | null;
|
|
17
21
|
homePage?: boolean;
|
|
18
|
-
|
|
22
|
+
screenshot?: boolean;
|
|
23
|
+
bypassProxy?: boolean;
|
|
24
|
+
cookieJar?: string | null;
|
|
25
|
+
headers?: Record<string, string> | string | null;
|
|
19
26
|
maxRetries?: number;
|
|
20
27
|
tokenCap?: number | null;
|
|
28
|
+
stream?: boolean | null;
|
|
29
|
+
discardTokenSecret?: string;
|
|
21
30
|
}
|
|
22
|
-
export declare function
|
|
23
|
-
export interface
|
|
24
|
-
token: string;
|
|
25
|
-
url: string;
|
|
26
|
-
message?: string;
|
|
27
|
-
agent?: Agent;
|
|
28
|
-
mode?: ScrapeMode;
|
|
29
|
-
proxyCountry?: string | null;
|
|
30
|
-
maxDepth?: number;
|
|
31
|
-
maxPages?: number;
|
|
32
|
-
limit?: number;
|
|
33
|
-
includePatterns?: string;
|
|
34
|
-
excludePatterns?: string;
|
|
35
|
-
}
|
|
36
|
-
export declare function createAiScraperApi({ token, url, message, agent, mode, proxyCountry, maxDepth, maxPages, limit, includePatterns, excludePatterns, fetchFn, }: CreateAiScraperOptions): Promise<ApiResponse>;
|
|
37
|
-
export interface RerunAiOptions extends FetchDependency {
|
|
38
|
-
token: string;
|
|
39
|
-
scraperId: string;
|
|
40
|
-
url: string;
|
|
41
|
-
maxDepth?: number;
|
|
42
|
-
maxPages?: number;
|
|
43
|
-
limit?: number;
|
|
44
|
-
includePatterns?: string;
|
|
45
|
-
excludePatterns?: string;
|
|
46
|
-
proxyCountry?: string;
|
|
47
|
-
maxRetry?: number;
|
|
48
|
-
timeout?: number;
|
|
49
|
-
}
|
|
50
|
-
export declare function rerunAiScraperApi({ token, scraperId, url, maxDepth, maxPages, limit, includePatterns, excludePatterns, proxyCountry, maxRetry, timeout, fetchFn, }: RerunAiOptions): Promise<ApiResponse>;
|
|
51
|
-
interface BulkRerunOptions extends FetchDependency {
|
|
52
|
-
token: string;
|
|
53
|
-
scraperId: string;
|
|
54
|
-
urls: string[];
|
|
55
|
-
}
|
|
56
|
-
export declare function bulkRerunAiScraperApi({ token, scraperId, urls, fetchFn, }: BulkRerunOptions): Promise<ApiResponse>;
|
|
57
|
-
interface ManualRerunOptions extends FetchDependency {
|
|
58
|
-
token: string;
|
|
59
|
-
scraperId: string;
|
|
60
|
-
url: string;
|
|
61
|
-
}
|
|
62
|
-
export declare function rerunManualScraperApi({ token, scraperId, url, fetchFn, }: ManualRerunOptions): Promise<ApiResponse>;
|
|
63
|
-
export declare function bulkRerunManualScraperApi({ token, scraperId, urls, fetchFn, }: BulkRerunOptions): Promise<ApiResponse>;
|
|
64
|
-
export interface GetResultsOptions extends FetchDependency {
|
|
65
|
-
token: string;
|
|
66
|
-
sortField?: string;
|
|
67
|
-
sortOrder?: string;
|
|
68
|
-
pageSize?: number;
|
|
69
|
-
page?: number;
|
|
70
|
-
search?: string | null;
|
|
71
|
-
dateRangeColumn?: string | null;
|
|
72
|
-
startAt?: string | null;
|
|
73
|
-
endAt?: string | null;
|
|
74
|
-
scraperId?: string | null;
|
|
75
|
-
status?: string | null;
|
|
76
|
-
type?: string | null;
|
|
77
|
-
url?: string | null;
|
|
78
|
-
}
|
|
79
|
-
export declare function getAllResultsApi({ token, sortField, sortOrder, pageSize, page, search, dateRangeColumn, startAt, endAt, scraperId, status, type, url, fetchFn, }: GetResultsOptions): Promise<ApiResponse>;
|
|
80
|
-
export declare function getResultByIdApi(token: string, resultId: string, options?: FetchDependency & {
|
|
81
|
-
includeHtml?: boolean;
|
|
82
|
-
}): Promise<ApiResponse>;
|
|
83
|
-
export declare function getSubscriptionAccountApi(token: string, options?: FetchDependency): Promise<ApiResponse>;
|
|
84
|
-
export interface AnalyticStatusesOptions extends FetchDependency {
|
|
85
|
-
token: string;
|
|
86
|
-
domain: string;
|
|
87
|
-
startDate: string;
|
|
88
|
-
endDate: string;
|
|
89
|
-
action?: string;
|
|
90
|
-
apiTokenName?: string;
|
|
91
|
-
}
|
|
92
|
-
export declare function getAnalyticStatusesApi({ token, domain, startDate, endDate, action, apiTokenName, fetchFn, }: AnalyticStatusesOptions): Promise<ApiResponse>;
|
|
93
|
-
export interface NormalizedSerpInput {
|
|
31
|
+
export declare function scraperApi({ token, url, html, markdown, timeout, geoCode, proxyCountry, blockResources, browserRendering, super: superMode, useProxy, proxy, waitForSelector, homePage, screenshot, bypassProxy, cookieJar, headers, maxRetries, tokenCap, stream, discardTokenSecret, fetchFn, }: ScraperApiOptions): Promise<ApiResponse>;
|
|
32
|
+
export interface NormalizedSearchInput {
|
|
94
33
|
query: string;
|
|
95
34
|
region: string | null;
|
|
96
35
|
language: string | null;
|
|
97
36
|
page: number | null;
|
|
98
37
|
}
|
|
99
|
-
export declare function
|
|
100
|
-
export interface
|
|
38
|
+
export declare function normalizeSearchInput(input: string): NormalizedSearchInput;
|
|
39
|
+
export interface WebSearchOptions extends FetchDependency {
|
|
101
40
|
token: string;
|
|
102
|
-
|
|
41
|
+
url: string;
|
|
42
|
+
raw?: boolean;
|
|
103
43
|
region?: string | null;
|
|
104
44
|
language?: string | null;
|
|
105
45
|
page?: number | null;
|
|
106
|
-
format?:
|
|
46
|
+
format?: SearchFormat;
|
|
107
47
|
renderJs?: boolean;
|
|
108
|
-
raw?: boolean;
|
|
109
48
|
timeout?: number;
|
|
110
49
|
}
|
|
111
|
-
export declare function
|
|
112
|
-
export declare function
|
|
50
|
+
export declare function webSearchApi({ token, url, raw, region, language, page, format, renderJs, timeout, fetchFn, }: WebSearchOptions): Promise<ApiResponse>;
|
|
51
|
+
export declare function getSubscriptionAccountApi(token: string, { fetchFn }?: FetchDependency): Promise<ApiResponse>;
|
|
113
52
|
export {};
|
|
114
53
|
//# sourceMappingURL=api.d.ts.map
|
package/dist/api.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAEA,OAAO,EAAW,KAAK,WAAW,EAAE,MAAM,WAAW,CAAC;AAGtD,MAAM,MAAM,
|
|
1
|
+
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAEA,OAAO,EAAW,KAAK,WAAW,EAAE,MAAM,WAAW,CAAC;AAGtD,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,MAAM,CAAC;AAE3C,UAAU,eAAe;IACvB,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACxB;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAUpE;AAaD,MAAM,WAAW,iBAAkB,SAAQ,eAAe;IACxD,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,cAAc,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAChC,gBAAgB,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAClC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC;IACjD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,MAAM,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,wBAAsB,UAAU,CAAC,EAC/B,KAAK,EACL,GAAG,EACH,IAAW,EACX,QAAgB,EAChB,OAAa,EACb,OAAc,EACd,YAAmB,EACnB,cAAsB,EACtB,gBAAuB,EACvB,KAAK,EAAE,SAAiB,EACxB,QAAe,EACf,KAAY,EACZ,eAAsB,EACtB,QAAgB,EAChB,UAAkB,EAClB,WAAmB,EACnB,SAAgB,EAChB,OAAc,EACd,UAAc,EACd,QAAe,EACf,MAAa,EACb,kBAAuB,EACvB,OAAO,GACR,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CA8C1C;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,qBAAqB,CAkBzE;AAED,MAAM,WAAW,gBAAiB,SAAQ,eAAe;IACvD,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAsB,YAAY,CAAC,EACjC,KAAK,EACL,GAAG,EACH,GAAU,EACV,MAAa,EACb,QAAe,EACf,IAAW,EACX,MAAe,EACf,QAAgB,EAChB,OAAa,EACb,OAAO,GACR,EAAE,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,CAezC;AAED,wBAAsB,yBAAyB,CAC7C,KAAK,EAAE,MAAM,EACb,EAAE,OAAO,EAAE,GAAE,eAAoB,GAChC,OAAO,CAAC,WAAW,CAAC,CAKtB"}
|
package/dist/api.js
CHANGED
|
@@ -2,224 +2,107 @@ import { normalizeBearerToken } from "./auth.js";
|
|
|
2
2
|
import { getApiEndpoints } from "./config.js";
|
|
3
3
|
import { request } from "./http.js";
|
|
4
4
|
import { isOAuthAccessToken } from "./oauth.js";
|
|
5
|
-
const CLI_FETCH_HEADERS = {
|
|
6
|
-
"User-Agent": "node",
|
|
7
|
-
"Accept-Language": "*",
|
|
8
|
-
"Sec-Fetch-Mode": "cors",
|
|
9
|
-
};
|
|
10
5
|
export function getAuthHeaders(token) {
|
|
11
6
|
const apiToken = normalizeBearerToken(token);
|
|
12
7
|
if (!apiToken)
|
|
13
8
|
throw new Error("API token is required");
|
|
14
|
-
if (isOAuthAccessToken(apiToken)) {
|
|
15
|
-
return { ...CLI_FETCH_HEADERS, Authorization: `Bearer ${apiToken}` };
|
|
16
|
-
}
|
|
17
9
|
return {
|
|
18
|
-
|
|
10
|
+
"User-Agent": "node",
|
|
11
|
+
"Accept-Language": "*",
|
|
12
|
+
"Sec-Fetch-Mode": "cors",
|
|
19
13
|
Authorization: `Bearer ${apiToken}`,
|
|
20
|
-
"x-api-token": apiToken,
|
|
14
|
+
...(!isOAuthAccessToken(apiToken) ? { "x-api-token": apiToken } : {}),
|
|
21
15
|
};
|
|
22
16
|
}
|
|
23
17
|
function compact(values) {
|
|
24
18
|
return Object.fromEntries(Object.entries(values).filter((entry) => entry[1] !== null && entry[1] !== undefined));
|
|
25
19
|
}
|
|
26
|
-
export async function
|
|
27
|
-
|
|
28
|
-
|
|
20
|
+
export async function scraperApi({ token, url, html = true, markdown = false, timeout = 120, geoCode = "US", proxyCountry = null, blockResources = false, browserRendering = null, super: superMode = false, useProxy = true, proxy = null, waitForSelector = null, homePage = false, screenshot = false, bypassProxy = false, cookieJar = null, headers = null, maxRetries = 3, tokenCap = null, stream = null, discardTokenSecret = "", fetchFn, }) {
|
|
21
|
+
if (!Number.isInteger(timeout) || timeout < 1 || timeout > 300) {
|
|
22
|
+
throw new Error("timeout must be between 1 and 300 seconds");
|
|
23
|
+
}
|
|
24
|
+
if (!Number.isInteger(maxRetries) || maxRetries < 0) {
|
|
25
|
+
throw new Error("max_retries must be a nonnegative integer");
|
|
26
|
+
}
|
|
27
|
+
if (tokenCap !== null && (!Number.isInteger(tokenCap) || tokenCap < 0)) {
|
|
28
|
+
throw new Error("token_cap must be a nonnegative integer");
|
|
29
|
+
}
|
|
30
|
+
const apiToken = normalizeBearerToken(token);
|
|
31
|
+
return request("POST", getApiEndpoints().scraperApiUrl, {
|
|
32
|
+
headers: {
|
|
33
|
+
...getAuthHeaders(apiToken),
|
|
34
|
+
...(discardTokenSecret
|
|
35
|
+
? { "x-discard-token-secret": discardTokenSecret }
|
|
36
|
+
: {}),
|
|
37
|
+
},
|
|
29
38
|
params: compact({
|
|
30
|
-
|
|
39
|
+
// Match the SDK's API-key query parameter; OAuth JWTs stay in Authorization.
|
|
40
|
+
token: isOAuthAccessToken(apiToken) ? null : apiToken,
|
|
31
41
|
timeout,
|
|
32
42
|
geoCode,
|
|
33
|
-
|
|
34
|
-
|
|
43
|
+
proxyCountry,
|
|
44
|
+
browserRendering: screenshot ? true : browserRendering,
|
|
45
|
+
super: superMode ? true : null,
|
|
46
|
+
useProxy: useProxy ? null : false,
|
|
47
|
+
proxy,
|
|
48
|
+
html,
|
|
49
|
+
markdown,
|
|
35
50
|
waitForSelector,
|
|
36
|
-
homePage,
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
51
|
+
warmStart: homePage ? true : null,
|
|
52
|
+
screenshot: screenshot ? true : null,
|
|
53
|
+
blockResources: blockResources ? true : null,
|
|
54
|
+
bypassProxy: bypassProxy ? true : null,
|
|
55
|
+
cookieJar,
|
|
56
|
+
headers: typeof headers === "object" && headers !== null
|
|
57
|
+
? JSON.stringify(headers)
|
|
58
|
+
: headers,
|
|
59
|
+
stream,
|
|
40
60
|
}),
|
|
61
|
+
json: compact({ url, maxRetries, tokenCap }),
|
|
41
62
|
timeout: timeout + 30,
|
|
42
63
|
fetchFn,
|
|
43
64
|
});
|
|
44
65
|
}
|
|
45
|
-
export
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
if (
|
|
50
|
-
|
|
51
|
-
}
|
|
52
|
-
if ((agent === "general" || agent === "listing") && !message?.trim()) {
|
|
53
|
-
throw new Error("An extraction message is required for general and listing agents");
|
|
54
|
-
}
|
|
55
|
-
if (agent === "map" && proxyCountry !== null && proxyCountry !== undefined) {
|
|
56
|
-
throw new Error("The map agent does not accept proxyCountry");
|
|
57
|
-
}
|
|
58
|
-
const payload = { url, agent };
|
|
59
|
-
if (mode !== undefined)
|
|
60
|
-
payload.mode = mode;
|
|
61
|
-
if (agent === "general" || agent === "listing") {
|
|
62
|
-
payload.message = message;
|
|
63
|
-
if (proxyCountry !== null && proxyCountry !== undefined) {
|
|
64
|
-
payload.proxyCountry = proxyCountry;
|
|
65
|
-
}
|
|
66
|
-
if (agent === "listing" && maxPages !== undefined) {
|
|
67
|
-
payload.maxPages = maxPages;
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
else {
|
|
71
|
-
if (message)
|
|
72
|
-
throw new Error("The map agent does not accept an extraction prompt");
|
|
73
|
-
if (maxDepth !== undefined)
|
|
74
|
-
payload.maxDepth = maxDepth;
|
|
75
|
-
if (maxPages !== undefined)
|
|
76
|
-
payload.maxPages = maxPages;
|
|
77
|
-
if (limit !== undefined)
|
|
78
|
-
payload.limit = limit;
|
|
79
|
-
if (includePatterns !== undefined)
|
|
80
|
-
payload.includePatterns = includePatterns;
|
|
81
|
-
if (excludePatterns !== undefined)
|
|
82
|
-
payload.excludePatterns = excludePatterns;
|
|
66
|
+
export function normalizeSearchInput(input) {
|
|
67
|
+
const value = String(input || "").trim();
|
|
68
|
+
if (!value)
|
|
69
|
+
throw new Error("A Google search query or URL is required");
|
|
70
|
+
if (!/^https?:\/\//i.test(value)) {
|
|
71
|
+
return { query: value, region: null, language: null, page: null };
|
|
83
72
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
73
|
+
const parsed = new URL(value);
|
|
74
|
+
const query = parsed.searchParams.get("q")?.trim();
|
|
75
|
+
if (!query)
|
|
76
|
+
throw new Error("Google search URL must contain a q parameter");
|
|
77
|
+
const startValue = parsed.searchParams.get("start") || "0";
|
|
78
|
+
const start = /^[+-]?\d+$/.test(startValue.trim()) ? Number(startValue) : 0;
|
|
79
|
+
return {
|
|
80
|
+
query,
|
|
81
|
+
region: parsed.searchParams.get("gl") || null,
|
|
82
|
+
language: parsed.searchParams.get("hl") || null,
|
|
83
|
+
page: Number.isFinite(start) && start > 0 ? Math.floor(start / 10) + 1 : null,
|
|
84
|
+
};
|
|
89
85
|
}
|
|
90
|
-
export async function
|
|
91
|
-
|
|
92
|
-
|
|
86
|
+
export async function webSearchApi({ token, url, raw = true, region = null, language = null, page = null, format = "json", renderJs = false, timeout = 600, fetchFn, }) {
|
|
87
|
+
const normalized = normalizeSearchInput(url);
|
|
88
|
+
return request("POST", getApiEndpoints().webSearchUrl, {
|
|
89
|
+
headers: getAuthHeaders(token),
|
|
93
90
|
json: compact({
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
excludePatterns,
|
|
101
|
-
proxyCountry,
|
|
102
|
-
maxRetry,
|
|
103
|
-
timeout,
|
|
91
|
+
query: normalized.query,
|
|
92
|
+
region: region || normalized.region,
|
|
93
|
+
language: language || normalized.language,
|
|
94
|
+
page: page || normalized.page,
|
|
95
|
+
format: raw ? "html" : format,
|
|
96
|
+
renderJs,
|
|
104
97
|
}),
|
|
98
|
+
timeout,
|
|
105
99
|
fetchFn,
|
|
106
100
|
});
|
|
107
101
|
}
|
|
108
|
-
export async function
|
|
109
|
-
return request("POST", getApiEndpoints().scrapersAiRerunBulk, {
|
|
110
|
-
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
111
|
-
json: { scraperId, urls },
|
|
112
|
-
fetchFn,
|
|
113
|
-
});
|
|
114
|
-
}
|
|
115
|
-
export async function rerunManualScraperApi({ token, scraperId, url, fetchFn, }) {
|
|
116
|
-
return request("POST", getApiEndpoints().scrapersManualRerun, {
|
|
117
|
-
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
118
|
-
json: { scraperId, url },
|
|
119
|
-
fetchFn,
|
|
120
|
-
});
|
|
121
|
-
}
|
|
122
|
-
export async function bulkRerunManualScraperApi({ token, scraperId, urls, fetchFn, }) {
|
|
123
|
-
return request("POST", getApiEndpoints().scrapersManualRerunBulk, {
|
|
124
|
-
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
125
|
-
json: { scraperId, urls },
|
|
126
|
-
fetchFn,
|
|
127
|
-
});
|
|
128
|
-
}
|
|
129
|
-
export async function getAllResultsApi({ token, sortField = "updatedAt", sortOrder = "DESC", pageSize = 10, page = 1, search = null, dateRangeColumn = null, startAt = null, endAt = null, scraperId = null, status = null, type = null, url = null, fetchFn, }) {
|
|
130
|
-
const params = {
|
|
131
|
-
sortField,
|
|
132
|
-
sortOrder,
|
|
133
|
-
pageSize,
|
|
134
|
-
page,
|
|
135
|
-
};
|
|
136
|
-
if (search)
|
|
137
|
-
params.search = search;
|
|
138
|
-
if (dateRangeColumn)
|
|
139
|
-
params.dateRangeColumn = dateRangeColumn;
|
|
140
|
-
if (startAt)
|
|
141
|
-
params.startAt = startAt;
|
|
142
|
-
if (endAt)
|
|
143
|
-
params.endAt = endAt;
|
|
144
|
-
if (scraperId)
|
|
145
|
-
params["filters[scraperId]"] = scraperId;
|
|
146
|
-
if (status)
|
|
147
|
-
params["filters[status]"] = status;
|
|
148
|
-
if (type)
|
|
149
|
-
params["filters[type]"] = type;
|
|
150
|
-
if (url)
|
|
151
|
-
params["filters[url]"] = url;
|
|
152
|
-
return request("GET", getApiEndpoints().results, {
|
|
153
|
-
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
154
|
-
params,
|
|
155
|
-
fetchFn,
|
|
156
|
-
});
|
|
157
|
-
}
|
|
158
|
-
export async function getResultByIdApi(token, resultId, options = {}) {
|
|
159
|
-
return request("GET", `${getApiEndpoints().results}/${resultId}`, {
|
|
160
|
-
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
161
|
-
params: { includeHtml: options.includeHtml ?? true },
|
|
162
|
-
fetchFn: options.fetchFn,
|
|
163
|
-
});
|
|
164
|
-
}
|
|
165
|
-
export async function getSubscriptionAccountApi(token, options = {}) {
|
|
102
|
+
export async function getSubscriptionAccountApi(token, { fetchFn } = {}) {
|
|
166
103
|
return request("GET", getApiEndpoints().subscriptionAccounts, {
|
|
167
104
|
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
168
|
-
fetchFn: options.fetchFn,
|
|
169
|
-
});
|
|
170
|
-
}
|
|
171
|
-
export async function getAnalyticStatusesApi({ token, domain, startDate, endDate, action = "", apiTokenName = "", fetchFn, }) {
|
|
172
|
-
return request("GET", getApiEndpoints().analyticStatuses, {
|
|
173
|
-
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
174
|
-
params: { domain, startDate, endDate, action, apiTokenName },
|
|
175
105
|
fetchFn,
|
|
176
106
|
});
|
|
177
107
|
}
|
|
178
|
-
export function normalizeSerpInput(input) {
|
|
179
|
-
const value = String(input || "").trim();
|
|
180
|
-
if (!value)
|
|
181
|
-
throw new Error("A Google search query or URL is required");
|
|
182
|
-
try {
|
|
183
|
-
const parsed = new URL(value);
|
|
184
|
-
const query = parsed.searchParams.get("q")?.trim();
|
|
185
|
-
if (!query)
|
|
186
|
-
throw new Error("Google search URL must contain a q parameter");
|
|
187
|
-
const start = Number(parsed.searchParams.get("start") || 0);
|
|
188
|
-
return {
|
|
189
|
-
query,
|
|
190
|
-
region: parsed.searchParams.get("gl"),
|
|
191
|
-
language: parsed.searchParams.get("hl"),
|
|
192
|
-
page: Number.isFinite(start) && start > 0 ? Math.floor(start / 10) + 1 : null,
|
|
193
|
-
};
|
|
194
|
-
}
|
|
195
|
-
catch (error) {
|
|
196
|
-
if (/^https?:\/\//i.test(value))
|
|
197
|
-
throw error;
|
|
198
|
-
return { query: value, region: null, language: null, page: null };
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
export async function googleSerpSyncApi({ token, queryOrUrl, region = null, language = null, page = null, format = "json", renderJs = false, raw = false, timeout = 120, fetchFn, }) {
|
|
202
|
-
const normalized = normalizeSerpInput(queryOrUrl);
|
|
203
|
-
const resolvedFormat = raw ? "html" : format;
|
|
204
|
-
const payload = compact({
|
|
205
|
-
query: normalized.query,
|
|
206
|
-
region: region || normalized.region,
|
|
207
|
-
language: language || normalized.language,
|
|
208
|
-
page: page || normalized.page,
|
|
209
|
-
format: resolvedFormat,
|
|
210
|
-
renderJs: Boolean(renderJs),
|
|
211
|
-
});
|
|
212
|
-
return request("POST", getApiEndpoints().googleSerpSync, {
|
|
213
|
-
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
214
|
-
json: payload,
|
|
215
|
-
timeout,
|
|
216
|
-
fetchFn,
|
|
217
|
-
});
|
|
218
|
-
}
|
|
219
|
-
export function parseBulkUrls(raw) {
|
|
220
|
-
return raw
|
|
221
|
-
.split(/[,\n]/g)
|
|
222
|
-
.map((part) => part.trim())
|
|
223
|
-
.filter(Boolean);
|
|
224
|
-
}
|
|
225
108
|
//# sourceMappingURL=api.js.map
|
package/dist/api.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"api.js","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAoB,MAAM,WAAW,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"api.js","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAoB,MAAM,WAAW,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAQhD,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC7C,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACxD,OAAO;QACL,YAAY,EAAE,MAAM;QACpB,iBAAiB,EAAE,GAAG;QACtB,gBAAgB,EAAE,MAAM;QACxB,aAAa,EAAE,UAAU,QAAQ,EAAE;QACnC,GAAG,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtE,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CACd,MAAoE;IAEpE,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAC3B,CAAC,KAAK,EAAgD,EAAE,CACtD,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS,CAC9C,CACF,CAAC;AACJ,CAAC;AA2BD,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,EAC/B,KAAK,EACL,GAAG,EACH,IAAI,GAAG,IAAI,EACX,QAAQ,GAAG,KAAK,EAChB,OAAO,GAAG,GAAG,EACb,OAAO,GAAG,IAAI,EACd,YAAY,GAAG,IAAI,EACnB,cAAc,GAAG,KAAK,EACtB,gBAAgB,GAAG,IAAI,EACvB,KAAK,EAAE,SAAS,GAAG,KAAK,EACxB,QAAQ,GAAG,IAAI,EACf,KAAK,GAAG,IAAI,EACZ,eAAe,GAAG,IAAI,EACtB,QAAQ,GAAG,KAAK,EAChB,UAAU,GAAG,KAAK,EAClB,WAAW,GAAG,KAAK,EACnB,SAAS,GAAG,IAAI,EAChB,OAAO,GAAG,IAAI,EACd,UAAU,GAAG,CAAC,EACd,QAAQ,GAAG,IAAI,EACf,MAAM,GAAG,IAAI,EACb,kBAAkB,GAAG,EAAE,EACvB,OAAO,GACW;IAClB,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,OAAO,GAAG,GAAG,EAAE,CAAC;QAC/D,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,QAAQ,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC7C,OAAO,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,CAAC,aAAa,EAAE;QACtD,OAAO,EAAE;YACP,GAAG,cAAc,CAAC,QAAQ,CAAC;YAC3B,GAAG,CAAC,kBAAkB;gBACpB,CAAC,CAAC,EAAE,wBAAwB,EAAE,kBAAkB,EAAE;gBAClD,CAAC,CAAC,EAAE,CAAC;SACR;QACD,MAAM,EAAE,OAAO,CAAC;YACd,6EAA6E;YAC7E,KAAK,EAAE,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ;YACrD,OAAO;YACP,OAAO;YACP,YAAY;YACZ,gBAAgB,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB;YACtD,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YAC9B,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK;YACjC,KAAK;YACL,IAAI;YACJ,QAAQ;YACR,eAAe;YACf,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YACjC,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YACpC,cAAc,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YAC5C,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YACtC,SAAS;YACT,OAAO,EACL,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI;gBAC7C,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;gBACzB,CAAC,CAAC,OAAO;YACb,MAAM;SACP,CAAC;QACF,IAAI,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;QAC5C,OAAO,EAAE,OAAO,GAAG,EAAE;QACrB,OAAO;KACR,CAAC,CAAC;AACL,CAAC;AASD,MAAM,UAAU,oBAAoB,CAAC,KAAa;IAChD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IACxE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACpE,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC;IACnD,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAC5E,MAAM,UAAU,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC;IAC3D,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5E,OAAO;QACL,KAAK;QACL,MAAM,EAAE,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI;QAC7C,QAAQ,EAAE,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI;QAC/C,IAAI,EACF,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;KAC1E,CAAC;AACJ,CAAC;AAcD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EACjC,KAAK,EACL,GAAG,EACH,GAAG,GAAG,IAAI,EACV,MAAM,GAAG,IAAI,EACb,QAAQ,GAAG,IAAI,EACf,IAAI,GAAG,IAAI,EACX,MAAM,GAAG,MAAM,EACf,QAAQ,GAAG,KAAK,EAChB,OAAO,GAAG,GAAG,EACb,OAAO,GACU;IACjB,MAAM,UAAU,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC;IAC7C,OAAO,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,CAAC,YAAY,EAAE;QACrD,OAAO,EAAE,cAAc,CAAC,KAAK,CAAC;QAC9B,IAAI,EAAE,OAAO,CAAC;YACZ,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,MAAM,EAAE,MAAM,IAAI,UAAU,CAAC,MAAM;YACnC,QAAQ,EAAE,QAAQ,IAAI,UAAU,CAAC,QAAQ;YACzC,IAAI,EAAE,IAAI,IAAI,UAAU,CAAC,IAAI;YAC7B,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC7B,QAAQ;SACT,CAAC;QACF,OAAO;QACP,OAAO;KACR,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,KAAa,EACb,EAAE,OAAO,EAAE,GAAoB,EAAE;IAEjC,OAAO,OAAO,CAAC,KAAK,EAAE,eAAe,EAAE,CAAC,oBAAoB,EAAE;QAC5D,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,EAAE;QACjE,OAAO;KACR,CAAC,CAAC;AACL,CAAC"}
|
package/dist/config.d.ts
CHANGED
|
@@ -1,20 +1,13 @@
|
|
|
1
|
-
export declare const VERSION = "0.1
|
|
2
|
-
export declare const TOOL_NAMES: readonly ["
|
|
1
|
+
export declare const VERSION = "0.2.1";
|
|
2
|
+
export declare const TOOL_NAMES: readonly ["scraperAPI", "webSearch", "status"];
|
|
3
3
|
export type ToolName = (typeof TOOL_NAMES)[number];
|
|
4
4
|
export type Transport = "stdio" | "http";
|
|
5
5
|
export interface ApiEndpoints {
|
|
6
6
|
apiBaseUrl: string;
|
|
7
|
-
|
|
7
|
+
scraperApiUrl: string;
|
|
8
8
|
syncScraperBaseUrl: string;
|
|
9
9
|
subscriptionAccounts: string;
|
|
10
|
-
|
|
11
|
-
scrapersAi: string;
|
|
12
|
-
scrapersAiRerun: string;
|
|
13
|
-
scrapersAiRerunBulk: string;
|
|
14
|
-
scrapersManualRerun: string;
|
|
15
|
-
scrapersManualRerunBulk: string;
|
|
16
|
-
results: string;
|
|
17
|
-
googleSerpSync: string;
|
|
10
|
+
webSearchUrl: string;
|
|
18
11
|
}
|
|
19
12
|
export declare function getApiEndpoints(): ApiEndpoints;
|
|
20
13
|
export declare function resolveTransport(value?: string | undefined): Transport;
|
|
@@ -27,7 +20,7 @@ export declare function httpRuntimeConfig(): {
|
|
|
27
20
|
port: number;
|
|
28
21
|
allowedOrigins: string[];
|
|
29
22
|
};
|
|
30
|
-
export declare const OAUTH_SCOPES: readonly ["scrape:read", "
|
|
23
|
+
export declare const OAUTH_SCOPES: readonly ["scrape:read", "account:read"];
|
|
31
24
|
export type OAuthScope = (typeof OAUTH_SCOPES)[number];
|
|
32
25
|
export declare const DOCUMENTATION_URL = "https://docs.mrscraper.com/docs/getting-started/mcp-server";
|
|
33
26
|
export interface OAuthConfig {
|
package/dist/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,eAAO,MAAM,UAAU,
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,eAAO,MAAM,UAAU,YAAI,YAAY,EAAE,WAAW,EAAE,QAAQ,CAAU,CAAC;AAEzE,MAAM,MAAM,QAAQ,GAAG,CAAC,OAAO,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnD,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,MAAM,CAAC;AAEzC,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD,wBAAgB,eAAe,IAAI,YAAY,CAqB9C;AAED,wBAAgB,gBAAgB,CAAC,KAAK,qBAAwB,GAAG,SAAS,CAQzE;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,GAAG,OAAO,CAIxE;AAED,wBAAgB,eAAe,IAAI,OAAO,CAEzC;AAED,eAAO,MAAM,uBAAuB,UAGnC,CAAC;AAEF,wBAAgB,mBAAmB,IAAI,MAAM,EAAE,CAM9C;AAED,wBAAgB,iBAAiB,IAAI;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,EAAE,CAAC;CAC1B,CAOA;AAED,eAAO,MAAM,YAAY,YAAI,aAAa,EAAE,cAAc,CAAU,CAAC;AAErE,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC;AAEvD,eAAO,MAAM,iBAAiB,+DACgC,CAAC;AAE/D,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAC;IAEjB,MAAM,EAAE,MAAM,CAAC;IAEf,OAAO,EAAE,MAAM,CAAC;IAEhB,SAAS,EAAE,MAAM,CAAC;IAElB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,wBAAgB,cAAc,IAAI,WAAW,CAqB5C"}
|
package/dist/config.js
CHANGED
|
@@ -1,33 +1,18 @@
|
|
|
1
|
-
export const VERSION = "0.1
|
|
2
|
-
export const TOOL_NAMES = [
|
|
3
|
-
"fetch",
|
|
4
|
-
"scrape",
|
|
5
|
-
"serp",
|
|
6
|
-
"status",
|
|
7
|
-
"rerun",
|
|
8
|
-
"results",
|
|
9
|
-
"result",
|
|
10
|
-
];
|
|
1
|
+
export const VERSION = "0.2.1";
|
|
2
|
+
export const TOOL_NAMES = ["scraperAPI", "webSearch", "status"];
|
|
11
3
|
function baseUrl(environmentName, fallback) {
|
|
12
4
|
return (process.env[environmentName] || fallback).replace(/\/+$/, "");
|
|
13
5
|
}
|
|
14
6
|
export function getApiEndpoints() {
|
|
15
7
|
const apiBaseUrl = baseUrl("MRSCRAPER_API_BASE_URL", "https://api.app.mrscraper.com/api/v1");
|
|
16
|
-
const
|
|
8
|
+
const scraperApiUrl = baseUrl("MRSCRAPER_FETCH_BASE_URL", "https://api.mrscraper.com");
|
|
17
9
|
const syncScraperBaseUrl = baseUrl("MRSCRAPER_SYNC_BASE_URL", "https://sync.scraper.mrscraper.com");
|
|
18
10
|
return {
|
|
19
11
|
apiBaseUrl,
|
|
20
|
-
|
|
12
|
+
scraperApiUrl,
|
|
21
13
|
syncScraperBaseUrl,
|
|
22
14
|
subscriptionAccounts: `${apiBaseUrl}/subscription-accounts`,
|
|
23
|
-
|
|
24
|
-
scrapersAi: `${apiBaseUrl}/scrapers-ai`,
|
|
25
|
-
scrapersAiRerun: `${apiBaseUrl}/scrapers-ai-rerun`,
|
|
26
|
-
scrapersAiRerunBulk: `${apiBaseUrl}/scrapers-ai-rerun/bulk`,
|
|
27
|
-
scrapersManualRerun: `${apiBaseUrl}/scrapers-manual-rerun`,
|
|
28
|
-
scrapersManualRerunBulk: `${apiBaseUrl}/scrapers-manual-rerun/bulk`,
|
|
29
|
-
results: `${apiBaseUrl}/results`,
|
|
30
|
-
googleSerpSync: `${syncScraperBaseUrl}/api/google/serp/v2/sync`,
|
|
15
|
+
webSearchUrl: `${syncScraperBaseUrl}/api/google/serp/v2/sync`,
|
|
31
16
|
};
|
|
32
17
|
}
|
|
33
18
|
export function resolveTransport(value = process.env.TRANSPORT) {
|
|
@@ -65,11 +50,7 @@ export function httpRuntimeConfig() {
|
|
|
65
50
|
}
|
|
66
51
|
return { host, port, allowedOrigins: parseAllowedOrigins() };
|
|
67
52
|
}
|
|
68
|
-
export const OAUTH_SCOPES = [
|
|
69
|
-
"scrape:read",
|
|
70
|
-
"scrape:write",
|
|
71
|
-
"account:read",
|
|
72
|
-
];
|
|
53
|
+
export const OAUTH_SCOPES = ["scrape:read", "account:read"];
|
|
73
54
|
export const DOCUMENTATION_URL = "https://docs.mrscraper.com/docs/getting-started/mcp-server";
|
|
74
55
|
export function getOAuthConfig() {
|
|
75
56
|
const issuer = baseUrl("MRSCRAPER_OAUTH_ISSUER", "https://api.app.mrscraper.com");
|
package/dist/config.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAE/B,MAAM,CAAC,MAAM,UAAU,GAAG
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAE/B,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,YAAY,EAAE,WAAW,EAAE,QAAQ,CAAU,CAAC;AAczE,SAAS,OAAO,CAAC,eAAuB,EAAE,QAAgB;IACxD,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,QAAQ,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,eAAe;IAC7B,MAAM,UAAU,GAAG,OAAO,CACxB,wBAAwB,EACxB,sCAAsC,CACvC,CAAC;IACF,MAAM,aAAa,GAAG,OAAO,CAC3B,0BAA0B,EAC1B,2BAA2B,CAC5B,CAAC;IACF,MAAM,kBAAkB,GAAG,OAAO,CAChC,yBAAyB,EACzB,oCAAoC,CACrC,CAAC;IAEF,OAAO;QACL,UAAU;QACV,aAAa;QACb,kBAAkB;QAClB,oBAAoB,EAAE,GAAG,UAAU,wBAAwB;QAC3D,YAAY,EAAE,GAAG,kBAAkB,0BAA0B;KAC9D,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS;IAC5D,MAAM,SAAS,GAAG,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC1D,IAAI,SAAS,KAAK,OAAO,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CACb,yBAAyB,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,iCAAiC,CACpF,CAAC;IACJ,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,QAAiB;IAC7D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACvC,OAAO,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,eAAe;IAC7B,OAAO,eAAe,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,CAAC,MAAM,uBAAuB,GAAG;IACrC,mBAAmB;IACnB,qBAAqB;CACtB,CAAC;AAEF,MAAM,UAAU,mBAAmB;IACjC,MAAM,UAAU,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,EAAE,CAAC;SAC7D,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;SACjD,MAAM,CAAC,OAAO,CAAC,CAAC;IACnB,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,uBAAuB,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,MAAM,UAAU,iBAAiB;IAK/B,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,WAAW,CAAC;IAC7C,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,MAAM,CAAC,CAAC;IAChD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,MAAM,EAAE,CAAC;QACzD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE,mBAAmB,EAAE,EAAE,CAAC;AAC/D,CAAC;AAED,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,cAAc,CAAU,CAAC;AAIrE,MAAM,CAAC,MAAM,iBAAiB,GAC5B,4DAA4D,CAAC;AAgB/D,MAAM,UAAU,cAAc;IAC5B,MAAM,MAAM,GAAG,OAAO,CACpB,wBAAwB,EACxB,+BAA+B,CAChC,CAAC;IACF,MAAM,SAAS,GAAG,OAAO,CACvB,0BAA0B,EAC1B,2BAA2B,CAC5B,CAAC;IACF,OAAO;QACL,OAAO,EAAE,eAAe,CAAC,iBAAiB,EAAE,IAAI,CAAC;QACjD,MAAM;QACN,OAAO,EAAE,OAAO,CACd,0BAA0B,EAC1B,GAAG,MAAM,wBAAwB,CAClC;QACD,SAAS;QACT,WAAW,EAAE,GAAG,SAAS,MAAM;QAC/B,eAAe,EAAE,YAAY;QAC7B,gBAAgB,EAAE,OAAO,CAAC,oBAAoB,EAAE,iBAAiB,CAAC;KACnE,CAAC;AACJ,CAAC"}
|
package/dist/http.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AAE9D,eAAO,MAAM,uBAAuB,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AAE9D,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAkC3C,MAAM,WAAW,WAAW;IAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,IAAI,EAAE,SAAS,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,SAAS,CAAC,CAAC;IACtE,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACxB;AAUD,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,OAAO,EACd,IAAI,kBAAwB,GAC3B,SAAS,CAqCX;AAyBD,wBAAsB,OAAO,CAC3B,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,kBAAuB,GAC/B,OAAO,CAAC,WAAW,CAAC,CAuEtB"}
|