@mrscraper/mcp 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +230 -353
- package/dist/api.d.ts +26 -75
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +72 -171
- package/dist/api.js.map +1 -1
- package/dist/app.js +1 -1
- package/dist/app.js.map +1 -1
- package/dist/config.d.ts +5 -12
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +6 -25
- package/dist/config.js.map +1 -1
- package/dist/http.d.ts.map +1 -1
- package/dist/http.js +5 -1
- package/dist/http.js.map +1 -1
- package/dist/index.d.ts +0 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -2
- package/dist/index.js.map +1 -1
- package/dist/scopes.d.ts.map +1 -1
- package/dist/scopes.js +2 -6
- package/dist/scopes.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +3 -16
- package/dist/server.js.map +1 -1
- package/dist/status.d.ts +0 -2
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +2 -30
- package/dist/status.js.map +1 -1
- package/dist/tools.d.ts +30 -80
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +184 -569
- package/dist/tools.js.map +1 -1
- package/dist/widgets/bundles.generated.d.ts.map +1 -1
- package/dist/widgets/bundles.generated.js +1 -2
- package/dist/widgets/bundles.generated.js.map +1 -1
- package/dist/widgets/index.d.ts +1 -7
- package/dist/widgets/index.d.ts.map +1 -1
- package/dist/widgets/index.js +1 -7
- package/dist/widgets/index.js.map +1 -1
- package/package.json +4 -4
- package/dist/compliance.d.ts +0 -4
- package/dist/compliance.d.ts.map +0 -1
- package/dist/compliance.js +0 -10
- package/dist/compliance.js.map +0 -1
package/dist/api.d.ts
CHANGED
|
@@ -1,102 +1,53 @@
|
|
|
1
1
|
import { type ApiResponse } from "./http.js";
|
|
2
|
-
export type
|
|
3
|
-
export type SerpFormat = "json" | "html";
|
|
2
|
+
export type SearchFormat = "json" | "html";
|
|
4
3
|
interface FetchDependency {
|
|
5
4
|
fetchFn?: typeof fetch;
|
|
6
5
|
}
|
|
7
6
|
export declare function getAuthHeaders(token: string): Record<string, string>;
|
|
8
|
-
export interface
|
|
7
|
+
export interface ScraperApiOptions extends FetchDependency {
|
|
9
8
|
token: string;
|
|
10
9
|
url: string;
|
|
10
|
+
html?: boolean;
|
|
11
|
+
markdown?: boolean;
|
|
11
12
|
timeout?: number;
|
|
12
|
-
geoCode?: string
|
|
13
|
-
|
|
13
|
+
geoCode?: string;
|
|
14
|
+
proxyCountry?: string | null;
|
|
15
|
+
blockResources?: boolean | null;
|
|
16
|
+
browserRendering?: boolean | null;
|
|
17
|
+
super?: boolean;
|
|
18
|
+
useProxy?: boolean;
|
|
19
|
+
proxy?: string | null;
|
|
14
20
|
waitForSelector?: string | null;
|
|
15
21
|
homePage?: boolean;
|
|
16
|
-
|
|
22
|
+
screenshot?: boolean;
|
|
23
|
+
bypassProxy?: boolean;
|
|
24
|
+
cookieJar?: string | null;
|
|
25
|
+
headers?: Record<string, string> | string | null;
|
|
17
26
|
maxRetries?: number;
|
|
18
27
|
tokenCap?: number | null;
|
|
28
|
+
stream?: boolean | null;
|
|
29
|
+
discardTokenSecret?: string;
|
|
19
30
|
}
|
|
20
|
-
export declare function
|
|
21
|
-
export interface
|
|
22
|
-
token: string;
|
|
23
|
-
url: string;
|
|
24
|
-
message?: string;
|
|
25
|
-
agent?: Agent;
|
|
26
|
-
proxyCountry?: string | null;
|
|
27
|
-
maxDepth?: number;
|
|
28
|
-
maxPages?: number;
|
|
29
|
-
limit?: number;
|
|
30
|
-
includePatterns?: string;
|
|
31
|
-
excludePatterns?: string;
|
|
32
|
-
}
|
|
33
|
-
export declare function createAiScraperApi({ token, url, message, agent, proxyCountry, maxDepth, maxPages, limit, includePatterns, excludePatterns, fetchFn, }: CreateAiScraperOptions): Promise<ApiResponse>;
|
|
34
|
-
export interface RerunAiOptions extends FetchDependency {
|
|
35
|
-
token: string;
|
|
36
|
-
scraperId: string;
|
|
37
|
-
url: string;
|
|
38
|
-
maxDepth?: number;
|
|
39
|
-
maxPages?: number;
|
|
40
|
-
limit?: number;
|
|
41
|
-
includePatterns?: string;
|
|
42
|
-
excludePatterns?: string;
|
|
43
|
-
}
|
|
44
|
-
export declare function rerunAiScraperApi({ token, scraperId, url, maxDepth, maxPages, limit, includePatterns, excludePatterns, fetchFn, }: RerunAiOptions): Promise<ApiResponse>;
|
|
45
|
-
interface BulkRerunOptions extends FetchDependency {
|
|
46
|
-
token: string;
|
|
47
|
-
scraperId: string;
|
|
48
|
-
urls: string[];
|
|
49
|
-
}
|
|
50
|
-
export declare function bulkRerunAiScraperApi({ token, scraperId, urls, fetchFn, }: BulkRerunOptions): Promise<ApiResponse>;
|
|
51
|
-
interface ManualRerunOptions extends FetchDependency {
|
|
52
|
-
token: string;
|
|
53
|
-
scraperId: string;
|
|
54
|
-
url: string;
|
|
55
|
-
}
|
|
56
|
-
export declare function rerunManualScraperApi({ token, scraperId, url, fetchFn, }: ManualRerunOptions): Promise<ApiResponse>;
|
|
57
|
-
export declare function bulkRerunManualScraperApi({ token, scraperId, urls, fetchFn, }: BulkRerunOptions): Promise<ApiResponse>;
|
|
58
|
-
export interface GetResultsOptions extends FetchDependency {
|
|
59
|
-
token: string;
|
|
60
|
-
sortField?: string;
|
|
61
|
-
sortOrder?: string;
|
|
62
|
-
pageSize?: number;
|
|
63
|
-
page?: number;
|
|
64
|
-
search?: string | null;
|
|
65
|
-
dateRangeColumn?: string | null;
|
|
66
|
-
startAt?: string | null;
|
|
67
|
-
endAt?: string | null;
|
|
68
|
-
}
|
|
69
|
-
export declare function getAllResultsApi({ token, sortField, sortOrder, pageSize, page, search, dateRangeColumn, startAt, endAt, fetchFn, }: GetResultsOptions): Promise<ApiResponse>;
|
|
70
|
-
export declare function getResultByIdApi(token: string, resultId: string, options?: FetchDependency): Promise<ApiResponse>;
|
|
71
|
-
export declare function getSubscriptionAccountApi(token: string, options?: FetchDependency): Promise<ApiResponse>;
|
|
72
|
-
export interface AnalyticStatusesOptions extends FetchDependency {
|
|
73
|
-
token: string;
|
|
74
|
-
domain: string;
|
|
75
|
-
startDate: string;
|
|
76
|
-
endDate: string;
|
|
77
|
-
action?: string;
|
|
78
|
-
apiTokenName?: string;
|
|
79
|
-
}
|
|
80
|
-
export declare function getAnalyticStatusesApi({ token, domain, startDate, endDate, action, apiTokenName, fetchFn, }: AnalyticStatusesOptions): Promise<ApiResponse>;
|
|
81
|
-
export interface NormalizedSerpInput {
|
|
31
|
+
export declare function scraperApi({ token, url, html, markdown, timeout, geoCode, proxyCountry, blockResources, browserRendering, super: superMode, useProxy, proxy, waitForSelector, homePage, screenshot, bypassProxy, cookieJar, headers, maxRetries, tokenCap, stream, discardTokenSecret, fetchFn, }: ScraperApiOptions): Promise<ApiResponse>;
|
|
32
|
+
export interface NormalizedSearchInput {
|
|
82
33
|
query: string;
|
|
83
34
|
region: string | null;
|
|
84
35
|
language: string | null;
|
|
85
36
|
page: number | null;
|
|
86
37
|
}
|
|
87
|
-
export declare function
|
|
88
|
-
export interface
|
|
38
|
+
export declare function normalizeSearchInput(input: string): NormalizedSearchInput;
|
|
39
|
+
export interface WebSearchOptions extends FetchDependency {
|
|
89
40
|
token: string;
|
|
90
|
-
|
|
41
|
+
url: string;
|
|
42
|
+
raw?: boolean;
|
|
91
43
|
region?: string | null;
|
|
92
44
|
language?: string | null;
|
|
93
45
|
page?: number | null;
|
|
94
|
-
format?:
|
|
46
|
+
format?: SearchFormat;
|
|
95
47
|
renderJs?: boolean;
|
|
96
|
-
raw?: boolean;
|
|
97
48
|
timeout?: number;
|
|
98
49
|
}
|
|
99
|
-
export declare function
|
|
100
|
-
export declare function
|
|
50
|
+
export declare function webSearchApi({ token, url, raw, region, language, page, format, renderJs, timeout, fetchFn, }: WebSearchOptions): Promise<ApiResponse>;
|
|
51
|
+
export declare function getSubscriptionAccountApi(token: string, { fetchFn }?: FetchDependency): Promise<ApiResponse>;
|
|
101
52
|
export {};
|
|
102
53
|
//# sourceMappingURL=api.d.ts.map
|
package/dist/api.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAEA,OAAO,EAAW,KAAK,WAAW,EAAE,MAAM,WAAW,CAAC;AAGtD,MAAM,MAAM,
|
|
1
|
+
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAEA,OAAO,EAAW,KAAK,WAAW,EAAE,MAAM,WAAW,CAAC;AAGtD,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,MAAM,CAAC;AAE3C,UAAU,eAAe;IACvB,OAAO,CAAC,EAAE,OAAO,KAAK,CAAC;CACxB;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAUpE;AAaD,MAAM,WAAW,iBAAkB,SAAQ,eAAe;IACxD,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,cAAc,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAChC,gBAAgB,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IAClC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC;IACjD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,MAAM,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,wBAAsB,UAAU,CAAC,EAC/B,KAAK,EACL,GAAG,EACH,IAAW,EACX,QAAgB,EAChB,OAAa,EACb,OAAc,EACd,YAAmB,EACnB,cAAsB,EACtB,gBAAuB,EACvB,KAAK,EAAE,SAAiB,EACxB,QAAe,EACf,KAAY,EACZ,eAAsB,EACtB,QAAgB,EAChB,UAAkB,EAClB,WAAmB,EACnB,SAAgB,EAChB,OAAc,EACd,UAAc,EACd,QAAe,EACf,MAAa,EACb,kBAAuB,EACvB,OAAO,GACR,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CA8C1C;AAED,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,GAAG,qBAAqB,CAkBzE;AAED,MAAM,WAAW,gBAAiB,SAAQ,eAAe;IACvD,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAsB,YAAY,CAAC,EACjC,KAAK,EACL,GAAG,EACH,GAAU,EACV,MAAa,EACb,QAAe,EACf,IAAW,EACX,MAAe,EACf,QAAgB,EAChB,OAAa,EACb,OAAO,GACR,EAAE,gBAAgB,GAAG,OAAO,CAAC,WAAW,CAAC,CAezC;AAED,wBAAsB,yBAAyB,CAC7C,KAAK,EAAE,MAAM,EACb,EAAE,OAAO,EAAE,GAAE,eAAoB,GAChC,OAAO,CAAC,WAAW,CAAC,CAKtB"}
|
package/dist/api.js
CHANGED
|
@@ -2,206 +2,107 @@ import { normalizeBearerToken } from "./auth.js";
|
|
|
2
2
|
import { getApiEndpoints } from "./config.js";
|
|
3
3
|
import { request } from "./http.js";
|
|
4
4
|
import { isOAuthAccessToken } from "./oauth.js";
|
|
5
|
-
const CLI_FETCH_HEADERS = {
|
|
6
|
-
"User-Agent": "node",
|
|
7
|
-
"Accept-Language": "*",
|
|
8
|
-
"Sec-Fetch-Mode": "cors",
|
|
9
|
-
};
|
|
10
5
|
export function getAuthHeaders(token) {
|
|
11
6
|
const apiToken = normalizeBearerToken(token);
|
|
12
7
|
if (!apiToken)
|
|
13
8
|
throw new Error("API token is required");
|
|
14
|
-
if (isOAuthAccessToken(apiToken)) {
|
|
15
|
-
return { ...CLI_FETCH_HEADERS, Authorization: `Bearer ${apiToken}` };
|
|
16
|
-
}
|
|
17
9
|
return {
|
|
18
|
-
|
|
10
|
+
"User-Agent": "node",
|
|
11
|
+
"Accept-Language": "*",
|
|
12
|
+
"Sec-Fetch-Mode": "cors",
|
|
19
13
|
Authorization: `Bearer ${apiToken}`,
|
|
20
|
-
"x-api-token": apiToken,
|
|
14
|
+
...(!isOAuthAccessToken(apiToken) ? { "x-api-token": apiToken } : {}),
|
|
21
15
|
};
|
|
22
16
|
}
|
|
23
17
|
function compact(values) {
|
|
24
18
|
return Object.fromEntries(Object.entries(values).filter((entry) => entry[1] !== null && entry[1] !== undefined));
|
|
25
19
|
}
|
|
26
|
-
export async function
|
|
27
|
-
|
|
28
|
-
|
|
20
|
+
export async function scraperApi({ token, url, html = true, markdown = false, timeout = 120, geoCode = "US", proxyCountry = null, blockResources = false, browserRendering = null, super: superMode = false, useProxy = true, proxy = null, waitForSelector = null, homePage = false, screenshot = false, bypassProxy = false, cookieJar = null, headers = null, maxRetries = 3, tokenCap = null, stream = null, discardTokenSecret = "", fetchFn, }) {
|
|
21
|
+
if (!Number.isInteger(timeout) || timeout < 1 || timeout > 300) {
|
|
22
|
+
throw new Error("timeout must be between 1 and 300 seconds");
|
|
23
|
+
}
|
|
24
|
+
if (!Number.isInteger(maxRetries) || maxRetries < 0) {
|
|
25
|
+
throw new Error("max_retries must be a nonnegative integer");
|
|
26
|
+
}
|
|
27
|
+
if (tokenCap !== null && (!Number.isInteger(tokenCap) || tokenCap < 0)) {
|
|
28
|
+
throw new Error("token_cap must be a nonnegative integer");
|
|
29
|
+
}
|
|
30
|
+
const apiToken = normalizeBearerToken(token);
|
|
31
|
+
return request("POST", getApiEndpoints().scraperApiUrl, {
|
|
32
|
+
headers: {
|
|
33
|
+
...getAuthHeaders(apiToken),
|
|
34
|
+
...(discardTokenSecret
|
|
35
|
+
? { "x-discard-token-secret": discardTokenSecret }
|
|
36
|
+
: {}),
|
|
37
|
+
},
|
|
29
38
|
params: compact({
|
|
30
|
-
|
|
39
|
+
// Match the SDK's API-key query parameter; OAuth JWTs stay in Authorization.
|
|
40
|
+
token: isOAuthAccessToken(apiToken) ? null : apiToken,
|
|
31
41
|
timeout,
|
|
32
42
|
geoCode,
|
|
33
|
-
|
|
43
|
+
proxyCountry,
|
|
44
|
+
browserRendering: screenshot ? true : browserRendering,
|
|
45
|
+
super: superMode ? true : null,
|
|
46
|
+
useProxy: useProxy ? null : false,
|
|
47
|
+
proxy,
|
|
48
|
+
html,
|
|
49
|
+
markdown,
|
|
34
50
|
waitForSelector,
|
|
35
|
-
homePage,
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
51
|
+
warmStart: homePage ? true : null,
|
|
52
|
+
screenshot: screenshot ? true : null,
|
|
53
|
+
blockResources: blockResources ? true : null,
|
|
54
|
+
bypassProxy: bypassProxy ? true : null,
|
|
55
|
+
cookieJar,
|
|
56
|
+
headers: typeof headers === "object" && headers !== null
|
|
57
|
+
? JSON.stringify(headers)
|
|
58
|
+
: headers,
|
|
59
|
+
stream,
|
|
39
60
|
}),
|
|
61
|
+
json: compact({ url, maxRetries, tokenCap }),
|
|
40
62
|
timeout: timeout + 30,
|
|
41
63
|
fetchFn,
|
|
42
64
|
});
|
|
43
65
|
}
|
|
44
|
-
export
|
|
45
|
-
if (!(agent === "general" || agent === "listing" || agent === "map")) {
|
|
46
|
-
throw new Error("agent must be general, listing, or map");
|
|
47
|
-
}
|
|
48
|
-
if ((agent === "general" || agent === "listing") && !message?.trim()) {
|
|
49
|
-
throw new Error("An extraction message is required for general and listing agents");
|
|
50
|
-
}
|
|
51
|
-
if (agent === "map" && proxyCountry !== null && proxyCountry !== undefined) {
|
|
52
|
-
throw new Error("The map agent does not accept proxyCountry");
|
|
53
|
-
}
|
|
54
|
-
const payload = { url, agent };
|
|
55
|
-
if (agent === "general" || agent === "listing") {
|
|
56
|
-
payload.message = message;
|
|
57
|
-
if (proxyCountry !== null && proxyCountry !== undefined) {
|
|
58
|
-
payload.proxyCountry = proxyCountry;
|
|
59
|
-
}
|
|
60
|
-
if (agent === "listing" && maxPages !== undefined) {
|
|
61
|
-
payload.maxPages = maxPages;
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
else {
|
|
65
|
-
if (message)
|
|
66
|
-
throw new Error("The map agent does not accept an extraction prompt");
|
|
67
|
-
if (maxDepth !== undefined)
|
|
68
|
-
payload.maxDepth = maxDepth;
|
|
69
|
-
if (maxPages !== undefined)
|
|
70
|
-
payload.maxPages = maxPages;
|
|
71
|
-
if (limit !== undefined)
|
|
72
|
-
payload.limit = limit;
|
|
73
|
-
if (includePatterns !== undefined)
|
|
74
|
-
payload.includePatterns = includePatterns;
|
|
75
|
-
if (excludePatterns !== undefined)
|
|
76
|
-
payload.excludePatterns = excludePatterns;
|
|
77
|
-
}
|
|
78
|
-
return request("POST", getApiEndpoints().scrapersAi, {
|
|
79
|
-
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
80
|
-
json: payload,
|
|
81
|
-
fetchFn,
|
|
82
|
-
});
|
|
83
|
-
}
|
|
84
|
-
export async function rerunAiScraperApi({ token, scraperId, url, maxDepth = 2, maxPages = 50, limit = 1_000, includePatterns = "", excludePatterns = "", fetchFn, }) {
|
|
85
|
-
return request("POST", getApiEndpoints().scrapersAiRerun, {
|
|
86
|
-
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
87
|
-
json: {
|
|
88
|
-
scraperId,
|
|
89
|
-
url,
|
|
90
|
-
maxDepth,
|
|
91
|
-
maxPages,
|
|
92
|
-
limit,
|
|
93
|
-
includePatterns,
|
|
94
|
-
excludePatterns,
|
|
95
|
-
},
|
|
96
|
-
fetchFn,
|
|
97
|
-
});
|
|
98
|
-
}
|
|
99
|
-
export async function bulkRerunAiScraperApi({ token, scraperId, urls, fetchFn, }) {
|
|
100
|
-
return request("POST", getApiEndpoints().scrapersAiRerunBulk, {
|
|
101
|
-
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
102
|
-
json: { scraperId, urls },
|
|
103
|
-
fetchFn,
|
|
104
|
-
});
|
|
105
|
-
}
|
|
106
|
-
export async function rerunManualScraperApi({ token, scraperId, url, fetchFn, }) {
|
|
107
|
-
return request("POST", getApiEndpoints().scrapersManualRerun, {
|
|
108
|
-
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
109
|
-
json: { scraperId, url },
|
|
110
|
-
fetchFn,
|
|
111
|
-
});
|
|
112
|
-
}
|
|
113
|
-
export async function bulkRerunManualScraperApi({ token, scraperId, urls, fetchFn, }) {
|
|
114
|
-
return request("POST", getApiEndpoints().scrapersManualRerunBulk, {
|
|
115
|
-
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
116
|
-
json: { scraperId, urls },
|
|
117
|
-
fetchFn,
|
|
118
|
-
});
|
|
119
|
-
}
|
|
120
|
-
export async function getAllResultsApi({ token, sortField = "updatedAt", sortOrder = "DESC", pageSize = 10, page = 1, search = null, dateRangeColumn = null, startAt = null, endAt = null, fetchFn, }) {
|
|
121
|
-
const params = {
|
|
122
|
-
sortField,
|
|
123
|
-
sortOrder,
|
|
124
|
-
pageSize,
|
|
125
|
-
page,
|
|
126
|
-
};
|
|
127
|
-
if (search)
|
|
128
|
-
params.search = search;
|
|
129
|
-
if (dateRangeColumn)
|
|
130
|
-
params.dateRangeColumn = dateRangeColumn;
|
|
131
|
-
if (startAt)
|
|
132
|
-
params.startAt = startAt;
|
|
133
|
-
if (endAt)
|
|
134
|
-
params.endAt = endAt;
|
|
135
|
-
return request("GET", getApiEndpoints().results, {
|
|
136
|
-
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
137
|
-
params,
|
|
138
|
-
fetchFn,
|
|
139
|
-
});
|
|
140
|
-
}
|
|
141
|
-
export async function getResultByIdApi(token, resultId, options = {}) {
|
|
142
|
-
return request("GET", `${getApiEndpoints().results}/${resultId}`, {
|
|
143
|
-
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
144
|
-
fetchFn: options.fetchFn,
|
|
145
|
-
});
|
|
146
|
-
}
|
|
147
|
-
export async function getSubscriptionAccountApi(token, options = {}) {
|
|
148
|
-
return request("GET", getApiEndpoints().subscriptionAccounts, {
|
|
149
|
-
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
150
|
-
fetchFn: options.fetchFn,
|
|
151
|
-
});
|
|
152
|
-
}
|
|
153
|
-
export async function getAnalyticStatusesApi({ token, domain, startDate, endDate, action = "", apiTokenName = "", fetchFn, }) {
|
|
154
|
-
return request("GET", getApiEndpoints().analyticStatuses, {
|
|
155
|
-
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
156
|
-
params: { domain, startDate, endDate, action, apiTokenName },
|
|
157
|
-
fetchFn,
|
|
158
|
-
});
|
|
159
|
-
}
|
|
160
|
-
export function normalizeSerpInput(input) {
|
|
66
|
+
export function normalizeSearchInput(input) {
|
|
161
67
|
const value = String(input || "").trim();
|
|
162
68
|
if (!value)
|
|
163
69
|
throw new Error("A Google search query or URL is required");
|
|
164
|
-
|
|
165
|
-
const parsed = new URL(value);
|
|
166
|
-
const query = parsed.searchParams.get("q")?.trim();
|
|
167
|
-
if (!query)
|
|
168
|
-
throw new Error("Google search URL must contain a q parameter");
|
|
169
|
-
const start = Number(parsed.searchParams.get("start") || 0);
|
|
170
|
-
return {
|
|
171
|
-
query,
|
|
172
|
-
region: parsed.searchParams.get("gl"),
|
|
173
|
-
language: parsed.searchParams.get("hl"),
|
|
174
|
-
page: Number.isFinite(start) && start > 0 ? Math.floor(start / 10) + 1 : null,
|
|
175
|
-
};
|
|
176
|
-
}
|
|
177
|
-
catch (error) {
|
|
178
|
-
if (/^https?:\/\//i.test(value))
|
|
179
|
-
throw error;
|
|
70
|
+
if (!/^https?:\/\//i.test(value)) {
|
|
180
71
|
return { query: value, region: null, language: null, page: null };
|
|
181
72
|
}
|
|
73
|
+
const parsed = new URL(value);
|
|
74
|
+
const query = parsed.searchParams.get("q")?.trim();
|
|
75
|
+
if (!query)
|
|
76
|
+
throw new Error("Google search URL must contain a q parameter");
|
|
77
|
+
const startValue = parsed.searchParams.get("start") || "0";
|
|
78
|
+
const start = /^[+-]?\d+$/.test(startValue.trim()) ? Number(startValue) : 0;
|
|
79
|
+
return {
|
|
80
|
+
query,
|
|
81
|
+
region: parsed.searchParams.get("gl") || null,
|
|
82
|
+
language: parsed.searchParams.get("hl") || null,
|
|
83
|
+
page: Number.isFinite(start) && start > 0 ? Math.floor(start / 10) + 1 : null,
|
|
84
|
+
};
|
|
182
85
|
}
|
|
183
|
-
export async function
|
|
184
|
-
const normalized =
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
196
|
-
json: payload,
|
|
86
|
+
export async function webSearchApi({ token, url, raw = true, region = null, language = null, page = null, format = "json", renderJs = false, timeout = 600, fetchFn, }) {
|
|
87
|
+
const normalized = normalizeSearchInput(url);
|
|
88
|
+
return request("POST", getApiEndpoints().webSearchUrl, {
|
|
89
|
+
headers: getAuthHeaders(token),
|
|
90
|
+
json: compact({
|
|
91
|
+
query: normalized.query,
|
|
92
|
+
region: region || normalized.region,
|
|
93
|
+
language: language || normalized.language,
|
|
94
|
+
page: page || normalized.page,
|
|
95
|
+
format: raw ? "html" : format,
|
|
96
|
+
renderJs,
|
|
97
|
+
}),
|
|
197
98
|
timeout,
|
|
198
99
|
fetchFn,
|
|
199
100
|
});
|
|
200
101
|
}
|
|
201
|
-
export function
|
|
202
|
-
return
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
102
|
+
export async function getSubscriptionAccountApi(token, { fetchFn } = {}) {
|
|
103
|
+
return request("GET", getApiEndpoints().subscriptionAccounts, {
|
|
104
|
+
headers: { accept: "application/json", ...getAuthHeaders(token) },
|
|
105
|
+
fetchFn,
|
|
106
|
+
});
|
|
206
107
|
}
|
|
207
108
|
//# sourceMappingURL=api.js.map
|
package/dist/api.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"api.js","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAoB,MAAM,WAAW,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"api.js","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAoB,MAAM,WAAW,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAQhD,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC7C,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACxD,OAAO;QACL,YAAY,EAAE,MAAM;QACpB,iBAAiB,EAAE,GAAG;QACtB,gBAAgB,EAAE,MAAM;QACxB,aAAa,EAAE,UAAU,QAAQ,EAAE;QACnC,GAAG,CAAC,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtE,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CACd,MAAoE;IAEpE,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAC3B,CAAC,KAAK,EAAgD,EAAE,CACtD,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,SAAS,CAC9C,CACF,CAAC;AACJ,CAAC;AA2BD,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,EAC/B,KAAK,EACL,GAAG,EACH,IAAI,GAAG,IAAI,EACX,QAAQ,GAAG,KAAK,EAChB,OAAO,GAAG,GAAG,EACb,OAAO,GAAG,IAAI,EACd,YAAY,GAAG,IAAI,EACnB,cAAc,GAAG,KAAK,EACtB,gBAAgB,GAAG,IAAI,EACvB,KAAK,EAAE,SAAS,GAAG,KAAK,EACxB,QAAQ,GAAG,IAAI,EACf,KAAK,GAAG,IAAI,EACZ,eAAe,GAAG,IAAI,EACtB,QAAQ,GAAG,KAAK,EAChB,UAAU,GAAG,KAAK,EAClB,WAAW,GAAG,KAAK,EACnB,SAAS,GAAG,IAAI,EAChB,OAAO,GAAG,IAAI,EACd,UAAU,GAAG,CAAC,EACd,QAAQ,GAAG,IAAI,EACf,MAAM,GAAG,IAAI,EACb,kBAAkB,GAAG,EAAE,EACvB,OAAO,GACW;IAClB,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,OAAO,GAAG,GAAG,EAAE,CAAC;QAC/D,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,QAAQ,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,QAAQ,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC7C,OAAO,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,CAAC,aAAa,EAAE;QACtD,OAAO,EAAE;YACP,GAAG,cAAc,CAAC,QAAQ,CAAC;YAC3B,GAAG,CAAC,kBAAkB;gBACpB,CAAC,CAAC,EAAE,wBAAwB,EAAE,kBAAkB,EAAE;gBAClD,CAAC,CAAC,EAAE,CAAC;SACR;QACD,MAAM,EAAE,OAAO,CAAC;YACd,6EAA6E;YAC7E,KAAK,EAAE,kBAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ;YACrD,OAAO;YACP,OAAO;YACP,YAAY;YACZ,gBAAgB,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB;YACtD,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YAC9B,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK;YACjC,KAAK;YACL,IAAI;YACJ,QAAQ;YACR,eAAe;YACf,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YACjC,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YACpC,cAAc,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YAC5C,WAAW,EAAE,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YACtC,SAAS;YACT,OAAO,EACL,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI;gBAC7C,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;gBACzB,CAAC,CAAC,OAAO;YACb,MAAM;SACP,CAAC;QACF,IAAI,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;QAC5C,OAAO,EAAE,OAAO,GAAG,EAAE;QACrB,OAAO;KACR,CAAC,CAAC;AACL,CAAC;AASD,MAAM,UAAU,oBAAoB,CAAC,KAAa;IAChD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACzC,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IACxE,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACjC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACpE,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC;IACnD,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAC5E,MAAM,UAAU,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC;IAC3D,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5E,OAAO;QACL,KAAK;QACL,MAAM,EAAE,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI;QAC7C,QAAQ,EAAE,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI;QAC/C,IAAI,EACF,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI;KAC1E,CAAC;AACJ,CAAC;AAcD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EACjC,KAAK,EACL,GAAG,EACH,GAAG,GAAG,IAAI,EACV,MAAM,GAAG,IAAI,EACb,QAAQ,GAAG,IAAI,EACf,IAAI,GAAG,IAAI,EACX,MAAM,GAAG,MAAM,EACf,QAAQ,GAAG,KAAK,EAChB,OAAO,GAAG,GAAG,EACb,OAAO,GACU;IACjB,MAAM,UAAU,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC;IAC7C,OAAO,OAAO,CAAC,MAAM,EAAE,eAAe,EAAE,CAAC,YAAY,EAAE;QACrD,OAAO,EAAE,cAAc,CAAC,KAAK,CAAC;QAC9B,IAAI,EAAE,OAAO,CAAC;YACZ,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,MAAM,EAAE,MAAM,IAAI,UAAU,CAAC,MAAM;YACnC,QAAQ,EAAE,QAAQ,IAAI,UAAU,CAAC,QAAQ;YACzC,IAAI,EAAE,IAAI,IAAI,UAAU,CAAC,IAAI;YAC7B,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;YAC7B,QAAQ;SACT,CAAC;QACF,OAAO;QACP,OAAO;KACR,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,KAAa,EACb,EAAE,OAAO,EAAE,GAAoB,EAAE;IAEjC,OAAO,OAAO,CAAC,KAAK,EAAE,eAAe,EAAE,CAAC,oBAAoB,EAAE;QAC5D,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,EAAE;QACjE,OAAO;KACR,CAAC,CAAC;AACL,CAAC"}
|
package/dist/app.js
CHANGED
|
@@ -19,7 +19,7 @@ function normalizeAllowedOrigins(origins) {
|
|
|
19
19
|
return parsed;
|
|
20
20
|
}
|
|
21
21
|
catch {
|
|
22
|
-
throw new Error(`Invalid MRSCRAPER_ALLOWED_ORIGINS entry ${JSON.stringify(origin)}; expected a full origin such as https://
|
|
22
|
+
throw new Error(`Invalid MRSCRAPER_ALLOWED_ORIGINS entry ${JSON.stringify(origin)}; expected a full origin such as https://www.scrapethissite.com`);
|
|
23
23
|
}
|
|
24
24
|
});
|
|
25
25
|
return {
|
package/dist/app.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"app.js","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mBAAmB,EACnB,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EACL,mCAAmC,EACnC,gBAAgB,EAChB,oCAAoC,GAIrC,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EACL,UAAU,EACV,OAAO,EACP,cAAc,EACd,eAAe,EACf,iBAAiB,GAElB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,gCAAgC,EAChC,uBAAuB,GACxB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,aAAa,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EAAE,mBAAmB,EAA6B,MAAM,aAAa,CAAC;AAkB7E,MAAM,sBAAsB,GAAG,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;AAEnE,SAAS,uBAAuB,CAAC,OAAiB;IAIhD,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACxC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;YAC/B,IACE,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAC9C,MAAM,CAAC,QAAQ,KAAK,GAAG;gBACvB,MAAM,CAAC,MAAM;gBACb,MAAM,CAAC,IAAI,EACX,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;YACxC,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CACb,2CAA2C,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"app.js","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mBAAmB,EACnB,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EACL,mCAAmC,EACnC,gBAAgB,EAChB,oCAAoC,GAIrC,MAAM,8BAA8B,CAAC;AAGtC,OAAO,EACL,UAAU,EACV,OAAO,EACP,cAAc,EACd,eAAe,EACf,iBAAiB,GAElB,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,gCAAgC,EAChC,uBAAuB,GACxB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,aAAa,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EAAE,mBAAmB,EAA6B,MAAM,aAAa,CAAC;AAkB7E,MAAM,sBAAsB,GAAG,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;AAEnE,SAAS,uBAAuB,CAAC,OAAiB;IAIhD,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;QACxC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;YAC/B,IACE,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAC9C,MAAM,CAAC,QAAQ,KAAK,GAAG;gBACvB,MAAM,CAAC,MAAM;gBACb,MAAM,CAAC,IAAI,EACX,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;YACxC,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CACb,2CAA2C,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,iEAAiE,CACnI,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO;QACL,OAAO,EAAE,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3D,SAAS,EAAE;YACT,GAAG,IAAI,GAAG,CAAC;gBACT,GAAG,sBAAsB;gBACzB,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC;aAC/C,CAAC;SACH;KACF,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,cAA2B;IACxD,OAAO,CAAC,OAAgB,EAAE,QAAkB,EAAE,IAAkB,EAAQ,EAAE;QACxE,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,IAAI,EAAE,CAAC;YACP,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;YAC/B,IACE,sBAAsB,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC;gBAChD,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,EACjC,CAAC;gBACD,IAAI,EAAE,CAAC;gBACP,OAAO;YACT,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,wEAAwE;QAC1E,CAAC;QACD,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAChD,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAgB,EAAE,aAAqB;IACnE,OAAO,CAAC,OAAgB,EAAE,SAAmB,EAAE,IAAkB,EAAQ,EAAE;QACzE,IACE,OAAO;YACP,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC;YAC3D,OAAO,CAAC,IAAI,KAAK,SAAS,EAC1B,CAAC;YACD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAChD,MAAM,QAAQ,GACZ,UAAU,CAAC,MAAM,GAAG,aAAa;gBAC/B,CAAC,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,iBAAiB;gBACxD,CAAC,CAAC,UAAU,CAAC;YACjB,OAAO,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,aAAa,QAAQ,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,IAAI,EAAE,CAAC;IACT,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,mBAA2B;IACpD,OAAO,CAAC,OAAgB,EAAE,QAAkB,EAAE,IAAkB,EAAQ,EAAE;QACxE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,IAAI,EAAE,CAAC;YACP,OAAO;QACT,CAAC;QACD,MAAM,QAAQ,GAAG,wBAAwB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACxD,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACrD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,IAAI,EAAE,CAAC;YACP,OAAO;QACT,CAAC;QACD,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpE,QAAQ;aACL,MAAM,CAAC,GAAG,CAAC;aACX,GAAG,CACF,kBAAkB,EAClB,6CAA6C,KAAK,yBAAyB,mBAAmB,KAAK;YACjG,6CAA6C,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAC1E;aACA,IAAI,CAAC;YACJ,KAAK,EAAE,oBAAoB;YAC3B,iBAAiB,EAAE,0BAA0B,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ;SACvE,CAAC,CAAC;IACP,CAAC,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,QAAiB;IACtC,OAAO,CAAC,OAAgB,EAAE,QAAkB,EAAE,IAAkB,EAAQ,EAAE;QACxE,IAAI,OAAO,CAAC,IAAI,KAAK,GAAG,EAAE,CAAC;YACzB,IAAI,EAAE,CAAC;YACP,OAAO;QACT,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,6BAA6B,EAAE,GAAG,CAAC,CAAC;QACjD,QAAQ,CAAC,GAAG,CAAC,8BAA8B,EAAE,GAAG,CAAC,CAAC;QAClD,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACjC,QAAQ,CAAC,GAAG,CAAC,8BAA8B,EAAE,cAAc,CAAC,CAAC;YAC7D,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YAC3B,OAAO;QACT,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;YAC7B,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YACxD,OAAO;QACT,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1B,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,OAAO,GAAmB,EAAE;IACxD,MAAM,OAAO,GAAG,iBAAiB,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAC1C,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,cAAc,CAAC;IACjE,MAAM,cAAc,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;IACxD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,cAAc,EAAE,CAAC;IAC5D,MAAM,iBAAiB,GAAG,WAAW,CAAC,OAAO;QAC3C,CAAC,CAAC,IAAI,GAAG,CAAC,WAAW,CAAC,WAAW,CAAC;QAClC,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,mBAAmB,GAAG,iBAAiB;QAC3C,CAAC,CAAC,oCAAoC,CAAC,iBAAiB,CAAC;QACzD,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,GAAG,GAAG,mBAAmB,CAAC;QAC9B,IAAI;QACJ,cAAc,EAAE,cAAc,CAAC,SAAS;KACzC,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,gBAAgB,CAAC,mBAAmB,CAAC,OAAO,CAAC,EAAE;QAC7D,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,KAAK,CAAC;KAC/D,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;IAC3C,MAAM,WAAW,GACf,OAAO,CAAC,WAAW;QACnB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,QAAQ,CAC3B,CAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAC7D,CAAC;IACJ,MAAM,aAAa,GAAG,MAAM,CAC1B,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,IAAI,CAC5E,CAAC;IACF,MAAM,aAAa,GACjB,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC;IAE9E,IAAI,iBAAiB,EAAE,CAAC;QACtB,MAAM,mBAAmB,GAAwB;YAC/C,aAAa,EACX,OAAO,CAAC,aAAa,IAAI,gCAAgC,CAAC,WAAW,CAAC;YACxE,iBAAiB;YACjB,eAAe,EAAE,CAAC,GAAG,WAAW,CAAC,eAAe,CAAC;YACjD,YAAY,EAAE,WAAW;YACzB,uBAAuB,EAAE,IAAI,GAAG,CAAC,WAAW,CAAC,gBAAgB,CAAC;SAC/D,CAAC;QACF,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC,mBAAmB,CAAC,CAAC,CAAC;QAEpD,GAAG,CAAC,GAAG,CACL,uCAAuC,EACvC,aAAa,CAAC,mCAAmC,CAAC,mBAAmB,CAAC,CAAC,CACxE,CAAC;IACJ,CAAC;IAED,GAAG,CAAC,GAAG,CAAC,qBAAqB,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC;IACvD,GAAG,CAAC,GAAG,CAAC,oBAAoB,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC;IAC1D,GAAG,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE;QACxC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IAC9E,CAAC,CAAC,CAAC;IACH,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,EAAE;QACvC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM,EAAE,OAAO;YACf,OAAO,EAAE,eAAe;YACxB,OAAO,EAAE,OAAO;YAChB,KAAK,EAAE,UAAU,CAAC,MAAM;SACzB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,eAAe,EAAE,CAAC;IAC7D,MAAM,IAAI,GAAG,WAAW;QACtB,CAAC,CAAC;YACE,iBAAiB,CAAC;gBAChB,QAAQ,EAAE,uBAAuB,CAAC;oBAChC,MAAM,EAAE,WAAW;oBACnB,aAAa,EAAE,OAAO,CAAC,aAAa;iBACrC,CAAC;gBACF,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxD,CAAC;YACF,GAAG,CAAC,mBAAmB;gBACrB,CAAC,CAAC,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,CAAC;gBAC1C,CAAC,CAAC,EAAE,CAAC;SACR;QACH,CAAC,CAAC,EAAE,CAAC;IACP,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE;QACzD,IAAI,CAAC;YACH,MAAM,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,CAAC;QACd,CAAC;IACH,CAAC,CAAC,CAAC;IACH,GAAG,CAAC,GAAG,CACL,CACE,KAAc,EACd,QAAiB,EACjB,QAAkB,EAClB,KAAmB,EACnB,EAAE;QACF,OAAO,CAAC,KAAK,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC1B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC;QAChE,CAAC;IACH,CAAC,CACF,CAAC;IACF,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AAC1B,CAAC"}
|
package/dist/config.d.ts
CHANGED
|
@@ -1,20 +1,13 @@
|
|
|
1
|
-
export declare const VERSION = "0.
|
|
2
|
-
export declare const TOOL_NAMES: readonly ["
|
|
1
|
+
export declare const VERSION = "0.2.0";
|
|
2
|
+
export declare const TOOL_NAMES: readonly ["scraperAPI", "webSearch", "status"];
|
|
3
3
|
export type ToolName = (typeof TOOL_NAMES)[number];
|
|
4
4
|
export type Transport = "stdio" | "http";
|
|
5
5
|
export interface ApiEndpoints {
|
|
6
6
|
apiBaseUrl: string;
|
|
7
|
-
|
|
7
|
+
scraperApiUrl: string;
|
|
8
8
|
syncScraperBaseUrl: string;
|
|
9
9
|
subscriptionAccounts: string;
|
|
10
|
-
|
|
11
|
-
scrapersAi: string;
|
|
12
|
-
scrapersAiRerun: string;
|
|
13
|
-
scrapersAiRerunBulk: string;
|
|
14
|
-
scrapersManualRerun: string;
|
|
15
|
-
scrapersManualRerunBulk: string;
|
|
16
|
-
results: string;
|
|
17
|
-
googleSerpSync: string;
|
|
10
|
+
webSearchUrl: string;
|
|
18
11
|
}
|
|
19
12
|
export declare function getApiEndpoints(): ApiEndpoints;
|
|
20
13
|
export declare function resolveTransport(value?: string | undefined): Transport;
|
|
@@ -27,7 +20,7 @@ export declare function httpRuntimeConfig(): {
|
|
|
27
20
|
port: number;
|
|
28
21
|
allowedOrigins: string[];
|
|
29
22
|
};
|
|
30
|
-
export declare const OAUTH_SCOPES: readonly ["scrape:read", "
|
|
23
|
+
export declare const OAUTH_SCOPES: readonly ["scrape:read", "account:read"];
|
|
31
24
|
export type OAuthScope = (typeof OAUTH_SCOPES)[number];
|
|
32
25
|
export declare const DOCUMENTATION_URL = "https://docs.mrscraper.com/docs/getting-started/mcp-server";
|
|
33
26
|
export interface OAuthConfig {
|
package/dist/config.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,eAAO,MAAM,UAAU,
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,eAAO,MAAM,UAAU,YAAI,YAAY,EAAE,WAAW,EAAE,QAAQ,CAAU,CAAC;AAEzE,MAAM,MAAM,QAAQ,GAAG,CAAC,OAAO,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnD,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,MAAM,CAAC;AAEzC,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;CACtB;AAMD,wBAAgB,eAAe,IAAI,YAAY,CAqB9C;AAED,wBAAgB,gBAAgB,CAAC,KAAK,qBAAwB,GAAG,SAAS,CAQzE;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,GAAG,OAAO,CAIxE;AAED,wBAAgB,eAAe,IAAI,OAAO,CAEzC;AAED,eAAO,MAAM,uBAAuB,UAGnC,CAAC;AAEF,wBAAgB,mBAAmB,IAAI,MAAM,EAAE,CAM9C;AAED,wBAAgB,iBAAiB,IAAI;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,EAAE,CAAC;CAC1B,CAOA;AAED,eAAO,MAAM,YAAY,YAAI,aAAa,EAAE,cAAc,CAAU,CAAC;AAErE,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC;AAEvD,eAAO,MAAM,iBAAiB,+DACgC,CAAC;AAE/D,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAC;IAEjB,MAAM,EAAE,MAAM,CAAC;IAEf,OAAO,EAAE,MAAM,CAAC;IAEhB,SAAS,EAAE,MAAM,CAAC;IAElB,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,wBAAgB,cAAc,IAAI,WAAW,CAqB5C"}
|
package/dist/config.js
CHANGED
|
@@ -1,33 +1,18 @@
|
|
|
1
|
-
export const VERSION = "0.
|
|
2
|
-
export const TOOL_NAMES = [
|
|
3
|
-
"fetch",
|
|
4
|
-
"scrape",
|
|
5
|
-
"serp",
|
|
6
|
-
"status",
|
|
7
|
-
"rerun",
|
|
8
|
-
"results",
|
|
9
|
-
"result",
|
|
10
|
-
];
|
|
1
|
+
export const VERSION = "0.2.0";
|
|
2
|
+
export const TOOL_NAMES = ["scraperAPI", "webSearch", "status"];
|
|
11
3
|
function baseUrl(environmentName, fallback) {
|
|
12
4
|
return (process.env[environmentName] || fallback).replace(/\/+$/, "");
|
|
13
5
|
}
|
|
14
6
|
export function getApiEndpoints() {
|
|
15
7
|
const apiBaseUrl = baseUrl("MRSCRAPER_API_BASE_URL", "https://api.app.mrscraper.com/api/v1");
|
|
16
|
-
const
|
|
8
|
+
const scraperApiUrl = baseUrl("MRSCRAPER_FETCH_BASE_URL", "https://api.mrscraper.com");
|
|
17
9
|
const syncScraperBaseUrl = baseUrl("MRSCRAPER_SYNC_BASE_URL", "https://sync.scraper.mrscraper.com");
|
|
18
10
|
return {
|
|
19
11
|
apiBaseUrl,
|
|
20
|
-
|
|
12
|
+
scraperApiUrl,
|
|
21
13
|
syncScraperBaseUrl,
|
|
22
14
|
subscriptionAccounts: `${apiBaseUrl}/subscription-accounts`,
|
|
23
|
-
|
|
24
|
-
scrapersAi: `${apiBaseUrl}/scrapers-ai`,
|
|
25
|
-
scrapersAiRerun: `${apiBaseUrl}/scrapers-ai-rerun`,
|
|
26
|
-
scrapersAiRerunBulk: `${apiBaseUrl}/scrapers-ai-rerun/bulk`,
|
|
27
|
-
scrapersManualRerun: `${apiBaseUrl}/scrapers-manual-rerun`,
|
|
28
|
-
scrapersManualRerunBulk: `${apiBaseUrl}/scrapers-manual-rerun/bulk`,
|
|
29
|
-
results: `${apiBaseUrl}/results`,
|
|
30
|
-
googleSerpSync: `${syncScraperBaseUrl}/api/google/serp/v2/sync`,
|
|
15
|
+
webSearchUrl: `${syncScraperBaseUrl}/api/google/serp/v2/sync`,
|
|
31
16
|
};
|
|
32
17
|
}
|
|
33
18
|
export function resolveTransport(value = process.env.TRANSPORT) {
|
|
@@ -65,11 +50,7 @@ export function httpRuntimeConfig() {
|
|
|
65
50
|
}
|
|
66
51
|
return { host, port, allowedOrigins: parseAllowedOrigins() };
|
|
67
52
|
}
|
|
68
|
-
export const OAUTH_SCOPES = [
|
|
69
|
-
"scrape:read",
|
|
70
|
-
"scrape:write",
|
|
71
|
-
"account:read",
|
|
72
|
-
];
|
|
53
|
+
export const OAUTH_SCOPES = ["scrape:read", "account:read"];
|
|
73
54
|
export const DOCUMENTATION_URL = "https://docs.mrscraper.com/docs/getting-started/mcp-server";
|
|
74
55
|
export function getOAuthConfig() {
|
|
75
56
|
const issuer = baseUrl("MRSCRAPER_OAUTH_ISSUER", "https://api.app.mrscraper.com");
|