@scrapi.ai/mcp-server 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +715 -0
- package/dist/http.d.ts +3 -0
- package/dist/http.d.ts.map +1 -0
- package/dist/http.js +89 -0
- package/dist/http.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +14 -0
- package/dist/index.js.map +1 -0
- package/dist/server.d.ts +3 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +19 -0
- package/dist/server.js.map +1 -0
- package/dist/tools/get-billing.d.ts +3 -0
- package/dist/tools/get-billing.d.ts.map +1 -0
- package/dist/tools/get-billing.js +179 -0
- package/dist/tools/get-billing.js.map +1 -0
- package/dist/tools/get-usage.d.ts +3 -0
- package/dist/tools/get-usage.d.ts.map +1 -0
- package/dist/tools/get-usage.js +96 -0
- package/dist/tools/get-usage.js.map +1 -0
- package/dist/tools/scrape-url.d.ts +3 -0
- package/dist/tools/scrape-url.d.ts.map +1 -0
- package/dist/tools/scrape-url.js +77 -0
- package/dist/tools/scrape-url.js.map +1 -0
- package/dist/tools/scrape-urls.d.ts +3 -0
- package/dist/tools/scrape-urls.d.ts.map +1 -0
- package/dist/tools/scrape-urls.js +96 -0
- package/dist/tools/scrape-urls.js.map +1 -0
- package/dist/tools/scraper-server-status.d.ts +3 -0
- package/dist/tools/scraper-server-status.d.ts.map +1 -0
- package/dist/tools/scraper-server-status.js +87 -0
- package/dist/tools/scraper-server-status.js.map +1 -0
- package/dist/utils/api.d.ts +150 -0
- package/dist/utils/api.d.ts.map +1 -0
- package/dist/utils/api.js +142 -0
- package/dist/utils/api.js.map +1 -0
- package/package.json +63 -0
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { scrapeUrl, formatCreditWarning } from "../utils/api.js";
|
|
3
|
+
const ScrapeUrlsSchema = z.object({
|
|
4
|
+
urls: z
|
|
5
|
+
.array(z.string().url())
|
|
6
|
+
.min(1)
|
|
7
|
+
.max(10)
|
|
8
|
+
.describe("URLs to scrape (max 10)"),
|
|
9
|
+
format: z
|
|
10
|
+
.enum(["markdown", "text"])
|
|
11
|
+
.default("markdown")
|
|
12
|
+
.describe("Output format: markdown (default) or text"),
|
|
13
|
+
});
|
|
14
|
+
export function registerScrapeUrlsTool(server) {
|
|
15
|
+
server.tool("scrape_urls", "Scrapes multiple webpages in parallel and returns the content in AI-readable Markdown format. Can access blocked sites through browser rendering.", ScrapeUrlsSchema.shape, async (params) => {
|
|
16
|
+
const { urls, format } = ScrapeUrlsSchema.parse(params);
|
|
17
|
+
try {
|
|
18
|
+
// Scrape all URLs in parallel
|
|
19
|
+
const results = await Promise.all(urls.map(async (url) => {
|
|
20
|
+
try {
|
|
21
|
+
const response = await scrapeUrl({
|
|
22
|
+
url,
|
|
23
|
+
format,
|
|
24
|
+
javascript: true,
|
|
25
|
+
});
|
|
26
|
+
if (!response.success) {
|
|
27
|
+
return {
|
|
28
|
+
url,
|
|
29
|
+
success: false,
|
|
30
|
+
error: response.error || "Failed to scrape the page.",
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
return {
|
|
34
|
+
url,
|
|
35
|
+
success: true,
|
|
36
|
+
title: response.data.title || "Untitled",
|
|
37
|
+
content: response.data.content,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
const message = error instanceof Error ? error.message : "Unknown error";
|
|
42
|
+
return {
|
|
43
|
+
url,
|
|
44
|
+
success: false,
|
|
45
|
+
error: message,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
}));
|
|
49
|
+
// Format results
|
|
50
|
+
const formattedResults = results.map((result, index) => {
|
|
51
|
+
if (result.success) {
|
|
52
|
+
return [
|
|
53
|
+
`## ${index + 1}. ${result.title}`,
|
|
54
|
+
"",
|
|
55
|
+
`> Source: ${result.url}`,
|
|
56
|
+
"",
|
|
57
|
+
result.content,
|
|
58
|
+
].join("\n");
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
return [
|
|
62
|
+
`## ${index + 1}. Error`,
|
|
63
|
+
"",
|
|
64
|
+
`> Source: ${result.url}`,
|
|
65
|
+
"",
|
|
66
|
+
`Error: ${result.error}`,
|
|
67
|
+
].join("\n");
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
const successCount = results.filter((r) => r.success).length;
|
|
71
|
+
const summary = `# Scrape Results (${successCount}/${results.length} successful)\n\n`;
|
|
72
|
+
const warning = formatCreditWarning();
|
|
73
|
+
return {
|
|
74
|
+
content: [
|
|
75
|
+
{
|
|
76
|
+
type: "text",
|
|
77
|
+
text: summary + formattedResults.join("\n\n---\n\n") + (warning || ""),
|
|
78
|
+
},
|
|
79
|
+
],
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
catch (error) {
|
|
83
|
+
const message = error instanceof Error ? error.message : "Unknown error";
|
|
84
|
+
return {
|
|
85
|
+
isError: true,
|
|
86
|
+
content: [
|
|
87
|
+
{
|
|
88
|
+
type: "text",
|
|
89
|
+
text: `Error: ${message}`,
|
|
90
|
+
},
|
|
91
|
+
],
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
//# sourceMappingURL=scrape-urls.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scrape-urls.js","sourceRoot":"","sources":["../../src/tools/scrape-urls.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAEjE,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,IAAI,EAAE,CAAC;SACJ,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC;SACvB,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,CAAC,yBAAyB,CAAC;IACtC,MAAM,EAAE,CAAC;SACN,IAAI,CAAC,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;SAC1B,OAAO,CAAC,UAAU,CAAC;SACnB,QAAQ,CAAC,2CAA2C,CAAC;CACzD,CAAC,CAAC;AAEH,MAAM,UAAU,sBAAsB,CAAC,MAAiB;IACtD,MAAM,CAAC,IAAI,CACT,aAAa,EACb,mJAAmJ,EACnJ,gBAAgB,CAAC,KAAK,EACtB,KAAK,EAAE,MAAM,EAAE,EAAE;QACf,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAExD,IAAI,CAAC;YACH,8BAA8B;YAC9B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC/B,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBACrB,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC;wBAC/B,GAAG;wBACH,MAAM;wBACN,UAAU,EAAE,IAAI;qBACjB,CAAC,CAAC;oBAEH,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;wBACtB,OAAO;4BACL,GAAG;4BACH,OAAO,EAAE,KAAK;4BACd,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,4BAA4B;yBACtD,CAAC;oBACJ,CAAC;oBAED,OAAO;wBACL,GAAG;wBACH,OAAO,EAAE,IAAI;wBACb,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,UAAU;wBACxC,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,OAAO;qBAC/B,CAAC;gBACJ,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;oBACzE,OAAO;wBACL,GAAG;wBACH,OAAO,EAAE,KAAK;wBACd,KAAK,EAAE,OAAO;qBACf,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,CACH,CAAC;YAEF,iBAAiB;YACjB,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;gBACrD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;oBACnB,OAAO;wBACL,MAAM,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,KAAK,EAAE;wBAClC,EAAE;wBACF,aAAa,MAAM,CAAC,GAAG,EAAE;wBACzB,EAAE;wBACF,MAAM,CAAC,OAAO;qBACf,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACf,CAAC;qBAAM,CAAC;oBACN,OAAO;wBACL,MAAM,KAAK,GAAG,CAAC,SAAS;wBACxB,EAAE;wBACF,aAAa,MAAM,CAAC,GAAG,EAAE;wBACzB,EAAE;wBACF,UAAU,MAAM,CAAC,KAAK,EAAE;qBACzB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACf,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;YAC7D,MAAM,OAAO,GAAG,qBAAqB,YAAY,IAAI,OAAO,CAAC,MAAM,kBAAkB,CAAC;YACtF,MAAM,OAAO,GAAG,mBAAmB,EAAE,CAAC;YAEtC,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;qBACvE;iBACF;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YACzE,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,UAAU,OAAO,EAAE;qBAC1B;iBACF;aACF,CAAC;QACJ,CAAC;IACH,CAAC,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scraper-server-status.d.ts","sourceRoot":"","sources":["../../src/tools/scraper-server-status.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAqBpE,wBAAgB,+BAA+B,CAAC,MAAM,EAAE,SAAS,QAiFhE"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { getScraperServerStatus } from "../utils/api.js";
|
|
2
|
+
function formatStatus(server) {
|
|
3
|
+
if (server.circuit_open)
|
|
4
|
+
return "OPEN";
|
|
5
|
+
if (server.healthy === true)
|
|
6
|
+
return "OK";
|
|
7
|
+
if (server.healthy === false)
|
|
8
|
+
return "FAIL";
|
|
9
|
+
return "N/A";
|
|
10
|
+
}
|
|
11
|
+
function formatTime(iso) {
|
|
12
|
+
if (!iso)
|
|
13
|
+
return "-";
|
|
14
|
+
const d = new Date(iso);
|
|
15
|
+
const mm = String(d.getMonth() + 1).padStart(2, "0");
|
|
16
|
+
const dd = String(d.getDate()).padStart(2, "0");
|
|
17
|
+
const hh = String(d.getHours()).padStart(2, "0");
|
|
18
|
+
const mi = String(d.getMinutes()).padStart(2, "0");
|
|
19
|
+
const ss = String(d.getSeconds()).padStart(2, "0");
|
|
20
|
+
return `${mm}/${dd} ${hh}:${mi}:${ss}`;
|
|
21
|
+
}
|
|
22
|
+
export function registerScraperServerStatusTool(server) {
|
|
23
|
+
server.tool("scraper_server_status", "Check the status of all ScraperServer instances. Shows server health, circuit breaker state, failure counts, and last success/failure times.", {}, async () => {
|
|
24
|
+
try {
|
|
25
|
+
const response = await getScraperServerStatus();
|
|
26
|
+
if (!response.success) {
|
|
27
|
+
return {
|
|
28
|
+
isError: true,
|
|
29
|
+
content: [
|
|
30
|
+
{
|
|
31
|
+
type: "text",
|
|
32
|
+
text: `Error: ${response.error || "Failed to retrieve ScraperServer status."}`,
|
|
33
|
+
},
|
|
34
|
+
],
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
const { total, available, servers } = response.data;
|
|
38
|
+
const lines = [
|
|
39
|
+
"## ScraperServer Status",
|
|
40
|
+
"",
|
|
41
|
+
`Total: ${total} | Available: ${available}`,
|
|
42
|
+
"",
|
|
43
|
+
"| Name | OS | Status | Failures | Last Success | Last Failure |",
|
|
44
|
+
"|------|----|--------|----------|--------------|--------------|",
|
|
45
|
+
];
|
|
46
|
+
for (const s of servers) {
|
|
47
|
+
const status = formatStatus(s);
|
|
48
|
+
lines.push(`| ${s.name} | ${s.os} | ${status} | ${s.failure_count} | ${formatTime(s.last_success)} | ${formatTime(s.last_failure)} |`);
|
|
49
|
+
}
|
|
50
|
+
// Show circuit breaker and error details for problem servers
|
|
51
|
+
const problems = servers.filter((s) => s.circuit_open || s.healthy === false);
|
|
52
|
+
if (problems.length > 0) {
|
|
53
|
+
lines.push("");
|
|
54
|
+
lines.push("### Issues");
|
|
55
|
+
for (const s of problems) {
|
|
56
|
+
if (s.circuit_open && s.circuit_open_until) {
|
|
57
|
+
lines.push(`- **${s.name}**: Circuit breaker open until ${formatTime(s.circuit_open_until)}`);
|
|
58
|
+
}
|
|
59
|
+
if (s.last_error) {
|
|
60
|
+
lines.push(`- **${s.name}**: ${s.last_error}`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return {
|
|
65
|
+
content: [
|
|
66
|
+
{
|
|
67
|
+
type: "text",
|
|
68
|
+
text: lines.join("\n"),
|
|
69
|
+
},
|
|
70
|
+
],
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
const message = error instanceof Error ? error.message : "Unknown error";
|
|
75
|
+
return {
|
|
76
|
+
isError: true,
|
|
77
|
+
content: [
|
|
78
|
+
{
|
|
79
|
+
type: "text",
|
|
80
|
+
text: `Error: ${message}`,
|
|
81
|
+
},
|
|
82
|
+
],
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
//# sourceMappingURL=scraper-server-status.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scraper-server-status.js","sourceRoot":"","sources":["../../src/tools/scraper-server-status.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,sBAAsB,EAAqB,MAAM,iBAAiB,CAAC;AAE5E,SAAS,YAAY,CAAC,MAAyB;IAC7C,IAAI,MAAM,CAAC,YAAY;QAAE,OAAO,MAAM,CAAC;IACvC,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IACzC,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK;QAAE,OAAO,MAAM,CAAC;IAC5C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,UAAU,CAAC,GAAkB;IACpC,IAAI,CAAC,GAAG;QAAE,OAAO,GAAG,CAAC;IACrB,MAAM,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;IACxB,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACrD,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IAChD,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACjD,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACnD,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACnD,OAAO,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,+BAA+B,CAAC,MAAiB;IAC/D,MAAM,CAAC,IAAI,CACT,uBAAuB,EACvB,8IAA8I,EAC9I,EAAE,EACF,KAAK,IAAI,EAAE;QACT,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,sBAAsB,EAAE,CAAC;YAEhD,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACtB,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAe;4BACrB,IAAI,EAAE,UAAU,QAAQ,CAAC,KAAK,IAAI,0CAA0C,EAAE;yBAC/E;qBACF;iBACF,CAAC;YACJ,CAAC;YAED,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,IAAI,CAAC;YAEpD,MAAM,KAAK,GAAa;gBACtB,yBAAyB;gBACzB,EAAE;gBACF,UAAU,KAAK,iBAAiB,SAAS,EAAE;gBAC3C,EAAE;gBACF,iEAAiE;gBACjE,iEAAiE;aAClE,CAAC;YAEF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACxB,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBAC/B,KAAK,CAAC,IAAI,CACR,KAAK,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,EAAE,MAAM,MAAM,MAAM,CAAC,CAAC,aAAa,MAAM,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAC3H,CAAC;YACJ,CAAC;YAED,6DAA6D;YAC7D,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAC7B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,OAAO,KAAK,KAAK,CAC7C,CAAC;YACF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACf,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBACzB,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;oBACzB,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,kBAAkB,EAAE,CAAC;wBAC3C,KAAK,CAAC,IAAI,CACR,OAAO,CAAC,CAAC,IAAI,kCAAkC,UAAU,CAAC,CAAC,CAAC,kBAAkB,CAAC,EAAE,CAClF,CAAC;oBACJ,CAAC;oBACD,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;wBACjB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;oBACjD,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;qBACvB;iBACF;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,OAAO,GACX,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;YAC3D,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAe;wBACrB,IAAI,EAAE,UAAU,OAAO,EAAE;qBAC1B;iBACF;aACF,CAAC;QACJ,CAAC;IACH,CAAC,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
export declare function getApiUrl(): string;
|
|
2
|
+
export declare function isMcpMode(): boolean;
|
|
3
|
+
export interface McpResponseHeaders {
|
|
4
|
+
creditsRemaining?: number;
|
|
5
|
+
rateLimitRemaining?: number;
|
|
6
|
+
rateLimitReset?: number;
|
|
7
|
+
burstCurrent?: number;
|
|
8
|
+
burstLimit?: number;
|
|
9
|
+
dailyWarning?: string;
|
|
10
|
+
idempotentReplayed?: boolean;
|
|
11
|
+
}
|
|
12
|
+
export declare function getLastMcpHeaders(): McpResponseHeaders;
|
|
13
|
+
export interface ScrapeUrlOptions {
|
|
14
|
+
url: string;
|
|
15
|
+
format?: "markdown" | "text";
|
|
16
|
+
javascript?: boolean;
|
|
17
|
+
timeout?: number;
|
|
18
|
+
}
|
|
19
|
+
export interface UsageInfo {
|
|
20
|
+
credits_used: number;
|
|
21
|
+
credits_remaining: number;
|
|
22
|
+
}
|
|
23
|
+
export interface ScrapeUrlResponse {
|
|
24
|
+
success: boolean;
|
|
25
|
+
data: {
|
|
26
|
+
content: string;
|
|
27
|
+
title: string;
|
|
28
|
+
url: string;
|
|
29
|
+
metadata?: {
|
|
30
|
+
description?: string;
|
|
31
|
+
og_image?: string;
|
|
32
|
+
};
|
|
33
|
+
};
|
|
34
|
+
usage?: UsageInfo;
|
|
35
|
+
error?: string;
|
|
36
|
+
}
|
|
37
|
+
export interface GetUsageResponse {
|
|
38
|
+
success: boolean;
|
|
39
|
+
data: {
|
|
40
|
+
plan: string;
|
|
41
|
+
credits_total: number;
|
|
42
|
+
credits_used: number;
|
|
43
|
+
credits_remaining: number;
|
|
44
|
+
reset_date: string;
|
|
45
|
+
};
|
|
46
|
+
error?: string;
|
|
47
|
+
}
|
|
48
|
+
export interface McpCreditsResponse {
|
|
49
|
+
success: boolean;
|
|
50
|
+
data: {
|
|
51
|
+
mode: "mcp" | "legacy";
|
|
52
|
+
subscription_credits?: number;
|
|
53
|
+
purchased_credits?: number;
|
|
54
|
+
total_remaining?: number;
|
|
55
|
+
plan?: string;
|
|
56
|
+
current_period_start?: string;
|
|
57
|
+
current_period_end?: string;
|
|
58
|
+
credits_total?: number;
|
|
59
|
+
credits_used?: number;
|
|
60
|
+
credits_remaining?: number;
|
|
61
|
+
reset_date?: string;
|
|
62
|
+
};
|
|
63
|
+
error?: string;
|
|
64
|
+
}
|
|
65
|
+
export interface McpSubscriptionResponse {
|
|
66
|
+
success: boolean;
|
|
67
|
+
data: {
|
|
68
|
+
subscription: {
|
|
69
|
+
plan: string;
|
|
70
|
+
plan_name: string | null;
|
|
71
|
+
status: string;
|
|
72
|
+
monthly_credits: number;
|
|
73
|
+
price_cents: number;
|
|
74
|
+
rate_limit_rpm: number;
|
|
75
|
+
burst_limit: number;
|
|
76
|
+
current_period_start: string | null;
|
|
77
|
+
current_period_end: string | null;
|
|
78
|
+
daily_spending_limit: number | null;
|
|
79
|
+
};
|
|
80
|
+
};
|
|
81
|
+
error?: string;
|
|
82
|
+
}
|
|
83
|
+
export interface McpPlansResponse {
|
|
84
|
+
success: boolean;
|
|
85
|
+
data: {
|
|
86
|
+
plans: Array<{
|
|
87
|
+
slug: string;
|
|
88
|
+
name: string;
|
|
89
|
+
monthly_credits: number;
|
|
90
|
+
price_cents: number;
|
|
91
|
+
rate_limit_rpm: number;
|
|
92
|
+
burst_limit: number;
|
|
93
|
+
}>;
|
|
94
|
+
};
|
|
95
|
+
error?: string;
|
|
96
|
+
}
|
|
97
|
+
export interface McpSpendingLimitsResponse {
|
|
98
|
+
success: boolean;
|
|
99
|
+
data: {
|
|
100
|
+
daily_limit: number | null;
|
|
101
|
+
current_usage: number;
|
|
102
|
+
};
|
|
103
|
+
error?: string;
|
|
104
|
+
}
|
|
105
|
+
export interface McpDailyUsageResponse {
|
|
106
|
+
success: boolean;
|
|
107
|
+
data: {
|
|
108
|
+
daily_usage: Array<{
|
|
109
|
+
date: string;
|
|
110
|
+
requests: number;
|
|
111
|
+
credits: number;
|
|
112
|
+
tools: Record<string, number>;
|
|
113
|
+
}>;
|
|
114
|
+
period: {
|
|
115
|
+
start: string;
|
|
116
|
+
end: string;
|
|
117
|
+
};
|
|
118
|
+
};
|
|
119
|
+
error?: string;
|
|
120
|
+
}
|
|
121
|
+
export interface ScraperServerInfo {
|
|
122
|
+
name: string;
|
|
123
|
+
os: string;
|
|
124
|
+
healthy: boolean | null;
|
|
125
|
+
failure_count: number;
|
|
126
|
+
circuit_open: boolean;
|
|
127
|
+
circuit_open_until: string | null;
|
|
128
|
+
last_success: string | null;
|
|
129
|
+
last_failure: string | null;
|
|
130
|
+
last_error: string | null;
|
|
131
|
+
}
|
|
132
|
+
export interface GetScraperServerStatusResponse {
|
|
133
|
+
success: boolean;
|
|
134
|
+
data: {
|
|
135
|
+
total: number;
|
|
136
|
+
available: number;
|
|
137
|
+
servers: ScraperServerInfo[];
|
|
138
|
+
};
|
|
139
|
+
error?: string;
|
|
140
|
+
}
|
|
141
|
+
export declare function scrapeUrl(options: ScrapeUrlOptions): Promise<ScrapeUrlResponse>;
|
|
142
|
+
export declare function getUsage(): Promise<GetUsageResponse>;
|
|
143
|
+
export declare function getScraperServerStatus(): Promise<GetScraperServerStatusResponse>;
|
|
144
|
+
export declare function getMcpCredits(): Promise<McpCreditsResponse>;
|
|
145
|
+
export declare function getMcpSubscription(): Promise<McpSubscriptionResponse>;
|
|
146
|
+
export declare function getMcpPlans(): Promise<McpPlansResponse>;
|
|
147
|
+
export declare function getMcpSpendingLimits(): Promise<McpSpendingLimitsResponse>;
|
|
148
|
+
export declare function getMcpDailyUsage(startDate?: string, endDate?: string): Promise<McpDailyUsageResponse>;
|
|
149
|
+
export declare function formatCreditWarning(): string | null;
|
|
150
|
+
//# sourceMappingURL=api.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../../src/utils/api.ts"],"names":[],"mappings":"AAMA,wBAAgB,SAAS,IAAI,MAAM,CAElC;AAUD,wBAAgB,SAAS,IAAI,OAAO,CAEnC;AAKD,MAAM,WAAW,kBAAkB;IACjC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,wBAAgB,iBAAiB,IAAI,kBAAkB,CAEtD;AAkED,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,UAAU,GAAG,MAAM,CAAC;IAC7B,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,SAAS;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,CAAC;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,GAAG,EAAE,MAAM,CAAC;QACZ,QAAQ,CAAC,EAAE;YACT,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;SACnB,CAAC;KACH,CAAC;IACF,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM,CAAC;QACb,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAGD,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE;QACJ,IAAI,EAAE,KAAK,GAAG,QAAQ,CAAC;QAEvB,oBAAoB,CAAC,EAAE,MAAM,CAAC;QAC9B,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,oBAAoB,CAAC,EAAE,MAAM,CAAC;QAC9B,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAE5B,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE;QACJ,YAAY,EAAE;YACZ,IAAI,EAAE,MAAM,CAAC;YACb,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;YACzB,MAAM,EAAE,MAAM,CAAC;YACf,eAAe,EAAE,MAAM,CAAC;YACxB,WAAW,EAAE,MAAM,CAAC;YACpB,cAAc,EAAE,MAAM,CAAC;YACvB,WAAW,EAAE,MAAM,CAAC;YACpB,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;YACpC,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;YAClC,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;SACrC,CAAC;KACH,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE;QACJ,KAAK,EAAE,KAAK,CAAC;YACX,IAAI,EAAE,MAAM,CAAC;YACb,IAAI,EAAE,MAAM,CAAC;YACb,eAAe,EAAE,MAAM,CAAC;YACxB,WAAW,EAAE,MAAM,CAAC;YACpB,cAAc,EAAE,MAAM,CAAC;YACvB,WAAW,EAAE,MAAM,CAAC;SACrB,CAAC,CAAC;KACJ,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE;QACJ,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE;QACJ,WAAW,EAAE,KAAK,CAAC;YACjB,IAAI,EAAE,MAAM,CAAC;YACb,QAAQ,EAAE,MAAM,CAAC;YACjB,OAAO,EAAE,MAAM,CAAC;YAChB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;SAC/B,CAAC,CAAC;QACH,MAAM,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,GAAG,EAAE,MAAM,CAAA;SAAE,CAAC;KACxC,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,OAAO,CAAC;IACtB,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,WAAW,8BAA8B;IAC7C,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE;QACJ,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,iBAAiB,EAAE,CAAC;KAC9B,CAAC;IACF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAMD,wBAAsB,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAWrF;AAED,wBAAsB,QAAQ,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAG1D;AAED,wBAAsB,sBAAsB,IAAI,OAAO,CAAC,8BAA8B,CAAC,CAGtF;AAMD,wBAAsB,aAAa,IAAI,OAAO,CAAC,kBAAkB,CAAC,CAGjE;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,uBAAuB,CAAC,CAG3E;AAED,wBAAsB,WAAW,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAG7D;AAED,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,yBAAyB,CAAC,CAG/E;AAED,wBAAsB,gBAAgB,CACpC,SAAS,CAAC,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,qBAAqB,CAAC,CAMhC;AAGD,wBAAgB,mBAAmB,IAAI,MAAM,GAAG,IAAI,CAmBnD"}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import axios from "axios";
|
|
2
|
+
const API_URL = process.env.SCRAPI_API_URL || process.env.HASHSCRAPER_API_URL || "https://api.hashscraper.com";
|
|
3
|
+
const API_KEY = process.env.SCRAPI_API_KEY || process.env.HASHSCRAPER_API_KEY;
|
|
4
|
+
// 디버그용: 현재 사용 중인 API URL 반환
|
|
5
|
+
export function getApiUrl() {
|
|
6
|
+
return API_URL;
|
|
7
|
+
}
|
|
8
|
+
function getApiKey() {
|
|
9
|
+
if (!API_KEY) {
|
|
10
|
+
throw new Error("SCRAPI_API_KEY environment variable is not set. (HASHSCRAPER_API_KEY also accepted)");
|
|
11
|
+
}
|
|
12
|
+
return API_KEY;
|
|
13
|
+
}
|
|
14
|
+
// MCP 모드 여부 (hsmcp_ 프리픽스)
|
|
15
|
+
export function isMcpMode() {
|
|
16
|
+
return getApiKey().startsWith("hsmcp_");
|
|
17
|
+
}
|
|
18
|
+
// 마지막 응답의 MCP 헤더 정보
|
|
19
|
+
let lastMcpHeaders = {};
|
|
20
|
+
export function getLastMcpHeaders() {
|
|
21
|
+
return lastMcpHeaders;
|
|
22
|
+
}
|
|
23
|
+
const client = axios.create({
|
|
24
|
+
baseURL: API_URL,
|
|
25
|
+
timeout: 60000, // 60 seconds timeout
|
|
26
|
+
});
|
|
27
|
+
// Request interceptor: Add API Key to header
|
|
28
|
+
client.interceptors.request.use((config) => {
|
|
29
|
+
const apiKey = getApiKey();
|
|
30
|
+
config.headers["Content-Type"] = "application/json; version=2";
|
|
31
|
+
config.headers["X-API-Key"] = apiKey;
|
|
32
|
+
return config;
|
|
33
|
+
});
|
|
34
|
+
// Response interceptor: Extract MCP headers + 429 retry
|
|
35
|
+
client.interceptors.response.use((response) => {
|
|
36
|
+
// MCP 헤더 추출
|
|
37
|
+
const headers = response.headers;
|
|
38
|
+
lastMcpHeaders = {
|
|
39
|
+
creditsRemaining: headers["x-mcp-credits-remaining"]
|
|
40
|
+
? parseInt(headers["x-mcp-credits-remaining"], 10)
|
|
41
|
+
: undefined,
|
|
42
|
+
rateLimitRemaining: headers["x-ratelimit-remaining"]
|
|
43
|
+
? parseInt(headers["x-ratelimit-remaining"], 10)
|
|
44
|
+
: undefined,
|
|
45
|
+
rateLimitReset: headers["x-ratelimit-reset"]
|
|
46
|
+
? parseInt(headers["x-ratelimit-reset"], 10)
|
|
47
|
+
: undefined,
|
|
48
|
+
burstCurrent: headers["x-burst-current"]
|
|
49
|
+
? parseInt(headers["x-burst-current"], 10)
|
|
50
|
+
: undefined,
|
|
51
|
+
burstLimit: headers["x-burst-limit"]
|
|
52
|
+
? parseInt(headers["x-burst-limit"], 10)
|
|
53
|
+
: undefined,
|
|
54
|
+
dailyWarning: headers["x-mcp-daily-warning"] || undefined,
|
|
55
|
+
idempotentReplayed: headers["x-idempotent-replayed"] === "true",
|
|
56
|
+
};
|
|
57
|
+
return response;
|
|
58
|
+
}, async (error) => {
|
|
59
|
+
// 429 Too Many Requests → Retry-After 헤더 기반 재시도 (최대 1회)
|
|
60
|
+
const config = error.config;
|
|
61
|
+
if (error.response?.status === 429 &&
|
|
62
|
+
config &&
|
|
63
|
+
!config.__retried) {
|
|
64
|
+
config.__retried = true;
|
|
65
|
+
const retryAfter = parseInt(error.response.headers["retry-after"] || "2", 10);
|
|
66
|
+
const waitMs = Math.min(retryAfter * 1000, 60000);
|
|
67
|
+
await new Promise((resolve) => setTimeout(resolve, waitMs));
|
|
68
|
+
return client.request(config);
|
|
69
|
+
}
|
|
70
|
+
return Promise.reject(error);
|
|
71
|
+
});
|
|
72
|
+
// ============================================
|
|
73
|
+
// API Functions
|
|
74
|
+
// ============================================
|
|
75
|
+
export async function scrapeUrl(options) {
|
|
76
|
+
const response = await client.post("/api/scrape", {
|
|
77
|
+
url: options.url,
|
|
78
|
+
format: options.format || "markdown",
|
|
79
|
+
options: {
|
|
80
|
+
javascript: options.javascript ?? true,
|
|
81
|
+
timeout: options.timeout || 30000,
|
|
82
|
+
},
|
|
83
|
+
});
|
|
84
|
+
return response.data;
|
|
85
|
+
}
|
|
86
|
+
export async function getUsage() {
|
|
87
|
+
const response = await client.get("/api/usage");
|
|
88
|
+
return response.data;
|
|
89
|
+
}
|
|
90
|
+
export async function getScraperServerStatus() {
|
|
91
|
+
const response = await client.get("/api/scraper_servers/status");
|
|
92
|
+
return response.data;
|
|
93
|
+
}
|
|
94
|
+
// ============================================
|
|
95
|
+
// MCP Billing API Functions
|
|
96
|
+
// ============================================
|
|
97
|
+
export async function getMcpCredits() {
|
|
98
|
+
const response = await client.get("/api/mcp/credits");
|
|
99
|
+
return response.data;
|
|
100
|
+
}
|
|
101
|
+
export async function getMcpSubscription() {
|
|
102
|
+
const response = await client.get("/api/mcp/subscription");
|
|
103
|
+
return response.data;
|
|
104
|
+
}
|
|
105
|
+
export async function getMcpPlans() {
|
|
106
|
+
const response = await client.get("/api/mcp/plans");
|
|
107
|
+
return response.data;
|
|
108
|
+
}
|
|
109
|
+
export async function getMcpSpendingLimits() {
|
|
110
|
+
const response = await client.get("/api/mcp/spending-limits");
|
|
111
|
+
return response.data;
|
|
112
|
+
}
|
|
113
|
+
export async function getMcpDailyUsage(startDate, endDate) {
|
|
114
|
+
const params = {};
|
|
115
|
+
if (startDate)
|
|
116
|
+
params.start_date = startDate;
|
|
117
|
+
if (endDate)
|
|
118
|
+
params.end_date = endDate;
|
|
119
|
+
const response = await client.get("/api/mcp/usage/daily", { params });
|
|
120
|
+
return response.data;
|
|
121
|
+
}
|
|
122
|
+
// 크레딧 경고 메시지 생성 (tool 응답에 추가)
|
|
123
|
+
export function formatCreditWarning() {
|
|
124
|
+
const h = lastMcpHeaders;
|
|
125
|
+
const warnings = [];
|
|
126
|
+
if (h.dailyWarning && h.dailyWarning !== "ok") {
|
|
127
|
+
if (h.dailyWarning === "warn_75")
|
|
128
|
+
warnings.push("Daily spending at 75% of limit");
|
|
129
|
+
else if (h.dailyWarning === "warn_90")
|
|
130
|
+
warnings.push("Daily spending at 90% of limit");
|
|
131
|
+
else if (h.dailyWarning === "limit_reached")
|
|
132
|
+
warnings.push("Daily spending limit reached");
|
|
133
|
+
}
|
|
134
|
+
if (h.creditsRemaining !== undefined && h.creditsRemaining <= 10) {
|
|
135
|
+
warnings.push(`Only ${h.creditsRemaining} credits remaining`);
|
|
136
|
+
}
|
|
137
|
+
if (h.rateLimitRemaining !== undefined && h.rateLimitRemaining <= 2) {
|
|
138
|
+
warnings.push(`Rate limit: ${h.rateLimitRemaining} requests remaining this minute`);
|
|
139
|
+
}
|
|
140
|
+
return warnings.length > 0 ? `\n\n> ⚠️ ${warnings.join(" | ")}` : null;
|
|
141
|
+
}
|
|
142
|
+
//# sourceMappingURL=api.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"api.js","sourceRoot":"","sources":["../../src/utils/api.ts"],"names":[],"mappings":"AAAA,OAAO,KAAqB,MAAM,OAAO,CAAC;AAE1C,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,6BAA6B,CAAC;AAC/G,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;AAE9E,4BAA4B;AAC5B,MAAM,UAAU,SAAS;IACvB,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,SAAS;IAChB,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,qFAAqF,CAAC,CAAC;IACzG,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,0BAA0B;AAC1B,MAAM,UAAU,SAAS;IACvB,OAAO,SAAS,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;AAC1C,CAAC;AAED,oBAAoB;AACpB,IAAI,cAAc,GAAuB,EAAE,CAAC;AAY5C,MAAM,UAAU,iBAAiB;IAC/B,OAAO,cAAc,CAAC;AACxB,CAAC;AAED,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC1B,OAAO,EAAE,OAAO;IAChB,OAAO,EAAE,KAAK,EAAE,qBAAqB;CACtC,CAAC,CAAC;AAEH,6CAA6C;AAC7C,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;IACzC,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,6BAA6B,CAAC;IAC/D,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC;IACrC,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,wDAAwD;AACxD,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAC9B,CAAC,QAAQ,EAAE,EAAE;IACX,YAAY;IACZ,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;IACjC,cAAc,GAAG;QACf,gBAAgB,EAAE,OAAO,CAAC,yBAAyB,CAAC;YAClD,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,yBAAyB,CAAC,EAAE,EAAE,CAAC;YAClD,CAAC,CAAC,SAAS;QACb,kBAAkB,EAAE,OAAO,CAAC,uBAAuB,CAAC;YAClD,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,uBAAuB,CAAC,EAAE,EAAE,CAAC;YAChD,CAAC,CAAC,SAAS;QACb,cAAc,EAAE,OAAO,CAAC,mBAAmB,CAAC;YAC1C,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,mBAAmB,CAAC,EAAE,EAAE,CAAC;YAC5C,CAAC,CAAC,SAAS;QACb,YAAY,EAAE,OAAO,CAAC,iBAAiB,CAAC;YACtC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,EAAE,CAAC;YAC1C,CAAC,CAAC,SAAS;QACb,UAAU,EAAE,OAAO,CAAC,eAAe,CAAC;YAClC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC;YACxC,CAAC,CAAC,SAAS;QACb,YAAY,EAAE,OAAO,CAAC,qBAAqB,CAAC,IAAI,SAAS;QACzD,kBAAkB,EAAE,OAAO,CAAC,uBAAuB,CAAC,KAAK,MAAM;KAChE,CAAC;IACF,OAAO,QAAQ,CAAC;AAClB,CAAC,EACD,KAAK,EAAE,KAAiB,EAAE,EAAE;IAC1B,wDAAwD;IACxD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC5B,IACE,KAAK,CAAC,QAAQ,EAAE,MAAM,KAAK,GAAG;QAC9B,MAAM;QACN,CAAE,MAAc,CAAC,SAAS,EAC1B,CAAC;QACA,MAAc,CAAC,SAAS,GAAG,IAAI,CAAC;QACjC,MAAM,UAAU,GAAG,QAAQ,CACzB,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,GAAG,EAC5C,EAAE,CACH,CAAC;QACF,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,EAAE,KAAK,CAAC,CAAC;QAClD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;QAC5D,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC,CACF,CAAC;AAiJF,+CAA+C;AAC/C,gBAAgB;AAChB,+CAA+C;AAE/C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB;IACvD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAoB,aAAa,EAAE;QACnE,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,UAAU;QACpC,OAAO,EAAE;YACP,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,IAAI;YACtC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,KAAK;SAClC;KACF,CAAC,CAAC;IAEH,OAAO,QAAQ,CAAC,IAAI,CAAC;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ;IAC5B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAAmB,YAAY,CAAC,CAAC;IAClE,OAAO,QAAQ,CAAC,IAAI,CAAC;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB;IAC1C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAAiC,6BAA6B,CAAC,CAAC;IACjG,OAAO,QAAQ,CAAC,IAAI,CAAC;AACvB,CAAC;AAED,+CAA+C;AAC/C,4BAA4B;AAC5B,+CAA+C;AAE/C,MAAM,CAAC,KAAK,UAAU,aAAa;IACjC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAAqB,kBAAkB,CAAC,CAAC;IAC1E,OAAO,QAAQ,CAAC,IAAI,CAAC;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAA0B,uBAAuB,CAAC,CAAC;IACpF,OAAO,QAAQ,CAAC,IAAI,CAAC;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAAmB,gBAAgB,CAAC,CAAC;IACtE,OAAO,QAAQ,CAAC,IAAI,CAAC;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB;IACxC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAA4B,0BAA0B,CAAC,CAAC;IACzF,OAAO,QAAQ,CAAC,IAAI,CAAC;AACvB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,SAAkB,EAClB,OAAgB;IAEhB,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,IAAI,SAAS;QAAE,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC;IAC7C,IAAI,OAAO;QAAE,MAAM,CAAC,QAAQ,GAAG,OAAO,CAAC;IACvC,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAAwB,sBAAsB,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IAC7F,OAAO,QAAQ,CAAC,IAAI,CAAC;AACvB,CAAC;AAED,8BAA8B;AAC9B,MAAM,UAAU,mBAAmB;IACjC,MAAM,CAAC,GAAG,cAAc,CAAC;IACzB,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;QAC9C,IAAI,CAAC,CAAC,YAAY,KAAK,SAAS;YAAE,QAAQ,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;aAC7E,IAAI,CAAC,CAAC,YAAY,KAAK,SAAS;YAAE,QAAQ,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;aAClF,IAAI,CAAC,CAAC,YAAY,KAAK,eAAe;YAAE,QAAQ,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC7F,CAAC;IAED,IAAI,CAAC,CAAC,gBAAgB,KAAK,SAAS,IAAI,CAAC,CAAC,gBAAgB,IAAI,EAAE,EAAE,CAAC;QACjE,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,gBAAgB,oBAAoB,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,CAAC,CAAC,kBAAkB,KAAK,SAAS,IAAI,CAAC,CAAC,kBAAkB,IAAI,CAAC,EAAE,CAAC;QACpE,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,kBAAkB,iCAAiC,CAAC,CAAC;IACtF,CAAC;IAED,OAAO,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AACzE,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@scrapi.ai/mcp-server",
|
|
3
|
+
"version": "2.0.0",
|
|
4
|
+
"description": "Scrapi MCP Server - Web scraping for AI agents. Bypass anti-bot systems and get clean, LLM-ready content via MCP protocol.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"bin": {
|
|
8
|
+
"scrapi": "dist/index.js",
|
|
9
|
+
"scrapi-http": "dist/http.js"
|
|
10
|
+
},
|
|
11
|
+
"files": [
|
|
12
|
+
"dist",
|
|
13
|
+
"README.md",
|
|
14
|
+
"LICENSE"
|
|
15
|
+
],
|
|
16
|
+
"scripts": {
|
|
17
|
+
"build": "tsc",
|
|
18
|
+
"dev": "tsc --watch",
|
|
19
|
+
"start": "node dist/index.js",
|
|
20
|
+
"start:http": "node dist/http.js",
|
|
21
|
+
"inspector": "npx @modelcontextprotocol/inspector node dist/index.js",
|
|
22
|
+
"prepublishOnly": "npm run build && chmod +x dist/index.js && chmod +x dist/http.js"
|
|
23
|
+
},
|
|
24
|
+
"keywords": [
|
|
25
|
+
"mcp",
|
|
26
|
+
"model-context-protocol",
|
|
27
|
+
"scrapi",
|
|
28
|
+
"web-scraping",
|
|
29
|
+
"scraping",
|
|
30
|
+
"ai",
|
|
31
|
+
"ai-agent",
|
|
32
|
+
"llm",
|
|
33
|
+
"anti-bot"
|
|
34
|
+
],
|
|
35
|
+
"author": "Scrapi",
|
|
36
|
+
"license": "MIT",
|
|
37
|
+
"homepage": "https://scrapi.ai",
|
|
38
|
+
"repository": {
|
|
39
|
+
"type": "git",
|
|
40
|
+
"url": "https://github.com/bamchi/scrapi-mcp-server.git"
|
|
41
|
+
},
|
|
42
|
+
"publishConfig": {
|
|
43
|
+
"access": "public"
|
|
44
|
+
},
|
|
45
|
+
"dependencies": {
|
|
46
|
+
"@modelcontextprotocol/sdk": "^1.25.0",
|
|
47
|
+
"@mozilla/readability": "^0.5.0",
|
|
48
|
+
"axios": "^1.7.0",
|
|
49
|
+
"jsdom": "^24.0.0",
|
|
50
|
+
"turndown": "^7.2.0",
|
|
51
|
+
"zod": "^3.23.0"
|
|
52
|
+
},
|
|
53
|
+
"devDependencies": {
|
|
54
|
+
"@types/express": "^5.0.6",
|
|
55
|
+
"@types/jsdom": "^21.1.6",
|
|
56
|
+
"@types/node": "^20.0.0",
|
|
57
|
+
"@types/turndown": "^5.0.4",
|
|
58
|
+
"typescript": "^5.4.0"
|
|
59
|
+
},
|
|
60
|
+
"engines": {
|
|
61
|
+
"node": ">=20.0.0"
|
|
62
|
+
}
|
|
63
|
+
}
|