alterlab-mcp-server 1.8.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -26
- package/dist/client.d.ts +2 -1
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +7 -4
- package/dist/client.js.map +1 -1
- package/dist/errors.js +2 -2
- package/dist/errors.js.map +1 -1
- package/dist/format.d.ts +5 -1
- package/dist/format.d.ts.map +1 -1
- package/dist/format.js +48 -0
- package/dist/format.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/tools/batch.d.ts +1213 -17
- package/dist/tools/batch.d.ts.map +1 -1
- package/dist/tools/batch.js +121 -33
- package/dist/tools/batch.js.map +1 -1
- package/dist/tools/common.d.ts +313 -0
- package/dist/tools/common.d.ts.map +1 -0
- package/dist/tools/common.js +336 -0
- package/dist/tools/common.js.map +1 -0
- package/dist/tools/crawl.d.ts +135 -18
- package/dist/tools/crawl.d.ts.map +1 -1
- package/dist/tools/crawl.js +141 -56
- package/dist/tools/crawl.js.map +1 -1
- package/dist/tools/estimate.d.ts +20 -5
- package/dist/tools/estimate.d.ts.map +1 -1
- package/dist/tools/estimate.js +25 -34
- package/dist/tools/estimate.js.map +1 -1
- package/dist/tools/extract.d.ts +4 -4
- package/dist/tools/extract.d.ts.map +1 -1
- package/dist/tools/extract.js +2 -0
- package/dist/tools/extract.js.map +1 -1
- package/dist/tools/job.d.ts +13 -0
- package/dist/tools/job.d.ts.map +1 -0
- package/dist/tools/job.js +62 -0
- package/dist/tools/job.js.map +1 -0
- package/dist/tools/map.d.ts.map +1 -1
- package/dist/tools/map.js +4 -1
- package/dist/tools/map.js.map +1 -1
- package/dist/tools/scrape.d.ts +362 -59
- package/dist/tools/scrape.d.ts.map +1 -1
- package/dist/tools/scrape.js +176 -215
- package/dist/tools/scrape.js.map +1 -1
- package/dist/tools/screenshot.d.ts +33 -0
- package/dist/tools/screenshot.d.ts.map +1 -1
- package/dist/tools/screenshot.js +46 -5
- package/dist/tools/screenshot.js.map +1 -1
- package/dist/tools/search.d.ts +8 -5
- package/dist/tools/search.d.ts.map +1 -1
- package/dist/tools/search.js +13 -5
- package/dist/tools/search.js.map +1 -1
- package/dist/tools/sessions.d.ts +180 -12
- package/dist/tools/sessions.d.ts.map +1 -1
- package/dist/tools/sessions.js +75 -21
- package/dist/tools/sessions.js.map +1 -1
- package/dist/tools/simple_batch.d.ts +3 -3
- package/dist/tools/simple_batch.d.ts.map +1 -1
- package/dist/tools/simple_batch.js +5 -4
- package/dist/tools/simple_batch.js.map +1 -1
- package/dist/tools/usage.js +9 -18
- package/dist/tools/usage.js.map +1 -1
- package/dist/types.d.ts +170 -52
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +5 -3
- package/server.json +15 -6
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getJobDescription = exports.getJobSchema = void 0;
|
|
4
|
+
exports.handleGetJob = handleGetJob;
|
|
5
|
+
const zod_1 = require("zod");
|
|
6
|
+
const errors_js_1 = require("../errors.js");
|
|
7
|
+
const format_js_1 = require("../format.js");
|
|
8
|
+
exports.getJobSchema = zod_1.z.object({
|
|
9
|
+
job_id: zod_1.z
|
|
10
|
+
.string()
|
|
11
|
+
.min(1)
|
|
12
|
+
.describe("Job ID returned by an async or downgraded scrape (alterlab_scrape reports it when a scrape outlives the ~85s synchronous window)"),
|
|
13
|
+
});
|
|
14
|
+
exports.getJobDescription = "Fetch the status and result of an asynchronous scrape job by job_id. " +
|
|
15
|
+
"Use this when alterlab_scrape or alterlab_screenshot returns a job_id instead of content " +
|
|
16
|
+
"(a long-running scrape that outlived the synchronous wait window). " +
|
|
17
|
+
"Status values: queued, processing, succeeded, failed, rejected (rejected = not processed due to insufficient balance; stop polling and check the balance). " +
|
|
18
|
+
"Polling is free; do not re-run the scrape, which would be billed again.";
|
|
19
|
+
const MAX_RAW_CHARS = 20_000;
|
|
20
|
+
async function handleGetJob(client, params) {
|
|
21
|
+
try {
|
|
22
|
+
const job = await client.getJob(params.job_id);
|
|
23
|
+
return { content: [{ type: "text", text: formatJob(job, params.job_id) }] };
|
|
24
|
+
}
|
|
25
|
+
catch (error) {
|
|
26
|
+
if (isApiError(error)) {
|
|
27
|
+
return (0, errors_js_1.formatErrorResult)(error, { url: params.job_id });
|
|
28
|
+
}
|
|
29
|
+
return (0, errors_js_1.formatErrorResult)(error, { url: params.job_id });
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
function formatJob(job, jobId) {
|
|
33
|
+
const status = String(job.status ?? "unknown");
|
|
34
|
+
const parts = [`**Job ${status.toUpperCase()}**\n`, `- Job ID: \`${jobId}\``];
|
|
35
|
+
if (job.error) {
|
|
36
|
+
parts.push(`- Error: ${typeof job.error === "string" ? job.error : JSON.stringify(job.error)}`);
|
|
37
|
+
}
|
|
38
|
+
const result = job.result;
|
|
39
|
+
if (result && typeof result === "object") {
|
|
40
|
+
if (result.billing && result.url) {
|
|
41
|
+
parts.push("\n" + (0, format_js_1.formatScrapeResponse)(result));
|
|
42
|
+
}
|
|
43
|
+
else {
|
|
44
|
+
const raw = JSON.stringify(result, null, 2);
|
|
45
|
+
parts.push("\n```json\n" +
|
|
46
|
+
raw.slice(0, MAX_RAW_CHARS) +
|
|
47
|
+
(raw.length > MAX_RAW_CHARS ? "\n... (truncated)" : "") +
|
|
48
|
+
"\n```");
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
else if (status === "queued" || status === "processing" || status === "pending") {
|
|
52
|
+
parts.push("\nStill running — call alterlab_get_job again in a few seconds.");
|
|
53
|
+
}
|
|
54
|
+
return parts.join("\n");
|
|
55
|
+
}
|
|
56
|
+
function isApiError(error) {
|
|
57
|
+
return (typeof error === "object" &&
|
|
58
|
+
error !== null &&
|
|
59
|
+
"status" in error &&
|
|
60
|
+
typeof error.status === "number");
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=job.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"job.js","sourceRoot":"","sources":["../../src/tools/job.ts"],"names":[],"mappings":";;;AAyBA,oCAaC;AAtCD,6BAAwB;AAGxB,4CAAgE;AAChE,4CAAoD;AAGvC,QAAA,YAAY,GAAG,OAAC,CAAC,MAAM,CAAC;IACnC,MAAM,EAAE,OAAC;SACN,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,QAAQ,CACP,kIAAkI,CACnI;CACJ,CAAC,CAAC;AAEU,QAAA,iBAAiB,GAC5B,uEAAuE;IACvE,2FAA2F;IAC3F,qEAAqE;IACrE,6JAA6J;IAC7J,yEAAyE,CAAC;AAE5E,MAAM,aAAa,GAAG,MAAM,CAAC;AAEtB,KAAK,UAAU,YAAY,CAChC,MAAsB,EACtB,MAAoC;IAEpC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC/C,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;IAC9E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,IAAA,6BAAiB,EAAC,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1D,CAAC;QACD,OAAO,IAAA,6BAAiB,EAAC,KAAc,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACnE,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,GAA4B,EAAE,KAAa;IAC5D,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAa,CAAC,SAAS,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC,CAAC;IAExF,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;QACd,KAAK,CAAC,IAAI,CACR,YAAY,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CACpF,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,GAAG,CAAC,MAA6C,CAAC;IACjE,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QACzC,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,IAAA,gCAAoB,EAAC,MAA0C,CAAC,CAAC,CAAC;QACtF,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAC5C,KAAK,CAAC,IAAI,CACR,aAAa;gBACX,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC;gBAC3B,CAAC,GAAG,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvD,OAAO,CACV,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,IAAI,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,YAAY,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QAClF,KAAK,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;IAChF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,QAAQ,IAAI,KAAK;QACjB,OAAQ,KAAkB,CAAC,MAAM,KAAK,QAAQ,CAC/C,CAAC;AACJ,CAAC"}
|
package/dist/tools/map.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"map.d.ts","sourceRoot":"","sources":["../../src/tools/map.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACzE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAGnD,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"map.d.ts","sourceRoot":"","sources":["../../src/tools/map.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACzE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAGnD,eAAO,MAAM,SAAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAiEpB,CAAC;AAEH,eAAO,MAAM,cAAc,QASkE,CAAC;AAE9F,wBAAsB,SAAS,CAC7B,MAAM,EAAE,cAAc,EACtB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,SAAS,CAAC,GAChC,OAAO,CAAC,cAAc,CAAC,CAwBzB"}
|
package/dist/tools/map.js
CHANGED
|
@@ -5,7 +5,7 @@ exports.handleMap = handleMap;
|
|
|
5
5
|
const zod_1 = require("zod");
|
|
6
6
|
const errors_js_1 = require("../errors.js");
|
|
7
7
|
exports.mapSchema = zod_1.z.object({
|
|
8
|
-
url: zod_1.z.string().url().describe("Starting URL for site discovery"),
|
|
8
|
+
url: zod_1.z.string().url().max(2083).describe("Starting URL for site discovery"),
|
|
9
9
|
max_pages: zod_1.z
|
|
10
10
|
.number()
|
|
11
11
|
.int()
|
|
@@ -22,10 +22,12 @@ exports.mapSchema = zod_1.z.object({
|
|
|
22
22
|
.describe("Link-following depth (0 = start page + sitemap only)"),
|
|
23
23
|
include_patterns: zod_1.z
|
|
24
24
|
.array(zod_1.z.string())
|
|
25
|
+
.max(50)
|
|
25
26
|
.optional()
|
|
26
27
|
.describe("Glob patterns — only include URLs whose path matches at least one (e.g., ['/docs/*'])"),
|
|
27
28
|
exclude_patterns: zod_1.z
|
|
28
29
|
.array(zod_1.z.string())
|
|
30
|
+
.max(50)
|
|
29
31
|
.optional()
|
|
30
32
|
.describe("Glob patterns — exclude URLs whose path matches any (e.g., ['/tag/*', '/page/*'])"),
|
|
31
33
|
search: zod_1.z
|
|
@@ -39,6 +41,7 @@ exports.mapSchema = zod_1.z.object({
|
|
|
39
41
|
.describe("Sitemap handling: include (parse sitemaps + follow links), skip (links only), only (sitemap URLs only)"),
|
|
40
42
|
sitemap_path: zod_1.z
|
|
41
43
|
.string()
|
|
44
|
+
.max(500)
|
|
42
45
|
.optional()
|
|
43
46
|
.describe("Explicit path to the sitemap file (e.g., '/sitemap_index.xml'). " +
|
|
44
47
|
"Use when the sitemap is not at the standard /sitemap.xml location."),
|
package/dist/tools/map.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"map.js","sourceRoot":"","sources":["../../src/tools/map.ts"],"names":[],"mappings":";;;
|
|
1
|
+
{"version":3,"file":"map.js","sourceRoot":"","sources":["../../src/tools/map.ts"],"names":[],"mappings":";;;AAmFA,8BA2BC;AA9GD,6BAAwB;AAGxB,4CAAgE;AAEnD,QAAA,SAAS,GAAG,OAAC,CAAC,MAAM,CAAC;IAChC,GAAG,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,iCAAiC,CAAC;IAC3E,SAAS,EAAE,OAAC;SACT,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,KAAK,CAAC;SACV,OAAO,CAAC,GAAG,CAAC;SACZ,QAAQ,CAAC,0BAA0B,CAAC;IACvC,SAAS,EAAE,OAAC;SACT,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,OAAO,CAAC,CAAC,CAAC;SACV,QAAQ,CAAC,sDAAsD,CAAC;IACnE,gBAAgB,EAAE,OAAC;SAChB,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;SACjB,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,EAAE;SACV,QAAQ,CACP,uFAAuF,CACxF;IACH,gBAAgB,EAAE,OAAC;SAChB,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;SACjB,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,EAAE;SACV,QAAQ,CACP,mFAAmF,CACpF;IACH,MAAM,EAAE,OAAC;SACN,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CACP,yFAAyF,CAC1F;IACH,OAAO,EAAE,OAAC;SACP,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;SACjC,OAAO,CAAC,SAAS,CAAC;SAClB,QAAQ,CACP,wGAAwG,CACzG;IACH,YAAY,EAAE,OAAC;SACZ,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CACP,kEAAkE;QAChE,oEAAoE,CACvE;IACH,gBAAgB,EAAE,OAAC;SAChB,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CACP,kFAAkF,CACnF;IACH,kBAAkB,EAAE,OAAC;SAClB,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,mDAAmD,CAAC;IAChE,cAAc,EAAE,OAAC;SACd,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,+BAA+B,CAAC;CAC7C,CAAC,CAAC;AAEU,QAAA,cAAc,GACzB,6FAA6F;IAC7F,4GAA4G;IAC5G,mFAAmF;IACnF,sFAAsF;IACtF,oEAAoE;IACpE,iFAAiF;IACjF,mDAAmD;IACnD,wEAAwE;IACxE,2FAA2F,CAAC;AAEvF,KAAK,UAAU,SAAS,CAC7B,MAAsB,EACtB,MAAiC;IAEjC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC;YAChC,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,YAAY,EAAE,MAAM,CAAC,YAAY;YACjC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;YAC7C,cAAc,EAAE,MAAM,CAAC,cAAc;SACtC,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;QACrD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,IAAA,6BAAiB,EAAC,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,IAAA,6BAAiB,EAAC,KAAc,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IAChE,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAiB,EAAE,QAAgB;IAC5D,MAAM,CAAC,GAAG,QAAmC,CAAC;IAC9C,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC;IAC5C,MAAM,YAAY,GAAG,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC;IAC9C,MAAM,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IAEjD,MAAM,KAAK,GAAa;QACtB,eAAe,QAAQ,MAAM;QAC7B,eAAe,KAAK,IAAI;QACxB,sBAAsB,SAAS,EAAE;QACjC,oBAAoB,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE;QACjD,mBAAmB,WAAW,EAAE;QAChC,EAAE;KACH,CAAC;IAEF,kBAAkB;IAClB,MAAM,SAAS,GAAG,CAAC,CAAC,UAAiD,CAAC;IACtE,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,gBAAgB,CAAC;YAC1D,CAAC,CAAC,SAAS,CAAC,gBAAgB;YAC5B,CAAC,CAAC,EAAE,CAAC;QACP,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CACR,mBAAmB,UAAU,CAAC,MAAM,mBAAmB,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5F,CAAC;QACJ,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CACR,kCAAkC,OAAO,CAAC,MAAM,OAAO,SAAS,MAAM,CACvE,CAAC;QACF,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,KAAgC,CAAC;YAC3C,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;YAChC,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;YACtC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;YACnC,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,MAAM,KAAK,GACT,CAAC,CAAC,eAAe,KAAK,IAAI,IAAI,CAAC,CAAC,eAAe,KAAK,SAAS;gBAC3D,CAAC,CAAC,gBAAgB,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;gBACzD,CAAC,CAAC,EAAE,CAAC;YACT,KAAK,CAAC,IAAI,CAAC,MAAM,MAAM,WAAW,KAAK,KAAK,GAAG,GAAG,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,SAAS,GAAG,EAAE,EAAE,CAAC;YACnB,KAAK,CAAC,IAAI,CAAC,aAAa,SAAS,GAAG,EAAE,YAAY,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IACpC,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,QAAQ,IAAI,KAAK;QACjB,OAAQ,KAAkB,CAAC,MAAM,KAAK,QAAQ,CAC/C,CAAC;AACJ,CAAC"}
|
package/dist/tools/scrape.d.ts
CHANGED
|
@@ -1,36 +1,8 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
|
|
3
3
|
import type { AlterLabClient } from "../client.js";
|
|
4
|
+
import type { UnifiedScrapeRequest } from "../types.js";
|
|
4
5
|
export declare const scrapeSchema: z.ZodObject<{
|
|
5
|
-
url: z.ZodString;
|
|
6
|
-
method: z.ZodDefault<z.ZodEnum<["GET", "POST"]>>;
|
|
7
|
-
body: z.ZodOptional<z.ZodString>;
|
|
8
|
-
content_type: z.ZodOptional<z.ZodEnum<["application/json", "application/x-www-form-urlencoded", "text/plain", "application/graphql"]>>;
|
|
9
|
-
mode: z.ZodDefault<z.ZodEnum<["auto", "html", "js", "pdf", "ocr"]>>;
|
|
10
|
-
formats: z.ZodDefault<z.ZodArray<z.ZodEnum<["text", "json", "json_v2", "html", "markdown", "rag", "raw", "content"]>, "many">>;
|
|
11
|
-
extraction_schema: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
12
|
-
extraction_model: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
13
|
-
extraction_provider: z.ZodOptional<z.ZodEnum<["openai", "anthropic", "openrouter", "groq"]>>;
|
|
14
|
-
extraction_prompt: z.ZodOptional<z.ZodString>;
|
|
15
|
-
extraction_profile: z.ZodOptional<z.ZodEnum<["auto", "product", "article", "job_posting", "faq", "recipe", "event", "ecommerce_homepage", "directory_listing"]>>;
|
|
16
|
-
template: z.ZodOptional<z.ZodString>;
|
|
17
|
-
evidence: z.ZodOptional<z.ZodBoolean>;
|
|
18
|
-
filter_content: z.ZodOptional<z.ZodBoolean>;
|
|
19
|
-
cache: z.ZodOptional<z.ZodBoolean>;
|
|
20
|
-
cache_ttl: z.ZodOptional<z.ZodNumber>;
|
|
21
|
-
render_js: z.ZodDefault<z.ZodUnion<[z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
22
|
-
use_proxy: z.ZodDefault<z.ZodBoolean>;
|
|
23
|
-
proxy_country: z.ZodOptional<z.ZodString>;
|
|
24
|
-
wait_for: z.ZodOptional<z.ZodString>;
|
|
25
|
-
timeout: z.ZodDefault<z.ZodNumber>;
|
|
26
|
-
max_response_bytes: z.ZodOptional<z.ZodDefault<z.ZodNumber>>;
|
|
27
|
-
include_raw_html: z.ZodDefault<z.ZodBoolean>;
|
|
28
|
-
session_id: z.ZodOptional<z.ZodString>;
|
|
29
|
-
cookies: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
30
|
-
scroll_to_load: z.ZodDefault<z.ZodBoolean>;
|
|
31
|
-
scroll_count: z.ZodOptional<z.ZodDefault<z.ZodNumber>>;
|
|
32
|
-
remove_cookie_banners: z.ZodDefault<z.ZodBoolean>;
|
|
33
|
-
block_images: z.ZodDefault<z.ZodBoolean>;
|
|
34
6
|
location: z.ZodOptional<z.ZodObject<{
|
|
35
7
|
country: z.ZodOptional<z.ZodString>;
|
|
36
8
|
language: z.ZodOptional<z.ZodString>;
|
|
@@ -44,8 +16,6 @@ export declare const scrapeSchema: z.ZodObject<{
|
|
|
44
16
|
prefer_cost: z.ZodOptional<z.ZodBoolean>;
|
|
45
17
|
prefer_speed: z.ZodOptional<z.ZodBoolean>;
|
|
46
18
|
fail_fast: z.ZodOptional<z.ZodBoolean>;
|
|
47
|
-
force_refresh: z.ZodOptional<z.ZodBoolean>;
|
|
48
|
-
promote_schema_org: z.ZodOptional<z.ZodBoolean>;
|
|
49
19
|
cost_controls: z.ZodOptional<z.ZodObject<{
|
|
50
20
|
force_tier: z.ZodOptional<z.ZodEnum<["1", "2", "3", "3.5", "4"]>>;
|
|
51
21
|
max_tier: z.ZodOptional<z.ZodEnum<["1", "2", "3", "3.5", "4"]>>;
|
|
@@ -72,22 +42,201 @@ export declare const scrapeSchema: z.ZodObject<{
|
|
|
72
42
|
time_budget?: number | undefined;
|
|
73
43
|
}>>;
|
|
74
44
|
estimate_first: z.ZodDefault<z.ZodBoolean>;
|
|
45
|
+
render_js: z.ZodOptional<z.ZodUnion<[z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
46
|
+
use_proxy: z.ZodOptional<z.ZodBoolean>;
|
|
47
|
+
use_own_proxy: z.ZodOptional<z.ZodBoolean>;
|
|
48
|
+
use_system_proxy: z.ZodOptional<z.ZodBoolean>;
|
|
49
|
+
proxy_integration_id: z.ZodOptional<z.ZodString>;
|
|
50
|
+
proxy_country: z.ZodOptional<z.ZodString>;
|
|
51
|
+
wait_condition: z.ZodOptional<z.ZodString>;
|
|
52
|
+
remove_cookie_banners: z.ZodOptional<z.ZodBoolean>;
|
|
53
|
+
include_iframes: z.ZodOptional<z.ZodBoolean>;
|
|
54
|
+
flatten_shadow_dom: z.ZodOptional<z.ZodBoolean>;
|
|
55
|
+
skip_tls_verification: z.ZodOptional<z.ZodBoolean>;
|
|
56
|
+
block_images: z.ZodOptional<z.ZodBoolean>;
|
|
57
|
+
scroll_to_load: z.ZodOptional<z.ZodBoolean>;
|
|
58
|
+
scroll_count: z.ZodOptional<z.ZodNumber>;
|
|
59
|
+
generate_pdf: z.ZodOptional<z.ZodBoolean>;
|
|
60
|
+
ocr: z.ZodOptional<z.ZodBoolean>;
|
|
61
|
+
section_filter: z.ZodOptional<z.ZodObject<{
|
|
62
|
+
min_content_blocks: z.ZodOptional<z.ZodNumber>;
|
|
63
|
+
exclude_content_types: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
|
|
64
|
+
content_only: z.ZodOptional<z.ZodBoolean>;
|
|
65
|
+
}, "strip", z.ZodTypeAny, {
|
|
66
|
+
min_content_blocks?: number | undefined;
|
|
67
|
+
exclude_content_types?: string[] | undefined;
|
|
68
|
+
content_only?: boolean | undefined;
|
|
69
|
+
}, {
|
|
70
|
+
min_content_blocks?: number | undefined;
|
|
71
|
+
exclude_content_types?: string[] | undefined;
|
|
72
|
+
content_only?: boolean | undefined;
|
|
73
|
+
}>>;
|
|
74
|
+
session_id: z.ZodOptional<z.ZodString>;
|
|
75
|
+
cookies: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
76
|
+
session_headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
77
|
+
sticky_session: z.ZodOptional<z.ZodString>;
|
|
78
|
+
sticky_session_ttl: z.ZodOptional<z.ZodNumber>;
|
|
79
|
+
actions: z.ZodOptional<z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
80
|
+
type: z.ZodLiteral<"wait">;
|
|
81
|
+
milliseconds: z.ZodOptional<z.ZodNumber>;
|
|
82
|
+
selector: z.ZodOptional<z.ZodString>;
|
|
83
|
+
}, "strip", z.ZodTypeAny, {
|
|
84
|
+
type: "wait";
|
|
85
|
+
milliseconds?: number | undefined;
|
|
86
|
+
selector?: string | undefined;
|
|
87
|
+
}, {
|
|
88
|
+
type: "wait";
|
|
89
|
+
milliseconds?: number | undefined;
|
|
90
|
+
selector?: string | undefined;
|
|
91
|
+
}>, z.ZodObject<{
|
|
92
|
+
type: z.ZodLiteral<"wait_for">;
|
|
93
|
+
selector: z.ZodString;
|
|
94
|
+
timeout: z.ZodOptional<z.ZodNumber>;
|
|
95
|
+
}, "strip", z.ZodTypeAny, {
|
|
96
|
+
type: "wait_for";
|
|
97
|
+
selector: string;
|
|
98
|
+
timeout?: number | undefined;
|
|
99
|
+
}, {
|
|
100
|
+
type: "wait_for";
|
|
101
|
+
selector: string;
|
|
102
|
+
timeout?: number | undefined;
|
|
103
|
+
}>, z.ZodObject<{
|
|
104
|
+
type: z.ZodLiteral<"click">;
|
|
105
|
+
selector: z.ZodString;
|
|
106
|
+
}, "strip", z.ZodTypeAny, {
|
|
107
|
+
type: "click";
|
|
108
|
+
selector: string;
|
|
109
|
+
}, {
|
|
110
|
+
type: "click";
|
|
111
|
+
selector: string;
|
|
112
|
+
}>, z.ZodObject<{
|
|
113
|
+
type: z.ZodLiteral<"fill">;
|
|
114
|
+
selector: z.ZodString;
|
|
115
|
+
value: z.ZodString;
|
|
116
|
+
}, "strip", z.ZodTypeAny, {
|
|
117
|
+
type: "fill";
|
|
118
|
+
value: string;
|
|
119
|
+
selector: string;
|
|
120
|
+
}, {
|
|
121
|
+
type: "fill";
|
|
122
|
+
value: string;
|
|
123
|
+
selector: string;
|
|
124
|
+
}>, z.ZodObject<{
|
|
125
|
+
type: z.ZodLiteral<"press">;
|
|
126
|
+
key: z.ZodString;
|
|
127
|
+
}, "strip", z.ZodTypeAny, {
|
|
128
|
+
type: "press";
|
|
129
|
+
key: string;
|
|
130
|
+
}, {
|
|
131
|
+
type: "press";
|
|
132
|
+
key: string;
|
|
133
|
+
}>, z.ZodObject<{
|
|
134
|
+
type: z.ZodLiteral<"scroll">;
|
|
135
|
+
direction: z.ZodOptional<z.ZodEnum<["down", "up"]>>;
|
|
136
|
+
amount: z.ZodOptional<z.ZodNumber>;
|
|
137
|
+
}, "strip", z.ZodTypeAny, {
|
|
138
|
+
type: "scroll";
|
|
139
|
+
direction?: "down" | "up" | undefined;
|
|
140
|
+
amount?: number | undefined;
|
|
141
|
+
}, {
|
|
142
|
+
type: "scroll";
|
|
143
|
+
direction?: "down" | "up" | undefined;
|
|
144
|
+
amount?: number | undefined;
|
|
145
|
+
}>, z.ZodObject<{
|
|
146
|
+
type: z.ZodLiteral<"infinite_scroll">;
|
|
147
|
+
max_scrolls: z.ZodOptional<z.ZodNumber>;
|
|
148
|
+
delay: z.ZodOptional<z.ZodNumber>;
|
|
149
|
+
end_selector: z.ZodOptional<z.ZodString>;
|
|
150
|
+
}, "strip", z.ZodTypeAny, {
|
|
151
|
+
type: "infinite_scroll";
|
|
152
|
+
max_scrolls?: number | undefined;
|
|
153
|
+
delay?: number | undefined;
|
|
154
|
+
end_selector?: string | undefined;
|
|
155
|
+
}, {
|
|
156
|
+
type: "infinite_scroll";
|
|
157
|
+
max_scrolls?: number | undefined;
|
|
158
|
+
delay?: number | undefined;
|
|
159
|
+
end_selector?: string | undefined;
|
|
160
|
+
}>, z.ZodObject<{
|
|
161
|
+
type: z.ZodLiteral<"screenshot">;
|
|
162
|
+
full_page: z.ZodOptional<z.ZodBoolean>;
|
|
163
|
+
}, "strip", z.ZodTypeAny, {
|
|
164
|
+
type: "screenshot";
|
|
165
|
+
full_page?: boolean | undefined;
|
|
166
|
+
}, {
|
|
167
|
+
type: "screenshot";
|
|
168
|
+
full_page?: boolean | undefined;
|
|
169
|
+
}>, z.ZodObject<{
|
|
170
|
+
type: z.ZodLiteral<"evaluate">;
|
|
171
|
+
script: z.ZodString;
|
|
172
|
+
}, "strip", z.ZodTypeAny, {
|
|
173
|
+
type: "evaluate";
|
|
174
|
+
script: string;
|
|
175
|
+
}, {
|
|
176
|
+
type: "evaluate";
|
|
177
|
+
script: string;
|
|
178
|
+
}>]>, "many">>;
|
|
179
|
+
url: z.ZodString;
|
|
180
|
+
method: z.ZodDefault<z.ZodEnum<["GET", "POST"]>>;
|
|
181
|
+
body: z.ZodOptional<z.ZodString>;
|
|
182
|
+
content_type: z.ZodOptional<z.ZodEnum<["application/json", "application/x-www-form-urlencoded", "text/plain", "application/graphql"]>>;
|
|
183
|
+
headers: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
184
|
+
mode: z.ZodDefault<z.ZodEnum<["auto", "html", "js", "pdf", "ocr"]>>;
|
|
185
|
+
formats: z.ZodDefault<z.ZodArray<z.ZodEnum<["text", "json", "json_v2", "html", "markdown", "rag", "content", "raw"]>, "many">>;
|
|
186
|
+
extraction_schema: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
187
|
+
extraction_model: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
188
|
+
extraction_provider: z.ZodOptional<z.ZodEnum<["openai", "anthropic", "openrouter", "groq"]>>;
|
|
189
|
+
extraction_prompt: z.ZodOptional<z.ZodString>;
|
|
190
|
+
extraction_profile: z.ZodOptional<z.ZodEnum<["auto", "product", "article", "job_posting", "faq", "recipe", "event", "ecommerce_homepage", "directory_listing"]>>;
|
|
191
|
+
extraction_template: z.ZodOptional<z.ZodEnum<["auto", "product", "article", "job_posting", "faq", "recipe", "event", "ecommerce_homepage", "directory_listing"]>>;
|
|
192
|
+
template: z.ZodOptional<z.ZodString>;
|
|
193
|
+
evidence: z.ZodOptional<z.ZodBoolean>;
|
|
194
|
+
filter_content: z.ZodOptional<z.ZodBoolean>;
|
|
195
|
+
promote_schema_org: z.ZodOptional<z.ZodBoolean>;
|
|
196
|
+
cache: z.ZodOptional<z.ZodBoolean>;
|
|
197
|
+
cache_ttl: z.ZodOptional<z.ZodNumber>;
|
|
198
|
+
force_refresh: z.ZodOptional<z.ZodBoolean>;
|
|
199
|
+
include_raw_html: z.ZodDefault<z.ZodBoolean>;
|
|
200
|
+
max_response_bytes: z.ZodOptional<z.ZodNumber>;
|
|
201
|
+
timeout: z.ZodDefault<z.ZodNumber>;
|
|
202
|
+
wait_for: z.ZodOptional<z.ZodString>;
|
|
203
|
+
wait_until: z.ZodOptional<z.ZodString>;
|
|
204
|
+
screenshot: z.ZodOptional<z.ZodBoolean>;
|
|
205
|
+
enable_scroll: z.ZodOptional<z.ZodNullable<z.ZodBoolean>>;
|
|
206
|
+
pdf_format: z.ZodOptional<z.ZodString>;
|
|
207
|
+
ocr_language: z.ZodOptional<z.ZodString>;
|
|
208
|
+
sync_policy: z.ZodOptional<z.ZodEnum<["auto", "wait_until_deadline"]>>;
|
|
209
|
+
embeddings: z.ZodOptional<z.ZodObject<{
|
|
210
|
+
enabled: z.ZodOptional<z.ZodBoolean>;
|
|
211
|
+
model: z.ZodOptional<z.ZodEnum<["text-embedding-3-small", "text-embedding-3-large"]>>;
|
|
212
|
+
chunk_size: z.ZodOptional<z.ZodNumber>;
|
|
213
|
+
chunk_overlap: z.ZodOptional<z.ZodNumber>;
|
|
214
|
+
}, "strip", z.ZodTypeAny, {
|
|
215
|
+
enabled?: boolean | undefined;
|
|
216
|
+
model?: "text-embedding-3-small" | "text-embedding-3-large" | undefined;
|
|
217
|
+
chunk_size?: number | undefined;
|
|
218
|
+
chunk_overlap?: number | undefined;
|
|
219
|
+
}, {
|
|
220
|
+
enabled?: boolean | undefined;
|
|
221
|
+
model?: "text-embedding-3-small" | "text-embedding-3-large" | undefined;
|
|
222
|
+
chunk_size?: number | undefined;
|
|
223
|
+
chunk_overlap?: number | undefined;
|
|
224
|
+
}>>;
|
|
75
225
|
}, "strip", z.ZodTypeAny, {
|
|
76
226
|
mode: "auto" | "html" | "js" | "pdf" | "ocr";
|
|
77
227
|
method: "POST" | "GET";
|
|
78
228
|
timeout: number;
|
|
79
229
|
url: string;
|
|
80
|
-
formats: ("content" | "text" | "
|
|
81
|
-
render_js: boolean | "auto";
|
|
82
|
-
use_proxy: boolean;
|
|
230
|
+
formats: ("content" | "text" | "json" | "json_v2" | "html" | "markdown" | "rag" | "raw")[];
|
|
83
231
|
include_raw_html: boolean;
|
|
84
|
-
scroll_to_load: boolean;
|
|
85
|
-
remove_cookie_banners: boolean;
|
|
86
|
-
block_images: boolean;
|
|
87
232
|
estimate_first: boolean;
|
|
233
|
+
headers?: Record<string, string> | undefined;
|
|
234
|
+
ocr?: boolean | undefined;
|
|
88
235
|
prefer_cost?: boolean | undefined;
|
|
89
236
|
prefer_speed?: boolean | undefined;
|
|
90
237
|
fail_fast?: boolean | undefined;
|
|
238
|
+
wait_for?: string | undefined;
|
|
239
|
+
screenshot?: boolean | undefined;
|
|
91
240
|
body?: string | undefined;
|
|
92
241
|
content_type?: "application/json" | "application/x-www-form-urlencoded" | "text/plain" | "application/graphql" | undefined;
|
|
93
242
|
extraction_schema?: Record<string, unknown> | undefined;
|
|
@@ -95,23 +244,30 @@ export declare const scrapeSchema: z.ZodObject<{
|
|
|
95
244
|
extraction_provider?: "openai" | "anthropic" | "openrouter" | "groq" | undefined;
|
|
96
245
|
extraction_prompt?: string | undefined;
|
|
97
246
|
extraction_profile?: "auto" | "product" | "article" | "job_posting" | "faq" | "recipe" | "event" | "ecommerce_homepage" | "directory_listing" | undefined;
|
|
247
|
+
extraction_template?: "auto" | "product" | "article" | "job_posting" | "faq" | "recipe" | "event" | "ecommerce_homepage" | "directory_listing" | undefined;
|
|
98
248
|
template?: string | undefined;
|
|
99
249
|
evidence?: boolean | undefined;
|
|
100
250
|
filter_content?: boolean | undefined;
|
|
251
|
+
promote_schema_org?: boolean | undefined;
|
|
101
252
|
cache?: boolean | undefined;
|
|
102
253
|
cache_ttl?: number | undefined;
|
|
103
|
-
|
|
104
|
-
wait_for?: string | undefined;
|
|
254
|
+
force_refresh?: boolean | undefined;
|
|
105
255
|
max_response_bytes?: number | undefined;
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
256
|
+
wait_until?: string | undefined;
|
|
257
|
+
enable_scroll?: boolean | null | undefined;
|
|
258
|
+
pdf_format?: string | undefined;
|
|
259
|
+
ocr_language?: string | undefined;
|
|
260
|
+
sync_policy?: "auto" | "wait_until_deadline" | undefined;
|
|
261
|
+
embeddings?: {
|
|
262
|
+
enabled?: boolean | undefined;
|
|
263
|
+
model?: "text-embedding-3-small" | "text-embedding-3-large" | undefined;
|
|
264
|
+
chunk_size?: number | undefined;
|
|
265
|
+
chunk_overlap?: number | undefined;
|
|
266
|
+
} | undefined;
|
|
109
267
|
location?: {
|
|
110
268
|
country?: string | undefined;
|
|
111
269
|
language?: string | undefined;
|
|
112
270
|
} | undefined;
|
|
113
|
-
force_refresh?: boolean | undefined;
|
|
114
|
-
promote_schema_org?: boolean | undefined;
|
|
115
271
|
cost_controls?: {
|
|
116
272
|
force_tier?: "1" | "2" | "3" | "3.5" | "4" | undefined;
|
|
117
273
|
max_tier?: "1" | "2" | "3" | "3.5" | "4" | undefined;
|
|
@@ -121,45 +277,110 @@ export declare const scrapeSchema: z.ZodObject<{
|
|
|
121
277
|
fail_fast?: boolean | undefined;
|
|
122
278
|
time_budget?: number | undefined;
|
|
123
279
|
} | undefined;
|
|
280
|
+
render_js?: boolean | "auto" | undefined;
|
|
281
|
+
use_proxy?: boolean | undefined;
|
|
282
|
+
use_own_proxy?: boolean | undefined;
|
|
283
|
+
use_system_proxy?: boolean | undefined;
|
|
284
|
+
proxy_integration_id?: string | undefined;
|
|
285
|
+
proxy_country?: string | undefined;
|
|
286
|
+
wait_condition?: string | undefined;
|
|
287
|
+
remove_cookie_banners?: boolean | undefined;
|
|
288
|
+
include_iframes?: boolean | undefined;
|
|
289
|
+
flatten_shadow_dom?: boolean | undefined;
|
|
290
|
+
skip_tls_verification?: boolean | undefined;
|
|
291
|
+
block_images?: boolean | undefined;
|
|
292
|
+
scroll_to_load?: boolean | undefined;
|
|
293
|
+
scroll_count?: number | undefined;
|
|
294
|
+
generate_pdf?: boolean | undefined;
|
|
295
|
+
section_filter?: {
|
|
296
|
+
min_content_blocks?: number | undefined;
|
|
297
|
+
exclude_content_types?: string[] | undefined;
|
|
298
|
+
content_only?: boolean | undefined;
|
|
299
|
+
} | undefined;
|
|
300
|
+
session_id?: string | undefined;
|
|
301
|
+
cookies?: Record<string, string> | undefined;
|
|
302
|
+
session_headers?: Record<string, string> | undefined;
|
|
303
|
+
sticky_session?: string | undefined;
|
|
304
|
+
sticky_session_ttl?: number | undefined;
|
|
305
|
+
actions?: ({
|
|
306
|
+
type: "wait";
|
|
307
|
+
milliseconds?: number | undefined;
|
|
308
|
+
selector?: string | undefined;
|
|
309
|
+
} | {
|
|
310
|
+
type: "wait_for";
|
|
311
|
+
selector: string;
|
|
312
|
+
timeout?: number | undefined;
|
|
313
|
+
} | {
|
|
314
|
+
type: "click";
|
|
315
|
+
selector: string;
|
|
316
|
+
} | {
|
|
317
|
+
type: "fill";
|
|
318
|
+
value: string;
|
|
319
|
+
selector: string;
|
|
320
|
+
} | {
|
|
321
|
+
type: "press";
|
|
322
|
+
key: string;
|
|
323
|
+
} | {
|
|
324
|
+
type: "scroll";
|
|
325
|
+
direction?: "down" | "up" | undefined;
|
|
326
|
+
amount?: number | undefined;
|
|
327
|
+
} | {
|
|
328
|
+
type: "infinite_scroll";
|
|
329
|
+
max_scrolls?: number | undefined;
|
|
330
|
+
delay?: number | undefined;
|
|
331
|
+
end_selector?: string | undefined;
|
|
332
|
+
} | {
|
|
333
|
+
type: "screenshot";
|
|
334
|
+
full_page?: boolean | undefined;
|
|
335
|
+
} | {
|
|
336
|
+
type: "evaluate";
|
|
337
|
+
script: string;
|
|
338
|
+
})[] | undefined;
|
|
124
339
|
}, {
|
|
125
340
|
url: string;
|
|
126
341
|
mode?: "auto" | "html" | "js" | "pdf" | "ocr" | undefined;
|
|
342
|
+
headers?: Record<string, string> | undefined;
|
|
127
343
|
method?: "POST" | "GET" | undefined;
|
|
128
344
|
timeout?: number | undefined;
|
|
345
|
+
ocr?: boolean | undefined;
|
|
129
346
|
prefer_cost?: boolean | undefined;
|
|
130
347
|
prefer_speed?: boolean | undefined;
|
|
131
348
|
fail_fast?: boolean | undefined;
|
|
349
|
+
wait_for?: string | undefined;
|
|
350
|
+
screenshot?: boolean | undefined;
|
|
132
351
|
body?: string | undefined;
|
|
133
352
|
content_type?: "application/json" | "application/x-www-form-urlencoded" | "text/plain" | "application/graphql" | undefined;
|
|
134
|
-
formats?: ("content" | "text" | "
|
|
353
|
+
formats?: ("content" | "text" | "json" | "json_v2" | "html" | "markdown" | "rag" | "raw")[] | undefined;
|
|
135
354
|
extraction_schema?: Record<string, unknown> | undefined;
|
|
136
355
|
extraction_model?: string | null | undefined;
|
|
137
356
|
extraction_provider?: "openai" | "anthropic" | "openrouter" | "groq" | undefined;
|
|
138
357
|
extraction_prompt?: string | undefined;
|
|
139
358
|
extraction_profile?: "auto" | "product" | "article" | "job_posting" | "faq" | "recipe" | "event" | "ecommerce_homepage" | "directory_listing" | undefined;
|
|
359
|
+
extraction_template?: "auto" | "product" | "article" | "job_posting" | "faq" | "recipe" | "event" | "ecommerce_homepage" | "directory_listing" | undefined;
|
|
140
360
|
template?: string | undefined;
|
|
141
361
|
evidence?: boolean | undefined;
|
|
142
362
|
filter_content?: boolean | undefined;
|
|
363
|
+
promote_schema_org?: boolean | undefined;
|
|
143
364
|
cache?: boolean | undefined;
|
|
144
365
|
cache_ttl?: number | undefined;
|
|
145
|
-
|
|
146
|
-
use_proxy?: boolean | undefined;
|
|
147
|
-
proxy_country?: string | undefined;
|
|
148
|
-
wait_for?: string | undefined;
|
|
149
|
-
max_response_bytes?: number | undefined;
|
|
366
|
+
force_refresh?: boolean | undefined;
|
|
150
367
|
include_raw_html?: boolean | undefined;
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
368
|
+
max_response_bytes?: number | undefined;
|
|
369
|
+
wait_until?: string | undefined;
|
|
370
|
+
enable_scroll?: boolean | null | undefined;
|
|
371
|
+
pdf_format?: string | undefined;
|
|
372
|
+
ocr_language?: string | undefined;
|
|
373
|
+
sync_policy?: "auto" | "wait_until_deadline" | undefined;
|
|
374
|
+
embeddings?: {
|
|
375
|
+
enabled?: boolean | undefined;
|
|
376
|
+
model?: "text-embedding-3-small" | "text-embedding-3-large" | undefined;
|
|
377
|
+
chunk_size?: number | undefined;
|
|
378
|
+
chunk_overlap?: number | undefined;
|
|
379
|
+
} | undefined;
|
|
157
380
|
location?: {
|
|
158
381
|
country?: string | undefined;
|
|
159
382
|
language?: string | undefined;
|
|
160
383
|
} | undefined;
|
|
161
|
-
force_refresh?: boolean | undefined;
|
|
162
|
-
promote_schema_org?: boolean | undefined;
|
|
163
384
|
cost_controls?: {
|
|
164
385
|
force_tier?: "1" | "2" | "3" | "3.5" | "4" | undefined;
|
|
165
386
|
max_tier?: "1" | "2" | "3" | "3.5" | "4" | undefined;
|
|
@@ -170,7 +391,89 @@ export declare const scrapeSchema: z.ZodObject<{
|
|
|
170
391
|
time_budget?: number | undefined;
|
|
171
392
|
} | undefined;
|
|
172
393
|
estimate_first?: boolean | undefined;
|
|
394
|
+
render_js?: boolean | "auto" | undefined;
|
|
395
|
+
use_proxy?: boolean | undefined;
|
|
396
|
+
use_own_proxy?: boolean | undefined;
|
|
397
|
+
use_system_proxy?: boolean | undefined;
|
|
398
|
+
proxy_integration_id?: string | undefined;
|
|
399
|
+
proxy_country?: string | undefined;
|
|
400
|
+
wait_condition?: string | undefined;
|
|
401
|
+
remove_cookie_banners?: boolean | undefined;
|
|
402
|
+
include_iframes?: boolean | undefined;
|
|
403
|
+
flatten_shadow_dom?: boolean | undefined;
|
|
404
|
+
skip_tls_verification?: boolean | undefined;
|
|
405
|
+
block_images?: boolean | undefined;
|
|
406
|
+
scroll_to_load?: boolean | undefined;
|
|
407
|
+
scroll_count?: number | undefined;
|
|
408
|
+
generate_pdf?: boolean | undefined;
|
|
409
|
+
section_filter?: {
|
|
410
|
+
min_content_blocks?: number | undefined;
|
|
411
|
+
exclude_content_types?: string[] | undefined;
|
|
412
|
+
content_only?: boolean | undefined;
|
|
413
|
+
} | undefined;
|
|
414
|
+
session_id?: string | undefined;
|
|
415
|
+
cookies?: Record<string, string> | undefined;
|
|
416
|
+
session_headers?: Record<string, string> | undefined;
|
|
417
|
+
sticky_session?: string | undefined;
|
|
418
|
+
sticky_session_ttl?: number | undefined;
|
|
419
|
+
actions?: ({
|
|
420
|
+
type: "wait";
|
|
421
|
+
milliseconds?: number | undefined;
|
|
422
|
+
selector?: string | undefined;
|
|
423
|
+
} | {
|
|
424
|
+
type: "wait_for";
|
|
425
|
+
selector: string;
|
|
426
|
+
timeout?: number | undefined;
|
|
427
|
+
} | {
|
|
428
|
+
type: "click";
|
|
429
|
+
selector: string;
|
|
430
|
+
} | {
|
|
431
|
+
type: "fill";
|
|
432
|
+
value: string;
|
|
433
|
+
selector: string;
|
|
434
|
+
} | {
|
|
435
|
+
type: "press";
|
|
436
|
+
key: string;
|
|
437
|
+
} | {
|
|
438
|
+
type: "scroll";
|
|
439
|
+
direction?: "down" | "up" | undefined;
|
|
440
|
+
amount?: number | undefined;
|
|
441
|
+
} | {
|
|
442
|
+
type: "infinite_scroll";
|
|
443
|
+
max_scrolls?: number | undefined;
|
|
444
|
+
delay?: number | undefined;
|
|
445
|
+
end_selector?: string | undefined;
|
|
446
|
+
} | {
|
|
447
|
+
type: "screenshot";
|
|
448
|
+
full_page?: boolean | undefined;
|
|
449
|
+
} | {
|
|
450
|
+
type: "evaluate";
|
|
451
|
+
script: string;
|
|
452
|
+
})[] | undefined;
|
|
173
453
|
}>;
|
|
174
454
|
export declare const scrapeDescription: string;
|
|
175
|
-
|
|
455
|
+
type ScrapeParams = z.infer<typeof scrapeSchema>;
|
|
456
|
+
/**
|
|
457
|
+
* Translate flat tool parameters into the API's UnifiedScrapeRequest body.
|
|
458
|
+
* Exported for tests.
|
|
459
|
+
*/
|
|
460
|
+
export declare function buildScrapeRequest(params: ScrapeParams): UnifiedScrapeRequest;
|
|
461
|
+
/**
|
|
462
|
+
* Cost-estimate request for the same scrape. The estimate endpoint prices from
|
|
463
|
+
* url, mode, method, formats and the cost-bearing advanced flags only.
|
|
464
|
+
*/
|
|
465
|
+
export declare function buildEstimateRequest(params: {
|
|
466
|
+
url: string;
|
|
467
|
+
mode?: ScrapeParams["mode"];
|
|
468
|
+
method?: ScrapeParams["method"];
|
|
469
|
+
formats?: ScrapeParams["formats"];
|
|
470
|
+
render_js?: ScrapeParams["render_js"];
|
|
471
|
+
use_proxy?: boolean;
|
|
472
|
+
use_own_proxy?: boolean;
|
|
473
|
+
screenshot?: boolean;
|
|
474
|
+
generate_pdf?: boolean;
|
|
475
|
+
ocr?: boolean;
|
|
476
|
+
}): UnifiedScrapeRequest;
|
|
477
|
+
export declare function handleScrape(client: AlterLabClient, params: ScrapeParams): Promise<CallToolResult>;
|
|
478
|
+
export {};
|
|
176
479
|
//# sourceMappingURL=scrape.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/tools/scrape.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACzE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/tools/scrape.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACzE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAGnD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAgBxD,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAqQvB,CAAC;AAEH,eAAO,MAAM,iBAAiB,QAyCkE,CAAC;AAEjG,KAAK,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAajD;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,YAAY,GAAG,oBAAoB,CA6C7E;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE;IAC3C,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;IAC5B,MAAM,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IAChC,OAAO,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC;IAClC,SAAS,CAAC,EAAE,YAAY,CAAC,WAAW,CAAC,CAAC;IACtC,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,GAAG,CAAC,EAAE,OAAO,CAAC;CACf,GAAG,oBAAoB,CAiBvB;AAED,wBAAsB,YAAY,CAChC,MAAM,EAAE,cAAc,EACtB,MAAM,EAAE,YAAY,GACnB,OAAO,CAAC,cAAc,CAAC,CA+BzB"}
|