alterlab-mcp-server 1.7.0 → 1.9.0

package/dist/tools/crawl.js

@@ -1,16 +1,37 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.crawlCancelDescription = exports.crawlCancelSchema = exports.crawlStatusDescription = exports.crawlStatusSchema = exports.crawlDescription = exports.crawlSchema = void 0;
+exports.buildCrawlRequest = buildCrawlRequest;
 exports.handleCrawl = handleCrawl;
 exports.handleCrawlStatus = handleCrawlStatus;
 exports.handleCrawlCancel = handleCrawlCancel;
 const zod_1 = require("zod");
 const errors_js_1 = require("../errors.js");
+const format_js_1 = require("../format.js");
+const common_js_1 = require("./common.js");
 // ============================================================================
 // Start Crawl
 // ============================================================================
+const crawlCostControlsSchema = zod_1.z
+    .object({
+    max_credits: zod_1.z
+        .number()
+        .int()
+        .positive()
+        .optional()
+        .describe("Hard spending cap in credits for the entire crawl"),
+    max_tier: common_js_1.tierEnum
+        .optional()
+        .describe("Maximum tier to use for page scrapes " +
+        "(1=curl $0.0002, 2=http $0.0003, 3=stealth $0.002, 3.5=lightjs $0.0025, 4=browser $0.004). " +
+        "render_js=true with a max_tier below 4 is rejected (render_js forces tier 4)."),
+    force_tier: common_js_1.tierEnum
+        .optional()
+        .describe("Force all pages to use this exact tier, bypassing per-page escalation"),
+})
+    .describe("Cost controls for the entire crawl — cap total spend or pin the scraping tier");
 exports.crawlSchema = zod_1.z.object({
-    url: zod_1.z.string().url().describe("Start URL for the crawl"),
+    url: zod_1.z.string().url().max(2083).describe("Start URL for the crawl"),
     max_pages: zod_1.z
         .number()
         .int()
@@ -27,37 +48,69 @@ exports.crawlSchema = zod_1.z.object({
         .describe("Maximum link-following depth from start URL (0 = start page only)"),
     include_patterns: zod_1.z
         .array(zod_1.z.string())
+        .max(50)
         .optional()
         .describe("Glob patterns — only scrape URLs whose path matches at least one (e.g., ['/blog/*', '/docs/*'])"),
     exclude_patterns: zod_1.z
         .array(zod_1.z.string())
+        .max(50)
         .optional()
         .describe("Glob patterns — skip URLs whose path matches any (e.g., ['/tag/*', '/author/*'])"),
     sitemap: zod_1.z
         .enum(["include", "skip", "only"])
         .default("include")
         .describe("Sitemap mode: include (default), skip (link extraction only), only (sitemap URLs only)"),
+    sitemap_path: zod_1.z
+        .string()
+        .optional()
+        .describe("Explicit path to the sitemap file (e.g., '/sitemap_index.xml'). " +
+        "Use when the sitemap is not at the standard /sitemap.xml location."),
+    budget: zod_1.z
+        .record(zod_1.z.string(), zod_1.z.number().int())
+        .optional()
+        .describe("Per-path page budget: glob pattern -> max pages, most specific pattern wins, '*' is the catch-all " +
+        '(e.g., {"/products/*": 100, "/blog/*": 50, "*": 5}). max_pages stays the hard cap.'),
+    priority: zod_1.z
+        .object({
+        keywords: zod_1.z.array(zod_1.z.string()).optional().describe("Crawl pages matching these keywords first"),
+        boost_patterns: zod_1.z.array(zod_1.z.string()).optional().describe("URL glob patterns to crawl earlier"),
+        demote_patterns: zod_1.z.array(zod_1.z.string()).optional().describe("URL glob patterns to crawl later"),
+    })
+        .optional()
+        .describe("Crawl prioritisation hints (keywords, boost_patterns, demote_patterns)"),
     formats: zod_1.z
-        .array(zod_1.z.enum(["text", "json", "json_v2", "html", "markdown"]))
+        .array(common_js_1.crawlFormatEnum)
         .optional()
-        .describe("Output formats for each scraped page"),
+        .describe("Output formats for each scraped page. " + common_js_1.FORMATS_DESCRIPTION),
+    extraction: zod_1.z
+        .literal("auto")
+        .optional()
+        .describe("Set to 'auto' for automatic page-type detection and extraction without a user-defined schema. " +
+        "Mutually exclusive with extraction_schema."),
     extraction_schema: zod_1.z
         .record(zod_1.z.unknown())
         .optional()
         .describe("JSON schema for structured extraction on each page"),
+    extraction_profile: common_js_1.extractionProfileEnum
+        .optional()
+        .describe("Pre-defined extraction profile applied to every crawled page. " +
+        "'auto' detects the page type automatically."),
+    extraction_template: common_js_1.extractionProfileEnum
+        .optional()
+        .describe("Alias for extraction_profile (mutually exclusive with it)"),
+    extraction_provider: common_js_1.extractionProviderEnum
+        .optional()
+        .describe("BYOK LLM provider used for extraction on each page"),
     extraction_model: zod_1.z
         .string()
         .nullish()
         .describe("Per-request LLM model override in provider-specific format (e.g. 'gpt-4o', 'claude-opus-4-5-20251101', 'llama3-70b-8192'). " +
         "Overrides the model saved in your BYOK key settings for this request only."),
-    render_js: zod_1.z
-        .union([zod_1.z.boolean(), zod_1.z.literal("auto")])
-        .default(false)
-        .describe("Render JavaScript on crawled pages. true=always (Tier 4), false=never, auto=smart detection per page"),
-    use_proxy: zod_1.z
-        .boolean()
-        .default(false)
-        .describe("Route all crawl requests through premium proxy"),
+    headers: zod_1.z
+        .record(zod_1.z.string(), zod_1.z.string())
+        .optional()
+        .describe("Custom HTTP headers injected into every page request during the crawl " +
+        '(e.g., {"Authorization": "Bearer token"}). Maximum 50 headers.'),
     max_concurrency: zod_1.z
         .number()
         .int()
@@ -73,45 +126,135 @@ exports.crawlSchema = zod_1.z.object({
         .boolean()
         .default(false)
         .describe("Include links to subdomains during discovery"),
-    block_images: zod_1.z
+    change_tracking: zod_1.z
+        .object({
+        enabled: zod_1.z.boolean().optional(),
+        tracking_id: zod_1.z
+            .string()
+            .max(200)
+            .optional()
+            .describe("Stable ID to compare this crawl against earlier crawls with the same ID"),
+        mode: zod_1.z.enum(["content", "structured"]).optional(),
+        include_diff: zod_1.z.boolean().optional().describe("Include a diff of changed pages"),
+    })
+        .optional()
+        .describe("Track page changes between crawls"),
+    render_js: zod_1.z
+        .union([zod_1.z.boolean(), zod_1.z.literal("auto")])
+        .optional()
+        .describe("Render JavaScript on crawled pages. true=always (Tier 4, $0.004/page), false=never, auto=smart detection per page"),
+    screenshot: zod_1.z
         .boolean()
-        .default(false)
-        .describe("Block image downloads during browser rendering on each crawled page. " +
-        "Reduces proxy bandwidth and speeds up crawls. Only effective with render_js=true."),
+        .optional()
+        .describe("Capture a screenshot of each crawled page (requires render_js)"),
+    use_proxy: zod_1.z
+        .boolean()
+        .optional()
+        .describe("Route all crawl requests through premium proxy (+$0.0002/page)"),
+    wait_for: zod_1.z
+        .string()
+        .optional()
+        .describe("CSS selector to wait for before extracting each page (e.g., '#main-content'). " +
+        "Applied to all pages in the crawl."),
+    timeout: zod_1.z
+        .number()
+        .int()
+        .min(1)
+        .max(300)
+        .optional()
+        .describe("Per-page request timeout in seconds (1-300)"),
+    delay: zod_1.z
+        .number()
+        .min(0)
+        .max(30)
+        .optional()
+        .describe("Politeness delay between page requests, in seconds (0-30)"),
+    depth_strategy: zod_1.z
+        .enum(["eager", "sequential"])
+        .optional()
+        .describe("Link-discovery strategy (API default 'sequential')"),
     webhook_url: zod_1.z
         .string()
         .url()
+        .max(2083)
         .optional()
         .describe("Webhook URL to notify on crawl completion"),
+    webhook: zod_1.z
+        .object({
+        url: zod_1.z.string().url().max(2083),
+        events: zod_1.z.array(zod_1.z.string()).optional().describe("Event names to deliver (default: all)"),
+        secret: zod_1.z
+            .string()
+            .min(8)
+            .max(256)
+            .optional()
+            .describe("Signing secret for webhook payloads (8-256 chars)"),
+        headers: zod_1.z.record(zod_1.z.string(), zod_1.z.string()).optional(),
+        metadata: zod_1.z.record(zod_1.z.unknown()).optional().describe("Opaque metadata echoed in every event"),
+    })
+        .optional()
+        .describe("Detailed webhook config (events, signing secret, headers); use instead of webhook_url"),
+    storage: zod_1.z
+        .enum(["redis", "s3", "r2"])
+        .optional()
+        .describe("Where crawl results are stored (API default 'redis')"),
+    cache: zod_1.z.boolean().optional().describe("Serve pages from the response cache when available"),
+    cache_ttl: zod_1.z
+        .number()
+        .int()
+        .min(60)
+        .max(86400)
+        .optional()
+        .describe("Cache TTL in seconds (60-86400). Requires cache=true"),
+    force_refresh: zod_1.z
+        .boolean()
+        .optional()
+        .describe("Bypass the cache and fetch every page fresh"),
+    cost_controls: crawlCostControlsSchema.optional(),
 });
-exports.crawlDescription = "Start an asynchronous crawl of an entire website. " +
-    "Discovers URLs via sitemap parsing and link extraction, then scrapes each page. " +
+exports.crawlDescription = "Crawl an entire website for RAG, index all pages on a site, or build a knowledge base from a domain. " +
+    "Automatically bypasses anti-bot protection on every page — works on Cloudflare-protected sites. " +
+    "Discovers URLs via sitemap parsing and link extraction, then scrapes each page asynchronously. " +
+    "Each page is billed at the tier that succeeds (tier 1 $0.0002 ... tier 4 $0.004). " +
+    "Returns content as clean markdown, text, or structured data. " +
     "Returns a crawl_id immediately — use alterlab_crawl_status to poll results. " +
-    "Use include_patterns/exclude_patterns to scope the crawl to specific sections. " +
+    "Use include_patterns/exclude_patterns to scope the crawl to specific sections; budget to allocate pages per path. " +
     "Use render_js='auto' for mixed sites to save 30-60% vs always rendering. " +
-    "Supports extraction_schema to extract structured data from every page.";
+    "Supports extraction_schema, extraction_profile or extraction='auto' to extract structured data from every page. " +
+    "Use cost_controls to cap total credits or pin the scraping tier for all pages. " +
+    "Use sitemap_path to specify a non-standard sitemap location. " +
+    "Use headers to inject custom HTTP headers (e.g., Authorization) into every page request. " +
+    "Use change_tracking to detect page changes between runs; webhook/webhook_url for completion events.";
+const CRAWL_ADVANCED_KEYS = [
+    "render_js",
+    "screenshot",
+    "use_proxy",
+    "wait_for",
+    "timeout",
+    "delay",
+    "depth_strategy",
+];
+/** Translate flat crawl tool parameters into the API's CrawlRequest. Exported for tests. */
+function buildCrawlRequest(params) {
+    const { extraction_model, cost_controls, ...rest } = params;
+    const body = {};
+    for (const [k, v] of Object.entries(rest)) {
+        if (v !== undefined && !CRAWL_ADVANCED_KEYS.includes(k)) {
+            body[k] = v;
+        }
+    }
+    if (extraction_model)
+        body.extraction_model = extraction_model;
+    if (cost_controls)
+        body.cost_controls = cost_controls;
+    const advanced = (0, common_js_1.pickAdvanced)(params, CRAWL_ADVANCED_KEYS);
+    if (Object.keys(advanced).length > 0)
+        body.advanced = advanced;
+    return body;
+}
 async function handleCrawl(client, params) {
     try {
-        const response = await client.startCrawl({
-            url: params.url,
-            max_pages: params.max_pages,
-            max_depth: params.max_depth,
-            include_patterns: params.include_patterns,
-            exclude_patterns: params.exclude_patterns,
-            sitemap: params.sitemap,
-            formats: params.formats,
-            extraction_schema: params.extraction_schema,
-            extraction_model: params.extraction_model ?? undefined,
-            max_concurrency: params.max_concurrency,
-            respect_robots: params.respect_robots,
-            include_subdomains: params.include_subdomains,
-            webhook_url: params.webhook_url,
-            advanced: {
-                render_js: params.render_js,
-                use_proxy: params.use_proxy,
-                block_images: params.block_images,
-            },
-        });
+        const response = await client.startCrawl(buildCrawlRequest(params));
         const text = `**Crawl Started**\n\n` +
             `- Crawl ID: \`${response.crawl_id}\`\n` +
             `- Status: ${response.status}\n` +
@@ -120,13 +263,18 @@ async function handleCrawl(client, params) {
             `- Max depth: ${params.max_depth}\n\n` +
             `Use \`alterlab_crawl_status\` with crawl_id \`${response.crawl_id}\` to poll for results.\n` +
             `Crawls run asynchronously — check back after a few minutes for large sites.`;
-        return { content: [{ type: "text", text }] };
+        const balanceWarningSuffix = (0, format_js_1.formatBalanceWarning)(client.getLastBalanceWarning());
+        return { content: [{ type: "text", text: text + balanceWarningSuffix }] };
     }
     catch (error) {
-        if (isApiError(error)) {
-            return (0, errors_js_1.formatErrorResult)(error, { url: params.url });
+        const balanceWarningSuffix = (0, format_js_1.formatBalanceWarning)(client.getLastBalanceWarning());
+        const result = isApiError(error)
+            ? (0, errors_js_1.formatErrorResult)(error, { url: params.url })
+            : (0, errors_js_1.formatErrorResult)(error, { url: params.url });
+        if (balanceWarningSuffix && result.content[0]?.type === "text") {
+            result.content[0].text += balanceWarningSuffix;
         }
-        return (0, errors_js_1.formatErrorResult)(error, { url: params.url });
+        return result;
     }
 }
 // ============================================================================

package/dist/tools/crawl.js.map

@@ -1 +1 @@
-{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/tools/crawl.ts"],"names":[],"mappings":";;;AA2GA,kCA2CC;AAgBD,8CAcC;AAeD,8CAoBC;AAvND,6BAAwB;AAGxB,4CAAgE;AAEhE,+EAA+E;AAC/E,cAAc;AACd,+EAA+E;AAElE,QAAA,WAAW,GAAG,OAAC,CAAC,MAAM,CAAC;IAClC,GAAG,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IACzD,SAAS,EAAE,OAAC;SACT,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,MAAM,CAAC;SACX,OAAO,CAAC,EAAE,CAAC;SACX,QAAQ,CAAC,mCAAmC,CAAC;IAChD,SAAS,EAAE,OAAC;SACT,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,OAAO,CAAC,CAAC,CAAC;SACV,QAAQ,CACP,mEAAmE,CACpE;IACH,gBAAgB,EAAE,OAAC;SAChB,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;SACjB,QAAQ,EAAE;SACV,QAAQ,CACP,iGAAiG,CAClG;IACH,gBAAgB,EAAE,OAAC;SAChB,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;SACjB,QAAQ,EAAE;SACV,QAAQ,CACP,kFAAkF,CACnF;IACH,OAAO,EAAE,OAAC;SACP,IAAI,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;SACjC,OAAO,CAAC,SAAS,CAAC;SAClB,QAAQ,CACP,wFAAwF,CACzF;IACH,OAAO,EAAE,OAAC;SACP,KAAK,CAAC,OAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;SAC9D,QAAQ,EAAE;SACV,QAAQ,CAAC,sCAAsC,CAAC;IACnD,iBAAiB,EAAE,OAAC;SACjB,MAAM,CAAC,OAAC,CAAC,OAAO,EAAE,CAAC;SACnB,QAAQ,EAAE;SACV,QAAQ,CAAC,oDAAoD,CAAC;IACjE,gBAAgB,EAAE,OAAC;SAChB,MAAM,EAAE;SACR,OAAO,EAAE;SACT,QAAQ,CACP,6HAA6H;QAC3H,4EAA4E,CAC/E;IACH,SAAS,EAAE,OAAC;SACT,KAAK,CAAC,CAAC,OAAC,CAAC,OAAO,EAAE,EAAE,OAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;SACvC,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CACP,sGAAsG,CACvG;IACH,SAAS,EAAE,OAAC;SACT,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,gDAAgD,CAAC;IAC7D,eAAe,EAAE,OAAC;SACf,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,OAAO,CAAC,EAAE,CAAC;SACX,QAAQ,CAAC,mDAAmD,CAAC;IAChE,cAAc,EAAE,OAAC;SACd,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,gDAAgD,CAAC;IAC7D,kBAAkB,EAAE,OAAC;SAClB,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,8CAA8C,CAAC;IAC3D,YAAY,EAAE,OAAC;SACZ,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CACP,uEAAuE;QACrE,mFAAmF,CACtF;IACH,WAAW,EAAE,OAAC;SACX,MAAM,EAAE;SACR,GAAG,EAAE;SACL,QAAQ,EAAE;SACV,QAAQ,CAAC,2CAA2C,CAAC;CACzD,CAAC,CAAC;AAEU,QAAA,gBAAgB,GAC3B,oDAAoD;IACpD,kFAAkF;IAClF,8EAA8E;IAC9E,iFAAiF;IACjF,2EAA2E;IAC3E,wEAAwE,CAAC;AAEpE,KAAK,UAAU,WAAW,CAC/B,MAAsB,EACtB,MAAmC;IAEnC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC;YACvC,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;YAC3C,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,IAAI,SAAS;YACtD,eAAe,EAAE,MAAM,CAAC,eAAe;YACvC,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;YAC7C,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,QAAQ,EAAE;gBACR,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;aAClC;SACF,CAAC,CAAC;QAEH,MAAM,IAAI,GACR,uBAAuB;YACvB,iBAAiB,QAAQ,CAAC,QAAQ,MAAM;YACxC,aAAa,QAAQ,CAAC,MAAM,IAAI;YAChC,gBAAgB,QAAQ,CAAC,GAAG,IAAI;YAChC,gBAAgB,MAAM,CAAC,SAAS,IAAI;YACpC,gBAAgB,MAAM,CAAC,SAAS,MAAM;YACtC,iDAAiD,QAAQ,CAAC,QAAQ,2BAA2B;YAC7F,6EAA6E,CAAC;QAEhF,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,IAAA,6BAAiB,EAAC,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,IAAA,6BAAiB,EAAC,KAAc,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IAChE,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,sBAAsB;AACtB,+EAA+E;AAElE,QAAA,iBAAiB,GAAG,OAAC,CAAC,MAAM,CAAC;IACxC,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;CACrE,CAAC,CAAC;AAEU,QAAA,sBAAsB,GACjC,gEAAgE;IAChE,+EAA+E;IAC/E,gEAAgE;IAChE,4DAA4D,CAAC;AAExD,KAAK,UAAU,iBAAiB,CACrC,MAAsB,EACtB,MAAyC;IAEzC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC9D,MAAM,IAAI,GAAG,yBAAyB,CAAC,QAAQ,CAAC,CAAC;QACjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,IAAA,6BAAiB,EAAC,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,IAAA,6BAAiB,EAAC,KAAc,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,eAAe;AACf,+EAA+E;AAElE,QAAA,iBAAiB,GAAG,OAAC,CAAC,MAAM,CAAC;IACxC,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAAC;CACpD,CAAC,CAAC;AAEU,QAAA,sBAAsB,GACjC,iEAAiE;IACjE,0EAA0E;IAC1E,qCAAqC,CAAC;AAEjC,KAAK,UAAU,iBAAiB,CACrC,MAAsB,EACtB,MAAyC;IAEzC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC3D,MAAM,IAAI,GACR,yBAAyB;YACzB,iBAAiB,MAAM,CAAC,QAAQ,MAAM;YACtC,kCAAkC,QAAQ,CAAC,aAAa,IAAI,SAAS,IAAI;YACzE,uBAAuB,QAAQ,CAAC,gBAAgB,IAAI,CAAC,MAAM;YAC3D,0FAA0F,CAAC;QAE7F,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,IAAA,6BAAiB,EAAC,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,IAAA,6BAAiB,EAAC,KAAc,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,UAAU;AACV,+EAA+E;AAE/E,SAAS,yBAAyB,CAAC,QAAiB;IAClD,MAAM,CAAC,GAAG,QAAmC,CAAC;IAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;IAChC,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC;IAClD,MAAM,UAAU,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAChE,MAAM,WAAW,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACnE,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAE1D,MAAM,KAAK,GAAa;QACtB,mBAAmB,MAAM,CAAC,WAAW,EAAE,MAAM;QAC7C,iBAAiB,OAAO,IAAI;QAC5B,UAAU,GAAG,EAAE;QACf,oBAAoB,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,MAAM,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;KAC1E,CAAC;IAEF,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,mBAAmB,WAAW,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QAChD,KAAK,CAAC,IAAI,CACR,6FAA6F,CAC9F,CAAC;IACJ,CAAC;SAAM,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,kBAAkB,OAAO,CAAC,MAAM,YAAY,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACrC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,IAA+B,CAAC;YAC1C,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;YACpC,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;YAC1C,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,MAAM,UAAU,KAAK,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,MAAM,GAAG,EAAE,aAAa,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;SAAM,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC;QAC1D,KAAK,CAAC,IAAI,CAAC,gBAAgB,KAAK,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,QAAQ,IAAI,KAAK;QACjB,OAAQ,KAAkB,CAAC,MAAM,KAAK,QAAQ,CAC/C,CAAC;AACJ,CAAC"}
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/tools/crawl.ts"],"names":[],"mappings":";;;AAqRA,8CAgBC;AAED,kCA6BC;AAgBD,8CAcC;AAeD,8CAoBC;AArYD,6BAAwB;AAGxB,4CAAgE;AAChE,4CAAoD;AAEpD,2CAOqB;AAErB,+EAA+E;AAC/E,cAAc;AACd,+EAA+E;AAE/E,MAAM,uBAAuB,GAAG,OAAC;KAC9B,MAAM,CAAC;IACN,WAAW,EAAE,OAAC;SACX,MAAM,EAAE;SACR,GAAG,EAAE;SACL,QAAQ,EAAE;SACV,QAAQ,EAAE;SACV,QAAQ,CAAC,mDAAmD,CAAC;IAChE,QAAQ,EAAE,oBAAQ;SACf,QAAQ,EAAE;SACV,QAAQ,CACP,uCAAuC;QACrC,6FAA6F;QAC7F,+EAA+E,CAClF;IACH,UAAU,EAAE,oBAAQ;SACjB,QAAQ,EAAE;SACV,QAAQ,CACP,uEAAuE,CACxE;CACJ,CAAC;KACD,QAAQ,CACP,+EAA+E,CAChF,CAAC;AAES,QAAA,WAAW,GAAG,OAAC,CAAC,MAAM,CAAC;IAClC,GAAG,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,yBAAyB,CAAC;IACnE,SAAS,EAAE,OAAC;SACT,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,MAAM,CAAC;SACX,OAAO,CAAC,EAAE,CAAC;SACX,QAAQ,CAAC,mCAAmC,CAAC;IAChD,SAAS,EAAE,OAAC;SACT,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,OAAO,CAAC,CAAC,CAAC;SACV,QAAQ,CACP,mEAAmE,CACpE;IACH,gBAAgB,EAAE,OAAC;SAChB,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;SACjB,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,EAAE;SACV,QAAQ,CACP,iGAAiG,CAClG;IACH,gBAAgB,EAAE,OAAC;SAChB,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;SACjB,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,EAAE;SACV,QAAQ,CACP,kFAAkF,CACnF;IACH,OAAO,EAAE,OAAC;SACP,IAAI,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;SACjC,OAAO,CAAC,SAAS,CAAC;SAClB,QAAQ,CACP,wFAAwF,CACzF;IACH,YAAY,EAAE,OAAC;SACZ,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,QAAQ,CACP,kEAAkE;QAChE,oEAAoE,CACvE;IACH,MAAM,EAAE,OAAC;SACN,MAAM,CAAC,OAAC,CAAC,MAAM,EAAE,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC;SACpC,QAAQ,EAAE;SACV,QAAQ,CACP,oGAAoG;QAClG,oFAAoF,CACvF;IACH,QAAQ,EAAE,OAAC;SACR,MAAM,CAAC;QACN,QAAQ,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,2CAA2C,CAAC;QAC9F,cAAc,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oCAAoC,CAAC;QAC7F,eAAe,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;KAC7F,CAAC;SACD,QAAQ,EAAE;SACV,QAAQ,CAAC,wEAAwE,CAAC;IACrF,OAAO,EAAE,OAAC;SACP,KAAK,CAAC,2BAAe,CAAC;SACtB,QAAQ,EAAE;SACV,QAAQ,CAAC,wCAAwC,GAAG,+BAAmB,CAAC;IAC3E,UAAU,EAAE,OAAC;SACV,OAAO,CAAC,MAAM,CAAC;SACf,QAAQ,EAAE;SACV,QAAQ,CACP,gGAAgG;QAC9F,4CAA4C,CAC/C;IACH,iBAAiB,EAAE,OAAC;SACjB,MAAM,CAAC,OAAC,CAAC,OAAO,EAAE,CAAC;SACnB,QAAQ,EAAE;SACV,QAAQ,CAAC,oDAAoD,CAAC;IACjE,kBAAkB,EAAE,iCAAqB;SACtC,QAAQ,EAAE;SACV,QAAQ,CACP,gEAAgE;QAC9D,6CAA6C,CAChD;IACH,mBAAmB,EAAE,iCAAqB;SACvC,QAAQ,EAAE;SACV,QAAQ,CAAC,2DAA2D,CAAC;IACxE,mBAAmB,EAAE,kCAAsB;SACxC,QAAQ,EAAE;SACV,QAAQ,CAAC,oDAAoD,CAAC;IACjE,gBAAgB,EAAE,OAAC;SAChB,MAAM,EAAE;SACR,OAAO,EAAE;SACT,QAAQ,CACP,6HAA6H;QAC3H,4EAA4E,CAC/E;IACH,OAAO,EAAE,OAAC;SACP,MAAM,CAAC,OAAC,CAAC,MAAM,EAAE,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC;SAC9B,QAAQ,EAAE;SACV,QAAQ,CACP,wEAAwE;QACtE,gEAAgE,CACnE;IACH,eAAe,EAAE,OAAC;SACf,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,OAAO,CAAC,EAAE,CAAC;SACX,QAAQ,CAAC,mDAAmD,CAAC;IAChE,cAAc,EAAE,OAAC;SACd,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,gDAAgD,CAAC;IAC7D,kBAAkB,EAAE,OAAC;SAClB,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,8CAA8C,CAAC;IAC3D,eAAe,EAAE,OAAC;SACf,MAAM,CAAC;QACN,OAAO,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;QAC/B,WAAW,EAAE,OAAC;aACX,MAAM,EAAE;aACR,GAAG,CAAC,GAAG,CAAC;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,yEAAyE,CAAC;QACtF,IAAI,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE;QAClD,YAAY,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,iCAAiC,CAAC;KACjF,CAAC;SACD,QAAQ,EAAE;SACV,QAAQ,CAAC,mCAAmC,CAAC;IAChD,SAAS,EAAE,OAAC;SACT,KAAK,CAAC,CAAC,OAAC,CAAC,OAAO,EAAE,EAAE,OAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;SACvC,QAAQ,EAAE;SACV,QAAQ,CACP,mHAAmH,CACpH;IACH,UAAU,EAAE,OAAC;SACV,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,gEAAgE,CAAC;IAC7E,SAAS,EAAE,OAAC;SACT,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,gEAAgE,CAAC;IAC7E,QAAQ,EAAE,OAAC;SACR,MAAM,EAAE;SACR,QAAQ,EAAE;SACV,QAAQ,CACP,gFAAgF;QAC9E,oCAAoC,CACvC;IACH,OAAO,EAAE,OAAC;SACP,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,6CAA6C,CAAC;IAC1D,KAAK,EAAE,OAAC;SACL,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,GAAG,CAAC,EAAE,CAAC;SACP,QAAQ,EAAE;SACV,QAAQ,CAAC,2DAA2D,CAAC;IACxE,cAAc,EAAE,OAAC;SACd,IAAI,CAAC,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;SAC7B,QAAQ,EAAE;SACV,QAAQ,CAAC,oDAAoD,CAAC;IACjE,WAAW,EAAE,OAAC;SACX,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,IAAI,CAAC;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,2CAA2C,CAAC;IACxD,OAAO,EAAE,OAAC;SACP,MAAM,CAAC;QACN,GAAG,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC;QAC/B,MAAM,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;QACxF,MAAM,EAAE,OAAC;aACN,MAAM,EAAE;aACR,GAAG,CAAC,CAAC,CAAC;aACN,GAAG,CAAC,GAAG,CAAC;aACR,QAAQ,EAAE;aACV,QAAQ,CAAC,mDAAmD,CAAC;QAChE,OAAO,EAAE,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,MAAM,EAAE,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;QACpD,QAAQ,EAAE,OAAC,CAAC,MAAM,CAAC,OAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,uCAAuC,CAAC;KAC7F,CAAC;SACD,QAAQ,EAAE;SACV,QAAQ,CAAC,uFAAuF,CAAC;IACpG,OAAO,EAAE,OAAC;SACP,IAAI,CAAC,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;SAC3B,QAAQ,EAAE;SACV,QAAQ,CAAC,sDAAsD,CAAC;IACnE,KAAK,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oDAAoD,CAAC;IAC5F,SAAS,EAAE,OAAC;SACT,MAAM,EAAE;SACR,GAAG,EAAE;SACL,GAAG,CAAC,EAAE,CAAC;SACP,GAAG,CAAC,KAAK,CAAC;SACV,QAAQ,EAAE;SACV,QAAQ,CAAC,sDAAsD,CAAC;IACnE,aAAa,EAAE,OAAC;SACb,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,6CAA6C,CAAC;IAC1D,aAAa,EAAE,uBAAuB,CAAC,QAAQ,EAAE;CAClD,CAAC,CAAC;AAEU,QAAA,gBAAgB,GAC3B,uGAAuG;IACvG,kGAAkG;IAClG,iGAAiG;IACjG,oFAAoF;IACpF,+DAA+D;IAC/D,8EAA8E;IAC9E,oHAAoH;IACpH,2EAA2E;IAC3E,kHAAkH;IAClH,iFAAiF;IACjF,+DAA+D;IAC/D,2FAA2F;IAC3F,qGAAqG,CAAC;AAExG,MAAM,mBAAmB,GAAG;IAC1B,WAAW;IACX,YAAY;IACZ,WAAW;IACX,UAAU;IACV,SAAS;IACT,OAAO;IACP,gBAAgB;CACR,CAAC;AAEX,4FAA4F;AAC5F,SAAgB,iBAAiB,CAAC,MAAmC;IACnE,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;IAC5D,MAAM,IAAI,GAA4B,EAAE,CAAC;IACzC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1C,IAAI,CAAC,KAAK,SAAS,IAAI,CAAE,mBAAyC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/E,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACd,CAAC;IACH,CAAC;IACD,IAAI,gBAAgB;QAAE,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;IAC/D,IAAI,aAAa;QAAE,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACtD,MAAM,QAAQ,GAAG,IAAA,wBAAY,EAC3B,MAA4C,EAC5C,mBAAmB,CACpB,CAAC;IACF,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC;QAAE,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC/D,OAAO,IAA+B,CAAC;AACzC,CAAC;AAEM,KAAK,UAAU,WAAW,CAC/B,MAAsB,EACtB,MAAmC;IAEnC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC;QAEpE,MAAM,IAAI,GACR,uBAAuB;YACvB,iBAAiB,QAAQ,CAAC,QAAQ,MAAM;YACxC,aAAa,QAAQ,CAAC,MAAM,IAAI;YAChC,gBAAgB,QAAQ,CAAC,GAAG,IAAI;YAChC,gBAAgB,MAAM,CAAC,SAAS,IAAI;YACpC,gBAAgB,MAAM,CAAC,SAAS,MAAM;YACtC,iDAAiD,QAAQ,CAAC,QAAQ,2BAA2B;YAC7F,6EAA6E,CAAC;QAEhF,MAAM,oBAAoB,GAAG,IAAA,gCAAoB,EAAC,MAAM,CAAC,qBAAqB,EAAE,CAAC,CAAC;QAClF,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,GAAG,oBAAoB,EAAE,CAAC,EAAE,CAAC;IAC5E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,oBAAoB,GAAG,IAAA,gCAAoB,EAAC,MAAM,CAAC,qBAAqB,EAAE,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC;YAC9B,CAAC,CAAC,IAAA,6BAAiB,EAAC,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;YAC/C,CAAC,CAAC,IAAA,6BAAiB,EAAC,KAAc,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;QAC3D,IAAI,oBAAoB,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,MAAM,EAAE,CAAC;YAC9D,MAAM,CAAC,OAAO,CAAC,CAAC,CAAoC,CAAC,IAAI,IAAI,oBAAoB,CAAC;QACrF,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,sBAAsB;AACtB,+EAA+E;AAElE,QAAA,iBAAiB,GAAG,OAAC,CAAC,MAAM,CAAC;IACxC,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;CACrE,CAAC,CAAC;AAEU,QAAA,sBAAsB,GACjC,gEAAgE;IAChE,+EAA+E;IAC/E,gEAAgE;IAChE,4DAA4D,CAAC;AAExD,KAAK,UAAU,iBAAiB,CACrC,MAAsB,EACtB,MAAyC;IAEzC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC9D,MAAM,IAAI,GAAG,yBAAyB,CAAC,QAAQ,CAAC,CAAC;QACjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,IAAA,6BAAiB,EAAC,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,IAAA,6BAAiB,EAAC,KAAc,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,eAAe;AACf,+EAA+E;AAElE,QAAA,iBAAiB,GAAG,OAAC,CAAC,MAAM,CAAC;IACxC,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,oBAAoB,CAAC;CACpD,CAAC,CAAC;AAEU,QAAA,sBAAsB,GACjC,iEAAiE;IACjE,0EAA0E;IAC1E,qCAAqC,CAAC;AAEjC,KAAK,UAAU,iBAAiB,CACrC,MAAsB,EACtB,MAAyC;IAEzC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC3D,MAAM,IAAI,GACR,yBAAyB;YACzB,iBAAiB,MAAM,CAAC,QAAQ,MAAM;YACtC,kCAAkC,QAAQ,CAAC,aAAa,IAAI,SAAS,IAAI;YACzE,uBAAuB,QAAQ,CAAC,gBAAgB,IAAI,CAAC,MAAM;YAC3D,0FAA0F,CAAC;QAE7F,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,IAAA,6BAAiB,EAAC,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,IAAA,6BAAiB,EAAC,KAAc,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,UAAU;AACV,+EAA+E;AAE/E,SAAS,yBAAyB,CAAC,QAAiB;IAClD,MAAM,CAAC,GAAG,QAAmC,CAAC;IAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;IAChC,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC;IAClD,MAAM,UAAU,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAChE,MAAM,WAAW,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACnE,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAE1D,MAAM,KAAK,GAAa;QACtB,mBAAmB,MAAM,CAAC,WAAW,EAAE,MAAM;QAC7C,iBAAiB,OAAO,IAAI;QAC5B,UAAU,GAAG,EAAE;QACf,oBAAoB,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,MAAM,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;KAC1E,CAAC;IAEF,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,mBAAmB,WAAW,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QAChD,KAAK,CAAC,IAAI,CACR,6FAA6F,CAC9F,CAAC;IACJ,CAAC;SAAM,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,kBAAkB,OAAO,CAAC,MAAM,YAAY,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACrC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,MAAM,CAAC,GAAG,IAA+B,CAAC;YAC1C,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;YACpC,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;YAC1C,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,MAAM,UAAU,KAAK,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,WAAW,OAAO,CAAC,MAAM,GAAG,EAAE,aAAa,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;SAAM,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC;QAC1D,KAAK,CAAC,IAAI,CAAC,gBAAgB,KAAK,EAAE,CAAC,CAAC;IACtC,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,QAAQ,IAAI,KAAK;QACjB,OAAQ,KAAkB,CAAC,MAAM,KAAK,QAAQ,CAC/C,CAAC;AACJ,CAAC"}

package/dist/tools/estimate.d.ts

@@ -4,21 +4,36 @@ import type { AlterLabClient } from "../client.js";
 export declare const estimateSchema: z.ZodObject<{
     url: z.ZodString;
     mode: z.ZodDefault<z.ZodEnum<["auto", "html", "js", "pdf", "ocr"]>>;
+    method: z.ZodOptional<z.ZodEnum<["GET", "POST"]>>;
     formats: z.ZodOptional<z.ZodArray<z.ZodEnum<["text", "json", "json_v2", "html", "markdown", "rag", "content", "raw"]>, "many">>;
-    render_js: z.ZodDefault<z.ZodBoolean>;
+    render_js: z.ZodDefault<z.ZodUnion<[z.ZodBoolean, z.ZodLiteral<"auto">]>>;
     use_proxy: z.ZodDefault<z.ZodBoolean>;
+    use_own_proxy: z.ZodOptional<z.ZodBoolean>;
+    screenshot: z.ZodOptional<z.ZodBoolean>;
+    generate_pdf: z.ZodOptional<z.ZodBoolean>;
+    ocr: z.ZodOptional<z.ZodBoolean>;
 }, "strip", z.ZodTypeAny, {
-    url: string;
     mode: "auto" | "html" | "js" | "pdf" | "ocr";
-    render_js: boolean;
+    url: string;
+    render_js: boolean | "auto";
     use_proxy: boolean;
-    formats?: ("content" | "text" | "html" | "json" | "json_v2" | "markdown" | "rag" | "raw")[] | undefined;
+    method?: "POST" | "GET" | undefined;
+    ocr?: boolean | undefined;
+    screenshot?: boolean | undefined;
+    formats?: ("content" | "text" | "json" | "json_v2" | "html" | "markdown" | "rag" | "raw")[] | undefined;
+    use_own_proxy?: boolean | undefined;
+    generate_pdf?: boolean | undefined;
 }, {
     url: string;
     mode?: "auto" | "html" | "js" | "pdf" | "ocr" | undefined;
-    formats?: ("content" | "text" | "html" | "json" | "json_v2" | "markdown" | "rag" | "raw")[] | undefined;
-    render_js?: boolean | undefined;
+    method?: "POST" | "GET" | undefined;
+    ocr?: boolean | undefined;
+    screenshot?: boolean | undefined;
+    formats?: ("content" | "text" | "json" | "json_v2" | "html" | "markdown" | "rag" | "raw")[] | undefined;
+    render_js?: boolean | "auto" | undefined;
     use_proxy?: boolean | undefined;
+    use_own_proxy?: boolean | undefined;
+    generate_pdf?: boolean | undefined;
 }>;
 export declare const estimateDescription: string;
 export declare function handleEstimate(client: AlterLabClient, params: z.infer<typeof estimateSchema>): Promise<CallToolResult>;

package/dist/tools/estimate.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"estimate.d.ts","sourceRoot":"","sources":["../../src/tools/estimate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACzE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAInD,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;EAqCzB,CAAC;AAEH,eAAO,MAAM,mBAAmB,QAG6B,CAAC;AAE9D,wBAAsB,cAAc,CAClC,MAAM,EAAE,cAAc,EACtB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,GACrC,OAAO,CAAC,cAAc,CAAC,CAoBzB"}
+{"version":3,"file":"estimate.d.ts","sourceRoot":"","sources":["../../src/tools/estimate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACzE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAMnD,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA+BzB,CAAC;AAEH,eAAO,MAAM,mBAAmB,QAMsE,CAAC;AAEvG,wBAAsB,cAAc,CAClC,MAAM,EAAE,cAAc,EACtB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,GACrC,OAAO,CAAC,cAAc,CAAC,CAYzB"}

package/dist/tools/estimate.js

@@ -5,52 +5,45 @@ exports.handleEstimate = handleEstimate;
 const zod_1 = require("zod");
 const errors_js_1 = require("../errors.js");
 const format_js_1 = require("../format.js");
+const common_js_1 = require("./common.js");
+const scrape_js_1 = require("./scrape.js");
 exports.estimateSchema = zod_1.z.object({
-    url: zod_1.z.string().url().describe("URL to estimate scraping cost for"),
-    mode: zod_1.z
-        .enum(["auto", "html", "js", "pdf", "ocr"])
-        .default("auto")
-        .describe("Scraping mode"),
+    url: zod_1.z.string().url().max(2083).describe("URL to estimate scraping cost for"),
+    mode: common_js_1.modeEnum.default("auto").describe("Scraping mode"),
+    method: zod_1.z
+        .enum(["GET", "POST"])
+        .optional()
+        .describe("HTTP method. POST requests cost 1.5x the base tier price."),
     formats: zod_1.z
-        .array(zod_1.z.enum([
-        "text",
-        "json",
-        "json_v2",
-        "html",
-        "markdown",
-        "rag",
-        "content",
-        "raw",
-    ]))
+        .array(common_js_1.allFormatEnum)
         .optional()
-        .describe("Output formats to include in the estimate. Affects cost prediction accuracy. " +
-        "'json_v2' returns a structured section tree. " +
-        "'rag' returns chunked text for retrieval-augmented generation. " +
-        "'content' returns body_markdown + content_hash + images + links. " +
-        "'raw' returns the response body byte-for-byte with no transformation."),
+        .describe("Output formats to include in the estimate. Passed to the API but does not currently affect the cost estimate — the endpoint uses URL-pattern heuristics only. " +
+        common_js_1.FORMATS_DESCRIPTION),
     render_js: zod_1.z
-        .boolean()
+        .union([zod_1.z.boolean(), zod_1.z.literal("auto")])
         .default(false)
-        .describe("Enable JS rendering (forces Tier 4 minimum — no separate add-on charge)"),
+        .describe("Enable JS rendering (forces Tier 4 minimum, $0.004 — no separate add-on charge)"),
     use_proxy: zod_1.z
         .boolean()
         .default(false)
-        .describe("Include premium proxy cost (+1 credit)"),
+        .describe("Include premium proxy cost (+$0.0002)"),
+    use_own_proxy: zod_1.z
+        .boolean()
+        .optional()
+        .describe("Price the request with your own proxy integration (20% BYOP discount)"),
+    screenshot: zod_1.z.boolean().optional().describe("Include the screenshot add-on (+$0.0002)"),
+    generate_pdf: zod_1.z.boolean().optional().describe("Include the PDF add-on (+$0.0004)"),
+    ocr: zod_1.z.boolean().optional().describe("Include the OCR add-on (+$0.001)"),
 });
-exports.estimateDescription = "Estimate the cost of scraping a URL without actually scraping it. " +
-    "Returns the predicted tier, cost per request, and confidence level. " +
-    "Use this to check costs before running expensive scrapes.";
+exports.estimateDescription = "Preview the cost of scraping a URL before committing — no credits charged. " +
+    "Returns the predicted anti-bot tier, estimated cost per request, and confidence level. " +
+    "Call this before scraping protected or unfamiliar sites to avoid surprises. " +
+    "Tier 1 (curl): $0.0002. Tier 2 (http): $0.0003. Tier 3 (stealth): $0.002. Tier 3.5 (lightjs): $0.0025. " +
+    "Tier 4 (full browser): $0.004. Tier 5 (captcha solver): $0.02. " +
+    "Use render_js, use_proxy, screenshot, generate_pdf, ocr and method to see how options affect cost.";
 async function handleEstimate(client, params) {
     try {
-        const estimate = await client.estimate({
-            url: params.url,
-            mode: params.mode,
-            formats: params.formats,
-            advanced: {
-                render_js: params.render_js,
-                use_proxy: params.use_proxy,
-            },
-        });
+        const estimate = await client.estimate((0, scrape_js_1.buildEstimateRequest)(params));
         return {
             content: [{ type: "text", text: (0, format_js_1.formatEstimateResponse)(estimate) }],
         };

package/dist/tools/estimate.js.map

@@ -1 +1 @@
-{"version":3,"file":"estimate.js","sourceRoot":"","sources":["../../src/tools/estimate.ts"],"names":[],"mappings":";;;AAkDA,wCAuBC;AAzED,6BAAwB;AAGxB,4CAAgE;AAChE,4CAAsD;AAEzC,QAAA,cAAc,GAAG,OAAC,CAAC,MAAM,CAAC;IACrC,GAAG,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IACnE,IAAI,EAAE,OAAC;SACJ,IAAI,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;SAC1C,OAAO,CAAC,MAAM,CAAC;SACf,QAAQ,CAAC,eAAe,CAAC;IAC5B,OAAO,EAAE,OAAC;SACP,KAAK,CACJ,OAAC,CAAC,IAAI,CAAC;QACL,MAAM;QACN,MAAM;QACN,SAAS;QACT,MAAM;QACN,UAAU;QACV,KAAK;QACL,SAAS;QACT,KAAK;KACN,CAAC,CACH;SACA,QAAQ,EAAE;SACV,QAAQ,CACP,+EAA+E;QAC7E,+CAA+C;QAC/C,iEAAiE;QACjE,mEAAmE;QACnE,uEAAuE,CAC1E;IACH,SAAS,EAAE,OAAC;SACT,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CACP,yEAAyE,CAC1E;IACH,SAAS,EAAE,OAAC;SACT,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,wCAAwC,CAAC;CACtD,CAAC,CAAC;AAEU,QAAA,mBAAmB,GAC9B,oEAAoE;IACpE,sEAAsE;IACtE,2DAA2D,CAAC;AAEvD,KAAK,UAAU,cAAc,CAClC,MAAsB,EACtB,MAAsC;IAEtC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC;YACrC,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,QAAQ,EAAE;gBACR,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;aAC5B;SACF,CAAC,CAAC;QACH,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAA,kCAAsB,EAAC,QAAQ,CAAC,EAAE,CAAC;SACpE,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,IAAA,6BAAiB,EAAC,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,IAAA,6BAAiB,EAAC,KAAc,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IAChE,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,QAAQ,IAAI,KAAK;QACjB,OAAQ,KAAkB,CAAC,MAAM,KAAK,QAAQ,CAC/C,CAAC;AACJ,CAAC"}
+{"version":3,"file":"estimate.js","sourceRoot":"","sources":["../../src/tools/estimate.ts"],"names":[],"mappings":";;;AAiDA,wCAeC;AAhED,6BAAwB;AAGxB,4CAAgE;AAChE,4CAAsD;AACtD,2CAA2E;AAC3E,2CAAmD;AAEtC,QAAA,cAAc,GAAG,OAAC,CAAC,MAAM,CAAC;IACrC,GAAG,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IAC7E,IAAI,EAAE,oBAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC;IACxD,MAAM,EAAE,OAAC;SACN,IAAI,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;SACrB,QAAQ,EAAE;SACV,QAAQ,CAAC,2DAA2D,CAAC;IACxE,OAAO,EAAE,OAAC;SACP,KAAK,CAAC,yBAAa,CAAC;SACpB,QAAQ,EAAE;SACV,QAAQ,CACP,gKAAgK;QAC9J,+BAAmB,CACtB;IACH,SAAS,EAAE,OAAC;SACT,KAAK,CAAC,CAAC,OAAC,CAAC,OAAO,EAAE,EAAE,OAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;SACvC,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CACP,iFAAiF,CAClF;IACH,SAAS,EAAE,OAAC;SACT,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,uCAAuC,CAAC;IACpD,aAAa,EAAE,OAAC;SACb,OAAO,EAAE;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,uEAAuE,CAAC;IACpF,UAAU,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;IACvF,YAAY,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IAClF,GAAG,EAAE,OAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;CACzE,CAAC,CAAC;AAEU,QAAA,mBAAmB,GAC9B,6EAA6E;IAC7E,yFAAyF;IACzF,8EAA8E;IAC9E,yGAAyG;IACzG,iEAAiE;IACjE,oGAAoG,CAAC;AAEhG,KAAK,UAAU,cAAc,CAClC,MAAsB,EACtB,MAAsC;IAEtC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,IAAA,gCAAoB,EAAC,MAAM,CAAC,CAAC,CAAC;QACrE,OAAO;YACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAA,kCAAsB,EAAC,QAAQ,CAAC,EAAE,CAAC;SACpE,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,IAAA,6BAAiB,EAAC,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,IAAA,6BAAiB,EAAC,KAAc,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;IAChE,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,QAAQ,IAAI,KAAK;QACjB,OAAQ,KAAkB,CAAC,MAAM,KAAK,QAAQ,CAC/C,CAAC;AACJ,CAAC"}

package/dist/tools/extract.d.ts

@@ -17,8 +17,8 @@ export declare const extractSchema: z.ZodObject<{
     cache_ttl: z.ZodOptional<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
     content: string;
-    formats: ("content" | "text" | "html" | "json" | "json_v2" | "markdown" | "rag" | "raw")[];
     content_type: "text" | "html" | "markdown";
+    formats: ("content" | "text" | "json" | "json_v2" | "html" | "markdown" | "rag" | "raw")[];
     evidence: boolean;
     cache: "auto" | "skip" | "only";
     extraction_schema?: Record<string, unknown> | undefined;
@@ -27,22 +27,22 @@ export declare const extractSchema: z.ZodObject<{
     extraction_prompt?: string | undefined;
     extraction_profile?: "auto" | "product" | "article" | "job_posting" | "faq" | "recipe" | "event" | "ecommerce_homepage" | "directory_listing" | undefined;
     extraction_template?: "auto" | "product" | "article" | "job_posting" | "faq" | "recipe" | "event" | "ecommerce_homepage" | "directory_listing" | undefined;
-    source_url?: string | undefined;
     cache_ttl?: number | undefined;
+    source_url?: string | undefined;
 }, {
     content: string;
-    formats?: ("content" | "text" | "html" | "json" | "json_v2" | "markdown" | "rag" | "raw")[] | undefined;
+    content_type?: "text" | "html" | "markdown" | undefined;
+    formats?: ("content" | "text" | "json" | "json_v2" | "html" | "markdown" | "rag" | "raw")[] | undefined;
     extraction_schema?: Record<string, unknown> | undefined;
     extraction_model?: string | null | undefined;
     extraction_provider?: "openai" | "anthropic" | "openrouter" | "groq" | undefined;
     extraction_prompt?: string | undefined;
     extraction_profile?: "auto" | "product" | "article" | "job_posting" | "faq" | "recipe" | "event" | "ecommerce_homepage" | "directory_listing" | undefined;
-    content_type?: "text" | "html" | "markdown" | undefined;
     extraction_template?: "auto" | "product" | "article" | "job_posting" | "faq" | "recipe" | "event" | "ecommerce_homepage" | "directory_listing" | undefined;
-    source_url?: string | undefined;
     evidence?: boolean | undefined;
     cache?: "auto" | "skip" | "only" | undefined;
     cache_ttl?: number | undefined;
+    source_url?: string | undefined;
 }>;
 export declare const extractDescription: string;
 export declare function handleExtract(client: AlterLabClient, params: z.infer<typeof extractSchema>): Promise<CallToolResult>;

package/dist/tools/extract.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/tools/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACzE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAGnD,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAgIxB,CAAC;AAEH,eAAO,MAAM,kBAAkB,QAQqD,CAAC;AAErF,wBAAsB,aAAa,CACjC,MAAM,EAAE,cAAc,EACtB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,GACpC,OAAO,CAAC,cAAc,CAAC,CA8BzB"}
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/tools/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACzE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAInD,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAkIxB,CAAC;AAEH,eAAO,MAAM,kBAAkB,QAQqD,CAAC;AAErF,wBAAsB,aAAa,CACjC,MAAM,EAAE,cAAc,EACtB,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,aAAa,CAAC,GACpC,OAAO,CAAC,cAAc,CAAC,CA+BzB"}

package/dist/tools/extract.js

@@ -4,10 +4,12 @@ exports.extractDescription = exports.extractSchema = void 0;
 exports.handleExtract = handleExtract;
 const zod_1 = require("zod");
 const errors_js_1 = require("../errors.js");
+const format_js_1 = require("../format.js");
 exports.extractSchema = zod_1.z.object({
     content: zod_1.z
         .string()
         .min(1)
+        .max(500_000)
         .describe("Raw content to extract from — HTML, text, or markdown. " +
         "Bring your own pre-fetched content; this endpoint does NOT scrape a URL."),
     content_type: zod_1.z
@@ -58,6 +60,7 @@ exports.extractSchema = zod_1.z.object({
         "Charged at LLM extraction rate when provided."),
     extraction_model: zod_1.z
         .string()
+        .max(256)
         .nullish()
         .describe("Per-request LLM model override in provider-specific format (e.g. 'gpt-4o', 'claude-opus-4-5-20251101', 'llama3-70b-8192'). " +
         "Overrides the model saved in your BYOK key settings for this request only."),
@@ -107,10 +110,10 @@ exports.extractSchema = zod_1.z.object({
         .describe("TTL for caching this extraction result, in seconds. " +
         "Defaults to server setting (3600s). Max 86400s (24 hours)."),
 });
-exports.extractDescription = "Extract structured data from raw HTML, text, or markdown content WITHOUT scraping. " +
-    "Bring your own pre-fetched content. Use this when you already have the page content " +
-    "and want to run AlterLab's extraction pipeline on it. " +
-    "For scraping + extraction in one step, use alterlab_scrape with formats=['json'] instead. " +
+exports.extractDescription = "Extract product data, scrape prices, get structured data from any page content, " +
+    "or pull specific fields like names, emails, and ratings from HTML. " +
+    "Runs AlterLab's extraction pipeline on raw HTML, text, or markdown you already have — does NOT scrape a URL. " +
+    "For scraping + extraction in one step, use alterlab_scrape with extraction_schema instead. " +
     "Profiles: 'product' (price, title, reviews), 'article' (title, author, body), " +
     "'job_posting', 'faq', 'recipe', 'event', 'ecommerce_homepage', 'directory_listing'. " +
     "Returns JSON data. Use extraction_prompt for natural language extraction (LLM-powered). " +
@@ -133,17 +136,18 @@ async function handleExtract(client, params) {
             cache_ttl: params.cache_ttl,
         });
         const text = formatStandaloneExtractResponse(response, params.source_url);
-        return { content: [{ type: "text", text }] };
+        const balanceWarningSuffix = (0, format_js_1.formatBalanceWarning)(client.getLastBalanceWarning());
+        return { content: [{ type: "text", text: text + balanceWarningSuffix }] };
     }
     catch (error) {
-        if (isApiError(error)) {
-            return (0, errors_js_1.formatErrorResult)(error, {
-                url: params.source_url ?? "raw content",
-            });
+        const balanceWarningSuffix = (0, format_js_1.formatBalanceWarning)(client.getLastBalanceWarning());
+        const result = isApiError(error)
+            ? (0, errors_js_1.formatErrorResult)(error, { url: params.source_url ?? "raw content" })
+            : (0, errors_js_1.formatErrorResult)(error, { url: params.source_url ?? "raw content" });
+        if (balanceWarningSuffix && result.content[0]?.type === "text") {
+            result.content[0].text += balanceWarningSuffix;
         }
-        return (0, errors_js_1.formatErrorResult)(error, {
-            url: params.source_url ?? "raw content",
-        });
+        return result;
     }
 }
 function formatStandaloneExtractResponse(response, sourceUrl) {