@nstdata-ai/mcp-server 0.0.0-stage → 1.0.2

package/dist/modules/proxy.js

@@ -0,0 +1,65 @@
+import { z } from 'zod';
+import { isIP } from 'node:net';
+import { isUtf8 } from 'node:buffer';
+import { validateHeaderName, validateHeaderValue } from 'node:http';
+import { guard, ToolFailure } from '../shared/errors.js';
+import { createProxyTransport } from '@nstdata-ai/proxy-checker/transport';
+const reservedHeaders = new Set(['host', 'connection', 'proxy-authorization', 'proxy-authenticate', 'proxy-connection', 'transfer-encoding', 'content-length', 'upgrade', 'te', 'trailer', 'keep-alive']);
+const targetHeaders = z.record(z.string().min(1).max(256), z.string().max(8192)).refine(headers => {
+    if (Object.keys(headers).length > 100)
+        return false;
+    try {
+        for (const [key, value] of Object.entries(headers)) {
+            if (reservedHeaders.has(key.toLowerCase()))
+                return false;
+            validateHeaderName(key);
+            validateHeaderValue(key, value);
+        }
+        return true;
+    }
+    catch {
+        return false;
+    }
+}, 'Invalid target headers or reserved connection/proxy header');
+export function registerProxyTools(server, config) {
+    const transport = createProxyTransport({ proxyURL: config.proxyURL, timeoutMs: config.httpTimeoutMs,
+        maxResponseBytes: config.maxResponseBytes, caFile: config.proxyCAFile });
+    server.registerTool('nstdata_proxy_fetch', {
+        description: 'Read an HTTP(S) page through the configured proxy. Does not follow redirects or retry; may consume billable traffic.',
+        inputSchema: z.strictObject({
+            url: z.url().max(8192).refine(value => { const url = new URL(value); return ['http:', 'https:'].includes(url.protocol) && !url.username && !url.password; }).describe('HTTP(S) target accessed through the configured proxy; no embedded credentials.'),
+            method: z.enum(['GET', 'HEAD']).default('GET').describe('GET returns content; HEAD requests response headers without a body.'),
+            format: z.enum(['text', 'base64']).default('text').describe('text requires UTF-8; base64 preserves binary bytes.'),
+            headers: targetHeaders.optional().describe('Optional target request headers. Send Authorization or Cookie only to trusted targets; proxy and connection headers are reserved.'),
+        }),
+        annotations: { readOnlyHint: true, idempotentHint: true, openWorldHint: true },
+    }, async (args, ctx) => guard(ctx, config, async (options) => {
+        const response = await transport.request(args, { signal: options.signal });
+        if (args.format === 'text' && !isUtf8(response.body))
+            throw new ToolFailure('INVALID_RESPONSE', 'The response is not valid UTF-8. Use base64 to read binary content.');
+        const output = { statusCode: response.statusCode, encoding: args.format === 'text' ? 'utf-8' : 'base64',
+            body: response.body.toString(args.format === 'text' ? 'utf8' : 'base64'),
+            headers: { 'content-type': response.headers['content-type'], location: response.headers.location, 'retry-after': response.headers['retry-after'] } };
+        return { content: [{ type: 'text', text: JSON.stringify(output) }], structuredContent: output,
+            isError: response.statusCode < 200 || response.statusCode >= 300 };
+    }));
+    server.registerTool('nstdata_proxy_check', {
+        description: 'Check the proxy once against NSTDATA_PROXY_CHECK_URL, or https://example.com/ by default. Returns observedIp only when the target echoes a valid IP. Startup does not run this check.',
+        inputSchema: z.strictObject({}),
+        annotations: { readOnlyHint: true, idempotentHint: true, openWorldHint: true },
+    }, async (_args, ctx) => guard(ctx, config, async (options) => {
+        const started = performance.now();
+        const response = await transport.request({ url: config.proxyCheckURL, method: 'GET' }, { signal: options.signal });
+        const text = response.body.toString('utf8').trim();
+        let address = text;
+        try {
+            const json = JSON.parse(text);
+            address = json?.ip ?? (json?.code === 200 && json?.err === false ? json.data?.ip : undefined);
+        }
+        catch { /* Plain IP responses are also supported. */ }
+        const output = { statusCode: response.statusCode, elapsedMs: performance.now() - started,
+            observedIp: typeof address === 'string' && isIP(address) ? address : undefined };
+        return { content: [{ type: 'text', text: JSON.stringify(output) }], structuredContent: output,
+            isError: response.statusCode < 200 || response.statusCode >= 300 };
+    }));
+}

package/dist/server.d.ts

@@ -0,0 +1,4 @@
+import { McpServer } from '@modelcontextprotocol/server';
+import { type Config } from './config.js';
+export declare function createServer(config: Config): McpServer;
+export { loadConfig, type Config } from './config.js';

package/dist/server.js

@@ -0,0 +1,19 @@
+import { createRequire } from 'node:module';
+import { McpServer } from '@modelcontextprotocol/server';
+import { NstDataClient } from '@nstdata-ai/crawl';
+import { validateConfig } from './config.js';
+import { registerCrawlTools } from './modules/crawl.js';
+import { registerProxyTools } from './modules/proxy.js';
+const { version } = createRequire(import.meta.url)('../package.json');
+export function createServer(config) {
+    config = validateConfig(config);
+    const server = new McpServer({ name: 'nstdata', version });
+    if (config.modules.includes('crawl')) {
+        const client = new NstDataClient(config.apiToken, { baseURL: config.baseURL });
+        registerCrawlTools(server, client, config);
+    }
+    if (config.modules.includes('proxy'))
+        registerProxyTools(server, config);
+    return server;
+}
+export { loadConfig } from './config.js';

package/dist/shared/errors.d.ts

@@ -0,0 +1,10 @@
+import { type RequestOptions } from '@nstdata-ai/crawl';
+import { type CallToolResult, type ServerContext } from '@modelcontextprotocol/server';
+import { type Config } from '../config.js';
+export declare class ToolFailure extends Error {
+    readonly code: string;
+    readonly details: Record<string, unknown>;
+    constructor(code: string, message: string, details?: Record<string, unknown>);
+}
+export declare function toolError(code: string, message: string, details?: Record<string, unknown>): CallToolResult;
+export declare function guard(ctx: ServerContext, config: Config, operation: (options: RequestOptions) => Promise<CallToolResult>, operationKind?: 'request' | 'task'): Promise<CallToolResult>;

package/dist/shared/errors.js

@@ -0,0 +1,58 @@
+import { NstDataApiError } from '@nstdata-ai/crawl';
+import { ProxyTransportError } from '@nstdata-ai/proxy-checker/transport';
+import { boundResult } from './output.js';
+export class ToolFailure extends Error {
+    code;
+    details;
+    constructor(code, message, details = {}) {
+        super(message);
+        this.code = code;
+        this.details = details;
+        this.name = 'ToolFailure';
+    }
+}
+export function toolError(code, message, details = {}) {
+    const output = { error: { code, message, ...details } };
+    return { isError: true, content: [{ type: 'text', text: JSON.stringify(output) }], structuredContent: output };
+}
+export async function guard(ctx, config, operation, operationKind = 'request') {
+    const unknownSubmission = operationKind === 'task'
+        ? ' The submission outcome is unknown; a remote task may still exist. Review it before submitting again.' : '';
+    const controller = new AbortController();
+    const abort = () => controller.abort(new DOMException('Local request cancelled', 'AbortError'));
+    if (ctx.mcpReq.signal.aborted)
+        abort();
+    else
+        ctx.mcpReq.signal.addEventListener('abort', abort, { once: true });
+    const timer = setTimeout(() => controller.abort(new DOMException('Tool deadline exceeded', 'TimeoutError')), config.httpTimeoutMs);
+    try {
+        return boundResult(await operation({ signal: controller.signal, timeoutMs: config.httpTimeoutMs, maxResponseBytes: config.maxResponseBytes }), config);
+    }
+    catch (error) {
+        if (error instanceof ToolFailure)
+            return toolError(error.code, error.message, error.details);
+        if (error instanceof NstDataApiError && error.message === 'Response exceeds configured maxResponseBytes') {
+            return toolError('RESPONSE_LIMIT', 'The response exceeds the configured byte limit. No automatic retry was performed.' + unknownSubmission);
+        }
+        if (controller.signal.aborted || (error instanceof Error && error.name === 'TimeoutError') ||
+            (error instanceof ProxyTransportError && ['TIMEOUT', 'CANCELLED'].includes(error.code))) {
+            const message = 'Local request stopped. No automatic retry was performed.'
+                + (operationKind === 'task' ? ' Any submitted remote task may still run; check its status before submitting again.' : '');
+            return toolError(error instanceof ProxyTransportError ? error.code : controller.signal.reason?.name === 'AbortError' ? 'CANCELLED' : 'TIMEOUT', message);
+        }
+        if (error instanceof ProxyTransportError)
+            return toolError(error.code, error.message, error.details);
+        const statusCode = error instanceof NstDataApiError ? error.statusCode : undefined;
+        const apiCode = error instanceof NstDataApiError ? error.code : undefined;
+        const classification = statusCode !== undefined && statusCode >= 400 ? statusCode : apiCode;
+        const code = classification === 401 || classification === 403 ? 'AUTHORIZATION_FAILED'
+            : classification === 429 ? 'RATE_LIMITED' : 'UPSTREAM_ERROR';
+        const message = 'The request failed. No automatic retry was performed.'
+            + (code === 'UPSTREAM_ERROR' && !(classification !== undefined && classification >= 400 && classification < 500) ? unknownSubmission : '');
+        return toolError(code, message, { statusCode, apiCode });
+    }
+    finally {
+        clearTimeout(timer);
+        ctx.mcpReq.signal.removeEventListener('abort', abort);
+    }
+}

package/dist/shared/output.d.ts

@@ -0,0 +1,4 @@
+import { type CallToolResult } from '@modelcontextprotocol/server';
+import { type Config } from '../config.js';
+export declare function binaryContent(bytes: Buffer | undefined): Record<string, string> | undefined;
+export declare function boundResult(result: CallToolResult, config: Config): CallToolResult;

package/dist/shared/output.js

@@ -0,0 +1,104 @@
+// Upstream JSON cannot mark itself as a locally generated byte envelope.
+const inlineBinary = Symbol('inlineBinary');
+export function binaryContent(bytes) {
+    return bytes === undefined ? undefined : { [inlineBinary]: true, encoding: 'base64', base64: bytes.toString('base64') };
+}
+export function boundResult(result, config) {
+    const secrets = [config.apiToken];
+    if (config.proxyURL) {
+        const proxy = new URL(config.proxyURL);
+        const username = decodeURIComponent(proxy.username), password = decodeURIComponent(proxy.password);
+        secrets.push(config.proxyURL, proxy.username, proxy.password, username, password);
+        if (username || password)
+            secrets.push(Buffer.from(`${username}:${password}`).toString('base64'));
+    }
+    const knownSecrets = secrets.filter((secret) => !!secret).sort((a, b) => b.length - a.length);
+    const redact = (value) => knownSecrets.reduce((text, secret) => text.replaceAll(secret, '<REDACTED>'), value);
+    const original = (result.structuredContent ?? {});
+    const literalFields = new Set(['encoding', 'format', 'unit', 'status']);
+    const literals = new Set(['utf-8', 'base64', 'codePoints', 'bytes', 'markdown', 'html', 'pdf', 'screenshot', 'rawData', 'processing', 'completed', 'failed', 'expired', 'scraping']);
+    const opaqueFields = new Set(['encoding', 'format', 'unit', 'status', 'id', 'taskId', 'crawlId', 'ref', 'markdownRef', 'htmlRef', 'pdfRef', 'screenshotRef', 'rawDataRef', 'nextCursor', 'prevCursor', 'errorCode']);
+    let sensitive = false;
+    const text = JSON.stringify(original, function (key, value) {
+        if (!(this[inlineBinary] === true && (key === 'encoding' || key === 'base64')) && knownSecrets.some(secret => key.includes(secret)))
+            sensitive = true;
+        if (typeof value !== 'string')
+            return value;
+        if (this === original && literalFields.has(key) && literals.has(value))
+            return value;
+        if (this[inlineBinary] === true && key === 'encoding')
+            return value;
+        if ((this === original || this[inlineBinary] === true) && this.encoding === 'base64' && (key === 'body' || key === 'base64')) {
+            const bytes = Buffer.from(value, 'base64');
+            if (knownSecrets.some(secret => bytes.includes(Buffer.from(secret))))
+                sensitive = true;
+            return value;
+        }
+        const filtered = redact(value);
+        if (opaqueFields.has(key) && filtered !== value) {
+            sensitive = true;
+            return value;
+        }
+        return filtered;
+    });
+    if (sensitive) {
+        const output = { error: { code: 'SENSITIVE_OUTPUT', message: 'A result containing configured credentials was withheld to preserve its structure and bytes.' } };
+        return { isError: true, content: [{ type: 'text', text: JSON.stringify(output) }], structuredContent: output };
+    }
+    const data = JSON.parse(text);
+    if (text.length <= config.maxOutputChars) {
+        return { ...result, content: [{ type: 'text', text }], structuredContent: data };
+    }
+    const recovery = {};
+    const pageList = Array.isArray(data.data);
+    const error = { code: 'OUTPUT_LIMIT', message: pageList
+            ? 'Page output exceeds the local limit. Retry with the same input cursor and a smaller limit. If recovery.page has the required references, read them before using recovery.continuation, which skips that page. Do not automatically repeat a task submission.'
+            : 'Result exceeds the local output limit. Reduce source content or use a supported smaller page/chunk/reference. Do not automatically repeat a task submission.', recovery };
+    const add = (key, value) => {
+        if (value === undefined)
+            return false;
+        const candidate = { ...recovery, [key]: value };
+        if (JSON.stringify({ error: { ...error, recovery: candidate } }).length > config.maxOutputChars)
+            return false;
+        Object.assign(recovery, candidate);
+        return true;
+    };
+    const scalar = (value) => typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean';
+    for (const key of ['statusCode', 'id', 'taskId', 'crawlId', 'ref', 'format', 'offset']) {
+        if (scalar(data[key]))
+            add(key, data[key]);
+    }
+    if (Array.isArray(data.data)) {
+        if (data.data.length === 1 && data.data[0] && typeof data.data[0] === 'object') {
+            const page = data.data[0];
+            const summary = {};
+            for (const key of ['id', 'taskId', 'url', 'status', 'success', 'errorCode']) {
+                if (scalar(page[key]) && add('page', { ...summary, [key]: page[key] }))
+                    summary[key] = page[key];
+            }
+            const content = page.data;
+            const references = {};
+            for (const key of ['markdownRef', 'htmlRef', 'pdfRef', 'screenshotRef', 'rawDataRef']) {
+                if (typeof content?.[key] === 'string' && content[key] && add('page', { ...summary, data: { ...references, [key]: content[key] } }))
+                    references[key] = content[key];
+            }
+            if (Object.keys(references).length) {
+                if (typeof content?.truncated === 'boolean')
+                    add('page', { ...summary, data: { ...references, truncated: content.truncated } });
+                const continuation = Object.fromEntries(['nextCursor', 'prevCursor']
+                    .filter(key => typeof data[key] === 'string' && data[key]).map(key => [key, data[key]]));
+                if (Object.keys(continuation).length)
+                    add('continuation', continuation);
+            }
+        }
+    }
+    else if (data.data && typeof data.data === 'object') {
+        const page = data.data;
+        for (const key of ['markdownRef', 'htmlRef', 'pdfRef', 'screenshotRef', 'rawDataRef', 'truncated']) {
+            if (scalar(page[key]))
+                add(key, page[key]);
+        }
+    }
+    const output = { error };
+    return { isError: true, content: [{ type: 'text', text: JSON.stringify(output) }], structuredContent: output };
+}

package/dist/stdio.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/stdio.js

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+import { serveStdio } from '@modelcontextprotocol/server/stdio';
+import { ProxyTransportError } from '@nstdata-ai/proxy-checker/transport';
+import { createRequire } from 'node:module';
+import { createServer, loadConfig } from './server.js';
+import { DEFAULT_PROXY_CHECK_URL, EnvironmentConfigError } from './config.js';
+const args = process.argv.slice(2);
+if (args.length === 1 && args[0] === '--help') {
+    console.log(`Usage: nstdata-mcp-server [--help | --version]
+
+With no arguments, start a local stdio MCP server.
+Set environment variables in your MCP client configuration:
+  NSTDATA_MODULES          crawl (default), proxy, or crawl,proxy
+  NSTDATA_API_TOKEN        Required when Crawl is enabled
+  NSTDATA_PROXY_URL        Required when Proxy is enabled; HTTP(S) gateway URL
+  NSTDATA_PROXY_CHECK_URL  Optional check target; default ${DEFAULT_PROXY_CHECK_URL}
+
+Startup does not submit tasks or check the proxy. Calls may consume credits or traffic.
+Configuration reference: https://github.com/nstdata-ai/nstdata-mcp-server#readme`);
+}
+else if (args.length === 1 && args[0] === '--version') {
+    const { version } = createRequire(import.meta.url)('../package.json');
+    console.log(version);
+}
+else if (args.length) {
+    console.error('Unsupported arguments. Use --help for usage.');
+    process.exitCode = 1;
+}
+else
+    try {
+        const config = loadConfig();
+        // Validate module resources before the lazy stdio negotiation receives input.
+        let initialServer;
+        try {
+            initialServer = createServer(config);
+        }
+        catch (error) {
+            if (config.proxyCAFile && error instanceof ProxyTransportError && error.code === 'INVALID_CONFIG') {
+                throw new EnvironmentConfigError(['proxyCAFile']);
+            }
+            throw error;
+        }
+        const handle = serveStdio(() => {
+            const server = initialServer ?? createServer(config);
+            initialServer = undefined;
+            return server;
+        }, {
+            onerror: () => { console.error('MCP protocol error.'); },
+        });
+        const shutdown = () => { void handle.close().catch(() => { process.exitCode = 1; }); };
+        process.once('SIGINT', shutdown);
+        process.once('SIGTERM', shutdown);
+    }
+    catch (error) {
+        console.error(error instanceof EnvironmentConfigError
+            ? error.message
+            : 'Unable to start Nstdata MCP. Check the documented environment configuration.');
+        process.exitCode = 1;
+    }

package/docs/mcp.md

@@ -0,0 +1,190 @@
+# MCP client setup and Crawl reference
+
+[中文](mcp_zh.md) · [Quick start](../README.md)
+
+## Client setup
+
+Install the packages and use the stdio template in [README.md](../README.md).
+
+- **Cursor:** project configuration is `.cursor/mcp.json`; user configuration is `~/.cursor/mcp.json`. A local entry may use `envFile` with an absolute path to a protected credential file. See the [Cursor MCP guide](https://cursor.com/docs/mcp).
+- **Claude Desktop:** follow the [local MCP setup guide](https://modelcontextprotocol.io/docs/develop/connect-local-servers) to merge the entry into the client configuration.
+
+The standard configuration uses `npx` to start the installed package. Preserve other server entries and keep credentials outside version control. Restart the MCP server after changing environment configuration.
+
+Crawl-only exposes six tools, Proxy-only two, and both modules eight. Verify tool discovery before submitting a task. Startup failures should be checked in client diagnostics/stderr: confirm the supported Node version, `npx` availability and required environment variables. If a graphical client cannot find `npx`, use its absolute executable path without changing the package argument.
+
+## Configuration
+
+| Variable | Default | Constraint |
+|---|---|---|
+| `NSTDATA_MODULES` | `crawl` | `crawl`, `proxy`, or both, comma-separated |
+| `NSTDATA_API_TOKEN` | — | Required when Crawl is enabled |
+| `NSTDATA_API_BASE_URL` | `https://api.nstdata.io` | Trusted HTTP(S) URL; no embedded credentials, query or fragment |
+| `NSTDATA_HTTP_TIMEOUT_MS` | `120000` | Integer 1–600000; local tool deadline, including body reads |
+| `NSTDATA_MAX_RESPONSE_BYTES` | `8388608` | Integer 256–67108864; each API/storage download |
+| `NSTDATA_MAX_OUTPUT_CHARS` | `16000` | Integer 1024–64000; each serialized JSON representation, in UTF-16 code units |
+| `NSTDATA_MAX_CRAWL_PAGES` | `10` | Integer 1–1000; default and maximum accepted `maxPages` |
+
+Proxy-specific variables and behavior are defined in [proxy.md](proxy.md). Disabled modules do not require their credentials.
+
+## Tool reference
+
+| Tool | Required input | Operation and result |
+|---|---|---|
+| `nstdata_crawl_scrape` | `url` | One sync/async single-page submission; returns status/content or an ID |
+| `nstdata_crawl_scrape_result` | `taskId` | One result query; retains status and available content/references |
+| `nstdata_crawl_start` | `url` | One bounded site submission; returns its actual ID |
+| `nstdata_crawl_status` | `crawlId` | One aggregate progress query; does not poll |
+| `nstdata_crawl_pages` | `crawlId` | One page of results with actual cursors and per-page status |
+| `nstdata_crawl_read_result` | `ref`, `format` | One bounded reference download and local chunk selection |
+
+Submission tools create potentially billable tasks. Reads may also consume service resources. Formats are `markdown` (default), `html`, `links`, `pdf`, `screenshot` and `rawData`; request 1–6 entries. Scrape `mode` defaults to `sync`; site page/depth controls do not apply to single-page scraping. Proxy tools are listed in [proxy.md](proxy.md#tools).
+
+## Workflows
+
+### Single page
+
+```json
+{
+  "name": "nstdata_crawl_scrape",
+  "arguments": {
+    "url": "https://example.com/",
+    "formats": [
+      "markdown"
+    ],
+    "mode": "sync",
+    "options": {
+      "onlyMainContent": true
+    }
+  }
+}
+```
+
+For asynchronous work, use `mode: "async"`. Save the returned `id` and pass it as `taskId` to `nstdata_crawl_scrape_result`. Each call performs one query; callers must bound polling. Consume completed synchronous results directly.
+
+Sync and result-query outputs retain `id` when supplied by the service. Query outputs also retain the requested `taskId`. A synchronous result ID alone does not guarantee that its content can be queried again.
+
+### Site crawl
+
+Call `nstdata_crawl_start` with an explicit `maxPages`. Save its `id` and pass it as `crawlId` to `nstdata_crawl_status` and `nstdata_crawl_pages`.
+
+For example, collect the Quotes homepage and its second page:
+
+```json
+{
+  "name": "nstdata_crawl_start",
+  "arguments": {
+    "url": "https://quotes.toscrape.com/",
+    "formats": [
+      "markdown"
+    ],
+    "maxPages": 2,
+    "maxDepth": 1,
+    "includeUrls": [
+      "/page/2/"
+    ],
+    "options": {
+      "onlyMainContent": false
+    }
+  }
+}
+```
+
+Use paths to select discovered links; submit the seed separately as `url`. Adding `/` to the filter can admit other paths. In this example, `onlyMainContent: true` returned only the seed, while `false` collected both pages. Full-page output can include navigation. Filter semantics are service-specific, and `maxPages` is an upper bound: verify the returned URLs, not just task completion or page count.
+
+Page inputs accept `limit` (1–100, default 10) and one of `afterCursor`/`beforeCursor`. Copy `nextCursor` or `prevCursor` unchanged. An empty page with a cursor is not the end; stop on repeated cursors or the caller's page/call limit. Aggregate completion does not mean every page succeeded. If a page list exceeds the output limit, repeat that read from the same input cursor with a smaller `limit` (down to 1), then continue using the successful response's cursor. Advancing past a withheld page would skip unread results.
+
+If one row still exceeds the cap, `error.recovery.page` retains its identity, status and available storage references when they fit the output budget. It is a summary, not the full row. Read the required references before using `error.recovery.continuation.nextCursor` or `prevCursor` to move past that row. Those cursors are omitted for multi-row errors or when no reference fits. If the summary cannot satisfy the request, stop and report the limit; do not skip the page or submit another crawl.
+
+### Read a reference
+
+Pass the value of `data.markdownRef` as **`ref`**, with `format: "markdown"`:
+
+```json
+{
+  "name": "nstdata_crawl_read_result",
+  "arguments": {
+    "ref": "RETURNED_MARKDOWN_REF",
+    "format": "markdown",
+    "offset": 0,
+    "length": 2000
+  }
+}
+```
+
+Use the same mapping for htmlRef→html, pdfRef→pdf, screenshotRef→screenshot and rawDataRef→rawData. References are opaque values, not public URLs. Only read references from an authorized task.
+
+`length` is 1–4000, default 2000. Text uses Unicode code-point offsets and UTF-8; invalid UTF-8 fails with `INVALID_RESPONSE` before slicing. Binary uses byte offsets and base64 output. Continue at `nextOffset` until absent. Decode binary chunks separately before joining bytes. Every call downloads the full reference within the response cap, then slices locally; this is not HTTP Range or persistent caching.
+
+## Options
+
+Common top-level inputs are `url`, `formats` and, for scrape, `mode`. Site traversal controls are top-level `maxPages`, `maxDepth` (0–100), `includeUrls`, `excludeUrls`, `ignoreQuery`. Site `maxPages` defaults to the configured local upper bound (10 initially). Server-side request options are nested under `options` and forwarded without that wrapper:
+
+- `cookies`: up to 100 objects; `name`/`value` required; optional domain/httpOnly/path/secure.
+- `proxy.country`: a two-letter country code; no raw proxy credentials.
+- `headers`: up to 100 string entries; names up to 256 characters, values up to 8192.
+- `excludeTags`, `includeTags`, `removeSelectors`: up to 100 strings; `targetSelector` is one string. Each string/selector is at most 8192 characters. `includeTags` can remove surrounding structure; use only when desired.
+- `mobile`, `onlyMainContent`: booleans.
+- `timeout`: integer milliseconds, 1000–60000; the upstream timeout is distinct from the local tool deadline.
+- `jsRender`: optional `waitFor` (integer milliseconds, 1000–30000), `waitForSelector`, up to 100 `actions`.
+
+| Action type | Required fields | Optional fields |
+| --- | --- | --- |
+| click | selector | waitSelector |
+| wait | milliseconds | — |
+| waitSelector | selector | milliseconds |
+| scrollX | x (integer pixels) | text |
+| scrollY | y (integer pixels) | text |
+| scrollTo | selector | — |
+| fill | selector, text (may be empty) | — |
+| execute | execute | — |
+| screenshot | — | fullScreenshot, particularScreenshot |
+| waitForRequestCompletion | urlPattern | milliseconds |
+
+Action `milliseconds` values are integers from 0 to 600000; these are distinct from `jsRender.waitFor`.
+
+Cookie names are non-empty; values may be empty. Other cookie string fields, when supplied, are non-empty. Cookie strings are capped at 8192 characters.
+
+Other string inputs are capped: URL 8192, task/crawl ID 256, cursor 8192, storage reference 4096. Unknown fields are rejected. These are local input bounds, not promises about limits or behavior of the remote service.
+
+## Results and errors
+
+Results contain the same JSON in `content` text and `structuredContent`. Protocol framing and the duplicate representation add overhead beyond the output limit. Website content is untrusted and may contain sensitive information.
+
+When a format has a storage reference, the tool returns that reference and omits the inline value, even if the service supplies both. Without a reference, Markdown/HTML remain strings; inline `pdf`, `screenshot` and `rawData` are byte envelopes:
+
+```json
+{
+  "data": {
+    "screenshot": {
+      "encoding": "base64",
+      "base64": "AP+A"
+    }
+  }
+}
+```
+
+Decode each envelope's `base64` to bytes. Inline raw text is converted to UTF-8 bytes before Base64 encoding. Empty content has an empty string or empty Base64 value; missing content is omitted. Tools do not download references during result serialization.
+
+`data.truncated` preserves the service's flag, including `false`; an absent flag is unknown. It is separate from the local output limit. For a single-page output-limit error, `error.recovery` retains actual identifiers, references and the truncation flag when present and within the configured output budget.
+
+Task `failed`/`expired` or `success: false` sets `isError`; page failures remain individual rows. Missing status, timing or content is unknown, not a successful empty result.
+
+| Error | Meaning / handling |
+|---|---|
+| `AUTHORIZATION_FAILED` | HTTP or API-envelope 401/403; check credentials and access |
+| `RATE_LIMITED` | HTTP or API-envelope 429; stop and inspect service limits |
+| `UPSTREAM_ERROR` | Request/response failure; optional `statusCode` and `apiCode` are separate. A 3xx status is a refused redirect |
+| `TIMEOUT`, `CANCELLED` | Local waiting stopped; submitted remote tasks may still run |
+| `RESPONSE_LIMIT` | A response exceeded the download cap |
+| `INVALID_RESPONSE` | Invalid response content, including malformed UTF-8 in a text reference |
+| `OUTPUT_LIMIT` | Serialized output exceeded the cap; use smaller pages/chunks or actual identifiers in `error.recovery` |
+| `SENSITIVE_OUTPUT` | A credential matched a property name, binary content or an opaque identifier; the result was withheld |
+
+No request is automatically retried. An uncertain submission without an ID requires review, not another creation. A failed read does not create or cancel a task.
+
+For task submissions, a connection failure, server error or unreadable/oversized response can leave the outcome unknown even when no ID was received. The error message identifies this uncertainty; check the service task history before submitting again. Successful async acknowledgements must contain an ID accepted by the result-query tools; a missing or unusable ID is an unknown outcome. Explicit task failures and synchronous content without a diagnostic ID remain distinct. Authentication and rate-limit rejections remain distinct from unknown outcomes.
+
+`OUTPUT_LIMIT` is not a download continuation API. Inline-only content may have no reference; an ID alone does not make it chunkable. Configured credentials are filtered from display text and checked in property names and decoded binary bytes, including nested inline envelopes. Matching names, opaque identifiers and binary payloads are withheld rather than altered; short credentials can also match ordinary field names. Other website secrets are not automatically detected. Raw HTTP bodies and transport messages are omitted from request errors.
+
+API/storage redirects are refused, including same-origin redirects. Configure a verified final API origin rather than following Location. For Proxy failures and limits, see [proxy.md](proxy.md).