crawlforge-mcp-server 5.2.9 → 5.3.0

package/src/tools/basic/_fetch.js

@@ -1,18 +1,13 @@
 /**
  * Shared HTTP fetch helper for basic tools.
- * Applies an AbortController timeout and a default User-Agent.
+ * Applies an AbortController timeout and the shared pre-fetch gate.
  */
 
 import { readBody } from 'crawlforge-extractors';
 import { config } from '../../constants/config.js';
-import { createRequire } from 'module';
 import { ssrfGuard, isSsrfError } from '../../utils/ssrfGuard.js';
-import { throttleHost } from '../../utils/hostRateLimiter.js';
-
-// Derive User-Agent from package version so it reflects the actual release.
-const _require = createRequire(import.meta.url);
-const _pkg = _require('../../../package.json');
-const CRAWLFORGE_UA = `CrawlForge/${_pkg.version} (+https://crawlforge.dev)`;
+import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
+import { preflightFetch } from '../../utils/robotsGate.js';
 
 /**
  * Fetch a URL with a configurable timeout and body-size cap.
@@ -23,19 +18,21 @@ const CRAWLFORGE_UA = `CrawlForge/${_pkg.version} (+https://crawlforge.dev)`;
  * default 25 MB).
  *
  * @param {string} url
- * @param {{ timeout?: number, headers?: Record<string,string> }} [options]
- * @returns {Promise<Response & { _body: string }>}
+ * @param {{ timeout?: number, headers?: Record<string,string>, userAgent?: string,
+ *   respectRobots?: boolean, tool?: string, apiKey?: string }} [options]
+ * @returns {Promise<Response & { _body: string, _warnings: string[] }>}
  */
 export async function fetchWithTimeout(url, options = {}) {
-  const { timeout = 10000, headers = {} } = options;
+  const { timeout = 10000, headers = {}, userAgent, respectRobots, tool, apiKey } = options;
   const maxBodySize = config.fetch.maxBodySize;
 
   // SSRF pre-flight (protocol / metadata host). Throws a clear error before any
   // connection is attempted; `guard.dispatcher` enforces IP rules at connect time.
   const guard = ssrfGuard(url);
 
-  // Per-host politeness throttle (before the timeout window starts).
-  await throttleHost(url);
+  // robots.txt / blocklist gate + per-host politeness throttle, before the
+  // timeout window starts. Throws if the gate refuses the URL.
+  const gate = await preflightFetch(url, { userAgent, respectRobots, tool, apiKey });
 
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), timeout);
@@ -57,7 +54,7 @@ export async function fetchWithTimeout(url, options = {}) {
       response = await fetch(url, {
         signal: controller.signal,
         headers: {
-          'User-Agent': CRAWLFORGE_UA,
+          ...gate.headers,
           ...headers
         },
         ...guard
@@ -72,6 +69,12 @@ export async function fetchWithTimeout(url, options = {}) {
       throw error;
     }
 
+    // A host asking us to back off is honoured on the *next* request to it,
+    // rather than retrying straight into the wall.
+    if (response.status === 429 || response.status === 503) {
+      noteRetryAfter(url, response.headers?.get?.('retry-after'));
+    }
+
     // Reading is delegated to crawlforge-extractors so the REST API applies
     // the same cap and the same charset handling to the same page.
     let bodyText;
@@ -89,7 +92,8 @@ export async function fetchWithTimeout(url, options = {}) {
       text: () => Promise.resolve(bodyText),
       json: () => Promise.resolve(JSON.parse(bodyText)),
       _body: bodyText,
-      _responseTime: Date.now() - startedAt
+      _responseTime: Date.now() - startedAt,
+      _warnings: gate.warnings
     });
   } finally {
     clearTimeout(timeoutId);

package/src/tools/basic/extractLinks.js

@@ -7,11 +7,16 @@ import { load } from 'cheerio';
 import { fetchWithTimeout } from './_fetch.js';
 
 /**
- * @param {{ url: string, filter_external?: boolean, base_url?: string }} params
+ * @param {{ url: string, filter_external?: boolean, base_url?: string,
+ *   user_agent?: string, respect_robots?: boolean }} params
  */
-export async function extractLinksHandler({ url, filter_external, base_url }) {
+export async function extractLinksHandler({ url, filter_external, base_url, user_agent, respect_robots }) {
   try {
-    const response = await fetchWithTimeout(url);
+    const response = await fetchWithTimeout(url, {
+      userAgent: user_agent,
+      respectRobots: respect_robots,
+      tool: 'extract_links'
+    });
     if (!response.ok) {
       throw new Error(`HTTP ${response.status}: ${response.statusText}`);
     }

package/src/tools/basic/extractMetadata.js

@@ -60,11 +60,15 @@ function parseMicrodata($) {
 }
 
 /**
- * @param {{ url: string }} params
+ * @param {{ url: string, user_agent?: string, respect_robots?: boolean }} params
  */
-export async function extractMetadataHandler({ url }) {
+export async function extractMetadataHandler({ url, user_agent, respect_robots }) {
   try {
-    const response = await fetchWithTimeout(url);
+    const response = await fetchWithTimeout(url, {
+      userAgent: user_agent,
+      respectRobots: respect_robots,
+      tool: 'extract_metadata'
+    });
     if (!response.ok) {
       throw new Error(`HTTP ${response.status}: ${response.statusText}`);
     }

package/src/tools/basic/extractText.js

@@ -74,11 +74,16 @@ export function readabilityToMarkdown(html, pageUrl) {
 }
 
 /**
- * @param {{ url: string, remove_scripts?: boolean, remove_styles?: boolean, output_format?: "text"|"markdown" }} params
+ * @param {{ url: string, remove_scripts?: boolean, remove_styles?: boolean,
+ *   output_format?: "text"|"markdown", user_agent?: string, respect_robots?: boolean }} params
  */
-export async function extractTextHandler({ url, remove_scripts, remove_styles, output_format }) {
+export async function extractTextHandler({ url, remove_scripts, remove_styles, output_format, user_agent, respect_robots }) {
   try {
-    const response = await fetchWithTimeout(url);
+    const response = await fetchWithTimeout(url, {
+      userAgent: user_agent,
+      respectRobots: respect_robots,
+      tool: 'extract_text'
+    });
     if (!response.ok) {
       throw new Error(`HTTP ${response.status}: ${response.statusText}`);
     }

package/src/tools/basic/fetchUrl.js

@@ -6,13 +6,17 @@
 import { fetchWithTimeout } from './_fetch.js';
 
 /**
- * @param {{ url: string, headers?: Record<string,string>, timeout?: number }} params
+ * @param {{ url: string, headers?: Record<string,string>, timeout?: number,
+ *   user_agent?: string, respect_robots?: boolean }} params
  */
-export async function fetchUrlHandler({ url, headers, timeout }) {
+export async function fetchUrlHandler({ url, headers, timeout, user_agent, respect_robots }) {
   try {
     const response = await fetchWithTimeout(url, {
       timeout: timeout || 10000,
-      headers: headers || {}
+      headers: headers || {},
+      userAgent: user_agent,
+      respectRobots: respect_robots,
+      tool: 'fetch_url'
     });
 
     const body = await response.text();

package/src/tools/basic/scrapeStructured.js

@@ -3,6 +3,8 @@
  * Extracted from server.js inline handler.
  * B1: Support attribute extraction (selector@attr), add max_results,
  *     fix elements_found to report real per-field DOM match counts.
+ * 3.2: Optional row_selector returns row-aligned records instead of the
+ *     default parallel arrays, which are matched independently per field.
  */
 
 import { load } from 'cheerio';
@@ -39,17 +41,88 @@ function parseSelectorSpec(raw) {
 }
 
 /**
- * @param {{ url: string, selectors: Record<string, string>, max_results?: number }} params
+ * Row-aligned extraction: every field is matched *inside* each row element, so
+ * record N of the output is row N of the page. A field a row lacks is null
+ * rather than borrowed from a neighbouring row, which is what the default
+ * parallel-array output does — there each field is matched independently across
+ * the whole document and the arrays are not row-aligned.
+ *
+ * @param {import('cheerio').CheerioAPI} $
+ * @param {{ row_selector: string, selectors: Record<string, string>, max_results?: number }} params
+ * @returns {{ records: object[], rowsFound: number, matchCounts: Record<string, number> }}
  */
-export async function scrapeStructuredHandler({ url, selectors, max_results }) {
+function extractRows($, { row_selector, selectors, max_results }) {
+  const allRows = $(row_selector);
+  // In row mode max_results caps rows, not per-field matches.
+  const rows = (max_results != null && max_results > 0)
+    ? allRows.slice(0, max_results)
+    : allRows;
+
+  const specs = Object.entries(selectors).map(([field, raw]) => [field, raw, parseSelectorSpec(raw)]);
+  const matchCounts = Object.fromEntries(specs.map(([field]) => [field, 0]));
+
+  const records = rows.toArray().map((rowEl) => {
+    const $row = $(rowEl);
+    const record = {};
+    for (const [field, raw, { selector, attribute }] of specs) {
+      try {
+        // A field selector may name the row element itself (row "tr.athing",
+        // field "tr@id"); .find() searches descendants only.
+        const el = $row.is(selector) ? $row : $row.find(selector).first();
+        if (el.length === 0) {
+          record[field] = null;
+          continue;
+        }
+        record[field] = attribute ? (el.attr(attribute) ?? null) : el.text().trim();
+        matchCounts[field] += 1;
+      } catch (selectorError) {
+        record[field] = {
+          error: `Invalid selector: ${raw}`,
+          message: selectorError.message
+        };
+      }
+    }
+    return record;
+  });
+
+  return { records, rowsFound: allRows.length, matchCounts };
+}
+
+/**
+ * @param {{ url: string, selectors: Record<string, string>, row_selector?: string,
+ *   max_results?: number, user_agent?: string, respect_robots?: boolean }} params
+ */
+export async function scrapeStructuredHandler({ url, selectors, row_selector, max_results, user_agent, respect_robots }) {
   try {
-    const response = await fetchWithTimeout(url);
+    const response = await fetchWithTimeout(url, {
+      userAgent: user_agent,
+      respectRobots: respect_robots,
+      tool: 'scrape_structured'
+    });
     if (!response.ok) {
       throw new Error(`HTTP ${response.status}: ${response.statusText}`);
     }
 
     const html = await response.text();
     const $ = load(html);
+
+    if (row_selector) {
+      const { records, rowsFound, matchCounts } = extractRows($, { row_selector, selectors, max_results });
+      return {
+        content: [{
+          type: 'text',
+          text: JSON.stringify({
+            data: records,
+            selectors_used: selectors,
+            row_selector,
+            rows_found: rowsFound,
+            elements_found: matchCounts,
+            url: response.url
+          }, null, 2)
+        }]
+      };
+    }
+
     const results = {};
     const matchCounts = {};
 

package/src/tools/crawl/_sessionContext.js

@@ -16,6 +16,7 @@
  */
 
 import { safeFetch } from '../../utils/ssrfGuard.js';
+import { preflightFetch } from '../../utils/robotsGate.js';
 
 /**
  * Parse a single Set-Cookie header value into a cookie object.
@@ -201,13 +202,20 @@ export class SessionContext {
    * any cookies it sets into the jar. Returns the response body text.
    *
    * @param {{ url: string, method?: string, headers?: Record<string,string>, body?: string }} req
+   * @param {{ userAgent?: string, respectRobots?: boolean }} [options]
    * @returns {Promise<{ status: number, body: string }>}
    */
-  async performInitialRequest(req) {
+  async performInitialRequest(req, options = {}) {
     const { url, method = 'GET', headers: extraHeaders = {}, body } = req;
 
+    const gate = await preflightFetch(url, {
+      respectRobots: options.respectRobots,
+      userAgent: options.userAgent,
+      tool: 'crawl_deep'
+    });
+
     const requestHeaders = this.applyToHeaders(url, {
-      'User-Agent': 'MCP-WebScraper/1.0',
+      ...gate.headers,
       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
       ...extraHeaders
     });

package/src/tools/crawl/crawlDeep.js

@@ -4,6 +4,7 @@ import { BFSCrawler } from '../../core/crawlers/BFSCrawler.js';
 import { DomainFilter } from '../../utils/domainFilter.js';
 import { CacheManager } from '../../core/cache/CacheManager.js';
 import { SessionContext } from './_sessionContext.js';
+import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
 
 const CrawlDeepSchema = z.object({
   url: z.string().url(),
@@ -79,7 +80,7 @@ const CrawlDeepSchema = z.object({
 export class CrawlDeepTool {
   constructor(options = {}) {
     const {
-      userAgent = 'MCP-WebScraper/1.0',
+      userAgent = CRAWLFORGE_USER_AGENT,
       timeout = 30000,
       cacheEnabled = true,
       cacheTTL = 3600000,
@@ -218,7 +219,10 @@ export class CrawlDeepTool {
 
         // Perform optional login / pre-crawl request
         if (validated.session.initialRequest) {
-          await sessionContext.performInitialRequest(validated.session.initialRequest);
+          await sessionContext.performInitialRequest(validated.session.initialRequest, {
+            userAgent: this.userAgent,
+            respectRobots: effectiveRespectRobots
+          });
         }
       }
 
@@ -263,7 +267,7 @@ export class CrawlDeepTool {
           results: this.formatResults(results.results, validated.extract_content, validated.content_max_length),
           errors: results.errors,
           stats: results.stats,
-          site_structure: this.analyzeSiteStructure(results.urls),
+          site_structure: this.analyzeSiteStructure(results.urls, results.results),
           domain_filter_config: domainFilter ? domainFilter.exportConfig() : null,
           link_analysis: results.linkAnalysis,
           session: sessionContext
@@ -344,23 +348,35 @@ export class CrawlDeepTool {
     });
   }
 
-  analyzeSiteStructure(urls) {
+  analyzeSiteStructure(urls, pages = []) {
     const structure = {
       total_pages: urls.length,
       depth_distribution: {},
+      path_depth_distribution: {},
       path_patterns: {},
       file_types: {},
       subdomains: new Set()
     };
-    
+
+    // depth_distribution reports crawl depth — how many links from the start URL each
+    // page was reached at — taken from the depth the crawler recorded on each result.
+    // URL path depth is a different measurement and keeps its own field below.
+    for (const page of pages) {
+      const depth = page?.depth;
+      if (typeof depth === 'number') {
+        structure.depth_distribution[depth] = (structure.depth_distribution[depth] || 0) + 1;
+      }
+    }
+
     for (const url of urls) {
       try {
         const urlObj = new URL(url);
-        
-        // Analyze depth
-        const depth = urlObj.pathname.split('/').filter(s => s).length;
-        structure.depth_distribution[depth] = (structure.depth_distribution[depth] || 0) + 1;
-        
+
+        // Analyze URL path depth
+        const pathDepth = urlObj.pathname.split('/').filter(s => s).length;
+        structure.path_depth_distribution[pathDepth] =
+          (structure.path_depth_distribution[pathDepth] || 0) + 1;
+
         // Analyze path patterns
         const pathSegments = urlObj.pathname.split('/').filter(s => s);
         if (pathSegments.length > 0) {
@@ -442,10 +458,11 @@ export class CrawlDeepTool {
    * Analyze site structure with enhanced link analysis
    * @param {Array} urls - Crawled URLs
    * @param {Object} linkAnalysis - Link analysis results
+   * @param {Array} [pages] - Crawl results, each carrying the depth it was reached at
    * @returns {Object} Enhanced site structure analysis
    */
-  analyzeEnhancedSiteStructure(urls, linkAnalysis = null) {
-    const basicStructure = this.analyzeSiteStructure(urls);
+  analyzeEnhancedSiteStructure(urls, linkAnalysis = null, pages = []) {
+    const basicStructure = this.analyzeSiteStructure(urls, pages);
     
     if (!linkAnalysis) {
       return basicStructure;

package/src/tools/crawl/mapSite.js

@@ -6,6 +6,8 @@ import { CacheManager } from '../../core/cache/CacheManager.js';
 import { SitemapParser } from '../../utils/sitemapParser.js';
 import { ResultRanker } from '../search/ranking/ResultRanker.js';
 import { safeFetch } from '../../utils/ssrfGuard.js';
+import { CRAWLFORGE_USER_AGENT } from '../../utils/fetchIdentity.js';
+import { preflightFetch } from '../../utils/robotsGate.js';
 
 // Lazy singleton — avoids creating a CacheManager timer per request
 let _ranker = null;
@@ -28,13 +30,17 @@ const MapSiteSchema = z.object({
     exclude_patterns: z.array(z.string()).optional().default([])
   }).optional(),
   import_filter_config: z.string().optional(), // JSON string of exported config
-  search: z.string().optional() // when set, rank URLs by relevance and emit ranked_urls
+  search: z.string().optional(), // when set, rank URLs by relevance and emit ranked_urls
+  // Compliance overrides, per request: identify as yourself for a target you
+  // have your own agreement with, and take responsibility for ignoring robots.
+  user_agent: z.string().optional(),
+  respect_robots: z.boolean().optional()
 });
 
 export class MapSiteTool {
   constructor(options = {}) {
     const {
-      userAgent = 'MCP-WebScraper/1.0',
+      userAgent = CRAWLFORGE_USER_AGENT,
       timeout = 10000,
       cacheEnabled = true,
       cacheTTL = 3600000
@@ -62,6 +68,14 @@ export class MapSiteTool {
       const urls = new Set();
       const metadata = new Map();
 
+      // Per-request identity/robots overrides, carried to every fetch this
+      // call makes. Not stored on the instance: one tool object serves
+      // concurrent requests.
+      const identity = {
+        userAgent: validated.user_agent,
+        respectRobots: validated.respect_robots
+      };
+
       // Create domain filter if configuration provided
       let domainFilter = null;
       if (validated.import_filter_config) {
@@ -99,7 +113,7 @@ export class MapSiteTool {
       }
 
       // Fetch and parse the main page for additional URLs
-      const pageUrls = await this.fetchPageUrls(validated.url, domainFilter);
+      const pageUrls = await this.fetchPageUrls(validated.url, domainFilter, identity);
       pageUrls.forEach(url => {
         if (urls.size < validated.max_urls) {
           urls.add(normalizeUrl(url));
@@ -111,7 +125,7 @@ export class MapSiteTool {
 
       // Fetch metadata if requested
       if (validated.include_metadata) {
-        await this.fetchMetadata(urlArray.slice(0, 50), metadata); // Limit metadata fetching
+        await this.fetchMetadata(urlArray.slice(0, 50), metadata, identity); // Limit metadata fetching
       }
 
       // Organize results
@@ -172,7 +186,9 @@ export class MapSiteTool {
       domainFilter: validated.domain_filter ?? null,
       importFilterConfig: validated.import_filter_config ?? null,
       includeMetadata: validated.include_metadata,
-      groupByPath: validated.group_by_path
+      groupByPath: validated.group_by_path,
+      userAgent: validated.user_agent ?? null,
+      respectRobots: validated.respect_robots ?? null
     });
   }
 
@@ -212,9 +228,9 @@ export class MapSiteTool {
     return Array.from(urls);
   }
 
-  async fetchPageUrls(url, domainFilter = null) {
+  async fetchPageUrls(url, domainFilter = null, identity = {}) {
     try {
-      const response = await this.fetchWithTimeout(url);
+      const response = await this.fetchWithTimeout(url, identity);
       if (!response.ok) {
         return [];
       }
@@ -246,15 +262,19 @@ export class MapSiteTool {
       });
 
       return Array.from(urls);
-    } catch {
+    } catch (error) {
+      // A gate refusal is the answer to the request, not a page we failed to
+      // read: surface it instead of returning an emptier map than the caller
+      // would notice.
+      if (error.code === 'ROBOTS_DISALLOWED' || error.code === 'HOST_BLOCKED') throw error;
       return [];
     }
   }
 
-  async fetchMetadata(urls, metadataMap) {
+  async fetchMetadata(urls, metadataMap, identity = {}) {
     const promises = urls.slice(0, 10).map(async (url) => {
       try {
-        const response = await this.fetchWithTimeout(url);
+        const response = await this.fetchWithTimeout(url, identity);
         if (response.ok) {
           const html = await response.text();
           const $ = load(html);
@@ -275,16 +295,21 @@ export class MapSiteTool {
     await Promise.allSettled(promises);
   }
 
-  async fetchWithTimeout(url) {
+  async fetchWithTimeout(url, { userAgent, respectRobots } = {}) {
+    // robots.txt / blocklist gate + per-host throttle. Throws if it refuses.
+    const gate = await preflightFetch(url, {
+      userAgent: userAgent || this.userAgent,
+      respectRobots,
+      tool: 'map_site'
+    });
+
     const controller = new AbortController();
     const timeoutId = setTimeout(() => controller.abort(), this.timeout);
 
     try {
       const response = await safeFetch(url, {
         signal: controller.signal,
-        headers: {
-          'User-Agent': this.userAgent
-        }
+        headers: gate.headers
       });
       clearTimeout(timeoutId);
       return response;

package/src/tools/extract/_fetchAndParse.js

@@ -6,15 +6,16 @@
  *   extractContent.js      (uses native fetch directly but can adopt this)
  *   processDocument.js     (URL sources)
  *
- * Returns { html, $, textContent, finalUrl } so callers don't repeat
+ * Returns { html, $, textContent, finalUrl, warnings } so callers don't repeat
  * the same fetch/cheerio/cleanup boilerplate.
  */
 
 import { load } from 'cheerio';
 import { safeFetch } from '../../utils/ssrfGuard.js';
 import { config } from '../../constants/config.js';
+import { noteRetryAfter } from '../../utils/hostRateLimiter.js';
+import { preflightFetch } from '../../utils/robotsGate.js';
 
-const DEFAULT_USER_AGENT = 'Mozilla/5.0 (compatible; CrawlForge-MCP/3.0)';
 const DEFAULT_TIMEOUT_MS = 15000;
 
 /**
@@ -132,27 +133,41 @@ export function flattenBodyText($) {
  *
  * @param {string} url
  * @param {Object} [options]
- * @param {string}   [options.userAgent]
+ * @param {string}   [options.userAgent]      — per-request identity override
+ * @param {boolean}  [options.respectRobots]  — false = explicit, audited override
+ * @param {string}   [options.tool]           — tool name, for the audit row
+ * @param {string}   [options.apiKey]         — hashed into the audit row
  * @param {number}   [options.timeoutMs]
  * @param {string[]} [options.stripTags]   — additional tags to strip (default: script, style, noscript, iframe, svg)
- * @returns {Promise<{ html: string, $: import('cheerio').CheerioAPI, textContent: string, finalUrl: string }>}
+ * @returns {Promise<{ html: string, $: import('cheerio').CheerioAPI, textContent: string, finalUrl: string, warnings: string[] }>}
  */
 export async function fetchAndParse(url, options = {}) {
   const {
-    userAgent = DEFAULT_USER_AGENT,
+    userAgent,
+    respectRobots,
+    tool,
+    apiKey,
     timeoutMs = DEFAULT_TIMEOUT_MS,
     stripTags = ['script', 'style', 'noscript', 'iframe', 'svg']
   } = options;
 
+  // robots.txt / blocklist gate + per-host throttle. Throws if it refuses.
+  const gate = await preflightFetch(url, { userAgent, respectRobots, tool, apiKey });
+  const warnings = gate.warnings;
+
   const response = await safeFetch(url, {
     headers: {
-      'User-Agent': userAgent,
+      ...gate.headers,
       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
     },
     signal: AbortSignal.timeout(timeoutMs)
   });
 
   if (!response.ok) {
+    // Honour a back-off the host asked for before giving up on this request.
+    if (response.status === 429 || response.status === 503) {
+      noteRetryAfter(url, response.headers?.get?.('retry-after'));
+    }
     throw new Error(`HTTP ${response.status}: ${response.statusText}`);
   }
 
@@ -171,7 +186,7 @@ export async function fetchAndParse(url, options = {}) {
   // HTML parser risks misinterpreting substrings (e.g. a "<script>" value
   // inside a JSON string) as real tags and stripping/mangling content.
   if (classification === 'text') {
-    return { html, $: load(''), textContent: html.trim(), finalUrl: response.url };
+    return { html, $: load(''), textContent: html.trim(), finalUrl: response.url, warnings };
   }
 
   const $ = load(html);
@@ -182,5 +197,5 @@ export async function fetchAndParse(url, options = {}) {
 
   const textContent = flattenBodyText($);
 
-  return { html, $, textContent, finalUrl: response.url };
+  return { html, $, textContent, finalUrl: response.url, warnings };
 }

package/src/tools/extract/analyzeContent.js

@@ -78,8 +78,10 @@ const AnalyzeContentResult = z.object({
     })).optional()
   }).optional(),
   readability: z.object({
-    score: z.number(),
-    level: z.string(),
+    // score/level are absent when Flesch does not apply (CJK) — see notApplicable
+    score: z.number().optional(),
+    level: z.string().optional(),
+    notApplicable: z.string().optional(),
     metrics: z.object({
       sentences: z.number(),
       words: z.number(),
@@ -561,7 +563,7 @@ export class AnalyzeContentTool {
    */
   compareReadability(results) {
     const readabilityScores = results
-      .filter(r => r.success && r.readability)
+      .filter(r => r.success && r.readability && typeof r.readability.score === 'number')
       .map(r => r.readability.score);
 
     if (readabilityScores.length === 0) return null;