crawlforge-mcp-server 6.1.0 → 6.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "6.1.0",
+  "version": "6.3.0",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 30 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -23,7 +23,7 @@ import { ListOllamaModelsTool } from "./src/tools/extract/listOllamaModels.js";
 import { BatchScrapeTool } from "./src/tools/advanced/BatchScrapeTool.js";
 import { ScrapeWithActionsTool } from "./src/tools/advanced/ScrapeWithActionsTool.js";
 import { DeepResearchTool } from "./src/tools/research/deepResearch.js";
-import { TrackChangesTool } from "./src/tools/tracking/trackChanges/index.js";
+import { TrackChangesTool, TRACK_CHANGES_INPUT_SHAPE } from "./src/tools/tracking/trackChanges/index.js";
 import { GenerateLLMsTxtTool } from "./src/tools/llmstxt/generateLLMsTxt.js";
 import { ScrapeTemplateTool } from "./src/tools/templates/ScrapeTemplateTool.js"; // D3.3
 import { UnifiedScrapeTool, SCRAPE_INPUT_SHAPE } from "./src/tools/scrape/unifiedScrape.js"; // D4 D1
@@ -107,7 +107,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
 // Create the server
 const server = new McpServer({
   name: "crawlforge",
-  version: "6.1.0",
+  version: "6.3.0",
   description: "Production-ready MCP server with 30 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
   homepage: "https://www.crawlforge.dev",
   icon: "https://www.crawlforge.dev/icon.png",
@@ -1138,103 +1138,11 @@ registerToolIfEnabled("agent", {
 
 // Tool: track_changes
 registerToolIfEnabled("track_changes", {
-  description: "Use this to monitor a URL for content changes over time - competitor pricing, regulation updates, product availability. Start with operation:\"create_baseline\", then periodically use operation:\"compare\" to diff; repeated compare calls on the same URL are expected. Supports webhooks and scheduled monitoring. Not for a one-off read (scrape). Cost: 3 credits. Example: track_changes({url: \"https://example.com/pricing\", operation: \"create_baseline\"})",
+  description: "Use this to monitor a URL for content changes over time - competitor pricing, regulation updates, product availability. Start with operation:\"create_baseline\", then periodically use operation:\"compare\" to diff; repeated compare calls on the same URL are expected. Supports webhooks and scheduled monitoring, and scheduledMonitorOptions.hosted:true runs the monitor on CrawlForge's servers with email and signed webhooks. Not for a one-off read (scrape). Cost: 3 credits. Example: track_changes({url: \"https://example.com/pricing\", operation: \"create_baseline\"})",
   annotations: { title: "Track Changes", readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true },
+  // The tool module owns the schema (G5); this is the same shape it validates with.
   inputSchema: {
-    url: z.string().url().optional().describe("The URL to track changes for (optional for list_scheduled_monitors)"),
-    operation: z.enum([
-      'create_baseline', 'compare', 'monitor', 'get_history', 'get_stats',
-      'create_scheduled_monitor', 'stop_scheduled_monitor', 'list_scheduled_monitors', 'get_dashboard',
-      'export_history', 'create_alert_rule', 'generate_trend_report', 'get_monitoring_templates'
-    ]).default('compare').describe("Tracking operation to perform"),
-    content: z.string().optional().describe("Content to compare against baseline"),
-    html: z.string().optional().describe("HTML content to compare against baseline"),
-    trackingOptions: z.object({
-      granularity: z.enum(['page', 'section', 'element', 'text']).default('section'),
-      trackText: z.boolean().default(true),
-      trackStructure: z.boolean().default(true),
-      trackAttributes: z.boolean().default(false),
-      trackImages: z.boolean().default(false),
-      trackLinks: z.boolean().default(true),
-      ignoreWhitespace: z.boolean().default(true),
-      ignoreCase: z.boolean().default(false),
-      customSelectors: z.array(z.string()).optional(),
-      excludeSelectors: z.array(z.string()).optional(),
-      significanceThresholds: z.object({
-        minor: z.number().min(0).max(1).default(0.1),
-        moderate: z.number().min(0).max(1).default(0.3),
-        major: z.number().min(0).max(1).default(0.7)
-      }).optional()
-    }).optional().describe("Options for how changes are tracked and compared"),
-    monitoringOptions: z.object({
-      enabled: z.boolean().default(false),
-      interval: z.number().min(60000).max(24 * 60 * 60 * 1000).default(300000),
-      maxRetries: z.number().min(0).max(5).default(3),
-      retryDelay: z.number().min(1000).max(60000).default(5000),
-      notificationThreshold: z.enum(['minor', 'moderate', 'major', 'critical']).default('moderate'),
-      enableWebhook: z.boolean().default(false),
-      webhookUrl: z.string().url().optional(),
-      webhookSecret: z.string().optional()
-    }).optional().describe("Monitoring schedule and notification settings"),
-    storageOptions: z.object({
-      enableSnapshots: z.boolean().default(true),
-      retainHistory: z.boolean().default(true),
-      maxHistoryEntries: z.number().min(1).max(1000).default(100),
-      compressionEnabled: z.boolean().default(true),
-      deltaStorageEnabled: z.boolean().default(true)
-    }).optional().describe("Storage and history retention settings"),
-    queryOptions: z.object({
-      limit: z.number().min(1).max(500).default(50),
-      offset: z.number().min(0).default(0),
-      startTime: z.number().optional(),
-      endTime: z.number().optional(),
-      includeContent: z.boolean().default(false),
-      significanceFilter: z.enum(['all', 'minor', 'moderate', 'major', 'critical']).optional()
-    }).optional().describe("Query options for history and stats retrieval"),
-    notificationOptions: z.object({
-      webhook: z.object({
-        enabled: z.boolean().default(false),
-        url: z.string().url().optional(),
-        method: z.enum(['POST', 'PUT']).default('POST'),
-        headers: z.record(z.string()).optional(),
-        signingSecret: z.string().optional(),
-        includeContent: z.boolean().default(false)
-      }).optional(),
-      slack: z.object({
-        enabled: z.boolean().default(false),
-        webhookUrl: z.string().url().optional(),
-        channel: z.string().optional(),
-        username: z.string().optional()
-      }).optional()
-    }).optional().describe("Notification configuration for webhooks and Slack"),
-    scheduledMonitorOptions: z.object({
-      schedule: z.string().optional().describe("Optional cron expression (power users)"),
-      templateId: z.string().optional(),
-      enabled: z.boolean().default(true),
-      interval: z.number().min(60000).optional().describe("Polling interval in ms (default 1h)"),
-      goal: z.string().optional().describe("Plain-English alert goal; an LLM judges whether a change matches (degrades to threshold if no LLM)"),
-      monitorId: z.string().optional().describe("Monitor id for stop_scheduled_monitor"),
-      notificationThreshold: z.enum(['minor', 'moderate', 'major', 'critical']).optional()
-    }).optional().describe("Scheduled monitoring: recurring compare + notify, optional plain-English goal"),
-    alertRuleOptions: z.object({
-      ruleId: z.string().optional(),
-      condition: z.string().optional(),
-      actions: z.array(z.enum(['webhook', 'email', 'slack'])).optional(),
-      throttle: z.number().min(0).optional(),
-      priority: z.enum(['low', 'medium', 'high']).optional()
-    }).optional().describe("Alert rule configuration for change notifications"),
-    exportOptions: z.object({
-      format: z.enum(['json', 'csv']).default('json'),
-      startTime: z.number().optional(),
-      endTime: z.number().optional(),
-      includeContent: z.boolean().default(false),
-      includeSnapshots: z.boolean().default(false)
-    }).optional().describe("Export options for change history data"),
-    dashboardOptions: z.object({
-      includeRecentAlerts: z.boolean().default(true),
-      includeTrends: z.boolean().default(true),
-      includeMonitorStatus: z.boolean().default(true)
-    }).optional().describe("Dashboard display options"),
+    ...TRACK_CHANGES_INPUT_SHAPE,
     ...COMPLIANCE_PARAMS
   }
 }, withAuth("track_changes", async (params) => {

package/src/cli/commands/monitor.js

@@ -61,19 +61,33 @@ export function register(program) {
     .option('--threshold <level>', 'Notification threshold: minor|moderate|major|critical', 'moderate')
     .option('--cron <expr>', 'Optional cron expression (advanced)')
     .option('--selector <css>', 'CSS selector to scope monitoring')
+    .option('--hosted', "Run the monitor on CrawlForge's servers (fires without this process; email + signed webhooks; 3 credits per compared target per check)")
+    .option('--email <addresses>', 'Comma-separated notification emails (sent by hosted monitors only)')
+    .option('--name <text>', 'Display name for a hosted monitor (default: the URL host)')
     .action(async (url, opts) => {
       const tool = new TrackChangesTool(getToolConfig('track_changes'));
+      const notificationOptions = {
+        ...(opts.webhook ? { webhook: { enabled: true, url: opts.webhook } } : {}),
+        ...(opts.email ? { email: { enabled: true, recipients: opts.email.split(',').map((s) => s.trim()).filter(Boolean) } } : {})
+      };
+      if (opts.email && !opts.hosted) {
+        process.stderr.write('Warning: local monitors do not send email; add --hosted for --email to take effect.\n');
+      }
       try {
         const res = await tool.execute({
           url,
           operation: 'create_scheduled_monitor',
           ...(opts.selector ? { trackingOptions: { customSelectors: [opts.selector] } } : {}),
-          ...(opts.webhook ? { notificationOptions: { webhook: { enabled: true, url: opts.webhook } } } : {}),
+          ...(Object.keys(notificationOptions).length ? { notificationOptions } : {}),
           scheduledMonitorOptions: {
             interval: Math.max(parseInt(opts.every, 10), 60) * 1000,
             ...(opts.goal ? { goal: opts.goal } : {}),
             ...(opts.cron ? { schedule: opts.cron } : {}),
-            notificationThreshold: opts.threshold
+            ...(opts.hosted ? { hosted: true } : {}),
+            ...(opts.name ? { name: opts.name } : {}),
+            // Local only: a hosted check has no significance threshold, and
+            // the option's default would otherwise warn on every hosted create.
+            ...(opts.hosted ? {} : { notificationThreshold: opts.threshold })
           }
         });
         emit(res);
@@ -86,7 +100,7 @@ export function register(program) {
 
   program
     .command('monitor:list')
-    .description('List persisted scheduled monitors')
+    .description('List scheduled monitors (local and hosted)')
     .action(async () => {
       const tool = new TrackChangesTool(getToolConfig('track_changes'));
       try {
@@ -100,7 +114,7 @@ export function register(program) {
 
   program
     .command('monitor:stop <id>')
-    .description('Stop and remove a scheduled monitor by id')
+    .description('Stop and remove a scheduled monitor by id (local or hosted)')
     .action(async (id) => {
       const tool = new TrackChangesTool(getToolConfig('track_changes'));
       try {

package/src/core/AgentOrchestrator.js

@@ -100,6 +100,62 @@ export function unverifiedValues(answer, sourceText) {
  */
 const CURRENT_STATE_RE = /\b(right now|currently|today|tonight|at the moment|as of (now|today)|latest|this (week|month|morning))\b|#\d+[^.?!]*\bnow\b/i;
 
+/** Version numbers as they appear in an answer, e.g. "2.11.4". */
+const VERSION_IN_TEXT = /\b\d+\.\d+(?:\.\d+){0,2}\b/g;
+
+/** Hosts that publish user discussion rather than a project's own statements. */
+const DISCUSSION_HOSTS = new Set([
+  'news.ycombinator.com', 'reddit.com', 'www.reddit.com', 'quora.com', 'www.quora.com',
+  'stackoverflow.com', 'serverfault.com', 'superuser.com', 'askubuntu.com',
+  'facebook.com', 'www.facebook.com', 'twitter.com', 'x.com'
+]);
+
+/**
+ * True when a URL is a discussion page — a forum, Q&A site or social post.
+ * Covers the hosted-forum conventions too: caddy.community, community.acme.io,
+ * forum.example.org, discuss.example.org, anything.stackexchange.com.
+ */
+export function isDiscussionSource(url) {
+  try {
+    const host = new URL(url).hostname.toLowerCase();
+    return DISCUSSION_HOSTS.has(host)
+      || /(^|\.)(community|forum|forums|discuss|discourse)\./.test(host)
+      || /\.(community|forum)$/.test(host)
+      || /\.stackexchange\.com$/.test(host);
+  } catch {
+    return false;
+  }
+}
+
+/** True when the task asks for a version or release number. */
+export function isVersionQuestion(prompt) {
+  return /\b(version|versions|release|releases|released|changelog)\b/i.test(prompt || '');
+}
+
+/**
+ * Version numbers the answer states that no non-discussion source supports.
+ *
+ * A forum thread carries real version strings, so the literal provenance check
+ * passes them: asked for the current Caddy release, the agent answered "2.4.3"
+ * off a caddy.community thread about a different project, while caddyserver.com
+ * and the GitHub repo — both fetched — stated no version at all (R19). A
+ * version is a claim about the project, and a discussion page is not the
+ * project speaking.
+ *
+ * Returns [] when nothing but discussion pages was fetched: there is then no
+ * better source to have preferred, and the provenance check already applies.
+ */
+export function unsupportedVersionClaims(answer, evidence) {
+  if (!answer) return [];
+  // URLs are citations, not claims — a version inside one is not an assertion.
+  const claims = [...new Set(answer.replace(/https?:\/\/\S+/g, ' ').match(VERSION_IN_TEXT) || [])];
+  if (claims.length === 0) return [];
+  const authoritative = (evidence || []).filter(e => !isDiscussionSource(e.url));
+  if (authoritative.length === 0) return [];
+  const supported = authoritative.map(e => e.text || '').join('\n');
+  return claims.filter(version => !supported.includes(version));
+}
+
 export function isCurrentStateTask(prompt) {
   return CURRENT_STATE_RE.test(prompt || '');
 }
@@ -459,6 +515,10 @@ export class AgentOrchestrator {
     let degraded = false;
     let degradedReason;
     let unverified = [];
+    let forumOnlyVersions = [];
+    // Distinct from `degraded`: a run can be degraded because its version
+    // claims are unsupported and still have had its provenance checked.
+    let synthesisFailed = false;
 
     try {
       // Wording matters for small local models (llama3.2-class): without the
@@ -515,8 +575,49 @@ export class AgentOrchestrator {
           `\n\nProvenance warning: ${unverified.map(v => `"${v}"`).join(', ')} ` +
           `${unverified.length === 1 ? 'does' : 'do'} not appear in the fetched sources and may be invented.`;
       }
+
+      // Asked for a current version, the agent answered "2.4.3" from a forum
+      // thread about a different project while the project's own site and its
+      // GitHub repo — both fetched — stated no version at all (R19). The
+      // literal provenance check cannot catch this: the string really is on a
+      // fetched page. A version only a discussion page supports gets the same
+      // one corrective rewrite, and what survives it is flagged rather than
+      // presented as the answer.
+      if (currentState && isVersionQuestion(prompt)) {
+        forumOnlyVersions = unsupportedVersionClaims(answer, evidence);
+        if (forumOnlyVersions.length > 0) {
+          const versionRetryPrompt =
+            `${synthesisPrompt}\n\nYour previous answer was:\n${fenceUntrusted(answer, 'previous answer')}\n` +
+            `It states ${forumOnlyVersions.map(v => `"${v}"`).join(', ')}, which ${forumOnlyVersions.length === 1 ? 'appears' : 'appear'} ` +
+            `only in forum or discussion pages, not in the project's own pages among the sources. ` +
+            `A forum post about a different project is not a release announcement. ` +
+            `Rewrite the answer using only versions stated by the project's own sources. ` +
+            `If none of them states a current version, say plainly that the fetched sources do not state one.`;
+          try {
+            const { text: rewritten } = await this._getSamplingClient().complete(versionRetryPrompt, { maxTokens: 1024 });
+            if (rewritten && rewritten.trim()) {
+              answer = rewritten;
+              forumOnlyVersions = unsupportedVersionClaims(answer, evidence);
+            }
+          } catch {
+            // Keep the previous answer; it is flagged below.
+          }
+        }
+        if (forumOnlyVersions.length > 0) {
+          answer +=
+            `\n\nSource warning: ${forumOnlyVersions.map(v => `"${v}"`).join(', ')} ` +
+            `${forumOnlyVersions.length === 1 ? 'appears' : 'appear'} only in discussion pages among the fetched sources, ` +
+            `not in the project's own. Treat ${forumOnlyVersions.length === 1 ? 'it' : 'them'} as unconfirmed.`;
+          degraded = true;
+          degradedReason =
+            `Version claim unsupported by the project's own sources: ` +
+            `${forumOnlyVersions.map(v => `"${v}"`).join(', ')} ${forumOnlyVersions.length === 1 ? 'appears' : 'appear'} ` +
+            `only in discussion pages. The authoritative pages fetched state no version.`;
+        }
+      }
     } catch (err) {
       degraded = true;
+      synthesisFailed = true;
       degradedReason = `LLM synthesis unavailable: ${err.message}`;
       // Return raw evidence so the host LLM can synthesize
       answer = null;
@@ -531,7 +632,11 @@ export class AgentOrchestrator {
       reason: degradedReason,
       steps: step,
       urls_fetched: urlsFetched,
-      provenance: { checked: !degraded, unverified }
+      provenance: {
+        checked: !synthesisFailed,
+        unverified,
+        ...(forumOnlyVersions.length > 0 ? { unsupported_versions: forumOnlyVersions } : {})
+      }
     };
   }
 

package/src/core/ResearchOrchestrator.js

@@ -2000,7 +2000,13 @@ export class ResearchOrchestrator extends EventEmitter {
         sessionId: this.researchState.sessionId,
         topic,
         synthesisMode: 'raw_evidence',
-        note: "This response contains raw research evidence with no AI synthesis. The calling LLM (you) should synthesize these sources to answer the user's question. To enable internal LLM synthesis instead, set OPENAI_API_KEY or ANTHROPIC_API_KEY in the MCP server environment.",
+        // Raw evidence has two quite different causes and the note used to
+        // report only one of them: a run that synthesized fine until its
+        // token budget ran out was still told to set an API key it already
+        // had (R19). Say which one actually happened.
+        note: this.researchState.tokenBudgetExceeded
+          ? `This response contains raw research evidence with no AI synthesis. Synthesis ran until the research token budget (${this.researchState.tokenBudgetChars.toLocaleString()} chars of source content) was exhausted, then stopped for the rest of the session. The calling LLM (you) should synthesize these sources. To get internal synthesis, narrow the topic or lower maxUrls, or raise RESEARCH_TOKEN_BUDGET_CHARS in the MCP server environment.`
+          : "This response contains raw research evidence with no AI synthesis. The calling LLM (you) should synthesize these sources to answer the user's question. To enable internal LLM synthesis instead, set OPENAI_API_KEY or ANTHROPIC_API_KEY in the MCP server environment.",
         sources,
         findings: [],
         researchSummary: {

package/src/core/analysis/ContentAnalyzer.js

@@ -143,6 +143,26 @@ export class ContentAnalyzer {
     return this.cjkScriptCounts(text).share >= 0.1;
   }
 
+  /**
+   * True when the text is written mostly in the Latin alphabet.
+   *
+   * The analyser's entity extraction (compromise) and its syllable counter are
+   * both English/Latin-only. Run over Russian they do not degrade, they invent:
+   * "дождь и порывистый" ("rain and gusty") came back as an organization, and
+   * every Cyrillic word counted as one syllable, which put a weather report at
+   * Flesch 100, "Very Easy" (R19). Text with too few Latin letters to work on
+   * is reported as not-applicable rather than analysed anyway.
+   *
+   * @param {string} text - Text to analyze
+   * @returns {boolean}
+   */
+  isLatinScriptText(text) {
+    const letters = (text || '').match(/\p{L}/gu);
+    if (!letters || letters.length === 0) return true; // nothing to misread
+    const latin = (text || '').match(/\p{Script=Latin}/gu);
+    return (latin ? latin.length : 0) / letters.length >= 0.5;
+  }
+
   /**
    * Tokenize text into words with Intl.Segmenter (dictionary-based for CJK
    * scripts, whitespace/boundary-based otherwise). Only word-like segments
@@ -534,6 +554,17 @@ export class ContentAnalyzer {
    */
   async extractEntities(text, options = {}) {
     try {
+      // compromise is an English model. On Russian it does not find fewer
+      // entities, it finds wrong ones — "дождь и порывистый" as an
+      // organization (R19). Say so rather than fill the categories with noise.
+      if (!this.isLatinScriptText(text)) {
+        return {
+          people: [], places: [], organizations: [], dates: [], money: [], other: [],
+          notApplicable: 'entity-extraction-requires-latin-script',
+          summary: { totalEntities: 0, uniqueEntities: 0, entityDensity: 0 }
+        };
+      }
+
       const doc = nlp(text);
 
       // compromise's out('array') keeps adjoining punctuation ("Craigslist.",
@@ -754,10 +785,13 @@ export class ContentAnalyzer {
         syllables: totalSyllables
       };
 
-      // Flesch is syllable-based and says nothing about CJK scripts. Report the
-      // metrics with an explicit reason instead of a fabricated score — a null
-      // return above already means "failed", so the two stay distinguishable.
-      if (isCjk) {
+      // Flesch is syllable-based, and countSyllables() only knows the Latin
+      // vowels — every Cyrillic, Greek or Arabic word scores one syllable, so
+      // the formula returns ~100 ("Very Easy") for any of them (R19). Report
+      // the metrics with an explicit reason instead of a fabricated score — a
+      // null return above already means "failed", so the two stay
+      // distinguishable.
+      if (isCjk || !this.isLatinScriptText(text)) {
         return {
           notApplicable: 'flesch-requires-syllable-based-language',
           metrics
@@ -1070,16 +1104,38 @@ export class ContentAnalyzer {
    * @returns {boolean} - True if stop word
    */
   isStopWord(word) {
-    const stopWords = [
-      'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by',
-      'from', 'as', 'is', 'was', 'are', 'were', 'be', 'been', 'being', 'have', 'has', 'had',
-      'do', 'does', 'did', 'will', 'would', 'could', 'should', 'may', 'might', 'must', 'can',
-      'this', 'that', 'these', 'those', 'i', 'you', 'he', 'she', 'it', 'we', 'they', 'me',
-      'him', 'her', 'us', 'them', 'my', 'your', 'his', 'its', 'our', 'their'
-    ];
-    
-    return stopWords.includes(word.toLowerCase());
+    return STOP_WORDS.has(word.toLowerCase());
   }
 }
 
+/**
+ * Stop words, by script. Russian was added after a weather report analysed with
+ * `analyze_content` returned bare prepositions and conjunctions among its
+ * topics — "над" ("over") ranked as one (R19). R17 had added Japanese the same
+ * way; every language whose function words are not on this list gets them back
+ * as topics.
+ */
+const STOP_WORDS = new Set([
+  // English
+  'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by',
+  'from', 'as', 'is', 'was', 'are', 'were', 'be', 'been', 'being', 'have', 'has', 'had',
+  'do', 'does', 'did', 'will', 'would', 'could', 'should', 'may', 'might', 'must', 'can',
+  'this', 'that', 'these', 'those', 'i', 'you', 'he', 'she', 'it', 'we', 'they', 'me',
+  'him', 'her', 'us', 'them', 'my', 'your', 'his', 'its', 'our', 'their',
+  // Russian — prepositions, conjunctions, particles and pronouns
+  'и', 'в', 'во', 'не', 'что', 'он', 'на', 'я', 'с', 'со', 'как', 'а', 'то', 'все', 'она',
+  'так', 'его', 'но', 'да', 'ты', 'к', 'у', 'же', 'вы', 'за', 'бы', 'по', 'только', 'ее',
+  'мне', 'было', 'вот', 'от', 'меня', 'еще', 'нет', 'о', 'из', 'ему', 'теперь', 'когда',
+  'даже', 'ну', 'вдруг', 'ли', 'если', 'уже', 'или', 'ни', 'быть', 'был', 'него', 'до',
+  'вас', 'нибудь', 'опять', 'уж', 'вам', 'ведь', 'там', 'потом', 'себя', 'ничего', 'ей',
+  'может', 'они', 'тут', 'где', 'есть', 'надо', 'ней', 'для', 'мы', 'тебя', 'их', 'чем',
+  'была', 'сам', 'чтоб', 'без', 'будто', 'чего', 'раз', 'тоже', 'себе', 'под', 'будет',
+  'ж', 'тогда', 'кто', 'этот', 'того', 'потому', 'этого', 'какой', 'совсем', 'ним',
+  'здесь', 'этом', 'один', 'почти', 'мой', 'тем', 'чтобы', 'нее', 'были', 'куда', 'зачем',
+  'всех', 'никогда', 'можно', 'при', 'наконец', 'два', 'об', 'другой', 'хоть', 'после',
+  'над', 'больше', 'тот', 'через', 'эти', 'нас', 'про', 'всего', 'них', 'какая', 'много',
+  'разве', 'три', 'эту', 'моя', 'впрочем', 'свою', 'этой', 'перед', 'иногда', 'лучше',
+  'чуть', 'том', 'нельзя', 'такой', 'им', 'более', 'всегда', 'конечно', 'всю', 'между'
+]);
+
 export default ContentAnalyzer;

package/src/core/llm/LLMManager.js

@@ -1,10 +1,32 @@
 import { OpenAIProvider } from './OpenAIProvider.js';
 import { extractionFormat } from '../../utils/extractionFormat.js';
+import { validateFieldsAgainstSchema } from '../../utils/schemaValidate.js';
 import { AnthropicProvider } from './AnthropicProvider.js';
 import { OllamaProvider } from './OllamaProvider.js';
 import { Logger } from '../../utils/Logger.js';
 import { isJudgementModel } from '../../utils/ollamaConfig.js';
 
+// Output-token ceilings for structured extraction. A schema of scalar fields
+// answers in a few hundred tokens; one asking for an array is writing rows,
+// and the same ceiling truncates it mid-object.
+const SCALAR_OUTPUT_CEILING = 2000;
+const ROW_OUTPUT_CEILING = 4000;
+const ROW_FIELD_ALLOWANCE = 1200;
+
+/**
+ * True when a JSON string that failed to parse simply stops partway — the
+ * signature of a response that hit its output-token limit, as opposed to one
+ * that is malformed from the start.
+ */
+function endsMidJson(text) {
+  const trimmed = text.trim();
+  if (!trimmed.startsWith('{') && !trimmed.startsWith('[')) return false;
+  // A complete document ends on its closing brace or bracket. Anything else
+  // — a dangling comma, an unterminated string, a half-written key — means
+  // the writer stopped rather than finished.
+  return !trimmed.endsWith('}') && !trimmed.endsWith(']');
+}
+
 /**
  * LLM Manager
  * Manages multiple LLM providers and provides unified interface
@@ -916,9 +938,17 @@ Return the list and nothing else.`;
       ? content.substring(0, maxContentLength) + '...'
       : content;
 
-    // Scale maxTokens with schema complexity
-    const schemaFields = Object.keys(schema.properties || {}).length;
-    const scaledTokens = Math.min(2000, Math.max(maxTokens, schemaFields * 100 + 500));
+    // Scale maxTokens with schema complexity. An array-valued property is one
+    // key but many rows of output, so counting keys alone budgets a 250-row
+    // table exactly like a single string: the response stops mid-object and
+    // the whole extraction is thrown away as unparseable (R19).
+    const fields = Object.values(schema.properties || {});
+    const arrayFields = fields.filter((field) => field?.type === 'array').length;
+    const ceiling = arrayFields > 0 ? ROW_OUTPUT_CEILING : SCALAR_OUTPUT_CEILING;
+    const scaledTokens = Math.min(
+      ceiling,
+      Math.max(maxTokens, fields.length * 100 + arrayFields * ROW_FIELD_ALLOWANCE + 500)
+    );
 
     const systemPrompt = `You are a structured data extraction expert. Extract data from the provided content and return ONLY valid JSON that conforms to the given JSON Schema. Use null for any field the content does not state — never guess, infer, or fill a value from memory. Do not include any explanation or markdown — only the raw JSON object.`;
 
@@ -933,76 +963,74 @@ ${truncatedContent}
 
 Extract the data and return valid JSON:`;
 
-    try {
-      const response = await this.generateCompletion(extractionPrompt, {
-        systemPrompt,
-        maxTokens: scaledTokens,
-        temperature: 0.1,
-        // Constrain the output to the caller's shape with every field
-        // nullable, so a model shown content that does not state a field can
-        // answer null instead of being decoded into an invented string. Small
-        // local models otherwise wrap the JSON in prose and the parse throws.
-        format: extractionFormat(schema)
-      });
+    // Two attempts, the second with twice the budget: the failure this guards
+    // against is a response cut off mid-object, which re-asking at the same
+    // size reproduces exactly. Mirrors the retry in synthesizeFindings().
+    let lastError;
+    for (let attempt = 0; attempt < 2; attempt++) {
+      const budget = attempt === 0 ? scaledTokens : Math.min(ceiling * 2, scaledTokens * 2);
+      try {
+        const response = await this.generateCompletion(extractionPrompt, {
+          systemPrompt,
+          maxTokens: budget,
+          temperature: 0.1,
+          // Constrain the output to the caller's shape with every field
+          // nullable, so a model shown content that does not state a field can
+          // answer null instead of being decoded into an invented string. Small
+          // local models otherwise wrap the JSON in prose and the parse throws.
+          format: extractionFormat(schema)
+        });
+
+        // Strip markdown code fences if present
+        const cleaned = response.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '').trim();
+        let parsed;
+        try {
+          parsed = JSON.parse(cleaned);
+        } catch (parseError) {
+          // Name the cause when the JSON simply stops: "unexpected end of
+          // input at position 2608" tells a caller nothing they can act on,
+          // whereas "cut off at the 1000-token limit" points at the schema.
+          throw endsMidJson(cleaned)
+            ? new Error(`model response was cut off at the ${budget}-token output limit (${cleaned.length} chars) — the schema asks for more rows than fit`)
+            : parseError;
+        }
 
-      // Strip markdown code fences if present
-      const cleaned = response.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '').trim();
-      const parsed = JSON.parse(cleaned);
-
-      // Lightweight validation
-      const validation = this.validateAgainstSchema(parsed, schema);
-      return {
-        data: parsed,
-        method: 'llm',
-        valid: validation.valid,
-        validationErrors: validation.errors
-      };
-    } catch (error) {
-      this.logger.warn('LLM structured extraction failed, using fallback', { error: error.message });
-      // Report which path produced the data. Callers previously labelled this
-      // result "llm", so a failed LLM call was returned as a high-confidence
-      // LLM extraction.
-      return { ...this.fallbackStructuredExtraction(content, schema), error: error.message };
+        const validation = this.validateAgainstSchema(parsed, schema);
+        return {
+          data: parsed,
+          method: 'llm',
+          valid: validation.valid,
+          validationErrors: validation.errors
+        };
+      } catch (error) {
+        lastError = error;
+        this.logger.warn('LLM structured extraction attempt failed', {
+          attempt: attempt + 1,
+          budget,
+          error: error.message
+        });
+      }
     }
+
+    this.logger.warn('LLM structured extraction failed, using fallback', { error: lastError.message });
+    // Report which path produced the data. Callers previously labelled this
+    // result "llm", so a failed LLM call was returned as a high-confidence
+    // LLM extraction.
+    return { ...this.fallbackStructuredExtraction(content, schema), error: lastError.message };
   }
 
   /**
-   * Validate a parsed object against a simple JSON Schema
+   * Validate a parsed object against a JSON Schema.
+   *
+   * Delegates to the shared validator so that it descends into `items` and
+   * nested `properties`. This used to be a hand-rolled one-level check, which
+   * passed `{countries: ["a string"]}` against
+   * `{countries: {type: 'array', items: {type: 'object'}}}` as valid — the
+   * top-level value really was an array, and nothing looked inside it, so an
+   * extraction full of stray page text was returned as `valid: true` (R19).
    */
   validateAgainstSchema(data, schema) {
-    const errors = [];
-    const properties = schema.properties || {};
-    const required = schema.required || [];
-
-    for (const field of required) {
-      // The decoder is told to answer null for a field the content never
-      // states, so a null here is "not filled in", the same as absent.
-      if (!(field in data) || data[field] === null) {
-        errors.push(`Missing required field: ${field}`);
-      }
-    }
-
-    for (const [key, fieldSchema] of Object.entries(properties)) {
-      if (key in data) {
-        const value = data[key];
-        // A null in a field the schema does not require is the honest answer
-        // for content that never states it, not a type error (typeof null is
-        // 'object', which used to read as "expected number, got object").
-        if (value === null) continue;
-        const expectedType = fieldSchema.type;
-        if (expectedType) {
-          const actualType = Array.isArray(value) ? 'array' : typeof value;
-          if (actualType !== expectedType) {
-            errors.push(`Field "${key}": expected ${expectedType}, got ${actualType}`);
-          }
-        }
-        if (fieldSchema.enum && !fieldSchema.enum.includes(value)) {
-          errors.push(`Field "${key}": value "${value}" not in enum ${JSON.stringify(fieldSchema.enum)}`);
-        }
-      }
-    }
-
-    return { valid: errors.length === 0, errors };
+    return validateFieldsAgainstSchema(data, schema);
   }
 
   /**