nothumanallowed 15.1.4 → 15.1.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nothumanallowed",
-  "version": "15.1.4",
+  "version": "15.1.6",
   "description": "NotHumanAllowed — 38 AI agents, 80 tools, Studio (visual agentic workflows). Email, calendar, browser automation, screen capture, canvas, cron/heartbeat, Alexandria E2E messaging, GitHub, Notion, Slack, voice chat, free AI (Liara), 28 languages. Zero-dependency CLI.",
   "type": "module",
   "bin": {

package/src/constants.mjs

@@ -5,7 +5,7 @@ import { fileURLToPath } from 'url';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
-export const VERSION = '15.0.12';
+export const VERSION = '15.1.6';
 export const BASE_URL = 'https://nothumanallowed.com/cli';
 export const API_BASE = 'https://nothumanallowed.com/api/v1';
 

package/src/server/routes/studio.mjs

@@ -9,14 +9,45 @@ import { sendJSON, sendError, parseBody } from '../index.mjs';
 import { loadConfig } from '../../config.mjs';
 import { NHA_DIR, AGENTS_DIR } from '../../constants.mjs';
 import { callLLM, callLLMStream, parseAgentFile } from '../../services/llm.mjs';
-import { webSearch, fetchUrl } from '../../services/web-tools.mjs';
+import { webSearch, fetchUrl, fetchUrlRich, headProbe } from '../../services/web-tools.mjs';
 
-// Agents that get web data pre-fetched before their LLM call
+// Agents that get web data pre-fetched (search + URL content) before their LLM call.
+// We err on the side of inclusion: any agent that writes analysis, copy, briefings,
+// or reports benefits from real-time data — without it they hallucinate plausible
+// but generic frameworks. Synthesis-only agents (CanvasAgent, GitHub/Email/Slack
+// connectors) are intentionally excluded.
 const WEB_TOOL_AGENTS = new Set([
-  'WebSearchAgent', 'TravelAgent', 'mercury', 'MERCURY',
-  'athena', 'ATHENA', 'oracle', 'ORACLE', 'cassandra', 'CASSANDRA',
-  'HERALD', 'DataAnalystAgent', 'herald', 'tempest', 'TEMPEST',
-  'epicure', 'EPICURE', 'cartographer', 'CARTOGRAPHER',
+  // Original research/intel set
+  'WebSearchAgent', 'TravelAgent',
+  'MERCURY', 'mercury',
+  'ATHENA',  'athena',
+  'ORACLE',  'oracle',
+  'CASSANDRA', 'cassandra',
+  'HERALD',  'herald',
+  'DataAnalystAgent',
+  'TEMPEST', 'tempest',
+  'EPICURE', 'epicure',
+  'CARTOGRAPHER', 'cartographer',
+  // Analysis & writing — they MUST ground on the target site, not invent.
+  'QUILL', 'quill',
+  'ECHO',  'echo',
+  'MURASAKI', 'murasaki',
+  'SCHEHERAZADE', 'scheherazade',
+  'MUSE', 'muse',
+  // Security / architecture / reasoning — also benefit from grounded context.
+  'FORGE', 'forge',
+  'PROMETHEUS', 'prometheus',
+  'SABER', 'saber',
+  'ADE', 'ade',
+  'ZERO', 'zero',
+  'SAURON', 'sauron',
+  'LOGOS', 'logos',
+  'VERITAS', 'veritas',
+  'LINK', 'link',
+  'NAVI', 'navi',
+  'EDI', 'edi',
+  // Polyglot needs source content to translate accurately.
+  'polyglot',
 ]);
 
 // ── Complete Agent Registry ─────────────────────────────────────────────────
@@ -94,20 +125,197 @@ async function runWebSearch(query) {
 }
 
 /**
- * Fetch a URL and return formatted content string.
+ * Fetch a URL and return a structured result. Caller decides how to format it.
+ * Returns { ok, url, content?, title?, metadata?, code?, reason? }.
  */
 async function runFetchUrl(url) {
   try {
-    const result = await fetchUrl(url);
-    if (result.error) return `Fetch failed: ${result.message}`;
-    const titlePart = result.title ? `Title: ${result.title}\n\n` : '';
-    const text = (result.body || '').slice(0, 5000);
-    return `Content from ${url}:\n\n${titlePart}${text}`;
+    const result = await fetchUrlRich(url, { maxChars: 16_000 });
+    if (result.error) {
+      return { ok: false, url, code: result.code || 'FETCH_FAILED', reason: result.message || 'unknown' };
+    }
+    const meta = result.metadata || {};
+    const ogBlock = Object.keys(meta.og || {}).length
+      ? `OpenGraph: ${Object.entries(meta.og).slice(0, 8).map(([k, v]) => `${k}=${String(v).slice(0, 200)}`).join(' | ')}\n`
+      : '';
+    const ldBlock = (meta.jsonLd || []).length
+      ? `Schema.org types: ${meta.jsonLd.slice(0, 6).map(j => j.type).join(', ')}\n`
+      : '';
+    const headingsBlock = meta.headings && (meta.headings.h1?.length || meta.headings.h2?.length)
+      ? `Headings H1: ${(meta.headings.h1 || []).slice(0, 5).join(' | ')}\n` +
+        (meta.headings.h2?.length ? `Headings H2: ${meta.headings.h2.slice(0, 8).join(' | ')}\n` : '')
+      : '';
+    const descBlock = meta.description ? `Description: ${meta.description}\n` : '';
+    return {
+      ok: true,
+      url: result.url || url,
+      title: result.title || '',
+      content: result.body || '',
+      metadata: meta,
+      preamble: `${result.title ? `Title: ${result.title}\n` : ''}${descBlock}${ogBlock}${ldBlock}${headingsBlock}`.trim(),
+    };
   } catch (e) {
-    return `Fetch error: ${e.message}`;
+    return { ok: false, url, code: 'EXCEPTION', reason: e.message || String(e) };
   }
 }
 
+/**
+ * Extract URLs from free-form text — robust matcher.
+ *
+ * Catches:
+ *   - Fully-qualified URLs: http(s)://example.com/path
+ *   - Bare domains with www: www.example.com
+ *   - Bare domains without www: example.com, example.it, example.co.uk
+ *   - Markdown-wrapped: [label](example.com) — the URL portion is matched
+ *
+ * Rejects:
+ *   - File extensions misread as domains: image.png, doc.pdf, app.js
+ *   - Single-word tokens without a dot
+ *   - Email addresses (the @ rules out the domain match)
+ *
+ * Returns normalized https:// URLs, deduplicated, max 6.
+ */
+const FILE_EXT_BLOCKLIST = new Set([
+  'png','jpg','jpeg','gif','webp','svg','ico','bmp','tiff',
+  'pdf','doc','docx','xls','xlsx','ppt','pptx','csv','txt','zip','tar','gz','rar','7z',
+  'mp3','mp4','wav','ogg','mov','avi','mkv',
+  'js','css','json','xml','yaml','yml','md','html','htm',
+]);
+
+function extractUrlsFromText(text) {
+  if (!text) return [];
+  const t = String(text);
+  const found = new Set();
+
+  // Full URLs
+  for (const m of t.matchAll(/https?:\/\/[^\s,)<>"'`]+/gi)) {
+    const clean = m[0].replace(/[.,;:!?)\]]+$/, '');
+    found.add(clean);
+  }
+
+  // Bare domains — TLD of 2..24 letters. Accepts 2-label (apple.com),
+  // 3-label (www.apple.com), and N-label domains.
+  const bareRe = /(?<![\/@\w])(?:www\.)?[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*\.([a-z]{2,24})\b(?:\/[^\s,()<>"'`]*)?/gi;
+  for (const m of t.matchAll(bareRe)) {
+    const candidate = m[0].replace(/[.,;:!?)\]]+$/, '');
+    if (candidate.startsWith('http')) continue;
+    const tld = (m[1] || '').toLowerCase();
+    if (FILE_EXT_BLOCKLIST.has(tld)) continue;
+    if (/^\d+(\.\d+)+$/.test(candidate)) continue;
+    found.add('https://' + candidate);
+  }
+
+  // Dedup: if "https://x.com/path" and "https://www.x.com/path" both present,
+  // keep the longer (more specific) one only.
+  const list = [...found];
+  const filtered = list.filter(url => {
+    const stripped = url.replace(/^https?:\/\/(www\.)?/, '');
+    return !list.some(other =>
+      other !== url &&
+      other.replace(/^https?:\/\/(www\.)?/, '') === stripped &&
+      other.length > url.length,
+    );
+  });
+
+  return filtered.slice(0, 6);
+}
+
+/**
+ * Automatic fallback: when the primary fetch fails, try to gather indirect
+ * intelligence about the domain via web search. Looks for:
+ *   - site:domain.com listings (whatever Google has indexed)
+ *   - "<domain> chi è" / "<domain> azienda settore" — public profiles, news
+ *   - Wikipedia / corporate registries
+ *
+ * Returns up to 3 fetched pages with real content, formatted for ground truth.
+ * Returns null if nothing useful was found.
+ */
+async function gatherIndirectIntel(failedUrl, sse) {
+  try {
+    const parsed = new URL(failedUrl);
+    const domain = parsed.hostname.replace(/^www\./, '');
+    const queries = [
+      `site:${domain}`,
+      `"${domain}" azienda settore prodotti`,
+      `${domain} chi è cosa fa`,
+    ];
+
+    if (sse) sse({ token: `[Fallback: ricerco info indirette su ${domain}...]` });
+
+    const allResults = [];
+    for (const q of queries) {
+      try {
+        const r = await webSearch(q, 5);
+        if (!r.error && Array.isArray(r.results)) {
+          for (const item of r.results) {
+            // Skip if the URL is the same failed domain (we already know it's down)
+            try {
+              const u = new URL(item.url);
+              if (u.hostname.replace(/^www\./, '') === domain) continue;
+            } catch { /* skip invalid URLs */ }
+            allResults.push({ ...item, fromQuery: q });
+          }
+        }
+      } catch { /* skip failed query */ }
+    }
+
+    if (allResults.length === 0) return null;
+
+    // Dedupe by URL, take top 4, fetch them
+    const seen = new Set();
+    const unique = allResults.filter(r => {
+      if (seen.has(r.url)) return false;
+      seen.add(r.url);
+      return true;
+    }).slice(0, 4);
+
+    if (sse) sse({ token: `[Fallback: ${unique.length} fonti indirette trovate, recupero contenuto...]` });
+
+    const fetched = await Promise.all(
+      unique.map(async (r) => {
+        try {
+          const full = await fetchUrlRich(r.url, { maxChars: 4_000, timeout: 8_000 });
+          if (full.error) return null;
+          return {
+            url: r.url,
+            title: full.title || r.title || '',
+            snippet: r.snippet || '',
+            content: (full.body || '').slice(0, 3_000),
+            description: full.metadata?.description || '',
+            fromQuery: r.fromQuery,
+          };
+        } catch { return null; }
+      })
+    );
+
+    const good = fetched.filter(Boolean);
+    if (good.length === 0) return null;
+
+    return good;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Strip transient status markers, internal think/tool blocks, and propagation
+ * noise before the text is injected into another agent's prompt or surfaced
+ * to the client. Parity with LegionX `CommunicationStream.stripTags`.
+ */
+function sanitizeAgentOutput(text) {
+  if (!text) return '';
+  return String(text)
+    .replace(/<think>[\s\S]*?<\/think>/gi, '')
+    .replace(/<thinking>[\s\S]*?<\/thinking>/gi, '')
+    .replace(/<tool[^>]*>[\s\S]*?<\/tool>/gi, '')
+    .replace(/<scratch[^>]*>[\s\S]*?<\/scratch>/gi, '')
+    .replace(/<reasoning>[\s\S]*?<\/reasoning>/gi, '')
+    .replace(/\[(?:Searching|Fetching|GET|POST|HEAD|Raccolta dati web|Parlamento)[^\]]*\]\s*/gi, '')
+    .replace(/​|/g, '') // zero-width / BOM
+    .replace(/\r\n/g, '\n')
+    .trim();
+}
+
 /**
  * Extract search queries from a task string.
  * Returns up to 3 queries covering different angles.
@@ -257,8 +465,24 @@ Select the optimal agent pipeline. Output ONLY the JSON.`;
         } catch {}
       }
 
-      const contextBlock = context ? `\n\n## CONTEXT FROM PREVIOUS STEPS:\n${context.slice(0, 12000)}` : '';
-      const proposalContextBlock = body.proposalContext ? `\n\n## OTHER AGENTS' PROPOSALS (CROSS-READING):\n${body.proposalContext.slice(0, 8000)}` : '';
+      const contextBlock = context ? `\n\n## CONTEXT FROM PREVIOUS STEPS:\n${sanitizeAgentOutput(context).slice(0, 12000)}` : '';
+
+      // Cross-reading block — when other agents have already produced proposals,
+      // we inject them with explicit deliberation instructions so the model
+      // engages with them (agree/disagree) rather than restating its own view.
+      const proposalContextBlock = body.proposalContext ? `
+
+## ALTRE PROPOSTE GIÀ COMPLETATE (CROSS-READING)
+
+I seguenti output sono proposte di altri agenti che hanno lavorato sullo stesso task. Leggile attentamente prima di rispondere.
+
+${sanitizeAgentOutput(body.proposalContext).slice(0, 12000)}
+
+### ISTRUZIONI DI DELIBERAZIONE (obbligatorie)
+1. **Identifica i punti in cui CONCORDI** con un agente precedente → marca con \`[✓ CONCORDA CON: NomeAgente]\` e cita il punto specifico.
+2. **Identifica i punti in cui DISSENTI** → marca con \`[✗ DISACCORDO CON: NomeAgente]\` e spiega il motivo con evidenze concrete (dai dati fetched, non opinioni).
+3. **NON ripetere ciò che hanno già detto bene** — aggiungi il TUO contributo specifico dalla TUA specializzazione.
+4. **Se i dati raccolti contraddicono un agente precedente**, segnalalo esplicitamente.` : '';
 
       const formatInstructions = `\n\nFORMATTING RULES (CRITICAL — your output will be rendered as HTML):
 - Use MARKDOWN TABLES with | pipes | for ALL tabular data. Example:
@@ -284,27 +508,97 @@ Select the optimal agent pipeline. Output ONLY the JSON.`;
       const userMessage = stepDef?.prompt || task;
 
       const useWebTools = WEB_TOOL_AGENTS.has(agent);
+      const isPrimaryResearch = !!body.isPrimaryResearch || !!stepDef?.isPrimaryResearch;
       let webDataBlock = '';
+      const fetchOutcomes = { ok: 0, failed: 0, attempted: 0, failures: [] };
 
-      // ALWAYS pre-fetch URLs found in the task — every agent needs real data
-      const taskUrls = [...(task || '').matchAll(/https?:\/\/[^\s,)]+/gi)].map(m => m[0]);
-      if (taskUrls.length > 0 && !webDataBlock) {
-        sse({ token: '[Fetching task URLs...]' });
+      // ── Pre-fetch URLs declared in the task ─────────────────────────
+      const taskUrls = extractUrlsFromText(task);
+      if (taskUrls.length > 0) {
+        sse({ token: `[Fetching ${taskUrls.length} URL(s) from task...]` });
         const urlFetches = await Promise.all(
-          taskUrls.slice(0, 3).map(async (url) => {
-            sse({ token: `[Fetching: ${url}]` });
-            return { url, content: await runFetchUrl(url) };
+          taskUrls.slice(0, 4).map(async (url) => {
+            sse({ token: `[GET ${url}]` });
+            const r = await runFetchUrl(url);
+            fetchOutcomes.attempted++;
+            if (r.ok) fetchOutcomes.ok++;
+            else { fetchOutcomes.failed++; fetchOutcomes.failures.push({ url: r.url, code: r.code, reason: r.reason }); }
+            return r;
           })
         );
-        const urlBlock = urlFetches.filter(f => f.content).map(f => `### Content from ${f.url}:\n${f.content}`).join('\n\n---\n\n');
-        if (urlBlock) {
-          webDataBlock = `\n\n## FETCHED WEBSITE CONTENT (THIS IS THE REAL DATA — base your analysis ONLY on this, do NOT invent):\n\n${urlBlock}`;
+
+        const okFetches = urlFetches.filter(f => f.ok);
+        const failedFetches = urlFetches.filter(f => !f.ok);
+
+        if (okFetches.length > 0) {
+          const okBlock = okFetches.map(f => {
+            const preamble = f.preamble ? `${f.preamble}\n\n` : '';
+            const content = (f.content || '').slice(0, 12_000);
+            return `### Content from ${f.url}\n${preamble}${content}`;
+          }).join('\n\n---\n\n');
+          webDataBlock = `
+
+## ✅ DATI REALI RACCOLTI DAL SITO (GROUND TRUTH — usa SOLO questi)
+
+Quanto segue è il contenuto effettivamente estratto dai siti specificati nel task.
+È OBBLIGATORIO basare l'analisi su questi dati e SOLO su questi. È VIETATO:
+- Inventare il settore, i prodotti, il target o il modello di business.
+- Aggiungere "esempi tipici del settore" o "best practice generiche" senza riferirsi ai dati sopra.
+- Scrivere "da verificare" o "supponiamo che" quando il dato è già presente nel blocco.
+
+${okBlock}`;
         }
+
+        // ── Automatic indirect-intel fallback ──
+        // Search the web for info ABOUT each failed domain (registries, news,
+        // Wikipedia, profiles). This gives the agent grounded data even when
+        // the primary site is down, instead of forcing it to invent a framework.
+        let recoveredAny = false;
+        if (failedFetches.length > 0) {
+          const failBlock = failedFetches.map(f => `- ${f.url}  →  ${f.code}: ${f.reason}`).join('\n');
+          webDataBlock += `\n\n## ⚠ FONTI PRIMARIE NON DISPONIBILI\n${failBlock}`;
+          sse({ data_unavailable: failedFetches.map(f => ({ url: f.url, code: f.code, reason: f.reason })) });
+
+          const indirectBlocks = [];
+          for (const f of failedFetches.slice(0, 3)) {
+            const intel = await gatherIndirectIntel(f.url, sse);
+            if (intel && intel.length > 0) {
+              recoveredAny = true;
+              const block = intel.map(i =>
+                `### ${i.title || '(senza titolo)'}\nURL: ${i.url}\nQuery: "${i.fromQuery}"\n${i.description ? `Descrizione: ${i.description}\n` : ''}\n${i.content}`,
+              ).join('\n\n---\n\n');
+              indirectBlocks.push(`#### Info indirette su ${f.url}:\n\n${block}`);
+            }
+          }
+
+          if (indirectBlocks.length > 0) {
+            webDataBlock += `\n\n## 🔎 FONTI INDIRETTE (raccolte automaticamente perché le fonti primarie sono down)\n\nIl sito principale non è raggiungibile, ma queste fonti pubbliche parlano dello stesso dominio/azienda. **Usa queste informazioni come ground truth** — sono dati reali, non inventati.\n\n${indirectBlocks.join('\n\n---\n\n')}`;
+            sse({ token: '[Fallback: recupero indiretto riuscito, procedo con i dati alternativi.]' });
+          } else {
+            webDataBlock += `\n\nNessuna fonte indiretta utile trovata sul web per i domini sopra. Per le sezioni che riguardano questi URL, scrivi esplicitamente "Fonte non raggiungibile — analisi non eseguibile". NON sostituire con framework generici, esempi tipici, o supposizioni basate sul nome del dominio.`;
+          }
+        }
+
+        // Fail-fast: only abort if primary research has NO data AT ALL —
+        // neither primary fetches nor indirect intel recovered anything.
+        if (isPrimaryResearch && okFetches.length === 0 && !recoveredAny && taskUrls.length > 0) {
+          clearInterval(keepalive);
+          sse({
+            aborted: true,
+            reason: 'PRIMARY_RESEARCH_FAILED',
+            message: 'Tutti gli URL del task sono irraggiungibili. Pipeline interrotta per evitare contenuto inventato.',
+            failures: fetchOutcomes.failures,
+          });
+          res.write('data: [DONE]\n\n');
+          res.end();
+          return;
+        }
+
         sse({ token: '\n' });
       }
 
-      // Pre-fetch web data BEFORE the LLM call so the model writes with real facts
-      if (useWebTools && !webDataBlock) {
+      // ── Web-search agents: search-driven data collection ────────────
+      if (useWebTools) {
         sse({ token: '[Raccolta dati web...]' });
 
         const queries = extractSearchQueries(task, stepDef?.prompt);
@@ -324,7 +618,11 @@ Select the optimal agent pipeline. Output ONLY the JSON.`;
         const fetchResults = await Promise.all(
           urlMatches.map(async (url) => {
             sse({ token: `[Fetching: ${url}]` });
-            return { url, content: await runFetchUrl(url) };
+            const r = await runFetchUrl(url);
+            fetchOutcomes.attempted++;
+            if (r.ok) fetchOutcomes.ok++;
+            else fetchOutcomes.failed++;
+            return r;
           })
         );
 
@@ -332,11 +630,19 @@ Select the optimal agent pipeline. Output ONLY the JSON.`;
           .map((s) => `### Search: "${s.query}"\n${s.result}`)
           .join('\n\n---\n\n');
 
-        const fetchBlock = fetchResults.length > 0
-          ? fetchResults.map((f) => `### Full content: ${f.url}\n${f.content}`).join('\n\n---\n\n')
+        const fetchBlock = fetchResults
+          .filter(f => f.ok)
+          .map(f => `### Full content: ${f.url}\n${f.preamble ? f.preamble + '\n\n' : ''}${(f.content || '').slice(0, 8_000)}`)
+          .join('\n\n---\n\n');
+
+        const failedFetches = fetchResults.filter(f => !f.ok);
+        const failBlock = failedFetches.length > 0
+          ? `\n\n## URL non raggiungibili durante la ricerca:\n${failedFetches.map(f => `- ${f.url}  →  ${f.code}`).join('\n')}`
           : '';
 
-        webDataBlock = `\n\n## REAL-TIME WEB DATA (use ONLY this data — do NOT invent prices or figures):\n\n${searchBlock}${fetchBlock ? '\n\n---\n\n' + fetchBlock : ''}`;
+        const collected = `${searchBlock}${fetchBlock ? '\n\n---\n\n' + fetchBlock : ''}${failBlock}`;
+        webDataBlock = (webDataBlock || '') +
+          `\n\n## REAL-TIME WEB DATA (use ONLY this data — do NOT invent prices, figures, brands, sectors):\n\n${collected}`;
 
         sse({ token: '\n' });
       }
@@ -355,7 +661,17 @@ Select the optimal agent pipeline. Output ONLY the JSON.`;
       }, { max_tokens: 16384 });
 
       clearInterval(keepalive);
-      sse({ done: true, output, tokensOut });
+      // Strip internal think/tool blocks before emitting the final output —
+      // the streamed tokens may contain them. Quality is computed on the clean
+      // text so think-blocks don't inflate output_length artificially.
+      const cleanOutput = sanitizeAgentOutput(output);
+      const dataQuality = computeDataQuality({
+        fetchOutcomes,
+        output: cleanOutput,
+        taskUrls,
+        useWebTools,
+      });
+      sse({ done: true, output: cleanOutput, tokensOut, dataQuality, fetchOutcomes });
       res.write('data: [DONE]\n\n');
       res.end();
     } catch (e) {
@@ -365,6 +681,30 @@ Select the optimal agent pipeline. Output ONLY the JSON.`;
     }
   });
 
+  // ── /api/studio/smoke-test — Pre-run reachability check ──────────────
+  router.post('/api/studio/smoke-test', async (req, res) => {
+    try {
+      const body = await parseBody(req);
+      let urls = Array.isArray(body.urls) ? body.urls : [];
+      if (!urls.length && typeof body.task === 'string') urls = extractUrlsFromText(body.task);
+      if (!urls.length) return sendJSON(res, 200, { probes: [], allOk: true });
+
+      const probes = await Promise.all(
+        urls.slice(0, 8).map(async (url) => {
+          try {
+            const r = await headProbe(url);
+            return { url, ok: r.ok, status: r.status, reason: r.reason || '', finalUrl: r.finalUrl };
+          } catch (e) {
+            return { url, ok: false, status: 0, reason: e.message || String(e) };
+          }
+        })
+      );
+
+      const allOk = probes.every(p => p.ok);
+      sendJSON(res, 200, { probes, allOk });
+    } catch (e) { sendError(res, 500, e.message); }
+  });
+
   // ── /api/studio/deliberate — Parliament Geth Consensus ───────────────
   router.post('/api/studio/deliberate', async (req, res) => {
     const body = await parseBody(req);
@@ -470,6 +810,66 @@ Select the optimal agent pipeline. Output ONLY the JSON.`;
   });
 }
 
+// ── Data-quality scoring (replaces fake sentiment "confidence") ────────
+/**
+ * Compute a 0..1 data-quality score from observable signals.
+ *
+ * Signals (weighted):
+ *   - fetch_success_ratio:  fraction of attempted fetches that succeeded   (0.45)
+ *   - declared_urls_ok:     primary URLs from the task were reachable      (0.25)
+ *   - output_length:        produced a substantive response (>= 600 chars) (0.10)
+ *   - no_error_strings:     output free of "fetch failed", "search failed" (0.10)
+ *   - structured_evidence:  output cites numbers, dates, or quoted strings (0.10)
+ *
+ * Returns { score: number, label: 'high'|'medium'|'low', signals: {...} }.
+ */
+export function computeDataQuality({ fetchOutcomes, output = '', taskUrls = [], useWebTools = false }) {
+  const out = String(output || '');
+  const signals = {};
+
+  const totalAttempted = fetchOutcomes?.attempted || 0;
+  const totalOk        = fetchOutcomes?.ok || 0;
+  if (totalAttempted > 0) {
+    signals.fetch_success_ratio = totalOk / totalAttempted;
+  } else if (useWebTools || taskUrls.length > 0) {
+    signals.fetch_success_ratio = 0;
+  } else {
+    signals.fetch_success_ratio = 1; // No fetches expected → neutral
+  }
+
+  const declaredFailures = (fetchOutcomes?.failures || []).filter(f => taskUrls.includes(f.url)).length;
+  if (taskUrls.length === 0) {
+    signals.declared_urls_ok = 1;
+  } else {
+    signals.declared_urls_ok = Math.max(0, 1 - declaredFailures / taskUrls.length);
+  }
+
+  signals.output_length = Math.min(1, out.length / 1_200);
+
+  const errorMarkers = /fetch failed|search failed|errore 500|errore 404|http_500|http_404|http_429|http_4\d{2}|timeout|network error/gi;
+  const errorCount = (out.match(errorMarkers) || []).length;
+  signals.no_error_strings = Math.max(0, 1 - errorCount / 5);
+
+  const hasNumbers  = /\b\d{1,4}(?:[.,]\d+)?(?:\s*(%|€|\$|£|¥|kg|cm|mm|km|°c))?/i.test(out);
+  const hasDates    = /\b(19|20)\d{2}\b|\b\d{1,2}[\/\-.]\d{1,2}[\/\-.](19|20)?\d{2}\b/.test(out);
+  const hasQuotes   = /"[^"]{8,}"|"[^"]{8,}"/.test(out);
+  const structScore = (hasNumbers ? 0.5 : 0) + (hasDates ? 0.25 : 0) + (hasQuotes ? 0.25 : 0);
+  signals.structured_evidence = Math.min(1, structScore);
+
+  const score =
+    0.45 * signals.fetch_success_ratio +
+    0.25 * signals.declared_urls_ok +
+    0.10 * signals.output_length +
+    0.10 * signals.no_error_strings +
+    0.10 * signals.structured_evidence;
+
+  let label = 'low';
+  if (score >= 0.70) label = 'high';
+  else if (score >= 0.50) label = 'medium';
+
+  return { score: Math.round(score * 100) / 100, label, signals };
+}
+
 // ── Keyword fallback planner (used when LLM is unavailable) ────────────
 
 function buildKeywordFallback(task, sanitizedTask, hasPdf, pdfName, it) {

package/src/services/tool-executor.mjs

@@ -2029,15 +2029,45 @@ export async function executeTool(action, params, config) {
       const url = params.url;
       if (!url) return 'A URL is required.';
 
-      const result = await wt.fetchUrl(url);
-      if (result.error) return `Fetch error: ${result.message}`;
+      // Use the rich variant so we get OpenGraph / JSON-LD / headings.
+      // The structured preamble dramatically reduces hallucination on sites
+      // where the body alone is ambiguous (e.g. SPA shells, B2B catalogs).
+      const result = await wt.fetchUrlRich(url);
+      if (result.error) {
+        return `Fetch error (${result.code || 'UNKNOWN'}): ${result.message}. ` +
+               `Try web_search to find an alternative source, or browser_open for JS-rendered pages.`;
+      }
 
+      const meta = result.metadata || {};
       const lines = [];
       if (result.title) lines.push(`Title: ${result.title}`);
       lines.push(`URL: ${result.url || url}`);
-      lines.push(`Status: ${result.status}`);
+      lines.push(`Status: ${result.status}  Bytes: ${result.bytes}  Attempts: ${result.attempts}`);
+      if (meta.description) lines.push(`Description: ${meta.description}`);
+      if (meta.lang) lines.push(`Lang: ${meta.lang}`);
+      if (meta.canonical) lines.push(`Canonical: ${meta.canonical}`);
+
+      const ogKeys = Object.keys(meta.og || {});
+      if (ogKeys.length) {
+        const ogPairs = ogKeys.slice(0, 8).map(k => `${k}=${String(meta.og[k]).slice(0, 240)}`);
+        lines.push(`OpenGraph: ${ogPairs.join(' | ')}`);
+      }
+
+      const ldTypes = (meta.jsonLd || []).map(j => j.type).filter(Boolean);
+      if (ldTypes.length) {
+        lines.push(`Schema.org types found: ${[...new Set(ldTypes)].join(', ')}`);
+        const ldNames = (meta.jsonLd || []).map(j => j.name).filter(Boolean).slice(0, 5);
+        if (ldNames.length) lines.push(`Schema.org entities: ${ldNames.join(' | ')}`);
+      }
+
+      if (meta.headings) {
+        if (meta.headings.h1?.length) lines.push(`H1: ${meta.headings.h1.slice(0, 3).join(' | ')}`);
+        if (meta.headings.h2?.length) lines.push(`H2: ${meta.headings.h2.slice(0, 6).join(' | ')}`);
+      }
+
       if (result.truncated) lines.push('[Content was truncated due to size limits]');
       lines.push('');
+      lines.push('--- MAIN CONTENT ---');
       lines.push(result.body);
 
       return lines.join('\n');
@@ -2047,13 +2077,37 @@ export async function executeTool(action, params, config) {
     case 'get_weather': {
       const location = (params.location || '').trim();
       if (!location) return 'A location is required (e.g. "Rome", "Viterbo, Italy").';
+      const encodedLoc = encodeURIComponent(location);
+      const WTTR_UA = 'Mozilla/5.0 (compatible; nha-weather/1.0; +https://nothumanallowed.com)';
+      const fetchWithRetry = async (attempts = 3) => {
+        let lastErr = null;
+        for (let i = 0; i < attempts; i++) {
+          try {
+            const r = await fetch(`https://wttr.in/${encodedLoc}?format=j1`, {
+              headers: { 'User-Agent': WTTR_UA, 'Accept': 'application/json' },
+              signal: AbortSignal.timeout(10_000),
+            });
+            if (r.ok) return r;
+            // 429/5xx → retry; 4xx other → don't retry
+            if (r.status === 429 || (r.status >= 500 && r.status < 600)) {
+              await new Promise(rs => setTimeout(rs, 600 * (i + 1) + Math.random() * 400));
+              continue;
+            }
+            return r;
+          } catch (e) {
+            lastErr = e;
+            if (i < attempts - 1) await new Promise(rs => setTimeout(rs, 600 * (i + 1)));
+          }
+        }
+        if (lastErr) throw lastErr;
+        return null;
+      };
       try {
-        const encodedLoc = encodeURIComponent(location);
-        const wttrRes = await fetch(`https://wttr.in/${encodedLoc}?format=j1`, {
-          headers: { 'User-Agent': 'nha-cli/1.0' },
-          signal: AbortSignal.timeout(8000),
-        });
-        if (!wttrRes.ok) return `Weather service returned ${wttrRes.status} for "${location}". Try a different location name.`;
+        const wttrRes = await fetchWithRetry(3);
+        if (!wttrRes || !wttrRes.ok) {
+          const status = wttrRes ? wttrRes.status : 'NET_ERR';
+          return `Weather service returned ${status} for "${location}". Try a more specific location (e.g. "Rome, Italy") or use web_search("weather ${location}") as fallback.`;
+        }
         const w = await wttrRes.json();
         const cur = w.current_condition?.[0];
         const area = w.nearest_area?.[0];