nothumanallowed 15.1.4 → 15.1.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nothumanallowed",
-  "version": "15.1.4",
+  "version": "15.1.5",
   "description": "NotHumanAllowed — 38 AI agents, 80 tools, Studio (visual agentic workflows). Email, calendar, browser automation, screen capture, canvas, cron/heartbeat, Alexandria E2E messaging, GitHub, Notion, Slack, voice chat, free AI (Liara), 28 languages. Zero-dependency CLI.",
   "type": "module",
   "bin": {

package/src/constants.mjs

@@ -5,7 +5,7 @@ import { fileURLToPath } from 'url';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
-export const VERSION = '15.0.12';
+export const VERSION = '15.1.5';
 export const BASE_URL = 'https://nothumanallowed.com/cli';
 export const API_BASE = 'https://nothumanallowed.com/api/v1';
 

package/src/server/routes/studio.mjs

@@ -9,7 +9,7 @@ import { sendJSON, sendError, parseBody } from '../index.mjs';
 import { loadConfig } from '../../config.mjs';
 import { NHA_DIR, AGENTS_DIR } from '../../constants.mjs';
 import { callLLM, callLLMStream, parseAgentFile } from '../../services/llm.mjs';
-import { webSearch, fetchUrl } from '../../services/web-tools.mjs';
+import { webSearch, fetchUrl, fetchUrlRich, headProbe } from '../../services/web-tools.mjs';
 
 // Agents that get web data pre-fetched before their LLM call
 const WEB_TOOL_AGENTS = new Set([
@@ -94,20 +94,65 @@ async function runWebSearch(query) {
 }
 
 /**
- * Fetch a URL and return formatted content string.
+ * Fetch a URL and return a structured result. Caller decides how to format it.
+ * Returns { ok, url, content?, title?, metadata?, code?, reason? }.
  */
 async function runFetchUrl(url) {
   try {
-    const result = await fetchUrl(url);
-    if (result.error) return `Fetch failed: ${result.message}`;
-    const titlePart = result.title ? `Title: ${result.title}\n\n` : '';
-    const text = (result.body || '').slice(0, 5000);
-    return `Content from ${url}:\n\n${titlePart}${text}`;
+    const result = await fetchUrlRich(url, { maxChars: 16_000 });
+    if (result.error) {
+      return { ok: false, url, code: result.code || 'FETCH_FAILED', reason: result.message || 'unknown' };
+    }
+    const meta = result.metadata || {};
+    const ogBlock = Object.keys(meta.og || {}).length
+      ? `OpenGraph: ${Object.entries(meta.og).slice(0, 8).map(([k, v]) => `${k}=${String(v).slice(0, 200)}`).join(' | ')}\n`
+      : '';
+    const ldBlock = (meta.jsonLd || []).length
+      ? `Schema.org types: ${meta.jsonLd.slice(0, 6).map(j => j.type).join(', ')}\n`
+      : '';
+    const headingsBlock = meta.headings && (meta.headings.h1?.length || meta.headings.h2?.length)
+      ? `Headings H1: ${(meta.headings.h1 || []).slice(0, 5).join(' | ')}\n` +
+        (meta.headings.h2?.length ? `Headings H2: ${meta.headings.h2.slice(0, 8).join(' | ')}\n` : '')
+      : '';
+    const descBlock = meta.description ? `Description: ${meta.description}\n` : '';
+    return {
+      ok: true,
+      url: result.url || url,
+      title: result.title || '',
+      content: result.body || '',
+      metadata: meta,
+      preamble: `${result.title ? `Title: ${result.title}\n` : ''}${descBlock}${ogBlock}${ldBlock}${headingsBlock}`.trim(),
+    };
   } catch (e) {
-    return `Fetch error: ${e.message}`;
+    return { ok: false, url, code: 'EXCEPTION', reason: e.message || String(e) };
   }
 }
 
+/** Extract HTTP/HTTPS URLs from a free-form string, deduplicated, max 6. */
+function extractUrlsFromText(text) {
+  const urls = [...(text || '').matchAll(/https?:\/\/[^\s,)<>"'`]+/gi)].map(m => m[0].replace(/[.,;:!?)\]]+$/, ''));
+  return [...new Set(urls)].slice(0, 6);
+}
+
+/**
+ * Strip transient status markers, internal think/tool blocks, and propagation
+ * noise before the text is injected into another agent's prompt or surfaced
+ * to the client. Parity with LegionX `CommunicationStream.stripTags`.
+ */
+function sanitizeAgentOutput(text) {
+  if (!text) return '';
+  return String(text)
+    .replace(/<think>[\s\S]*?<\/think>/gi, '')
+    .replace(/<thinking>[\s\S]*?<\/thinking>/gi, '')
+    .replace(/<tool[^>]*>[\s\S]*?<\/tool>/gi, '')
+    .replace(/<scratch[^>]*>[\s\S]*?<\/scratch>/gi, '')
+    .replace(/<reasoning>[\s\S]*?<\/reasoning>/gi, '')
+    .replace(/\[(?:Searching|Fetching|GET|POST|HEAD|Raccolta dati web|Parlamento)[^\]]*\]\s*/gi, '')
+    .replace(/​|/g, '') // zero-width / BOM
+    .replace(/\r\n/g, '\n')
+    .trim();
+}
+
 /**
  * Extract search queries from a task string.
  * Returns up to 3 queries covering different angles.
@@ -257,8 +302,24 @@ Select the optimal agent pipeline. Output ONLY the JSON.`;
         } catch {}
       }
 
-      const contextBlock = context ? `\n\n## CONTEXT FROM PREVIOUS STEPS:\n${context.slice(0, 12000)}` : '';
-      const proposalContextBlock = body.proposalContext ? `\n\n## OTHER AGENTS' PROPOSALS (CROSS-READING):\n${body.proposalContext.slice(0, 8000)}` : '';
+      const contextBlock = context ? `\n\n## CONTEXT FROM PREVIOUS STEPS:\n${sanitizeAgentOutput(context).slice(0, 12000)}` : '';
+
+      // Cross-reading block — when other agents have already produced proposals,
+      // we inject them with explicit deliberation instructions so the model
+      // engages with them (agree/disagree) rather than restating its own view.
+      const proposalContextBlock = body.proposalContext ? `
+
+## ALTRE PROPOSTE GIÀ COMPLETATE (CROSS-READING)
+
+I seguenti output sono proposte di altri agenti che hanno lavorato sullo stesso task. Leggile attentamente prima di rispondere.
+
+${sanitizeAgentOutput(body.proposalContext).slice(0, 12000)}
+
+### ISTRUZIONI DI DELIBERAZIONE (obbligatorie)
+1. **Identifica i punti in cui CONCORDI** con un agente precedente → marca con \`[✓ CONCORDA CON: NomeAgente]\` e cita il punto specifico.
+2. **Identifica i punti in cui DISSENTI** → marca con \`[✗ DISACCORDO CON: NomeAgente]\` e spiega il motivo con evidenze concrete (dai dati fetched, non opinioni).
+3. **NON ripetere ciò che hanno già detto bene** — aggiungi il TUO contributo specifico dalla TUA specializzazione.
+4. **Se i dati raccolti contraddicono un agente precedente**, segnalalo esplicitamente.` : '';
 
       const formatInstructions = `\n\nFORMATTING RULES (CRITICAL — your output will be rendered as HTML):
 - Use MARKDOWN TABLES with | pipes | for ALL tabular data. Example:
@@ -284,27 +345,62 @@ Select the optimal agent pipeline. Output ONLY the JSON.`;
       const userMessage = stepDef?.prompt || task;
 
       const useWebTools = WEB_TOOL_AGENTS.has(agent);
+      const isPrimaryResearch = !!body.isPrimaryResearch || !!stepDef?.isPrimaryResearch;
       let webDataBlock = '';
+      const fetchOutcomes = { ok: 0, failed: 0, attempted: 0, failures: [] };
 
-      // ALWAYS pre-fetch URLs found in the task — every agent needs real data
-      const taskUrls = [...(task || '').matchAll(/https?:\/\/[^\s,)]+/gi)].map(m => m[0]);
-      if (taskUrls.length > 0 && !webDataBlock) {
-        sse({ token: '[Fetching task URLs...]' });
+      // ── Pre-fetch URLs declared in the task ─────────────────────────
+      const taskUrls = extractUrlsFromText(task);
+      if (taskUrls.length > 0) {
+        sse({ token: `[Fetching ${taskUrls.length} URL(s) from task...]` });
         const urlFetches = await Promise.all(
-          taskUrls.slice(0, 3).map(async (url) => {
-            sse({ token: `[Fetching: ${url}]` });
-            return { url, content: await runFetchUrl(url) };
+          taskUrls.slice(0, 4).map(async (url) => {
+            sse({ token: `[GET ${url}]` });
+            const r = await runFetchUrl(url);
+            fetchOutcomes.attempted++;
+            if (r.ok) fetchOutcomes.ok++;
+            else { fetchOutcomes.failed++; fetchOutcomes.failures.push({ url: r.url, code: r.code, reason: r.reason }); }
+            return r;
           })
         );
-        const urlBlock = urlFetches.filter(f => f.content).map(f => `### Content from ${f.url}:\n${f.content}`).join('\n\n---\n\n');
-        if (urlBlock) {
-          webDataBlock = `\n\n## FETCHED WEBSITE CONTENT (THIS IS THE REAL DATA — base your analysis ONLY on this, do NOT invent):\n\n${urlBlock}`;
+
+        const okFetches = urlFetches.filter(f => f.ok);
+        const failedFetches = urlFetches.filter(f => !f.ok);
+
+        if (okFetches.length > 0) {
+          const okBlock = okFetches.map(f => {
+            const preamble = f.preamble ? `${f.preamble}\n\n` : '';
+            const content = (f.content || '').slice(0, 12_000);
+            return `### Content from ${f.url}\n${preamble}${content}`;
+          }).join('\n\n---\n\n');
+          webDataBlock = `\n\n## FETCHED WEBSITE CONTENT (THIS IS THE GROUND TRUTH — base your analysis EXCLUSIVELY on this. Do NOT invent products, sectors, brands, or facts that are not in this block.):\n\n${okBlock}`;
         }
+
+        if (failedFetches.length > 0) {
+          const failBlock = failedFetches.map(f => `- ${f.url}  →  ${f.code}: ${f.reason}`).join('\n');
+          webDataBlock += `\n\n## ❌ URL NON RAGGIUNGIBILI (NON inventare contenuto su questi siti — scrivi esplicitamente "fonte non disponibile" nelle sezioni che li riguardano):\n${failBlock}`;
+          sse({ data_unavailable: failedFetches.map(f => ({ url: f.url, code: f.code, reason: f.reason })) });
+        }
+
+        // Fail-fast: primary research agent and ALL declared URLs failed → abort.
+        if (isPrimaryResearch && okFetches.length === 0 && taskUrls.length > 0) {
+          clearInterval(keepalive);
+          sse({
+            aborted: true,
+            reason: 'PRIMARY_RESEARCH_FAILED',
+            message: 'Tutti gli URL del task sono irraggiungibili. Pipeline interrotta per evitare contenuto inventato.',
+            failures: fetchOutcomes.failures,
+          });
+          res.write('data: [DONE]\n\n');
+          res.end();
+          return;
+        }
+
         sse({ token: '\n' });
       }
 
-      // Pre-fetch web data BEFORE the LLM call so the model writes with real facts
-      if (useWebTools && !webDataBlock) {
+      // ── Web-search agents: search-driven data collection ────────────
+      if (useWebTools) {
         sse({ token: '[Raccolta dati web...]' });
 
         const queries = extractSearchQueries(task, stepDef?.prompt);
@@ -324,7 +420,11 @@ Select the optimal agent pipeline. Output ONLY the JSON.`;
         const fetchResults = await Promise.all(
           urlMatches.map(async (url) => {
             sse({ token: `[Fetching: ${url}]` });
-            return { url, content: await runFetchUrl(url) };
+            const r = await runFetchUrl(url);
+            fetchOutcomes.attempted++;
+            if (r.ok) fetchOutcomes.ok++;
+            else fetchOutcomes.failed++;
+            return r;
           })
         );
 
@@ -332,11 +432,19 @@ Select the optimal agent pipeline. Output ONLY the JSON.`;
           .map((s) => `### Search: "${s.query}"\n${s.result}`)
           .join('\n\n---\n\n');
 
-        const fetchBlock = fetchResults.length > 0
-          ? fetchResults.map((f) => `### Full content: ${f.url}\n${f.content}`).join('\n\n---\n\n')
+        const fetchBlock = fetchResults
+          .filter(f => f.ok)
+          .map(f => `### Full content: ${f.url}\n${f.preamble ? f.preamble + '\n\n' : ''}${(f.content || '').slice(0, 8_000)}`)
+          .join('\n\n---\n\n');
+
+        const failedFetches = fetchResults.filter(f => !f.ok);
+        const failBlock = failedFetches.length > 0
+          ? `\n\n## URL non raggiungibili durante la ricerca:\n${failedFetches.map(f => `- ${f.url}  →  ${f.code}`).join('\n')}`
           : '';
 
-        webDataBlock = `\n\n## REAL-TIME WEB DATA (use ONLY this data — do NOT invent prices or figures):\n\n${searchBlock}${fetchBlock ? '\n\n---\n\n' + fetchBlock : ''}`;
+        const collected = `${searchBlock}${fetchBlock ? '\n\n---\n\n' + fetchBlock : ''}${failBlock}`;
+        webDataBlock = (webDataBlock || '') +
+          `\n\n## REAL-TIME WEB DATA (use ONLY this data — do NOT invent prices, figures, brands, sectors):\n\n${collected}`;
 
         sse({ token: '\n' });
       }
@@ -355,7 +463,17 @@ Select the optimal agent pipeline. Output ONLY the JSON.`;
       }, { max_tokens: 16384 });
 
       clearInterval(keepalive);
-      sse({ done: true, output, tokensOut });
+      // Strip internal think/tool blocks before emitting the final output —
+      // the streamed tokens may contain them. Quality is computed on the clean
+      // text so think-blocks don't inflate output_length artificially.
+      const cleanOutput = sanitizeAgentOutput(output);
+      const dataQuality = computeDataQuality({
+        fetchOutcomes,
+        output: cleanOutput,
+        taskUrls,
+        useWebTools,
+      });
+      sse({ done: true, output: cleanOutput, tokensOut, dataQuality, fetchOutcomes });
       res.write('data: [DONE]\n\n');
       res.end();
     } catch (e) {
@@ -365,6 +483,30 @@ Select the optimal agent pipeline. Output ONLY the JSON.`;
     }
   });
 
+  // ── /api/studio/smoke-test — Pre-run reachability check ──────────────
+  router.post('/api/studio/smoke-test', async (req, res) => {
+    try {
+      const body = await parseBody(req);
+      let urls = Array.isArray(body.urls) ? body.urls : [];
+      if (!urls.length && typeof body.task === 'string') urls = extractUrlsFromText(body.task);
+      if (!urls.length) return sendJSON(res, 200, { probes: [], allOk: true });
+
+      const probes = await Promise.all(
+        urls.slice(0, 8).map(async (url) => {
+          try {
+            const r = await headProbe(url);
+            return { url, ok: r.ok, status: r.status, reason: r.reason || '', finalUrl: r.finalUrl };
+          } catch (e) {
+            return { url, ok: false, status: 0, reason: e.message || String(e) };
+          }
+        })
+      );
+
+      const allOk = probes.every(p => p.ok);
+      sendJSON(res, 200, { probes, allOk });
+    } catch (e) { sendError(res, 500, e.message); }
+  });
+
   // ── /api/studio/deliberate — Parliament Geth Consensus ───────────────
   router.post('/api/studio/deliberate', async (req, res) => {
     const body = await parseBody(req);
@@ -470,6 +612,66 @@ Select the optimal agent pipeline. Output ONLY the JSON.`;
   });
 }
 
+// ── Data-quality scoring (replaces fake sentiment "confidence") ────────
+/**
+ * Compute a 0..1 data-quality score from observable signals.
+ *
+ * Signals (weighted):
+ *   - fetch_success_ratio:  fraction of attempted fetches that succeeded   (0.45)
+ *   - declared_urls_ok:     primary URLs from the task were reachable      (0.25)
+ *   - output_length:        produced a substantive response (>= 600 chars) (0.10)
+ *   - no_error_strings:     output free of "fetch failed", "search failed" (0.10)
+ *   - structured_evidence:  output cites numbers, dates, or quoted strings (0.10)
+ *
+ * Returns { score: number, label: 'high'|'medium'|'low', signals: {...} }.
+ */
+export function computeDataQuality({ fetchOutcomes, output = '', taskUrls = [], useWebTools = false }) {
+  const out = String(output || '');
+  const signals = {};
+
+  const totalAttempted = fetchOutcomes?.attempted || 0;
+  const totalOk        = fetchOutcomes?.ok || 0;
+  if (totalAttempted > 0) {
+    signals.fetch_success_ratio = totalOk / totalAttempted;
+  } else if (useWebTools || taskUrls.length > 0) {
+    signals.fetch_success_ratio = 0;
+  } else {
+    signals.fetch_success_ratio = 1; // No fetches expected → neutral
+  }
+
+  const declaredFailures = (fetchOutcomes?.failures || []).filter(f => taskUrls.includes(f.url)).length;
+  if (taskUrls.length === 0) {
+    signals.declared_urls_ok = 1;
+  } else {
+    signals.declared_urls_ok = Math.max(0, 1 - declaredFailures / taskUrls.length);
+  }
+
+  signals.output_length = Math.min(1, out.length / 1_200);
+
+  const errorMarkers = /fetch failed|search failed|errore 500|errore 404|http_500|http_404|http_429|http_4\d{2}|timeout|network error/gi;
+  const errorCount = (out.match(errorMarkers) || []).length;
+  signals.no_error_strings = Math.max(0, 1 - errorCount / 5);
+
+  const hasNumbers  = /\b\d{1,4}(?:[.,]\d+)?(?:\s*(%|€|\$|£|¥|kg|cm|mm|km|°c))?/i.test(out);
+  const hasDates    = /\b(19|20)\d{2}\b|\b\d{1,2}[\/\-.]\d{1,2}[\/\-.](19|20)?\d{2}\b/.test(out);
+  const hasQuotes   = /"[^"]{8,}"|"[^"]{8,}"/.test(out);
+  const structScore = (hasNumbers ? 0.5 : 0) + (hasDates ? 0.25 : 0) + (hasQuotes ? 0.25 : 0);
+  signals.structured_evidence = Math.min(1, structScore);
+
+  const score =
+    0.45 * signals.fetch_success_ratio +
+    0.25 * signals.declared_urls_ok +
+    0.10 * signals.output_length +
+    0.10 * signals.no_error_strings +
+    0.10 * signals.structured_evidence;
+
+  let label = 'low';
+  if (score >= 0.70) label = 'high';
+  else if (score >= 0.50) label = 'medium';
+
+  return { score: Math.round(score * 100) / 100, label, signals };
+}
+
 // ── Keyword fallback planner (used when LLM is unavailable) ────────────
 
 function buildKeywordFallback(task, sanitizedTask, hasPdf, pdfName, it) {

package/src/services/tool-executor.mjs

@@ -2029,15 +2029,45 @@ export async function executeTool(action, params, config) {
       const url = params.url;
       if (!url) return 'A URL is required.';
 
-      const result = await wt.fetchUrl(url);
-      if (result.error) return `Fetch error: ${result.message}`;
+      // Use the rich variant so we get OpenGraph / JSON-LD / headings.
+      // The structured preamble dramatically reduces hallucination on sites
+      // where the body alone is ambiguous (e.g. SPA shells, B2B catalogs).
+      const result = await wt.fetchUrlRich(url);
+      if (result.error) {
+        return `Fetch error (${result.code || 'UNKNOWN'}): ${result.message}. ` +
+               `Try web_search to find an alternative source, or browser_open for JS-rendered pages.`;
+      }
 
+      const meta = result.metadata || {};
       const lines = [];
       if (result.title) lines.push(`Title: ${result.title}`);
       lines.push(`URL: ${result.url || url}`);
-      lines.push(`Status: ${result.status}`);
+      lines.push(`Status: ${result.status}  Bytes: ${result.bytes}  Attempts: ${result.attempts}`);
+      if (meta.description) lines.push(`Description: ${meta.description}`);
+      if (meta.lang) lines.push(`Lang: ${meta.lang}`);
+      if (meta.canonical) lines.push(`Canonical: ${meta.canonical}`);
+
+      const ogKeys = Object.keys(meta.og || {});
+      if (ogKeys.length) {
+        const ogPairs = ogKeys.slice(0, 8).map(k => `${k}=${String(meta.og[k]).slice(0, 240)}`);
+        lines.push(`OpenGraph: ${ogPairs.join(' | ')}`);
+      }
+
+      const ldTypes = (meta.jsonLd || []).map(j => j.type).filter(Boolean);
+      if (ldTypes.length) {
+        lines.push(`Schema.org types found: ${[...new Set(ldTypes)].join(', ')}`);
+        const ldNames = (meta.jsonLd || []).map(j => j.name).filter(Boolean).slice(0, 5);
+        if (ldNames.length) lines.push(`Schema.org entities: ${ldNames.join(' | ')}`);
+      }
+
+      if (meta.headings) {
+        if (meta.headings.h1?.length) lines.push(`H1: ${meta.headings.h1.slice(0, 3).join(' | ')}`);
+        if (meta.headings.h2?.length) lines.push(`H2: ${meta.headings.h2.slice(0, 6).join(' | ')}`);
+      }
+
       if (result.truncated) lines.push('[Content was truncated due to size limits]');
       lines.push('');
+      lines.push('--- MAIN CONTENT ---');
       lines.push(result.body);
 
       return lines.join('\n');
@@ -2047,13 +2077,37 @@ export async function executeTool(action, params, config) {
     case 'get_weather': {
       const location = (params.location || '').trim();
       if (!location) return 'A location is required (e.g. "Rome", "Viterbo, Italy").';
+      const encodedLoc = encodeURIComponent(location);
+      const WTTR_UA = 'Mozilla/5.0 (compatible; nha-weather/1.0; +https://nothumanallowed.com)';
+      const fetchWithRetry = async (attempts = 3) => {
+        let lastErr = null;
+        for (let i = 0; i < attempts; i++) {
+          try {
+            const r = await fetch(`https://wttr.in/${encodedLoc}?format=j1`, {
+              headers: { 'User-Agent': WTTR_UA, 'Accept': 'application/json' },
+              signal: AbortSignal.timeout(10_000),
+            });
+            if (r.ok) return r;
+            // 429/5xx → retry; 4xx other → don't retry
+            if (r.status === 429 || (r.status >= 500 && r.status < 600)) {
+              await new Promise(rs => setTimeout(rs, 600 * (i + 1) + Math.random() * 400));
+              continue;
+            }
+            return r;
+          } catch (e) {
+            lastErr = e;
+            if (i < attempts - 1) await new Promise(rs => setTimeout(rs, 600 * (i + 1)));
+          }
+        }
+        if (lastErr) throw lastErr;
+        return null;
+      };
       try {
-        const encodedLoc = encodeURIComponent(location);
-        const wttrRes = await fetch(`https://wttr.in/${encodedLoc}?format=j1`, {
-          headers: { 'User-Agent': 'nha-cli/1.0' },
-          signal: AbortSignal.timeout(8000),
-        });
-        if (!wttrRes.ok) return `Weather service returned ${wttrRes.status} for "${location}". Try a different location name.`;
+        const wttrRes = await fetchWithRetry(3);
+        if (!wttrRes || !wttrRes.ok) {
+          const status = wttrRes ? wttrRes.status : 'NET_ERR';
+          return `Weather service returned ${status} for "${location}". Try a more specific location (e.g. "Rome, Italy") or use web_search("weather ${location}") as fallback.`;
+        }
         const w = await wttrRes.json();
         const cur = w.current_condition?.[0];
         const area = w.nearest_area?.[0];