@vierratale/ai 0.1.0-beta.10 → 0.1.0-beta.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vierratale/ai",
-  "version": "0.1.0-beta.10",
+  "version": "0.1.0-beta.12",
   "description": "VierrataleAI - Intelligent terminal assistant",
   "type": "module",
   "files": [

package/src/cli.js

@@ -18,6 +18,7 @@ import { Session } from './session.js';
 import { ChatUI, FrameThrottle } from './ui/chatbox.js';
 import { logger } from './utils/logger.js';
 import { runWithTools, TOOL_RESULTS_PROMPT, looksLikeOperationRequest } from './cmd/agent.js';
+import { loadTodos, addTodo, updateTodo, clearTodos, saveTodos, formatTodos, todosFile } from './cmd/todos.js';
 
 function parseArgs(args) {
   const parsed = { provider: null, model: null, clear: false, version: false, help: false, install: false };
@@ -54,6 +55,7 @@ Commands (in chat):
   /search <query>         Search the web and summarize with AI
   /fetch <url>            Open a link and summarize its content
   /download <url>         Download a file, folder listing or archive
+  /todos                  List the todo list (add <text> | done/undone <n> | clear)
   /log [n|path]           Show the last n log entries (default 30), or the log file path
   /clear                  Clear screen
   /new                    Start a new empty session (keeps old ones)
@@ -79,6 +81,7 @@ export const SLASH_COMMANDS = [
   { name: 'fetch', args: '<url>', desc: 'Open a link and summarize' },
   { name: 'download', args: '<url>', desc: 'Download a file / folder / archive' },
   { name: 'system', desc: 'Show or set the system prompt' },
+  { name: 'todos', args: '[add <text> | done <n> | undone <n> | clear]', desc: 'Manage the todo list' },
   { name: 'sessions', desc: 'List saved sessions' },
   { name: 'session', args: 'new | open | delete | backup | restore', desc: 'Manage sessions' },
   { name: 'log', args: '[n | path]', desc: 'Show recent log entries' },
@@ -656,7 +659,7 @@ async function chat(provider, systemPrompt) {
     if (cmd === '/help') {
       chatUI.clearNotices();
       chatUI.notify(
-        `Commands:\n  /search <query> · /fetch <url> · /model [name] · /provider [name]\n  /models · /log [n|path] · /new · /sessions · /session new|open|delete|backup|restore\n  /clear · /help · /quit`
+        `Commands:\n  /search <query> · /fetch <url> · /model [name] · /provider [name]\n  /models · /todos · /log [n|path] · /new · /sessions · /session new|open|delete|backup|restore\n  /clear · /help · /quit`
       );
       chatUI.render(messages);
       continue;
@@ -977,6 +980,11 @@ async function chat(provider, systemPrompt) {
       continue;
     }
 
+    if (cmd === '/todos' || cmd.startsWith('/todos ')) {
+      handleTodosCommand(trimmed, chatUI, messages, process.cwd());
+      continue;
+    }
+
     const intent = detectIntent(trimmed);
     if (intent.type === 'self') {
       const model = Config.getEffectiveModel(provider.name);
@@ -1125,6 +1133,49 @@ async function ensureRawLocalModel(realModel, notify) {
 export { looksLikeFileRequest, requestedFileName, extractFallbackFile, storyFileName, responseWantsFile, responseIsSoloCode, languageFileName, fallbackHasUsefulCode, validateWrittenCode, looksLikeJunkCode, warnIfJunkCode, writeFilesFromResponse }; 
 // Command/tool layer: asks the model to plan filesystem/shell operations,
 // executes them with the sandboxed executor, then streams the final answer.
+function handleTodosCommand(trimmed, chatUI, messages, cwd) {
+  chatUI.clearNotices();
+  const rest = trimmed.slice('/todos'.length).trim();
+  let out;
+  if (!rest) {
+    out = `Todos (${todosFile(cwd)}):\n${formatTodos(loadTodos(cwd))}`;
+  } else if (/^add\s+/i.test(rest)) {
+    const err = addTodo(cwd, rest.replace(/^add\s+/i, '').trim());
+    out = err && err.error ? err.error : `Todos (${todosFile(cwd)}):\n${formatTodos(loadTodos(cwd))}`;
+  } else if (/^done\s+\d+$/i.test(rest)) {
+    const err = updateTodo(cwd, parseInt(rest.replace(/^done\s+/i, ''), 10), { done: true });
+    out = err && err.error ? err.error : `Todos (${todosFile(cwd)}):\n${formatTodos(loadTodos(cwd))}`;
+  } else if (/^undone\s+\d+$/i.test(rest)) {
+    const err = updateTodo(cwd, parseInt(rest.replace(/^undone\s+/i, ''), 10), { done: false });
+    out = err && err.error ? err.error : `Todos (${todosFile(cwd)}):\n${formatTodos(loadTodos(cwd))}`;
+  } else if (/^remove\s+\d+$/i.test(rest)) {
+    const n = parseInt(rest.replace(/^remove\s+/i, ''), 10);
+    const todos = loadTodos(cwd);
+    if (!Number.isInteger(n - 1) || n - 1 < 0 || n - 1 >= todos.length) {
+      out = `No todo #${n}.`;
+    } else {
+      const next = todos.filter((_, i) => i !== n - 1);
+      saveTodos(cwd, next);
+      out = `Todos (${todosFile(cwd)}):\n${formatTodos(loadTodos(cwd))}`;
+    }
+  } else if (/^clear$/i.test(rest)) {
+    clearTodos(cwd);
+    out = `Todos cleared (${todosFile(cwd)}). (no todos yet)`;
+  } else if (/^\d+$/.test(rest)) {
+    const n = parseInt(rest, 10);
+    const todos = loadTodos(cwd);
+    const err = (Number.isInteger(n - 1) && n - 1 >= 0 && n - 1 < todos.length)
+      ? updateTodo(cwd, n, { done: !todos[n - 1].done })
+      : { error: `No todo #${n}.` };
+    out = err && err.error ? err.error : `Todos (${todosFile(cwd)}):\n${formatTodos(loadTodos(cwd))}`;
+  } else {
+    out = 'Usage: /todos [add <text> | done <n> | undone <n> | remove <n> | <n> (toggle) | clear]';
+  }
+  chatUI.notify(out);
+  logger.log('CMD', `todos: ${rest || '(list)'}`);
+  chatUI.render(messages);
+}
+
 async function answerWithTools(messages, provider, requestText, systemPrompt, confirm, chatUI, cwd, timeoutMs) {
   let results = [];
   try {

package/src/cmd/agent.js

@@ -19,6 +19,10 @@ Reply with EXACTLY ONE JSON array of the tool calls needed to do the job, in ord
 {"name":"list_directory","path":"."}
 {"name":"delete_file","path":"old.txt"}
 {"name":"download_url","url":"https://example.com/file.zip","dir":"downloads"}
+{"name":"todo_add","text":"install dependencies"}
+{"name":"todo_list"}
+{"name":"todo_update","index":1,"done":true}
+{"name":"todo_clear"}
 
 Rules:
 - run_command runs WITHOUT a shell. Allowed commands: mkdir touch ls pwd cat cp mv rm npm npx node python python3 git echo printf.
@@ -27,6 +31,7 @@ Rules:
 - Prefer create_file over redirecting output.
 - If you need to create a folder first, plan create_directory before create_file inside it.
 - To download a file, web asset, archive or folder listing from the web, use download_url with the full URL. dir is optional (defaults to ~/Downloads).
+- For multi-step work, track your progress with todo_add/todo_list/todo_update (e.g. todo_add for each remaining subtask, todo_update to mark steps done).
 - Code you create must be runnable. For Python servers bind to 127.0.0.1, read PORT from the environment (default 8000), and fall back to a free port (port 0) when the default is taken. HTML/CSS/JS must be self-contained and reference each other by relative filename.
 - If NO tools are needed, reply exactly: NONE
 - Output the JSON array only — no prose, no markdown fences unless the array is inside them.`;
@@ -37,6 +42,20 @@ ${resultsJson}
 
 Give the user a short, natural reply (in the user's own language) telling them what happened. Keep the whole reply under 40 words. Do NOT tell them to run the commands yourself. If they asked for a file containing code and files were created with content, you may also emit FILE: blocks if more files are still required.`;
 
+// Auto error fixing: shown to the model when a tool call fails so it can emit
+// a corrected plan instead of giving up. Bounded by MAX_AUTO_FIXES per run.
+const MAX_AUTO_FIXES = 3;
+
+const TOOL_FIX_PROMPT = (call, result) => `A tool call made while working in the workspace FAILED. Fix the problem so the original request can complete.
+
+Failed tool call:
+${JSON.stringify(call, null, 2)}
+
+Failure:
+${JSON.stringify({ success: false, exitCode: result.exitCode, stdout: result.stdout, stderr: result.stderr }, null, 2)}
+
+Emit a NEW tool plan (a JSON array of tool calls) that corrects the failure — for example re-create the file with the error fixed, run a command to repair it, or download/read what is missing. If your previous plan skipped a required step (like creating a parent folder), include it. Use todo tools to track remaining subtasks if useful. Reply with exactly one JSON array, or the single word NONE if no tool call can fix this.`;
+
 // ---------------------------------------------------------------------------
 // Request detection
 // ---------------------------------------------------------------------------
@@ -84,6 +103,18 @@ export function normalizeToolCall(raw) {
   } else if (name === 'download_url') {
     if (typeof raw.url !== 'string' || raw.url.trim() === '') return null;
     params = { url: raw.url, dir: typeof raw.dir === 'string' && raw.dir.trim() ? raw.dir.trim() : undefined };
+  } else if (name === 'todo_add') {
+    if (typeof raw.text !== 'string' || raw.text.trim() === '') return null;
+    params = { text: raw.text };
+  } else if (name === 'todo_update') {
+    if (!Number.isInteger(raw.index) && !/^\d+$/.test(String(raw.index ?? ''))) return null;
+    params = {
+      index: Number(raw.index),
+      done: typeof raw.done === 'boolean' ? raw.done : undefined,
+      text: typeof raw.text === 'string' && raw.text.trim() ? raw.text : undefined,
+    };
+  } else if (name === 'todo_list' || name === 'todo_clear') {
+    params = {};
   } else {
     return null;
   }
@@ -1631,6 +1662,75 @@ export async function planOnce(provider, messages, requestText, options = {}) {
   return heuristic;
 }
 
+async function executeToolCall({ executor, call, chatUI, messages, cwd }) {
+  const label = call.name === 'run_command' ? call.params.command
+    : call.name === 'download_url' ? `download_url(${call.params.url})`
+    : (call.name === 'todo_add' || call.name === 'todo_update')
+      ? `${call.name}(${JSON.stringify(call.params ?? {})})`
+      : `${call.name}(${call.params.path})`;
+  if (chatUI) {
+    chatUI.setStatus(`Running: ${label}`);
+    chatUI.setThinking(true);
+    chatUI.render(messages);
+  }
+  const result = await dispatchTool(executor, call);
+  logger.log('EXEC', `${label} -> ${result.exitCode}${result.timedOut ? ' [timeout]' : ''}`);
+  if (result.success) {
+    if (chatUI) chatUI.notify(`\u2713 ${label}`);
+    if (result.stdout.trim()) logger.log('EXEC', `${label} stdout: ${result.stdout.trim().slice(0, 500)}`);
+    if (result.stderr.trim()) logger.log('WARN', `${label} stderr: ${result.stderr.trim().slice(0, 500)}`);
+    if (call.name === 'create_file' && /\.py$/i.test(call.params.path)) {
+      const check = spawnSync('python3', [
+        '-c', 'import ast,sys; ast.parse(open(sys.argv[1], encoding="utf-8").read())',
+        join(cwd || '.', call.params.path),
+      ], { encoding: 'utf-8', timeout: 30000 });
+      if (check.status === 1) {
+        const detail = (check.stderr || '').trim().split('\n').slice(-2).join(' ');
+        if (chatUI) chatUI.notify(`\u2717 python check failed: ${call.params.path} (${detail})`);
+        logger.log('WARN', `${label} failed python syntax check: ${detail}`);
+      }
+    }
+  } else {
+    if (chatUI) chatUI.notify(`\u2717 ${label}: ${(result.stderr || 'failed').trim().slice(0, 300)}`);
+    logger.log('ERROR', `${label} failed: ${(result.stderr || result.stdout || 'exit ' + result.exitCode).trim().slice(0, 500)}`);
+  }
+  return { label, result };
+}
+
+// Ask the model for a corrected tool plan after a failed call. Returns []
+// when the model emits nothing usable. Does NOT mutate the session messages.
+async function planAutoFix(provider, messages, requestText, call, result) {
+  const history = (messages && messages.length ? messages.slice(-6) : [{ role: 'user', content: String(requestText || '') }]);
+  const deadline = Config.get('planTimeoutMs');
+  const controller = new AbortController();
+  let text = '';
+  let timer;
+  try {
+    text = await Promise.race([
+      provider.complete([...history, { role: 'user', content: TOOL_FIX_PROMPT(call, result) }], {
+        model: Config.getEffectiveModel(provider.name),
+        system_prompt: TOOL_PLAN_PROMPT,
+        maxTokens: 400,
+        signal: controller.signal,
+      }),
+      new Promise((_resolve, reject) => {
+        timer = setTimeout(() => {
+          if (!controller.signal.aborted) controller.abort();
+          reject(Object.assign(new Error(`auto-fix exceeded ${deadline}ms`), { name: 'DeadlineError' }));
+        }, deadline);
+      }),
+    ]);
+  } catch (err) {
+    logger.log('FIX', `auto-fix plan failed: ${err.message}`);
+    return [];
+  } finally {
+    clearTimeout(timer);
+  }
+  const plan = parseToolPlan(text);
+  if (plan.length) logger.log('FIX', `auto-fix plan: ${plan.map((p) => p.name).join(', ')}`);
+  return plan;
+}
+
 export async function runWithTools({
   provider,
   messages,
@@ -1648,6 +1748,7 @@ export async function runWithTools({
   const executor = makeExecutor({ cwd, timeoutMs });
   const results = [];
   const plannedByModel = [];
+  let autoFixes = 0;
 
   for (let round = 0; round < MAX_PLANNING_ROUNDS; round++) {
     let plan;
@@ -1662,38 +1763,25 @@ export async function runWithTools({
     const steps = Math.min(plan.length, MAX_STEPS - results.length);
     let allOk = true;
     for (const call of plan.slice(0, steps)) {
-      const label = call.name === 'run_command' ? call.params.command
-        : call.name === 'download_url' ? `download_url(${call.params.url})`
-        : `${call.name}(${call.params.path})`;
-      if (chatUI) {
-        chatUI.setStatus(`Running: ${label}`);
-        chatUI.setThinking(true);
-        chatUI.render(messages);
-      }
-      const result = await dispatchTool(executor, call);
+      const { result } = await executeToolCall({ executor, call, chatUI, messages, cwd });
       results.push({ name: call.name, params: call.params, result });
-      logger.log('EXEC', `${label} -> ${result.exitCode}${result.timedOut ? ' [timeout]' : ''}`);
-      if (result.success) {
-        if (chatUI) chatUI.notify(`\u2713 ${label}`);
-        if (result.stdout.trim()) logger.log('EXEC', `${label} stdout: ${result.stdout.trim().slice(0, 500)}`);
-        if (result.stderr.trim()) logger.log('WARN', `${label} stderr: ${result.stderr.trim().slice(0, 500)}`);
-        if (call.name === 'create_file' && /\.py$/i.test(call.params.path)) {
-          const check = spawnSync('python3', [
-            '-c', 'import ast,sys; ast.parse(open(sys.argv[1], encoding="utf-8").read())',
-            join(cwd || '.', call.params.path),
-          ], { encoding: 'utf-8', timeout: 30000 });
-          if (check.status === 1) {
-            const detail = (check.stderr || '').trim().split('\n').slice(-2).join(' ');
-            if (chatUI) chatUI.notify(`\u2717 python check failed: ${call.params.path} (${detail})`);
-            logger.log('WARN', `${label} failed python syntax check: ${detail}`);
-          }
+      if (!result.success) allOk = false;
+      if (result.timedOut && !result.success) break;
+
+      if (!result.success && autoFixes < MAX_AUTO_FIXES && results.length < MAX_STEPS) {
+        autoFixes++;
+        if (chatUI) chatUI.setStatus('Auto-fixing failed step…');
+        const fixPlan = await planAutoFix(provider, messages, requestText, call, result);
+        const fixSteps = Math.min(fixPlan.length, MAX_STEPS - results.length);
+        let fixedOk = fixPlan.length > 0;
+        for (const fcall of fixPlan.slice(0, fixSteps)) {
+          const f = await executeToolCall({ executor, call: fcall, chatUI, messages, cwd });
+          results.push({ name: fcall.name, params: fcall.params, result: f.result });
+          if (!f.result.success) { fixedOk = false; break; }
         }
-      } else {
-        allOk = false;
-        if (chatUI) chatUI.notify(`\u2717 ${label}: ${(result.stderr || 'failed').trim().slice(0, 300)}`);
-        logger.log('ERROR', `${label} failed: ${(result.stderr || result.stdout || 'exit ' + result.exitCode).trim().slice(0, 500)}`);
+        if (fixedOk && chatUI) chatUI.notify('Auto-fixed after failure.');
+        if (fixedOk) allOk = true;
       }
-      if (result.timedOut && !result.success) break;
     }
     if (allOk || !results.length || round >= MAX_PLANNING_ROUNDS - 1) break;
   }

package/src/cmd/todos.js

@@ -0,0 +1,64 @@
+import { readFileSync, writeFileSync } from 'fs';
+import { join } from 'path';
+
+const FILE_NAME = '.todos.json';
+
+export function todosFile(cwd) {
+  return join(cwd || '.', FILE_NAME);
+}
+
+export function loadTodos(cwd) {
+  try {
+    const data = JSON.parse(readFileSync(todosFile(cwd), 'utf-8'));
+    if (!Array.isArray(data)) return [];
+    return data.map((t, i) => {
+      if (t && typeof t === 'object') {
+        return {
+          text: String(t.text ?? '').trim(),
+          done: Boolean(t.done),
+          createdAt: t.createdAt ?? null,
+          updatedAt: t.updatedAt ?? null,
+        };
+      }
+      return { text: String(t ?? '').trim(), done: false, createdAt: null, updatedAt: null };
+    }).filter((t) => t.text.length > 0);
+  } catch {
+    return [];
+  }
+}
+
+export function saveTodos(cwd, todos) {
+  writeFileSync(todosFile(cwd), JSON.stringify(todos, null, 2) + '\n', 'utf-8');
+}
+
+export function formatTodos(todos) {
+  if (!todos || !todos.length) return '(no todos yet)';
+  return todos.map((t, i) => `${i + 1}. ${t.done ? '[x]' : '[ ]'} ${t.text}`).join('\n');
+}
+
+export function addTodo(cwd, text) {
+  const clean = String(text ?? '').trim();
+  if (!clean) return { error: 'Missing todo text.' };
+  const todos = loadTodos(cwd);
+  todos.push({ text: clean, done: false, createdAt: new Date().toISOString(), updatedAt: null });
+  saveTodos(cwd, todos);
+  return { okay: true };
+}
+
+export function updateTodo(cwd, index, { done, text } = {}) {
+  const todos = loadTodos(cwd);
+  const i = Number(index) - 1;
+  if (!Number.isInteger(i) || i < 0 || i >= todos.length) {
+    return { error: `No todo #${index}.` };
+  }
+  if (typeof done === 'boolean') todos[i].done = done;
+  if (typeof text === 'string' && text.trim()) todos[i].text = text.trim();
+  todos[i].updatedAt = new Date().toISOString();
+  saveTodos(cwd, todos);
+  return { okay: true };
+}
+
+export function clearTodos(cwd) {
+  saveTodos(cwd, []);
+  return { okay: true };
+}

package/src/cmd/tools.js

@@ -3,6 +3,7 @@ import { resolve, sep } from 'path';
 import { toolResult, SafeCommandExecutor } from './executor.js';
 import { Downloader } from '../utils/downloader.js';
 import { Config } from '../config.js';
+import { loadTodos, addTodo, updateTodo, clearTodos, formatTodos } from './todos.js';
 
 // Resolve a path relative to the workspace and require it to stay inside.
 // Throws if the path escapes the workspace (or hits the workspace root for
@@ -124,6 +125,35 @@ export async function downloadUrl(cwd, url, dir) {
   }
 }
 
+export function todoAdd(cwd, text) {
+  const err = addTodo(cwd, text);
+  if (err && err.error) return fail({ command: `todo_add("${text}")`, stderr: err.error, cwd });
+  return ok({ command: `todo_add("${text}")`, stdout: formatTodos(loadTodos(cwd)), cwd });
+}
+
+export function todoList(cwd) {
+  try {
+    return ok({ command: 'todo_list', stdout: formatTodos(loadTodos(cwd)), cwd });
+  } catch (err) {
+    return fail({ command: 'todo_list', stderr: err.message, cwd });
+  }
+}
+
+export function todoUpdate(cwd, index, done, text) {
+  const err = updateTodo(cwd, index, { done, text });
+  if (err && err.error) return fail({ command: `todo_update(${index})`, stderr: err.error, cwd });
+  return ok({ command: `todo_update(${index})`, stdout: formatTodos(loadTodos(cwd)), cwd });
+}
+
+export function todoClear(cwd) {
+  try {
+    clearTodos(cwd);
+    return ok({ command: 'todo_clear', stdout: '(no todos yet)', cwd });
+  } catch (err) {
+    return fail({ command: 'todo_clear', stderr: err.message, cwd });
+  }
+}
+
 // Structured tool registry shared with the agent loop.
 export const TOOL_EXECUTORS = {
   run_command: (executor, params) => executor.runCommand(params.command),
@@ -133,6 +163,10 @@ export const TOOL_EXECUTORS = {
   list_directory: (executor, params) => listDirectory(executor.cwd, params.path || '.'),
   delete_file: (executor, params) => deleteFile(executor.cwd, params.path),
   download_url: (executor, params) => downloadUrl(executor.cwd, params.url, params.dir),
+  todo_add: (executor, params) => todoAdd(executor.cwd, params ? params.text : undefined),
+  todo_list: (executor, params) => todoList(executor.cwd),
+  todo_update: (executor, params) => todoUpdate(executor.cwd, params ? params.index : undefined, params ? params.done : undefined, params ? params.text : undefined),
+  todo_clear: (executor, params) => todoClear(executor.cwd),
 };
 
 export function makeExecutor({ cwd, timeoutMs } = {}) {

package/src/ui/branding.js

@@ -1,6 +1,6 @@
 export const Branding = {
   APP_NAME: 'VierrataleAI',
-  VERSION: '0.1.0-beta.10',
+  VERSION: '0.1.0-beta.12',
 
   colors: {
     primary: '\x1b[38;2;124;58;237m',

package/src/utils/webfetch.js

@@ -2,6 +2,10 @@ export const UA =
   'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' +
   '(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
 
+const ALT_UA =
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 ' +
+  '(KHTML, like Gecko) Version/17.4 Safari/605.1.15';
+
 const MAX_BYTES = 200000; // 200KB cap
 const MAX_TEXT = 8000;    // ~8k chars of readable text
 
@@ -16,10 +20,18 @@ const BLOCK_MARKERS = [
   'client challenge',
   'checking your browser',
   'enable javascript and cookies',
-  'cf-chl',
+  'verify you are human',
   'challenge-platform',
   'attention required!',
-  'powershell install',
+  'unusual traffic',
+  'access denied',
+  'too many requests',
+  'cf-error-details',
+  'atomic challenge',
+  'incapsula',
+  'rh-captcha',
+  'privacy pass',
+  'error code: 1020',
 ];
 
 // Browser-like headers so header-based bot checks see a real browser.
@@ -39,6 +51,12 @@ export function browserHeaders(accept = 'text/html,application/xhtml+xml,applica
   };
 }
 
+function altHeaders() {
+  const h = browserHeaders();
+  h['User-Agent'] = ALT_UA;
+  return h;
+}
+
 // File types the AI can download off a fetched page.
 const ASSET_TYPES = {
   pdf: 'pdf', zip: 'zip', tar: 'tar', gz: 'gz', '7z': '7z', rar: 'rar',
@@ -61,6 +79,9 @@ export function isBlockedBody(html) {
   return BLOCK_MARKERS.some((m) => s.includes(m));
 }
 
+// Optional namespace prefix for XML tags (e.g. <sm:loc>).
+const NSTAG = (name) => `(?:[a-z][\\w.-]*:)?${name}`;
+
 export class WebFetch {
   static normalizeUrl(input) {
     const trimmed = (input || '').trim();
@@ -81,52 +102,63 @@ export class WebFetch {
     }
   }
 
+  // Fetch any URL and return readable text regardless of content type.
   static async fetch(rawUrl, maxText = MAX_TEXT) {
     const url = this.normalizeUrl(rawUrl);
     if (!url) {
       throw new Error('Invalid URL. Provide a valid http(s) address.');
     }
 
-    let finalUrl = url;
-    let html = '';
-    let blocked = false;
-    try {
-      const res = await this._fetchHtml(url);
-      finalUrl = res.url;
-      html = res.html;
-      const status = res.status;
-      blocked = status !== 200 || isBlockedBody(html);
-    } catch (err) {
-      blocked = true;
-      // _fetchHtml errors are network/cert/PROTOCOL issues; a fallback may
-      // still succeed (e.g. registry API is reachable even when the HTML
-      // page's CDN is not).
+    let res = await this._get(url, browserHeaders());
+
+    // Transport-level failure (TLS, no https) → retry plain http.
+    if (!res.ok) {
+      const alt = url.replace(/^https:/i, 'http:');
+      if (alt !== url) res = await this._get(alt, browserHeaders());
     }
 
-    // Bot-walled npm/pypi pages degrade to their open JSON API.
-    if (blocked) {
-      const registry = await this._registryFallback(finalUrl || url, maxText);
+    // Hard wall with a challenge challenge → one retry with a different UA.
+    if (res.ok && (res.status === 403 || res.status === 429) && isBlockedBody(res.text)) {
+      const retried = await this._get(url, altHeaders());
+      if (retried.ok && retried.status === 200 && !isBlockedBody(retried.text)) res = retried;
+    }
+
+    if (!res.ok) {
+      const registry = await this._registryFallback(res.url || url, maxText);
       if (registry) return registry;
+      throw new Error(`Could not reach ${res.url || url}`);
     }
 
+    const status = res.status;
+    const bodyText = res.text;
+    const blocked = status >= 400 || isBlockedBody(bodyText);
+    if (blocked) {
+      const registry = await this._registryFallback(res.url || url, maxText);
+      if (registry) return registry;
+    }
     if (blocked) {
-      const status = 'blocked';
-      throw new Error(`Request blocked (${status}) for ${finalUrl || url}`);
+      throw new Error(`Request blocked (HTTP ${status}) for ${res.url || url}`);
     }
 
-    return {
-      url: finalUrl,
-      title: this._extractTitle(html),
-      text: this._extractText(html).slice(0, maxText),
-      assets: this._extractAssets(html, finalUrl),
-    };
+    const kind = this._classify(res.url, res.contentType, bodyText);
+    return this._render(kind, res, bodyText, maxText);
+  }
+
+  static async _get(url, headers) {
+    try {
+      const r = await this._fetchBytes(url, headers);
+      const text = this._decodeText(r.body, r.contentType);
+      return { ok: true, url: r.url, status: r.status, contentType: r.contentType, body: r.body, text };
+    } catch {
+      return { ok: false, url };
+    }
   }
 
-  static async _fetchHtml(url) {
+  static async _fetchBytes(url, headers) {
     let resp;
     try {
       resp = await fetch(url, {
-        headers: browserHeaders(),
+        headers,
         redirect: 'follow',
         signal: AbortSignal.timeout(15000),
       });
@@ -134,20 +166,316 @@ export class WebFetch {
       throw new Error(`Could not reach ${url}`);
     }
 
-    const reader = resp.body.getReader();
-    const decoder = new TextDecoder('utf-8', { fatal: false });
-    let html = '';
-    let received = 0;
-    while (received < MAX_BYTES) {
-      const { done, value } = await reader.read();
-      if (done) break;
-      received += value.length;
-      html += decoder.decode(value, { stream: true });
+    const chunks = [];
+    let total = 0;
+    const reader = resp.body && resp.body.getReader();
+    if (reader) {
+      while (total < MAX_BYTES) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        total += value.byteLength;
+        chunks.push(Buffer.from(value));
+      }
+      reader.cancel();
     }
-    reader.cancel();
-    html += decoder.decode();
+    return {
+      url: resp.url || url,
+      status: resp.status,
+      contentType: String(resp.headers.get('content-type') || '').toLowerCase(),
+      body: Buffer.concat(chunks, total),
+    };
+  }
+
+  // --- charset-aware decoding ------------------------------------------- //
 
-    return { url: resp.url || url, html, status: resp.status };
+  static _sniffCharset(contentType, bytes) {
+    const m = /charset=(["']?)([a-z0-9._-]+)\1/i.exec(contentType || '');
+    if (m) return m[2].toLowerCase();
+    const head = bytes.slice(0, 2048).toString('latin1').toLowerCase();
+    const meta = /<meta[^>]+charset=["']?\s*([a-z0-9._-]+)/i.exec(head);
+    if (meta) return meta[1].toLowerCase();
+    const httpEquiv = /<meta[^>]+http-equiv=["']content-type["'][^>]+content=["']text\/html; charset=([a-z0-9._-]+)/i.exec(head);
+    return httpEquiv ? httpEquiv[1].toLowerCase() : '';
+  }
+
+  static _decodeText(bytes, contentType) {
+    const charset = this._sniffCharset(contentType, bytes);
+    if (charset) {
+      try {
+        return new TextDecoder(charset, { fatal: false }).decode(bytes);
+      } catch {}
+    }
+    let text = new TextDecoder('utf-8', { fatal: false }).decode(bytes);
+    if (this._badDecode(text, bytes)) {
+      try {
+        text = new TextDecoder('windows-1252', { fatal: false }).decode(bytes);
+      } catch {}
+    }
+    return text;
+  }
+
+  static _badDecode(text, bytes) {
+    if (!text || !bytes || bytes.length === 0) return false;
+    let bad = 0;
+    for (let i = 0; i < text.length; i++) if (text.charCodeAt(i) === 0xfffd) bad++;
+    return bad / bytes.length > 0.001;
+  }
+
+  // --- content-type classification --------------------------------------- //
+
+  static _classify(url, contentType, text) {
+    const meta = ((contentType || '').split(';')[0] || '').trim().toLowerCase();
+    const head = (text || '').slice(0, 240).trimStart();
+    if (/\bjson\b/.test(meta) || /\+\w*json\b/.test(meta)) return 'json';
+    if (/\b(?:rss|atom|xml)\b/.test(meta) || meta === 'application/sitemap+xml') return 'xml';
+    if (/\/xhtml\+html$/.test(meta) || /html/.test(meta)) return 'html';
+    if (/markdown/.test(meta)) return 'markdown';
+    if (meta === 'application/pdf') return 'pdf';
+    if (/^application\/(?:json|x-ndjson|xml|rss|atom)/.test(meta)) return 'json';
+    if (/^text\//.test(meta) && !/html/.test(meta)) return 'text';
+    if (head.startsWith('{') || head.startsWith('[')) return 'json';
+    if (head.startsWith('<?xml') || /^<!DOCTYPE\s+(?!html)/i.test(head)) return 'xml';
+    if (/^<!doctype html/i.test(head) || /<html\b/i.test(head.slice(0, 400))) return 'html';
+    if (/^(image|video|audio|font)\//.test(meta)) return 'binary';
+    if (/^application\//.test(meta)) return 'binary';
+    if (/\/pdf$/.test(url.split('?')[0].toLowerCase())) return 'pdf';
+    return 'text';
+  }
+
+  static _render(kind, res, bodyText, maxText) {
+    const url = res.url;
+
+    if (kind === 'json') {
+      let flat = bodyText.trim();
+      try {
+        flat = this._flattenJson(JSON.parse(bodyText));
+      } catch {}
+      return { url, title: this._titleFromUrl(url), text: flat.slice(0, maxText), assets: [] };
+    }
+
+    if (kind === 'xml') {
+      return {
+        url,
+        title: this._titleFromUrl(url),
+        text: this._xmlToText(bodyText).slice(0, maxText),
+        assets: this._extractAssets(bodyText, url),
+      };
+    }
+
+    if (kind === 'text' || kind === 'markdown') {
+      return { url, title: this._titleFromUrl(url), text: bodyText.trim().slice(0, maxText), assets: [] };
+    }
+
+    if (kind === 'pdf' || kind === 'binary') {
+      let parsed;
+      try {
+        parsed = new URL(url);
+      } catch {
+        parsed = null;
+      }
+      let type = detectAssetType(url) || 'file';
+      if (kind === 'pdf') type = 'pdf';
+      return {
+        url,
+        title: this._titleFromUrl(url),
+        text: this._binaryNote(res, kind, type),
+        assets: [{ url, name: this._assetName(parsed, type), type }],
+      };
+    }
+
+    // HTML
+    const desc = this._extractDescription(bodyText);
+    let text = this._extractText(bodyText);
+    if (text.length < 2400 && text.length < maxText) {
+      const embedded = this._extractEmbeddedJson(bodyText);
+      if (embedded !== null && embedded !== undefined) {
+        const flat = this._flattenJson(embedded).trim();
+        if (flat) text = `${text}\n\n[embedded page data]\n${flat}`.trim();
+      }
+      const links = this._extractLinks(bodyText, url);
+      if (links.length >= 3 && text.length < 2400) {
+        text = `${text}\n\n[Links on page]\n${links.slice(0, 25).map((l) => `- ${l.text}: ${l.url}`).join('\n')}`.trim();
+      }
+    }
+    let out = text.slice(0, maxText);
+    if (desc && out.length < maxText - 600 && !out.includes(desc.slice(0, 40))) {
+      out = `${desc.slice(0, 400)}\n\n${out}`;
+    }
+    let title = this._extractTitle(bodyText);
+    if (!title) title = this._titleFromUrl(url);
+    return { url, title, text: out, assets: this._extractAssets(bodyText, url) };
+  }
+
+  static _binaryNote(res, kind, type) {
+    const ct = res.contentType || 'unknown content type';
+    const size = res.body ? `~${(res.body.length / 1024).toFixed(1)} KB` : '?';
+    return [
+      `This resource is a ${kind === 'pdf' ? 'PDF' : 'binary/non-text file'} (${ct}, ${size}) and cannot be read as text.`,
+      `Use the download_url tool to save it locally: ${res.url}${type !== 'file' ? ` (type: ${type})` : ''}`,
+    ].join('\n');
+  }
+
+  static _titleFromUrl(url) {
+    try {
+      const u = new URL(url);
+      const seg = decodeURIComponent(u.pathname.split('/').filter(Boolean).pop() || '');
+      const base = u.hostname.replace(/^www\./i, '');
+      if (seg && !/\.(html?|php|aspx?)$/i.test(seg)) return base;
+      const cleaned = seg.replace(/\.(html?|php|aspx?)$/i, '').replace(/[-_]+/g, ' ').trim();
+      return cleaned || base;
+    } catch {
+      return url;
+    }
+  }
+
+  static _flattenJson(data, maxLines = 600) {
+    const out = [];
+    const walk = (v, label) => {
+      if (out.length >= maxLines) return;
+      if (v === null || v === undefined) {
+        if (label) out.push(`${label}: null`);
+        return;
+      }
+      if (Array.isArray(v)) {
+        if (label) out.push(`${label} (${v.length})`);
+        for (let i = 0; i < v.length; i++) walk(v[i], label ? `${label}[${i}]` : `[${i}]`);
+      } else if (typeof v === 'object') {
+        if (label) out.push(`${label} {${Object.keys(v).length}}`);
+        for (const k of Object.keys(v)) walk(v[k], label ? `${label}.${k}` : k);
+      } else {
+        const s = typeof v === 'string' ? v.replace(/\s+/g, ' ').trim() : String(v);
+        out.push(`${label ? label + ': ' : ''}${s.slice(0, 400)}`);
+      }
+    };
+    walk(data, '');
+    return out.join('\n');
+  }
+
+  // --- XML / feeds / sitemaps -------------------------------------------- //
+
+  static _xmlToText(xml) {
+    const doc = (xml || '').trimStart();
+    if (/<sitemapindex\b/i.test(doc) || /<urlset\b/i.test(doc)) {
+      const locs = [];
+      const re = new RegExp(`<${NSTAG('loc')}(?:[^>]*)>([\\s\\S]*?)<\\/${NSTAG('loc')}>`, 'gi');
+      let m;
+      while ((m = re.exec(xml)) !== null) {
+        const u = this._stripEntities(m[1]).replace(/\s+/g, ' ').trim();
+        if (u && /^https?:\/\//i.test(u) && !locs.includes(u)) locs.push(u);
+        if (locs.length >= 100) break;
+      }
+      if (locs.length) {
+        return `Sitemap (${locs.length} URLs)\n${locs.slice(0, 60).map((u) => `* ${u}`).join('\n')}`;
+      }
+    }
+    if (/<(?:rss|feed)\b/i.test(doc)) return this._feedToText(xml);
+    return this._extractText(xml).slice(0, MAX_TEXT);
+  }
+
+  static _feedToText(xml) {
+    const lines = [];
+    const top = new RegExp(`<${NSTAG('channel')}[\\s\\S]*?<\\/${NSTAG('channel')}>|<${NSTAG('feed')}[\\s\\S]*?<\\/${NSTAG('feed')}>`, 'i');
+    const topMatch = top.exec(xml);
+    let title = '';
+    if (topMatch) {
+      const t = new RegExp(`<${NSTAG('title')}(?:[^>]*)>([\\s\\S]*?)<\\/${NSTAG('title')}>`, 'i').exec(topMatch[0]);
+      if (t) title = this._stripTags(t[1]);
+    }
+    if (title) lines.push(title);
+
+    const itemRe = new RegExp(`<${NSTAG('item')}[\\s\\S]*?<\\/${NSTAG('item')}>|<${NSTAG('entry')}[\\s\\S]*?<\\/${NSTAG('entry')}>`, 'gi');
+    let m;
+    let count = 0;
+    while ((m = itemRe.exec(xml)) !== null && count < 40) {
+      const b = m[0];
+      count++;
+      const it = new RegExp(`<${NSTAG('title')}(?:[^>]*)>([\\s\\S]*?)<\\/${NSTAG('title')}>`, 'i').exec(b);
+      const il = new RegExp(`<${NSTAG('link')}(?:[^>]*)>([\\s\\S]*?)<\\/${NSTAG('link')}>`, 'i').exec(b) ||
+                 new RegExp(`<${NSTAG('link')}(?:[^>]*)\\bhref=["']([^"']+)["']`, 'i').exec(b);
+      const di = /<(?:[a-z][\w.-]*:)?(?:description|summary|subtitle)[^>]*>([\s\S]*?)<\/(?:[a-z][\w.-]*:)?(?:description|summary|subtitle)>/i.exec(b);
+      const item = [];
+      if (it) item.push(this._stripTags(it[1]));
+      if (il) item.push(`  ${this._stripTags(il[1]).replace(/\s+/g, ' ').trim()}`);
+      if (di && di[1].trim()) {
+        const d = this._stripTags(di[1]);
+        if (d) item.push(`  ${d}`);
+      }
+      if (item.length) lines.push(item.join('\n'));
+    }
+    if (lines.length === 0) return this._extractText(xml).slice(0, MAX_TEXT);
+    return lines.join('\n\n');
+  }
+
+  // --- SPA / embedded JSON ------------------------------------------------ //
+
+  static _extractEmbeddedJson(html) {
+    const next = /<script[^>]+id=["']__NEXT_DATA__["'][^>]*>([\s\S]*?)<\/script>/i.exec(html);
+    if (next && next[1].trim()) {
+      try {
+        return JSON.parse(next[1].trim());
+      } catch {}
+    }
+    const ld = /<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
+    let c;
+    while ((c = ld.exec(html)) !== null) {
+      if (!c[1].trim()) continue;
+      try {
+        return JSON.parse(c[1].trim());
+      } catch {}
+    }
+    const st = /window\.__PRELOADED_STATE__\s*=\s*/i.exec(html);
+    if (st) {
+      let i = st.index + st[0].length;
+      let depth = 0;
+      const start = i;
+      while (i < html.length) {
+        const ch = html[i];
+        if (ch === '{') depth++;
+        else if (ch === '}') {
+          depth--;
+          if (depth === 0) break;
+        }
+        i++;
+      }
+      if (depth === 0) {
+        try {
+          return JSON.parse(html.slice(start, i + 1).replace(/;?\s*$/, '').trim());
+        } catch {}
+      }
+    }
+    return null;
+  }
+
+  static _extractDescription(html) {
+    const m1 = /<meta[^>]+name=["']description["'][^>]+content=["']([^"']*)["'][^>]*>/i.exec(html);
+    if (m1) return this._stripTags(m1[1]);
+    const m2 = /<meta[^>]+content=["']([^"']*)["'][^>]+name=["']description["'][^>]*>/i.exec(html);
+    return m2 ? this._stripTags(m2[1]) : '';
+  }
+
+  static _extractLinks(html, baseUrl) {
+    const links = [];
+    const seen = new Set();
+    const re = /<a\b[^>]*href=["']([^"']+)["'][^>]*>((?:[^<]|<(?!\/a\b)[^>]+>)*?)<\/a>/gi;
+    let m;
+    while ((m = re.exec(html)) !== null) {
+      const href = m[1].trim().split('#')[0];
+      if (!href || /^(javascript|mailto|tel|data):/i.test(href)) continue;
+      let resolved;
+      try {
+        const u = new URL(href, baseUrl);
+        if (u.protocol !== 'http:' && u.protocol !== 'https:') continue;
+        resolved = u.toString();
+      } catch {
+        continue;
+      }
+      if (seen.has(resolved)) continue;
+      seen.add(resolved);
+      const text = this._stripTags(m[2]).slice(0, 80);
+      links.push({ text: text || resolved, url: resolved });
+      if (links.length >= 30) break;
+    }
+    return links;
   }
 
   // --- Registry fallback (npm + PyPI) via their open JSON APIs ---------- //
@@ -280,6 +608,7 @@ export class WebFetch {
   }
 
   static _assetName(url, type) {
+    if (!url) return `download.${type}`;
     const path = url.pathname.split('/').filter(Boolean);
     let name = path.length ? decodeURIComponent(path[path.length - 1]) : '';
     if (!name || type === 'unknown') name = `${url.hostname.replace(/[^a-z0-9.-]/gi, '_')}.${type}`;
@@ -293,17 +622,19 @@ export class WebFetch {
   }
 
   static _extractText(html) {
-    // Strip block tags' contents that add no readable value.
-    let s = html.replace(/<(script|style|noscript|svg|head|iframe|form|nav|footer|aside)[^>]*>[\s\S]*?<\/\1>/gi, ' ');
+    // Drop comments and tag-blocks that add no readable value.
+    let s = String(html || '')
+      .replace(/<!--[\s\S]*?-->/g, ' ')
+      .replace(/<(script|style|noscript|svg|head|iframe|form|nav|footer|aside|template|object)[^>]*>[\s\S]*?<\/\1>/gi, ' ');
 
     // Force spacing around block-level elements so words don't merge.
-    s = s.replace(/<\/(p|div|h[1-6]|li|tr|br|section|article)>/gi, '\n');
+    s = s.replace(/<\/(p|div|h[1-6]|li|tr|br|section|article|blockquote|pre|table)>/gi, '\n');
     s = s.replace(/<(br|li|tr)[^>]*>/gi, '\n');
 
     // Remove remaining tags.
     s = s.replace(/<[^>]+>/g, ' ');
 
-    // College entities.
+    // Collapse entities.
     s = this._stripEntities(s);
 
     // Collapse whitespace and trim lines.
@@ -316,6 +647,10 @@ export class WebFetch {
       .trim();
   }
 
+  static _stripTags(text) {
+    return this._stripEntities(text);
+  }
+
   static _stripEntities(text) {
     const map = {
       '&amp;': '&', '&lt;': '<', '&gt;': '>', '&quot;': '"',