@vierratale/ai 0.1.0-beta.9 → 0.1.0-fixfetch-beta9.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vierratale/ai",
-  "version": "0.1.0-beta.9",
+  "version": "0.1.0-fixfetch-beta9.5",
   "description": "VierrataleAI - Intelligent terminal assistant",
   "type": "module",
   "files": [
@@ -18,7 +18,7 @@
   },
   "scripts": {
     "start": "node bin/vierrataleai.js",
-    "test": "node --test test/",
+    "test": "node --test \"test/*.test.js\"",
     "postinstall": "node scripts/postinstall.js"
   },
   "keywords": ["ai", "chatbot", "terminal", "assistant", "llm"],

package/src/config.js

@@ -18,7 +18,7 @@ const DEFAULTS = {
   temperature: 0.7,
   maxTokens: 4096,
   keepAlive: '5m',
-  numThreads: 6,
+  numThreads: 8,
   commandTimeoutMs: 60000,
   planTimeoutMs: 45000,
 };

package/src/ui/branding.js

@@ -1,6 +1,6 @@
 export const Branding = {
   APP_NAME: 'VierrataleAI',
-  VERSION: '0.1.0-beta.9',
+  VERSION: '0.1.0-fixfetch-beta9.5',
 
   colors: {
     primary: '\x1b[38;2;124;58;237m',

package/src/utils/webfetch.js

@@ -1,4 +1,6 @@
-const UA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36';
+const UA =
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' +
+  '(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
 
 const MAX_BYTES = 200000; // 200KB cap
 const MAX_TEXT = 8000;    // ~8k chars of readable text
@@ -8,6 +10,40 @@ const BLOCK_TAGS = new Set([
   'nav', 'footer', 'aside', 'iframe', 'form', 'button', 'noscript',
 ]);
 
+// Body markers that indicate an anti-bot wall instead of real content.
+const BLOCK_MARKERS = [
+  'just a moment',
+  'client challenge',
+  'checking your browser',
+  'enable javascript and cookies',
+  'cf-chl',
+  'challenge-platform',
+  'attention required!',
+  'powershell install',
+];
+
+// Browser-like headers so header-based bot checks see a real browser.
+function browserHeaders(accept = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8') {
+  return {
+    'User-Agent': UA,
+    Accept: accept,
+    'Accept-Language': 'en-US,en;q=0.9',
+    Connection: 'keep-alive',
+    'Upgrade-Insecure-Requests': '1',
+    'Sec-Fetch-Dest': accept.startsWith('application/json') ? 'empty' : 'document',
+    'Sec-Fetch-Mode': accept.startsWith('application/json') ? 'cors' : 'navigate',
+    'Sec-Fetch-Site': 'none',
+    'Sec-Fetch-User': accept.startsWith('application/json') ? '?0' : '?1',
+    'Cache-Control': 'no-cache',
+    Pragma: 'no-cache',
+  };
+}
+
+export function isBlockedBody(html) {
+  const s = String(html || '').toLowerCase().slice(0, 4000);
+  return BLOCK_MARKERS.some((m) => s.includes(m));
+}
+
 export class WebFetch {
   static normalizeUrl(input) {
     const trimmed = (input || '').trim();
@@ -34,10 +70,45 @@ export class WebFetch {
       throw new Error('Invalid URL. Provide a valid http(s) address.');
     }
 
+    let finalUrl = url;
+    let html = '';
+    let blocked = false;
+    try {
+      const res = await this._fetchHtml(url);
+      finalUrl = res.url;
+      html = res.html;
+      const status = res.status;
+      blocked = status !== 200 || isBlockedBody(html);
+    } catch (err) {
+      blocked = true;
+      // _fetchHtml errors are network/cert/PROTOCOL issues; a fallback may
+      // still succeed (e.g. registry API is reachable even when the HTML
+      // page's CDN is not).
+    }
+
+    // Bot-walled npm/pypi pages degrade to their open JSON API.
+    if (blocked) {
+      const registry = await this._registryFallback(finalUrl || url, maxText);
+      if (registry) return registry;
+    }
+
+    if (blocked) {
+      const status = 'blocked';
+      throw new Error(`Request blocked (${status}) for ${finalUrl || url}`);
+    }
+
+    return {
+      url: finalUrl,
+      title: this._extractTitle(html),
+      text: this._extractText(html).slice(0, maxText),
+    };
+  }
+
+  static async _fetchHtml(url) {
     let resp;
     try {
       resp = await fetch(url, {
-        headers: { 'User-Agent': UA, Accept: 'text/html,application/xhtml+xml' },
+        headers: browserHeaders(),
         redirect: 'follow',
         signal: AbortSignal.timeout(15000),
       });
@@ -45,11 +116,6 @@ export class WebFetch {
       throw new Error(`Could not reach ${url}`);
     }
 
-    if (!resp.ok) {
-      throw new Error(`Request failed (${resp.status}) for ${url}`);
-    }
-
-    // Refuse to read the whole body past the cap.
     const reader = resp.body.getReader();
     const decoder = new TextDecoder('utf-8', { fatal: false });
     let html = '';
@@ -63,13 +129,95 @@ export class WebFetch {
     reader.cancel();
     html += decoder.decode();
 
+    return { url: resp.url || url, html, status: resp.status };
+  }
+
+  // --- Registry fallback (npm + PyPI) via their open JSON APIs ---------- //
+
+  static async _registryFallback(url, maxText) {
+    const npm = this._npmSpec(url);
+    if (npm) {
+      try {
+        return await this._npmFallback(npm, url, maxText);
+      } catch {}
+    }
+    const pypi = this._pypiSpec(url);
+    if (pypi) {
+      try {
+        return await this._pypiFallback(pypi, url, maxText);
+      } catch {}
+    }
+    return null;
+  }
+
+  static _npmSpec(url) {
+    const m = /npmjs\.\w+\/package\/([@a-zA-Z0-9._~-]+(?:\/[@a-zA-Z0-9._~-]+)?)/i.exec(url);
+    return m ? m[1] : null;
+  }
+
+  static _pypiSpec(url) {
+    const m = /pypi\.org\/project\/([a-zA-Z0-9._-]+)/i.exec(url);
+    return m ? m[1] : null;
+  }
+
+  static async _fetchJson(apiUrl) {
+    const resp = await fetch(apiUrl, {
+      headers: browserHeaders('application/json'),
+      redirect: 'follow',
+      signal: AbortSignal.timeout(15000),
+    });
+    if (!resp.ok) throw new Error(`API ${resp.status}`);
+    return resp.json();
+  }
+
+  static async _npmFallback(spec, pageUrl, maxText) {
+    const pkg = encodeURIComponent(spec); // '@vierratale/ai' -> %40vierratale%2Fai
+    const data = await this._fetchJson(`https://registry.npmjs.org/${pkg}`);
+    const version =
+      (data['dist-tags'] && (data['dist-tags'].latest || data['dist-tags'].beta)) ||
+      Object.keys(data.versions || {})[0];
+    const meta = (data.versions && data.versions[version]) || {};
+    const deps = meta.dependencies || data.dependencies || {};
+    const lines = [
+      `${data.name} (npm package)`,
+      `Latest version: ${version}`,
+      meta.description || data.description || '',
+      `License: ${meta.license || data.license || 'N/A'}`,
+      `Dependencies: ${Object.keys(deps).length}`,
+      '',
+    ];
+    const readme = String(data.readme || '').trim();
+    if (readme) lines.push(readme.slice(0, maxText));
     return {
-      url: resp.url || url,
-      title: this._extractTitle(html),
-      text: this._extractText(html).slice(0, maxText),
+      url: pageUrl,
+      title: `${data.name} · npm`,
+      text: lines.filter(Boolean).join('\n').slice(0, maxText),
+    };
+  }
+
+  static async _pypiFallback(spec, pageUrl, maxText) {
+    const data = await this._fetchJson(`https://pypi.org/pypi/${encodeURIComponent(spec)}/json`);
+    const info = data.info || {};
+    const lines = [
+      `${info.name} (PyPI package)`,
+      `Version: ${info.version}`,
+      info.summary || '',
+      `Requires Python: ${info.requires_python || 'N/A'}`,
+      `License: ${info.license || 'N/A'}`,
+      `Home: ${info.home_page || info.project_url || 'N/A'}`,
+      '',
+    ];
+    const desc = String(info.description || '').trim();
+    if (desc) lines.push(desc.slice(0, maxText));
+    return {
+      url: pageUrl,
+      title: `${info.name} · PyPI`,
+      text: lines.filter(Boolean).join('\n').slice(0, maxText),
     };
   }
 
+  // --- extraction -------------------------------------------------------- //
+
   static _extractTitle(html) {
     const m = /<title[^>]*>([^<]*)<\/title>/i.exec(html);
     return m ? this._stripEntities(m[1].trim()) : '';
@@ -112,4 +260,4 @@ export class WebFetch {
       .replace(/\s+/g, ' ')
       .trim();
   }
-}
+}