@vierratale/ai 0.1.0-beta.1 → 0.1.0-beta.10

package/src/utils/math.js

@@ -0,0 +1,142 @@
+// utils/math.js - Deterministic, safe math evaluator.
+// Parses arithmetic ( + - * / ^ % parentheses, decimals, scientific notation )
+// WITHOUT using eval(), so it can never execute arbitrary code. Returns null
+// when the input is not a clean math expression, so callers fall back to the
+// model.
+
+// Extract a bare arithmetic expression from a line of chat text.
+export function extractMathExpression(text) {
+  const s = String(text || '').trim();
+  // Strip a leading question/math verb phrase if present.
+  const cleaned = s.replace(
+    /^(?:what(?:'s| is| are)|what's|compute|calculate|solve|evaluate|equals?)\b\s*/i,
+    ''
+  ).replace(/\??$/, '').trim();
+  const expr = cleaned.replace(/[?,!.]$/, '').trim();
+  if (!expr) return null;
+  // Guard: must be purely numeric + arithmetic operators + parens + spaces.
+  if (!/^[0-9+\-*\/^%().\s]+$/.test(expr)) return null;
+  if (!/\d/.test(expr)) return null;
+  if (/[a-zA-Z]/.test(expr)) return null;
+  // Must contain at least one operator (otherwise "2" alone isn't a math ask).
+  if (!/[\+\-\*\/\^%]/.test(expr)) return null;
+  return expr;
+}
+
+// Tokenize an expression into numbers and operators.
+function tokenize(expr) {
+  const tokens = [];
+  let i = 0;
+  const n = expr.length;
+  while (i < n) {
+    const c = expr[i];
+    if (/\s/.test(c)) { i++; continue; }
+    if ('+-*/^%()'.includes(c)) {
+      tokens.push(c);
+      i++;
+      continue;
+    }
+    // read a number (int, decimal, scientific)
+    const num = expr.slice(i).match(/^(\d+(?:\.\d+)?|\.\d+)([eE][+-]?\d+)?/);
+    if (!num) return null;
+    tokens.push(parseFloat(num[0]));
+    i += num[0].length;
+  }
+  return tokens;
+}
+
+// Recursive-descent parser with correct precedence:
+//   expr   := term (('+'|'-') term)*
+//   term   := power (('*'|'/'|'%') power)*
+//   power  := unary ('^' power)?        right-associative
+//   unary  := ('+'|'-') unary | factor
+//   factor := '(' expr ')' | number
+class Parser {
+  constructor(tokens) {
+    this.tokens = tokens;
+    this.pos = 0;
+  }
+  peek() { return this.tokens[this.pos]; }
+  next() { return this.tokens[this.pos++]; }
+  expect(val) {
+    const t = this.next();
+    if (t !== val) throw new Error('syntax error');
+  }
+  parseExpr() { return this.parseTerm(); }
+  parseTerm() {
+    let left = this.parseMul();
+    while (['+', '-'].includes(this.peek())) {
+      const op = this.next();
+      const right = this.parseMul();
+      left = op === '+' ? left + right : left - right;
+    }
+    return left;
+  }
+  parseMul() {
+    let left = this.parsePower();
+    while (['*', '/', '%'].includes(this.peek())) {
+      const op = this.next();
+      const right = this.parsePower();
+      if (op === '*') left = left * right;
+      else if (op === '/') left = left / right;
+      else left = left % right;
+    }
+    return left;
+  }
+  parsePower() {
+    let left = this.parseUnary();
+    if (this.peek() === '^') {
+      this.next();
+      const right = this.parsePower();
+      left = Math.pow(left, right);
+    }
+    return left;
+  }
+  parseUnary() {
+    if (this.peek() === '+') { this.next(); return this.parseUnary(); }
+    if (this.peek() === '-') {
+      this.next();
+      return -this.parseUnary();
+    }
+    return this.parseFactor();
+  }
+  parseFactor() {
+    const t = this.peek();
+    if (t === '(') {
+      this.next();
+      const v = this.parseExpr();
+      this.expect(')');
+      return v;
+    }
+    if (typeof t === 'number') {
+      this.next();
+      return t;
+    }
+    throw new Error('syntax error');
+  }
+}
+
+export function evaluateMath(expr) {
+  try {
+    const tokens = tokenize(expr);
+    if (!tokens || !tokens.length) return null;
+    const p = new Parser(tokens);
+    const result = p.parseExpr();
+    if (p.pos !== tokens.length) return null; // leftover tokens = not a clean expr
+    if (!Number.isFinite(result)) return null;
+    // Round to avoid floating point noise like 0.30000000000000004
+    const rounded = Math.round((result + Number.EPSILON) * 1e10) / 1e10;
+    return Number(rounded.toPrecision(12));
+  } catch {
+    return null;
+  }
+}
+
+// Front-door: given a chat line, return a clean math answer or null.
+export function trySolveMath(text) {
+  const expr = extractMathExpression(text);
+  if (!expr) return null;
+  const value = evaluateMath(expr);
+  if (value === null) return null;
+  return `${expr} = ${value}`;
+}

package/src/utils/webfetch.js

@@ -0,0 +1,332 @@
+export const UA =
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' +
+  '(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
+
+const MAX_BYTES = 200000; // 200KB cap
+const MAX_TEXT = 8000;    // ~8k chars of readable text
+
+const BLOCK_TAGS = new Set([
+  'script', 'style', 'noscript', 'svg', 'head', 'title',
+  'nav', 'footer', 'aside', 'iframe', 'form', 'button', 'noscript',
+]);
+
+// Body markers that indicate an anti-bot wall instead of real content.
+const BLOCK_MARKERS = [
+  'just a moment',
+  'client challenge',
+  'checking your browser',
+  'enable javascript and cookies',
+  'cf-chl',
+  'challenge-platform',
+  'attention required!',
+  'powershell install',
+];
+
+// Browser-like headers so header-based bot checks see a real browser.
+export function browserHeaders(accept = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8') {
+  return {
+    'User-Agent': UA,
+    Accept: accept,
+    'Accept-Language': 'en-US,en;q=0.9',
+    Connection: 'keep-alive',
+    'Upgrade-Insecure-Requests': '1',
+    'Sec-Fetch-Dest': accept.startsWith('application/json') ? 'empty' : 'document',
+    'Sec-Fetch-Mode': accept.startsWith('application/json') ? 'cors' : 'navigate',
+    'Sec-Fetch-Site': 'none',
+    'Sec-Fetch-User': accept.startsWith('application/json') ? '?0' : '?1',
+    'Cache-Control': 'no-cache',
+    Pragma: 'no-cache',
+  };
+}
+
+// File types the AI can download off a fetched page.
+const ASSET_TYPES = {
+  pdf: 'pdf', zip: 'zip', tar: 'tar', gz: 'gz', '7z': '7z', rar: 'rar',
+  doc: 'doc', docx: 'docx', xls: 'xls', xlsx: 'xlsx', ppt: 'ppt', pptx: 'pptx',
+  mp3: 'mp3', mp4: 'mp4', wav: 'wav', ogg: 'ogg', webm: 'webm', mov: 'mov',
+  csv: 'csv', json: 'json', ipynb: 'ipynb', txt: 'txt', md: 'md',
+  png: 'png', jpg: 'jpg', jpeg: 'jpeg', gif: 'gif', webp: 'webp', svg: 'svg', ico: 'ico',
+  py: 'py', js: 'js', ts: 'ts', sh: 'sh',
+};
+const ASSET_TYPE_RE = /\.([a-z0-9]+)(?:[?#].*)?$/i;
+
+export function detectAssetType(url) {
+  const m = ASSET_TYPE_RE.exec(url);
+  if (!m) return null;
+  return ASSET_TYPES[m[1].toLowerCase()] || null;
+}
+
+export function isBlockedBody(html) {
+  const s = String(html || '').toLowerCase().slice(0, 4000);
+  return BLOCK_MARKERS.some((m) => s.includes(m));
+}
+
+export class WebFetch {
+  static normalizeUrl(input) {
+    const trimmed = (input || '').trim();
+    if (!trimmed) return null;
+    let url = trimmed;
+    if (/^[a-z][a-z0-9+.-]*:\/\//i.test(url)) {
+      // Already has an explicit scheme — must be http(s).
+      if (!/^https?:\/\//i.test(url)) return null;
+    } else if (!/^https?:\/\//i.test(url)) {
+      url = `https://${url}`;
+    }
+    try {
+      const u = new URL(url);
+      if (u.protocol !== 'http:' && u.protocol !== 'https:') return null;
+      return u.toString();
+    } catch {
+      return null;
+    }
+  }
+
+  static async fetch(rawUrl, maxText = MAX_TEXT) {
+    const url = this.normalizeUrl(rawUrl);
+    if (!url) {
+      throw new Error('Invalid URL. Provide a valid http(s) address.');
+    }
+
+    let finalUrl = url;
+    let html = '';
+    let blocked = false;
+    try {
+      const res = await this._fetchHtml(url);
+      finalUrl = res.url;
+      html = res.html;
+      const status = res.status;
+      blocked = status !== 200 || isBlockedBody(html);
+    } catch (err) {
+      blocked = true;
+      // _fetchHtml errors are network/cert/PROTOCOL issues; a fallback may
+      // still succeed (e.g. registry API is reachable even when the HTML
+      // page's CDN is not).
+    }
+
+    // Bot-walled npm/pypi pages degrade to their open JSON API.
+    if (blocked) {
+      const registry = await this._registryFallback(finalUrl || url, maxText);
+      if (registry) return registry;
+    }
+
+    if (blocked) {
+      const status = 'blocked';
+      throw new Error(`Request blocked (${status}) for ${finalUrl || url}`);
+    }
+
+    return {
+      url: finalUrl,
+      title: this._extractTitle(html),
+      text: this._extractText(html).slice(0, maxText),
+      assets: this._extractAssets(html, finalUrl),
+    };
+  }
+
+  static async _fetchHtml(url) {
+    let resp;
+    try {
+      resp = await fetch(url, {
+        headers: browserHeaders(),
+        redirect: 'follow',
+        signal: AbortSignal.timeout(15000),
+      });
+    } catch {
+      throw new Error(`Could not reach ${url}`);
+    }
+
+    const reader = resp.body.getReader();
+    const decoder = new TextDecoder('utf-8', { fatal: false });
+    let html = '';
+    let received = 0;
+    while (received < MAX_BYTES) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      received += value.length;
+      html += decoder.decode(value, { stream: true });
+    }
+    reader.cancel();
+    html += decoder.decode();
+
+    return { url: resp.url || url, html, status: resp.status };
+  }
+
+  // --- Registry fallback (npm + PyPI) via their open JSON APIs ---------- //
+
+  static async _registryFallback(url, maxText) {
+    const npm = this._npmSpec(url);
+    if (npm) {
+      try {
+        return await this._npmFallback(npm, url, maxText);
+      } catch {}
+    }
+    const pypi = this._pypiSpec(url);
+    if (pypi) {
+      try {
+        return await this._pypiFallback(pypi, url, maxText);
+      } catch {}
+    }
+    return null;
+  }
+
+  static _npmSpec(url) {
+    const m = /npmjs\.\w+\/package\/([@a-zA-Z0-9._~-]+(?:\/[@a-zA-Z0-9._~-]+)?)/i.exec(url);
+    return m ? m[1] : null;
+  }
+
+  static _pypiSpec(url) {
+    const m = /pypi\.org\/project\/([a-zA-Z0-9._-]+)/i.exec(url);
+    return m ? m[1] : null;
+  }
+
+  static async _fetchJson(apiUrl) {
+    const resp = await fetch(apiUrl, {
+      headers: browserHeaders('application/json'),
+      redirect: 'follow',
+      signal: AbortSignal.timeout(15000),
+    });
+    if (!resp.ok) throw new Error(`API ${resp.status}`);
+    return resp.json();
+  }
+
+  static async _npmFallback(spec, pageUrl, maxText) {
+    const pkg = encodeURIComponent(spec); // '@vierratale/ai' -> %40vierratale%2Fai
+    const data = await this._fetchJson(`https://registry.npmjs.org/${pkg}`);
+    const version =
+      (data['dist-tags'] && (data['dist-tags'].latest || data['dist-tags'].beta)) ||
+      Object.keys(data.versions || {})[0];
+    const meta = (data.versions && data.versions[version]) || {};
+    const deps = meta.dependencies || data.dependencies || {};
+    const lines = [
+      `${data.name} (npm package)`,
+      `Latest version: ${version}`,
+      meta.description || data.description || '',
+      `License: ${meta.license || data.license || 'N/A'}`,
+      `Dependencies: ${Object.keys(deps).length}`,
+      '',
+    ];
+    const readme = String(data.readme || '').trim();
+    if (readme) lines.push(readme.slice(0, maxText));
+    return {
+      url: pageUrl,
+      title: `${data.name} · npm`,
+      text: lines.filter(Boolean).join('\n').slice(0, maxText),
+    };
+  }
+
+  static async _pypiFallback(spec, pageUrl, maxText) {
+    const data = await this._fetchJson(`https://pypi.org/pypi/${encodeURIComponent(spec)}/json`);
+    const info = data.info || {};
+    const lines = [
+      `${info.name} (PyPI package)`,
+      `Version: ${info.version}`,
+      info.summary || '',
+      `Requires Python: ${info.requires_python || 'N/A'}`,
+      `License: ${info.license || 'N/A'}`,
+      `Home: ${info.home_page || info.project_url || 'N/A'}`,
+      '',
+    ];
+    const desc = String(info.description || '').trim();
+    if (desc) lines.push(desc.slice(0, maxText));
+    return {
+      url: pageUrl,
+      title: `${info.name} · PyPI`,
+      text: lines.filter(Boolean).join('\n').slice(0, maxText),
+    };
+  }
+
+  // --- extraction -------------------------------------------------------- //
+
+  static _extractAssets(html, baseUrl) {
+    const found = [];
+    const seen = new Set();
+    const refs = [];
+    const hrefRe = /<a[^>]+href=["']([^"']+)["'][^>]*>/gi;
+    let m;
+    while ((m = hrefRe.exec(html)) !== null) refs.push([m[1], 'a']);
+    const srcRe = /<(?:img|source|video|audio|embed|iframe)[^>]+src=["']([^"']+)["'][^>]*>/gi;
+    while ((m = srcRe.exec(html)) !== null) refs.push([m[1], 'src']);
+    const dataRe = /<(?:a|source)[^>]+data-(?:file|download|src)=["']([^"']+)["'][^>]*>/gi;
+    while ((m = dataRe.exec(html)) !== null) refs.push([m[1], 'data']);
+
+    for (const [raw, attr] of refs) {
+      const resolved = this._resolveAssetUrl(raw, baseUrl);
+      if (!resolved || seen.has(resolved)) continue;
+      const rel = attr === 'a' || attr === 'data' ? raw : raw;
+      if (rel.startsWith('tel:') || rel.startsWith('mailto:') || rel.startsWith('javascript:')) continue;
+      const type = detectAssetType(resolved.pathname);
+      if (!type) continue;
+      if (resolved.pathname.endsWith('.html') || resolved.pathname.endsWith('.htm')) continue;
+      seen.add(resolved.href);
+      found.push({
+        url: resolved.href,
+        name: this._assetName(resolved, type),
+        type,
+      });
+      if (found.length >= 30) break;
+    }
+    return found.sort((x, y) => x.type.localeCompare(y.type));
+  }
+
+  static _resolveAssetUrl(raw, baseUrl) {
+    try {
+      const href = String(raw || '').trim().split('#')[0];
+      if (!href || /^data:/i.test(href)) return null;
+      const url = new URL(href, baseUrl);
+      if (url.protocol !== 'http:' && url.protocol !== 'https:') return null;
+      return url;
+    } catch {
+      return null;
+    }
+  }
+
+  static _assetName(url, type) {
+    const path = url.pathname.split('/').filter(Boolean);
+    let name = path.length ? decodeURIComponent(path[path.length - 1]) : '';
+    if (!name || type === 'unknown') name = `${url.hostname.replace(/[^a-z0-9.-]/gi, '_')}.${type}`;
+    if (!name.includes('.')) name = `${name}.${type}`;
+    return name.slice(0, 120);
+  }
+
+  static _extractTitle(html) {
+    const m = /<title[^>]*>([^<]*)<\/title>/i.exec(html);
+    return m ? this._stripEntities(m[1].trim()) : '';
+  }
+
+  static _extractText(html) {
+    // Strip block tags' contents that add no readable value.
+    let s = html.replace(/<(script|style|noscript|svg|head|iframe|form|nav|footer|aside)[^>]*>[\s\S]*?<\/\1>/gi, ' ');
+
+    // Force spacing around block-level elements so words don't merge.
+    s = s.replace(/<\/(p|div|h[1-6]|li|tr|br|section|article)>/gi, '\n');
+    s = s.replace(/<(br|li|tr)[^>]*>/gi, '\n');
+
+    // Remove remaining tags.
+    s = s.replace(/<[^>]+>/g, ' ');
+
+    // College entities.
+    s = this._stripEntities(s);
+
+    // Collapse whitespace and trim lines.
+    return s
+      .replace(/[ \t]+/g, ' ')
+      .replace(/ *\n */g, '\n')
+      .replace(/\n{3,}/g, '\n\n')
+      .replace(/[ \t]+/g, ' ')
+      .replace(/\u00a0/g, ' ')
+      .trim();
+  }
+
+  static _stripEntities(text) {
+    const map = {
+      '&amp;': '&', '&lt;': '<', '&gt;': '>', '&quot;': '"',
+      '&#39;': "'", '&#039;': "'", '&apos;': "'", '&nbsp;': ' ',
+      '&ndash;': '–', '&mdash;': '—', '&hellip;': '...', '&copy;': '©',
+      '&#x27;': "'", '&rsquo;': '’', '&lsquo;': '‘', '&ldquo;': '“', '&rdquo;': '”',
+    };
+    return String(text)
+      .replace(/<[^>]*>/g, '')
+      .replace(/&[a-zA-Z0-9#]+;/g, (m) => map[m] ?? '')
+      .replace(/\s+/g, ' ')
+      .trim();
+  }
+}