jaitus 0.3.0

package/src/agent/websearch.js

@@ -0,0 +1,522 @@
+// ─── Searching the web ──────────────────────────────────────────────────────────
+//
+// The old arrangement asked one engine, and if it returned anything at all, that was
+// the answer. Asking Bing for "ink react terminal library" returns a hospital in
+// Kolkata, a Netflix trailer and Flipkart — it drops the qualifying words and answers
+// the first one, with local bias — and because it returned *ten* results the fallbacks
+// were never consulted. A confident wrong answer is worse than an error, and the agent
+// then fetched those pages and reasoned about them.
+//
+// Two changes fix it:
+//
+//   • **Ask several engines at once and merge.** Agreement between independent engines
+//     is the only relevance signal available without an API key, and it is a good one:
+//     a URL two engines both return for a query is almost never a hospital.
+//   • **Score the results against the query.** A result whose title and snippet contain
+//     none of the distinctive words in the query is dropped, however confidently it was
+//     returned. That single filter removes every one of the bad results above.
+//
+// A real search API is better than all of this and is used when one is configured. The
+// scrapers are the free path, not the good one, and the output says which was used so
+// nobody mistakes one for the other.
+
+const BROWSER_HEADERS = {
+  'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+  'accept': 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.8',
+  'accept-language': 'en-US,en;q=0.9',
+};
+
+const TIMEOUT = 12_000;
+const stripTags = (s) => String(s ?? '')
+  .replace(/<[^>]*>/g, '')
+  .replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
+  .replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&nbsp;/g, ' ')
+  .replace(/\s+/g, ' ')
+  .trim();
+
+// ─── Relevance ──────────────────────────────────────────────────────────────────
+
+const STOP = new Set('a an and are as at be but by for from how in is it of on or that the to was what when where which who why with your you i we can do does'.split(' '));
+
+/** The words in a query that a relevant result ought to mention. */
+function terms(query) {
+  return String(query).toLowerCase().match(/[a-z0-9][a-z0-9.+#-]*/g)?.filter((w) => w.length > 1 && !STOP.has(w)) || [];
+}
+
+/**
+ * How well one result answers the query.
+ *
+ * Two numbers, and COVERAGE is the one that decides. Searching for
+ * "ink react terminal library" returns a Wikipedia article about a university library
+ * that happens to mention terminals: it scores well on any measure that adds up how
+ * often the query's words appear, because it genuinely contains two of them a lot. What
+ * it does not contain is "ink" or "react" — and a result missing the distinctive half
+ * of the query is not a weak answer, it is a different subject.
+ *
+ * So coverage — the fraction of query terms present at all — gates the result, and
+ * placement only orders the ones that survive. Title and URL count double there,
+ * because a page whose title carries the words is about them, whereas a page that
+ * mentions them in a snippet may only be adjacent to them.
+ */
+function relevance(result, wanted) {
+  if (!wanted.length) return { coverage: 1, placement: 1 };
+  const title = String(result.title || '').toLowerCase();
+  const snippet = String(result.snippet || '').toLowerCase();
+  const url = String(result.url || '').toLowerCase();
+  let present = 0;
+  let placement = 0;
+  for (const term of wanted) {
+    if (title.includes(term)) { present += 1; placement += 2; }
+    else if (url.includes(term)) { present += 1; placement += 2; }
+    else if (snippet.includes(term)) { present += 1; placement += 1; }
+  }
+  return { coverage: present / wanted.length, placement: placement / (wanted.length * 2) };
+}
+
+/**
+ * How much of the query a result has to contain.
+ *
+ * A one- or two-word query is usually a proper noun and every result will contain it,
+ * so the bar is low. The longer and more specific the query, the more of it a genuine
+ * answer carries — and the more aggressively a partial match should be dropped.
+ */
+const coverageFloor = (termCount) => (termCount <= 2 ? 0.5 : termCount <= 4 ? 0.6 : 0.5);
+
+// ─── Engines ────────────────────────────────────────────────────────────────────
+
+async function get(url, extraHeaders) {
+  const res = await fetch(url, { headers: { ...BROWSER_HEADERS, ...extraHeaders }, redirect: 'follow', signal: AbortSignal.timeout(TIMEOUT) });
+  if (!res.ok) throw new Error(`HTTP ${res.status}`);
+  return res;
+}
+
+/** Bing. `ensearch=1` + an explicit market is what stops the localised answer. */
+async function bing(query) {
+  const res = await get(`https://www.bing.com/search?q=${encodeURIComponent(query)}&count=20&setmkt=en-US&setlang=en&ensearch=1`);
+  const html = await res.text();
+  const out = [];
+  for (const block of html.split('<li class="b_algo').slice(1)) {
+    const href = /<h2[^>]*><a[^>]+href="([^"]+)"/.exec(block);
+    const h2 = /<h2[^>]*>([\s\S]*?)<\/h2>/.exec(block);
+    if (!href || !h2) continue;
+    const url = unwrapBing(href[1]);
+    const title = stripTags(h2[1]);
+    const snippet = stripTags((/<p[^>]*>([\s\S]*?)<\/p>/.exec(block) || [])[1] || '');
+    if (url && title) out.push({ url, title, snippet });
+  }
+  return out;
+}
+
+/** Bing wraps every link in a ck/a redirect whose `u=a1<base64url>` is the real URL. */
+function unwrapBing(href) {
+  const clean = href.replace(/&amp;/g, '&');
+  if (!clean.includes('bing.com/ck/')) return clean;
+  const u = /[?&]u=a1([^&]+)/.exec(clean);
+  if (!u) return clean;
+  let b64 = u[1].replace(/-/g, '+').replace(/_/g, '/');
+  while (b64.length % 4) b64 += '=';
+  try {
+    const decoded = Buffer.from(b64, 'base64').toString('utf8');
+    return /^https?:\/\//.test(decoded) ? decoded : clean;
+  } catch { return clean; }
+}
+
+async function duckduckgo(query) {
+  const res = await get(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`);
+  const html = await res.text();
+  const out = [];
+  const re = /<a[^>]+class="result__a"[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/g;
+  const snippets = [...html.matchAll(/<a[^>]+class="result__snippet"[^>]*>([\s\S]*?)<\/a>/g)].map((m) => stripTags(m[1]));
+  let m;
+  let i = 0;
+  while ((m = re.exec(html))) {
+    let href = m[1].replace(/&amp;/g, '&');
+    const uddg = /[?&]uddg=([^&]+)/.exec(href);
+    if (uddg) href = decodeURIComponent(uddg[1]);
+    if (href.startsWith('//')) href = `https:${href}`;
+    const title = stripTags(m[2]);
+    if (title && /^https?:\/\//.test(href)) out.push({ url: href, title, snippet: snippets[i] || '' });
+    i += 1;
+  }
+  return out;
+}
+
+async function mojeek(query) {
+  const res = await get(`https://www.mojeek.com/search?q=${encodeURIComponent(query)}`);
+  const html = await res.text();
+  const out = [];
+  // Mojeek's markup moves; match the result list rather than one class name that was
+  // right the day it was written.
+  for (const m of html.matchAll(/<a[^>]+href="(https?:\/\/[^"]+)"[^>]*>\s*<?h2?[^>]*>?([\s\S]{0,180}?)<\/(?:h2|a)>/g)) {
+    const title = stripTags(m[2]);
+    if (title.length > 3 && !/mojeek/i.test(m[1])) out.push({ url: m[1].replace(/&amp;/g, '&'), title, snippet: '' });
+    if (out.length >= 15) break;
+  }
+  return out;
+}
+
+async function startpage(query) {
+  const res = await get(`https://www.startpage.com/sp/search?query=${encodeURIComponent(query)}`);
+  const html = await res.text();
+  const out = [];
+  for (const m of html.matchAll(/<a[^>]+class="[^"]*result-link[^"]*"[^>]+href="(https?:\/\/[^"]+)"[^>]*>([\s\S]*?)<\/a>/g)) {
+    const title = stripTags(m[2]);
+    if (title) out.push({ url: m[1].replace(/&amp;/g, '&'), title, snippet: '' });
+  }
+  return out;
+}
+
+// ─── The indexes that actually know the answer ──────────────────────────────────
+//
+// Almost every search a coding agent runs is about a package, a repository, a web API
+// or an error someone else has already hit. Those things all have real, public, keyless
+// search APIs that return ranked, structured, correct results in under a second — which
+// is a different category of thing from scraping a general search engine's HTML and
+// hoping it did not decide the query was about printer ink.
+//
+// They are queried alongside the general engines rather than instead of them: a genuine
+// index that returns nothing costs half a second, and one that returns something has
+// almost certainly returned the right thing.
+
+async function npmSearch(query) {
+  const res = await get(`https://registry.npmjs.org/-/v1/search?text=${encodeURIComponent(query)}&size=6`, { accept: 'application/json' });
+  const body = await res.json();
+  return (body?.objects || []).map((o) => ({
+    url: o.package?.links?.npm || `https://www.npmjs.com/package/${o.package?.name}`,
+    title: `${o.package?.name}${o.package?.version ? ` ${o.package.version}` : ''} (npm)`,
+    snippet: [o.package?.description, o.package?.links?.repository].filter(Boolean).join(' — ').slice(0, 300),
+  })).filter((r) => r.url);
+}
+
+async function githubSearch(query) {
+  const res = await get(`https://api.github.com/search/repositories?q=${encodeURIComponent(query)}&per_page=6`, {
+    accept: 'application/vnd.github+json',
+  });
+  const body = await res.json();
+  return (body?.items || []).map((o) => ({
+    url: o.html_url,
+    title: `${o.full_name} · ★${o.stargazers_count} (GitHub)`,
+    snippet: String(o.description || '').slice(0, 300),
+    // GitHub's best-match ranking will happily put a three-star demo above the library
+    // it demonstrates. Popularity is a weak signal but it is the right tiebreak here:
+    // when two repos match a query equally, the one everyone uses is the one meant.
+    boost: Math.log10(1 + (Number(o.stargazers_count) || 0)) / 2,
+  }));
+}
+
+async function stackoverflow(query) {
+  const res = await get(
+    `https://api.stackexchange.com/2.3/search/advanced?order=desc&sort=relevance&q=${encodeURIComponent(query)}&site=stackoverflow&pagesize=6&filter=default`,
+    { accept: 'application/json' },
+  );
+  const body = await res.json();
+  return (body?.items || []).map((o) => ({
+    url: o.link,
+    title: `${stripTags(o.title)}${o.is_answered ? ' ✓answered' : ''} (Stack Overflow)`,
+    snippet: `score ${o.score}, ${o.answer_count} answer${o.answer_count === 1 ? '' : 's'}`,
+  }));
+}
+
+async function mdn(query) {
+  const res = await get(`https://developer.mozilla.org/api/v1/search?q=${encodeURIComponent(query)}`, { accept: 'application/json' });
+  const body = await res.json();
+  return (body?.documents || []).slice(0, 6).map((d) => ({
+    url: `https://developer.mozilla.org${d.mdn_url}`,
+    title: `${d.title} (MDN)`,
+    snippet: String(d.summary || '').slice(0, 300),
+  }));
+}
+
+async function crates(query) {
+  const res = await get(`https://crates.io/api/v1/crates?q=${encodeURIComponent(query)}&per_page=5`, { accept: 'application/json' });
+  const body = await res.json();
+  // Named `crate`, not `c`: `c` is the colour helper everywhere else in this codebase
+  // and a whole-source check greps for `c.<name>` to catch typo'd colours.
+  return (body?.crates || []).map((crate) => ({
+    url: `https://crates.io/crates/${crate.name}`,
+    title: `${crate.name} ${crate.max_version || ''} (crates.io)`,
+    snippet: String(crate.description || '').slice(0, 300),
+  }));
+}
+
+async function pypi(query) {
+  // PyPI has no search API any more; the JSON endpoint is exact-name only, which is
+  // still useful when the query names a package and useless otherwise.
+  const name = String(query).trim().split(/\s+/)[0].replace(/[^a-zA-Z0-9._-]/g, '');
+  if (!name) return [];
+  const res = await get(`https://pypi.org/pypi/${encodeURIComponent(name)}/json`, { accept: 'application/json' });
+  const body = await res.json();
+  if (!body?.info) return [];
+  return [{
+    url: body.info.package_url || `https://pypi.org/project/${name}/`,
+    title: `${body.info.name} ${body.info.version} (PyPI)`,
+    snippet: String(body.info.summary || '').slice(0, 300),
+  }];
+}
+
+/** Which language-specific indexes are worth asking, from what the query mentions. */
+function languageEngines(query) {
+  const q = query.toLowerCase();
+  const out = [];
+  if (/\b(rust|cargo|crate|tokio|serde)\b/.test(q)) out.push(['crates.io', () => crates(query)]);
+  if (/\b(python|pip|pypi|django|flask|pandas|numpy)\b/.test(q)) out.push(['pypi', () => pypi(query)]);
+  return out;
+}
+
+/** Wikipedia's own search: narrow, but exact, and it answers when the others do not. */
+async function wikipedia(query) {
+  const res = await get(
+    `https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch=${encodeURIComponent(query)}&srlimit=5&format=json&origin=*`,
+    { accept: 'application/json' },
+  );
+  const body = await res.json();
+  return (body?.query?.search || []).map((hit) => ({
+    url: `https://en.wikipedia.org/wiki/${encodeURIComponent(String(hit.title).replace(/ /g, '_'))}`,
+    title: String(hit.title),
+    snippet: stripTags(hit.snippet || ''),
+  }));
+}
+
+/** Google News RSS: dated, news-only, and the right index for "what happened today". */
+async function googleNews(query, { headlines = false } = {}) {
+  const endpoint = headlines
+    ? 'https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en'
+    : `https://news.google.com/rss/search?q=${encodeURIComponent(query)}&hl=en-US&gl=US&ceid=US:en`;
+  const res = await get(endpoint, { accept: 'application/rss+xml, application/xml;q=0.9' });
+  const xml = await res.text();
+  const out = [];
+  for (const item of xml.match(/<item>[\s\S]*?<\/item>/g) || []) {
+    const title = xmlText(item, 'title');
+    const url = xmlText(item, 'link');
+    const date = xmlText(item, 'pubDate');
+    if (title && /^https?:\/\//.test(url)) {
+      out.push({ url, title, snippet: [date, stripTags(xmlText(item, 'description'))].filter(Boolean).join(' — ').slice(0, 300) });
+    }
+    if (out.length >= 12) break;
+  }
+  return out;
+}
+
+function xmlText(source, name) {
+  const match = new RegExp(`<${name}>([\\s\\S]*?)<\\/${name}>`, 'i').exec(source);
+  if (!match) return '';
+  return match[1]
+    .replace(/^<!\[CDATA\[([\s\S]*)\]\]>$/, '$1')
+    .replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"').replace(/&#39;/g, "'")
+    .trim();
+}
+
+// ─── Paid APIs, when one is configured ──────────────────────────────────────────
+//
+// Any of these beats every scraper above, and none of them is required. The key is read
+// from settings or the environment so it never has to be typed into a prompt.
+
+async function brave(query, key) {
+  const res = await get(`https://api.search.brave.com/res/v1/web/search?q=${encodeURIComponent(query)}&count=10`, {
+    accept: 'application/json', 'x-subscription-token': key,
+  });
+  const body = await res.json();
+  return (body?.web?.results || []).map((r) => ({ url: r.url, title: r.title, snippet: stripTags(r.description || '') }));
+}
+
+async function tavily(query, key) {
+  const res = await fetch('https://api.tavily.com/search', {
+    method: 'POST',
+    headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` },
+    body: JSON.stringify({ query, max_results: 10 }),
+    signal: AbortSignal.timeout(TIMEOUT),
+  });
+  if (!res.ok) throw new Error(`HTTP ${res.status}`);
+  const body = await res.json();
+  return (body?.results || []).map((r) => ({ url: r.url, title: r.title, snippet: String(r.content || '').slice(0, 300) }));
+}
+
+async function serper(query, key) {
+  const res = await fetch('https://google.serper.dev/search', {
+    method: 'POST',
+    headers: { 'content-type': 'application/json', 'X-API-KEY': key },
+    body: JSON.stringify({ q: query, num: 10 }),
+    signal: AbortSignal.timeout(TIMEOUT),
+  });
+  if (!res.ok) throw new Error(`HTTP ${res.status}`);
+  const body = await res.json();
+  return (body?.organic || []).map((r) => ({ url: r.link, title: r.title, snippet: r.snippet || '' }));
+}
+
+function configuredApi(settings) {
+  const pick = (a, b) => (typeof a === 'string' && a.trim() ? a.trim() : (typeof b === 'string' && b.trim() ? b.trim() : ''));
+  const braveKey = pick(settings?.braveSearchKey, process.env.BRAVE_SEARCH_API_KEY);
+  if (braveKey) return { name: 'brave', run: (q) => brave(q, braveKey) };
+  const tavilyKey = pick(settings?.tavilyKey, process.env.TAVILY_API_KEY);
+  if (tavilyKey) return { name: 'tavily', run: (q) => tavily(q, tavilyKey) };
+  const serperKey = pick(settings?.serperKey, process.env.SERPER_API_KEY);
+  if (serperKey) return { name: 'serper', run: (q) => serper(q, serperKey) };
+  return null;
+}
+
+// ─── The search ─────────────────────────────────────────────────────────────────
+
+const isNews = (q) => /\b(news|headlines?|breaking|today|latest|this week|current events?)\b/i.test(q);
+const isHeadlines = (q) => !q.toLowerCase()
+  .replace(/\b(top|latest|today|breaking|news|headlines?|current|events?|world|global|updates?|the|me|show|give)\b/g, '')
+  .replace(/[^a-z0-9]+/g, '')
+  .length;
+
+/** Two URLs that differ only in tracking junk or a trailing slash are one URL. */
+function canonical(url) {
+  try {
+    const u = new URL(url);
+    u.hash = '';
+    for (const key of [...u.searchParams.keys()]) {
+      if (/^(utm_|fbclid|gclid|ref|source|mc_cid|mc_eid)/i.test(key)) u.searchParams.delete(key);
+    }
+    return `${u.protocol}//${u.host.replace(/^www\./, '')}${u.pathname.replace(/\/$/, '')}${u.search}`.toLowerCase();
+  } catch {
+    return String(url).toLowerCase();
+  }
+}
+
+/**
+ * Search the web.
+ *
+ * Returns `{ results, engines, notes }`. `engines` names what actually answered, which
+ * matters because the quality difference between a configured API and a scrape is
+ * large and the caller should be able to say so.
+ */
+export async function searchWeb(query, { settings = null, limit = 8, deadlineMs = 6000 } = {}) {
+  const q = String(query || '').trim();
+  if (!q) throw new Error('No query given.');
+
+  const api = configuredApi(settings);
+  const wanted = terms(q);
+  const notes = [];
+
+  // A configured API is trusted on its own — it is a ranked answer, not a scrape, and
+  // second-guessing it with a keyword filter would throw away good results.
+  if (api) {
+    try {
+      const results = await api.run(q);
+      if (results.length) return { results: results.slice(0, limit), engines: [api.name], notes };
+      notes.push(`${api.name} returned nothing; falling back to the free engines.`);
+    } catch (error) {
+      notes.push(`${api.name} failed (${error.message}); falling back to the free engines.`);
+    }
+  }
+
+  const engines = isNews(q)
+    ? [['google-news', () => googleNews(q, { headlines: isHeadlines(q) })], ['bing', () => bing(q)], ['duckduckgo', () => duckduckgo(q)]]
+    : [
+      ['bing', () => bing(q)],
+      ['duckduckgo', () => duckduckgo(q)],
+      ['mojeek', () => mojeek(q)],
+      ['startpage', () => startpage(q)],
+      // Real indexes, always asked. They are faster than every scraper here and when
+      // they answer at all they answer correctly, which is not something that can be
+      // said of a general engine's free HTML endpoint.
+      ['npm', () => npmSearch(q)],
+      ['github', () => githubSearch(q)],
+      ['stackoverflow', () => stackoverflow(q)],
+      ['mdn', () => mdn(q)],
+      ...languageEngines(q),
+    ];
+
+  // Wikipedia is deliberately NOT in the parallel set. It answers every query with
+  // something, because almost any pair of English words appears in some article, and
+  // those answers crowd out real ones — a search for a React library returned a
+  // university library. It is a fallback for when nothing else answered, not a peer.
+  const merged = new Map();
+  const answered = [];
+  const failures = [];
+  const absorb = (name, results) => {
+    if (!results?.length) { failures.push(`${name}: no results`); return; }
+    answered.push(name);
+    results.forEach((result, position) => {
+      const key = canonical(result.url);
+      const existing = merged.get(key);
+      if (existing) {
+        existing.engines.add(name);
+        existing.best = Math.min(existing.best, position);
+        existing.boost = Math.max(existing.boost || 0, result.boost || 0);
+        if (!existing.snippet && result.snippet) existing.snippet = result.snippet;
+      } else {
+        merged.set(key, { ...result, engines: new Set([name]), best: position });
+      }
+    });
+  };
+
+  // In parallel, not in sequence — sequentially, the first engine to return anything
+  // won, including when what it returned was irrelevant. Bounded by a deadline as well
+  // as by each engine's own timeout: one unreachable engine should cost a second of
+  // patience, not the full timeout, when two others have already answered.
+  let settle = () => {};
+  const enough = new Promise((resolve) => { settle = resolve; });
+  let answeredCount = 0;
+  await Promise.race([
+    Promise.allSettled(engines.map(async ([name, run]) => {
+      try {
+        absorb(name, await run());
+        // Once a few independent engines have answered there is enough to rank. Waiting
+        // for the slowest — which is usually one that is unreachable from this network
+        // and will time out — costs seconds for nothing.
+        if (++answeredCount >= 3) { const t = setTimeout(settle, 700); if (t.unref) t.unref(); }
+      } catch (error) { failures.push(`${name}: ${error.message}`); }
+    })),
+    enough,
+    new Promise((resolve) => { const t = setTimeout(resolve, deadlineMs); if (t.unref) t.unref(); }),
+  ]);
+
+  const floor = coverageFloor(wanted.length);
+  const rank = (list) => list.map((r) => {
+    const { coverage, placement } = relevance(r, wanted);
+    return {
+      ...r,
+      coverage,
+      // Agreement between independent engines is the strongest signal available without
+      // an API key; position within one engine's list is the weakest.
+      score: placement * 3 + coverage * 2 + (r.engines.size - 1) * 1.5 - r.best * 0.05 + (r.boost || 0),
+    };
+  }).sort((a, b) => b.score - a.score);
+
+  let scored = rank([...merged.values()]);
+  let kept = scored.filter((r) => r.coverage >= floor);
+
+  if (!kept.length) {
+    try {
+      absorb('wikipedia', await wikipedia(q));
+      scored = rank([...merged.values()]);
+      kept = scored.filter((r) => r.coverage >= floor);
+    } catch (error) { failures.push(`wikipedia: ${error.message}`); }
+  }
+
+  if (!merged.size) {
+    const error = new Error(`No engine returned results (${failures.join('; ')}).`);
+    error.failures = failures;
+    throw error;
+  }
+
+  if (!kept.length) {
+    // Everything missed most of the query. Return the closest few anyway — but say so,
+    // because the difference between "here are the answers" and "nothing matched, these
+    // are the nearest" is the difference between a correct answer and a confident wrong
+    // one, and the model cannot tell them apart unless it is told.
+    kept = scored.slice(0, Math.min(3, scored.length));
+    notes.push(`No result contained enough of the query to be trusted (best covered ${Math.round((scored[0]?.coverage || 0) * 100)}% of its terms). These are the closest and may be about something else — rephrase, or fetch a known URL directly.`);
+  }
+
+  if (failures.length) notes.push(`Engines that did not answer: ${failures.join('; ')}.`);
+  return { results: kept.slice(0, limit), engines: answered, notes };
+}
+
+/** Format for the model: numbered, with the URL on its own line so it can be copied. */
+export function formatResults({ results, engines, notes }) {
+  const lines = results.map((r, i) => {
+    const agreement = r.engines && r.engines.size > 1 ? ` [${r.engines.size} engines agree]` : '';
+    return `${i + 1}. ${r.title}${agreement}\n   ${r.url}${r.snippet ? `\n   ${r.snippet}` : ''}`;
+  });
+  const footer = [
+    `\n(searched via ${engines.join(', ') || 'none'})`,
+    ...notes.map((n) => `Note: ${n}`),
+  ].join('\n');
+  return `${lines.join('\n')}\n${footer}`;
+}

package/src/api.js

@@ -0,0 +1,72 @@
+// ─── Talking to Jaitus ──────────────────────────────────────────────────────────
+//
+// The CLI is the only client of the gateway. Every failure it gets back is turned into
+// a JaitusError here, at the edge, so no caller ever has an HTTP body in its hands to
+// print — see errors.js for why that matters.
+
+import { endpoints } from './config.js';
+import { signedIn, withAuth } from './session.js';
+import { fromGatewayBody, fromNetwork, JaitusError } from './errors.js';
+
+export class NotSignedIn extends Error {}
+
+export function createClient() {
+  const { baseUrl } = endpoints();
+  if (!signedIn()) throw new NotSignedIn('Not signed in. Run `jaitus login` first.');
+
+  const base = { 'Content-Type': 'application/json', 'user-agent': 'jaitus-cli' };
+  const headers = (token) => ({ ...base, authorization: `Bearer ${token}` });
+
+  return {
+    baseUrl,
+
+    /** POST /v1/messages. Returns the raw Response so the caller can stream it. */
+    async messages(body, { signal } = {}) {
+      try {
+        return await withAuth((token) => fetch(`${baseUrl}/v1/messages`, {
+          method: 'POST',
+          headers: headers(token),
+          body: JSON.stringify(body),
+          signal,
+        }));
+      } catch (cause) {
+        if (cause?.name === 'JaitusError' || cause?.name === 'AbortError') throw cause;
+        throw fromNetwork(cause);
+      }
+    },
+
+    /** A dashboard API call, returning parsed JSON. */
+    async api(pathname, init = {}) {
+      let res;
+      try {
+        res = await withAuth((token) => fetch(`${baseUrl}/api${pathname}`, {
+          ...init,
+          headers: { ...headers(token), ...(init.headers || {}) },
+        }));
+      } catch (cause) {
+        if (cause?.name === 'JaitusError') throw cause;
+        throw fromNetwork(cause);
+      }
+      const body = await res.json().catch(() => ({}));
+      return { ok: res.ok, status: res.status, body };
+    },
+
+    /** Model ids the gateway advertises. */
+    async models() {
+      let res;
+      try {
+        res = await withAuth((token) => fetch(`${baseUrl}/v1/models`, { headers: headers(token) }));
+      } catch (cause) {
+        if (cause?.name === 'JaitusError') throw cause;
+        throw fromNetwork(cause);
+      }
+      const body = await res.json().catch(() => ({}));
+      if (!res.ok) throw fromGatewayBody(res.status, body, { requestId: res.headers.get('x-request-id') || '' });
+      const models = (body?.data || []).map((m) => m?.id).filter(Boolean);
+      // An empty list is a service fault, not something the user can fix — and naming
+      // the gateway's model map to them would be naming our internals.
+      if (!models.length) throw new JaitusError({ code: 'unavailable', status: 502 });
+      return models;
+    },
+  };
+}

package/src/background.js

@@ -0,0 +1,85 @@
+// ─── Background shell jobs (§16) ───────────────────────────────────────────────
+//
+// Long-running commands (dev servers, builds, watchers) run detached with
+// output streaming to a per-job log file. The agent keeps working and polls
+// with background_read; /tasks lists jobs with attach/stop. `! cmd &` and
+// Ctrl+B start one from the prompt; background_run does it from the model.
+
+import { spawn } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+const jobs = new Map();
+let seq = 0;
+
+function logDir() {
+  const d = path.join(os.homedir(), '.jaitus', 'background');
+  fs.mkdirSync(d, { recursive: true });
+  return d;
+}
+
+export function startBackground(command, cwd) {
+  const id = `bg${++seq}-${Date.now().toString(36)}`;
+  const log = path.join(logDir(), `${id}.log`);
+  const out = fs.openSync(log, 'w');
+  const child = spawn(command, {
+    cwd, shell: true, stdio: ['ignore', out, out],
+    detached: process.platform !== 'win32',
+    windowsHide: true,
+  });
+  const job = { id, command, pid: child.pid || null, log, startedAt: Date.now(), exit: null };
+  child.on('error', (e) => {
+    try { fs.appendFileSync(log, `\n[failed to start: ${e.message}]\n`); } catch { /* log gone */ }
+    job.exit = 'error';
+  });
+  child.on('close', (code) => {
+    job.exit = code ?? 'signal';
+    try { fs.appendFileSync(log, `\n[exit ${job.exit}]\n`); } catch { /* log gone */ }
+  });
+  try { child.unref(); } catch { /* stay attached */ }
+  job.child = child;
+  jobs.set(id, job);
+  return { id, pid: job.pid, log };
+}
+
+export function listBackground() {
+  return [...jobs.values()].map((j) => ({
+    id: j.id, command: j.command, pid: j.pid,
+    running: j.exit === null, startedAt: j.startedAt,
+    exit: j.exit === null ? null : j.exit,
+  }));
+}
+
+function findJob(id) {
+  const direct = jobs.get(id);
+  if (direct) return direct;
+  const prefix = [...jobs.values()].filter((j) => j.id.startsWith(id));
+  return prefix.length === 1 ? prefix[0] : null;
+}
+
+export function readBackground(id, tail = 40) {
+  const job = findJob(id);
+  if (!job) return null;
+  let lines = [];
+  try {
+    lines = fs.readFileSync(job.log, 'utf8').split('\n');
+  } catch { /* empty so far */ }
+  return {
+    id: job.id, command: job.command, running: job.exit === null, exit: job.exit,
+    tail: lines.slice(-Math.max(1, tail)).join('\n').slice(-6000) || '(no output yet)',
+  };
+}
+
+export function stopBackground(id) {
+  const job = findJob(id);
+  if (!job) return false;
+  if (job.exit !== null) return true;
+  try {
+    job.child?.kill();
+    if (job.child?.pid && process.platform !== 'win32') {
+      try { process.kill(-job.child.pid); } catch { /* already gone */ }
+    }
+  } catch { /* already gone */ }
+  return true;
+}