@superjackfruitlabs/salaam 0.0.4 → 0.0.5

package/src/web-guard.ts

@@ -0,0 +1,195 @@
+import { BlockList, isIP } from 'node:net';
+import { lookup as dnsLookup } from 'node:dns/promises';
+import { request as httpRequest, type IncomingMessage } from 'node:http';
+import { request as httpsRequest } from 'node:https';
+
+/**
+ * Outbound HTTP for web search and fetch. These run in the daemon, outside the command sandbox, so
+ * every connection is checked at connect time: the URL's literal address, every address its name
+ * resolves to (a DNS answer that mixes public and private is refused, so rebinding cannot land on
+ * a private one), and every redirect hop. Bodies are capped and requests time out.
+ */
+const denied = new BlockList();
+for (const [net, bits] of [
+  ['0.0.0.0', 8],
+  ['10.0.0.0', 8],
+  ['100.64.0.0', 10],
+  ['127.0.0.0', 8],
+  ['169.254.0.0', 16],
+  ['172.16.0.0', 12],
+  ['192.0.0.0', 24],
+  ['192.0.2.0', 24],
+  ['192.168.0.0', 16],
+  ['198.18.0.0', 15],
+  ['198.51.100.0', 24],
+  ['203.0.113.0', 24],
+  ['224.0.0.0', 4],
+  ['240.0.0.0', 4],
+] as const)
+  denied.addSubnet(net, bits, 'ipv4');
+for (const [net, bits] of [
+  ['::', 128],
+  ['::1', 128],
+  ['64:ff9b::', 96],
+  ['100::', 64],
+  ['2001:db8::', 32],
+  ['fc00::', 7],
+  ['fe80::', 10],
+  ['ff00::', 8],
+] as const)
+  denied.addSubnet(net, bits, 'ipv6');
+
+/** True only for globally routable unicast addresses. */
+export function isPublicAddress(address: string): boolean {
+  const bare = address.replace(/^\[|\]$/g, '').split('%')[0]!;
+  const family = isIP(bare);
+  if (!family) return false;
+  if (family === 6) {
+    const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(bare);
+    if (mapped) return isPublicAddress(mapped[1]!);
+    if (/^::ffff:/i.test(bare)) return false;
+    return !denied.check(bare, 'ipv6');
+  }
+  return !denied.check(bare, 'ipv4');
+}
+
+export interface Resolved {
+  address: string;
+  family: number;
+}
+export interface GuardOptions {
+  method?: 'GET' | 'POST';
+  headers?: Record<string, string>;
+  body?: string;
+  maxBytes?: number;
+  timeoutMs?: number;
+  maxRedirects?: number;
+  /** Injected resolver (tests); defaults to the system resolver, all addresses. */
+  lookup?: (host: string) => Promise<Resolved[]>;
+  /** Tests only: a loopback test server may be reached. */
+  allowLoopbackForTests?: boolean;
+  /** Tests only: loopback is allowed on the first hop but not after a redirect. */
+  allowLoopbackOnlyFirstHop?: boolean;
+}
+export interface GuardedResponse {
+  status: number;
+  url: string;
+  body: string;
+  truncated: boolean;
+  contentType: string;
+}
+
+const LOOPBACK = new Set(['127.0.0.1', '::1']);
+function allowed(address: string, loopbackOk: boolean): boolean {
+  return isPublicAddress(address) || (loopbackOk && LOOPBACK.has(address));
+}
+function refuse(host: string): Error {
+  return new Error(`Refused: ${host} is not a public address`);
+}
+
+export async function guardedRequest(
+  rawUrl: string,
+  opts: GuardOptions,
+  hop = 0,
+): Promise<GuardedResponse> {
+  let url: URL;
+  try {
+    url = new URL(rawUrl);
+  } catch {
+    throw new Error('Not a valid URL');
+  }
+  if (url.protocol !== 'http:' && url.protocol !== 'https:')
+    throw new Error('Only http or https URLs can be fetched');
+  if (url.username || url.password) throw new Error('URLs with credentials are refused');
+  const loopbackOk = !!opts.allowLoopbackForTests && !(opts.allowLoopbackOnlyFirstHop && hop > 0);
+  const host = url.hostname.replace(/^\[|\]$/g, '');
+  if (isIP(host) && !allowed(host, loopbackOk)) throw refuse(host);
+  const resolve = opts.lookup ?? (async (h: string) => dnsLookup(h, { all: true, verbatim: true }));
+  // Resolve once here, refuse if any answer is private, then connect only to a checked address.
+  const addresses: Resolved[] = isIP(host)
+    ? [{ address: host, family: isIP(host) }]
+    : await resolve(host);
+  if (!addresses.length || addresses.some((a) => !allowed(a.address, loopbackOk)))
+    throw refuse(host);
+  const pinned = addresses[0]!;
+  const maxBytes = opts.maxBytes ?? 2_000_000;
+  const timeoutMs = opts.timeoutMs ?? 15_000;
+  const res = await new Promise<IncomingMessage>((resolveRes, reject) => {
+    const req = (url.protocol === 'https:' ? httpsRequest : httpRequest)(
+      url,
+      {
+        method: opts.method ?? 'GET',
+        headers: {
+          'user-agent': 'salaam-web/1 (+https://salaam.sh)',
+          ...(opts.body ? { 'content-type': 'application/json' } : {}),
+          ...opts.headers,
+        },
+        // The connection goes to the address checked above, never a fresh DNS answer.
+        lookup: (_h, options, cb) =>
+          (options as { all?: boolean }).all
+            ? (cb as (e: null, a: Resolved[]) => void)(null, [pinned])
+            : (cb as (e: null, a: string, f: number) => void)(null, pinned.address, pinned.family),
+        timeout: timeoutMs,
+      },
+      resolveRes,
+    );
+    const timer = setTimeout(() => req.destroy(new Error('Request timed out')), timeoutMs);
+    req.on('timeout', () => req.destroy(new Error('Request timed out')));
+    req.on('error', (e) => {
+      clearTimeout(timer);
+      reject(e);
+    });
+    req.on('response', () => clearTimeout(timer));
+    if (opts.body) req.write(opts.body);
+    req.end();
+  });
+  const status = res.statusCode ?? 0;
+  if ([301, 302, 303, 307, 308].includes(status) && res.headers.location) {
+    res.resume();
+    if (hop >= (opts.maxRedirects ?? 5)) throw new Error('Too many redirects');
+    const next = new URL(res.headers.location, url).toString();
+    const keep = status === 307 || status === 308;
+    return guardedRequest(
+      next,
+      { ...opts, ...(keep ? {} : { method: 'GET', body: undefined }) },
+      hop + 1,
+    );
+  }
+  const chunks: Buffer[] = [];
+  let size = 0;
+  let truncated = false;
+  await new Promise<void>((done, fail) => {
+    const timer = setTimeout(() => res.destroy(new Error('Request timed out')), timeoutMs);
+    res.on('data', (c: Buffer) => {
+      if (truncated) return;
+      const room = maxBytes - size;
+      if (c.length >= room) {
+        chunks.push(c.subarray(0, room));
+        size += room;
+        truncated = true;
+        clearTimeout(timer);
+        res.destroy();
+        done();
+        return;
+      }
+      chunks.push(c);
+      size += c.length;
+    });
+    res.on('end', () => {
+      clearTimeout(timer);
+      done();
+    });
+    res.on('error', (e) => {
+      clearTimeout(timer);
+      if (truncated) done();
+      else fail(e);
+    });
+  });
+  return {
+    status,
+    url: url.toString(),
+    body: Buffer.concat(chunks).toString('utf8'),
+    truncated,
+    contentType: String(res.headers['content-type'] ?? ''),
+  };
+}

package/src/web-search.ts

@@ -0,0 +1,402 @@
+import { readFileSync, statSync } from 'node:fs';
+import { guardedRequest, type GuardedResponse } from './web-guard.js';
+import { decodeEntities, stripTags } from './web-fetch.js';
+
+/**
+ * Built-in web search for interactive sessions. Operator ruling 2026-10-31: "Official free APIs
+ * go as default and then other providers can be configured as per the user requirement. It could
+ * searxng or brave or tavily or exa." Keyless official APIs answer by default, routed by kind;
+ * a configured provider answers general queries first. Backends are tried in order, falling
+ * through on an error or an empty answer, and every attempt is recorded (never a key).
+ */
+export type SearchKind = 'general' | 'code' | 'package' | 'paper' | 'wiki';
+export const SEARCH_KINDS: readonly SearchKind[] = ['general', 'code', 'package', 'paper', 'wiki'];
+export interface SearchResult {
+  title: string;
+  url: string;
+  snippet: string;
+  source: string;
+}
+export interface Attempt {
+  backend: string;
+  ok: boolean;
+  count?: number;
+  ms: number;
+  error?: string;
+}
+export interface SearchOutcome {
+  query: string;
+  kind: SearchKind;
+  results: SearchResult[];
+  tried: Attempt[];
+  cached?: boolean;
+}
+export type Transport = (
+  url: string,
+  opts?: { method?: 'GET' | 'POST'; headers?: Record<string, string>; body?: string },
+) => Promise<GuardedResponse>;
+export type Provider =
+  { type: 'searxng'; url: string } | { type: 'brave' | 'tavily' | 'exa'; key: string };
+export interface WebSearchOptions {
+  transport?: Transport;
+  providers?: Provider[];
+  githubToken?: string;
+  now?: () => number;
+}
+
+type Search = (q: string, limit: number) => Promise<SearchResult[]>;
+interface Backend {
+  name: string;
+  intervalMs: number;
+  run: Search;
+}
+const KEYLESS = ['wikipedia', 'stackexchange', 'github', 'npm', 'pypi', 'arxiv', 'hackernews'];
+const ROUTES: Record<Exclude<SearchKind, 'general'>, string[]> = {
+  code: ['github', 'stackexchange'],
+  package: ['npm', 'pypi'],
+  paper: ['arxiv'],
+  wiki: ['wikipedia'],
+};
+const GENERAL_KEYLESS = ['wikipedia', 'stackexchange', 'hackernews'];
+const CACHE_TTL_MS = 10 * 60_000;
+const CACHE_MAX = 200;
+
+const clean = (s: unknown, max = 400) =>
+  decodeEntities(stripTags(String(s ?? '')))
+    .replace(/\s+/g, ' ')
+    .trim()
+    .slice(0, max);
+const q = encodeURIComponent;
+
+export class WebSearch {
+  private readonly transport: Transport;
+  private readonly providers: Provider[];
+  private readonly now: () => number;
+  private readonly backends = new Map<string, Backend>();
+  private readonly last = new Map<string, number>();
+  private readonly cache = new Map<string, { at: number; value: SearchOutcome }>();
+  constructor(opts: WebSearchOptions = {}) {
+    this.transport =
+      opts.transport ??
+      ((url, o = {}) => guardedRequest(url, { ...o, maxBytes: 1_000_000, timeoutMs: 12_000 }));
+    this.providers = opts.providers ?? [];
+    this.now = opts.now ?? Date.now;
+    const json = async (url: string, o?: Parameters<Transport>[1]) => {
+      const r = await this.transport(url, {
+        ...o,
+        headers: { accept: 'application/json', ...o?.headers },
+      });
+      if (r.status < 200 || r.status >= 300) throw new Error(`HTTP ${r.status}`);
+      return JSON.parse(r.body) as any;
+    };
+    const add = (name: string, run: Search, intervalMs = 1000) =>
+      this.backends.set(name, { name, run, intervalMs });
+    add('wikipedia', async (s, n) =>
+      (
+        (
+          await json(
+            `https://en.wikipedia.org/w/api.php?action=query&list=search&format=json&utf8=1&srlimit=${n}&srsearch=${q(s)}`,
+          )
+        ).query?.search ?? []
+      ).map((h: any) => ({
+        title: clean(h.title, 200),
+        url:
+          'https://en.wikipedia.org/wiki/' + encodeURIComponent(String(h.title).replace(/ /g, '_')),
+        snippet: clean(h.snippet),
+        source: 'wikipedia',
+      })),
+    );
+    add('stackexchange', async (s, n) =>
+      (
+        (
+          await json(
+            `https://api.stackexchange.com/2.3/search/advanced?order=desc&sort=relevance&site=stackoverflow&pagesize=${n}&q=${q(s)}`,
+          )
+        ).items ?? []
+      ).map((h: any) => ({
+        title: clean(h.title, 200),
+        url: String(h.link),
+        snippet: `score ${h.score ?? 0}${h.is_answered ? ', answered' : ''}`,
+        source: 'stackexchange',
+      })),
+    );
+    add(
+      'github',
+      async (s, n) =>
+        (
+          (
+            await json(`https://api.github.com/search/repositories?per_page=${n}&q=${q(s)}`, {
+              headers: {
+                accept: 'application/vnd.github+json',
+                ...(opts.githubToken ? { authorization: 'Bearer ' + opts.githubToken } : {}),
+              },
+            })
+          ).items ?? []
+        ).map((h: any) => ({
+          title: clean(h.full_name, 200),
+          url: String(h.html_url),
+          snippet: `${clean(h.description)} (${h.stargazers_count ?? 0} stars)`.trim(),
+          source: 'github',
+        })),
+      // Unauthenticated repository search allows 10 requests a minute.
+      opts.githubToken ? 2000 : 6000,
+    );
+    add('npm', async (s, n) =>
+      (
+        (await json(`https://registry.npmjs.org/-/v1/search?size=${n}&text=${q(s)}`)).objects ?? []
+      ).map((o: any) => ({
+        title: `${clean(o.package?.name, 200)}@${clean(o.package?.version, 40)}`,
+        url: String(o.package?.links?.npm ?? `https://www.npmjs.com/package/${o.package?.name}`),
+        snippet: clean(o.package?.description),
+        source: 'npm',
+      })),
+    );
+    // PyPI has no search API; its official JSON API answers an exact project name.
+    add('pypi', async (s) => {
+      const name = s.trim();
+      if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$/.test(name)) return [];
+      const r = await this.transport(`https://pypi.org/pypi/${q(name)}/json`, {
+        headers: { accept: 'application/json' },
+      });
+      if (r.status === 404) return [];
+      if (r.status < 200 || r.status >= 300) throw new Error(`HTTP ${r.status}`);
+      const info = (JSON.parse(r.body) as any).info ?? {};
+      return [
+        {
+          title: `${clean(info.name, 200)} ${clean(info.version, 40)}`.trim(),
+          url: String(info.package_url ?? `https://pypi.org/project/${name}/`),
+          snippet: clean(info.summary),
+          source: 'pypi',
+        },
+      ];
+    });
+    add(
+      'arxiv',
+      async (s, n) => {
+        const r = await this.transport(
+          `https://export.arxiv.org/api/query?max_results=${n}&search_query=all:${q(s)}`,
+          { headers: { accept: 'application/atom+xml' } },
+        );
+        if (r.status < 200 || r.status >= 300) throw new Error(`HTTP ${r.status}`);
+        return [...r.body.matchAll(/<entry>([\s\S]*?)<\/entry>/g)].map((m) => {
+          const tag = (t: string) =>
+            new RegExp(`<${t}[^>]*>([\\s\\S]*?)</${t}>`).exec(m[1]!)?.[1] ?? '';
+          return {
+            title: clean(tag('title'), 300),
+            url: clean(tag('id'), 300),
+            snippet: clean(tag('summary')),
+            source: 'arxiv',
+          };
+        });
+      },
+      3000,
+    );
+    add('hackernews', async (s, n) =>
+      (
+        (await json(`https://hn.algolia.com/api/v1/search?hitsPerPage=${n}&query=${q(s)}`)).hits ??
+        []
+      ).map((h: any) => ({
+        title: clean(h.title ?? h.story_title, 200),
+        url: String(h.url ?? `https://news.ycombinator.com/item?id=${h.objectID}`),
+        snippet: `${h.points ?? 0} points on Hacker News`,
+        source: 'hackernews',
+      })),
+    );
+    this.providers.forEach((p, i) => {
+      const name =
+        this.providers.findIndex((o) => o.type === p.type) === i ? p.type : `${p.type}-${i}`;
+      if (p.type === 'searxng')
+        add(name, async (s, n) =>
+          ((await json(`${p.url.replace(/\/+$/, '')}/search?format=json&q=${q(s)}`)).results ?? [])
+            .slice(0, n)
+            .map((h: any) => ({
+              title: clean(h.title, 200),
+              url: String(h.url),
+              snippet: clean(h.content),
+              source: p.type,
+            })),
+        );
+      else if (p.type === 'brave')
+        add(name, async (s, n) =>
+          (
+            (
+              await json(`https://api.search.brave.com/res/v1/web/search?count=${n}&q=${q(s)}`, {
+                headers: { 'x-subscription-token': p.key },
+              })
+            ).web?.results ?? []
+          ).map((h: any) => ({
+            title: clean(h.title, 200),
+            url: String(h.url),
+            snippet: clean(h.description),
+            source: p.type,
+          })),
+        );
+      else if (p.type === 'tavily')
+        add(name, async (s, n) =>
+          (
+            (
+              await json('https://api.tavily.com/search', {
+                method: 'POST',
+                headers: { authorization: 'Bearer ' + p.key },
+                body: JSON.stringify({ query: s, max_results: n }),
+              })
+            ).results ?? []
+          ).map((h: any) => ({
+            title: clean(h.title, 200),
+            url: String(h.url),
+            snippet: clean(h.content),
+            source: p.type,
+          })),
+        );
+      else
+        add(name, async (s, n) =>
+          (
+            (
+              await json('https://api.exa.ai/search', {
+                method: 'POST',
+                headers: { 'x-api-key': p.key },
+                body: JSON.stringify({ query: s, numResults: n }),
+              })
+            ).results ?? []
+          ).map((h: any) => ({
+            title: clean(h.title, 200),
+            url: String(h.url),
+            snippet: clean(h.text ?? h.summary),
+            source: p.type,
+          })),
+        );
+    });
+  }
+  /** What is enabled, for health and manifests: provider types only, never keys or URLs' secrets. */
+  describe(): { providers: string[]; keyless: string[] } {
+    return { providers: this.providers.map((p) => p.type), keyless: [...KEYLESS] };
+  }
+  private order(kind: SearchKind): string[] {
+    if (kind !== 'general') return ROUTES[kind];
+    const providerNames = [...this.backends.keys()].filter((n) => !KEYLESS.includes(n));
+    return [...providerNames, ...GENERAL_KEYLESS];
+  }
+  async search(input: {
+    query: string;
+    kind?: SearchKind;
+    limit?: number;
+  }): Promise<SearchOutcome> {
+    const query = String(input.query ?? '')
+      .trim()
+      .slice(0, 300);
+    if (!query) throw new Error('An empty query cannot be searched');
+    const kind = input.kind ?? 'general';
+    if (!SEARCH_KINDS.includes(kind)) throw new Error('Unknown search kind');
+    const limit = Math.max(1, Math.min(10, Math.floor(input.limit ?? 5)));
+    const key = `${kind}\u0000${query.toLowerCase()}\u0000${limit}`;
+    const hit = this.cache.get(key);
+    if (hit && this.now() - hit.at < CACHE_TTL_MS) return { ...hit.value, cached: true };
+    const results: SearchResult[] = [];
+    const seen = new Set<string>();
+    const tried: Attempt[] = [];
+    for (const name of this.order(kind)) {
+      if (results.length >= limit) break;
+      const b = this.backends.get(name)!;
+      const last = this.last.get(name);
+      if (last !== undefined && this.now() - last < b.intervalMs) {
+        tried.push({ backend: name, ok: false, ms: 0, error: 'rate-limited' });
+        continue;
+      }
+      this.last.set(name, this.now());
+      const started = Date.now();
+      try {
+        const got = await b.run(query, limit - results.length);
+        let count = 0;
+        for (const r of got) {
+          if (!/^https?:\/\//.test(r.url) || seen.has(r.url)) continue;
+          seen.add(r.url);
+          results.push(r);
+          count++;
+          if (results.length >= limit) break;
+        }
+        tried.push({ backend: name, ok: true, count, ms: Date.now() - started });
+      } catch (error) {
+        tried.push({
+          backend: name,
+          ok: false,
+          ms: Date.now() - started,
+          error: String((error as Error).message ?? error).slice(0, 200),
+        });
+      }
+    }
+    const value: SearchOutcome = { query, kind, results, tried };
+    // Cache only a complete answer: one assembled around a skipped or failing backend would
+    // keep that gap for the cache's lifetime.
+    if (results.length && tried.every((t) => t.ok)) {
+      if (this.cache.size >= CACHE_MAX) this.cache.delete(this.cache.keys().next().value!);
+      this.cache.set(key, { at: this.now(), value });
+    }
+    return value;
+  }
+}
+
+/** Reads the operator's web search config: providers in order, keys from env or a private file. */
+export function loadWebSearchConfig(
+  path: string,
+  env: Record<string, string | undefined> = process.env,
+): { providers: Provider[]; githubToken?: string } {
+  let raw: unknown;
+  try {
+    raw = JSON.parse(readFileSync(path, 'utf8'));
+  } catch {
+    throw new Error(`Web search config ${path} is not readable JSON`);
+  }
+  const top = raw as Record<string, unknown>;
+  if (!top || typeof top !== 'object' || Array.isArray(top))
+    throw new Error('Web search config must be an object');
+  for (const k of Object.keys(top))
+    if (!['providers', 'github_token_env'].includes(k))
+      throw new Error(`Web search config: unknown field ${k}`);
+  const secret = (p: Record<string, unknown>, label: string): string => {
+    const hasEnv = typeof p.key_env === 'string' && p.key_env;
+    const hasFile = typeof p.key_file === 'string' && p.key_file;
+    if ('key' in p || (!hasEnv && !hasFile) || (hasEnv && hasFile))
+      throw new Error(`${label} needs exactly one of key_env or key_file (never the key itself)`);
+    if (hasEnv) {
+      const v = env[p.key_env as string];
+      if (!v) throw new Error(`${label}: ${p.key_env} is not set`);
+      return v.trim();
+    }
+    const file = p.key_file as string;
+    const st = statSync(file);
+    if (st.mode & 0o077) throw new Error(`${label}: ${file} is readable by others; chmod 600 it`);
+    return readFileSync(file, 'utf8').trim();
+  };
+  const providers: Provider[] = [];
+  const list = top.providers ?? [];
+  if (!Array.isArray(list) || list.length > 8)
+    throw new Error('providers must be a list of up to 8');
+  list.forEach((entry, i) => {
+    const p = entry as Record<string, unknown>;
+    const label = `providers[${i}]`;
+    const type = p?.type;
+    if (type === 'searxng') {
+      for (const k of Object.keys(p))
+        if (!['type', 'url'].includes(k)) throw new Error(`${label}: unknown field ${k}`);
+      let u: URL;
+      try {
+        u = new URL(String(p.url));
+      } catch {
+        throw new Error(`${label}: url must be http or https`);
+      }
+      if (u.protocol !== 'https:' && u.protocol !== 'http:')
+        throw new Error(`${label}: url must be http or https`);
+      providers.push({ type, url: u.toString() });
+    } else if (type === 'brave' || type === 'tavily' || type === 'exa') {
+      for (const k of Object.keys(p))
+        if (!['type', 'key_env', 'key_file', 'key'].includes(k))
+          throw new Error(`${label}: unknown field ${k}`);
+      providers.push({ type, key: secret(p, label) });
+    } else throw new Error(`${label}: unknown provider (searxng, brave, tavily or exa)`);
+  });
+  const ghEnv = top.github_token_env;
+  if (ghEnv !== undefined && typeof ghEnv !== 'string')
+    throw new Error('github_token_env must be a name');
+  const githubToken = ghEnv ? env[ghEnv] : undefined;
+  return { providers, ...(githubToken ? { githubToken } : {}) };
+}