@sylphx/lookout 0.1.0

package/src/extract.ts

@@ -0,0 +1,274 @@
+export type CiteSpan = {
+  text: string;
+  start: number;
+  end: number;
+  kind: string;
+};
+
+export type ExtractResult = {
+  ok: boolean;
+  url?: string;
+  title?: string;
+  description?: string;
+  canonicalUrl?: string;
+  author?: string;
+  siteName?: string;
+  language?: string;
+  textExcerpt?: string;
+  headings: { level: number; text: string }[];
+  links: { href: string; text: string }[];
+  jsonLd: unknown[];
+  tables: { rows: string[][] }[];
+  spans: CiteSpan[];
+  route: string;
+  warnings: string[];
+};
+
+function stripTags(html: string): string {
+  return html
+    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
+    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
+    .replace(/<noscript[\s\S]*?<\/noscript>/gi, ' ')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function decodeBasicEntities(s: string): string {
+  return s
+    .replace(/&amp;/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, ' ');
+}
+
+/** Prefer <article>/<main>, else body, else full document for readable text. */
+function pickContentHtml(html: string): { html: string; route: string } {
+  const article = html.match(/<article\b[^>]*>([\s\S]*?)<\/article>/i);
+  if (article?.[1] && stripTags(article[1]).length > 40) {
+    return { html: article[1], route: 'html_article' };
+  }
+  const main = html.match(/<main\b[^>]*>([\s\S]*?)<\/main>/i);
+  if (main?.[1] && stripTags(main[1]).length > 40) {
+    return { html: main[1], route: 'html_main' };
+  }
+  const body = html.match(/<body\b[^>]*>([\s\S]*?)<\/body>/i);
+  if (body?.[1]) {
+    return { html: body[1], route: 'html_body' };
+  }
+  return { html, route: 'html_dom_light' };
+}
+
+
+function resolveHref(href: string, base?: string): string {
+  try {
+    if (!base) return href;
+    return new URL(href, base).toString();
+  } catch {
+    return href;
+  }
+}
+
+export function extractFromHtml(html: string, url?: string): ExtractResult {
+  const warnings: string[] = [];
+  const spans: CiteSpan[] = [];
+  const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+  let title = titleMatch ? decodeBasicEntities(stripTags(titleMatch[1] ?? '')) : undefined;
+  if (title && titleMatch && titleMatch.index !== undefined) {
+    const raw = titleMatch[1] ?? '';
+    const start = titleMatch.index + titleMatch[0].indexOf(raw);
+    spans.push({ text: title, start, end: start + raw.length, kind: 'title' });
+  }
+
+  let language: string | undefined;
+  const langMatch = html.match(/<html\b[^>]*\blang=["']([^"']+)["'][^>]*>/i);
+  if (langMatch?.[1]) {
+    language = langMatch[1].trim();
+  }
+
+  let description: string | undefined;
+  const metaDesc =
+    html.match(/<meta[^>]+name=["']description["'][^>]+content=["']([^"']+)["'][^>]*>/i) ||
+    html.match(/<meta[^>]+content=["']([^"']+)["'][^>]+name=["']description["'][^>]*>/i);
+  if (metaDesc) {
+    description = decodeBasicEntities(stripTags(metaDesc[1] ?? ''));
+    if (metaDesc.index !== undefined) {
+      spans.push({
+        text: description,
+        start: metaDesc.index,
+        end: metaDesc.index + metaDesc[0].length,
+        kind: 'meta_description',
+      });
+    }
+  }
+
+  if (!description) {
+    const ogDesc =
+      html.match(/<meta[^>]+property=["']og:description["'][^>]+content=["']([^"']+)["'][^>]*>/i) ||
+      html.match(/<meta[^>]+content=["']([^"']+)["'][^>]+property=["']og:description["'][^>]*>/i);
+    if (ogDesc?.[1]) {
+      description = decodeBasicEntities(stripTags(ogDesc[1]));
+      spans.push({
+        text: description,
+        start: ogDesc.index ?? 0,
+        end: (ogDesc.index ?? 0) + ogDesc[0].length,
+        kind: 'og_description',
+      });
+    }
+  }
+
+  const ogTitle =
+    html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["'][^>]*>/i) ||
+    html.match(/<meta[^>]+content=["']([^"']+)["'][^>]+property=["']og:title["'][^>]*>/i);
+  if (ogTitle?.[1] && !title) {
+    title = decodeBasicEntities(stripTags(ogTitle[1]));
+    spans.push({ text: title, start: ogTitle.index ?? 0, end: (ogTitle.index ?? 0) + ogTitle[0].length, kind: 'og_title' });
+  } else if (ogTitle?.[1] && title && title !== decodeBasicEntities(stripTags(ogTitle[1]))) {
+    spans.push({
+      text: decodeBasicEntities(stripTags(ogTitle[1])),
+      start: ogTitle.index ?? 0,
+      end: (ogTitle.index ?? 0) + ogTitle[0].length,
+      kind: 'og_title',
+    });
+  }
+
+  let canonicalUrl: string | undefined;
+  const canonical =
+    html.match(/<link[^>]+rel=["']canonical["'][^>]+href=["']([^"']+)["'][^>]*>/i) ||
+    html.match(/<link[^>]+href=["']([^"']+)["'][^>]+rel=["']canonical["'][^>]*>/i);
+  if (canonical?.[1]) {
+    canonicalUrl = resolveHref(decodeBasicEntities(canonical[1].trim()), url);
+    spans.push({
+      text: canonicalUrl,
+      start: canonical.index ?? 0,
+      end: (canonical.index ?? 0) + canonical[0].length,
+      kind: 'canonical',
+    });
+  }
+
+  let author: string | undefined;
+  const authorMeta =
+    html.match(/<meta[^>]+name=["']author["'][^>]+content=["']([^"']+)["'][^>]*>/i) ||
+    html.match(/<meta[^>]+content=["']([^"']+)["'][^>]+name=["']author["'][^>]*>/i);
+  if (authorMeta?.[1]) {
+    author = decodeBasicEntities(stripTags(authorMeta[1]));
+    spans.push({
+      text: author,
+      start: authorMeta.index ?? 0,
+      end: (authorMeta.index ?? 0) + authorMeta[0].length,
+      kind: 'author',
+    });
+  }
+
+  let siteName: string | undefined;
+  const siteMeta =
+    html.match(/<meta[^>]+property=["']og:site_name["'][^>]+content=["']([^"']+)["'][^>]*>/i) ||
+    html.match(/<meta[^>]+content=["']([^"']+)["'][^>]+property=["']og:site_name["'][^>]*>/i);
+  if (siteMeta?.[1]) {
+    siteName = decodeBasicEntities(stripTags(siteMeta[1]));
+  }
+
+  const jsonLd: unknown[] = [];
+  const ldRe = /<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
+  let m: RegExpExecArray | null;
+  while ((m = ldRe.exec(html)) !== null) {
+    try {
+      jsonLd.push(JSON.parse(m[1] ?? ''));
+    } catch {
+      warnings.push('json_ld_parse_error');
+    }
+  }
+
+  const tables: { rows: string[][] }[] = [];
+  const tableRe = /<table[^>]*>([\s\S]*?)<\/table>/gi;
+  while ((m = tableRe.exec(html)) !== null) {
+    const rows: string[][] = [];
+    const rowRe = /<tr[^>]*>([\s\S]*?)<\/tr>/gi;
+    let r: RegExpExecArray | null;
+    const tableHtml = m[1] ?? '';
+    while ((r = rowRe.exec(tableHtml)) !== null) {
+      const cells: string[] = [];
+      const cellRe = /<t[dh][^>]*>([\s\S]*?)<\/t[dh]>/gi;
+      let c: RegExpExecArray | null;
+      while ((c = cellRe.exec(r[1] ?? '')) !== null) {
+        cells.push(decodeBasicEntities(stripTags(c[1] ?? '')));
+      }
+      if (cells.length) rows.push(cells);
+    }
+    if (rows.length) tables.push({ rows });
+    if (tables.length >= 8) break;
+  }
+
+  const headings: { level: number; text: string }[] = [];
+  const hRe = /<h([1-6])\b[^>]*>([\s\S]*?)<\/h\1>/gi;
+  while ((m = hRe.exec(html)) !== null) {
+    const level = Number(m[1] ?? 1);
+    const text = decodeBasicEntities(stripTags(m[2] ?? ''));
+    if (!text) continue;
+    headings.push({ level, text });
+    if (m.index !== undefined) {
+      spans.push({
+        text,
+        start: m.index,
+        end: m.index + m[0].length,
+        kind: `heading_h${level}`,
+      });
+    }
+    if (headings.length >= 30) break;
+  }
+
+  const links: { href: string; text: string }[] = [];
+  const aRe = /<a\b[^>]*href=["']([^"'#][^"']*)["'][^>]*>([\s\S]*?)<\/a>/gi;
+  while ((m = aRe.exec(html)) !== null) {
+    const rawHref = (m[1] ?? '').trim();
+    const text = decodeBasicEntities(stripTags(m[2] ?? '')).slice(0, 200);
+    if (!rawHref || rawHref.startsWith('javascript:')) continue;
+    const href = resolveHref(rawHref, url);
+    links.push({ href, text });
+    if (links.length >= 40) break;
+  }
+
+  const content = pickContentHtml(html);
+  const textExcerpt = decodeBasicEntities(stripTags(content.html)).slice(0, 4000);
+  if (textExcerpt) {
+    spans.push({
+      text: textExcerpt.slice(0, 280),
+      start: 0,
+      end: Math.min(280, html.length),
+      kind: 'excerpt',
+    });
+  } else {
+    warnings.push('empty_text_excerpt');
+  }
+
+
+  const robots =
+    html.match(/<meta[^>]+name=["']robots["'][^>]+content=["']([^"']+)["'][^>]*>/i) ||
+    html.match(/<meta[^>]+content=["']([^"']+)["'][^>]+name=["']robots["'][^>]*>/i);
+  if (robots?.[1] && /noindex/i.test(robots[1])) {
+    warnings.push('robots_noindex');
+  }
+  if (!title) warnings.push('missing_title');
+  if (!description) warnings.push('missing_description');
+
+  return {
+    ok: true,
+    url,
+    title,
+    description,
+    canonicalUrl,
+    author,
+    siteName,
+    language,
+    textExcerpt,
+    headings,
+    links,
+    jsonLd,
+    tables,
+    spans,
+    route: content.route,
+    warnings,
+  };
+}

package/src/fetch.ts

@@ -0,0 +1,145 @@
+import { assertSafeUrl } from './ssrf.ts';
+
+export type FetchResult = {
+  ok: boolean;
+  url: string;
+  finalUrl: string;
+  status?: number;
+  contentType?: string;
+  body?: string;
+  route: string;
+  warnings: string[];
+  code?: string;
+  message?: string;
+  redirects?: string[];
+  truncated?: boolean;
+};
+
+const DEFAULT_MAX_BYTES = 1_500_000;
+const DEFAULT_TIMEOUT_MS = 20_000;
+const MAX_REDIRECTS = 5;
+
+export async function webFetch(
+  rawUrl: string,
+  options: { maxBytes?: number; timeoutMs?: number; userAgent?: string } = {},
+): Promise<FetchResult> {
+  const warnings: string[] = [];
+  const redirects: string[] = [];
+  let current = rawUrl;
+  const envMax = process.env.LOOKOUT_FETCH_MAX_BYTES?.trim();
+  const envTimeout = process.env.LOOKOUT_FETCH_TIMEOUT_MS?.trim();
+  const maxBytes =
+    options.maxBytes ??
+    (envMax && Number.isFinite(Number(envMax)) ? Number(envMax) : DEFAULT_MAX_BYTES);
+  const timeoutMs =
+    options.timeoutMs ??
+    (envTimeout && Number.isFinite(Number(envTimeout)) ? Number(envTimeout) : DEFAULT_TIMEOUT_MS);
+  const ua =
+    options.userAgent ??
+    process.env.LOOKOUT_USER_AGENT?.trim() ??
+    'Lookout/0.1 (+https://github.com/SylphxAI/lookout; local-first agent web instrument)';
+
+  for (let i = 0; i <= MAX_REDIRECTS; i++) {
+    const safety = assertSafeUrl(current);
+    if (!safety.ok) {
+      return {
+        ok: false,
+        url: rawUrl,
+        finalUrl: current,
+        route: 'http',
+        warnings,
+        code: safety.code,
+        message: safety.message,
+        redirects,
+      };
+    }
+
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+      const res = await fetch(safety.url, {
+        method: 'GET',
+        redirect: 'manual',
+        signal: controller.signal,
+        headers: {
+          'user-agent': ua,
+          accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,text/plain;q=0.8,*/*;q=0.5',
+        },
+      });
+
+      if ([301, 302, 303, 307, 308].includes(res.status)) {
+        const loc = res.headers.get('location');
+        if (!loc) {
+          return {
+            ok: false,
+            url: rawUrl,
+            finalUrl: current,
+            status: res.status,
+            route: 'http',
+            warnings,
+            code: 'REDIRECT_MISSING_LOCATION',
+            message: 'Redirect without Location header',
+            redirects,
+          };
+        }
+        const next = new URL(loc, safety.url).toString();
+        redirects.push(next);
+        current = next;
+        continue;
+      }
+
+      const contentType = res.headers.get('content-type') ?? undefined;
+      const buf = new Uint8Array(await res.arrayBuffer());
+      let truncated = false;
+      let slice = buf;
+      if (buf.byteLength > maxBytes) {
+        slice = buf.slice(0, maxBytes);
+        truncated = true;
+        warnings.push(`Response truncated to ${maxBytes} bytes`);
+      }
+      const body = new TextDecoder('utf-8', { fatal: false }).decode(slice);
+      if (!res.ok) {
+        warnings.push(`HTTP ${res.status}`);
+      }
+      return {
+        ok: res.ok,
+        url: rawUrl,
+        finalUrl: current,
+        status: res.status,
+        contentType,
+        body,
+        route: 'http',
+        warnings,
+        redirects,
+        truncated,
+        code: res.ok ? undefined : 'HTTP_ERROR',
+        message: res.ok ? undefined : `HTTP ${res.status}`,
+      };
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      return {
+        ok: false,
+        url: rawUrl,
+        finalUrl: current,
+        route: 'http',
+        warnings,
+        code: message.includes('abort') ? 'TIMEOUT' : 'FETCH_FAILED',
+        message,
+        redirects,
+      };
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  return {
+    ok: false,
+    url: rawUrl,
+    finalUrl: current,
+    route: 'http',
+    warnings,
+    code: 'TOO_MANY_REDIRECTS',
+    message: `Exceeded ${MAX_REDIRECTS} redirects`,
+    redirects,
+  };
+}

package/src/mcp.ts

@@ -0,0 +1,118 @@
+/**
+ * Lookout MCP server — stdio tools:
+ * core: web_search, web_fetch, web_extract, web_cache
+ * advanced: web_crawl, web_research
+ */
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { z } from 'zod';
+import { LookoutEngine } from './engine.ts';
+
+const engine = new LookoutEngine();
+
+function textResult(envelope: unknown, isError = false) {
+  return {
+    content: [{ type: 'text' as const, text: JSON.stringify(envelope, null, 2) }],
+    isError,
+  };
+}
+
+const server = new McpServer({
+  name: 'lookout',
+  version: '0.1.0',
+});
+
+server.tool(
+  'web_search',
+  'Local-first web search via public adapters (no API key). Returns ranked hits with score explanations.',
+  {
+    query: z.union([z.string(), z.array(z.string())]).describe('Search query or queries'),
+    useCache: z.boolean().optional(),
+    hostsInclude: z.array(z.string()).optional().describe('Keep only hits whose host matches any entry'),
+    hostsExclude: z.array(z.string()).optional().describe('Drop hits whose host matches any entry'),
+  },
+  async (args) => {
+    const envelope = await engine.handle('web_search', args as Record<string, unknown>);
+    return textResult(envelope, envelope.status !== 'ok');
+  },
+);
+
+server.tool(
+  'web_fetch',
+  'Fetch a URL with SSRF protection, redirect limits, and citeable body prefix spans.',
+  {
+    url: z.string().url().describe('http(s) URL to fetch'),
+    useCache: z.boolean().optional(),
+    maxBytes: z.number().int().positive().optional(),
+    timeoutMs: z.number().int().positive().optional(),
+    respectRobots: z.boolean().optional().describe('If true, honor robots.txt Disallow (default false for fetch)'),
+  },
+  async (args) => {
+    const envelope = await engine.handle('web_fetch', args as Record<string, unknown>);
+    return textResult(envelope, envelope.status !== 'ok');
+  },
+);
+
+server.tool(
+  'web_extract',
+  'Extract title, description, headings, links, JSON-LD, tables, and cite spans from HTML or by URL.',
+  {
+    url: z.string().url().optional(),
+    html: z.string().optional(),
+    useCache: z.boolean().optional(),
+  },
+  async (args) => {
+    const envelope = await engine.handle('web_extract', args as Record<string, unknown>);
+    return textResult(envelope, envelope.status !== 'ok');
+  },
+);
+
+server.tool(
+  'web_cache',
+  'Query, stats, or clear the local Lookout cache.',
+  {
+    op: z.enum(['query', 'stats', 'clear', 'prune']).optional(),
+    operation: z.enum(['query', 'stats', 'clear', 'prune']).optional(),
+    query: z.string().optional(),
+    limit: z.number().int().positive().optional(),
+    maxAgeMs: z.number().int().nonnegative().optional(),
+  },
+  async (args) => {
+    const envelope = await engine.handle('web_cache', args as Record<string, unknown>);
+    return textResult(envelope, envelope.status !== 'ok');
+  },
+);
+
+server.tool(
+  'web_crawl',
+  'Depth-limited same-origin crawl (advanced). SSRF-safe; not a full-site crawler.',
+  {
+    url: z.string().url(),
+    maxDepth: z.number().int().min(0).max(3).optional(),
+    maxPages: z.number().int().min(1).max(25).optional(),
+    respectRobots: z.boolean().optional().describe('Honor robots.txt User-agent: * Disallow (default true)'),
+    useSitemap: z.boolean().optional().describe('If true, seed same-origin URLs from /sitemap.xml'),
+  },
+  async (args) => {
+    const envelope = await engine.handle('web_crawl', args as Record<string, unknown>);
+    return textResult(envelope, envelope.status !== 'ok');
+  },
+);
+
+server.tool(
+  'web_research',
+  'Advanced: search then fetch/extract top public pages with citeable excerpts (local-first, no API key).',
+  {
+    query: z.string().describe('Research question or keywords'),
+    maxPages: z.number().int().min(1).max(6).optional(),
+    hostsInclude: z.array(z.string()).optional().describe('Keep only research hits from these hosts'),
+    hostsExclude: z.array(z.string()).optional().describe('Drop research hits from these hosts'),
+  },
+  async (args) => {
+    const envelope = await engine.handle('web_research', args as Record<string, unknown>);
+    return textResult(envelope, envelope.status !== 'ok');
+  },
+);
+
+const transport = new StdioServerTransport();
+await server.connect(transport);

package/src/research.ts

@@ -0,0 +1,98 @@
+import { webSearch, filterHitsByHosts, type SearchHit, type SearchResult } from './search.ts';
+import { webFetch, type FetchResult } from './fetch.ts';
+import { extractFromHtml } from './extract.ts';
+
+export type ResearchPage = {
+  url: string;
+  title?: string;
+  score?: number;
+  engine?: string;
+  fetchOk: boolean;
+  extractRoute?: string;
+  textExcerpt?: string;
+  headings?: { level: number; text: string }[];
+  warnings: string[];
+  spans?: { text: string; kind: string }[];
+};
+
+export type ResearchResult = {
+  query: string;
+  hits: SearchHit[];
+  pages: ResearchPage[];
+  warnings: string[];
+  route: string;
+};
+
+export type ResearchDeps = {
+  searchFn?: (query: string) => Promise<SearchResult>;
+  fetchFn?: (url: string, options?: { maxBytes?: number }) => Promise<FetchResult>;
+};
+
+export async function webResearch(
+  query: string,
+  options: {
+    maxPages?: number;
+    maxBytes?: number;
+    hostsInclude?: string[];
+    hostsExclude?: string[];
+  } & ResearchDeps = {},
+): Promise<ResearchResult> {
+  const maxPages = Math.min(Math.max(options.maxPages ?? 3, 1), 6);
+  const maxBytes = options.maxBytes ?? 512_000;
+  const searchFn = options.searchFn ?? webSearch;
+  const fetchFn = options.fetchFn ?? webFetch;
+  const search = await searchFn(query);
+  const warnings = [...(search.warnings ?? [])];
+  let hits = search.hits;
+  if ((options.hostsInclude?.length ?? 0) > 0 || (options.hostsExclude?.length ?? 0) > 0) {
+    const before = hits.length;
+    hits = filterHitsByHosts(hits, {
+      include: options.hostsInclude,
+      exclude: options.hostsExclude,
+    });
+    warnings.push(
+      `host_filter kept ${hits.length}/${before} for research (include=${(options.hostsInclude ?? []).join(',') || '*'}; exclude=${(options.hostsExclude ?? []).join(',') || 'none'})`,
+    );
+  }
+  const pages: ResearchPage[] = [];
+
+  for (const hit of hits.slice(0, maxPages)) {
+    const page: ResearchPage = {
+      url: hit.url,
+      title: hit.title,
+      score: hit.score,
+      engine: hit.engine,
+      fetchOk: false,
+      warnings: [],
+    };
+    try {
+      const fetched = await fetchFn(hit.url, { maxBytes });
+      if (!fetched.ok || !fetched.body) {
+        page.warnings.push(fetched.message ?? fetched.code ?? 'fetch_failed');
+        pages.push(page);
+        continue;
+      }
+      page.fetchOk = true;
+      const extracted = extractFromHtml(fetched.body, hit.url);
+      page.extractRoute = extracted.route;
+      page.textExcerpt = extracted.textExcerpt?.slice(0, 1200);
+      page.headings = extracted.headings.slice(0, 8);
+      page.spans = extracted.spans.slice(0, 6).map((s) => ({ text: s.text.slice(0, 200), kind: s.kind }));
+      page.warnings.push(...extracted.warnings);
+      if (!page.title && extracted.title) page.title = extracted.title;
+    } catch (e) {
+      page.warnings.push(e instanceof Error ? e.message : 'research_page_error');
+    }
+    pages.push(page);
+  }
+
+  if (pages.length === 0) warnings.push('no_research_pages');
+
+  return {
+    query,
+    hits,
+    pages,
+    warnings,
+    route: 'search_then_extract',
+  };
+}