@selesai/code 0.13.3 → 0.13.5

package/dist/extensions/pi-web-agent/src/search/brave.ts

@@ -1,114 +1,114 @@
-import { buildSearchPresentation } from '../presentation/search-presentation.js';
-import type { SearchResult, WebSearchResponse } from '../types.js';
-
-type BraveWebResult = {
-  title?: unknown;
-  url?: unknown;
-  description?: unknown;
-};
-
-type BraveSearchResponse = {
-  web?: {
-    results?: BraveWebResult[];
-  };
-};
-
-const BRAVE_WEB_SEARCH_URL = 'https://api.search.brave.com/res/v1/web/search';
-
-function resultWithPresentation(result: WebSearchResponse): WebSearchResponse {
-  return { ...result, presentation: buildSearchPresentation(result) };
-}
-
-function normalizeResults(response: BraveSearchResponse): SearchResult[] {
-  return (response.web?.results ?? []).flatMap((item) => {
-    if (typeof item.title !== 'string' || typeof item.url !== 'string') {
-      return [];
-    }
-
-    return [
-      {
-        title: item.title,
-        url: item.url,
-        snippet: typeof item.description === 'string' ? item.description : ''
-      }
-    ];
-  });
-}
-
-function buildBraveUrl(query: string) {
-  const url = new URL(BRAVE_WEB_SEARCH_URL);
-  url.searchParams.set('q', query);
-  return url.toString();
-}
-
-export function createBraveSearchTool({
-  apiKey,
-  fetchImpl = fetch
-}: {
-  apiKey?: string;
-  fetchImpl?: typeof fetch;
-}) {
-  return async function braveSearch({ query }: { query: string }): Promise<WebSearchResponse> {
-    const normalizedQuery = query.trim();
-
-    if (!normalizedQuery) {
-      return resultWithPresentation({
-        status: 'error',
-        results: [],
-        metadata: { backend: 'brave', cacheHit: false },
-        error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
-      });
-    }
-
-    if (!apiKey?.trim()) {
-      return resultWithPresentation({
-        status: 'error',
-        results: [],
-        metadata: { backend: 'brave', cacheHit: false },
-        error: {
-          code: 'BACKEND_CONFIG_INVALID',
-          message: 'Brave search requires PI_WEB_AGENT_BRAVE_API_KEY.'
-        }
-      });
-    }
-
-    try {
-      const response = await fetchImpl(buildBraveUrl(normalizedQuery), {
-        headers: {
-          Accept: 'application/json',
-          'X-Subscription-Token': apiKey
-        }
-      });
-
-      if (!response.ok) {
-        throw new Error(`HTTP ${response.status}`);
-      }
-
-      const parsed = (await response.json()) as BraveSearchResponse;
-      const results = normalizeResults(parsed);
-
-      if (results.length === 0) {
-        return resultWithPresentation({
-          status: 'error',
-          results: [],
-          metadata: { backend: 'brave', cacheHit: false },
-          error: { code: 'NO_RESULTS', message: 'Brave returned no usable results for this query.' }
-        });
-      }
-
-      return resultWithPresentation({
-        status: 'ok',
-        results,
-        metadata: { backend: 'brave', cacheHit: false }
-      });
-    } catch (error) {
-      const rawMessage = error instanceof Error ? error.message : String(error);
-      return resultWithPresentation({
-        status: 'error',
-        results: [],
-        metadata: { backend: 'brave', cacheHit: false },
-        error: { code: 'FETCH_FAILED', message: `Brave search request failed: ${rawMessage}` }
-      });
-    }
-  };
-}
+import { buildSearchPresentation } from '../presentation/search-presentation.js';
+import type { SearchResult, WebSearchResponse } from '../types.js';
+
+type BraveWebResult = {
+  title?: unknown;
+  url?: unknown;
+  description?: unknown;
+};
+
+type BraveSearchResponse = {
+  web?: {
+    results?: BraveWebResult[];
+  };
+};
+
+const BRAVE_WEB_SEARCH_URL = 'https://api.search.brave.com/res/v1/web/search';
+
+function resultWithPresentation(result: WebSearchResponse): WebSearchResponse {
+  return { ...result, presentation: buildSearchPresentation(result) };
+}
+
+function normalizeResults(response: BraveSearchResponse): SearchResult[] {
+  return (response.web?.results ?? []).flatMap((item) => {
+    if (typeof item.title !== 'string' || typeof item.url !== 'string') {
+      return [];
+    }
+
+    return [
+      {
+        title: item.title,
+        url: item.url,
+        snippet: typeof item.description === 'string' ? item.description : ''
+      }
+    ];
+  });
+}
+
+function buildBraveUrl(query: string) {
+  const url = new URL(BRAVE_WEB_SEARCH_URL);
+  url.searchParams.set('q', query);
+  return url.toString();
+}
+
+export function createBraveSearchTool({
+  apiKey,
+  fetchImpl = fetch
+}: {
+  apiKey?: string;
+  fetchImpl?: typeof fetch;
+}) {
+  return async function braveSearch({ query }: { query: string }): Promise<WebSearchResponse> {
+    const normalizedQuery = query.trim();
+
+    if (!normalizedQuery) {
+      return resultWithPresentation({
+        status: 'error',
+        results: [],
+        metadata: { backend: 'brave', cacheHit: false },
+        error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
+      });
+    }
+
+    if (!apiKey?.trim()) {
+      return resultWithPresentation({
+        status: 'error',
+        results: [],
+        metadata: { backend: 'brave', cacheHit: false },
+        error: {
+          code: 'BACKEND_CONFIG_INVALID',
+          message: 'Brave search requires PI_WEB_AGENT_BRAVE_API_KEY.'
+        }
+      });
+    }
+
+    try {
+      const response = await fetchImpl(buildBraveUrl(normalizedQuery), {
+        headers: {
+          Accept: 'application/json',
+          'X-Subscription-Token': apiKey
+        }
+      });
+
+      if (!response.ok) {
+        throw new Error(`HTTP ${response.status}`);
+      }
+
+      const parsed = (await response.json()) as BraveSearchResponse;
+      const results = normalizeResults(parsed);
+
+      if (results.length === 0) {
+        return resultWithPresentation({
+          status: 'error',
+          results: [],
+          metadata: { backend: 'brave', cacheHit: false },
+          error: { code: 'NO_RESULTS', message: 'Brave returned no usable results for this query.' }
+        });
+      }
+
+      return resultWithPresentation({
+        status: 'ok',
+        results,
+        metadata: { backend: 'brave', cacheHit: false }
+      });
+    } catch (error) {
+      const rawMessage = error instanceof Error ? error.message : String(error);
+      return resultWithPresentation({
+        status: 'error',
+        results: [],
+        metadata: { backend: 'brave', cacheHit: false },
+        error: { code: 'FETCH_FAILED', message: `Brave search request failed: ${rawMessage}` }
+      });
+    }
+  };
+}

package/dist/extensions/pi-web-agent/src/search/duckduckgo.ts

@@ -33,12 +33,39 @@ export function buildSearchUrl(query: string): string {
   return `https://html.duckduckgo.com/html/?${params.toString()}`;
 }
 
-export async function fetchDuckDuckGoHtml(query: string): Promise<string> {
-  const response = await fetch(buildSearchUrl(query));
-  if (!response.ok) {
-    throw new Error(`DuckDuckGo request failed with ${response.status}`);
+export const DUCKDUCKGO_HEADERS = {
+  'User-Agent':
+    'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0',
+  Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+  'Accept-Language': 'en-US,en;q=0.9'
+} as const;
+
+const defaultSleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
+
+export async function fetchDuckDuckGoHtml(
+  query: string,
+  {
+    fetchImpl = fetch,
+    retries = 1,
+    sleep = defaultSleep
+  }: { fetchImpl?: typeof fetch; retries?: number; sleep?: (ms: number) => Promise<void> } = {}
+): Promise<string> {
+  let lastError: unknown;
+  for (let attempt = 0; attempt <= retries; attempt += 1) {
+    try {
+      const response = await fetchImpl(buildSearchUrl(query), { headers: { ...DUCKDUCKGO_HEADERS } });
+      if (!response.ok) {
+        throw new Error(`DuckDuckGo request failed with ${response.status}`);
+      }
+      return response.text();
+    } catch (error) {
+      lastError = error;
+      if (attempt < retries) {
+        await sleep(500);
+      }
+    }
   }
-  return response.text();
+  throw lastError instanceof Error ? lastError : new Error('DuckDuckGo request failed');
 }
 
 export function parseDuckDuckGoResults(html: string): ParsedDuckDuckGoResults {

package/dist/extensions/pi-web-agent/src/search/exa.ts

@@ -0,0 +1,109 @@
+import { buildSearchPresentation } from '../presentation/search-presentation.js';
+import type { SearchResult, WebSearchResponse } from '../types.js';
+
+type ExaSearchResult = {
+  title?: unknown;
+  url?: unknown;
+  text?: unknown;
+};
+
+type ExaSearchResponse = {
+  results?: ExaSearchResult[];
+};
+
+const EXA_SEARCH_URL = 'https://api.exa.ai/search';
+
+function resultWithPresentation(result: WebSearchResponse): WebSearchResponse {
+  return { ...result, presentation: buildSearchPresentation(result) };
+}
+
+function normalizeResults(response: ExaSearchResponse): SearchResult[] {
+  return (response.results ?? []).flatMap((item) => {
+    if (typeof item.title !== 'string' || typeof item.url !== 'string') {
+      return [];
+    }
+
+    return [
+      {
+        title: item.title,
+        url: item.url,
+        snippet: typeof item.text === 'string' ? item.text : ''
+      }
+    ];
+  });
+}
+
+export function createExaSearchTool({
+  apiKey,
+  fetchImpl = fetch
+}: {
+  apiKey?: string;
+  fetchImpl?: typeof fetch;
+}) {
+  return async function exaSearch({ query }: { query: string }): Promise<WebSearchResponse> {
+    const normalizedQuery = query.trim();
+
+    if (!normalizedQuery) {
+      return resultWithPresentation({
+        status: 'error',
+        results: [],
+        metadata: { backend: 'exa', cacheHit: false },
+        error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
+      });
+    }
+
+    if (!apiKey?.trim()) {
+      return resultWithPresentation({
+        status: 'error',
+        results: [],
+        metadata: { backend: 'exa', cacheHit: false },
+        error: {
+          code: 'BACKEND_CONFIG_INVALID',
+          message: 'Exa search requires EXA_API_KEY.'
+        }
+      });
+    }
+
+    try {
+      const response = await fetchImpl(EXA_SEARCH_URL, {
+        method: 'POST',
+        headers: {
+          Accept: 'application/json',
+          'Content-Type': 'application/json',
+          'x-api-key': apiKey
+        },
+        body: JSON.stringify({ query: normalizedQuery, numResults: 10 })
+      });
+
+      if (!response.ok) {
+        throw new Error(`HTTP ${response.status}`);
+      }
+
+      const parsed = (await response.json()) as ExaSearchResponse;
+      const results = normalizeResults(parsed);
+
+      if (results.length === 0) {
+        return resultWithPresentation({
+          status: 'error',
+          results: [],
+          metadata: { backend: 'exa', cacheHit: false },
+          error: { code: 'NO_RESULTS', message: 'Exa returned no usable results for this query.' }
+        });
+      }
+
+      return resultWithPresentation({
+        status: 'ok',
+        results,
+        metadata: { backend: 'exa', cacheHit: false }
+      });
+    } catch (error) {
+      const rawMessage = error instanceof Error ? error.message : String(error);
+      return resultWithPresentation({
+        status: 'error',
+        results: [],
+        metadata: { backend: 'exa', cacheHit: false },
+        error: { code: 'FETCH_FAILED', message: `Exa search request failed: ${rawMessage}` }
+      });
+    }
+  };
+}

package/dist/extensions/pi-web-agent/src/search/fanout.ts

@@ -0,0 +1,154 @@
+import { buildSearchPresentation } from '../presentation/search-presentation.js';
+import { canonicalizeUrl } from '../orchestration/url.js';
+import type { FanoutMetadata, FanoutMode, SearchProviderName, SearchResult, WebSearchResponse } from '../types.js';
+
+export type FanoutProvider = {
+  name: SearchProviderName;
+  search: (input: { query: string }) => Promise<WebSearchResponse>;
+};
+
+const FANOUT_MIN_RESULTS = 3;
+
+type RankedEntry = {
+  result: SearchResult;
+  providers: Set<SearchProviderName>;
+  bestRank: number;
+};
+
+function hostOf(url: string): string | undefined {
+  try {
+    return new URL(url).hostname.toLowerCase().replace(/^www\./, '');
+  } catch {
+    return undefined;
+  }
+}
+
+function primaryLooksWeak(results: SearchResult[]): boolean {
+  if (results.length < FANOUT_MIN_RESULTS) return true;
+  const hosts = new Set(results.map((r) => hostOf(r.url)).filter(Boolean));
+  return hosts.size <= 1;
+}
+
+function merge(lists: Array<{ name: SearchProviderName; results: SearchResult[] }>): SearchResult[] {
+  const byKey = new Map<string, RankedEntry>();
+
+  for (const { name, results } of lists) {
+    const seenThisProvider = new Set<string>();
+    results.forEach((result, index) => {
+      const key = canonicalizeUrl(result.url) ?? result.url;
+      if (seenThisProvider.has(key)) return; // ignore duplicates within a single provider's own list
+      seenThisProvider.add(key);
+      const existing = byKey.get(key);
+      if (!existing) {
+        byKey.set(key, { result: { ...result }, providers: new Set([name]), bestRank: index });
+        return;
+      }
+      existing.providers.add(name);
+      existing.bestRank = Math.min(existing.bestRank, index);
+      if ((result.title?.length ?? 0) > (existing.result.title?.length ?? 0)) existing.result.title = result.title;
+      if ((result.snippet?.length ?? 0) > (existing.result.snippet?.length ?? 0)) existing.result.snippet = result.snippet;
+    });
+  }
+
+  return [...byKey.values()]
+    .sort((a, b) => b.providers.size - a.providers.size || a.bestRank - b.bestRank)
+    .map((entry) => entry.result);
+}
+
+function withPresentation(result: WebSearchResponse): WebSearchResponse {
+  return { ...result, presentation: buildSearchPresentation(result) };
+}
+
+const FANOUT_PROVIDER_TIMEOUT_MS = 8000;
+
+/** Resolve to undefined if the provider doesn't answer in time, so one slow/unreachable
+ *  provider (e.g. a down self-hosted SearXNG) can't stall the whole fanout across passes. */
+function withTimeout(
+  promise: Promise<WebSearchResponse>,
+  ms: number
+): Promise<WebSearchResponse | undefined> {
+  return new Promise((resolve) => {
+    const timer = setTimeout(() => resolve(undefined), ms);
+    promise.then(
+      (value) => {
+        clearTimeout(timer);
+        resolve(value);
+      },
+      () => {
+        clearTimeout(timer);
+        resolve(undefined);
+      }
+    );
+  });
+}
+
+export function createFanoutSearch({
+  providers,
+  mode,
+  timeoutMs = FANOUT_PROVIDER_TIMEOUT_MS
+}: {
+  providers: FanoutProvider[];
+  mode: Exclude<FanoutMode, 'off'>;
+  timeoutMs?: number;
+}) {
+  return async function fanoutSearch({ query }: { query: string }): Promise<WebSearchResponse> {
+    const [primary, ...rest] = providers;
+
+    async function runSet(set: FanoutProvider[]) {
+      const settled = await Promise.all(set.map((p) => withTimeout(p.search({ query }), timeoutMs)));
+      const contributing: Array<{ name: SearchProviderName; results: SearchResult[] }> = [];
+      const skipped: SearchProviderName[] = [];
+      set.forEach((provider, i) => {
+        const value = settled[i];
+        if (value && value.status === 'ok' && value.results.length > 0) {
+          contributing.push({ name: provider.name, results: value.results });
+        } else {
+          skipped.push(provider.name);
+        }
+      });
+      return { contributing, skipped };
+    }
+
+    function finalize(
+      lists: Array<{ name: SearchProviderName; results: SearchResult[] }>,
+      skipped: SearchProviderName[],
+      resolvedMode: Exclude<FanoutMode, 'off'>
+    ): WebSearchResponse {
+      if (lists.length === 0) {
+        const fanout: FanoutMetadata = { mode: resolvedMode, providers: [] };
+        if (skipped.length > 0) fanout.skipped = skipped;
+        return withPresentation({
+          status: 'error',
+          results: [],
+          metadata: { backend: primary.name, cacheHit: false, fanout },
+          error: { code: 'FANOUT_NO_RESULTS', message: 'No fanout provider returned usable results.' }
+        });
+      }
+      const fanout: FanoutMetadata = { mode: resolvedMode, providers: lists.map((l) => l.name) };
+      if (skipped.length > 0) fanout.skipped = skipped;
+      return withPresentation({
+        status: 'ok',
+        results: merge(lists),
+        metadata: { backend: primary.name, cacheHit: false, fanout }
+      });
+    }
+
+    if (mode === 'auto') {
+      const primaryOutcome = await withTimeout(primary.search({ query }), timeoutMs);
+      const primaryResults = primaryOutcome?.status === 'ok' ? primaryOutcome.results : [];
+      if (primaryResults.length > 0 && !primaryLooksWeak(primaryResults)) {
+        return withPresentation(primaryOutcome as WebSearchResponse); // strong primary: no fanout
+      }
+      const { contributing, skipped } = await runSet(rest);
+      const lists = [
+        ...(primaryResults.length ? [{ name: primary.name, results: primaryResults }] : []),
+        ...contributing
+      ];
+      const allSkipped = [...(primaryResults.length ? [] : [primary.name]), ...skipped];
+      return finalize(lists, allSkipped, 'auto');
+    }
+
+    const { contributing, skipped } = await runSet(providers);
+    return finalize(contributing, skipped, 'on');
+  };
+}