@selesai/code 0.1.19 → 0.2.1

package/dist/extensions/pi-web-agent/src/extension.ts

@@ -0,0 +1,126 @@
+import type { ExtensionAPI } from '@earendil-works/pi-coding-agent';
+import { DEFAULT_BACKEND_CONFIG, type BackendConfig } from './backends/config.js';
+import { Type } from 'typebox';
+import { registerWebAgentConfigCommands } from './commands/web-agent-config.js';
+import { DEFAULT_PRESENTATION_CONFIG, resolvePresentationMode } from './presentation/config.js';
+import { createResearchWorkflow } from './orchestration/index.js';
+import { loadPresentationConfigLayers } from './presentation/config-store.js';
+import { selectPresentationView } from './presentation/select-view.js';
+import type {
+  PresentationConfig,
+  PresentationEnvelope,
+  PresentationToolName
+} from './presentation/types.js';
+import { createWebExploreTool } from './tools/web-explore.js';
+import type { WebExploreResponse } from './types.js';
+import { getUpdateChangelogNotice } from './changelog-notice.js';
+
+type ToolResultWithPresentation = {
+  presentation?: PresentationEnvelope;
+};
+
+async function loadWebAgentConfig(pi: ExtensionAPI) {
+  const store = (
+    pi as ExtensionAPI & {
+      __presentationConfigStore?: {
+        load?: typeof loadPresentationConfigLayers;
+      };
+    }
+  ).__presentationConfigStore;
+
+  return store?.load?.() ?? loadPresentationConfigLayers();
+}
+
+async function getEffectivePresentationConfig(pi: ExtensionAPI): Promise<PresentationConfig> {
+  try {
+    const loaded = await loadWebAgentConfig(pi);
+    return loaded.effectiveConfig;
+  } catch {
+    return DEFAULT_PRESENTATION_CONFIG;
+  }
+}
+
+async function getEffectiveBackendConfig(pi: ExtensionAPI): Promise<BackendConfig> {
+  try {
+    const loaded = await loadWebAgentConfig(pi);
+    return loaded.effectiveBackends ?? DEFAULT_BACKEND_CONFIG;
+  } catch {
+    return DEFAULT_BACKEND_CONFIG;
+  }
+}
+
+async function renderToolText(
+  pi: ExtensionAPI,
+  toolName: PresentationToolName,
+  details: ToolResultWithPresentation
+): Promise<string> {
+  const config = await getEffectivePresentationConfig(pi);
+  const mode = resolvePresentationMode(toolName, config);
+  return selectPresentationView(details.presentation, mode) ?? JSON.stringify(details, null, 2);
+}
+
+export default function extension(pi: ExtensionAPI) {
+  registerWebAgentConfigCommands(pi);
+
+  const injectedWebExplore = (pi as ExtensionAPI & { __webExploreTool?: ReturnType<typeof createWebExploreTool> }).__webExploreTool;
+  let cachedBackendKey: string | undefined;
+  let cachedWebExplore: ReturnType<typeof createWebExploreTool> | undefined;
+
+  async function getConfiguredWebExplore() {
+    if (injectedWebExplore) return injectedWebExplore;
+
+    const backendConfig = await getEffectiveBackendConfig(pi);
+    const backendKey = JSON.stringify(backendConfig);
+    if (!cachedWebExplore || cachedBackendKey !== backendKey) {
+      cachedBackendKey = backendKey;
+      cachedWebExplore = createWebExploreTool({
+        explore: createResearchWorkflow({ backendConfig })
+      });
+    }
+
+    return cachedWebExplore;
+  }
+
+  pi.on('session_start', async (_event, ctx) => {
+    try {
+      const notice = await getUpdateChangelogNotice();
+      if (notice) {
+        ctx.ui.notify(`pi-web-agent updated\n\n${notice}`, 'info');
+      }
+    } catch {
+      // Never block extension startup on changelog display.
+    }
+  });
+
+  pi.on('before_agent_start', async (event) => ({
+    systemPrompt:
+      `${event.systemPrompt}\n\n` +
+      'For web research questions that require finding and comparing sources, use web_explore. ' +
+      'web_explore handles search, fetch, source ranking, and headless escalation internally. ' +
+      'If more web evidence is needed after web_explore, call web_explore again with a narrower query; do not use shell/network commands such as curl, Invoke-WebRequest, npm view/search/pack, or direct HTTP URLs for web research.'
+  }));
+
+  pi.registerTool({
+    name: 'web_explore',
+    label: 'Web Explore',
+    description:
+      'Research a web question using bounded search/fetch passes, source ranking, and targeted headless escalation. Use this for web research, current docs/discussion lookups, and recommendation summaries.',
+    parameters: Type.Object({
+      query: Type.String({ description: 'Web research question to explore.' })
+    }),
+    async execute(_toolCallId, params) {
+      const webExplore = await getConfiguredWebExplore();
+      const result: WebExploreResponse = await webExplore({ query: params.query });
+      return {
+        content: [
+          {
+            type: 'text',
+            text: await renderToolText(pi, 'web_explore', result)
+          }
+        ],
+        details: result,
+        isError: result.status === 'error'
+      };
+    }
+  });
+}

package/dist/extensions/pi-web-agent/src/extract/readability.ts

@@ -0,0 +1,118 @@
+import { Readability } from '@mozilla/readability';
+import { JSDOM, VirtualConsole } from 'jsdom';
+import type { ExtractedContent } from '../types.js';
+
+export type ReadableExtractionMode = 'readability' | 'fallback';
+
+export type SafeReadableExtraction = {
+  mode: ReadableExtractionMode;
+  content: ExtractedContent;
+};
+
+export function extractReadableContent(html: string, maxLength = 4000): ExtractedContent {
+  let stylesheetError: Error | undefined;
+  const virtualConsole = new VirtualConsole();
+
+  virtualConsole.on('jsdomError', (error) => {
+    if (!stylesheetError && error.message.includes('Could not parse CSS stylesheet')) {
+      stylesheetError = error;
+    }
+  });
+
+  const dom = new JSDOM(html, {
+    url: 'https://example.com',
+    virtualConsole
+  });
+
+  if (stylesheetError) {
+    throw stylesheetError;
+  }
+
+  const article = new Readability(dom.window.document).parse();
+  const rawText = (article?.textContent ?? dom.window.document.body.textContent ?? '').trim();
+  const text = rawText.slice(0, maxLength);
+
+  const fallbackTitle = dom.window.document.title || undefined;
+
+  return {
+    title: article?.title ?? fallbackTitle,
+    byline: article?.byline || undefined,
+    text
+  };
+}
+
+function decodeHtmlEntities(text: string): string {
+  return text
+    .replace(/ /gi, ' ')
+    .replace(/&/gi, '&')
+    .replace(/&lt;/gi, '<')
+    .replace(/&gt;/gi, '>')
+    .replace(/&quot;/gi, '"')
+    .replace(/&#39;/gi, "'")
+    .replace(/&#x27;/gi, "'")
+    .replace(/&#x2F;/gi, '/')
+    .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)))
+    .replace(/&#x([\da-f]+);/gi, (_, code) => String.fromCharCode(parseInt(code, 16)));
+}
+
+function extractTitle(html: string): string | undefined {
+  const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+  if (!match) return undefined;
+  return decodeHtmlEntities(match[1].replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim()) || undefined;
+}
+
+function stripTagContent(html: string, tagName: string): string {
+  return html.replace(new RegExp(`<${tagName}\\b[^>]*>[\\s\\S]*?<\\/${tagName}>`, 'gi'), ' ');
+}
+
+function extractPreferredSection(html: string): string {
+  const mainMatch = html.match(/<main\b[^>]*>([\s\S]*?)<\/main>/i);
+  if (mainMatch) return mainMatch[1];
+
+  const articleMatch = html.match(/<article\b[^>]*>([\s\S]*?)<\/article>/i);
+  if (articleMatch) return articleMatch[1];
+
+  const bodyMatch = html.match(/<body\b[^>]*>([\s\S]*?)<\/body>/i);
+  if (bodyMatch) return bodyMatch[1];
+
+  return html;
+}
+
+function extractFallbackText(html: string, maxLength: number): ExtractedContent {
+  const title = extractTitle(html);
+  let section = extractPreferredSection(html);
+
+  section = stripTagContent(section, 'script');
+  section = stripTagContent(section, 'style');
+  section = stripTagContent(section, 'noscript');
+  section = stripTagContent(section, 'svg');
+  section = stripTagContent(section, 'template');
+
+  const text = decodeHtmlEntities(section)
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim()
+    .slice(0, maxLength);
+
+  return {
+    title,
+    text
+  };
+}
+
+export function extractReadableContentSafely(
+  html: string,
+  maxLength = 4000
+): SafeReadableExtraction {
+  try {
+    return {
+      mode: 'readability',
+      content: extractReadableContent(html, maxLength)
+    };
+  } catch {
+    return {
+      mode: 'fallback',
+      content: extractFallbackText(html, maxLength)
+    };
+  }
+}

package/dist/extensions/pi-web-agent/src/fetch/browser-resolution.ts

@@ -0,0 +1,199 @@
+type SupportedPlatform = NodeJS.Platform | string;
+type BrowserName = 'configured' | 'chrome' | 'edge' | 'brave' | 'chromium';
+
+type BrowserCandidate = {
+  browser: Exclude<BrowserName, 'configured'>;
+  paths: string[];
+  commands: string[];
+};
+
+const WINDOWS_CANDIDATES: BrowserCandidate[] = [
+  {
+    browser: 'chrome',
+    paths: [
+      'C:/Program Files/Google/Chrome/Application/chrome.exe',
+      'C:/Program Files (x86)/Google/Chrome/Application/chrome.exe'
+    ],
+    commands: ['chrome.exe', 'chrome', 'google-chrome']
+  },
+  {
+    browser: 'edge',
+    paths: [
+      'C:/Program Files/Microsoft/Edge/Application/msedge.exe',
+      'C:/Program Files (x86)/Microsoft/Edge/Application/msedge.exe'
+    ],
+    commands: ['msedge.exe', 'msedge', 'microsoft-edge']
+  },
+  {
+    browser: 'brave',
+    paths: [
+      'C:/Program Files/BraveSoftware/Brave-Browser/Application/brave.exe',
+      'C:/Program Files (x86)/BraveSoftware/Brave-Browser/Application/brave.exe'
+    ],
+    commands: ['brave.exe', 'brave']
+  },
+  {
+    browser: 'chromium',
+    paths: [
+      'C:/Program Files/Chromium/Application/chrome.exe',
+      'C:/Program Files (x86)/Chromium/Application/chrome.exe'
+    ],
+    commands: ['chromium.exe', 'chromium']
+  }
+];
+
+const MACOS_CANDIDATES: BrowserCandidate[] = [
+  {
+    browser: 'chrome',
+    paths: ['/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'],
+    commands: ['google-chrome', 'chrome']
+  },
+  {
+    browser: 'edge',
+    paths: ['/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge'],
+    commands: ['microsoft-edge', 'msedge']
+  },
+  {
+    browser: 'brave',
+    paths: ['/Applications/Brave Browser.app/Contents/MacOS/Brave Browser'],
+    commands: ['brave-browser', 'brave']
+  },
+  {
+    browser: 'chromium',
+    paths: ['/Applications/Chromium.app/Contents/MacOS/Chromium'],
+    commands: ['chromium', 'chromium-browser']
+  }
+];
+
+const LINUX_CANDIDATES: BrowserCandidate[] = [
+  {
+    browser: 'chrome',
+    paths: ['/usr/bin/google-chrome', '/usr/local/bin/google-chrome', '/opt/google/chrome/chrome'],
+    commands: ['google-chrome', 'google-chrome-stable', 'chrome']
+  },
+  {
+    browser: 'edge',
+    paths: ['/usr/bin/microsoft-edge', '/opt/microsoft/msedge/msedge'],
+    commands: ['microsoft-edge', 'microsoft-edge-stable', 'msedge']
+  },
+  {
+    browser: 'brave',
+    paths: ['/usr/bin/brave-browser', '/usr/local/bin/brave-browser', '/opt/brave.com/brave/brave'],
+    commands: ['brave-browser', 'brave']
+  },
+  {
+    browser: 'chromium',
+    paths: ['/usr/bin/chromium', '/usr/bin/chromium-browser', '/snap/bin/chromium'],
+    commands: ['chromium', 'chromium-browser']
+  }
+];
+
+export type BrowserResolutionResult =
+  | {
+      ok: true;
+      executablePath: string;
+      browser: BrowserName;
+    }
+  | {
+      ok: false;
+      error: {
+        code: 'BROWSER_NOT_FOUND' | 'CONFIGURED_BROWSER_NOT_FOUND';
+        message: string;
+      };
+    };
+
+export async function resolveBrowserExecutable({
+  configuredPath,
+  platform = process.platform,
+  env = process.env,
+  fileExists = defaultFileExists
+}: {
+  configuredPath?: string;
+  platform?: SupportedPlatform;
+  env?: NodeJS.ProcessEnv | Record<string, string | undefined>;
+  fileExists?: (path: string) => Promise<boolean>;
+}): Promise<BrowserResolutionResult> {
+  if (configuredPath) {
+    if (await fileExists(configuredPath)) {
+      return {
+        ok: true,
+        executablePath: configuredPath,
+        browser: 'configured'
+      };
+    }
+
+    return {
+      ok: false,
+      error: {
+        code: 'CONFIGURED_BROWSER_NOT_FOUND',
+        message: `Configured browser path was not found: ${configuredPath}`
+      }
+    };
+  }
+
+  const candidates = getCandidatesForPlatform(platform);
+
+  for (const candidate of candidates) {
+    for (const path of candidate.paths) {
+      if (await fileExists(path)) {
+        return { ok: true, executablePath: path, browser: candidate.browser };
+      }
+    }
+  }
+
+  for (const candidate of candidates) {
+    for (const path of getPathCommandCandidates(candidate.commands, platform, env)) {
+      if (await fileExists(path)) {
+        return { ok: true, executablePath: path, browser: candidate.browser };
+      }
+    }
+  }
+
+  return {
+    ok: false,
+    error: {
+      code: 'BROWSER_NOT_FOUND',
+      message: 'No compatible local browser was found for headless fetch.'
+    }
+  };
+}
+
+function getCandidatesForPlatform(platform: SupportedPlatform): BrowserCandidate[] {
+  if (platform === 'win32') return WINDOWS_CANDIDATES;
+  if (platform === 'darwin') return MACOS_CANDIDATES;
+  if (platform === 'linux') return LINUX_CANDIDATES;
+
+  return [...WINDOWS_CANDIDATES, ...MACOS_CANDIDATES, ...LINUX_CANDIDATES];
+}
+
+function getPathCommandCandidates(
+  commands: string[],
+  platform: SupportedPlatform,
+  env: NodeJS.ProcessEnv | Record<string, string | undefined>
+): string[] {
+  const pathValue = env.PATH ?? env.Path ?? env.path;
+  if (!pathValue) return [];
+
+  const delimiter = platform === 'win32' ? ';' : ':';
+  const dirs = pathValue.split(delimiter).filter(Boolean);
+  const extensions = platform === 'win32' ? ['', '.exe'] : [''];
+
+  return dirs.flatMap((dir) =>
+    commands.flatMap((command) =>
+      extensions.map((extension) => {
+        const normalizedCommand = command.toLowerCase().endsWith(extension) ? command : `${command}${extension}`;
+        return `${dir.replace(/[\\/]$/, '')}/${normalizedCommand}`;
+      })
+    )
+  );
+}
+
+async function defaultFileExists(path: string): Promise<boolean> {
+  try {
+    const { access } = await import('node:fs/promises');
+    await access(path);
+    return true;
+  } catch {
+    return false;
+  }
+}

package/dist/extensions/pi-web-agent/src/fetch/firecrawl-fetch.ts

@@ -0,0 +1,100 @@
+import type { FirecrawlOptions } from '../backends/config.js';
+import type { WebFetchResponse } from '../types.js';
+
+type FirecrawlResponse = {
+  success?: boolean;
+  data?: {
+    markdown?: unknown;
+    html?: unknown;
+    metadata?: {
+      title?: unknown;
+      sourceURL?: unknown;
+    };
+  };
+  error?: unknown;
+};
+
+function buildScrapeUrl(baseUrl: string) {
+  return new URL('/v1/scrape', baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`).toString();
+}
+
+function errorMessage(error: unknown) {
+  return error instanceof Error ? error.message : String(error);
+}
+
+export function createFirecrawlFetcher({
+  baseUrl,
+  apiKey,
+  options,
+  fetchImpl = fetch
+}: {
+  baseUrl: string;
+  apiKey?: string;
+  options?: FirecrawlOptions;
+  fetchImpl?: typeof fetch;
+}) {
+  return async function firecrawlFetch(url: string): Promise<WebFetchResponse> {
+    try {
+      const headers: Record<string, string> = { 'content-type': 'application/json' };
+      if (apiKey) {
+        headers.Authorization = `Bearer ${apiKey}`;
+      }
+
+      const body = {
+        url,
+        formats: options?.formats ?? ['markdown'],
+        ...(options?.onlyMainContent !== undefined ? { onlyMainContent: options.onlyMainContent } : {})
+      };
+
+      const response = await fetchImpl(buildScrapeUrl(baseUrl), {
+        method: 'POST',
+        headers,
+        body: JSON.stringify(body)
+      });
+
+      if (!response.ok) {
+        throw new Error(`HTTP ${response.status}`);
+      }
+
+      const parsed = (await response.json()) as FirecrawlResponse;
+      if (parsed.success === false) {
+        throw new Error(typeof parsed.error === 'string' ? parsed.error : 'Firecrawl scrape failed.');
+      }
+
+      const text = typeof parsed.data?.markdown === 'string'
+        ? parsed.data.markdown
+        : typeof parsed.data?.html === 'string'
+          ? parsed.data.html
+          : '';
+      const resolvedUrl = typeof parsed.data?.metadata?.sourceURL === 'string'
+        ? parsed.data.metadata.sourceURL
+        : url;
+      const title = typeof parsed.data?.metadata?.title === 'string'
+        ? parsed.data.metadata.title
+        : undefined;
+
+      if (!text.trim()) {
+        return {
+          status: 'needs_headless',
+          url: resolvedUrl,
+          metadata: { method: 'firecrawl', cacheHit: false },
+          error: { code: 'WEAK_EXTRACTION', message: 'Firecrawl did not return useful page text.' }
+        };
+      }
+
+      return {
+        status: 'ok',
+        url: resolvedUrl,
+        content: { title, text },
+        metadata: { method: 'firecrawl', cacheHit: false, truncated: text.length >= 4000 }
+      };
+    } catch (error) {
+      return {
+        status: 'error',
+        url,
+        metadata: { method: 'firecrawl', cacheHit: false },
+        error: { code: 'FETCH_FAILED', message: `Firecrawl scrape failed: ${errorMessage(error)}` }
+      };
+    }
+  };
+}

package/dist/extensions/pi-web-agent/src/fetch/headless-fetch.ts

@@ -0,0 +1,117 @@
+import { chromium } from 'playwright';
+import { extractReadableContentSafely } from '../extract/readability.js';
+import { resolveBrowserExecutable, type BrowserResolutionResult } from './browser-resolution.js';
+import type { WebFetchHeadlessResponse } from '../types.js';
+
+function cleanupRenderedText(text: string): string {
+  let cleaned = text.replace(/(Show more)(\s+\1){1,}/gi, '$1');
+  cleaned = cleaned.replace(/(Privacy Terms)(\s+\1){1,}/gi, '$1');
+  cleaned = cleaned.replace(/\s+/g, ' ').trim();
+  return cleaned;
+}
+
+export async function headlessFetch(
+  url: string,
+  {
+    configuredPath,
+    resolveBrowser = (options?: { configuredPath?: string }) =>
+      resolveBrowserExecutable({ configuredPath: options?.configuredPath }),
+    launchBrowser = ({ executablePath, headless }: { executablePath?: string; headless: true }) =>
+      chromium.launch(executablePath ? { executablePath, headless } : { headless }),
+    now = () => Date.now()
+  }: {
+    configuredPath?: string;
+    resolveBrowser?: (options?: { configuredPath?: string }) => Promise<BrowserResolutionResult>;
+    launchBrowser?: (options: { executablePath?: string; headless: true }) => Promise<{
+      newContext: () => Promise<{ newPage: () => Promise<any>; close: () => Promise<void> }>;
+      close: () => Promise<void>;
+    }>;
+    now?: () => number;
+  } = {}
+): Promise<WebFetchHeadlessResponse> {
+  const resolved = await resolveBrowser({ configuredPath });
+  if (!resolved.ok && resolved.error.code === 'CONFIGURED_BROWSER_NOT_FOUND') {
+    return {
+      status: 'error',
+      url,
+      metadata: { method: 'headless', cacheHit: false },
+      error: resolved.error
+    };
+  }
+
+  const browserName = resolved.ok ? resolved.browser : 'chromium';
+  const launchOptions = resolved.ok
+    ? { executablePath: resolved.executablePath, headless: true as const }
+    : { headless: true as const };
+
+  let browser: Awaited<ReturnType<typeof launchBrowser>> | undefined;
+  let context: Awaited<ReturnType<Awaited<ReturnType<typeof launchBrowser>>['newContext']>> | undefined;
+  let page: Awaited<ReturnType<Awaited<ReturnType<Awaited<ReturnType<typeof launchBrowser>>['newContext']>>['newPage']>> | undefined;
+
+  try {
+    browser = await launchBrowser(launchOptions);
+    context = await browser.newContext();
+    page = await context.newPage();
+
+    const startedAt = now();
+    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
+    await page.waitForLoadState('load', { timeout: 10000 });
+    await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => undefined);
+    const html = await page.content();
+    const finishedAt = now();
+
+    const extraction = extractReadableContentSafely(html);
+    const cleanedContent = {
+      ...extraction.content,
+      text: cleanupRenderedText(extraction.content.text)
+    };
+
+    if (!cleanedContent.text || cleanedContent.text.length < 40) {
+      return {
+        status: 'blocked',
+        url,
+        metadata: {
+          method: 'headless',
+          cacheHit: false,
+          browser: browserName,
+          navigationMs: finishedAt - startedAt
+        },
+        error: {
+          code: 'HEADLESS_EXTRACTION_WEAK',
+          message: 'Rendered page did not produce enough readable content.'
+        }
+      };
+    }
+
+    return {
+      status: 'ok',
+      url,
+      content: cleanedContent,
+      metadata: {
+        method: 'headless',
+        cacheHit: false,
+        browser: browserName,
+        navigationMs: finishedAt - startedAt,
+        truncated: cleanedContent.text.length >= 4000
+      }
+    };
+  } catch (error) {
+    return {
+      status: 'error',
+      url,
+      metadata: {
+        method: 'headless',
+        cacheHit: false,
+        browser: browserName
+      },
+      error: {
+        code: 'HEADLESS_NAVIGATION_FAILED',
+        message: error instanceof Error ? error.message : 'Unknown headless navigation failure.'
+      }
+    };
+  } finally {
+    await page?.close?.().catch(() => undefined);
+    await context?.close?.().catch(() => undefined);
+    await browser?.close?.().catch(() => undefined);
+  }
+}