@selesai/code 0.13.4 → 0.13.6

package/dist/extensions/pi-web-agent/src/fetch/firecrawl-fetch.ts

@@ -1,100 +1,100 @@
-import type { FirecrawlOptions } from '../backends/config.js';
-import type { WebFetchResponse } from '../types.js';
-
-type FirecrawlResponse = {
-  success?: boolean;
-  data?: {
-    markdown?: unknown;
-    html?: unknown;
-    metadata?: {
-      title?: unknown;
-      sourceURL?: unknown;
-    };
-  };
-  error?: unknown;
-};
-
-function buildScrapeUrl(baseUrl: string) {
-  return new URL('/v1/scrape', baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`).toString();
-}
-
-function errorMessage(error: unknown) {
-  return error instanceof Error ? error.message : String(error);
-}
-
-export function createFirecrawlFetcher({
-  baseUrl,
-  apiKey,
-  options,
-  fetchImpl = fetch
-}: {
-  baseUrl: string;
-  apiKey?: string;
-  options?: FirecrawlOptions;
-  fetchImpl?: typeof fetch;
-}) {
-  return async function firecrawlFetch(url: string): Promise<WebFetchResponse> {
-    try {
-      const headers: Record<string, string> = { 'content-type': 'application/json' };
-      if (apiKey) {
-        headers.Authorization = `Bearer ${apiKey}`;
-      }
-
-      const body = {
-        url,
-        formats: options?.formats ?? ['markdown'],
-        ...(options?.onlyMainContent !== undefined ? { onlyMainContent: options.onlyMainContent } : {})
-      };
-
-      const response = await fetchImpl(buildScrapeUrl(baseUrl), {
-        method: 'POST',
-        headers,
-        body: JSON.stringify(body)
-      });
-
-      if (!response.ok) {
-        throw new Error(`HTTP ${response.status}`);
-      }
-
-      const parsed = (await response.json()) as FirecrawlResponse;
-      if (parsed.success === false) {
-        throw new Error(typeof parsed.error === 'string' ? parsed.error : 'Firecrawl scrape failed.');
-      }
-
-      const text = typeof parsed.data?.markdown === 'string'
-        ? parsed.data.markdown
-        : typeof parsed.data?.html === 'string'
-          ? parsed.data.html
-          : '';
-      const resolvedUrl = typeof parsed.data?.metadata?.sourceURL === 'string'
-        ? parsed.data.metadata.sourceURL
-        : url;
-      const title = typeof parsed.data?.metadata?.title === 'string'
-        ? parsed.data.metadata.title
-        : undefined;
-
-      if (!text.trim()) {
-        return {
-          status: 'needs_headless',
-          url: resolvedUrl,
-          metadata: { method: 'firecrawl', cacheHit: false },
-          error: { code: 'WEAK_EXTRACTION', message: 'Firecrawl did not return useful page text.' }
-        };
-      }
-
-      return {
-        status: 'ok',
-        url: resolvedUrl,
-        content: { title, text },
-        metadata: { method: 'firecrawl', cacheHit: false, truncated: text.length >= 4000 }
-      };
-    } catch (error) {
-      return {
-        status: 'error',
-        url,
-        metadata: { method: 'firecrawl', cacheHit: false },
-        error: { code: 'FETCH_FAILED', message: `Firecrawl scrape failed: ${errorMessage(error)}` }
-      };
-    }
-  };
-}
+import type { FirecrawlOptions } from '../backends/config.js';
+import type { WebFetchResponse } from '../types.js';
+
+type FirecrawlResponse = {
+  success?: boolean;
+  data?: {
+    markdown?: unknown;
+    html?: unknown;
+    metadata?: {
+      title?: unknown;
+      sourceURL?: unknown;
+    };
+  };
+  error?: unknown;
+};
+
+function buildScrapeUrl(baseUrl: string) {
+  return new URL('/v1/scrape', baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`).toString();
+}
+
+function errorMessage(error: unknown) {
+  return error instanceof Error ? error.message : String(error);
+}
+
+export function createFirecrawlFetcher({
+  baseUrl,
+  apiKey,
+  options,
+  fetchImpl = fetch
+}: {
+  baseUrl: string;
+  apiKey?: string;
+  options?: FirecrawlOptions;
+  fetchImpl?: typeof fetch;
+}) {
+  return async function firecrawlFetch(url: string): Promise<WebFetchResponse> {
+    try {
+      const headers: Record<string, string> = { 'content-type': 'application/json' };
+      if (apiKey) {
+        headers.Authorization = `Bearer ${apiKey}`;
+      }
+
+      const body = {
+        url,
+        formats: options?.formats ?? ['markdown'],
+        ...(options?.onlyMainContent !== undefined ? { onlyMainContent: options.onlyMainContent } : {})
+      };
+
+      const response = await fetchImpl(buildScrapeUrl(baseUrl), {
+        method: 'POST',
+        headers,
+        body: JSON.stringify(body)
+      });
+
+      if (!response.ok) {
+        throw new Error(`HTTP ${response.status}`);
+      }
+
+      const parsed = (await response.json()) as FirecrawlResponse;
+      if (parsed.success === false) {
+        throw new Error(typeof parsed.error === 'string' ? parsed.error : 'Firecrawl scrape failed.');
+      }
+
+      const text = typeof parsed.data?.markdown === 'string'
+        ? parsed.data.markdown
+        : typeof parsed.data?.html === 'string'
+          ? parsed.data.html
+          : '';
+      const resolvedUrl = typeof parsed.data?.metadata?.sourceURL === 'string'
+        ? parsed.data.metadata.sourceURL
+        : url;
+      const title = typeof parsed.data?.metadata?.title === 'string'
+        ? parsed.data.metadata.title
+        : undefined;
+
+      if (!text.trim()) {
+        return {
+          status: 'needs_headless',
+          url: resolvedUrl,
+          metadata: { method: 'firecrawl', cacheHit: false },
+          error: { code: 'WEAK_EXTRACTION', message: 'Firecrawl did not return useful page text.' }
+        };
+      }
+
+      return {
+        status: 'ok',
+        url: resolvedUrl,
+        content: { title, text },
+        metadata: { method: 'firecrawl', cacheHit: false, truncated: text.length >= 4000 }
+      };
+    } catch (error) {
+      return {
+        status: 'error',
+        url,
+        metadata: { method: 'firecrawl', cacheHit: false },
+        error: { code: 'FETCH_FAILED', message: `Firecrawl scrape failed: ${errorMessage(error)}` }
+      };
+    }
+  };
+}

package/dist/extensions/pi-web-agent/src/fetch/headless-fetch.ts

@@ -1,117 +1,117 @@
-import { chromium } from 'playwright';
-import { extractReadableContentSafely } from '../extract/readability.js';
-import { resolveBrowserExecutable, type BrowserResolutionResult } from './browser-resolution.js';
-import type { WebFetchHeadlessResponse } from '../types.js';
-
-function cleanupRenderedText(text: string): string {
-  let cleaned = text.replace(/(Show more)(\s+\1){1,}/gi, '$1');
-  cleaned = cleaned.replace(/(Privacy Terms)(\s+\1){1,}/gi, '$1');
-  cleaned = cleaned.replace(/\s+/g, ' ').trim();
-  return cleaned;
-}
-
-export async function headlessFetch(
-  url: string,
-  {
-    configuredPath,
-    resolveBrowser = (options?: { configuredPath?: string }) =>
-      resolveBrowserExecutable({ configuredPath: options?.configuredPath }),
-    launchBrowser = ({ executablePath, headless }: { executablePath?: string; headless: true }) =>
-      chromium.launch(executablePath ? { executablePath, headless } : { headless }),
-    now = () => Date.now()
-  }: {
-    configuredPath?: string;
-    resolveBrowser?: (options?: { configuredPath?: string }) => Promise<BrowserResolutionResult>;
-    launchBrowser?: (options: { executablePath?: string; headless: true }) => Promise<{
-      newContext: () => Promise<{ newPage: () => Promise<any>; close: () => Promise<void> }>;
-      close: () => Promise<void>;
-    }>;
-    now?: () => number;
-  } = {}
-): Promise<WebFetchHeadlessResponse> {
-  const resolved = await resolveBrowser({ configuredPath });
-  if (!resolved.ok && resolved.error.code === 'CONFIGURED_BROWSER_NOT_FOUND') {
-    return {
-      status: 'error',
-      url,
-      metadata: { method: 'headless', cacheHit: false },
-      error: resolved.error
-    };
-  }
-
-  const browserName = resolved.ok ? resolved.browser : 'chromium';
-  const launchOptions = resolved.ok
-    ? { executablePath: resolved.executablePath, headless: true as const }
-    : { headless: true as const };
-
-  let browser: Awaited<ReturnType<typeof launchBrowser>> | undefined;
-  let context: Awaited<ReturnType<Awaited<ReturnType<typeof launchBrowser>>['newContext']>> | undefined;
-  let page: Awaited<ReturnType<Awaited<ReturnType<Awaited<ReturnType<typeof launchBrowser>>['newContext']>>['newPage']>> | undefined;
-
-  try {
-    browser = await launchBrowser(launchOptions);
-    context = await browser.newContext();
-    page = await context.newPage();
-
-    const startedAt = now();
-    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
-    await page.waitForLoadState('load', { timeout: 10000 });
-    await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => undefined);
-    const html = await page.content();
-    const finishedAt = now();
-
-    const extraction = extractReadableContentSafely(html);
-    const cleanedContent = {
-      ...extraction.content,
-      text: cleanupRenderedText(extraction.content.text)
-    };
-
-    if (!cleanedContent.text || cleanedContent.text.length < 40) {
-      return {
-        status: 'blocked',
-        url,
-        metadata: {
-          method: 'headless',
-          cacheHit: false,
-          browser: browserName,
-          navigationMs: finishedAt - startedAt
-        },
-        error: {
-          code: 'HEADLESS_EXTRACTION_WEAK',
-          message: 'Rendered page did not produce enough readable content.'
-        }
-      };
-    }
-
-    return {
-      status: 'ok',
-      url,
-      content: cleanedContent,
-      metadata: {
-        method: 'headless',
-        cacheHit: false,
-        browser: browserName,
-        navigationMs: finishedAt - startedAt,
-        truncated: cleanedContent.text.length >= 4000
-      }
-    };
-  } catch (error) {
-    return {
-      status: 'error',
-      url,
-      metadata: {
-        method: 'headless',
-        cacheHit: false,
-        browser: browserName
-      },
-      error: {
-        code: 'HEADLESS_NAVIGATION_FAILED',
-        message: error instanceof Error ? error.message : 'Unknown headless navigation failure.'
-      }
-    };
-  } finally {
-    await page?.close?.().catch(() => undefined);
-    await context?.close?.().catch(() => undefined);
-    await browser?.close?.().catch(() => undefined);
-  }
-}
+import { chromium } from 'playwright';
+import { extractReadableContentSafely } from '../extract/readability.js';
+import { resolveBrowserExecutable, type BrowserResolutionResult } from './browser-resolution.js';
+import type { WebFetchHeadlessResponse } from '../types.js';
+
+function cleanupRenderedText(text: string): string {
+  let cleaned = text.replace(/(Show more)(\s+\1){1,}/gi, '$1');
+  cleaned = cleaned.replace(/(Privacy Terms)(\s+\1){1,}/gi, '$1');
+  cleaned = cleaned.replace(/\s+/g, ' ').trim();
+  return cleaned;
+}
+
+export async function headlessFetch(
+  url: string,
+  {
+    configuredPath,
+    resolveBrowser = (options?: { configuredPath?: string }) =>
+      resolveBrowserExecutable({ configuredPath: options?.configuredPath }),
+    launchBrowser = ({ executablePath, headless }: { executablePath?: string; headless: true }) =>
+      chromium.launch(executablePath ? { executablePath, headless } : { headless }),
+    now = () => Date.now()
+  }: {
+    configuredPath?: string;
+    resolveBrowser?: (options?: { configuredPath?: string }) => Promise<BrowserResolutionResult>;
+    launchBrowser?: (options: { executablePath?: string; headless: true }) => Promise<{
+      newContext: () => Promise<{ newPage: () => Promise<any>; close: () => Promise<void> }>;
+      close: () => Promise<void>;
+    }>;
+    now?: () => number;
+  } = {}
+): Promise<WebFetchHeadlessResponse> {
+  const resolved = await resolveBrowser({ configuredPath });
+  if (!resolved.ok && resolved.error.code === 'CONFIGURED_BROWSER_NOT_FOUND') {
+    return {
+      status: 'error',
+      url,
+      metadata: { method: 'headless', cacheHit: false },
+      error: resolved.error
+    };
+  }
+
+  const browserName = resolved.ok ? resolved.browser : 'chromium';
+  const launchOptions = resolved.ok
+    ? { executablePath: resolved.executablePath, headless: true as const }
+    : { headless: true as const };
+
+  let browser: Awaited<ReturnType<typeof launchBrowser>> | undefined;
+  let context: Awaited<ReturnType<Awaited<ReturnType<typeof launchBrowser>>['newContext']>> | undefined;
+  let page: Awaited<ReturnType<Awaited<ReturnType<Awaited<ReturnType<typeof launchBrowser>>['newContext']>>['newPage']>> | undefined;
+
+  try {
+    browser = await launchBrowser(launchOptions);
+    context = await browser.newContext();
+    page = await context.newPage();
+
+    const startedAt = now();
+    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
+    await page.waitForLoadState('load', { timeout: 10000 });
+    await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => undefined);
+    const html = await page.content();
+    const finishedAt = now();
+
+    const extraction = extractReadableContentSafely(html);
+    const cleanedContent = {
+      ...extraction.content,
+      text: cleanupRenderedText(extraction.content.text)
+    };
+
+    if (!cleanedContent.text || cleanedContent.text.length < 40) {
+      return {
+        status: 'blocked',
+        url,
+        metadata: {
+          method: 'headless',
+          cacheHit: false,
+          browser: browserName,
+          navigationMs: finishedAt - startedAt
+        },
+        error: {
+          code: 'HEADLESS_EXTRACTION_WEAK',
+          message: 'Rendered page did not produce enough readable content.'
+        }
+      };
+    }
+
+    return {
+      status: 'ok',
+      url,
+      content: cleanedContent,
+      metadata: {
+        method: 'headless',
+        cacheHit: false,
+        browser: browserName,
+        navigationMs: finishedAt - startedAt,
+        truncated: cleanedContent.text.length >= 4000
+      }
+    };
+  } catch (error) {
+    return {
+      status: 'error',
+      url,
+      metadata: {
+        method: 'headless',
+        cacheHit: false,
+        browser: browserName
+      },
+      error: {
+        code: 'HEADLESS_NAVIGATION_FAILED',
+        message: error instanceof Error ? error.message : 'Unknown headless navigation failure.'
+      }
+    };
+  } finally {
+    await page?.close?.().catch(() => undefined);
+    await context?.close?.().catch(() => undefined);
+    await browser?.close?.().catch(() => undefined);
+  }
+}

package/dist/extensions/pi-web-agent/src/fetch/http-fetch.ts

@@ -1,67 +1,67 @@
-import { extractReadableContentSafely } from '../extract/readability.js';
-import type { WebFetchResponse } from '../types.js';
-
-function looksLikeScriptShell(html: string): boolean {
-  const lower = html.toLowerCase();
-  return lower.includes('<script') && (lower.includes('id="app"') || lower.includes('id="root"'));
-}
-
-function isWeakHttpContent(options: { html: string; title?: string; text: string }): boolean {
-  const normalizedText = options.text.replace(/\s+/g, ' ').trim();
-  const normalizedHtml = options.html.replace(/\s+/g, ' ').trim();
-  const textLength = normalizedText.length;
-  const htmlLength = normalizedHtml.length;
-  const hasGenericShellMarker = /enable javascript|javascript required|please turn on javascript/i.test(
-    options.html
-  );
-  const veryShortBody = textLength > 0 && textLength < 120;
-  const lowDensity = htmlLength > 0 && textLength / htmlLength < 0.02;
-
-  return veryShortBody && (lowDensity || hasGenericShellMarker);
-}
-
-export function createHttpFetcher({
-  fetchImpl = fetch
-}: {
-  fetchImpl?: typeof fetch;
-} = {}) {
-  return async function httpFetch(url: string): Promise<WebFetchResponse> {
-    const response = await fetchImpl(url);
-    const contentType = response.headers.get('content-type') ?? '';
-
-    if (!contentType.includes('text/html')) {
-      return {
-        status: 'unsupported',
-        url: response.url,
-        metadata: { method: 'http', cacheHit: false, contentType }
-      };
-    }
-
-    const html = await response.text();
-    const extraction = extractReadableContentSafely(html);
-    const content = extraction.content;
-
-    if (
-      looksLikeScriptShell(html) ||
-      content.text.length < 40 ||
-      isWeakHttpContent({ html, title: content.title, text: content.text })
-    ) {
-      return {
-        status: 'needs_headless',
-        url: response.url,
-        metadata: { method: 'http', cacheHit: false, contentType },
-        error: {
-          code: 'WEAK_EXTRACTION',
-          message: 'HTTP extraction was not reliable enough.'
-        }
-      };
-    }
-
-    return {
-      status: 'ok',
-      url: response.url,
-      content,
-      metadata: { method: 'http', cacheHit: false, contentType, truncated: content.text.length >= 4000 }
-    };
-  };
-}
+import { extractReadableContentSafely } from '../extract/readability.js';
+import type { WebFetchResponse } from '../types.js';
+
+function looksLikeScriptShell(html: string): boolean {
+  const lower = html.toLowerCase();
+  return lower.includes('<script') && (lower.includes('id="app"') || lower.includes('id="root"'));
+}
+
+function isWeakHttpContent(options: { html: string; title?: string; text: string }): boolean {
+  const normalizedText = options.text.replace(/\s+/g, ' ').trim();
+  const normalizedHtml = options.html.replace(/\s+/g, ' ').trim();
+  const textLength = normalizedText.length;
+  const htmlLength = normalizedHtml.length;
+  const hasGenericShellMarker = /enable javascript|javascript required|please turn on javascript/i.test(
+    options.html
+  );
+  const veryShortBody = textLength > 0 && textLength < 120;
+  const lowDensity = htmlLength > 0 && textLength / htmlLength < 0.02;
+
+  return veryShortBody && (lowDensity || hasGenericShellMarker);
+}
+
+export function createHttpFetcher({
+  fetchImpl = fetch
+}: {
+  fetchImpl?: typeof fetch;
+} = {}) {
+  return async function httpFetch(url: string): Promise<WebFetchResponse> {
+    const response = await fetchImpl(url);
+    const contentType = response.headers.get('content-type') ?? '';
+
+    if (!contentType.includes('text/html')) {
+      return {
+        status: 'unsupported',
+        url: response.url,
+        metadata: { method: 'http', cacheHit: false, contentType }
+      };
+    }
+
+    const html = await response.text();
+    const extraction = extractReadableContentSafely(html);
+    const content = extraction.content;
+
+    if (
+      looksLikeScriptShell(html) ||
+      content.text.length < 40 ||
+      isWeakHttpContent({ html, title: content.title, text: content.text })
+    ) {
+      return {
+        status: 'needs_headless',
+        url: response.url,
+        metadata: { method: 'http', cacheHit: false, contentType },
+        error: {
+          code: 'WEAK_EXTRACTION',
+          message: 'HTTP extraction was not reliable enough.'
+        }
+      };
+    }
+
+    return {
+      status: 'ok',
+      url: response.url,
+      content,
+      metadata: { method: 'http', cacheHit: false, contentType, truncated: content.text.length >= 4000 }
+    };
+  };
+}