@selesai/code 0.1.19 → 0.2.1

package/dist/extensions/pi-web-agent/src/fetch/http-fetch.ts

@@ -0,0 +1,67 @@
+import { extractReadableContentSafely } from '../extract/readability.js';
+import type { WebFetchResponse } from '../types.js';
+
+function looksLikeScriptShell(html: string): boolean {
+  const lower = html.toLowerCase();
+  return lower.includes('<script') && (lower.includes('id="app"') || lower.includes('id="root"'));
+}
+
+function isWeakHttpContent(options: { html: string; title?: string; text: string }): boolean {
+  const normalizedText = options.text.replace(/\s+/g, ' ').trim();
+  const normalizedHtml = options.html.replace(/\s+/g, ' ').trim();
+  const textLength = normalizedText.length;
+  const htmlLength = normalizedHtml.length;
+  const hasGenericShellMarker = /enable javascript|javascript required|please turn on javascript/i.test(
+    options.html
+  );
+  const veryShortBody = textLength > 0 && textLength < 120;
+  const lowDensity = htmlLength > 0 && textLength / htmlLength < 0.02;
+
+  return veryShortBody && (lowDensity || hasGenericShellMarker);
+}
+
+export function createHttpFetcher({
+  fetchImpl = fetch
+}: {
+  fetchImpl?: typeof fetch;
+} = {}) {
+  return async function httpFetch(url: string): Promise<WebFetchResponse> {
+    const response = await fetchImpl(url);
+    const contentType = response.headers.get('content-type') ?? '';
+
+    if (!contentType.includes('text/html')) {
+      return {
+        status: 'unsupported',
+        url: response.url,
+        metadata: { method: 'http', cacheHit: false, contentType }
+      };
+    }
+
+    const html = await response.text();
+    const extraction = extractReadableContentSafely(html);
+    const content = extraction.content;
+
+    if (
+      looksLikeScriptShell(html) ||
+      content.text.length < 40 ||
+      isWeakHttpContent({ html, title: content.title, text: content.text })
+    ) {
+      return {
+        status: 'needs_headless',
+        url: response.url,
+        metadata: { method: 'http', cacheHit: false, contentType },
+        error: {
+          code: 'WEAK_EXTRACTION',
+          message: 'HTTP extraction was not reliable enough.'
+        }
+      };
+    }
+
+    return {
+      status: 'ok',
+      url: response.url,
+      content,
+      metadata: { method: 'http', cacheHit: false, contentType, truncated: content.text.length >= 4000 }
+    };
+  };
+}

package/dist/extensions/pi-web-agent/src/orchestration/answer-synthesizer.ts

@@ -0,0 +1,60 @@
+import type { ResearchEvidence } from './research-types.js';
+import type { EvidenceCaveatReason } from './evidence-quality.js';
+
+function normalizeSummary(summary: string) {
+  return summary.replace(/\s+/g, ' ').trim();
+}
+
+function sentenceForReason(reason: EvidenceCaveatReason): string {
+  switch (reason) {
+    case 'community-only':
+      return 'the strongest readable sources were mostly community/practical context';
+    case 'low-diversity':
+      return 'the source set was narrow';
+    case 'unreadable-direct-source':
+      return 'one or more linked sources could not be read reliably';
+    case 'unreadable-thread-source':
+      return 'one or more thread sources could not be read reliably';
+    case 'possible-conflict':
+      return 'readable sources include cautionary or possibly conflicting guidance';
+    case 'bot-check':
+      return 'some candidate sources showed bot-check or security verification pages';
+  }
+}
+
+function joinReasons(reasons: string[]) {
+  if (reasons.length === 0) return '';
+  if (reasons.length === 1) return reasons[0];
+  return `${reasons.slice(0, -1).join(', ')}, and ${reasons.at(-1)}`;
+}
+
+function caveatText(partial: boolean, caveatReasons: EvidenceCaveatReason[] = []) {
+  if (!partial) return undefined;
+  const specificReasons = caveatReasons.map(sentenceForReason);
+  if (specificReasons.length > 0) {
+    return `Evidence is partial: ${joinReasons(specificReasons)}.`;
+  }
+  return 'Evidence is partial, so this answer is based on the strongest source found within the bounded research budget.';
+}
+
+export function synthesizeAnswer({
+  evidence,
+  partial,
+  caveatReasons = []
+}: {
+  evidence: ResearchEvidence[];
+  partial: boolean;
+  caveatReasons?: EvidenceCaveatReason[];
+}) {
+  const findings = evidence.slice(0, 5).map((item) => {
+    const summary = normalizeSummary(item.summary);
+    return item.sourceKind === 'community' || item.sourceKind === 'issue-thread'
+      ? `Community/practical context: ${summary}`
+      : summary;
+  });
+
+  return {
+    findings,
+    caveat: caveatText(partial, caveatReasons)
+  };
+}

package/dist/extensions/pi-web-agent/src/orchestration/candidate-selector.ts

@@ -0,0 +1,50 @@
+import type { SearchResult } from '../types.js';
+import { classifySourceProfile } from './source-profile.js';
+
+function wantsDiscussionSources(query = '') {
+  return /reddit|forum|forums|discussion|thread|comments|community|user experience|people recommend/i.test(query);
+}
+
+function candidateScore(result: SearchResult, query?: string) {
+  const url = result.url.toLowerCase();
+  const profile = classifySourceProfile(result.url);
+  const wantsThreads = wantsDiscussionSources(query);
+
+  if (profile.kind === 'official-docs') return 0;
+  if (profile.kind === 'official-api') return 1;
+
+  if (wantsThreads) {
+    if (profile.kind === 'forum-thread') return 2;
+    if (profile.kind === 'issue-thread') return 3;
+    if (url.includes('github.com/')) return 4;
+    if (profile.kind === 'package-page') return 6;
+    return 5;
+  }
+
+  if (profile.kind === 'issue-thread') return 2;
+  if (url.includes('github.com/')) return 3;
+  if (profile.kind === 'package-page') return 5;
+  return 4;
+}
+
+export function selectCandidates({
+  query,
+  results,
+  seenUrls,
+  maxCandidates
+}: {
+  query?: string;
+  results: SearchResult[];
+  seenUrls: Set<string>;
+  maxCandidates: number;
+}) {
+  const deduped = new Map<string, SearchResult>();
+  for (const result of results) {
+    if (seenUrls.has(result.url)) continue;
+    if (!deduped.has(result.url)) deduped.set(result.url, result);
+  }
+
+  return [...deduped.values()]
+    .sort((left, right) => candidateScore(left, query) - candidateScore(right, query))
+    .slice(0, maxCandidates);
+}

package/dist/extensions/pi-web-agent/src/orchestration/direct-url.ts

@@ -0,0 +1,52 @@
+const TRACKING_PARAMS = new Set([
+  'utm_source',
+  'utm_medium',
+  'utm_campaign',
+  'utm_term',
+  'utm_content',
+  'utm_name',
+  'fbclid',
+  'gclid'
+]);
+
+function stripTrailingPunctuation(raw: string): string {
+  let next = raw.trim();
+
+  while (/[),.;!?\]]$/.test(next)) {
+    const last = next.at(-1);
+    if (last === ')' && next.includes('(') && next.lastIndexOf('(') > next.lastIndexOf(')')) break;
+    next = next.slice(0, -1);
+  }
+
+  return next;
+}
+
+function normalizeDirectUrl(raw: string): string | undefined {
+  try {
+    const url = new URL(stripTrailingPunctuation(raw));
+    if (url.protocol !== 'http:' && url.protocol !== 'https:') return undefined;
+
+    for (const key of [...url.searchParams.keys()]) {
+      if (TRACKING_PARAMS.has(key.toLowerCase())) {
+        url.searchParams.delete(key);
+      }
+    }
+
+    url.hash = '';
+    return url.toString().replace(/\/$/, '');
+  } catch {
+    return undefined;
+  }
+}
+
+export function extractDirectUrls(query: string): string[] {
+  const matches = query.match(/https?:\/\/\S+/gi) ?? [];
+  const urls = new Set<string>();
+
+  for (const match of matches) {
+    const normalized = normalizeDirectUrl(match);
+    if (normalized) urls.add(normalized);
+  }
+
+  return [...urls];
+}

package/dist/extensions/pi-web-agent/src/orchestration/evidence-quality.ts

@@ -0,0 +1,105 @@
+import type { ResearchEvidence, ResearchGap, ResearchLowValueOutcome } from './research-types.js';
+
+export type EvidenceCaveatReason =
+  | 'community-only'
+  | 'low-diversity'
+  | 'unreadable-direct-source'
+  | 'unreadable-thread-source'
+  | 'possible-conflict'
+  | 'bot-check';
+
+export type EvidenceQualityReport = {
+  counts: {
+    total: number;
+    official: number;
+    community: number;
+    thread: number;
+    packagePage: number;
+    distinctHosts: number;
+  };
+  flags: {
+    hasOfficialEvidence: boolean;
+    hasOnlyCommunityEvidence: boolean;
+    hasLowDiversity: boolean;
+    hasUnreadableDirectSource: boolean;
+    hasUnreadableThreadSource: boolean;
+    hasPossibleConflict: boolean;
+  };
+  caveatReasons: EvidenceCaveatReason[];
+};
+
+function hostname(url: string) {
+  try {
+    return new URL(url).hostname.toLowerCase().replace(/^www\./, '');
+  } catch {
+    return url.toLowerCase();
+  }
+}
+
+function hasConflictMarkers(evidence: ResearchEvidence[]) {
+  const combined = evidence
+    .flatMap((item) => [item.summary, ...item.supports])
+    .join('\n')
+    .toLowerCase();
+
+  const caution = /\bdeprecated\b|not recommended|use at your own risk|should not/.test(combined);
+  const positiveText = combined.replace(/not recommended/g, '');
+  const positive = /\brecommended\b/.test(positiveText);
+  return positive && caution;
+}
+
+function addReason(reasons: EvidenceCaveatReason[], reason: EvidenceCaveatReason, enabled: boolean) {
+  if (enabled && !reasons.includes(reason)) reasons.push(reason);
+}
+
+export function analyzeEvidenceQuality({
+  evidence,
+  gaps,
+  lowValueOutcomes
+}: {
+  evidence: ResearchEvidence[];
+  gaps: ResearchGap[];
+  lowValueOutcomes: ResearchLowValueOutcome[];
+}): EvidenceQualityReport {
+  const official = evidence.filter((item) => item.sourceKind === 'official-docs' || item.sourceKind === 'official-api').length;
+  const community = evidence.filter((item) => item.sourceKind === 'community').length;
+  const thread = evidence.filter((item) => item.sourceKind === 'issue-thread' || item.sourceKind === 'official-discussion').length;
+  const packagePage = evidence.filter((item) => item.sourceKind === 'package-page').length;
+  const distinctHosts = new Set(evidence.map((item) => hostname(item.url))).size;
+
+  const hasOfficialEvidence = official > 0;
+  const hasOnlyCommunityEvidence = evidence.length > 0 && official === 0;
+  const hasLowDiversity = evidence.length > 1 && distinctHosts <= 1;
+  const hasUnreadableDirectSource = gaps.some((gap) => /Direct URL could not be read reliably/i.test(gap.message));
+  const hasUnreadableThreadSource = gaps.some((gap) => /Thread source could not be read reliably/i.test(gap.message));
+  const hasPossibleConflict = hasConflictMarkers(evidence);
+  const hasBotCheck = lowValueOutcomes.some((outcome) => outcome.kind === 'bot-check');
+
+  const caveatReasons: EvidenceCaveatReason[] = [];
+  addReason(caveatReasons, 'community-only', hasOnlyCommunityEvidence);
+  addReason(caveatReasons, 'low-diversity', hasLowDiversity);
+  addReason(caveatReasons, 'unreadable-direct-source', hasUnreadableDirectSource);
+  addReason(caveatReasons, 'unreadable-thread-source', hasUnreadableThreadSource);
+  addReason(caveatReasons, 'possible-conflict', hasPossibleConflict);
+  addReason(caveatReasons, 'bot-check', hasBotCheck);
+
+  return {
+    counts: {
+      total: evidence.length,
+      official,
+      community,
+      thread,
+      packagePage,
+      distinctHosts
+    },
+    flags: {
+      hasOfficialEvidence,
+      hasOnlyCommunityEvidence,
+      hasLowDiversity,
+      hasUnreadableDirectSource,
+      hasUnreadableThreadSource,
+      hasPossibleConflict
+    },
+    caveatReasons
+  };
+}

package/dist/extensions/pi-web-agent/src/orchestration/evidence-ranker.ts

@@ -0,0 +1,45 @@
+import type { ResearchEvidence } from './research-types.js';
+
+function sourceRank(sourceKind: ResearchEvidence['sourceKind']) {
+  switch (sourceKind) {
+    case 'official-docs':
+      return 0;
+    case 'official-api':
+      return 1;
+    case 'official-discussion':
+      return 2;
+    case 'issue-thread':
+      return 3;
+    case 'community':
+      return 4;
+    case 'package-page':
+      return 5;
+    default:
+      return 6;
+  }
+}
+
+export function rankEvidence(evidence: ResearchEvidence[]) {
+  const bestByUrl = new Map<string, ResearchEvidence>();
+  for (const item of evidence) {
+    const current = bestByUrl.get(item.url);
+    if (!current || sourceRank(item.sourceKind) < sourceRank(current.sourceKind)) {
+      bestByUrl.set(item.url, item);
+    }
+  }
+
+  return [...bestByUrl.values()].sort((left, right) => sourceRank(left.sourceKind) - sourceRank(right.sourceKind));
+}
+
+export function hasOfficialEvidence(evidence: ResearchEvidence[]) {
+  return evidence.some((item) => item.sourceKind === 'official-docs' || item.sourceKind === 'official-api');
+}
+
+export function strongEvidenceCount(evidence: ResearchEvidence[]) {
+  return evidence.filter(
+    (item) =>
+      item.sourceKind === 'official-docs' ||
+      item.sourceKind === 'official-api' ||
+      item.sourceKind === 'official-discussion'
+  ).length;
+}

package/dist/extensions/pi-web-agent/src/orchestration/index.ts

@@ -0,0 +1,28 @@
+import { createBackendSet } from '../backends/factory.js';
+import type { BackendConfig } from '../backends/config.js';
+import type { WebFetchHeadlessResponse, WebFetchResponse, WebSearchResponse } from '../types.js';
+import { createResearchOrchestrator } from './research-orchestrator.js';
+import { createResearchWorker } from './research-worker.js';
+
+export function createResearchWorkflow({
+  backendConfig,
+  search,
+  fetchPage,
+  headlessFetch
+}: {
+  backendConfig?: BackendConfig;
+  search?: (input: { query: string }) => Promise<WebSearchResponse>;
+  fetchPage?: (input: { url: string }) => Promise<WebFetchResponse>;
+  headlessFetch?: (input: { url: string }) => Promise<WebFetchHeadlessResponse>;
+} = {}) {
+  const backends = createBackendSet(backendConfig);
+  const resolvedSearch = search ?? backends.search;
+  const resolvedFetchPage = fetchPage ?? backends.fetchPage;
+  const resolvedHeadlessFetch = headlessFetch ?? backends.headlessFetch;
+  const worker = createResearchWorker({ search: resolvedSearch, fetchPage: resolvedFetchPage });
+  return createResearchOrchestrator({
+    worker,
+    fetchDirect: resolvedFetchPage,
+    headlessFetch: resolvedHeadlessFetch
+  });
+}

package/dist/extensions/pi-web-agent/src/orchestration/query-planner.ts

@@ -0,0 +1,47 @@
+export type QueryPlanInput = {
+  originalQuery: string;
+  passIndex: number;
+  previousQueries: string[];
+  gaps: string[];
+};
+
+function normalizeWhitespace(value: string) {
+  return value.replace(/\s+/g, ' ').trim();
+}
+
+function extractImportantWords(query: string) {
+  return normalizeWhitespace(query)
+    .replace(/\b(find|current|tell|me|how|to|the|and|with|for|in|a|an)\b/gi, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function officialSiteQuery(query: string) {
+  const lower = query.toLowerCase();
+  const terms = extractImportantWords(query);
+  if (lower.includes('vitest')) return `site:vitest.dev ${terms}`;
+  if (lower.includes('playwright')) return `site:playwright.dev ${terms}`;
+  if (lower.includes('microsoft edge') || lower.includes('edge')) return `site:learn.microsoft.com ${terms}`;
+  return `${terms} official docs`;
+}
+
+function implementationQuery(query: string) {
+  return `site:github.com ${extractImportantWords(query)}`;
+}
+
+export function planSearchQueries(input: QueryPlanInput): string[] {
+  const planned =
+    input.passIndex === 0
+      ? [input.originalQuery]
+      : [
+          officialSiteQuery(input.originalQuery),
+          implementationQuery(input.originalQuery),
+          `${extractImportantWords(input.originalQuery)} discussion`
+        ];
+
+  const previous = new Set(input.previousQueries.map(normalizeWhitespace));
+  return planned
+    .map(normalizeWhitespace)
+    .filter((query) => query && !previous.has(query))
+    .slice(0, 3);
+}