@selesai/code 0.13.3 → 0.13.5

package/dist/extensions/pi-web-agent/src/orchestration/query-planner.ts

@@ -1,47 +1,47 @@
-export type QueryPlanInput = {
-  originalQuery: string;
-  passIndex: number;
-  previousQueries: string[];
-  gaps: string[];
-};
-
-function normalizeWhitespace(value: string) {
-  return value.replace(/\s+/g, ' ').trim();
-}
-
-function extractImportantWords(query: string) {
-  return normalizeWhitespace(query)
-    .replace(/\b(find|current|tell|me|how|to|the|and|with|for|in|a|an)\b/gi, ' ')
-    .replace(/\s+/g, ' ')
-    .trim();
-}
-
-function officialSiteQuery(query: string) {
-  const lower = query.toLowerCase();
-  const terms = extractImportantWords(query);
-  if (lower.includes('vitest')) return `site:vitest.dev ${terms}`;
-  if (lower.includes('playwright')) return `site:playwright.dev ${terms}`;
-  if (lower.includes('microsoft edge') || lower.includes('edge')) return `site:learn.microsoft.com ${terms}`;
-  return `${terms} official docs`;
-}
-
-function implementationQuery(query: string) {
-  return `site:github.com ${extractImportantWords(query)}`;
-}
-
-export function planSearchQueries(input: QueryPlanInput): string[] {
-  const planned =
-    input.passIndex === 0
-      ? [input.originalQuery]
-      : [
-          officialSiteQuery(input.originalQuery),
-          implementationQuery(input.originalQuery),
-          `${extractImportantWords(input.originalQuery)} discussion`
-        ];
-
-  const previous = new Set(input.previousQueries.map(normalizeWhitespace));
-  return planned
-    .map(normalizeWhitespace)
-    .filter((query) => query && !previous.has(query))
-    .slice(0, 3);
-}
+export type QueryPlanInput = {
+  originalQuery: string;
+  passIndex: number;
+  previousQueries: string[];
+  gaps: string[];
+};
+
+function normalizeWhitespace(value: string) {
+  return value.replace(/\s+/g, ' ').trim();
+}
+
+function extractImportantWords(query: string) {
+  return normalizeWhitespace(query)
+    .replace(/\b(find|current|tell|me|how|to|the|and|with|for|in|a|an)\b/gi, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function officialSiteQuery(query: string) {
+  const lower = query.toLowerCase();
+  const terms = extractImportantWords(query);
+  if (lower.includes('vitest')) return `site:vitest.dev ${terms}`;
+  if (lower.includes('playwright')) return `site:playwright.dev ${terms}`;
+  if (lower.includes('microsoft edge') || lower.includes('edge')) return `site:learn.microsoft.com ${terms}`;
+  return `${terms} official docs`;
+}
+
+function implementationQuery(query: string) {
+  return `site:github.com ${extractImportantWords(query)}`;
+}
+
+export function planSearchQueries(input: QueryPlanInput): string[] {
+  const planned =
+    input.passIndex === 0
+      ? [input.originalQuery]
+      : [
+          officialSiteQuery(input.originalQuery),
+          implementationQuery(input.originalQuery),
+          `${extractImportantWords(input.originalQuery)} discussion`
+        ];
+
+  const previous = new Set(input.previousQueries.map(normalizeWhitespace));
+  return planned
+    .map(normalizeWhitespace)
+    .filter((query) => query && !previous.has(query))
+    .slice(0, 3);
+}

package/dist/extensions/pi-web-agent/src/orchestration/research-orchestrator.ts

@@ -1,4 +1,4 @@
-import type { WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
+import type { SearchProviderName, WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
 import { rankEvidence } from './evidence-ranker.js';
 import { planSearchQueries } from './query-planner.js';
 import { classifySourceProfile } from './source-profile.js';
@@ -25,6 +25,10 @@ function summarizeText(text: string, maxLength = 180): string {
   return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
 }
 
+function isReaderMethod(method: string): boolean {
+  return method === 'github' || method === 'pdf' || method === 'youtube';
+}
+
 function isBotCheckContent({ title = '', text }: { title?: string; text: string }) {
   return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(
     `${title}\n${text}`
@@ -35,6 +39,17 @@ function evidenceFromFetch(result: WebFetchResponse): ResearchEvidence | null {
   if (result.status !== 'ok' || !result.content?.text.trim()) return null;
   if (isBotCheckContent({ title: result.content.title, text: result.content.text })) return null;
 
+  if (isReaderMethod(result.metadata.method)) {
+    return {
+      title: result.content.title ?? result.url,
+      url: result.url,
+      sourceKind: 'primary-content',
+      method: result.metadata.method,
+      summary: result.content.text,
+      supports: [result.content.text]
+    };
+  }
+
   return {
     title: result.content.title ?? result.url,
     url: result.url,
@@ -101,7 +116,9 @@ function buildMetadata({
   allLowValueOutcomes,
   headlessAttempts,
   exhaustedBudget,
-  caveatReasons = []
+  caveatReasons = [],
+  fanoutProviders,
+  fanoutSkipped
 }: {
   previousQueries: string[];
   allEvidence: ResearchEvidence[];
@@ -110,13 +127,17 @@ function buildMetadata({
   headlessAttempts: number;
   exhaustedBudget: boolean;
   caveatReasons?: EvidenceCaveatReason[];
+  fanoutProviders?: SearchProviderName[];
+  fanoutSkipped?: SearchProviderName[];
 }) {
   return {
     searchPasses: previousQueries.length,
     fetchedPages: allEvidence.length + allGaps.length + allLowValueOutcomes.length,
     headlessAttempts,
     exhaustedBudget,
-    caveatReasons
+    caveatReasons,
+    fanoutProviders,
+    fanoutSkipped
   };
 }
 
@@ -170,6 +191,14 @@ export function createResearchOrchestrator({
       const suggestedHeadlessUrls: string[] = [];
       let headlessAttempts = 0;
       let lastPass: ResearchWorkerResult | undefined;
+      const fanoutProvidersSeen = new Set<SearchProviderName>();
+      const fanoutSkippedSeen = new Set<SearchProviderName>();
+
+      function fanoutSnapshot() {
+        const providers = fanoutProvidersSeen.size ? [...fanoutProvidersSeen] : undefined;
+        const skipped = [...fanoutSkippedSeen].filter((p) => !fanoutProvidersSeen.has(p));
+        return { fanoutProviders: providers, fanoutSkipped: skipped.length ? skipped : undefined };
+      }
 
       if (fetchDirect) {
         for (const url of extractDirectUrls(query).slice(0, 3)) {
@@ -204,6 +233,36 @@ export function createResearchOrchestrator({
         }
       }
 
+      if (allEvidence.some((item) => item.sourceKind === 'primary-content')) {
+        const ranked = rankEvidence(allEvidence.filter((item) => item.sourceKind !== 'package-page'));
+        const quality = analyzeEvidenceQuality({
+          evidence: ranked,
+          gaps: allGaps,
+          lowValueOutcomes: allLowValueOutcomes
+        });
+        return {
+          decision: decisionForAnswer({ action: 'answer', query, ranked, exhaustedBudget: false }),
+          evidence: ranked,
+          workerPass: combinedWorkerPass({
+            lastPass,
+            previousQueries,
+            allGaps,
+            allLowValueOutcomes,
+            exhaustedBudget: false
+          }),
+          metadata: buildMetadata({
+            previousQueries,
+            allEvidence,
+            allGaps,
+            allLowValueOutcomes,
+            headlessAttempts,
+            exhaustedBudget: false,
+            caveatReasons: quality.caveatReasons,
+            ...fanoutSnapshot()
+          })
+        };
+      }
+
       for (let passIndex = 0; passIndex < DEFAULT_MAX_PASSES; passIndex++) {
         const queries = planSearchQueries({
           originalQuery: query,
@@ -221,6 +280,8 @@ export function createResearchOrchestrator({
           });
 
           lastPass = pass;
+          pass.fanoutProviders?.forEach((p) => fanoutProvidersSeen.add(p));
+          pass.fanoutSkipped?.forEach((p) => fanoutSkippedSeen.add(p));
           allEvidence.push(...pass.evidence);
           allGaps.push(...pass.gaps);
           allLowValueOutcomes.push(...pass.lowValueOutcomes);
@@ -287,7 +348,8 @@ export function createResearchOrchestrator({
                   allLowValueOutcomes,
                   headlessAttempts,
                   exhaustedBudget,
-                  caveatReasons: updatedQuality.caveatReasons
+                  caveatReasons: updatedQuality.caveatReasons,
+                  ...fanoutSnapshot()
                 })
               };
             }
@@ -314,7 +376,8 @@ export function createResearchOrchestrator({
                 allLowValueOutcomes,
                 headlessAttempts,
                 exhaustedBudget: false,
-                caveatReasons: quality.caveatReasons
+                caveatReasons: quality.caveatReasons,
+                ...fanoutSnapshot()
               })
             };
           }
@@ -338,7 +401,8 @@ export function createResearchOrchestrator({
                 allLowValueOutcomes,
                 headlessAttempts,
                 exhaustedBudget,
-                caveatReasons: quality.caveatReasons
+                caveatReasons: quality.caveatReasons,
+                ...fanoutSnapshot()
               })
             };
           }
@@ -368,7 +432,8 @@ export function createResearchOrchestrator({
           allLowValueOutcomes,
           headlessAttempts,
           exhaustedBudget: true,
-          caveatReasons: quality.caveatReasons
+          caveatReasons: quality.caveatReasons,
+          ...fanoutSnapshot()
         })
       };
     }

package/dist/extensions/pi-web-agent/src/orchestration/research-types.ts

@@ -1,4 +1,7 @@
+import type { SearchProviderName } from '../types.js';
+
 export type ResearchSourceKind =
+  | 'primary-content'
   | 'official-docs'
   | 'official-api'
   | 'official-discussion'
@@ -7,7 +10,7 @@ export type ResearchSourceKind =
   | 'package-page'
   | 'other';
 
-export type ResearchMethod = 'search' | 'http' | 'headless' | 'firecrawl';
+export type ResearchMethod = 'search' | 'http' | 'headless' | 'firecrawl' | 'github' | 'pdf' | 'youtube';
 
 export type ResearchEvidence = {
   title: string;
@@ -36,6 +39,8 @@ export type ResearchWorkerResult = {
   lowValueOutcomes: ResearchLowValueOutcome[];
   suggestedHeadlessUrl?: string;
   exhaustedBudget: boolean;
+  fanoutProviders?: SearchProviderName[];
+  fanoutSkipped?: SearchProviderName[];
 };
 
 export type ResearchRunMetadata = {
@@ -43,6 +48,8 @@ export type ResearchRunMetadata = {
   fetchedPages: number;
   headlessAttempts: number;
   exhaustedBudget: boolean;
+  fanoutProviders?: SearchProviderName[];
+  fanoutSkipped?: SearchProviderName[];
 };
 
 export type ResearchOrchestratorDecision =

package/dist/extensions/pi-web-agent/src/orchestration/research-worker.ts

@@ -17,6 +17,10 @@ function summarizeText(text: string, maxLength = 180): string {
   return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
 }
 
+function isReaderMethod(method: string): boolean {
+  return method === 'github' || method === 'pdf' || method === 'youtube';
+}
+
 function isBotCheckContent({ title = '', text }: { title?: string; text: string }) {
   return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(
     `${title}\n${text}`
@@ -28,6 +32,19 @@ function evidenceFromFetch(fetched: WebFetchResponse, fallbackTitle: string) {
   if (fetched.status !== 'ok' || !content) return null;
   if (isBotCheckContent({ title: content.title, text: content.text })) return null;
 
+  // A successful reader read with usable text is primary content, exempt from the
+  // package-page filter below.
+  if (isReaderMethod(fetched.metadata.method) && content.text.trim()) {
+    return {
+      title: content.title ?? fallbackTitle,
+      url: fetched.url,
+      sourceKind: 'primary-content',
+      method: fetched.metadata.method,
+      summary: content.text,
+      supports: [content.text]
+    } satisfies ResearchEvidence;
+  }
+
   const sourceKind = classifySource(fetched.url);
   if (sourceKind === 'package-page') {
     return null;
@@ -98,6 +115,8 @@ export function createResearchWorker({
       }
 
       const searchResult = await search({ query });
+      const fanoutProviders = searchResult.metadata.fanout?.providers;
+      const fanoutSkipped = searchResult.metadata.fanout?.skipped;
       if (searchResult.status !== 'ok') {
         return {
           searchQueries,
@@ -110,7 +129,9 @@ export function createResearchWorker({
           ],
           lowValueOutcomes,
           suggestedHeadlessUrl,
-          exhaustedBudget: false
+          exhaustedBudget: false,
+          fanoutProviders,
+          fanoutSkipped
         };
       }
 
@@ -126,7 +147,9 @@ export function createResearchWorker({
             }
           ],
           suggestedHeadlessUrl,
-          exhaustedBudget: false
+          exhaustedBudget: false,
+          fanoutProviders,
+          fanoutSkipped
         };
       }
 
@@ -174,7 +197,9 @@ export function createResearchWorker({
         gaps,
         lowValueOutcomes,
         suggestedHeadlessUrl,
-        exhaustedBudget: false
+        exhaustedBudget: false,
+        fanoutProviders,
+        fanoutSkipped
       };
     }
   };

package/dist/extensions/pi-web-agent/src/orchestration/source-profile.ts

@@ -32,6 +32,10 @@ function parseUrl(rawUrl: string): URL | undefined {
   }
 }
 
+export function hostOf(rawUrl: string): string | undefined {
+  return parseUrl(rawUrl)?.hostname.toLowerCase().replace(/^www\./, '');
+}
+
 function isOfficialApi(host: string, path: string) {
   return (
     (host === 'playwright.dev' && path.startsWith('/docs/api/')) ||

package/dist/extensions/pi-web-agent/src/orchestration/stop-decider.ts

@@ -1,81 +1,81 @@
-import type { ResearchEvidence } from './research-types.js';
-import type { EvidenceQualityReport } from './evidence-quality.js';
-import { hasOfficialEvidence, strongEvidenceCount } from './evidence-ranker.js';
-
-export type ResearchStepDecision =
-  | { action: 'answer' }
-  | { action: 'answer-with-caveat' }
-  | { action: 'search-again' }
-  | { action: 'headless'; url: string };
-
-function activeCaveatReasons(evidence: ResearchEvidence[], quality?: EvidenceQualityReport) {
-  const reasons = quality?.caveatReasons ?? [];
-  if (!hasOfficialDocsAndApi(evidence)) return reasons;
-  return reasons.filter((reason) => reason !== 'low-diversity');
-}
-
-function hasQualityConcern(evidence: ResearchEvidence[], quality?: EvidenceQualityReport) {
-  return activeCaveatReasons(evidence, quality).length > 0;
-}
-
-function hasOfficialDocsAndApi(evidence: ResearchEvidence[]) {
-  return evidence.some((item) => item.sourceKind === 'official-docs') &&
-    evidence.some((item) => item.sourceKind === 'official-api');
-}
-
-function shouldSearchForBetterQuality({
-  evidence,
-  quality,
-  passIndex,
-  maxPasses
-}: {
-  evidence: ResearchEvidence[];
-  quality?: EvidenceQualityReport;
-  passIndex: number;
-  maxPasses: number;
-}) {
-  if (!quality) return false;
-  if (passIndex + 1 >= maxPasses) return false;
-  if (quality.flags.hasOnlyCommunityEvidence) return true;
-  if (quality.flags.hasLowDiversity && !hasOfficialDocsAndApi(evidence)) return true;
-  return false;
-}
-
-export function decideNextResearchStep({
-  evidence,
-  suggestedHeadlessUrls,
-  passIndex,
-  maxPasses,
-  headlessAttempts,
-  maxHeadlessAttempts,
-  quality
-}: {
-  evidence: ResearchEvidence[];
-  suggestedHeadlessUrls: string[];
-  passIndex: number;
-  maxPasses: number;
-  headlessAttempts: number;
-  maxHeadlessAttempts: number;
-  quality?: EvidenceQualityReport;
-}): ResearchStepDecision {
-  const strongEnough = strongEvidenceCount(evidence) >= 2 && hasOfficialEvidence(evidence);
-
-  if (strongEnough && shouldSearchForBetterQuality({ evidence, quality, passIndex, maxPasses })) {
-    return { action: 'search-again' };
-  }
-
-  if (strongEnough) {
-    return hasQualityConcern(evidence, quality) ? { action: 'answer-with-caveat' } : { action: 'answer' };
-  }
-
-  const headlessUrl = suggestedHeadlessUrls.find((url) => !url.includes('npmjs.com/package/'));
-  if (headlessUrl && headlessAttempts < maxHeadlessAttempts) {
-    return { action: 'headless', url: headlessUrl };
-  }
-
-  if (passIndex + 1 < maxPasses) {
-    return { action: 'search-again' };
-  }
-
-  return { action: 'answer-with-caveat' };
-}
+import type { ResearchEvidence } from './research-types.js';
+import type { EvidenceQualityReport } from './evidence-quality.js';
+import { hasOfficialEvidence, strongEvidenceCount } from './evidence-ranker.js';
+
+export type ResearchStepDecision =
+  | { action: 'answer' }
+  | { action: 'answer-with-caveat' }
+  | { action: 'search-again' }
+  | { action: 'headless'; url: string };
+
+function activeCaveatReasons(evidence: ResearchEvidence[], quality?: EvidenceQualityReport) {
+  const reasons = quality?.caveatReasons ?? [];
+  if (!hasOfficialDocsAndApi(evidence)) return reasons;
+  return reasons.filter((reason) => reason !== 'low-diversity');
+}
+
+function hasQualityConcern(evidence: ResearchEvidence[], quality?: EvidenceQualityReport) {
+  return activeCaveatReasons(evidence, quality).length > 0;
+}
+
+function hasOfficialDocsAndApi(evidence: ResearchEvidence[]) {
+  return evidence.some((item) => item.sourceKind === 'official-docs') &&
+    evidence.some((item) => item.sourceKind === 'official-api');
+}
+
+function shouldSearchForBetterQuality({
+  evidence,
+  quality,
+  passIndex,
+  maxPasses
+}: {
+  evidence: ResearchEvidence[];
+  quality?: EvidenceQualityReport;
+  passIndex: number;
+  maxPasses: number;
+}) {
+  if (!quality) return false;
+  if (passIndex + 1 >= maxPasses) return false;
+  if (quality.flags.hasOnlyCommunityEvidence) return true;
+  if (quality.flags.hasLowDiversity && !hasOfficialDocsAndApi(evidence)) return true;
+  return false;
+}
+
+export function decideNextResearchStep({
+  evidence,
+  suggestedHeadlessUrls,
+  passIndex,
+  maxPasses,
+  headlessAttempts,
+  maxHeadlessAttempts,
+  quality
+}: {
+  evidence: ResearchEvidence[];
+  suggestedHeadlessUrls: string[];
+  passIndex: number;
+  maxPasses: number;
+  headlessAttempts: number;
+  maxHeadlessAttempts: number;
+  quality?: EvidenceQualityReport;
+}): ResearchStepDecision {
+  const strongEnough = strongEvidenceCount(evidence) >= 2 && hasOfficialEvidence(evidence);
+
+  if (strongEnough && shouldSearchForBetterQuality({ evidence, quality, passIndex, maxPasses })) {
+    return { action: 'search-again' };
+  }
+
+  if (strongEnough) {
+    return hasQualityConcern(evidence, quality) ? { action: 'answer-with-caveat' } : { action: 'answer' };
+  }
+
+  const headlessUrl = suggestedHeadlessUrls.find((url) => !url.includes('npmjs.com/package/'));
+  if (headlessUrl && headlessAttempts < maxHeadlessAttempts) {
+    return { action: 'headless', url: headlessUrl };
+  }
+
+  if (passIndex + 1 < maxPasses) {
+    return { action: 'search-again' };
+  }
+
+  return { action: 'answer-with-caveat' };
+}

package/dist/extensions/pi-web-agent/src/orchestration/url.ts

@@ -0,0 +1,35 @@
+const TRACKING_PARAMS = new Set([
+  'utm_source',
+  'utm_medium',
+  'utm_campaign',
+  'utm_term',
+  'utm_content',
+  'utm_name',
+  'fbclid',
+  'gclid'
+]);
+
+/** Canonical form for dedupe/comparison. Returns undefined for non-http(s) or unparseable input. */
+export function canonicalizeUrl(raw: string, options: { canonicalizeHost?: boolean } = {}): string | undefined {
+  const { canonicalizeHost = true } = options;
+  let url: URL;
+  try {
+    url = new URL(raw);
+  } catch {
+    return undefined;
+  }
+  if (url.protocol !== 'http:' && url.protocol !== 'https:') return undefined;
+
+  if (canonicalizeHost) {
+    url.hostname = url.hostname.replace(/^www\./, '');
+  }
+  for (const key of [...url.searchParams.keys()]) {
+    if (TRACKING_PARAMS.has(key.toLowerCase())) {
+      url.searchParams.delete(key);
+    }
+  }
+  url.hash = '';
+  return url.toString().replace(/\/$/, '');
+}
+
+export { TRACKING_PARAMS };