@demigodmode/pi-web-agent 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -18,6 +18,22 @@ The format is intentionally simple and release-oriented.
18
18
  ### Breaking
19
19
  - None.
20
20
 
21
+ ## [1.3.0] - 2026-06-04
22
+ ### Added
23
+ - Added direct URL handling in `web_explore` so linked pages are read before search results.
24
+ - Added forum/thread source classification for Reddit-style discussions, forums, Stack Overflow, and GitHub issues/discussions.
25
+ - Added Playwright-managed Chromium fallback when no local Chromium-family browser is detected.
26
+
27
+ ### Changed
28
+ - Discussion-oriented queries now prefer forum/thread results over generic pages.
29
+ - `/web-agent doctor` now reports the local-browser headless backend and managed Chromium fallback.
30
+
31
+ ### Fixed
32
+ - Preserved direct/thread fetch gaps in bounded research results so unreadable thread sources get explicit caveats.
33
+
34
+ ### Breaking
35
+ - None.
36
+
21
37
  ## [1.2.0] - 2026-06-01
22
38
  ### Added
23
39
  - Added backend provider and fallback editing to `/web-agent settings`.
package/README.md CHANGED
@@ -8,7 +8,7 @@
8
8
 
9
9
  Most agent web tools blur search, fetch, browser rendering, and research into one vague thing. `pi-web-agent` exposes one public research tool, `web_explore`, and keeps search/fetch/headless work inside that bounded workflow.
10
10
 
11
- The point is keeping the model-facing boundary simple: ask `web_explore` to research a question, and it handles discovery, HTTP reads, targeted browser rendering, source ranking, and caveats internally.
11
+ The point is keeping the model-facing boundary simple: ask `web_explore` to research a question, and it handles direct links, discovery, HTTP reads, targeted browser rendering, source ranking, and caveats internally.
12
12
 
13
13
  That sounds obvious, but a lot of agent tooling gets fuzzy right there. This package is meant to be stricter about what it actually did and more willing to say when a read was not good enough to trust.
14
14
 
@@ -20,9 +20,9 @@ Compatibility notice: current `pi-web-agent` requires Pi 0.74+ because Pi packag
20
20
  pi install npm:@demigodmode/pi-web-agent
21
21
  ```
22
22
 
23
- After installing, reload or restart Pi. Run `/web-agent` for the action menu, or `/web-agent doctor` to check whether the package loaded cleanly and whether headless rendering can find a browser.
23
+ After installing, reload or restart Pi. Run `/web-agent` for the action menu, or `/web-agent doctor` to check whether the package loaded cleanly and which web backends are configured.
24
24
 
25
- Headless rendering currently requires a detectable Chromium-family browser: Chrome, Chromium, Edge, or Brave. Firefox/Safari-only systems can still use search and plain HTTP reads, but browser-rendered fallback pages need a supported Chromium-family browser for now.
25
+ Headless rendering first tries a detectable Chromium-family browser: Chrome, Chromium, Edge, or Brave. If none is found, it falls back to Playwright-managed Chromium and still launches headless. Firefox/Safari-only systems can still use search and plain HTTP reads; browser-rendered fallback uses Chromium.
26
26
 
27
27
  Later on, update installed packages with:
28
28
 
@@ -109,7 +109,7 @@ Example:
109
109
  }
110
110
  ```
111
111
 
112
- Backend config is also supported. Defaults remain DuckDuckGo search, plain HTTP fetch, and local browser headless fallback.
112
+ Backend config is also supported. Defaults remain DuckDuckGo search, plain HTTP fetch, and local-browser headless fallback with managed Chromium fallback configured.
113
113
 
114
114
  Backend settings can be changed from:
115
115
 
@@ -83,5 +83,8 @@ export async function checkBackendHealth(config, { fetchImpl = fetch, timeoutMs
83
83
  if (config.fetch.fallback) {
84
84
  lines.push(`fetch fallback: ${config.fetch.fallback}`);
85
85
  }
86
+ if (config.headless.provider === 'local-browser') {
87
+ lines.push('headless backend: local-browser (managed Chromium fallback configured)');
88
+ }
86
89
  return lines;
87
90
  }
@@ -6,7 +6,8 @@ export declare function headlessFetch(url: string, { configuredPath, resolveBrow
6
6
  configuredPath?: string;
7
7
  }) => Promise<BrowserResolutionResult>;
8
8
  launchBrowser?: (options: {
9
- executablePath: string;
9
+ executablePath?: string;
10
+ headless: true;
10
11
  }) => Promise<{
11
12
  newContext: () => Promise<{
12
13
  newPage: () => Promise<any>;
@@ -1,4 +1,4 @@
1
- import { chromium } from 'playwright-core';
1
+ import { chromium } from 'playwright';
2
2
  import { extractReadableContentSafely } from '../extract/readability.js';
3
3
  import { resolveBrowserExecutable } from './browser-resolution.js';
4
4
  function cleanupRenderedText(text) {
@@ -7,9 +7,9 @@ function cleanupRenderedText(text) {
7
7
  cleaned = cleaned.replace(/\s+/g, ' ').trim();
8
8
  return cleaned;
9
9
  }
10
- export async function headlessFetch(url, { configuredPath, resolveBrowser = (options) => resolveBrowserExecutable({ configuredPath: options?.configuredPath }), launchBrowser = ({ executablePath }) => chromium.launch({ executablePath, headless: true }), now = () => Date.now() } = {}) {
10
+ export async function headlessFetch(url, { configuredPath, resolveBrowser = (options) => resolveBrowserExecutable({ configuredPath: options?.configuredPath }), launchBrowser = ({ executablePath, headless }) => chromium.launch(executablePath ? { executablePath, headless } : { headless }), now = () => Date.now() } = {}) {
11
11
  const resolved = await resolveBrowser({ configuredPath });
12
- if (!resolved.ok) {
12
+ if (!resolved.ok && resolved.error.code === 'CONFIGURED_BROWSER_NOT_FOUND') {
13
13
  return {
14
14
  status: 'error',
15
15
  url,
@@ -17,11 +17,15 @@ export async function headlessFetch(url, { configuredPath, resolveBrowser = (opt
17
17
  error: resolved.error
18
18
  };
19
19
  }
20
+ const browserName = resolved.ok ? resolved.browser : 'chromium';
21
+ const launchOptions = resolved.ok
22
+ ? { executablePath: resolved.executablePath, headless: true }
23
+ : { headless: true };
20
24
  let browser;
21
25
  let context;
22
26
  let page;
23
27
  try {
24
- browser = await launchBrowser({ executablePath: resolved.executablePath });
28
+ browser = await launchBrowser(launchOptions);
25
29
  context = await browser.newContext();
26
30
  page = await context.newPage();
27
31
  const startedAt = now();
@@ -42,7 +46,7 @@ export async function headlessFetch(url, { configuredPath, resolveBrowser = (opt
42
46
  metadata: {
43
47
  method: 'headless',
44
48
  cacheHit: false,
45
- browser: resolved.browser,
49
+ browser: browserName,
46
50
  navigationMs: finishedAt - startedAt
47
51
  },
48
52
  error: {
@@ -58,7 +62,7 @@ export async function headlessFetch(url, { configuredPath, resolveBrowser = (opt
58
62
  metadata: {
59
63
  method: 'headless',
60
64
  cacheHit: false,
61
- browser: resolved.browser,
65
+ browser: browserName,
62
66
  navigationMs: finishedAt - startedAt,
63
67
  truncated: cleanedContent.text.length >= 4000
64
68
  }
@@ -71,7 +75,7 @@ export async function headlessFetch(url, { configuredPath, resolveBrowser = (opt
71
75
  metadata: {
72
76
  method: 'headless',
73
77
  cacheHit: false,
74
- browser: resolved.browser
78
+ browser: browserName
75
79
  },
76
80
  error: {
77
81
  code: 'HEADLESS_NAVIGATION_FAILED',
@@ -1,5 +1,6 @@
1
1
  import type { SearchResult } from '../types.js';
2
- export declare function selectCandidates({ results, seenUrls, maxCandidates }: {
2
+ export declare function selectCandidates({ query, results, seenUrls, maxCandidates }: {
3
+ query?: string;
3
4
  results: SearchResult[];
4
5
  seenUrls: Set<string>;
5
6
  maxCandidates: number;
@@ -1,16 +1,35 @@
1
- function candidateScore(result) {
1
+ import { classifySourceProfile } from './source-profile.js';
2
+ function wantsDiscussionSources(query = '') {
3
+ return /reddit|forum|forums|discussion|thread|comments|community|user experience|people recommend/i.test(query);
4
+ }
5
+ function candidateScore(result, query) {
2
6
  const url = result.url.toLowerCase();
3
- if (url.includes('playwright.dev/docs') || url.includes('vitest.dev/guide') || url.includes('learn.microsoft.com'))
7
+ const profile = classifySourceProfile(result.url);
8
+ const wantsThreads = wantsDiscussionSources(query);
9
+ if (profile.kind === 'official-docs')
4
10
  return 0;
5
- if (url.includes('github.com/') && (url.includes('/issues/') || url.includes('/discussions/')))
11
+ if (profile.kind === 'official-api')
6
12
  return 1;
7
- if (url.includes('github.com/'))
13
+ if (wantsThreads) {
14
+ if (profile.kind === 'forum-thread')
15
+ return 2;
16
+ if (profile.kind === 'issue-thread')
17
+ return 3;
18
+ if (url.includes('github.com/'))
19
+ return 4;
20
+ if (profile.kind === 'package-page')
21
+ return 6;
22
+ return 5;
23
+ }
24
+ if (profile.kind === 'issue-thread')
8
25
  return 2;
9
- if (url.includes('npmjs.com/package/'))
10
- return 4;
11
- return 3;
26
+ if (url.includes('github.com/'))
27
+ return 3;
28
+ if (profile.kind === 'package-page')
29
+ return 5;
30
+ return 4;
12
31
  }
13
- export function selectCandidates({ results, seenUrls, maxCandidates }) {
32
+ export function selectCandidates({ query, results, seenUrls, maxCandidates }) {
14
33
  const deduped = new Map();
15
34
  for (const result of results) {
16
35
  if (seenUrls.has(result.url))
@@ -19,6 +38,6 @@ export function selectCandidates({ results, seenUrls, maxCandidates }) {
19
38
  deduped.set(result.url, result);
20
39
  }
21
40
  return [...deduped.values()]
22
- .sort((left, right) => candidateScore(left) - candidateScore(right))
41
+ .sort((left, right) => candidateScore(left, query) - candidateScore(right, query))
23
42
  .slice(0, maxCandidates);
24
43
  }
@@ -0,0 +1 @@
1
+ export declare function extractDirectUrls(query: string): string[];
@@ -0,0 +1,47 @@
1
+ const TRACKING_PARAMS = new Set([
2
+ 'utm_source',
3
+ 'utm_medium',
4
+ 'utm_campaign',
5
+ 'utm_term',
6
+ 'utm_content',
7
+ 'utm_name',
8
+ 'fbclid',
9
+ 'gclid'
10
+ ]);
11
+ function stripTrailingPunctuation(raw) {
12
+ let next = raw.trim();
13
+ while (/[),.;!?\]]$/.test(next)) {
14
+ const last = next.at(-1);
15
+ if (last === ')' && next.includes('(') && next.lastIndexOf('(') > next.lastIndexOf(')'))
16
+ break;
17
+ next = next.slice(0, -1);
18
+ }
19
+ return next;
20
+ }
21
+ function normalizeDirectUrl(raw) {
22
+ try {
23
+ const url = new URL(stripTrailingPunctuation(raw));
24
+ if (url.protocol !== 'http:' && url.protocol !== 'https:')
25
+ return undefined;
26
+ for (const key of [...url.searchParams.keys()]) {
27
+ if (TRACKING_PARAMS.has(key.toLowerCase())) {
28
+ url.searchParams.delete(key);
29
+ }
30
+ }
31
+ url.hash = '';
32
+ return url.toString().replace(/\/$/, '');
33
+ }
34
+ catch {
35
+ return undefined;
36
+ }
37
+ }
38
+ export function extractDirectUrls(query) {
39
+ const matches = query.match(/https?:\/\/\S+/gi) ?? [];
40
+ const urls = new Set();
41
+ for (const match of matches) {
42
+ const normalized = normalizeDirectUrl(match);
43
+ if (normalized)
44
+ urls.add(normalized);
45
+ }
46
+ return [...urls];
47
+ }
@@ -7,5 +7,9 @@ export function createResearchWorkflow({ backendConfig, search, fetchPage, headl
7
7
  const resolvedFetchPage = fetchPage ?? backends.fetchPage;
8
8
  const resolvedHeadlessFetch = headlessFetch ?? backends.headlessFetch;
9
9
  const worker = createResearchWorker({ search: resolvedSearch, fetchPage: resolvedFetchPage });
10
- return createResearchOrchestrator({ worker, headlessFetch: resolvedHeadlessFetch });
10
+ return createResearchOrchestrator({
11
+ worker,
12
+ fetchDirect: resolvedFetchPage,
13
+ headlessFetch: resolvedHeadlessFetch
14
+ });
11
15
  }
@@ -1,6 +1,6 @@
1
- import type { WebFetchHeadlessResponse } from '../types.js';
1
+ import type { WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
2
2
  import type { ResearchEvidence, ResearchOrchestratorDecision, ResearchWorkerResult } from './research-types.js';
3
- export declare function createResearchOrchestrator({ worker, headlessFetch }: {
3
+ export declare function createResearchOrchestrator({ worker, fetchDirect, headlessFetch }: {
4
4
  worker: {
5
5
  run: (input: {
6
6
  query: string;
@@ -8,6 +8,9 @@ export declare function createResearchOrchestrator({ worker, headlessFetch }: {
8
8
  maxFetches: number;
9
9
  }) => Promise<ResearchWorkerResult>;
10
10
  };
11
+ fetchDirect?: (input: {
12
+ url: string;
13
+ }) => Promise<WebFetchResponse>;
11
14
  headlessFetch: (input: {
12
15
  url: string;
13
16
  }) => Promise<WebFetchHeadlessResponse>;
@@ -1,23 +1,13 @@
1
1
  import { rankEvidence } from './evidence-ranker.js';
2
2
  import { planSearchQueries } from './query-planner.js';
3
+ import { classifySourceProfile } from './source-profile.js';
4
+ import { extractDirectUrls } from './direct-url.js';
3
5
  import { decideNextResearchStep } from './stop-decider.js';
4
6
  const DEFAULT_MAX_PASSES = 3;
5
7
  const DEFAULT_MAX_FETCHES_PER_PASS = 4;
6
8
  const DEFAULT_MAX_HEADLESS_ATTEMPTS = 2;
7
9
  function classifyEvidenceUrl(url) {
8
- if (url.includes('/docs/api/') || url.includes('/config/'))
9
- return 'official-api';
10
- if (url.includes('playwright.dev/docs') || url.includes('vitest.dev/guide/'))
11
- return 'official-docs';
12
- if (url.includes('github.com/vitest-dev/vitest') && url.includes('/docs/'))
13
- return 'official-docs';
14
- if (url.includes('learn.microsoft.com'))
15
- return 'official-docs';
16
- if (url.includes('github.com/') && url.includes('/issues/'))
17
- return 'issue-thread';
18
- if (url.includes('npmjs.com/package/'))
19
- return 'package-page';
20
- return 'community';
10
+ return classifySourceProfile(url).sourceKind;
21
11
  }
22
12
  function summarizeText(text, maxLength = 180) {
23
13
  return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
@@ -25,6 +15,20 @@ function summarizeText(text, maxLength = 180) {
25
15
  function isBotCheckContent({ title = '', text }) {
26
16
  return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(`${title}\n${text}`);
27
17
  }
18
+ function evidenceFromFetch(result) {
19
+ if (result.status !== 'ok' || !result.content?.text.trim())
20
+ return null;
21
+ if (isBotCheckContent({ title: result.content.title, text: result.content.text }))
22
+ return null;
23
+ return {
24
+ title: result.content.title ?? result.url,
25
+ url: result.url,
26
+ sourceKind: classifyEvidenceUrl(result.url),
27
+ method: result.metadata.method,
28
+ summary: summarizeText(result.content.text),
29
+ supports: [summarizeText(result.content.text, 120)]
30
+ };
31
+ }
28
32
  function evidenceFromHeadless(result) {
29
33
  if (result.status !== 'ok' || !result.content?.text.trim())
30
34
  return null;
@@ -39,15 +43,28 @@ function evidenceFromHeadless(result) {
39
43
  supports: [summarizeText(result.content.text, 120)]
40
44
  };
41
45
  }
42
- function fallbackWorkerPass({ previousQueries, allGaps, allLowValueOutcomes, exhaustedBudget }) {
46
+ function combinedWorkerPass({ lastPass, previousQueries, allGaps, allLowValueOutcomes, exhaustedBudget }) {
43
47
  return {
44
- searchQueries: previousQueries,
45
- evidence: [],
48
+ searchQueries: lastPass?.searchQueries ?? previousQueries,
49
+ evidence: lastPass?.evidence ?? [],
46
50
  gaps: allGaps,
47
51
  lowValueOutcomes: allLowValueOutcomes,
52
+ suggestedHeadlessUrl: lastPass?.suggestedHeadlessUrl,
48
53
  exhaustedBudget
49
54
  };
50
55
  }
56
+ function directUnreadableMessage(url) {
57
+ return classifySourceProfile(url).kind === 'forum-thread'
58
+ ? `Thread source could not be read reliably: ${url}`
59
+ : `Direct URL could not be read reliably: ${url}`;
60
+ }
61
+ function shouldRetryDirectWithHeadless(result, evidence) {
62
+ if (result.status === 'needs_headless')
63
+ return true;
64
+ if (result.status !== 'ok' || evidence)
65
+ return false;
66
+ return classifySourceProfile(result.url).shouldPreferHeadlessWhenWeak;
67
+ }
51
68
  function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget }) {
52
69
  return {
53
70
  searchPasses: previousQueries.length,
@@ -70,7 +87,7 @@ function decisionForAnswer(action, query, ranked) {
70
87
  followupQuery: query
71
88
  };
72
89
  }
73
- export function createResearchOrchestrator({ worker, headlessFetch }) {
90
+ export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch }) {
74
91
  return {
75
92
  async run({ query }) {
76
93
  const allEvidence = [];
@@ -80,6 +97,41 @@ export function createResearchOrchestrator({ worker, headlessFetch }) {
80
97
  const suggestedHeadlessUrls = [];
81
98
  let headlessAttempts = 0;
82
99
  let lastPass;
100
+ if (fetchDirect) {
101
+ for (const url of extractDirectUrls(query).slice(0, 3)) {
102
+ const directResult = await fetchDirect({ url });
103
+ const directEvidence = evidenceFromFetch(directResult);
104
+ if (directEvidence) {
105
+ allEvidence.push(directEvidence);
106
+ continue;
107
+ }
108
+ if (shouldRetryDirectWithHeadless(directResult, directEvidence)) {
109
+ if (headlessAttempts < DEFAULT_MAX_HEADLESS_ATTEMPTS) {
110
+ headlessAttempts++;
111
+ const headlessResult = await headlessFetch({ url: directResult.url });
112
+ const headlessEvidence = evidenceFromHeadless(headlessResult);
113
+ if (headlessEvidence) {
114
+ allEvidence.push(headlessEvidence);
115
+ }
116
+ else {
117
+ allGaps.push({ kind: 'fetch-failed', message: directUnreadableMessage(directResult.url) });
118
+ }
119
+ }
120
+ else {
121
+ allGaps.push({ kind: 'fetch-failed', message: directUnreadableMessage(directResult.url) });
122
+ }
123
+ }
124
+ else if (directResult.status !== 'ok') {
125
+ allGaps.push({
126
+ kind: 'fetch-failed',
127
+ message: directResult.error?.message ?? `Direct URL fetch failed for ${directResult.url}`
128
+ });
129
+ }
130
+ else {
131
+ allGaps.push({ kind: 'fetch-failed', message: directUnreadableMessage(directResult.url) });
132
+ }
133
+ }
134
+ }
83
135
  for (let passIndex = 0; passIndex < DEFAULT_MAX_PASSES; passIndex++) {
84
136
  const queries = planSearchQueries({
85
137
  originalQuery: query,
@@ -127,7 +179,13 @@ export function createResearchOrchestrator({ worker, headlessFetch }) {
127
179
  return {
128
180
  decision: decisionForAnswer(updatedDecision.action === 'answer' ? 'answer' : 'answer-with-caveat', query, updatedRanked),
129
181
  evidence: updatedRanked,
130
- workerPass: lastPass,
182
+ workerPass: combinedWorkerPass({
183
+ lastPass,
184
+ previousQueries,
185
+ allGaps,
186
+ allLowValueOutcomes,
187
+ exhaustedBudget: updatedDecision.action !== 'answer'
188
+ }),
131
189
  metadata: buildMetadata({
132
190
  previousQueries,
133
191
  allEvidence,
@@ -146,7 +204,13 @@ export function createResearchOrchestrator({ worker, headlessFetch }) {
146
204
  approvedEvidence: ranked
147
205
  },
148
206
  evidence: ranked,
149
- workerPass: lastPass,
207
+ workerPass: combinedWorkerPass({
208
+ lastPass,
209
+ previousQueries,
210
+ allGaps,
211
+ allLowValueOutcomes,
212
+ exhaustedBudget: false
213
+ }),
150
214
  metadata: buildMetadata({
151
215
  previousQueries,
152
216
  allEvidence,
@@ -161,7 +225,13 @@ export function createResearchOrchestrator({ worker, headlessFetch }) {
161
225
  return {
162
226
  decision: decisionForAnswer(decision.action, query, ranked),
163
227
  evidence: ranked,
164
- workerPass: lastPass,
228
+ workerPass: combinedWorkerPass({
229
+ lastPass,
230
+ previousQueries,
231
+ allGaps,
232
+ allLowValueOutcomes,
233
+ exhaustedBudget: decision.action === 'answer-with-caveat'
234
+ }),
165
235
  metadata: buildMetadata({
166
236
  previousQueries,
167
237
  allEvidence,
@@ -178,13 +248,13 @@ export function createResearchOrchestrator({ worker, headlessFetch }) {
178
248
  return {
179
249
  decision: decisionForAnswer('answer-with-caveat', query, ranked),
180
250
  evidence: ranked,
181
- workerPass: lastPass ??
182
- fallbackWorkerPass({
183
- previousQueries,
184
- allGaps,
185
- allLowValueOutcomes,
186
- exhaustedBudget: true
187
- }),
251
+ workerPass: combinedWorkerPass({
252
+ lastPass,
253
+ previousQueries,
254
+ allGaps,
255
+ allLowValueOutcomes,
256
+ exhaustedBudget: true
257
+ }),
188
258
  metadata: buildMetadata({
189
259
  previousQueries,
190
260
  allEvidence,
@@ -1,18 +1,7 @@
1
1
  import { selectCandidates } from './candidate-selector.js';
2
+ import { classifySourceProfile } from './source-profile.js';
2
3
  function classifySource(url) {
3
- if (url.includes('/docs/api/') || url.includes('/config/'))
4
- return 'official-api';
5
- if (url.includes('playwright.dev/docs') || url.includes('vitest.dev/guide/'))
6
- return 'official-docs';
7
- if (url.includes('github.com/vitest-dev/vitest') && url.includes('/docs/'))
8
- return 'official-docs';
9
- if (url.includes('learn.microsoft.com'))
10
- return 'official-docs';
11
- if (url.includes('github.com/') && url.includes('/issues/'))
12
- return 'issue-thread';
13
- if (url.includes('npmjs.com/package/'))
14
- return 'package-page';
15
- return 'community';
4
+ return classifySourceProfile(url).sourceKind;
16
5
  }
17
6
  function summarizeText(text, maxLength = 180) {
18
7
  return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
@@ -95,6 +84,7 @@ export function createResearchWorker({ search, fetchPage }) {
95
84
  };
96
85
  }
97
86
  const candidates = selectCandidates({
87
+ query,
98
88
  results: searchResult.results,
99
89
  seenUrls: new Set(evidence.map((item) => item.url)),
100
90
  maxCandidates: maxFetches
@@ -0,0 +1,8 @@
1
+ import type { ResearchSourceKind } from './research-types.js';
2
+ export type SourceProfileKind = 'official-docs' | 'official-api' | 'issue-thread' | 'forum-thread' | 'package-page' | 'community';
3
+ export type SourceProfile = {
4
+ kind: SourceProfileKind;
5
+ sourceKind: ResearchSourceKind;
6
+ shouldPreferHeadlessWhenWeak: boolean;
7
+ };
8
+ export declare function classifySourceProfile(rawUrl: string): SourceProfile;
@@ -0,0 +1,60 @@
1
+ const COMMUNITY_FORUM_HOST_RE = /(^|\.)(community|forum|forums|discuss|discourse)\./;
2
+ function profile(kind, sourceKind, shouldPreferHeadlessWhenWeak) {
3
+ return { kind, sourceKind, shouldPreferHeadlessWhenWeak };
4
+ }
5
+ function parseUrl(rawUrl) {
6
+ try {
7
+ return new URL(rawUrl);
8
+ }
9
+ catch {
10
+ return undefined;
11
+ }
12
+ }
13
+ function isOfficialApi(host, path) {
14
+ return ((host === 'playwright.dev' && path.startsWith('/docs/api/')) ||
15
+ (host === 'vitest.dev' && path.startsWith('/config/')));
16
+ }
17
+ function isOfficialDocs(host, path) {
18
+ return ((host === 'playwright.dev' && path.startsWith('/docs/')) ||
19
+ (host === 'vitest.dev' && path.startsWith('/guide/')) ||
20
+ (host === 'github.com' && path.startsWith('/vitest-dev/vitest/') && path.includes('/docs/')) ||
21
+ host === 'learn.microsoft.com');
22
+ }
23
+ function isIssueThread(host, path) {
24
+ return host === 'github.com' && (path.includes('/issues/') || path.includes('/discussions/'));
25
+ }
26
+ function hasForumThreadPath(path) {
27
+ return (path.includes('/forum/') ||
28
+ path.includes('/forums/') ||
29
+ path.includes('/t/') ||
30
+ path.includes('/topic/') ||
31
+ path.includes('/threads/'));
32
+ }
33
+ function isForumThread(host, path) {
34
+ return ((host === 'reddit.com' && path.includes('/comments/')) ||
35
+ (host === 'stackoverflow.com' && path.startsWith('/questions/')) ||
36
+ (COMMUNITY_FORUM_HOST_RE.test(`${host}.`) && hasForumThreadPath(path)));
37
+ }
38
+ export function classifySourceProfile(rawUrl) {
39
+ const parsed = parseUrl(rawUrl);
40
+ if (!parsed)
41
+ return profile('community', 'community', false);
42
+ const host = parsed.hostname.toLowerCase().replace(/^www\./, '');
43
+ const path = parsed.pathname.toLowerCase();
44
+ if (isOfficialApi(host, path)) {
45
+ return profile('official-api', 'official-api', false);
46
+ }
47
+ if (isOfficialDocs(host, path)) {
48
+ return profile('official-docs', 'official-docs', false);
49
+ }
50
+ if (isIssueThread(host, path)) {
51
+ return profile('issue-thread', 'issue-thread', true);
52
+ }
53
+ if (isForumThread(host, path)) {
54
+ return profile('forum-thread', 'community', true);
55
+ }
56
+ if (host === 'npmjs.com' && path.startsWith('/package/')) {
57
+ return profile('package-page', 'package-page', false);
58
+ }
59
+ return profile('community', 'community', false);
60
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@demigodmode/pi-web-agent",
3
- "version": "1.2.0",
3
+ "version": "1.3.0",
4
4
  "description": "Pi package for reliable web access with explicit search, fetch, and headless boundaries.",
5
5
  "type": "module",
6
6
  "main": "./dist/extension.js",
@@ -57,7 +57,7 @@
57
57
  "@mozilla/readability": "^0.6.0",
58
58
  "cheerio": "^1.1.0",
59
59
  "jsdom": "^26.0.0",
60
- "playwright-core": "^1.54.0",
60
+ "playwright": "^1.60.0",
61
61
  "typebox": "^1.1.37"
62
62
  },
63
63
  "devDependencies": {