@demigodmode/pi-web-agent 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +4 -4
- package/dist/backends/doctor.js +3 -0
- package/dist/fetch/headless-fetch.d.ts +2 -1
- package/dist/fetch/headless-fetch.js +11 -7
- package/dist/orchestration/candidate-selector.d.ts +2 -1
- package/dist/orchestration/candidate-selector.js +28 -9
- package/dist/orchestration/direct-url.d.ts +1 -0
- package/dist/orchestration/direct-url.js +47 -0
- package/dist/orchestration/index.js +5 -1
- package/dist/orchestration/research-orchestrator.d.ts +5 -2
- package/dist/orchestration/research-orchestrator.js +97 -27
- package/dist/orchestration/research-worker.js +3 -13
- package/dist/orchestration/source-profile.d.ts +8 -0
- package/dist/orchestration/source-profile.js +60 -0
- package/package.json +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -18,6 +18,22 @@ The format is intentionally simple and release-oriented.
|
|
|
18
18
|
### Breaking
|
|
19
19
|
- None.
|
|
20
20
|
|
|
21
|
+
## [1.3.0] - 2026-06-04
|
|
22
|
+
### Added
|
|
23
|
+
- Added direct URL handling in `web_explore` so linked pages are read before search results.
|
|
24
|
+
- Added forum/thread source classification for Reddit-style discussions, forums, Stack Overflow, and GitHub issues/discussions.
|
|
25
|
+
- Added Playwright-managed Chromium fallback when no local Chromium-family browser is detected.
|
|
26
|
+
|
|
27
|
+
### Changed
|
|
28
|
+
- Discussion-oriented queries now prefer forum/thread results over generic pages.
|
|
29
|
+
- `/web-agent doctor` now reports the local-browser headless backend and managed Chromium fallback.
|
|
30
|
+
|
|
31
|
+
### Fixed
|
|
32
|
+
- Preserved direct/thread fetch gaps in bounded research results so unreadable thread sources get explicit caveats.
|
|
33
|
+
|
|
34
|
+
### Breaking
|
|
35
|
+
- None.
|
|
36
|
+
|
|
21
37
|
## [1.2.0] - 2026-06-01
|
|
22
38
|
### Added
|
|
23
39
|
- Added backend provider and fallback editing to `/web-agent settings`.
|
package/README.md
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
Most agent web tools blur search, fetch, browser rendering, and research into one vague thing. `pi-web-agent` exposes one public research tool, `web_explore`, and keeps search/fetch/headless work inside that bounded workflow.
|
|
10
10
|
|
|
11
|
-
The point is keeping the model-facing boundary simple: ask `web_explore` to research a question, and it handles discovery, HTTP reads, targeted browser rendering, source ranking, and caveats internally.
|
|
11
|
+
The point is keeping the model-facing boundary simple: ask `web_explore` to research a question, and it handles direct links, discovery, HTTP reads, targeted browser rendering, source ranking, and caveats internally.
|
|
12
12
|
|
|
13
13
|
That sounds obvious, but a lot of agent tooling gets fuzzy right there. This package is meant to be stricter about what it actually did and more willing to say when a read was not good enough to trust.
|
|
14
14
|
|
|
@@ -20,9 +20,9 @@ Compatibility notice: current `pi-web-agent` requires Pi 0.74+ because Pi packag
|
|
|
20
20
|
pi install npm:@demigodmode/pi-web-agent
|
|
21
21
|
```
|
|
22
22
|
|
|
23
|
-
After installing, reload or restart Pi. Run `/web-agent` for the action menu, or `/web-agent doctor` to check whether the package loaded cleanly and
|
|
23
|
+
After installing, reload or restart Pi. Run `/web-agent` for the action menu, or `/web-agent doctor` to check whether the package loaded cleanly and which web backends are configured.
|
|
24
24
|
|
|
25
|
-
Headless rendering
|
|
25
|
+
Headless rendering first tries a detectable Chromium-family browser: Chrome, Chromium, Edge, or Brave. If none is found, it falls back to Playwright-managed Chromium and still launches headless. Firefox/Safari-only systems can still use search and plain HTTP reads; browser-rendered fallback uses Chromium.
|
|
26
26
|
|
|
27
27
|
Later on, update installed packages with:
|
|
28
28
|
|
|
@@ -109,7 +109,7 @@ Example:
|
|
|
109
109
|
}
|
|
110
110
|
```
|
|
111
111
|
|
|
112
|
-
Backend config is also supported. Defaults remain DuckDuckGo search, plain HTTP fetch, and local
|
|
112
|
+
Backend config is also supported. Defaults remain DuckDuckGo search, plain HTTP fetch, and local-browser headless fallback with managed Chromium fallback configured.
|
|
113
113
|
|
|
114
114
|
Backend settings can be changed from:
|
|
115
115
|
|
package/dist/backends/doctor.js
CHANGED
|
@@ -83,5 +83,8 @@ export async function checkBackendHealth(config, { fetchImpl = fetch, timeoutMs
|
|
|
83
83
|
if (config.fetch.fallback) {
|
|
84
84
|
lines.push(`fetch fallback: ${config.fetch.fallback}`);
|
|
85
85
|
}
|
|
86
|
+
if (config.headless.provider === 'local-browser') {
|
|
87
|
+
lines.push('headless backend: local-browser (managed Chromium fallback configured)');
|
|
88
|
+
}
|
|
86
89
|
return lines;
|
|
87
90
|
}
|
|
@@ -6,7 +6,8 @@ export declare function headlessFetch(url: string, { configuredPath, resolveBrow
|
|
|
6
6
|
configuredPath?: string;
|
|
7
7
|
}) => Promise<BrowserResolutionResult>;
|
|
8
8
|
launchBrowser?: (options: {
|
|
9
|
-
executablePath
|
|
9
|
+
executablePath?: string;
|
|
10
|
+
headless: true;
|
|
10
11
|
}) => Promise<{
|
|
11
12
|
newContext: () => Promise<{
|
|
12
13
|
newPage: () => Promise<any>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { chromium } from 'playwright
|
|
1
|
+
import { chromium } from 'playwright';
|
|
2
2
|
import { extractReadableContentSafely } from '../extract/readability.js';
|
|
3
3
|
import { resolveBrowserExecutable } from './browser-resolution.js';
|
|
4
4
|
function cleanupRenderedText(text) {
|
|
@@ -7,9 +7,9 @@ function cleanupRenderedText(text) {
|
|
|
7
7
|
cleaned = cleaned.replace(/\s+/g, ' ').trim();
|
|
8
8
|
return cleaned;
|
|
9
9
|
}
|
|
10
|
-
export async function headlessFetch(url, { configuredPath, resolveBrowser = (options) => resolveBrowserExecutable({ configuredPath: options?.configuredPath }), launchBrowser = ({ executablePath }) => chromium.launch({ executablePath, headless:
|
|
10
|
+
export async function headlessFetch(url, { configuredPath, resolveBrowser = (options) => resolveBrowserExecutable({ configuredPath: options?.configuredPath }), launchBrowser = ({ executablePath, headless }) => chromium.launch(executablePath ? { executablePath, headless } : { headless }), now = () => Date.now() } = {}) {
|
|
11
11
|
const resolved = await resolveBrowser({ configuredPath });
|
|
12
|
-
if (!resolved.ok) {
|
|
12
|
+
if (!resolved.ok && resolved.error.code === 'CONFIGURED_BROWSER_NOT_FOUND') {
|
|
13
13
|
return {
|
|
14
14
|
status: 'error',
|
|
15
15
|
url,
|
|
@@ -17,11 +17,15 @@ export async function headlessFetch(url, { configuredPath, resolveBrowser = (opt
|
|
|
17
17
|
error: resolved.error
|
|
18
18
|
};
|
|
19
19
|
}
|
|
20
|
+
const browserName = resolved.ok ? resolved.browser : 'chromium';
|
|
21
|
+
const launchOptions = resolved.ok
|
|
22
|
+
? { executablePath: resolved.executablePath, headless: true }
|
|
23
|
+
: { headless: true };
|
|
20
24
|
let browser;
|
|
21
25
|
let context;
|
|
22
26
|
let page;
|
|
23
27
|
try {
|
|
24
|
-
browser = await launchBrowser(
|
|
28
|
+
browser = await launchBrowser(launchOptions);
|
|
25
29
|
context = await browser.newContext();
|
|
26
30
|
page = await context.newPage();
|
|
27
31
|
const startedAt = now();
|
|
@@ -42,7 +46,7 @@ export async function headlessFetch(url, { configuredPath, resolveBrowser = (opt
|
|
|
42
46
|
metadata: {
|
|
43
47
|
method: 'headless',
|
|
44
48
|
cacheHit: false,
|
|
45
|
-
browser:
|
|
49
|
+
browser: browserName,
|
|
46
50
|
navigationMs: finishedAt - startedAt
|
|
47
51
|
},
|
|
48
52
|
error: {
|
|
@@ -58,7 +62,7 @@ export async function headlessFetch(url, { configuredPath, resolveBrowser = (opt
|
|
|
58
62
|
metadata: {
|
|
59
63
|
method: 'headless',
|
|
60
64
|
cacheHit: false,
|
|
61
|
-
browser:
|
|
65
|
+
browser: browserName,
|
|
62
66
|
navigationMs: finishedAt - startedAt,
|
|
63
67
|
truncated: cleanedContent.text.length >= 4000
|
|
64
68
|
}
|
|
@@ -71,7 +75,7 @@ export async function headlessFetch(url, { configuredPath, resolveBrowser = (opt
|
|
|
71
75
|
metadata: {
|
|
72
76
|
method: 'headless',
|
|
73
77
|
cacheHit: false,
|
|
74
|
-
browser:
|
|
78
|
+
browser: browserName
|
|
75
79
|
},
|
|
76
80
|
error: {
|
|
77
81
|
code: 'HEADLESS_NAVIGATION_FAILED',
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { SearchResult } from '../types.js';
|
|
2
|
-
export declare function selectCandidates({ results, seenUrls, maxCandidates }: {
|
|
2
|
+
export declare function selectCandidates({ query, results, seenUrls, maxCandidates }: {
|
|
3
|
+
query?: string;
|
|
3
4
|
results: SearchResult[];
|
|
4
5
|
seenUrls: Set<string>;
|
|
5
6
|
maxCandidates: number;
|
|
@@ -1,16 +1,35 @@
|
|
|
1
|
-
|
|
1
|
+
import { classifySourceProfile } from './source-profile.js';
|
|
2
|
+
function wantsDiscussionSources(query = '') {
|
|
3
|
+
return /reddit|forum|forums|discussion|thread|comments|community|user experience|people recommend/i.test(query);
|
|
4
|
+
}
|
|
5
|
+
function candidateScore(result, query) {
|
|
2
6
|
const url = result.url.toLowerCase();
|
|
3
|
-
|
|
7
|
+
const profile = classifySourceProfile(result.url);
|
|
8
|
+
const wantsThreads = wantsDiscussionSources(query);
|
|
9
|
+
if (profile.kind === 'official-docs')
|
|
4
10
|
return 0;
|
|
5
|
-
if (
|
|
11
|
+
if (profile.kind === 'official-api')
|
|
6
12
|
return 1;
|
|
7
|
-
if (
|
|
13
|
+
if (wantsThreads) {
|
|
14
|
+
if (profile.kind === 'forum-thread')
|
|
15
|
+
return 2;
|
|
16
|
+
if (profile.kind === 'issue-thread')
|
|
17
|
+
return 3;
|
|
18
|
+
if (url.includes('github.com/'))
|
|
19
|
+
return 4;
|
|
20
|
+
if (profile.kind === 'package-page')
|
|
21
|
+
return 6;
|
|
22
|
+
return 5;
|
|
23
|
+
}
|
|
24
|
+
if (profile.kind === 'issue-thread')
|
|
8
25
|
return 2;
|
|
9
|
-
if (url.includes('
|
|
10
|
-
return
|
|
11
|
-
|
|
26
|
+
if (url.includes('github.com/'))
|
|
27
|
+
return 3;
|
|
28
|
+
if (profile.kind === 'package-page')
|
|
29
|
+
return 5;
|
|
30
|
+
return 4;
|
|
12
31
|
}
|
|
13
|
-
export function selectCandidates({ results, seenUrls, maxCandidates }) {
|
|
32
|
+
export function selectCandidates({ query, results, seenUrls, maxCandidates }) {
|
|
14
33
|
const deduped = new Map();
|
|
15
34
|
for (const result of results) {
|
|
16
35
|
if (seenUrls.has(result.url))
|
|
@@ -19,6 +38,6 @@ export function selectCandidates({ results, seenUrls, maxCandidates }) {
|
|
|
19
38
|
deduped.set(result.url, result);
|
|
20
39
|
}
|
|
21
40
|
return [...deduped.values()]
|
|
22
|
-
.sort((left, right) => candidateScore(left) - candidateScore(right))
|
|
41
|
+
.sort((left, right) => candidateScore(left, query) - candidateScore(right, query))
|
|
23
42
|
.slice(0, maxCandidates);
|
|
24
43
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function extractDirectUrls(query: string): string[];
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
const TRACKING_PARAMS = new Set([
|
|
2
|
+
'utm_source',
|
|
3
|
+
'utm_medium',
|
|
4
|
+
'utm_campaign',
|
|
5
|
+
'utm_term',
|
|
6
|
+
'utm_content',
|
|
7
|
+
'utm_name',
|
|
8
|
+
'fbclid',
|
|
9
|
+
'gclid'
|
|
10
|
+
]);
|
|
11
|
+
function stripTrailingPunctuation(raw) {
|
|
12
|
+
let next = raw.trim();
|
|
13
|
+
while (/[),.;!?\]]$/.test(next)) {
|
|
14
|
+
const last = next.at(-1);
|
|
15
|
+
if (last === ')' && next.includes('(') && next.lastIndexOf('(') > next.lastIndexOf(')'))
|
|
16
|
+
break;
|
|
17
|
+
next = next.slice(0, -1);
|
|
18
|
+
}
|
|
19
|
+
return next;
|
|
20
|
+
}
|
|
21
|
+
function normalizeDirectUrl(raw) {
|
|
22
|
+
try {
|
|
23
|
+
const url = new URL(stripTrailingPunctuation(raw));
|
|
24
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:')
|
|
25
|
+
return undefined;
|
|
26
|
+
for (const key of [...url.searchParams.keys()]) {
|
|
27
|
+
if (TRACKING_PARAMS.has(key.toLowerCase())) {
|
|
28
|
+
url.searchParams.delete(key);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
url.hash = '';
|
|
32
|
+
return url.toString().replace(/\/$/, '');
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
return undefined;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
export function extractDirectUrls(query) {
|
|
39
|
+
const matches = query.match(/https?:\/\/\S+/gi) ?? [];
|
|
40
|
+
const urls = new Set();
|
|
41
|
+
for (const match of matches) {
|
|
42
|
+
const normalized = normalizeDirectUrl(match);
|
|
43
|
+
if (normalized)
|
|
44
|
+
urls.add(normalized);
|
|
45
|
+
}
|
|
46
|
+
return [...urls];
|
|
47
|
+
}
|
|
@@ -7,5 +7,9 @@ export function createResearchWorkflow({ backendConfig, search, fetchPage, headl
|
|
|
7
7
|
const resolvedFetchPage = fetchPage ?? backends.fetchPage;
|
|
8
8
|
const resolvedHeadlessFetch = headlessFetch ?? backends.headlessFetch;
|
|
9
9
|
const worker = createResearchWorker({ search: resolvedSearch, fetchPage: resolvedFetchPage });
|
|
10
|
-
return createResearchOrchestrator({
|
|
10
|
+
return createResearchOrchestrator({
|
|
11
|
+
worker,
|
|
12
|
+
fetchDirect: resolvedFetchPage,
|
|
13
|
+
headlessFetch: resolvedHeadlessFetch
|
|
14
|
+
});
|
|
11
15
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import type { WebFetchHeadlessResponse } from '../types.js';
|
|
1
|
+
import type { WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
|
|
2
2
|
import type { ResearchEvidence, ResearchOrchestratorDecision, ResearchWorkerResult } from './research-types.js';
|
|
3
|
-
export declare function createResearchOrchestrator({ worker, headlessFetch }: {
|
|
3
|
+
export declare function createResearchOrchestrator({ worker, fetchDirect, headlessFetch }: {
|
|
4
4
|
worker: {
|
|
5
5
|
run: (input: {
|
|
6
6
|
query: string;
|
|
@@ -8,6 +8,9 @@ export declare function createResearchOrchestrator({ worker, headlessFetch }: {
|
|
|
8
8
|
maxFetches: number;
|
|
9
9
|
}) => Promise<ResearchWorkerResult>;
|
|
10
10
|
};
|
|
11
|
+
fetchDirect?: (input: {
|
|
12
|
+
url: string;
|
|
13
|
+
}) => Promise<WebFetchResponse>;
|
|
11
14
|
headlessFetch: (input: {
|
|
12
15
|
url: string;
|
|
13
16
|
}) => Promise<WebFetchHeadlessResponse>;
|
|
@@ -1,23 +1,13 @@
|
|
|
1
1
|
import { rankEvidence } from './evidence-ranker.js';
|
|
2
2
|
import { planSearchQueries } from './query-planner.js';
|
|
3
|
+
import { classifySourceProfile } from './source-profile.js';
|
|
4
|
+
import { extractDirectUrls } from './direct-url.js';
|
|
3
5
|
import { decideNextResearchStep } from './stop-decider.js';
|
|
4
6
|
const DEFAULT_MAX_PASSES = 3;
|
|
5
7
|
const DEFAULT_MAX_FETCHES_PER_PASS = 4;
|
|
6
8
|
const DEFAULT_MAX_HEADLESS_ATTEMPTS = 2;
|
|
7
9
|
function classifyEvidenceUrl(url) {
|
|
8
|
-
|
|
9
|
-
return 'official-api';
|
|
10
|
-
if (url.includes('playwright.dev/docs') || url.includes('vitest.dev/guide/'))
|
|
11
|
-
return 'official-docs';
|
|
12
|
-
if (url.includes('github.com/vitest-dev/vitest') && url.includes('/docs/'))
|
|
13
|
-
return 'official-docs';
|
|
14
|
-
if (url.includes('learn.microsoft.com'))
|
|
15
|
-
return 'official-docs';
|
|
16
|
-
if (url.includes('github.com/') && url.includes('/issues/'))
|
|
17
|
-
return 'issue-thread';
|
|
18
|
-
if (url.includes('npmjs.com/package/'))
|
|
19
|
-
return 'package-page';
|
|
20
|
-
return 'community';
|
|
10
|
+
return classifySourceProfile(url).sourceKind;
|
|
21
11
|
}
|
|
22
12
|
function summarizeText(text, maxLength = 180) {
|
|
23
13
|
return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
|
|
@@ -25,6 +15,20 @@ function summarizeText(text, maxLength = 180) {
|
|
|
25
15
|
function isBotCheckContent({ title = '', text }) {
|
|
26
16
|
return /performing security verification|security service|verify you are not a bot|just a moment|checking your browser/i.test(`${title}\n${text}`);
|
|
27
17
|
}
|
|
18
|
+
function evidenceFromFetch(result) {
|
|
19
|
+
if (result.status !== 'ok' || !result.content?.text.trim())
|
|
20
|
+
return null;
|
|
21
|
+
if (isBotCheckContent({ title: result.content.title, text: result.content.text }))
|
|
22
|
+
return null;
|
|
23
|
+
return {
|
|
24
|
+
title: result.content.title ?? result.url,
|
|
25
|
+
url: result.url,
|
|
26
|
+
sourceKind: classifyEvidenceUrl(result.url),
|
|
27
|
+
method: result.metadata.method,
|
|
28
|
+
summary: summarizeText(result.content.text),
|
|
29
|
+
supports: [summarizeText(result.content.text, 120)]
|
|
30
|
+
};
|
|
31
|
+
}
|
|
28
32
|
function evidenceFromHeadless(result) {
|
|
29
33
|
if (result.status !== 'ok' || !result.content?.text.trim())
|
|
30
34
|
return null;
|
|
@@ -39,15 +43,28 @@ function evidenceFromHeadless(result) {
|
|
|
39
43
|
supports: [summarizeText(result.content.text, 120)]
|
|
40
44
|
};
|
|
41
45
|
}
|
|
42
|
-
function
|
|
46
|
+
function combinedWorkerPass({ lastPass, previousQueries, allGaps, allLowValueOutcomes, exhaustedBudget }) {
|
|
43
47
|
return {
|
|
44
|
-
searchQueries: previousQueries,
|
|
45
|
-
evidence: [],
|
|
48
|
+
searchQueries: lastPass?.searchQueries ?? previousQueries,
|
|
49
|
+
evidence: lastPass?.evidence ?? [],
|
|
46
50
|
gaps: allGaps,
|
|
47
51
|
lowValueOutcomes: allLowValueOutcomes,
|
|
52
|
+
suggestedHeadlessUrl: lastPass?.suggestedHeadlessUrl,
|
|
48
53
|
exhaustedBudget
|
|
49
54
|
};
|
|
50
55
|
}
|
|
56
|
+
function directUnreadableMessage(url) {
|
|
57
|
+
return classifySourceProfile(url).kind === 'forum-thread'
|
|
58
|
+
? `Thread source could not be read reliably: ${url}`
|
|
59
|
+
: `Direct URL could not be read reliably: ${url}`;
|
|
60
|
+
}
|
|
61
|
+
function shouldRetryDirectWithHeadless(result, evidence) {
|
|
62
|
+
if (result.status === 'needs_headless')
|
|
63
|
+
return true;
|
|
64
|
+
if (result.status !== 'ok' || evidence)
|
|
65
|
+
return false;
|
|
66
|
+
return classifySourceProfile(result.url).shouldPreferHeadlessWhenWeak;
|
|
67
|
+
}
|
|
51
68
|
function buildMetadata({ previousQueries, allEvidence, allGaps, allLowValueOutcomes, headlessAttempts, exhaustedBudget }) {
|
|
52
69
|
return {
|
|
53
70
|
searchPasses: previousQueries.length,
|
|
@@ -70,7 +87,7 @@ function decisionForAnswer(action, query, ranked) {
|
|
|
70
87
|
followupQuery: query
|
|
71
88
|
};
|
|
72
89
|
}
|
|
73
|
-
export function createResearchOrchestrator({ worker, headlessFetch }) {
|
|
90
|
+
export function createResearchOrchestrator({ worker, fetchDirect, headlessFetch }) {
|
|
74
91
|
return {
|
|
75
92
|
async run({ query }) {
|
|
76
93
|
const allEvidence = [];
|
|
@@ -80,6 +97,41 @@ export function createResearchOrchestrator({ worker, headlessFetch }) {
|
|
|
80
97
|
const suggestedHeadlessUrls = [];
|
|
81
98
|
let headlessAttempts = 0;
|
|
82
99
|
let lastPass;
|
|
100
|
+
if (fetchDirect) {
|
|
101
|
+
for (const url of extractDirectUrls(query).slice(0, 3)) {
|
|
102
|
+
const directResult = await fetchDirect({ url });
|
|
103
|
+
const directEvidence = evidenceFromFetch(directResult);
|
|
104
|
+
if (directEvidence) {
|
|
105
|
+
allEvidence.push(directEvidence);
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
if (shouldRetryDirectWithHeadless(directResult, directEvidence)) {
|
|
109
|
+
if (headlessAttempts < DEFAULT_MAX_HEADLESS_ATTEMPTS) {
|
|
110
|
+
headlessAttempts++;
|
|
111
|
+
const headlessResult = await headlessFetch({ url: directResult.url });
|
|
112
|
+
const headlessEvidence = evidenceFromHeadless(headlessResult);
|
|
113
|
+
if (headlessEvidence) {
|
|
114
|
+
allEvidence.push(headlessEvidence);
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
allGaps.push({ kind: 'fetch-failed', message: directUnreadableMessage(directResult.url) });
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
allGaps.push({ kind: 'fetch-failed', message: directUnreadableMessage(directResult.url) });
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
else if (directResult.status !== 'ok') {
|
|
125
|
+
allGaps.push({
|
|
126
|
+
kind: 'fetch-failed',
|
|
127
|
+
message: directResult.error?.message ?? `Direct URL fetch failed for ${directResult.url}`
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
else {
|
|
131
|
+
allGaps.push({ kind: 'fetch-failed', message: directUnreadableMessage(directResult.url) });
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
83
135
|
for (let passIndex = 0; passIndex < DEFAULT_MAX_PASSES; passIndex++) {
|
|
84
136
|
const queries = planSearchQueries({
|
|
85
137
|
originalQuery: query,
|
|
@@ -127,7 +179,13 @@ export function createResearchOrchestrator({ worker, headlessFetch }) {
|
|
|
127
179
|
return {
|
|
128
180
|
decision: decisionForAnswer(updatedDecision.action === 'answer' ? 'answer' : 'answer-with-caveat', query, updatedRanked),
|
|
129
181
|
evidence: updatedRanked,
|
|
130
|
-
workerPass:
|
|
182
|
+
workerPass: combinedWorkerPass({
|
|
183
|
+
lastPass,
|
|
184
|
+
previousQueries,
|
|
185
|
+
allGaps,
|
|
186
|
+
allLowValueOutcomes,
|
|
187
|
+
exhaustedBudget: updatedDecision.action !== 'answer'
|
|
188
|
+
}),
|
|
131
189
|
metadata: buildMetadata({
|
|
132
190
|
previousQueries,
|
|
133
191
|
allEvidence,
|
|
@@ -146,7 +204,13 @@ export function createResearchOrchestrator({ worker, headlessFetch }) {
|
|
|
146
204
|
approvedEvidence: ranked
|
|
147
205
|
},
|
|
148
206
|
evidence: ranked,
|
|
149
|
-
workerPass:
|
|
207
|
+
workerPass: combinedWorkerPass({
|
|
208
|
+
lastPass,
|
|
209
|
+
previousQueries,
|
|
210
|
+
allGaps,
|
|
211
|
+
allLowValueOutcomes,
|
|
212
|
+
exhaustedBudget: false
|
|
213
|
+
}),
|
|
150
214
|
metadata: buildMetadata({
|
|
151
215
|
previousQueries,
|
|
152
216
|
allEvidence,
|
|
@@ -161,7 +225,13 @@ export function createResearchOrchestrator({ worker, headlessFetch }) {
|
|
|
161
225
|
return {
|
|
162
226
|
decision: decisionForAnswer(decision.action, query, ranked),
|
|
163
227
|
evidence: ranked,
|
|
164
|
-
workerPass:
|
|
228
|
+
workerPass: combinedWorkerPass({
|
|
229
|
+
lastPass,
|
|
230
|
+
previousQueries,
|
|
231
|
+
allGaps,
|
|
232
|
+
allLowValueOutcomes,
|
|
233
|
+
exhaustedBudget: decision.action === 'answer-with-caveat'
|
|
234
|
+
}),
|
|
165
235
|
metadata: buildMetadata({
|
|
166
236
|
previousQueries,
|
|
167
237
|
allEvidence,
|
|
@@ -178,13 +248,13 @@ export function createResearchOrchestrator({ worker, headlessFetch }) {
|
|
|
178
248
|
return {
|
|
179
249
|
decision: decisionForAnswer('answer-with-caveat', query, ranked),
|
|
180
250
|
evidence: ranked,
|
|
181
|
-
workerPass:
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
251
|
+
workerPass: combinedWorkerPass({
|
|
252
|
+
lastPass,
|
|
253
|
+
previousQueries,
|
|
254
|
+
allGaps,
|
|
255
|
+
allLowValueOutcomes,
|
|
256
|
+
exhaustedBudget: true
|
|
257
|
+
}),
|
|
188
258
|
metadata: buildMetadata({
|
|
189
259
|
previousQueries,
|
|
190
260
|
allEvidence,
|
|
@@ -1,18 +1,7 @@
|
|
|
1
1
|
import { selectCandidates } from './candidate-selector.js';
|
|
2
|
+
import { classifySourceProfile } from './source-profile.js';
|
|
2
3
|
function classifySource(url) {
|
|
3
|
-
|
|
4
|
-
return 'official-api';
|
|
5
|
-
if (url.includes('playwright.dev/docs') || url.includes('vitest.dev/guide/'))
|
|
6
|
-
return 'official-docs';
|
|
7
|
-
if (url.includes('github.com/vitest-dev/vitest') && url.includes('/docs/'))
|
|
8
|
-
return 'official-docs';
|
|
9
|
-
if (url.includes('learn.microsoft.com'))
|
|
10
|
-
return 'official-docs';
|
|
11
|
-
if (url.includes('github.com/') && url.includes('/issues/'))
|
|
12
|
-
return 'issue-thread';
|
|
13
|
-
if (url.includes('npmjs.com/package/'))
|
|
14
|
-
return 'package-page';
|
|
15
|
-
return 'community';
|
|
4
|
+
return classifySourceProfile(url).sourceKind;
|
|
16
5
|
}
|
|
17
6
|
function summarizeText(text, maxLength = 180) {
|
|
18
7
|
return text.replace(/\s+/g, ' ').trim().slice(0, maxLength);
|
|
@@ -95,6 +84,7 @@ export function createResearchWorker({ search, fetchPage }) {
|
|
|
95
84
|
};
|
|
96
85
|
}
|
|
97
86
|
const candidates = selectCandidates({
|
|
87
|
+
query,
|
|
98
88
|
results: searchResult.results,
|
|
99
89
|
seenUrls: new Set(evidence.map((item) => item.url)),
|
|
100
90
|
maxCandidates: maxFetches
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { ResearchSourceKind } from './research-types.js';
|
|
2
|
+
export type SourceProfileKind = 'official-docs' | 'official-api' | 'issue-thread' | 'forum-thread' | 'package-page' | 'community';
|
|
3
|
+
export type SourceProfile = {
|
|
4
|
+
kind: SourceProfileKind;
|
|
5
|
+
sourceKind: ResearchSourceKind;
|
|
6
|
+
shouldPreferHeadlessWhenWeak: boolean;
|
|
7
|
+
};
|
|
8
|
+
export declare function classifySourceProfile(rawUrl: string): SourceProfile;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
const COMMUNITY_FORUM_HOST_RE = /(^|\.)(community|forum|forums|discuss|discourse)\./;
|
|
2
|
+
function profile(kind, sourceKind, shouldPreferHeadlessWhenWeak) {
|
|
3
|
+
return { kind, sourceKind, shouldPreferHeadlessWhenWeak };
|
|
4
|
+
}
|
|
5
|
+
function parseUrl(rawUrl) {
|
|
6
|
+
try {
|
|
7
|
+
return new URL(rawUrl);
|
|
8
|
+
}
|
|
9
|
+
catch {
|
|
10
|
+
return undefined;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
function isOfficialApi(host, path) {
|
|
14
|
+
return ((host === 'playwright.dev' && path.startsWith('/docs/api/')) ||
|
|
15
|
+
(host === 'vitest.dev' && path.startsWith('/config/')));
|
|
16
|
+
}
|
|
17
|
+
function isOfficialDocs(host, path) {
|
|
18
|
+
return ((host === 'playwright.dev' && path.startsWith('/docs/')) ||
|
|
19
|
+
(host === 'vitest.dev' && path.startsWith('/guide/')) ||
|
|
20
|
+
(host === 'github.com' && path.startsWith('/vitest-dev/vitest/') && path.includes('/docs/')) ||
|
|
21
|
+
host === 'learn.microsoft.com');
|
|
22
|
+
}
|
|
23
|
+
function isIssueThread(host, path) {
|
|
24
|
+
return host === 'github.com' && (path.includes('/issues/') || path.includes('/discussions/'));
|
|
25
|
+
}
|
|
26
|
+
function hasForumThreadPath(path) {
|
|
27
|
+
return (path.includes('/forum/') ||
|
|
28
|
+
path.includes('/forums/') ||
|
|
29
|
+
path.includes('/t/') ||
|
|
30
|
+
path.includes('/topic/') ||
|
|
31
|
+
path.includes('/threads/'));
|
|
32
|
+
}
|
|
33
|
+
function isForumThread(host, path) {
|
|
34
|
+
return ((host === 'reddit.com' && path.includes('/comments/')) ||
|
|
35
|
+
(host === 'stackoverflow.com' && path.startsWith('/questions/')) ||
|
|
36
|
+
(COMMUNITY_FORUM_HOST_RE.test(`${host}.`) && hasForumThreadPath(path)));
|
|
37
|
+
}
|
|
38
|
+
export function classifySourceProfile(rawUrl) {
|
|
39
|
+
const parsed = parseUrl(rawUrl);
|
|
40
|
+
if (!parsed)
|
|
41
|
+
return profile('community', 'community', false);
|
|
42
|
+
const host = parsed.hostname.toLowerCase().replace(/^www\./, '');
|
|
43
|
+
const path = parsed.pathname.toLowerCase();
|
|
44
|
+
if (isOfficialApi(host, path)) {
|
|
45
|
+
return profile('official-api', 'official-api', false);
|
|
46
|
+
}
|
|
47
|
+
if (isOfficialDocs(host, path)) {
|
|
48
|
+
return profile('official-docs', 'official-docs', false);
|
|
49
|
+
}
|
|
50
|
+
if (isIssueThread(host, path)) {
|
|
51
|
+
return profile('issue-thread', 'issue-thread', true);
|
|
52
|
+
}
|
|
53
|
+
if (isForumThread(host, path)) {
|
|
54
|
+
return profile('forum-thread', 'community', true);
|
|
55
|
+
}
|
|
56
|
+
if (host === 'npmjs.com' && path.startsWith('/package/')) {
|
|
57
|
+
return profile('package-page', 'package-page', false);
|
|
58
|
+
}
|
|
59
|
+
return profile('community', 'community', false);
|
|
60
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@demigodmode/pi-web-agent",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.0",
|
|
4
4
|
"description": "Pi package for reliable web access with explicit search, fetch, and headless boundaries.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/extension.js",
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"@mozilla/readability": "^0.6.0",
|
|
58
58
|
"cheerio": "^1.1.0",
|
|
59
59
|
"jsdom": "^26.0.0",
|
|
60
|
-
"playwright
|
|
60
|
+
"playwright": "^1.60.0",
|
|
61
61
|
"typebox": "^1.1.37"
|
|
62
62
|
},
|
|
63
63
|
"devDependencies": {
|