@demigodmode/pi-web-agent 0.2.2 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +119 -63
- package/dist/commands/web-agent-config.d.ts +23 -0
- package/dist/commands/web-agent-config.js +254 -0
- package/dist/extension.js +113 -4
- package/dist/presentation/config-store.d.ts +23 -0
- package/dist/presentation/config-store.js +64 -0
- package/dist/presentation/config.d.ts +7 -0
- package/dist/presentation/config.js +44 -0
- package/dist/presentation/explore-presentation.d.ts +3 -0
- package/dist/presentation/explore-presentation.js +34 -0
- package/dist/presentation/fetch-presentation.d.ts +5 -0
- package/dist/presentation/fetch-presentation.js +40 -0
- package/dist/presentation/search-presentation.d.ts +3 -0
- package/dist/presentation/search-presentation.js +30 -0
- package/dist/presentation/select-view.d.ts +2 -0
- package/dist/presentation/select-view.js +12 -0
- package/dist/presentation/types.d.ts +50 -0
- package/dist/presentation/types.js +1 -0
- package/dist/search/duckduckgo.d.ts +6 -1
- package/dist/search/duckduckgo.js +11 -1
- package/dist/tools/web-explore.d.ts +6 -16
- package/dist/tools/web-explore.js +12 -10
- package/dist/tools/web-fetch-headless.js +11 -2
- package/dist/tools/web-fetch.js +11 -2
- package/dist/tools/web-search.js +99 -12
- package/dist/types.d.ts +15 -0
- package/package.json +1 -1
- package/dist/scripts/live-web-eval.d.ts +0 -1
- package/dist/scripts/live-web-eval.js +0 -411
- package/dist/src/cache/ttl-cache.d.ts +0 -8
- package/dist/src/cache/ttl-cache.js +0 -21
- package/dist/src/extension.d.ts +0 -2
- package/dist/src/extension.js +0 -155
- package/dist/src/extract/readability.d.ts +0 -8
- package/dist/src/extract/readability.js +0 -93
- package/dist/src/fetch/browser-resolution.d.ts +0 -15
- package/dist/src/fetch/browser-resolution.js +0 -55
- package/dist/src/fetch/headless-fetch.d.ts +0 -18
- package/dist/src/fetch/headless-fetch.js +0 -87
- package/dist/src/fetch/http-fetch.d.ts +0 -4
- package/dist/src/fetch/http-fetch.js +0 -50
- package/dist/src/orchestration/index.d.ts +0 -41
- package/dist/src/orchestration/index.js +0 -9
- package/dist/src/orchestration/research-orchestrator.d.ts +0 -43
- package/dist/src/orchestration/research-orchestrator.js +0 -87
- package/dist/src/orchestration/research-types.d.ts +0 -41
- package/dist/src/orchestration/research-types.js +0 -1
- package/dist/src/orchestration/research-worker.d.ts +0 -16
- package/dist/src/orchestration/research-worker.js +0 -131
- package/dist/src/search/duckduckgo.d.ts +0 -9
- package/dist/src/search/duckduckgo.js +0 -52
- package/dist/src/tools/web-explore.d.ts +0 -44
- package/dist/src/tools/web-explore.js +0 -50
- package/dist/src/tools/web-fetch-headless.d.ts +0 -6
- package/dist/src/tools/web-fetch-headless.js +0 -14
- package/dist/src/tools/web-fetch.d.ts +0 -6
- package/dist/src/tools/web-fetch.js +0 -14
- package/dist/src/tools/web-search.d.ts +0 -10
- package/dist/src/tools/web-search.js +0 -103
- package/dist/src/types.d.ts +0 -48
- package/dist/src/types.js +0 -7
- package/dist/tests/cache/ttl-cache.test.d.ts +0 -1
- package/dist/tests/cache/ttl-cache.test.js +0 -19
- package/dist/tests/contracts.test.d.ts +0 -1
- package/dist/tests/contracts.test.js +0 -65
- package/dist/tests/extension.test.d.ts +0 -1
- package/dist/tests/extension.test.js +0 -123
- package/dist/tests/extract/readability.test.d.ts +0 -1
- package/dist/tests/extract/readability.test.js +0 -79
- package/dist/tests/fetch/browser-resolution.test.d.ts +0 -1
- package/dist/tests/fetch/browser-resolution.test.js +0 -37
- package/dist/tests/fetch/headless-fetch.smoke.test.d.ts +0 -1
- package/dist/tests/fetch/headless-fetch.smoke.test.js +0 -17
- package/dist/tests/fetch/headless-fetch.test.d.ts +0 -1
- package/dist/tests/fetch/headless-fetch.test.js +0 -150
- package/dist/tests/fetch/http-fetch.test.d.ts +0 -1
- package/dist/tests/fetch/http-fetch.test.js +0 -129
- package/dist/tests/orchestration/research-orchestrator.test.d.ts +0 -1
- package/dist/tests/orchestration/research-orchestrator.test.js +0 -298
- package/dist/tests/orchestration/research-worker.test.d.ts +0 -1
- package/dist/tests/orchestration/research-worker.test.js +0 -171
- package/dist/tests/orchestration/research-workflow.test.d.ts +0 -1
- package/dist/tests/orchestration/research-workflow.test.js +0 -119
- package/dist/tests/package-manifest.test.d.ts +0 -1
- package/dist/tests/package-manifest.test.js +0 -29
- package/dist/tests/release-foundation.test.d.ts +0 -1
- package/dist/tests/release-foundation.test.js +0 -16
- package/dist/tests/release-script.test.d.ts +0 -1
- package/dist/tests/release-script.test.js +0 -72
- package/dist/tests/search/duckduckgo.test.d.ts +0 -1
- package/dist/tests/search/duckduckgo.test.js +0 -103
- package/dist/tests/tools/web-explore.test.d.ts +0 -1
- package/dist/tests/tools/web-explore.test.js +0 -163
- package/dist/tests/tools/web-fetch-headless.test.d.ts +0 -1
- package/dist/tests/tools/web-fetch-headless.test.js +0 -31
- package/dist/tests/tools/web-fetch.test.d.ts +0 -1
- package/dist/tests/tools/web-fetch.test.js +0 -27
- package/dist/tests/tools/web-search.test.d.ts +0 -1
- package/dist/tests/tools/web-search.test.js +0 -125
- package/dist/vitest.config.d.ts +0 -2
- package/dist/vitest.config.js +0 -13
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { DEFAULT_PRESENTATION_CONFIG, extractPresentationConfigOverride, mergePresentationConfigLayers } from './config.js';
|
|
4
|
+
export function getPresentationConfigPaths(options = {}) {
|
|
5
|
+
const homeDir = options.homeDir ?? process.env.USERPROFILE ?? process.env.HOME ?? '';
|
|
6
|
+
const projectDir = options.projectDir ?? process.cwd();
|
|
7
|
+
return {
|
|
8
|
+
globalPath: path.join(homeDir, '.pi', 'agent', 'extensions', 'pi-web-agent', 'config.json'),
|
|
9
|
+
projectPath: path.join(projectDir, '.pi', 'extensions', 'pi-web-agent', 'config.json')
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
async function readPresentationConfigFile(filePath) {
|
|
13
|
+
try {
|
|
14
|
+
const rawText = await readFile(filePath, 'utf8');
|
|
15
|
+
const parsed = JSON.parse(rawText);
|
|
16
|
+
return {
|
|
17
|
+
path: filePath,
|
|
18
|
+
exists: true,
|
|
19
|
+
rawConfig: extractPresentationConfigOverride(parsed)
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
catch (error) {
|
|
23
|
+
if (error?.code === 'ENOENT') {
|
|
24
|
+
return { path: filePath, exists: false };
|
|
25
|
+
}
|
|
26
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
27
|
+
return {
|
|
28
|
+
path: filePath,
|
|
29
|
+
exists: true,
|
|
30
|
+
error: message
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
function serializePresentationConfigOverride(config) {
|
|
35
|
+
const presentation = {};
|
|
36
|
+
if (config.defaultMode) {
|
|
37
|
+
presentation.defaultMode = config.defaultMode;
|
|
38
|
+
}
|
|
39
|
+
if (Object.keys(config.tools).length > 0) {
|
|
40
|
+
presentation.tools = config.tools;
|
|
41
|
+
}
|
|
42
|
+
return { presentation };
|
|
43
|
+
}
|
|
44
|
+
export async function loadPresentationConfigLayers(options = {}) {
|
|
45
|
+
const { globalPath, projectPath } = getPresentationConfigPaths(options);
|
|
46
|
+
const global = await readPresentationConfigFile(globalPath);
|
|
47
|
+
const project = await readPresentationConfigFile(projectPath);
|
|
48
|
+
return {
|
|
49
|
+
global,
|
|
50
|
+
project,
|
|
51
|
+
effectiveConfig: mergePresentationConfigLayers(DEFAULT_PRESENTATION_CONFIG, global.rawConfig, project.rawConfig)
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
export async function savePresentationConfigScope(options, scope, config) {
|
|
55
|
+
const { globalPath, projectPath } = getPresentationConfigPaths(options);
|
|
56
|
+
const filePath = scope === 'global' ? globalPath : projectPath;
|
|
57
|
+
await mkdir(path.dirname(filePath), { recursive: true });
|
|
58
|
+
await writeFile(filePath, JSON.stringify(serializePresentationConfigOverride(config), null, 2) + '\n', 'utf8');
|
|
59
|
+
}
|
|
60
|
+
export async function resetPresentationConfigScope(options, scope) {
|
|
61
|
+
const { globalPath, projectPath } = getPresentationConfigPaths(options);
|
|
62
|
+
const filePath = scope === 'global' ? globalPath : projectPath;
|
|
63
|
+
await rm(filePath, { force: true });
|
|
64
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { type PresentationConfig, type PresentationConfigFile, type PresentationConfigOverride, type PresentationMode, type PresentationToolName } from './types.js';
|
|
2
|
+
export declare const DEFAULT_PRESENTATION_CONFIG: PresentationConfig;
|
|
3
|
+
export declare function isPresentationMode(value: unknown): value is PresentationMode;
|
|
4
|
+
export declare function extractPresentationConfigOverride(file: PresentationConfigFile | null | undefined): PresentationConfigOverride;
|
|
5
|
+
export declare function normalizePresentationConfigFile(file: PresentationConfigFile | null | undefined): PresentationConfig;
|
|
6
|
+
export declare function mergePresentationConfigLayers(defaults: PresentationConfig, globalConfig?: PresentationConfigOverride, projectConfig?: PresentationConfigOverride): PresentationConfig;
|
|
7
|
+
export declare function resolvePresentationMode(toolName: PresentationToolName, config?: PresentationConfig): PresentationMode;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { PRESENTATION_MODES } from './types.js';
|
|
2
|
+
const PRESENTATION_MODE_SET = new Set(PRESENTATION_MODES);
|
|
3
|
+
export const DEFAULT_PRESENTATION_CONFIG = {
|
|
4
|
+
defaultMode: 'compact',
|
|
5
|
+
tools: {}
|
|
6
|
+
};
|
|
7
|
+
export function isPresentationMode(value) {
|
|
8
|
+
return typeof value === 'string' && PRESENTATION_MODE_SET.has(value);
|
|
9
|
+
}
|
|
10
|
+
export function extractPresentationConfigOverride(file) {
|
|
11
|
+
const presentation = file?.presentation;
|
|
12
|
+
const tools = Object.fromEntries(Object.entries(presentation?.tools ?? {}).flatMap(([toolName, value]) => {
|
|
13
|
+
if (!value || !isPresentationMode(value.mode)) {
|
|
14
|
+
return [];
|
|
15
|
+
}
|
|
16
|
+
return [[toolName, { mode: value.mode }]];
|
|
17
|
+
}));
|
|
18
|
+
return {
|
|
19
|
+
defaultMode: isPresentationMode(presentation?.defaultMode)
|
|
20
|
+
? presentation.defaultMode
|
|
21
|
+
: undefined,
|
|
22
|
+
tools
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
export function normalizePresentationConfigFile(file) {
|
|
26
|
+
const override = extractPresentationConfigOverride(file);
|
|
27
|
+
return {
|
|
28
|
+
defaultMode: override.defaultMode ?? DEFAULT_PRESENTATION_CONFIG.defaultMode,
|
|
29
|
+
tools: override.tools
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
export function mergePresentationConfigLayers(defaults, globalConfig, projectConfig) {
|
|
33
|
+
return {
|
|
34
|
+
defaultMode: projectConfig?.defaultMode ?? globalConfig?.defaultMode ?? defaults.defaultMode,
|
|
35
|
+
tools: {
|
|
36
|
+
...defaults.tools,
|
|
37
|
+
...globalConfig?.tools,
|
|
38
|
+
...projectConfig?.tools
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
export function resolvePresentationMode(toolName, config = DEFAULT_PRESENTATION_CONFIG) {
|
|
43
|
+
return config.tools[toolName]?.mode ?? config.defaultMode;
|
|
44
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
export function buildExplorePresentation(result) {
|
|
2
|
+
if (result.status === 'error') {
|
|
3
|
+
return {
|
|
4
|
+
mode: 'compact',
|
|
5
|
+
views: {
|
|
6
|
+
compact: `Research failed: ${result.error?.message ?? 'Unknown research failure.'}`
|
|
7
|
+
}
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
const preview = result.findings.map((finding) => `- ${finding}`).join('\n');
|
|
11
|
+
const verbose = [
|
|
12
|
+
'Findings',
|
|
13
|
+
...result.findings.map((finding) => `- ${finding}`),
|
|
14
|
+
'',
|
|
15
|
+
'Sources',
|
|
16
|
+
...result.sources.map((source) => `- ${source.title}: ${source.url}`),
|
|
17
|
+
result.caveat ? `\nCaveat\n${result.caveat}` : undefined
|
|
18
|
+
]
|
|
19
|
+
.filter((line) => line !== undefined)
|
|
20
|
+
.join('\n');
|
|
21
|
+
return {
|
|
22
|
+
mode: 'compact',
|
|
23
|
+
views: {
|
|
24
|
+
compact: `Reviewed ${result.sources.length} sources · synthesized answer with ${result.findings.length} findings`,
|
|
25
|
+
preview,
|
|
26
|
+
verbose
|
|
27
|
+
},
|
|
28
|
+
metrics: {
|
|
29
|
+
sourceCount: result.sources.length,
|
|
30
|
+
resultCount: result.findings.length
|
|
31
|
+
},
|
|
32
|
+
sources: result.sources
|
|
33
|
+
};
|
|
34
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { WebFetchHeadlessResponse, WebFetchResponse } from '../types.js';
|
|
2
|
+
import type { PresentationEnvelope } from './types.js';
|
|
3
|
+
type FetchLike = WebFetchResponse | WebFetchHeadlessResponse;
|
|
4
|
+
export declare function buildFetchPresentation(result: FetchLike): PresentationEnvelope;
|
|
5
|
+
export {};
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
function countWords(text) {
|
|
2
|
+
return text?.trim() ? text.trim().split(/\s+/).length : undefined;
|
|
3
|
+
}
|
|
4
|
+
function firstExcerpt(text, maxChars = 240) {
|
|
5
|
+
if (!text) {
|
|
6
|
+
return undefined;
|
|
7
|
+
}
|
|
8
|
+
return text.length <= maxChars ? text : `${text.slice(0, maxChars).trimEnd()}…`;
|
|
9
|
+
}
|
|
10
|
+
export function buildFetchPresentation(result) {
|
|
11
|
+
const wordCount = countWords(result.content?.text);
|
|
12
|
+
const compact = result.status === 'ok'
|
|
13
|
+
? `Fetched page · article extracted${wordCount ? ` · ${wordCount} words` : ''}`
|
|
14
|
+
: result.status === 'needs_headless'
|
|
15
|
+
? `Needs headless rendering: ${result.error?.message ?? 'Headless rendering recommended.'}`
|
|
16
|
+
: `Fetch failed: ${result.error?.message ?? 'Unknown fetch failure.'}`;
|
|
17
|
+
return {
|
|
18
|
+
mode: 'compact',
|
|
19
|
+
views: {
|
|
20
|
+
compact,
|
|
21
|
+
preview: result.content?.title
|
|
22
|
+
? `${result.content.title}\n${firstExcerpt(result.content.text) ?? ''}`.trim()
|
|
23
|
+
: firstExcerpt(result.content?.text),
|
|
24
|
+
verbose: result.status === 'ok'
|
|
25
|
+
? [
|
|
26
|
+
`URL: ${result.url}`,
|
|
27
|
+
result.content?.title ? `Title: ${result.content.title}` : undefined,
|
|
28
|
+
firstExcerpt(result.content?.text, 500)
|
|
29
|
+
]
|
|
30
|
+
.filter(Boolean)
|
|
31
|
+
.join('\n')
|
|
32
|
+
: undefined
|
|
33
|
+
},
|
|
34
|
+
metrics: {
|
|
35
|
+
wordCount,
|
|
36
|
+
cacheHit: result.metadata.cacheHit
|
|
37
|
+
},
|
|
38
|
+
sources: [{ title: result.content?.title ?? result.url, url: result.url }]
|
|
39
|
+
};
|
|
40
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
function formatCompact(result) {
|
|
2
|
+
if (result.status === 'error') {
|
|
3
|
+
return `Search failed: ${result.error?.message ?? 'Unknown search failure.'}`;
|
|
4
|
+
}
|
|
5
|
+
const suffix = result.results.length === 1 ? 'result' : 'results';
|
|
6
|
+
return `Found ${result.results.length} ${suffix}`;
|
|
7
|
+
}
|
|
8
|
+
export function buildSearchPresentation(result) {
|
|
9
|
+
const preview = result.results
|
|
10
|
+
.slice(0, 3)
|
|
11
|
+
.map((item, index) => `${index + 1}. ${item.title}`)
|
|
12
|
+
.join('\n');
|
|
13
|
+
const verbose = result.results
|
|
14
|
+
.slice(0, 5)
|
|
15
|
+
.map((item, index) => `${index + 1}. ${item.title}\n ${item.url}\n ${item.snippet}`)
|
|
16
|
+
.join('\n');
|
|
17
|
+
return {
|
|
18
|
+
mode: 'compact',
|
|
19
|
+
views: {
|
|
20
|
+
compact: formatCompact(result),
|
|
21
|
+
preview: preview || undefined,
|
|
22
|
+
verbose: verbose || undefined
|
|
23
|
+
},
|
|
24
|
+
metrics: {
|
|
25
|
+
resultCount: result.results.length,
|
|
26
|
+
cacheHit: result.metadata.cacheHit
|
|
27
|
+
},
|
|
28
|
+
sources: result.results.map((item) => ({ title: item.title, url: item.url }))
|
|
29
|
+
};
|
|
30
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export function selectPresentationView(envelope, requestedMode) {
|
|
2
|
+
if (!envelope) {
|
|
3
|
+
return undefined;
|
|
4
|
+
}
|
|
5
|
+
if (requestedMode === 'preview' && envelope.views.preview) {
|
|
6
|
+
return envelope.views.preview;
|
|
7
|
+
}
|
|
8
|
+
if (requestedMode === 'verbose' && envelope.views.verbose) {
|
|
9
|
+
return envelope.views.verbose;
|
|
10
|
+
}
|
|
11
|
+
return envelope.views.compact;
|
|
12
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
export declare const PRESENTATION_MODES: readonly ["compact", "preview", "verbose"];
|
|
2
|
+
export type PresentationMode = (typeof PRESENTATION_MODES)[number];
|
|
3
|
+
export type PresentationViews = {
|
|
4
|
+
compact: string;
|
|
5
|
+
preview?: string;
|
|
6
|
+
verbose?: string;
|
|
7
|
+
};
|
|
8
|
+
export type PresentationMetrics = {
|
|
9
|
+
durationMs?: number;
|
|
10
|
+
resultCount?: number;
|
|
11
|
+
sourceCount?: number;
|
|
12
|
+
wordCount?: number;
|
|
13
|
+
statusCode?: number;
|
|
14
|
+
cacheHit?: boolean;
|
|
15
|
+
truncated?: boolean;
|
|
16
|
+
};
|
|
17
|
+
export type PresentationSource = {
|
|
18
|
+
title: string;
|
|
19
|
+
url: string;
|
|
20
|
+
domain?: string;
|
|
21
|
+
};
|
|
22
|
+
export type PresentationEnvelope = {
|
|
23
|
+
mode: PresentationMode;
|
|
24
|
+
views: PresentationViews;
|
|
25
|
+
metrics?: PresentationMetrics;
|
|
26
|
+
sources?: PresentationSource[];
|
|
27
|
+
debug?: Record<string, unknown>;
|
|
28
|
+
};
|
|
29
|
+
export type PresentationToolName = 'web_search' | 'web_fetch' | 'web_fetch_headless' | 'web_explore';
|
|
30
|
+
export type PresentationScope = 'global' | 'project';
|
|
31
|
+
export type PresentationToolOverrideMode = PresentationMode;
|
|
32
|
+
export type PresentationToolConfig = {
|
|
33
|
+
mode: PresentationToolOverrideMode;
|
|
34
|
+
};
|
|
35
|
+
export type PresentationConfig = {
|
|
36
|
+
defaultMode: PresentationMode;
|
|
37
|
+
tools: Partial<Record<PresentationToolName, PresentationToolConfig>>;
|
|
38
|
+
};
|
|
39
|
+
export type PresentationConfigOverride = {
|
|
40
|
+
defaultMode?: PresentationMode;
|
|
41
|
+
tools: Partial<Record<PresentationToolName, PresentationToolConfig>>;
|
|
42
|
+
};
|
|
43
|
+
export type PresentationConfigFile = {
|
|
44
|
+
presentation?: {
|
|
45
|
+
defaultMode?: unknown;
|
|
46
|
+
tools?: Partial<Record<PresentationToolName, {
|
|
47
|
+
mode?: unknown;
|
|
48
|
+
}>>;
|
|
49
|
+
};
|
|
50
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export const PRESENTATION_MODES = ['compact', 'preview', 'verbose'];
|
|
@@ -1,4 +1,9 @@
|
|
|
1
1
|
import type { SearchResult } from '../types.js';
|
|
2
|
+
export type ParsedDuckDuckGoResults = {
|
|
3
|
+
results: SearchResult[];
|
|
4
|
+
noResults: boolean;
|
|
5
|
+
hasResultContainers: boolean;
|
|
6
|
+
};
|
|
2
7
|
export declare function buildSearchUrl(query: string): string;
|
|
3
8
|
export declare function fetchDuckDuckGoHtml(query: string): Promise<string>;
|
|
4
|
-
export declare function parseDuckDuckGoResults(html: string):
|
|
9
|
+
export declare function parseDuckDuckGoResults(html: string): ParsedDuckDuckGoResults;
|
|
@@ -30,7 +30,8 @@ export async function fetchDuckDuckGoHtml(query) {
|
|
|
30
30
|
}
|
|
31
31
|
export function parseDuckDuckGoResults(html) {
|
|
32
32
|
const $ = cheerio.load(html);
|
|
33
|
-
|
|
33
|
+
const resultContainers = $('.result');
|
|
34
|
+
const results = resultContainers
|
|
34
35
|
.map((_, element) => {
|
|
35
36
|
const title = $(element).find('.result__a').first().text().trim();
|
|
36
37
|
const url = normalizeDuckDuckGoUrl($(element).find('.result__a').first().attr('href')?.trim() ?? '');
|
|
@@ -39,4 +40,13 @@ export function parseDuckDuckGoResults(html) {
|
|
|
39
40
|
})
|
|
40
41
|
.get()
|
|
41
42
|
.filter((value) => value !== null);
|
|
43
|
+
const text = $.text().toLowerCase();
|
|
44
|
+
const noResults = text.includes('no results found') ||
|
|
45
|
+
text.includes('no more results') ||
|
|
46
|
+
text.includes('did not match any documents');
|
|
47
|
+
return {
|
|
48
|
+
results,
|
|
49
|
+
noResults,
|
|
50
|
+
hasResultContainers: resultContainers.length > 0
|
|
51
|
+
};
|
|
42
52
|
}
|
|
@@ -22,23 +22,13 @@ export declare function createWebExploreTool({ explore }?: {
|
|
|
22
22
|
}): ({ query }: {
|
|
23
23
|
query: string;
|
|
24
24
|
}) => Promise<{
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
sources: never[];
|
|
28
|
-
error: {
|
|
29
|
-
code: string;
|
|
30
|
-
message: string;
|
|
31
|
-
};
|
|
32
|
-
caveat?: undefined;
|
|
33
|
-
text?: undefined;
|
|
34
|
-
} | {
|
|
35
|
-
status: "ok";
|
|
25
|
+
presentation: import("../presentation/types.js").PresentationEnvelope;
|
|
26
|
+
status: "ok" | "error";
|
|
36
27
|
findings: string[];
|
|
37
|
-
sources: {
|
|
28
|
+
sources: Array<{
|
|
38
29
|
title: string;
|
|
39
30
|
url: string;
|
|
40
|
-
}
|
|
41
|
-
caveat
|
|
42
|
-
|
|
43
|
-
error?: undefined;
|
|
31
|
+
}>;
|
|
32
|
+
caveat?: string;
|
|
33
|
+
error?: import("../types.js").ToolError;
|
|
44
34
|
}>;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createResearchWorkflow } from '../orchestration/index.js';
|
|
2
|
+
import { buildExplorePresentation } from '../presentation/explore-presentation.js';
|
|
2
3
|
function findingFromEvidence(evidence, index) {
|
|
3
4
|
if (evidence.summary.includes('Use channel')) {
|
|
4
5
|
return 'Use channel for branded Chrome or Edge when possible.';
|
|
@@ -12,23 +13,21 @@ function findingFromEvidence(evidence, index) {
|
|
|
12
13
|
}
|
|
13
14
|
return evidence.summary || `Finding ${index + 1}`;
|
|
14
15
|
}
|
|
15
|
-
function formatExploreText({ findings, sources, caveat }) {
|
|
16
|
-
const findingLines = findings.map((finding) => `- ${finding}`).join('\n');
|
|
17
|
-
const sourceLines = sources.map((source) => `- ${source.title}: ${source.url}`).join('\n');
|
|
18
|
-
const caveatBlock = caveat ? `\n\nCaveat\n${caveat}` : '';
|
|
19
|
-
return `Findings\n${findingLines}\n\nSources\n${sourceLines}${caveatBlock}`;
|
|
20
|
-
}
|
|
21
16
|
export function createWebExploreTool({ explore = createResearchWorkflow() } = {}) {
|
|
22
17
|
const runExplore = typeof explore === 'function' ? explore : explore.run.bind(explore);
|
|
23
18
|
return async function webExplore({ query }) {
|
|
24
19
|
const normalizedQuery = query.trim();
|
|
25
20
|
if (!normalizedQuery) {
|
|
26
|
-
|
|
21
|
+
const result = {
|
|
27
22
|
status: 'error',
|
|
28
23
|
findings: [],
|
|
29
24
|
sources: [],
|
|
30
25
|
error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
|
|
31
26
|
};
|
|
27
|
+
return {
|
|
28
|
+
...result,
|
|
29
|
+
presentation: buildExplorePresentation(result)
|
|
30
|
+
};
|
|
32
31
|
}
|
|
33
32
|
const result = await runExplore({ query: normalizedQuery });
|
|
34
33
|
const findings = result.evidence.slice(0, 5).map(findingFromEvidence);
|
|
@@ -39,12 +38,15 @@ export function createWebExploreTool({ explore = createResearchWorkflow() } = {}
|
|
|
39
38
|
const caveat = result.decision.action === 'answer'
|
|
40
39
|
? undefined
|
|
41
40
|
: 'Evidence is partial, so this answer is based on the strongest source found so far.';
|
|
42
|
-
|
|
41
|
+
const shaped = {
|
|
43
42
|
status: 'ok',
|
|
44
43
|
findings,
|
|
45
44
|
sources,
|
|
46
|
-
caveat
|
|
47
|
-
|
|
45
|
+
caveat
|
|
46
|
+
};
|
|
47
|
+
return {
|
|
48
|
+
...shaped,
|
|
49
|
+
presentation: buildExplorePresentation(shaped)
|
|
48
50
|
};
|
|
49
51
|
};
|
|
50
52
|
}
|
|
@@ -1,14 +1,23 @@
|
|
|
1
1
|
import { headlessFetch } from '../fetch/headless-fetch.js';
|
|
2
|
+
import { buildFetchPresentation } from '../presentation/fetch-presentation.js';
|
|
2
3
|
export function createWebFetchHeadlessTool({ fetchPage = headlessFetch } = {}) {
|
|
3
4
|
return async function webFetchHeadless({ url }) {
|
|
4
5
|
if (!/^https?:\/\//.test(url)) {
|
|
5
|
-
|
|
6
|
+
const result = {
|
|
6
7
|
status: 'unsupported',
|
|
7
8
|
url,
|
|
8
9
|
metadata: { method: 'headless', cacheHit: false },
|
|
9
10
|
error: { code: 'UNSUPPORTED_URL', message: 'Only http and https URLs are supported.' }
|
|
10
11
|
};
|
|
12
|
+
return {
|
|
13
|
+
...result,
|
|
14
|
+
presentation: buildFetchPresentation(result)
|
|
15
|
+
};
|
|
11
16
|
}
|
|
12
|
-
|
|
17
|
+
const result = await fetchPage(url);
|
|
18
|
+
return {
|
|
19
|
+
...result,
|
|
20
|
+
presentation: buildFetchPresentation(result)
|
|
21
|
+
};
|
|
13
22
|
};
|
|
14
23
|
}
|
package/dist/tools/web-fetch.js
CHANGED
|
@@ -1,14 +1,23 @@
|
|
|
1
1
|
import { createHttpFetcher } from '../fetch/http-fetch.js';
|
|
2
|
+
import { buildFetchPresentation } from '../presentation/fetch-presentation.js';
|
|
2
3
|
export function createWebFetchTool({ fetchPage = createHttpFetcher() } = {}) {
|
|
3
4
|
return async function webFetch({ url }) {
|
|
4
5
|
if (!/^https?:\/\//.test(url)) {
|
|
5
|
-
|
|
6
|
+
const result = {
|
|
6
7
|
status: 'unsupported',
|
|
7
8
|
url,
|
|
8
9
|
metadata: { method: 'http', cacheHit: false },
|
|
9
10
|
error: { code: 'UNSUPPORTED_URL', message: 'Only http and https URLs are supported.' }
|
|
10
11
|
};
|
|
12
|
+
return {
|
|
13
|
+
...result,
|
|
14
|
+
presentation: buildFetchPresentation(result)
|
|
15
|
+
};
|
|
11
16
|
}
|
|
12
|
-
|
|
17
|
+
const result = await fetchPage(url);
|
|
18
|
+
return {
|
|
19
|
+
...result,
|
|
20
|
+
presentation: buildFetchPresentation(result)
|
|
21
|
+
};
|
|
13
22
|
};
|
|
14
23
|
}
|
package/dist/tools/web-search.js
CHANGED
|
@@ -1,43 +1,130 @@
|
|
|
1
1
|
import { createCacheKey, createTtlCache } from '../cache/ttl-cache.js';
|
|
2
|
+
import { buildSearchPresentation } from '../presentation/search-presentation.js';
|
|
2
3
|
import { fetchDuckDuckGoHtml, parseDuckDuckGoResults } from '../search/duckduckgo.js';
|
|
4
|
+
function classifySearchFailure(error) {
|
|
5
|
+
const rawMessage = error instanceof Error ? error.message : 'Unknown search failure.';
|
|
6
|
+
const normalized = rawMessage.toLowerCase();
|
|
7
|
+
if (normalized.includes('blocked') ||
|
|
8
|
+
normalized.includes('rate limit') ||
|
|
9
|
+
normalized.includes('rate-limit') ||
|
|
10
|
+
normalized.includes('403') ||
|
|
11
|
+
normalized.includes('429') ||
|
|
12
|
+
normalized.includes('captcha') ||
|
|
13
|
+
normalized.includes('challenge')) {
|
|
14
|
+
return {
|
|
15
|
+
code: 'BLOCKED',
|
|
16
|
+
message: 'DuckDuckGo search appears to be blocked or rate limited.'
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
return {
|
|
20
|
+
code: 'FETCH_FAILED',
|
|
21
|
+
message: `DuckDuckGo search request failed: ${rawMessage}`
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
function htmlLooksBlocked(html) {
|
|
25
|
+
const normalized = html.toLowerCase();
|
|
26
|
+
return (normalized.includes('captcha') ||
|
|
27
|
+
normalized.includes('challenge') ||
|
|
28
|
+
normalized.includes('verify you are human') ||
|
|
29
|
+
normalized.includes('are you a robot') ||
|
|
30
|
+
normalized.includes('unusual traffic'));
|
|
31
|
+
}
|
|
3
32
|
export function createWebSearchTool({ searchHtml = fetchDuckDuckGoHtml, cache = createTtlCache({ ttlMs: 30_000 }) } = {}) {
|
|
4
33
|
return async function webSearch({ query }) {
|
|
5
34
|
const normalizedQuery = query.trim();
|
|
6
35
|
if (!normalizedQuery) {
|
|
7
|
-
|
|
36
|
+
const result = {
|
|
8
37
|
status: 'error',
|
|
9
38
|
results: [],
|
|
10
39
|
metadata: { backend: 'duckduckgo', cacheHit: false },
|
|
11
40
|
error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
|
|
12
41
|
};
|
|
42
|
+
return {
|
|
43
|
+
...result,
|
|
44
|
+
presentation: buildSearchPresentation(result)
|
|
45
|
+
};
|
|
13
46
|
}
|
|
14
47
|
const cacheKey = createCacheKey(['web_search', normalizedQuery]);
|
|
15
48
|
const cached = cache.get(cacheKey);
|
|
16
49
|
if (cached) {
|
|
17
|
-
|
|
50
|
+
const result = {
|
|
18
51
|
...cached,
|
|
19
52
|
metadata: { ...cached.metadata, cacheHit: true }
|
|
20
53
|
};
|
|
54
|
+
return {
|
|
55
|
+
...result,
|
|
56
|
+
presentation: buildSearchPresentation(result)
|
|
57
|
+
};
|
|
21
58
|
}
|
|
22
59
|
try {
|
|
23
60
|
const html = await searchHtml(normalizedQuery);
|
|
61
|
+
const parsed = parseDuckDuckGoResults(html);
|
|
62
|
+
if (parsed.results.length > 0) {
|
|
63
|
+
const result = {
|
|
64
|
+
status: 'ok',
|
|
65
|
+
results: parsed.results,
|
|
66
|
+
metadata: { backend: 'duckduckgo', cacheHit: false }
|
|
67
|
+
};
|
|
68
|
+
cache.set(cacheKey, result);
|
|
69
|
+
return {
|
|
70
|
+
...result,
|
|
71
|
+
presentation: buildSearchPresentation(result)
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
if (parsed.noResults) {
|
|
75
|
+
const result = {
|
|
76
|
+
status: 'error',
|
|
77
|
+
results: [],
|
|
78
|
+
metadata: { backend: 'duckduckgo', cacheHit: false },
|
|
79
|
+
error: {
|
|
80
|
+
code: 'NO_RESULTS',
|
|
81
|
+
message: 'DuckDuckGo returned no usable results for this query.'
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
return {
|
|
85
|
+
...result,
|
|
86
|
+
presentation: buildSearchPresentation(result)
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
if (htmlLooksBlocked(html)) {
|
|
90
|
+
const result = {
|
|
91
|
+
status: 'error',
|
|
92
|
+
results: [],
|
|
93
|
+
metadata: { backend: 'duckduckgo', cacheHit: false },
|
|
94
|
+
error: {
|
|
95
|
+
code: 'BLOCKED',
|
|
96
|
+
message: 'DuckDuckGo search appears to be blocked or rate limited.'
|
|
97
|
+
}
|
|
98
|
+
};
|
|
99
|
+
return {
|
|
100
|
+
...result,
|
|
101
|
+
presentation: buildSearchPresentation(result)
|
|
102
|
+
};
|
|
103
|
+
}
|
|
24
104
|
const result = {
|
|
25
|
-
status: '
|
|
26
|
-
results:
|
|
27
|
-
metadata: { backend: 'duckduckgo', cacheHit: false }
|
|
105
|
+
status: 'error',
|
|
106
|
+
results: [],
|
|
107
|
+
metadata: { backend: 'duckduckgo', cacheHit: false },
|
|
108
|
+
error: {
|
|
109
|
+
code: 'PARSE_FAILED',
|
|
110
|
+
message: 'DuckDuckGo returned a page, but it did not match the expected results format.'
|
|
111
|
+
}
|
|
112
|
+
};
|
|
113
|
+
return {
|
|
114
|
+
...result,
|
|
115
|
+
presentation: buildSearchPresentation(result)
|
|
28
116
|
};
|
|
29
|
-
cache.set(cacheKey, result);
|
|
30
|
-
return result;
|
|
31
117
|
}
|
|
32
118
|
catch (error) {
|
|
33
|
-
|
|
119
|
+
const result = {
|
|
34
120
|
status: 'error',
|
|
35
121
|
results: [],
|
|
36
122
|
metadata: { backend: 'duckduckgo', cacheHit: false },
|
|
37
|
-
error:
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
123
|
+
error: classifySearchFailure(error)
|
|
124
|
+
};
|
|
125
|
+
return {
|
|
126
|
+
...result,
|
|
127
|
+
presentation: buildSearchPresentation(result)
|
|
41
128
|
};
|
|
42
129
|
}
|
|
43
130
|
};
|
package/dist/types.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { PresentationEnvelope } from './presentation/types.js';
|
|
1
2
|
export declare const TOOL_STATUSES: readonly ["ok", "needs_headless", "blocked", "unsupported", "error"];
|
|
2
3
|
export type ToolStatus = (typeof TOOL_STATUSES)[number];
|
|
3
4
|
export type SearchResult = {
|
|
@@ -30,6 +31,7 @@ export type WebSearchResponse = {
|
|
|
30
31
|
status: 'ok' | 'error';
|
|
31
32
|
results: SearchResult[];
|
|
32
33
|
metadata: SearchMetadata;
|
|
34
|
+
presentation?: PresentationEnvelope;
|
|
33
35
|
error?: ToolError;
|
|
34
36
|
};
|
|
35
37
|
export type WebFetchResponse = {
|
|
@@ -37,6 +39,7 @@ export type WebFetchResponse = {
|
|
|
37
39
|
url: string;
|
|
38
40
|
content?: ExtractedContent;
|
|
39
41
|
metadata: FetchMetadata;
|
|
42
|
+
presentation?: PresentationEnvelope;
|
|
40
43
|
error?: ToolError;
|
|
41
44
|
};
|
|
42
45
|
export type WebFetchHeadlessResponse = {
|
|
@@ -44,5 +47,17 @@ export type WebFetchHeadlessResponse = {
|
|
|
44
47
|
url: string;
|
|
45
48
|
content?: ExtractedContent;
|
|
46
49
|
metadata: FetchMetadata;
|
|
50
|
+
presentation?: PresentationEnvelope;
|
|
51
|
+
error?: ToolError;
|
|
52
|
+
};
|
|
53
|
+
export type WebExploreResponse = {
|
|
54
|
+
status: 'ok' | 'error';
|
|
55
|
+
findings: string[];
|
|
56
|
+
sources: Array<{
|
|
57
|
+
title: string;
|
|
58
|
+
url: string;
|
|
59
|
+
}>;
|
|
60
|
+
caveat?: string;
|
|
61
|
+
presentation?: PresentationEnvelope;
|
|
47
62
|
error?: ToolError;
|
|
48
63
|
};
|