@fffangchenghao/dsh-web-search 0.19.2 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +160 -0
- package/README.md +195 -34
- package/README.zh-CN.md +171 -34
- package/SECURITY.md +42 -7
- package/client.js +131 -9
- package/docs/store-evidence.md +10 -5
- package/lib/advanced-search.d.ts +107 -0
- package/lib/advanced-search.js +255 -0
- package/lib/bridge.d.ts +16 -0
- package/lib/bridge.js +160 -0
- package/lib/cache.d.ts +11 -4
- package/lib/cache.js +9 -5
- package/lib/config.d.ts +19 -3
- package/lib/config.js +24 -0
- package/lib/domain-tool.d.ts +5 -0
- package/lib/domain-tool.js +45 -6
- package/lib/engine-test.d.ts +116 -0
- package/lib/engine-test.js +256 -0
- package/lib/engines.d.ts +11 -1
- package/lib/engines.js +122 -20
- package/lib/errors.d.ts +16 -2
- package/lib/errors.js +40 -1
- package/lib/index.d.ts +24 -2
- package/lib/index.js +304 -20
- package/lib/inject.d.ts +43 -0
- package/lib/inject.js +1 -0
- package/lib/markdown.js +18 -1
- package/lib/mirrors.d.ts +23 -0
- package/lib/mirrors.js +69 -0
- package/lib/official-feed.d.ts +114 -0
- package/lib/official-feed.js +288 -0
- package/lib/official-support.d.ts +74 -0
- package/lib/official-support.js +134 -0
- package/lib/platform-tool.d.ts +77 -0
- package/lib/platform-tool.js +173 -0
- package/lib/platforms.d.ts +178 -0
- package/lib/platforms.js +1035 -0
- package/lib/query-router.d.ts +29 -0
- package/lib/query-router.js +58 -0
- package/lib/read-page.js +2 -1
- package/lib/research-cache.d.ts +115 -0
- package/lib/research-cache.js +275 -0
- package/lib/research.d.ts +45 -4
- package/lib/research.js +505 -22
- package/lib/search-many.d.ts +7 -1
- package/lib/search-many.js +7 -2
- package/lib/system-prompt.d.ts +27 -0
- package/lib/system-prompt.js +50 -0
- package/lib/tool-suite.d.ts +97 -0
- package/lib/tool-suite.js +224 -0
- package/lib/transport.d.ts +33 -2
- package/lib/transport.js +42 -9
- package/lib/types.d.ts +20 -0
- package/lib/types.js +9 -1
- package/lib/untrusted.d.ts +33 -0
- package/lib/untrusted.js +42 -0
- package/lib/verticals.d.ts +3 -1
- package/lib/verticals.js +26 -7
- package/package.json +5 -3
- package/skills/dsh-web-search/SKILL.md +21 -6
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Detect the narrow, unambiguous case where a `search_many` query names a
|
|
3
|
+
* package registry. These queries answer far better from the registry itself
|
|
4
|
+
* than from general web results, but routing every query that contains a short
|
|
5
|
+
* English word would be worse than doing nothing. Only an explicit registry
|
|
6
|
+
* name plus package intent is routed.
|
|
7
|
+
*
|
|
8
|
+
* @module @fangchenghao/dsh-web-search/query-router
|
|
9
|
+
*/
|
|
10
|
+
import type { VerticalId } from './verticals.ts';
|
|
11
|
+
/** A package query that can be answered by one existing vertical source. */
|
|
12
|
+
export interface PackageQueryRoute {
|
|
13
|
+
/** Existing `domain_search` source id. */
|
|
14
|
+
id: Extract<VerticalId, 'crates' | 'npm' | 'pypi'>;
|
|
15
|
+
/** Query text with the registry name and release filler removed. */
|
|
16
|
+
query: string;
|
|
17
|
+
/** Package-name-shaped tokens extracted from the query. */
|
|
18
|
+
names: string[];
|
|
19
|
+
/** Human-readable source name for the result note. */
|
|
20
|
+
label: string;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Route an explicitly registry-scoped package query.
|
|
24
|
+
*
|
|
25
|
+
* @param query - one raw `search_many` query.
|
|
26
|
+
* @returns the existing source id and a cleaned query, or `undefined` when the
|
|
27
|
+
* query is not unambiguously about a package registry.
|
|
28
|
+
*/
|
|
29
|
+
export declare function routePackageQuery(query: string): PackageQueryRoute | undefined;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/** Words that signal package intent even when no version number is present. */
|
|
2
|
+
const PACKAGE_INTENT = /(?:package|packages|library|dependency|dependencies|crate|crates|module|版本|最新|依赖|包|库|发布)/iu;
|
|
3
|
+
/** Latin package-name-shaped tokens, used to clean Chinese filler words. */
|
|
4
|
+
const PACKAGE_TOKEN = /[A-Za-z][A-Za-z0-9_.-]{0,}/gu;
|
|
5
|
+
/** Filler that should never be sent to a registry search. */
|
|
6
|
+
const STOP_WORDS = new Set([
|
|
7
|
+
'a', 'an', 'and', 'are', 'at', 'by', 'compare', 'dependency', 'dependencies', 'do',
|
|
8
|
+
'does', 'for', 'from', 'how', 'in', 'install', 'is', 'latest', 'library', 'module',
|
|
9
|
+
'newest', 'of', 'on', 'or', 'package', 'packages', 'release', 'the', 'to', 'version',
|
|
10
|
+
'versions', 'vs', 'what', 'which', 'with',
|
|
11
|
+
]);
|
|
12
|
+
/**
|
|
13
|
+
* Route an explicitly registry-scoped package query.
|
|
14
|
+
*
|
|
15
|
+
* @param query - one raw `search_many` query.
|
|
16
|
+
* @returns the existing source id and a cleaned query, or `undefined` when the
|
|
17
|
+
* query is not unambiguously about a package registry.
|
|
18
|
+
*/
|
|
19
|
+
export function routePackageQuery(query) {
|
|
20
|
+
if (typeof query !== 'string')
|
|
21
|
+
return undefined;
|
|
22
|
+
const value = query.trim();
|
|
23
|
+
if (value.length === 0)
|
|
24
|
+
return undefined;
|
|
25
|
+
let id;
|
|
26
|
+
let label = '';
|
|
27
|
+
if (/\bcrates?\.io\b/iu.test(value)) {
|
|
28
|
+
id = 'crates';
|
|
29
|
+
label = 'crates.io';
|
|
30
|
+
}
|
|
31
|
+
else if (/\bnpm(?:js)?\b/iu.test(value)) {
|
|
32
|
+
id = 'npm';
|
|
33
|
+
label = 'npm';
|
|
34
|
+
}
|
|
35
|
+
else if (/\bpypi\b/iu.test(value)) {
|
|
36
|
+
id = 'pypi';
|
|
37
|
+
label = 'PyPI';
|
|
38
|
+
}
|
|
39
|
+
if (id === undefined || !PACKAGE_INTENT.test(value))
|
|
40
|
+
return undefined;
|
|
41
|
+
const cleaned = value
|
|
42
|
+
.replace(/\b(?:crates?\.io|npm(?:js)?|pypi)\b/giu, ' ')
|
|
43
|
+
.replace(/\b(?:latest|newest)\s+(?:version|release)\b/giu, ' ')
|
|
44
|
+
.replace(/最新版本|最新版|最新|版本号|版本|发布/gu, ' ')
|
|
45
|
+
.replace(/\s+/gu, ' ')
|
|
46
|
+
.trim();
|
|
47
|
+
// Keep Latin package names when the surrounding sentence is Chinese; the
|
|
48
|
+
// registry APIs are much better at `serde tokio` than at `上 serde 和 tokio`.
|
|
49
|
+
const tokens = cleaned.match(PACKAGE_TOKEN) ?? [];
|
|
50
|
+
const names = tokens.filter((token) => {
|
|
51
|
+
const lower = token.toLowerCase();
|
|
52
|
+
return !STOP_WORDS.has(lower) && !lower.includes('crates.io');
|
|
53
|
+
});
|
|
54
|
+
const routedQuery = names.length > 0 ? names.join(' ') : cleaned;
|
|
55
|
+
if (routedQuery.length === 0)
|
|
56
|
+
return undefined;
|
|
57
|
+
return { id, query: routedQuery, names: names.slice(0, 5), label };
|
|
58
|
+
}
|
package/lib/read-page.js
CHANGED
|
@@ -40,6 +40,7 @@
|
|
|
40
40
|
import { createFetchProvider } from "./fetch.js";
|
|
41
41
|
import { DEFAULT_FETCH_MAX_LENGTH, renderFetchText } from "./markdown.js";
|
|
42
42
|
import { DEFAULT_RENDER_TIMEOUT_MS } from "./render.js";
|
|
43
|
+
import { wrapUntrustedContent } from "./untrusted.js";
|
|
43
44
|
/** The name this tool prefers. */
|
|
44
45
|
export const READ_PAGE_TOOL_NAME = 'read_page';
|
|
45
46
|
/** The name used when another plugin already registered `read_page`. */
|
|
@@ -383,7 +384,7 @@ export function createReadPageDefinition(name, readSettings, options = {}) {
|
|
|
383
384
|
parameters: parameterSchema(),
|
|
384
385
|
output: {
|
|
385
386
|
schema: outputSchema(),
|
|
386
|
-
render: (_args, value) => [{ type: 'text', text: formatReadPageValue(value) }],
|
|
387
|
+
render: (_args, value) => [{ type: 'text', text: wrapUntrustedContent(formatReadPageValue(value)) }],
|
|
387
388
|
},
|
|
388
389
|
// A single fetch may try direct, tunnel, a browser and extraction in turn.
|
|
389
390
|
timeoutMs: 60_000,
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import type { ToolContent, ToolJsonSchema } from './domain-tool.ts';
|
|
2
|
+
/** The name this tool prefers. */
|
|
3
|
+
export declare const CACHE_SEARCH_TOOL_NAME = "cache_search";
|
|
4
|
+
/** The name used when another plugin already registered `cache_search`. */
|
|
5
|
+
export declare const CACHE_SEARCH_FALLBACK_NAME = "web_cache_search";
|
|
6
|
+
/** How long a fetched body stays searchable. */
|
|
7
|
+
export declare const CACHE_SEARCH_TTL_MS: number;
|
|
8
|
+
/** Bounded memory: enough for a normal research session, not a crawler. */
|
|
9
|
+
export declare const CACHE_SEARCH_MAX_BODIES = 24;
|
|
10
|
+
/** Per-body storage cap, in characters. */
|
|
11
|
+
export declare const CACHE_SEARCH_MAX_CHARS = 400000;
|
|
12
|
+
/** Snippet budget returned for one matching body. */
|
|
13
|
+
export declare const CACHE_SEARCH_SNIPPET_CHARS = 1600;
|
|
14
|
+
/** Default and maximum result count for one query. */
|
|
15
|
+
export declare const CACHE_SEARCH_DEFAULT_LIMIT = 3;
|
|
16
|
+
export declare const CACHE_SEARCH_MAX_LIMIT = 10;
|
|
17
|
+
/** One body stored after a successful research fetch. */
|
|
18
|
+
export interface CachedResearchBody {
|
|
19
|
+
url: string;
|
|
20
|
+
title?: string;
|
|
21
|
+
text: string;
|
|
22
|
+
fetchedAt: number;
|
|
23
|
+
queries: string[];
|
|
24
|
+
}
|
|
25
|
+
/** One matching body and the passage that matched. */
|
|
26
|
+
export interface CachedResearchMatch {
|
|
27
|
+
url: string;
|
|
28
|
+
title?: string;
|
|
29
|
+
snippet: string;
|
|
30
|
+
score: number;
|
|
31
|
+
fetchedAt: number;
|
|
32
|
+
}
|
|
33
|
+
/** What one cache search returns. */
|
|
34
|
+
export interface CacheSearchValue {
|
|
35
|
+
query: string;
|
|
36
|
+
matches: CachedResearchMatch[];
|
|
37
|
+
failures: string[];
|
|
38
|
+
}
|
|
39
|
+
/** A registry definition in the host's raw JSON-Schema dialect. */
|
|
40
|
+
export interface CacheSearchDefinition {
|
|
41
|
+
name: string;
|
|
42
|
+
description: string;
|
|
43
|
+
parameters: ToolJsonSchema;
|
|
44
|
+
output: {
|
|
45
|
+
schema: ToolJsonSchema;
|
|
46
|
+
render: (args: unknown, value: CacheSearchValue) => ToolContent[];
|
|
47
|
+
};
|
|
48
|
+
timeoutMs?: number;
|
|
49
|
+
isConcurrencySafe?: () => boolean;
|
|
50
|
+
execute: (args: unknown, exec: {
|
|
51
|
+
signal?: AbortSignal;
|
|
52
|
+
}) => Promise<CacheSearchValue>;
|
|
53
|
+
}
|
|
54
|
+
/** The slice of the tool registry this module needs. */
|
|
55
|
+
export interface CacheSearchRegistryLike {
|
|
56
|
+
register(definition: CacheSearchDefinition): unknown;
|
|
57
|
+
get?(name: string, scope?: unknown): unknown;
|
|
58
|
+
}
|
|
59
|
+
/** Test seams. */
|
|
60
|
+
export interface ResearchBodyCacheOptions {
|
|
61
|
+
ttlMs?: number;
|
|
62
|
+
maxBodies?: number;
|
|
63
|
+
maxChars?: number;
|
|
64
|
+
now?: () => number;
|
|
65
|
+
}
|
|
66
|
+
/** Lowercase search terms; CJK runs become bigrams, matching research's lexical gate. */
|
|
67
|
+
export declare function cacheTokens(query: string): string[];
|
|
68
|
+
/** Split a page into paragraph windows small enough to quote. */
|
|
69
|
+
export declare function cacheWindows(text: string, size?: number): string[];
|
|
70
|
+
/** In-memory, bounded, expiring cache of research bodies. */
|
|
71
|
+
export declare class ResearchBodyCache {
|
|
72
|
+
private readonly entries;
|
|
73
|
+
private readonly ttlMs;
|
|
74
|
+
private readonly maxBodies;
|
|
75
|
+
private readonly maxChars;
|
|
76
|
+
private readonly now;
|
|
77
|
+
constructor(options?: ResearchBodyCacheOptions);
|
|
78
|
+
/** Store one successful research body, replacing an older copy of the URL. */
|
|
79
|
+
put(body: {
|
|
80
|
+
url: string;
|
|
81
|
+
title?: string;
|
|
82
|
+
text: string;
|
|
83
|
+
queries?: readonly string[];
|
|
84
|
+
}): void;
|
|
85
|
+
/** Number of live entries, after dropping expired rows. */
|
|
86
|
+
size(): number;
|
|
87
|
+
/** Drop expired entries; called by both `put` and `search`. */
|
|
88
|
+
private evict;
|
|
89
|
+
/**
|
|
90
|
+
* Search all cached bodies lexically and return the best passage per body.
|
|
91
|
+
* @param query - what to look for.
|
|
92
|
+
* @param limit - maximum matches, clamped to 1...10.
|
|
93
|
+
* @returns matches ordered by score, then recency.
|
|
94
|
+
*/
|
|
95
|
+
search(query: string, limit?: number): CachedResearchMatch[];
|
|
96
|
+
/** Remove every entry; used by tests and disposal. */
|
|
97
|
+
clear(): void;
|
|
98
|
+
}
|
|
99
|
+
/** Render results for the model, inside the untrusted-content boundary. */
|
|
100
|
+
export declare function formatCacheSearchValue(value: CacheSearchValue): string;
|
|
101
|
+
/**
|
|
102
|
+
* Build the `cache_search` definition.
|
|
103
|
+
* @param name - preferred or fallback tool name.
|
|
104
|
+
* @param cache - live cache populated by research.
|
|
105
|
+
*/
|
|
106
|
+
export declare function createCacheSearchDefinition(name: string, cache: ResearchBodyCache): CacheSearchDefinition;
|
|
107
|
+
/** Pick the preferred name unless another tool already owns it. */
|
|
108
|
+
export declare function pickCacheSearchToolName(tools: CacheSearchRegistryLike): string;
|
|
109
|
+
/**
|
|
110
|
+
* Register the cache-search tool; every failure is contained so the rest of the
|
|
111
|
+
* plugin keeps mounting.
|
|
112
|
+
*/
|
|
113
|
+
export declare function registerCacheSearchTool(tools: CacheSearchRegistryLike, cache: ResearchBodyCache, options?: {
|
|
114
|
+
warn?: (message: string) => void;
|
|
115
|
+
}): string | undefined;
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
import { wrapUntrustedContent } from "./untrusted.js";
|
|
2
|
+
/** The name this tool prefers. */
|
|
3
|
+
export const CACHE_SEARCH_TOOL_NAME = 'cache_search';
|
|
4
|
+
/** The name used when another plugin already registered `cache_search`. */
|
|
5
|
+
export const CACHE_SEARCH_FALLBACK_NAME = 'web_cache_search';
|
|
6
|
+
/** How long a fetched body stays searchable. */
|
|
7
|
+
export const CACHE_SEARCH_TTL_MS = 7 * 24 * 60 * 60 * 1_000;
|
|
8
|
+
/** Bounded memory: enough for a normal research session, not a crawler. */
|
|
9
|
+
export const CACHE_SEARCH_MAX_BODIES = 24;
|
|
10
|
+
/** Per-body storage cap, in characters. */
|
|
11
|
+
export const CACHE_SEARCH_MAX_CHARS = 400_000;
|
|
12
|
+
/** Snippet budget returned for one matching body. */
|
|
13
|
+
export const CACHE_SEARCH_SNIPPET_CHARS = 1_600;
|
|
14
|
+
/** Default and maximum result count for one query. */
|
|
15
|
+
export const CACHE_SEARCH_DEFAULT_LIMIT = 3;
|
|
16
|
+
export const CACHE_SEARCH_MAX_LIMIT = 10;
|
|
17
|
+
/** True for a value usable as a record. */
|
|
18
|
+
function isRecord(value) {
|
|
19
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
20
|
+
}
|
|
21
|
+
/** Lowercase search terms; CJK runs become bigrams, matching research's lexical gate. */
|
|
22
|
+
export function cacheTokens(query) {
|
|
23
|
+
const text = query.toLowerCase();
|
|
24
|
+
const tokens = new Set();
|
|
25
|
+
for (const word of text.match(/[a-z][a-z0-9.+#-]{2,}/gu) ?? [])
|
|
26
|
+
tokens.add(word);
|
|
27
|
+
for (const run of text.match(/[\u4e00-\u9fff]{2,}/gu) ?? []) {
|
|
28
|
+
if (run.length === 2) {
|
|
29
|
+
tokens.add(run);
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
for (let index = 0; index < run.length - 1; index += 1)
|
|
33
|
+
tokens.add(run.slice(index, index + 2));
|
|
34
|
+
}
|
|
35
|
+
return [...tokens];
|
|
36
|
+
}
|
|
37
|
+
/** Split a page into paragraph windows small enough to quote. */
|
|
38
|
+
export function cacheWindows(text, size = 700) {
|
|
39
|
+
const blocks = text.replace(/\r/g, '').replace(/[ \t]+/g, ' ').split(/\n{2,}/u)
|
|
40
|
+
.map((block) => block.trim()).filter((block) => block.length > 0);
|
|
41
|
+
const windows = [];
|
|
42
|
+
for (const block of blocks) {
|
|
43
|
+
if (block.length <= size) {
|
|
44
|
+
windows.push(block);
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
for (let offset = 0; offset < block.length; offset += size)
|
|
48
|
+
windows.push(block.slice(offset, offset + size));
|
|
49
|
+
}
|
|
50
|
+
return windows;
|
|
51
|
+
}
|
|
52
|
+
/** Score one window by distinct query-term hits and a small repetition bonus. */
|
|
53
|
+
function scoreWindow(window, tokens) {
|
|
54
|
+
const haystack = window.toLowerCase();
|
|
55
|
+
let score = 0;
|
|
56
|
+
for (const token of tokens) {
|
|
57
|
+
if (!haystack.includes(token))
|
|
58
|
+
continue;
|
|
59
|
+
score += 1;
|
|
60
|
+
let at = haystack.indexOf(token) + token.length;
|
|
61
|
+
while (at < haystack.length) {
|
|
62
|
+
const next = haystack.indexOf(token, at);
|
|
63
|
+
if (next < 0)
|
|
64
|
+
break;
|
|
65
|
+
score += 0.1;
|
|
66
|
+
at = next + token.length;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return score;
|
|
70
|
+
}
|
|
71
|
+
/** In-memory, bounded, expiring cache of research bodies. */
|
|
72
|
+
export class ResearchBodyCache {
|
|
73
|
+
entries = new Map();
|
|
74
|
+
ttlMs;
|
|
75
|
+
maxBodies;
|
|
76
|
+
maxChars;
|
|
77
|
+
now;
|
|
78
|
+
constructor(options = {}) {
|
|
79
|
+
this.ttlMs = options.ttlMs ?? CACHE_SEARCH_TTL_MS;
|
|
80
|
+
this.maxBodies = options.maxBodies ?? CACHE_SEARCH_MAX_BODIES;
|
|
81
|
+
this.maxChars = options.maxChars ?? CACHE_SEARCH_MAX_CHARS;
|
|
82
|
+
this.now = options.now ?? Date.now;
|
|
83
|
+
}
|
|
84
|
+
/** Store one successful research body, replacing an older copy of the URL. */
|
|
85
|
+
put(body) {
|
|
86
|
+
if (body.text.trim().length === 0)
|
|
87
|
+
return;
|
|
88
|
+
const text = body.text.length > this.maxChars ? body.text.slice(0, this.maxChars) : body.text;
|
|
89
|
+
const entry = {
|
|
90
|
+
url: body.url,
|
|
91
|
+
...(body.title === undefined || body.title.length === 0 ? {} : { title: body.title }),
|
|
92
|
+
text,
|
|
93
|
+
fetchedAt: this.now(),
|
|
94
|
+
queries: [...(body.queries ?? [])],
|
|
95
|
+
};
|
|
96
|
+
this.entries.delete(body.url);
|
|
97
|
+
this.entries.set(body.url, entry);
|
|
98
|
+
while (this.entries.size > this.maxBodies) {
|
|
99
|
+
const oldest = this.entries.keys().next().value;
|
|
100
|
+
if (oldest === undefined)
|
|
101
|
+
break;
|
|
102
|
+
this.entries.delete(oldest);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
/** Number of live entries, after dropping expired rows. */
|
|
106
|
+
size() {
|
|
107
|
+
this.evict();
|
|
108
|
+
return this.entries.size;
|
|
109
|
+
}
|
|
110
|
+
/** Drop expired entries; called by both `put` and `search`. */
|
|
111
|
+
evict() {
|
|
112
|
+
const cutoff = this.now() - this.ttlMs;
|
|
113
|
+
for (const [url, entry] of this.entries) {
|
|
114
|
+
if (entry.fetchedAt < cutoff)
|
|
115
|
+
this.entries.delete(url);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Search all cached bodies lexically and return the best passage per body.
|
|
120
|
+
* @param query - what to look for.
|
|
121
|
+
* @param limit - maximum matches, clamped to 1...10.
|
|
122
|
+
* @returns matches ordered by score, then recency.
|
|
123
|
+
*/
|
|
124
|
+
search(query, limit = CACHE_SEARCH_DEFAULT_LIMIT) {
|
|
125
|
+
this.evict();
|
|
126
|
+
const tokens = cacheTokens(query);
|
|
127
|
+
if (tokens.length === 0)
|
|
128
|
+
return [];
|
|
129
|
+
const matches = [];
|
|
130
|
+
for (const entry of this.entries.values()) {
|
|
131
|
+
const windows = cacheWindows(entry.text);
|
|
132
|
+
const best = windows
|
|
133
|
+
.map((window, index) => ({ window, index, score: scoreWindow(window, tokens) }))
|
|
134
|
+
.filter((row) => row.score > 0)
|
|
135
|
+
.sort((a, b) => b.score - a.score || a.index - b.index)
|
|
136
|
+
.slice(0, 2);
|
|
137
|
+
if (best.length === 0)
|
|
138
|
+
continue;
|
|
139
|
+
const snippet = best.map((row) => row.window).join('\n\n…\n\n').slice(0, CACHE_SEARCH_SNIPPET_CHARS);
|
|
140
|
+
matches.push({
|
|
141
|
+
url: entry.url,
|
|
142
|
+
...(entry.title === undefined ? {} : { title: entry.title }),
|
|
143
|
+
snippet,
|
|
144
|
+
score: best[0]?.score ?? 0,
|
|
145
|
+
fetchedAt: entry.fetchedAt,
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
return matches.sort((a, b) => b.score - a.score || b.fetchedAt - a.fetchedAt)
|
|
149
|
+
.slice(0, Math.max(1, Math.min(CACHE_SEARCH_MAX_LIMIT, Math.floor(limit))));
|
|
150
|
+
}
|
|
151
|
+
/** Remove every entry; used by tests and disposal. */
|
|
152
|
+
clear() {
|
|
153
|
+
this.entries.clear();
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
/** Render results for the model, inside the untrusted-content boundary. */
|
|
157
|
+
export function formatCacheSearchValue(value) {
|
|
158
|
+
const head = `Research cache search: ${JSON.stringify(value.query)} — ${value.matches.length} match(es).`;
|
|
159
|
+
const lines = value.matches.map((match, index) => [
|
|
160
|
+
`${index + 1}. ${match.title ?? match.url}`,
|
|
161
|
+
` ${match.url}`,
|
|
162
|
+
` ${match.snippet}`,
|
|
163
|
+
].join('\n'));
|
|
164
|
+
if (value.matches.length === 0)
|
|
165
|
+
lines.push('No matching passage in the bodies fetched by research in this process.');
|
|
166
|
+
return [head, '', ...lines].join('\n').trimEnd();
|
|
167
|
+
}
|
|
168
|
+
/** Parameter schema. */
|
|
169
|
+
function parameterSchema() {
|
|
170
|
+
return {
|
|
171
|
+
type: 'object',
|
|
172
|
+
additionalProperties: false,
|
|
173
|
+
properties: {
|
|
174
|
+
query: { type: 'string', description: 'Words to find in the full texts already fetched by research.' },
|
|
175
|
+
limit: {
|
|
176
|
+
type: 'integer',
|
|
177
|
+
description: `Matches wanted, 1–${CACHE_SEARCH_MAX_LIMIT}; defaults to ${CACHE_SEARCH_DEFAULT_LIMIT}.`,
|
|
178
|
+
},
|
|
179
|
+
},
|
|
180
|
+
required: ['query'],
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
/** Output schema. */
|
|
184
|
+
function outputSchema() {
|
|
185
|
+
return {
|
|
186
|
+
type: 'object',
|
|
187
|
+
additionalProperties: false,
|
|
188
|
+
properties: {
|
|
189
|
+
query: { type: 'string' },
|
|
190
|
+
matches: {
|
|
191
|
+
type: 'array',
|
|
192
|
+
items: {
|
|
193
|
+
type: 'object',
|
|
194
|
+
additionalProperties: false,
|
|
195
|
+
properties: {
|
|
196
|
+
url: { type: 'string' },
|
|
197
|
+
title: { type: 'string' },
|
|
198
|
+
snippet: { type: 'string' },
|
|
199
|
+
score: { type: 'number' },
|
|
200
|
+
fetchedAt: { type: 'number' },
|
|
201
|
+
},
|
|
202
|
+
required: ['url', 'snippet', 'score', 'fetchedAt'],
|
|
203
|
+
},
|
|
204
|
+
},
|
|
205
|
+
failures: { type: 'array', items: { type: 'string' } },
|
|
206
|
+
},
|
|
207
|
+
required: ['query', 'matches', 'failures'],
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
/** Parse and validate arguments. */
|
|
211
|
+
function parseArgs(args) {
|
|
212
|
+
if (!isRecord(args) || typeof args['query'] !== 'string' || args['query'].trim().length === 0) {
|
|
213
|
+
throw new Error('cache_search requires a non-empty query');
|
|
214
|
+
}
|
|
215
|
+
const raw = args['limit'];
|
|
216
|
+
if (raw !== undefined && (typeof raw !== 'number' || !Number.isInteger(raw) || raw < 1 || raw > CACHE_SEARCH_MAX_LIMIT)) {
|
|
217
|
+
throw new Error(`cache_search limit must be an integer between 1 and ${CACHE_SEARCH_MAX_LIMIT}`);
|
|
218
|
+
}
|
|
219
|
+
return { query: args['query'].trim(), limit: raw ?? CACHE_SEARCH_DEFAULT_LIMIT };
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Build the `cache_search` definition.
|
|
223
|
+
* @param name - preferred or fallback tool name.
|
|
224
|
+
* @param cache - live cache populated by research.
|
|
225
|
+
*/
|
|
226
|
+
export function createCacheSearchDefinition(name, cache) {
|
|
227
|
+
return {
|
|
228
|
+
name,
|
|
229
|
+
description: 'Search the full texts that the research tool already fetched in this process, without fetching them again. Use it when a research answer needs an exact detail, quote or section and the evidence pack only shows an excerpt. Returns matching passages with their source URLs.',
|
|
230
|
+
parameters: parameterSchema(),
|
|
231
|
+
output: {
|
|
232
|
+
schema: outputSchema(),
|
|
233
|
+
render: (_args, value) => [{ type: 'text', text: wrapUntrustedContent(formatCacheSearchValue(value)) }],
|
|
234
|
+
},
|
|
235
|
+
timeoutMs: 10_000,
|
|
236
|
+
isConcurrencySafe: () => true,
|
|
237
|
+
async execute(args, exec) {
|
|
238
|
+
const parsed = parseArgs(args);
|
|
239
|
+
if (exec.signal?.aborted === true)
|
|
240
|
+
throw new Error('cache_search was cancelled');
|
|
241
|
+
return { query: parsed.query, matches: cache.search(parsed.query, parsed.limit), failures: [] };
|
|
242
|
+
},
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
/** Pick the preferred name unless another tool already owns it. */
|
|
246
|
+
export function pickCacheSearchToolName(tools) {
|
|
247
|
+
if (typeof tools.get !== 'function')
|
|
248
|
+
return CACHE_SEARCH_TOOL_NAME;
|
|
249
|
+
try {
|
|
250
|
+
return tools.get(CACHE_SEARCH_TOOL_NAME) === undefined ? CACHE_SEARCH_TOOL_NAME : CACHE_SEARCH_FALLBACK_NAME;
|
|
251
|
+
}
|
|
252
|
+
catch {
|
|
253
|
+
return CACHE_SEARCH_TOOL_NAME;
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* Register the cache-search tool; every failure is contained so the rest of the
|
|
258
|
+
* plugin keeps mounting.
|
|
259
|
+
*/
|
|
260
|
+
export function registerCacheSearchTool(tools, cache, options = {}) {
|
|
261
|
+
const name = pickCacheSearchToolName(tools);
|
|
262
|
+
try {
|
|
263
|
+
tools.register(createCacheSearchDefinition(name, cache));
|
|
264
|
+
return name;
|
|
265
|
+
}
|
|
266
|
+
catch (error) {
|
|
267
|
+
try {
|
|
268
|
+
options.warn?.(`cache_search registration skipped: ${error instanceof Error ? error.message : String(error)}`);
|
|
269
|
+
}
|
|
270
|
+
catch {
|
|
271
|
+
// A diagnostic must not break mounting.
|
|
272
|
+
}
|
|
273
|
+
return undefined;
|
|
274
|
+
}
|
|
275
|
+
}
|
package/lib/research.d.ts
CHANGED
|
@@ -7,8 +7,11 @@
|
|
|
7
7
|
* two or three rows that matter, fetch them without timing out, and keep track
|
|
8
8
|
* of which sentence came from which URL. This tool performs exactly that step:
|
|
9
9
|
* it runs up to three searches through the normal engine path, de-duplicates
|
|
10
|
-
* the rows,
|
|
11
|
-
*
|
|
10
|
+
* the rows, ranks them by query relevance and then source authority, fetches
|
|
11
|
+
* the leading pages through the same ladder `web_fetch` uses, and returns one
|
|
12
|
+
* Markdown document in which every excerpt sits under its URL. A credible
|
|
13
|
+
* government/academic/official-documentation page that matches the query leads
|
|
14
|
+
* the pack; relevance still gates out official pages that match none of it.
|
|
12
15
|
*
|
|
13
16
|
* Failure is part of the contract. A page that will not load is reported by
|
|
14
17
|
* URL and reason, the remaining pages still arrive, and a total fetch failure
|
|
@@ -32,7 +35,9 @@ export declare const RESEARCH_MAX_SOURCES = 5;
|
|
|
32
35
|
/** Distinct queries accepted in one call. */
|
|
33
36
|
export declare const RESEARCH_MAX_QUERIES = 3;
|
|
34
37
|
/** Characters kept from each fetched page. */
|
|
35
|
-
export declare const RESEARCH_EXCERPT_CHARS =
|
|
38
|
+
export declare const RESEARCH_EXCERPT_CHARS = 1800;
|
|
39
|
+
/** Below this much readable text a page is boilerplate, not evidence. */
|
|
40
|
+
export declare const RESEARCH_MIN_TEXT_CHARS = 160;
|
|
36
41
|
/** Pages fetched at the same time. */
|
|
37
42
|
export declare const RESEARCH_PAGE_CONCURRENCY = 3;
|
|
38
43
|
/** Wall-clock budget for the whole call, below the tool's host timeout. */
|
|
@@ -111,6 +116,17 @@ export interface ResearchRegistryLike {
|
|
|
111
116
|
export interface ResearchOptions {
|
|
112
117
|
/** Registration diagnostics. */
|
|
113
118
|
warn?: (message: string) => void;
|
|
119
|
+
/**
|
|
120
|
+
* Called once for every page that survives validation, with the full body.
|
|
121
|
+
* Used by the optional cache_search tool; a throwing sink must never fail the
|
|
122
|
+
* research call.
|
|
123
|
+
*/
|
|
124
|
+
onPage?: (page: {
|
|
125
|
+
url: string;
|
|
126
|
+
title?: string;
|
|
127
|
+
text: string;
|
|
128
|
+
queries: readonly string[];
|
|
129
|
+
}) => void;
|
|
114
130
|
}
|
|
115
131
|
/** A successful registration plus the exact way to undo it. */
|
|
116
132
|
export interface ResearchRegistration {
|
|
@@ -127,6 +143,31 @@ export interface ResearchRegistration {
|
|
|
127
143
|
* @returns a stable key, or `undefined` for a non-http(s) URL.
|
|
128
144
|
*/
|
|
129
145
|
export declare function canonicalKey(raw: string): string | undefined;
|
|
146
|
+
/**
|
|
147
|
+
* Select the passages of a fetched page that actually answer the research
|
|
148
|
+
* question, instead of always returning the first N characters.
|
|
149
|
+
*
|
|
150
|
+
* The search result's snippet is a summary, not the page: the fact can be
|
|
151
|
+
* 20 KB down (a Kubernetes field, an RFC section). This second pass scores
|
|
152
|
+
* paragraph windows against the same query tokens and keeps the best ones in
|
|
153
|
+
* reading order. It is deterministic and lexical; no model call and no extra
|
|
154
|
+
* network request. When nothing matches, the opening is still returned, so a
|
|
155
|
+
* weakly matched page is not silently turned into an empty excerpt.
|
|
156
|
+
*
|
|
157
|
+
* @param text - full page text from the fetch layer.
|
|
158
|
+
* @param tokens - query tokens from all research queries.
|
|
159
|
+
* @param max - excerpt budget in characters.
|
|
160
|
+
* @returns a bounded excerpt, with best passages first preserved in page order.
|
|
161
|
+
*/
|
|
162
|
+
export declare function selectExcerpt(text: string, tokens: readonly string[], max: number): string;
|
|
163
|
+
/**
|
|
164
|
+
* Score a URL's source authority from 0 (generic page) to 5 (government,
|
|
165
|
+
* academic or standards body). Credible sources lead only after they pass the
|
|
166
|
+
* relevance gate, so an unrelated official page is not fetched as evidence.
|
|
167
|
+
* @param raw - candidate URL from any engine.
|
|
168
|
+
* @returns the authority score; malformed and non-HTTP(S) URLs score 0.
|
|
169
|
+
*/
|
|
170
|
+
export declare function authorityScore(raw: string): number;
|
|
130
171
|
/**
|
|
131
172
|
* Format the evidence pack the model reads.
|
|
132
173
|
* @param value - what the call produced.
|
|
@@ -140,7 +181,7 @@ export declare function formatResearchValue(value: ResearchValue): string;
|
|
|
140
181
|
* @param fetchPage - fetches one URL through the plugin's fetch ladder.
|
|
141
182
|
* @returns a registry-ready definition.
|
|
142
183
|
*/
|
|
143
|
-
export declare function createResearchDefinition(name: string, search: ResearchSearcher, fetchPage: ResearchFetcher): ResearchDefinition;
|
|
184
|
+
export declare function createResearchDefinition(name: string, search: ResearchSearcher, fetchPage: ResearchFetcher, options?: ResearchOptions): ResearchDefinition;
|
|
144
185
|
/**
|
|
145
186
|
* Pick the tool name this deployment can accept.
|
|
146
187
|
* @param tools - the live registry.
|