@fffangchenghao/dsh-web-search 0.24.2 → 0.24.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +72 -0
- package/README.md +5 -5
- package/README.zh-CN.md +5 -5
- package/SECURITY.md +13 -7
- package/client.js +1 -1
- package/docs/comparison.md +1 -1
- package/docs/images/web-tools-demo.svg +1 -1
- package/docs/latency-and-sources.md +1 -1
- package/docs/store-evidence.md +1 -1
- package/lib/bridge.js +53 -2
- package/lib/cache-persist.js +9 -1
- package/lib/cache.js +15 -6
- package/lib/fetch.js +59 -7
- package/lib/index.js +2 -2
- package/lib/markdown.js +35 -5
- package/lib/official-release.js +6 -0
- package/lib/official-sources.js +4 -2
- package/lib/official-support.js +4 -0
- package/lib/parsers.js +38 -21
- package/lib/platforms.js +65 -2
- package/lib/proxy.d.ts +9 -6
- package/lib/proxy.js +66 -23
- package/lib/rank.js +2 -2
- package/lib/read-page.js +30 -11
- package/lib/research.d.ts +5 -0
- package/lib/research.js +76 -12
- package/lib/transport.js +3 -2
- package/lib/verticals.js +11 -5
- package/package.json +1 -1
package/lib/research.js
CHANGED
|
@@ -169,9 +169,13 @@ export function selectExcerpt(text, tokens, max) {
|
|
|
169
169
|
const add = (index) => {
|
|
170
170
|
const window = windows[index];
|
|
171
171
|
if (window === undefined || picked.has(index))
|
|
172
|
-
return;
|
|
172
|
+
return false;
|
|
173
|
+
const addition = window.length + 2;
|
|
174
|
+
if (used + addition > max)
|
|
175
|
+
return false;
|
|
173
176
|
picked.add(index);
|
|
174
|
-
used +=
|
|
177
|
+
used += addition;
|
|
178
|
+
return true;
|
|
175
179
|
};
|
|
176
180
|
// Regulations often put the effective date in the final article, after the
|
|
177
181
|
// definitions. A pure relevance sort can spend the budget on the definitions
|
|
@@ -233,10 +237,12 @@ export function selectExcerpt(text, tokens, max) {
|
|
|
233
237
|
}
|
|
234
238
|
const selected = [...picked].sort((a, b) => a - b).map((index) => windows[index] ?? '');
|
|
235
239
|
const excerpt = selected.join('\n\n…\n\n');
|
|
236
|
-
//
|
|
237
|
-
//
|
|
238
|
-
//
|
|
239
|
-
|
|
240
|
+
// The tail/legal/opening safeguards must not bypass the caller's excerpt
|
|
241
|
+
// budget. If a single window still overshoots, cut it explicitly; losing a
|
|
242
|
+
// suffix is better than returning a multi-thousand-character excerpt.
|
|
243
|
+
if (excerpt.length <= max)
|
|
244
|
+
return excerpt;
|
|
245
|
+
return max > 0 ? `${excerpt.slice(0, Math.max(0, max - 1)).trimEnd()}…` : '';
|
|
240
246
|
}
|
|
241
247
|
/** Walk `worker` over items with at most `limit` in flight, preserving order. */
|
|
242
248
|
async function mapLimit(items, limit, worker) {
|
|
@@ -254,6 +260,26 @@ async function mapLimit(items, limit, worker) {
|
|
|
254
260
|
await Promise.all(runners);
|
|
255
261
|
return results;
|
|
256
262
|
}
|
|
263
|
+
/** Race a promise against a signal while observing both outcomes. */
|
|
264
|
+
function withAbort(promise, signal) {
|
|
265
|
+
if (signal.aborted) {
|
|
266
|
+
return Promise.reject(signal.reason instanceof Error ? signal.reason : new Error('research was aborted'));
|
|
267
|
+
}
|
|
268
|
+
return new Promise((resolve, reject) => {
|
|
269
|
+
const onAbort = () => {
|
|
270
|
+
signal.removeEventListener('abort', onAbort);
|
|
271
|
+
reject(signal.reason instanceof Error ? signal.reason : new Error('research was aborted'));
|
|
272
|
+
};
|
|
273
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
274
|
+
promise.then((value) => {
|
|
275
|
+
signal.removeEventListener('abort', onAbort);
|
|
276
|
+
resolve(value);
|
|
277
|
+
}, (error) => {
|
|
278
|
+
signal.removeEventListener('abort', onAbort);
|
|
279
|
+
reject(error);
|
|
280
|
+
});
|
|
281
|
+
});
|
|
282
|
+
}
|
|
257
283
|
/** Words that carry no retrieval signal in a research query. */
|
|
258
284
|
const QUERY_STOPWORDS = new Set([
|
|
259
285
|
'the', 'a', 'an', 'and', 'or', 'of', 'to', 'in', 'on', 'for', 'with', 'from',
|
|
@@ -607,6 +633,15 @@ function outputSchema() {
|
|
|
607
633
|
required: ['url', 'reason'],
|
|
608
634
|
},
|
|
609
635
|
},
|
|
636
|
+
searchFailures: {
|
|
637
|
+
type: 'array',
|
|
638
|
+
items: {
|
|
639
|
+
type: 'object',
|
|
640
|
+
additionalProperties: false,
|
|
641
|
+
properties: { query: { type: 'string' }, reason: { type: 'string' } },
|
|
642
|
+
required: ['query', 'reason'],
|
|
643
|
+
},
|
|
644
|
+
},
|
|
610
645
|
found: { type: 'integer' },
|
|
611
646
|
truncated: { type: 'boolean' },
|
|
612
647
|
},
|
|
@@ -644,10 +679,20 @@ export function formatResearchValue(value) {
|
|
|
644
679
|
'## Found but not fetched',
|
|
645
680
|
...value.sources.map((source) => `- ${source.title !== undefined && source.title.length > 0 ? `${source.title} — ` : ''}${source.url}`),
|
|
646
681
|
];
|
|
682
|
+
const searchFailures = value.searchFailures === undefined || value.searchFailures.length === 0
|
|
683
|
+
? []
|
|
684
|
+
: ['## Search failures', ...value.searchFailures.map((failure) => `- ${JSON.stringify(failure.query)}: ${failure.reason}`)];
|
|
647
685
|
const failures = value.failures.length === 0
|
|
648
686
|
? []
|
|
649
687
|
: ['## Fetch failures', ...value.failures.map((failure) => `- ${failure.url}: ${failure.reason}`)];
|
|
650
|
-
return [
|
|
688
|
+
return [
|
|
689
|
+
...head,
|
|
690
|
+
'',
|
|
691
|
+
...sections,
|
|
692
|
+
...(notFetched.length === 0 ? [] : ['', ...notFetched]),
|
|
693
|
+
...(searchFailures.length === 0 ? [] : ['', ...searchFailures]),
|
|
694
|
+
...(failures.length === 0 ? [] : ['', ...failures]),
|
|
695
|
+
]
|
|
651
696
|
.join('\n')
|
|
652
697
|
.trimEnd();
|
|
653
698
|
}
|
|
@@ -673,15 +718,26 @@ export function createResearchDefinition(name, search, fetchPage, options = {})
|
|
|
673
718
|
const parsed = parseArgs(args);
|
|
674
719
|
const signal = exec.signal;
|
|
675
720
|
const startedAt = Date.now();
|
|
721
|
+
// The tool's declared 45 s budget must cover the searches too. Passing
|
|
722
|
+
// only the host signal let a slow search consume the whole 60 s host
|
|
723
|
+
// timeout before the page ladder ever saw `remaining()`.
|
|
724
|
+
const deadline = AbortSignal.timeout(RESEARCH_BUDGET_MS);
|
|
725
|
+
const workSignal = signal === undefined ? deadline : AbortSignal.any([signal, deadline]);
|
|
676
726
|
const remaining = () => Math.max(0, RESEARCH_BUDGET_MS - (Date.now() - startedAt));
|
|
677
727
|
const cancelled = () => signal !== undefined && signal.aborted;
|
|
678
728
|
const searchLimit = Math.max(5, parsed.maxSources);
|
|
679
729
|
const queriesUsed = [...parsed.queries];
|
|
680
730
|
const runSearch = async (query) => {
|
|
681
731
|
try {
|
|
682
|
-
|
|
732
|
+
if (workSignal.aborted === true) {
|
|
733
|
+
const reason = workSignal.reason;
|
|
734
|
+
throw reason instanceof Error ? reason : new Error('research was aborted');
|
|
735
|
+
}
|
|
736
|
+
return await withAbort(search(query, searchLimit, workSignal), workSignal);
|
|
683
737
|
}
|
|
684
738
|
catch (error) {
|
|
739
|
+
if (signal?.aborted === true)
|
|
740
|
+
throw error;
|
|
685
741
|
return {
|
|
686
742
|
sources: [],
|
|
687
743
|
failures: [error instanceof Error ? error.message : String(error)],
|
|
@@ -691,12 +747,15 @@ export function createResearchDefinition(name, search, fetchPage, options = {})
|
|
|
691
747
|
const outcomes = await mapLimit(parsed.queries, 2, runSearch);
|
|
692
748
|
if (cancelled())
|
|
693
749
|
throw new Error('research was cancelled');
|
|
750
|
+
const searchFailures = parsed.queries.flatMap((query, index) => (outcomes[index]?.failures ?? []).map((reason) => ({ query, reason })));
|
|
694
751
|
let candidates = interleave(outcomes.map((outcome) => outcome.sources));
|
|
695
752
|
const hints = officialHints(parsed.queries);
|
|
696
753
|
// Long questions often need the issuing body's own site, which the open
|
|
697
754
|
// engine may not rank. Spend unused query slots on a site-specific query
|
|
698
755
|
// instead of letting the evidence pack stop at an aggregator.
|
|
699
756
|
for (const hint of hints) {
|
|
757
|
+
if (remaining() === 0)
|
|
758
|
+
break;
|
|
700
759
|
// A host match is enough for most issuers, but a generic W3C page does
|
|
701
760
|
// not cover a specific success criterion; let the targeted query run.
|
|
702
761
|
// A generic host match is enough for most issuers, but not for the
|
|
@@ -713,6 +772,8 @@ export function createResearchDefinition(name, search, fetchPage, options = {})
|
|
|
713
772
|
const extra = await runSearch(hinted);
|
|
714
773
|
if (cancelled())
|
|
715
774
|
throw new Error('research was cancelled');
|
|
775
|
+
for (const reason of extra.failures)
|
|
776
|
+
searchFailures.push({ query: hinted, reason });
|
|
716
777
|
// A site: operator is advisory on some engines; keep only rows that
|
|
717
778
|
// really belong to the issuer's domain.
|
|
718
779
|
outcomes.push({
|
|
@@ -801,11 +862,13 @@ export function createResearchDefinition(name, search, fetchPage, options = {})
|
|
|
801
862
|
failures.push({ url: source.url, reason: 'no time left in the research budget' });
|
|
802
863
|
return;
|
|
803
864
|
}
|
|
804
|
-
const bound =
|
|
805
|
-
? AbortSignal.timeout(budget)
|
|
806
|
-
: AbortSignal.any([signal, AbortSignal.timeout(budget)]);
|
|
865
|
+
const bound = AbortSignal.any([workSignal, AbortSignal.timeout(budget)]);
|
|
807
866
|
try {
|
|
808
|
-
|
|
867
|
+
if (bound.aborted === true) {
|
|
868
|
+
const reason = bound.reason;
|
|
869
|
+
throw reason instanceof Error ? reason : new Error('research budget exhausted');
|
|
870
|
+
}
|
|
871
|
+
const fetched = await withAbort(fetchPage(source.url, bound), bound);
|
|
809
872
|
if (looksLikeHtmlDocument(fetched.text)) {
|
|
810
873
|
failures.push({ url: source.url, reason: 'the page returned a raw HTML document, not readable text' });
|
|
811
874
|
continue;
|
|
@@ -888,6 +951,7 @@ export function createResearchDefinition(name, search, fetchPage, options = {})
|
|
|
888
951
|
...(source.title === undefined || source.title.length === 0 ? {} : { title: source.title }),
|
|
889
952
|
})),
|
|
890
953
|
failures,
|
|
954
|
+
...(searchFailures.length === 0 ? {} : { searchFailures }),
|
|
891
955
|
found: candidates.length,
|
|
892
956
|
truncated: candidates.length > parsed.maxSources,
|
|
893
957
|
};
|
package/lib/transport.js
CHANGED
|
@@ -199,9 +199,10 @@ async function attemptWithRoute(host, query, signal, timeoutMs, proxy, timeRange
|
|
|
199
199
|
}
|
|
200
200
|
if (signal?.aborted === true || proxy === undefined)
|
|
201
201
|
return first;
|
|
202
|
+
// When the failed first route was the proxy, the alternate is `undefined`,
|
|
203
|
+
// which means direct — not "no alternate". The old guard here removed the
|
|
204
|
+
// direct fallback exactly when a proxy-first route failed.
|
|
202
205
|
const alternateRoute = firstRoute === proxy ? undefined : proxy;
|
|
203
|
-
if (alternateRoute === undefined)
|
|
204
|
-
return first;
|
|
205
206
|
const retry = await attempt(host, url, signal, timeoutMs, alternateRoute, query);
|
|
206
207
|
if (retry.ok && retry.route !== undefined)
|
|
207
208
|
health?.noteRouteSuccess(host.id, retry.route);
|
package/lib/verticals.js
CHANGED
|
@@ -152,10 +152,13 @@ function openAlexAbstract(value) {
|
|
|
152
152
|
for (const [word, positions] of Object.entries(value)) {
|
|
153
153
|
if (!Array.isArray(positions) || word.trim().length === 0)
|
|
154
154
|
continue;
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
155
|
+
// The inverted index maps one word to every occurrence. Keeping only the
|
|
156
|
+
// first position silently deleted repeated words from the abstract.
|
|
157
|
+
for (const item of positions) {
|
|
158
|
+
if (typeof item === 'number' && Number.isSafeInteger(item) && item >= 0) {
|
|
159
|
+
words.push({ word: word.trim(), position: item });
|
|
160
|
+
}
|
|
161
|
+
}
|
|
159
162
|
}
|
|
160
163
|
words.sort((a, b) => a.position - b.position);
|
|
161
164
|
return words.map((entry) => entry.word).join(' ');
|
|
@@ -1654,8 +1657,11 @@ export async function searchVerticals(query, options = {}) {
|
|
|
1654
1657
|
const answers = await Promise.all(specs.map(async (spec) => {
|
|
1655
1658
|
try {
|
|
1656
1659
|
const built = spec.build(trimmed, limit);
|
|
1660
|
+
// arXiv's builder already encodes fields/booleans correctly. A date
|
|
1661
|
+
// window only asks arXiv to sort newest first; replacing its
|
|
1662
|
+
// `search_query` mangled `ti:`/`cat:` queries into a quoted phrase.
|
|
1657
1663
|
const url = spec.id === 'arxiv' && options.timeRange !== undefined && options.timeRange !== 'off'
|
|
1658
|
-
? `${built
|
|
1664
|
+
? `${built}&sortBy=submittedDate&sortOrder=descending`
|
|
1659
1665
|
: built;
|
|
1660
1666
|
const payload = await fetchPayload(spec, url, options, timeoutMs);
|
|
1661
1667
|
const sources = spec.parse(payload, url, limit);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fffangchenghao/dsh-web-search",
|
|
3
|
-
"version": "0.24.
|
|
3
|
+
"version": "0.24.4",
|
|
4
4
|
"description": "DSH bundle: key-free web search, page fetch and research for DeepSeek Harness; eight engines fused with RRF, 34 vertical sources, arXiv, Windows/macOS system-proxy support and a dependency-free MCP server.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|