@fffangchenghao/dsh-web-search 0.24.2 → 0.24.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/research.js CHANGED
@@ -169,9 +169,13 @@ export function selectExcerpt(text, tokens, max) {
169
169
  const add = (index) => {
170
170
  const window = windows[index];
171
171
  if (window === undefined || picked.has(index))
172
- return;
172
+ return false;
173
+ const addition = window.length + 2;
174
+ if (used + addition > max)
175
+ return false;
173
176
  picked.add(index);
174
- used += window.length + 2;
177
+ used += addition;
178
+ return true;
175
179
  };
176
180
  // Regulations often put the effective date in the final article, after the
177
181
  // definitions. A pure relevance sort can spend the budget on the definitions
@@ -233,10 +237,12 @@ export function selectExcerpt(text, tokens, max) {
233
237
  }
234
238
  const selected = [...picked].sort((a, b) => a - b).map((index) => windows[index] ?? '');
235
239
  const excerpt = selected.join('\n\n…\n\n');
236
- // Do not truncate from the end here: when tailNeeded is true, the end is the
237
- // effective date the caller asked for. The selected windows are already
238
- // bounded to roughly max plus one window.
239
- return excerpt;
240
+ // The tail/legal/opening safeguards must not bypass the caller's excerpt
241
+ // budget. If a single window still overshoots, cut it explicitly; losing a
242
+ // suffix is better than returning a multi-thousand-character excerpt.
243
+ if (excerpt.length <= max)
244
+ return excerpt;
245
+ return max > 0 ? `${excerpt.slice(0, Math.max(0, max - 1)).trimEnd()}…` : '';
240
246
  }
241
247
  /** Walk `worker` over items with at most `limit` in flight, preserving order. */
242
248
  async function mapLimit(items, limit, worker) {
@@ -254,6 +260,26 @@ async function mapLimit(items, limit, worker) {
254
260
  await Promise.all(runners);
255
261
  return results;
256
262
  }
263
+ /** Race a promise against a signal while observing both outcomes. */
264
+ function withAbort(promise, signal) {
265
+ if (signal.aborted) {
266
+ return Promise.reject(signal.reason instanceof Error ? signal.reason : new Error('research was aborted'));
267
+ }
268
+ return new Promise((resolve, reject) => {
269
+ const onAbort = () => {
270
+ signal.removeEventListener('abort', onAbort);
271
+ reject(signal.reason instanceof Error ? signal.reason : new Error('research was aborted'));
272
+ };
273
+ signal.addEventListener('abort', onAbort, { once: true });
274
+ promise.then((value) => {
275
+ signal.removeEventListener('abort', onAbort);
276
+ resolve(value);
277
+ }, (error) => {
278
+ signal.removeEventListener('abort', onAbort);
279
+ reject(error);
280
+ });
281
+ });
282
+ }
257
283
  /** Words that carry no retrieval signal in a research query. */
258
284
  const QUERY_STOPWORDS = new Set([
259
285
  'the', 'a', 'an', 'and', 'or', 'of', 'to', 'in', 'on', 'for', 'with', 'from',
@@ -607,6 +633,15 @@ function outputSchema() {
607
633
  required: ['url', 'reason'],
608
634
  },
609
635
  },
636
+ searchFailures: {
637
+ type: 'array',
638
+ items: {
639
+ type: 'object',
640
+ additionalProperties: false,
641
+ properties: { query: { type: 'string' }, reason: { type: 'string' } },
642
+ required: ['query', 'reason'],
643
+ },
644
+ },
610
645
  found: { type: 'integer' },
611
646
  truncated: { type: 'boolean' },
612
647
  },
@@ -644,10 +679,20 @@ export function formatResearchValue(value) {
644
679
  '## Found but not fetched',
645
680
  ...value.sources.map((source) => `- ${source.title !== undefined && source.title.length > 0 ? `${source.title} — ` : ''}${source.url}`),
646
681
  ];
682
+ const searchFailures = value.searchFailures === undefined || value.searchFailures.length === 0
683
+ ? []
684
+ : ['## Search failures', ...value.searchFailures.map((failure) => `- ${JSON.stringify(failure.query)}: ${failure.reason}`)];
647
685
  const failures = value.failures.length === 0
648
686
  ? []
649
687
  : ['## Fetch failures', ...value.failures.map((failure) => `- ${failure.url}: ${failure.reason}`)];
650
- return [...head, '', ...sections, ...(notFetched.length === 0 ? [] : ['', ...notFetched]), ...(failures.length === 0 ? [] : ['', ...failures])]
688
+ return [
689
+ ...head,
690
+ '',
691
+ ...sections,
692
+ ...(notFetched.length === 0 ? [] : ['', ...notFetched]),
693
+ ...(searchFailures.length === 0 ? [] : ['', ...searchFailures]),
694
+ ...(failures.length === 0 ? [] : ['', ...failures]),
695
+ ]
651
696
  .join('\n')
652
697
  .trimEnd();
653
698
  }
@@ -673,15 +718,26 @@ export function createResearchDefinition(name, search, fetchPage, options = {})
673
718
  const parsed = parseArgs(args);
674
719
  const signal = exec.signal;
675
720
  const startedAt = Date.now();
721
+ // The tool's declared 45 s budget must cover the searches too. Passing
722
+ // only the host signal let a slow search consume the whole 60 s host
723
+ // timeout before the page ladder ever saw `remaining()`.
724
+ const deadline = AbortSignal.timeout(RESEARCH_BUDGET_MS);
725
+ const workSignal = signal === undefined ? deadline : AbortSignal.any([signal, deadline]);
676
726
  const remaining = () => Math.max(0, RESEARCH_BUDGET_MS - (Date.now() - startedAt));
677
727
  const cancelled = () => signal !== undefined && signal.aborted;
678
728
  const searchLimit = Math.max(5, parsed.maxSources);
679
729
  const queriesUsed = [...parsed.queries];
680
730
  const runSearch = async (query) => {
681
731
  try {
682
- return await search(query, searchLimit, signal);
732
+ if (workSignal.aborted === true) {
733
+ const reason = workSignal.reason;
734
+ throw reason instanceof Error ? reason : new Error('research was aborted');
735
+ }
736
+ return await withAbort(search(query, searchLimit, workSignal), workSignal);
683
737
  }
684
738
  catch (error) {
739
+ if (signal?.aborted === true)
740
+ throw error;
685
741
  return {
686
742
  sources: [],
687
743
  failures: [error instanceof Error ? error.message : String(error)],
@@ -691,12 +747,15 @@ export function createResearchDefinition(name, search, fetchPage, options = {})
691
747
  const outcomes = await mapLimit(parsed.queries, 2, runSearch);
692
748
  if (cancelled())
693
749
  throw new Error('research was cancelled');
750
+ const searchFailures = parsed.queries.flatMap((query, index) => (outcomes[index]?.failures ?? []).map((reason) => ({ query, reason })));
694
751
  let candidates = interleave(outcomes.map((outcome) => outcome.sources));
695
752
  const hints = officialHints(parsed.queries);
696
753
  // Long questions often need the issuing body's own site, which the open
697
754
  // engine may not rank. Spend unused query slots on a site-specific query
698
755
  // instead of letting the evidence pack stop at an aggregator.
699
756
  for (const hint of hints) {
757
+ if (remaining() === 0)
758
+ break;
700
759
  // A host match is enough for most issuers, but a generic W3C page does
701
760
  // not cover a specific success criterion; let the targeted query run.
702
761
  // A generic host match is enough for most issuers, but not for the
@@ -713,6 +772,8 @@ export function createResearchDefinition(name, search, fetchPage, options = {})
713
772
  const extra = await runSearch(hinted);
714
773
  if (cancelled())
715
774
  throw new Error('research was cancelled');
775
+ for (const reason of extra.failures)
776
+ searchFailures.push({ query: hinted, reason });
716
777
  // A site: operator is advisory on some engines; keep only rows that
717
778
  // really belong to the issuer's domain.
718
779
  outcomes.push({
@@ -801,11 +862,13 @@ export function createResearchDefinition(name, search, fetchPage, options = {})
801
862
  failures.push({ url: source.url, reason: 'no time left in the research budget' });
802
863
  return;
803
864
  }
804
- const bound = signal === undefined
805
- ? AbortSignal.timeout(budget)
806
- : AbortSignal.any([signal, AbortSignal.timeout(budget)]);
865
+ const bound = AbortSignal.any([workSignal, AbortSignal.timeout(budget)]);
807
866
  try {
808
- const fetched = await fetchPage(source.url, bound);
867
+ if (bound.aborted === true) {
868
+ const reason = bound.reason;
869
+ throw reason instanceof Error ? reason : new Error('research budget exhausted');
870
+ }
871
+ const fetched = await withAbort(fetchPage(source.url, bound), bound);
809
872
  if (looksLikeHtmlDocument(fetched.text)) {
810
873
  failures.push({ url: source.url, reason: 'the page returned a raw HTML document, not readable text' });
811
874
  continue;
@@ -888,6 +951,7 @@ export function createResearchDefinition(name, search, fetchPage, options = {})
888
951
  ...(source.title === undefined || source.title.length === 0 ? {} : { title: source.title }),
889
952
  })),
890
953
  failures,
954
+ ...(searchFailures.length === 0 ? {} : { searchFailures }),
891
955
  found: candidates.length,
892
956
  truncated: candidates.length > parsed.maxSources,
893
957
  };
package/lib/transport.js CHANGED
@@ -199,9 +199,10 @@ async function attemptWithRoute(host, query, signal, timeoutMs, proxy, timeRange
199
199
  }
200
200
  if (signal?.aborted === true || proxy === undefined)
201
201
  return first;
202
+ // When the failed first route was the proxy, the alternate is `undefined`,
203
+ // which means direct — not "no alternate". The old guard here removed the
204
+ // direct fallback exactly when a proxy-first route failed.
202
205
  const alternateRoute = firstRoute === proxy ? undefined : proxy;
203
- if (alternateRoute === undefined)
204
- return first;
205
206
  const retry = await attempt(host, url, signal, timeoutMs, alternateRoute, query);
206
207
  if (retry.ok && retry.route !== undefined)
207
208
  health?.noteRouteSuccess(host.id, retry.route);
package/lib/verticals.js CHANGED
@@ -152,10 +152,13 @@ function openAlexAbstract(value) {
152
152
  for (const [word, positions] of Object.entries(value)) {
153
153
  if (!Array.isArray(positions) || word.trim().length === 0)
154
154
  continue;
155
- const position = positions.find((item) => typeof item === 'number' && Number.isFinite(item));
156
- if (position === undefined)
157
- continue;
158
- words.push({ word: word.trim(), position });
155
+ // The inverted index maps one word to every occurrence. Keeping only the
156
+ // first position silently deleted repeated words from the abstract.
157
+ for (const item of positions) {
158
+ if (typeof item === 'number' && Number.isSafeInteger(item) && item >= 0) {
159
+ words.push({ word: word.trim(), position: item });
160
+ }
161
+ }
159
162
  }
160
163
  words.sort((a, b) => a.position - b.position);
161
164
  return words.map((entry) => entry.word).join(' ');
@@ -1654,8 +1657,11 @@ export async function searchVerticals(query, options = {}) {
1654
1657
  const answers = await Promise.all(specs.map(async (spec) => {
1655
1658
  try {
1656
1659
  const built = spec.build(trimmed, limit);
1660
+ // arXiv's builder already encodes fields/booleans correctly. A date
1661
+ // window only asks arXiv to sort newest first; replacing its
1662
+ // `search_query` mangled `ti:`/`cat:` queries into a quoted phrase.
1657
1663
  const url = spec.id === 'arxiv' && options.timeRange !== undefined && options.timeRange !== 'off'
1658
- ? `${built.replace(/search_query=[^&]*/u, `search_query=${encodeURIComponent(`all:"${trimmed}"`)}`)}&sortBy=submittedDate&sortOrder=descending`
1664
+ ? `${built}&sortBy=submittedDate&sortOrder=descending`
1659
1665
  : built;
1660
1666
  const payload = await fetchPayload(spec, url, options, timeoutMs);
1661
1667
  const sources = spec.parse(payload, url, limit);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@fffangchenghao/dsh-web-search",
3
- "version": "0.24.2",
3
+ "version": "0.24.4",
4
4
  "description": "DSH bundle: key-free web search, page fetch and research for DeepSeek Harness; eight engines fused with RRF, 34 vertical sources, arXiv, Windows/macOS system-proxy support and a dependency-free MCP server.",
5
5
  "type": "module",
6
6
  "main": "lib/index.js",