@staticn0va/wigolo 0.6.5 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -48
- package/SKILL.md +22 -22
- package/assets/skills/wigolo/rules/cache-first.md +1 -1
- package/assets/skills/wigolo/rules/synthesis.md +1 -1
- package/assets/skills/wigolo-fetch/SKILL.md +1 -1
- package/assets/skills/wigolo-find-similar/SKILL.md +2 -2
- package/assets/skills/wigolo-search/SKILL.md +3 -3
- package/dist/cache/store.d.ts +9 -1
- package/dist/cache/store.d.ts.map +1 -1
- package/dist/cache/store.js +30 -4
- package/dist/cache/store.js.map +1 -1
- package/dist/cli/doctor.d.ts +3 -3
- package/dist/cli/doctor.d.ts.map +1 -1
- package/dist/cli/doctor.js +67 -13
- package/dist/cli/doctor.js.map +1 -1
- package/dist/cli/health.js +1 -1
- package/dist/cli/health.js.map +1 -1
- package/dist/cli/status.js +1 -1
- package/dist/cli/status.js.map +1 -1
- package/dist/cli/tui/hooks/useInstall.js +2 -2
- package/dist/cli/tui/hooks/useInstall.js.map +1 -1
- package/dist/cli/tui/hooks/useVerify.js +3 -3
- package/dist/cli/tui/hooks/useVerify.js.map +1 -1
- package/dist/cli/tui/status-format.d.ts +1 -1
- package/dist/cli/tui/status-format.d.ts.map +1 -1
- package/dist/cli/tui/status-format.js +5 -5
- package/dist/cli/tui/status-format.js.map +1 -1
- package/dist/cli/tui/status-python.d.ts +1 -1
- package/dist/cli/tui/status-python.d.ts.map +1 -1
- package/dist/cli/tui/status-python.js +17 -1
- package/dist/cli/tui/status-python.js.map +1 -1
- package/dist/cli/tui/verify-suggestions.d.ts +1 -1
- package/dist/cli/tui/verify-suggestions.d.ts.map +1 -1
- package/dist/cli/tui/verify-suggestions.js +5 -5
- package/dist/cli/tui/verify-suggestions.js.map +1 -1
- package/dist/cli/tui/verify.d.ts +2 -2
- package/dist/cli/tui/verify.d.ts.map +1 -1
- package/dist/cli/tui/verify.js +34 -8
- package/dist/cli/tui/verify.js.map +1 -1
- package/dist/cli/uninstall.js +2 -2
- package/dist/cli/uninstall.js.map +1 -1
- package/dist/cli/warmup.d.ts.map +1 -1
- package/dist/cli/warmup.js +29 -25
- package/dist/cli/warmup.js.map +1 -1
- package/dist/config.d.ts +6 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +15 -2
- package/dist/config.js.map +1 -1
- package/dist/crawl/dedup.d.ts +1 -0
- package/dist/crawl/dedup.d.ts.map +1 -1
- package/dist/crawl/dedup.js +47 -1
- package/dist/crawl/dedup.js.map +1 -1
- package/dist/extraction/boilerplate.d.ts +15 -0
- package/dist/extraction/boilerplate.d.ts.map +1 -0
- package/dist/extraction/boilerplate.js +49 -0
- package/dist/extraction/boilerplate.js.map +1 -0
- package/dist/extraction/defuddle.d.ts.map +1 -1
- package/dist/extraction/defuddle.js +7 -3
- package/dist/extraction/defuddle.js.map +1 -1
- package/dist/extraction/jsonld.js +1 -1
- package/dist/extraction/jsonld.js.map +1 -1
- package/dist/extraction/lang-hints.d.ts +2 -0
- package/dist/extraction/lang-hints.d.ts.map +1 -0
- package/dist/extraction/lang-hints.js +28 -0
- package/dist/extraction/lang-hints.js.map +1 -0
- package/dist/extraction/llm/anthropic.d.ts +3 -0
- package/dist/extraction/llm/anthropic.d.ts.map +1 -0
- package/dist/extraction/llm/anthropic.js +33 -0
- package/dist/extraction/llm/anthropic.js.map +1 -0
- package/dist/extraction/llm/cache.d.ts +5 -0
- package/dist/extraction/llm/cache.d.ts.map +1 -0
- package/dist/extraction/llm/cache.js +35 -0
- package/dist/extraction/llm/cache.js.map +1 -0
- package/dist/extraction/llm/gemini.d.ts +3 -0
- package/dist/extraction/llm/gemini.d.ts.map +1 -0
- package/dist/extraction/llm/gemini.js +35 -0
- package/dist/extraction/llm/gemini.js.map +1 -0
- package/dist/extraction/llm/groq.d.ts +3 -0
- package/dist/extraction/llm/groq.d.ts.map +1 -0
- package/dist/extraction/llm/groq.js +63 -0
- package/dist/extraction/llm/groq.js.map +1 -0
- package/dist/extraction/llm/hash.d.ts +3 -0
- package/dist/extraction/llm/hash.d.ts.map +1 -0
- package/dist/extraction/llm/hash.js +22 -0
- package/dist/extraction/llm/hash.js.map +1 -0
- package/dist/extraction/llm/openai.d.ts +3 -0
- package/dist/extraction/llm/openai.d.ts.map +1 -0
- package/dist/extraction/llm/openai.js +38 -0
- package/dist/extraction/llm/openai.js.map +1 -0
- package/dist/extraction/llm/select.d.ts +5 -0
- package/dist/extraction/llm/select.d.ts.map +1 -0
- package/dist/extraction/llm/select.js +27 -0
- package/dist/extraction/llm/select.js.map +1 -0
- package/dist/extraction/llm/types.d.ts +24 -0
- package/dist/extraction/llm/types.d.ts.map +1 -0
- package/dist/extraction/llm/types.js +2 -0
- package/dist/extraction/llm/types.js.map +1 -0
- package/dist/extraction/llm/validate.d.ts +6 -0
- package/dist/extraction/llm/validate.d.ts.map +1 -0
- package/dist/extraction/llm/validate.js +63 -0
- package/dist/extraction/llm/validate.js.map +1 -0
- package/dist/extraction/llm-fallback.d.ts +17 -0
- package/dist/extraction/llm-fallback.d.ts.map +1 -0
- package/dist/extraction/llm-fallback.js +129 -0
- package/dist/extraction/llm-fallback.js.map +1 -0
- package/dist/extraction/markdown.d.ts +9 -0
- package/dist/extraction/markdown.d.ts.map +1 -1
- package/dist/extraction/markdown.js +52 -3
- package/dist/extraction/markdown.js.map +1 -1
- package/dist/extraction/pipeline.d.ts.map +1 -1
- package/dist/extraction/pipeline.js +17 -5
- package/dist/extraction/pipeline.js.map +1 -1
- package/dist/extraction/readability.d.ts.map +1 -1
- package/dist/extraction/readability.js +2 -3
- package/dist/extraction/readability.js.map +1 -1
- package/dist/extraction/schema.d.ts +12 -0
- package/dist/extraction/schema.d.ts.map +1 -1
- package/dist/extraction/schema.js +81 -11
- package/dist/extraction/schema.js.map +1 -1
- package/dist/extraction/site-extractors/docs-generic.d.ts.map +1 -1
- package/dist/extraction/site-extractors/docs-generic.js +2 -3
- package/dist/extraction/site-extractors/docs-generic.js.map +1 -1
- package/dist/extraction/site-extractors/github.d.ts.map +1 -1
- package/dist/extraction/site-extractors/github.js +4 -5
- package/dist/extraction/site-extractors/github.js.map +1 -1
- package/dist/extraction/site-extractors/mdn.d.ts.map +1 -1
- package/dist/extraction/site-extractors/mdn.js +2 -3
- package/dist/extraction/site-extractors/mdn.js.map +1 -1
- package/dist/extraction/site-extractors/stackoverflow.d.ts.map +1 -1
- package/dist/extraction/site-extractors/stackoverflow.js +3 -4
- package/dist/extraction/site-extractors/stackoverflow.js.map +1 -1
- package/dist/extraction/structured-data.d.ts +4 -0
- package/dist/extraction/structured-data.d.ts.map +1 -0
- package/dist/extraction/structured-data.js +203 -0
- package/dist/extraction/structured-data.js.map +1 -0
- package/dist/fetch/router.d.ts +2 -1
- package/dist/fetch/router.d.ts.map +1 -1
- package/dist/fetch/router.js +19 -1
- package/dist/fetch/router.js.map +1 -1
- package/dist/instructions.d.ts +8 -8
- package/dist/instructions.d.ts.map +1 -1
- package/dist/instructions.js +48 -41
- package/dist/instructions.js.map +1 -1
- package/dist/logger.d.ts +1 -1
- package/dist/logger.d.ts.map +1 -1
- package/dist/research/brief.js +1 -1
- package/dist/research/brief.js.map +1 -1
- package/dist/search/evidence.d.ts +25 -0
- package/dist/search/evidence.d.ts.map +1 -0
- package/dist/search/evidence.js +260 -0
- package/dist/search/evidence.js.map +1 -0
- package/dist/search/highlights.d.ts +11 -2
- package/dist/search/highlights.d.ts.map +1 -1
- package/dist/search/highlights.js +131 -48
- package/dist/search/highlights.js.map +1 -1
- package/dist/search/multi-query.d.ts +1 -0
- package/dist/search/multi-query.d.ts.map +1 -1
- package/dist/search/multi-query.js +13 -0
- package/dist/search/multi-query.js.map +1 -1
- package/dist/search/rerank.d.ts +3 -2
- package/dist/search/rerank.d.ts.map +1 -1
- package/dist/search/rerank.js +16 -44
- package/dist/search/rerank.js.map +1 -1
- package/dist/search/reranker/download.d.ts +9 -0
- package/dist/search/reranker/download.d.ts.map +1 -0
- package/dist/search/reranker/download.js +77 -0
- package/dist/search/reranker/download.js.map +1 -0
- package/dist/search/reranker/models.d.ts +14 -0
- package/dist/search/reranker/models.d.ts.map +1 -0
- package/dist/search/reranker/models.js +37 -0
- package/dist/search/reranker/models.js.map +1 -0
- package/dist/search/reranker/onnx.d.ts +13 -0
- package/dist/search/reranker/onnx.d.ts.map +1 -0
- package/dist/search/reranker/onnx.js +70 -0
- package/dist/search/reranker/onnx.js.map +1 -0
- package/dist/search/reranker/recency-boost.d.ts +3 -0
- package/dist/search/reranker/recency-boost.d.ts.map +1 -0
- package/dist/search/reranker/recency-boost.js +12 -0
- package/dist/search/reranker/recency-boost.js.map +1 -0
- package/dist/search/reranker/recency.d.ts +3 -0
- package/dist/search/reranker/recency.d.ts.map +1 -0
- package/dist/search/reranker/recency.js +26 -0
- package/dist/search/reranker/recency.js.map +1 -0
- package/dist/search/reranker/tokenizer.d.ts +30 -0
- package/dist/search/reranker/tokenizer.d.ts.map +1 -0
- package/dist/search/reranker/tokenizer.js +49 -0
- package/dist/search/reranker/tokenizer.js.map +1 -0
- package/dist/search/tokens.d.ts +3 -0
- package/dist/search/tokens.d.ts.map +1 -0
- package/dist/search/tokens.js +38 -0
- package/dist/search/tokens.js.map +1 -0
- package/dist/search/truncate.d.ts +4 -0
- package/dist/search/truncate.d.ts.map +1 -1
- package/dist/search/truncate.js +13 -0
- package/dist/search/truncate.js.map +1 -1
- package/dist/server/backend-status.js +2 -2
- package/dist/server/backend-status.js.map +1 -1
- package/dist/server/tool-schemas.d.ts +503 -0
- package/dist/server/tool-schemas.d.ts.map +1 -0
- package/dist/server/tool-schemas.js +425 -0
- package/dist/server/tool-schemas.js.map +1 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +14 -339
- package/dist/server.js.map +1 -1
- package/dist/tools/agent.d.ts.map +1 -1
- package/dist/tools/agent.js +36 -0
- package/dist/tools/agent.js.map +1 -1
- package/dist/tools/crawl.d.ts.map +1 -1
- package/dist/tools/crawl.js +37 -2
- package/dist/tools/crawl.js.map +1 -1
- package/dist/tools/extract.d.ts.map +1 -1
- package/dist/tools/extract.js +19 -3
- package/dist/tools/extract.js.map +1 -1
- package/dist/tools/fetch.d.ts.map +1 -1
- package/dist/tools/fetch.js +44 -7
- package/dist/tools/fetch.js.map +1 -1
- package/dist/tools/find-similar.d.ts.map +1 -1
- package/dist/tools/find-similar.js +32 -1
- package/dist/tools/find-similar.js.map +1 -1
- package/dist/tools/research.d.ts.map +1 -1
- package/dist/tools/research.js +34 -1
- package/dist/tools/research.js.map +1 -1
- package/dist/tools/search.d.ts.map +1 -1
- package/dist/tools/search.js +98 -54
- package/dist/tools/search.js.map +1 -1
- package/dist/types.d.ts +65 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +1 -1
- package/dist/types.js.map +1 -1
- package/dist/util/mode.d.ts +4 -0
- package/dist/util/mode.d.ts.map +1 -0
- package/dist/util/mode.js +13 -0
- package/dist/util/mode.js.map +1 -0
- package/package.json +10 -4
- package/dist/search/flashrank.d.ts +0 -12
- package/dist/search/flashrank.d.ts.map +0 -1
- package/dist/search/flashrank.js +0 -64
- package/dist/search/flashrank.js.map +0 -1
package/dist/tools/crawl.js
CHANGED
|
@@ -2,16 +2,21 @@ import { Crawler } from '../crawl/crawler.js';
|
|
|
2
2
|
import { deduplicatePages } from '../crawl/dedup.js';
|
|
3
3
|
import { mapUrls } from '../crawl/mapper.js';
|
|
4
4
|
import { handleFetch } from './fetch.js';
|
|
5
|
+
import { buildEvidenceFromMarkdown, applyAggregateMarkdownBudget, } from '../search/evidence.js';
|
|
6
|
+
import { countTokens } from '../search/tokens.js';
|
|
5
7
|
import { createLogger } from '../logger.js';
|
|
6
8
|
const log = createLogger('crawl');
|
|
7
9
|
const DEFAULT_MAX_TOTAL_CHARS = 100000;
|
|
10
|
+
const DEFAULT_MAX_TOKENS_OUT = 4000;
|
|
8
11
|
export async function handleCrawl(input, router) {
|
|
9
12
|
try {
|
|
10
13
|
// Map strategy: lightweight URL-only discovery, skip full crawl pipeline
|
|
11
14
|
if (input.strategy === 'map') {
|
|
12
15
|
return handleMapStrategy(input, router);
|
|
13
16
|
}
|
|
14
|
-
|
|
17
|
+
// Crawler needs full markdown internally for dedup; opt in explicitly so
|
|
18
|
+
// handleFetch's default strip does not steal page bodies mid-crawl.
|
|
19
|
+
const fetchFn = async (url) => handleFetch({ url, use_auth: input.use_auth, include_full_markdown: true }, router);
|
|
15
20
|
const rawFetchFn = async (url) => router.fetch(url, { renderJs: 'never' });
|
|
16
21
|
const crawler = new Crawler(fetchFn, rawFetchFn);
|
|
17
22
|
const result = await crawler.crawl(input);
|
|
@@ -40,12 +45,14 @@ export async function handleCrawl(input, router) {
|
|
|
40
45
|
returned: budgetedPages.length,
|
|
41
46
|
totalChars: charCount,
|
|
42
47
|
});
|
|
43
|
-
|
|
48
|
+
const out = {
|
|
44
49
|
pages: budgetedPages,
|
|
45
50
|
total_found: result.total_found,
|
|
46
51
|
crawled: result.crawled,
|
|
47
52
|
...(result.links ? { links: result.links } : {}),
|
|
48
53
|
};
|
|
54
|
+
await attachEvidence(out, input);
|
|
55
|
+
return out;
|
|
49
56
|
}
|
|
50
57
|
catch (err) {
|
|
51
58
|
log.error('Crawl failed', { url: input.url, error: String(err) });
|
|
@@ -57,6 +64,34 @@ export async function handleCrawl(input, router) {
|
|
|
57
64
|
};
|
|
58
65
|
}
|
|
59
66
|
}
|
|
67
|
+
async function attachEvidence(out, input) {
|
|
68
|
+
if (out.pages.length === 0)
|
|
69
|
+
return;
|
|
70
|
+
const includeFull = input.include_full_markdown ?? false;
|
|
71
|
+
const maxTokensOut = input.max_tokens_out ?? DEFAULT_MAX_TOKENS_OUT;
|
|
72
|
+
let used = 0;
|
|
73
|
+
for (const page of out.pages) {
|
|
74
|
+
if (!page.markdown)
|
|
75
|
+
continue;
|
|
76
|
+
const remaining = maxTokensOut - used;
|
|
77
|
+
if (remaining <= 0)
|
|
78
|
+
break;
|
|
79
|
+
const evs = await buildEvidenceFromMarkdown(page.title || page.url, page.title, page.url, page.markdown, { maxItems: 1, maxTokensOut: remaining });
|
|
80
|
+
if (evs.length > 0) {
|
|
81
|
+
page.evidence = evs;
|
|
82
|
+
for (const ev of evs)
|
|
83
|
+
used += countTokens(ev.excerpt);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
if (!includeFull) {
|
|
87
|
+
for (const page of out.pages) {
|
|
88
|
+
page.markdown = '';
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
else {
|
|
92
|
+
applyAggregateMarkdownBudget(out.pages, (p) => p.markdown ?? '', (p, body) => { p.markdown = body; }, { maxTokensOut });
|
|
93
|
+
}
|
|
94
|
+
}
|
|
60
95
|
async function handleMapStrategy(input, router) {
|
|
61
96
|
const httpFetchFn = async (url) => {
|
|
62
97
|
const raw = await router.fetch(url, { renderJs: 'never' });
|
package/dist/tools/crawl.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/tools/crawl.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/tools/crawl.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EACL,yBAAyB,EACzB,4BAA4B,GAC7B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;AAElC,MAAM,uBAAuB,GAAG,MAAM,CAAC;AACvC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAiB,EACjB,MAAmB;IAEnB,IAAI,CAAC;QACH,yEAAyE;QACzE,IAAI,KAAK,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;YAC7B,OAAO,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC1C,CAAC;QAED,yEAAyE;QACzE,oEAAoE;QACpE,MAAM,OAAO,GAAG,KAAK,EAAE,GAAW,EAAE,EAAE,CACpC,WAAW,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,qBAAqB,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC;QAEtF,MAAM,UAAU,GAAG,KAAK,EAAE,GAAW,EAAE,EAAE,CACvC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAE3C,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE1C,8EAA8E;QAC9E,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;QAC3C,MAAM,YAAY,GAAG,gBAAgB,CACnC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,EAC/D,MAAM,CACP,CAAC;QAEF,uCAAuC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3C,GAAG,IAAI;YACP,QAAQ,EAAE,YAAY,CAAC,CAAC,CAAC,EAAE,QAAQ,IAAI,IAAI,CAAC,QAAQ;SACrD,CAAC,CAAC,CAAC;QAEJ,iCAAiC;QACjC,MAAM,aAAa,GAAG,KAAK,CAAC,eAAe,IAAI,uBAAuB,CAAC;QACvE,MAAM,aAAa,GAAG,EAAE,CAAC;QACzB,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,aAAa,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjF,MAAM;YACR,CAAC;YACD,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzB,SAAS,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QACpC,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE;YACzB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,QAAQ,EAAE,aAAa,CAAC,MAAM;YAC9B,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC;QAEH,MAAM,GAAG,GAAgB;YACvB,KAAK,EAAE,aAAa;YACpB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACjD,CAAC;QAEF,MAAM,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACjC,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO;YACL,KAAK,EAAE,EAAE;YACT,WAAW,EAAE,CAAC;YACd,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,GAAgB,EAAE,KAAiB;IAC/D,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IACnC,MAAM,WAAW,GAAG,KAAK,CAAC,qBAAqB,IAAI,KAAK,CAAC;IACzD,MAAM,YAAY,GAAG,KAAK,CAAC,cAAc,IAAI,sBAAsB,CAAC;IAEpE,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,SAAS;QAC7B,MAAM,SAAS,GAAG,YAAY,GAAG,IAAI,CAAC;QACtC,IAAI,SAAS,IAAI,CAAC;YAAE,MAAM;QAC1B,MAAM,GAAG,GAAG,MAAM,yBAAyB,CACzC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EACtB,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,GAAG,EACR,IAAI,CAAC,QAAQ,EACb,EAAE,QAAQ,EAAE,CAAC,EAAE,YAAY,EAAE,SAAS,EAAE,CACzC,CAAC;QACF,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;YACpB,KAAK,MAAM,EAAE,IAAI,GAAG;gBAAE,IAAI,IAAI,WAAW,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YAC7B,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,4BAA4B,CAC1B,GAAG,CAAC,KAAK,EACT,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,EACvB,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,EACnC,EAAE,YAAY,EAAE,CACjB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,KAAiB,EACjB,MAAmB;IAEnB,MAAM,WAAW,GAAG,KAAK,EAAE,GAAW,EAAE,EAAE;QACxC,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAC3D,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC;IAChF,CAAC,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,OAAO,CAC7B;YACE,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;YACxC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;SACzC,EACD,WAAW,CACZ,CAAC;QAEF,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE;YACvB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,aAAa,EAAE,SAAS,CAAC,aAAa;SACvC,CAAC,CAAC;QAEH,OAAO;YACL,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,aAAa,EAAE,SAAS,CAAC,aAAa;YACtC,OAAO,EAAE,CAAC;YACV,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACvD,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,qBAAqB,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzE,OAAO;YACL,IAAI,EAAE,EAAE;YACR,WAAW,EAAE,CAAC;YACd,aAAa,EAAE,KAAK;YACpB,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/tools/extract.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC/D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/tools/extract.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC/D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAkCtD,wBAAsB,aAAa,CACjC,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,aAAa,CAAC,CAoFxB"}
|
package/dist/tools/extract.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { extractMetadata, extractSelector, extractTables } from '../extraction/extract.js';
|
|
2
|
-
import { extractWithSchema } from '../extraction/schema.js';
|
|
2
|
+
import { extractWithSchema, extractWithSchemaDetailedAsync, } from '../extraction/schema.js';
|
|
3
3
|
import { extractJsonLd } from '../extraction/jsonld.js';
|
|
4
4
|
import { extractStructured } from '../extraction/structured.js';
|
|
5
5
|
import { getCachedContent, isExpired } from '../cache/store.js';
|
|
@@ -44,9 +44,25 @@ export async function handleExtract(input, router) {
|
|
|
44
44
|
case 'structured':
|
|
45
45
|
data = extractStructured(html);
|
|
46
46
|
break;
|
|
47
|
-
case 'schema':
|
|
48
|
-
|
|
47
|
+
case 'schema': {
|
|
48
|
+
const schema = input.schema;
|
|
49
|
+
if (Array.isArray(schema.required) && schema.required.length > 0) {
|
|
50
|
+
const detailed = await extractWithSchemaDetailedAsync(html, schema);
|
|
51
|
+
data = detailed.values;
|
|
52
|
+
if (detailed.warnings.length > 0) {
|
|
53
|
+
return {
|
|
54
|
+
data,
|
|
55
|
+
source_url: sourceUrl,
|
|
56
|
+
mode,
|
|
57
|
+
warnings: detailed.warnings,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
data = extractWithSchema(html, schema);
|
|
63
|
+
}
|
|
49
64
|
break;
|
|
65
|
+
}
|
|
50
66
|
case 'metadata':
|
|
51
67
|
default: {
|
|
52
68
|
const meta = extractMetadata(html);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/tools/extract.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC3F,OAAO,
|
|
1
|
+
{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/tools/extract.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC3F,OAAO,EACL,iBAAiB,EACjB,8BAA8B,GAC/B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;AAEpC,KAAK,UAAU,WAAW,CACxB,KAAmB,EACnB,MAAmB;IAEnB,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC;QACd,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3C,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;YACjC,GAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;YAClD,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC;QACzD,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACxC,QAAQ,EAAE,MAAM;YAChB,OAAO,EAAE,KAAK;SACf,CAAC,CAAC;QACH,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC;IACrD,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAK,EAAE,CAAC;AAC/B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,KAAmB,EACnB,MAAmB;IAEnB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,UAAU,CAAC;IAEtC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC;IAC1E,CAAC;IAED,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;QAC/C,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,kDAAkD,EAAE,CAAC;IACvF,CAAC;IAED,IAAI,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACrE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,mEAAmE,EAAE,CAAC;IACxG,CAAC;IAED,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAE7D,IAAI,IAA2B,CAAC;QAEhC,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,UAAU;gBACb,IAAI,GAAG,eAAe,CAAC,IAAI,EAAE,KAAK,CAAC,YAAa,EAAE,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,CAAC;gBAC3E,MAAM;YACR,KAAK,QAAQ;gBACX,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;gBAC3B,MAAM;YACR,KAAK,YAAY;gBACf,IAAI,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC;gBAC/B,MAAM;YACR,KAAK,QAAQ,CAAC,CAAC,CAAC;gBACd,MAAM,MAAM,GAAG,KAAK,CAAC,MAAO,CAAC;gBAC7B,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACjE,MAAM,QAAQ,GAAG,MAAM,8BAA8B,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;oBACpE,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC;oBACvB,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACjC,OAAO;4BACL,IAAI;4BACJ,UAAU,EAAE,SAAS;4BACrB,IAAI;4BACJ,QAAQ,EAAE,QAAQ,CAAC,QAAQ;yBAC5B,CAAC;oBACJ,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,IAAI,GAAG,iBAAiB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBACzC,CAAC;gBACD,MAAM;YACR,CAAC;YACD,KAAK,UAAU,CAAC;YAChB,OAAO,CAAC,CAAC,CAAC;gBACR,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;gBACnC,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;gBACnC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;gBACvB,CAAC;gBACD,IAAI,GAAG,IAAI,CAAC;gBACZ,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IAC/C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,gBAAgB,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpE,IAAI,YAAmC,CAAC;QACxC,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,UAAU;gBACb,YAAY,GAAG,EAAE,CAAC;gBAClB,MAAM;YACR,KAAK,QAAQ;gBACX,YAAY,GAAG,EAAE,CAAC;gBAClB,MAAM;YACR,KAAK,YAAY;gBACf,YAAY,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC;gBACjG,MAAM;YACR;gBACE,YAAY,GAAG,EAAE,CAAC;QACtB,CAAC;QACD,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,UAAU,EAAE,KAAK,CAAC,GAAG;YACrB,IAAI;YACJ,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../src/tools/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAiB,MAAM,aAAa,CAAC;AAC1E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../src/tools/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAiB,MAAM,aAAa,CAAC;AAC1E,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AA2EtD,wBAAsB,WAAW,CAC/B,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,WAAW,CAAC,CA+FtB"}
|
package/dist/tools/fetch.js
CHANGED
|
@@ -1,11 +1,33 @@
|
|
|
1
1
|
import { extractContent } from '../extraction/pipeline.js';
|
|
2
|
-
import { getCachedContent, cacheContent,
|
|
2
|
+
import { getCachedContent, cacheContent, isCacheUsable } from '../cache/store.js';
|
|
3
|
+
import { getConfig } from '../config.js';
|
|
3
4
|
import { extractSection } from '../extraction/markdown.js';
|
|
4
5
|
import { detectChange } from '../cache/change-detector.js';
|
|
5
6
|
import { getEmbeddingService } from '../embedding/embed.js';
|
|
6
|
-
import { truncateSmartly } from '../search/truncate.js';
|
|
7
|
+
import { truncateSmartly, applyOutputBudget } from '../search/truncate.js';
|
|
8
|
+
import { buildEvidenceFromMarkdown } from '../search/evidence.js';
|
|
9
|
+
import { resolveMode } from '../util/mode.js';
|
|
7
10
|
import { createLogger } from '../logger.js';
|
|
8
11
|
const log = createLogger('fetch');
|
|
12
|
+
const DEFAULT_MAX_TOKENS_OUT = 4000;
|
|
13
|
+
async function attachEvidence(output, input, markdown) {
|
|
14
|
+
if (!markdown)
|
|
15
|
+
return;
|
|
16
|
+
const includeFull = input.include_full_markdown ?? false;
|
|
17
|
+
const maxTokensOut = input.max_tokens_out ?? DEFAULT_MAX_TOKENS_OUT;
|
|
18
|
+
const evidence = await buildEvidenceFromMarkdown(output.title || output.url, output.title, output.url, markdown, { maxTokensOut, maxItems: 1 });
|
|
19
|
+
if (evidence.length > 0)
|
|
20
|
+
output.evidence = evidence;
|
|
21
|
+
if (!includeFull) {
|
|
22
|
+
output.markdown = '';
|
|
23
|
+
}
|
|
24
|
+
else if (output.markdown) {
|
|
25
|
+
output.markdown = applyOutputBudget(output.markdown, {
|
|
26
|
+
maxTokensOut: input.max_tokens_out,
|
|
27
|
+
maxChars: input.max_chars,
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
}
|
|
9
31
|
function formatCachedResponse(cached, input) {
|
|
10
32
|
let markdown = cached.markdown;
|
|
11
33
|
let sectionMatched;
|
|
@@ -31,23 +53,35 @@ function formatCachedResponse(cached, input) {
|
|
|
31
53
|
links: JSON.parse(cached.links || '[]'),
|
|
32
54
|
images: JSON.parse(cached.images || '[]'),
|
|
33
55
|
cached: true,
|
|
56
|
+
cached_at: cached.fetchedAt,
|
|
34
57
|
};
|
|
35
58
|
}
|
|
36
59
|
export async function handleFetch(input, router) {
|
|
60
|
+
const mode = resolveMode(input.mode);
|
|
37
61
|
try {
|
|
38
62
|
if (!input.force_refresh) {
|
|
39
63
|
const cached = getCachedContent(input.url);
|
|
40
|
-
if (cached &&
|
|
41
|
-
|
|
42
|
-
|
|
64
|
+
if (cached && (!input.actions || input.actions.length === 0)) {
|
|
65
|
+
const staleMaxSeconds = mode === 'fast' ? getConfig().fastStaleMaxHours * 3600 : 0;
|
|
66
|
+
const { usable, stale } = isCacheUsable(cached, { staleMaxSeconds });
|
|
67
|
+
if (usable) {
|
|
68
|
+
log.info('Serving from cache', { url: input.url, stale });
|
|
69
|
+
const out = formatCachedResponse(cached, input);
|
|
70
|
+
if (stale)
|
|
71
|
+
out.stale = true;
|
|
72
|
+
const fullMarkdown = out.markdown;
|
|
73
|
+
await attachEvidence(out, input, fullMarkdown);
|
|
74
|
+
return out;
|
|
75
|
+
}
|
|
43
76
|
}
|
|
44
77
|
}
|
|
45
78
|
const raw = await router.fetch(input.url, {
|
|
46
|
-
renderJs: input.render_js ?? 'auto',
|
|
79
|
+
renderJs: mode === 'fast' ? 'never' : (input.render_js ?? 'auto'),
|
|
47
80
|
useAuth: input.use_auth ?? false,
|
|
48
81
|
headers: input.headers,
|
|
49
82
|
screenshot: input.screenshot,
|
|
50
83
|
actions: input.actions,
|
|
84
|
+
mode,
|
|
51
85
|
});
|
|
52
86
|
const extraction = await extractContent(raw.html, raw.finalUrl, {
|
|
53
87
|
maxChars: input.max_chars,
|
|
@@ -81,7 +115,7 @@ export async function handleFetch(input, router) {
|
|
|
81
115
|
const finalMarkdown = input.max_content_chars !== undefined
|
|
82
116
|
? truncateSmartly(extraction.markdown, input.max_content_chars)
|
|
83
117
|
: extraction.markdown;
|
|
84
|
-
|
|
118
|
+
const out = {
|
|
85
119
|
url: raw.finalUrl,
|
|
86
120
|
title: extraction.title,
|
|
87
121
|
markdown: finalMarkdown,
|
|
@@ -91,12 +125,15 @@ export async function handleFetch(input, router) {
|
|
|
91
125
|
screenshot: raw.screenshot,
|
|
92
126
|
cached: false,
|
|
93
127
|
action_results: raw.actionResults,
|
|
128
|
+
...(raw.jsRequired ? { js_required: true } : {}),
|
|
94
129
|
...(changeResult?.changed ? {
|
|
95
130
|
changed: true,
|
|
96
131
|
previous_hash: changeResult.previousHash,
|
|
97
132
|
diff_summary: changeResult.diffSummary,
|
|
98
133
|
} : {}),
|
|
99
134
|
};
|
|
135
|
+
await attachEvidence(out, input, finalMarkdown);
|
|
136
|
+
return out;
|
|
100
137
|
}
|
|
101
138
|
catch (err) {
|
|
102
139
|
log.error('Fetch failed', { url: input.url, error: String(err) });
|
package/dist/tools/fetch.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/tools/fetch.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,
|
|
1
|
+
{"version":3,"file":"fetch.js","sourceRoot":"","sources":["../../src/tools/fetch.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClF,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC3E,OAAO,EAAE,yBAAyB,EAAE,MAAM,uBAAuB,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;AAElC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,KAAK,UAAU,cAAc,CAC3B,MAAmB,EACnB,KAAiB,EACjB,QAAgB;IAEhB,IAAI,CAAC,QAAQ;QAAE,OAAO;IACtB,MAAM,WAAW,GAAG,KAAK,CAAC,qBAAqB,IAAI,KAAK,CAAC;IACzD,MAAM,YAAY,GAAG,KAAK,CAAC,cAAc,IAAI,sBAAsB,CAAC;IACpE,MAAM,QAAQ,GAAG,MAAM,yBAAyB,CAC9C,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,GAAG,EAC1B,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,GAAG,EACV,QAAQ,EACR,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC,EAAE,CAC9B,CAAC;IACF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC;IACpD,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,CAAC,QAAQ,GAAG,EAAE,CAAC;IACvB,CAAC;SAAM,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QAC3B,MAAM,CAAC,QAAQ,GAAG,iBAAiB,CAAC,MAAM,CAAC,QAAQ,EAAE;YACnD,YAAY,EAAE,KAAK,CAAC,cAAc;YAClC,QAAQ,EAAE,KAAK,CAAC,SAAS;SAC1B,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAqB,EAAE,KAAiB;IACpE,IAAI,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IAC/B,IAAI,cAAmC,CAAC;IAExC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,aAAa,CAAC,CAAC;QAC5E,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC;QAC1B,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC;IAClC,CAAC;IAED,IAAI,KAAK,CAAC,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;QACzD,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC;IAED,IAAI,KAAK,CAAC,iBAAiB,KAAK,SAAS,EAAE,CAAC;QAC1C,QAAQ,GAAG,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC;IAChE,CAAC;IAED,OAAO;QACL,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,QAAQ;QACR,QAAQ,EAAE;YACR,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC;YACtC,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC7E;QACD,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC;QACvC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC;QACzC,MAAM,EAAE,IAAI;QACZ,SAAS,EAAE,MAAM,CAAC,SAAS;KAC5B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAiB,EACjB,MAAmB;IAEnB,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,MAAM,IAAI,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC7D,MAAM,eAAe,GAAG,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACnF,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,aAAa,CAAC,MAAM,EAAE,EAAE,eAAe,EAAE,CAAC,CAAC;gBACrE,IAAI,MAAM,EAAE,CAAC;oBACX,GAAG,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;oBAC1D,MAAM,GAAG,GAAG,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;oBAChD,IAAI,KAAK;wBAAE,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC;oBAC5B,MAAM,YAAY,GAAG,GAAG,CAAC,QAAQ,CAAC;oBAClC,MAAM,cAAc,CAAC,GAAG,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;oBAC/C,OAAO,GAAG,CAAC;gBACb,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YACxC,QAAQ,EAAE,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC;YACjE,OAAO,EAAE,KAAK,CAAC,QAAQ,IAAI,KAAK;YAChC,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,IAAI;SACL,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE;YAC9D,QAAQ,EAAE,KAAK,CAAC,SAAS;YACzB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,YAAY,EAAE,KAAK,CAAC,aAAa;YACjC,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,SAAS,EAAE,GAAG,CAAC,SAAS;SACzB,CAAC,CAAC;QAEH,IAAI,YAA2F,CAAC;QAChG,IAAI,CAAC;YACH,YAAY,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC;QACjE,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CAAC,yBAAyB,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACjF,CAAC;QAED,IAAI,CAAC;YACH,YAAY,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QAChC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CAAC,iCAAiC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzF,CAAC;QAED,IAAI,CAAC;YACH,MAAM,gBAAgB,GAAG,mBAAmB,EAAE,CAAC;YAC/C,IAAI,gBAAgB,CAAC,WAAW,EAAE,EAAE,CAAC;gBACnC,gBAAgB,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC;YACjE,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9D,CAAC;QAED,MAAM,aAAa,GAAG,KAAK,CAAC,iBAAiB,KAAK,SAAS;YACzD,CAAC,CAAC,eAAe,CAAC,UAAU,CAAC,QAAQ,EAAE,KAAK,CAAC,iBAAiB,CAAC;YAC/D,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC;QAExB,MAAM,GAAG,GAAgB;YACvB,GAAG,EAAE,GAAG,CAAC,QAAQ;YACjB,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,QAAQ,EAAE,aAAa;YACvB,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,MAAM,EAAE,KAAK;YACb,cAAc,EAAE,GAAG,CAAC,aAAa;YACjC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAChD,GAAG,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC1B,OAAO,EAAE,IAAI;gBACb,aAAa,EAAE,YAAY,CAAC,YAAY;gBACxC,YAAY,EAAE,YAAY,CAAC,WAAW;aACvC,CAAC,CAAC,CAAC,EAAE,CAAC;SACR,CAAC;QAEF,MAAM,cAAc,CAAC,GAAG,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;QAChD,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO;YACL,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,KAAK,EAAE,EAAE;YACT,QAAQ,EAAE,EAAE;YACZ,QAAQ,EAAE,EAAE;YACZ,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"find-similar.d.ts","sourceRoot":"","sources":["../../src/tools/find-similar.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"find-similar.d.ts","sourceRoot":"","sources":["../../src/tools/find-similar.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,YAAY,EAEb,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAcjE,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,gBAAgB,EACvB,OAAO,EAAE,YAAY,EAAE,EACvB,MAAM,EAAE,WAAW,EACnB,aAAa,CAAC,EAAE,aAAa,GAC5B,OAAO,CAAC,iBAAiB,CAAC,CA+C5B"}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { findSimilar } from '../search/find-similar.js';
|
|
2
|
+
import { buildEvidenceFromMarkdown, applyTokenBudget, applyAggregateMarkdownBudget, } from '../search/evidence.js';
|
|
2
3
|
import { createLogger } from '../logger.js';
|
|
3
4
|
const log = createLogger('search');
|
|
4
5
|
const MAX_RESULTS_CAP = 50;
|
|
6
|
+
const DEFAULT_MAX_TOKENS_OUT = 4000;
|
|
5
7
|
export async function handleFindSimilar(input, engines, router, backendStatus) {
|
|
6
8
|
try {
|
|
7
9
|
const url = input.url?.trim();
|
|
@@ -30,7 +32,9 @@ export async function handleFindSimilar(input, engines, router, backendStatus) {
|
|
|
30
32
|
includeCache: sanitizedInput.include_cache,
|
|
31
33
|
includeWeb: sanitizedInput.include_web,
|
|
32
34
|
});
|
|
33
|
-
|
|
35
|
+
const out = await findSimilar(sanitizedInput, engines, router, backendStatus);
|
|
36
|
+
await attachEvidence(out, input);
|
|
37
|
+
return out;
|
|
34
38
|
}
|
|
35
39
|
catch (err) {
|
|
36
40
|
log.error('handleFindSimilar failed', { error: String(err) });
|
|
@@ -45,4 +49,31 @@ export async function handleFindSimilar(input, engines, router, backendStatus) {
|
|
|
45
49
|
};
|
|
46
50
|
}
|
|
47
51
|
}
|
|
52
|
+
async function attachEvidence(out, input) {
|
|
53
|
+
if (out.results.length === 0)
|
|
54
|
+
return;
|
|
55
|
+
const includeFull = input.include_full_markdown ?? false;
|
|
56
|
+
const maxTokensOut = input.max_tokens_out ?? DEFAULT_MAX_TOKENS_OUT;
|
|
57
|
+
const query = input.concept?.trim() || input.url?.trim() || out.results[0].title;
|
|
58
|
+
const collected = [];
|
|
59
|
+
for (const r of out.results) {
|
|
60
|
+
if (!r.markdown)
|
|
61
|
+
continue;
|
|
62
|
+
const evs = await buildEvidenceFromMarkdown(query, r.title, r.url, r.markdown, {
|
|
63
|
+
maxItems: 1,
|
|
64
|
+
});
|
|
65
|
+
collected.push(...evs);
|
|
66
|
+
}
|
|
67
|
+
const budgeted = applyTokenBudget(collected, maxTokensOut);
|
|
68
|
+
if (budgeted.length > 0)
|
|
69
|
+
out.evidence = budgeted;
|
|
70
|
+
if (!includeFull) {
|
|
71
|
+
for (const r of out.results) {
|
|
72
|
+
r.markdown = '';
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
applyAggregateMarkdownBudget(out.results, (r) => r.markdown ?? '', (r, body) => { r.markdown = body; }, { maxTokensOut });
|
|
77
|
+
}
|
|
78
|
+
}
|
|
48
79
|
//# sourceMappingURL=find-similar.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"find-similar.js","sourceRoot":"","sources":["../../src/tools/find-similar.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"find-similar.js","sourceRoot":"","sources":["../../src/tools/find-similar.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EACL,yBAAyB,EACzB,gBAAgB,EAChB,4BAA4B,GAC7B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAEnC,MAAM,eAAe,GAAG,EAAE,CAAC;AAC3B,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,KAAuB,EACvB,OAAuB,EACvB,MAAmB,EACnB,aAA6B;IAE7B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;QAEtC,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;YACrB,OAAO;gBACL,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,MAAM;gBACd,UAAU,EAAE,CAAC;gBACb,WAAW,EAAE,CAAC;gBACd,mBAAmB,EAAE,KAAK;gBAC1B,KAAK,EAAE,wCAAwC;gBAC/C,aAAa,EAAE,CAAC;aACjB,CAAC;QACJ,CAAC;QAED,MAAM,cAAc,GAAqB;YACvC,GAAG,KAAK;YACR,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC5B,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,eAAe,CAAC;gBAC9C,CAAC,CAAC,SAAS;SACd,CAAC;QAEF,GAAG,CAAC,IAAI,CAAC,sBAAsB,EAAE;YAC/B,MAAM,EAAE,CAAC,CAAC,GAAG;YACb,UAAU,EAAE,CAAC,CAAC,OAAO;YACrB,UAAU,EAAE,cAAc,CAAC,WAAW;YACtC,YAAY,EAAE,cAAc,CAAC,aAAa;YAC1C,UAAU,EAAE,cAAc,CAAC,WAAW;SACvC,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,cAAc,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;QAC9E,MAAM,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACjC,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9D,OAAO;YACL,OAAO,EAAE,EAAE;YACX,MAAM,EAAE,MAAM;YACd,UAAU,EAAE,CAAC;YACb,WAAW,EAAE,CAAC;YACd,mBAAmB,EAAE,KAAK;YAC1B,KAAK,EAAE,+BAA+B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YACxF,aAAa,EAAE,CAAC;SACjB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,GAAsB,EACtB,KAAuB;IAEvB,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IACrC,MAAM,WAAW,GAAG,KAAK,CAAC,qBAAqB,IAAI,KAAK,CAAC;IACzD,MAAM,YAAY,GAAG,KAAK,CAAC,cAAc,IAAI,sBAAsB,CAAC;IACpE,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAEjF,MAAM,SAAS,GAAmB,EAAE,CAAC;IACrC,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,QAAQ;YAAE,SAAS;QAC1B,MAAM,GAAG,GAAG,MAAM,yBAAyB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE;YAC7E,QAAQ,EAAE,CAAC;SACZ,CAAC,CAAC;QACH,SAAS,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAC3D,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAEjD,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAC5B,CAAC,CAAC,QAAQ,GAAG,EAAE,CAAC;QAClB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,4BAA4B,CAC1B,GAAG,CAAC,OAAO,EACX,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,EACvB,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,EACnC,EAAE,YAAY,EAAE,CACjB,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"research.d.ts","sourceRoot":"","sources":["../../src/tools/research.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"research.d.ts","sourceRoot":"","sources":["../../src/tools/research.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAEV,aAAa,EACb,cAAc,EACd,YAAY,EACb,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAQnE,wBAAsB,cAAc,CAClC,KAAK,EAAE,aAAa,EACpB,OAAO,EAAE,YAAY,EAAE,EACvB,MAAM,EAAE,WAAW,EACnB,cAAc,CAAC,EAAE,OAAO,EACxB,MAAM,CAAC,EAAE,qBAAqB,GAC7B,OAAO,CAAC,cAAc,CAAC,CA6CzB"}
|
package/dist/tools/research.js
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import { createLogger } from '../logger.js';
|
|
2
2
|
import { runResearchPipeline } from '../research/pipeline.js';
|
|
3
|
+
import { buildEvidenceFromMarkdown, applyTokenBudget, applyAggregateMarkdownBudget, } from '../search/evidence.js';
|
|
4
|
+
import { applyOutputBudget } from '../search/truncate.js';
|
|
3
5
|
const log = createLogger('research');
|
|
4
6
|
const VALID_DEPTHS = new Set(['quick', 'standard', 'comprehensive']);
|
|
5
7
|
const MAX_SOURCES_LIMIT = 50;
|
|
8
|
+
const DEFAULT_MAX_TOKENS_OUT = 4000;
|
|
6
9
|
export async function handleResearch(input, engines, router, _backendStatus, server) {
|
|
7
10
|
const start = Date.now();
|
|
8
11
|
try {
|
|
@@ -25,7 +28,9 @@ export async function handleResearch(input, engines, router, _backendStatus, ser
|
|
|
25
28
|
depth: input.depth ?? 'standard',
|
|
26
29
|
max_sources: input.max_sources,
|
|
27
30
|
});
|
|
28
|
-
|
|
31
|
+
const out = await runResearchPipeline(input, engines, router, server);
|
|
32
|
+
await attachEvidence(out, input);
|
|
33
|
+
return out;
|
|
29
34
|
}
|
|
30
35
|
catch (err) {
|
|
31
36
|
log.error('research handler failed', {
|
|
@@ -35,6 +40,34 @@ export async function handleResearch(input, engines, router, _backendStatus, ser
|
|
|
35
40
|
return errorResult(err instanceof Error ? err.message : String(err), input, start);
|
|
36
41
|
}
|
|
37
42
|
}
|
|
43
|
+
async function attachEvidence(out, input) {
|
|
44
|
+
// Always honour max_tokens_out for the report text, regardless of sources.
|
|
45
|
+
if (input.max_tokens_out !== undefined && out.report) {
|
|
46
|
+
out.report = applyOutputBudget(out.report, { maxTokensOut: input.max_tokens_out });
|
|
47
|
+
}
|
|
48
|
+
if (out.sources.length === 0)
|
|
49
|
+
return;
|
|
50
|
+
const includeFull = input.include_full_markdown ?? false;
|
|
51
|
+
const maxTokensOut = input.max_tokens_out ?? DEFAULT_MAX_TOKENS_OUT;
|
|
52
|
+
const collected = [];
|
|
53
|
+
for (const s of out.sources) {
|
|
54
|
+
if (!s.markdown_content)
|
|
55
|
+
continue;
|
|
56
|
+
const evs = await buildEvidenceFromMarkdown(input.question, s.title, s.url, s.markdown_content, { maxItems: 1 });
|
|
57
|
+
collected.push(...evs);
|
|
58
|
+
}
|
|
59
|
+
const budgeted = applyTokenBudget(collected, maxTokensOut);
|
|
60
|
+
if (budgeted.length > 0)
|
|
61
|
+
out.evidence = budgeted;
|
|
62
|
+
if (!includeFull) {
|
|
63
|
+
for (const s of out.sources) {
|
|
64
|
+
s.markdown_content = '';
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
applyAggregateMarkdownBudget(out.sources, (s) => s.markdown_content ?? '', (s, body) => { s.markdown_content = body; }, { maxTokensOut });
|
|
69
|
+
}
|
|
70
|
+
}
|
|
38
71
|
function errorResult(error, input, start) {
|
|
39
72
|
return {
|
|
40
73
|
report: '',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"research.js","sourceRoot":"","sources":["../../src/tools/research.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"research.js","sourceRoot":"","sources":["../../src/tools/research.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EACL,yBAAyB,EACzB,gBAAgB,EAChB,4BAA4B,GAC7B,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAU1D,MAAM,GAAG,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC;AAErC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,eAAe,CAAC,CAAC,CAAC;AACrE,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAEpC,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAoB,EACpB,OAAuB,EACvB,MAAmB,EACnB,cAAwB,EACxB,MAA8B;IAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,IAAI,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChG,OAAO,WAAW,CAAC,qDAAqD,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QAC1F,CAAC;QAED,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YAClD,OAAO,WAAW,CAChB,+DAA+D,KAAK,CAAC,KAAK,GAAG,EAC7E,KAAK,EACL,KAAK,CACN,CAAC;QACJ,CAAC;QAED,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACpC,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,IAAI,KAAK,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;gBACnE,OAAO,WAAW,CAAC,uCAAuC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YAC5E,CAAC;YACD,IAAI,KAAK,CAAC,WAAW,GAAG,iBAAiB,EAAE,CAAC;gBAC1C,OAAO,WAAW,CAAC,+BAA+B,iBAAiB,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YACvF,CAAC;QACH,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE;YACpC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YACtC,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,UAAU;YAChC,WAAW,EAAE,KAAK,CAAC,WAAW;SAC/B,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,MAAM,mBAAmB,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QACtE,MAAM,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACjC,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,yBAAyB,EAAE;YACnC,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YACvC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC,CAAC;QACH,OAAO,WAAW,CAChB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAChD,KAAK,EACL,KAAK,CACN,CAAC;IACJ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,GAAmB,EAAE,KAAoB;IACrE,2EAA2E;IAC3E,IAAI,KAAK,CAAC,cAAc,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QACrD,GAAG,CAAC,MAAM,GAAG,iBAAiB,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,YAAY,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IACrC,MAAM,WAAW,GAAG,KAAK,CAAC,qBAAqB,IAAI,KAAK,CAAC;IACzD,MAAM,YAAY,GAAG,KAAK,CAAC,cAAc,IAAI,sBAAsB,CAAC;IAEpE,MAAM,SAAS,GAAmB,EAAE,CAAC;IACrC,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,gBAAgB;YAAE,SAAS;QAClC,MAAM,GAAG,GAAG,MAAM,yBAAyB,CACzC,KAAK,CAAC,QAAQ,EACd,CAAC,CAAC,KAAK,EACP,CAAC,CAAC,GAAG,EACL,CAAC,CAAC,gBAAgB,EAClB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAChB,CAAC;QACF,SAAS,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IAC3D,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAEjD,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAC5B,CAAC,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;SAAM,CAAC;QACN,4BAA4B,CAC1B,GAAG,CAAC,OAAO,EACX,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,IAAI,EAAE,EAC/B,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAAC,CAAC,EAC3C,EAAE,YAAY,EAAE,CACjB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,KAAa,EAAE,KAAoB,EAAE,KAAa;IACrE,OAAO;QACL,MAAM,EAAE,EAAE;QACV,SAAS,EAAE,EAAE;QACb,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,EAAE;QACf,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,UAAU;QAChC,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;QACjC,kBAAkB,EAAE,KAAK;QACzB,KAAK;KACN,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/tools/search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAoB,YAAY,EAAmB,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAChI,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAOjE,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/tools/search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,YAAY,EAAoB,YAAY,EAAmB,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAChI,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAOjE,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAqBnE,wBAAsB,YAAY,CAChC,KAAK,EAAE,WAAW,EAClB,OAAO,EAAE,YAAY,EAAE,EACvB,MAAM,EAAE,WAAW,EACnB,aAAa,CAAC,EAAE,aAAa,EAC7B,cAAc,CAAC,EAAE,qBAAqB,EACtC,UAAU,CAAC,EAAE,gBAAgB,GAC5B,OAAO,CAAC,YAAY,CAAC,CAiYvB"}
|