@framers/agentos 0.1.179 → 0.1.181
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/web-search/WebSearchService.d.ts +31 -0
- package/dist/web-search/WebSearchService.d.ts.map +1 -0
- package/dist/web-search/WebSearchService.js +108 -0
- package/dist/web-search/WebSearchService.js.map +1 -0
- package/dist/web-search/dedup.d.ts +19 -0
- package/dist/web-search/dedup.d.ts.map +1 -0
- package/dist/web-search/dedup.js +107 -0
- package/dist/web-search/dedup.js.map +1 -0
- package/dist/web-search/fusion.d.ts +29 -0
- package/dist/web-search/fusion.d.ts.map +1 -0
- package/dist/web-search/fusion.js +50 -0
- package/dist/web-search/fusion.js.map +1 -0
- package/dist/web-search/index.d.ts +27 -0
- package/dist/web-search/index.d.ts.map +1 -0
- package/dist/web-search/index.js +30 -0
- package/dist/web-search/index.js.map +1 -0
- package/dist/web-search/providers/BraveProvider.d.ts +15 -0
- package/dist/web-search/providers/BraveProvider.d.ts.map +1 -0
- package/dist/web-search/providers/BraveProvider.js +23 -0
- package/dist/web-search/providers/BraveProvider.js.map +1 -0
- package/dist/web-search/providers/FirecrawlProvider.d.ts +17 -0
- package/dist/web-search/providers/FirecrawlProvider.d.ts.map +1 -0
- package/dist/web-search/providers/FirecrawlProvider.js +38 -0
- package/dist/web-search/providers/FirecrawlProvider.js.map +1 -0
- package/dist/web-search/providers/SerperProvider.d.ts +15 -0
- package/dist/web-search/providers/SerperProvider.d.ts.map +1 -0
- package/dist/web-search/providers/SerperProvider.js +27 -0
- package/dist/web-search/providers/SerperProvider.js.map +1 -0
- package/dist/web-search/providers/TavilyProvider.d.ts +15 -0
- package/dist/web-search/providers/TavilyProvider.d.ts.map +1 -0
- package/dist/web-search/providers/TavilyProvider.js +32 -0
- package/dist/web-search/providers/TavilyProvider.js.map +1 -0
- package/dist/web-search/providers/index.d.ts +5 -0
- package/dist/web-search/providers/index.d.ts.map +1 -0
- package/dist/web-search/providers/index.js +5 -0
- package/dist/web-search/providers/index.js.map +1 -0
- package/dist/web-search/types.d.ts +61 -0
- package/dist/web-search/types.d.ts.map +1 -0
- package/dist/web-search/types.js +2 -0
- package/dist/web-search/types.js.map +1 -0
- package/package.json +31 -1
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module web-search/WebSearchService
|
|
3
|
+
*
|
|
4
|
+
* Multi-provider web search orchestrator.
|
|
5
|
+
* Pipeline: parallel search → semantic dedup → RRF fusion → optional neural reranking.
|
|
6
|
+
*/
|
|
7
|
+
import type { IWebSearchProvider, WebSearchConfig, EnrichedSearchResult } from './types';
|
|
8
|
+
export declare class WebSearchService {
|
|
9
|
+
private providers;
|
|
10
|
+
private readonly config;
|
|
11
|
+
constructor(config?: WebSearchConfig);
|
|
12
|
+
/** Register a search provider. */
|
|
13
|
+
registerProvider(provider: IWebSearchProvider): void;
|
|
14
|
+
/** Check if any providers are registered and available. */
|
|
15
|
+
hasProviders(): boolean;
|
|
16
|
+
/** List registered provider IDs. */
|
|
17
|
+
listProviders(): string[];
|
|
18
|
+
/**
|
|
19
|
+
* Full search pipeline: parallel query → dedup → RRF → optional rerank.
|
|
20
|
+
*
|
|
21
|
+
* @param query - Search query string
|
|
22
|
+
* @param options - Override maxResults or disable reranking
|
|
23
|
+
*/
|
|
24
|
+
search(query: string, options?: {
|
|
25
|
+
maxResults?: number;
|
|
26
|
+
rerank?: boolean;
|
|
27
|
+
}): Promise<EnrichedSearchResult[]>;
|
|
28
|
+
/** Convenience: return a function matching the legacy SearchFn signature. */
|
|
29
|
+
asSearchFn(): (query: string) => Promise<EnrichedSearchResult[]>;
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=WebSearchService.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"WebSearchService.d.ts","sourceRoot":"","sources":["../../src/web-search/WebSearchService.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,KAAK,EACV,kBAAkB,EAClB,eAAe,EACf,oBAAoB,EAErB,MAAM,SAAS,CAAC;AAIjB,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,SAAS,CAA4B;IAC7C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAEH;gBAER,MAAM,CAAC,EAAE,eAAe;IAQpC,kCAAkC;IAClC,gBAAgB,CAAC,QAAQ,EAAE,kBAAkB,GAAG,IAAI;IAIpD,2DAA2D;IAC3D,YAAY,IAAI,OAAO;IAIvB,oCAAoC;IACpC,aAAa,IAAI,MAAM,EAAE;IAIzB;;;;;OAKG;IACG,MAAM,CACV,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE,GAClD,OAAO,CAAC,oBAAoB,EAAE,CAAC;IAsFlC,6EAA6E;IAC7E,UAAU,IAAI,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,oBAAoB,EAAE,CAAC;CAGjE"}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { normalizeUrl, computeWeightedRRF } from './fusion.js';
|
|
2
|
+
import { semanticDedup } from './dedup.js';
|
|
3
|
+
export class WebSearchService {
|
|
4
|
+
constructor(config) {
|
|
5
|
+
this.providers = [];
|
|
6
|
+
this.config = {
|
|
7
|
+
dedupThreshold: config?.dedupThreshold ?? 0.85,
|
|
8
|
+
maxResults: config?.maxResults ?? 12,
|
|
9
|
+
...config,
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
/** Register a search provider. */
|
|
13
|
+
registerProvider(provider) {
|
|
14
|
+
this.providers.push(provider);
|
|
15
|
+
}
|
|
16
|
+
/** Check if any providers are registered and available. */
|
|
17
|
+
hasProviders() {
|
|
18
|
+
return this.providers.some((p) => p.isAvailable());
|
|
19
|
+
}
|
|
20
|
+
/** List registered provider IDs. */
|
|
21
|
+
listProviders() {
|
|
22
|
+
return this.providers.map((p) => p.providerId);
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Full search pipeline: parallel query → dedup → RRF → optional rerank.
|
|
26
|
+
*
|
|
27
|
+
* @param query - Search query string
|
|
28
|
+
* @param options - Override maxResults or disable reranking
|
|
29
|
+
*/
|
|
30
|
+
async search(query, options) {
|
|
31
|
+
const available = this.providers.filter((p) => p.isAvailable());
|
|
32
|
+
if (available.length === 0)
|
|
33
|
+
return [];
|
|
34
|
+
// ── Parallel search across all available providers ──
|
|
35
|
+
const settled = await Promise.allSettled(available.map((p) => p.search(query)));
|
|
36
|
+
// ── Collect as RRF candidates ──
|
|
37
|
+
const allCandidates = [];
|
|
38
|
+
for (let i = 0; i < available.length; i++) {
|
|
39
|
+
const outcome = settled[i];
|
|
40
|
+
if (outcome.status !== 'fulfilled')
|
|
41
|
+
continue;
|
|
42
|
+
const provider = available[i];
|
|
43
|
+
for (let rank = 0; rank < outcome.value.length; rank++) {
|
|
44
|
+
const r = outcome.value[rank];
|
|
45
|
+
allCandidates.push({
|
|
46
|
+
url: r.url,
|
|
47
|
+
normalizedUrl: normalizeUrl(r.url),
|
|
48
|
+
title: r.title,
|
|
49
|
+
snippet: r.snippet,
|
|
50
|
+
content: r.content,
|
|
51
|
+
providerRanks: new Map([[provider.providerId, rank]]),
|
|
52
|
+
providerSources: [provider.providerId],
|
|
53
|
+
rrfScore: 0,
|
|
54
|
+
relevanceScore: r.relevanceScore,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
if (allCandidates.length === 0)
|
|
59
|
+
return [];
|
|
60
|
+
// ── Semantic dedup ──
|
|
61
|
+
const deduped = semanticDedup(allCandidates, this.config.dedupThreshold);
|
|
62
|
+
// ── RRF scoring with provider weights ──
|
|
63
|
+
const weights = {};
|
|
64
|
+
for (const p of available) {
|
|
65
|
+
weights[p.providerId] = p.weight;
|
|
66
|
+
}
|
|
67
|
+
computeWeightedRRF(deduped, weights);
|
|
68
|
+
// ── Slice to max results ──
|
|
69
|
+
const maxResults = options?.maxResults ?? this.config.maxResults;
|
|
70
|
+
const enriched = deduped.slice(0, maxResults).map((c) => ({
|
|
71
|
+
url: c.url,
|
|
72
|
+
title: c.title,
|
|
73
|
+
snippet: c.snippet,
|
|
74
|
+
content: c.content,
|
|
75
|
+
relevanceScore: c.relevanceScore,
|
|
76
|
+
providerSources: c.providerSources,
|
|
77
|
+
rrfScore: c.rrfScore,
|
|
78
|
+
}));
|
|
79
|
+
// ── Neural reranking (optional) ──
|
|
80
|
+
const shouldRerank = options?.rerank !== false;
|
|
81
|
+
if (shouldRerank && this.config.reranker && this.config.rerankChain) {
|
|
82
|
+
try {
|
|
83
|
+
const chunks = enriched.map((r, i) => ({
|
|
84
|
+
id: `search-${i}`,
|
|
85
|
+
originalDocumentId: r.url,
|
|
86
|
+
content: r.content ?? r.snippet,
|
|
87
|
+
relevanceScore: r.rrfScore,
|
|
88
|
+
metadata: { url: r.url, title: r.title },
|
|
89
|
+
}));
|
|
90
|
+
const reranked = await this.config.reranker.rerankChain(query, chunks, this.config.rerankChain);
|
|
91
|
+
const resultMap = new Map(enriched.map((r, i) => [`search-${i}`, r]));
|
|
92
|
+
return reranked.map((chunk) => {
|
|
93
|
+
const original = resultMap.get(chunk.id);
|
|
94
|
+
return { ...original, rerankScore: chunk.relevanceScore ?? 0 };
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
// Reranking failure is non-fatal
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return enriched;
|
|
102
|
+
}
|
|
103
|
+
/** Convenience: return a function matching the legacy SearchFn signature. */
|
|
104
|
+
asSearchFn() {
|
|
105
|
+
return (query) => this.search(query);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
//# sourceMappingURL=WebSearchService.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"WebSearchService.js","sourceRoot":"","sources":["../../src/web-search/WebSearchService.ts"],"names":[],"mappings":"AAYA,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,MAAM,OAAO,gBAAgB;IAM3B,YAAY,MAAwB;QAL5B,cAAS,GAAyB,EAAE,CAAC;QAM3C,IAAI,CAAC,MAAM,GAAG;YACZ,cAAc,EAAE,MAAM,EAAE,cAAc,IAAI,IAAI;YAC9C,UAAU,EAAE,MAAM,EAAE,UAAU,IAAI,EAAE;YACpC,GAAG,MAAM;SACV,CAAC;IACJ,CAAC;IAED,kCAAkC;IAClC,gBAAgB,CAAC,QAA4B;QAC3C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,2DAA2D;IAC3D,YAAY;QACV,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,oCAAoC;IACpC,aAAa;QACX,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;IACjD,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,MAAM,CACV,KAAa,EACb,OAAmD;QAEnD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QAChE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,uDAAuD;QACvD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CACtC,CAAC;QAEF,kCAAkC;QAClC,MAAM,aAAa,GAAmB,EAAE,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAC3B,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW;gBAAE,SAAS;YAC7C,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YAC9B,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;gBACvD,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC9B,aAAa,CAAC,IAAI,CAAC;oBACjB,GAAG,EAAE,CAAC,CAAC,GAAG;oBACV,aAAa,EAAE,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC;oBAClC,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,aAAa,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC;oBACrD,eAAe,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC;oBACtC,QAAQ,EAAE,CAAC;oBACX,cAAc,EAAE,CAAC,CAAC,cAAc;iBACjC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAE1C,uBAAuB;QACvB,MAAM,OAAO,GAAG,aAAa,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QAEzE,0CAA0C;QAC1C,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YAC1B,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QACnC,CAAC;QACD,kBAAkB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAErC,6BAA6B;QAC7B,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;QACjE,MAAM,QAAQ,GAA2B,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAChF,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,cAAc,EAAE,CAAC,CAAC,cAAc;YAChC,eAAe,EAAE,CAAC,CAAC,eAAe;YAClC,QAAQ,EAAE,CAAC,CAAC,QAAQ;SACrB,CAAC,CAAC,CAAC;QAEJ,oCAAoC;QACpC,MAAM,YAAY,GAAG,OAAO,EAAE,MAAM,KAAK,KAAK,CAAC;QAC/C,IAAI,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YACpE,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;oBACrC,EAAE,EAAE,UAAU,CAAC,EAAE;oBACjB,kBAAkB,EAAE,CAAC,CAAC,GAAG;oBACzB,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO;oBAC/B,cAAc,EAAE,CAAC,CAAC,QAAQ;oBAC1B,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE;iBACzC,CAAC,CAAC,CAAC;gBAEJ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CACrD,KAAK,EACL,MAAM,EACN,IAAI,CAAC,MAAM,CAAC,WAAW,CACxB,CAAC;gBAEF,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBACtE,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;oBAC5B,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAE,CAAC;oBAC1C,OAAO,EAAE,GAAG,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,cAAc,IAAI,CAAC,EAAE,CAAC;gBACjE,CAAC,CAAC,CAAC;YACL,CAAC;YAAC,MAAM,CAAC;gBACP,iCAAiC;YACnC,CAAC;QACH,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,6EAA6E;IAC7E,UAAU;QACR,OAAO,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC/C,CAAC;CACF"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { RRFCandidate } from './types';
|
|
2
|
+
/** Build vocabulary index from all texts. */
|
|
3
|
+
export declare function buildVocabulary(texts: string[]): Map<string, number>;
|
|
4
|
+
/** Build a TF vector normalized by document length. */
|
|
5
|
+
export declare function buildTfVector(text: string, vocabulary: Map<string, number>): number[];
|
|
6
|
+
/**
|
|
7
|
+
* Semantic dedup: merge near-duplicate results across providers.
|
|
8
|
+
*
|
|
9
|
+
* Two-pass:
|
|
10
|
+
* 1. URL normalization merge (exact match after stripping protocol/www/tracking)
|
|
11
|
+
* 2. TF-IDF cosine similarity on snippets (catches same article at different URLs)
|
|
12
|
+
*
|
|
13
|
+
* Keeps the version with richest content (prefers full markdown over snippet-only).
|
|
14
|
+
*
|
|
15
|
+
* @param candidates - Raw candidates from all providers
|
|
16
|
+
* @param threshold - Cosine similarity threshold for dedup (default 0.85)
|
|
17
|
+
*/
|
|
18
|
+
export declare function semanticDedup(candidates: RRFCandidate[], threshold?: number): RRFCandidate[];
|
|
19
|
+
//# sourceMappingURL=dedup.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["../../src/web-search/dedup.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAG5C,6CAA6C;AAC7C,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAQpE;AAED,uDAAuD;AACvD,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,EAAE,CASrF;AAwBD;;;;;;;;;;;GAWG;AACH,wBAAgB,aAAa,CAC3B,UAAU,EAAE,YAAY,EAAE,EAC1B,SAAS,GAAE,MAAa,GACvB,YAAY,EAAE,CA4ChB"}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module web-search/dedup
|
|
3
|
+
*
|
|
4
|
+
* Semantic deduplication for web search results.
|
|
5
|
+
* Uses TF-IDF cosine similarity to detect near-duplicates across providers.
|
|
6
|
+
* Pure functions, no IO.
|
|
7
|
+
*/
|
|
8
|
+
import { cosineSimilarity } from '../rag/citation/cosine.js';
|
|
9
|
+
import { normalizeUrl } from './fusion.js';
|
|
10
|
+
/** Build vocabulary index from all texts. */
|
|
11
|
+
export function buildVocabulary(texts) {
|
|
12
|
+
const vocab = new Map();
|
|
13
|
+
for (const text of texts) {
|
|
14
|
+
for (const word of text.toLowerCase().split(/\W+/).filter(Boolean)) {
|
|
15
|
+
if (!vocab.has(word))
|
|
16
|
+
vocab.set(word, vocab.size);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
return vocab;
|
|
20
|
+
}
|
|
21
|
+
/** Build a TF vector normalized by document length. */
|
|
22
|
+
export function buildTfVector(text, vocabulary) {
|
|
23
|
+
const words = text.toLowerCase().split(/\W+/).filter(Boolean);
|
|
24
|
+
const vec = new Array(vocabulary.size).fill(0);
|
|
25
|
+
for (const w of words) {
|
|
26
|
+
const idx = vocabulary.get(w);
|
|
27
|
+
if (idx !== undefined)
|
|
28
|
+
vec[idx]++;
|
|
29
|
+
}
|
|
30
|
+
const len = words.length || 1;
|
|
31
|
+
return vec.map((v) => v / len);
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Merge a source candidate into a target candidate.
|
|
35
|
+
* Combines provider ranks and keeps the richest content.
|
|
36
|
+
*/
|
|
37
|
+
function mergeCandidates(target, source) {
|
|
38
|
+
for (const [prov, rank] of source.providerRanks) {
|
|
39
|
+
if (!target.providerRanks.has(prov)) {
|
|
40
|
+
target.providerRanks.set(prov, rank);
|
|
41
|
+
target.providerSources.push(prov);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (source.content && !target.content) {
|
|
45
|
+
target.content = source.content;
|
|
46
|
+
}
|
|
47
|
+
if (source.relevanceScore &&
|
|
48
|
+
(!target.relevanceScore || source.relevanceScore > target.relevanceScore)) {
|
|
49
|
+
target.relevanceScore = source.relevanceScore;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Semantic dedup: merge near-duplicate results across providers.
|
|
54
|
+
*
|
|
55
|
+
* Two-pass:
|
|
56
|
+
* 1. URL normalization merge (exact match after stripping protocol/www/tracking)
|
|
57
|
+
* 2. TF-IDF cosine similarity on snippets (catches same article at different URLs)
|
|
58
|
+
*
|
|
59
|
+
* Keeps the version with richest content (prefers full markdown over snippet-only).
|
|
60
|
+
*
|
|
61
|
+
* @param candidates - Raw candidates from all providers
|
|
62
|
+
* @param threshold - Cosine similarity threshold for dedup (default 0.85)
|
|
63
|
+
*/
|
|
64
|
+
export function semanticDedup(candidates, threshold = 0.85) {
|
|
65
|
+
if (candidates.length <= 1)
|
|
66
|
+
return candidates;
|
|
67
|
+
// Pass 1: merge by normalized URL
|
|
68
|
+
const urlMap = new Map();
|
|
69
|
+
for (const c of candidates) {
|
|
70
|
+
const normUrl = c.normalizedUrl || normalizeUrl(c.url);
|
|
71
|
+
const existing = urlMap.get(normUrl);
|
|
72
|
+
if (existing) {
|
|
73
|
+
mergeCandidates(existing, c);
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
urlMap.set(normUrl, { ...c, normalizedUrl: normUrl });
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
const urlDeduped = Array.from(urlMap.values());
|
|
80
|
+
if (urlDeduped.length <= 1)
|
|
81
|
+
return urlDeduped;
|
|
82
|
+
// Pass 2: semantic similarity on snippets
|
|
83
|
+
const snippets = urlDeduped.map((c) => c.snippet);
|
|
84
|
+
const vocab = buildVocabulary(snippets);
|
|
85
|
+
if (vocab.size === 0)
|
|
86
|
+
return urlDeduped;
|
|
87
|
+
const vectors = snippets.map((s) => buildTfVector(s, vocab));
|
|
88
|
+
const merged = new Set();
|
|
89
|
+
const result = [];
|
|
90
|
+
for (let i = 0; i < urlDeduped.length; i++) {
|
|
91
|
+
if (merged.has(i))
|
|
92
|
+
continue;
|
|
93
|
+
const current = urlDeduped[i];
|
|
94
|
+
for (let j = i + 1; j < urlDeduped.length; j++) {
|
|
95
|
+
if (merged.has(j))
|
|
96
|
+
continue;
|
|
97
|
+
const sim = cosineSimilarity(vectors[i], vectors[j]);
|
|
98
|
+
if (sim >= threshold) {
|
|
99
|
+
mergeCandidates(current, urlDeduped[j]);
|
|
100
|
+
merged.add(j);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
result.push(current);
|
|
104
|
+
}
|
|
105
|
+
return result;
|
|
106
|
+
}
|
|
107
|
+
//# sourceMappingURL=dedup.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dedup.js","sourceRoot":"","sources":["../../src/web-search/dedup.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE1D,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAExC,6CAA6C;AAC7C,MAAM,UAAU,eAAe,CAAC,KAAe;IAC7C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACnE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,aAAa,CAAC,IAAY,EAAE,UAA+B;IACzE,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC9D,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC9B,IAAI,GAAG,KAAK,SAAS;YAAE,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;IACpC,CAAC;IACD,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IAC9B,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;AACjC,CAAC;AAED;;;GAGG;AACH,SAAS,eAAe,CAAC,MAAoB,EAAE,MAAoB;IACjE,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;QAChD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACpC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACrC,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAClC,CAAC;IACD,IACE,MAAM,CAAC,cAAc;QACrB,CAAC,CAAC,MAAM,CAAC,cAAc,IAAI,MAAM,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC,EACzE,CAAC;QACD,MAAM,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;IAChD,CAAC;AACH,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,aAAa,CAC3B,UAA0B,EAC1B,YAAoB,IAAI;IAExB,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,UAAU,CAAC;IAE9C,kCAAkC;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,CAAC,CAAC,aAAa,IAAI,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACrC,IAAI,QAAQ,EAAE,CAAC;YACb,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,GAAG,CAAC,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/C,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,UAAU,CAAC;IAE9C,0CAA0C;IAC1C,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAClD,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,UAAU,CAAC;IAExC,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,MAAM,MAAM,GAAmB,EAAE,CAAC;IAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,SAAS;QAC5B,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,SAAS;YAC5B,MAAM,GAAG,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YACrD,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;gBACrB,eAAe,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAChB,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module web-search/fusion
|
|
3
|
+
*
|
|
4
|
+
* URL normalization and Reciprocal Rank Fusion (RRF) scoring.
|
|
5
|
+
* Pure functions, no IO.
|
|
6
|
+
*/
|
|
7
|
+
import type { RRFCandidate } from './types';
|
|
8
|
+
/** Normalize a URL for dedup comparison. */
|
|
9
|
+
export declare function normalizeUrl(url: string): string;
|
|
10
|
+
/**
|
|
11
|
+
* Compute RRF scores for candidates.
|
|
12
|
+
*
|
|
13
|
+
* Formula: score(d) = Σ (provider_weight / (k + rank_in_provider))
|
|
14
|
+
* Results appearing in multiple providers get naturally boosted.
|
|
15
|
+
*
|
|
16
|
+
* @param candidates - Candidates with providerRanks populated
|
|
17
|
+
* @param k - RRF constant (default 60)
|
|
18
|
+
* @returns Same candidates with rrfScore computed, sorted descending
|
|
19
|
+
*/
|
|
20
|
+
export declare function computeRRF(candidates: RRFCandidate[], k?: number): RRFCandidate[];
|
|
21
|
+
/**
|
|
22
|
+
* Compute weighted RRF scores using per-provider weights.
|
|
23
|
+
*
|
|
24
|
+
* @param candidates - Candidates with providerRanks populated
|
|
25
|
+
* @param weights - Provider ID → weight multiplier map
|
|
26
|
+
* @param k - RRF constant (default 60)
|
|
27
|
+
*/
|
|
28
|
+
export declare function computeWeightedRRF(candidates: RRFCandidate[], weights: Record<string, number>, k?: number): RRFCandidate[];
|
|
29
|
+
//# sourceMappingURL=fusion.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fusion.d.ts","sourceRoot":"","sources":["../../src/web-search/fusion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAK5C,4CAA4C;AAC5C,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAOhD;AAED;;;;;;;;;GASG;AACH,wBAAgB,UAAU,CAAC,UAAU,EAAE,YAAY,EAAE,EAAE,CAAC,GAAE,MAAkB,GAAG,YAAY,EAAE,CAW5F;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAChC,UAAU,EAAE,YAAY,EAAE,EAC1B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC/B,CAAC,GAAE,MAAkB,GACpB,YAAY,EAAE,CAUhB"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/** Default RRF constant — prevents top-ranked results from dominating. */
|
|
2
|
+
const DEFAULT_K = 60;
|
|
3
|
+
/** Normalize a URL for dedup comparison. */
|
|
4
|
+
export function normalizeUrl(url) {
|
|
5
|
+
return url
|
|
6
|
+
.replace(/^https?:\/\//, '')
|
|
7
|
+
.replace(/^www\./, '')
|
|
8
|
+
.replace(/\/+$/, '')
|
|
9
|
+
.replace(/[?&](utm_\w+|ref|source|fbclid|gclid)=[^&]*/g, '')
|
|
10
|
+
.replace(/\?$/, '');
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Compute RRF scores for candidates.
|
|
14
|
+
*
|
|
15
|
+
* Formula: score(d) = Σ (provider_weight / (k + rank_in_provider))
|
|
16
|
+
* Results appearing in multiple providers get naturally boosted.
|
|
17
|
+
*
|
|
18
|
+
* @param candidates - Candidates with providerRanks populated
|
|
19
|
+
* @param k - RRF constant (default 60)
|
|
20
|
+
* @returns Same candidates with rrfScore computed, sorted descending
|
|
21
|
+
*/
|
|
22
|
+
export function computeRRF(candidates, k = DEFAULT_K) {
|
|
23
|
+
for (const candidate of candidates) {
|
|
24
|
+
let score = 0;
|
|
25
|
+
for (const [, rank] of candidate.providerRanks) {
|
|
26
|
+
// Weight is baked into the candidate via the provider that created it
|
|
27
|
+
score += 1 / (k + rank);
|
|
28
|
+
}
|
|
29
|
+
candidate.rrfScore = score;
|
|
30
|
+
}
|
|
31
|
+
return candidates.sort((a, b) => b.rrfScore - a.rrfScore);
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Compute weighted RRF scores using per-provider weights.
|
|
35
|
+
*
|
|
36
|
+
* @param candidates - Candidates with providerRanks populated
|
|
37
|
+
* @param weights - Provider ID → weight multiplier map
|
|
38
|
+
* @param k - RRF constant (default 60)
|
|
39
|
+
*/
|
|
40
|
+
export function computeWeightedRRF(candidates, weights, k = DEFAULT_K) {
|
|
41
|
+
for (const candidate of candidates) {
|
|
42
|
+
let score = 0;
|
|
43
|
+
for (const [provider, rank] of candidate.providerRanks) {
|
|
44
|
+
score += (weights[provider] ?? 1.0) / (k + rank);
|
|
45
|
+
}
|
|
46
|
+
candidate.rrfScore = score;
|
|
47
|
+
}
|
|
48
|
+
return candidates.sort((a, b) => b.rrfScore - a.rrfScore);
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=fusion.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fusion.js","sourceRoot":"","sources":["../../src/web-search/fusion.ts"],"names":[],"mappings":"AAQA,0EAA0E;AAC1E,MAAM,SAAS,GAAG,EAAE,CAAC;AAErB,4CAA4C;AAC5C,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,OAAO,GAAG;SACP,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;SACrB,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;SACnB,OAAO,CAAC,8CAA8C,EAAE,EAAE,CAAC;SAC3D,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AACxB,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,UAAU,CAAC,UAA0B,EAAE,IAAY,SAAS;IAC1E,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,CAAC,EAAE,IAAI,CAAC,IAAI,SAAS,CAAC,aAAa,EAAE,CAAC;YAC/C,sEAAsE;YACtE,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QAC1B,CAAC;QACD,SAAS,CAAC,QAAQ,GAAG,KAAK,CAAC;IAC7B,CAAC;IAED,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;AAC5D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAChC,UAA0B,EAC1B,OAA+B,EAC/B,IAAY,SAAS;IAErB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,SAAS,CAAC,aAAa,EAAE,CAAC;YACvD,KAAK,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;QACnD,CAAC;QACD,SAAS,CAAC,QAAQ,GAAG,KAAK,CAAC;IAC7B,CAAC;IAED,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;AAC5D,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module @framers/agentos/web-search
|
|
3
|
+
*
|
|
4
|
+
* Multi-provider web search with RRF fusion, semantic dedup, and neural reranking.
|
|
5
|
+
* Supports Firecrawl (primary), Tavily, Serper, and Brave search providers.
|
|
6
|
+
*
|
|
7
|
+
* @example
|
|
8
|
+
* ```typescript
|
|
9
|
+
* import { WebSearchService, FirecrawlProvider, TavilyProvider } from '../web-search';
|
|
10
|
+
*
|
|
11
|
+
* const service = new WebSearchService();
|
|
12
|
+
* service.registerProvider(new FirecrawlProvider(process.env.FIRECRAWL_API_KEY!));
|
|
13
|
+
* service.registerProvider(new TavilyProvider(process.env.TAVILY_API_KEY!));
|
|
14
|
+
*
|
|
15
|
+
* const results = await service.search('quantum computing breakthroughs 2026');
|
|
16
|
+
* // Results are deduped, RRF-scored, and optionally reranked
|
|
17
|
+
* ```
|
|
18
|
+
*/
|
|
19
|
+
export type { IWebSearchProvider, WebSearchResult, EnrichedSearchResult, WebSearchConfig, RRFCandidate, } from './types';
|
|
20
|
+
export { WebSearchService } from './WebSearchService';
|
|
21
|
+
export { FirecrawlProvider } from './providers/FirecrawlProvider';
|
|
22
|
+
export { TavilyProvider } from './providers/TavilyProvider';
|
|
23
|
+
export { SerperProvider } from './providers/SerperProvider';
|
|
24
|
+
export { BraveProvider } from './providers/BraveProvider';
|
|
25
|
+
export { normalizeUrl, computeRRF, computeWeightedRRF } from './fusion';
|
|
26
|
+
export { semanticDedup, buildVocabulary, buildTfVector } from './dedup';
|
|
27
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/web-search/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAGH,YAAY,EACV,kBAAkB,EAClB,eAAe,EACf,oBAAoB,EACpB,eAAe,EACf,YAAY,GACb,MAAM,SAAS,CAAC;AAGjB,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAGtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAG1D,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAGxE,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module @framers/agentos/web-search
|
|
3
|
+
*
|
|
4
|
+
* Multi-provider web search with RRF fusion, semantic dedup, and neural reranking.
|
|
5
|
+
* Supports Firecrawl (primary), Tavily, Serper, and Brave search providers.
|
|
6
|
+
*
|
|
7
|
+
* @example
|
|
8
|
+
* ```typescript
|
|
9
|
+
* import { WebSearchService, FirecrawlProvider, TavilyProvider } from '../web-search/index.js';
|
|
10
|
+
*
|
|
11
|
+
* const service = new WebSearchService();
|
|
12
|
+
* service.registerProvider(new FirecrawlProvider(process.env.FIRECRAWL_API_KEY!));
|
|
13
|
+
* service.registerProvider(new TavilyProvider(process.env.TAVILY_API_KEY!));
|
|
14
|
+
*
|
|
15
|
+
* const results = await service.search('quantum computing breakthroughs 2026');
|
|
16
|
+
* // Results are deduped, RRF-scored, and optionally reranked
|
|
17
|
+
* ```
|
|
18
|
+
*/
|
|
19
|
+
// Service
|
|
20
|
+
export { WebSearchService } from './WebSearchService.js';
|
|
21
|
+
// Providers
|
|
22
|
+
export { FirecrawlProvider } from './providers/FirecrawlProvider.js';
|
|
23
|
+
export { TavilyProvider } from './providers/TavilyProvider.js';
|
|
24
|
+
export { SerperProvider } from './providers/SerperProvider.js';
|
|
25
|
+
export { BraveProvider } from './providers/BraveProvider.js';
|
|
26
|
+
// Fusion utilities
|
|
27
|
+
export { normalizeUrl, computeRRF, computeWeightedRRF } from './fusion.js';
|
|
28
|
+
// Dedup utilities
|
|
29
|
+
export { semanticDedup, buildVocabulary, buildTfVector } from './dedup.js';
|
|
30
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/web-search/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAWH,UAAU;AACV,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAEtD,YAAY;AACZ,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAE1D,mBAAmB;AACnB,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAExE,kBAAkB;AAClB,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module web-search/providers/BraveProvider
|
|
3
|
+
*
|
|
4
|
+
* Brave Search API provider.
|
|
5
|
+
*/
|
|
6
|
+
import type { IWebSearchProvider, WebSearchResult } from '../types';
|
|
7
|
+
export declare class BraveProvider implements IWebSearchProvider {
|
|
8
|
+
private readonly apiKey;
|
|
9
|
+
readonly providerId: "brave";
|
|
10
|
+
readonly weight = 1;
|
|
11
|
+
constructor(apiKey: string);
|
|
12
|
+
isAvailable(): boolean;
|
|
13
|
+
search(query: string, limit?: number): Promise<WebSearchResult[]>;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=BraveProvider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BraveProvider.d.ts","sourceRoot":"","sources":["../../../src/web-search/providers/BraveProvider.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,KAAK,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAEpE,qBAAa,aAAc,YAAW,kBAAkB;IAI1C,OAAO,CAAC,QAAQ,CAAC,MAAM;IAHnC,QAAQ,CAAC,UAAU,EAAG,OAAO,CAAU;IACvC,QAAQ,CAAC,MAAM,KAAO;gBAEO,MAAM,EAAE,MAAM;IAE3C,WAAW,IAAI,OAAO;IAIhB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAU,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CAa3E"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export class BraveProvider {
|
|
2
|
+
constructor(apiKey) {
|
|
3
|
+
this.apiKey = apiKey;
|
|
4
|
+
this.providerId = 'brave';
|
|
5
|
+
this.weight = 1.0;
|
|
6
|
+
}
|
|
7
|
+
isAvailable() {
|
|
8
|
+
return this.apiKey.length > 0;
|
|
9
|
+
}
|
|
10
|
+
async search(query, limit = 5) {
|
|
11
|
+
const params = new URLSearchParams({ q: query, count: String(limit) });
|
|
12
|
+
const res = await fetch(`https://api.search.brave.com/res/v1/web/search?${params}`, {
|
|
13
|
+
headers: { 'X-Subscription-Token': this.apiKey },
|
|
14
|
+
});
|
|
15
|
+
const data = await res.json();
|
|
16
|
+
return (data.web?.results ?? []).map((r) => ({
|
|
17
|
+
url: String(r.url ?? ''),
|
|
18
|
+
title: String(r.title ?? ''),
|
|
19
|
+
snippet: String(r.description ?? ''),
|
|
20
|
+
}));
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=BraveProvider.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BraveProvider.js","sourceRoot":"","sources":["../../../src/web-search/providers/BraveProvider.ts"],"names":[],"mappings":"AAOA,MAAM,OAAO,aAAa;IAIxB,YAA6B,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;QAHlC,eAAU,GAAG,OAAgB,CAAC;QAC9B,WAAM,GAAG,GAAG,CAAC;IAEwB,CAAC;IAE/C,WAAW;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,QAAgB,CAAC;QAC3C,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACvE,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,kDAAkD,MAAM,EAAE,EAAE;YAClF,OAAO,EAAE,EAAE,sBAAsB,EAAE,IAAI,CAAC,MAAM,EAAE;SACjD,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAA0B,EAAE,EAAE,CAAC,CAAC;YACpE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC;YACxB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC5B,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC;SACrC,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module web-search/providers/FirecrawlProvider
|
|
3
|
+
*
|
|
4
|
+
* Firecrawl search provider — search + scrape in one API call.
|
|
5
|
+
* Returns full extracted markdown content alongside search results.
|
|
6
|
+
* Primary provider with 1.5x RRF weight.
|
|
7
|
+
*/
|
|
8
|
+
import type { IWebSearchProvider, WebSearchResult } from '../types';
|
|
9
|
+
export declare class FirecrawlProvider implements IWebSearchProvider {
|
|
10
|
+
private readonly apiKey;
|
|
11
|
+
readonly providerId: "firecrawl";
|
|
12
|
+
readonly weight = 1.5;
|
|
13
|
+
constructor(apiKey: string);
|
|
14
|
+
isAvailable(): boolean;
|
|
15
|
+
search(query: string, limit?: number): Promise<WebSearchResult[]>;
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=FirecrawlProvider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FirecrawlProvider.d.ts","sourceRoot":"","sources":["../../../src/web-search/providers/FirecrawlProvider.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAEpE,qBAAa,iBAAkB,YAAW,kBAAkB;IAI9C,OAAO,CAAC,QAAQ,CAAC,MAAM;IAHnC,QAAQ,CAAC,UAAU,EAAG,WAAW,CAAU;IAC3C,QAAQ,CAAC,MAAM,OAAO;gBAEO,MAAM,EAAE,MAAM;IAE3C,WAAW,IAAI,OAAO;IAIhB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAU,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CA6B3E"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
export class FirecrawlProvider {
|
|
2
|
+
constructor(apiKey) {
|
|
3
|
+
this.apiKey = apiKey;
|
|
4
|
+
this.providerId = 'firecrawl';
|
|
5
|
+
this.weight = 1.5;
|
|
6
|
+
}
|
|
7
|
+
isAvailable() {
|
|
8
|
+
return this.apiKey.length > 0;
|
|
9
|
+
}
|
|
10
|
+
async search(query, limit = 5) {
|
|
11
|
+
const res = await fetch('https://api.firecrawl.dev/v1/search', {
|
|
12
|
+
method: 'POST',
|
|
13
|
+
headers: {
|
|
14
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
15
|
+
'Content-Type': 'application/json',
|
|
16
|
+
},
|
|
17
|
+
body: JSON.stringify({
|
|
18
|
+
query,
|
|
19
|
+
limit,
|
|
20
|
+
scrapeOptions: { formats: ['markdown'], onlyMainContent: true },
|
|
21
|
+
}),
|
|
22
|
+
});
|
|
23
|
+
if (!res.ok) {
|
|
24
|
+
const errText = await res.text().catch(() => '');
|
|
25
|
+
throw new Error(`Firecrawl search failed (${res.status}): ${errText}`);
|
|
26
|
+
}
|
|
27
|
+
const data = await res.json();
|
|
28
|
+
if (!data.success || !Array.isArray(data.data))
|
|
29
|
+
return [];
|
|
30
|
+
return data.data.map((r) => ({
|
|
31
|
+
url: String(r.url ?? r.metadata?.sourceURL ?? ''),
|
|
32
|
+
title: String(r.title ?? r.metadata?.title ?? ''),
|
|
33
|
+
snippet: String(r.description ?? r.metadata?.description ?? ''),
|
|
34
|
+
content: typeof r.markdown === 'string' ? r.markdown : undefined,
|
|
35
|
+
}));
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=FirecrawlProvider.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"FirecrawlProvider.js","sourceRoot":"","sources":["../../../src/web-search/providers/FirecrawlProvider.ts"],"names":[],"mappings":"AASA,MAAM,OAAO,iBAAiB;IAI5B,YAA6B,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;QAHlC,eAAU,GAAG,WAAoB,CAAC;QAClC,WAAM,GAAG,GAAG,CAAC;IAEwB,CAAC;IAE/C,WAAW;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,QAAgB,CAAC;QAC3C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,qCAAqC,EAAE;YAC7D,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;gBACtC,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK;gBACL,KAAK;gBACL,aAAa,EAAE,EAAE,OAAO,EAAE,CAAC,UAAU,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE;aAChE,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YACjD,MAAM,IAAI,KAAK,CAAC,4BAA4B,GAAG,CAAC,MAAM,MAAM,OAAO,EAAE,CAAC,CAAC;QACzE,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,EAAE,CAAC;QAE1D,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAA0B,EAAE,EAAE,CAAC,CAAC;YACpD,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,IAAK,CAAC,CAAC,QAAoC,EAAE,SAAS,IAAI,EAAE,CAAC;YAC9E,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,IAAK,CAAC,CAAC,QAAoC,EAAE,KAAK,IAAI,EAAE,CAAC;YAC9E,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,IAAK,CAAC,CAAC,QAAoC,EAAE,WAAW,IAAI,EAAE,CAAC;YAC5F,OAAO,EAAE,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;SACjE,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module web-search/providers/SerperProvider
|
|
3
|
+
*
|
|
4
|
+
* Serper (Google Search) provider.
|
|
5
|
+
*/
|
|
6
|
+
import type { IWebSearchProvider, WebSearchResult } from '../types';
|
|
7
|
+
export declare class SerperProvider implements IWebSearchProvider {
|
|
8
|
+
private readonly apiKey;
|
|
9
|
+
readonly providerId: "serper";
|
|
10
|
+
readonly weight = 1;
|
|
11
|
+
constructor(apiKey: string);
|
|
12
|
+
isAvailable(): boolean;
|
|
13
|
+
search(query: string, limit?: number): Promise<WebSearchResult[]>;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=SerperProvider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SerperProvider.d.ts","sourceRoot":"","sources":["../../../src/web-search/providers/SerperProvider.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,KAAK,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAEpE,qBAAa,cAAe,YAAW,kBAAkB;IAI3C,OAAO,CAAC,QAAQ,CAAC,MAAM;IAHnC,QAAQ,CAAC,UAAU,EAAG,QAAQ,CAAU;IACxC,QAAQ,CAAC,MAAM,KAAO;gBAEO,MAAM,EAAE,MAAM;IAE3C,WAAW,IAAI,OAAO;IAIhB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAU,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CAiB3E"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
export class SerperProvider {
|
|
2
|
+
constructor(apiKey) {
|
|
3
|
+
this.apiKey = apiKey;
|
|
4
|
+
this.providerId = 'serper';
|
|
5
|
+
this.weight = 1.0;
|
|
6
|
+
}
|
|
7
|
+
isAvailable() {
|
|
8
|
+
return this.apiKey.length > 0;
|
|
9
|
+
}
|
|
10
|
+
async search(query, limit = 5) {
|
|
11
|
+
const res = await fetch('https://google.serper.dev/search', {
|
|
12
|
+
method: 'POST',
|
|
13
|
+
headers: {
|
|
14
|
+
'X-API-KEY': this.apiKey,
|
|
15
|
+
'Content-Type': 'application/json',
|
|
16
|
+
},
|
|
17
|
+
body: JSON.stringify({ q: query, num: limit }),
|
|
18
|
+
});
|
|
19
|
+
const data = await res.json();
|
|
20
|
+
return (data.organic ?? []).map((r) => ({
|
|
21
|
+
url: String(r.link ?? ''),
|
|
22
|
+
title: String(r.title ?? ''),
|
|
23
|
+
snippet: String(r.snippet ?? ''),
|
|
24
|
+
}));
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=SerperProvider.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SerperProvider.js","sourceRoot":"","sources":["../../../src/web-search/providers/SerperProvider.ts"],"names":[],"mappings":"AAOA,MAAM,OAAO,cAAc;IAIzB,YAA6B,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;QAHlC,eAAU,GAAG,QAAiB,CAAC;QAC/B,WAAM,GAAG,GAAG,CAAC;IAEwB,CAAC;IAE/C,WAAW;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,QAAgB,CAAC;QAC3C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,kCAAkC,EAAE;YAC1D,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,WAAW,EAAE,IAAI,CAAC,MAAM;gBACxB,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;SAC/C,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAA0B,EAAE,EAAE,CAAC,CAAC;YAC/D,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;YACzB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC5B,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;SACjC,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module web-search/providers/TavilyProvider
|
|
3
|
+
*
|
|
4
|
+
* Tavily AI-optimized search provider.
|
|
5
|
+
*/
|
|
6
|
+
import type { IWebSearchProvider, WebSearchResult } from '../types';
|
|
7
|
+
export declare class TavilyProvider implements IWebSearchProvider {
|
|
8
|
+
private readonly apiKey;
|
|
9
|
+
readonly providerId: "tavily";
|
|
10
|
+
readonly weight = 1;
|
|
11
|
+
constructor(apiKey: string);
|
|
12
|
+
isAvailable(): boolean;
|
|
13
|
+
search(query: string, limit?: number): Promise<WebSearchResult[]>;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=TavilyProvider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"TavilyProvider.d.ts","sourceRoot":"","sources":["../../../src/web-search/providers/TavilyProvider.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,KAAK,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAEpE,qBAAa,cAAe,YAAW,kBAAkB;IAI3C,OAAO,CAAC,QAAQ,CAAC,MAAM;IAHnC,QAAQ,CAAC,UAAU,EAAG,QAAQ,CAAU;IACxC,QAAQ,CAAC,MAAM,KAAO;gBAEO,MAAM,EAAE,MAAM;IAE3C,WAAW,IAAI,OAAO;IAIhB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAU,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CAsB3E"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export class TavilyProvider {
|
|
2
|
+
constructor(apiKey) {
|
|
3
|
+
this.apiKey = apiKey;
|
|
4
|
+
this.providerId = 'tavily';
|
|
5
|
+
this.weight = 1.0;
|
|
6
|
+
}
|
|
7
|
+
isAvailable() {
|
|
8
|
+
return this.apiKey.length > 0;
|
|
9
|
+
}
|
|
10
|
+
async search(query, limit = 5) {
|
|
11
|
+
const res = await fetch('https://api.tavily.com/search', {
|
|
12
|
+
method: 'POST',
|
|
13
|
+
headers: { 'Content-Type': 'application/json' },
|
|
14
|
+
body: JSON.stringify({
|
|
15
|
+
api_key: this.apiKey,
|
|
16
|
+
query,
|
|
17
|
+
search_depth: 'advanced',
|
|
18
|
+
include_answer: false,
|
|
19
|
+
include_raw_content: false,
|
|
20
|
+
max_results: limit,
|
|
21
|
+
}),
|
|
22
|
+
});
|
|
23
|
+
const data = await res.json();
|
|
24
|
+
return (data.results ?? []).map((r) => ({
|
|
25
|
+
url: String(r.url ?? ''),
|
|
26
|
+
title: String(r.title ?? ''),
|
|
27
|
+
snippet: String(r.content ?? ''),
|
|
28
|
+
relevanceScore: typeof r.score === 'number' ? r.score : 0.5,
|
|
29
|
+
}));
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=TavilyProvider.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"TavilyProvider.js","sourceRoot":"","sources":["../../../src/web-search/providers/TavilyProvider.ts"],"names":[],"mappings":"AAOA,MAAM,OAAO,cAAc;IAIzB,YAA6B,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;QAHlC,eAAU,GAAG,QAAiB,CAAC;QAC/B,WAAM,GAAG,GAAG,CAAC;IAEwB,CAAC;IAE/C,WAAW;QACT,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,QAAgB,CAAC;QAC3C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,+BAA+B,EAAE;YACvD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,OAAO,EAAE,IAAI,CAAC,MAAM;gBACpB,KAAK;gBACL,YAAY,EAAE,UAAU;gBACxB,cAAc,EAAE,KAAK;gBACrB,mBAAmB,EAAE,KAAK;gBAC1B,WAAW,EAAE,KAAK;aACnB,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAA0B,EAAE,EAAE,CAAC,CAAC;YAC/D,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC;YACxB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC5B,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC;YAChC,cAAc,EAAE,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG;SAC5D,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/web-search/providers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/web-search/providers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module web-search/types
|
|
3
|
+
*
|
|
4
|
+
* Core types for the AgentOS multi-provider web search pipeline.
|
|
5
|
+
*/
|
|
6
|
+
import type { RerankChainStage } from '../rag/reranking/IRerankerService';
|
|
7
|
+
import type { RerankerService } from '../rag/reranking/RerankerService';
|
|
8
|
+
/** A single raw result from a search provider. */
|
|
9
|
+
export interface WebSearchResult {
|
|
10
|
+
url: string;
|
|
11
|
+
title: string;
|
|
12
|
+
snippet: string;
|
|
13
|
+
/** Full extracted page content (Firecrawl returns this; others do not). */
|
|
14
|
+
content?: string;
|
|
15
|
+
/** Provider-specific relevance score. */
|
|
16
|
+
relevanceScore?: number;
|
|
17
|
+
}
|
|
18
|
+
/** Enriched result after cross-provider fusion and optional reranking. */
|
|
19
|
+
export interface EnrichedSearchResult extends WebSearchResult {
|
|
20
|
+
/** Which providers returned this result. */
|
|
21
|
+
providerSources: string[];
|
|
22
|
+
/** Reciprocal Rank Fusion score across providers. */
|
|
23
|
+
rrfScore: number;
|
|
24
|
+
/** Neural reranker relevance score (0-1). Populated after reranking. */
|
|
25
|
+
rerankScore?: number;
|
|
26
|
+
}
|
|
27
|
+
/** Contract for pluggable search providers. */
|
|
28
|
+
export interface IWebSearchProvider {
|
|
29
|
+
/** Unique provider identifier (e.g., 'firecrawl', 'tavily'). */
|
|
30
|
+
readonly providerId: string;
|
|
31
|
+
/** RRF weight multiplier. Firecrawl = 1.5, others = 1.0. */
|
|
32
|
+
readonly weight: number;
|
|
33
|
+
/** Execute a search query. Returns raw results from this provider. */
|
|
34
|
+
search(query: string, limit?: number): Promise<WebSearchResult[]>;
|
|
35
|
+
/** Check if this provider has valid configuration (API key present). */
|
|
36
|
+
isAvailable(): boolean;
|
|
37
|
+
}
|
|
38
|
+
/** Configuration for the WebSearchService. */
|
|
39
|
+
export interface WebSearchConfig {
|
|
40
|
+
/** Optional RerankerService for neural reranking after RRF. */
|
|
41
|
+
reranker?: RerankerService;
|
|
42
|
+
/** Rerank chain stages (e.g., Cohere → LLM Judge). */
|
|
43
|
+
rerankChain?: RerankChainStage[];
|
|
44
|
+
/** Semantic dedup cosine similarity threshold (default 0.85). */
|
|
45
|
+
dedupThreshold?: number;
|
|
46
|
+
/** Max results to return (default 12). */
|
|
47
|
+
maxResults?: number;
|
|
48
|
+
}
|
|
49
|
+
/** Internal candidate during RRF fusion. */
|
|
50
|
+
export interface RRFCandidate {
|
|
51
|
+
url: string;
|
|
52
|
+
normalizedUrl: string;
|
|
53
|
+
title: string;
|
|
54
|
+
snippet: string;
|
|
55
|
+
content?: string;
|
|
56
|
+
providerRanks: Map<string, number>;
|
|
57
|
+
providerSources: string[];
|
|
58
|
+
rrfScore: number;
|
|
59
|
+
relevanceScore?: number;
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/web-search/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AAExE,kDAAkD;AAClD,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,2EAA2E;IAC3E,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yCAAyC;IACzC,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,0EAA0E;AAC1E,MAAM,WAAW,oBAAqB,SAAQ,eAAe;IAC3D,4CAA4C;IAC5C,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,qDAAqD;IACrD,QAAQ,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,+CAA+C;AAC/C,MAAM,WAAW,kBAAkB;IACjC,gEAAgE;IAChE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,4DAA4D;IAC5D,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,sEAAsE;IACtE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IAClE,wEAAwE;IACxE,WAAW,IAAI,OAAO,CAAC;CACxB;AAED,8CAA8C;AAC9C,MAAM,WAAW,eAAe;IAC9B,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,eAAe,CAAC;IAC3B,sDAAsD;IACtD,WAAW,CAAC,EAAE,gBAAgB,EAAE,CAAC;IACjC,iEAAiE;IACjE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,0CAA0C;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,4CAA4C;AAC5C,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,aAAa,EAAE,MAAM,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/web-search/types.ts"],"names":[],"mappings":""}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@framers/agentos",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.181",
|
|
4
4
|
"description": "Modular AgentOS orchestration library",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -32,6 +32,31 @@
|
|
|
32
32
|
"default": "./dist/api/streamText.js",
|
|
33
33
|
"types": "./dist/api/streamText.d.ts"
|
|
34
34
|
},
|
|
35
|
+
"./api/generateImage": {
|
|
36
|
+
"import": "./dist/api/generateImage.js",
|
|
37
|
+
"default": "./dist/api/generateImage.js",
|
|
38
|
+
"types": "./dist/api/generateImage.d.ts"
|
|
39
|
+
},
|
|
40
|
+
"./api/generateObject": {
|
|
41
|
+
"import": "./dist/api/generateObject.js",
|
|
42
|
+
"default": "./dist/api/generateObject.js",
|
|
43
|
+
"types": "./dist/api/generateObject.d.ts"
|
|
44
|
+
},
|
|
45
|
+
"./api/generateMusic": {
|
|
46
|
+
"import": "./dist/api/generateMusic.js",
|
|
47
|
+
"default": "./dist/api/generateMusic.js",
|
|
48
|
+
"types": "./dist/api/generateMusic.d.ts"
|
|
49
|
+
},
|
|
50
|
+
"./api/generateSFX": {
|
|
51
|
+
"import": "./dist/api/generateSFX.js",
|
|
52
|
+
"default": "./dist/api/generateSFX.js",
|
|
53
|
+
"types": "./dist/api/generateSFX.d.ts"
|
|
54
|
+
},
|
|
55
|
+
"./api/generateVideo": {
|
|
56
|
+
"import": "./dist/api/generateVideo.js",
|
|
57
|
+
"default": "./dist/api/generateVideo.js",
|
|
58
|
+
"types": "./dist/api/generateVideo.d.ts"
|
|
59
|
+
},
|
|
35
60
|
"./core/llm/routing/IModelRouter": {
|
|
36
61
|
"import": "./dist/core/llm/routing/IModelRouter.js",
|
|
37
62
|
"default": "./dist/core/llm/routing/IModelRouter.js",
|
|
@@ -138,6 +163,11 @@
|
|
|
138
163
|
"default": "./dist/rag/reranking/index.js",
|
|
139
164
|
"types": "./dist/rag/reranking/index.d.ts"
|
|
140
165
|
},
|
|
166
|
+
"./web-search": {
|
|
167
|
+
"import": "./dist/web-search/index.js",
|
|
168
|
+
"default": "./dist/web-search/index.js",
|
|
169
|
+
"types": "./dist/web-search/index.d.ts"
|
|
170
|
+
},
|
|
141
171
|
"./rag/graphrag": {
|
|
142
172
|
"import": "./dist/rag/graphrag/index.js",
|
|
143
173
|
"default": "./dist/rag/graphrag/index.js",
|