@thanh01.pmt/research-kit 1.4.1 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/academic/index.cjs +13 -12
- package/dist/academic/index.cjs.map +1 -1
- package/dist/academic/index.d.cts +1 -1
- package/dist/academic/index.d.ts +1 -1
- package/dist/academic/index.js +1 -1
- package/dist/agent/index.cjs +269 -108
- package/dist/agent/index.cjs.map +1 -1
- package/dist/agent/index.d.cts +4 -4
- package/dist/agent/index.d.ts +4 -4
- package/dist/agent/index.js +3 -3
- package/dist/{catalogBuilder-D-UiM37y.d.ts → catalogBuilder-BN3VwEsE.d.ts} +15 -2
- package/dist/{catalogBuilder-BBCa9jqI.d.cts → catalogBuilder-mba5BIHD.d.cts} +15 -2
- package/dist/{chunk-UCPWBRZ2.js → chunk-5HSLUAM6.js} +11 -3
- package/dist/chunk-5HSLUAM6.js.map +1 -0
- package/dist/{chunk-4S6UMYBA.js → chunk-6QW5TFDJ.js} +56 -328
- package/dist/chunk-6QW5TFDJ.js.map +1 -0
- package/dist/chunk-BPZLZKVJ.js +1012 -0
- package/dist/chunk-BPZLZKVJ.js.map +1 -0
- package/dist/{chunk-7BGR4E37.js → chunk-LTXXARCA.js} +3 -3
- package/dist/{chunk-7BGR4E37.js.map → chunk-LTXXARCA.js.map} +1 -1
- package/dist/{chunk-6U2H2DMQ.js → chunk-QLBHTCV6.js} +95 -35
- package/dist/chunk-QLBHTCV6.js.map +1 -0
- package/dist/{chunk-SDDZFFL6.js → chunk-SB7WYK7A.js} +15 -14
- package/dist/chunk-SB7WYK7A.js.map +1 -0
- package/dist/harvester/index.cjs +219 -86
- package/dist/harvester/index.cjs.map +1 -1
- package/dist/harvester/index.d.cts +2 -2
- package/dist/harvester/index.d.ts +2 -2
- package/dist/harvester/index.js +3 -3
- package/dist/{index-_aiOh8jX.d.ts → index-9Z-uMwdI.d.ts} +12 -4
- package/dist/{index-BOwJ1jON.d.cts → index-eTcF4XVo.d.cts} +12 -4
- package/dist/index.cjs +2585 -2355
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +12 -12
- package/dist/{literatureReview-Bvlqk4DB.d.cts → literatureReview-BUtPL7bj.d.cts} +1 -0
- package/dist/{literatureReview-Bn2iEGRO.d.ts → literatureReview-CHz-JfSR.d.ts} +1 -0
- package/dist/pipeline/index.cjs +322 -128
- package/dist/pipeline/index.cjs.map +1 -1
- package/dist/pipeline/index.d.cts +15 -3
- package/dist/pipeline/index.d.ts +15 -3
- package/dist/pipeline/index.js +4 -4
- package/dist/{socialPulseHarvester-CN4MwrpY.d.cts → socialPulseHarvester-BCFxGwx2.d.cts} +4 -1
- package/dist/{socialPulseHarvester-IbdT3zD1.d.ts → socialPulseHarvester-DjMvBufT.d.ts} +4 -1
- package/package.json +1 -1
- package/dist/chunk-4S6UMYBA.js.map +0 -1
- package/dist/chunk-6U2H2DMQ.js.map +0 -1
- package/dist/chunk-MJIREPKJ.js +0 -581
- package/dist/chunk-MJIREPKJ.js.map +0 -1
- package/dist/chunk-SDDZFFL6.js.map +0 -1
- package/dist/chunk-UCPWBRZ2.js.map +0 -1
package/dist/agent/index.cjs
CHANGED
|
@@ -121,6 +121,18 @@ function normalizeUrlKey(rawUrl) {
|
|
|
121
121
|
return rawUrl.trim().replace(/\/$/, "").toLowerCase();
|
|
122
122
|
}
|
|
123
123
|
}
|
|
124
|
+
function computeCanonicalId(url) {
|
|
125
|
+
try {
|
|
126
|
+
let hash = 0;
|
|
127
|
+
const clean = url.trim().replace(/^https?:\/\//i, "").replace(/[\/?#].*$/, "").toLowerCase();
|
|
128
|
+
for (let i = 0; i < clean.length; i++) {
|
|
129
|
+
hash = hash * 31 + clean.charCodeAt(i) >>> 0;
|
|
130
|
+
}
|
|
131
|
+
return `SRC-${hash.toString(16).padStart(8, "0")}`;
|
|
132
|
+
} catch {
|
|
133
|
+
return `SRC-${Math.random().toString(36).slice(2, 10)}`;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
124
136
|
var EvidenceLedger = class {
|
|
125
137
|
items = [];
|
|
126
138
|
urlIndex = /* @__PURE__ */ new Map();
|
|
@@ -155,6 +167,7 @@ var EvidenceLedger = class {
|
|
|
155
167
|
}
|
|
156
168
|
const item = {
|
|
157
169
|
id: this.allocateId(),
|
|
170
|
+
canonicalId: computeCanonicalId(doc.url),
|
|
158
171
|
sourceType,
|
|
159
172
|
title: doc.title || "Untitled Document",
|
|
160
173
|
url: doc.url,
|
|
@@ -183,6 +196,7 @@ var EvidenceLedger = class {
|
|
|
183
196
|
const snippet = (hit.content || "").replace(/\s+/g, " ").trim();
|
|
184
197
|
const item = {
|
|
185
198
|
id: this.allocateId(),
|
|
199
|
+
canonicalId: computeCanonicalId(hit.url),
|
|
186
200
|
sourceType: hit.engine === "academic" ? "academic" : "web",
|
|
187
201
|
title: hit.title || "Search Result",
|
|
188
202
|
url: hit.url,
|
|
@@ -213,6 +227,7 @@ var EvidenceLedger = class {
|
|
|
213
227
|
const citationBonus = paper.citationCount >= 50 ? 9 : paper.citationCount >= 10 ? 8 : 7;
|
|
214
228
|
const item = {
|
|
215
229
|
id: this.allocateId(),
|
|
230
|
+
canonicalId: computeCanonicalId(url),
|
|
216
231
|
sourceType: "academic",
|
|
217
232
|
title: paper.title,
|
|
218
233
|
url,
|
|
@@ -242,6 +257,7 @@ var EvidenceLedger = class {
|
|
|
242
257
|
}
|
|
243
258
|
const item = {
|
|
244
259
|
id: this.allocateId(),
|
|
260
|
+
canonicalId: computeCanonicalId(pulse.url),
|
|
245
261
|
sourceType: "social_pulse",
|
|
246
262
|
title: pulse.title,
|
|
247
263
|
url: pulse.url,
|
|
@@ -272,6 +288,7 @@ var EvidenceLedger = class {
|
|
|
272
288
|
const excerpt = `${resource.description} [Provider: ${resource.provider}${resource.license ? `, License: ${resource.license}` : ""}]`;
|
|
273
289
|
const item = {
|
|
274
290
|
id: this.allocateId(),
|
|
291
|
+
canonicalId: computeCanonicalId(resource.url),
|
|
275
292
|
sourceType: "stem_oer",
|
|
276
293
|
title: resource.title,
|
|
277
294
|
url: resource.url,
|
|
@@ -296,6 +313,7 @@ var EvidenceLedger = class {
|
|
|
296
313
|
const rawExcerpt = item.keyExcerpts[0] || item.title;
|
|
297
314
|
return {
|
|
298
315
|
id: item.id,
|
|
316
|
+
canonicalId: item.canonicalId || computeCanonicalId(item.url),
|
|
299
317
|
sourceType: item.sourceType,
|
|
300
318
|
title: item.title.slice(0, 140),
|
|
301
319
|
url: item.url,
|
|
@@ -303,7 +321,7 @@ var EvidenceLedger = class {
|
|
|
303
321
|
headings: item.headings.slice(0, 5),
|
|
304
322
|
codeBlockCount: item.codeBlocks.length,
|
|
305
323
|
tableCount: item.tables.length,
|
|
306
|
-
briefExcerpt: rawExcerpt.slice(0,
|
|
324
|
+
briefExcerpt: rawExcerpt.slice(0, 450)
|
|
307
325
|
};
|
|
308
326
|
}
|
|
309
327
|
getItems() {
|
|
@@ -733,7 +751,7 @@ ${receiptsText}`,
|
|
|
733
751
|
const missing = [];
|
|
734
752
|
targetCheckList.forEach((attr, idx) => {
|
|
735
753
|
const ans = res.answers[`attr_${idx}`];
|
|
736
|
-
if (ans && ans.type === "noul" && ans.probability >= 0.55) {
|
|
754
|
+
if (ans && (ans.type === "noul" && ans.probability >= 0.55 || ans.passed === true || ans.choice === "yes")) {
|
|
737
755
|
satisfied.push(attr);
|
|
738
756
|
} else {
|
|
739
757
|
missing.push(attr);
|
|
@@ -2088,6 +2106,18 @@ var HealthAuditor = class {
|
|
|
2088
2106
|
};
|
|
2089
2107
|
|
|
2090
2108
|
// src/harvester/multiSourceSearch.ts
|
|
2109
|
+
function isUrlAllowedByDomainFilter(url, domainFilter) {
|
|
2110
|
+
if (!domainFilter || domainFilter.length === 0) return true;
|
|
2111
|
+
try {
|
|
2112
|
+
const hostname = new URL(url).hostname.toLowerCase();
|
|
2113
|
+
return domainFilter.some((d) => {
|
|
2114
|
+
const cleanD = d.toLowerCase().trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "");
|
|
2115
|
+
return hostname === cleanD || hostname.endsWith(`.${cleanD}`);
|
|
2116
|
+
});
|
|
2117
|
+
} catch {
|
|
2118
|
+
return false;
|
|
2119
|
+
}
|
|
2120
|
+
}
|
|
2091
2121
|
var MultiSourceSearchEngine = class {
|
|
2092
2122
|
tavilyKey;
|
|
2093
2123
|
exaKey;
|
|
@@ -2099,6 +2129,20 @@ var MultiSourceSearchEngine = class {
|
|
|
2099
2129
|
allowEnvFallback;
|
|
2100
2130
|
customAdapter;
|
|
2101
2131
|
academicEngine;
|
|
2132
|
+
lastErrors = [];
|
|
2133
|
+
getLastErrors() {
|
|
2134
|
+
return [...this.lastErrors];
|
|
2135
|
+
}
|
|
2136
|
+
recordError(engine, error, status, traceErrors) {
|
|
2137
|
+
const errObj = {
|
|
2138
|
+
engine,
|
|
2139
|
+
error,
|
|
2140
|
+
status,
|
|
2141
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
2142
|
+
};
|
|
2143
|
+
this.lastErrors.push(errObj);
|
|
2144
|
+
traceErrors?.push(errObj);
|
|
2145
|
+
}
|
|
2102
2146
|
constructor(options = {}) {
|
|
2103
2147
|
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
2104
2148
|
this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
|
|
@@ -2179,97 +2223,128 @@ var MultiSourceSearchEngine = class {
|
|
|
2179
2223
|
const brightDataZone = options.brightDataZone || this.brightDataZone;
|
|
2180
2224
|
const searxngHost = options.searxngHost || this.searxngHost;
|
|
2181
2225
|
if (preferred === "searxng" && searxngHost) {
|
|
2182
|
-
const searxHits = await this.querySearxng(
|
|
2226
|
+
const searxHits = await this.querySearxng(
|
|
2227
|
+
query,
|
|
2228
|
+
searxngHost,
|
|
2229
|
+
limit,
|
|
2230
|
+
options.searxngHeaders,
|
|
2231
|
+
options.timeRange,
|
|
2232
|
+
options.domainFilter
|
|
2233
|
+
);
|
|
2183
2234
|
if (searxHits.length > 0) return searxHits;
|
|
2184
2235
|
}
|
|
2185
2236
|
if (preferred === "brightdata" && brightDataKey) {
|
|
2186
2237
|
const bdHits = await this.queryBrightData(query, brightDataKey, limit, {
|
|
2187
2238
|
zone: brightDataZone,
|
|
2188
|
-
engine: options.brightDataEngine
|
|
2239
|
+
engine: options.brightDataEngine,
|
|
2240
|
+
domainFilter: options.domainFilter
|
|
2189
2241
|
});
|
|
2190
2242
|
if (bdHits.length > 0) return bdHits;
|
|
2191
2243
|
}
|
|
2192
2244
|
if (preferred === "exa" && exaKey) {
|
|
2193
|
-
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
|
|
2245
|
+
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter, options.timeRange);
|
|
2194
2246
|
if (exaHits.length > 0) return exaHits;
|
|
2195
2247
|
}
|
|
2196
2248
|
if (preferred === "serper" && serperKey) {
|
|
2197
|
-
const serperHits = await this.querySerper(query, serperKey, limit);
|
|
2249
|
+
const serperHits = await this.querySerper(query, serperKey, limit, options.timeRange, options.domainFilter);
|
|
2198
2250
|
if (serperHits.length > 0) return serperHits;
|
|
2199
2251
|
}
|
|
2252
|
+
if (preferred === "tavily" && tavilyKey) {
|
|
2253
|
+
const tavilyHits = await this.queryTavily(query, tavilyKey, limit, options.timeRange, options.domainFilter);
|
|
2254
|
+
if (tavilyHits.length > 0) return tavilyHits;
|
|
2255
|
+
}
|
|
2200
2256
|
if (preferred === "duckduckgo") {
|
|
2201
|
-
const
|
|
2202
|
-
if (
|
|
2257
|
+
const ddgHits = await this.queryDuckDuckGo(query, limit, options.domainFilter);
|
|
2258
|
+
if (ddgHits.length > 0) return ddgHits;
|
|
2203
2259
|
}
|
|
2204
|
-
if (tavilyKey &&
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
api_key: tavilyKey,
|
|
2208
|
-
query,
|
|
2209
|
-
search_depth: "advanced",
|
|
2210
|
-
include_answer: true,
|
|
2211
|
-
max_results: limit
|
|
2212
|
-
};
|
|
2213
|
-
if (options.timeRange) {
|
|
2214
|
-
body.time_range = options.timeRange;
|
|
2215
|
-
}
|
|
2216
|
-
if (options.domainFilter && options.domainFilter.length > 0) {
|
|
2217
|
-
body.include_domains = options.domainFilter;
|
|
2218
|
-
}
|
|
2219
|
-
const res = await fetch("https://api.tavily.com/search", {
|
|
2220
|
-
method: "POST",
|
|
2221
|
-
headers: { "Content-Type": "application/json" },
|
|
2222
|
-
body: JSON.stringify(body),
|
|
2223
|
-
signal: AbortSignal.timeout(12e3)
|
|
2224
|
-
});
|
|
2225
|
-
if (res.ok) {
|
|
2226
|
-
const data = await res.json();
|
|
2227
|
-
if (Array.isArray(data?.results) && data.results.length > 0) {
|
|
2228
|
-
return data.results.map((r) => ({
|
|
2229
|
-
title: r.title || "Untitled",
|
|
2230
|
-
url: r.url,
|
|
2231
|
-
content: r.content || "",
|
|
2232
|
-
score: r.score,
|
|
2233
|
-
engine: "tavily"
|
|
2234
|
-
}));
|
|
2235
|
-
}
|
|
2236
|
-
}
|
|
2237
|
-
} catch (e) {
|
|
2238
|
-
console.warn("[MultiSourceSearchEngine] Tavily attempt failed:", e?.message || e);
|
|
2239
|
-
}
|
|
2260
|
+
if (tavilyKey && preferred !== "tavily") {
|
|
2261
|
+
const tavilyHits = await this.queryTavily(query, tavilyKey, limit, options.timeRange, options.domainFilter);
|
|
2262
|
+
if (tavilyHits.length > 0) return tavilyHits;
|
|
2240
2263
|
}
|
|
2241
|
-
if (exaKey) {
|
|
2264
|
+
if (exaKey && preferred !== "exa") {
|
|
2242
2265
|
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter, options.timeRange);
|
|
2243
2266
|
if (exaHits.length > 0) return exaHits;
|
|
2244
2267
|
}
|
|
2245
|
-
if (serperKey) {
|
|
2246
|
-
const serperHits = await this.querySerper(query, serperKey, limit, options.timeRange);
|
|
2268
|
+
if (serperKey && preferred !== "serper") {
|
|
2269
|
+
const serperHits = await this.querySerper(query, serperKey, limit, options.timeRange, options.domainFilter);
|
|
2247
2270
|
if (serperHits.length > 0) return serperHits;
|
|
2248
2271
|
}
|
|
2249
|
-
if (brightDataKey) {
|
|
2272
|
+
if (brightDataKey && preferred !== "brightdata") {
|
|
2250
2273
|
const bdHits = await this.queryBrightData(query, brightDataKey, limit, {
|
|
2251
2274
|
zone: brightDataZone,
|
|
2252
|
-
engine: options.brightDataEngine
|
|
2275
|
+
engine: options.brightDataEngine,
|
|
2276
|
+
domainFilter: options.domainFilter
|
|
2253
2277
|
});
|
|
2254
2278
|
if (bdHits.length > 0) return bdHits;
|
|
2255
2279
|
}
|
|
2256
|
-
if (searxngHost) {
|
|
2257
|
-
const searxHits = await this.querySearxng(
|
|
2280
|
+
if (searxngHost && preferred !== "searxng") {
|
|
2281
|
+
const searxHits = await this.querySearxng(
|
|
2282
|
+
query,
|
|
2283
|
+
searxngHost,
|
|
2284
|
+
limit,
|
|
2285
|
+
options.searxngHeaders,
|
|
2286
|
+
options.timeRange,
|
|
2287
|
+
options.domainFilter
|
|
2288
|
+
);
|
|
2258
2289
|
if (searxHits.length > 0) return searxHits;
|
|
2259
2290
|
}
|
|
2260
|
-
|
|
2261
|
-
|
|
2291
|
+
if (preferred !== "duckduckgo") {
|
|
2292
|
+
const ddgHits = await this.queryDuckDuckGo(query, limit, options.domainFilter);
|
|
2293
|
+
if (ddgHits.length > 0) return ddgHits;
|
|
2294
|
+
}
|
|
2295
|
+
return [];
|
|
2296
|
+
}
|
|
2297
|
+
async queryTavily(query, tavilyKey, limit, timeRange, domainFilter) {
|
|
2298
|
+
try {
|
|
2299
|
+
const body = {
|
|
2300
|
+
api_key: tavilyKey,
|
|
2301
|
+
query,
|
|
2302
|
+
search_depth: "advanced",
|
|
2303
|
+
include_answer: true,
|
|
2304
|
+
max_results: limit
|
|
2305
|
+
};
|
|
2306
|
+
if (timeRange) {
|
|
2307
|
+
body.time_range = timeRange;
|
|
2308
|
+
}
|
|
2309
|
+
if (domainFilter && domainFilter.length > 0) {
|
|
2310
|
+
body.include_domains = domainFilter;
|
|
2311
|
+
}
|
|
2312
|
+
const res = await fetch("https://api.tavily.com/search", {
|
|
2313
|
+
method: "POST",
|
|
2314
|
+
headers: { "Content-Type": "application/json" },
|
|
2315
|
+
body: JSON.stringify(body),
|
|
2316
|
+
signal: AbortSignal.timeout(12e3)
|
|
2317
|
+
});
|
|
2318
|
+
if (res.ok) {
|
|
2319
|
+
const data = await res.json();
|
|
2320
|
+
if (Array.isArray(data?.results) && data.results.length > 0) {
|
|
2321
|
+
return data.results.map((r) => ({
|
|
2322
|
+
title: r.title || "Untitled",
|
|
2323
|
+
url: r.url,
|
|
2324
|
+
content: r.content || "",
|
|
2325
|
+
score: r.score,
|
|
2326
|
+
engine: "tavily"
|
|
2327
|
+
}));
|
|
2328
|
+
}
|
|
2329
|
+
}
|
|
2330
|
+
} catch (e) {
|
|
2331
|
+
console.warn("[MultiSourceSearchEngine] Tavily attempt failed:", e?.message || e);
|
|
2332
|
+
}
|
|
2262
2333
|
return [];
|
|
2263
2334
|
}
|
|
2264
2335
|
async queryBrightData(query, key, limit = 5, options = {}) {
|
|
2265
2336
|
try {
|
|
2266
2337
|
const zone = options.zone || this.brightDataZone || "cli_unlocker";
|
|
2267
2338
|
const engine = options.engine || "bing";
|
|
2268
|
-
let
|
|
2339
|
+
let effectiveQuery = query;
|
|
2340
|
+
if (options.domainFilter && options.domainFilter.length === 1) {
|
|
2341
|
+
effectiveQuery = `${query} site:${options.domainFilter[0]}`;
|
|
2342
|
+
}
|
|
2343
|
+
let searchUrl = `https://www.bing.com/search?q=${encodeURIComponent(effectiveQuery)}&brd_json=1`;
|
|
2269
2344
|
if (engine === "google") {
|
|
2270
|
-
searchUrl = `https://www.google.com/search?q=${encodeURIComponent(
|
|
2345
|
+
searchUrl = `https://www.google.com/search?q=${encodeURIComponent(effectiveQuery)}&brd_json=1&hl=en&gl=us`;
|
|
2271
2346
|
} else if (engine === "yandex") {
|
|
2272
|
-
searchUrl = `https://www.yandex.com/search?q=${encodeURIComponent(
|
|
2347
|
+
searchUrl = `https://www.yandex.com/search?q=${encodeURIComponent(effectiveQuery)}&brd_json=1`;
|
|
2273
2348
|
}
|
|
2274
2349
|
const res = await fetch("https://api.brightdata.com/request", {
|
|
2275
2350
|
method: "POST",
|
|
@@ -2285,27 +2360,48 @@ var MultiSourceSearchEngine = class {
|
|
|
2285
2360
|
signal: AbortSignal.timeout(15e3)
|
|
2286
2361
|
});
|
|
2287
2362
|
if (res.ok) {
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2363
|
+
let data;
|
|
2364
|
+
try {
|
|
2365
|
+
if (typeof res.json === "function") {
|
|
2366
|
+
data = await res.json();
|
|
2367
|
+
} else if (typeof res.text === "function") {
|
|
2368
|
+
const rawText = await res.text();
|
|
2369
|
+
data = JSON.parse(rawText);
|
|
2370
|
+
}
|
|
2371
|
+
} catch {
|
|
2372
|
+
data = null;
|
|
2298
2373
|
}
|
|
2374
|
+
const hits = [];
|
|
2375
|
+
if (data && Array.isArray(data?.organic)) {
|
|
2376
|
+
for (let idx = 0; idx < data.organic.length; idx++) {
|
|
2377
|
+
const item = data.organic[idx];
|
|
2378
|
+
const hitUrl = item.link || item.url;
|
|
2379
|
+
if (!hitUrl || isUrlAllowedByDomainFilter(hitUrl, options.domainFilter)) {
|
|
2380
|
+
hits.push({
|
|
2381
|
+
title: item.title || "Untitled",
|
|
2382
|
+
url: hitUrl,
|
|
2383
|
+
content: item.description || item.snippet || "",
|
|
2384
|
+
score: item.rank ? 1 / item.rank : 1 / (idx + 1),
|
|
2385
|
+
engine: "brightdata"
|
|
2386
|
+
});
|
|
2387
|
+
}
|
|
2388
|
+
}
|
|
2389
|
+
}
|
|
2390
|
+
if (hits.length > 0) return hits.slice(0, limit);
|
|
2299
2391
|
}
|
|
2300
2392
|
} catch (e) {
|
|
2301
2393
|
console.warn("[MultiSourceSearchEngine] Bright Data attempt failed:", e?.message || e);
|
|
2302
2394
|
}
|
|
2303
2395
|
return [];
|
|
2304
2396
|
}
|
|
2305
|
-
async querySearxng(query, searxngHost, limit, customHeaders, timeRange) {
|
|
2397
|
+
async querySearxng(query, searxngHost, limit, customHeaders, timeRange, domainFilter) {
|
|
2306
2398
|
try {
|
|
2307
2399
|
const cleanHost = searxngHost.replace(/\/$/, "");
|
|
2308
|
-
let
|
|
2400
|
+
let effectiveQuery = query;
|
|
2401
|
+
if (domainFilter && domainFilter.length === 1) {
|
|
2402
|
+
effectiveQuery = `${query} site:${domainFilter[0]}`;
|
|
2403
|
+
}
|
|
2404
|
+
let searchUrl = `${cleanHost}/search?q=${encodeURIComponent(effectiveQuery)}&format=json`;
|
|
2309
2405
|
if (timeRange) {
|
|
2310
2406
|
searchUrl += `&time_range=${encodeURIComponent(timeRange)}`;
|
|
2311
2407
|
}
|
|
@@ -2321,7 +2417,7 @@ var MultiSourceSearchEngine = class {
|
|
|
2321
2417
|
if (res.ok) {
|
|
2322
2418
|
const data = await res.json();
|
|
2323
2419
|
if (Array.isArray(data?.results) && data.results.length > 0) {
|
|
2324
|
-
return data.results.slice(0, limit).map((r) => ({
|
|
2420
|
+
return data.results.filter((r) => !r.url || isUrlAllowedByDomainFilter(r.url, domainFilter)).slice(0, limit).map((r) => ({
|
|
2325
2421
|
title: r.title || "Untitled",
|
|
2326
2422
|
url: r.url,
|
|
2327
2423
|
content: r.content || r.snippet || "",
|
|
@@ -2379,9 +2475,13 @@ var MultiSourceSearchEngine = class {
|
|
|
2379
2475
|
}
|
|
2380
2476
|
return [];
|
|
2381
2477
|
}
|
|
2382
|
-
async querySerper(query, serperKey, limit, timeRange) {
|
|
2478
|
+
async querySerper(query, serperKey, limit, timeRange, domainFilter) {
|
|
2383
2479
|
try {
|
|
2384
|
-
|
|
2480
|
+
let effectiveQuery = query;
|
|
2481
|
+
if (domainFilter && domainFilter.length === 1) {
|
|
2482
|
+
effectiveQuery = `${query} site:${domainFilter[0]}`;
|
|
2483
|
+
}
|
|
2484
|
+
const body = { q: effectiveQuery, num: limit };
|
|
2385
2485
|
if (timeRange) {
|
|
2386
2486
|
const tbsMap = { day: "qdr:d", week: "qdr:w", month: "qdr:m", year: "qdr:y" };
|
|
2387
2487
|
if (tbsMap[timeRange]) {
|
|
@@ -2400,7 +2500,7 @@ var MultiSourceSearchEngine = class {
|
|
|
2400
2500
|
if (res.ok) {
|
|
2401
2501
|
const data = await res.json();
|
|
2402
2502
|
if (Array.isArray(data?.organic) && data.organic.length > 0) {
|
|
2403
|
-
return data.organic.slice(0, limit).map((r) => ({
|
|
2503
|
+
return data.organic.filter((r) => !r.link || isUrlAllowedByDomainFilter(r.link, domainFilter)).slice(0, limit).map((r) => ({
|
|
2404
2504
|
title: r.title || "Untitled",
|
|
2405
2505
|
url: r.link,
|
|
2406
2506
|
content: r.snippet || "",
|
|
@@ -2413,9 +2513,13 @@ var MultiSourceSearchEngine = class {
|
|
|
2413
2513
|
}
|
|
2414
2514
|
return [];
|
|
2415
2515
|
}
|
|
2416
|
-
async queryDuckDuckGo(query, limit) {
|
|
2516
|
+
async queryDuckDuckGo(query, limit, domainFilter) {
|
|
2417
2517
|
try {
|
|
2418
|
-
|
|
2518
|
+
let effectiveQuery = query;
|
|
2519
|
+
if (domainFilter && domainFilter.length === 1) {
|
|
2520
|
+
effectiveQuery = `${query} site:${domainFilter[0]}`;
|
|
2521
|
+
}
|
|
2522
|
+
const ddgUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(effectiveQuery)}`;
|
|
2419
2523
|
const res = await fetch(ddgUrl, {
|
|
2420
2524
|
headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" },
|
|
2421
2525
|
signal: AbortSignal.timeout(8e3)
|
|
@@ -2423,20 +2527,45 @@ var MultiSourceSearchEngine = class {
|
|
|
2423
2527
|
if (res.ok) {
|
|
2424
2528
|
const html = await res.text();
|
|
2425
2529
|
const hits = [];
|
|
2426
|
-
const
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
const
|
|
2430
|
-
|
|
2431
|
-
const
|
|
2432
|
-
const
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2530
|
+
const resultBlocks = html.split(/class="[^"]*result__body/i);
|
|
2531
|
+
for (const block of resultBlocks.slice(1)) {
|
|
2532
|
+
if (hits.length >= limit) break;
|
|
2533
|
+
const titleMatch = block.match(/<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
|
2534
|
+
if (!titleMatch) continue;
|
|
2535
|
+
const rawHref = titleMatch[1]?.trim() || "";
|
|
2536
|
+
const cleanUrl = rawHref.startsWith("//duckduckgo.com/l/?uddg=") || rawHref.includes("uddg=") ? decodeURIComponent(rawHref.split("uddg=")[1]?.split("&")[0] || "") : rawHref;
|
|
2537
|
+
const rawTitle = titleMatch[2]?.replace(/<[^>]+>/g, "").trim() || query;
|
|
2538
|
+
const snippetMatch = block.match(/<(?:a|div)[^>]*class="[^"]*result__snippet[^"]*"[^>]*>([\s\S]*?)<\/(?:a|div)>/i);
|
|
2539
|
+
const snippet = snippetMatch ? snippetMatch[1]?.replace(/<[^>]+>/g, "").trim() || "" : "";
|
|
2540
|
+
if (cleanUrl && !cleanUrl.includes("duckduckgo.com") && cleanUrl.startsWith("http")) {
|
|
2541
|
+
if (isUrlAllowedByDomainFilter(cleanUrl, domainFilter)) {
|
|
2542
|
+
hits.push({
|
|
2543
|
+
title: rawTitle,
|
|
2544
|
+
url: cleanUrl,
|
|
2545
|
+
content: snippet,
|
|
2546
|
+
engine: "duckduckgo"
|
|
2547
|
+
});
|
|
2548
|
+
}
|
|
2549
|
+
}
|
|
2550
|
+
}
|
|
2551
|
+
if (hits.length === 0) {
|
|
2552
|
+
const resultRegex = /<a[^>]*class="result__a"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
|
|
2553
|
+
let match;
|
|
2554
|
+
while ((match = resultRegex.exec(html)) !== null && hits.length < limit) {
|
|
2555
|
+
const rawTitle = match[1]?.replace(/<[^>]+>/g, "").trim() || query;
|
|
2556
|
+
const rawUrl = match[2]?.trim();
|
|
2557
|
+
const cleanUrl = rawUrl?.startsWith("//duckduckgo.com/l/?uddg=") ? decodeURIComponent(rawUrl.split("uddg=")[1]?.split("&")[0] || "") : rawUrl;
|
|
2558
|
+
const snippet = match[3]?.replace(/<[^>]+>/g, "").trim() || "";
|
|
2559
|
+
if (cleanUrl && !cleanUrl.includes("duckduckgo.com")) {
|
|
2560
|
+
if (isUrlAllowedByDomainFilter(cleanUrl, domainFilter)) {
|
|
2561
|
+
hits.push({
|
|
2562
|
+
title: rawTitle,
|
|
2563
|
+
url: cleanUrl,
|
|
2564
|
+
content: snippet,
|
|
2565
|
+
engine: "duckduckgo"
|
|
2566
|
+
});
|
|
2567
|
+
}
|
|
2568
|
+
}
|
|
2440
2569
|
}
|
|
2441
2570
|
}
|
|
2442
2571
|
if (hits.length > 0) return hits;
|
|
@@ -3191,6 +3320,28 @@ function describeZodSchemaKeys(schema) {
|
|
|
3191
3320
|
}
|
|
3192
3321
|
return void 0;
|
|
3193
3322
|
}
|
|
3323
|
+
function extractJsonEnvelope(raw) {
|
|
3324
|
+
if (!raw || typeof raw !== "string") return null;
|
|
3325
|
+
const trimmed = raw.trim();
|
|
3326
|
+
const unFenced = trimmed.replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/i, "").trim();
|
|
3327
|
+
try {
|
|
3328
|
+
JSON.parse(unFenced);
|
|
3329
|
+
return unFenced;
|
|
3330
|
+
} catch {
|
|
3331
|
+
}
|
|
3332
|
+
const start = unFenced.indexOf("{");
|
|
3333
|
+
const end = unFenced.lastIndexOf("}");
|
|
3334
|
+
if (start !== -1 && end > start) {
|
|
3335
|
+
const slice = unFenced.slice(start, end + 1);
|
|
3336
|
+
try {
|
|
3337
|
+
JSON.parse(slice);
|
|
3338
|
+
return slice;
|
|
3339
|
+
} catch {
|
|
3340
|
+
return slice;
|
|
3341
|
+
}
|
|
3342
|
+
}
|
|
3343
|
+
return null;
|
|
3344
|
+
}
|
|
3194
3345
|
function resolveResearchProfile(contract) {
|
|
3195
3346
|
if (contract.profile && contract.profile !== "auto") {
|
|
3196
3347
|
return contract.profile;
|
|
@@ -3599,25 +3750,23 @@ ${currentMissingGaps.length > 0 ? currentMissingGaps.map((g) => `- ${g}`).join("
|
|
|
3599
3750
|
if (hit.content && hit.content.trim().length > 35) {
|
|
3600
3751
|
const hitScore = computeNativeHitScore(hit, 7, 6);
|
|
3601
3752
|
ledger.addSearchHit(hit, hitScore);
|
|
3602
|
-
substantiveBytes +=
|
|
3603
|
-
maxScrapePerRound === 0 ? 500 : 100,
|
|
3604
|
-
Buffer.byteLength(hit.content, "utf-8")
|
|
3605
|
-
);
|
|
3753
|
+
substantiveBytes += Buffer.byteLength(hit.content, "utf-8");
|
|
3606
3754
|
}
|
|
3607
3755
|
}
|
|
3608
3756
|
} else if (channel === "academic") {
|
|
3609
|
-
const
|
|
3610
|
-
const
|
|
3611
|
-
this.academicEngine.searchPapers(
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
for (const p of
|
|
3757
|
+
const targetQueries = queries.length > 0 ? queries.slice(0, 2) : [contract.objective];
|
|
3758
|
+
const paperLists = await Promise.all(
|
|
3759
|
+
targetQueries.map((q) => this.academicEngine.searchPapers(q, { limit: 3 }).catch(() => []))
|
|
3760
|
+
);
|
|
3761
|
+
const collectedPapers = paperLists.flat();
|
|
3762
|
+
for (const p of collectedPapers.slice(0, 5)) {
|
|
3615
3763
|
const receipt = ledger.addAcademicPaper(p);
|
|
3616
3764
|
const item = ledger.getById(receipt.id);
|
|
3617
3765
|
substantiveBytes += Math.max(850, item?.rawByteSize || 0);
|
|
3618
3766
|
}
|
|
3619
3767
|
} else if (channel === "social_pulse") {
|
|
3620
|
-
const
|
|
3768
|
+
const pulseQuery = queries[0] || contract.objective;
|
|
3769
|
+
const pulseReport = await this.socialPulseHarvester.harvest(pulseQuery, {
|
|
3621
3770
|
recencyDays,
|
|
3622
3771
|
limitPerPlatform: 3
|
|
3623
3772
|
});
|
|
@@ -3626,7 +3775,8 @@ ${currentMissingGaps.length > 0 ? currentMissingGaps.map((g) => `- ${g}`).join("
|
|
|
3626
3775
|
substantiveBytes += Math.max(450, Buffer.byteLength(item.snippet || "", "utf-8"));
|
|
3627
3776
|
}
|
|
3628
3777
|
} else if (channel === "stem_oer") {
|
|
3629
|
-
const
|
|
3778
|
+
const oerQuery = queries[0] || contract.objective;
|
|
3779
|
+
const oerCatalog = await this.stemResearcher.findResources(oerQuery, {
|
|
3630
3780
|
limit: 4
|
|
3631
3781
|
});
|
|
3632
3782
|
for (const res of oerCatalog.resources.slice(0, 4)) {
|
|
@@ -3823,7 +3973,9 @@ Generate ${maxQueries} high-precision search queries to fulfill ${round === 1 ?
|
|
|
3823
3973
|
if (receipts.length === 0) {
|
|
3824
3974
|
missingGaps.push(...targetCheckList);
|
|
3825
3975
|
} else {
|
|
3826
|
-
const receiptsText = receipts.map(
|
|
3976
|
+
const receiptsText = receipts.map(
|
|
3977
|
+
(r) => `- [${r.id}] ${r.title} (headings: ${r.headings.join(", ") || "N/A"}, codeBlocks: ${r.codeBlockCount}, tables: ${r.tableCount}): ${r.briefExcerpt}`
|
|
3978
|
+
).join("\n");
|
|
3827
3979
|
if (decision) {
|
|
3828
3980
|
const decEval = await evaluateAttributeCoverageWithDecision({
|
|
3829
3981
|
objective: contract.objective,
|
|
@@ -3892,7 +4044,7 @@ ${receiptsText}`,
|
|
|
3892
4044
|
const detBaseline = this.claimVerifier.verifyClaimsDeterministic(claims, sourceCorpus);
|
|
3893
4045
|
const res = await options.decision({
|
|
3894
4046
|
context: `Verified Evidence Corpus:
|
|
3895
|
-
${sourceCorpus.slice(0,
|
|
4047
|
+
${sourceCorpus.slice(0, 24e3)}`,
|
|
3896
4048
|
questions: claims.map((claim, idx) => ({
|
|
3897
4049
|
id: `claim_${idx}`,
|
|
3898
4050
|
type: "choice",
|
|
@@ -4003,9 +4155,9 @@ Return ONLY a valid JSON object with the exact envelope:`,
|
|
|
4003
4155
|
temperature: 0.2,
|
|
4004
4156
|
jsonMode: true
|
|
4005
4157
|
});
|
|
4006
|
-
const
|
|
4007
|
-
if (
|
|
4008
|
-
const parsed = JSON.parse(
|
|
4158
|
+
const jsonCandidate = extractJsonEnvelope(raw);
|
|
4159
|
+
if (jsonCandidate) {
|
|
4160
|
+
const parsed = JSON.parse(jsonCandidate);
|
|
4009
4161
|
markdownReport = typeof parsed.markdownReport === "string" ? parsed.markdownReport : "";
|
|
4010
4162
|
tier1Summary = typeof parsed.tier1Summary === "string" ? parsed.tier1Summary : "";
|
|
4011
4163
|
tier2TechnicalDigest = typeof parsed.tier2TechnicalDigest === "string" ? parsed.tier2TechnicalDigest : "";
|
|
@@ -4029,9 +4181,9 @@ Return ONLY a valid JSON object with the exact envelope:`,
|
|
|
4029
4181
|
temperature: 0.1,
|
|
4030
4182
|
jsonMode: true
|
|
4031
4183
|
});
|
|
4032
|
-
const
|
|
4033
|
-
if (
|
|
4034
|
-
const repParsed = JSON.parse(
|
|
4184
|
+
const repCandidate = extractJsonEnvelope(repairedRaw);
|
|
4185
|
+
if (repCandidate) {
|
|
4186
|
+
const repParsed = JSON.parse(repCandidate);
|
|
4035
4187
|
const repCheck = contract.outputSchema.safeParse(repParsed);
|
|
4036
4188
|
if (repCheck.success) {
|
|
4037
4189
|
synthesizedData = repCheck.data;
|
|
@@ -4080,6 +4232,13 @@ ${JSON.stringify(synthesizedData || {})}`;
|
|
|
4080
4232
|
markdownReport = markdownReport.replace(regex, "[UNVERIFIED-CITATION-REMOVED]");
|
|
4081
4233
|
tier1Summary = tier1Summary.replace(regex, "");
|
|
4082
4234
|
tier2TechnicalDigest = tier2TechnicalDigest.replace(regex, "");
|
|
4235
|
+
if (synthesizedData) {
|
|
4236
|
+
try {
|
|
4237
|
+
const rawJson = JSON.stringify(synthesizedData);
|
|
4238
|
+
synthesizedData = JSON.parse(rawJson.replace(regex, ""));
|
|
4239
|
+
} catch {
|
|
4240
|
+
}
|
|
4241
|
+
}
|
|
4083
4242
|
}
|
|
4084
4243
|
}
|
|
4085
4244
|
if (contract.acceptanceCriteria?.forbiddenTerms?.length) {
|
|
@@ -4264,9 +4423,11 @@ exports.ResearchSubAgent = ResearchSubAgent;
|
|
|
4264
4423
|
exports.SOURCE_CHANNELS = SOURCE_CHANNELS;
|
|
4265
4424
|
exports.classifyChannelStatus = classifyChannelStatus;
|
|
4266
4425
|
exports.classifyModeAndPrior = classifyModeAndPrior;
|
|
4426
|
+
exports.computeCanonicalId = computeCanonicalId;
|
|
4267
4427
|
exports.computeContractComplexityScore = computeContractComplexityScore;
|
|
4268
4428
|
exports.createResearchAgentTool = createResearchAgentTool;
|
|
4269
4429
|
exports.evaluateAttributeCoverageWithDecision = evaluateAttributeCoverageWithDecision;
|
|
4430
|
+
exports.extractJsonEnvelope = extractJsonEnvelope;
|
|
4270
4431
|
exports.inferFallbackFacetProbabilities = inferFallbackFacetProbabilities;
|
|
4271
4432
|
exports.isSemanticRankedHit = isSemanticRankedHit;
|
|
4272
4433
|
exports.rankChannelsByExpectedPrior = rankChannelsByExpectedPrior;
|