@thanh01.pmt/research-kit 1.4.0 → 1.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/dist/academic/index.cjs +17 -16
  2. package/dist/academic/index.cjs.map +1 -1
  3. package/dist/academic/index.d.cts +1 -1
  4. package/dist/academic/index.d.ts +1 -1
  5. package/dist/academic/index.js +3 -3
  6. package/dist/agent/index.cjs +287 -126
  7. package/dist/agent/index.cjs.map +1 -1
  8. package/dist/agent/index.d.cts +4 -4
  9. package/dist/agent/index.d.ts +4 -4
  10. package/dist/agent/index.js +7 -7
  11. package/dist/{catalogBuilder-D-UiM37y.d.ts → catalogBuilder-BN3VwEsE.d.ts} +15 -2
  12. package/dist/{catalogBuilder-BBCa9jqI.d.cts → catalogBuilder-mba5BIHD.d.cts} +15 -2
  13. package/dist/{chunk-RYLZYZRF.js → chunk-5HSLUAM6.js} +11 -3
  14. package/dist/chunk-5HSLUAM6.js.map +1 -0
  15. package/dist/{chunk-RAMDHDRJ.js → chunk-6QW5TFDJ.js} +70 -342
  16. package/dist/chunk-6QW5TFDJ.js.map +1 -0
  17. package/dist/chunk-BPZLZKVJ.js +1012 -0
  18. package/dist/chunk-BPZLZKVJ.js.map +1 -0
  19. package/dist/{chunk-AKEI7H2C.js → chunk-EWIAA2CC.js} +4 -4
  20. package/dist/{chunk-AKEI7H2C.js.map → chunk-EWIAA2CC.js.map} +1 -1
  21. package/dist/{chunk-QQZQNBJC.js → chunk-JV6Z6WUQ.js} +4 -4
  22. package/dist/{chunk-QQZQNBJC.js.map → chunk-JV6Z6WUQ.js.map} +1 -1
  23. package/dist/{chunk-4O3FXMXO.js → chunk-LTXXARCA.js} +5 -5
  24. package/dist/{chunk-4O3FXMXO.js.map → chunk-LTXXARCA.js.map} +1 -1
  25. package/dist/{chunk-UHKDJZNL.js → chunk-PFECHAAR.js} +3 -3
  26. package/dist/{chunk-UHKDJZNL.js.map → chunk-PFECHAAR.js.map} +1 -1
  27. package/dist/{chunk-MFRQ2VJZ.js → chunk-QLBHTCV6.js} +98 -38
  28. package/dist/chunk-QLBHTCV6.js.map +1 -0
  29. package/dist/{chunk-SDDZFFL6.js → chunk-SB7WYK7A.js} +15 -14
  30. package/dist/chunk-SB7WYK7A.js.map +1 -0
  31. package/dist/{chunk-SHBDLP64.js → chunk-SEHHDO7X.js} +5 -5
  32. package/dist/{chunk-SHBDLP64.js.map → chunk-SEHHDO7X.js.map} +1 -1
  33. package/dist/evidence/index.cjs.map +1 -1
  34. package/dist/evidence/index.js +2 -2
  35. package/dist/harvester/index.cjs +225 -92
  36. package/dist/harvester/index.cjs.map +1 -1
  37. package/dist/harvester/index.d.cts +2 -2
  38. package/dist/harvester/index.d.ts +2 -2
  39. package/dist/harvester/index.js +5 -5
  40. package/dist/{index-CGa7lf3L.d.ts → index-9Z-uMwdI.d.ts} +13 -5
  41. package/dist/{index-DrjfrbGw.d.cts → index-eTcF4XVo.d.cts} +13 -5
  42. package/dist/index.cjs +2578 -2348
  43. package/dist/index.cjs.map +1 -1
  44. package/dist/index.d.cts +5 -5
  45. package/dist/index.d.ts +5 -5
  46. package/dist/index.js +19 -19
  47. package/dist/{literatureReview-Bvlqk4DB.d.cts → literatureReview-BUtPL7bj.d.cts} +1 -0
  48. package/dist/{literatureReview-Bn2iEGRO.d.ts → literatureReview-CHz-JfSR.d.ts} +1 -0
  49. package/dist/pipeline/index.cjs +326 -132
  50. package/dist/pipeline/index.cjs.map +1 -1
  51. package/dist/pipeline/index.d.cts +15 -3
  52. package/dist/pipeline/index.d.ts +15 -3
  53. package/dist/pipeline/index.js +7 -7
  54. package/dist/{socialPulseHarvester-CN4MwrpY.d.cts → socialPulseHarvester-BCFxGwx2.d.cts} +4 -1
  55. package/dist/{socialPulseHarvester-IbdT3zD1.d.ts → socialPulseHarvester-DjMvBufT.d.ts} +4 -1
  56. package/package.json +1 -1
  57. package/dist/chunk-2RMBLKP5.js +0 -581
  58. package/dist/chunk-2RMBLKP5.js.map +0 -1
  59. package/dist/chunk-MFRQ2VJZ.js.map +0 -1
  60. package/dist/chunk-RAMDHDRJ.js.map +0 -1
  61. package/dist/chunk-RYLZYZRF.js.map +0 -1
  62. package/dist/chunk-SDDZFFL6.js.map +0 -1
@@ -6,7 +6,7 @@ var zod = require('zod');
6
6
  var RESEARCH_MODE_PRESETS = {
7
7
  search: {
8
8
  mode: "search",
9
- timeoutMs: 8e3,
9
+ timeoutMs: 15e3,
10
10
  maxChannels: 1,
11
11
  channelConcurrency: 1,
12
12
  maxRounds: 1,
@@ -16,7 +16,7 @@ var RESEARCH_MODE_PRESETS = {
16
16
  },
17
17
  research: {
18
18
  mode: "research",
19
- timeoutMs: 9e4,
19
+ timeoutMs: 18e4,
20
20
  maxChannels: 2,
21
21
  channelConcurrency: 2,
22
22
  maxRounds: 1,
@@ -26,7 +26,7 @@ var RESEARCH_MODE_PRESETS = {
26
26
  },
27
27
  "deep-research": {
28
28
  mode: "deep-research",
29
- timeoutMs: 3e5,
29
+ timeoutMs: 6e5,
30
30
  maxChannels: 5,
31
31
  channelConcurrency: 2,
32
32
  maxRounds: 3,
@@ -40,7 +40,7 @@ var RESEARCH_PROFILE_PRESETS = {
40
40
  maxRounds: 1,
41
41
  maxQueriesPerRound: 3,
42
42
  maxScrapePerRound: 3,
43
- timeoutMs: 9e4,
43
+ timeoutMs: 18e4,
44
44
  enableWeb: true,
45
45
  enableAcademic: false,
46
46
  enableSocialPulse: false,
@@ -50,7 +50,7 @@ var RESEARCH_PROFILE_PRESETS = {
50
50
  maxRounds: 3,
51
51
  maxQueriesPerRound: 4,
52
52
  maxScrapePerRound: 4,
53
- timeoutMs: 3e5,
53
+ timeoutMs: 6e5,
54
54
  enableWeb: true,
55
55
  enableAcademic: true,
56
56
  enableSocialPulse: true,
@@ -60,7 +60,7 @@ var RESEARCH_PROFILE_PRESETS = {
60
60
  maxRounds: 1,
61
61
  maxQueriesPerRound: 3,
62
62
  maxScrapePerRound: 3,
63
- timeoutMs: 9e4,
63
+ timeoutMs: 18e4,
64
64
  enableWeb: true,
65
65
  enableAcademic: false,
66
66
  enableSocialPulse: true,
@@ -70,7 +70,7 @@ var RESEARCH_PROFILE_PRESETS = {
70
70
  maxRounds: 1,
71
71
  maxQueriesPerRound: 3,
72
72
  maxScrapePerRound: 2,
73
- timeoutMs: 9e4,
73
+ timeoutMs: 18e4,
74
74
  enableWeb: true,
75
75
  enableAcademic: true,
76
76
  enableSocialPulse: false,
@@ -80,7 +80,7 @@ var RESEARCH_PROFILE_PRESETS = {
80
80
  maxRounds: 1,
81
81
  maxQueriesPerRound: 3,
82
82
  maxScrapePerRound: 2,
83
- timeoutMs: 9e4,
83
+ timeoutMs: 18e4,
84
84
  enableWeb: true,
85
85
  enableAcademic: false,
86
86
  enableSocialPulse: true,
@@ -90,7 +90,7 @@ var RESEARCH_PROFILE_PRESETS = {
90
90
  maxRounds: 1,
91
91
  maxQueriesPerRound: 3,
92
92
  maxScrapePerRound: 3,
93
- timeoutMs: 9e4,
93
+ timeoutMs: 18e4,
94
94
  enableWeb: true,
95
95
  enableAcademic: false,
96
96
  enableSocialPulse: false,
@@ -100,7 +100,7 @@ var RESEARCH_PROFILE_PRESETS = {
100
100
  maxRounds: 1,
101
101
  maxQueriesPerRound: 3,
102
102
  maxScrapePerRound: 3,
103
- timeoutMs: 9e4,
103
+ timeoutMs: 18e4,
104
104
  enableWeb: true,
105
105
  enableAcademic: false,
106
106
  enableSocialPulse: false,
@@ -121,6 +121,18 @@ function normalizeUrlKey(rawUrl) {
121
121
  return rawUrl.trim().replace(/\/$/, "").toLowerCase();
122
122
  }
123
123
  }
124
+ function computeCanonicalId(url) {
125
+ try {
126
+ let hash = 0;
127
+ const clean = url.trim().replace(/^https?:\/\//i, "").replace(/[\/?#].*$/, "").toLowerCase();
128
+ for (let i = 0; i < clean.length; i++) {
129
+ hash = hash * 31 + clean.charCodeAt(i) >>> 0;
130
+ }
131
+ return `SRC-${hash.toString(16).padStart(8, "0")}`;
132
+ } catch {
133
+ return `SRC-${Math.random().toString(36).slice(2, 10)}`;
134
+ }
135
+ }
124
136
  var EvidenceLedger = class {
125
137
  items = [];
126
138
  urlIndex = /* @__PURE__ */ new Map();
@@ -155,6 +167,7 @@ var EvidenceLedger = class {
155
167
  }
156
168
  const item = {
157
169
  id: this.allocateId(),
170
+ canonicalId: computeCanonicalId(doc.url),
158
171
  sourceType,
159
172
  title: doc.title || "Untitled Document",
160
173
  url: doc.url,
@@ -183,6 +196,7 @@ var EvidenceLedger = class {
183
196
  const snippet = (hit.content || "").replace(/\s+/g, " ").trim();
184
197
  const item = {
185
198
  id: this.allocateId(),
199
+ canonicalId: computeCanonicalId(hit.url),
186
200
  sourceType: hit.engine === "academic" ? "academic" : "web",
187
201
  title: hit.title || "Search Result",
188
202
  url: hit.url,
@@ -213,6 +227,7 @@ var EvidenceLedger = class {
213
227
  const citationBonus = paper.citationCount >= 50 ? 9 : paper.citationCount >= 10 ? 8 : 7;
214
228
  const item = {
215
229
  id: this.allocateId(),
230
+ canonicalId: computeCanonicalId(url),
216
231
  sourceType: "academic",
217
232
  title: paper.title,
218
233
  url,
@@ -242,6 +257,7 @@ var EvidenceLedger = class {
242
257
  }
243
258
  const item = {
244
259
  id: this.allocateId(),
260
+ canonicalId: computeCanonicalId(pulse.url),
245
261
  sourceType: "social_pulse",
246
262
  title: pulse.title,
247
263
  url: pulse.url,
@@ -272,6 +288,7 @@ var EvidenceLedger = class {
272
288
  const excerpt = `${resource.description} [Provider: ${resource.provider}${resource.license ? `, License: ${resource.license}` : ""}]`;
273
289
  const item = {
274
290
  id: this.allocateId(),
291
+ canonicalId: computeCanonicalId(resource.url),
275
292
  sourceType: "stem_oer",
276
293
  title: resource.title,
277
294
  url: resource.url,
@@ -296,6 +313,7 @@ var EvidenceLedger = class {
296
313
  const rawExcerpt = item.keyExcerpts[0] || item.title;
297
314
  return {
298
315
  id: item.id,
316
+ canonicalId: item.canonicalId || computeCanonicalId(item.url),
299
317
  sourceType: item.sourceType,
300
318
  title: item.title.slice(0, 140),
301
319
  url: item.url,
@@ -303,7 +321,7 @@ var EvidenceLedger = class {
303
321
  headings: item.headings.slice(0, 5),
304
322
  codeBlockCount: item.codeBlocks.length,
305
323
  tableCount: item.tables.length,
306
- briefExcerpt: rawExcerpt.slice(0, 300)
324
+ briefExcerpt: rawExcerpt.slice(0, 450)
307
325
  };
308
326
  }
309
327
  getItems() {
@@ -733,7 +751,7 @@ ${receiptsText}`,
733
751
  const missing = [];
734
752
  targetCheckList.forEach((attr, idx) => {
735
753
  const ans = res.answers[`attr_${idx}`];
736
- if (ans && ans.type === "noul" && ans.probability >= 0.55) {
754
+ if (ans && (ans.type === "noul" && ans.probability >= 0.55 || ans.passed === true || ans.choice === "yes")) {
737
755
  satisfied.push(attr);
738
756
  } else {
739
757
  missing.push(attr);
@@ -994,7 +1012,7 @@ var AcademicSearchEngine = class {
994
1012
  }
995
1013
  const res = await fetch(`https://api.semanticscholar.org/graph/v1/paper/search?${params.toString()}`, {
996
1014
  headers,
997
- signal: AbortSignal.timeout(6e3)
1015
+ signal: AbortSignal.timeout(12e3)
998
1016
  });
999
1017
  if (res.ok) {
1000
1018
  const data = await res.json();
@@ -1025,7 +1043,7 @@ var AcademicSearchEngine = class {
1025
1043
  try {
1026
1044
  const res = await fetch(`https://api.openalex.org/works?search=${encodeURIComponent(query)}&per-page=${limit}`, {
1027
1045
  headers: { "User-Agent": "OrchableComposer/1.0 (mailto:team@orchable.app)" },
1028
- signal: AbortSignal.timeout(6e3)
1046
+ signal: AbortSignal.timeout(12e3)
1029
1047
  });
1030
1048
  if (res.ok) {
1031
1049
  const data = await res.json();
@@ -1112,7 +1130,7 @@ var OpenAlexClient = class {
1112
1130
  headers: {
1113
1131
  "User-Agent": `OrchableResearchKit/1.3 (mailto:${this.mailto})`
1114
1132
  },
1115
- signal: AbortSignal.timeout(7e3)
1133
+ signal: AbortSignal.timeout(12e3)
1116
1134
  });
1117
1135
  if (res.ok) {
1118
1136
  const data = await res.json();
@@ -1152,7 +1170,7 @@ var OpenAlexClient = class {
1152
1170
  headers: {
1153
1171
  "User-Agent": `OrchableResearchKit/1.3 (mailto:${this.mailto})`
1154
1172
  },
1155
- signal: AbortSignal.timeout(7e3)
1173
+ signal: AbortSignal.timeout(12e3)
1156
1174
  });
1157
1175
  if (res.ok) {
1158
1176
  const data = await res.json();
@@ -1691,7 +1709,7 @@ var WebScraper = class {
1691
1709
  }
1692
1710
  const res = await fetch(jinaUrl, {
1693
1711
  headers,
1694
- signal: AbortSignal.timeout(8e3)
1712
+ signal: AbortSignal.timeout(15e3)
1695
1713
  });
1696
1714
  if (res.ok) {
1697
1715
  const markdown = await res.text();
@@ -1733,7 +1751,7 @@ var WebScraper = class {
1733
1751
  try {
1734
1752
  const res = await fetch(targetUrl, {
1735
1753
  headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" },
1736
- signal: AbortSignal.timeout(1e4)
1754
+ signal: AbortSignal.timeout(15e3)
1737
1755
  });
1738
1756
  if (res.ok) {
1739
1757
  const rawHtml = await res.text();
@@ -2088,6 +2106,18 @@ var HealthAuditor = class {
2088
2106
  };
2089
2107
 
2090
2108
  // src/harvester/multiSourceSearch.ts
2109
+ function isUrlAllowedByDomainFilter(url, domainFilter) {
2110
+ if (!domainFilter || domainFilter.length === 0) return true;
2111
+ try {
2112
+ const hostname = new URL(url).hostname.toLowerCase();
2113
+ return domainFilter.some((d) => {
2114
+ const cleanD = d.toLowerCase().trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "");
2115
+ return hostname === cleanD || hostname.endsWith(`.${cleanD}`);
2116
+ });
2117
+ } catch {
2118
+ return false;
2119
+ }
2120
+ }
2091
2121
  var MultiSourceSearchEngine = class {
2092
2122
  tavilyKey;
2093
2123
  exaKey;
@@ -2099,6 +2129,20 @@ var MultiSourceSearchEngine = class {
2099
2129
  allowEnvFallback;
2100
2130
  customAdapter;
2101
2131
  academicEngine;
2132
+ lastErrors = [];
2133
+ getLastErrors() {
2134
+ return [...this.lastErrors];
2135
+ }
2136
+ recordError(engine, error, status, traceErrors) {
2137
+ const errObj = {
2138
+ engine,
2139
+ error,
2140
+ status,
2141
+ timestamp: (/* @__PURE__ */ new Date()).toISOString()
2142
+ };
2143
+ this.lastErrors.push(errObj);
2144
+ traceErrors?.push(errObj);
2145
+ }
2102
2146
  constructor(options = {}) {
2103
2147
  this.allowEnvFallback = options.allowEnvFallback ?? true;
2104
2148
  this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
@@ -2179,97 +2223,128 @@ var MultiSourceSearchEngine = class {
2179
2223
  const brightDataZone = options.brightDataZone || this.brightDataZone;
2180
2224
  const searxngHost = options.searxngHost || this.searxngHost;
2181
2225
  if (preferred === "searxng" && searxngHost) {
2182
- const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders);
2226
+ const searxHits = await this.querySearxng(
2227
+ query,
2228
+ searxngHost,
2229
+ limit,
2230
+ options.searxngHeaders,
2231
+ options.timeRange,
2232
+ options.domainFilter
2233
+ );
2183
2234
  if (searxHits.length > 0) return searxHits;
2184
2235
  }
2185
2236
  if (preferred === "brightdata" && brightDataKey) {
2186
2237
  const bdHits = await this.queryBrightData(query, brightDataKey, limit, {
2187
2238
  zone: brightDataZone,
2188
- engine: options.brightDataEngine
2239
+ engine: options.brightDataEngine,
2240
+ domainFilter: options.domainFilter
2189
2241
  });
2190
2242
  if (bdHits.length > 0) return bdHits;
2191
2243
  }
2192
2244
  if (preferred === "exa" && exaKey) {
2193
- const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
2245
+ const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter, options.timeRange);
2194
2246
  if (exaHits.length > 0) return exaHits;
2195
2247
  }
2196
2248
  if (preferred === "serper" && serperKey) {
2197
- const serperHits = await this.querySerper(query, serperKey, limit);
2249
+ const serperHits = await this.querySerper(query, serperKey, limit, options.timeRange, options.domainFilter);
2198
2250
  if (serperHits.length > 0) return serperHits;
2199
2251
  }
2252
+ if (preferred === "tavily" && tavilyKey) {
2253
+ const tavilyHits = await this.queryTavily(query, tavilyKey, limit, options.timeRange, options.domainFilter);
2254
+ if (tavilyHits.length > 0) return tavilyHits;
2255
+ }
2200
2256
  if (preferred === "duckduckgo") {
2201
- const ddgHits2 = await this.queryDuckDuckGo(query, limit);
2202
- if (ddgHits2.length > 0) return ddgHits2;
2257
+ const ddgHits = await this.queryDuckDuckGo(query, limit, options.domainFilter);
2258
+ if (ddgHits.length > 0) return ddgHits;
2203
2259
  }
2204
- if (tavilyKey && (!preferred || preferred === "tavily")) {
2205
- try {
2206
- const body = {
2207
- api_key: tavilyKey,
2208
- query,
2209
- search_depth: "advanced",
2210
- include_answer: true,
2211
- max_results: limit
2212
- };
2213
- if (options.timeRange) {
2214
- body.time_range = options.timeRange;
2215
- }
2216
- if (options.domainFilter && options.domainFilter.length > 0) {
2217
- body.include_domains = options.domainFilter;
2218
- }
2219
- const res = await fetch("https://api.tavily.com/search", {
2220
- method: "POST",
2221
- headers: { "Content-Type": "application/json" },
2222
- body: JSON.stringify(body),
2223
- signal: AbortSignal.timeout(12e3)
2224
- });
2225
- if (res.ok) {
2226
- const data = await res.json();
2227
- if (Array.isArray(data?.results) && data.results.length > 0) {
2228
- return data.results.map((r) => ({
2229
- title: r.title || "Untitled",
2230
- url: r.url,
2231
- content: r.content || "",
2232
- score: r.score,
2233
- engine: "tavily"
2234
- }));
2235
- }
2236
- }
2237
- } catch (e) {
2238
- console.warn("[MultiSourceSearchEngine] Tavily attempt failed:", e?.message || e);
2239
- }
2260
+ if (tavilyKey && preferred !== "tavily") {
2261
+ const tavilyHits = await this.queryTavily(query, tavilyKey, limit, options.timeRange, options.domainFilter);
2262
+ if (tavilyHits.length > 0) return tavilyHits;
2240
2263
  }
2241
- if (exaKey) {
2264
+ if (exaKey && preferred !== "exa") {
2242
2265
  const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter, options.timeRange);
2243
2266
  if (exaHits.length > 0) return exaHits;
2244
2267
  }
2245
- if (serperKey) {
2246
- const serperHits = await this.querySerper(query, serperKey, limit, options.timeRange);
2268
+ if (serperKey && preferred !== "serper") {
2269
+ const serperHits = await this.querySerper(query, serperKey, limit, options.timeRange, options.domainFilter);
2247
2270
  if (serperHits.length > 0) return serperHits;
2248
2271
  }
2249
- if (brightDataKey) {
2272
+ if (brightDataKey && preferred !== "brightdata") {
2250
2273
  const bdHits = await this.queryBrightData(query, brightDataKey, limit, {
2251
2274
  zone: brightDataZone,
2252
- engine: options.brightDataEngine
2275
+ engine: options.brightDataEngine,
2276
+ domainFilter: options.domainFilter
2253
2277
  });
2254
2278
  if (bdHits.length > 0) return bdHits;
2255
2279
  }
2256
- if (searxngHost) {
2257
- const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders, options.timeRange);
2280
+ if (searxngHost && preferred !== "searxng") {
2281
+ const searxHits = await this.querySearxng(
2282
+ query,
2283
+ searxngHost,
2284
+ limit,
2285
+ options.searxngHeaders,
2286
+ options.timeRange,
2287
+ options.domainFilter
2288
+ );
2258
2289
  if (searxHits.length > 0) return searxHits;
2259
2290
  }
2260
- const ddgHits = await this.queryDuckDuckGo(query, limit);
2261
- if (ddgHits.length > 0) return ddgHits;
2291
+ if (preferred !== "duckduckgo") {
2292
+ const ddgHits = await this.queryDuckDuckGo(query, limit, options.domainFilter);
2293
+ if (ddgHits.length > 0) return ddgHits;
2294
+ }
2295
+ return [];
2296
+ }
2297
+ async queryTavily(query, tavilyKey, limit, timeRange, domainFilter) {
2298
+ try {
2299
+ const body = {
2300
+ api_key: tavilyKey,
2301
+ query,
2302
+ search_depth: "advanced",
2303
+ include_answer: true,
2304
+ max_results: limit
2305
+ };
2306
+ if (timeRange) {
2307
+ body.time_range = timeRange;
2308
+ }
2309
+ if (domainFilter && domainFilter.length > 0) {
2310
+ body.include_domains = domainFilter;
2311
+ }
2312
+ const res = await fetch("https://api.tavily.com/search", {
2313
+ method: "POST",
2314
+ headers: { "Content-Type": "application/json" },
2315
+ body: JSON.stringify(body),
2316
+ signal: AbortSignal.timeout(12e3)
2317
+ });
2318
+ if (res.ok) {
2319
+ const data = await res.json();
2320
+ if (Array.isArray(data?.results) && data.results.length > 0) {
2321
+ return data.results.map((r) => ({
2322
+ title: r.title || "Untitled",
2323
+ url: r.url,
2324
+ content: r.content || "",
2325
+ score: r.score,
2326
+ engine: "tavily"
2327
+ }));
2328
+ }
2329
+ }
2330
+ } catch (e) {
2331
+ console.warn("[MultiSourceSearchEngine] Tavily attempt failed:", e?.message || e);
2332
+ }
2262
2333
  return [];
2263
2334
  }
2264
2335
  async queryBrightData(query, key, limit = 5, options = {}) {
2265
2336
  try {
2266
2337
  const zone = options.zone || this.brightDataZone || "cli_unlocker";
2267
2338
  const engine = options.engine || "bing";
2268
- let searchUrl = `https://www.bing.com/search?q=${encodeURIComponent(query)}&brd_json=1`;
2339
+ let effectiveQuery = query;
2340
+ if (options.domainFilter && options.domainFilter.length === 1) {
2341
+ effectiveQuery = `${query} site:${options.domainFilter[0]}`;
2342
+ }
2343
+ let searchUrl = `https://www.bing.com/search?q=${encodeURIComponent(effectiveQuery)}&brd_json=1`;
2269
2344
  if (engine === "google") {
2270
- searchUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}&brd_json=1&hl=en&gl=us`;
2345
+ searchUrl = `https://www.google.com/search?q=${encodeURIComponent(effectiveQuery)}&brd_json=1&hl=en&gl=us`;
2271
2346
  } else if (engine === "yandex") {
2272
- searchUrl = `https://www.yandex.com/search?q=${encodeURIComponent(query)}&brd_json=1`;
2347
+ searchUrl = `https://www.yandex.com/search?q=${encodeURIComponent(effectiveQuery)}&brd_json=1`;
2273
2348
  }
2274
2349
  const res = await fetch("https://api.brightdata.com/request", {
2275
2350
  method: "POST",
@@ -2285,27 +2360,48 @@ var MultiSourceSearchEngine = class {
2285
2360
  signal: AbortSignal.timeout(15e3)
2286
2361
  });
2287
2362
  if (res.ok) {
2288
- const data = await res.json();
2289
- const organic = Array.isArray(data?.organic) ? data.organic : [];
2290
- if (organic.length > 0) {
2291
- return organic.slice(0, limit).map((r, idx) => ({
2292
- title: r.title || "Untitled",
2293
- url: r.link,
2294
- content: r.description || r.snippet || "",
2295
- score: r.rank ? 1 / r.rank : 1 / (idx + 1),
2296
- engine: "brightdata"
2297
- }));
2363
+ let data;
2364
+ try {
2365
+ if (typeof res.json === "function") {
2366
+ data = await res.json();
2367
+ } else if (typeof res.text === "function") {
2368
+ const rawText = await res.text();
2369
+ data = JSON.parse(rawText);
2370
+ }
2371
+ } catch {
2372
+ data = null;
2373
+ }
2374
+ const hits = [];
2375
+ if (data && Array.isArray(data?.organic)) {
2376
+ for (let idx = 0; idx < data.organic.length; idx++) {
2377
+ const item = data.organic[idx];
2378
+ const hitUrl = item.link || item.url;
2379
+ if (!hitUrl || isUrlAllowedByDomainFilter(hitUrl, options.domainFilter)) {
2380
+ hits.push({
2381
+ title: item.title || "Untitled",
2382
+ url: hitUrl,
2383
+ content: item.description || item.snippet || "",
2384
+ score: item.rank ? 1 / item.rank : 1 / (idx + 1),
2385
+ engine: "brightdata"
2386
+ });
2387
+ }
2388
+ }
2298
2389
  }
2390
+ if (hits.length > 0) return hits.slice(0, limit);
2299
2391
  }
2300
2392
  } catch (e) {
2301
2393
  console.warn("[MultiSourceSearchEngine] Bright Data attempt failed:", e?.message || e);
2302
2394
  }
2303
2395
  return [];
2304
2396
  }
2305
- async querySearxng(query, searxngHost, limit, customHeaders, timeRange) {
2397
+ async querySearxng(query, searxngHost, limit, customHeaders, timeRange, domainFilter) {
2306
2398
  try {
2307
2399
  const cleanHost = searxngHost.replace(/\/$/, "");
2308
- let searchUrl = `${cleanHost}/search?q=${encodeURIComponent(query)}&format=json`;
2400
+ let effectiveQuery = query;
2401
+ if (domainFilter && domainFilter.length === 1) {
2402
+ effectiveQuery = `${query} site:${domainFilter[0]}`;
2403
+ }
2404
+ let searchUrl = `${cleanHost}/search?q=${encodeURIComponent(effectiveQuery)}&format=json`;
2309
2405
  if (timeRange) {
2310
2406
  searchUrl += `&time_range=${encodeURIComponent(timeRange)}`;
2311
2407
  }
@@ -2321,7 +2417,7 @@ var MultiSourceSearchEngine = class {
2321
2417
  if (res.ok) {
2322
2418
  const data = await res.json();
2323
2419
  if (Array.isArray(data?.results) && data.results.length > 0) {
2324
- return data.results.slice(0, limit).map((r) => ({
2420
+ return data.results.filter((r) => !r.url || isUrlAllowedByDomainFilter(r.url, domainFilter)).slice(0, limit).map((r) => ({
2325
2421
  title: r.title || "Untitled",
2326
2422
  url: r.url,
2327
2423
  content: r.content || r.snippet || "",
@@ -2379,9 +2475,13 @@ var MultiSourceSearchEngine = class {
2379
2475
  }
2380
2476
  return [];
2381
2477
  }
2382
- async querySerper(query, serperKey, limit, timeRange) {
2478
+ async querySerper(query, serperKey, limit, timeRange, domainFilter) {
2383
2479
  try {
2384
- const body = { q: query, num: limit };
2480
+ let effectiveQuery = query;
2481
+ if (domainFilter && domainFilter.length === 1) {
2482
+ effectiveQuery = `${query} site:${domainFilter[0]}`;
2483
+ }
2484
+ const body = { q: effectiveQuery, num: limit };
2385
2485
  if (timeRange) {
2386
2486
  const tbsMap = { day: "qdr:d", week: "qdr:w", month: "qdr:m", year: "qdr:y" };
2387
2487
  if (tbsMap[timeRange]) {
@@ -2400,7 +2500,7 @@ var MultiSourceSearchEngine = class {
2400
2500
  if (res.ok) {
2401
2501
  const data = await res.json();
2402
2502
  if (Array.isArray(data?.organic) && data.organic.length > 0) {
2403
- return data.organic.slice(0, limit).map((r) => ({
2503
+ return data.organic.filter((r) => !r.link || isUrlAllowedByDomainFilter(r.link, domainFilter)).slice(0, limit).map((r) => ({
2404
2504
  title: r.title || "Untitled",
2405
2505
  url: r.link,
2406
2506
  content: r.snippet || "",
@@ -2413,9 +2513,13 @@ var MultiSourceSearchEngine = class {
2413
2513
  }
2414
2514
  return [];
2415
2515
  }
2416
- async queryDuckDuckGo(query, limit) {
2516
+ async queryDuckDuckGo(query, limit, domainFilter) {
2417
2517
  try {
2418
- const ddgUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
2518
+ let effectiveQuery = query;
2519
+ if (domainFilter && domainFilter.length === 1) {
2520
+ effectiveQuery = `${query} site:${domainFilter[0]}`;
2521
+ }
2522
+ const ddgUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(effectiveQuery)}`;
2419
2523
  const res = await fetch(ddgUrl, {
2420
2524
  headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" },
2421
2525
  signal: AbortSignal.timeout(8e3)
@@ -2423,20 +2527,45 @@ var MultiSourceSearchEngine = class {
2423
2527
  if (res.ok) {
2424
2528
  const html = await res.text();
2425
2529
  const hits = [];
2426
- const resultRegex = /<a[^>]*class="result__a"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
2427
- let match;
2428
- while ((match = resultRegex.exec(html)) !== null && hits.length < limit) {
2429
- const rawTitle = match[1]?.replace(/<[^>]+>/g, "").trim() || query;
2430
- const rawUrl = match[2]?.trim();
2431
- const cleanUrl = rawUrl?.startsWith("//duckduckgo.com/l/?uddg=") ? decodeURIComponent(rawUrl.split("uddg=")[1]?.split("&")[0] || "") : rawUrl;
2432
- const snippet = match[3]?.replace(/<[^>]+>/g, "").trim() || "";
2433
- if (cleanUrl && !cleanUrl.includes("duckduckgo.com")) {
2434
- hits.push({
2435
- title: rawTitle,
2436
- url: cleanUrl,
2437
- content: snippet,
2438
- engine: "duckduckgo"
2439
- });
2530
+ const resultBlocks = html.split(/class="[^"]*result__body/i);
2531
+ for (const block of resultBlocks.slice(1)) {
2532
+ if (hits.length >= limit) break;
2533
+ const titleMatch = block.match(/<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
2534
+ if (!titleMatch) continue;
2535
+ const rawHref = titleMatch[1]?.trim() || "";
2536
+ const cleanUrl = rawHref.startsWith("//duckduckgo.com/l/?uddg=") || rawHref.includes("uddg=") ? decodeURIComponent(rawHref.split("uddg=")[1]?.split("&")[0] || "") : rawHref;
2537
+ const rawTitle = titleMatch[2]?.replace(/<[^>]+>/g, "").trim() || query;
2538
+ const snippetMatch = block.match(/<(?:a|div)[^>]*class="[^"]*result__snippet[^"]*"[^>]*>([\s\S]*?)<\/(?:a|div)>/i);
2539
+ const snippet = snippetMatch ? snippetMatch[1]?.replace(/<[^>]+>/g, "").trim() || "" : "";
2540
+ if (cleanUrl && !cleanUrl.includes("duckduckgo.com") && cleanUrl.startsWith("http")) {
2541
+ if (isUrlAllowedByDomainFilter(cleanUrl, domainFilter)) {
2542
+ hits.push({
2543
+ title: rawTitle,
2544
+ url: cleanUrl,
2545
+ content: snippet,
2546
+ engine: "duckduckgo"
2547
+ });
2548
+ }
2549
+ }
2550
+ }
2551
+ if (hits.length === 0) {
2552
+ const resultRegex = /<a[^>]*class="result__a"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
2553
+ let match;
2554
+ while ((match = resultRegex.exec(html)) !== null && hits.length < limit) {
2555
+ const rawTitle = match[1]?.replace(/<[^>]+>/g, "").trim() || query;
2556
+ const rawUrl = match[2]?.trim();
2557
+ const cleanUrl = rawUrl?.startsWith("//duckduckgo.com/l/?uddg=") ? decodeURIComponent(rawUrl.split("uddg=")[1]?.split("&")[0] || "") : rawUrl;
2558
+ const snippet = match[3]?.replace(/<[^>]+>/g, "").trim() || "";
2559
+ if (cleanUrl && !cleanUrl.includes("duckduckgo.com")) {
2560
+ if (isUrlAllowedByDomainFilter(cleanUrl, domainFilter)) {
2561
+ hits.push({
2562
+ title: rawTitle,
2563
+ url: cleanUrl,
2564
+ content: snippet,
2565
+ engine: "duckduckgo"
2566
+ });
2567
+ }
2568
+ }
2440
2569
  }
2441
2570
  }
2442
2571
  if (hits.length > 0) return hits;
@@ -2520,7 +2649,7 @@ var SocialPulseHarvester = class {
2520
2649
  });
2521
2650
  const res = await fetch(`https://hn.algolia.com/api/v1/search?${params.toString()}`, {
2522
2651
  headers: { "User-Agent": "OrchableResearchKit/1.3" },
2523
- signal: AbortSignal.timeout(6500)
2652
+ signal: AbortSignal.timeout(12e3)
2524
2653
  });
2525
2654
  if (res.ok) {
2526
2655
  const data = await res.json();
@@ -2567,7 +2696,7 @@ var SocialPulseHarvester = class {
2567
2696
  Accept: "application/vnd.github+json",
2568
2697
  "User-Agent": "OrchableResearchKit/1.3"
2569
2698
  },
2570
- signal: AbortSignal.timeout(6500)
2699
+ signal: AbortSignal.timeout(12e3)
2571
2700
  });
2572
2701
  if (res.ok) {
2573
2702
  const data = await res.json();
@@ -3191,6 +3320,28 @@ function describeZodSchemaKeys(schema) {
3191
3320
  }
3192
3321
  return void 0;
3193
3322
  }
3323
+ function extractJsonEnvelope(raw) {
3324
+ if (!raw || typeof raw !== "string") return null;
3325
+ const trimmed = raw.trim();
3326
+ const unFenced = trimmed.replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/i, "").trim();
3327
+ try {
3328
+ JSON.parse(unFenced);
3329
+ return unFenced;
3330
+ } catch {
3331
+ }
3332
+ const start = unFenced.indexOf("{");
3333
+ const end = unFenced.lastIndexOf("}");
3334
+ if (start !== -1 && end > start) {
3335
+ const slice = unFenced.slice(start, end + 1);
3336
+ try {
3337
+ JSON.parse(slice);
3338
+ return slice;
3339
+ } catch {
3340
+ return slice;
3341
+ }
3342
+ }
3343
+ return null;
3344
+ }
3194
3345
  function resolveResearchProfile(contract) {
3195
3346
  if (contract.profile && contract.profile !== "auto") {
3196
3347
  return contract.profile;
@@ -3599,25 +3750,23 @@ ${currentMissingGaps.length > 0 ? currentMissingGaps.map((g) => `- ${g}`).join("
3599
3750
  if (hit.content && hit.content.trim().length > 35) {
3600
3751
  const hitScore = computeNativeHitScore(hit, 7, 6);
3601
3752
  ledger.addSearchHit(hit, hitScore);
3602
- substantiveBytes += Math.max(
3603
- maxScrapePerRound === 0 ? 500 : 100,
3604
- Buffer.byteLength(hit.content, "utf-8")
3605
- );
3753
+ substantiveBytes += Buffer.byteLength(hit.content, "utf-8");
3606
3754
  }
3607
3755
  }
3608
3756
  } else if (channel === "academic") {
3609
- const primaryQuery = queries[0] || contract.objective;
3610
- const [s2Papers, oaPapers] = await Promise.all([
3611
- this.academicEngine.searchPapers(primaryQuery, { limit: 3 }).catch(() => []),
3612
- this.openAlexClient.searchWorks(primaryQuery, { limit: 3 }).catch(() => [])
3613
- ]);
3614
- for (const p of [...s2Papers, ...oaPapers].slice(0, 5)) {
3757
+ const targetQueries = queries.length > 0 ? queries.slice(0, 2) : [contract.objective];
3758
+ const paperLists = await Promise.all(
3759
+ targetQueries.map((q) => this.academicEngine.searchPapers(q, { limit: 3 }).catch(() => []))
3760
+ );
3761
+ const collectedPapers = paperLists.flat();
3762
+ for (const p of collectedPapers.slice(0, 5)) {
3615
3763
  const receipt = ledger.addAcademicPaper(p);
3616
3764
  const item = ledger.getById(receipt.id);
3617
3765
  substantiveBytes += Math.max(850, item?.rawByteSize || 0);
3618
3766
  }
3619
3767
  } else if (channel === "social_pulse") {
3620
- const pulseReport = await this.socialPulseHarvester.harvest(contract.objective, {
3768
+ const pulseQuery = queries[0] || contract.objective;
3769
+ const pulseReport = await this.socialPulseHarvester.harvest(pulseQuery, {
3621
3770
  recencyDays,
3622
3771
  limitPerPlatform: 3
3623
3772
  });
@@ -3626,7 +3775,8 @@ ${currentMissingGaps.length > 0 ? currentMissingGaps.map((g) => `- ${g}`).join("
3626
3775
  substantiveBytes += Math.max(450, Buffer.byteLength(item.snippet || "", "utf-8"));
3627
3776
  }
3628
3777
  } else if (channel === "stem_oer") {
3629
- const oerCatalog = await this.stemResearcher.findResources(contract.objective, {
3778
+ const oerQuery = queries[0] || contract.objective;
3779
+ const oerCatalog = await this.stemResearcher.findResources(oerQuery, {
3630
3780
  limit: 4
3631
3781
  });
3632
3782
  for (const res of oerCatalog.resources.slice(0, 4)) {
@@ -3823,7 +3973,9 @@ Generate ${maxQueries} high-precision search queries to fulfill ${round === 1 ?
3823
3973
  if (receipts.length === 0) {
3824
3974
  missingGaps.push(...targetCheckList);
3825
3975
  } else {
3826
- const receiptsText = receipts.map((r) => `- [${r.id}] ${r.title} (headings: ${r.headings.join(", ")}): ${r.briefExcerpt}`).join("\n");
3976
+ const receiptsText = receipts.map(
3977
+ (r) => `- [${r.id}] ${r.title} (headings: ${r.headings.join(", ") || "N/A"}, codeBlocks: ${r.codeBlockCount}, tables: ${r.tableCount}): ${r.briefExcerpt}`
3978
+ ).join("\n");
3827
3979
  if (decision) {
3828
3980
  const decEval = await evaluateAttributeCoverageWithDecision({
3829
3981
  objective: contract.objective,
@@ -3892,7 +4044,7 @@ ${receiptsText}`,
3892
4044
  const detBaseline = this.claimVerifier.verifyClaimsDeterministic(claims, sourceCorpus);
3893
4045
  const res = await options.decision({
3894
4046
  context: `Verified Evidence Corpus:
3895
- ${sourceCorpus.slice(0, 6e3)}`,
4047
+ ${sourceCorpus.slice(0, 24e3)}`,
3896
4048
  questions: claims.map((claim, idx) => ({
3897
4049
  id: `claim_${idx}`,
3898
4050
  type: "choice",
@@ -4003,9 +4155,9 @@ Return ONLY a valid JSON object with the exact envelope:`,
4003
4155
  temperature: 0.2,
4004
4156
  jsonMode: true
4005
4157
  });
4006
- const match = raw.match(/\{[\s\S]*\}/);
4007
- if (match) {
4008
- const parsed = JSON.parse(match[0]);
4158
+ const jsonCandidate = extractJsonEnvelope(raw);
4159
+ if (jsonCandidate) {
4160
+ const parsed = JSON.parse(jsonCandidate);
4009
4161
  markdownReport = typeof parsed.markdownReport === "string" ? parsed.markdownReport : "";
4010
4162
  tier1Summary = typeof parsed.tier1Summary === "string" ? parsed.tier1Summary : "";
4011
4163
  tier2TechnicalDigest = typeof parsed.tier2TechnicalDigest === "string" ? parsed.tier2TechnicalDigest : "";
@@ -4029,9 +4181,9 @@ Return ONLY a valid JSON object with the exact envelope:`,
4029
4181
  temperature: 0.1,
4030
4182
  jsonMode: true
4031
4183
  });
4032
- const repMatch = repairedRaw.match(/\{[\s\S]*\}/);
4033
- if (repMatch) {
4034
- const repParsed = JSON.parse(repMatch[0]);
4184
+ const repCandidate = extractJsonEnvelope(repairedRaw);
4185
+ if (repCandidate) {
4186
+ const repParsed = JSON.parse(repCandidate);
4035
4187
  const repCheck = contract.outputSchema.safeParse(repParsed);
4036
4188
  if (repCheck.success) {
4037
4189
  synthesizedData = repCheck.data;
@@ -4080,6 +4232,13 @@ ${JSON.stringify(synthesizedData || {})}`;
4080
4232
  markdownReport = markdownReport.replace(regex, "[UNVERIFIED-CITATION-REMOVED]");
4081
4233
  tier1Summary = tier1Summary.replace(regex, "");
4082
4234
  tier2TechnicalDigest = tier2TechnicalDigest.replace(regex, "");
4235
+ if (synthesizedData) {
4236
+ try {
4237
+ const rawJson = JSON.stringify(synthesizedData);
4238
+ synthesizedData = JSON.parse(rawJson.replace(regex, ""));
4239
+ } catch {
4240
+ }
4241
+ }
4083
4242
  }
4084
4243
  }
4085
4244
  if (contract.acceptanceCriteria?.forbiddenTerms?.length) {
@@ -4264,9 +4423,11 @@ exports.ResearchSubAgent = ResearchSubAgent;
4264
4423
  exports.SOURCE_CHANNELS = SOURCE_CHANNELS;
4265
4424
  exports.classifyChannelStatus = classifyChannelStatus;
4266
4425
  exports.classifyModeAndPrior = classifyModeAndPrior;
4426
+ exports.computeCanonicalId = computeCanonicalId;
4267
4427
  exports.computeContractComplexityScore = computeContractComplexityScore;
4268
4428
  exports.createResearchAgentTool = createResearchAgentTool;
4269
4429
  exports.evaluateAttributeCoverageWithDecision = evaluateAttributeCoverageWithDecision;
4430
+ exports.extractJsonEnvelope = extractJsonEnvelope;
4270
4431
  exports.inferFallbackFacetProbabilities = inferFallbackFacetProbabilities;
4271
4432
  exports.isSemanticRankedHit = isSemanticRankedHit;
4272
4433
  exports.rankChannelsByExpectedPrior = rankChannelsByExpectedPrior;