@thanh01.pmt/research-kit 1.4.1 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/academic/index.cjs +13 -12
- package/dist/academic/index.cjs.map +1 -1
- package/dist/academic/index.d.cts +1 -1
- package/dist/academic/index.d.ts +1 -1
- package/dist/academic/index.js +1 -1
- package/dist/agent/index.cjs +269 -108
- package/dist/agent/index.cjs.map +1 -1
- package/dist/agent/index.d.cts +4 -4
- package/dist/agent/index.d.ts +4 -4
- package/dist/agent/index.js +3 -3
- package/dist/{catalogBuilder-D-UiM37y.d.ts → catalogBuilder-BN3VwEsE.d.ts} +15 -2
- package/dist/{catalogBuilder-BBCa9jqI.d.cts → catalogBuilder-mba5BIHD.d.cts} +15 -2
- package/dist/{chunk-UCPWBRZ2.js → chunk-5HSLUAM6.js} +11 -3
- package/dist/chunk-5HSLUAM6.js.map +1 -0
- package/dist/{chunk-4S6UMYBA.js → chunk-6QW5TFDJ.js} +56 -328
- package/dist/chunk-6QW5TFDJ.js.map +1 -0
- package/dist/chunk-BPZLZKVJ.js +1012 -0
- package/dist/chunk-BPZLZKVJ.js.map +1 -0
- package/dist/{chunk-7BGR4E37.js → chunk-LTXXARCA.js} +3 -3
- package/dist/{chunk-7BGR4E37.js.map → chunk-LTXXARCA.js.map} +1 -1
- package/dist/{chunk-6U2H2DMQ.js → chunk-QLBHTCV6.js} +95 -35
- package/dist/chunk-QLBHTCV6.js.map +1 -0
- package/dist/{chunk-SDDZFFL6.js → chunk-SB7WYK7A.js} +15 -14
- package/dist/chunk-SB7WYK7A.js.map +1 -0
- package/dist/harvester/index.cjs +219 -86
- package/dist/harvester/index.cjs.map +1 -1
- package/dist/harvester/index.d.cts +2 -2
- package/dist/harvester/index.d.ts +2 -2
- package/dist/harvester/index.js +3 -3
- package/dist/{index-_aiOh8jX.d.ts → index-9Z-uMwdI.d.ts} +12 -4
- package/dist/{index-BOwJ1jON.d.cts → index-eTcF4XVo.d.cts} +12 -4
- package/dist/index.cjs +2585 -2355
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +12 -12
- package/dist/{literatureReview-Bvlqk4DB.d.cts → literatureReview-BUtPL7bj.d.cts} +1 -0
- package/dist/{literatureReview-Bn2iEGRO.d.ts → literatureReview-CHz-JfSR.d.ts} +1 -0
- package/dist/pipeline/index.cjs +322 -128
- package/dist/pipeline/index.cjs.map +1 -1
- package/dist/pipeline/index.d.cts +15 -3
- package/dist/pipeline/index.d.ts +15 -3
- package/dist/pipeline/index.js +4 -4
- package/dist/{socialPulseHarvester-CN4MwrpY.d.cts → socialPulseHarvester-BCFxGwx2.d.cts} +4 -1
- package/dist/{socialPulseHarvester-IbdT3zD1.d.ts → socialPulseHarvester-DjMvBufT.d.ts} +4 -1
- package/package.json +1 -1
- package/dist/chunk-4S6UMYBA.js.map +0 -1
- package/dist/chunk-6U2H2DMQ.js.map +0 -1
- package/dist/chunk-MJIREPKJ.js +0 -581
- package/dist/chunk-MJIREPKJ.js.map +0 -1
- package/dist/chunk-SDDZFFL6.js.map +0 -1
- package/dist/chunk-UCPWBRZ2.js.map +0 -1
package/dist/harvester/index.cjs
CHANGED
|
@@ -329,6 +329,18 @@ var AcademicSearchEngine = class {
|
|
|
329
329
|
};
|
|
330
330
|
|
|
331
331
|
// src/harvester/multiSourceSearch.ts
|
|
332
|
+
function isUrlAllowedByDomainFilter(url, domainFilter) {
|
|
333
|
+
if (!domainFilter || domainFilter.length === 0) return true;
|
|
334
|
+
try {
|
|
335
|
+
const hostname = new URL(url).hostname.toLowerCase();
|
|
336
|
+
return domainFilter.some((d) => {
|
|
337
|
+
const cleanD = d.toLowerCase().trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "");
|
|
338
|
+
return hostname === cleanD || hostname.endsWith(`.${cleanD}`);
|
|
339
|
+
});
|
|
340
|
+
} catch {
|
|
341
|
+
return false;
|
|
342
|
+
}
|
|
343
|
+
}
|
|
332
344
|
var MultiSourceSearchEngine = class {
|
|
333
345
|
tavilyKey;
|
|
334
346
|
exaKey;
|
|
@@ -340,6 +352,20 @@ var MultiSourceSearchEngine = class {
|
|
|
340
352
|
allowEnvFallback;
|
|
341
353
|
customAdapter;
|
|
342
354
|
academicEngine;
|
|
355
|
+
lastErrors = [];
|
|
356
|
+
getLastErrors() {
|
|
357
|
+
return [...this.lastErrors];
|
|
358
|
+
}
|
|
359
|
+
recordError(engine, error, status, traceErrors) {
|
|
360
|
+
const errObj = {
|
|
361
|
+
engine,
|
|
362
|
+
error,
|
|
363
|
+
status,
|
|
364
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
365
|
+
};
|
|
366
|
+
this.lastErrors.push(errObj);
|
|
367
|
+
traceErrors?.push(errObj);
|
|
368
|
+
}
|
|
343
369
|
constructor(options = {}) {
|
|
344
370
|
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
345
371
|
this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
|
|
@@ -420,97 +446,128 @@ var MultiSourceSearchEngine = class {
|
|
|
420
446
|
const brightDataZone = options.brightDataZone || this.brightDataZone;
|
|
421
447
|
const searxngHost = options.searxngHost || this.searxngHost;
|
|
422
448
|
if (preferred === "searxng" && searxngHost) {
|
|
423
|
-
const searxHits = await this.querySearxng(
|
|
449
|
+
const searxHits = await this.querySearxng(
|
|
450
|
+
query,
|
|
451
|
+
searxngHost,
|
|
452
|
+
limit,
|
|
453
|
+
options.searxngHeaders,
|
|
454
|
+
options.timeRange,
|
|
455
|
+
options.domainFilter
|
|
456
|
+
);
|
|
424
457
|
if (searxHits.length > 0) return searxHits;
|
|
425
458
|
}
|
|
426
459
|
if (preferred === "brightdata" && brightDataKey) {
|
|
427
460
|
const bdHits = await this.queryBrightData(query, brightDataKey, limit, {
|
|
428
461
|
zone: brightDataZone,
|
|
429
|
-
engine: options.brightDataEngine
|
|
462
|
+
engine: options.brightDataEngine,
|
|
463
|
+
domainFilter: options.domainFilter
|
|
430
464
|
});
|
|
431
465
|
if (bdHits.length > 0) return bdHits;
|
|
432
466
|
}
|
|
433
467
|
if (preferred === "exa" && exaKey) {
|
|
434
|
-
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
|
|
468
|
+
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter, options.timeRange);
|
|
435
469
|
if (exaHits.length > 0) return exaHits;
|
|
436
470
|
}
|
|
437
471
|
if (preferred === "serper" && serperKey) {
|
|
438
|
-
const serperHits = await this.querySerper(query, serperKey, limit);
|
|
472
|
+
const serperHits = await this.querySerper(query, serperKey, limit, options.timeRange, options.domainFilter);
|
|
439
473
|
if (serperHits.length > 0) return serperHits;
|
|
440
474
|
}
|
|
475
|
+
if (preferred === "tavily" && tavilyKey) {
|
|
476
|
+
const tavilyHits = await this.queryTavily(query, tavilyKey, limit, options.timeRange, options.domainFilter);
|
|
477
|
+
if (tavilyHits.length > 0) return tavilyHits;
|
|
478
|
+
}
|
|
441
479
|
if (preferred === "duckduckgo") {
|
|
442
|
-
const
|
|
443
|
-
if (
|
|
480
|
+
const ddgHits = await this.queryDuckDuckGo(query, limit, options.domainFilter);
|
|
481
|
+
if (ddgHits.length > 0) return ddgHits;
|
|
444
482
|
}
|
|
445
|
-
if (tavilyKey &&
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
api_key: tavilyKey,
|
|
449
|
-
query,
|
|
450
|
-
search_depth: "advanced",
|
|
451
|
-
include_answer: true,
|
|
452
|
-
max_results: limit
|
|
453
|
-
};
|
|
454
|
-
if (options.timeRange) {
|
|
455
|
-
body.time_range = options.timeRange;
|
|
456
|
-
}
|
|
457
|
-
if (options.domainFilter && options.domainFilter.length > 0) {
|
|
458
|
-
body.include_domains = options.domainFilter;
|
|
459
|
-
}
|
|
460
|
-
const res = await fetch("https://api.tavily.com/search", {
|
|
461
|
-
method: "POST",
|
|
462
|
-
headers: { "Content-Type": "application/json" },
|
|
463
|
-
body: JSON.stringify(body),
|
|
464
|
-
signal: AbortSignal.timeout(12e3)
|
|
465
|
-
});
|
|
466
|
-
if (res.ok) {
|
|
467
|
-
const data = await res.json();
|
|
468
|
-
if (Array.isArray(data?.results) && data.results.length > 0) {
|
|
469
|
-
return data.results.map((r) => ({
|
|
470
|
-
title: r.title || "Untitled",
|
|
471
|
-
url: r.url,
|
|
472
|
-
content: r.content || "",
|
|
473
|
-
score: r.score,
|
|
474
|
-
engine: "tavily"
|
|
475
|
-
}));
|
|
476
|
-
}
|
|
477
|
-
}
|
|
478
|
-
} catch (e) {
|
|
479
|
-
console.warn("[MultiSourceSearchEngine] Tavily attempt failed:", e?.message || e);
|
|
480
|
-
}
|
|
483
|
+
if (tavilyKey && preferred !== "tavily") {
|
|
484
|
+
const tavilyHits = await this.queryTavily(query, tavilyKey, limit, options.timeRange, options.domainFilter);
|
|
485
|
+
if (tavilyHits.length > 0) return tavilyHits;
|
|
481
486
|
}
|
|
482
|
-
if (exaKey) {
|
|
487
|
+
if (exaKey && preferred !== "exa") {
|
|
483
488
|
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter, options.timeRange);
|
|
484
489
|
if (exaHits.length > 0) return exaHits;
|
|
485
490
|
}
|
|
486
|
-
if (serperKey) {
|
|
487
|
-
const serperHits = await this.querySerper(query, serperKey, limit, options.timeRange);
|
|
491
|
+
if (serperKey && preferred !== "serper") {
|
|
492
|
+
const serperHits = await this.querySerper(query, serperKey, limit, options.timeRange, options.domainFilter);
|
|
488
493
|
if (serperHits.length > 0) return serperHits;
|
|
489
494
|
}
|
|
490
|
-
if (brightDataKey) {
|
|
495
|
+
if (brightDataKey && preferred !== "brightdata") {
|
|
491
496
|
const bdHits = await this.queryBrightData(query, brightDataKey, limit, {
|
|
492
497
|
zone: brightDataZone,
|
|
493
|
-
engine: options.brightDataEngine
|
|
498
|
+
engine: options.brightDataEngine,
|
|
499
|
+
domainFilter: options.domainFilter
|
|
494
500
|
});
|
|
495
501
|
if (bdHits.length > 0) return bdHits;
|
|
496
502
|
}
|
|
497
|
-
if (searxngHost) {
|
|
498
|
-
const searxHits = await this.querySearxng(
|
|
503
|
+
if (searxngHost && preferred !== "searxng") {
|
|
504
|
+
const searxHits = await this.querySearxng(
|
|
505
|
+
query,
|
|
506
|
+
searxngHost,
|
|
507
|
+
limit,
|
|
508
|
+
options.searxngHeaders,
|
|
509
|
+
options.timeRange,
|
|
510
|
+
options.domainFilter
|
|
511
|
+
);
|
|
499
512
|
if (searxHits.length > 0) return searxHits;
|
|
500
513
|
}
|
|
501
|
-
|
|
502
|
-
|
|
514
|
+
if (preferred !== "duckduckgo") {
|
|
515
|
+
const ddgHits = await this.queryDuckDuckGo(query, limit, options.domainFilter);
|
|
516
|
+
if (ddgHits.length > 0) return ddgHits;
|
|
517
|
+
}
|
|
518
|
+
return [];
|
|
519
|
+
}
|
|
520
|
+
async queryTavily(query, tavilyKey, limit, timeRange, domainFilter) {
|
|
521
|
+
try {
|
|
522
|
+
const body = {
|
|
523
|
+
api_key: tavilyKey,
|
|
524
|
+
query,
|
|
525
|
+
search_depth: "advanced",
|
|
526
|
+
include_answer: true,
|
|
527
|
+
max_results: limit
|
|
528
|
+
};
|
|
529
|
+
if (timeRange) {
|
|
530
|
+
body.time_range = timeRange;
|
|
531
|
+
}
|
|
532
|
+
if (domainFilter && domainFilter.length > 0) {
|
|
533
|
+
body.include_domains = domainFilter;
|
|
534
|
+
}
|
|
535
|
+
const res = await fetch("https://api.tavily.com/search", {
|
|
536
|
+
method: "POST",
|
|
537
|
+
headers: { "Content-Type": "application/json" },
|
|
538
|
+
body: JSON.stringify(body),
|
|
539
|
+
signal: AbortSignal.timeout(12e3)
|
|
540
|
+
});
|
|
541
|
+
if (res.ok) {
|
|
542
|
+
const data = await res.json();
|
|
543
|
+
if (Array.isArray(data?.results) && data.results.length > 0) {
|
|
544
|
+
return data.results.map((r) => ({
|
|
545
|
+
title: r.title || "Untitled",
|
|
546
|
+
url: r.url,
|
|
547
|
+
content: r.content || "",
|
|
548
|
+
score: r.score,
|
|
549
|
+
engine: "tavily"
|
|
550
|
+
}));
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
} catch (e) {
|
|
554
|
+
console.warn("[MultiSourceSearchEngine] Tavily attempt failed:", e?.message || e);
|
|
555
|
+
}
|
|
503
556
|
return [];
|
|
504
557
|
}
|
|
505
558
|
async queryBrightData(query, key, limit = 5, options = {}) {
|
|
506
559
|
try {
|
|
507
560
|
const zone = options.zone || this.brightDataZone || "cli_unlocker";
|
|
508
561
|
const engine = options.engine || "bing";
|
|
509
|
-
let
|
|
562
|
+
let effectiveQuery = query;
|
|
563
|
+
if (options.domainFilter && options.domainFilter.length === 1) {
|
|
564
|
+
effectiveQuery = `${query} site:${options.domainFilter[0]}`;
|
|
565
|
+
}
|
|
566
|
+
let searchUrl = `https://www.bing.com/search?q=${encodeURIComponent(effectiveQuery)}&brd_json=1`;
|
|
510
567
|
if (engine === "google") {
|
|
511
|
-
searchUrl = `https://www.google.com/search?q=${encodeURIComponent(
|
|
568
|
+
searchUrl = `https://www.google.com/search?q=${encodeURIComponent(effectiveQuery)}&brd_json=1&hl=en&gl=us`;
|
|
512
569
|
} else if (engine === "yandex") {
|
|
513
|
-
searchUrl = `https://www.yandex.com/search?q=${encodeURIComponent(
|
|
570
|
+
searchUrl = `https://www.yandex.com/search?q=${encodeURIComponent(effectiveQuery)}&brd_json=1`;
|
|
514
571
|
}
|
|
515
572
|
const res = await fetch("https://api.brightdata.com/request", {
|
|
516
573
|
method: "POST",
|
|
@@ -526,27 +583,48 @@ var MultiSourceSearchEngine = class {
|
|
|
526
583
|
signal: AbortSignal.timeout(15e3)
|
|
527
584
|
});
|
|
528
585
|
if (res.ok) {
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
586
|
+
let data;
|
|
587
|
+
try {
|
|
588
|
+
if (typeof res.json === "function") {
|
|
589
|
+
data = await res.json();
|
|
590
|
+
} else if (typeof res.text === "function") {
|
|
591
|
+
const rawText = await res.text();
|
|
592
|
+
data = JSON.parse(rawText);
|
|
593
|
+
}
|
|
594
|
+
} catch {
|
|
595
|
+
data = null;
|
|
596
|
+
}
|
|
597
|
+
const hits = [];
|
|
598
|
+
if (data && Array.isArray(data?.organic)) {
|
|
599
|
+
for (let idx = 0; idx < data.organic.length; idx++) {
|
|
600
|
+
const item = data.organic[idx];
|
|
601
|
+
const hitUrl = item.link || item.url;
|
|
602
|
+
if (!hitUrl || isUrlAllowedByDomainFilter(hitUrl, options.domainFilter)) {
|
|
603
|
+
hits.push({
|
|
604
|
+
title: item.title || "Untitled",
|
|
605
|
+
url: hitUrl,
|
|
606
|
+
content: item.description || item.snippet || "",
|
|
607
|
+
score: item.rank ? 1 / item.rank : 1 / (idx + 1),
|
|
608
|
+
engine: "brightdata"
|
|
609
|
+
});
|
|
610
|
+
}
|
|
611
|
+
}
|
|
539
612
|
}
|
|
613
|
+
if (hits.length > 0) return hits.slice(0, limit);
|
|
540
614
|
}
|
|
541
615
|
} catch (e) {
|
|
542
616
|
console.warn("[MultiSourceSearchEngine] Bright Data attempt failed:", e?.message || e);
|
|
543
617
|
}
|
|
544
618
|
return [];
|
|
545
619
|
}
|
|
546
|
-
async querySearxng(query, searxngHost, limit, customHeaders, timeRange) {
|
|
620
|
+
async querySearxng(query, searxngHost, limit, customHeaders, timeRange, domainFilter) {
|
|
547
621
|
try {
|
|
548
622
|
const cleanHost = searxngHost.replace(/\/$/, "");
|
|
549
|
-
let
|
|
623
|
+
let effectiveQuery = query;
|
|
624
|
+
if (domainFilter && domainFilter.length === 1) {
|
|
625
|
+
effectiveQuery = `${query} site:${domainFilter[0]}`;
|
|
626
|
+
}
|
|
627
|
+
let searchUrl = `${cleanHost}/search?q=${encodeURIComponent(effectiveQuery)}&format=json`;
|
|
550
628
|
if (timeRange) {
|
|
551
629
|
searchUrl += `&time_range=${encodeURIComponent(timeRange)}`;
|
|
552
630
|
}
|
|
@@ -562,7 +640,7 @@ var MultiSourceSearchEngine = class {
|
|
|
562
640
|
if (res.ok) {
|
|
563
641
|
const data = await res.json();
|
|
564
642
|
if (Array.isArray(data?.results) && data.results.length > 0) {
|
|
565
|
-
return data.results.slice(0, limit).map((r) => ({
|
|
643
|
+
return data.results.filter((r) => !r.url || isUrlAllowedByDomainFilter(r.url, domainFilter)).slice(0, limit).map((r) => ({
|
|
566
644
|
title: r.title || "Untitled",
|
|
567
645
|
url: r.url,
|
|
568
646
|
content: r.content || r.snippet || "",
|
|
@@ -620,9 +698,13 @@ var MultiSourceSearchEngine = class {
|
|
|
620
698
|
}
|
|
621
699
|
return [];
|
|
622
700
|
}
|
|
623
|
-
async querySerper(query, serperKey, limit, timeRange) {
|
|
701
|
+
async querySerper(query, serperKey, limit, timeRange, domainFilter) {
|
|
624
702
|
try {
|
|
625
|
-
|
|
703
|
+
let effectiveQuery = query;
|
|
704
|
+
if (domainFilter && domainFilter.length === 1) {
|
|
705
|
+
effectiveQuery = `${query} site:${domainFilter[0]}`;
|
|
706
|
+
}
|
|
707
|
+
const body = { q: effectiveQuery, num: limit };
|
|
626
708
|
if (timeRange) {
|
|
627
709
|
const tbsMap = { day: "qdr:d", week: "qdr:w", month: "qdr:m", year: "qdr:y" };
|
|
628
710
|
if (tbsMap[timeRange]) {
|
|
@@ -641,7 +723,7 @@ var MultiSourceSearchEngine = class {
|
|
|
641
723
|
if (res.ok) {
|
|
642
724
|
const data = await res.json();
|
|
643
725
|
if (Array.isArray(data?.organic) && data.organic.length > 0) {
|
|
644
|
-
return data.organic.slice(0, limit).map((r) => ({
|
|
726
|
+
return data.organic.filter((r) => !r.link || isUrlAllowedByDomainFilter(r.link, domainFilter)).slice(0, limit).map((r) => ({
|
|
645
727
|
title: r.title || "Untitled",
|
|
646
728
|
url: r.link,
|
|
647
729
|
content: r.snippet || "",
|
|
@@ -654,9 +736,13 @@ var MultiSourceSearchEngine = class {
|
|
|
654
736
|
}
|
|
655
737
|
return [];
|
|
656
738
|
}
|
|
657
|
-
async queryDuckDuckGo(query, limit) {
|
|
739
|
+
async queryDuckDuckGo(query, limit, domainFilter) {
|
|
658
740
|
try {
|
|
659
|
-
|
|
741
|
+
let effectiveQuery = query;
|
|
742
|
+
if (domainFilter && domainFilter.length === 1) {
|
|
743
|
+
effectiveQuery = `${query} site:${domainFilter[0]}`;
|
|
744
|
+
}
|
|
745
|
+
const ddgUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(effectiveQuery)}`;
|
|
660
746
|
const res = await fetch(ddgUrl, {
|
|
661
747
|
headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" },
|
|
662
748
|
signal: AbortSignal.timeout(8e3)
|
|
@@ -664,20 +750,45 @@ var MultiSourceSearchEngine = class {
|
|
|
664
750
|
if (res.ok) {
|
|
665
751
|
const html = await res.text();
|
|
666
752
|
const hits = [];
|
|
667
|
-
const
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
const
|
|
671
|
-
|
|
672
|
-
const
|
|
673
|
-
const
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
753
|
+
const resultBlocks = html.split(/class="[^"]*result__body/i);
|
|
754
|
+
for (const block of resultBlocks.slice(1)) {
|
|
755
|
+
if (hits.length >= limit) break;
|
|
756
|
+
const titleMatch = block.match(/<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
|
757
|
+
if (!titleMatch) continue;
|
|
758
|
+
const rawHref = titleMatch[1]?.trim() || "";
|
|
759
|
+
const cleanUrl = rawHref.startsWith("//duckduckgo.com/l/?uddg=") || rawHref.includes("uddg=") ? decodeURIComponent(rawHref.split("uddg=")[1]?.split("&")[0] || "") : rawHref;
|
|
760
|
+
const rawTitle = titleMatch[2]?.replace(/<[^>]+>/g, "").trim() || query;
|
|
761
|
+
const snippetMatch = block.match(/<(?:a|div)[^>]*class="[^"]*result__snippet[^"]*"[^>]*>([\s\S]*?)<\/(?:a|div)>/i);
|
|
762
|
+
const snippet = snippetMatch ? snippetMatch[1]?.replace(/<[^>]+>/g, "").trim() || "" : "";
|
|
763
|
+
if (cleanUrl && !cleanUrl.includes("duckduckgo.com") && cleanUrl.startsWith("http")) {
|
|
764
|
+
if (isUrlAllowedByDomainFilter(cleanUrl, domainFilter)) {
|
|
765
|
+
hits.push({
|
|
766
|
+
title: rawTitle,
|
|
767
|
+
url: cleanUrl,
|
|
768
|
+
content: snippet,
|
|
769
|
+
engine: "duckduckgo"
|
|
770
|
+
});
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
if (hits.length === 0) {
|
|
775
|
+
const resultRegex = /<a[^>]*class="result__a"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
|
|
776
|
+
let match;
|
|
777
|
+
while ((match = resultRegex.exec(html)) !== null && hits.length < limit) {
|
|
778
|
+
const rawTitle = match[1]?.replace(/<[^>]+>/g, "").trim() || query;
|
|
779
|
+
const rawUrl = match[2]?.trim();
|
|
780
|
+
const cleanUrl = rawUrl?.startsWith("//duckduckgo.com/l/?uddg=") ? decodeURIComponent(rawUrl.split("uddg=")[1]?.split("&")[0] || "") : rawUrl;
|
|
781
|
+
const snippet = match[3]?.replace(/<[^>]+>/g, "").trim() || "";
|
|
782
|
+
if (cleanUrl && !cleanUrl.includes("duckduckgo.com")) {
|
|
783
|
+
if (isUrlAllowedByDomainFilter(cleanUrl, domainFilter)) {
|
|
784
|
+
hits.push({
|
|
785
|
+
title: rawTitle,
|
|
786
|
+
url: cleanUrl,
|
|
787
|
+
content: snippet,
|
|
788
|
+
engine: "duckduckgo"
|
|
789
|
+
});
|
|
790
|
+
}
|
|
791
|
+
}
|
|
681
792
|
}
|
|
682
793
|
}
|
|
683
794
|
if (hits.length > 0) return hits;
|
|
@@ -1653,12 +1764,19 @@ Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered fr
|
|
|
1653
1764
|
concepts: sortedConcepts.length > 0 ? sortedConcepts : concepts,
|
|
1654
1765
|
conceptDagTsv: this.formatConceptDagTsv(sortedConcepts.length > 0 ? sortedConcepts : concepts),
|
|
1655
1766
|
isLlmRefined: true,
|
|
1767
|
+
status: "success",
|
|
1656
1768
|
unrefinedCandidatesCount: detBundle.unrefinedCandidatesCount
|
|
1657
1769
|
};
|
|
1658
1770
|
}
|
|
1659
1771
|
}
|
|
1660
1772
|
} catch (e) {
|
|
1661
1773
|
console.warn("[PedagogicalTermExtractor] Host LLM term refinement failed, using deterministic extraction:", e?.message || e);
|
|
1774
|
+
detBundle.status = "partial";
|
|
1775
|
+
detBundle.errors = detBundle.errors || [];
|
|
1776
|
+
detBundle.errors.push(`Host LLM refinement failed: ${e?.message || String(e)}`);
|
|
1777
|
+
}
|
|
1778
|
+
if (!detBundle.status) {
|
|
1779
|
+
detBundle.status = detBundle.concepts.length > 0 ? "partial" : "failed";
|
|
1662
1780
|
}
|
|
1663
1781
|
return detBundle;
|
|
1664
1782
|
}
|
|
@@ -1682,12 +1800,27 @@ Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered fr
|
|
|
1682
1800
|
}
|
|
1683
1801
|
};
|
|
1684
1802
|
|
|
1803
|
+
// src/agent/evidenceLedger.ts
|
|
1804
|
+
function computeCanonicalId(url) {
|
|
1805
|
+
try {
|
|
1806
|
+
let hash = 0;
|
|
1807
|
+
const clean = url.trim().replace(/^https?:\/\//i, "").replace(/[\/?#].*$/, "").toLowerCase();
|
|
1808
|
+
for (let i = 0; i < clean.length; i++) {
|
|
1809
|
+
hash = hash * 31 + clean.charCodeAt(i) >>> 0;
|
|
1810
|
+
}
|
|
1811
|
+
return `SRC-${hash.toString(16).padStart(8, "0")}`;
|
|
1812
|
+
} catch {
|
|
1813
|
+
return `SRC-${Math.random().toString(36).slice(2, 10)}`;
|
|
1814
|
+
}
|
|
1815
|
+
}
|
|
1816
|
+
|
|
1685
1817
|
// src/harvester/catalogBuilder.ts
|
|
1686
1818
|
var CatalogBuilder = class {
|
|
1687
1819
|
buildBundle(topic, domain, items, options = {}) {
|
|
1688
1820
|
const defaultLang = inferPrimaryCodeLanguage(topic, options.context, domain);
|
|
1689
1821
|
const indexList = items.map((item, idx) => ({
|
|
1690
1822
|
id: `RES-${String(idx + 1).padStart(2, "0")}`,
|
|
1823
|
+
canonicalId: computeCanonicalId(item.doc.url),
|
|
1691
1824
|
title: item.doc.title,
|
|
1692
1825
|
url: item.doc.url,
|
|
1693
1826
|
domain,
|