@thanh01.pmt/research-kit 1.4.1 → 1.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/dist/academic/index.cjs +13 -12
  2. package/dist/academic/index.cjs.map +1 -1
  3. package/dist/academic/index.d.cts +1 -1
  4. package/dist/academic/index.d.ts +1 -1
  5. package/dist/academic/index.js +1 -1
  6. package/dist/agent/index.cjs +269 -108
  7. package/dist/agent/index.cjs.map +1 -1
  8. package/dist/agent/index.d.cts +4 -4
  9. package/dist/agent/index.d.ts +4 -4
  10. package/dist/agent/index.js +3 -3
  11. package/dist/{catalogBuilder-D-UiM37y.d.ts → catalogBuilder-BN3VwEsE.d.ts} +15 -2
  12. package/dist/{catalogBuilder-BBCa9jqI.d.cts → catalogBuilder-mba5BIHD.d.cts} +15 -2
  13. package/dist/{chunk-UCPWBRZ2.js → chunk-5HSLUAM6.js} +11 -3
  14. package/dist/chunk-5HSLUAM6.js.map +1 -0
  15. package/dist/{chunk-4S6UMYBA.js → chunk-6QW5TFDJ.js} +56 -328
  16. package/dist/chunk-6QW5TFDJ.js.map +1 -0
  17. package/dist/chunk-BPZLZKVJ.js +1012 -0
  18. package/dist/chunk-BPZLZKVJ.js.map +1 -0
  19. package/dist/{chunk-7BGR4E37.js → chunk-LTXXARCA.js} +3 -3
  20. package/dist/{chunk-7BGR4E37.js.map → chunk-LTXXARCA.js.map} +1 -1
  21. package/dist/{chunk-6U2H2DMQ.js → chunk-QLBHTCV6.js} +95 -35
  22. package/dist/chunk-QLBHTCV6.js.map +1 -0
  23. package/dist/{chunk-SDDZFFL6.js → chunk-SB7WYK7A.js} +15 -14
  24. package/dist/chunk-SB7WYK7A.js.map +1 -0
  25. package/dist/harvester/index.cjs +219 -86
  26. package/dist/harvester/index.cjs.map +1 -1
  27. package/dist/harvester/index.d.cts +2 -2
  28. package/dist/harvester/index.d.ts +2 -2
  29. package/dist/harvester/index.js +3 -3
  30. package/dist/{index-_aiOh8jX.d.ts → index-9Z-uMwdI.d.ts} +12 -4
  31. package/dist/{index-BOwJ1jON.d.cts → index-eTcF4XVo.d.cts} +12 -4
  32. package/dist/index.cjs +2585 -2355
  33. package/dist/index.cjs.map +1 -1
  34. package/dist/index.d.cts +5 -5
  35. package/dist/index.d.ts +5 -5
  36. package/dist/index.js +12 -12
  37. package/dist/{literatureReview-Bvlqk4DB.d.cts → literatureReview-BUtPL7bj.d.cts} +1 -0
  38. package/dist/{literatureReview-Bn2iEGRO.d.ts → literatureReview-CHz-JfSR.d.ts} +1 -0
  39. package/dist/pipeline/index.cjs +322 -128
  40. package/dist/pipeline/index.cjs.map +1 -1
  41. package/dist/pipeline/index.d.cts +15 -3
  42. package/dist/pipeline/index.d.ts +15 -3
  43. package/dist/pipeline/index.js +4 -4
  44. package/dist/{socialPulseHarvester-CN4MwrpY.d.cts → socialPulseHarvester-BCFxGwx2.d.cts} +4 -1
  45. package/dist/{socialPulseHarvester-IbdT3zD1.d.ts → socialPulseHarvester-DjMvBufT.d.ts} +4 -1
  46. package/package.json +1 -1
  47. package/dist/chunk-4S6UMYBA.js.map +0 -1
  48. package/dist/chunk-6U2H2DMQ.js.map +0 -1
  49. package/dist/chunk-MJIREPKJ.js +0 -581
  50. package/dist/chunk-MJIREPKJ.js.map +0 -1
  51. package/dist/chunk-SDDZFFL6.js.map +0 -1
  52. package/dist/chunk-UCPWBRZ2.js.map +0 -1
@@ -550,6 +550,18 @@ var AcademicSearchEngine = class {
550
550
  };
551
551
 
552
552
  // src/harvester/multiSourceSearch.ts
553
+ function isUrlAllowedByDomainFilter(url, domainFilter) {
554
+ if (!domainFilter || domainFilter.length === 0) return true;
555
+ try {
556
+ const hostname = new URL(url).hostname.toLowerCase();
557
+ return domainFilter.some((d) => {
558
+ const cleanD = d.toLowerCase().trim().replace(/^https?:\/\//, "").replace(/\/.*$/, "");
559
+ return hostname === cleanD || hostname.endsWith(`.${cleanD}`);
560
+ });
561
+ } catch {
562
+ return false;
563
+ }
564
+ }
553
565
  var MultiSourceSearchEngine = class {
554
566
  tavilyKey;
555
567
  exaKey;
@@ -561,6 +573,20 @@ var MultiSourceSearchEngine = class {
561
573
  allowEnvFallback;
562
574
  customAdapter;
563
575
  academicEngine;
576
+ lastErrors = [];
577
+ getLastErrors() {
578
+ return [...this.lastErrors];
579
+ }
580
+ recordError(engine, error, status, traceErrors) {
581
+ const errObj = {
582
+ engine,
583
+ error,
584
+ status,
585
+ timestamp: (/* @__PURE__ */ new Date()).toISOString()
586
+ };
587
+ this.lastErrors.push(errObj);
588
+ traceErrors?.push(errObj);
589
+ }
564
590
  constructor(options = {}) {
565
591
  this.allowEnvFallback = options.allowEnvFallback ?? true;
566
592
  this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
@@ -641,97 +667,128 @@ var MultiSourceSearchEngine = class {
641
667
  const brightDataZone = options.brightDataZone || this.brightDataZone;
642
668
  const searxngHost = options.searxngHost || this.searxngHost;
643
669
  if (preferred === "searxng" && searxngHost) {
644
- const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders);
670
+ const searxHits = await this.querySearxng(
671
+ query,
672
+ searxngHost,
673
+ limit,
674
+ options.searxngHeaders,
675
+ options.timeRange,
676
+ options.domainFilter
677
+ );
645
678
  if (searxHits.length > 0) return searxHits;
646
679
  }
647
680
  if (preferred === "brightdata" && brightDataKey) {
648
681
  const bdHits = await this.queryBrightData(query, brightDataKey, limit, {
649
682
  zone: brightDataZone,
650
- engine: options.brightDataEngine
683
+ engine: options.brightDataEngine,
684
+ domainFilter: options.domainFilter
651
685
  });
652
686
  if (bdHits.length > 0) return bdHits;
653
687
  }
654
688
  if (preferred === "exa" && exaKey) {
655
- const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
689
+ const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter, options.timeRange);
656
690
  if (exaHits.length > 0) return exaHits;
657
691
  }
658
692
  if (preferred === "serper" && serperKey) {
659
- const serperHits = await this.querySerper(query, serperKey, limit);
693
+ const serperHits = await this.querySerper(query, serperKey, limit, options.timeRange, options.domainFilter);
660
694
  if (serperHits.length > 0) return serperHits;
661
695
  }
696
+ if (preferred === "tavily" && tavilyKey) {
697
+ const tavilyHits = await this.queryTavily(query, tavilyKey, limit, options.timeRange, options.domainFilter);
698
+ if (tavilyHits.length > 0) return tavilyHits;
699
+ }
662
700
  if (preferred === "duckduckgo") {
663
- const ddgHits2 = await this.queryDuckDuckGo(query, limit);
664
- if (ddgHits2.length > 0) return ddgHits2;
701
+ const ddgHits = await this.queryDuckDuckGo(query, limit, options.domainFilter);
702
+ if (ddgHits.length > 0) return ddgHits;
665
703
  }
666
- if (tavilyKey && (!preferred || preferred === "tavily")) {
667
- try {
668
- const body = {
669
- api_key: tavilyKey,
670
- query,
671
- search_depth: "advanced",
672
- include_answer: true,
673
- max_results: limit
674
- };
675
- if (options.timeRange) {
676
- body.time_range = options.timeRange;
677
- }
678
- if (options.domainFilter && options.domainFilter.length > 0) {
679
- body.include_domains = options.domainFilter;
680
- }
681
- const res = await fetch("https://api.tavily.com/search", {
682
- method: "POST",
683
- headers: { "Content-Type": "application/json" },
684
- body: JSON.stringify(body),
685
- signal: AbortSignal.timeout(12e3)
686
- });
687
- if (res.ok) {
688
- const data = await res.json();
689
- if (Array.isArray(data?.results) && data.results.length > 0) {
690
- return data.results.map((r) => ({
691
- title: r.title || "Untitled",
692
- url: r.url,
693
- content: r.content || "",
694
- score: r.score,
695
- engine: "tavily"
696
- }));
697
- }
698
- }
699
- } catch (e) {
700
- console.warn("[MultiSourceSearchEngine] Tavily attempt failed:", e?.message || e);
701
- }
704
+ if (tavilyKey && preferred !== "tavily") {
705
+ const tavilyHits = await this.queryTavily(query, tavilyKey, limit, options.timeRange, options.domainFilter);
706
+ if (tavilyHits.length > 0) return tavilyHits;
702
707
  }
703
- if (exaKey) {
708
+ if (exaKey && preferred !== "exa") {
704
709
  const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter, options.timeRange);
705
710
  if (exaHits.length > 0) return exaHits;
706
711
  }
707
- if (serperKey) {
708
- const serperHits = await this.querySerper(query, serperKey, limit, options.timeRange);
712
+ if (serperKey && preferred !== "serper") {
713
+ const serperHits = await this.querySerper(query, serperKey, limit, options.timeRange, options.domainFilter);
709
714
  if (serperHits.length > 0) return serperHits;
710
715
  }
711
- if (brightDataKey) {
716
+ if (brightDataKey && preferred !== "brightdata") {
712
717
  const bdHits = await this.queryBrightData(query, brightDataKey, limit, {
713
718
  zone: brightDataZone,
714
- engine: options.brightDataEngine
719
+ engine: options.brightDataEngine,
720
+ domainFilter: options.domainFilter
715
721
  });
716
722
  if (bdHits.length > 0) return bdHits;
717
723
  }
718
- if (searxngHost) {
719
- const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders, options.timeRange);
724
+ if (searxngHost && preferred !== "searxng") {
725
+ const searxHits = await this.querySearxng(
726
+ query,
727
+ searxngHost,
728
+ limit,
729
+ options.searxngHeaders,
730
+ options.timeRange,
731
+ options.domainFilter
732
+ );
720
733
  if (searxHits.length > 0) return searxHits;
721
734
  }
722
- const ddgHits = await this.queryDuckDuckGo(query, limit);
723
- if (ddgHits.length > 0) return ddgHits;
735
+ if (preferred !== "duckduckgo") {
736
+ const ddgHits = await this.queryDuckDuckGo(query, limit, options.domainFilter);
737
+ if (ddgHits.length > 0) return ddgHits;
738
+ }
739
+ return [];
740
+ }
741
+ async queryTavily(query, tavilyKey, limit, timeRange, domainFilter) {
742
+ try {
743
+ const body = {
744
+ api_key: tavilyKey,
745
+ query,
746
+ search_depth: "advanced",
747
+ include_answer: true,
748
+ max_results: limit
749
+ };
750
+ if (timeRange) {
751
+ body.time_range = timeRange;
752
+ }
753
+ if (domainFilter && domainFilter.length > 0) {
754
+ body.include_domains = domainFilter;
755
+ }
756
+ const res = await fetch("https://api.tavily.com/search", {
757
+ method: "POST",
758
+ headers: { "Content-Type": "application/json" },
759
+ body: JSON.stringify(body),
760
+ signal: AbortSignal.timeout(12e3)
761
+ });
762
+ if (res.ok) {
763
+ const data = await res.json();
764
+ if (Array.isArray(data?.results) && data.results.length > 0) {
765
+ return data.results.map((r) => ({
766
+ title: r.title || "Untitled",
767
+ url: r.url,
768
+ content: r.content || "",
769
+ score: r.score,
770
+ engine: "tavily"
771
+ }));
772
+ }
773
+ }
774
+ } catch (e) {
775
+ console.warn("[MultiSourceSearchEngine] Tavily attempt failed:", e?.message || e);
776
+ }
724
777
  return [];
725
778
  }
726
779
  async queryBrightData(query, key, limit = 5, options = {}) {
727
780
  try {
728
781
  const zone = options.zone || this.brightDataZone || "cli_unlocker";
729
782
  const engine = options.engine || "bing";
730
- let searchUrl = `https://www.bing.com/search?q=${encodeURIComponent(query)}&brd_json=1`;
783
+ let effectiveQuery = query;
784
+ if (options.domainFilter && options.domainFilter.length === 1) {
785
+ effectiveQuery = `${query} site:${options.domainFilter[0]}`;
786
+ }
787
+ let searchUrl = `https://www.bing.com/search?q=${encodeURIComponent(effectiveQuery)}&brd_json=1`;
731
788
  if (engine === "google") {
732
- searchUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}&brd_json=1&hl=en&gl=us`;
789
+ searchUrl = `https://www.google.com/search?q=${encodeURIComponent(effectiveQuery)}&brd_json=1&hl=en&gl=us`;
733
790
  } else if (engine === "yandex") {
734
- searchUrl = `https://www.yandex.com/search?q=${encodeURIComponent(query)}&brd_json=1`;
791
+ searchUrl = `https://www.yandex.com/search?q=${encodeURIComponent(effectiveQuery)}&brd_json=1`;
735
792
  }
736
793
  const res = await fetch("https://api.brightdata.com/request", {
737
794
  method: "POST",
@@ -747,27 +804,48 @@ var MultiSourceSearchEngine = class {
747
804
  signal: AbortSignal.timeout(15e3)
748
805
  });
749
806
  if (res.ok) {
750
- const data = await res.json();
751
- const organic = Array.isArray(data?.organic) ? data.organic : [];
752
- if (organic.length > 0) {
753
- return organic.slice(0, limit).map((r, idx) => ({
754
- title: r.title || "Untitled",
755
- url: r.link,
756
- content: r.description || r.snippet || "",
757
- score: r.rank ? 1 / r.rank : 1 / (idx + 1),
758
- engine: "brightdata"
759
- }));
807
+ let data;
808
+ try {
809
+ if (typeof res.json === "function") {
810
+ data = await res.json();
811
+ } else if (typeof res.text === "function") {
812
+ const rawText = await res.text();
813
+ data = JSON.parse(rawText);
814
+ }
815
+ } catch {
816
+ data = null;
760
817
  }
818
+ const hits = [];
819
+ if (data && Array.isArray(data?.organic)) {
820
+ for (let idx = 0; idx < data.organic.length; idx++) {
821
+ const item = data.organic[idx];
822
+ const hitUrl = item.link || item.url;
823
+ if (!hitUrl || isUrlAllowedByDomainFilter(hitUrl, options.domainFilter)) {
824
+ hits.push({
825
+ title: item.title || "Untitled",
826
+ url: hitUrl,
827
+ content: item.description || item.snippet || "",
828
+ score: item.rank ? 1 / item.rank : 1 / (idx + 1),
829
+ engine: "brightdata"
830
+ });
831
+ }
832
+ }
833
+ }
834
+ if (hits.length > 0) return hits.slice(0, limit);
761
835
  }
762
836
  } catch (e) {
763
837
  console.warn("[MultiSourceSearchEngine] Bright Data attempt failed:", e?.message || e);
764
838
  }
765
839
  return [];
766
840
  }
767
- async querySearxng(query, searxngHost, limit, customHeaders, timeRange) {
841
+ async querySearxng(query, searxngHost, limit, customHeaders, timeRange, domainFilter) {
768
842
  try {
769
843
  const cleanHost = searxngHost.replace(/\/$/, "");
770
- let searchUrl = `${cleanHost}/search?q=${encodeURIComponent(query)}&format=json`;
844
+ let effectiveQuery = query;
845
+ if (domainFilter && domainFilter.length === 1) {
846
+ effectiveQuery = `${query} site:${domainFilter[0]}`;
847
+ }
848
+ let searchUrl = `${cleanHost}/search?q=${encodeURIComponent(effectiveQuery)}&format=json`;
771
849
  if (timeRange) {
772
850
  searchUrl += `&time_range=${encodeURIComponent(timeRange)}`;
773
851
  }
@@ -783,7 +861,7 @@ var MultiSourceSearchEngine = class {
783
861
  if (res.ok) {
784
862
  const data = await res.json();
785
863
  if (Array.isArray(data?.results) && data.results.length > 0) {
786
- return data.results.slice(0, limit).map((r) => ({
864
+ return data.results.filter((r) => !r.url || isUrlAllowedByDomainFilter(r.url, domainFilter)).slice(0, limit).map((r) => ({
787
865
  title: r.title || "Untitled",
788
866
  url: r.url,
789
867
  content: r.content || r.snippet || "",
@@ -841,9 +919,13 @@ var MultiSourceSearchEngine = class {
841
919
  }
842
920
  return [];
843
921
  }
844
- async querySerper(query, serperKey, limit, timeRange) {
922
+ async querySerper(query, serperKey, limit, timeRange, domainFilter) {
845
923
  try {
846
- const body = { q: query, num: limit };
924
+ let effectiveQuery = query;
925
+ if (domainFilter && domainFilter.length === 1) {
926
+ effectiveQuery = `${query} site:${domainFilter[0]}`;
927
+ }
928
+ const body = { q: effectiveQuery, num: limit };
847
929
  if (timeRange) {
848
930
  const tbsMap = { day: "qdr:d", week: "qdr:w", month: "qdr:m", year: "qdr:y" };
849
931
  if (tbsMap[timeRange]) {
@@ -862,7 +944,7 @@ var MultiSourceSearchEngine = class {
862
944
  if (res.ok) {
863
945
  const data = await res.json();
864
946
  if (Array.isArray(data?.organic) && data.organic.length > 0) {
865
- return data.organic.slice(0, limit).map((r) => ({
947
+ return data.organic.filter((r) => !r.link || isUrlAllowedByDomainFilter(r.link, domainFilter)).slice(0, limit).map((r) => ({
866
948
  title: r.title || "Untitled",
867
949
  url: r.link,
868
950
  content: r.snippet || "",
@@ -875,9 +957,13 @@ var MultiSourceSearchEngine = class {
875
957
  }
876
958
  return [];
877
959
  }
878
- async queryDuckDuckGo(query, limit) {
960
+ async queryDuckDuckGo(query, limit, domainFilter) {
879
961
  try {
880
- const ddgUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
962
+ let effectiveQuery = query;
963
+ if (domainFilter && domainFilter.length === 1) {
964
+ effectiveQuery = `${query} site:${domainFilter[0]}`;
965
+ }
966
+ const ddgUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(effectiveQuery)}`;
881
967
  const res = await fetch(ddgUrl, {
882
968
  headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" },
883
969
  signal: AbortSignal.timeout(8e3)
@@ -885,20 +971,45 @@ var MultiSourceSearchEngine = class {
885
971
  if (res.ok) {
886
972
  const html = await res.text();
887
973
  const hits = [];
888
- const resultRegex = /<a[^>]*class="result__a"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
889
- let match;
890
- while ((match = resultRegex.exec(html)) !== null && hits.length < limit) {
891
- const rawTitle = match[1]?.replace(/<[^>]+>/g, "").trim() || query;
892
- const rawUrl = match[2]?.trim();
893
- const cleanUrl = rawUrl?.startsWith("//duckduckgo.com/l/?uddg=") ? decodeURIComponent(rawUrl.split("uddg=")[1]?.split("&")[0] || "") : rawUrl;
894
- const snippet = match[3]?.replace(/<[^>]+>/g, "").trim() || "";
895
- if (cleanUrl && !cleanUrl.includes("duckduckgo.com")) {
896
- hits.push({
897
- title: rawTitle,
898
- url: cleanUrl,
899
- content: snippet,
900
- engine: "duckduckgo"
901
- });
974
+ const resultBlocks = html.split(/class="[^"]*result__body/i);
975
+ for (const block of resultBlocks.slice(1)) {
976
+ if (hits.length >= limit) break;
977
+ const titleMatch = block.match(/<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
978
+ if (!titleMatch) continue;
979
+ const rawHref = titleMatch[1]?.trim() || "";
980
+ const cleanUrl = rawHref.startsWith("//duckduckgo.com/l/?uddg=") || rawHref.includes("uddg=") ? decodeURIComponent(rawHref.split("uddg=")[1]?.split("&")[0] || "") : rawHref;
981
+ const rawTitle = titleMatch[2]?.replace(/<[^>]+>/g, "").trim() || query;
982
+ const snippetMatch = block.match(/<(?:a|div)[^>]*class="[^"]*result__snippet[^"]*"[^>]*>([\s\S]*?)<\/(?:a|div)>/i);
983
+ const snippet = snippetMatch ? snippetMatch[1]?.replace(/<[^>]+>/g, "").trim() || "" : "";
984
+ if (cleanUrl && !cleanUrl.includes("duckduckgo.com") && cleanUrl.startsWith("http")) {
985
+ if (isUrlAllowedByDomainFilter(cleanUrl, domainFilter)) {
986
+ hits.push({
987
+ title: rawTitle,
988
+ url: cleanUrl,
989
+ content: snippet,
990
+ engine: "duckduckgo"
991
+ });
992
+ }
993
+ }
994
+ }
995
+ if (hits.length === 0) {
996
+ const resultRegex = /<a[^>]*class="result__a"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
997
+ let match;
998
+ while ((match = resultRegex.exec(html)) !== null && hits.length < limit) {
999
+ const rawTitle = match[1]?.replace(/<[^>]+>/g, "").trim() || query;
1000
+ const rawUrl = match[2]?.trim();
1001
+ const cleanUrl = rawUrl?.startsWith("//duckduckgo.com/l/?uddg=") ? decodeURIComponent(rawUrl.split("uddg=")[1]?.split("&")[0] || "") : rawUrl;
1002
+ const snippet = match[3]?.replace(/<[^>]+>/g, "").trim() || "";
1003
+ if (cleanUrl && !cleanUrl.includes("duckduckgo.com")) {
1004
+ if (isUrlAllowedByDomainFilter(cleanUrl, domainFilter)) {
1005
+ hits.push({
1006
+ title: rawTitle,
1007
+ url: cleanUrl,
1008
+ content: snippet,
1009
+ engine: "duckduckgo"
1010
+ });
1011
+ }
1012
+ }
902
1013
  }
903
1014
  }
904
1015
  if (hits.length > 0) return hits;
@@ -1289,12 +1400,27 @@ ${p.replace(/<[^>]+>/g, " ").trim()}
1289
1400
  }
1290
1401
  };
1291
1402
 
1403
+ // src/agent/evidenceLedger.ts
1404
+ function computeCanonicalId(url) {
1405
+ try {
1406
+ let hash = 0;
1407
+ const clean = url.trim().replace(/^https?:\/\//i, "").replace(/[\/?#].*$/, "").toLowerCase();
1408
+ for (let i = 0; i < clean.length; i++) {
1409
+ hash = hash * 31 + clean.charCodeAt(i) >>> 0;
1410
+ }
1411
+ return `SRC-${hash.toString(16).padStart(8, "0")}`;
1412
+ } catch {
1413
+ return `SRC-${Math.random().toString(36).slice(2, 10)}`;
1414
+ }
1415
+ }
1416
+
1292
1417
  // src/harvester/catalogBuilder.ts
1293
1418
  var CatalogBuilder = class {
1294
1419
  buildBundle(topic, domain, items, options = {}) {
1295
1420
  const defaultLang = inferPrimaryCodeLanguage(topic, options.context, domain);
1296
1421
  const indexList = items.map((item, idx) => ({
1297
1422
  id: `RES-${String(idx + 1).padStart(2, "0")}`,
1423
+ canonicalId: computeCanonicalId(item.doc.url),
1298
1424
  title: item.doc.title,
1299
1425
  url: item.doc.url,
1300
1426
  domain,
@@ -1936,12 +2062,19 @@ Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered fr
1936
2062
  concepts: sortedConcepts.length > 0 ? sortedConcepts : concepts,
1937
2063
  conceptDagTsv: this.formatConceptDagTsv(sortedConcepts.length > 0 ? sortedConcepts : concepts),
1938
2064
  isLlmRefined: true,
2065
+ status: "success",
1939
2066
  unrefinedCandidatesCount: detBundle.unrefinedCandidatesCount
1940
2067
  };
1941
2068
  }
1942
2069
  }
1943
2070
  } catch (e) {
1944
2071
  console.warn("[PedagogicalTermExtractor] Host LLM term refinement failed, using deterministic extraction:", e?.message || e);
2072
+ detBundle.status = "partial";
2073
+ detBundle.errors = detBundle.errors || [];
2074
+ detBundle.errors.push(`Host LLM refinement failed: ${e?.message || String(e)}`);
2075
+ }
2076
+ if (!detBundle.status) {
2077
+ detBundle.status = detBundle.concepts.length > 0 ? "partial" : "failed";
1945
2078
  }
1946
2079
  return detBundle;
1947
2080
  }
@@ -2251,34 +2384,34 @@ function mapTopicToCS2023(topic, context) {
2251
2384
  ...context?.inScopeTopics || []
2252
2385
  ].filter(Boolean).join(" ").toLowerCase();
2253
2386
  if (/\b(concurren|async|actor|thread|distributed|parallel|đồng thời|bất đồng bộ|đa luồng)\b/.test(combined)) {
2254
- return { area: CS2023_KNOWLEDGE_AREAS.PDC, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.95 };
2387
+ return { area: CS2023_KNOWLEDGE_AREAS.PDC, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.95, sourceProvider: "internal_catalog" };
2255
2388
  }
2256
2389
  if (/\b(ai|machine learning|deep learning|neural|llm|computer vision|trí tuệ nhân tạo|học máy|mạng nơ-ron)\b/.test(combined)) {
2257
- return { area: CS2023_KNOWLEDGE_AREAS.AI, k12Strand: CSTA_K12_STRANDS.DA, matchConfidence: 0.95 };
2390
+ return { area: CS2023_KNOWLEDGE_AREAS.AI, k12Strand: CSTA_K12_STRANDS.DA, matchConfidence: 0.95, sourceProvider: "internal_catalog" };
2258
2391
  }
2259
2392
  if (/\b(arduino|esp32|raspberry|micro:?bit|mblock|makecode|lego|sensor|circuit|gpio|iot|hardware|robotics|cảm biến|vi điều khiển|mạch điện|điện tử)\b/.test(combined)) {
2260
- return { area: CS2023_KNOWLEDGE_AREAS.AR, k12Strand: CSTA_K12_STRANDS.CS, matchConfidence: 0.92 };
2393
+ return { area: CS2023_KNOWLEDGE_AREAS.AR, k12Strand: CSTA_K12_STRANDS.CS, matchConfidence: 0.92, sourceProvider: "internal_catalog" };
2261
2394
  }
2262
2395
  if (/\b(roblox|unity|godot|pygame|game|2d|3d|sprite|collision|physics|shader|trò chơi|làm game)\b/.test(combined)) {
2263
- return { area: CS2023_KNOWLEDGE_AREAS.GIT, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.92 };
2396
+ return { area: CS2023_KNOWLEDGE_AREAS.GIT, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.92, sourceProvider: "internal_catalog" };
2264
2397
  }
2265
2398
  if (/\b(swiftui|ui|ux|interface|frontend|accessibility|figma|giao diện|tương tác người-máy)\b/.test(combined)) {
2266
- return { area: CS2023_KNOWLEDGE_AREAS.HCI, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.9 };
2399
+ return { area: CS2023_KNOWLEDGE_AREAS.HCI, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.9, sourceProvider: "internal_catalog" };
2267
2400
  }
2268
2401
  if (/\b(os|operating system|kernel|virtual memory|linux|hệ điều hành)\b/.test(combined)) {
2269
- return { area: CS2023_KNOWLEDGE_AREAS.OS, k12Strand: CSTA_K12_STRANDS.CS, matchConfidence: 0.9 };
2402
+ return { area: CS2023_KNOWLEDGE_AREAS.OS, k12Strand: CSTA_K12_STRANDS.CS, matchConfidence: 0.9, sourceProvider: "internal_catalog" };
2270
2403
  }
2271
2404
  if (/\b(security|auth|crypto|cyber|jwt|oauth|bảo mật|an toàn thông tin|mã hóa)\b/.test(combined)) {
2272
- return { area: CS2023_KNOWLEDGE_AREAS.SEC, k12Strand: CSTA_K12_STRANDS.NI, matchConfidence: 0.9 };
2405
+ return { area: CS2023_KNOWLEDGE_AREAS.SEC, k12Strand: CSTA_K12_STRANDS.NI, matchConfidence: 0.9, sourceProvider: "internal_catalog" };
2273
2406
  }
2274
2407
  if (/\b(sql|database|postgres|supabase|mongodb|data model|cơ sở dữ liệu|truy vấn)\b/.test(combined)) {
2275
- return { area: CS2023_KNOWLEDGE_AREAS.DM, k12Strand: CSTA_K12_STRANDS.DA, matchConfidence: 0.9 };
2408
+ return { area: CS2023_KNOWLEDGE_AREAS.DM, k12Strand: CSTA_K12_STRANDS.DA, matchConfidence: 0.9, sourceProvider: "internal_catalog" };
2276
2409
  }
2277
2410
  if (/\b(architecture|design pattern|software engineer|clean code|testing|tdd|ci\/cd|kiến trúc phần mềm|kiểm thử)\b/.test(combined)) {
2278
- return { area: CS2023_KNOWLEDGE_AREAS.SE, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.88 };
2411
+ return { area: CS2023_KNOWLEDGE_AREAS.SE, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.88, sourceProvider: "internal_catalog" };
2279
2412
  }
2280
2413
  if (/\b(algorithm|data structure|tree|graph|dynamic programming|sorting|thuật toán|cấu trúc dữ liệu|đồ thị|quy hoạch động)\b/.test(combined)) {
2281
- return { area: CS2023_KNOWLEDGE_AREAS.AL, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.9, isCsApplicable: true };
2414
+ return { area: CS2023_KNOWLEDGE_AREAS.AL, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.9, isCsApplicable: true, sourceProvider: "internal_catalog" };
2282
2415
  }
2283
2416
  const isExplicitNonCoding = /\b(noncoding|non-coding|phi lập trình|không lập trình|applied science|khoa học ứng dụng|general science|khoa học tự nhiên|applied-noncoding)\b/i.test(combined);
2284
2417
  if (isExplicitNonCoding) {
@@ -2286,10 +2419,11 @@ function mapTopicToCS2023(topic, context) {
2286
2419
  area: CS2023_KNOWLEDGE_AREAS.SPD,
2287
2420
  k12Strand: CSTA_K12_STRANDS.IC,
2288
2421
  matchConfidence: 0.25,
2289
- isCsApplicable: false
2422
+ isCsApplicable: false,
2423
+ sourceProvider: "internal_catalog"
2290
2424
  };
2291
2425
  }
2292
- return { area: CS2023_KNOWLEDGE_AREAS.SDF, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.45, isCsApplicable: true };
2426
+ return { area: CS2023_KNOWLEDGE_AREAS.SDF, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.45, isCsApplicable: true, sourceProvider: "internal_catalog" };
2293
2427
  }
2294
2428
 
2295
2429
  // src/academic/citationManager.ts
@@ -2776,7 +2910,7 @@ var TechnicalGroundingEngine = class {
2776
2910
  } catch {
2777
2911
  }
2778
2912
  }
2779
- const hitsToScrape = rawHits.slice(0, 3);
2913
+ const hitsToScrape = rawHits.slice(0, 5);
2780
2914
  const scrapeResults = await Promise.allSettled(
2781
2915
  hitsToScrape.map(async (h) => {
2782
2916
  try {
@@ -2854,36 +2988,40 @@ var TechnicalGroundingEngine = class {
2854
2988
  pinoutTables: preservedTables.slice(0, 4),
2855
2989
  keyExcerpts: preservedKeyExcerpts.slice(0, 6)
2856
2990
  };
2857
- if (llm && scrapedSnippets.length > 0) {
2991
+ if (llm) {
2858
2992
  try {
2859
2993
  const roleByDomain = domain === "hardware" ? "Hardware & Embedded Systems Curriculum Specialist" : domain === "software" ? "Software & Technology Curriculum Specialist" : "Applied Subject Matter & Curriculum Specialist";
2994
+ const sourcesText = scrapedSnippets.length > 0 ? scrapedSnippets.map((s, i) => {
2995
+ const parts = [`[Source ${i + 1}: ${s.title} (${s.url})]
2996
+ ${s.excerpt}`];
2997
+ if (s.codeBlocks && s.codeBlocks.length > 0) {
2998
+ parts.push(
2999
+ "Code Snippets:\n" + s.codeBlocks.map((c) => `\`\`\`${c.language || ""}
3000
+ ${c.code}
3001
+ \`\`\``).join("\n")
3002
+ );
3003
+ }
3004
+ if (s.tables && s.tables.length > 0) {
3005
+ parts.push("Tables/Pinouts:\n" + s.tables.join("\n\n"));
3006
+ }
3007
+ return parts.join("\n");
3008
+ }).join("\n\n") : "[No live web sources retrieved. Synthesize domain constraints, exemplars, and common pitfalls based strictly on domain engineering standards].";
2860
3009
  const prompt = [
2861
3010
  `You are a ${roleByDomain}. Analyze these verified research excerpts for topic: "${topic}".`,
2862
3011
  `Domain: ${domain}, Hardware: ${hardware || "N/A"}, Software/Env: ${software || "N/A"}, Student Age Band: ${ageBand[0]}-${ageBand[1]} years old.`,
2863
3012
  `
2864
3013
  Harvested Sources:
2865
- ${scrapedSnippets.map((s, i) => {
2866
- const parts = [`[Source ${i + 1}: ${s.title} (${s.url})]
2867
- ${s.excerpt}`];
2868
- if (s.codeBlocks && s.codeBlocks.length > 0) {
2869
- parts.push("Code Snippets:\n" + s.codeBlocks.map((c) => `\`\`\`${c.language || ""}
2870
- ${c.code}
2871
- \`\`\``).join("\n"));
2872
- }
2873
- if (s.tables && s.tables.length > 0) {
2874
- parts.push("Tables/Pinouts:\n" + s.tables.join("\n\n"));
2875
- }
2876
- return parts.join("\n");
2877
- }).join("\n\n")}`,
3014
+ ${sourcesText}`,
2878
3015
  `
2879
- Synthesize a strictly valid JSON object based ONLY on the evidence above. If no evidence exists for a field, leave it as an empty array [] without inventing facts:`,
3016
+ Synthesize a strictly valid JSON object based on the evidence above. If no evidence exists for a field, leave it as an empty array [] without inventing facts:`,
2880
3017
  `{`,
2881
3018
  ` "compatibilityVerdict": boolean,`,
2882
3019
  ` "compatibilityNotes": string[],`,
2883
3020
  ` "cognitiveReadiness": "feasible" | "challenging" | "excessive",`,
2884
3021
  ` "potentialBarriers": string[],`,
2885
- ` "exemplars": [{"title": string, "description": string, "targetAudience": string, "hardwareRequirements": string[], "softwareRequirements": string[]}],`,
2886
- ` "technicalBottlenecks": string[]`,
3022
+ ` "exemplars": [{"title": string, "description": string, "targetAudience": string, "hardwareRequirements": string[], "softwareRequirements": string[], "sourceUrl": string}],`,
3023
+ ` "technicalBottlenecks": string[],`,
3024
+ ` "components": [{"name": string, "pinout": string, "voltage": string, "libraries": string[], "safety_rules": string[], "wiring_notes": string}]`,
2887
3025
  `}`
2888
3026
  ].join("\n");
2889
3027
  const raw = await llm(prompt, {
@@ -2891,9 +3029,18 @@ Synthesize a strictly valid JSON object based ONLY on the evidence above. If no
2891
3029
  temperature: 0.2,
2892
3030
  jsonMode: true
2893
3031
  });
2894
- const jsonMatch = raw.match(/\{[\s\S]*\}/);
2895
- if (jsonMatch) {
2896
- const parsed = JSON.parse(jsonMatch[0]);
3032
+ const unFenced = raw.trim().replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/i, "").trim();
3033
+ const firstBrace = unFenced.indexOf("{");
3034
+ const lastBrace = unFenced.lastIndexOf("}");
3035
+ const candidateJson = firstBrace !== -1 && lastBrace > firstBrace ? unFenced.slice(firstBrace, lastBrace + 1) : unFenced;
3036
+ let parsed;
3037
+ try {
3038
+ parsed = JSON.parse(candidateJson);
3039
+ } catch {
3040
+ const match = raw.match(/\{[\s\S]*\}/);
3041
+ parsed = match ? JSON.parse(match[0]) : null;
3042
+ }
3043
+ if (parsed) {
2897
3044
  factsheet.compatibilityMatrix.verifiedCompatible = Boolean(parsed.compatibilityVerdict ?? true);
2898
3045
  if (Array.isArray(parsed.compatibilityNotes)) factsheet.compatibilityMatrix.notes = parsed.compatibilityNotes.map(String);
2899
3046
  if (["feasible", "challenging", "excessive"].includes(parsed.cognitiveReadiness)) {
@@ -2901,24 +3048,44 @@ Synthesize a strictly valid JSON object based ONLY on the evidence above. If no
2901
3048
  }
2902
3049
  if (Array.isArray(parsed.potentialBarriers)) factsheet.ageFeasibility.potentialBarriers = parsed.potentialBarriers.map(String);
2903
3050
  if (Array.isArray(parsed.exemplars) && parsed.exemplars.length > 0) {
2904
- factsheet.exemplarProjects = parsed.exemplars.map((ex) => ({
2905
- title: String(ex.title || "Verified Project"),
2906
- description: String(ex.description || ""),
2907
- targetAudience: String(ex.targetAudience || `Ages ${ageBand[0]}-${ageBand[1]}`),
2908
- hardwareRequirements: Array.isArray(ex.hardwareRequirements) ? ex.hardwareRequirements.map(String) : void 0,
2909
- softwareRequirements: Array.isArray(ex.softwareRequirements) ? ex.softwareRequirements.map(String) : void 0,
2910
- sourceUrl: scrapedSnippets[0]?.url
3051
+ factsheet.exemplarProjects = parsed.exemplars.map((ex, idx) => {
3052
+ const matchedSnippet = scrapedSnippets.find(
3053
+ (s) => s.title && ex.title && (s.title.toLowerCase().includes(ex.title.toLowerCase()) || ex.title.toLowerCase().includes(s.title.toLowerCase()))
3054
+ );
3055
+ const fallbackUrl = scrapedSnippets.length > 0 ? scrapedSnippets[idx % scrapedSnippets.length]?.url || scrapedSnippets[0]?.url : void 0;
3056
+ return {
3057
+ title: String(ex.title || "Verified Project"),
3058
+ description: String(ex.description || ""),
3059
+ targetAudience: String(ex.targetAudience || `Ages ${ageBand[0]}-${ageBand[1]}`),
3060
+ hardwareRequirements: Array.isArray(ex.hardwareRequirements) ? ex.hardwareRequirements.map(String) : void 0,
3061
+ softwareRequirements: Array.isArray(ex.softwareRequirements) ? ex.softwareRequirements.map(String) : void 0,
3062
+ sourceUrl: ex.sourceUrl ? String(ex.sourceUrl) : matchedSnippet?.url || fallbackUrl
3063
+ };
3064
+ });
3065
+ }
3066
+ if (Array.isArray(parsed.components) && parsed.components.length > 0) {
3067
+ factsheet.components = parsed.components.map((c) => ({
3068
+ name: String(c.name || "Component"),
3069
+ pinout: String(c.pinout || ""),
3070
+ voltage: c.voltage ? String(c.voltage) : "5V",
3071
+ libraries: Array.isArray(c.libraries) ? c.libraries.map(String) : [],
3072
+ safety_rules: Array.isArray(c.safety_rules) ? c.safety_rules.map(String) : Array.isArray(c.safetyPrecautions) ? c.safetyPrecautions.map(String) : [],
3073
+ wiring_notes: c.wiring_notes ? String(c.wiring_notes) : c.circuitSchematicNotes ? String(c.circuitSchematicNotes) : void 0
2911
3074
  }));
2912
3075
  }
2913
3076
  if (Array.isArray(parsed.technicalBottlenecks)) factsheet.technicalBottlenecks = parsed.technicalBottlenecks.map(String);
2914
- if (factsheet.exemplarProjects.length > 0 || factsheet.technicalBottlenecks.length > 0) {
3077
+ if (factsheet.exemplarProjects.length > 0 || factsheet.technicalBottlenecks.length > 0 || factsheet.components && factsheet.components.length > 0) {
2915
3078
  factsheet.evidenceBacked = true;
3079
+ factsheet.status = "success";
2916
3080
  }
2917
3081
  }
2918
3082
  } catch (err) {
2919
3083
  console.warn("[TechnicalGroundingEngine] LLM synthesis error:", err?.message || err);
3084
+ factsheet.errors = factsheet.errors || [];
3085
+ factsheet.errors.push(`LLM synthesis failed: ${err?.message || String(err)}`);
2920
3086
  }
2921
- } else if (scrapedSnippets.length > 0 && scrapedSnippets[0]?.excerpt?.trim()) {
3087
+ }
3088
+ if ((!factsheet.exemplarProjects || factsheet.exemplarProjects.length === 0) && scrapedSnippets.length > 0 && scrapedSnippets[0]?.excerpt?.trim()) {
2922
3089
  const first = scrapedSnippets[0];
2923
3090
  if (first.title || first.excerpt) {
2924
3091
  factsheet.exemplarProjects = [
@@ -2934,6 +3101,13 @@ Synthesize a strictly valid JSON object based ONLY on the evidence above. If no
2934
3101
  factsheet.evidenceBacked = true;
2935
3102
  }
2936
3103
  }
3104
+ if (!factsheet.status) {
3105
+ if (factsheet.exemplarProjects.length > 0 || factsheet.evidenceBacked) {
3106
+ factsheet.status = factsheet.errors && factsheet.errors.length > 0 ? "partial" : "success";
3107
+ } else {
3108
+ factsheet.status = factsheet.errors && factsheet.errors.length > 0 ? "failed" : "partial";
3109
+ }
3110
+ }
2937
3111
  factsheet.markdownReport = this.renderMarkdownReport(factsheet);
2938
3112
  return factsheet;
2939
3113
  }
@@ -2993,6 +3167,26 @@ ${c.code}
2993
3167
  if (fs.pinoutTables && fs.pinoutTables.length > 0) {
2994
3168
  lines.push(`
2995
3169
  ## 7. Pinout & Specification Tables`, ...fs.pinoutTables);
3170
+ }
3171
+ if (fs.components && fs.components.length > 0) {
3172
+ lines.push(
3173
+ `
3174
+ ## 8. Hardware Specifications & Pinout Reference`,
3175
+ `| Component | Pinout | Voltage | Libraries | Safety |`,
3176
+ `|---|---|---|---|---|`,
3177
+ ...fs.components.map((c) => {
3178
+ const name = (c.name || "Component").replace(/\|/g, "/");
3179
+ const pinout = (c.pinout || "N/A").replace(/\|/g, "/");
3180
+ const voltage = (c.voltage || "5V").replace(/\|/g, "/");
3181
+ const libs = (c.libraries && c.libraries.length > 0 ? c.libraries.join(", ") : "Standard").replace(/\|/g, "/");
3182
+ const safety = (c.safety_rules && c.safety_rules.length > 0 ? c.safety_rules.join("; ") : "Safe operating margins").replace(/\|/g, "/");
3183
+ return `| ${name} | ${pinout} | ${voltage} | ${libs} | ${safety} |`;
3184
+ })
3185
+ );
3186
+ }
3187
+ if (fs.errors && fs.errors.length > 0) {
3188
+ lines.push(`
3189
+ > \u26A0\uFE0F **Status Note:** [${fs.status?.toUpperCase() || "PARTIAL"}] ${fs.errors.join("; ")}`);
2996
3190
  }
2997
3191
  return lines.join("\n");
2998
3192
  }