@thanh01.pmt/research-kit 1.2.1 → 1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/academic/index.cjs +11 -2
- package/dist/academic/index.cjs.map +1 -1
- package/dist/academic/index.d.cts +3 -3
- package/dist/academic/index.d.ts +3 -3
- package/dist/academic/index.js +1 -1
- package/dist/{catalogBuilder-DHV6y1gD.d.ts → catalogBuilder-B4eiLS_S.d.cts} +20 -6
- package/dist/{catalogBuilder-BaVRvg-9.d.cts → catalogBuilder-BNB_gaWP.d.ts} +20 -6
- package/dist/{chunk-X3X26W2I.js → chunk-HESXWQEF.js} +12 -7
- package/dist/chunk-HESXWQEF.js.map +1 -0
- package/dist/{chunk-QFYM2NV4.js → chunk-KXJDQBSJ.js} +3 -3
- package/dist/{chunk-QFYM2NV4.js.map → chunk-KXJDQBSJ.js.map} +1 -1
- package/dist/{chunk-RW7VS7NI.js → chunk-P6RHBJY2.js} +88 -25
- package/dist/chunk-P6RHBJY2.js.map +1 -0
- package/dist/{chunk-ON6VFCJO.js → chunk-QQZQNBJC.js} +51 -2
- package/dist/chunk-QQZQNBJC.js.map +1 -0
- package/dist/{chunk-HZNSZDSM.js → chunk-SAHBBO3U.js} +99 -11
- package/dist/chunk-SAHBBO3U.js.map +1 -0
- package/dist/{chunk-UKGR45VZ.js → chunk-SDDZFFL6.js} +13 -4
- package/dist/chunk-SDDZFFL6.js.map +1 -0
- package/dist/chunk-YICMTZS5.js +527 -0
- package/dist/chunk-YICMTZS5.js.map +1 -0
- package/dist/config/index.cjs +10 -5
- package/dist/config/index.cjs.map +1 -1
- package/dist/config/index.d.cts +1 -1
- package/dist/config/index.d.ts +1 -1
- package/dist/config/index.js +1 -1
- package/dist/evidence/index.cjs.map +1 -1
- package/dist/evidence/index.d.cts +2 -2
- package/dist/evidence/index.d.ts +2 -2
- package/dist/evidence/index.js +2 -2
- package/dist/harvester/index.cjs +674 -9
- package/dist/harvester/index.cjs.map +1 -1
- package/dist/harvester/index.d.cts +87 -5
- package/dist/harvester/index.d.ts +87 -5
- package/dist/harvester/index.js +3 -3
- package/dist/{index-DlngvZBr.d.cts → index-9rDiFuR4.d.cts} +43 -12
- package/dist/{index-DlngvZBr.d.ts → index-9rDiFuR4.d.ts} +43 -12
- package/dist/index.cjs +783 -35
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +13 -11
- package/dist/index.d.ts +13 -11
- package/dist/index.js +19 -12
- package/dist/index.js.map +1 -1
- package/dist/{literatureReview-C7hsjrn4.d.cts → literatureReview-BwidpYJo.d.cts} +2 -1
- package/dist/{literatureReview-DWl13r_3.d.ts → literatureReview-DOzgm9Q2.d.ts} +2 -1
- package/dist/pipeline/index.cjs +256 -32
- package/dist/pipeline/index.cjs.map +1 -1
- package/dist/pipeline/index.d.cts +10 -4
- package/dist/pipeline/index.d.ts +10 -4
- package/dist/pipeline/index.js +5 -5
- package/dist/{webScraper-yO9P-oLb.d.cts → webScraper-CAQjE46m.d.cts} +11 -2
- package/dist/{webScraper-DBFUIqoO.d.ts → webScraper-CXIpvyA6.d.ts} +11 -2
- package/package.json +9 -8
- package/LICENSE +0 -21
- package/dist/chunk-HZNSZDSM.js.map +0 -1
- package/dist/chunk-LSMSIPTZ.js +0 -3
- package/dist/chunk-LSMSIPTZ.js.map +0 -1
- package/dist/chunk-ON6VFCJO.js.map +0 -1
- package/dist/chunk-RW7VS7NI.js.map +0 -1
- package/dist/chunk-UKGR45VZ.js.map +0 -1
- package/dist/chunk-X3X26W2I.js.map +0 -1
package/dist/pipeline/index.cjs
CHANGED
|
@@ -554,6 +554,8 @@ var MultiSourceSearchEngine = class {
|
|
|
554
554
|
tavilyKey;
|
|
555
555
|
exaKey;
|
|
556
556
|
serperKey;
|
|
557
|
+
brightDataKey;
|
|
558
|
+
brightDataZone;
|
|
557
559
|
searxngHost;
|
|
558
560
|
searxngHeaders;
|
|
559
561
|
allowEnvFallback;
|
|
@@ -564,6 +566,8 @@ var MultiSourceSearchEngine = class {
|
|
|
564
566
|
this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
|
|
565
567
|
this.exaKey = options.apiKeys?.exa || options.exaKey || (this.allowEnvFallback ? process.env.EXA_API_KEY : void 0);
|
|
566
568
|
this.serperKey = options.apiKeys?.serper || options.serperKey || (this.allowEnvFallback ? process.env.SERPER_API_KEY : void 0);
|
|
569
|
+
this.brightDataKey = options.apiKeys?.brightdata || options.apiKeys?.brightData || options.brightDataKey || (this.allowEnvFallback ? process.env.BRIGHTDATA_API_KEY || process.env.BRIGHT_DATA_KEY : void 0);
|
|
570
|
+
this.brightDataZone = options.brightDataZone || (this.allowEnvFallback ? process.env.BRIGHTDATA_SERP_ZONE || process.env.BRIGHTDATA_UNLOCKER_ZONE : void 0) || "cli_unlocker";
|
|
567
571
|
this.searxngHost = options.searxngHost || (this.allowEnvFallback ? process.env.SEARXNG_URL || process.env.SEARXNG_HOST : void 0);
|
|
568
572
|
const cfId = process.env.CF_ACCESS_CLIENT_ID;
|
|
569
573
|
const cfSecret = process.env.CF_ACCESS_CLIENT_SECRET;
|
|
@@ -581,6 +585,12 @@ var MultiSourceSearchEngine = class {
|
|
|
581
585
|
if (keys.tavily !== void 0) this.tavilyKey = keys.tavily;
|
|
582
586
|
if (keys.exa !== void 0) this.exaKey = keys.exa;
|
|
583
587
|
if (keys.serper !== void 0) this.serperKey = keys.serper;
|
|
588
|
+
if (keys.brightdata !== void 0) this.brightDataKey = keys.brightdata;
|
|
589
|
+
if (keys.brightData !== void 0) this.brightDataKey = keys.brightData;
|
|
590
|
+
}
|
|
591
|
+
setBrightData(key, zone) {
|
|
592
|
+
if (key !== void 0) this.brightDataKey = key;
|
|
593
|
+
if (zone !== void 0) this.brightDataZone = zone;
|
|
584
594
|
}
|
|
585
595
|
setSearxng(host, headers) {
|
|
586
596
|
this.searxngHost = host;
|
|
@@ -627,11 +637,20 @@ var MultiSourceSearchEngine = class {
|
|
|
627
637
|
const tavilyKey = options.apiKeys?.tavily || options.tavilyKey || this.tavilyKey;
|
|
628
638
|
const exaKey = options.apiKeys?.exa || options.exaKey || this.exaKey;
|
|
629
639
|
const serperKey = options.apiKeys?.serper || options.serperKey || this.serperKey;
|
|
640
|
+
const brightDataKey = options.apiKeys?.brightdata || options.apiKeys?.brightData || options.brightDataKey || this.brightDataKey;
|
|
641
|
+
const brightDataZone = options.brightDataZone || this.brightDataZone;
|
|
630
642
|
const searxngHost = options.searxngHost || this.searxngHost;
|
|
631
643
|
if (preferred === "searxng" && searxngHost) {
|
|
632
644
|
const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders);
|
|
633
645
|
if (searxHits.length > 0) return searxHits;
|
|
634
646
|
}
|
|
647
|
+
if (preferred === "brightdata" && brightDataKey) {
|
|
648
|
+
const bdHits = await this.queryBrightData(query, brightDataKey, limit, {
|
|
649
|
+
zone: brightDataZone,
|
|
650
|
+
engine: options.brightDataEngine
|
|
651
|
+
});
|
|
652
|
+
if (bdHits.length > 0) return bdHits;
|
|
653
|
+
}
|
|
635
654
|
if (preferred === "exa" && exaKey) {
|
|
636
655
|
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
|
|
637
656
|
if (exaHits.length > 0) return exaHits;
|
|
@@ -653,6 +672,9 @@ var MultiSourceSearchEngine = class {
|
|
|
653
672
|
include_answer: true,
|
|
654
673
|
max_results: limit
|
|
655
674
|
};
|
|
675
|
+
if (options.timeRange) {
|
|
676
|
+
body.time_range = options.timeRange;
|
|
677
|
+
}
|
|
656
678
|
if (options.domainFilter && options.domainFilter.length > 0) {
|
|
657
679
|
body.include_domains = options.domainFilter;
|
|
658
680
|
}
|
|
@@ -679,25 +701,76 @@ var MultiSourceSearchEngine = class {
|
|
|
679
701
|
}
|
|
680
702
|
}
|
|
681
703
|
if (exaKey) {
|
|
682
|
-
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
|
|
704
|
+
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter, options.timeRange);
|
|
683
705
|
if (exaHits.length > 0) return exaHits;
|
|
684
706
|
}
|
|
685
707
|
if (serperKey) {
|
|
686
|
-
const serperHits = await this.querySerper(query, serperKey, limit);
|
|
708
|
+
const serperHits = await this.querySerper(query, serperKey, limit, options.timeRange);
|
|
687
709
|
if (serperHits.length > 0) return serperHits;
|
|
688
710
|
}
|
|
711
|
+
if (brightDataKey) {
|
|
712
|
+
const bdHits = await this.queryBrightData(query, brightDataKey, limit, {
|
|
713
|
+
zone: brightDataZone,
|
|
714
|
+
engine: options.brightDataEngine
|
|
715
|
+
});
|
|
716
|
+
if (bdHits.length > 0) return bdHits;
|
|
717
|
+
}
|
|
689
718
|
if (searxngHost) {
|
|
690
|
-
const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders);
|
|
719
|
+
const searxHits = await this.querySearxng(query, searxngHost, limit, options.searxngHeaders, options.timeRange);
|
|
691
720
|
if (searxHits.length > 0) return searxHits;
|
|
692
721
|
}
|
|
693
722
|
const ddgHits = await this.queryDuckDuckGo(query, limit);
|
|
694
723
|
if (ddgHits.length > 0) return ddgHits;
|
|
695
724
|
return [];
|
|
696
725
|
}
|
|
697
|
-
async
|
|
726
|
+
async queryBrightData(query, key, limit = 5, options = {}) {
|
|
727
|
+
try {
|
|
728
|
+
const zone = options.zone || this.brightDataZone || "cli_unlocker";
|
|
729
|
+
const engine = options.engine || "bing";
|
|
730
|
+
let searchUrl = `https://www.bing.com/search?q=${encodeURIComponent(query)}&brd_json=1`;
|
|
731
|
+
if (engine === "google") {
|
|
732
|
+
searchUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}&brd_json=1&hl=en&gl=us`;
|
|
733
|
+
} else if (engine === "yandex") {
|
|
734
|
+
searchUrl = `https://www.yandex.com/search?q=${encodeURIComponent(query)}&brd_json=1`;
|
|
735
|
+
}
|
|
736
|
+
const res = await fetch("https://api.brightdata.com/request", {
|
|
737
|
+
method: "POST",
|
|
738
|
+
headers: {
|
|
739
|
+
Authorization: `Bearer ${key}`,
|
|
740
|
+
"Content-Type": "application/json"
|
|
741
|
+
},
|
|
742
|
+
body: JSON.stringify({
|
|
743
|
+
zone,
|
|
744
|
+
url: searchUrl,
|
|
745
|
+
format: "raw"
|
|
746
|
+
}),
|
|
747
|
+
signal: AbortSignal.timeout(15e3)
|
|
748
|
+
});
|
|
749
|
+
if (res.ok) {
|
|
750
|
+
const data = await res.json();
|
|
751
|
+
const organic = Array.isArray(data?.organic) ? data.organic : [];
|
|
752
|
+
if (organic.length > 0) {
|
|
753
|
+
return organic.slice(0, limit).map((r, idx) => ({
|
|
754
|
+
title: r.title || "Untitled",
|
|
755
|
+
url: r.link,
|
|
756
|
+
content: r.description || r.snippet || "",
|
|
757
|
+
score: r.rank ? 1 / r.rank : 1 / (idx + 1),
|
|
758
|
+
engine: "brightdata"
|
|
759
|
+
}));
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
} catch (e) {
|
|
763
|
+
console.warn("[MultiSourceSearchEngine] Bright Data attempt failed:", e?.message || e);
|
|
764
|
+
}
|
|
765
|
+
return [];
|
|
766
|
+
}
|
|
767
|
+
async querySearxng(query, searxngHost, limit, customHeaders, timeRange) {
|
|
698
768
|
try {
|
|
699
769
|
const cleanHost = searxngHost.replace(/\/$/, "");
|
|
700
|
-
|
|
770
|
+
let searchUrl = `${cleanHost}/search?q=${encodeURIComponent(query)}&format=json`;
|
|
771
|
+
if (timeRange) {
|
|
772
|
+
searchUrl += `&time_range=${encodeURIComponent(timeRange)}`;
|
|
773
|
+
}
|
|
701
774
|
const headers = {
|
|
702
775
|
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)",
|
|
703
776
|
...this.searxngHeaders || {},
|
|
@@ -724,7 +797,7 @@ var MultiSourceSearchEngine = class {
|
|
|
724
797
|
}
|
|
725
798
|
return [];
|
|
726
799
|
}
|
|
727
|
-
async queryExa(query, exaKey, limit, domainFilter) {
|
|
800
|
+
async queryExa(query, exaKey, limit, domainFilter, timeRange) {
|
|
728
801
|
try {
|
|
729
802
|
const body = {
|
|
730
803
|
query,
|
|
@@ -734,6 +807,14 @@ var MultiSourceSearchEngine = class {
|
|
|
734
807
|
if (domainFilter && domainFilter.length > 0) {
|
|
735
808
|
body.includeDomains = domainFilter;
|
|
736
809
|
}
|
|
810
|
+
if (timeRange) {
|
|
811
|
+
const now = /* @__PURE__ */ new Date();
|
|
812
|
+
if (timeRange === "day") now.setDate(now.getDate() - 1);
|
|
813
|
+
else if (timeRange === "week") now.setDate(now.getDate() - 7);
|
|
814
|
+
else if (timeRange === "month") now.setMonth(now.getMonth() - 1);
|
|
815
|
+
else if (timeRange === "year") now.setFullYear(now.getFullYear() - 1);
|
|
816
|
+
body.startPublishedDate = now.toISOString();
|
|
817
|
+
}
|
|
737
818
|
const res = await fetch("https://api.exa.ai/search", {
|
|
738
819
|
method: "POST",
|
|
739
820
|
headers: {
|
|
@@ -760,15 +841,22 @@ var MultiSourceSearchEngine = class {
|
|
|
760
841
|
}
|
|
761
842
|
return [];
|
|
762
843
|
}
|
|
763
|
-
async querySerper(query, serperKey, limit) {
|
|
844
|
+
async querySerper(query, serperKey, limit, timeRange) {
|
|
764
845
|
try {
|
|
846
|
+
const body = { q: query, num: limit };
|
|
847
|
+
if (timeRange) {
|
|
848
|
+
const tbsMap = { day: "qdr:d", week: "qdr:w", month: "qdr:m", year: "qdr:y" };
|
|
849
|
+
if (tbsMap[timeRange]) {
|
|
850
|
+
body.tbs = tbsMap[timeRange];
|
|
851
|
+
}
|
|
852
|
+
}
|
|
765
853
|
const res = await fetch("https://google.serper.dev/search", {
|
|
766
854
|
method: "POST",
|
|
767
855
|
headers: {
|
|
768
856
|
"X-API-KEY": serperKey,
|
|
769
857
|
"Content-Type": "application/json"
|
|
770
858
|
},
|
|
771
|
-
body: JSON.stringify(
|
|
859
|
+
body: JSON.stringify(body),
|
|
772
860
|
signal: AbortSignal.timeout(1e4)
|
|
773
861
|
});
|
|
774
862
|
if (res.ok) {
|
|
@@ -1017,6 +1105,8 @@ function extractSubstantiveExcerpts(markdown, maxExcerpts = 6) {
|
|
|
1017
1105
|
var WebScraper = class {
|
|
1018
1106
|
crawl4aiHost;
|
|
1019
1107
|
jinaKey;
|
|
1108
|
+
brightDataKey;
|
|
1109
|
+
brightDataZone;
|
|
1020
1110
|
allowEnvFallback;
|
|
1021
1111
|
customAdapter;
|
|
1022
1112
|
youtubeService;
|
|
@@ -1024,6 +1114,8 @@ var WebScraper = class {
|
|
|
1024
1114
|
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
1025
1115
|
this.crawl4aiHost = options.crawl4aiHost || (this.allowEnvFallback ? process.env.CRAWL4AI_HOST : void 0) || "http://localhost:11235";
|
|
1026
1116
|
this.jinaKey = options.jinaKey || (this.allowEnvFallback ? process.env.JINA_API_KEY : void 0);
|
|
1117
|
+
this.brightDataKey = options.brightDataKey || (this.allowEnvFallback ? process.env.BRIGHTDATA_API_KEY || process.env.BRIGHT_DATA_KEY : void 0);
|
|
1118
|
+
this.brightDataZone = options.brightDataZone || (this.allowEnvFallback ? process.env.BRIGHTDATA_UNLOCKER_ZONE : void 0) || "cli_unlocker";
|
|
1027
1119
|
this.customAdapter = options.customAdapter;
|
|
1028
1120
|
this.youtubeService = new YouTubeIngestionService();
|
|
1029
1121
|
}
|
|
@@ -1033,8 +1125,15 @@ var WebScraper = class {
|
|
|
1033
1125
|
setJinaKey(key) {
|
|
1034
1126
|
this.jinaKey = key;
|
|
1035
1127
|
}
|
|
1128
|
+
setBrightData(key, zone) {
|
|
1129
|
+
if (key !== void 0) this.brightDataKey = key;
|
|
1130
|
+
if (zone !== void 0) this.brightDataZone = zone;
|
|
1131
|
+
}
|
|
1036
1132
|
async scrapeUrl(targetUrl, options = {}) {
|
|
1037
1133
|
const adapter = options.customAdapter || this.customAdapter;
|
|
1134
|
+
const bdKey = options.brightDataKey || this.brightDataKey;
|
|
1135
|
+
const bdZone = options.brightDataZone || this.brightDataZone;
|
|
1136
|
+
const preferred = options.preferredProvider;
|
|
1038
1137
|
if (adapter) {
|
|
1039
1138
|
try {
|
|
1040
1139
|
const result = await adapter.crawl(targetUrl);
|
|
@@ -1055,6 +1154,13 @@ var WebScraper = class {
|
|
|
1055
1154
|
console.warn(`[WebScraper] YouTube transcript extraction failed for ${targetUrl}:`, e?.message || e);
|
|
1056
1155
|
}
|
|
1057
1156
|
}
|
|
1157
|
+
if (preferred === "brightdata" && bdKey) {
|
|
1158
|
+
try {
|
|
1159
|
+
return await this.scrapeWithBrightData(targetUrl, bdKey, bdZone);
|
|
1160
|
+
} catch (e) {
|
|
1161
|
+
console.warn(`[WebScraper] Preferred Bright Data failed for ${targetUrl}:`, e?.message || e);
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1058
1164
|
try {
|
|
1059
1165
|
const jinaUrl = `https://r.jina.ai/${targetUrl}`;
|
|
1060
1166
|
const jinaKey = options.jinaKey || this.jinaKey;
|
|
@@ -1099,6 +1205,13 @@ var WebScraper = class {
|
|
|
1099
1205
|
} catch {
|
|
1100
1206
|
}
|
|
1101
1207
|
}
|
|
1208
|
+
if (bdKey) {
|
|
1209
|
+
try {
|
|
1210
|
+
return await this.scrapeWithBrightData(targetUrl, bdKey, bdZone);
|
|
1211
|
+
} catch (e) {
|
|
1212
|
+
console.warn(`[WebScraper] Bright Data fallback failed for ${targetUrl}:`, e?.message || e);
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1102
1215
|
try {
|
|
1103
1216
|
const res = await fetch(targetUrl, {
|
|
1104
1217
|
headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" },
|
|
@@ -1123,6 +1236,30 @@ ${p.replace(/<[^>]+>/g, " ").trim()}
|
|
|
1123
1236
|
}
|
|
1124
1237
|
throw new Error(`Unable to fetch content from URL: ${targetUrl}`);
|
|
1125
1238
|
}
|
|
1239
|
+
async scrapeWithBrightData(targetUrl, key, zone) {
|
|
1240
|
+
const reqZone = zone || this.brightDataZone || "cli_unlocker";
|
|
1241
|
+
const res = await fetch("https://api.brightdata.com/request", {
|
|
1242
|
+
method: "POST",
|
|
1243
|
+
headers: {
|
|
1244
|
+
Authorization: `Bearer ${key}`,
|
|
1245
|
+
"Content-Type": "application/json"
|
|
1246
|
+
},
|
|
1247
|
+
body: JSON.stringify({
|
|
1248
|
+
zone: reqZone,
|
|
1249
|
+
url: targetUrl,
|
|
1250
|
+
format: "raw",
|
|
1251
|
+
data_format: "markdown"
|
|
1252
|
+
}),
|
|
1253
|
+
signal: AbortSignal.timeout(2e4)
|
|
1254
|
+
});
|
|
1255
|
+
if (res.ok) {
|
|
1256
|
+
const markdown = await res.text();
|
|
1257
|
+
if (markdown && markdown.length > 30) {
|
|
1258
|
+
return this.packageDocument(targetUrl, markdown, "brightdata");
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
throw new Error(`Bright Data Web Unlocker failed with HTTP ${res.status}`);
|
|
1262
|
+
}
|
|
1126
1263
|
packageDocument(url, markdown, provider, explicitTitle) {
|
|
1127
1264
|
const titleMatch = markdown.match(/^#\s+(.+)$/m) || markdown.match(/^Title:\s*(.+)$/im);
|
|
1128
1265
|
const title = (explicitTitle || (titleMatch ? titleMatch[1].trim() : url.split("/").filter(Boolean).pop() || "Document")).slice(0, 180);
|
|
@@ -1739,7 +1876,7 @@ ${corpusPreview}`,
|
|
|
1739
1876
|
`
|
|
1740
1877
|
Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered from foundational to advanced.`,
|
|
1741
1878
|
`IMPORTANT FOR VIETNAMESE TERMS: Keep multi-syllable Vietnamese compound terms intact (e.g. "Bi\u1EBFn tr\u1EA1ng th\xE1i", "C\u1EA3m bi\u1EBFn \xE1nh s\xE1ng", "V\xF2ng l\u1EB7p s\u1EF1 ki\u1EC7n") and pair them with accurate English canonical terms.`,
|
|
1742
|
-
`Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"], "facets": {"computationalPrinciple": "...", "mentalModel": "...", "syntaxAndTools": ["..."]}, "commonMisconceptions": ["..."]}`
|
|
1879
|
+
`Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"], "facets": {"computationalPrinciple": "...", "mentalModel": "...", "syntaxAndTools": ["..."]}, "commonMisconceptions": ["..."], "troubleshootingHints": ["2-minute quick fix for common student pitfall"]}`
|
|
1743
1880
|
].filter(Boolean).join("\n");
|
|
1744
1881
|
const rawResponse = await options.llm(prompt, {
|
|
1745
1882
|
system: "You are a Principal Computer Science Curriculum Architect & Bilingual Terminology Specialist (Vietnamese-English). Output strictly valid JSON array without markdown commentary.",
|
|
@@ -2014,7 +2151,15 @@ var CSTA_K12_STRANDS = {
|
|
|
2014
2151
|
nameVi: "Thu\u1EADt to\xE1n & L\u1EADp tr\xECnh",
|
|
2015
2152
|
description: "Algorithms, variables, control structures, modularity, and iterative program development.",
|
|
2016
2153
|
subconcepts: ["Algorithms", "Variables", "Control", "Modularity", "Program Development"]
|
|
2017
|
-
}
|
|
2154
|
+
},
|
|
2155
|
+
IC: {
|
|
2156
|
+
code: "IC",
|
|
2157
|
+
name: "Impacts of Computing",
|
|
2158
|
+
nameVi: "T\xE1c \u0111\u1ED9ng c\u1EE7a Tin h\u1ECDc & AI",
|
|
2159
|
+
description: "Culture, social interactions, safety, law, and ethics of computing and AI systems.",
|
|
2160
|
+
subconcepts: ["Culture", "Social Interactions", "Safety, Law, & Ethics"]
|
|
2161
|
+
}
|
|
2162
|
+
};
|
|
2018
2163
|
var CS2023_KNOWLEDGE_AREAS = {
|
|
2019
2164
|
AL: {
|
|
2020
2165
|
code: "AL",
|
|
@@ -2081,7 +2226,14 @@ var CS2023_KNOWLEDGE_AREAS = {
|
|
|
2081
2226
|
name: "Security & Cryptography",
|
|
2082
2227
|
description: "Authentication, cryptography, secure coding, threat modeling, and network defense.",
|
|
2083
2228
|
coreTopics: ["Symmetric & Asymmetric Encryption", "OAuth & JWT Auth", "Secure Coding & Vulnerabilities", "Identity & Access Management"]
|
|
2084
|
-
}
|
|
2229
|
+
},
|
|
2230
|
+
SPD: {
|
|
2231
|
+
code: "SPD",
|
|
2232
|
+
name: "Specialized Platform Development",
|
|
2233
|
+
description: "Web, mobile, cloud, IoT, and robotics platform engineering.",
|
|
2234
|
+
coreTopics: ["Mobile & Desktop App Lifecycle", "Web Full-Stack Architecture", "IoT & Edge Device Connectivity", "Platform SDK Integration"]
|
|
2235
|
+
}
|
|
2236
|
+
};
|
|
2085
2237
|
function mapTopicToCS2023(topic, context) {
|
|
2086
2238
|
const combined = [
|
|
2087
2239
|
topic,
|
|
@@ -2119,9 +2271,18 @@ function mapTopicToCS2023(topic, context) {
|
|
|
2119
2271
|
return { area: CS2023_KNOWLEDGE_AREAS.SE, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.88 };
|
|
2120
2272
|
}
|
|
2121
2273
|
if (/\b(algorithm|data structure|tree|graph|dynamic programming|sorting|thuật toán|cấu trúc dữ liệu|đồ thị|quy hoạch động)\b/.test(combined)) {
|
|
2122
|
-
return { area: CS2023_KNOWLEDGE_AREAS.AL, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.9 };
|
|
2274
|
+
return { area: CS2023_KNOWLEDGE_AREAS.AL, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.9, isCsApplicable: true };
|
|
2275
|
+
}
|
|
2276
|
+
const isExplicitNonCoding = /\b(noncoding|non-coding|phi lập trình|không lập trình|applied science|khoa học ứng dụng|general science|khoa học tự nhiên|applied-noncoding)\b/i.test(combined);
|
|
2277
|
+
if (isExplicitNonCoding) {
|
|
2278
|
+
return {
|
|
2279
|
+
area: CS2023_KNOWLEDGE_AREAS.SPD,
|
|
2280
|
+
k12Strand: CSTA_K12_STRANDS.IC,
|
|
2281
|
+
matchConfidence: 0.25,
|
|
2282
|
+
isCsApplicable: false
|
|
2283
|
+
};
|
|
2123
2284
|
}
|
|
2124
|
-
return { area: CS2023_KNOWLEDGE_AREAS.SDF, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.
|
|
2285
|
+
return { area: CS2023_KNOWLEDGE_AREAS.SDF, k12Strand: CSTA_K12_STRANDS.AP, matchConfidence: 0.45, isCsApplicable: true };
|
|
2125
2286
|
}
|
|
2126
2287
|
|
|
2127
2288
|
// src/academic/citationManager.ts
|
|
@@ -2614,35 +2775,68 @@ var TechnicalGroundingEngine = class {
|
|
|
2614
2775
|
try {
|
|
2615
2776
|
const doc = await this.scraper.scrapeUrl(h.url);
|
|
2616
2777
|
if (doc && doc.markdownContent) {
|
|
2778
|
+
const substantiveExcerpt = doc.keyExcerpts && doc.keyExcerpts.length > 0 ? doc.keyExcerpts.slice(0, 3).join("\n\n") : doc.markdownContent.slice(0, 1500);
|
|
2617
2779
|
return {
|
|
2618
2780
|
title: h.title,
|
|
2619
2781
|
url: h.url,
|
|
2620
|
-
excerpt:
|
|
2782
|
+
excerpt: substantiveExcerpt,
|
|
2783
|
+
codeBlocks: (doc.extractedCodeBlocks || []).slice(0, 2),
|
|
2784
|
+
tables: (doc.extractedTables || []).slice(0, 2),
|
|
2785
|
+
keyExcerpts: (doc.keyExcerpts || []).slice(0, 4)
|
|
2621
2786
|
};
|
|
2622
2787
|
}
|
|
2623
2788
|
} catch {
|
|
2624
2789
|
}
|
|
2625
|
-
return {
|
|
2790
|
+
return {
|
|
2791
|
+
title: h.title,
|
|
2792
|
+
url: h.url,
|
|
2793
|
+
excerpt: h.content.slice(0, 500),
|
|
2794
|
+
codeBlocks: [],
|
|
2795
|
+
tables: [],
|
|
2796
|
+
keyExcerpts: []
|
|
2797
|
+
};
|
|
2626
2798
|
})
|
|
2627
2799
|
);
|
|
2628
2800
|
const scrapedSnippets = scrapeResults.map(
|
|
2629
|
-
(res, idx) => res.status === "fulfilled" ? res.value : {
|
|
2801
|
+
(res, idx) => res.status === "fulfilled" ? res.value : {
|
|
2802
|
+
title: hitsToScrape[idx].title,
|
|
2803
|
+
url: hitsToScrape[idx].url,
|
|
2804
|
+
excerpt: hitsToScrape[idx].content.slice(0, 500),
|
|
2805
|
+
codeBlocks: [],
|
|
2806
|
+
tables: [],
|
|
2807
|
+
keyExcerpts: []
|
|
2808
|
+
}
|
|
2630
2809
|
);
|
|
2631
|
-
const
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2810
|
+
const preservedCodeSnippets = [];
|
|
2811
|
+
const preservedTables = [];
|
|
2812
|
+
const preservedKeyExcerpts = [];
|
|
2813
|
+
for (const s of scrapedSnippets) {
|
|
2814
|
+
if (s.codeBlocks && s.codeBlocks.length > 0) {
|
|
2815
|
+
preservedCodeSnippets.push(...s.codeBlocks);
|
|
2816
|
+
}
|
|
2817
|
+
if (s.tables && s.tables.length > 0) {
|
|
2818
|
+
preservedTables.push(...s.tables);
|
|
2819
|
+
}
|
|
2820
|
+
if (s.keyExcerpts && s.keyExcerpts.length > 0) {
|
|
2821
|
+
preservedKeyExcerpts.push(...s.keyExcerpts);
|
|
2822
|
+
}
|
|
2823
|
+
}
|
|
2824
|
+
const defaultExemplars = scrapedSnippets.length > 0 ? [
|
|
2637
2825
|
{
|
|
2638
|
-
title: `${topic}
|
|
2639
|
-
description:
|
|
2826
|
+
title: scrapedSnippets[0].title || `${topic} Practical Foundation`,
|
|
2827
|
+
description: scrapedSnippets[0].excerpt ? scrapedSnippets[0].excerpt.slice(0, 220) : `Hands-on implementation project for ${topic}.`,
|
|
2640
2828
|
targetAudience: `Students aged ${ageBand[0]}-${ageBand[1]}`,
|
|
2641
|
-
sourceUrl: scrapedSnippets[0]
|
|
2829
|
+
sourceUrl: scrapedSnippets[0].url,
|
|
2642
2830
|
hardwareRequirements: hardware ? [hardware] : void 0,
|
|
2643
2831
|
softwareRequirements: software ? [software] : void 0
|
|
2644
2832
|
}
|
|
2645
|
-
];
|
|
2833
|
+
] : [];
|
|
2834
|
+
const defaultBottlenecks = scrapedSnippets.length > 0 ? domain === "hardware" || Boolean(hardware) ? [
|
|
2835
|
+
"Power supply decoupling and current sufficiency when powering sensors or actuators.",
|
|
2836
|
+
"Logic level shifting and hardware pin mapping conflicts."
|
|
2837
|
+
] : [
|
|
2838
|
+
"SDK/runtime version compatibility and local environment dependencies."
|
|
2839
|
+
] : [];
|
|
2646
2840
|
let factsheet = {
|
|
2647
2841
|
topic,
|
|
2648
2842
|
domain,
|
|
@@ -2650,18 +2844,21 @@ var TechnicalGroundingEngine = class {
|
|
|
2650
2844
|
hardwarePlatform: hardware || void 0,
|
|
2651
2845
|
softwareEnv: software || void 0,
|
|
2652
2846
|
verifiedCompatible: true,
|
|
2653
|
-
notes: [`Targeting ${hardware || "General Platform"} with ${software || "Standard SDK"}.`]
|
|
2847
|
+
notes: hardware || software ? [`Targeting ${hardware || "General Platform"} with ${software || "Standard SDK"}.`] : [`Baseline technical verification for ${topic}.`]
|
|
2654
2848
|
},
|
|
2655
2849
|
ageFeasibility: {
|
|
2656
2850
|
recommendedMinAge: ageBand[0],
|
|
2657
2851
|
recommendedMaxAge: ageBand[1],
|
|
2658
2852
|
cognitiveReadinessVerdict: "feasible",
|
|
2659
|
-
potentialBarriers: [
|
|
2853
|
+
potentialBarriers: []
|
|
2660
2854
|
},
|
|
2661
2855
|
exemplarProjects: defaultExemplars,
|
|
2662
2856
|
technicalBottlenecks: defaultBottlenecks,
|
|
2663
2857
|
groundingSources: scrapedSnippets.map((s) => ({ title: s.title, url: s.url })),
|
|
2664
|
-
markdownReport: ""
|
|
2858
|
+
markdownReport: "",
|
|
2859
|
+
codeSnippets: preservedCodeSnippets.slice(0, 4),
|
|
2860
|
+
pinoutTables: preservedTables.slice(0, 4),
|
|
2861
|
+
keyExcerpts: preservedKeyExcerpts.slice(0, 6)
|
|
2665
2862
|
};
|
|
2666
2863
|
if (llm && scrapedSnippets.length > 0) {
|
|
2667
2864
|
try {
|
|
@@ -2670,8 +2867,19 @@ var TechnicalGroundingEngine = class {
|
|
|
2670
2867
|
`Hardware: ${hardware || "N/A"}, Software/Env: ${software || "N/A"}, Student Age Band: ${ageBand[0]}-${ageBand[1]} years old.`,
|
|
2671
2868
|
`
|
|
2672
2869
|
Harvested Sources:
|
|
2673
|
-
${scrapedSnippets.map((s, i) =>
|
|
2674
|
-
${s.
|
|
2870
|
+
${scrapedSnippets.map((s, i) => {
|
|
2871
|
+
const parts = [`[Source ${i + 1}: ${s.title} (${s.url})]
|
|
2872
|
+
${s.excerpt}`];
|
|
2873
|
+
if (s.codeBlocks && s.codeBlocks.length > 0) {
|
|
2874
|
+
parts.push("Code Snippets:\n" + s.codeBlocks.map((c) => `\`\`\`${c.language || ""}
|
|
2875
|
+
${c.code}
|
|
2876
|
+
\`\`\``).join("\n"));
|
|
2877
|
+
}
|
|
2878
|
+
if (s.tables && s.tables.length > 0) {
|
|
2879
|
+
parts.push("Tables/Pinouts:\n" + s.tables.join("\n\n"));
|
|
2880
|
+
}
|
|
2881
|
+
return parts.join("\n");
|
|
2882
|
+
}).join("\n\n")}`,
|
|
2675
2883
|
`
|
|
2676
2884
|
Synthesize a strictly valid JSON object with:`,
|
|
2677
2885
|
`{`,
|
|
@@ -2710,7 +2918,7 @@ Synthesize a strictly valid JSON object with:`,
|
|
|
2710
2918
|
if (Array.isArray(parsed.technicalBottlenecks)) factsheet.technicalBottlenecks = parsed.technicalBottlenecks.map(String);
|
|
2711
2919
|
}
|
|
2712
2920
|
} catch (err) {
|
|
2713
|
-
console.warn("[TechnicalGroundingEngine] LLM synthesis
|
|
2921
|
+
console.warn("[TechnicalGroundingEngine] LLM synthesis error:", err?.message || err);
|
|
2714
2922
|
}
|
|
2715
2923
|
}
|
|
2716
2924
|
factsheet.markdownReport = this.renderMarkdownReport(factsheet);
|
|
@@ -2742,6 +2950,22 @@ Synthesize a strictly valid JSON object with:`,
|
|
|
2742
2950
|
## 5. Grounding Citations`,
|
|
2743
2951
|
...fs.groundingSources.map((s, idx) => `${idx + 1}. [${s.title}](${s.url})`)
|
|
2744
2952
|
];
|
|
2953
|
+
if (fs.codeSnippets && fs.codeSnippets.length > 0) {
|
|
2954
|
+
lines.push(
|
|
2955
|
+
`
|
|
2956
|
+
## 6. Verified Code Snippets & Syntax Reference`,
|
|
2957
|
+
...fs.codeSnippets.map((c) => `\`\`\`${c.language || "text"}
|
|
2958
|
+
${c.code}
|
|
2959
|
+
\`\`\``)
|
|
2960
|
+
);
|
|
2961
|
+
}
|
|
2962
|
+
if (fs.pinoutTables && fs.pinoutTables.length > 0) {
|
|
2963
|
+
lines.push(
|
|
2964
|
+
`
|
|
2965
|
+
## 7. Pinout & Specification Tables`,
|
|
2966
|
+
...fs.pinoutTables
|
|
2967
|
+
);
|
|
2968
|
+
}
|
|
2745
2969
|
return lines.join("\n");
|
|
2746
2970
|
}
|
|
2747
2971
|
};
|