@thanh01.pmt/research-kit 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +213 -0
- package/dist/academic/index.cjs +332 -127
- package/dist/academic/index.cjs.map +1 -1
- package/dist/academic/index.d.cts +3 -3
- package/dist/academic/index.d.ts +3 -3
- package/dist/academic/index.js +2 -2
- package/dist/{catalogBuilder-8A-Dg_A-.d.ts → catalogBuilder-CmmQzUnf.d.ts} +11 -3
- package/dist/{catalogBuilder-feU79wQQ.d.cts → catalogBuilder-DgXYxL2F.d.cts} +11 -3
- package/dist/chunk-2JEVBCIF.js +3 -0
- package/dist/{chunk-KGCDYFFD.js.map → chunk-2JEVBCIF.js.map} +1 -1
- package/dist/chunk-2XIEWCUE.js +244 -0
- package/dist/chunk-2XIEWCUE.js.map +1 -0
- package/dist/chunk-AUGVVZTL.js +514 -0
- package/dist/chunk-AUGVVZTL.js.map +1 -0
- package/dist/chunk-DOXSPRSP.js +559 -0
- package/dist/chunk-DOXSPRSP.js.map +1 -0
- package/dist/chunk-GKF3NDEX.js +162 -0
- package/dist/chunk-GKF3NDEX.js.map +1 -0
- package/dist/chunk-J3VCGH7Z.js +3 -0
- package/dist/chunk-J3VCGH7Z.js.map +1 -0
- package/dist/{chunk-67V2GJGK.js → chunk-J7XKDMGL.js} +7 -5
- package/dist/chunk-J7XKDMGL.js.map +1 -0
- package/dist/chunk-KAFQVI2R.js +333 -0
- package/dist/chunk-KAFQVI2R.js.map +1 -0
- package/dist/chunk-XIF5YNI7.js +799 -0
- package/dist/chunk-XIF5YNI7.js.map +1 -0
- package/dist/config/index.cjs +5 -3
- package/dist/config/index.cjs.map +1 -1
- package/dist/config/index.d.cts +1 -1
- package/dist/config/index.d.ts +1 -1
- package/dist/config/index.js +1 -1
- package/dist/evidence/index.cjs +337 -21
- package/dist/evidence/index.cjs.map +1 -1
- package/dist/evidence/index.d.cts +44 -7
- package/dist/evidence/index.d.ts +44 -7
- package/dist/evidence/index.js +3 -2
- package/dist/harvester/index.cjs +805 -64
- package/dist/harvester/index.cjs.map +1 -1
- package/dist/harvester/index.d.cts +79 -8
- package/dist/harvester/index.d.ts +79 -8
- package/dist/harvester/index.js +3 -2
- package/dist/{index-BU5n1ip_.d.ts → index-t8pahlJ4.d.cts} +104 -15
- package/dist/{index-BU5n1ip_.d.cts → index-t8pahlJ4.d.ts} +104 -15
- package/dist/index.cjs +2001 -252
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +156 -8
- package/dist/index.d.ts +156 -8
- package/dist/index.js +237 -8
- package/dist/index.js.map +1 -1
- package/dist/literatureReview-BTPFe7eb.d.cts +64 -0
- package/dist/literatureReview-hGbfrBmw.d.ts +64 -0
- package/dist/pipeline/index.cjs +1570 -241
- package/dist/pipeline/index.cjs.map +1 -1
- package/dist/pipeline/index.d.cts +49 -9
- package/dist/pipeline/index.d.ts +49 -9
- package/dist/pipeline/index.js +5 -4
- package/dist/webScraper-BhiQ8s9x.d.ts +68 -0
- package/dist/webScraper-D9Ldus9p.d.cts +68 -0
- package/package.json +8 -9
- package/dist/chunk-47L7ZHMY.js +0 -316
- package/dist/chunk-47L7ZHMY.js.map +0 -1
- package/dist/chunk-63X2HPCI.js +0 -47
- package/dist/chunk-63X2HPCI.js.map +0 -1
- package/dist/chunk-67V2GJGK.js.map +0 -1
- package/dist/chunk-6Q2XCUG7.js +0 -40
- package/dist/chunk-6Q2XCUG7.js.map +0 -1
- package/dist/chunk-KGCDYFFD.js +0 -3
- package/dist/chunk-MHAYD555.js +0 -239
- package/dist/chunk-MHAYD555.js.map +0 -1
- package/dist/chunk-MPLDSHFM.js +0 -396
- package/dist/chunk-MPLDSHFM.js.map +0 -1
- package/dist/chunk-QZNLB5X4.js +0 -65
- package/dist/chunk-QZNLB5X4.js.map +0 -1
- package/dist/literatureReview-C4AgVljZ.d.ts +0 -43
- package/dist/literatureReview-Dlods4Aw.d.cts +0 -43
- package/dist/webScraper-pKzdXhfI.d.ts +0 -46
- package/dist/webScraper-yBcYOHoL.d.cts +0 -46
|
@@ -0,0 +1,799 @@
|
|
|
1
|
+
// src/harvester/domainClassifier.ts
|
|
2
|
+
var HARDWARE_PATTERNS = [
|
|
3
|
+
/\b(esp32|esp8266|arduino|raspberry\s*pi|stm32|micro:?bit|mblock|makecode|lego|spike\s*prime|mindstorms|ev3|wedo|vex)\b/i,
|
|
4
|
+
/\b(sensor|sensors|circuit|circuits|gpio|breadboard|microcontroller|firmware|embedded|iot|led|servo|ultrasonic|ldr|dht11|dht22|i2c|spi|uart|pwm|adc|actuator|robotics)\b/i,
|
|
5
|
+
/(cảm biến|vi điều khiển|mạch điện|linh kiện|điện tử|nhúng|động cơ servo|đèn led|bảng mạch)/i
|
|
6
|
+
];
|
|
7
|
+
var PLATFORM_PATTERNS = [
|
|
8
|
+
/\b(supabase|firebase|aws|amazon web services|gcp|google cloud|azure|cloudflare|vercel|netlify|docker|kubernetes|k8s|terraform|backend-as-a-service|baas|ci\/cd|github actions)\b/i,
|
|
9
|
+
/(điện toán đám mây|hạ tầng cloud|triển khai hệ thống|container hóa)/i
|
|
10
|
+
];
|
|
11
|
+
var ACADEMIC_PATTERNS = [
|
|
12
|
+
/\b(theorem|complexity|np-hard|np-complete|pedagogy|bloom|constructivism|formal verification|computational theory|automata|turing|lambda calculus|type theory|big-o|asymptotic)\b/i,
|
|
13
|
+
/(lý thuyết tính toán|độ phức tạp thuật toán|chứng minh hình thức|sư phạm|tải nhận thức)/i
|
|
14
|
+
];
|
|
15
|
+
function inferPrimaryCodeLanguage(topic, context, domain) {
|
|
16
|
+
const combined = [
|
|
17
|
+
topic,
|
|
18
|
+
context?.courseTitle,
|
|
19
|
+
context?.hardwarePlatform,
|
|
20
|
+
context?.softwareEnv,
|
|
21
|
+
...context?.inScopeTopics || []
|
|
22
|
+
].filter(Boolean).join(" ").toLowerCase();
|
|
23
|
+
if (/\b(swift|swiftui|xcode|playground)\b/.test(combined)) return "swift";
|
|
24
|
+
if (/\b(lua|roblox|lovr|love2d)\b/.test(combined)) return "lua";
|
|
25
|
+
if (/\b(python|pygame|jupyter|pandas|numpy|pytorch|tensorflow|micropython)\b/.test(combined)) return "python";
|
|
26
|
+
if (/\b(typescript|tsx|nextjs|next\.js|angular|nestjs)\b/.test(combined)) return "typescript";
|
|
27
|
+
if (/\b(javascript|nodejs|node\.js|react|vue|svelte|p5\.js|makecode)\b/.test(combined)) return "javascript";
|
|
28
|
+
if (/\b(c\+\+|cpp|arduino|ino|esp-idf|stm32)\b/.test(combined)) return "cpp";
|
|
29
|
+
if (/\b(c#|csharp|unity|\.net|dotnet)\b/.test(combined)) return "csharp";
|
|
30
|
+
if (/\b(gdscript|godot)\b/.test(combined)) return "gdscript";
|
|
31
|
+
if (/\b(java|spring|android|kotlin)\b/.test(combined)) return "java";
|
|
32
|
+
if (/\b(rust|cargo)\b/.test(combined)) return "rust";
|
|
33
|
+
if (/\b(golang|\bgo\b)\b/.test(combined)) return "go";
|
|
34
|
+
if (/\b(sql|postgres|postgresql|mysql|sqlite)\b/.test(combined)) return "sql";
|
|
35
|
+
if (/\b(scratch|mblock|blockly)\b/.test(combined)) return "text";
|
|
36
|
+
if (domain === "hardware") return "cpp";
|
|
37
|
+
return "text";
|
|
38
|
+
}
|
|
39
|
+
function classifyTechnicalDomain(topic, explicitDomain, context) {
|
|
40
|
+
if (explicitDomain && ["hardware", "software", "platform", "academic"].includes(explicitDomain.toLowerCase())) {
|
|
41
|
+
const domain = explicitDomain.toLowerCase();
|
|
42
|
+
return {
|
|
43
|
+
domain,
|
|
44
|
+
confidence: 1,
|
|
45
|
+
rationale: "Explicitly configured by user flag.",
|
|
46
|
+
recommendedExtractors: getExtractorsForDomain(domain),
|
|
47
|
+
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, domain)
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
const combined = [
|
|
51
|
+
topic,
|
|
52
|
+
context?.courseTitle,
|
|
53
|
+
context?.positioning,
|
|
54
|
+
context?.hardwarePlatform,
|
|
55
|
+
context?.softwareEnv,
|
|
56
|
+
...context?.inScopeTopics || []
|
|
57
|
+
].filter(Boolean).join(" ");
|
|
58
|
+
if (HARDWARE_PATTERNS.some((re) => re.test(combined))) {
|
|
59
|
+
return {
|
|
60
|
+
domain: "hardware",
|
|
61
|
+
confidence: 0.95,
|
|
62
|
+
rationale: "Topic or context contains microcontrollers, sensors, robotics kits, or physical electronics keywords.",
|
|
63
|
+
recommendedExtractors: getExtractorsForDomain("hardware"),
|
|
64
|
+
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "hardware")
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
if (PLATFORM_PATTERNS.some((re) => re.test(combined))) {
|
|
68
|
+
return {
|
|
69
|
+
domain: "platform",
|
|
70
|
+
confidence: 0.9,
|
|
71
|
+
rationale: "Topic represents a cloud infrastructure, hosting, DevOps, or BaaS platform.",
|
|
72
|
+
recommendedExtractors: getExtractorsForDomain("platform"),
|
|
73
|
+
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "platform")
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
if (ACADEMIC_PATTERNS.some((re) => re.test(combined))) {
|
|
77
|
+
return {
|
|
78
|
+
domain: "academic",
|
|
79
|
+
confidence: 0.9,
|
|
80
|
+
rationale: "Topic focuses on theoretical foundations, pedagogical models, or formal proofs.",
|
|
81
|
+
recommendedExtractors: getExtractorsForDomain("academic"),
|
|
82
|
+
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "academic")
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
return {
|
|
86
|
+
domain: "software",
|
|
87
|
+
confidence: 0.85,
|
|
88
|
+
rationale: "Topic is a programming language, game engine, client/server library, or software application.",
|
|
89
|
+
recommendedExtractors: getExtractorsForDomain("software"),
|
|
90
|
+
primaryCodeLanguage: inferPrimaryCodeLanguage(topic, context, "software")
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
function getExtractorsForDomain(domain) {
|
|
94
|
+
switch (domain) {
|
|
95
|
+
case "hardware":
|
|
96
|
+
return ["PinoutTableExtractor", "DatasheetSpecsExtractor", "CircuitSchematicExtractor"];
|
|
97
|
+
case "platform":
|
|
98
|
+
return ["ArchitectureDiagramExtractor", "CLICommandExtractor", "SDKIntegrationExtractor"];
|
|
99
|
+
case "academic":
|
|
100
|
+
return ["SemanticScholarExtractor", "ArXivExtractor", "TheoremFormulaExtractor"];
|
|
101
|
+
case "software":
|
|
102
|
+
default:
|
|
103
|
+
return ["APISignatureExtractor", "CodeExampleExtractor", "DependencyTreeExtractor"];
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// src/harvester/multiSourceSearch.ts
|
|
108
|
+
var MultiSourceSearchEngine = class {
|
|
109
|
+
tavilyKey;
|
|
110
|
+
exaKey;
|
|
111
|
+
serperKey;
|
|
112
|
+
allowEnvFallback;
|
|
113
|
+
customAdapter;
|
|
114
|
+
constructor(options = {}) {
|
|
115
|
+
this.allowEnvFallback = options.allowEnvFallback ?? true;
|
|
116
|
+
this.tavilyKey = options.apiKeys?.tavily || options.tavilyKey || (this.allowEnvFallback ? process.env.TAVILY_API_KEY : void 0);
|
|
117
|
+
this.exaKey = options.apiKeys?.exa || options.exaKey || (this.allowEnvFallback ? process.env.EXA_API_KEY : void 0);
|
|
118
|
+
this.serperKey = options.apiKeys?.serper || options.serperKey || (this.allowEnvFallback ? process.env.SERPER_API_KEY : void 0);
|
|
119
|
+
this.customAdapter = options.customAdapter;
|
|
120
|
+
}
|
|
121
|
+
setCustomAdapter(adapter) {
|
|
122
|
+
this.customAdapter = adapter;
|
|
123
|
+
}
|
|
124
|
+
setApiKeys(keys) {
|
|
125
|
+
if (keys.tavily !== void 0) this.tavilyKey = keys.tavily;
|
|
126
|
+
if (keys.exa !== void 0) this.exaKey = keys.exa;
|
|
127
|
+
if (keys.serper !== void 0) this.serperKey = keys.serper;
|
|
128
|
+
}
|
|
129
|
+
async search(query, options = {}) {
|
|
130
|
+
const limit = options.limit ?? 5;
|
|
131
|
+
const adapter = options.customAdapter || this.customAdapter;
|
|
132
|
+
const preferred = options.preferredEngine;
|
|
133
|
+
if (adapter && (!preferred || preferred === "custom")) {
|
|
134
|
+
try {
|
|
135
|
+
const customResults = await adapter.search(query, { limit, domainFilter: options.domainFilter });
|
|
136
|
+
if (Array.isArray(customResults) && customResults.length > 0) {
|
|
137
|
+
return customResults.map((r) => ({
|
|
138
|
+
title: r.title,
|
|
139
|
+
url: r.url,
|
|
140
|
+
content: r.content,
|
|
141
|
+
score: r.score,
|
|
142
|
+
engine: "custom"
|
|
143
|
+
}));
|
|
144
|
+
}
|
|
145
|
+
} catch (e) {
|
|
146
|
+
console.warn("[MultiSourceSearchEngine] Custom search adapter failed:", e?.message || e);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
const tavilyKey = options.apiKeys?.tavily || options.tavilyKey || this.tavilyKey;
|
|
150
|
+
const exaKey = options.apiKeys?.exa || options.exaKey || this.exaKey;
|
|
151
|
+
const serperKey = options.apiKeys?.serper || options.serperKey || this.serperKey;
|
|
152
|
+
if (preferred === "exa" && exaKey) {
|
|
153
|
+
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
|
|
154
|
+
if (exaHits.length > 0) return exaHits;
|
|
155
|
+
}
|
|
156
|
+
if (preferred === "serper" && serperKey) {
|
|
157
|
+
const serperHits = await this.querySerper(query, serperKey, limit);
|
|
158
|
+
if (serperHits.length > 0) return serperHits;
|
|
159
|
+
}
|
|
160
|
+
if (preferred === "duckduckgo") {
|
|
161
|
+
const ddgHits2 = await this.queryDuckDuckGo(query, limit);
|
|
162
|
+
if (ddgHits2.length > 0) return ddgHits2;
|
|
163
|
+
}
|
|
164
|
+
if (tavilyKey && (!preferred || preferred === "tavily")) {
|
|
165
|
+
try {
|
|
166
|
+
const body = {
|
|
167
|
+
api_key: tavilyKey,
|
|
168
|
+
query,
|
|
169
|
+
search_depth: "advanced",
|
|
170
|
+
include_answer: true,
|
|
171
|
+
max_results: limit
|
|
172
|
+
};
|
|
173
|
+
if (options.domainFilter && options.domainFilter.length > 0) {
|
|
174
|
+
body.include_domains = options.domainFilter;
|
|
175
|
+
}
|
|
176
|
+
const res = await fetch("https://api.tavily.com/search", {
|
|
177
|
+
method: "POST",
|
|
178
|
+
headers: { "Content-Type": "application/json" },
|
|
179
|
+
body: JSON.stringify(body),
|
|
180
|
+
signal: AbortSignal.timeout(12e3)
|
|
181
|
+
});
|
|
182
|
+
if (res.ok) {
|
|
183
|
+
const data = await res.json();
|
|
184
|
+
if (Array.isArray(data?.results) && data.results.length > 0) {
|
|
185
|
+
return data.results.map((r) => ({
|
|
186
|
+
title: r.title || "Untitled",
|
|
187
|
+
url: r.url,
|
|
188
|
+
content: r.content || "",
|
|
189
|
+
score: r.score,
|
|
190
|
+
engine: "tavily"
|
|
191
|
+
}));
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
} catch (e) {
|
|
195
|
+
console.warn("[MultiSourceSearchEngine] Tavily attempt failed:", e?.message || e);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
if (exaKey) {
|
|
199
|
+
const exaHits = await this.queryExa(query, exaKey, limit, options.domainFilter);
|
|
200
|
+
if (exaHits.length > 0) return exaHits;
|
|
201
|
+
}
|
|
202
|
+
if (serperKey) {
|
|
203
|
+
const serperHits = await this.querySerper(query, serperKey, limit);
|
|
204
|
+
if (serperHits.length > 0) return serperHits;
|
|
205
|
+
}
|
|
206
|
+
const ddgHits = await this.queryDuckDuckGo(query, limit);
|
|
207
|
+
if (ddgHits.length > 0) return ddgHits;
|
|
208
|
+
return [];
|
|
209
|
+
}
|
|
210
|
+
async queryExa(query, exaKey, limit, domainFilter) {
|
|
211
|
+
try {
|
|
212
|
+
const body = {
|
|
213
|
+
query,
|
|
214
|
+
numResults: limit,
|
|
215
|
+
useAutoprompt: true
|
|
216
|
+
};
|
|
217
|
+
if (domainFilter && domainFilter.length > 0) {
|
|
218
|
+
body.includeDomains = domainFilter;
|
|
219
|
+
}
|
|
220
|
+
const res = await fetch("https://api.exa.ai/search", {
|
|
221
|
+
method: "POST",
|
|
222
|
+
headers: {
|
|
223
|
+
"x-api-key": exaKey,
|
|
224
|
+
"Content-Type": "application/json"
|
|
225
|
+
},
|
|
226
|
+
body: JSON.stringify(body),
|
|
227
|
+
signal: AbortSignal.timeout(1e4)
|
|
228
|
+
});
|
|
229
|
+
if (res.ok) {
|
|
230
|
+
const data = await res.json();
|
|
231
|
+
if (Array.isArray(data?.results) && data.results.length > 0) {
|
|
232
|
+
return data.results.map((r) => ({
|
|
233
|
+
title: r.title || "Untitled",
|
|
234
|
+
url: r.url,
|
|
235
|
+
content: r.text || r.snippet || "",
|
|
236
|
+
score: r.score,
|
|
237
|
+
engine: "exa"
|
|
238
|
+
}));
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
} catch (e) {
|
|
242
|
+
console.warn("[MultiSourceSearchEngine] Exa attempt failed:", e?.message || e);
|
|
243
|
+
}
|
|
244
|
+
return [];
|
|
245
|
+
}
|
|
246
|
+
async querySerper(query, serperKey, limit) {
|
|
247
|
+
try {
|
|
248
|
+
const res = await fetch("https://google.serper.dev/search", {
|
|
249
|
+
method: "POST",
|
|
250
|
+
headers: {
|
|
251
|
+
"X-API-KEY": serperKey,
|
|
252
|
+
"Content-Type": "application/json"
|
|
253
|
+
},
|
|
254
|
+
body: JSON.stringify({ q: query, num: limit }),
|
|
255
|
+
signal: AbortSignal.timeout(1e4)
|
|
256
|
+
});
|
|
257
|
+
if (res.ok) {
|
|
258
|
+
const data = await res.json();
|
|
259
|
+
if (Array.isArray(data?.organic) && data.organic.length > 0) {
|
|
260
|
+
return data.organic.slice(0, limit).map((r) => ({
|
|
261
|
+
title: r.title || "Untitled",
|
|
262
|
+
url: r.link,
|
|
263
|
+
content: r.snippet || "",
|
|
264
|
+
engine: "serper"
|
|
265
|
+
}));
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
} catch (e) {
|
|
269
|
+
console.warn("[MultiSourceSearchEngine] Serper attempt failed:", e?.message || e);
|
|
270
|
+
}
|
|
271
|
+
return [];
|
|
272
|
+
}
|
|
273
|
+
async queryDuckDuckGo(query, limit) {
|
|
274
|
+
try {
|
|
275
|
+
const ddgUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
|
|
276
|
+
const res = await fetch(ddgUrl, {
|
|
277
|
+
headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" },
|
|
278
|
+
signal: AbortSignal.timeout(8e3)
|
|
279
|
+
});
|
|
280
|
+
if (res.ok) {
|
|
281
|
+
const html = await res.text();
|
|
282
|
+
const hits = [];
|
|
283
|
+
const resultRegex = /<a[^>]*class="result__a"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a class="result__url" href="([^"]+)">[\s\S]*?<a class="result__snippet[^>]*>([\s\S]*?)<\/a>/g;
|
|
284
|
+
let match;
|
|
285
|
+
while ((match = resultRegex.exec(html)) !== null && hits.length < limit) {
|
|
286
|
+
const rawTitle = match[1]?.replace(/<[^>]+>/g, "").trim() || query;
|
|
287
|
+
const rawUrl = match[2]?.trim();
|
|
288
|
+
const cleanUrl = rawUrl?.startsWith("//duckduckgo.com/l/?uddg=") ? decodeURIComponent(rawUrl.split("uddg=")[1]?.split("&")[0] || "") : rawUrl;
|
|
289
|
+
const snippet = match[3]?.replace(/<[^>]+>/g, "").trim() || "";
|
|
290
|
+
if (cleanUrl && !cleanUrl.includes("duckduckgo.com")) {
|
|
291
|
+
hits.push({
|
|
292
|
+
title: rawTitle,
|
|
293
|
+
url: cleanUrl,
|
|
294
|
+
content: snippet,
|
|
295
|
+
engine: "duckduckgo"
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
if (hits.length > 0) return hits;
|
|
300
|
+
}
|
|
301
|
+
} catch (e) {
|
|
302
|
+
console.warn("[MultiSourceSearchEngine] DuckDuckGo attempt failed:", e?.message || e);
|
|
303
|
+
}
|
|
304
|
+
return [];
|
|
305
|
+
}
|
|
306
|
+
};
|
|
307
|
+
|
|
308
|
+
// src/harvester/termExtractor.ts
|
|
309
|
+
var VIETNAMESE_STOPWORDS = /* @__PURE__ */ new Set([
|
|
310
|
+
"l\xE0",
|
|
311
|
+
"c\u1EE7a",
|
|
312
|
+
"trong",
|
|
313
|
+
"\u0111\u01B0\u1EE3c",
|
|
314
|
+
"nh\u1EEFng",
|
|
315
|
+
"c\xE1c",
|
|
316
|
+
"cho",
|
|
317
|
+
"v\u1EDBi",
|
|
318
|
+
"khi",
|
|
319
|
+
"n\u1EBFu",
|
|
320
|
+
"th\xEC",
|
|
321
|
+
"v\xE0",
|
|
322
|
+
"ho\u1EB7c",
|
|
323
|
+
"\u0111\u1EC3",
|
|
324
|
+
"t\u1EEB",
|
|
325
|
+
"\u0111\u1EBFn",
|
|
326
|
+
"v\u1EC1",
|
|
327
|
+
"n\xE0y",
|
|
328
|
+
"\u0111\xF3",
|
|
329
|
+
"m\u1ED9t",
|
|
330
|
+
"nh\u01B0",
|
|
331
|
+
"s\u1EBD",
|
|
332
|
+
"\u0111ang",
|
|
333
|
+
"\u0111\xE3",
|
|
334
|
+
"kh\xF4ng",
|
|
335
|
+
"c\xF3",
|
|
336
|
+
"ra",
|
|
337
|
+
"v\xE0o",
|
|
338
|
+
"b\u1EB1ng",
|
|
339
|
+
"theo",
|
|
340
|
+
"tr\xEAn",
|
|
341
|
+
"d\u01B0\u1EDBi",
|
|
342
|
+
"qua",
|
|
343
|
+
"nh\u1EDD",
|
|
344
|
+
"v\xEC",
|
|
345
|
+
"do",
|
|
346
|
+
"t\u1EA1i",
|
|
347
|
+
"b\u1EDFi",
|
|
348
|
+
"nh\u01B0ng",
|
|
349
|
+
"m\xE0",
|
|
350
|
+
"c\u0169ng",
|
|
351
|
+
"v\u1EABn",
|
|
352
|
+
"r\u1EA5t",
|
|
353
|
+
"qu\xE1",
|
|
354
|
+
"h\u01A1n",
|
|
355
|
+
"nh\u1EA5t",
|
|
356
|
+
"c\u1EA7n",
|
|
357
|
+
"ph\u1EA3i",
|
|
358
|
+
"n\xEAn",
|
|
359
|
+
"th\u1EC3",
|
|
360
|
+
"vi\u1EC7c",
|
|
361
|
+
"c\xE1ch",
|
|
362
|
+
"ph\u1EA7n",
|
|
363
|
+
"b\xE0i",
|
|
364
|
+
"bu\u1ED5i",
|
|
365
|
+
"h\u1ECDc",
|
|
366
|
+
"sinh",
|
|
367
|
+
"gi\xE1o",
|
|
368
|
+
"vi\xEAn",
|
|
369
|
+
"ng\u01B0\u1EDDi",
|
|
370
|
+
"d\xF9ng",
|
|
371
|
+
"ch\xFAng",
|
|
372
|
+
"ta",
|
|
373
|
+
"b\u1EA1n",
|
|
374
|
+
"em",
|
|
375
|
+
"gi\xFAp",
|
|
376
|
+
"l\xE0m",
|
|
377
|
+
"t\u1EA1o",
|
|
378
|
+
"s\u1EED",
|
|
379
|
+
"d\u1EE5ng",
|
|
380
|
+
"th\u1EF1c",
|
|
381
|
+
"hi\u1EC7n",
|
|
382
|
+
"hi\u1EC3u",
|
|
383
|
+
"bi\u1EBFt",
|
|
384
|
+
"xem",
|
|
385
|
+
"th\xEAm",
|
|
386
|
+
"sau",
|
|
387
|
+
"tr\u01B0\u1EDBc"
|
|
388
|
+
]);
|
|
389
|
+
var ENGLISH_STOPWORDS = /* @__PURE__ */ new Set([
|
|
390
|
+
"the",
|
|
391
|
+
"and",
|
|
392
|
+
"for",
|
|
393
|
+
"with",
|
|
394
|
+
"that",
|
|
395
|
+
"this",
|
|
396
|
+
"from",
|
|
397
|
+
"into",
|
|
398
|
+
"your",
|
|
399
|
+
"you",
|
|
400
|
+
"are",
|
|
401
|
+
"was",
|
|
402
|
+
"were",
|
|
403
|
+
"have",
|
|
404
|
+
"has",
|
|
405
|
+
"had",
|
|
406
|
+
"will",
|
|
407
|
+
"would",
|
|
408
|
+
"can",
|
|
409
|
+
"could",
|
|
410
|
+
"should",
|
|
411
|
+
"about",
|
|
412
|
+
"more",
|
|
413
|
+
"other",
|
|
414
|
+
"such",
|
|
415
|
+
"than",
|
|
416
|
+
"then",
|
|
417
|
+
"them",
|
|
418
|
+
"these",
|
|
419
|
+
"those",
|
|
420
|
+
"their",
|
|
421
|
+
"what",
|
|
422
|
+
"when",
|
|
423
|
+
"where",
|
|
424
|
+
"which",
|
|
425
|
+
"while",
|
|
426
|
+
"who",
|
|
427
|
+
"how",
|
|
428
|
+
"why",
|
|
429
|
+
"each",
|
|
430
|
+
"some",
|
|
431
|
+
"many",
|
|
432
|
+
"most",
|
|
433
|
+
"also",
|
|
434
|
+
"only",
|
|
435
|
+
"just",
|
|
436
|
+
"very",
|
|
437
|
+
"using",
|
|
438
|
+
"used",
|
|
439
|
+
"use",
|
|
440
|
+
"overview",
|
|
441
|
+
"introduction",
|
|
442
|
+
"getting",
|
|
443
|
+
"started",
|
|
444
|
+
"guide",
|
|
445
|
+
"tutorial",
|
|
446
|
+
"example",
|
|
447
|
+
"examples",
|
|
448
|
+
"documentation"
|
|
449
|
+
]);
|
|
450
|
+
var VIETNAMESE_TECHNICAL_HEADS = /^(cảm biến|vi điều khiển|vòng lặp|biến|hàm|mảng|chuỗi|đối tượng|lớp|phương thức|thuật toán|cấu trúc|mạch điện|tín hiệu|động cơ|đèn led|trạng thái|sự kiện|luồng|bất đồng bộ|giao diện|cơ sở dữ liệu|truy vấn|bảo mật|mã hóa|mạng|giao thức|khái niệm|mô hình|kiến trúc|kiểm thử|gỡ lỗi)\b/i;
|
|
451
|
+
var SYNTAX_OR_API_PATTERN = /(?:`[^`]+`|\b[a-z]+[A-Z][a-zA-Z0-9]*\b|\b[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+\b|@[A-Za-z]+|\b[a-z_]+\(\)|\b(?:pinMode|digitalWrite|digitalRead|analogRead|analogWrite|async|await|actor|struct|class|func|def|let|const|var)\b)/;
|
|
452
|
+
var HARDWARE_COMPONENT_PATTERN = /\b(esp32|arduino|micro:?bit|raspberry|sensor|led|servo|ultrasonic|ldr|dht11|dht22|breadboard|gpio|pwm|i2c|spi|uart|cảm biến|động cơ|mạch|đèn led|vi điều khiển)\b/i;
|
|
453
|
+
function normalizeTermKey(raw) {
|
|
454
|
+
return raw.normalize("NFC").toLowerCase().replace(/[`*_~#"']/g, "").replace(/\s+/g, " ").trim();
|
|
455
|
+
}
|
|
456
|
+
function cleanDisplayTerm(raw) {
|
|
457
|
+
return raw.normalize("NFC").replace(/^[-*•#\d.)\s]+/, "").replace(/[`*_~]+/g, "").replace(/\s+/g, " ").trim();
|
|
458
|
+
}
|
|
459
|
+
function isNoiseTerm(term) {
|
|
460
|
+
const clean = normalizeTermKey(term);
|
|
461
|
+
if (clean.length < 2 || clean.length > 65) return true;
|
|
462
|
+
if (/^\d+$/.test(clean)) return true;
|
|
463
|
+
if (/^https?:\/\//i.test(clean)) return true;
|
|
464
|
+
if (ENGLISH_STOPWORDS.has(clean) || VIETNAMESE_STOPWORDS.has(clean)) return true;
|
|
465
|
+
const words = clean.split(" ");
|
|
466
|
+
if (words.length > 6) return true;
|
|
467
|
+
if (words.length === 1 && clean.length < 3 && !/^(ai|ui|ux|os|io|db|id|ip)$/i.test(clean)) return true;
|
|
468
|
+
return false;
|
|
469
|
+
}
|
|
470
|
+
function classifyMarrAndCategory(term, domain) {
|
|
471
|
+
if (HARDWARE_COMPONENT_PATTERN.test(term)) {
|
|
472
|
+
return { category: "hardware_component", marrLayer: "CIO", bloomLevel: "apply" };
|
|
473
|
+
}
|
|
474
|
+
if (SYNTAX_OR_API_PATTERN.test(term)) {
|
|
475
|
+
return { category: "api_syntax", marrLayer: "SIO", bloomLevel: "apply" };
|
|
476
|
+
}
|
|
477
|
+
if (/\b(architecture|pattern|model|design|debugging|verification|mô hình|kiến trúc|tư duy|quy trình)\b/i.test(term)) {
|
|
478
|
+
return { category: "pedagogical_pattern", marrLayer: "ULO", bloomLevel: "analyze" };
|
|
479
|
+
}
|
|
480
|
+
return {
|
|
481
|
+
category: "core_concept",
|
|
482
|
+
marrLayer: domain === "academic" ? "ULO" : "CIO",
|
|
483
|
+
bloomLevel: "understand"
|
|
484
|
+
};
|
|
485
|
+
}
|
|
486
|
+
function findSentenceDefinition(term, markdown) {
|
|
487
|
+
const clean = cleanDisplayTerm(term);
|
|
488
|
+
const escaped = clean.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
489
|
+
const sentences = markdown.replace(/```[\s\S]*?```/g, " ").split(/(?<=[.!?。\n])\s+/).map((s) => s.replace(/^[-*•#>\s]+/, "").trim()).filter((s) => s.length >= 25 && s.length <= 280);
|
|
490
|
+
const re = new RegExp(escaped, "i");
|
|
491
|
+
const match = sentences.find((s) => re.test(s));
|
|
492
|
+
return match || `Core concept in ${clean} extracted from primary technical reference.`;
|
|
493
|
+
}
|
|
494
|
+
var PedagogicalTermExtractor = class {
|
|
495
|
+
extractDeterministic(topic, domain, docs, maxTerms = 15) {
|
|
496
|
+
const candidates = /* @__PURE__ */ new Map();
|
|
497
|
+
let orderCounter = 0;
|
|
498
|
+
const upsertCandidate = (rawTerm, weight, resourceId, markdown, pair) => {
|
|
499
|
+
const display = cleanDisplayTerm(rawTerm);
|
|
500
|
+
if (isNoiseTerm(display)) return;
|
|
501
|
+
const key = normalizeTermKey(display);
|
|
502
|
+
const existing = candidates.get(key);
|
|
503
|
+
if (existing) {
|
|
504
|
+
existing.score += weight;
|
|
505
|
+
existing.sourceIds.add(resourceId);
|
|
506
|
+
if (pair?.termVi && !existing.termVi) existing.termVi = pair.termVi;
|
|
507
|
+
if (pair?.termEn && !existing.termEn) existing.termEn = pair.termEn;
|
|
508
|
+
} else {
|
|
509
|
+
candidates.set(key, {
|
|
510
|
+
display,
|
|
511
|
+
termVi: pair?.termVi,
|
|
512
|
+
termEn: pair?.termEn,
|
|
513
|
+
score: weight,
|
|
514
|
+
firstSeenOrder: orderCounter++,
|
|
515
|
+
sourceIds: /* @__PURE__ */ new Set([resourceId]),
|
|
516
|
+
definition: findSentenceDefinition(display, markdown)
|
|
517
|
+
});
|
|
518
|
+
}
|
|
519
|
+
};
|
|
520
|
+
for (const { resourceId, doc } of docs) {
|
|
521
|
+
const md = doc.markdownContent || "";
|
|
522
|
+
const mdWithoutCode = md.replace(/```[\s\S]*?```/g, "\n");
|
|
523
|
+
const bilingualMatches = mdWithoutCode.matchAll(
|
|
524
|
+
/([A-ZÀÁẠẢÃÂẦẤẬẨẪĂẰẮẶẲẴÈÉẸẺẼÊỀẾỆỂỄÌÍỊỈĨÒÓỌỎÕÔỒỐỘỔỖƠỜỚỢỞỠÙÚỤỦŨƯỪỨỰỬỮỲÝỴỶỸĐ][a-zA-Z0-9\u00C0-\u1EF9\s/-]{2,40}?)\s*\(([A-Za-z0-9\u00C0-\u1EF9\s/_:-]{2,40})\)/g
|
|
525
|
+
);
|
|
526
|
+
for (const m of bilingualMatches) {
|
|
527
|
+
const left = cleanDisplayTerm(m[1] || "");
|
|
528
|
+
const right = cleanDisplayTerm(m[2] || "");
|
|
529
|
+
if (!isNoiseTerm(left) && !isNoiseTerm(right)) {
|
|
530
|
+
const leftHasVi = /[\u00C0-\u1EF9]/.test(left);
|
|
531
|
+
const rightHasVi = /[\u00C0-\u1EF9]/.test(right);
|
|
532
|
+
if (leftHasVi && !rightHasVi) {
|
|
533
|
+
upsertCandidate(`${left} (${right})`, 6, resourceId, mdWithoutCode, { termVi: left, termEn: right });
|
|
534
|
+
} else if (!leftHasVi && rightHasVi) {
|
|
535
|
+
upsertCandidate(`${right} (${left})`, 6, resourceId, mdWithoutCode, { termVi: right, termEn: left });
|
|
536
|
+
} else {
|
|
537
|
+
upsertCandidate(left, 4, resourceId, mdWithoutCode);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
for (const heading of doc.extractedHeadings || []) {
|
|
542
|
+
const cleaned = cleanDisplayTerm(heading.replace(/^\d+(\.\d+)*\s*/, ""));
|
|
543
|
+
if (!isNoiseTerm(cleaned)) {
|
|
544
|
+
upsertCandidate(cleaned, 4, resourceId, mdWithoutCode);
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
for (const m of mdWithoutCode.matchAll(/\*\*([^*\n]{3,45})\*\*/g)) {
|
|
548
|
+
upsertCandidate(m[1] || "", 3.5, resourceId, mdWithoutCode);
|
|
549
|
+
}
|
|
550
|
+
for (const m of mdWithoutCode.matchAll(/`([a-zA-Z_@][a-zA-Z0-9_.:()-]{2,35})`/g)) {
|
|
551
|
+
upsertCandidate(m[1] || "", 3, resourceId, mdWithoutCode);
|
|
552
|
+
}
|
|
553
|
+
for (const line of mdWithoutCode.split("\n")) {
|
|
554
|
+
const clauses = line.split(/[.,;:!?()[\]{}"“”‘’/\\|]+/);
|
|
555
|
+
for (const clause of clauses) {
|
|
556
|
+
const trimmed = clause.trim();
|
|
557
|
+
const viMatch = trimmed.match(
|
|
558
|
+
/\b((?:cảm biến|vi điều khiển|vòng lặp|biến|hàm|mảng|đối tượng|thuật toán|cấu trúc|mạch điện|tín hiệu|động cơ|đèn led|trạng thái|sự kiện|luồng|bất đồng bộ|giao diện|cơ sở dữ liệu|truy vấn|bảo mật|mã hóa|giao thức|mô hình|kiến trúc|kiểm thử|gỡ lỗi)(?:\s+[a-zA-Z0-9\u00C0-\u1EF9_-]+){1,3})\b/i
|
|
559
|
+
);
|
|
560
|
+
if (viMatch?.[1]) {
|
|
561
|
+
const tokens = viMatch[1].trim().split(/\s+/);
|
|
562
|
+
while (tokens.length > 1 && VIETNAMESE_STOPWORDS.has(tokens[tokens.length - 1].toLowerCase())) {
|
|
563
|
+
tokens.pop();
|
|
564
|
+
}
|
|
565
|
+
const phrase = tokens.join(" ");
|
|
566
|
+
if (tokens.length >= 2 && VIETNAMESE_TECHNICAL_HEADS.test(phrase)) {
|
|
567
|
+
upsertCandidate(phrase, 3.5, resourceId, mdWithoutCode, { termVi: phrase });
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
const sorted = [...candidates.values()].sort((a, b) => b.score - a.score || a.firstSeenOrder - b.firstSeenOrder).slice(0, maxTerms);
|
|
574
|
+
const orderedByAppearance = [...sorted].sort((a, b) => a.firstSeenOrder - b.firstSeenOrder);
|
|
575
|
+
const maxScore = Math.max(1, sorted[0]?.score || 1);
|
|
576
|
+
const concepts = orderedByAppearance.map((item, idx) => {
|
|
577
|
+
const id = `C_${String(idx + 1).padStart(2, "0")}`;
|
|
578
|
+
const { category, marrLayer, bloomLevel } = classifyMarrAndCategory(item.display, domain);
|
|
579
|
+
const prerequisites = [];
|
|
580
|
+
return {
|
|
581
|
+
id,
|
|
582
|
+
term: item.display,
|
|
583
|
+
termVi: item.termVi,
|
|
584
|
+
termEn: item.termEn,
|
|
585
|
+
category,
|
|
586
|
+
marrLayer,
|
|
587
|
+
bloomLevel,
|
|
588
|
+
salienceScore: Number(Math.min(10, Math.max(1, item.score / maxScore * 10)).toFixed(1)),
|
|
589
|
+
definition: item.definition,
|
|
590
|
+
prerequisites,
|
|
591
|
+
sourceIds: [...item.sourceIds]
|
|
592
|
+
};
|
|
593
|
+
});
|
|
594
|
+
return {
|
|
595
|
+
concepts,
|
|
596
|
+
conceptDagTsv: this.formatConceptDagTsv(concepts),
|
|
597
|
+
isLlmRefined: false,
|
|
598
|
+
unrefinedCandidatesCount: candidates.size
|
|
599
|
+
};
|
|
600
|
+
}
|
|
601
|
+
async extractWithOptionalLlm(topic, domain, docs, options = {}) {
|
|
602
|
+
const detBundle = this.extractDeterministic(topic, domain, docs, options.maxTerms ?? 15);
|
|
603
|
+
if (!options.llm || docs.length === 0) {
|
|
604
|
+
return detBundle;
|
|
605
|
+
}
|
|
606
|
+
try {
|
|
607
|
+
const corpusPreview = docs.slice(0, 6).map(
|
|
608
|
+
(d) => `[${d.resourceId}] ${d.doc.title} (${d.doc.url}):
|
|
609
|
+
${(d.doc.keyExcerpts || []).slice(0, 3).join("\n")}`
|
|
610
|
+
).join("\n\n");
|
|
611
|
+
const candidateSeed = detBundle.concepts.slice(0, 12).map((c) => `${c.term} (${c.sourceIds.join(",")})`).join("; ");
|
|
612
|
+
const prompt = [
|
|
613
|
+
`Analyze the harvested technical & pedagogical corpus for topic "${topic}" (Domain: ${domain}).`,
|
|
614
|
+
options.context?.ageBand ? `Target Learner Age Band: ${options.context.ageBand[0]}-${options.context.ageBand[1]} years old.` : "",
|
|
615
|
+
options.context?.hardwarePlatform ? `Hardware Platform: ${options.context.hardwarePlatform}.` : "",
|
|
616
|
+
options.context?.softwareEnv ? `Software/Tools: ${options.context.softwareEnv}.` : "",
|
|
617
|
+
`Deterministic Candidate Terms: ${candidateSeed}`,
|
|
618
|
+
`
|
|
619
|
+
Harvested Source Excerpts:
|
|
620
|
+
${corpusPreview}`,
|
|
621
|
+
`
|
|
622
|
+
Return ONLY a valid JSON array of 6 to 12 atomic pedagogical concepts ordered from foundational to advanced.`,
|
|
623
|
+
`IMPORTANT FOR VIETNAMESE TERMS: Keep multi-syllable Vietnamese compound terms intact (e.g. "Bi\u1EBFn tr\u1EA1ng th\xE1i", "C\u1EA3m bi\u1EBFn \xE1nh s\xE1ng", "V\xF2ng l\u1EB7p s\u1EF1 ki\u1EC7n") and pair them with accurate English canonical terms.`,
|
|
624
|
+
`Each item in the JSON array must have: {"id": "C_01", "term": "Display Name", "termVi": "Thu\u1EADt ng\u1EEF Ti\u1EBFng Vi\u1EC7t chu\u1EA9n", "termEn": "Canonical English Term", "category": "core_concept"|"api_syntax"|"hardware_component"|"pedagogical_pattern", "marrLayer": "ULO"|"CIO"|"SIO", "bloomLevel": "understand"|"apply"|"analyze", "salienceScore": 9.0, "definition": "1-sentence clear definition grounded in sources", "prerequisites": ["C_01"], "sourceIds": ["RES-01"]}`
|
|
625
|
+
].filter(Boolean).join("\n");
|
|
626
|
+
const rawResponse = await options.llm(prompt, {
|
|
627
|
+
system: "You are a Principal Computer Science Curriculum Architect & Bilingual Terminology Specialist (Vietnamese-English). Output strictly valid JSON array without markdown commentary.",
|
|
628
|
+
temperature: 0.2,
|
|
629
|
+
jsonMode: true
|
|
630
|
+
});
|
|
631
|
+
const jsonMatch = rawResponse.match(/\[[\s\S]*\]/);
|
|
632
|
+
if (jsonMatch) {
|
|
633
|
+
const parsed = JSON.parse(jsonMatch[0]);
|
|
634
|
+
if (Array.isArray(parsed) && parsed.length > 0) {
|
|
635
|
+
const validIds = new Set(parsed.map((item, idx) => String(item.id || `C_${String(idx + 1).padStart(2, "0")}`)));
|
|
636
|
+
const concepts = parsed.map((item, idx) => {
|
|
637
|
+
const id = String(item.id || `C_${String(idx + 1).padStart(2, "0")}`);
|
|
638
|
+
const prereqs = Array.isArray(item.prerequisites) ? item.prerequisites.map(String).filter((p) => p !== id && validIds.has(p)) : [];
|
|
639
|
+
return {
|
|
640
|
+
id,
|
|
641
|
+
term: String(item.term || item.termVi || item.termEn || `Concept ${idx + 1}`),
|
|
642
|
+
termVi: item.termVi ? String(item.termVi) : void 0,
|
|
643
|
+
termEn: item.termEn ? String(item.termEn) : void 0,
|
|
644
|
+
category: ["core_concept", "api_syntax", "hardware_component", "pedagogical_pattern"].includes(item.category) ? item.category : "core_concept",
|
|
645
|
+
marrLayer: ["ULO", "CIO", "SIO"].includes(item.marrLayer) ? item.marrLayer : "CIO",
|
|
646
|
+
bloomLevel: ["remember", "understand", "apply", "analyze", "evaluate", "create"].includes(item.bloomLevel) ? item.bloomLevel : "understand",
|
|
647
|
+
salienceScore: Number(item.salienceScore) || 8,
|
|
648
|
+
definition: String(item.definition || ""),
|
|
649
|
+
prerequisites: prereqs,
|
|
650
|
+
sourceIds: Array.isArray(item.sourceIds) && item.sourceIds.length > 0 ? item.sourceIds.map(String) : ["RES-01"]
|
|
651
|
+
};
|
|
652
|
+
});
|
|
653
|
+
return {
|
|
654
|
+
concepts,
|
|
655
|
+
conceptDagTsv: this.formatConceptDagTsv(concepts),
|
|
656
|
+
isLlmRefined: true,
|
|
657
|
+
unrefinedCandidatesCount: detBundle.unrefinedCandidatesCount
|
|
658
|
+
};
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
} catch (e) {
|
|
662
|
+
console.warn("[PedagogicalTermExtractor] Host LLM term refinement failed, using deterministic extraction:", e?.message || e);
|
|
663
|
+
}
|
|
664
|
+
return detBundle;
|
|
665
|
+
}
|
|
666
|
+
formatConceptDagTsv(concepts) {
|
|
667
|
+
const header = "id term term_vi term_en marr_layer bloom_level salience prerequisites sources definition";
|
|
668
|
+
const rows = concepts.map(
|
|
669
|
+
(c) => [
|
|
670
|
+
c.id,
|
|
671
|
+
c.term.replace(/\t|\n/g, " "),
|
|
672
|
+
(c.termVi || "").replace(/\t|\n/g, " "),
|
|
673
|
+
(c.termEn || "").replace(/\t|\n/g, " "),
|
|
674
|
+
c.marrLayer,
|
|
675
|
+
c.bloomLevel,
|
|
676
|
+
c.salienceScore,
|
|
677
|
+
c.prerequisites.join(","),
|
|
678
|
+
c.sourceIds.join(","),
|
|
679
|
+
c.definition.replace(/\t|\n/g, " ")
|
|
680
|
+
].join(" ")
|
|
681
|
+
);
|
|
682
|
+
return [header, ...rows].join("\n");
|
|
683
|
+
}
|
|
684
|
+
};
|
|
685
|
+
|
|
686
|
+
// src/harvester/catalogBuilder.ts
|
|
687
|
+
var CatalogBuilder = class {
|
|
688
|
+
buildBundle(topic, domain, items, options = {}) {
|
|
689
|
+
const defaultLang = inferPrimaryCodeLanguage(topic, options.context, domain);
|
|
690
|
+
const indexList = items.map((item, idx) => ({
|
|
691
|
+
id: `RES-${String(idx + 1).padStart(2, "0")}`,
|
|
692
|
+
title: item.doc.title,
|
|
693
|
+
url: item.doc.url,
|
|
694
|
+
domain,
|
|
695
|
+
relevanceScore: item.score,
|
|
696
|
+
byteSize: item.doc.byteSize,
|
|
697
|
+
tablesCount: item.doc.tablesCount,
|
|
698
|
+
codeSnippetsCount: item.doc.extractedCodeSnippets.length,
|
|
699
|
+
summary: item.summary,
|
|
700
|
+
keyTakeaways: item.keyTakeaways || item.doc.keyExcerpts?.slice(0, 3) || []
|
|
701
|
+
}));
|
|
702
|
+
const resourceIndexJson = JSON.stringify(
|
|
703
|
+
{
|
|
704
|
+
topic,
|
|
705
|
+
domain,
|
|
706
|
+
harvestDate: (/* @__PURE__ */ new Date()).toISOString(),
|
|
707
|
+
totalResources: indexList.length,
|
|
708
|
+
resources: indexList
|
|
709
|
+
},
|
|
710
|
+
null,
|
|
711
|
+
2
|
|
712
|
+
);
|
|
713
|
+
const catalogLines = [
|
|
714
|
+
`# \u{1F4CB} Resource Catalog: ${topic}`,
|
|
715
|
+
`> **Domain:** \`${domain.toUpperCase()}\` | **Total Resources:** ${indexList.length}`,
|
|
716
|
+
`
|
|
717
|
+
| ID | Resource Title | Relevance (1-10) | Code Snippets | Tables | Source Link |`,
|
|
718
|
+
`|---|---|---|---|---|---|`,
|
|
719
|
+
...indexList.map(
|
|
720
|
+
(r) => `| **${r.id}** | ${r.title.replace(/\|/g, "/")} | \u2B50 \`${r.relevanceScore}/10\` | \`${r.codeSnippetsCount}\` | \`${r.tablesCount}\` | [Link](${r.url}) |`
|
|
721
|
+
),
|
|
722
|
+
`
|
|
723
|
+
## Resource Summaries & Technical Ground Truth
|
|
724
|
+
`,
|
|
725
|
+
...indexList.map((r) => {
|
|
726
|
+
const takeaways = r.keyTakeaways && r.keyTakeaways.length > 0 ? r.keyTakeaways.map((t) => ` - ${t}`).join("\n") : ` - ${r.summary}`;
|
|
727
|
+
return `### ${r.id}: ${r.title}
|
|
728
|
+
- **URL:** ${r.url}
|
|
729
|
+
- **Relevance:** ${r.relevanceScore}/10
|
|
730
|
+
- **Key Excerpts:**
|
|
731
|
+
${takeaways}
|
|
732
|
+
`;
|
|
733
|
+
})
|
|
734
|
+
];
|
|
735
|
+
const codeSections = items.flatMap((item) => {
|
|
736
|
+
if (item.doc.extractedCodeBlocks && item.doc.extractedCodeBlocks.length > 0) {
|
|
737
|
+
return item.doc.extractedCodeBlocks.slice(0, 2).map((b) => {
|
|
738
|
+
const lang = b.language || defaultLang;
|
|
739
|
+
return `\`\`\`${lang}
|
|
740
|
+
${b.code}
|
|
741
|
+
\`\`\``;
|
|
742
|
+
});
|
|
743
|
+
}
|
|
744
|
+
return item.doc.extractedCodeSnippets.slice(0, 2).map((c) => `\`\`\`${defaultLang}
|
|
745
|
+
${c}
|
|
746
|
+
\`\`\``);
|
|
747
|
+
});
|
|
748
|
+
const conceptLines = options.concepts && options.concepts.length > 0 ? [
|
|
749
|
+
`| ID | Concept / Term | Vietnamese | English | Marr | Bloom | Prereqs | Sources |`,
|
|
750
|
+
`|---|---|---|---|---|---|---|---|`,
|
|
751
|
+
...options.concepts.map(
|
|
752
|
+
(c) => `| \`${c.id}\` | **${c.term.replace(/\|/g, "/")}** | ${c.termVi || "\u2014"} | ${c.termEn || "\u2014"} | \`${c.marrLayer}\` | \`${c.bloomLevel}\` | ${c.prerequisites.join(", ") || "\u2014"} | ${c.sourceIds.join(", ")} |`
|
|
753
|
+
)
|
|
754
|
+
] : [`- Canonical technical breakdown for **${topic}** derived from ${items.length} verified primary sources.`];
|
|
755
|
+
const kgLines = [
|
|
756
|
+
`# \u{1F9EC} Distilled Knowledge Graph: ${topic}`,
|
|
757
|
+
`> **Domain:** \`${domain.toUpperCase()}\` | **Ground Truth Verification:** PASSED`,
|
|
758
|
+
`
|
|
759
|
+
## 1. Architectural Blueprint & Concept Hierarchy (Marr & Bloom)`,
|
|
760
|
+
...conceptLines,
|
|
761
|
+
`
|
|
762
|
+
## 2. Core APIs & Code Primitives`,
|
|
763
|
+
...codeSections.length > 0 ? codeSections : ["*No explicit code snippets captured in harvested sources.*"],
|
|
764
|
+
`
|
|
765
|
+
## 3. Extracted Reference Tables`,
|
|
766
|
+
...items.flatMap((i) => (i.doc.extractedTables || []).slice(0, 1)).length > 0 ? items.flatMap((i) => (i.doc.extractedTables || []).slice(0, 1)) : ["*No markdown reference tables extracted.*"]
|
|
767
|
+
];
|
|
768
|
+
return {
|
|
769
|
+
resourceIndexJson,
|
|
770
|
+
resourceCatalogMd: catalogLines.join("\n"),
|
|
771
|
+
knowledgeGraphMd: kgLines.join("\n"),
|
|
772
|
+
conceptDagTsv: options.conceptDagTsv
|
|
773
|
+
};
|
|
774
|
+
}
|
|
775
|
+
};
|
|
776
|
+
|
|
777
|
+
// src/harvester/healthAuditor.ts
|
|
778
|
+
var HealthAuditor = class {
|
|
779
|
+
auditDocument(doc) {
|
|
780
|
+
const issues = [];
|
|
781
|
+
const warnings = [];
|
|
782
|
+
if (!doc.markdownContent || doc.markdownContent.length < 80) {
|
|
783
|
+
issues.push("Document content is too short or empty (< 80 characters).");
|
|
784
|
+
}
|
|
785
|
+
if (doc.markdownContent.includes("403 Forbidden") || doc.markdownContent.includes("Cloudflare Ray ID") || doc.markdownContent.includes("Access Denied")) {
|
|
786
|
+
issues.push("Document blocked by firewall / Cloudflare / WAF.");
|
|
787
|
+
}
|
|
788
|
+
if (doc.extractedCodeSnippets.length === 0 && !doc.markdownContent.includes("|")) {
|
|
789
|
+
warnings.push("Document does not contain any code snippets or reference tables.");
|
|
790
|
+
}
|
|
791
|
+
const passed = issues.length === 0;
|
|
792
|
+
const score = passed ? warnings.length > 0 ? 80 : 100 : 20;
|
|
793
|
+
return { passed, score, issues, warnings };
|
|
794
|
+
}
|
|
795
|
+
};
|
|
796
|
+
|
|
797
|
+
export { CatalogBuilder, HealthAuditor, MultiSourceSearchEngine, PedagogicalTermExtractor, classifyTechnicalDomain, inferPrimaryCodeLanguage };
|
|
798
|
+
//# sourceMappingURL=chunk-XIF5YNI7.js.map
|
|
799
|
+
//# sourceMappingURL=chunk-XIF5YNI7.js.map
|